experiment with more route perf tracking

common/counters/forums.go

@@ -27,7 +27,7 @@ func NewDefaultForumViewCounter() (*DefaultForumViewCounter, error) {
 	co := &DefaultForumViewCounter{
 		oddMap:  make(map[int]*RWMutexCounterBucket),
 		evenMap: make(map[int]*RWMutexCounterBucket),
-		insert:  acc.Insert("viewchunks_forums").Columns("count, createdAt, forum").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
+		insert:  acc.Insert("viewchunks_forums").Columns("count,createdAt,forum").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
 	}
 	c.AddScheduledFifteenMinuteTask(co.Tick) // There could be a lot of routes, so we don't want to be running this every second
 	//c.AddScheduledSecondTask(co.Tick)
@@ -38,17 +38,17 @@ func NewDefaultForumViewCounter() (*DefaultForumViewCounter, error) {
 func (co *DefaultForumViewCounter) Tick() error {
 	cLoop := func(l *sync.RWMutex, m map[int]*RWMutexCounterBucket) error {
 		l.RLock()
-		for forumID, forum := range m {
+		for fid, f := range m {
 			l.RUnlock()
 			var count int
-			forum.RLock()
+			f.RLock()
-			count = forum.counter
+			count = f.counter
-			forum.RUnlock()
+			f.RUnlock()
 			// TODO: Only delete the bucket when it's zero to avoid hitting popular forums?
 			l.Lock()
-			delete(m, forumID)
+			delete(m, fid)
 			l.Unlock()
-			err := co.insertChunk(count, forumID)
+			err := co.insertChunk(count, fid)
 			if err != nil {
 				return errors.Wrap(errors.WithStack(err),"forum counter")
 			}
@@ -64,7 +64,7 @@ func (co *DefaultForumViewCounter) Tick() error {
 	return cLoop(&co.evenLock,co.evenMap)
 }
 
-func (co *DefaultForumViewCounter) insertChunk(count int, forum int) error {
+func (co *DefaultForumViewCounter) insertChunk(count, forum int) error {
 	if count == 0 {
 		return nil
 	}
@@ -73,34 +73,34 @@ func (co *DefaultForumViewCounter) insertChunk(count int, forum int) error {
 	return err
 }
 
-func (co *DefaultForumViewCounter) Bump(forumID int) {
+func (co *DefaultForumViewCounter) Bump(fid int) {
 	// Is the ID even?
-	if forumID%2 == 0 {
+	if fid%2 == 0 {
 		co.evenLock.RLock()
-		forum, ok := co.evenMap[forumID]
+		f, ok := co.evenMap[fid]
 		co.evenLock.RUnlock()
 		if ok {
-			forum.Lock()
+			f.Lock()
-			forum.counter++
+			f.counter++
-			forum.Unlock()
+			f.Unlock()
 		} else {
 			co.evenLock.Lock()
-			co.evenMap[forumID] = &RWMutexCounterBucket{counter: 1}
+			co.evenMap[fid] = &RWMutexCounterBucket{counter: 1}
 			co.evenLock.Unlock()
 		}
 		return
 	}
 
 	co.oddLock.RLock()
-	forum, ok := co.oddMap[forumID]
+	f, ok := co.oddMap[fid]
 	co.oddLock.RUnlock()
 	if ok {
-		forum.Lock()
+		f.Lock()
-		forum.counter++
+		f.counter++
-		forum.Unlock()
+		f.Unlock()
 	} else {
 		co.oddLock.Lock()
-		co.oddMap[forumID] = &RWMutexCounterBucket{counter: 1}
+		co.oddMap[fid] = &RWMutexCounterBucket{counter: 1}
 		co.oddLock.Unlock()
 	}
 }

common/counters/routes.go

@@ -7,6 +7,7 @@ import (
 
 	c "github.com/Azareal/Gosora/common"
 	qgen "github.com/Azareal/Gosora/query_gen"
+	"github.com/Azareal/Gosora/uutils"
 	"github.com/pkg/errors"
 )
 
@@ -150,6 +151,32 @@ func (co *DefaultRouteViewCounter) Bump2(route int, t time.Time) {
 		return
 	}
 	micro := int(time.Since(t).Microseconds())
+	//co.PerfCounter.Push(since, true)
+	b.Lock()
+	b.counter++
+	if micro != b.avg {
+		if b.avg == 0 {
+			b.avg = micro
+		} else {
+			b.avg = (micro + b.avg) / 2
+		}
+	}
+	b.Unlock()
+}
+
+// TODO: Eliminate the lock?
+func (co *DefaultRouteViewCounter) Bump3(route int, nano int64) {
+	if c.Config.DisableAnalytics {
+		return
+	}
+	// TODO: Test this check
+	b := co.buckets[route]
+	c.DebugDetail("buckets[", route, "]: ", b)
+	if len(co.buckets) <= route || route < 0 {
+		return
+	}
+	micro := int((uutils.Nanotime() - nano) / 1000)
+	//co.PerfCounter.Push(since, true)
 	b.Lock()
 	b.counter++
 	if micro != b.avg {

router_gen/main.go

@@ -83,6 +83,9 @@ func main() {
 			vcpy := route.Vars
 			route.Vars = []string{"h"}
 			route.Vars = append(route.Vars, vcpy...)
+		} else if route.Name != "common.RouteWebsockets" {
+			//out += "\n\t\t\tsa := time.Now()"
+			out += "\n\t\t\tcn := uutils.Nanotime()"
 		}
 		out += "\n\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user"
 		for _, item := range route.Vars {
@@ -91,8 +94,10 @@ func main() {
 		out += `)`
 		if !route.Action && !route.NoHead {
 			out += "\n\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)"
-		} else {
+		} else if route.Name != "common.RouteWebsockets" {
-			out += "\n\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
+			//out += "\n\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
+			//out += "\n\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", sa)"
+			out += "\n\t\t\tco.RouteViewCounter.Bump3(" + strconv.Itoa(allRouteMap[route.Name]) + ", cn)"
 		}
 	}
 
@@ -149,6 +154,8 @@ func main() {
 				vcpy := route.Vars
 				route.Vars = []string{"h"}
 				route.Vars = append(route.Vars, vcpy...)
+			} else {
+				out += "\n\t\t\t\t\tcn := uutils.Nanotime()"
 			}
 			out += "\n\t\t\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user"
 			for _, item := range route.Vars {
@@ -158,7 +165,8 @@ func main() {
 			if !route.Action && !route.NoHead && !group.NoHead {
 				out += "\n\t\t\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)"
 			} else {
-				out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
+				//out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
+				out += "\n\t\t\t\t\tco.RouteViewCounter.Bump3(" + strconv.Itoa(allRouteMap[route.Name]) + ", cn)"
 			}
 		}
 
@@ -341,9 +349,11 @@ import (
 	"errors"
 	"os"
 	"net/http"
+	"time"
 
 	c "github.com/Azareal/Gosora/common"
 	co "github.com/Azareal/Gosora/common/counters"
+	"github.com/Azareal/Gosora/uutils"
 	"github.com/Azareal/Gosora/routes"
 	"github.com/Azareal/Gosora/routes/panel"
 )
@@ -384,6 +394,7 @@ var markToAgent = map[string]string{ {{range $index, $element := .AllAgentMarkNa
 
 // TODO: Stop spilling these into the package scope?
 func init() {
+	_ = time.Now()
 	co.SetRouteMapEnum(routeMapEnum)
 	co.SetReverseRouteMapEnum(reverseRouteMapEnum)
 	co.SetAgentMapEnum(agentMapEnum)
@@ -666,14 +677,14 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		var items []string
 		var buffer []byte
 		var os int
-		for _, it := range StringToBytes(ua) {
+		for _, it := range uutils.StringToBytes(ua) {
 			if (it > 64 && it < 91) || (it > 96 && it < 123) {
 				buffer = append(buffer, it)
 			} else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == '_' || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' || (it == ':' && bytes.Equal(buffer,[]byte("http"))) || it == ',' || it == '/' {
 				if len(buffer) != 0 {
 					if len(buffer) > 2 {
 						// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
-						switch(BytesToString(buffer)) {
+						switch(uutils.BytesToString(buffer)) {
 						case "Windows":
 							os = {{.AllOSMap.windows}}
 						case "Linux":

routes/forum.go

@@ -21,7 +21,7 @@ var forumStmts ForumStmts
 func init() {
 	c.DbInits.Add(func(acc *qgen.Accumulator) error {
 		forumStmts = ForumStmts{
-			getTopics: acc.Select("topics").Columns("tid, title, content, createdBy, is_closed, sticky, createdAt, lastReplyAt, lastReplyBy, lastReplyID, parentID, views, postCount, likeCount").Where("parentID = ?").Orderby("sticky DESC, lastReplyAt DESC, createdBy DESC").Limit("?,?").Prepare(),
+			getTopics: acc.Select("topics").Columns("tid, title, content, createdBy, is_closed, sticky, createdAt, lastReplyAt, lastReplyBy, lastReplyID, parentID, views, postCount, likeCount").Where("parentID=?").Orderby("sticky DESC, lastReplyAt DESC, createdBy DESC").Limit("?,?").Prepare(),
 		}
 		return acc.FirstError()
 	})

routes/panel/analytics.go

@@ -834,6 +834,9 @@ func AnalyticsRoutesPerf(w http.ResponseWriter, r *http.Request, user c.User) c.
 		if inEx(ovitem.name) {
 			continue
 		}
+		if strings.HasPrefix(ovitem.name,"panel.") {
+			continue
+		}
 		var viewList []int64
 		for _, value := range revLabelList {
 			viewList = append(viewList, ovitem.viewMap[value])

tmpl_client/stub.go

@@ -1,9 +1,10 @@
 package tmpl
 
 import (
-	"reflect"
+	//"reflect"
-	"runtime"
+	//"runtime"
-	"unsafe"
+	//"unsafe"
+	"github.com/Azareal/Gosora/uutils"
 )
 
 var GetFrag = func(name string) [][]byte {
@@ -14,6 +15,9 @@ type WriteString interface {
 	WriteString(s string) (n int, err error)
 }
 
+var StringToBytes = uutils.StringToBytes
+
+/*
 func StringToBytes(s string) (bytes []byte) {
 	str := (*reflect.StringHeader)(unsafe.Pointer(&s))
 	slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
@@ -23,3 +27,4 @@ func StringToBytes(s string) (bytes []byte) {
 	runtime.KeepAlive(&s)
 	return bytes
 }
+*/

tmpl_stub.go

@@ -1,11 +1,14 @@
 package main
 
 import (
-	"reflect"
+	//"reflect"
-	"runtime"
+	//"runtime"
-	"unsafe"
+	//"unsafe"
+	"github.com/Azareal/Gosora/uutils"
 )
 
+// TODO: Add a safe build mode for things like Google Appengine
+
 var GetFrag = func(name string) [][]byte {
 	return nil
 }
@@ -14,6 +17,11 @@ type WriteString interface {
 	WriteString(s string) (n int, err error)
 }
 
+var StringToBytes = uutils.StringToBytes
+var BytesToString = uutils.BytesToString
+var Nanotime = uutils.Nanotime
+
+/*
 func StringToBytes(s string) (bytes []byte) {
 	str := (*reflect.StringHeader)(unsafe.Pointer(&s))
 	slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
@@ -32,3 +40,11 @@ func BytesToString(bytes []byte) (s string) {
 	runtime.KeepAlive(&bytes)
 	return s
 }
+
+//go:noescape
+//go:linkname nanotime runtime.nanotime
+func nanotime() int64
+
+func Nanotime() int64 {
+	return nanotime()
+}*/

uutils/utils.go

@@ -0,0 +1,36 @@
+package uutils
+
+import (
+	"reflect"
+	"runtime"
+	"unsafe"
+)
+
+// TODO: Add a safe build mode for things like Google Appengine
+
+func StringToBytes(s string) (bytes []byte) {
+	str := (*reflect.StringHeader)(unsafe.Pointer(&s))
+	slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
+	slice.Data = str.Data
+	slice.Len = str.Len
+	slice.Cap = str.Len
+	runtime.KeepAlive(&s)
+	return bytes
+}
+
+func BytesToString(bytes []byte) (s string) {
+	slice := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
+	str := (*reflect.StringHeader)(unsafe.Pointer(&s))
+	str.Data = slice.Data
+	str.Len = slice.Len
+	runtime.KeepAlive(&bytes)
+	return s
+}
+
+//go:noescape
+//go:linkname nanotime runtime.nanotime
+func nanotime() int64
+
+func Nanotime() int64 {
+	return nanotime()
+}