From a25ee29197955ff628dfaed502aa43635ad98043 Mon Sep 17 00:00:00 2001 From: Azareal Date: Tue, 9 Jan 2018 07:39:29 +0000 Subject: [PATCH] We now track global user agent stats and have a currently simple Control Panel interface for that. Fixed a possible out of bounds panic in DefaultRouteViewCounter.Bump() --- common/common.go | 11 ++ common/counters.go | 62 ++++++- common/extend.go | 1 + common/pages.go | 14 ++ gen_router.go | 211 ++++++++++++++--------- main.go | 4 + panel_routes.go | 159 +++++++++++------ query_gen/tables.go | 10 ++ router_gen/main.go | 49 ++++++ router_gen/routes.go | 1 + schema/mssql/query_viewchunks_agents.sql | 5 + schema/mysql/query_viewchunks_agents.sql | 5 + schema/pgsql/query_viewchunks_agents.sql | 5 + templates/panel-analytics-agents.html | 18 ++ templates/panel-inner-menu.html | 3 + themes/cosora/public/panel.css | 4 +- 16 files changed, 427 insertions(+), 135 deletions(-) create mode 100644 schema/mssql/query_viewchunks_agents.sql create mode 100644 schema/mysql/query_viewchunks_agents.sql create mode 100644 schema/pgsql/query_viewchunks_agents.sql create mode 100644 templates/panel-analytics-agents.html diff --git a/common/common.go b/common/common.go index 106730d3..32db2a31 100644 --- a/common/common.go +++ b/common/common.go @@ -119,3 +119,14 @@ func SetRouteMapEnum(rme map[string]int) { func SetReverseRouteMapEnum(rrme map[int]string) { reverseRouteMapEnum = rrme } + +var agentMapEnum map[string]int +var reverseAgentMapEnum map[int]string + +func SetAgentMapEnum(ame map[string]int) { + agentMapEnum = ame +} + +func SetReverseAgentMapEnum(rame map[int]string) { + reverseAgentMapEnum = rame +} diff --git a/common/counters.go b/common/counters.go index f003d302..0c9e0745 100644 --- a/common/counters.go +++ b/common/counters.go @@ -9,6 +9,7 @@ import ( ) var GlobalViewCounter *ChunkedViewCounter +var AgentViewCounter *DefaultAgentViewCounter var RouteViewCounter *DefaultRouteViewCounter var TopicViewCounter *DefaultTopicViewCounter @@ -64,7 +65,64 @@ type RWMutexCounterBucket struct { sync.RWMutex } -// The name of the struct clashes with the name of the variable, so we're adding Impl to the end +type DefaultAgentViewCounter struct { + agentBuckets []*RWMutexCounterBucket //[AgentID]count + insert *sql.Stmt +} + +func NewDefaultAgentViewCounter() (*DefaultAgentViewCounter, error) { + acc := qgen.Builder.Accumulator() + var agentBuckets = make([]*RWMutexCounterBucket, len(agentMapEnum)) + for bucketID, _ := range agentBuckets { + agentBuckets[bucketID] = &RWMutexCounterBucket{counter: 0} + } + counter := &DefaultAgentViewCounter{ + agentBuckets: agentBuckets, + insert: acc.Insert("viewchunks_agents").Columns("count, createdAt, browser").Fields("?,UTC_TIMESTAMP(),?").Prepare(), + } + AddScheduledFifteenMinuteTask(counter.Tick) + //AddScheduledSecondTask(counter.Tick) + AddShutdownTask(counter.Tick) + return counter, acc.FirstError() +} + +func (counter *DefaultAgentViewCounter) Tick() error { + for agentID, agentBucket := range counter.agentBuckets { + var count int + agentBucket.RLock() + count = agentBucket.counter + agentBucket.counter = 0 + agentBucket.RUnlock() + + err := counter.insertChunk(count, agentID) // TODO: Bulk insert for speed? + if err != nil { + return err + } + } + return nil +} + +func (counter *DefaultAgentViewCounter) insertChunk(count int, agent int) error { + if count == 0 { + return nil + } + var agentName = reverseAgentMapEnum[agent] + debugLogf("Inserting a viewchunk with a count of %d for agent %s (%d)", count, agentName, agent) + _, err := counter.insert.Exec(count, agentName) + return err +} + +func (counter *DefaultAgentViewCounter) Bump(agent int) { + // TODO: Test this check + debugLog("counter.agentBuckets[", agent, "]: ", counter.agentBuckets[agent]) + if len(counter.agentBuckets) <= agent || agent < 0 { + return + } + counter.agentBuckets[agent].Lock() + counter.agentBuckets[agent].counter++ + counter.agentBuckets[agent].Unlock() +} + type DefaultRouteViewCounter struct { routeBuckets []*RWMutexCounterBucket //[RouteID]count insert *sql.Stmt @@ -115,7 +173,7 @@ func (counter *DefaultRouteViewCounter) insertChunk(count int, route int) error func (counter *DefaultRouteViewCounter) Bump(route int) { // TODO: Test this check debugLog("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route]) - if len(counter.routeBuckets) <= route { + if len(counter.routeBuckets) <= route || route < 0 { return } counter.routeBuckets[route].Lock() diff --git a/common/extend.go b/common/extend.go index 62b4cff6..8700c655 100644 --- a/common/extend.go +++ b/common/extend.go @@ -98,6 +98,7 @@ var PreRenderHooks = map[string][]func(http.ResponseWriter, *http.Request, *User "pre_render_panel_edit_forum": nil, "pre_render_panel_analytics": nil, "pre_render_panel_analytics_routes": nil, + "pre_render_panel_analytics_agents": nil, "pre_render_panel_analytics_route_views": nil, "pre_render_panel_settings": nil, "pre_render_panel_setting": nil, diff --git a/common/pages.go b/common/pages.go index 681ec2e4..928b3dbe 100644 --- a/common/pages.go +++ b/common/pages.go @@ -184,6 +184,20 @@ type PanelAnalyticsRoutesPage struct { ItemList []PanelAnalyticsRoutesItem } +type PanelAnalyticsAgentsItem struct { + Agent string + Count int +} + +type PanelAnalyticsAgentsPage struct { + Title string + CurrentUser User + Header *HeaderVars + Stats PanelStats + Zone string + ItemList []PanelAnalyticsAgentsItem +} + type PanelAnalyticsRoutePage struct { Title string CurrentUser User diff --git a/gen_router.go b/gen_router.go index e33b1e10..34219fc2 100644 --- a/gen_router.go +++ b/gen_router.go @@ -52,6 +52,7 @@ var RouteMap = map[string]interface{}{ "routePanelUsersEditSubmit": routePanelUsersEditSubmit, "routePanelAnalyticsViews": routePanelAnalyticsViews, "routePanelAnalyticsRoutes": routePanelAnalyticsRoutes, + "routePanelAnalyticsAgents": routePanelAnalyticsAgents, "routePanelAnalyticsRouteViews": routePanelAnalyticsRouteViews, "routePanelGroups": routePanelGroups, "routePanelGroupsEdit": routePanelGroupsEdit, @@ -119,32 +120,33 @@ var routeMapEnum = map[string]int{ "routePanelUsersEditSubmit": 34, "routePanelAnalyticsViews": 35, "routePanelAnalyticsRoutes": 36, - "routePanelAnalyticsRouteViews": 37, - "routePanelGroups": 38, - "routePanelGroupsEdit": 39, - "routePanelGroupsEditPerms": 40, - "routePanelGroupsEditSubmit": 41, - "routePanelGroupsEditPermsSubmit": 42, - "routePanelGroupsCreateSubmit": 43, - "routePanelBackups": 44, - "routePanelLogsMod": 45, - "routePanelDebug": 46, - "routePanel": 47, - "routeAccountEditCritical": 48, - "routeAccountEditCriticalSubmit": 49, - "routeAccountEditAvatar": 50, - "routeAccountEditAvatarSubmit": 51, - "routeAccountEditUsername": 52, - "routeAccountEditUsernameSubmit": 53, - "routeAccountEditEmail": 54, - "routeAccountEditEmailTokenSubmit": 55, - "routeProfile": 56, - "routeBanSubmit": 57, - "routeUnban": 58, - "routeActivate": 59, - "routeIps": 60, - "routeDynamic": 61, - "routeUploads": 62, + "routePanelAnalyticsAgents": 37, + "routePanelAnalyticsRouteViews": 38, + "routePanelGroups": 39, + "routePanelGroupsEdit": 40, + "routePanelGroupsEditPerms": 41, + "routePanelGroupsEditSubmit": 42, + "routePanelGroupsEditPermsSubmit": 43, + "routePanelGroupsCreateSubmit": 44, + "routePanelBackups": 45, + "routePanelLogsMod": 46, + "routePanelDebug": 47, + "routePanel": 48, + "routeAccountEditCritical": 49, + "routeAccountEditCriticalSubmit": 50, + "routeAccountEditAvatar": 51, + "routeAccountEditAvatarSubmit": 52, + "routeAccountEditUsername": 53, + "routeAccountEditUsernameSubmit": 54, + "routeAccountEditEmail": 55, + "routeAccountEditEmailTokenSubmit": 56, + "routeProfile": 57, + "routeBanSubmit": 58, + "routeUnban": 59, + "routeActivate": 60, + "routeIps": 61, + "routeDynamic": 62, + "routeUploads": 63, } var reverseRouteMapEnum = map[int]string{ 0: "routeAPI", @@ -184,38 +186,69 @@ var reverseRouteMapEnum = map[int]string{ 34: "routePanelUsersEditSubmit", 35: "routePanelAnalyticsViews", 36: "routePanelAnalyticsRoutes", - 37: "routePanelAnalyticsRouteViews", - 38: "routePanelGroups", - 39: "routePanelGroupsEdit", - 40: "routePanelGroupsEditPerms", - 41: "routePanelGroupsEditSubmit", - 42: "routePanelGroupsEditPermsSubmit", - 43: "routePanelGroupsCreateSubmit", - 44: "routePanelBackups", - 45: "routePanelLogsMod", - 46: "routePanelDebug", - 47: "routePanel", - 48: "routeAccountEditCritical", - 49: "routeAccountEditCriticalSubmit", - 50: "routeAccountEditAvatar", - 51: "routeAccountEditAvatarSubmit", - 52: "routeAccountEditUsername", - 53: "routeAccountEditUsernameSubmit", - 54: "routeAccountEditEmail", - 55: "routeAccountEditEmailTokenSubmit", - 56: "routeProfile", - 57: "routeBanSubmit", - 58: "routeUnban", - 59: "routeActivate", - 60: "routeIps", - 61: "routeDynamic", - 62: "routeUploads", + 37: "routePanelAnalyticsAgents", + 38: "routePanelAnalyticsRouteViews", + 39: "routePanelGroups", + 40: "routePanelGroupsEdit", + 41: "routePanelGroupsEditPerms", + 42: "routePanelGroupsEditSubmit", + 43: "routePanelGroupsEditPermsSubmit", + 44: "routePanelGroupsCreateSubmit", + 45: "routePanelBackups", + 46: "routePanelLogsMod", + 47: "routePanelDebug", + 48: "routePanel", + 49: "routeAccountEditCritical", + 50: "routeAccountEditCriticalSubmit", + 51: "routeAccountEditAvatar", + 52: "routeAccountEditAvatarSubmit", + 53: "routeAccountEditUsername", + 54: "routeAccountEditUsernameSubmit", + 55: "routeAccountEditEmail", + 56: "routeAccountEditEmailTokenSubmit", + 57: "routeProfile", + 58: "routeBanSubmit", + 59: "routeUnban", + 60: "routeActivate", + 61: "routeIps", + 62: "routeDynamic", + 63: "routeUploads", +} +var agentMapEnum = map[string]int{ + "unknown": 0, + "firefox": 1, + "chrome": 2, + "opera": 3, + "safari": 4, + "edge": 5, + "internet-explorer": 6, + "googlebot": 7, + "yandex": 8, + "bing": 9, + "baidu": 10, + "duckduckgo": 11, +} +var reverseAgentMapEnum = map[int]string{ + 0: "unknown", + 1: "firefox", + 2: "chrome", + 3: "opera", + 4: "safari", + 5: "edge", + 6: "internet-explorer", + 7: "googlebot", + 8: "yandex", + 9: "bing", + 10: "baidu", + 11: "duckduckgo", } // TODO: Stop spilling these into the package scope? func init() { common.SetRouteMapEnum(routeMapEnum) common.SetReverseRouteMapEnum(reverseRouteMapEnum) + common.SetAgentMapEnum(agentMapEnum) + common.SetReverseAgentMapEnum(reverseAgentMapEnum) } type GenRouter struct { @@ -301,6 +334,25 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // Increment the global view counter common.GlobalViewCounter.Bump() + + // Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like. + // TODO: Add a setting to disable this? + // TODO: Use a more efficient detector instead of smashing every possible combination in + ua := strings.TrimSuffix(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36") // Noise, no one's going to be running this and it complicates implementing an efficient UA parser, particularly the more efficient right-to-left one I have in mind + switch { + case strings.Contains(ua,"Google"): + common.AgentViewCounter.Bump(7) + case strings.Contains(ua,"OPR"): // Pretends to be Chrome, needs to run before that + common.AgentViewCounter.Bump(3) + case strings.Contains(ua,"Chrome"): + common.AgentViewCounter.Bump(2) + case strings.Contains(ua,"Firefox"): + common.AgentViewCounter.Bump(1) + case strings.Contains(ua,"Safari"): + common.AgentViewCounter.Bump(4) + default: + common.AgentViewCounter.Bump(0) + } // Deal with the session stuff, etc. user, ok := common.PreRoute(w, req) @@ -592,17 +644,20 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { case "/panel/analytics/routes/": common.RouteViewCounter.Bump(36) err = routePanelAnalyticsRoutes(w,req,user) - case "/panel/analytics/route/": + case "/panel/analytics/agents/": common.RouteViewCounter.Bump(37) + err = routePanelAnalyticsAgents(w,req,user) + case "/panel/analytics/route/": + common.RouteViewCounter.Bump(38) err = routePanelAnalyticsRouteViews(w,req,user,extraData) case "/panel/groups/": - common.RouteViewCounter.Bump(38) + common.RouteViewCounter.Bump(39) err = routePanelGroups(w,req,user) case "/panel/groups/edit/": - common.RouteViewCounter.Bump(39) + common.RouteViewCounter.Bump(40) err = routePanelGroupsEdit(w,req,user,extraData) case "/panel/groups/edit/perms/": - common.RouteViewCounter.Bump(40) + common.RouteViewCounter.Bump(41) err = routePanelGroupsEditPerms(w,req,user,extraData) case "/panel/groups/edit/submit/": err = common.NoSessionMismatch(w,req,user) @@ -611,7 +666,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(41) + common.RouteViewCounter.Bump(42) err = routePanelGroupsEditSubmit(w,req,user,extraData) case "/panel/groups/edit/perms/submit/": err = common.NoSessionMismatch(w,req,user) @@ -620,7 +675,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(42) + common.RouteViewCounter.Bump(43) err = routePanelGroupsEditPermsSubmit(w,req,user,extraData) case "/panel/groups/create/": err = common.NoSessionMismatch(w,req,user) @@ -629,7 +684,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(43) + common.RouteViewCounter.Bump(44) err = routePanelGroupsCreateSubmit(w,req,user) case "/panel/backups/": err = common.SuperAdminOnly(w,req,user) @@ -638,10 +693,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(44) + common.RouteViewCounter.Bump(45) err = routePanelBackups(w,req,user,extraData) case "/panel/logs/mod/": - common.RouteViewCounter.Bump(45) + common.RouteViewCounter.Bump(46) err = routePanelLogsMod(w,req,user) case "/panel/debug/": err = common.AdminOnly(w,req,user) @@ -650,10 +705,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(46) + common.RouteViewCounter.Bump(47) err = routePanelDebug(w,req,user) default: - common.RouteViewCounter.Bump(47) + common.RouteViewCounter.Bump(48) err = routePanel(w,req,user) } if err != nil { @@ -668,7 +723,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(48) + common.RouteViewCounter.Bump(49) err = routeAccountEditCritical(w,req,user) case "/user/edit/critical/submit/": err = common.NoSessionMismatch(w,req,user) @@ -683,7 +738,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(49) + common.RouteViewCounter.Bump(50) err = routeAccountEditCriticalSubmit(w,req,user) case "/user/edit/avatar/": err = common.MemberOnly(w,req,user) @@ -692,7 +747,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(50) + common.RouteViewCounter.Bump(51) err = routeAccountEditAvatar(w,req,user) case "/user/edit/avatar/submit/": err = common.MemberOnly(w,req,user) @@ -701,7 +756,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(51) + common.RouteViewCounter.Bump(52) err = routeAccountEditAvatarSubmit(w,req,user) case "/user/edit/username/": err = common.MemberOnly(w,req,user) @@ -710,7 +765,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(52) + common.RouteViewCounter.Bump(53) err = routeAccountEditUsername(w,req,user) case "/user/edit/username/submit/": err = common.NoSessionMismatch(w,req,user) @@ -725,7 +780,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(53) + common.RouteViewCounter.Bump(54) err = routeAccountEditUsernameSubmit(w,req,user) case "/user/edit/email/": err = common.MemberOnly(w,req,user) @@ -734,7 +789,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(54) + common.RouteViewCounter.Bump(55) err = routeAccountEditEmail(w,req,user) case "/user/edit/token/": err = common.NoSessionMismatch(w,req,user) @@ -749,11 +804,11 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(55) + common.RouteViewCounter.Bump(56) err = routeAccountEditEmailTokenSubmit(w,req,user,extraData) default: req.URL.Path += extraData - common.RouteViewCounter.Bump(56) + common.RouteViewCounter.Bump(57) err = routeProfile(w,req,user) } if err != nil { @@ -774,7 +829,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(57) + common.RouteViewCounter.Bump(58) err = routeBanSubmit(w,req,user,extraData) case "/users/unban/": err = common.NoSessionMismatch(w,req,user) @@ -789,7 +844,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(58) + common.RouteViewCounter.Bump(59) err = routeUnban(w,req,user,extraData) case "/users/activate/": err = common.NoSessionMismatch(w,req,user) @@ -804,7 +859,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(59) + common.RouteViewCounter.Bump(60) err = routeActivate(w,req,user,extraData) case "/users/ips/": err = common.MemberOnly(w,req,user) @@ -813,7 +868,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(60) + common.RouteViewCounter.Bump(61) err = routeIps(w,req,user) } if err != nil { @@ -830,7 +885,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { common.NotFound(w,req) return } - common.RouteViewCounter.Bump(62) + common.RouteViewCounter.Bump(63) req.URL.Path += extraData // TODO: Find a way to propagate errors up from this? router.UploadHandler(w,req) // TODO: Count these views @@ -874,7 +929,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { router.RUnlock() if ok { - common.RouteViewCounter.Bump(61) // TODO: Be more specific about *which* dynamic route it is + common.RouteViewCounter.Bump(62) // TODO: Be more specific about *which* dynamic route it is req.URL.Path += extraData err = handle(w,req,user) if err != nil { diff --git a/main.go b/main.go index 82c76032..ca50706d 100644 --- a/main.go +++ b/main.go @@ -84,6 +84,10 @@ func afterDBInit() (err error) { if err != nil { return err } + common.AgentViewCounter, err = common.NewDefaultAgentViewCounter() + if err != nil { + return err + } common.RouteViewCounter, err = common.NewDefaultRouteViewCounter() if err != nil { return err diff --git a/panel_routes.go b/panel_routes.go index dd6e413c..1edb7146 100644 --- a/panel_routes.go +++ b/panel_routes.go @@ -579,59 +579,6 @@ func routePanelAnalyticsViews(w http.ResponseWriter, r *http.Request, user commo return nil } -func routePanelAnalyticsRoutes(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { - headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) - if ferr != nil { - return ferr - } - var routeMap = make(map[string]int) - - acc := qgen.Builder.Accumulator() - rows, err := acc.Select("viewchunks").Columns("count, route").Where("route != ''").DateCutoff("createdAt", 1, "day").Query() - if err != nil && err != ErrNoRows { - return common.InternalError(err, w, r) - } - defer rows.Close() - - for rows.Next() { - var count int - var route string - err := rows.Scan(&count, &route) - if err != nil { - return common.InternalError(err, w, r) - } - - log.Print("count: ", count) - log.Print("route: ", route) - routeMap[route] += count - } - err = rows.Err() - if err != nil { - return common.InternalError(err, w, r) - } - - // TODO: Sort this slice - var routeItems []common.PanelAnalyticsRoutesItem - for route, count := range routeMap { - routeItems = append(routeItems, common.PanelAnalyticsRoutesItem{ - Route: route, - Count: count, - }) - } - - pi := common.PanelAnalyticsRoutesPage{common.GetTitlePhrase("panel-analytics"), user, headerVars, stats, "analytics", routeItems} - if common.PreRenderHooks["pre_render_panel_analytics_routes"] != nil { - if common.RunPreRenderHook("pre_render_panel_analytics_routes", w, r, &user, &pi) { - return nil - } - } - err = common.Templates.ExecuteTemplate(w, "panel-analytics-routes.html", pi) - if err != nil { - return common.InternalError(err, w, r) - } - return nil -} - func routePanelAnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user common.User, route string) common.RouteError { headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) if ferr != nil { @@ -730,6 +677,112 @@ func routePanelAnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user return nil } +func routePanelAnalyticsRoutes(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { + headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) + if ferr != nil { + return ferr + } + var routeMap = make(map[string]int) + + acc := qgen.Builder.Accumulator() + rows, err := acc.Select("viewchunks").Columns("count, route").Where("route != ''").DateCutoff("createdAt", 1, "day").Query() + if err != nil && err != ErrNoRows { + return common.InternalError(err, w, r) + } + defer rows.Close() + + for rows.Next() { + var count int + var route string + err := rows.Scan(&count, &route) + if err != nil { + return common.InternalError(err, w, r) + } + + log.Print("count: ", count) + log.Print("route: ", route) + routeMap[route] += count + } + err = rows.Err() + if err != nil { + return common.InternalError(err, w, r) + } + + // TODO: Sort this slice + var routeItems []common.PanelAnalyticsRoutesItem + for route, count := range routeMap { + routeItems = append(routeItems, common.PanelAnalyticsRoutesItem{ + Route: route, + Count: count, + }) + } + + pi := common.PanelAnalyticsRoutesPage{common.GetTitlePhrase("panel-analytics"), user, headerVars, stats, "analytics", routeItems} + if common.PreRenderHooks["pre_render_panel_analytics_routes"] != nil { + if common.RunPreRenderHook("pre_render_panel_analytics_routes", w, r, &user, &pi) { + return nil + } + } + err = common.Templates.ExecuteTemplate(w, "panel-analytics-routes.html", pi) + if err != nil { + return common.InternalError(err, w, r) + } + return nil +} + +func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { + headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) + if ferr != nil { + return ferr + } + var agentMap = make(map[string]int) + + acc := qgen.Builder.Accumulator() + rows, err := acc.Select("viewchunks_agents").Columns("count, browser").DateCutoff("createdAt", 1, "day").Query() + if err != nil && err != ErrNoRows { + return common.InternalError(err, w, r) + } + defer rows.Close() + + for rows.Next() { + var count int + var agent string + err := rows.Scan(&count, &agent) + if err != nil { + return common.InternalError(err, w, r) + } + + log.Print("count: ", count) + log.Print("agent: ", agent) + agentMap[agent] += count + } + err = rows.Err() + if err != nil { + return common.InternalError(err, w, r) + } + + // TODO: Sort this slice + var agentItems []common.PanelAnalyticsAgentsItem + for agent, count := range agentMap { + agentItems = append(agentItems, common.PanelAnalyticsAgentsItem{ + Agent: agent, + Count: count, + }) + } + + pi := common.PanelAnalyticsAgentsPage{common.GetTitlePhrase("panel-analytics"), user, headerVars, stats, "analytics", agentItems} + if common.PreRenderHooks["pre_render_panel_analytics_agents"] != nil { + if common.RunPreRenderHook("pre_render_panel_analytics_agents", w, r, &user, &pi) { + return nil + } + } + err = common.Templates.ExecuteTemplate(w, "panel-analytics-agents.html", pi) + if err != nil { + return common.InternalError(err, w, r) + } + return nil +} + func routePanelSettings(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) if ferr != nil { diff --git a/query_gen/tables.go b/query_gen/tables.go index 8a10cda0..587a14f6 100644 --- a/query_gen/tables.go +++ b/query_gen/tables.go @@ -365,6 +365,16 @@ func createTables(adapter qgen.Adapter) error { []qgen.DBTableKey{}, ) + qgen.Install.CreateTable("viewchunks_agents", "", "", + []qgen.DBTableColumn{ + qgen.DBTableColumn{"count", "int", 0, false, false, "0"}, + qgen.DBTableColumn{"createdAt", "datetime", 0, false, false, ""}, + qgen.DBTableColumn{"browser", "varchar", 200, false, false, ""}, // googlebot, firefox, opera, etc. + //qgen.DBTableColumn{"version","varchar",0,false,false,""}, // the version of the browser or bot + }, + []qgen.DBTableKey{}, + ) + /* qgen.Install.CreateTable("viewchunks_forums", "", "", []qgen.DBTableColumn{ diff --git a/router_gen/main.go b/router_gen/main.go index c4b272c1..b22c2c6e 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -17,6 +17,8 @@ type TmplVars struct { RouteGroups []*RouteGroup AllRouteNames []string AllRouteMap map[string]int + AllAgentNames []string + AllAgentMap map[string]int } func main() { @@ -155,6 +157,26 @@ func main() { mapIt("routeUploads") tmplVars.AllRouteNames = allRouteNames tmplVars.AllRouteMap = allRouteMap + tmplVars.AllAgentNames = []string{ + "unknown", + "firefox", + "chrome", + "opera", + "safari", + "edge", + "internet-explorer", + + "googlebot", + "yandex", + "bing", + "baidu", + "duckduckgo", + } + + tmplVars.AllAgentMap = make(map[string]int) + for id, agent := range tmplVars.AllAgentNames { + tmplVars.AllAgentMap[agent] = id + } var fileData = `// Code generated by. DO NOT EDIT. /* This file was automatically generated by the software. Please don't edit it as your changes may be overwritten at any moment. */ @@ -183,11 +205,19 @@ var routeMapEnum = map[string]int{ {{range $index, $element := .AllRouteNames}} var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}} {{$index}}: "{{$element}}",{{end}} } +var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}} + "{{$element}}": {{$index}},{{end}} +} +var reverseAgentMapEnum = map[int]string{ {{range $index, $element := .AllAgentNames}} + {{$index}}: "{{$element}}",{{end}} +} // TODO: Stop spilling these into the package scope? func init() { common.SetRouteMapEnum(routeMapEnum) common.SetReverseRouteMapEnum(reverseRouteMapEnum) + common.SetAgentMapEnum(agentMapEnum) + common.SetReverseAgentMapEnum(reverseAgentMapEnum) } type GenRouter struct { @@ -273,6 +303,25 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // Increment the global view counter common.GlobalViewCounter.Bump() + + // Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like. + // TODO: Add a setting to disable this? + // TODO: Use a more efficient detector instead of smashing every possible combination in + ua := strings.TrimSuffix(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36") // Noise, no one's going to be running this and it complicates implementing an efficient UA parser, particularly the more efficient right-to-left one I have in mind + switch { + case strings.Contains(ua,"Google"): + common.AgentViewCounter.Bump({{.AllAgentMap.googlebot}}) + case strings.Contains(ua,"OPR"): // Pretends to be Chrome, needs to run before that + common.AgentViewCounter.Bump({{.AllAgentMap.opera}}) + case strings.Contains(ua,"Chrome"): + common.AgentViewCounter.Bump({{.AllAgentMap.chrome}}) + case strings.Contains(ua,"Firefox"): + common.AgentViewCounter.Bump({{.AllAgentMap.firefox}}) + case strings.Contains(ua,"Safari"): + common.AgentViewCounter.Bump({{.AllAgentMap.safari}}) + default: + common.AgentViewCounter.Bump({{.AllAgentMap.unknown}}) + } // Deal with the session stuff, etc. user, ok := common.PreRoute(w, req) diff --git a/router_gen/routes.go b/router_gen/routes.go index f48bc4d5..6a4be933 100644 --- a/router_gen/routes.go +++ b/router_gen/routes.go @@ -92,6 +92,7 @@ func buildPanelRoutes() { View("routePanelAnalyticsViews", "/panel/analytics/views/").Before("ParseForm"), View("routePanelAnalyticsRoutes", "/panel/analytics/routes/"), + View("routePanelAnalyticsAgents", "/panel/analytics/agents/"), View("routePanelAnalyticsRouteViews", "/panel/analytics/route/", "extraData"), View("routePanelGroups", "/panel/groups/"), diff --git a/schema/mssql/query_viewchunks_agents.sql b/schema/mssql/query_viewchunks_agents.sql new file mode 100644 index 00000000..d8318f62 --- /dev/null +++ b/schema/mssql/query_viewchunks_agents.sql @@ -0,0 +1,5 @@ +CREATE TABLE [viewchunks_agents] ( + [count] int DEFAULT 0 not null, + [createdAt] datetime not null, + [browser] nvarchar (200) not null +); \ No newline at end of file diff --git a/schema/mysql/query_viewchunks_agents.sql b/schema/mysql/query_viewchunks_agents.sql new file mode 100644 index 00000000..510f84a0 --- /dev/null +++ b/schema/mysql/query_viewchunks_agents.sql @@ -0,0 +1,5 @@ +CREATE TABLE `viewchunks_agents` ( + `count` int DEFAULT 0 not null, + `createdAt` datetime not null, + `browser` varchar(200) not null +); \ No newline at end of file diff --git a/schema/pgsql/query_viewchunks_agents.sql b/schema/pgsql/query_viewchunks_agents.sql new file mode 100644 index 00000000..b74921cf --- /dev/null +++ b/schema/pgsql/query_viewchunks_agents.sql @@ -0,0 +1,5 @@ +CREATE TABLE `viewchunks_agents` ( + `count` int DEFAULT 0 not null, + `createdAt` timestamp not null, + `browser` varchar (200) not null +); \ No newline at end of file diff --git a/templates/panel-analytics-agents.html b/templates/panel-analytics-agents.html new file mode 100644 index 00000000..81ea6b8f --- /dev/null +++ b/templates/panel-analytics-agents.html @@ -0,0 +1,18 @@ +{{template "header.html" . }} +
+{{template "panel-menu.html" . }} +
+
+
Agents (24 hours)
+
+
+ {{range .ItemList}} +
+ {{.Agent}} + {{.Count}} views +
+ {{end}} +
+
+
+{{template "footer.html" . }} diff --git a/templates/panel-inner-menu.html b/templates/panel-inner-menu.html index 966acc2c..927afbcd 100644 --- a/templates/panel-inner-menu.html +++ b/templates/panel-inner-menu.html @@ -35,6 +35,9 @@
Routes
+
+ Agents +
Crawlers
diff --git a/themes/cosora/public/panel.css b/themes/cosora/public/panel.css index 334571d3..1ca5c114 100644 --- a/themes/cosora/public/panel.css +++ b/themes/cosora/public/panel.css @@ -168,10 +168,10 @@ padding-top: 16px; } .ct-series-a .ct-bar, .ct-series-a .ct-line, .ct-series-a .ct-point, .ct-series-a .ct-slice-donut { - stroke: hsl(359,98%,33%) !important; + stroke: hsl(359,98%,53%) !important; } .ct-point { - stroke: hsl(359,98%,53%) !important; + stroke: hsl(359,98%,33%) !important; } .ct-point:hover { stroke: hsl(359,98%,50%) !important;