From 2455e951aac290c41cd123d603f6bbc7f62b4834 Mon Sep 17 00:00:00 2001 From: Azareal Date: Sun, 4 Feb 2018 08:15:20 +0000 Subject: [PATCH] We now have analytics for the operating systems used by users. Route view bumping logs have been moved from regular debug mode to super debug to reduce the amount of noise. Added user friendly agent names. Reduced the amount of logging outside of debug mode. We now track Mobile Safari and the Samsung Browser. We now track SeznamBot, TwitterBot, and Discourse's Bot. We now track Trident. UAs are now filtered to reduce the amount of bad activity. Added more bad phrases for bad routes. Added the viewchunks_systems table. Began work on referrer tracking. --- common/common.go | 11 + common/counters.go | 61 ++- common/extend.go | 44 +- common/pages.go | 6 +- common/phrases.go | 38 +- common/requests.go | 75 +++ gen_router.go | 535 ++++++++++++-------- langs/english.json | 40 ++ main.go | 4 + panel_routes.go | 194 ++++++- query_gen/tables.go | 9 + router_gen/main.go | 126 ++++- router_gen/routes.go | 2 + schema/mssql/query_viewchunks_systems.sql | 5 + schema/mysql/query_viewchunks_systems.sql | 5 + schema/pgsql/query_viewchunks_systems.sql | 5 + templates/panel-inner-menu.html | 3 + templates/panel_analytics_agent_views.html | 2 +- templates/panel_analytics_agents.html | 2 +- templates/panel_analytics_system_views.html | 51 ++ templates/panel_analytics_systems.html | 29 ++ 21 files changed, 945 insertions(+), 302 deletions(-) create mode 100644 common/requests.go create mode 100644 schema/mssql/query_viewchunks_systems.sql create mode 100644 schema/mysql/query_viewchunks_systems.sql create mode 100644 schema/pgsql/query_viewchunks_systems.sql create mode 100644 templates/panel_analytics_system_views.html create mode 100644 templates/panel_analytics_systems.html diff --git a/common/common.go b/common/common.go index 32db2a31..c02a8cdb 100644 --- a/common/common.go +++ b/common/common.go @@ -130,3 +130,14 @@ func SetAgentMapEnum(ame map[string]int) { func SetReverseAgentMapEnum(rame map[int]string) { reverseAgentMapEnum = rame } + +var osMapEnum map[string]int +var reverseOSMapEnum map[int]string + +func SetOSMapEnum(osme map[string]int) { + osMapEnum = osme +} + +func SetReverseOSMapEnum(rosme map[int]string) { + reverseOSMapEnum = rosme +} diff --git a/common/counters.go b/common/counters.go index 865150c8..c342ea1a 100644 --- a/common/counters.go +++ b/common/counters.go @@ -11,6 +11,7 @@ import ( // Global counters var GlobalViewCounter *DefaultViewCounter var AgentViewCounter *DefaultAgentViewCounter +var OSViewCounter *DefaultOSViewCounter var RouteViewCounter *DefaultRouteViewCounter var PostCounter *DefaultPostCounter var TopicCounter *DefaultTopicCounter @@ -222,6 +223,64 @@ func (counter *DefaultAgentViewCounter) Bump(agent int) { counter.agentBuckets[agent].Unlock() } +type DefaultOSViewCounter struct { + osBuckets []*RWMutexCounterBucket //[OSID]count + insert *sql.Stmt +} + +func NewDefaultOSViewCounter() (*DefaultOSViewCounter, error) { + acc := qgen.Builder.Accumulator() + var osBuckets = make([]*RWMutexCounterBucket, len(osMapEnum)) + for bucketID, _ := range osBuckets { + osBuckets[bucketID] = &RWMutexCounterBucket{counter: 0} + } + counter := &DefaultOSViewCounter{ + osBuckets: osBuckets, + insert: acc.Insert("viewchunks_systems").Columns("count, createdAt, system").Fields("?,UTC_TIMESTAMP(),?").Prepare(), + } + AddScheduledFifteenMinuteTask(counter.Tick) + //AddScheduledSecondTask(counter.Tick) + AddShutdownTask(counter.Tick) + return counter, acc.FirstError() +} + +func (counter *DefaultOSViewCounter) Tick() error { + for osID, osBucket := range counter.osBuckets { + var count int + osBucket.RLock() + count = osBucket.counter + osBucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters? + osBucket.RUnlock() + + err := counter.insertChunk(count, osID) // TODO: Bulk insert for speed? + if err != nil { + return err + } + } + return nil +} + +func (counter *DefaultOSViewCounter) insertChunk(count int, os int) error { + if count == 0 { + return nil + } + var osName = reverseOSMapEnum[os] + debugLogf("Inserting a viewchunk with a count of %d for OS %s (%d)", count, osName, os) + _, err := counter.insert.Exec(count, osName) + return err +} + +func (counter *DefaultOSViewCounter) Bump(os int) { + // TODO: Test this check + debugDetail("counter.osBuckets[", os, "]: ", counter.osBuckets[os]) + if len(counter.osBuckets) <= os || os < 0 { + return + } + counter.osBuckets[os].Lock() + counter.osBuckets[os].counter++ + counter.osBuckets[os].Unlock() +} + type DefaultRouteViewCounter struct { routeBuckets []*RWMutexCounterBucket //[RouteID]count insert *sql.Stmt @@ -271,7 +330,7 @@ func (counter *DefaultRouteViewCounter) insertChunk(count int, route int) error func (counter *DefaultRouteViewCounter) Bump(route int) { // TODO: Test this check - debugLog("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route]) + debugDetail("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route]) if len(counter.routeBuckets) <= route || route < 0 { return } diff --git a/common/extend.go b/common/extend.go index 562bb8fb..678d0301 100644 --- a/common/extend.go +++ b/common/extend.go @@ -92,27 +92,29 @@ var PreRenderHooks = map[string][]func(http.ResponseWriter, *http.Request, *User "pre_render_ban": nil, "pre_render_ip_search": nil, - "pre_render_panel_dashboard": nil, - "pre_render_panel_forums": nil, - "pre_render_panel_delete_forum": nil, - "pre_render_panel_edit_forum": nil, - "pre_render_panel_analytics_views": nil, - "pre_render_panel_analytics_routes": nil, - "pre_render_panel_analytics_agents": nil, - "pre_render_panel_analytics_route_views": nil, - "pre_render_panel_analytics_agent_views": nil, - "pre_render_panel_settings": nil, - "pre_render_panel_setting": nil, - "pre_render_panel_word_filters": nil, - "pre_render_panel_word_filters_edit": nil, - "pre_render_panel_plugins": nil, - "pre_render_panel_users": nil, - "pre_render_panel_edit_user": nil, - "pre_render_panel_groups": nil, - "pre_render_panel_edit_group": nil, - "pre_render_panel_edit_group_perms": nil, - "pre_render_panel_themes": nil, - "pre_render_panel_modlogs": nil, + "pre_render_panel_dashboard": nil, + "pre_render_panel_forums": nil, + "pre_render_panel_delete_forum": nil, + "pre_render_panel_edit_forum": nil, + "pre_render_panel_analytics_views": nil, + "pre_render_panel_analytics_routes": nil, + "pre_render_panel_analytics_agents": nil, + "pre_render_panel_analytics_systems": nil, + "pre_render_panel_analytics_route_views": nil, + "pre_render_panel_analytics_agent_views": nil, + "pre_render_panel_analytics_system_views": nil, + "pre_render_panel_settings": nil, + "pre_render_panel_setting": nil, + "pre_render_panel_word_filters": nil, + "pre_render_panel_word_filters_edit": nil, + "pre_render_panel_plugins": nil, + "pre_render_panel_users": nil, + "pre_render_panel_edit_user": nil, + "pre_render_panel_groups": nil, + "pre_render_panel_edit_group": nil, + "pre_render_panel_edit_group_perms": nil, + "pre_render_panel_themes": nil, + "pre_render_panel_modlogs": nil, "pre_render_error": nil, // Note: This hook isn't run for a few errors whose templates are computed at startup and reused, such as InternalError. This hook is also not available in JS mode. "pre_render_security_error": nil, diff --git a/common/pages.go b/common/pages.go index 11d96dd1..4576b17c 100644 --- a/common/pages.go +++ b/common/pages.go @@ -190,8 +190,9 @@ type PanelAnalyticsRoutesPage struct { } type PanelAnalyticsAgentsItem struct { - Agent string - Count int + Agent string + FriendlyAgent string + Count int } type PanelAnalyticsAgentsPage struct { @@ -223,6 +224,7 @@ type PanelAnalyticsAgentPage struct { Stats PanelStats Zone string Agent string + FriendlyAgent string PrimaryGraph PanelTimeGraph TimeRange string } diff --git a/common/phrases.go b/common/phrases.go index a132fbc3..fc947a3c 100644 --- a/common/phrases.go +++ b/common/phrases.go @@ -34,16 +34,18 @@ type LevelPhrases struct { // ! For the sake of thread safety, you must never modify a *LanguagePack directly, but to create a copy of it and overwrite the entry in the sync.Map type LanguagePack struct { - Name string - Phrases map[string]string // Should we use a sync map or a struct for these? It would be nice, if we could keep all the phrases consistent. - Levels LevelPhrases - GlobalPerms map[string]string - LocalPerms map[string]string - SettingLabels map[string]string - PermPresets map[string]string - Accounts map[string]string // TODO: Apply these phrases in the software proper - Errors map[string]map[string]string // map[category]map[name]value - PageTitles map[string]string + Name string + Phrases map[string]string // Should we use a sync map or a struct for these? It would be nice, if we could keep all the phrases consistent. + Levels LevelPhrases + GlobalPerms map[string]string + LocalPerms map[string]string + SettingLabels map[string]string + PermPresets map[string]string + Accounts map[string]string // TODO: Apply these phrases in the software proper + UserAgents map[string]string + OperatingSystems map[string]string + Errors map[string]map[string]string // map[category]map[name]value + PageTitles map[string]string } // TODO: Add the ability to edit language JSON files from the Control Panel and automatically scan the files for changes @@ -154,6 +156,22 @@ func GetAccountPhrase(name string) string { return res } +func GetUserAgentPhrase(name string) (string, bool) { + res, ok := currentLangPack.Load().(*LanguagePack).UserAgents[name] + if !ok { + return "", false + } + return res, true +} + +func GetOSPhrase(name string) (string, bool) { + res, ok := currentLangPack.Load().(*LanguagePack).OperatingSystems[name] + if !ok { + return "", false + } + return res, true +} + // TODO: Does comma ok work with multi-dimensional maps? func GetErrorPhrase(category string, name string) string { res, ok := currentLangPack.Load().(*LanguagePack).Errors[category][name] diff --git a/common/requests.go b/common/requests.go new file mode 100644 index 00000000..9aeae62b --- /dev/null +++ b/common/requests.go @@ -0,0 +1,75 @@ +package common + +import ( + "sync" + "sync/atomic" +) + +// Add ReferrerItems here after they've had zero views for a while +var referrersToDelete = make(map[string]ReferrerDeletable) + +type ReferrerDeletable struct { + item *ReferrerItem + scheduledAt int64 //unixtime +} + +type ReferrerItem struct { + Counter int64 +} + +// ? We'll track referrer domains here rather than the exact URL they arrived from for now, we'll think about expanding later +// ? Referrers are fluid and ever-changing so we have to use string keys rather than 'enum' ints +type DefaultReferrerTracker struct { + odd map[string]*ReferrerItem + even map[string]*ReferrerItem + oddLock sync.RWMutex + evenLock sync.RWMutex +} + +func NewDefaultReferrerTracker() *DefaultReferrerTracker { + return &DefaultReferrerTracker{ + odd: make(map[string]*ReferrerItem), + even: make(map[string]*ReferrerItem), + } +} + +func (ref *DefaultReferrerTracker) Tick() (err error) { + for _, del := range referrersToDelete { + _ = del + // TODO: Calculate the gap between now and the times they were scheduled + } + // TODO: Run the queries and schedule zero view refs for deletion from memory + return nil +} + +func (ref *DefaultReferrerTracker) Bump(referrer string) { + if referrer == "" { + return + } + var refItem *ReferrerItem + + // Slightly crude and rudimentary, but it should give a basic degree of sharding + if referrer[0]%2 == 0 { + ref.evenLock.RLock() + refItem = ref.even[referrer] + ref.evenLock.RUnlock() + if ref != nil { + atomic.AddInt64(&refItem.Counter, 1) + } else { + ref.evenLock.Lock() + ref.even[referrer] = &ReferrerItem{Counter: 1} + ref.evenLock.Unlock() + } + } else { + ref.oddLock.RLock() + refItem = ref.odd[referrer] + ref.oddLock.RUnlock() + if ref != nil { + atomic.AddInt64(&refItem.Counter, 1) + } else { + ref.oddLock.Lock() + ref.odd[referrer] = &ReferrerItem{Counter: 1} + ref.oddLock.Unlock() + } + } +} diff --git a/gen_router.go b/gen_router.go index 88cc173f..34fc2649 100644 --- a/gen_router.go +++ b/gen_router.go @@ -56,8 +56,10 @@ var RouteMap = map[string]interface{}{ "routePanelAnalyticsViews": routePanelAnalyticsViews, "routePanelAnalyticsRoutes": routePanelAnalyticsRoutes, "routePanelAnalyticsAgents": routePanelAnalyticsAgents, + "routePanelAnalyticsSystems": routePanelAnalyticsSystems, "routePanelAnalyticsRouteViews": routePanelAnalyticsRouteViews, "routePanelAnalyticsAgentViews": routePanelAnalyticsAgentViews, + "routePanelAnalyticsSystemViews": routePanelAnalyticsSystemViews, "routePanelAnalyticsPosts": routePanelAnalyticsPosts, "routePanelAnalyticsTopics": routePanelAnalyticsTopics, "routePanelGroups": routePanelGroups, @@ -154,60 +156,62 @@ var routeMapEnum = map[string]int{ "routePanelAnalyticsViews": 37, "routePanelAnalyticsRoutes": 38, "routePanelAnalyticsAgents": 39, - "routePanelAnalyticsRouteViews": 40, - "routePanelAnalyticsAgentViews": 41, - "routePanelAnalyticsPosts": 42, - "routePanelAnalyticsTopics": 43, - "routePanelGroups": 44, - "routePanelGroupsEdit": 45, - "routePanelGroupsEditPerms": 46, - "routePanelGroupsEditSubmit": 47, - "routePanelGroupsEditPermsSubmit": 48, - "routePanelGroupsCreateSubmit": 49, - "routePanelBackups": 50, - "routePanelLogsMod": 51, - "routePanelDebug": 52, - "routePanelDashboard": 53, - "routes.AccountEditCritical": 54, - "routeAccountEditCriticalSubmit": 55, - "routeAccountEditAvatar": 56, - "routeAccountEditAvatarSubmit": 57, - "routeAccountEditUsername": 58, - "routeAccountEditUsernameSubmit": 59, - "routeAccountEditEmail": 60, - "routeAccountEditEmailTokenSubmit": 61, - "routeProfile": 62, - "routes.BanUserSubmit": 63, - "routes.UnbanUser": 64, - "routes.ActivateUser": 65, - "routes.IPSearch": 66, - "routes.CreateTopicSubmit": 67, - "routes.EditTopicSubmit": 68, - "routes.DeleteTopicSubmit": 69, - "routes.StickTopicSubmit": 70, - "routes.UnstickTopicSubmit": 71, - "routes.LockTopicSubmit": 72, - "routes.UnlockTopicSubmit": 73, - "routes.MoveTopicSubmit": 74, - "routeLikeTopicSubmit": 75, - "routes.ViewTopic": 76, - "routeCreateReplySubmit": 77, - "routes.ReplyEditSubmit": 78, - "routes.ReplyDeleteSubmit": 79, - "routeReplyLikeSubmit": 80, - "routeProfileReplyCreateSubmit": 81, - "routes.ProfileReplyEditSubmit": 82, - "routes.ProfileReplyDeleteSubmit": 83, - "routes.PollVote": 84, - "routes.PollResults": 85, - "routes.AccountLogin": 86, - "routes.AccountRegister": 87, - "routeLogout": 88, - "routes.AccountLoginSubmit": 89, - "routes.AccountRegisterSubmit": 90, - "routeDynamic": 91, - "routeUploads": 92, - "BadRoute": 93, + "routePanelAnalyticsSystems": 40, + "routePanelAnalyticsRouteViews": 41, + "routePanelAnalyticsAgentViews": 42, + "routePanelAnalyticsSystemViews": 43, + "routePanelAnalyticsPosts": 44, + "routePanelAnalyticsTopics": 45, + "routePanelGroups": 46, + "routePanelGroupsEdit": 47, + "routePanelGroupsEditPerms": 48, + "routePanelGroupsEditSubmit": 49, + "routePanelGroupsEditPermsSubmit": 50, + "routePanelGroupsCreateSubmit": 51, + "routePanelBackups": 52, + "routePanelLogsMod": 53, + "routePanelDebug": 54, + "routePanelDashboard": 55, + "routes.AccountEditCritical": 56, + "routeAccountEditCriticalSubmit": 57, + "routeAccountEditAvatar": 58, + "routeAccountEditAvatarSubmit": 59, + "routeAccountEditUsername": 60, + "routeAccountEditUsernameSubmit": 61, + "routeAccountEditEmail": 62, + "routeAccountEditEmailTokenSubmit": 63, + "routeProfile": 64, + "routes.BanUserSubmit": 65, + "routes.UnbanUser": 66, + "routes.ActivateUser": 67, + "routes.IPSearch": 68, + "routes.CreateTopicSubmit": 69, + "routes.EditTopicSubmit": 70, + "routes.DeleteTopicSubmit": 71, + "routes.StickTopicSubmit": 72, + "routes.UnstickTopicSubmit": 73, + "routes.LockTopicSubmit": 74, + "routes.UnlockTopicSubmit": 75, + "routes.MoveTopicSubmit": 76, + "routeLikeTopicSubmit": 77, + "routes.ViewTopic": 78, + "routeCreateReplySubmit": 79, + "routes.ReplyEditSubmit": 80, + "routes.ReplyDeleteSubmit": 81, + "routeReplyLikeSubmit": 82, + "routeProfileReplyCreateSubmit": 83, + "routes.ProfileReplyEditSubmit": 84, + "routes.ProfileReplyDeleteSubmit": 85, + "routes.PollVote": 86, + "routes.PollResults": 87, + "routes.AccountLogin": 88, + "routes.AccountRegister": 89, + "routeLogout": 90, + "routes.AccountLoginSubmit": 91, + "routes.AccountRegisterSubmit": 92, + "routeDynamic": 93, + "routeUploads": 94, + "BadRoute": 95, } var reverseRouteMapEnum = map[int]string{ 0: "routeAPI", @@ -250,60 +254,78 @@ var reverseRouteMapEnum = map[int]string{ 37: "routePanelAnalyticsViews", 38: "routePanelAnalyticsRoutes", 39: "routePanelAnalyticsAgents", - 40: "routePanelAnalyticsRouteViews", - 41: "routePanelAnalyticsAgentViews", - 42: "routePanelAnalyticsPosts", - 43: "routePanelAnalyticsTopics", - 44: "routePanelGroups", - 45: "routePanelGroupsEdit", - 46: "routePanelGroupsEditPerms", - 47: "routePanelGroupsEditSubmit", - 48: "routePanelGroupsEditPermsSubmit", - 49: "routePanelGroupsCreateSubmit", - 50: "routePanelBackups", - 51: "routePanelLogsMod", - 52: "routePanelDebug", - 53: "routePanelDashboard", - 54: "routes.AccountEditCritical", - 55: "routeAccountEditCriticalSubmit", - 56: "routeAccountEditAvatar", - 57: "routeAccountEditAvatarSubmit", - 58: "routeAccountEditUsername", - 59: "routeAccountEditUsernameSubmit", - 60: "routeAccountEditEmail", - 61: "routeAccountEditEmailTokenSubmit", - 62: "routeProfile", - 63: "routes.BanUserSubmit", - 64: "routes.UnbanUser", - 65: "routes.ActivateUser", - 66: "routes.IPSearch", - 67: "routes.CreateTopicSubmit", - 68: "routes.EditTopicSubmit", - 69: "routes.DeleteTopicSubmit", - 70: "routes.StickTopicSubmit", - 71: "routes.UnstickTopicSubmit", - 72: "routes.LockTopicSubmit", - 73: "routes.UnlockTopicSubmit", - 74: "routes.MoveTopicSubmit", - 75: "routeLikeTopicSubmit", - 76: "routes.ViewTopic", - 77: "routeCreateReplySubmit", - 78: "routes.ReplyEditSubmit", - 79: "routes.ReplyDeleteSubmit", - 80: "routeReplyLikeSubmit", - 81: "routeProfileReplyCreateSubmit", - 82: "routes.ProfileReplyEditSubmit", - 83: "routes.ProfileReplyDeleteSubmit", - 84: "routes.PollVote", - 85: "routes.PollResults", - 86: "routes.AccountLogin", - 87: "routes.AccountRegister", - 88: "routeLogout", - 89: "routes.AccountLoginSubmit", - 90: "routes.AccountRegisterSubmit", - 91: "routeDynamic", - 92: "routeUploads", - 93: "BadRoute", + 40: "routePanelAnalyticsSystems", + 41: "routePanelAnalyticsRouteViews", + 42: "routePanelAnalyticsAgentViews", + 43: "routePanelAnalyticsSystemViews", + 44: "routePanelAnalyticsPosts", + 45: "routePanelAnalyticsTopics", + 46: "routePanelGroups", + 47: "routePanelGroupsEdit", + 48: "routePanelGroupsEditPerms", + 49: "routePanelGroupsEditSubmit", + 50: "routePanelGroupsEditPermsSubmit", + 51: "routePanelGroupsCreateSubmit", + 52: "routePanelBackups", + 53: "routePanelLogsMod", + 54: "routePanelDebug", + 55: "routePanelDashboard", + 56: "routes.AccountEditCritical", + 57: "routeAccountEditCriticalSubmit", + 58: "routeAccountEditAvatar", + 59: "routeAccountEditAvatarSubmit", + 60: "routeAccountEditUsername", + 61: "routeAccountEditUsernameSubmit", + 62: "routeAccountEditEmail", + 63: "routeAccountEditEmailTokenSubmit", + 64: "routeProfile", + 65: "routes.BanUserSubmit", + 66: "routes.UnbanUser", + 67: "routes.ActivateUser", + 68: "routes.IPSearch", + 69: "routes.CreateTopicSubmit", + 70: "routes.EditTopicSubmit", + 71: "routes.DeleteTopicSubmit", + 72: "routes.StickTopicSubmit", + 73: "routes.UnstickTopicSubmit", + 74: "routes.LockTopicSubmit", + 75: "routes.UnlockTopicSubmit", + 76: "routes.MoveTopicSubmit", + 77: "routeLikeTopicSubmit", + 78: "routes.ViewTopic", + 79: "routeCreateReplySubmit", + 80: "routes.ReplyEditSubmit", + 81: "routes.ReplyDeleteSubmit", + 82: "routeReplyLikeSubmit", + 83: "routeProfileReplyCreateSubmit", + 84: "routes.ProfileReplyEditSubmit", + 85: "routes.ProfileReplyDeleteSubmit", + 86: "routes.PollVote", + 87: "routes.PollResults", + 88: "routes.AccountLogin", + 89: "routes.AccountRegister", + 90: "routeLogout", + 91: "routes.AccountLoginSubmit", + 92: "routes.AccountRegisterSubmit", + 93: "routeDynamic", + 94: "routeUploads", + 95: "BadRoute", +} +var osMapEnum = map[string]int{ + "unknown": 0, + "windows": 1, + "linux": 2, + "mac": 3, + "android": 4, + "iphone": 5, +} +var reverseOSMapEnum = map[int]string{ + 0: "unknown", + 1: "windows", + 2: "linux", + 3: "mac", + 4: "android", + 5: "iphone", } var agentMapEnum = map[string]int{ "unknown": 0, @@ -313,22 +335,27 @@ var agentMapEnum = map[string]int{ "safari": 4, "edge": 5, "internetexplorer": 6, - "androidchrome": 7, - "mobilesafari": 8, - "ucbrowser": 9, - "googlebot": 10, - "yandex": 11, - "bing": 12, - "baidu": 13, - "duckduckgo": 14, - "discord": 15, - "cloudflare": 16, - "uptimebot": 17, - "lynx": 18, - "blank": 19, - "malformed": 20, - "suspicious": 21, - "zgrab": 22, + "trident": 7, + "androidchrome": 8, + "mobilesafari": 9, + "samsung": 10, + "ucbrowser": 11, + "googlebot": 12, + "yandex": 13, + "bing": 14, + "baidu": 15, + "duckduckgo": 16, + "seznambot": 17, + "discord": 18, + "twitter": 19, + "cloudflare": 20, + "uptimebot": 21, + "discourse": 22, + "lynx": 23, + "blank": 24, + "malformed": 25, + "suspicious": 26, + "zgrab": 27, } var reverseAgentMapEnum = map[int]string{ 0: "unknown", @@ -338,31 +365,37 @@ var reverseAgentMapEnum = map[int]string{ 4: "safari", 5: "edge", 6: "internetexplorer", - 7: "androidchrome", - 8: "mobilesafari", - 9: "ucbrowser", - 10: "googlebot", - 11: "yandex", - 12: "bing", - 13: "baidu", - 14: "duckduckgo", - 15: "discord", - 16: "cloudflare", - 17: "uptimebot", - 18: "lynx", - 19: "blank", - 20: "malformed", - 21: "suspicious", - 22: "zgrab", + 7: "trident", + 8: "androidchrome", + 9: "mobilesafari", + 10: "samsung", + 11: "ucbrowser", + 12: "googlebot", + 13: "yandex", + 14: "bing", + 15: "baidu", + 16: "duckduckgo", + 17: "seznambot", + 18: "discord", + 19: "twitter", + 20: "cloudflare", + 21: "uptimebot", + 22: "discourse", + 23: "lynx", + 24: "blank", + 25: "malformed", + 26: "suspicious", + 27: "zgrab", } var markToAgent = map[string]string{ "OPR":"opera", "Chrome":"chrome", "Firefox":"firefox", "MSIE":"internetexplorer", - //"Trident":"internetexplorer", + "Trident":"trident", // Hack to support IE11 "Edge":"edge", "Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this + "SamsungBrowser":"samsung", "UCBrowser":"ucbrowser", "Google":"googlebot", @@ -372,9 +405,12 @@ var markToAgent = map[string]string{ "Baiduspider":"baidu", "bingbot":"bing", "BingPreview":"bing", + "SeznamBot":"seznambot", "CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots? "Uptimebot":"uptimebot", "Discordbot":"discord", + "Twitterbot":"twitter", + "Discourse":"discourse", "zgrab":"zgrab", } @@ -390,6 +426,8 @@ func init() { common.SetReverseRouteMapEnum(reverseRouteMapEnum) common.SetAgentMapEnum(agentMapEnum) common.SetReverseAgentMapEnum(reverseAgentMapEnum) + common.SetOSMapEnum(osMapEnum) + common.SetReverseOSMapEnum(reverseOSMapEnum) } type GenRouter struct { @@ -456,7 +494,7 @@ func (router *GenRouter) DumpRequest(req *http.Request) { func (router *GenRouter) SuspiciousRequest(req *http.Request) { log.Print("Suspicious Request") router.DumpRequest(req) - common.AgentViewCounter.Bump(21) + common.AgentViewCounter.Bump(26) } // TODO: Pass the default route or config struct to the router rather than accessing it via a package global @@ -484,7 +522,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { w.Write([]byte("")) log.Print("Malformed Request") router.DumpRequest(req) - common.AgentViewCounter.Bump(20) + common.AgentViewCounter.Bump(25) return } @@ -512,19 +550,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if common.Dev.SuperDebug { log.Print("before routes.StaticFile") - log.Print("Method: ", req.Method) - for key, value := range req.Header { - for _, vvalue := range value { - log.Print("Header '" + key + "': " + vvalue + "!!") - } - } - log.Print("prefix: ", prefix) - log.Print("req.Host: ", req.Host) - log.Print("req.URL.Path: ", req.URL.Path) - log.Print("req.URL.RawQuery: ", req.URL.RawQuery) - log.Print("extraData: ", extraData) - log.Print("req.Referer(): ", req.Referer()) - log.Print("req.RemoteAddr: ", req.RemoteAddr) + router.DumpRequest(req) } if prefix == "/static" { @@ -544,12 +570,24 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // TODO: Use a more efficient detector instead of smashing every possible combination in ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another if ua == "" { - common.AgentViewCounter.Bump(19) + common.AgentViewCounter.Bump(24) if common.Dev.DebugMode { log.Print("Blank UA: ", req.UserAgent()) router.DumpRequest(req) } } else { + var runeEquals = func(a []rune, b []rune) bool { + if len(a) != len(b) { + return false + } + for i, item := range a { + if item != b[i] { + return false + } + } + return true + } + // WIP UA Parser var indices []int var items []string @@ -557,10 +595,20 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { for index, item := range ua { if (item > 64 && item < 91) || (item > 96 && item < 123) { buffer = append(buffer, item) - } else if len(buffer) != 0 { - items = append(items, string(buffer)) - indices = append(indices, index - 1) - buffer = buffer[:0] + } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && runeEquals(buffer,[]rune("http"))) || item == ',' || item == '/' { + if len(buffer) != 0 { + items = append(items, string(buffer)) + indices = append(indices, index - 1) + buffer = buffer[:0] + } + } else { + // TODO: Test this + items = items[:0] + indices = indices[:0] + router.SuspiciousRequest(req) + log.Print("UA Buffer: ", buffer) + log.Print("UA Buffer String: ", string(buffer)) + break } } @@ -575,19 +623,47 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } } + if common.Dev.SuperDebug { + log.Print("parsed agent: ", agent) + } - if common.Dev.DebugMode { - log.Print("parsed agent: ",agent) + var os string + for _, mark := range items { + switch(mark) { + case "Windows": + os = "windows" + case "Linux": + os = "linux" + case "Mac": + os = "mac" + case "iPhone": + os = "iphone" + case "Android": + os = "android" + } + } + if os == "" { + os = "unknown" + } + if common.Dev.SuperDebug { + log.Print("os: ", os) + log.Printf("items: %+v\n",items) } // Special handling switch(agent) { case "chrome": - for _, mark := range items { - if mark == "Android" { - agent = "androidchrome" - break - } + if os == "android" { + agent = "androidchrome" + } + case "safari": + if os == "iphone" { + agent = "mobilesafari" + } + case "trident": + // Hack to support IE11, change this after we start logging versions + if strings.Contains(ua,"rv:11") { + agent = "internetexplorer" } case "zgrab": router.SuspiciousRequest(req) @@ -602,6 +678,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } else { common.AgentViewCounter.Bump(agentMapEnum[agent]) } + common.OSViewCounter.Bump(osMapEnum[os]) } // Deal with the session stuff, etc. @@ -925,12 +1002,24 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { common.RouteViewCounter.Bump(39) err = routePanelAnalyticsAgents(w,req,user) - case "/panel/analytics/route/": + case "/panel/analytics/systems/": + err = common.ParseForm(w,req,user) + if err != nil { + router.handleError(err,w,req,user) + return + } + common.RouteViewCounter.Bump(40) + err = routePanelAnalyticsSystems(w,req,user) + case "/panel/analytics/route/": + common.RouteViewCounter.Bump(41) err = routePanelAnalyticsRouteViews(w,req,user,extraData) case "/panel/analytics/agent/": - common.RouteViewCounter.Bump(41) + common.RouteViewCounter.Bump(42) err = routePanelAnalyticsAgentViews(w,req,user,extraData) + case "/panel/analytics/system/": + common.RouteViewCounter.Bump(43) + err = routePanelAnalyticsSystemViews(w,req,user,extraData) case "/panel/analytics/posts/": err = common.ParseForm(w,req,user) if err != nil { @@ -938,7 +1027,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(42) + common.RouteViewCounter.Bump(44) err = routePanelAnalyticsPosts(w,req,user) case "/panel/analytics/topics/": err = common.ParseForm(w,req,user) @@ -947,16 +1036,16 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(43) + common.RouteViewCounter.Bump(45) err = routePanelAnalyticsTopics(w,req,user) case "/panel/groups/": - common.RouteViewCounter.Bump(44) + common.RouteViewCounter.Bump(46) err = routePanelGroups(w,req,user) case "/panel/groups/edit/": - common.RouteViewCounter.Bump(45) + common.RouteViewCounter.Bump(47) err = routePanelGroupsEdit(w,req,user,extraData) case "/panel/groups/edit/perms/": - common.RouteViewCounter.Bump(46) + common.RouteViewCounter.Bump(48) err = routePanelGroupsEditPerms(w,req,user,extraData) case "/panel/groups/edit/submit/": err = common.NoSessionMismatch(w,req,user) @@ -965,7 +1054,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(47) + common.RouteViewCounter.Bump(49) err = routePanelGroupsEditSubmit(w,req,user,extraData) case "/panel/groups/edit/perms/submit/": err = common.NoSessionMismatch(w,req,user) @@ -974,7 +1063,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(48) + common.RouteViewCounter.Bump(50) err = routePanelGroupsEditPermsSubmit(w,req,user,extraData) case "/panel/groups/create/": err = common.NoSessionMismatch(w,req,user) @@ -983,7 +1072,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(49) + common.RouteViewCounter.Bump(51) err = routePanelGroupsCreateSubmit(w,req,user) case "/panel/backups/": err = common.SuperAdminOnly(w,req,user) @@ -992,10 +1081,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(50) + common.RouteViewCounter.Bump(52) err = routePanelBackups(w,req,user,extraData) case "/panel/logs/mod/": - common.RouteViewCounter.Bump(51) + common.RouteViewCounter.Bump(53) err = routePanelLogsMod(w,req,user) case "/panel/debug/": err = common.AdminOnly(w,req,user) @@ -1004,10 +1093,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(52) + common.RouteViewCounter.Bump(54) err = routePanelDebug(w,req,user) default: - common.RouteViewCounter.Bump(53) + common.RouteViewCounter.Bump(55) err = routePanelDashboard(w,req,user) } if err != nil { @@ -1022,7 +1111,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(54) + common.RouteViewCounter.Bump(56) err = routes.AccountEditCritical(w,req,user) case "/user/edit/critical/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1037,7 +1126,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(55) + common.RouteViewCounter.Bump(57) err = routeAccountEditCriticalSubmit(w,req,user) case "/user/edit/avatar/": err = common.MemberOnly(w,req,user) @@ -1046,7 +1135,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(56) + common.RouteViewCounter.Bump(58) err = routeAccountEditAvatar(w,req,user) case "/user/edit/avatar/submit/": err = common.MemberOnly(w,req,user) @@ -1066,7 +1155,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(57) + common.RouteViewCounter.Bump(59) err = routeAccountEditAvatarSubmit(w,req,user) case "/user/edit/username/": err = common.MemberOnly(w,req,user) @@ -1075,7 +1164,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(58) + common.RouteViewCounter.Bump(60) err = routeAccountEditUsername(w,req,user) case "/user/edit/username/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1090,7 +1179,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(59) + common.RouteViewCounter.Bump(61) err = routeAccountEditUsernameSubmit(w,req,user) case "/user/edit/email/": err = common.MemberOnly(w,req,user) @@ -1099,7 +1188,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(60) + common.RouteViewCounter.Bump(62) err = routeAccountEditEmail(w,req,user) case "/user/edit/token/": err = common.NoSessionMismatch(w,req,user) @@ -1114,11 +1203,11 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(61) + common.RouteViewCounter.Bump(63) err = routeAccountEditEmailTokenSubmit(w,req,user,extraData) default: req.URL.Path += extraData - common.RouteViewCounter.Bump(62) + common.RouteViewCounter.Bump(64) err = routeProfile(w,req,user) } if err != nil { @@ -1139,7 +1228,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(63) + common.RouteViewCounter.Bump(65) err = routes.BanUserSubmit(w,req,user,extraData) case "/users/unban/": err = common.NoSessionMismatch(w,req,user) @@ -1154,7 +1243,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(64) + common.RouteViewCounter.Bump(66) err = routes.UnbanUser(w,req,user,extraData) case "/users/activate/": err = common.NoSessionMismatch(w,req,user) @@ -1169,7 +1258,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(65) + common.RouteViewCounter.Bump(67) err = routes.ActivateUser(w,req,user,extraData) case "/users/ips/": err = common.MemberOnly(w,req,user) @@ -1178,7 +1267,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(66) + common.RouteViewCounter.Bump(68) err = routes.IPSearch(w,req,user) } if err != nil { @@ -1204,7 +1293,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(67) + common.RouteViewCounter.Bump(69) err = routes.CreateTopicSubmit(w,req,user) case "/topic/edit/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1219,7 +1308,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(68) + common.RouteViewCounter.Bump(70) err = routes.EditTopicSubmit(w,req,user,extraData) case "/topic/delete/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1235,7 +1324,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } req.URL.Path += extraData - common.RouteViewCounter.Bump(69) + common.RouteViewCounter.Bump(71) err = routes.DeleteTopicSubmit(w,req,user) case "/topic/stick/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1250,7 +1339,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(70) + common.RouteViewCounter.Bump(72) err = routes.StickTopicSubmit(w,req,user,extraData) case "/topic/unstick/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1265,7 +1354,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(71) + common.RouteViewCounter.Bump(73) err = routes.UnstickTopicSubmit(w,req,user,extraData) case "/topic/lock/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1281,7 +1370,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } req.URL.Path += extraData - common.RouteViewCounter.Bump(72) + common.RouteViewCounter.Bump(74) err = routes.LockTopicSubmit(w,req,user) case "/topic/unlock/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1296,7 +1385,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(73) + common.RouteViewCounter.Bump(75) err = routes.UnlockTopicSubmit(w,req,user,extraData) case "/topic/move/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1311,7 +1400,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(74) + common.RouteViewCounter.Bump(76) err = routes.MoveTopicSubmit(w,req,user,extraData) case "/topic/like/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1326,10 +1415,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(75) + common.RouteViewCounter.Bump(77) err = routeLikeTopicSubmit(w,req,user,extraData) default: - common.RouteViewCounter.Bump(76) + common.RouteViewCounter.Bump(78) err = routes.ViewTopic(w,req,user, extraData) } if err != nil { @@ -1355,7 +1444,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(77) + common.RouteViewCounter.Bump(79) err = routeCreateReplySubmit(w,req,user) case "/reply/edit/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1370,7 +1459,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(78) + common.RouteViewCounter.Bump(80) err = routes.ReplyEditSubmit(w,req,user,extraData) case "/reply/delete/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1385,7 +1474,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(79) + common.RouteViewCounter.Bump(81) err = routes.ReplyDeleteSubmit(w,req,user,extraData) case "/reply/like/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1400,7 +1489,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(80) + common.RouteViewCounter.Bump(82) err = routeReplyLikeSubmit(w,req,user,extraData) } if err != nil { @@ -1421,7 +1510,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(81) + common.RouteViewCounter.Bump(83) err = routeProfileReplyCreateSubmit(w,req,user) case "/profile/reply/edit/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1436,7 +1525,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(82) + common.RouteViewCounter.Bump(84) err = routes.ProfileReplyEditSubmit(w,req,user,extraData) case "/profile/reply/delete/submit/": err = common.NoSessionMismatch(w,req,user) @@ -1451,7 +1540,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(83) + common.RouteViewCounter.Bump(85) err = routes.ProfileReplyDeleteSubmit(w,req,user,extraData) } if err != nil { @@ -1472,10 +1561,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(84) + common.RouteViewCounter.Bump(86) err = routes.PollVote(w,req,user,extraData) case "/poll/results/": - common.RouteViewCounter.Bump(85) + common.RouteViewCounter.Bump(87) err = routes.PollResults(w,req,user,extraData) } if err != nil { @@ -1484,10 +1573,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { case "/accounts": switch(req.URL.Path) { case "/accounts/login/": - common.RouteViewCounter.Bump(86) + common.RouteViewCounter.Bump(88) err = routes.AccountLogin(w,req,user) case "/accounts/create/": - common.RouteViewCounter.Bump(87) + common.RouteViewCounter.Bump(89) err = routes.AccountRegister(w,req,user) case "/accounts/logout/": err = common.NoSessionMismatch(w,req,user) @@ -1502,7 +1591,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(88) + common.RouteViewCounter.Bump(90) err = routeLogout(w,req,user) case "/accounts/login/submit/": err = common.ParseForm(w,req,user) @@ -1511,7 +1600,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(89) + common.RouteViewCounter.Bump(91) err = routes.AccountLoginSubmit(w,req,user) case "/accounts/create/submit/": err = common.ParseForm(w,req,user) @@ -1520,7 +1609,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - common.RouteViewCounter.Bump(90) + common.RouteViewCounter.Bump(92) err = routes.AccountRegisterSubmit(w,req,user) } if err != nil { @@ -1537,7 +1626,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { common.NotFound(w,req) return } - common.RouteViewCounter.Bump(92) + common.RouteViewCounter.Bump(94) req.URL.Path += extraData // TODO: Find a way to propagate errors up from this? router.UploadHandler(w,req) // TODO: Count these views @@ -1580,7 +1669,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { router.RUnlock() if ok { - common.RouteViewCounter.Bump(91) // TODO: Be more specific about *which* dynamic route it is + common.RouteViewCounter.Bump(93) // TODO: Be more specific about *which* dynamic route it is req.URL.Path += extraData err = handle(w,req,user) if err != nil { @@ -1591,10 +1680,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // TODO: Log all bad routes for the admin to figure out where users are going wrong? lowerPath := strings.ToLower(req.URL.Path) - if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") { + if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") { router.SuspiciousRequest(req) } - common.RouteViewCounter.Bump(93) + common.RouteViewCounter.Bump(95) common.NotFound(w,req) } } diff --git a/langs/english.json b/langs/english.json index cc3ae649..007aafed 100644 --- a/langs/english.json +++ b/langs/english.json @@ -89,5 +89,45 @@ "panel_mod_logs":"Moderation Logs", "panel_admin_logs":"Administration Logs", "panel_debug":"Debug" + }, + + "UserAgents": { + "chrome": "Google Chrome", + "firefox":"Mozilla Firefox", + "opera":"Opera", + "safari":"Safari", + "edge": "Edge", + "internetexplorer":"MS Internet Explorer", + "trident":"Trident Engine", + "androidchrome":"Chrome for Android", + "mobilesafari":"Mobile Safari", + "samsung":"Samsung Browser", + "ucbrowser":"UCBrowser", + + "googlebot":"Googlebot", + "yandex":"Yandex", + "bing":"Bing", + "baidu":"Baidu", + "duckduckgo":"DuckDuckBot", + "seznambot":"SeznamBot", + "discord":"Discord", + "twitter":"Twitterbot", + "cloudflare":"Cloudflare Alwayson", + "uptimebot":"Uptimebot", + "discourse":"Discourse Forum Onebox", + "lynx":"Lynx", + + "zgrab":"Zgrab Application Scanner", + "suspicious":"Suspicious", + "unknown":"Unknown", + "blank":"Blank", + "malformed":"Malformed" + }, + "OperatingSystems": { + "windows": "Microsoft Windows", + "linux":"Linux", + "android": "Android", + "iphone":"iPhone", + "unknown":"Unknown" } } \ No newline at end of file diff --git a/main.go b/main.go index f171f810..96ebbfcd 100644 --- a/main.go +++ b/main.go @@ -105,6 +105,10 @@ func afterDBInit() (err error) { if err != nil { return err } + common.OSViewCounter, err = common.NewDefaultOSViewCounter() + if err != nil { + return err + } common.RouteViewCounter, err = common.NewDefaultRouteViewCounter() if err != nil { return err diff --git a/panel_routes.go b/panel_routes.go index 412e0098..474cef33 100644 --- a/panel_routes.go +++ b/panel_routes.go @@ -799,11 +799,14 @@ func routePanelAnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user if err != nil { return common.InternalError(err, w, r) } - log.Print("count: ", count) - log.Print("createdAt: ", createdAt) var unixCreatedAt = createdAt.Unix() - log.Print("unixCreatedAt: ", unixCreatedAt) + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("createdAt: ", createdAt) + log.Print("unixCreatedAt: ", unixCreatedAt) + } + for _, value := range labelList { if unixCreatedAt > value { viewMap[value] += count @@ -822,10 +825,98 @@ func routePanelAnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} log.Printf("graph: %+v\n", graph) - pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", html.EscapeString(agent), graph, timeRange.Range} + // ? Only allow valid agents? The problem with this is that agents wind up getting renamed and it would take a migration to get them all up to snuff + agent = html.EscapeString(agent) + friendlyAgent, ok := common.GetUserAgentPhrase(agent) + if !ok { + friendlyAgent = agent + } + + pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", agent, friendlyAgent, graph, timeRange.Range} return panelRenderTemplate("panel_analytics_agent_views", w, r, user, &pi) } +func routePanelAnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user common.User, system string) common.RouteError { + headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) + if ferr != nil { + return ferr + } + headerVars.Stylesheets = append(headerVars.Stylesheets, "chartist/chartist.min.css") + headerVars.Scripts = append(headerVars.Scripts, "chartist/chartist.min.js") + + timeRange, err := panelAnalyticsTimeRange(r.FormValue("timeRange")) + if err != nil { + return common.LocalError(err.Error(), w, r, user) + } + + var revLabelList []int64 + var labelList []int64 + var viewMap = make(map[int64]int64) + var currentTime = time.Now().Unix() + + for i := 1; i <= timeRange.Slices; i++ { + var label = currentTime - int64(i*timeRange.SliceWidth) + revLabelList = append(revLabelList, label) + viewMap[label] = 0 + } + for _, value := range revLabelList { + labelList = append(labelList, value) + } + + var viewList []int64 + log.Print("in routePanelAnalyticsSystemViews") + + acc := qgen.Builder.Accumulator() + // TODO: Verify the agent is valid + rows, err := acc.Select("viewchunks_systems").Columns("count, createdAt").Where("system = ?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(system) + if err != nil && err != ErrNoRows { + return common.InternalError(err, w, r) + } + defer rows.Close() + + for rows.Next() { + var count int64 + var createdAt time.Time + err := rows.Scan(&count, &createdAt) + if err != nil { + return common.InternalError(err, w, r) + } + + var unixCreatedAt = createdAt.Unix() + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("createdAt: ", createdAt) + log.Print("unixCreatedAt: ", unixCreatedAt) + } + + for _, value := range labelList { + if unixCreatedAt > value { + viewMap[value] += count + break + } + } + } + err = rows.Err() + if err != nil { + return common.InternalError(err, w, r) + } + + for _, value := range revLabelList { + viewList = append(viewList, viewMap[value]) + } + graph := common.PanelTimeGraph{Series: viewList, Labels: labelList} + log.Printf("graph: %+v\n", graph) + + system = html.EscapeString(system) + friendlySystem, ok := common.GetOSPhrase(system) + if !ok { + friendlySystem = system + } + + pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", system, friendlySystem, graph, timeRange.Range} + return panelRenderTemplate("panel_analytics_system_views", w, r, user, &pi) +} + func routePanelAnalyticsTopics(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) if ferr != nil { @@ -870,11 +961,14 @@ func routePanelAnalyticsTopics(w http.ResponseWriter, r *http.Request, user comm if err != nil { return common.InternalError(err, w, r) } - log.Print("count: ", count) - log.Print("createdAt: ", createdAt) var unixCreatedAt = createdAt.Unix() - log.Print("unixCreatedAt: ", unixCreatedAt) + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("createdAt: ", createdAt) + log.Print("unixCreatedAt: ", unixCreatedAt) + } + for _, value := range labelList { if unixCreatedAt > value { viewMap[value] += count @@ -943,11 +1037,14 @@ func routePanelAnalyticsPosts(w http.ResponseWriter, r *http.Request, user commo if err != nil { return common.InternalError(err, w, r) } - log.Print("count: ", count) - log.Print("createdAt: ", createdAt) var unixCreatedAt = createdAt.Unix() - log.Print("unixCreatedAt: ", unixCreatedAt) + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("createdAt: ", createdAt) + log.Print("unixCreatedAt: ", unixCreatedAt) + } + for _, value := range labelList { if unixCreatedAt > value { viewMap[value] += count @@ -999,8 +1096,10 @@ func routePanelAnalyticsRoutes(w http.ResponseWriter, r *http.Request, user comm return common.InternalError(err, w, r) } - log.Print("count: ", count) - log.Print("route: ", route) + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("route: ", route) + } routeMap[route] += count } err = rows.Err() @@ -1048,8 +1147,10 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm return common.InternalError(err, w, r) } - log.Print("count: ", count) - log.Print("agent: ", agent) + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("agent: ", agent) + } agentMap[agent] += count } err = rows.Err() @@ -1060,9 +1161,14 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm // TODO: Sort this slice var agentItems []common.PanelAnalyticsAgentsItem for agent, count := range agentMap { + aAgent, ok := common.GetUserAgentPhrase(agent) + if !ok { + aAgent = agent + } agentItems = append(agentItems, common.PanelAnalyticsAgentsItem{ - Agent: agent, - Count: count, + Agent: agent, + FriendlyAgent: aAgent, + Count: count, }) } @@ -1070,6 +1176,62 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm return panelRenderTemplate("panel_analytics_agents", w, r, user, &pi) } +func routePanelAnalyticsSystems(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { + headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) + if ferr != nil { + return ferr + } + var osMap = make(map[string]int) + + timeRange, err := panelAnalyticsTimeRange(r.FormValue("timeRange")) + if err != nil { + return common.LocalError(err.Error(), w, r, user) + } + + acc := qgen.Builder.Accumulator() + rows, err := acc.Select("viewchunks_systems").Columns("count, system").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query() + if err != nil && err != ErrNoRows { + return common.InternalError(err, w, r) + } + defer rows.Close() + + for rows.Next() { + var count int + var system string + err := rows.Scan(&count, &system) + if err != nil { + return common.InternalError(err, w, r) + } + + if common.Dev.DebugMode { + log.Print("count: ", count) + log.Print("system: ", system) + } + osMap[system] += count + } + err = rows.Err() + if err != nil { + return common.InternalError(err, w, r) + } + + // TODO: Sort this slice + var systemItems []common.PanelAnalyticsAgentsItem + for system, count := range osMap { + sSystem, ok := common.GetOSPhrase(system) + if !ok { + sSystem = system + } + systemItems = append(systemItems, common.PanelAnalyticsAgentsItem{ + Agent: system, + FriendlyAgent: sSystem, + Count: count, + }) + } + + pi := common.PanelAnalyticsAgentsPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", systemItems, timeRange.Range} + return panelRenderTemplate("panel_analytics_systems", w, r, user, &pi) +} + func routePanelSettings(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError { headerVars, stats, ferr := common.PanelUserCheck(w, r, &user) if ferr != nil { diff --git a/query_gen/tables.go b/query_gen/tables.go index 4b191fb3..6a9b433a 100644 --- a/query_gen/tables.go +++ b/query_gen/tables.go @@ -412,6 +412,15 @@ func createTables(adapter qgen.Adapter) error { []qgen.DBTableKey{}, ) + qgen.Install.CreateTable("viewchunks_systems", "", "", + []qgen.DBTableColumn{ + qgen.DBTableColumn{"count", "int", 0, false, false, "0"}, + qgen.DBTableColumn{"createdAt", "datetime", 0, false, false, ""}, + qgen.DBTableColumn{"system", "varchar", 200, false, false, ""}, // windows, android, bot, etc. + }, + []qgen.DBTableKey{}, + ) + /* qgen.Install.CreateTable("viewchunks_forums", "", "", []qgen.DBTableColumn{ diff --git a/router_gen/main.go b/router_gen/main.go index 6697a5c1..3f4f271c 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -19,6 +19,8 @@ type TmplVars struct { AllRouteMap map[string]int AllAgentNames []string AllAgentMap map[string]int + AllOSNames []string + AllOSMap map[string]int } func main() { @@ -158,6 +160,20 @@ func main() { mapIt("BadRoute") tmplVars.AllRouteNames = allRouteNames tmplVars.AllRouteMap = allRouteMap + + tmplVars.AllOSNames = []string{ + "unknown", + "windows", + "linux", + "mac", + "android", + "iphone", + } + tmplVars.AllOSMap = make(map[string]int) + for id, os := range tmplVars.AllOSNames { + tmplVars.AllOSMap[os] = id + } + tmplVars.AllAgentNames = []string{ "unknown", "firefox", @@ -166,9 +182,11 @@ func main() { "safari", "edge", "internetexplorer", + "trident", // Hack to support IE11 "androidchrome", - "mobilesafari", // Coming soon + "mobilesafari", + "samsung", "ucbrowser", "googlebot", @@ -176,9 +194,12 @@ func main() { "bing", "baidu", "duckduckgo", + "seznambot", "discord", + "twitter", "cloudflare", "uptimebot", + "discourse", "lynx", "blank", "malformed", @@ -219,6 +240,12 @@ var routeMapEnum = map[string]int{ {{range $index, $element := .AllRouteNames}} var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}} {{$index}}: "{{$element}}",{{end}} } +var osMapEnum = map[string]int{ {{range $index, $element := .AllOSNames}} + "{{$element}}": {{$index}},{{end}} +} +var reverseOSMapEnum = map[int]string{ {{range $index, $element := .AllOSNames}} + {{$index}}: "{{$element}}",{{end}} +} var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}} "{{$element}}": {{$index}},{{end}} } @@ -230,9 +257,10 @@ var markToAgent = map[string]string{ "Chrome":"chrome", "Firefox":"firefox", "MSIE":"internetexplorer", - //"Trident":"internetexplorer", + "Trident":"trident", // Hack to support IE11 "Edge":"edge", "Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this + "SamsungBrowser":"samsung", "UCBrowser":"ucbrowser", "Google":"googlebot", @@ -242,9 +270,12 @@ var markToAgent = map[string]string{ "Baiduspider":"baidu", "bingbot":"bing", "BingPreview":"bing", + "SeznamBot":"seznambot", "CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots? "Uptimebot":"uptimebot", "Discordbot":"discord", + "Twitterbot":"twitter", + "Discourse":"discourse", "zgrab":"zgrab", } @@ -260,6 +291,8 @@ func init() { common.SetReverseRouteMapEnum(reverseRouteMapEnum) common.SetAgentMapEnum(agentMapEnum) common.SetReverseAgentMapEnum(reverseAgentMapEnum) + common.SetOSMapEnum(osMapEnum) + common.SetReverseOSMapEnum(reverseOSMapEnum) } type GenRouter struct { @@ -382,19 +415,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if common.Dev.SuperDebug { log.Print("before routes.StaticFile") - log.Print("Method: ", req.Method) - for key, value := range req.Header { - for _, vvalue := range value { - log.Print("Header '" + key + "': " + vvalue + "!!") - } - } - log.Print("prefix: ", prefix) - log.Print("req.Host: ", req.Host) - log.Print("req.URL.Path: ", req.URL.Path) - log.Print("req.URL.RawQuery: ", req.URL.RawQuery) - log.Print("extraData: ", extraData) - log.Print("req.Referer(): ", req.Referer()) - log.Print("req.RemoteAddr: ", req.RemoteAddr) + router.DumpRequest(req) } if prefix == "/static" { @@ -420,6 +441,18 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { router.DumpRequest(req) } } else { + var runeEquals = func(a []rune, b []rune) bool { + if len(a) != len(b) { + return false + } + for i, item := range a { + if item != b[i] { + return false + } + } + return true + } + // WIP UA Parser var indices []int var items []string @@ -427,10 +460,20 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { for index, item := range ua { if (item > 64 && item < 91) || (item > 96 && item < 123) { buffer = append(buffer, item) - } else if len(buffer) != 0 { - items = append(items, string(buffer)) - indices = append(indices, index - 1) - buffer = buffer[:0] + } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && runeEquals(buffer,[]rune("http"))) || item == ',' || item == '/' { + if len(buffer) != 0 { + items = append(items, string(buffer)) + indices = append(indices, index - 1) + buffer = buffer[:0] + } + } else { + // TODO: Test this + items = items[:0] + indices = indices[:0] + router.SuspiciousRequest(req) + log.Print("UA Buffer: ", buffer) + log.Print("UA Buffer String: ", string(buffer)) + break } } @@ -445,19 +488,47 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } } + if common.Dev.SuperDebug { + log.Print("parsed agent: ", agent) + } - if common.Dev.DebugMode { - log.Print("parsed agent: ",agent) + var os string + for _, mark := range items { + switch(mark) { + case "Windows": + os = "windows" + case "Linux": + os = "linux" + case "Mac": + os = "mac" + case "iPhone": + os = "iphone" + case "Android": + os = "android" + } + } + if os == "" { + os = "unknown" + } + if common.Dev.SuperDebug { + log.Print("os: ", os) + log.Printf("items: %+v\n",items) } // Special handling switch(agent) { case "chrome": - for _, mark := range items { - if mark == "Android" { - agent = "androidchrome" - break - } + if os == "android" { + agent = "androidchrome" + } + case "safari": + if os == "iphone" { + agent = "mobilesafari" + } + case "trident": + // Hack to support IE11, change this after we start logging versions + if strings.Contains(ua,"rv:11") { + agent = "internetexplorer" } case "zgrab": router.SuspiciousRequest(req) @@ -472,6 +543,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } else { common.AgentViewCounter.Bump(agentMapEnum[agent]) } + common.OSViewCounter.Bump(osMapEnum[os]) } // Deal with the session stuff, etc. @@ -551,7 +623,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { // TODO: Log all bad routes for the admin to figure out where users are going wrong? lowerPath := strings.ToLower(req.URL.Path) - if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") { + if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") { router.SuspiciousRequest(req) } common.RouteViewCounter.Bump({{.AllRouteMap.BadRoute}}) diff --git a/router_gen/routes.go b/router_gen/routes.go index 812db555..ada460d0 100644 --- a/router_gen/routes.go +++ b/router_gen/routes.go @@ -166,8 +166,10 @@ func buildPanelRoutes() { View("routePanelAnalyticsViews", "/panel/analytics/views/").Before("ParseForm"), View("routePanelAnalyticsRoutes", "/panel/analytics/routes/").Before("ParseForm"), View("routePanelAnalyticsAgents", "/panel/analytics/agents/").Before("ParseForm"), + View("routePanelAnalyticsSystems", "/panel/analytics/systems/").Before("ParseForm"), View("routePanelAnalyticsRouteViews", "/panel/analytics/route/", "extraData"), View("routePanelAnalyticsAgentViews", "/panel/analytics/agent/", "extraData"), + View("routePanelAnalyticsSystemViews", "/panel/analytics/system/", "extraData"), View("routePanelAnalyticsPosts", "/panel/analytics/posts/").Before("ParseForm"), View("routePanelAnalyticsTopics", "/panel/analytics/topics/").Before("ParseForm"), diff --git a/schema/mssql/query_viewchunks_systems.sql b/schema/mssql/query_viewchunks_systems.sql new file mode 100644 index 00000000..09db462d --- /dev/null +++ b/schema/mssql/query_viewchunks_systems.sql @@ -0,0 +1,5 @@ +CREATE TABLE [viewchunks_systems] ( + [count] int DEFAULT 0 not null, + [createdAt] datetime not null, + [system] nvarchar (200) not null +); \ No newline at end of file diff --git a/schema/mysql/query_viewchunks_systems.sql b/schema/mysql/query_viewchunks_systems.sql new file mode 100644 index 00000000..f65c2fb5 --- /dev/null +++ b/schema/mysql/query_viewchunks_systems.sql @@ -0,0 +1,5 @@ +CREATE TABLE `viewchunks_systems` ( + `count` int DEFAULT 0 not null, + `createdAt` datetime not null, + `system` varchar(200) not null +); \ No newline at end of file diff --git a/schema/pgsql/query_viewchunks_systems.sql b/schema/pgsql/query_viewchunks_systems.sql new file mode 100644 index 00000000..7ae425a7 --- /dev/null +++ b/schema/pgsql/query_viewchunks_systems.sql @@ -0,0 +1,5 @@ +CREATE TABLE `viewchunks_systems` ( + `count` int DEFAULT 0 not null, + `createdAt` timestamp not null, + `system` varchar (200) not null +); \ No newline at end of file diff --git a/templates/panel-inner-menu.html b/templates/panel-inner-menu.html index cf144959..ff831c7c 100644 --- a/templates/panel-inner-menu.html +++ b/templates/panel-inner-menu.html @@ -41,6 +41,9 @@
Agents
+
+ Systems +
Crawlers
diff --git a/templates/panel_analytics_agent_views.html b/templates/panel_analytics_agent_views.html index 8186a913..9bcdef67 100644 --- a/templates/panel_analytics_agent_views.html +++ b/templates/panel_analytics_agent_views.html @@ -5,7 +5,7 @@
- {{.Agent}} Views + {{.FriendlyAgent}} Views + + + + + + +
+
+
+
+
+
+ + + +{{template "footer.html" . }} diff --git a/templates/panel_analytics_systems.html b/templates/panel_analytics_systems.html new file mode 100644 index 00000000..36997967 --- /dev/null +++ b/templates/panel_analytics_systems.html @@ -0,0 +1,29 @@ +{{template "header.html" . }} +
+{{template "panel-menu.html" . }} +
+
+
+
+ Operating Systems + +
+
+
+
+ {{range .ItemList}} +
+ {{.FriendlyAgent}} + {{.Count}} views +
+ {{end}} +
+
+
+{{template "footer.html" . }}