a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

We now have analytics for the operating systems used by users. 

Route view bumping logs have been moved from regular debug mode to super debug to reduce the amount of noise.
Added user friendly agent names.
Reduced the amount of logging outside of debug mode.
We now track Mobile Safari and the Samsung Browser.
We now track SeznamBot, TwitterBot, and Discourse's Bot.
We now track Trident.
UAs are now filtered to reduce the amount of bad activity.
Added more bad phrases for bad routes.

Added the viewchunks_systems table.
Began work on referrer tracking.
This commit is contained in:
Azareal 2018-02-04 08:15:20 +00:00
parent 017bce9c09
commit 2455e951aa
21 changed files with 945 additions and 302 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
common/common.go
View File

@ -130,3 +130,14 @@ func SetAgentMapEnum(ame map[string]int) {
func SetReverseAgentMapEnum(rame map[int]string) {
reverseAgentMapEnum = rame
}
var osMapEnum map[string]int
var reverseOSMapEnum map[int]string
func SetOSMapEnum(osme map[string]int) {
osMapEnum = osme
}
func SetReverseOSMapEnum(rosme map[int]string) {
reverseOSMapEnum = rosme
}

61
common/counters.go
View File

@ -11,6 +11,7 @@ import (
// Global counters
var GlobalViewCounter *DefaultViewCounter
var AgentViewCounter *DefaultAgentViewCounter
var OSViewCounter *DefaultOSViewCounter
var RouteViewCounter *DefaultRouteViewCounter
var PostCounter *DefaultPostCounter
var TopicCounter *DefaultTopicCounter
@ -222,6 +223,64 @@ func (counter *DefaultAgentViewCounter) Bump(agent int) {
counter.agentBuckets[agent].Unlock()
}
type DefaultOSViewCounter struct {
osBuckets []*RWMutexCounterBucket //[OSID]count
insert *sql.Stmt
}
func NewDefaultOSViewCounter() (*DefaultOSViewCounter, error) {
acc := qgen.Builder.Accumulator()
var osBuckets = make([]*RWMutexCounterBucket, len(osMapEnum))
for bucketID, _ := range osBuckets {
osBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
}
counter := &DefaultOSViewCounter{
osBuckets: osBuckets,
insert: acc.Insert("viewchunks_systems").Columns("count, createdAt, system").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
}
AddScheduledFifteenMinuteTask(counter.Tick)
//AddScheduledSecondTask(counter.Tick)
AddShutdownTask(counter.Tick)
return counter, acc.FirstError()
}
func (counter *DefaultOSViewCounter) Tick() error {
for osID, osBucket := range counter.osBuckets {
var count int
osBucket.RLock()
count = osBucket.counter
osBucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters?
osBucket.RUnlock()
err := counter.insertChunk(count, osID) // TODO: Bulk insert for speed?
if err != nil {
return err
}
}
return nil
}
func (counter *DefaultOSViewCounter) insertChunk(count int, os int) error {
if count == 0 {
return nil
}
var osName = reverseOSMapEnum[os]
debugLogf("Inserting a viewchunk with a count of %d for OS %s (%d)", count, osName, os)
_, err := counter.insert.Exec(count, osName)
return err
}
func (counter *DefaultOSViewCounter) Bump(os int) {
// TODO: Test this check
debugDetail("counter.osBuckets[", os, "]: ", counter.osBuckets[os])
if len(counter.osBuckets) <= os || os < 0 {
return
}
counter.osBuckets[os].Lock()
counter.osBuckets[os].counter++
counter.osBuckets[os].Unlock()
}
type DefaultRouteViewCounter struct {
routeBuckets []*RWMutexCounterBucket //[RouteID]count
insert *sql.Stmt
@ -271,7 +330,7 @@ func (counter *DefaultRouteViewCounter) insertChunk(count int, route int) error
func (counter *DefaultRouteViewCounter) Bump(route int) {
// TODO: Test this check
debugLog("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route])
debugDetail("counter.routeBuckets[", route, "]: ", counter.routeBuckets[route])
if len(counter.routeBuckets) <= route || route < 0 {
return
}

2
common/extend.go
View File

@ -99,8 +99,10 @@ var PreRenderHooks = map[string][]func(http.ResponseWriter, *http.Request, *User
"pre_render_panel_analytics_views": nil,
"pre_render_panel_analytics_routes": nil,
"pre_render_panel_analytics_agents": nil,
"pre_render_panel_analytics_systems": nil,
"pre_render_panel_analytics_route_views": nil,
"pre_render_panel_analytics_agent_views": nil,
"pre_render_panel_analytics_system_views": nil,
"pre_render_panel_settings": nil,
"pre_render_panel_setting": nil,
"pre_render_panel_word_filters": nil,

2
common/pages.go
View File

@ -191,6 +191,7 @@ type PanelAnalyticsRoutesPage struct {
type PanelAnalyticsAgentsItem struct {
Agent string
FriendlyAgent string
Count int
}
@ -223,6 +224,7 @@ type PanelAnalyticsAgentPage struct {
Stats PanelStats
Zone string
Agent string
FriendlyAgent string
PrimaryGraph PanelTimeGraph
TimeRange string
}

18
common/phrases.go
View File

@ -42,6 +42,8 @@ type LanguagePack struct {
SettingLabels map[string]string
PermPresets map[string]string
Accounts map[string]string // TODO: Apply these phrases in the software proper
UserAgents map[string]string
OperatingSystems map[string]string
Errors map[string]map[string]string // map[category]map[name]value
PageTitles map[string]string
}
@ -154,6 +156,22 @@ func GetAccountPhrase(name string) string {
return res
}
func GetUserAgentPhrase(name string) (string, bool) {
res, ok := currentLangPack.Load().(*LanguagePack).UserAgents[name]
if !ok {
return "", false
}
return res, true
}
func GetOSPhrase(name string) (string, bool) {
res, ok := currentLangPack.Load().(*LanguagePack).OperatingSystems[name]
if !ok {
return "", false
}
return res, true
}
// TODO: Does comma ok work with multi-dimensional maps?
func GetErrorPhrase(category string, name string) string {
res, ok := currentLangPack.Load().(*LanguagePack).Errors[category][name]

75
common/requests.go Normal file
View File

@ -0,0 +1,75 @@
package common
import (
"sync"
"sync/atomic"
)
// Add ReferrerItems here after they've had zero views for a while
var referrersToDelete = make(map[string]ReferrerDeletable)
type ReferrerDeletable struct {
item *ReferrerItem
scheduledAt int64 //unixtime
}
type ReferrerItem struct {
Counter int64
}
// ? We'll track referrer domains here rather than the exact URL they arrived from for now, we'll think about expanding later
// ? Referrers are fluid and ever-changing so we have to use string keys rather than 'enum' ints
type DefaultReferrerTracker struct {
odd map[string]*ReferrerItem
even map[string]*ReferrerItem
oddLock sync.RWMutex
evenLock sync.RWMutex
}
func NewDefaultReferrerTracker() *DefaultReferrerTracker {
return &DefaultReferrerTracker{
odd: make(map[string]*ReferrerItem),
even: make(map[string]*ReferrerItem),
}
}
func (ref *DefaultReferrerTracker) Tick() (err error) {
for _, del := range referrersToDelete {
_ = del
// TODO: Calculate the gap between now and the times they were scheduled
}
// TODO: Run the queries and schedule zero view refs for deletion from memory
return nil
}
func (ref *DefaultReferrerTracker) Bump(referrer string) {
if referrer == "" {
return
}
var refItem *ReferrerItem
// Slightly crude and rudimentary, but it should give a basic degree of sharding
if referrer[0]%2 == 0 {
ref.evenLock.RLock()
refItem = ref.even[referrer]
ref.evenLock.RUnlock()
if ref != nil {
atomic.AddInt64(&refItem.Counter, 1)
} else {
ref.evenLock.Lock()
ref.even[referrer] = &ReferrerItem{Counter: 1}
ref.evenLock.Unlock()
}
} else {
ref.oddLock.RLock()
refItem = ref.odd[referrer]
ref.oddLock.RUnlock()
if ref != nil {
atomic.AddInt64(&refItem.Counter, 1)
} else {
ref.oddLock.Lock()
ref.odd[referrer] = &ReferrerItem{Counter: 1}
ref.oddLock.Unlock()
}
}
}

525
gen_router.go
View File

@ -56,8 +56,10 @@ var RouteMap = map[string]interface{}{
"routePanelAnalyticsViews": routePanelAnalyticsViews,
"routePanelAnalyticsRoutes": routePanelAnalyticsRoutes,
"routePanelAnalyticsAgents": routePanelAnalyticsAgents,
"routePanelAnalyticsSystems": routePanelAnalyticsSystems,
"routePanelAnalyticsRouteViews": routePanelAnalyticsRouteViews,
"routePanelAnalyticsAgentViews": routePanelAnalyticsAgentViews,
"routePanelAnalyticsSystemViews": routePanelAnalyticsSystemViews,
"routePanelAnalyticsPosts": routePanelAnalyticsPosts,
"routePanelAnalyticsTopics": routePanelAnalyticsTopics,
"routePanelGroups": routePanelGroups,
@ -154,60 +156,62 @@ var routeMapEnum = map[string]int{
"routePanelAnalyticsViews": 37,
"routePanelAnalyticsRoutes": 38,
"routePanelAnalyticsAgents": 39,
"routePanelAnalyticsRouteViews": 40,
"routePanelAnalyticsAgentViews": 41,
"routePanelAnalyticsPosts": 42,
"routePanelAnalyticsTopics": 43,
"routePanelGroups": 44,
"routePanelGroupsEdit": 45,
"routePanelGroupsEditPerms": 46,
"routePanelGroupsEditSubmit": 47,
"routePanelGroupsEditPermsSubmit": 48,
"routePanelGroupsCreateSubmit": 49,
"routePanelBackups": 50,
"routePanelLogsMod": 51,
"routePanelDebug": 52,
"routePanelDashboard": 53,
"routes.AccountEditCritical": 54,
"routeAccountEditCriticalSubmit": 55,
"routeAccountEditAvatar": 56,
"routeAccountEditAvatarSubmit": 57,
"routeAccountEditUsername": 58,
"routeAccountEditUsernameSubmit": 59,
"routeAccountEditEmail": 60,
"routeAccountEditEmailTokenSubmit": 61,
"routeProfile": 62,
"routes.BanUserSubmit": 63,
"routes.UnbanUser": 64,
"routes.ActivateUser": 65,
"routes.IPSearch": 66,
"routes.CreateTopicSubmit": 67,
"routes.EditTopicSubmit": 68,
"routes.DeleteTopicSubmit": 69,
"routes.StickTopicSubmit": 70,
"routes.UnstickTopicSubmit": 71,
"routes.LockTopicSubmit": 72,
"routes.UnlockTopicSubmit": 73,
"routes.MoveTopicSubmit": 74,
"routeLikeTopicSubmit": 75,
"routes.ViewTopic": 76,
"routeCreateReplySubmit": 77,
"routes.ReplyEditSubmit": 78,
"routes.ReplyDeleteSubmit": 79,
"routeReplyLikeSubmit": 80,
"routeProfileReplyCreateSubmit": 81,
"routes.ProfileReplyEditSubmit": 82,
"routes.ProfileReplyDeleteSubmit": 83,
"routes.PollVote": 84,
"routes.PollResults": 85,
"routes.AccountLogin": 86,
"routes.AccountRegister": 87,
"routeLogout": 88,
"routes.AccountLoginSubmit": 89,
"routes.AccountRegisterSubmit": 90,
"routeDynamic": 91,
"routeUploads": 92,
"BadRoute": 93,
"routePanelAnalyticsSystems": 40,
"routePanelAnalyticsRouteViews": 41,
"routePanelAnalyticsAgentViews": 42,
"routePanelAnalyticsSystemViews": 43,
"routePanelAnalyticsPosts": 44,
"routePanelAnalyticsTopics": 45,
"routePanelGroups": 46,
"routePanelGroupsEdit": 47,
"routePanelGroupsEditPerms": 48,
"routePanelGroupsEditSubmit": 49,
"routePanelGroupsEditPermsSubmit": 50,
"routePanelGroupsCreateSubmit": 51,
"routePanelBackups": 52,
"routePanelLogsMod": 53,
"routePanelDebug": 54,
"routePanelDashboard": 55,
"routes.AccountEditCritical": 56,
"routeAccountEditCriticalSubmit": 57,
"routeAccountEditAvatar": 58,
"routeAccountEditAvatarSubmit": 59,
"routeAccountEditUsername": 60,
"routeAccountEditUsernameSubmit": 61,
"routeAccountEditEmail": 62,
"routeAccountEditEmailTokenSubmit": 63,
"routeProfile": 64,
"routes.BanUserSubmit": 65,
"routes.UnbanUser": 66,
"routes.ActivateUser": 67,
"routes.IPSearch": 68,
"routes.CreateTopicSubmit": 69,
"routes.EditTopicSubmit": 70,
"routes.DeleteTopicSubmit": 71,
"routes.StickTopicSubmit": 72,
"routes.UnstickTopicSubmit": 73,
"routes.LockTopicSubmit": 74,
"routes.UnlockTopicSubmit": 75,
"routes.MoveTopicSubmit": 76,
"routeLikeTopicSubmit": 77,
"routes.ViewTopic": 78,
"routeCreateReplySubmit": 79,
"routes.ReplyEditSubmit": 80,
"routes.ReplyDeleteSubmit": 81,
"routeReplyLikeSubmit": 82,
"routeProfileReplyCreateSubmit": 83,
"routes.ProfileReplyEditSubmit": 84,
"routes.ProfileReplyDeleteSubmit": 85,
"routes.PollVote": 86,
"routes.PollResults": 87,
"routes.AccountLogin": 88,
"routes.AccountRegister": 89,
"routeLogout": 90,
"routes.AccountLoginSubmit": 91,
"routes.AccountRegisterSubmit": 92,
"routeDynamic": 93,
"routeUploads": 94,
"BadRoute": 95,
}
var reverseRouteMapEnum = map[int]string{
0: "routeAPI",
@ -250,60 +254,78 @@ var reverseRouteMapEnum = map[int]string{
37: "routePanelAnalyticsViews",
38: "routePanelAnalyticsRoutes",
39: "routePanelAnalyticsAgents",
40: "routePanelAnalyticsRouteViews",
41: "routePanelAnalyticsAgentViews",
42: "routePanelAnalyticsPosts",
43: "routePanelAnalyticsTopics",
44: "routePanelGroups",
45: "routePanelGroupsEdit",
46: "routePanelGroupsEditPerms",
47: "routePanelGroupsEditSubmit",
48: "routePanelGroupsEditPermsSubmit",
49: "routePanelGroupsCreateSubmit",
50: "routePanelBackups",
51: "routePanelLogsMod",
52: "routePanelDebug",
53: "routePanelDashboard",
54: "routes.AccountEditCritical",
55: "routeAccountEditCriticalSubmit",
56: "routeAccountEditAvatar",
57: "routeAccountEditAvatarSubmit",
58: "routeAccountEditUsername",
59: "routeAccountEditUsernameSubmit",
60: "routeAccountEditEmail",
61: "routeAccountEditEmailTokenSubmit",
62: "routeProfile",
63: "routes.BanUserSubmit",
64: "routes.UnbanUser",
65: "routes.ActivateUser",
66: "routes.IPSearch",
67: "routes.CreateTopicSubmit",
68: "routes.EditTopicSubmit",
69: "routes.DeleteTopicSubmit",
70: "routes.StickTopicSubmit",
71: "routes.UnstickTopicSubmit",
72: "routes.LockTopicSubmit",
73: "routes.UnlockTopicSubmit",
74: "routes.MoveTopicSubmit",
75: "routeLikeTopicSubmit",
76: "routes.ViewTopic",
77: "routeCreateReplySubmit",
78: "routes.ReplyEditSubmit",
79: "routes.ReplyDeleteSubmit",
80: "routeReplyLikeSubmit",
81: "routeProfileReplyCreateSubmit",
82: "routes.ProfileReplyEditSubmit",
83: "routes.ProfileReplyDeleteSubmit",
84: "routes.PollVote",
85: "routes.PollResults",
86: "routes.AccountLogin",
87: "routes.AccountRegister",
88: "routeLogout",
89: "routes.AccountLoginSubmit",
90: "routes.AccountRegisterSubmit",
91: "routeDynamic",
92: "routeUploads",
93: "BadRoute",
40: "routePanelAnalyticsSystems",
41: "routePanelAnalyticsRouteViews",
42: "routePanelAnalyticsAgentViews",
43: "routePanelAnalyticsSystemViews",
44: "routePanelAnalyticsPosts",
45: "routePanelAnalyticsTopics",
46: "routePanelGroups",
47: "routePanelGroupsEdit",
48: "routePanelGroupsEditPerms",
49: "routePanelGroupsEditSubmit",
50: "routePanelGroupsEditPermsSubmit",
51: "routePanelGroupsCreateSubmit",
52: "routePanelBackups",
53: "routePanelLogsMod",
54: "routePanelDebug",
55: "routePanelDashboard",
56: "routes.AccountEditCritical",
57: "routeAccountEditCriticalSubmit",
58: "routeAccountEditAvatar",
59: "routeAccountEditAvatarSubmit",
60: "routeAccountEditUsername",
61: "routeAccountEditUsernameSubmit",
62: "routeAccountEditEmail",
63: "routeAccountEditEmailTokenSubmit",
64: "routeProfile",
65: "routes.BanUserSubmit",
66: "routes.UnbanUser",
67: "routes.ActivateUser",
68: "routes.IPSearch",
69: "routes.CreateTopicSubmit",
70: "routes.EditTopicSubmit",
71: "routes.DeleteTopicSubmit",
72: "routes.StickTopicSubmit",
73: "routes.UnstickTopicSubmit",
74: "routes.LockTopicSubmit",
75: "routes.UnlockTopicSubmit",
76: "routes.MoveTopicSubmit",
77: "routeLikeTopicSubmit",
78: "routes.ViewTopic",
79: "routeCreateReplySubmit",
80: "routes.ReplyEditSubmit",
81: "routes.ReplyDeleteSubmit",
82: "routeReplyLikeSubmit",
83: "routeProfileReplyCreateSubmit",
84: "routes.ProfileReplyEditSubmit",
85: "routes.ProfileReplyDeleteSubmit",
86: "routes.PollVote",
87: "routes.PollResults",
88: "routes.AccountLogin",
89: "routes.AccountRegister",
90: "routeLogout",
91: "routes.AccountLoginSubmit",
92: "routes.AccountRegisterSubmit",
93: "routeDynamic",
94: "routeUploads",
95: "BadRoute",
}
var osMapEnum = map[string]int{
"unknown": 0,
"windows": 1,
"linux": 2,
"mac": 3,
"android": 4,
"iphone": 5,
}
var reverseOSMapEnum = map[int]string{
0: "unknown",
1: "windows",
2: "linux",
3: "mac",
4: "android",
5: "iphone",
}
var agentMapEnum = map[string]int{
"unknown": 0,
@ -313,22 +335,27 @@ var agentMapEnum = map[string]int{
"safari": 4,
"edge": 5,
"internetexplorer": 6,
"androidchrome": 7,
"mobilesafari": 8,
"ucbrowser": 9,
"googlebot": 10,
"yandex": 11,
"bing": 12,
"baidu": 13,
"duckduckgo": 14,
"discord": 15,
"cloudflare": 16,
"uptimebot": 17,
"lynx": 18,
"blank": 19,
"malformed": 20,
"suspicious": 21,
"zgrab": 22,
"trident": 7,
"androidchrome": 8,
"mobilesafari": 9,
"samsung": 10,
"ucbrowser": 11,
"googlebot": 12,
"yandex": 13,
"bing": 14,
"baidu": 15,
"duckduckgo": 16,
"seznambot": 17,
"discord": 18,
"twitter": 19,
"cloudflare": 20,
"uptimebot": 21,
"discourse": 22,
"lynx": 23,
"blank": 24,
"malformed": 25,
"suspicious": 26,
"zgrab": 27,
}
var reverseAgentMapEnum = map[int]string{
0: "unknown",
@ -338,31 +365,37 @@ var reverseAgentMapEnum = map[int]string{
4: "safari",
5: "edge",
6: "internetexplorer",
7: "androidchrome",
8: "mobilesafari",
9: "ucbrowser",
10: "googlebot",
11: "yandex",
12: "bing",
13: "baidu",
14: "duckduckgo",
15: "discord",
16: "cloudflare",
17: "uptimebot",
18: "lynx",
19: "blank",
20: "malformed",
21: "suspicious",
22: "zgrab",
7: "trident",
8: "androidchrome",
9: "mobilesafari",
10: "samsung",
11: "ucbrowser",
12: "googlebot",
13: "yandex",
14: "bing",
15: "baidu",
16: "duckduckgo",
17: "seznambot",
18: "discord",
19: "twitter",
20: "cloudflare",
21: "uptimebot",
22: "discourse",
23: "lynx",
24: "blank",
25: "malformed",
26: "suspicious",
27: "zgrab",
}
var markToAgent = map[string]string{
"OPR":"opera",
"Chrome":"chrome",
"Firefox":"firefox",
"MSIE":"internetexplorer",
//"Trident":"internetexplorer",
"Trident":"trident", // Hack to support IE11
"Edge":"edge",
"Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this
"SamsungBrowser":"samsung",
"UCBrowser":"ucbrowser",
"Google":"googlebot",
@ -372,9 +405,12 @@ var markToAgent = map[string]string{
"Baiduspider":"baidu",
"bingbot":"bing",
"BingPreview":"bing",
"SeznamBot":"seznambot",
"CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots?
"Uptimebot":"uptimebot",
"Discordbot":"discord",
"Twitterbot":"twitter",
"Discourse":"discourse",
"zgrab":"zgrab",
}
@ -390,6 +426,8 @@ func init() {
common.SetReverseRouteMapEnum(reverseRouteMapEnum)
common.SetAgentMapEnum(agentMapEnum)
common.SetReverseAgentMapEnum(reverseAgentMapEnum)
common.SetOSMapEnum(osMapEnum)
common.SetReverseOSMapEnum(reverseOSMapEnum)
}
type GenRouter struct {
@ -456,7 +494,7 @@ func (router *GenRouter) DumpRequest(req *http.Request) {
func (router *GenRouter) SuspiciousRequest(req *http.Request) {
log.Print("Suspicious Request")
router.DumpRequest(req)
common.AgentViewCounter.Bump(21)
common.AgentViewCounter.Bump(26)
}
// TODO: Pass the default route or config struct to the router rather than accessing it via a package global
@ -484,7 +522,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Write([]byte(""))
log.Print("Malformed Request")
router.DumpRequest(req)
common.AgentViewCounter.Bump(20)
common.AgentViewCounter.Bump(25)
return
}
@ -512,19 +550,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if common.Dev.SuperDebug {
log.Print("before routes.StaticFile")
log.Print("Method: ", req.Method)
for key, value := range req.Header {
for _, vvalue := range value {
log.Print("Header '" + key + "': " + vvalue + "!!")
}
}
log.Print("prefix: ", prefix)
log.Print("req.Host: ", req.Host)
log.Print("req.URL.Path: ", req.URL.Path)
log.Print("req.URL.RawQuery: ", req.URL.RawQuery)
log.Print("extraData: ", extraData)
log.Print("req.Referer(): ", req.Referer())
log.Print("req.RemoteAddr: ", req.RemoteAddr)
router.DumpRequest(req)
}
if prefix == "/static" {
@ -544,12 +570,24 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// TODO: Use a more efficient detector instead of smashing every possible combination in
ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another
if ua == "" {
common.AgentViewCounter.Bump(19)
common.AgentViewCounter.Bump(24)
if common.Dev.DebugMode {
log.Print("Blank UA: ", req.UserAgent())
router.DumpRequest(req)
}
} else {
var runeEquals = func(a []rune, b []rune) bool {
if len(a) != len(b) {
return false
}
for i, item := range a {
if item != b[i] {
return false
}
}
return true
}
// WIP UA Parser
var indices []int
var items []string
@ -557,11 +595,21 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
for index, item := range ua {
if (item > 64 && item < 91) || (item > 96 && item < 123) {
buffer = append(buffer, item)
} else if len(buffer) != 0 {
} else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && runeEquals(buffer,[]rune("http"))) || item == ',' || item == '/' {
if len(buffer) != 0 {
items = append(items, string(buffer))
indices = append(indices, index - 1)
buffer = buffer[:0]
}
} else {
// TODO: Test this
items = items[:0]
indices = indices[:0]
router.SuspiciousRequest(req)
log.Print("UA Buffer: ", buffer)
log.Print("UA Buffer String: ", string(buffer))
break
}
}
// Iterate over this in reverse as the real UA tends to be on the right side
@ -575,19 +623,47 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}
}
}
if common.Dev.SuperDebug {
log.Print("parsed agent: ", agent)
}
if common.Dev.DebugMode {
log.Print("parsed agent: ",agent)
var os string
for _, mark := range items {
switch(mark) {
case "Windows":
os = "windows"
case "Linux":
os = "linux"
case "Mac":
os = "mac"
case "iPhone":
os = "iphone"
case "Android":
os = "android"
}
}
if os == "" {
os = "unknown"
}
if common.Dev.SuperDebug {
log.Print("os: ", os)
log.Printf("items: %+v\n",items)
}
// Special handling
switch(agent) {
case "chrome":
for _, mark := range items {
if mark == "Android" {
if os == "android" {
agent = "androidchrome"
break
}
case "safari":
if os == "iphone" {
agent = "mobilesafari"
}
case "trident":
// Hack to support IE11, change this after we start logging versions
if strings.Contains(ua,"rv:11") {
agent = "internetexplorer"
}
case "zgrab":
router.SuspiciousRequest(req)
@ -602,6 +678,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} else {
common.AgentViewCounter.Bump(agentMapEnum[agent])
}
common.OSViewCounter.Bump(osMapEnum[os])
}
// Deal with the session stuff, etc.
@ -925,12 +1002,24 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
common.RouteViewCounter.Bump(39)
err = routePanelAnalyticsAgents(w,req,user)
case "/panel/analytics/route/":
case "/panel/analytics/systems/":
err = common.ParseForm(w,req,user)
if err != nil {
router.handleError(err,w,req,user)
return
}
common.RouteViewCounter.Bump(40)
err = routePanelAnalyticsSystems(w,req,user)
case "/panel/analytics/route/":
common.RouteViewCounter.Bump(41)
err = routePanelAnalyticsRouteViews(w,req,user,extraData)
case "/panel/analytics/agent/":
common.RouteViewCounter.Bump(41)
common.RouteViewCounter.Bump(42)
err = routePanelAnalyticsAgentViews(w,req,user,extraData)
case "/panel/analytics/system/":
common.RouteViewCounter.Bump(43)
err = routePanelAnalyticsSystemViews(w,req,user,extraData)
case "/panel/analytics/posts/":
err = common.ParseForm(w,req,user)
if err != nil {
@ -938,7 +1027,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(42)
common.RouteViewCounter.Bump(44)
err = routePanelAnalyticsPosts(w,req,user)
case "/panel/analytics/topics/":
err = common.ParseForm(w,req,user)
@ -947,16 +1036,16 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(43)
common.RouteViewCounter.Bump(45)
err = routePanelAnalyticsTopics(w,req,user)
case "/panel/groups/":
common.RouteViewCounter.Bump(44)
common.RouteViewCounter.Bump(46)
err = routePanelGroups(w,req,user)
case "/panel/groups/edit/":
common.RouteViewCounter.Bump(45)
common.RouteViewCounter.Bump(47)
err = routePanelGroupsEdit(w,req,user,extraData)
case "/panel/groups/edit/perms/":
common.RouteViewCounter.Bump(46)
common.RouteViewCounter.Bump(48)
err = routePanelGroupsEditPerms(w,req,user,extraData)
case "/panel/groups/edit/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -965,7 +1054,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(47)
common.RouteViewCounter.Bump(49)
err = routePanelGroupsEditSubmit(w,req,user,extraData)
case "/panel/groups/edit/perms/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -974,7 +1063,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(48)
common.RouteViewCounter.Bump(50)
err = routePanelGroupsEditPermsSubmit(w,req,user,extraData)
case "/panel/groups/create/":
err = common.NoSessionMismatch(w,req,user)
@ -983,7 +1072,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(49)
common.RouteViewCounter.Bump(51)
err = routePanelGroupsCreateSubmit(w,req,user)
case "/panel/backups/":
err = common.SuperAdminOnly(w,req,user)
@ -992,10 +1081,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(50)
common.RouteViewCounter.Bump(52)
err = routePanelBackups(w,req,user,extraData)
case "/panel/logs/mod/":
common.RouteViewCounter.Bump(51)
common.RouteViewCounter.Bump(53)
err = routePanelLogsMod(w,req,user)
case "/panel/debug/":
err = common.AdminOnly(w,req,user)
@ -1004,10 +1093,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(52)
common.RouteViewCounter.Bump(54)
err = routePanelDebug(w,req,user)
default:
common.RouteViewCounter.Bump(53)
common.RouteViewCounter.Bump(55)
err = routePanelDashboard(w,req,user)
}
if err != nil {
@ -1022,7 +1111,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(54)
common.RouteViewCounter.Bump(56)
err = routes.AccountEditCritical(w,req,user)
case "/user/edit/critical/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1037,7 +1126,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(55)
common.RouteViewCounter.Bump(57)
err = routeAccountEditCriticalSubmit(w,req,user)
case "/user/edit/avatar/":
err = common.MemberOnly(w,req,user)
@ -1046,7 +1135,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(56)
common.RouteViewCounter.Bump(58)
err = routeAccountEditAvatar(w,req,user)
case "/user/edit/avatar/submit/":
err = common.MemberOnly(w,req,user)
@ -1066,7 +1155,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(57)
common.RouteViewCounter.Bump(59)
err = routeAccountEditAvatarSubmit(w,req,user)
case "/user/edit/username/":
err = common.MemberOnly(w,req,user)
@ -1075,7 +1164,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(58)
common.RouteViewCounter.Bump(60)
err = routeAccountEditUsername(w,req,user)
case "/user/edit/username/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1090,7 +1179,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(59)
common.RouteViewCounter.Bump(61)
err = routeAccountEditUsernameSubmit(w,req,user)
case "/user/edit/email/":
err = common.MemberOnly(w,req,user)
@ -1099,7 +1188,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(60)
common.RouteViewCounter.Bump(62)
err = routeAccountEditEmail(w,req,user)
case "/user/edit/token/":
err = common.NoSessionMismatch(w,req,user)
@ -1114,11 +1203,11 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(61)
common.RouteViewCounter.Bump(63)
err = routeAccountEditEmailTokenSubmit(w,req,user,extraData)
default:
req.URL.Path += extraData
common.RouteViewCounter.Bump(62)
common.RouteViewCounter.Bump(64)
err = routeProfile(w,req,user)
}
if err != nil {
@ -1139,7 +1228,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(63)
common.RouteViewCounter.Bump(65)
err = routes.BanUserSubmit(w,req,user,extraData)
case "/users/unban/":
err = common.NoSessionMismatch(w,req,user)
@ -1154,7 +1243,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(64)
common.RouteViewCounter.Bump(66)
err = routes.UnbanUser(w,req,user,extraData)
case "/users/activate/":
err = common.NoSessionMismatch(w,req,user)
@ -1169,7 +1258,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(65)
common.RouteViewCounter.Bump(67)
err = routes.ActivateUser(w,req,user,extraData)
case "/users/ips/":
err = common.MemberOnly(w,req,user)
@ -1178,7 +1267,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(66)
common.RouteViewCounter.Bump(68)
err = routes.IPSearch(w,req,user)
}
if err != nil {
@ -1204,7 +1293,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(67)
common.RouteViewCounter.Bump(69)
err = routes.CreateTopicSubmit(w,req,user)
case "/topic/edit/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1219,7 +1308,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(68)
common.RouteViewCounter.Bump(70)
err = routes.EditTopicSubmit(w,req,user,extraData)
case "/topic/delete/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1235,7 +1324,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}
req.URL.Path += extraData
common.RouteViewCounter.Bump(69)
common.RouteViewCounter.Bump(71)
err = routes.DeleteTopicSubmit(w,req,user)
case "/topic/stick/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1250,7 +1339,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(70)
common.RouteViewCounter.Bump(72)
err = routes.StickTopicSubmit(w,req,user,extraData)
case "/topic/unstick/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1265,7 +1354,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(71)
common.RouteViewCounter.Bump(73)
err = routes.UnstickTopicSubmit(w,req,user,extraData)
case "/topic/lock/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1281,7 +1370,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}
req.URL.Path += extraData
common.RouteViewCounter.Bump(72)
common.RouteViewCounter.Bump(74)
err = routes.LockTopicSubmit(w,req,user)
case "/topic/unlock/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1296,7 +1385,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(73)
common.RouteViewCounter.Bump(75)
err = routes.UnlockTopicSubmit(w,req,user,extraData)
case "/topic/move/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1311,7 +1400,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(74)
common.RouteViewCounter.Bump(76)
err = routes.MoveTopicSubmit(w,req,user,extraData)
case "/topic/like/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1326,10 +1415,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(75)
common.RouteViewCounter.Bump(77)
err = routeLikeTopicSubmit(w,req,user,extraData)
default:
common.RouteViewCounter.Bump(76)
common.RouteViewCounter.Bump(78)
err = routes.ViewTopic(w,req,user, extraData)
}
if err != nil {
@ -1355,7 +1444,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(77)
common.RouteViewCounter.Bump(79)
err = routeCreateReplySubmit(w,req,user)
case "/reply/edit/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1370,7 +1459,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(78)
common.RouteViewCounter.Bump(80)
err = routes.ReplyEditSubmit(w,req,user,extraData)
case "/reply/delete/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1385,7 +1474,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(79)
common.RouteViewCounter.Bump(81)
err = routes.ReplyDeleteSubmit(w,req,user,extraData)
case "/reply/like/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1400,7 +1489,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(80)
common.RouteViewCounter.Bump(82)
err = routeReplyLikeSubmit(w,req,user,extraData)
}
if err != nil {
@ -1421,7 +1510,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(81)
common.RouteViewCounter.Bump(83)
err = routeProfileReplyCreateSubmit(w,req,user)
case "/profile/reply/edit/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1436,7 +1525,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(82)
common.RouteViewCounter.Bump(84)
err = routes.ProfileReplyEditSubmit(w,req,user,extraData)
case "/profile/reply/delete/submit/":
err = common.NoSessionMismatch(w,req,user)
@ -1451,7 +1540,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(83)
common.RouteViewCounter.Bump(85)
err = routes.ProfileReplyDeleteSubmit(w,req,user,extraData)
}
if err != nil {
@ -1472,10 +1561,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(84)
common.RouteViewCounter.Bump(86)
err = routes.PollVote(w,req,user,extraData)
case "/poll/results/":
common.RouteViewCounter.Bump(85)
common.RouteViewCounter.Bump(87)
err = routes.PollResults(w,req,user,extraData)
}
if err != nil {
@ -1484,10 +1573,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
case "/accounts":
switch(req.URL.Path) {
case "/accounts/login/":
common.RouteViewCounter.Bump(86)
common.RouteViewCounter.Bump(88)
err = routes.AccountLogin(w,req,user)
case "/accounts/create/":
common.RouteViewCounter.Bump(87)
common.RouteViewCounter.Bump(89)
err = routes.AccountRegister(w,req,user)
case "/accounts/logout/":
err = common.NoSessionMismatch(w,req,user)
@ -1502,7 +1591,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(88)
common.RouteViewCounter.Bump(90)
err = routeLogout(w,req,user)
case "/accounts/login/submit/":
err = common.ParseForm(w,req,user)
@ -1511,7 +1600,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(89)
common.RouteViewCounter.Bump(91)
err = routes.AccountLoginSubmit(w,req,user)
case "/accounts/create/submit/":
err = common.ParseForm(w,req,user)
@ -1520,7 +1609,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
common.RouteViewCounter.Bump(90)
common.RouteViewCounter.Bump(92)
err = routes.AccountRegisterSubmit(w,req,user)
}
if err != nil {
@ -1537,7 +1626,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
common.NotFound(w,req)
return
}
common.RouteViewCounter.Bump(92)
common.RouteViewCounter.Bump(94)
req.URL.Path += extraData
// TODO: Find a way to propagate errors up from this?
router.UploadHandler(w,req) // TODO: Count these views
@ -1580,7 +1669,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
router.RUnlock()
if ok {
common.RouteViewCounter.Bump(91) // TODO: Be more specific about *which* dynamic route it is
common.RouteViewCounter.Bump(93) // TODO: Be more specific about *which* dynamic route it is
req.URL.Path += extraData
err = handle(w,req,user)
if err != nil {
@ -1591,10 +1680,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// TODO: Log all bad routes for the admin to figure out where users are going wrong?
lowerPath := strings.ToLower(req.URL.Path)
if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") {
if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") {
router.SuspiciousRequest(req)
}
common.RouteViewCounter.Bump(93)
common.RouteViewCounter.Bump(95)
common.NotFound(w,req)
}
}

40
langs/english.json
View File

@ -89,5 +89,45 @@
"panel_mod_logs":"Moderation Logs",
"panel_admin_logs":"Administration Logs",
"panel_debug":"Debug"
},
"UserAgents": {
"chrome": "Google Chrome",
"firefox":"Mozilla Firefox",
"opera":"Opera",
"safari":"Safari",
"edge": "Edge",
"internetexplorer":"MS Internet Explorer",
"trident":"Trident Engine",
"androidchrome":"Chrome for Android",
"mobilesafari":"Mobile Safari",
"samsung":"Samsung Browser",
"ucbrowser":"UCBrowser",
"googlebot":"Googlebot",
"yandex":"Yandex",
"bing":"Bing",
"baidu":"Baidu",
"duckduckgo":"DuckDuckBot",
"seznambot":"SeznamBot",
"discord":"Discord",
"twitter":"Twitterbot",
"cloudflare":"Cloudflare Alwayson",
"uptimebot":"Uptimebot",
"discourse":"Discourse Forum Onebox",
"lynx":"Lynx",
"zgrab":"Zgrab Application Scanner",
"suspicious":"Suspicious",
"unknown":"Unknown",
"blank":"Blank",
"malformed":"Malformed"
},
"OperatingSystems": {
"windows": "Microsoft Windows",
"linux":"Linux",
"android": "Android",
"iphone":"iPhone",
"unknown":"Unknown"
}
}

4
main.go
View File

@ -105,6 +105,10 @@ func afterDBInit() (err error) {
if err != nil {
return err
}
common.OSViewCounter, err = common.NewDefaultOSViewCounter()
if err != nil {
return err
}
common.RouteViewCounter, err = common.NewDefaultRouteViewCounter()
if err != nil {
return err

176
panel_routes.go
View File

@ -799,11 +799,14 @@ func routePanelAnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user
if err != nil {
return common.InternalError(err, w, r)
}
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
var unixCreatedAt = createdAt.Unix()
if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
log.Print("unixCreatedAt: ", unixCreatedAt)
}
for _, value := range labelList {
if unixCreatedAt > value {
viewMap[value] += count
@ -822,10 +825,98 @@ func routePanelAnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user
graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
log.Printf("graph: %+v\n", graph)
pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", html.EscapeString(agent), graph, timeRange.Range}
// ? Only allow valid agents? The problem with this is that agents wind up getting renamed and it would take a migration to get them all up to snuff
agent = html.EscapeString(agent)
friendlyAgent, ok := common.GetUserAgentPhrase(agent)
if !ok {
friendlyAgent = agent
}
pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", agent, friendlyAgent, graph, timeRange.Range}
return panelRenderTemplate("panel_analytics_agent_views", w, r, user, &pi)
}
func routePanelAnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user common.User, system string) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
return ferr
}
headerVars.Stylesheets = append(headerVars.Stylesheets, "chartist/chartist.min.css")
headerVars.Scripts = append(headerVars.Scripts, "chartist/chartist.min.js")
timeRange, err := panelAnalyticsTimeRange(r.FormValue("timeRange"))
if err != nil {
return common.LocalError(err.Error(), w, r, user)
}
var revLabelList []int64
var labelList []int64
var viewMap = make(map[int64]int64)
var currentTime = time.Now().Unix()
for i := 1; i <= timeRange.Slices; i++ {
var label = currentTime - int64(i*timeRange.SliceWidth)
revLabelList = append(revLabelList, label)
viewMap[label] = 0
}
for _, value := range revLabelList {
labelList = append(labelList, value)
}
var viewList []int64
log.Print("in routePanelAnalyticsSystemViews")
acc := qgen.Builder.Accumulator()
// TODO: Verify the agent is valid
rows, err := acc.Select("viewchunks_systems").Columns("count, createdAt").Where("system = ?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(system)
if err != nil && err != ErrNoRows {
return common.InternalError(err, w, r)
}
defer rows.Close()
for rows.Next() {
var count int64
var createdAt time.Time
err := rows.Scan(&count, &createdAt)
if err != nil {
return common.InternalError(err, w, r)
}
var unixCreatedAt = createdAt.Unix()
if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
log.Print("unixCreatedAt: ", unixCreatedAt)
}
for _, value := range labelList {
if unixCreatedAt > value {
viewMap[value] += count
break
}
}
}
err = rows.Err()
if err != nil {
return common.InternalError(err, w, r)
}
for _, value := range revLabelList {
viewList = append(viewList, viewMap[value])
}
graph := common.PanelTimeGraph{Series: viewList, Labels: labelList}
log.Printf("graph: %+v\n", graph)
system = html.EscapeString(system)
friendlySystem, ok := common.GetOSPhrase(system)
if !ok {
friendlySystem = system
}
pi := common.PanelAnalyticsAgentPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", system, friendlySystem, graph, timeRange.Range}
return panelRenderTemplate("panel_analytics_system_views", w, r, user, &pi)
}
func routePanelAnalyticsTopics(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
@ -870,11 +961,14 @@ func routePanelAnalyticsTopics(w http.ResponseWriter, r *http.Request, user comm
if err != nil {
return common.InternalError(err, w, r)
}
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
var unixCreatedAt = createdAt.Unix()
if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
log.Print("unixCreatedAt: ", unixCreatedAt)
}
for _, value := range labelList {
if unixCreatedAt > value {
viewMap[value] += count
@ -943,11 +1037,14 @@ func routePanelAnalyticsPosts(w http.ResponseWriter, r *http.Request, user commo
if err != nil {
return common.InternalError(err, w, r)
}
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
var unixCreatedAt = createdAt.Unix()
if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("createdAt: ", createdAt)
log.Print("unixCreatedAt: ", unixCreatedAt)
}
for _, value := range labelList {
if unixCreatedAt > value {
viewMap[value] += count
@ -999,8 +1096,10 @@ func routePanelAnalyticsRoutes(w http.ResponseWriter, r *http.Request, user comm
return common.InternalError(err, w, r)
}
if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("route: ", route)
}
routeMap[route] += count
}
err = rows.Err()
@ -1048,8 +1147,10 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm
return common.InternalError(err, w, r)
}
if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("agent: ", agent)
}
agentMap[agent] += count
}
err = rows.Err()
@ -1060,8 +1161,13 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm
// TODO: Sort this slice
var agentItems []common.PanelAnalyticsAgentsItem
for agent, count := range agentMap {
aAgent, ok := common.GetUserAgentPhrase(agent)
if !ok {
aAgent = agent
}
agentItems = append(agentItems, common.PanelAnalyticsAgentsItem{
Agent: agent,
FriendlyAgent: aAgent,
Count: count,
})
}
@ -1070,6 +1176,62 @@ func routePanelAnalyticsAgents(w http.ResponseWriter, r *http.Request, user comm
return panelRenderTemplate("panel_analytics_agents", w, r, user, &pi)
}
func routePanelAnalyticsSystems(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {
return ferr
}
var osMap = make(map[string]int)
timeRange, err := panelAnalyticsTimeRange(r.FormValue("timeRange"))
if err != nil {
return common.LocalError(err.Error(), w, r, user)
}
acc := qgen.Builder.Accumulator()
rows, err := acc.Select("viewchunks_systems").Columns("count, system").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
if err != nil && err != ErrNoRows {
return common.InternalError(err, w, r)
}
defer rows.Close()
for rows.Next() {
var count int
var system string
err := rows.Scan(&count, &system)
if err != nil {
return common.InternalError(err, w, r)
}
if common.Dev.DebugMode {
log.Print("count: ", count)
log.Print("system: ", system)
}
osMap[system] += count
}
err = rows.Err()
if err != nil {
return common.InternalError(err, w, r)
}
// TODO: Sort this slice
var systemItems []common.PanelAnalyticsAgentsItem
for system, count := range osMap {
sSystem, ok := common.GetOSPhrase(system)
if !ok {
sSystem = system
}
systemItems = append(systemItems, common.PanelAnalyticsAgentsItem{
Agent: system,
FriendlyAgent: sSystem,
Count: count,
})
}
pi := common.PanelAnalyticsAgentsPage{common.GetTitlePhrase("panel_analytics"), user, headerVars, stats, "analytics", systemItems, timeRange.Range}
return panelRenderTemplate("panel_analytics_systems", w, r, user, &pi)
}
func routePanelSettings(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, stats, ferr := common.PanelUserCheck(w, r, &user)
if ferr != nil {

9
query_gen/tables.go
View File

@ -412,6 +412,15 @@ func createTables(adapter qgen.Adapter) error {
[]qgen.DBTableKey{},
)
qgen.Install.CreateTable("viewchunks_systems", "", "",
[]qgen.DBTableColumn{
qgen.DBTableColumn{"count", "int", 0, false, false, "0"},
qgen.DBTableColumn{"createdAt", "datetime", 0, false, false, ""},
qgen.DBTableColumn{"system", "varchar", 200, false, false, ""}, // windows, android, bot, etc.
},
[]qgen.DBTableKey{},
)
/*
qgen.Install.CreateTable("viewchunks_forums", "", "",
[]qgen.DBTableColumn{

116
router_gen/main.go
View File

@ -19,6 +19,8 @@ type TmplVars struct {
AllRouteMap map[string]int
AllAgentNames []string
AllAgentMap map[string]int
AllOSNames []string
AllOSMap map[string]int
}
func main() {
@ -158,6 +160,20 @@ func main() {
mapIt("BadRoute")
tmplVars.AllRouteNames = allRouteNames
tmplVars.AllRouteMap = allRouteMap
tmplVars.AllOSNames = []string{
"unknown",
"windows",
"linux",
"mac",
"android",
"iphone",
}
tmplVars.AllOSMap = make(map[string]int)
for id, os := range tmplVars.AllOSNames {
tmplVars.AllOSMap[os] = id
}
tmplVars.AllAgentNames = []string{
"unknown",
"firefox",
@ -166,9 +182,11 @@ func main() {
"safari",
"edge",
"internetexplorer",
"trident", // Hack to support IE11
"androidchrome",
"mobilesafari", // Coming soon
"mobilesafari",
"samsung",
"ucbrowser",
"googlebot",
@ -176,9 +194,12 @@ func main() {
"bing",
"baidu",
"duckduckgo",
"seznambot",
"discord",
"twitter",
"cloudflare",
"uptimebot",
"discourse",
"lynx",
"blank",
"malformed",
@ -219,6 +240,12 @@ var routeMapEnum = map[string]int{ {{range $index, $element := .AllRouteNames}}
var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}}
{{$index}}: "{{$element}}",{{end}}
}
var osMapEnum = map[string]int{ {{range $index, $element := .AllOSNames}}
"{{$element}}": {{$index}},{{end}}
}
var reverseOSMapEnum = map[int]string{ {{range $index, $element := .AllOSNames}}
{{$index}}: "{{$element}}",{{end}}
}
var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}}
"{{$element}}": {{$index}},{{end}}
}
@ -230,9 +257,10 @@ var markToAgent = map[string]string{
"Chrome":"chrome",
"Firefox":"firefox",
"MSIE":"internetexplorer",
//"Trident":"internetexplorer",
"Trident":"trident", // Hack to support IE11
"Edge":"edge",
"Lynx":"lynx", // There's a rare android variant of lynx which isn't covered by this
"SamsungBrowser":"samsung",
"UCBrowser":"ucbrowser",
"Google":"googlebot",
@ -242,9 +270,12 @@ var markToAgent = map[string]string{
"Baiduspider":"baidu",
"bingbot":"bing",
"BingPreview":"bing",
"SeznamBot":"seznambot",
"CloudFlare":"cloudflare", // Track alwayson specifically in case there are other bots?
"Uptimebot":"uptimebot",
"Discordbot":"discord",
"Twitterbot":"twitter",
"Discourse":"discourse",
"zgrab":"zgrab",
}
@ -260,6 +291,8 @@ func init() {
common.SetReverseRouteMapEnum(reverseRouteMapEnum)
common.SetAgentMapEnum(agentMapEnum)
common.SetReverseAgentMapEnum(reverseAgentMapEnum)
common.SetOSMapEnum(osMapEnum)
common.SetReverseOSMapEnum(reverseOSMapEnum)
}
type GenRouter struct {
@ -382,19 +415,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if common.Dev.SuperDebug {
log.Print("before routes.StaticFile")
log.Print("Method: ", req.Method)
for key, value := range req.Header {
for _, vvalue := range value {
log.Print("Header '" + key + "': " + vvalue + "!!")
}
}
log.Print("prefix: ", prefix)
log.Print("req.Host: ", req.Host)
log.Print("req.URL.Path: ", req.URL.Path)
log.Print("req.URL.RawQuery: ", req.URL.RawQuery)
log.Print("extraData: ", extraData)
log.Print("req.Referer(): ", req.Referer())
log.Print("req.RemoteAddr: ", req.RemoteAddr)
router.DumpRequest(req)
}
if prefix == "/static" {
@ -420,6 +441,18 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
router.DumpRequest(req)
}
} else {
var runeEquals = func(a []rune, b []rune) bool {
if len(a) != len(b) {
return false
}
for i, item := range a {
if item != b[i] {
return false
}
}
return true
}
// WIP UA Parser
var indices []int
var items []string
@ -427,11 +460,21 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
for index, item := range ua {
if (item > 64 && item < 91) || (item > 96 && item < 123) {
buffer = append(buffer, item)
} else if len(buffer) != 0 {
} else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == '.' || item == '+' || (item == ':' && runeEquals(buffer,[]rune("http"))) || item == ',' || item == '/' {
if len(buffer) != 0 {
items = append(items, string(buffer))
indices = append(indices, index - 1)
buffer = buffer[:0]
}
} else {
// TODO: Test this
items = items[:0]
indices = indices[:0]
router.SuspiciousRequest(req)
log.Print("UA Buffer: ", buffer)
log.Print("UA Buffer String: ", string(buffer))
break
}
}
// Iterate over this in reverse as the real UA tends to be on the right side
@ -445,19 +488,47 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}
}
}
if common.Dev.SuperDebug {
log.Print("parsed agent: ", agent)
}
if common.Dev.DebugMode {
log.Print("parsed agent: ",agent)
var os string
for _, mark := range items {
switch(mark) {
case "Windows":
os = "windows"
case "Linux":
os = "linux"
case "Mac":
os = "mac"
case "iPhone":
os = "iphone"
case "Android":
os = "android"
}
}
if os == "" {
os = "unknown"
}
if common.Dev.SuperDebug {
log.Print("os: ", os)
log.Printf("items: %+v\n",items)
}
// Special handling
switch(agent) {
case "chrome":
for _, mark := range items {
if mark == "Android" {
if os == "android" {
agent = "androidchrome"
break
}
case "safari":
if os == "iphone" {
agent = "mobilesafari"
}
case "trident":
// Hack to support IE11, change this after we start logging versions
if strings.Contains(ua,"rv:11") {
agent = "internetexplorer"
}
case "zgrab":
router.SuspiciousRequest(req)
@ -472,6 +543,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} else {
common.AgentViewCounter.Bump(agentMapEnum[agent])
}
common.OSViewCounter.Bump(osMapEnum[os])
}
// Deal with the session stuff, etc.
@ -551,7 +623,7 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// TODO: Log all bad routes for the admin to figure out where users are going wrong?
lowerPath := strings.ToLower(req.URL.Path)
if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") {
if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") {
router.SuspiciousRequest(req)
}
common.RouteViewCounter.Bump({{.AllRouteMap.BadRoute}})

2
router_gen/routes.go
View File

@ -166,8 +166,10 @@ func buildPanelRoutes() {
View("routePanelAnalyticsViews", "/panel/analytics/views/").Before("ParseForm"),
View("routePanelAnalyticsRoutes", "/panel/analytics/routes/").Before("ParseForm"),
View("routePanelAnalyticsAgents", "/panel/analytics/agents/").Before("ParseForm"),
View("routePanelAnalyticsSystems", "/panel/analytics/systems/").Before("ParseForm"),
View("routePanelAnalyticsRouteViews", "/panel/analytics/route/", "extraData"),
View("routePanelAnalyticsAgentViews", "/panel/analytics/agent/", "extraData"),
View("routePanelAnalyticsSystemViews", "/panel/analytics/system/", "extraData"),
View("routePanelAnalyticsPosts", "/panel/analytics/posts/").Before("ParseForm"),
View("routePanelAnalyticsTopics", "/panel/analytics/topics/").Before("ParseForm"),

5
schema/mssql/query_viewchunks_systems.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE [viewchunks_systems] (
[count] int DEFAULT 0 not null,
[createdAt] datetime not null,
[system] nvarchar (200) not null
);

5
schema/mysql/query_viewchunks_systems.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE `viewchunks_systems` (
`count` int DEFAULT 0 not null,
`createdAt` datetime not null,
`system` varchar(200) not null
);

5
schema/pgsql/query_viewchunks_systems.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE `viewchunks_systems` (
`count` int DEFAULT 0 not null,
`createdAt` timestamp not null,
`system` varchar (200) not null
);

3
templates/panel-inner-menu.html
View File

@ -41,6 +41,9 @@
<div class="rowitem passive submenu">
<a href="/panel/analytics/agents/">Agents</a>
</div>
<div class="rowitem passive submenu">
<a href="/panel/analytics/systems/">Systems</a>
</div>
<div class="rowitem passive submenu">
<a href="/panel/analytics/crawlers/">Crawlers</a>
</div>

2
templates/panel_analytics_agent_views.html
View File

@ -5,7 +5,7 @@
<form id="timeRangeForm" name="timeRangeForm" action="/panel/analytics/agent/{{.Agent}}" method="get">
<div class="colstack_item colstack_head">
<div class="rowitem">
<a>{{.Agent}} Views</a>
<a>{{.FriendlyAgent}} Views</a>
<select class="timeRangeSelector to_right" name="timeRange">
<option val="one-month"{{if eq .TimeRange "one-month"}} selected{{end}}>1 month</option>
<option val="two-days"{{if eq .TimeRange "two-days"}} selected{{end}}>2 days</option>

2
templates/panel_analytics_agents.html
View File

@ -19,7 +19,7 @@
<div id="panel_analytics_agents" class="colstack_item rowlist">
{{range .ItemList}}
<div class="rowitem panel_compactrow editable_parent">
<a href="/panel/analytics/agent/{{.Agent}}" class="panel_upshift">{{.Agent}}</a>
<a href="/panel/analytics/agent/{{.Agent}}" class="panel_upshift">{{.FriendlyAgent}}</a>
<span class="panel_compacttext to_right">{{.Count}} views</span>
</div>
{{end}}

51
templates/panel_analytics_system_views.html Normal file
View File

@ -0,0 +1,51 @@
{{template "header.html" . }}
<div class="colstack panel_stack">
{{template "panel-menu.html" . }}
<main id="panel_dashboard_right" class="colstack_right">
<form id="timeRangeForm" name="timeRangeForm" action="/panel/analytics/system/{{.Agent}}" method="get">
<div class="colstack_item colstack_head">
<div class="rowitem">
<a>{{.FriendlyAgent}} Views</a>
<select class="timeRangeSelector to_right" name="timeRange">
<option val="one-month"{{if eq .TimeRange "one-month"}} selected{{end}}>1 month</option>
<option val="two-days"{{if eq .TimeRange "two-days"}} selected{{end}}>2 days</option>
<option val="one-day"{{if eq .TimeRange "one-day"}} selected{{end}}>1 day</option>
<option val="twelve-hours"{{if eq .TimeRange "twelve-hours"}} selected{{end}}>12 hours</option>
<option val="six-hours"{{if eq .TimeRange "six-hours"}} selected{{end}}>6 hours</option>
</select>
</div>
</div>
</form>
<div id="panel_analytics_systems" class="colstack_graph_holder">
<div class="ct-chart"></div>
</div>
</main>
</div>
<script>
let labels = [];
let rawLabels = [{{range .PrimaryGraph.Labels}}
{{.}},{{end}}
];
for(const i in rawLabels) {
let date = new Date(rawLabels[i]*1000);
console.log("date: ", date);
let minutes = "0" + date.getMinutes();
let label = date.getHours() + ":" + minutes.substr(-2);
console.log("label:", label);
labels.push(label);
}
labels = labels.reverse()
let seriesData = [{{range .PrimaryGraph.Series}}
{{.}},{{end}}
];
seriesData = seriesData.reverse();
Chartist.Line('.ct-chart', {
labels: labels,
series: [seriesData],
}, {
height: '250px',
});
</script>
{{template "footer.html" . }}

29
templates/panel_analytics_systems.html Normal file
View File

@ -0,0 +1,29 @@
{{template "header.html" . }}
<div class="colstack panel_stack">
{{template "panel-menu.html" . }}
<main id="panel_dashboard_right" class="colstack_right">
<form id="timeRangeForm" name="timeRangeForm" action="/panel/analytics/systems/" method="get">
<div class="colstack_item colstack_head">
<div class="rowitem">
<a>Operating Systems</a>
<select class="timeRangeSelector to_right" name="timeRange">
<option val="one-month"{{if eq .TimeRange "one-month"}} selected{{end}}>1 month</option>
<option val="two-days"{{if eq .TimeRange "two-days"}} selected{{end}}>2 days</option>
<option val="one-day"{{if eq .TimeRange "one-day"}} selected{{end}}>1 day</option>
<option val="twelve-hours"{{if eq .TimeRange "twelve-hours"}} selected{{end}}>12 hours</option>
<option val="six-hours"{{if eq .TimeRange "six-hours"}} selected{{end}}>6 hours</option>
</select>
</div>
</div>
</form>
<div id="panel_analytics_systems" class="colstack_item rowlist">
{{range .ItemList}}
<div class="rowitem panel_compactrow editable_parent">
<a href="/panel/analytics/system/{{.Agent}}" class="panel_upshift">{{.FriendlyAgent}}</a>
<span class="panel_compacttext to_right">{{.Count}} views</span>
</div>
{{end}}
</div>
</main>
</div>
{{template "footer.html" . }}