From 64335cf1ef9463fe6a1dbe11e8a81f34f8a29a9d Mon Sep 17 00:00:00 2001 From: Azareal Date: Tue, 25 Feb 2020 14:25:50 +1000 Subject: [PATCH] optimise lang counter optimise dumprequest add more suspicious phrases capture more router gen errors --- common/counters/langs.go | 33 +++++++++-------------- gen_router.go | 46 +++++++++++++++----------------- router_gen/main.go | 57 +++++++++++++++++++++------------------- 3 files changed, 64 insertions(+), 72 deletions(-) diff --git a/common/counters/langs.go b/common/counters/langs.go index 987ad4d7..fe181884 100644 --- a/common/counters/langs.go +++ b/common/counters/langs.go @@ -2,6 +2,7 @@ package counters import ( "database/sql" + "sync/atomic" c "github.com/Azareal/Gosora/common" qgen "github.com/Azareal/Gosora/query_gen" @@ -12,7 +13,7 @@ var LangViewCounter *DefaultLangViewCounter var langCodes = []string{ "unknown", - "none", + "", "af", "ar", "az", @@ -98,26 +99,22 @@ var langCodes = []string{ } type DefaultLangViewCounter struct { - buckets []*RWMutexCounterBucket //[OSID]count + //buckets []*MutexCounterBucket //[OSID]count + buckets []int64 //[OSID]count codesToIndices map[string]int insert *sql.Stmt } func NewDefaultLangViewCounter(acc *qgen.Accumulator) (*DefaultLangViewCounter, error) { - langBuckets := make([]*RWMutexCounterBucket, len(langCodes)) - for bucketID, _ := range langBuckets { - langBuckets[bucketID] = &RWMutexCounterBucket{counter: 0} - } codesToIndices := make(map[string]int, len(langCodes)) for index, code := range langCodes { codesToIndices[code] = index } - co := &DefaultLangViewCounter{ - buckets: langBuckets, + buckets: make([]int64, len(langCodes)), codesToIndices: codesToIndices, - insert: acc.Insert("viewchunks_langs").Columns("count, createdAt, lang").Fields("?,UTC_TIMESTAMP(),?").Prepare(), + insert: acc.Insert("viewchunks_langs").Columns("count,createdAt,lang").Fields("?,UTC_TIMESTAMP(),?").Prepare(), } c.AddScheduledFifteenMinuteTask(co.Tick) @@ -127,13 +124,8 @@ func NewDefaultLangViewCounter(acc *qgen.Accumulator) (*DefaultLangViewCounter, } func (co *DefaultLangViewCounter) Tick() error { - for id, bucket := range co.buckets { - var count int - bucket.RLock() - count = bucket.counter - bucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters? - bucket.RUnlock() - + for id := 0; id < len(co.buckets); id++ { + count := atomic.SwapInt64(&co.buckets[id], 0) err := co.insertChunk(count, id) // TODO: Bulk insert for speed? if err != nil { return errors.Wrap(errors.WithStack(err), "langview counter") @@ -142,11 +134,14 @@ func (co *DefaultLangViewCounter) Tick() error { return nil } -func (co *DefaultLangViewCounter) insertChunk(count int, id int) error { +func (co *DefaultLangViewCounter) insertChunk(count int64, id int) error { if count == 0 { return nil } langCode := langCodes[id] + if langCode == "" { + langCode = "none" + } c.DebugLogf("Inserting a vchunk with a count of %d for lang %s (%d)", count, langCode, id) _, err := co.insert.Exec(count, langCode) return err @@ -166,9 +161,7 @@ func (co *DefaultLangViewCounter) Bump(langCode string) (validCode bool) { if len(co.buckets) <= id || id < 0 { return validCode } - co.buckets[id].Lock() - co.buckets[id].counter++ - co.buckets[id].Unlock() + atomic.AddInt64(&co.buckets[id], 1) return validCode } diff --git a/gen_router.go b/gen_router.go index 6e5bcd58..1688d543 100644 --- a/gen_router.go +++ b/gen_router.go @@ -742,10 +742,10 @@ func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *ht func (r *GenRouter) Handle(_ string, _ http.Handler) { } -func (r *GenRouter) HandleFunc(pattern string, handle func(http.ResponseWriter, *http.Request, c.User) c.RouteError) { +func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, c.User) c.RouteError) { r.Lock() defer r.Unlock() - r.extraRoutes[pattern] = handle + r.extraRoutes[pattern] = h } func (r *GenRouter) RemoveFunc(pattern string) error { @@ -770,11 +770,11 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) { r.requestLogger.Print(prepend + "\nUA: " + c.SanitiseSingleLine(req.UserAgent()) + "\n" + "Method: " + c.SanitiseSingleLine(req.Method) + "\n" + heads + - "req.Host: " + c.SanitiseSingleLine(req.Host) + "\n" + - "req.URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" + - "req.URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" + - "req.Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" + - "req.RemoteAddr: " + req.RemoteAddr + "\n") + "Host: " + c.SanitiseSingleLine(req.Host) + "\n" + + "URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" + + "URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" + + "Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" + + "RemoteAddr: " + req.RemoteAddr + "\n") } func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) { @@ -785,8 +785,8 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) { co.AgentViewCounter.Bump(29) } -func isLocalHost(host string) bool { - return host=="localhost" || host=="127.0.0.1" || host=="::1" +func isLocalHost(h string) bool { + return h=="localhost" || h=="127.0.0.1" || h=="::1" } // TODO: Pass the default path or config struct to the router rather than accessing it via a package global @@ -943,10 +943,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { var items []string var buffer []byte var os string - for _, item := range StringToBytes(ua) { - if (item > 64 && item < 91) || (item > 96 && item < 123) { - buffer = append(buffer, item) - } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == ':' || item == '.' || item == '+' || item == '~' || item == '@' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' { + for _, it := range StringToBytes(ua) { + if (it > 64 && it < 91) || (it > 96 && it < 123) { + buffer = append(buffer, it) + } else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == '_' || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' || (it == ':' && bytes.Equal(buffer,[]byte("http"))) || it == ',' || it == '/' { if len(buffer) != 0 { if len(buffer) > 2 { // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append @@ -972,7 +972,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } else { // TODO: Test this items = items[:0] - r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(item))+" in UA") + r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA") r.requestLogger.Print("UA Buffer: ", buffer) r.requestLogger.Print("UA Buffer String: ", string(buffer)) break @@ -1050,16 +1050,12 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { break } c.DebugDetail("llLang:", llLang) - if llLang == "" { - co.LangViewCounter.Bump("none") - } else { - validCode := co.LangViewCounter.Bump(llLang) - if !validCode { - r.DumpRequest(req,"Invalid ISO Code") - } + validCode := co.LangViewCounter.Bump(llLang) + if !validCode { + r.DumpRequest(req,"Invalid ISO Code") } } else { - co.LangViewCounter.Bump("none") + co.LangViewCounter.Bump("") } if !c.Config.RefNoTrack { @@ -1114,7 +1110,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { //c.StoppedServer("Profile end") } -func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix string, extraData string) c.RouteError { +func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix, extraData string) c.RouteError { var err c.RouteError switch(prefix) { case "/overview": @@ -2657,8 +2653,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c return handle(w,req,user) } - lowerPath := strings.ToLower(req.URL.Path) - if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") { + lp := strings.ToLower(req.URL.Path) + if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { r.SuspiciousRequest(req,"Bad Route") } else { r.DumpRequest(req,"Bad Route") diff --git a/router_gen/main.go b/router_gen/main.go index f1d1b5af..1d9f79df 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -441,10 +441,10 @@ func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *ht func (r *GenRouter) Handle(_ string, _ http.Handler) { } -func (r *GenRouter) HandleFunc(pattern string, handle func(http.ResponseWriter, *http.Request, c.User) c.RouteError) { +func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, c.User) c.RouteError) { r.Lock() defer r.Unlock() - r.extraRoutes[pattern] = handle + r.extraRoutes[pattern] = h } func (r *GenRouter) RemoveFunc(pattern string) error { @@ -469,11 +469,11 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) { r.requestLogger.Print(prepend + "\nUA: " + c.SanitiseSingleLine(req.UserAgent()) + "\n" + "Method: " + c.SanitiseSingleLine(req.Method) + "\n" + heads + - "req.Host: " + c.SanitiseSingleLine(req.Host) + "\n" + - "req.URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" + - "req.URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" + - "req.Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" + - "req.RemoteAddr: " + req.RemoteAddr + "\n") + "Host: " + c.SanitiseSingleLine(req.Host) + "\n" + + "URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" + + "URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" + + "Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" + + "RemoteAddr: " + req.RemoteAddr + "\n") } func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) { @@ -484,8 +484,8 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) { co.AgentViewCounter.Bump({{.AllAgentMap.suspicious}}) } -func isLocalHost(host string) bool { - return host=="localhost" || host=="127.0.0.1" || host=="::1" +func isLocalHost(h string) bool { + return h=="localhost" || h=="127.0.0.1" || h=="::1" } // TODO: Pass the default path or config struct to the router rather than accessing it via a package global @@ -619,6 +619,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { r.requestLogger.Print("before PreRoute") } + /*if c.Dev.QuicPort != 0 { + w.Header().Set("Alt-Svc", "quic=\":"+strconv.Itoa(c.Dev.QuicPort)+"\"; ma=2592000; v=\"44,43,39\", h3-23=\":"+strconv.Itoa(c.Dev.QuicPort)+"\"; ma=3600, h3-24=\":"+strconv.Itoa(c.Dev.QuicPort)+"\"; ma=3600, h2=\":443\"; ma=3600") + }*/ + // Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like. // TODO: Add a setting to disable this? // TODO: Use a more efficient detector instead of smashing every possible combination in @@ -638,10 +642,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { var items []string var buffer []byte var os string - for _, item := range StringToBytes(ua) { - if (item > 64 && item < 91) || (item > 96 && item < 123) { - buffer = append(buffer, item) - } else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == ':' || item == '.' || item == '+' || item == '~' || item == '@' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' { + for _, it := range StringToBytes(ua) { + if (it > 64 && it < 91) || (it > 96 && it < 123) { + buffer = append(buffer, it) + } else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == '_' || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' || (it == ':' && bytes.Equal(buffer,[]byte("http"))) || it == ',' || it == '/' { if len(buffer) != 0 { if len(buffer) > 2 { // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append @@ -667,7 +671,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { } else { // TODO: Test this items = items[:0] - r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(item))+" in UA") + r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA") r.requestLogger.Print("UA Buffer: ", buffer) r.requestLogger.Print("UA Buffer String: ", string(buffer)) break @@ -745,16 +749,12 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { break } c.DebugDetail("llLang:", llLang) - if llLang == "" { - co.LangViewCounter.Bump("none") - } else { - validCode := co.LangViewCounter.Bump(llLang) - if !validCode { - r.DumpRequest(req,"Invalid ISO Code") - } + validCode := co.LangViewCounter.Bump(llLang) + if !validCode { + r.DumpRequest(req,"Invalid ISO Code") } } else { - co.LangViewCounter.Bump("none") + co.LangViewCounter.Bump("") } if !c.Config.RefNoTrack { @@ -809,7 +809,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { //c.StoppedServer("Profile end") } -func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix string, extraData string) c.RouteError { +func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix, extraData string) c.RouteError { var err c.RouteError switch(prefix) {` + out + ` /*case "/sitemaps": // TODO: Count these views @@ -869,8 +869,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c return handle(w,req,user) } - lowerPath := strings.ToLower(req.URL.Path) - if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") { + lp := strings.ToLower(req.URL.Path) + if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { r.SuspiciousRequest(req,"Bad Route") } else { r.DumpRequest(req,"Bad Route") @@ -891,7 +891,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c log.Println("Successfully generated the router") } -func writeFile(name string, content string) { +func writeFile(name, content string) { f, err := os.Create(name) if err != nil { log.Fatal(err) @@ -900,6 +900,9 @@ func writeFile(name string, content string) { if err != nil { log.Fatal(err) } - f.Sync() + err = f.Sync() + if err != nil { + log.Fatal(err) + } f.Close() }