a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

optimise lang counter 

optimise dumprequest
add more suspicious phrases
capture more router gen errors
This commit is contained in:
Azareal 2020-02-25 14:25:50 +10:00
parent 1ea023bb6d
commit 64335cf1ef
3 changed files with 64 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
common/counters/langs.go
View File

@ -2,6 +2,7 @@ package counters
import ( import (
"database/sql" "database/sql"
"sync/atomic"
c "github.com/Azareal/Gosora/common" c "github.com/Azareal/Gosora/common"
qgen "github.com/Azareal/Gosora/query_gen" qgen "github.com/Azareal/Gosora/query_gen"
@ -12,7 +13,7 @@ var LangViewCounter *DefaultLangViewCounter
var langCodes = []string{ var langCodes = []string{
"unknown", "unknown",
"none", "",
"af", "af",
"ar", "ar",
"az", "az",
@ -98,26 +99,22 @@ var langCodes = []string{
} }
type DefaultLangViewCounter struct { type DefaultLangViewCounter struct {
buckets []*RWMutexCounterBucket //[OSID]count //buckets []*MutexCounterBucket //[OSID]count
buckets []int64 //[OSID]count
codesToIndices map[string]int codesToIndices map[string]int
insert *sql.Stmt insert *sql.Stmt
} }
func NewDefaultLangViewCounter(acc *qgen.Accumulator) (*DefaultLangViewCounter, error) { func NewDefaultLangViewCounter(acc *qgen.Accumulator) (*DefaultLangViewCounter, error) {
langBuckets := make([]*RWMutexCounterBucket, len(langCodes))
for bucketID, _ := range langBuckets {
langBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
}
codesToIndices := make(map[string]int, len(langCodes)) codesToIndices := make(map[string]int, len(langCodes))
for index, code := range langCodes { for index, code := range langCodes {
codesToIndices[code] = index codesToIndices[code] = index
} }
co := &DefaultLangViewCounter{ co := &DefaultLangViewCounter{
buckets: langBuckets, buckets: make([]int64, len(langCodes)),
codesToIndices: codesToIndices, codesToIndices: codesToIndices,
insert: acc.Insert("viewchunks_langs").Columns("count, createdAt, lang").Fields("?,UTC_TIMESTAMP(),?").Prepare(), insert: acc.Insert("viewchunks_langs").Columns("count,createdAt,lang").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
} }
c.AddScheduledFifteenMinuteTask(co.Tick) c.AddScheduledFifteenMinuteTask(co.Tick)
@ -127,13 +124,8 @@ func NewDefaultLangViewCounter(acc *qgen.Accumulator) (*DefaultLangViewCounter,
} }
func (co *DefaultLangViewCounter) Tick() error { func (co *DefaultLangViewCounter) Tick() error {
for id, bucket := range co.buckets { for id := 0; id < len(co.buckets); id++ {
var count int count := atomic.SwapInt64(&co.buckets[id], 0)
bucket.RLock()
count = bucket.counter
bucket.counter = 0 // TODO: Add a SetZero method to reduce the amount of duplicate code between the OS and agent counters?
bucket.RUnlock()
err := co.insertChunk(count, id) // TODO: Bulk insert for speed? err := co.insertChunk(count, id) // TODO: Bulk insert for speed?
if err != nil { if err != nil {
return errors.Wrap(errors.WithStack(err), "langview counter") return errors.Wrap(errors.WithStack(err), "langview counter")
@ -142,11 +134,14 @@ func (co *DefaultLangViewCounter) Tick() error {
return nil return nil
} }
func (co *DefaultLangViewCounter) insertChunk(count int, id int) error { func (co *DefaultLangViewCounter) insertChunk(count int64, id int) error {
if count == 0 { if count == 0 {
return nil return nil
} }
langCode := langCodes[id] langCode := langCodes[id]
if langCode == "" {
langCode = "none"
}
c.DebugLogf("Inserting a vchunk with a count of %d for lang %s (%d)", count, langCode, id) c.DebugLogf("Inserting a vchunk with a count of %d for lang %s (%d)", count, langCode, id)
_, err := co.insert.Exec(count, langCode) _, err := co.insert.Exec(count, langCode)
return err return err
@ -166,9 +161,7 @@ func (co *DefaultLangViewCounter) Bump(langCode string) (validCode bool) {
if len(co.buckets) <= id || id < 0 { if len(co.buckets) <= id || id < 0 {
return validCode return validCode
} }
co.buckets[id].Lock() atomic.AddInt64(&co.buckets[id], 1)
co.buckets[id].counter++
co.buckets[id].Unlock()
return validCode return validCode
} }

46
gen_router.go
View File

@ -742,10 +742,10 @@ func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *ht
func (r *GenRouter) Handle(_ string, _ http.Handler) { func (r *GenRouter) Handle(_ string, _ http.Handler) {
} }
func (r *GenRouter) HandleFunc(pattern string, handle func(http.ResponseWriter, *http.Request, c.User) c.RouteError) { func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, c.User) c.RouteError) {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()
r.extraRoutes[pattern] = handle r.extraRoutes[pattern] = h
} }
func (r *GenRouter) RemoveFunc(pattern string) error { func (r *GenRouter) RemoveFunc(pattern string) error {
@ -770,11 +770,11 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) {
r.requestLogger.Print(prepend + r.requestLogger.Print(prepend +
"\nUA: " + c.SanitiseSingleLine(req.UserAgent()) + "\n" + "\nUA: " + c.SanitiseSingleLine(req.UserAgent()) + "\n" +
"Method: " + c.SanitiseSingleLine(req.Method) + "\n" + heads + "Method: " + c.SanitiseSingleLine(req.Method) + "\n" + heads +
"req.Host: " + c.SanitiseSingleLine(req.Host) + "\n" + "Host: " + c.SanitiseSingleLine(req.Host) + "\n" +
"req.URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" + "URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" +
"req.URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" + "URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" +
"req.Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" + "Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" +
"req.RemoteAddr: " + req.RemoteAddr + "\n") "RemoteAddr: " + req.RemoteAddr + "\n")
} }
func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) { func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) {
@ -785,8 +785,8 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) {
co.AgentViewCounter.Bump(29) co.AgentViewCounter.Bump(29)
} }
func isLocalHost(host string) bool { func isLocalHost(h string) bool {
return host=="localhost" || host=="127.0.0.1" || host=="::1" return h=="localhost" || h=="127.0.0.1" || h=="::1"
} }
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global // TODO: Pass the default path or config struct to the router rather than accessing it via a package global
@ -943,10 +943,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
var items []string var items []string
var buffer []byte var buffer []byte
var os string var os string
for _, item := range StringToBytes(ua) { for _, it := range StringToBytes(ua) {
if (item > 64 && item < 91) || (item > 96 && item < 123) { if (it > 64 && it < 91) || (it > 96 && it < 123) {
buffer = append(buffer, item) buffer = append(buffer, it)
} else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == ':' || item == '.' || item == '+' || item == '~' || item == '@' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' { } else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == '_' || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' || (it == ':' && bytes.Equal(buffer,[]byte("http"))) || it == ',' || it == '/' {
if len(buffer) != 0 { if len(buffer) != 0 {
if len(buffer) > 2 { if len(buffer) > 2 {
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
@ -972,7 +972,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} else { } else {
// TODO: Test this // TODO: Test this
items = items[:0] items = items[:0]
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(item))+" in UA") r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
r.requestLogger.Print("UA Buffer: ", buffer) r.requestLogger.Print("UA Buffer: ", buffer)
r.requestLogger.Print("UA Buffer String: ", string(buffer)) r.requestLogger.Print("UA Buffer String: ", string(buffer))
break break
@ -1050,16 +1050,12 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
break break
} }
c.DebugDetail("llLang:", llLang) c.DebugDetail("llLang:", llLang)
if llLang == "" { validCode := co.LangViewCounter.Bump(llLang)
co.LangViewCounter.Bump("none") if !validCode {
} else { r.DumpRequest(req,"Invalid ISO Code")
validCode := co.LangViewCounter.Bump(llLang)
if !validCode {
r.DumpRequest(req,"Invalid ISO Code")
}
} }
} else { } else {
co.LangViewCounter.Bump("none") co.LangViewCounter.Bump("")
} }
if !c.Config.RefNoTrack { if !c.Config.RefNoTrack {
@ -1114,7 +1110,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
//c.StoppedServer("Profile end") //c.StoppedServer("Profile end")
} }
func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix string, extraData string) c.RouteError { func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix, extraData string) c.RouteError {
var err c.RouteError var err c.RouteError
switch(prefix) { switch(prefix) {
case "/overview": case "/overview":
@ -2657,8 +2653,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
return handle(w,req,user) return handle(w,req,user)
} }
lowerPath := strings.ToLower(req.URL.Path) lp := strings.ToLower(req.URL.Path)
if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") { if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") {
r.SuspiciousRequest(req,"Bad Route") r.SuspiciousRequest(req,"Bad Route")
} else { } else {
r.DumpRequest(req,"Bad Route") r.DumpRequest(req,"Bad Route")

57
router_gen/main.go
View File

@ -441,10 +441,10 @@ func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *ht
func (r *GenRouter) Handle(_ string, _ http.Handler) { func (r *GenRouter) Handle(_ string, _ http.Handler) {
} }
func (r *GenRouter) HandleFunc(pattern string, handle func(http.ResponseWriter, *http.Request, c.User) c.RouteError) { func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, c.User) c.RouteError) {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()
r.extraRoutes[pattern] = handle r.extraRoutes[pattern] = h
} }
func (r *GenRouter) RemoveFunc(pattern string) error { func (r *GenRouter) RemoveFunc(pattern string) error {
@ -469,11 +469,11 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) {
r.requestLogger.Print(prepend + r.requestLogger.Print(prepend +
"\nUA: " + c.SanitiseSingleLine(req.UserAgent()) + "\n" + "\nUA: " + c.SanitiseSingleLine(req.UserAgent()) + "\n" +
"Method: " + c.SanitiseSingleLine(req.Method) + "\n" + heads + "Method: " + c.SanitiseSingleLine(req.Method) + "\n" + heads +
"req.Host: " + c.SanitiseSingleLine(req.Host) + "\n" + "Host: " + c.SanitiseSingleLine(req.Host) + "\n" +
"req.URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" + "URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" +
"req.URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" + "URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" +
"req.Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" + "Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" +
"req.RemoteAddr: " + req.RemoteAddr + "\n") "RemoteAddr: " + req.RemoteAddr + "\n")
} }
func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) { func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) {
@ -484,8 +484,8 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, prepend string) {
co.AgentViewCounter.Bump({{.AllAgentMap.suspicious}}) co.AgentViewCounter.Bump({{.AllAgentMap.suspicious}})
} }
func isLocalHost(host string) bool { func isLocalHost(h string) bool {
return host=="localhost" || host=="127.0.0.1" || host=="::1" return h=="localhost" || h=="127.0.0.1" || h=="::1"
} }
// TODO: Pass the default path or config struct to the router rather than accessing it via a package global // TODO: Pass the default path or config struct to the router rather than accessing it via a package global
@ -619,6 +619,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
r.requestLogger.Print("before PreRoute") r.requestLogger.Print("before PreRoute")
} }
/*if c.Dev.QuicPort != 0 {
w.Header().Set("Alt-Svc", "quic=\":"+strconv.Itoa(c.Dev.QuicPort)+"\"; ma=2592000; v=\"44,43,39\", h3-23=\":"+strconv.Itoa(c.Dev.QuicPort)+"\"; ma=3600, h3-24=\":"+strconv.Itoa(c.Dev.QuicPort)+"\"; ma=3600, h2=\":443\"; ma=3600")
}*/
// Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like. // Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like.
// TODO: Add a setting to disable this? // TODO: Add a setting to disable this?
// TODO: Use a more efficient detector instead of smashing every possible combination in // TODO: Use a more efficient detector instead of smashing every possible combination in
@ -638,10 +642,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
var items []string var items []string
var buffer []byte var buffer []byte
var os string var os string
for _, item := range StringToBytes(ua) { for _, it := range StringToBytes(ua) {
if (item > 64 && item < 91) || (item > 96 && item < 123) { if (it > 64 && it < 91) || (it > 96 && it < 123) {
buffer = append(buffer, item) buffer = append(buffer, it)
} else if item == ' ' || item == '(' || item == ')' || item == '-' || (item > 47 && item < 58) || item == '_' || item == ';' || item == ':' || item == '.' || item == '+' || item == '~' || item == '@' || (item == ':' && bytes.Equal(buffer,[]byte("http"))) || item == ',' || item == '/' { } else if it == ' ' || it == '(' || it == ')' || it == '-' || (it > 47 && it < 58) || it == '_' || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' || (it == ':' && bytes.Equal(buffer,[]byte("http"))) || it == ',' || it == '/' {
if len(buffer) != 0 { if len(buffer) != 0 {
if len(buffer) > 2 { if len(buffer) > 2 {
// Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append
@ -667,7 +671,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} else { } else {
// TODO: Test this // TODO: Test this
items = items[:0] items = items[:0]
r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(item))+" in UA") r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA")
r.requestLogger.Print("UA Buffer: ", buffer) r.requestLogger.Print("UA Buffer: ", buffer)
r.requestLogger.Print("UA Buffer String: ", string(buffer)) r.requestLogger.Print("UA Buffer String: ", string(buffer))
break break
@ -745,16 +749,12 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
break break
} }
c.DebugDetail("llLang:", llLang) c.DebugDetail("llLang:", llLang)
if llLang == "" { validCode := co.LangViewCounter.Bump(llLang)
co.LangViewCounter.Bump("none") if !validCode {
} else { r.DumpRequest(req,"Invalid ISO Code")
validCode := co.LangViewCounter.Bump(llLang)
if !validCode {
r.DumpRequest(req,"Invalid ISO Code")
}
} }
} else { } else {
co.LangViewCounter.Bump("none") co.LangViewCounter.Bump("")
} }
if !c.Config.RefNoTrack { if !c.Config.RefNoTrack {
@ -809,7 +809,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
//c.StoppedServer("Profile end") //c.StoppedServer("Profile end")
} }
func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix string, extraData string) c.RouteError { func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix, extraData string) c.RouteError {
var err c.RouteError var err c.RouteError
switch(prefix) {` + out + ` switch(prefix) {` + out + `
/*case "/sitemaps": // TODO: Count these views /*case "/sitemaps": // TODO: Count these views
@ -869,8 +869,8 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
return handle(w,req,user) return handle(w,req,user)
} }
lowerPath := strings.ToLower(req.URL.Path) lp := strings.ToLower(req.URL.Path)
if strings.Contains(lowerPath,"admin") || strings.Contains(lowerPath,"sql") || strings.Contains(lowerPath,"manage") || strings.Contains(lowerPath,"//") || strings.Contains(lowerPath,"\\\\") || strings.Contains(lowerPath,"wp") || strings.Contains(lowerPath,"wordpress") || strings.Contains(lowerPath,"config") || strings.Contains(lowerPath,"setup") || strings.Contains(lowerPath,"install") || strings.Contains(lowerPath,"update") || strings.Contains(lowerPath,"php") { if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") {
r.SuspiciousRequest(req,"Bad Route") r.SuspiciousRequest(req,"Bad Route")
} else { } else {
r.DumpRequest(req,"Bad Route") r.DumpRequest(req,"Bad Route")
@ -891,7 +891,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
log.Println("Successfully generated the router") log.Println("Successfully generated the router")
} }
func writeFile(name string, content string) { func writeFile(name, content string) {
f, err := os.Create(name) f, err := os.Create(name)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
@ -900,6 +900,9 @@ func writeFile(name string, content string) {
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
f.Sync() err = f.Sync()
if err != nil {
log.Fatal(err)
}
f.Close() f.Close()
} }