From 22ec566a2c328220b8668ad1a1be5624d1e3a806 Mon Sep 17 00:00:00 2001 From: Azareal Date: Sat, 21 Mar 2020 17:44:33 +1000 Subject: [PATCH] add website datenbank user agent reduce redundancy and fix sublties in StaticFile() --- common/settings.go | 28 +++--- gen_router.go | 192 ++++++++++++++++++++-------------------- langs/english.json | 1 + router_gen/main.go | 3 + routes/attachments.go | 8 +- routes/misc.go | 29 +++--- routes/panel/forums.go | 2 +- routes/panel/logs.go | 16 ++-- routes/panel/pages.go | 18 ++-- routes/panel/plugins.go | 8 +- 10 files changed, 155 insertions(+), 150 deletions(-) diff --git a/common/settings.go b/common/settings.go index 749777f7..08ef2030 100644 --- a/common/settings.go +++ b/common/settings.go @@ -16,7 +16,7 @@ var SettingBox atomic.Value // An atomic value pointing to a SettingBox type SettingMap map[string]interface{} type SettingStore interface { - ParseSetting(sname, scontent, stype, sconstraint string) string + ParseSetting(name, content, typ, constraint string) string BypassGet(name string) (*Setting, error) BypassGetAll(name string) ([]*Setting, error) } @@ -55,10 +55,10 @@ func init() { }) } -func (s *Setting) Copy() (out *Setting) { - out = &Setting{Name: ""} - *out = *s - return out +func (s *Setting) Copy() (o *Setting) { + o = &Setting{Name: ""} + *o = *s + return o } func LoadSettings() error { @@ -80,18 +80,18 @@ func LoadSettings() error { } // TODO: Add better support for HTML attributes (html-attribute). E.g. Meta descriptions. -func (sBox SettingMap) ParseSetting(sname, scontent, stype, constraint string) (err error) { +func (sBox SettingMap) ParseSetting(name, content, typ, constraint string) (err error) { ssBox := map[string]interface{}(sBox) - switch stype { + switch typ { case "bool": - ssBox[sname] = (scontent == "1") + ssBox[name] = (content == "1") case "int": - ssBox[sname], err = strconv.Atoi(scontent) + ssBox[name], err = strconv.Atoi(content) if err != nil { return errors.New("You were supposed to enter an integer x.x") } case "int64": - ssBox[sname], err = strconv.ParseInt(scontent, 10, 64) + ssBox[name], err = strconv.ParseInt(content, 10, 64) if err != nil { return errors.New("You were supposed to enter an integer x.x") } @@ -107,17 +107,17 @@ func (sBox SettingMap) ParseSetting(sname, scontent, stype, constraint string) ( return errors.New("Invalid contraint! The constraint field wasn't an integer!") } - value, err := strconv.Atoi(scontent) + val, err := strconv.Atoi(content) if err != nil { return errors.New("Only integers are allowed in this setting x.x") } - if value < con1 || value > con2 { + if val < con1 || val > con2 { return errors.New("Only integers between a certain range are allowed in this setting") } - ssBox[sname] = value + ssBox[name] = val default: - ssBox[sname] = scontent + ssBox[name] = content } return nil } diff --git a/gen_router.go b/gen_router.go index a80cade4..52962968 100644 --- a/gen_router.go +++ b/gen_router.go @@ -590,36 +590,37 @@ var agentMapEnum = map[string]int{ "exabot": 16, "mojeek": 17, "cliqz": 18, - "baidu": 19, - "sogou": 20, - "toutiao": 21, - "haosou": 22, - "duckduckgo": 23, - "seznambot": 24, - "discord": 25, - "twitter": 26, - "facebook": 27, - "cloudflare": 28, - "archive_org": 29, - "uptimebot": 30, - "slackbot": 31, - "apple": 32, - "discourse": 33, - "alexa": 34, - "lynx": 35, - "blank": 36, - "malformed": 37, - "suspicious": 38, - "semrush": 39, - "dotbot": 40, - "ahrefs": 41, - "proximic": 42, - "majestic": 43, - "blexbot": 44, - "aspiegel": 45, - "mail_ru": 46, - "zgrab": 47, - "curl": 48, + "datenbank": 19, + "baidu": 20, + "sogou": 21, + "toutiao": 22, + "haosou": 23, + "duckduckgo": 24, + "seznambot": 25, + "discord": 26, + "twitter": 27, + "facebook": 28, + "cloudflare": 29, + "archive_org": 30, + "uptimebot": 31, + "slackbot": 32, + "apple": 33, + "discourse": 34, + "alexa": 35, + "lynx": 36, + "blank": 37, + "malformed": 38, + "suspicious": 39, + "semrush": 40, + "dotbot": 41, + "ahrefs": 42, + "proximic": 43, + "majestic": 44, + "blexbot": 45, + "aspiegel": 46, + "mail_ru": 47, + "zgrab": 48, + "curl": 49, } var reverseAgentMapEnum = map[int]string{ 0: "unknown", @@ -641,36 +642,37 @@ var reverseAgentMapEnum = map[int]string{ 16: "exabot", 17: "mojeek", 18: "cliqz", - 19: "baidu", - 20: "sogou", - 21: "toutiao", - 22: "haosou", - 23: "duckduckgo", - 24: "seznambot", - 25: "discord", - 26: "twitter", - 27: "facebook", - 28: "cloudflare", - 29: "archive_org", - 30: "uptimebot", - 31: "slackbot", - 32: "apple", - 33: "discourse", - 34: "alexa", - 35: "lynx", - 36: "blank", - 37: "malformed", - 38: "suspicious", - 39: "semrush", - 40: "dotbot", - 41: "ahrefs", - 42: "proximic", - 43: "majestic", - 44: "blexbot", - 45: "aspiegel", - 46: "mail_ru", - 47: "zgrab", - 48: "curl", + 19: "datenbank", + 20: "baidu", + 21: "sogou", + 22: "toutiao", + 23: "haosou", + 24: "duckduckgo", + 25: "seznambot", + 26: "discord", + 27: "twitter", + 28: "facebook", + 29: "cloudflare", + 30: "archive_org", + 31: "uptimebot", + 32: "slackbot", + 33: "apple", + 34: "discourse", + 35: "alexa", + 36: "lynx", + 37: "blank", + 38: "malformed", + 39: "suspicious", + 40: "semrush", + 41: "dotbot", + 42: "ahrefs", + 43: "proximic", + 44: "majestic", + 45: "blexbot", + 46: "aspiegel", + 47: "mail_ru", + 48: "zgrab", + 49: "curl", } var markToAgent = map[string]string{ "OPR": "opera", @@ -699,6 +701,7 @@ var markToAgent = map[string]string{ "Exabot": "exabot", "MojeekBot": "mojeek", "Cliqzbot": "cliqz", + "netEstate": "datenbank", "SeznamBot": "seznambot", "CloudFlare": "cloudflare", "archive": "archive_org", @@ -731,18 +734,18 @@ var markToID = map[string]int{ "MSIE": 6, "Trident": 7, "Edge": 5, - "Lynx": 35, + "Lynx": 36, "SamsungBrowser": 10, "UCBrowser": 11, "Google": 12, "Googlebot": 12, "yandex": 13, - "DuckDuckBot": 23, - "DuckDuckGo": 23, - "Baiduspider": 19, - "Sogou": 20, - "ToutiaoSpider": 21, - "360Spider": 22, + "DuckDuckBot": 24, + "DuckDuckGo": 24, + "Baiduspider": 20, + "Sogou": 21, + "ToutiaoSpider": 22, + "360Spider": 23, "bingbot": 14, "BingPreview": 14, "msnbot": 14, @@ -750,29 +753,30 @@ var markToID = map[string]int{ "Exabot": 16, "MojeekBot": 17, "Cliqzbot": 18, - "SeznamBot": 24, - "CloudFlare": 28, - "archive": 29, - "Uptimebot": 30, - "Slackbot": 31, - "Slack": 31, - "Discordbot": 25, - "Twitterbot": 26, - "facebookexternalhit": 27, - "Facebot": 27, - "Applebot": 32, - "Discourse": 33, - "ia_archiver": 34, - "SemrushBot": 39, - "DotBot": 40, - "AhrefsBot": 41, - "proximic": 42, - "MJ12bot": 43, - "BLEXBot": 44, - "AspiegelBot": 45, - "RU_Bot": 46, - "zgrab": 47, - "curl": 48, + "netEstate": 19, + "SeznamBot": 25, + "CloudFlare": 29, + "archive": 30, + "Uptimebot": 31, + "Slackbot": 32, + "Slack": 32, + "Discordbot": 26, + "Twitterbot": 27, + "facebookexternalhit": 28, + "Facebot": 28, + "Applebot": 33, + "Discourse": 34, + "ia_archiver": 35, + "SemrushBot": 40, + "DotBot": 41, + "AhrefsBot": 42, + "proximic": 43, + "MJ12bot": 44, + "BLEXBot": 45, + "AspiegelBot": 46, + "RU_Bot": 47, + "zgrab": 48, + "curl": 49, } /*var agentRank = map[string]int{ "opera":9, @@ -902,7 +906,7 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { pre += "\n" } r.DumpRequest(req,pre+"Suspicious Request") - co.AgentViewCounter.Bump(38) + co.AgentViewCounter.Bump(39) } func isLocalHost(h string) bool { @@ -917,7 +921,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { w.WriteHeader(200) // 400 w.Write([]byte("")) r.DumpRequest(req,"Malformed Request T"+strconv.Itoa(typ)) - co.AgentViewCounter.Bump(37) + co.AgentViewCounter.Bump(38) } // Split the Host and Port string @@ -1060,7 +1064,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another if ua == "" { - co.AgentViewCounter.Bump(36) + co.AgentViewCounter.Bump(37) if c.Dev.DebugMode { var pre string for _, char := range req.UserAgent() { @@ -1141,7 +1145,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { if strings.Contains(ua,"rv:11") { agent = 6 } - case 47: + case 48: r.SuspiciousRequest(req,"Vulnerability Scanner") } diff --git a/langs/english.json b/langs/english.json index 0e170a52..b002d17c 100644 --- a/langs/english.json +++ b/langs/english.json @@ -204,6 +204,7 @@ "exabot":"Exabot", "mojeek":"MojeekBot", "cliqz":"Cliqzbot", + "datenbank":"Website Datenbank", "sogou":"Sogou", "toutiao":"Toutiao", "haosou":"Qihoo 360 Search", diff --git a/router_gen/main.go b/router_gen/main.go index 45cbc623..89554f6d 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -250,6 +250,7 @@ func main() { "exabot", "mojeek", "cliqz", + "datenbank", "baidu", "sogou", "toutiao", @@ -315,6 +316,7 @@ func main() { "Exabot", "MojeekBot", "Cliqzbot", + "netEstate", "SeznamBot", "CloudFlare", "archive", //archive.org_bot @@ -369,6 +371,7 @@ func main() { "Exabot": "exabot", "MojeekBot": "mojeek", "Cliqzbot": "cliqz", + "netEstate":"datenbank", "SeznamBot": "seznambot", "CloudFlare": "cloudflare", // Track alwayson specifically in case there are other bots? "archive": "archive_org", //archive.org_bot diff --git a/routes/attachments.go b/routes/attachments.go index df6e4817..4b09a158 100644 --- a/routes/attachments.go +++ b/routes/attachments.go @@ -88,7 +88,7 @@ func ShowAttachment(w http.ResponseWriter, r *http.Request, user *c.User, filena return nil } -func deleteAttachment(w http.ResponseWriter, r *http.Request, user *c.User, aid int, js bool) c.RouteError { +func deleteAttachment(w http.ResponseWriter, r *http.Request, u *c.User, aid int, js bool) c.RouteError { err := c.DeleteAttachment(aid) if err == sql.ErrNoRows { return c.NotFoundJSQ(w, r, nil, js) @@ -101,15 +101,15 @@ func deleteAttachment(w http.ResponseWriter, r *http.Request, user *c.User, aid // TODO: Stop duplicating this code // TODO: Use a transaction here // TODO: Move this function to neutral ground -func uploadAttachment(w http.ResponseWriter, r *http.Request, user *c.User, sid int, stable string, oid int, otable, extra string) (pathMap map[string]string, rerr c.RouteError) { +func uploadAttachment(w http.ResponseWriter, r *http.Request, u *c.User, sid int, stable string, oid int, otable, extra string) (pathMap map[string]string, rerr c.RouteError) { pathMap = make(map[string]string) - files, rerr := uploadFilesWithHash(w, r, user, "./attachs/") + files, rerr := uploadFilesWithHash(w, r, u, "./attachs/") if rerr != nil { return nil, rerr } for _, filename := range files { - aid, err := c.Attachments.Add(sid, stable, oid, otable, user.ID, filename, extra) + aid, err := c.Attachments.Add(sid, stable, oid, otable, u.ID, filename, extra) if err != nil { return nil, c.InternalError(err, w, r) } diff --git a/routes/misc.go b/routes/misc.go index feb99135..772f25a1 100644 --- a/routes/misc.go +++ b/routes/misc.go @@ -30,26 +30,23 @@ func StaticFile(w http.ResponseWriter, r *http.Request) { if file.Length > 300 { if h.Get("Range") != "" { + h.Set("Vary", "Accept-Encoding") + if len(file.Sha256) != 0 { + h.Set("Cache-Control", cacheControlMaxAgeWeek) + } else { + h.Set("Cache-Control", cacheControlMaxAge) //Cache-Control: max-age=31536000 + } + if file.GzipLength > 300 && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { - if len(file.Sha256) != 0 { - h.Set("Cache-Control", cacheControlMaxAgeWeek) - } else { - h.Set("Cache-Control", cacheControlMaxAge) //Cache-Control: max-age=31536000 - } h.Set("Content-Encoding", "gzip") h.Set("Content-Length", file.StrGzipLength) http.ServeContent(w, r, r.URL.Path, file.Info.ModTime(), bytes.NewReader(file.GzipData)) return - } else if file.GzipLength == 0 { - if len(file.Sha256) != 0 { - h.Set("Cache-Control", cacheControlMaxAgeWeek) - } else { - h.Set("Cache-Control", cacheControlMaxAge) //Cache-Control: max-age=31536000 - } - h.Set("Content-Length", file.StrLength) - http.ServeContent(w, r, r.URL.Path, file.Info.ModTime(), bytes.NewReader(file.Data)) - return } + + h.Set("Content-Length", file.StrLength) + http.ServeContent(w, r, r.URL.Path, file.Info.ModTime(), bytes.NewReader(file.Data)) + return } } @@ -79,13 +76,13 @@ func StaticFile(w http.ResponseWriter, r *http.Request) { // Other options instead of io.Copy: io.CopyN(), w.Write(), http.ServeContent() } -func Overview(w http.ResponseWriter, r *http.Request, user *c.User, h *c.Header) c.RouteError { +func Overview(w http.ResponseWriter, r *http.Request, u *c.User, h *c.Header) c.RouteError { h.Title = phrases.GetTitlePhrase("overview") h.Zone = "overview" return renderTemplate("overview", w, r, h, c.Page{h, tList, nil}) } -func CustomPage(w http.ResponseWriter, r *http.Request, user *c.User, h *c.Header, name string) c.RouteError { +func CustomPage(w http.ResponseWriter, r *http.Request, u *c.User, h *c.Header, name string) c.RouteError { h.Zone = "custom_page" name = c.SanitiseSingleLine(name) page, err := c.Pages.GetByName(name) diff --git a/routes/panel/forums.go b/routes/panel/forums.go index f30f2860..f916d42c 100644 --- a/routes/panel/forums.go +++ b/routes/panel/forums.go @@ -309,7 +309,7 @@ func ForumsEditPermsSubmit(w http.ResponseWriter, r *http.Request, user *c.User, } // A helper function for the Advanced portion of the Forum Perms Editor -func forumPermsExtractDash(paramList string) (fid int, gid int, err error) { +func forumPermsExtractDash(paramList string) (fid, gid int, err error) { params := strings.Split(paramList, "-") if len(params) != 2 { return fid, gid, errors.New("Parameter count mismatch") diff --git a/routes/panel/logs.go b/routes/panel/logs.go index c546fae4..a697db28 100644 --- a/routes/panel/logs.go +++ b/routes/panel/logs.go @@ -12,8 +12,8 @@ import ( ) // TODO: Link the usernames for successful registrations to the profiles -func LogsRegs(w http.ResponseWriter, r *http.Request, user *c.User) c.RouteError { - basePage, ferr := buildBasePage(w, r, user, "registration_logs", "logs") +func LogsRegs(w http.ResponseWriter, r *http.Request, u *c.User) c.RouteError { + basePage, ferr := buildBasePage(w, r, u, "registration_logs", "logs") if ferr != nil { return ferr } @@ -115,8 +115,8 @@ func adminlogsElementType(action, elementType string, elementID int, actor *c.Us switch elementType { // TODO: Record more detail for this, e.g. which field/s was changed case "user": - targetUser := handleUnknownUser(c.Users.Get(elementID)) - out = p.GetTmplPhrasef("panel_logs_admin_action_user_"+action, targetUser.Link, targetUser.Name, actor.Link, actor.Name) + tu := handleUnknownUser(c.Users.Get(elementID)) + out = p.GetTmplPhrasef("panel_logs_admin_action_user_"+action, tu.Link, tu.Name, actor.Link, actor.Name) case "group": g, err := c.Groups.Get(elementID) if err != nil { @@ -168,8 +168,8 @@ func adminlogsElementType(action, elementType string, elementID int, actor *c.Us return out } -func LogsMod(w http.ResponseWriter, r *http.Request, user *c.User) c.RouteError { - basePage, ferr := buildBasePage(w, r, user, "mod_logs", "logs") +func LogsMod(w http.ResponseWriter, r *http.Request, u *c.User) c.RouteError { + basePage, ferr := buildBasePage(w, r, u, "mod_logs", "logs") if ferr != nil { return ferr } @@ -193,8 +193,8 @@ func LogsMod(w http.ResponseWriter, r *http.Request, user *c.User) c.RouteError return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "", "", "panel_modlogs", pi}) } -func LogsAdmin(w http.ResponseWriter, r *http.Request, user *c.User) c.RouteError { - basePage, ferr := buildBasePage(w, r, user, "admin_logs", "logs") +func LogsAdmin(w http.ResponseWriter, r *http.Request, u *c.User) c.RouteError { + basePage, ferr := buildBasePage(w, r, u, "admin_logs", "logs") if ferr != nil { return ferr } diff --git a/routes/panel/pages.go b/routes/panel/pages.go index 50807161..63957852 100644 --- a/routes/panel/pages.go +++ b/routes/panel/pages.go @@ -8,8 +8,8 @@ import ( c "github.com/Azareal/Gosora/common" ) -func Pages(w http.ResponseWriter, r *http.Request, user *c.User) c.RouteError { - basePage, ferr := buildBasePage(w, r, user, "pages", "pages") +func Pages(w http.ResponseWriter, r *http.Request, u *c.User) c.RouteError { + basePage, ferr := buildBasePage(w, r, u, "pages", "pages") if ferr != nil { return ferr } @@ -71,8 +71,8 @@ func PagesCreateSubmit(w http.ResponseWriter, r *http.Request, user *c.User) c.R return nil } -func PagesEdit(w http.ResponseWriter, r *http.Request, user *c.User, spid string) c.RouteError { - basePage, ferr := buildBasePage(w, r, user, "pages_edit", "pages") +func PagesEdit(w http.ResponseWriter, r *http.Request, u *c.User, spid string) c.RouteError { + basePage, ferr := buildBasePage(w, r, u, "pages_edit", "pages") if ferr != nil { return ferr } @@ -82,7 +82,7 @@ func PagesEdit(w http.ResponseWriter, r *http.Request, user *c.User, spid string pid, err := strconv.Atoi(spid) if err != nil { - return c.LocalError("Page ID needs to be an integer", w, r, user) + return c.LocalError("Page ID needs to be an integer", w, r, u) } page, err := c.Pages.Get(pid) if err == sql.ErrNoRows { @@ -138,21 +138,21 @@ func PagesEditSubmit(w http.ResponseWriter, r *http.Request, user *c.User, spid return nil } -func PagesDeleteSubmit(w http.ResponseWriter, r *http.Request, user *c.User, spid string) c.RouteError { - _, ferr := c.SimplePanelUserCheck(w, r, user) +func PagesDeleteSubmit(w http.ResponseWriter, r *http.Request, u *c.User, spid string) c.RouteError { + _, ferr := c.SimplePanelUserCheck(w, r, u) if ferr != nil { return ferr } pid, err := strconv.Atoi(spid) if err != nil { - return c.LocalError("Page ID needs to be an integer", w, r, user) + return c.LocalError("Page ID needs to be an integer", w, r, u) } err = c.Pages.Delete(pid) if err != nil { return c.InternalError(err, w, r) } - err = c.AdminLogs.Create("delete", pid, "page", user.GetIP(), user.ID) + err = c.AdminLogs.Create("delete", pid, "page", u.GetIP(), u.ID) if err != nil { return c.InternalError(err, w, r) } diff --git a/routes/panel/plugins.go b/routes/panel/plugins.go index 17b23a3c..bc23286a 100644 --- a/routes/panel/plugins.go +++ b/routes/panel/plugins.go @@ -8,13 +8,13 @@ import ( c "github.com/Azareal/Gosora/common" ) -func Plugins(w http.ResponseWriter, r *http.Request, user *c.User) c.RouteError { - basePage, ferr := buildBasePage(w, r, user, "plugins", "plugins") +func Plugins(w http.ResponseWriter, r *http.Request, u *c.User) c.RouteError { + basePage, ferr := buildBasePage(w, r, u, "plugins", "plugins") if ferr != nil { return ferr } - if !user.Perms.ManagePlugins { - return c.NoPermissions(w, r, user) + if !u.Perms.ManagePlugins { + return c.NoPermissions(w, r, u) } var pluginList []interface{}