add telegram, domcop, maui and alt aspiegel user agents
reduce boilerplate in routergen
This commit is contained in:
Azareal
parent
7211ed4f0e
commit
75d1d6fcbe
241
gen_router.go
241
gen_router.go
@ -600,42 +600,45 @@ var agentMapEnum = map[string]int{
|
||||
"duckduckgo": 24,
|
||||
"seznambot": 25,
|
||||
"discord": 26,
|
||||
"twitter": 27,
|
||||
"facebook": 28,
|
||||
"cloudflare": 29,
|
||||
"archive_org": 30,
|
||||
"uptimebot": 31,
|
||||
"slackbot": 32,
|
||||
"apple": 33,
|
||||
"discourse": 34,
|
||||
"mattermost": 35,
|
||||
"alexa": 36,
|
||||
"lynx": 37,
|
||||
"blank": 38,
|
||||
"malformed": 39,
|
||||
"suspicious": 40,
|
||||
"semrush": 41,
|
||||
"dotbot": 42,
|
||||
"ahrefs": 43,
|
||||
"proximic": 44,
|
||||
"megaindex": 45,
|
||||
"majestic": 46,
|
||||
"cocolyze": 47,
|
||||
"babbar": 48,
|
||||
"surdotly": 49,
|
||||
"netcraft": 50,
|
||||
"blexbot": 51,
|
||||
"burf": 52,
|
||||
"aspiegel": 53,
|
||||
"mail_ru": 54,
|
||||
"ccbot": 55,
|
||||
"zgrab": 56,
|
||||
"cloudsystemnetworks": 57,
|
||||
"curl": 58,
|
||||
"python": 59,
|
||||
"go": 60,
|
||||
"headlesschrome": 61,
|
||||
"awesome_bot": 62,
|
||||
"telegram": 27,
|
||||
"twitter": 28,
|
||||
"facebook": 29,
|
||||
"cloudflare": 30,
|
||||
"archive_org": 31,
|
||||
"uptimebot": 32,
|
||||
"slackbot": 33,
|
||||
"apple": 34,
|
||||
"discourse": 35,
|
||||
"mattermost": 36,
|
||||
"alexa": 37,
|
||||
"lynx": 38,
|
||||
"blank": 39,
|
||||
"malformed": 40,
|
||||
"suspicious": 41,
|
||||
"semrush": 42,
|
||||
"dotbot": 43,
|
||||
"ahrefs": 44,
|
||||
"proximic": 45,
|
||||
"megaindex": 46,
|
||||
"majestic": 47,
|
||||
"cocolyze": 48,
|
||||
"babbar": 49,
|
||||
"surdotly": 50,
|
||||
"domcop": 51,
|
||||
"netcraft": 52,
|
||||
"blexbot": 53,
|
||||
"burf": 54,
|
||||
"aspiegel": 55,
|
||||
"mail_ru": 56,
|
||||
"ccbot": 57,
|
||||
"zgrab": 58,
|
||||
"cloudsystemnetworks": 59,
|
||||
"maui": 60,
|
||||
"curl": 61,
|
||||
"python": 62,
|
||||
"go": 63,
|
||||
"headlesschrome": 64,
|
||||
"awesome_bot": 65,
|
||||
}
|
||||
var reverseAgentMapEnum = map[int]string{
|
||||
0: "unknown",
|
||||
@ -665,42 +668,45 @@ var reverseAgentMapEnum = map[int]string{
|
||||
24: "duckduckgo",
|
||||
25: "seznambot",
|
||||
26: "discord",
|
||||
27: "twitter",
|
||||
28: "facebook",
|
||||
29: "cloudflare",
|
||||
30: "archive_org",
|
||||
31: "uptimebot",
|
||||
32: "slackbot",
|
||||
33: "apple",
|
||||
34: "discourse",
|
||||
35: "mattermost",
|
||||
36: "alexa",
|
||||
37: "lynx",
|
||||
38: "blank",
|
||||
39: "malformed",
|
||||
40: "suspicious",
|
||||
41: "semrush",
|
||||
42: "dotbot",
|
||||
43: "ahrefs",
|
||||
44: "proximic",
|
||||
45: "megaindex",
|
||||
46: "majestic",
|
||||
47: "cocolyze",
|
||||
48: "babbar",
|
||||
49: "surdotly",
|
||||
50: "netcraft",
|
||||
51: "blexbot",
|
||||
52: "burf",
|
||||
53: "aspiegel",
|
||||
54: "mail_ru",
|
||||
55: "ccbot",
|
||||
56: "zgrab",
|
||||
57: "cloudsystemnetworks",
|
||||
58: "curl",
|
||||
59: "python",
|
||||
60: "go",
|
||||
61: "headlesschrome",
|
||||
62: "awesome_bot",
|
||||
27: "telegram",
|
||||
28: "twitter",
|
||||
29: "facebook",
|
||||
30: "cloudflare",
|
||||
31: "archive_org",
|
||||
32: "uptimebot",
|
||||
33: "slackbot",
|
||||
34: "apple",
|
||||
35: "discourse",
|
||||
36: "mattermost",
|
||||
37: "alexa",
|
||||
38: "lynx",
|
||||
39: "blank",
|
||||
40: "malformed",
|
||||
41: "suspicious",
|
||||
42: "semrush",
|
||||
43: "dotbot",
|
||||
44: "ahrefs",
|
||||
45: "proximic",
|
||||
46: "megaindex",
|
||||
47: "majestic",
|
||||
48: "cocolyze",
|
||||
49: "babbar",
|
||||
50: "surdotly",
|
||||
51: "domcop",
|
||||
52: "netcraft",
|
||||
53: "blexbot",
|
||||
54: "burf",
|
||||
55: "aspiegel",
|
||||
56: "mail_ru",
|
||||
57: "ccbot",
|
||||
58: "zgrab",
|
||||
59: "cloudsystemnetworks",
|
||||
60: "maui",
|
||||
61: "curl",
|
||||
62: "python",
|
||||
63: "go",
|
||||
64: "headlesschrome",
|
||||
65: "awesome_bot",
|
||||
}
|
||||
var markToAgent = map[string]string{
|
||||
"OPR": "opera",
|
||||
@ -737,6 +743,7 @@ var markToAgent = map[string]string{
|
||||
"Slackbot": "slackbot",
|
||||
"Slack": "slackbot",
|
||||
"Discordbot": "discord",
|
||||
"TelegramBot": "telegram",
|
||||
"Twitterbot": "twitter",
|
||||
"facebookexternalhit": "facebook",
|
||||
"Facebot": "facebook",
|
||||
@ -754,6 +761,7 @@ var markToAgent = map[string]string{
|
||||
"Cocolyzebot": "cocolyze",
|
||||
"Barkrowler": "babbar",
|
||||
"SurdotlyBot": "surdotly",
|
||||
"DomCopBot": "domcop",
|
||||
"NetcraftSurveyAgent": "netcraft",
|
||||
"BLEXBot": "blexbot",
|
||||
"Burf": "burf",
|
||||
@ -762,6 +770,7 @@ var markToAgent = map[string]string{
|
||||
"CCBot": "ccbot",
|
||||
"zgrab": "zgrab",
|
||||
"Nimbostratus": "cloudsystemnetworks",
|
||||
"MauiBot": "maui",
|
||||
"curl": "curl",
|
||||
"python": "python",
|
||||
"Go": "go",
|
||||
@ -776,7 +785,7 @@ var markToID = map[string]int{
|
||||
"MSIE": 6,
|
||||
"Trident": 7,
|
||||
"Edge": 5,
|
||||
"Lynx": 37,
|
||||
"Lynx": 38,
|
||||
"SamsungBrowser": 10,
|
||||
"UCBrowser": 11,
|
||||
"Google": 12,
|
||||
@ -797,42 +806,45 @@ var markToID = map[string]int{
|
||||
"Cliqzbot": 18,
|
||||
"netEstate": 19,
|
||||
"SeznamBot": 25,
|
||||
"CloudFlare": 29,
|
||||
"archive": 30,
|
||||
"Uptimebot": 31,
|
||||
"Slackbot": 32,
|
||||
"Slack": 32,
|
||||
"CloudFlare": 30,
|
||||
"archive": 31,
|
||||
"Uptimebot": 32,
|
||||
"Slackbot": 33,
|
||||
"Slack": 33,
|
||||
"Discordbot": 26,
|
||||
"Twitterbot": 27,
|
||||
"facebookexternalhit": 28,
|
||||
"Facebot": 28,
|
||||
"Applebot": 33,
|
||||
"Discourse": 34,
|
||||
"mattermost": 35,
|
||||
"ia_archiver": 36,
|
||||
"SemrushBot": 41,
|
||||
"DotBot": 42,
|
||||
"AhrefsBot": 43,
|
||||
"proximic": 44,
|
||||
"MegaIndex": 45,
|
||||
"MJ12bot": 46,
|
||||
"mj12bot": 46,
|
||||
"Cocolyzebot": 47,
|
||||
"Barkrowler": 48,
|
||||
"SurdotlyBot": 49,
|
||||
"NetcraftSurveyAgent": 50,
|
||||
"BLEXBot": 51,
|
||||
"Burf": 52,
|
||||
"AspiegelBot": 53,
|
||||
"RU_Bot": 54,
|
||||
"CCBot": 55,
|
||||
"zgrab": 56,
|
||||
"Nimbostratus": 57,
|
||||
"curl": 58,
|
||||
"python": 59,
|
||||
"Go": 60,
|
||||
"HeadlessChrome": 61,
|
||||
"awesome_bot": 62,
|
||||
"TelegramBot": 27,
|
||||
"Twitterbot": 28,
|
||||
"facebookexternalhit": 29,
|
||||
"Facebot": 29,
|
||||
"Applebot": 34,
|
||||
"Discourse": 35,
|
||||
"mattermost": 36,
|
||||
"ia_archiver": 37,
|
||||
"SemrushBot": 42,
|
||||
"DotBot": 43,
|
||||
"AhrefsBot": 44,
|
||||
"proximic": 45,
|
||||
"MegaIndex": 46,
|
||||
"MJ12bot": 47,
|
||||
"mj12bot": 47,
|
||||
"Cocolyzebot": 48,
|
||||
"Barkrowler": 49,
|
||||
"SurdotlyBot": 50,
|
||||
"DomCopBot": 51,
|
||||
"NetcraftSurveyAgent": 52,
|
||||
"BLEXBot": 53,
|
||||
"Burf": 54,
|
||||
"AspiegelBot": 55,
|
||||
"RU_Bot": 56,
|
||||
"CCBot": 57,
|
||||
"zgrab": 58,
|
||||
"Nimbostratus": 59,
|
||||
"MauiBot": 60,
|
||||
"curl": 61,
|
||||
"python": 62,
|
||||
"Go": 63,
|
||||
"HeadlessChrome": 64,
|
||||
"awesome_bot": 65,
|
||||
}
|
||||
/*var agentRank = map[string]int{
|
||||
"opera":9,
|
||||
@ -902,11 +914,10 @@ func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &GenRouter{
|
||||
UploadHandler: func(w http.ResponseWriter, req *http.Request) {
|
||||
UploadHandler: func(w http.ResponseWriter, r *http.Request) {
|
||||
writ := NewWriterIntercept(w)
|
||||
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,req)
|
||||
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r)
|
||||
},
|
||||
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
|
||||
requestLogger: log.New(f, "", log.LstdFlags),
|
||||
@ -968,7 +979,7 @@ func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) {
|
||||
pre += "\n"
|
||||
}
|
||||
r.DumpRequest(req,pre+"Suspicious Request")
|
||||
co.AgentViewCounter.Bump(40)
|
||||
co.AgentViewCounter.Bump(41)
|
||||
}
|
||||
|
||||
func isLocalHost(h string) bool {
|
||||
@ -987,7 +998,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
w.WriteHeader(200) // 400
|
||||
w.Write([]byte(""))
|
||||
r.DumpRequest(req,"Malformed Request T"+strconv.Itoa(typ))
|
||||
co.AgentViewCounter.Bump(39)
|
||||
co.AgentViewCounter.Bump(40)
|
||||
}
|
||||
|
||||
// Split the Host and Port string
|
||||
@ -1126,7 +1137,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another
|
||||
if ua == "" {
|
||||
co.AgentViewCounter.Bump(38)
|
||||
co.AgentViewCounter.Bump(39)
|
||||
if c.Dev.DebugMode {
|
||||
var pre string
|
||||
for _, char := range req.UserAgent() {
|
||||
@ -1216,7 +1227,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
if strings.Contains(ua,"rv:11") {
|
||||
agent = 6
|
||||
}
|
||||
case 56:
|
||||
case 58:
|
||||
r.SuspiciousRequest(req,"Vuln Scanner")
|
||||
}
|
||||
|
||||
|
||||
@ -212,6 +212,7 @@
|
||||
"duckduckgo":"DuckDuckBot",
|
||||
"seznambot":"SeznamBot",
|
||||
"discord":"Discord",
|
||||
"telegram":"TelegramBot",
|
||||
"twitter":"Twitterbot",
|
||||
"cloudflare":"Cloudflare Alwayson",
|
||||
"archive_org":"Archive.org",
|
||||
@ -233,6 +234,7 @@
|
||||
"cocolyze":"Cocolyze",
|
||||
"babbar":"Babbar",
|
||||
"surdotly":"Surdotly",
|
||||
"domcop":"DomCopBot",
|
||||
"netcraft":"Netcraft",
|
||||
"blexbot":"BLEXBot",
|
||||
"burf":"Burf.co",
|
||||
@ -241,6 +243,7 @@
|
||||
"ccbot":"CCBot",
|
||||
"zgrab":"Zgrab App Scanner",
|
||||
"cloudsystemnetworks":"Nimbostratus / Cloud System Networks",
|
||||
"maui":"MauiBot",
|
||||
"curl":"curl",
|
||||
"python":"Python Bot",
|
||||
"go":"Go Bot",
|
||||
|
||||
@ -258,6 +258,7 @@ func main() {
|
||||
"duckduckgo",
|
||||
"seznambot",
|
||||
"discord",
|
||||
"telegram",
|
||||
"twitter",
|
||||
"facebook",
|
||||
"cloudflare",
|
||||
@ -281,6 +282,7 @@ func main() {
|
||||
"cocolyze",
|
||||
"babbar",
|
||||
"surdotly",
|
||||
"domcop",
|
||||
"netcraft",
|
||||
"blexbot",
|
||||
"burf",
|
||||
@ -289,6 +291,7 @@ func main() {
|
||||
"ccbot",
|
||||
"zgrab",
|
||||
"cloudsystemnetworks",
|
||||
"maui",
|
||||
"curl",
|
||||
"python",
|
||||
"go",
|
||||
@ -337,6 +340,7 @@ func main() {
|
||||
"Slackbot",
|
||||
"Slack",
|
||||
"Discordbot",
|
||||
"TelegramBot",
|
||||
"Twitterbot",
|
||||
"facebookexternalhit",
|
||||
"Facebot",
|
||||
@ -355,6 +359,7 @@ func main() {
|
||||
"Cocolyzebot",
|
||||
"Barkrowler",
|
||||
"SurdotlyBot",
|
||||
"DomCopBot",
|
||||
"NetcraftSurveyAgent",
|
||||
"BLEXBot",
|
||||
"Burf",
|
||||
@ -363,6 +368,7 @@ func main() {
|
||||
"CCBot",
|
||||
"zgrab",
|
||||
"Nimbostratus",
|
||||
"MauiBot",
|
||||
"curl",
|
||||
"python",
|
||||
"Go",
|
||||
@ -406,6 +412,7 @@ func main() {
|
||||
"Slackbot": "slackbot",
|
||||
"Slack": "slackbot",
|
||||
"Discordbot": "discord",
|
||||
"TelegramBot": "telegram",
|
||||
"Twitterbot": "twitter",
|
||||
"facebookexternalhit": "facebook",
|
||||
"Facebot": "facebook",
|
||||
@ -424,14 +431,17 @@ func main() {
|
||||
"Cocolyzebot": "cocolyze",
|
||||
"Barkrowler": "babbar",
|
||||
"SurdotlyBot": "surdotly",
|
||||
"DomCopBot":"domcop",
|
||||
"NetcraftSurveyAgent": "netcraft",
|
||||
"BLEXBot": "blexbot",
|
||||
"Burf": "burf",
|
||||
"AspiegelBot": "aspiegel",
|
||||
"PetalBot": "aspiegel",
|
||||
"RU_Bot": "mail_ru", // Mail.RU_Bot
|
||||
"CCBot": "ccbot",
|
||||
"zgrab": "zgrab",
|
||||
"Nimbostratus": "cloudsystemnetworks",
|
||||
"MauiBot":"maui",
|
||||
"curl": "curl",
|
||||
"python": "python",
|
||||
"Go": "go",
|
||||
@ -477,29 +487,29 @@ var RouteMap = map[string]interface{}{ {{range .AllRouteNames}}
|
||||
}
|
||||
|
||||
// ! NEVER RELY ON THESE REMAINING THE SAME BETWEEN COMMITS
|
||||
var routeMapEnum = map[string]int{ {{range $index, $element := .AllRouteNames}}
|
||||
"{{$element.Plain}}": {{$index}},{{end}}
|
||||
var routeMapEnum = map[string]int{ {{range $index, $el := .AllRouteNames}}
|
||||
"{{$el.Plain}}": {{$index}},{{end}}
|
||||
}
|
||||
var reverseRouteMapEnum = map[int]string{ {{range $index, $element := .AllRouteNames}}
|
||||
{{$index}}: "{{$element.Plain}}",{{end}}
|
||||
var reverseRouteMapEnum = map[int]string{ {{range $index, $el := .AllRouteNames}}
|
||||
{{$index}}: "{{$el.Plain}}",{{end}}
|
||||
}
|
||||
var osMapEnum = map[string]int{ {{range $index, $element := .AllOSNames}}
|
||||
"{{$element}}": {{$index}},{{end}}
|
||||
var osMapEnum = map[string]int{ {{range $index, $el := .AllOSNames}}
|
||||
"{{$el}}": {{$index}},{{end}}
|
||||
}
|
||||
var reverseOSMapEnum = map[int]string{ {{range $index, $element := .AllOSNames}}
|
||||
{{$index}}: "{{$element}}",{{end}}
|
||||
var reverseOSMapEnum = map[int]string{ {{range $index, $el := .AllOSNames}}
|
||||
{{$index}}: "{{$el}}",{{end}}
|
||||
}
|
||||
var agentMapEnum = map[string]int{ {{range $index, $element := .AllAgentNames}}
|
||||
"{{$element}}": {{$index}},{{end}}
|
||||
var agentMapEnum = map[string]int{ {{range $index, $el := .AllAgentNames}}
|
||||
"{{$el}}": {{$index}},{{end}}
|
||||
}
|
||||
var reverseAgentMapEnum = map[int]string{ {{range $index, $element := .AllAgentNames}}
|
||||
{{$index}}: "{{$element}}",{{end}}
|
||||
var reverseAgentMapEnum = map[int]string{ {{range $index, $el := .AllAgentNames}}
|
||||
{{$index}}: "{{$el}}",{{end}}
|
||||
}
|
||||
var markToAgent = map[string]string{ {{range $index, $element := .AllAgentMarkNames}}
|
||||
"{{$element}}": "{{index $.AllAgentMarks $element}}",{{end}}
|
||||
var markToAgent = map[string]string{ {{range $index, $el := .AllAgentMarkNames}}
|
||||
"{{$el}}": "{{index $.AllAgentMarks $el}}",{{end}}
|
||||
}
|
||||
var markToID = map[string]int{ {{range $index, $element := .AllAgentMarkNames}}
|
||||
"{{$element}}": {{index $.AllAgentMarkIDs $element}},{{end}}
|
||||
var markToID = map[string]int{ {{range $index, $el := .AllAgentMarkNames}}
|
||||
"{{$el}}": {{index $.AllAgentMarkIDs $el}},{{end}}
|
||||
}
|
||||
/*var agentRank = map[string]int{
|
||||
"opera":9,
|
||||
@ -569,11 +579,10 @@ func NewGenRouter(uploads http.Handler) (*GenRouter, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &GenRouter{
|
||||
UploadHandler: func(w http.ResponseWriter, req *http.Request) {
|
||||
UploadHandler: func(w http.ResponseWriter, r *http.Request) {
|
||||
writ := NewWriterIntercept(w)
|
||||
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,req)
|
||||
http.StripPrefix("/uploads/",uploads).ServeHTTP(writ,r)
|
||||
},
|
||||
extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
|
||||
requestLogger: log.New(f, "", log.LstdFlags),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user