From 49a3837621ddda39ac67036be4586ac45e599f41 Mon Sep 17 00:00:00 2001
From: Azareal <azareal@users.noreply.github.com>
Date: Sat, 4 Apr 2020 19:03:11 +1000
Subject: [PATCH] add netcraft user agent and alt mj12bot matcher

---
 gen_router.go      | 64 ++++++++++++++++++++++++++++------------------
 langs/english.json |  1 +
 router_gen/main.go | 35 ++++++++++++++-----------
 3 files changed, 60 insertions(+), 40 deletions(-)

diff --git a/gen_router.go b/gen_router.go
index f44d42a1..8d918dda 100644
--- a/gen_router.go
+++ b/gen_router.go
@@ -617,14 +617,17 @@ var agentMapEnum = map[string]int{
 	"ahrefs": 43,
 	"proximic": 44,
 	"majestic": 45,
-	"blexbot": 46,
-	"aspiegel": 47,
-	"mail_ru": 48,
-	"ccbot": 49,
-	"zgrab": 50,
-	"curl": 51,
-	"python": 52,
-	"go": 53,
+	"netcraft": 46,
+	"blexbot": 47,
+	"burf": 48,
+	"aspiegel": 49,
+	"mail_ru": 50,
+	"ccbot": 51,
+	"zgrab": 52,
+	"cloudsystemnetworks": 53,
+	"curl": 54,
+	"python": 55,
+	"go": 56,
 }
 var reverseAgentMapEnum = map[int]string{ 
 	0: "unknown",
@@ -673,14 +676,17 @@ var reverseAgentMapEnum = map[int]string{
 	43: "ahrefs",
 	44: "proximic",
 	45: "majestic",
-	46: "blexbot",
-	47: "aspiegel",
-	48: "mail_ru",
-	49: "ccbot",
-	50: "zgrab",
-	51: "curl",
-	52: "python",
-	53: "go",
+	46: "netcraft",
+	47: "blexbot",
+	48: "burf",
+	49: "aspiegel",
+	50: "mail_ru",
+	51: "ccbot",
+	52: "zgrab",
+	53: "cloudsystemnetworks",
+	54: "curl",
+	55: "python",
+	56: "go",
 }
 var markToAgent = map[string]string{ 
 	"OPR": "opera",
@@ -729,11 +735,15 @@ var markToAgent = map[string]string{
 	"AhrefsBot": "ahrefs",
 	"proximic": "proximic",
 	"MJ12bot": "majestic",
+	"mj12bot": "majestic",
+	"NetcraftSurveyAgent": "netcraft",
 	"BLEXBot": "blexbot",
+	"Burf": "burf",
 	"AspiegelBot": "aspiegel",
 	"RU_Bot": "mail_ru",
 	"CCBot": "ccbot",
 	"zgrab": "zgrab",
+	"Nimbostratus": "cloudsystemnetworks",
 	"curl": "curl",
 	"python": "python",
 	"Go": "go",
@@ -785,14 +795,18 @@ var markToID = map[string]int{
 	"AhrefsBot": 43,
 	"proximic": 44,
 	"MJ12bot": 45,
-	"BLEXBot": 46,
-	"AspiegelBot": 47,
-	"RU_Bot": 48,
-	"CCBot": 49,
-	"zgrab": 50,
-	"curl": 51,
-	"python": 52,
-	"Go": 53,
+	"mj12bot": 45,
+	"NetcraftSurveyAgent": 46,
+	"BLEXBot": 47,
+	"Burf": 48,
+	"AspiegelBot": 49,
+	"RU_Bot": 50,
+	"CCBot": 51,
+	"zgrab": 52,
+	"Nimbostratus": 53,
+	"curl": 54,
+	"python": 55,
+	"Go": 56,
 }
 /*var agentRank = map[string]int{
 	"opera":9,
@@ -1164,7 +1178,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			if strings.Contains(ua,"rv:11") {
 				agent = 6
 			}
-		case 50:
+		case 52:
 			r.SuspiciousRequest(req,"Vulnerability Scanner")
 		}
 		
diff --git a/langs/english.json b/langs/english.json
index 4873e6de..ed952d06 100644
--- a/langs/english.json
+++ b/langs/english.json
@@ -229,6 +229,7 @@
 		"ahrefs":"Ahrefs",
 		"proximic":"Comscore",
 		"majestic":"MJ12bot",
+		"netcraft":"Netcraft",
 		"blexbot":"BLEXBot",
 		"burf":"Burf.co",
 		"aspiegel":"Aspiegel",
diff --git a/router_gen/main.go b/router_gen/main.go
index df9bb6a0..52a6b059 100644
--- a/router_gen/main.go
+++ b/router_gen/main.go
@@ -277,6 +277,7 @@ func main() {
 		"ahrefs",
 		"proximic",
 		"majestic",
+		"netcraft",
 		"blexbot",
 		"burf",
 		"aspiegel",
@@ -343,6 +344,8 @@ func main() {
 		"AhrefsBot",
 		"proximic",
 		"MJ12bot",
+		"mj12bot",
+		"NetcraftSurveyAgent",
 		"BLEXBot",
 		"Burf",
 		"AspiegelBot",
@@ -399,21 +402,23 @@ func main() {
 		"mattermost":          "mattermost",
 		"ia_archiver":         "alexa",
 
-		"SemrushBot":   "semrush",
-		"DotBot":       "dotbot",
-		"AhrefsBot":    "ahrefs",
-		"proximic":     "proximic",
-		"MJ12bot":      "majestic",
-		"BLEXBot":      "blexbot",
-		"Burf":         "burf",
-		"AspiegelBot":  "aspiegel",
-		"RU_Bot":       "mail_ru", // Mail.RU_Bot
-		"CCBot":        "ccbot",
-		"zgrab":        "zgrab",
-		"Nimbostratus": "cloudsystemnetworks",
-		"curl":         "curl",
-		"python":       "python",
-		"Go":           "go",
+		"SemrushBot":          "semrush",
+		"DotBot":              "dotbot",
+		"AhrefsBot":           "ahrefs",
+		"proximic":            "proximic",
+		"MJ12bot":             "majestic",
+		"mj12bot":             "majestic",
+		"NetcraftSurveyAgent": "netcraft",
+		"BLEXBot":             "blexbot",
+		"Burf":                "burf",
+		"AspiegelBot":         "aspiegel",
+		"RU_Bot":              "mail_ru", // Mail.RU_Bot
+		"CCBot":               "ccbot",
+		"zgrab":               "zgrab",
+		"Nimbostratus":        "cloudsystemnetworks",
+		"curl":                "curl",
+		"python":              "python",
+		"Go":                  "go",
 	}
 
 	tmplVars.AllAgentMarkIDs = make(map[string]int)