diff --git a/bot_routes.go b/bot_routes.go
new file mode 100644
index 00000000..a9299dba
--- /dev/null
+++ b/bot_routes.go
@@ -0,0 +1,278 @@
+package main
+
+import (
+ "log"
+ "net/http"
+ "strconv"
+ "strings"
+
+ "./common"
+ "./query_gen/lib"
+)
+
+// TODO: Make this a static file somehow? Is it possible for us to put this file somewhere else?
+// TODO: Add an API so that plugins can register disallowed areas. E.g. /guilds/join for plugin_guilds
+func routeRobotsTxt(w http.ResponseWriter, r *http.Request) common.RouteError {
+ _, _ = w.Write([]byte(`User-agent: *
+Disallow: /panel/
+Disallow: /topics/create/
+Disallow: /user/edit/
+Disallow: /accounts/
+Disallow: /report/
+`))
+ return nil
+}
+
+var xmlInternalError = []byte(`
+A problem has occured`)
+var sitemapPageCap = 40000 // 40k, bump it up to 50k once we gzip this? Does brotli work on sitemaps?
+
+func writeXMLHeader(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/xml")
+ w.Write([]byte("\n"))
+}
+
+// TODO: Keep track of when a sitemap was last modifed and add a lastmod element for it
+func routeSitemapXml(w http.ResponseWriter, r *http.Request) common.RouteError {
+ var sslBit string
+ if common.Site.EnableSsl {
+ sslBit = "s"
+ }
+ var sitemapItem = func(path string) {
+ w.Write([]byte(`
+ http` + sslBit + `://` + common.Site.URL + "/" + path + `
+
+`))
+ }
+ writeXMLHeader(w, r)
+ w.Write([]byte("\n"))
+ sitemapItem("sitemaps/topics.xml")
+ sitemapItem("sitemaps/forums.xml")
+ //sitemapItem("sitemaps/users.xml")
+ w.Write([]byte(""))
+
+ return nil
+}
+
+type FuzzyRoute struct {
+ Path string
+ Handle func(http.ResponseWriter, *http.Request, int) common.RouteError
+}
+
+// TODO: Add a sitemap API and clean things up
+// TODO: ^-- Make sure that the API is concurrent
+// TODO: Add a social group sitemap
+var sitemapRoutes = map[string]func(http.ResponseWriter, *http.Request) common.RouteError{
+ "forums.xml": routeSitemapForums,
+ "topics.xml": routeSitemapTopics,
+}
+
+// TODO: Use a router capable of parsing this rather than hard-coding the logic in
+var fuzzySitemapRoutes = map[string]FuzzyRoute{
+ "topics_page_": FuzzyRoute{"topics_page_(%d).xml", routeSitemapTopic},
+}
+
+func sitemapSwitch(w http.ResponseWriter, r *http.Request) common.RouteError {
+ var path = r.URL.Path[len("/sitemaps/"):]
+ for name, fuzzy := range fuzzySitemapRoutes {
+ if strings.HasPrefix(path, name) && strings.HasSuffix(path, ".xml") {
+ var spath = strings.TrimPrefix(path, name)
+ spath = strings.TrimSuffix(spath, ".xml")
+ page, err := strconv.Atoi(spath)
+ if err != nil {
+ if common.Dev.DebugMode {
+ log.Printf("Unable to convert string '%s' to integer in fuzzy route", spath)
+ }
+ return common.NotFound(w, r)
+ }
+ return fuzzy.Handle(w, r, page)
+ }
+ }
+
+ route, ok := sitemapRoutes[path]
+ if !ok {
+ return common.NotFound(w, r)
+ }
+ return route(w, r)
+}
+
+func routeSitemapForums(w http.ResponseWriter, r *http.Request) common.RouteError {
+ var sslBit string
+ if common.Site.EnableSsl {
+ sslBit = "s"
+ }
+ var sitemapItem = func(path string) {
+ w.Write([]byte(`
+ http` + sslBit + `://` + common.Site.URL + path + `
+
+`))
+ }
+
+ group, err := common.Groups.Get(common.GuestUser.Group)
+ if err != nil {
+ log.Print("The guest group doesn't exist for some reason")
+ // TODO: Add XML error handling to errors.go
+ w.WriteHeader(500)
+ w.Write(xmlInternalError)
+ return common.HandledRouteError()
+ }
+
+ writeXMLHeader(w, r)
+ w.Write([]byte("\n"))
+
+ for _, fid := range group.CanSee {
+ // Avoid data races by copying the struct into something we can freely mold without worrying about breaking something somewhere else
+ var forum = common.Forums.DirtyGet(fid).Copy()
+ if forum.ParentID == 0 && forum.Name != "" && forum.Active {
+ sitemapItem(common.BuildForumURL(common.NameToSlug(forum.Name), forum.ID))
+ }
+ }
+
+ w.Write([]byte(""))
+ return nil
+}
+
+// TODO: Add a global ratelimit. 10 50MB files (smaller if compressed better) per minute?
+// ? We might have problems with banned users, if they have fewer ViewTopic permissions than guests as they'll be able to see this list. Then again, a banned user could just logout to see it
+func routeSitemapTopics(w http.ResponseWriter, r *http.Request) common.RouteError {
+ var sslBit string
+ if common.Site.EnableSsl {
+ sslBit = "s"
+ }
+ var sitemapItem = func(path string) {
+ w.Write([]byte(`
+ http` + sslBit + `://` + common.Site.URL + "/" + path + `
+
+`))
+ }
+ writeXMLHeader(w, r)
+
+ group, err := common.Groups.Get(common.GuestUser.Group)
+ if err != nil {
+ log.Print("The guest group doesn't exist for some reason")
+ // TODO: Add XML error handling to errors.go
+ w.WriteHeader(500)
+ w.Write(xmlInternalError)
+ return common.HandledRouteError()
+ }
+
+ var argList []interface{}
+ var qlist string
+ for _, fid := range group.CanSee {
+ forum := common.Forums.DirtyGet(fid)
+ if forum.Name != "" && forum.Active {
+ argList = append(argList, strconv.Itoa(fid))
+ qlist += "?,"
+ }
+ }
+ if qlist != "" {
+ qlist = qlist[0 : len(qlist)-1]
+ }
+
+ // TODO: Abstract this
+ topicCountStmt, err := qgen.Builder.SimpleCount("topics", "parentID IN("+qlist+")", "")
+ if err != nil {
+ // TODO: Add XML error handling to errors.go
+ w.WriteHeader(500)
+ w.Write(xmlInternalError)
+ common.LogError(err)
+ return common.HandledRouteError()
+ }
+ defer topicCountStmt.Close()
+
+ var topicCount int
+ err = topicCountStmt.QueryRow(argList...).Scan(&topicCount)
+ if err != nil && err != ErrNoRows {
+ // TODO: Add XML error handling to errors.go
+ w.WriteHeader(500)
+ w.Write(xmlInternalError)
+ common.LogError(err)
+ return common.HandledRouteError()
+ }
+
+ var pageCount = topicCount / sitemapPageCap
+ //log.Print("topicCount", topicCount)
+ //log.Print("pageCount", pageCount)
+ w.Write([]byte("\n"))
+ for i := 0; i <= pageCount; i++ {
+ sitemapItem("sitemaps/topics_page_" + strconv.Itoa(i) + ".xml")
+ }
+ w.Write([]byte(""))
+ return nil
+}
+
+func routeSitemapTopic(w http.ResponseWriter, r *http.Request, page int) common.RouteError {
+ /*var sslBit string
+ if common.Site.EnableSsl {
+ sslBit = "s"
+ }
+ var sitemapItem = func(path string) {
+ w.Write([]byte(`
+ http` + sslBit + `://` + common.Site.URL + "/" + path + `
+
+ `))
+ }*/
+
+ group, err := common.Groups.Get(common.GuestUser.Group)
+ if err != nil {
+ log.Print("The guest group doesn't exist for some reason")
+ // TODO: Add XML error handling to errors.go
+ w.WriteHeader(500)
+ w.Write(xmlInternalError)
+ return common.HandledRouteError()
+ }
+
+ var argList []interface{}
+ var qlist string
+ for _, fid := range group.CanSee {
+ forum := common.Forums.DirtyGet(fid)
+ if forum.Name != "" && forum.Active {
+ argList = append(argList, strconv.Itoa(fid))
+ qlist += "?,"
+ }
+ }
+ if qlist != "" {
+ qlist = qlist[0 : len(qlist)-1]
+ }
+
+ // TODO: Abstract this
+ topicCountStmt, err := qgen.Builder.SimpleCount("topics", "parentID IN("+qlist+")", "")
+ if err != nil {
+ // TODO: Add XML error handling to errors.go
+ w.WriteHeader(500)
+ w.Write(xmlInternalError)
+ common.LogError(err)
+ return common.HandledRouteError()
+ }
+ defer topicCountStmt.Close()
+
+ var topicCount int
+ err = topicCountStmt.QueryRow(argList...).Scan(&topicCount)
+ if err != nil && err != ErrNoRows {
+ // TODO: Add XML error handling to errors.go
+ w.WriteHeader(500)
+ w.Write(xmlInternalError)
+ common.LogError(err)
+ return common.HandledRouteError()
+ }
+
+ var pageCount = topicCount / sitemapPageCap
+ //log.Print("topicCount", topicCount)
+ //log.Print("pageCount", pageCount)
+ //log.Print("page",page)
+ if page > pageCount {
+ page = pageCount
+ }
+
+ writeXMLHeader(w, r)
+ w.Write([]byte("\n"))
+
+ w.Write([]byte(""))
+ return nil
+}
+
+func routeSitemapUsers(w http.ResponseWriter, r *http.Request) common.RouteError {
+ writeXMLHeader(w, r)
+ w.Write([]byte("\n"))
+ return nil
+}
diff --git a/common/counters.go b/common/counters.go
index 7d37809b..8853dee5 100644
--- a/common/counters.go
+++ b/common/counters.go
@@ -10,7 +10,8 @@ import (
)
var GlobalViewCounter *ChunkedViewCounter
-var RouteViewCounter *RouteViewCounterImpl
+var RouteViewCounter *DefaultRouteViewCounter
+var TopicViewCounter *DefaultTopicViewCounter
type ChunkedViewCounter struct {
buckets [2]int64
@@ -64,18 +65,18 @@ type RWMutexCounterBucket struct {
}
// The name of the struct clashes with the name of the variable, so we're adding Impl to the end
-type RouteViewCounterImpl struct {
+type DefaultRouteViewCounter struct {
routeBuckets []*RWMutexCounterBucket //[RouteID]count
insert *sql.Stmt
}
-func NewRouteViewCounter() (*RouteViewCounterImpl, error) {
+func NewDefaultRouteViewCounter() (*DefaultRouteViewCounter, error) {
acc := qgen.Builder.Accumulator()
var routeBuckets = make([]*RWMutexCounterBucket, len(routeMapEnum))
for bucketID, _ := range routeBuckets {
routeBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
}
- counter := &RouteViewCounterImpl{
+ counter := &DefaultRouteViewCounter{
routeBuckets: routeBuckets,
insert: acc.Insert("viewchunks").Columns("count, createdAt, route").Fields("?,UTC_TIMESTAMP(),?").Prepare(),
}
@@ -84,7 +85,7 @@ func NewRouteViewCounter() (*RouteViewCounterImpl, error) {
return counter, acc.FirstError()
}
-func (counter *RouteViewCounterImpl) Tick() error {
+func (counter *DefaultRouteViewCounter) Tick() error {
for routeID, routeBucket := range counter.routeBuckets {
var count int
routeBucket.RLock()
@@ -100,7 +101,7 @@ func (counter *RouteViewCounterImpl) Tick() error {
return nil
}
-func (counter *RouteViewCounterImpl) insertChunk(count int, route int) error {
+func (counter *DefaultRouteViewCounter) insertChunk(count int, route int) error {
if count == 0 {
return nil
}
@@ -110,7 +111,7 @@ func (counter *RouteViewCounterImpl) insertChunk(count int, route int) error {
return err
}
-func (counter *RouteViewCounterImpl) Bump(route int) {
+func (counter *DefaultRouteViewCounter) Bump(route int) {
// TODO: Test this check
log.Print("counter.routeBuckets[route]: ", counter.routeBuckets[route])
if len(counter.routeBuckets) <= route {
@@ -140,7 +141,7 @@ type ForumViewCounter struct {
}*/
// TODO: Use two odd-even maps for now, and move to something more concurrent later, maybe a sharded map?
-type TopicViewCounter struct {
+type DefaultTopicViewCounter struct {
oddTopics map[int]*RWMutexCounterBucket // map[tid]struct{counter,sync.RWMutex}
evenTopics map[int]*RWMutexCounterBucket
oddLock sync.RWMutex
@@ -149,19 +150,19 @@ type TopicViewCounter struct {
update *sql.Stmt
}
-func NewTopicViewCounter() (*TopicViewCounter, error) {
+func NewDefaultTopicViewCounter() (*DefaultTopicViewCounter, error) {
acc := qgen.Builder.Accumulator()
- counter := &TopicViewCounter{
+ counter := &DefaultTopicViewCounter{
oddTopics: make(map[int]*RWMutexCounterBucket),
evenTopics: make(map[int]*RWMutexCounterBucket),
- update: acc.Update("topics").Set("views = ?").Where("tid = ?").Prepare(), // TODO: Add the views column to the topics table
+ update: acc.Update("topics").Set("views = views + ?").Where("tid = ?").Prepare(),
}
AddScheduledFifteenMinuteTask(counter.Tick) // There could be a lot of routes, so we don't want to be running this every second
//AddScheduledSecondTask(counter.Tick)
return counter, acc.FirstError()
}
-func (counter *TopicViewCounter) Tick() error {
+func (counter *DefaultTopicViewCounter) Tick() error {
counter.oddLock.RLock()
for topicID, topic := range counter.oddTopics {
var count int
@@ -191,7 +192,7 @@ func (counter *TopicViewCounter) Tick() error {
return nil
}
-func (counter *TopicViewCounter) insertChunk(count int, topicID int) error {
+func (counter *DefaultTopicViewCounter) insertChunk(count int, topicID int) error {
if count == 0 {
return nil
}
@@ -200,7 +201,7 @@ func (counter *TopicViewCounter) insertChunk(count int, topicID int) error {
return err
}
-func (counter *TopicViewCounter) Bump(topicID int) {
+func (counter *DefaultTopicViewCounter) Bump(topicID int) {
// Is the ID even?
if topicID%2 == 0 {
counter.evenLock.Lock()
diff --git a/common/utils.go b/common/utils.go
index 55e644e7..cc6ef524 100644
--- a/common/utils.go
+++ b/common/utils.go
@@ -181,6 +181,8 @@ func ConvertFriendlyUnit(num int) (int, string) {
}
}
+// TODO: Make slugs optional for certain languages across the entirety of Gosora?
+// TODO: Let plugins replace NameToSlug and the URL building logic with their own
func NameToSlug(name string) (slug string) {
name = strings.TrimSpace(name)
name = strings.Replace(name, " ", " ", -1)
diff --git a/gen_router.go b/gen_router.go
index a626f87f..3d0ee519 100644
--- a/gen_router.go
+++ b/gen_router.go
@@ -783,6 +783,12 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if err != nil {
router.handleError(err,w,req,user)
}
+ /*case "/sitemaps": // TODO: Count these views
+ req.URL.Path += extraData
+ err = sitemapSwitch(w,req)
+ if err != nil {
+ router.handleError(err,w,req,user)
+ }*/
case "/uploads":
if extraData == "" {
common.NotFound(w,req)
@@ -801,6 +807,12 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
router.handleError(err,w,req,user)
}
return
+ /*case "sitemap.xml":
+ err = routeSitemapXml(w,req) // TODO: Count these views
+ if err != nil {
+ router.handleError(err,w,req,user)
+ }
+ return*/
}
if extraData != "" {
diff --git a/main.go b/main.go
index 1b3cd9b4..e6128765 100644
--- a/main.go
+++ b/main.go
@@ -84,7 +84,11 @@ func afterDBInit() (err error) {
if err != nil {
return err
}
- common.RouteViewCounter, err = common.NewRouteViewCounter()
+ common.RouteViewCounter, err = common.NewDefaultRouteViewCounter()
+ if err != nil {
+ return err
+ }
+ common.TopicViewCounter, err = common.NewDefaultTopicViewCounter()
if err != nil {
return err
}
diff --git a/query_gen/tables.go b/query_gen/tables.go
index 4393129a..9f935ac3 100644
--- a/query_gen/tables.go
+++ b/query_gen/tables.go
@@ -159,7 +159,7 @@ func createTables(adapter qgen.Adapter) error {
qgen.DBTableColumn{"postCount", "int", 0, false, false, "1"},
qgen.DBTableColumn{"likeCount", "int", 0, false, false, "0"},
qgen.DBTableColumn{"words", "int", 0, false, false, "0"},
- //qgen.DBTableColumn{"views", "int", 0, false, false, "0"},
+ qgen.DBTableColumn{"views", "int", 0, false, false, "0"},
qgen.DBTableColumn{"css_class", "varchar", 100, false, false, "''"},
qgen.DBTableColumn{"data", "varchar", 200, false, false, "''"},
},
diff --git a/router_gen/main.go b/router_gen/main.go
index 83c80876..72db1971 100644
--- a/router_gen/main.go
+++ b/router_gen/main.go
@@ -283,6 +283,12 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
var err common.RouteError
switch(prefix) {` + out + `
+ /*case "/sitemaps": // TODO: Count these views
+ req.URL.Path += extraData
+ err = sitemapSwitch(w,req)
+ if err != nil {
+ router.handleError(err,w,req,user)
+ }*/
case "/uploads":
if extraData == "" {
common.NotFound(w,req)
@@ -301,6 +307,12 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
router.handleError(err,w,req,user)
}
return
+ /*case "sitemap.xml":
+ err = routeSitemapXml(w,req) // TODO: Count these views
+ if err != nil {
+ router.handleError(err,w,req,user)
+ }
+ return*/
}
if extraData != "" {
diff --git a/routes.go b/routes.go
index d14ee323..bd39a64e 100644
--- a/routes.go
+++ b/routes.go
@@ -7,11 +7,10 @@
package main
import (
- "log"
- //"fmt"
"bytes"
"html"
"io"
+ "log"
"net/http"
"strconv"
"strings"
@@ -74,19 +73,6 @@ func routeStatic(w http.ResponseWriter, r *http.Request) {
// Other options instead of io.Copy: io.CopyN(), w.Write(), http.ServeContent()
}
-// TODO: Make this a static file somehow? Is it possible for us to put this file somewhere else?
-// TODO: Add a sitemap
-// TODO: Add an API so that plugins can register disallowed areas. E.g. /guilds/join for plugin_guilds
-func routeRobotsTxt(w http.ResponseWriter, r *http.Request) common.RouteError {
- _, _ = w.Write([]byte(`User-agent: *
-Disallow: /panel/
-Disallow: /topics/create/
-Disallow: /user/edit/
-Disallow: /accounts/
-`))
- return nil
-}
-
func routeOverview(w http.ResponseWriter, r *http.Request, user common.User) common.RouteError {
headerVars, ferr := common.UserCheck(w, r, &user)
if ferr != nil {
@@ -150,7 +136,7 @@ func routeTopics(w http.ResponseWriter, r *http.Request, user common.User) commo
return common.LocalError("Something weird happened", w, r, user)
}
- // TODO: Make CanSee a method on *Group with a canSee field?
+ // TODO: Make CanSee a method on *Group with a canSee field? Have a CanSee method on *User to cover the case of superadmins?
var canSee []int
if user.IsSuperAdmin {
canSee, err = common.Forums.GetAllVisibleIDs()
@@ -168,19 +154,16 @@ func routeTopics(w http.ResponseWriter, r *http.Request, user common.User) commo
for _, fid := range canSee {
forum := common.Forums.DirtyGet(fid)
if forum.Name != "" && forum.Active {
- if forum.ParentType == "" || forum.ParentType == "forum" {
- // Optimise Quick Topic away for guests
- if user.Loggedin {
- fcopy := forum.Copy()
- // TODO: Add a hook here for plugin_guilds
- forumList = append(forumList, fcopy)
- }
+ // This bit's for quick topic, as we don't want unbound forums (e.g. ones in plugin_socialgroups) showing up
+ if (forum.ParentType == "" || forum.ParentType == "forum") && user.Loggedin {
+ fcopy := forum.Copy()
+ // TODO: Add a hook here for plugin_guilds
+ forumList = append(forumList, fcopy)
}
// ? - Should we be showing plugin_guilds posts on /topics/?
// ? - Would it be useful, if we could post in social groups from /topics/?
argList = append(argList, strconv.Itoa(fid))
qlist += "?,"
-
}
}
@@ -190,6 +173,7 @@ func routeTopics(w http.ResponseWriter, r *http.Request, user common.User) commo
}
qlist = qlist[0 : len(qlist)-1]
+ // TODO: Abstract this
topicCountStmt, err := qgen.Builder.SimpleCount("topics", "parentID IN("+qlist+")", "")
if err != nil {
return common.InternalError(err, w, r)
diff --git a/routes/filler.txt b/routes/filler.txt
new file mode 100644
index 00000000..20e14b1e
--- /dev/null
+++ b/routes/filler.txt
@@ -0,0 +1 @@
+This file is here so that Git will include this folder in the repository.
\ No newline at end of file
diff --git a/schema/mssql/query_topics.sql b/schema/mssql/query_topics.sql
index 7113bf33..3e9a789d 100644
--- a/schema/mssql/query_topics.sql
+++ b/schema/mssql/query_topics.sql
@@ -14,6 +14,7 @@ CREATE TABLE [topics] (
[postCount] int DEFAULT 1 not null,
[likeCount] int DEFAULT 0 not null,
[words] int DEFAULT 0 not null,
+ [views] int DEFAULT 0 not null,
[css_class] nvarchar (100) DEFAULT '' not null,
[data] nvarchar (200) DEFAULT '' not null,
primary key([tid])
diff --git a/schema/mysql/query_topics.sql b/schema/mysql/query_topics.sql
index 6475add6..9713466c 100644
--- a/schema/mysql/query_topics.sql
+++ b/schema/mysql/query_topics.sql
@@ -14,6 +14,7 @@ CREATE TABLE `topics` (
`postCount` int DEFAULT 1 not null,
`likeCount` int DEFAULT 0 not null,
`words` int DEFAULT 0 not null,
+ `views` int DEFAULT 0 not null,
`css_class` varchar(100) DEFAULT '' not null,
`data` varchar(200) DEFAULT '' not null,
primary key(`tid`)
diff --git a/schema/pgsql/query_topics.sql b/schema/pgsql/query_topics.sql
index 0c8d84f8..9d256376 100644
--- a/schema/pgsql/query_topics.sql
+++ b/schema/pgsql/query_topics.sql
@@ -14,6 +14,7 @@ CREATE TABLE `topics` (
`postCount` int DEFAULT 1 not null,
`likeCount` int DEFAULT 0 not null,
`words` int DEFAULT 0 not null,
+ `views` int DEFAULT 0 not null,
`css_class` varchar (100) DEFAULT '' not null,
`data` varchar (200) DEFAULT '' not null,
primary key(`tid`)
diff --git a/template_list.go b/template_list.go
index a5997f9c..0d43f1c6 100644
--- a/template_list.go
+++ b/template_list.go
@@ -548,7 +548,7 @@ var profile_14 = []byte(`
+var profile_16 = []byte(`&type=user" class="profile_menu_item report_item" aria-label="Report User" title="Report User">
@@ -559,7 +559,7 @@ var profile_16 = []byte(`&type=user" class="profile_menu_item report_item">Repor
var profile_17 = []byte(`