a
/
gosora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

track favicon stats 

experiment with tracking average route performance
temporary error route stub
optimise dumprequest
add DisableAnalytics config setting
fix double hyphens in slugs being mistaken for sql injection
more querygen tests

You wil need to run the updater / patcher for this commit.
This commit is contained in:
Azareal 2020-02-26 20:34:38 +10:00
parent 04fdf4c318
commit ea1037bd63
19 changed files with 841 additions and 705 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/query_gen/tables.go
View File

@ -662,6 +662,7 @@ func createTables(adapter qgen.Adapter) (err error) {
createTable("viewchunks", "", "",
[]tC{
tC{"count", "int", 0, false, false, "0"},
tC{"avg", "int", 0, false, false, "0"},
tC{"createdAt", "datetime", 0, false, false, ""},
tC{"route", "varchar", 200, false, false, ""}, // todo: set a default empty here
}, nil,

2
common/counters/agents.go
View File

@ -50,7 +50,7 @@ func (co *DefaultAgentViewCounter) insertChunk(count int64, agent int) error {
func (co *DefaultAgentViewCounter) Bump(agent int) {
// TODO: Test this check
c.DebugDetail("co.buckets[", agent, "]: ", co.buckets[agent])
c.DebugDetail("buckets[", agent, "]: ", co.buckets[agent])
if len(co.buckets) <= agent || agent < 0 {
return
}

2
common/counters/langs.go
View File

@ -157,7 +157,7 @@ func (co *DefaultLangViewCounter) Bump(langCode string) (validCode bool) {
}
// TODO: Test this check
c.DebugDetail("co.buckets[", id, "]: ", co.buckets[id])
c.DebugDetail("buckets[", id, "]: ", co.buckets[id])
if len(co.buckets) <= id || id < 0 {
return validCode
}

2
common/counters/referrers.go
View File

@ -85,7 +85,7 @@ func (ref *DefaultReferrerTracker) insertChunk(referrer string, count int64) err
if count == 0 {
return nil
}
c.DebugDetailf("Inserting a vchunk with a count of %d for referrer %s", count, referrer)
c.DebugDetailf("Inserting a vchunk with a count of %d for ref %s", count, referrer)
_, err := ref.insert.Exec(count, referrer)
return err
}

101
common/counters/routes.go
View File

@ -2,6 +2,8 @@ package counters
import (
"database/sql"
"sync"
"time"
c "github.com/Azareal/Gosora/common"
qgen "github.com/Azareal/Gosora/query_gen"
@ -10,56 +12,68 @@ import (
var RouteViewCounter *DefaultRouteViewCounter
type RVBucket struct {
counter int
avg int
sync.Mutex
}
// TODO: Make this lockless?
type DefaultRouteViewCounter struct {
buckets []*RWMutexCounterBucket //[RouteID]count
buckets []*RVBucket //[RouteID]count
insert *sql.Stmt
insert5 *sql.Stmt
}
func NewDefaultRouteViewCounter(acc *qgen.Accumulator) (*DefaultRouteViewCounter, error) {
routeBuckets := make([]*RWMutexCounterBucket, len(routeMapEnum))
routeBuckets := make([]*RVBucket, len(routeMapEnum))
for bucketID, _ := range routeBuckets {
routeBuckets[bucketID] = &RWMutexCounterBucket{counter: 0}
routeBuckets[bucketID] = &RVBucket{counter: 0, avg: 0}
}
fields := "?,UTC_TIMESTAMP(),?"
fields := "?,?,UTC_TIMESTAMP(),?"
co := &DefaultRouteViewCounter{
buckets: routeBuckets,
insert: acc.Insert("viewchunks").Columns("count,createdAt,route").Fields(fields).Prepare(),
insert5: acc.BulkInsert("viewchunks").Columns("count,createdAt,route").Fields(fields,fields,fields,fields,fields).Prepare(),
insert: acc.Insert("viewchunks").Columns("count,avg,createdAt,route").Fields(fields).Prepare(),
insert5: acc.BulkInsert("viewchunks").Columns("count,avg,createdAt,route").Fields(fields, fields, fields, fields, fields).Prepare(),
}
if !c.Config.DisableAnalytics {
c.AddScheduledFifteenMinuteTask(co.Tick) // There could be a lot of routes, so we don't want to be running this every second
//c.AddScheduledSecondTask(co.Tick)
c.AddShutdownTask(co.Tick)
}
c.AddScheduledFifteenMinuteTask(co.Tick) // There could be a lot of routes, so we don't want to be running this every second
//c.AddScheduledSecondTask(co.Tick)
c.AddShutdownTask(co.Tick)
return co, acc.FirstError()
}
type RVCount struct {
RouteID int
Count int
Count int
Avg int
}
func (co *DefaultRouteViewCounter) Tick() error {
func (co *DefaultRouteViewCounter) Tick() (err error) {
var tb []RVCount
for routeID, b := range co.buckets {
var count int
b.RLock()
var count, avg int
b.Lock()
count = b.counter
b.counter = 0
b.RUnlock()
avg = b.avg
b.avg = 0
b.Unlock()
if count == 0 {
continue
}
tb = append(tb, RVCount{routeID,count})
tb = append(tb, RVCount{routeID, count, avg})
}
// TODO: Expand on this?
var i int
if len(tb) >= 5 {
for ; len(tb) > (i+5); i += 5 {
err := co.insert5Chunk(tb[i:i+5])
for ; len(tb) > (i + 5); i += 5 {
err := co.insert5Chunk(tb[i : i+5])
if err != nil {
c.DebugLogf("tb: %+v\n", tb)
c.DebugLog("i: ", i)
@ -69,7 +83,8 @@ func (co *DefaultRouteViewCounter) Tick() error {
}
for ; len(tb) > i; i++ {
err := co.insertChunk(tb[i].Count, tb[i].RouteID)
it := tb[i]
err = co.insertChunk(it.Count, it.Avg, it.RouteID)
if err != nil {
c.DebugLogf("tb: %+v\n", tb)
c.DebugLog("i: ", i)
@ -80,32 +95,40 @@ func (co *DefaultRouteViewCounter) Tick() error {
return nil
}
func (co *DefaultRouteViewCounter) insertChunk(count, route int) error {
func (co *DefaultRouteViewCounter) insertChunk(count, avg, route int) error {
routeName := reverseRouteMapEnum[route]
c.DebugLogf("Inserting a vchunk with a count of %d for route %s (%d)", count, routeName, route)
_, err := co.insert.Exec(count, routeName)
c.DebugLogf("Inserting a vchunk with a count of %d, avg of %d for route %s (%d)", count, avg, routeName, route)
_, err := co.insert.Exec(count, avg, routeName)
return err
}
func (co *DefaultRouteViewCounter) insert5Chunk(rvs []RVCount) error {
args := make([]interface{}, len(rvs) * 2)
args := make([]interface{}, len(rvs)*3)
i := 0
for _, rv := range rvs {
routeName := reverseRouteMapEnum[rv.RouteID]
c.DebugLogf("Queueing a vchunk with a count of %d for routes %s (%d)", rv.Count, routeName, rv.RouteID)
if rv.Avg == 0 {
c.DebugLogf("Queueing a vchunk with a count of %d for routes %s (%d)", rv.Count, routeName, rv.RouteID)
} else {
c.DebugLogf("Queueing a vchunk with count %d, avg %d for routes %s (%d)", rv.Count, rv.Avg, routeName, rv.RouteID)
}
args[i] = rv.Count
args[i+1] = routeName
i += 2
args[i+1] = rv.Avg
args[i+2] = routeName
i += 3
}
c.DebugLogf("args: %+v\n", args)
c.DebugDetailf("args: %+v\n", args)
_, err := co.insert5.Exec(args...)
return err
}
func (co *DefaultRouteViewCounter) Bump(route int) {
if c.Config.DisableAnalytics {
return
}
// TODO: Test this check
b := co.buckets[route]
c.DebugDetail("co.buckets[", route, "]: ", b)
c.DebugDetail("buckets[", route, "]: ", b)
if len(co.buckets) <= route || route < 0 {
return
}
@ -114,3 +137,27 @@ func (co *DefaultRouteViewCounter) Bump(route int) {
b.counter++
b.Unlock()
}
// TODO: Eliminate the lock?
func (co *DefaultRouteViewCounter) Bump2(route int, t time.Time) {
if c.Config.DisableAnalytics {
return
}
// TODO: Test this check
b := co.buckets[route]
c.DebugDetail("buckets[", route, "]: ", b)
if len(co.buckets) <= route || route < 0 {
return
}
micro := int(time.Since(t).Microseconds())
b.Lock()
b.counter++
if micro != b.avg {
if b.avg == 0 {
b.avg = micro
} else {
b.avg = (micro + b.avg) / 2
}
}
b.Unlock()
}

6
common/counters/topics.go
View File

@ -5,7 +5,7 @@ import (
"sync/atomic"
c "github.com/Azareal/Gosora/common"
"github.com/Azareal/Gosora/query_gen"
qgen "github.com/Azareal/Gosora/query_gen"
"github.com/pkg/errors"
)
@ -22,7 +22,7 @@ func NewTopicCounter() (*DefaultTopicCounter, error) {
acc := qgen.NewAcc()
co := &DefaultTopicCounter{
currentBucket: 0,
insert: acc.Insert("topicchunks").Columns("count, createdAt").Fields("?,UTC_TIMESTAMP()").Prepare(),
insert: acc.Insert("topicchunks").Columns("count,createdAt").Fields("?,UTC_TIMESTAMP()").Prepare(),
}
c.AddScheduledFifteenMinuteTask(co.Tick)
//c.AddScheduledSecondTask(co.Tick)
@ -44,7 +44,7 @@ func (co *DefaultTopicCounter) Tick() (err error) {
atomic.AddInt64(&co.buckets[oldBucket], -previousViewChunk)
err = co.insertChunk(previousViewChunk)
if err != nil {
return errors.Wrap(errors.WithStack(err),"topics counter")
return errors.Wrap(errors.WithStack(err), "topics counter")
}
return nil
}

6
common/reply_store.go
View File

@ -97,7 +97,7 @@ func (s *SQLReplyStore) Exists(id int) bool {
}
// TODO: Write a test for this
func (s *SQLReplyStore) Create(t *Topic, content, ip string, uid int) (rid int, err error) {
func (s *SQLReplyStore) Create(t *Topic, content, ip string, uid int) (id int, err error) {
if Config.DisablePostIP {
ip = ""
}
@ -110,8 +110,8 @@ func (s *SQLReplyStore) Create(t *Topic, content, ip string, uid int) (rid int,
if err != nil {
return 0, err
}
rid = int(lastID)
return rid, t.AddReply(rid, uid)
id = int(lastID)
return id, t.AddReply(id, uid)
}
// TODO: Write a test for this

1
common/site.go
View File

@ -113,6 +113,7 @@ type config struct {
EnableCDNPush bool
DisableNoavatarRange bool
DisableDefaultNoavatar bool
DisableAnalytics bool
RefNoTrack bool
RefNoRef bool

2
docs/configuration.md
View File

@ -116,6 +116,8 @@ DisableNoavatarRange - This switch lets you disable the noavatar algorithm which
DisableDefaultNoavatar - This switch lets you disable the default noavatar algorithm which may intercept noavatars for increased efficiency. Default: false
DisableAnalytics - This switch lets you disable the analytics subsystem. Default: false
RefNoTrack - This switch disables tracking the referrers of users who click from another site to your site and the referrers of any requests to resources from other sites as-well.
RefNoRef - This switch makes it so that if a user clicks on a link, then the incoming site won't know which site they're coming from.

1196
gen_router.go
View File

File diff suppressed because it is too large Load Diff

59
main.go
View File

@ -32,6 +32,7 @@ import (
qgen "github.com/Azareal/Gosora/query_gen"
"github.com/Azareal/Gosora/routes"
"github.com/fsnotify/fsnotify"
//"github.com/lucas-clemente/quic-go/http3"
"github.com/pkg/errors"
)
@ -272,21 +273,37 @@ func storeInit() (err error) {
}
log.Print("Initialising the view counters")
co.GlobalViewCounter, err = co.NewGlobalViewCounter(acc)
if err != nil {
return errors.WithStack(err)
}
co.AgentViewCounter, err = co.NewDefaultAgentViewCounter(acc)
if err != nil {
return errors.WithStack(err)
}
co.OSViewCounter, err = co.NewDefaultOSViewCounter(acc)
if err != nil {
return errors.WithStack(err)
}
co.LangViewCounter, err = co.NewDefaultLangViewCounter(acc)
if err != nil {
return errors.WithStack(err)
if !c.Config.DisableAnalytics {
co.GlobalViewCounter, err = co.NewGlobalViewCounter(acc)
if err != nil {
return errors.WithStack(err)
}
co.AgentViewCounter, err = co.NewDefaultAgentViewCounter(acc)
if err != nil {
return errors.WithStack(err)
}
co.OSViewCounter, err = co.NewDefaultOSViewCounter(acc)
if err != nil {
return errors.WithStack(err)
}
co.LangViewCounter, err = co.NewDefaultLangViewCounter(acc)
if err != nil {
return errors.WithStack(err)
}
if !c.Config.RefNoTrack {
co.ReferrerTracker, err = co.NewDefaultReferrerTracker()
if err != nil {
return errors.WithStack(err)
}
}
co.MemoryCounter, err = co.NewMemoryCounter(acc)
if err != nil {
return errors.WithStack(err)
}
co.PerfCounter, err = co.NewDefaultPerfCounter(acc)
if err != nil {
return errors.WithStack(err)
}
}
co.RouteViewCounter, err = co.NewDefaultRouteViewCounter(acc)
if err != nil {
@ -308,18 +325,6 @@ func storeInit() (err error) {
if err != nil {
return errors.WithStack(err)
}
co.ReferrerTracker, err = co.NewDefaultReferrerTracker()
if err != nil {
return errors.WithStack(err)
}
co.MemoryCounter, err = co.NewMemoryCounter(acc)
if err != nil {
return errors.WithStack(err)
}
co.PerfCounter, err = co.NewDefaultPerfCounter(acc)
if err != nil {
return errors.WithStack(err)
}
log.Print("Initialising the meta store")
c.Meta, err = meta.NewDefaultMetaStore(acc)

30
patcher/patches.go
View File

@ -50,6 +50,7 @@ func init() {
addPatch(30, patch30)
addPatch(31, patch31)
addPatch(32, patch32)
addPatch(33, patch33)
}
func patch0(scanner *bufio.Scanner) (err error) {
@ -428,22 +429,22 @@ func patch10(scanner *bufio.Scanner) error {
err = acc().Select("topics").Cols("tid").EachInt(func(tid int) error {
stid := itoa(tid)
count, err := acc().Count("attachments").Where("originTable = 'topics' and originID = " + stid).Total()
count, err := acc().Count("attachments").Where("originTable = 'topics' and originID=" + stid).Total()
if err != nil {
return err
}
hasReply := false
err = acc().Select("replies").Cols("rid").Where("tid = " + stid).Orderby("rid DESC").Limit("1").EachInt(func(rid int) error {
err = acc().Select("replies").Cols("rid").Where("tid=" + stid).Orderby("rid DESC").Limit("1").EachInt(func(rid int) error {
hasReply = true
_, err := acc().Update("topics").Set("lastReplyID = ?, attachCount = ?").Where("tid = "+stid).Exec(rid, count)
_, err := acc().Update("topics").Set("lastReplyID=?, attachCount=?").Where("tid="+stid).Exec(rid, count)
return err
})
if err != nil {
return err
}
if !hasReply {
_, err = acc().Update("topics").Set("attachCount = ?").Where("tid = " + stid).Exec(count)
_, err = acc().Update("topics").Set("attachCount=?").Where("tid=" + stid).Exec(count)
}
return err
})
@ -894,14 +895,17 @@ func patch31(scanner *bufio.Scanner) error {
return nil
}
func patch32(scanner *bufio.Scanner) error {
return execStmt(qgen.Builder.CreateTable("perfchunks", "", "",
[]tC{
tC{"low", "int", 0, false, false, "0"},
tC{"high", "int", 0, false, false, "0"},
tC{"avg", "int", 0, false, false, "0"},
tC{"createdAt", "datetime", 0, false, false, ""},
}, nil,
))
}
[]tC{
tC{"low", "int", 0, false, false, "0"},
tC{"high", "int", 0, false, false, "0"},
tC{"avg", "int", 0, false, false, "0"},
tC{"createdAt", "datetime", 0, false, false, ""},
}, nil,
))
}
func patch33(scanner *bufio.Scanner) error {
return execStmt(qgen.Builder.AddColumn("viewchunks", tC{"avg", "int", 0, false, false, "0"}, nil))
}

6
query_gen/utils_test.go
View File

@ -31,6 +31,8 @@ func TestProcessWhere(t *testing.T) {
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"})
whs = processWhere("uid = '1'")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"})
whs = processWhere("uid = 't'")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "t"})
whs = processWhere("uid = ''")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, ""})
whs = processWhere("uid = '")
@ -42,8 +44,12 @@ func TestProcessWhere(t *testing.T) {
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "1"})
whs = processWhere("uid=0")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "0"})
whs = processWhere("uid=20")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenNumber, "20"})
whs = processWhere("uid='1'")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "1"})
whs = processWhere("uid='t'")
expectTokens(t, whs, MT{TokenColumn, "uid"}, MT{TokenOp, "="}, MT{TokenString, "t"})
whs = processWhere("uid")
expectTokens(t, whs, MT{TokenColumn, "uid"})

98
router_gen/main.go
View File

@ -47,23 +47,24 @@ func main() {
allRouteNames = append(allRouteNames, RouteName{name, strings.Replace(name, "common.", "c.", -1)})
allRouteMap[name] = len(allRouteNames) - 1
}
mapIt("routes.Error")
countToIndents := func(indent int) (indentor string) {
for i := 0; i < indent; i++ {
indentor += "\t"
}
return indentor
}
runBefore := func(runnables []Runnable, indent int) (out string) {
indentor := countToIndents(indent)
runBefore := func(runnables []Runnable, indentCount int) (out string) {
indent := countToIndents(indentCount)
if len(runnables) > 0 {
for _, runnable := range runnables {
if runnable.Literal {
out += "\n\t" + indentor + runnable.Contents
out += "\n\t" + indent + runnable.Contents
} else {
out += "\n" + indentor + "err = c." + runnable.Contents + "(w,req,user)\n" +
indentor + "if err != nil {\n" +
indentor + "\treturn err\n" +
indentor + "}\n" + indentor
out += "\n" + indent + "err = c." + runnable.Contents + "(w,req,user)\n" +
indent + "if err != nil {\n" +
indent + "\treturn err\n" +
indent + "}\n" + indent
}
}
}
@ -74,13 +75,13 @@ func main() {
mapIt(route.Name)
end := len(route.Path) - 1
out += "\n\t\tcase \"" + route.Path[0:end] + "\":"
//out += "\n\t\t\tid = " + strconv.Itoa(allRouteMap[route.Name])
out += runBefore(route.RunBefore, 4)
out += "\n\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
if !route.Action && !route.NoHead {
out += "\n\t\t\thead, err := c.UserCheck(w,req,&user)"
out += "\n\t\t\th, err := c.UserCheck(w,req,&user)"
out += "\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}"
vcpy := route.Vars
route.Vars = []string{"head"}
route.Vars = []string{"h"}
route.Vars = append(route.Vars, vcpy...)
}
out += "\n\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user"
@ -88,6 +89,11 @@ func main() {
out += "," + item
}
out += `)`
if !route.Action && !route.NoHead {
out += "\n\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)"
} else {
out += "\n\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
}
}
for _, group := range r.routeGroups {
@ -105,6 +111,7 @@ func main() {
mapIt(route.Name)
out += "\n\t\t\t\tcase \"" + route.Path + "\":"
//out += "\n\t\t\t\t\tid = " + strconv.Itoa(allRouteMap[route.Name])
if len(route.RunBefore) > 0 {
skipRunnable:
for _, runnable := range route.RunBefore {
@ -136,12 +143,11 @@ func main() {
}
}
}
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
if !route.Action && !route.NoHead && !group.NoHead {
out += "\n\t\t\t\thead, err := c.UserCheck(w,req,&user)"
out += "\n\t\t\t\th, err := c.UserCheck(w,req,&user)"
out += "\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}"
vcpy := route.Vars
route.Vars = []string{"head"}
route.Vars = []string{"h"}
route.Vars = append(route.Vars, vcpy...)
}
out += "\n\t\t\t\t\terr = " + strings.Replace(route.Name, "common.", "c.", -1) + "(w,req,user"
@ -149,18 +155,23 @@ func main() {
out += "," + item
}
out += ")"
if !route.Action && !route.NoHead && !group.NoHead {
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[route.Name]) + ", h.StartedAt)"
} else {
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[route.Name]) + ")"
}
}
if defaultRoute.Name != "" {
mapIt(defaultRoute.Name)
out += "\n\t\t\t\tdefault:"
//out += "\n\t\t\t\t\tid = " + strconv.Itoa(allRouteMap[defaultRoute.Name])
out += runBefore(defaultRoute.RunBefore, 4)
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[defaultRoute.Name]) + ")"
if !defaultRoute.Action && !defaultRoute.NoHead && !group.NoHead {
out += "\n\t\t\t\t\thead, err := c.UserCheck(w,req,&user)"
out += "\n\t\t\t\t\th, err := c.UserCheck(w,req,&user)"
out += "\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}"
vcpy := defaultRoute.Vars
defaultRoute.Vars = []string{"head"}
defaultRoute.Vars = []string{"h"}
defaultRoute.Vars = append(defaultRoute.Vars, vcpy...)
}
out += "\n\t\t\t\t\terr = " + strings.Replace(defaultRoute.Name, "common.", "c.", -1) + "(w,req,user"
@ -168,6 +179,11 @@ func main() {
out += ", " + item
}
out += ")"
if !defaultRoute.Action && !defaultRoute.NoHead && !group.NoHead {
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump2(" + strconv.Itoa(allRouteMap[defaultRoute.Name]) + ", h.StartedAt)"
} else {
out += "\n\t\t\t\t\tco.RouteViewCounter.Bump(" + strconv.Itoa(allRouteMap[defaultRoute.Name]) + ")"
}
}
out += `
}`
@ -180,6 +196,7 @@ func main() {
mapIt("routes.RobotsTxt")
mapIt("routes.SitemapXml")
mapIt("routes.OpenSearchXml")
mapIt("routes.Favicon")
mapIt("routes.BadRoute")
mapIt("routes.HTTPSRedirect")
tmplVars.AllRouteNames = allRouteNames
@ -462,7 +479,7 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) {
var heads string
for key, value := range req.Header {
for _, vvalue := range value {
heads += "Header '" + c.SanitiseSingleLine(key) + "': " + c.SanitiseSingleLine(vvalue) + "!\n"
heads += "Header '" + c.SanitiseSingleLine(key) + "': " + c.SanitiseSingleLine(vvalue) + "\n"
}
}
@ -472,7 +489,7 @@ func (r *GenRouter) DumpRequest(req *http.Request, prepend string) {
"Host: " + c.SanitiseSingleLine(req.Host) + "\n" +
"URL.Path: " + c.SanitiseSingleLine(req.URL.Path) + "\n" +
"URL.RawQuery: " + c.SanitiseSingleLine(req.URL.RawQuery) + "\n" +
"Referer(): " + c.SanitiseSingleLine(req.Referer()) + "\n" +
"Referer: " + c.SanitiseSingleLine(req.Referer()) + "\n" +
"RemoteAddr: " + req.RemoteAddr + "\n")
}
@ -562,9 +579,9 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
break
}
}
lowerPath := strings.ToLower(req.URL.Path)
lp := strings.ToLower(req.URL.Path)
// TODO: Flag any requests which has a dot with anything but a number after that
if strings.Contains(req.URL.Path,"..") || strings.Contains(req.URL.Path,"--") || strings.Contains(lowerPath,".php") || strings.Contains(lowerPath,".asp") || strings.Contains(lowerPath,".cgi") || strings.Contains(lowerPath,".py") || strings.Contains(lowerPath,".sql") || strings.Contains(lowerPath,".action") {
if strings.Contains(lp,"..")/* || strings.Contains(lp,"--")*/ || strings.Contains(lp,".php") || strings.Contains(lp,".asp") || strings.Contains(lp,".cgi") || strings.Contains(lp,".py") || strings.Contains(lp,".sql") || strings.Contains(lp,".action") {
r.SuspiciousRequest(req,"Bad snippet in path")
}
@ -572,6 +589,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/" {
req.URL.Path = c.Config.DefaultPath
}
//log.Print("URL.Path: ", req.URL.Path)
var prefix, extraData string
prefix = req.URL.Path[0:strings.IndexByte(req.URL.Path[1:],'/') + 1]
@ -603,10 +621,14 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
r.DumpRequest(req,"before routes.StaticFile")
}
// Increment the request counter
co.GlobalViewCounter.Bump()
if !c.Config.DisableAnalytics {
co.GlobalViewCounter.Bump()
}
if prefix == "/s" { //old prefix: /static
co.RouteViewCounter.Bump({{index .AllRouteMap "routes.StaticFile"}})
if !c.Config.DisableAnalytics {
co.RouteViewCounter.Bump({{index .AllRouteMap "routes.StaticFile"}})
}
req.URL.Path += extraData
routes.StaticFile(w, req)
return
@ -626,8 +648,10 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like.
// TODO: Add a setting to disable this?
// TODO: Use a more efficient detector instead of smashing every possible combination in
ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another
var agent string
if !c.Config.DisableAnalytics {
ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another
if ua == "" {
co.AgentViewCounter.Bump({{.AllAgentMap.blank}})
if c.Dev.DebugMode {
@ -736,8 +760,7 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// TODO: Do we want to track missing language headers too? Maybe as it's own type, e.g. "noheader"?
lang := req.Header.Get("Accept-Language")
if lang != "" {
lang = strings.TrimSpace(lang)
lLang := strings.Split(lang,"-")
lLang := strings.Split(strings.TrimSpace(lang),"-")
tLang := strings.Split(strings.Split(lLang[0],";")[0],",")
c.DebugDetail("tLang:", tLang)
var llLang string
@ -764,11 +787,14 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
ref = strings.TrimPrefix(strings.TrimPrefix(ref,"http://"),"https://")
ref = strings.Split(ref,"/")[0]
portless := strings.Split(ref,":")[0]
// TODO: Handle c.Site.Host in uppercase too?
if portless != "localhost" && portless != "127.0.0.1" && portless != c.Site.Host {
co.ReferrerTracker.Bump(ref)
}
}
}
}
// Deal with the session stuff, etc.
user, ok := c.PreRoute(w, req)
@ -804,12 +830,15 @@ func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if ferr != nil {
r.handleError(ferr,w,req,user)
}
/*if !c.Config.DisableAnalytics {
co.RouteViewCounter.Bump(id)
}*/
hTbl.VhookNoRet("router_end", w, req, user, prefix, extraData)
//c.StoppedServer("Profile end")
}
func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix, extraData string) c.RouteError {
func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c.User, prefix, extraData string) /*(id int, orerr */c.RouteError/*)*/ {
var err c.RouteError
switch(prefix) {` + out + `
/*case "/sitemaps": // TODO: Count these views
@ -817,6 +846,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
err = sitemapSwitch(w,req)*/
case "/uploads":
if extraData == "" {
co.RouteViewCounter.Bump({{index .AllRouteMap "routes.UploadedFile"}})
return c.NotFound(w,req,nil)
}
gzw, ok := w.(c.GzipResponseWriter)
@ -826,10 +856,10 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
h.Del("Content-Type")
h.Del("Content-Encoding")
}
co.RouteViewCounter.Bump({{index .AllRouteMap "routes.UploadedFile"}})
req.URL.Path += extraData
// TODO: Find a way to propagate errors up from this?
r.UploadHandler(w,req) // TODO: Count these views
co.RouteViewCounter.Bump({{index .AllRouteMap "routes.UploadedFile"}})
return nil
case "":
// Stop the favicons, robots.txt file, etc. resolving to the topics list
@ -848,6 +878,7 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
}
req.URL.Path = "/s/favicon.ico"
routes.StaticFile(w,req)
co.RouteViewCounter.Bump({{index .AllRouteMap "routes.Favicon"}})
return nil
case "opensearch.xml":
co.RouteViewCounter.Bump({{index .AllRouteMap "routes.OpenSearchXml"}})
@ -856,17 +887,19 @@ func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user c
co.RouteViewCounter.Bump({{index .AllRouteMap "routes.SitemapXml"}})
return routes.SitemapXml(w,req)*/
}
co.RouteViewCounter.Bump({{index .AllRouteMap "routes.Error"}})
return c.NotFound(w,req,nil)
default:
// A fallback for dynamic routes, e.g. ones declared by plugins
r.RLock()
handle, ok := r.extraRoutes[req.URL.Path]
h, ok := r.extraRoutes[req.URL.Path]
r.RUnlock()
if ok {
co.RouteViewCounter.Bump({{index .AllRouteMap "routes.DynamicRoute"}}) // TODO: Be more specific about *which* dynamic route it is
req.URL.Path += extraData
return handle(w,req,user)
// TODO: Be more specific about *which* dynamic route it is
co.RouteViewCounter.Bump({{index .AllRouteMap "routes.DynamicRoute"}})
return h(w,req,user)
}
lp := strings.ToLower(req.URL.Path)
@ -904,5 +937,8 @@ func writeFile(name, content string) {
if err != nil {
log.Fatal(err)
}
f.Close()
err = f.Close()
if err != nil {
log.Fatal(err)
}
}

25
router_gen/route_impl.go
View File

@ -63,8 +63,9 @@ func (r *RouteImpl) NoGzip() *RouteImpl {
return r.LitBeforeMultiline(`gzw, ok := w.(c.GzipResponseWriter)
if ok {
w = gzw.ResponseWriter
w.Header().Del("Content-Type")
w.Header().Del("Content-Encoding")
h := w.Header()
h.Del("Content-Type")
h.Del("Content-Encoding")
}`)
}
@ -77,15 +78,15 @@ func blankRoute() *RouteImpl {
return &RouteImpl{"", "", false, false, []string{}, []Runnable{}, nil}
}
func route(fname string, path string, action bool, special bool, args ...string) *RouteImpl {
func route(fname, path string, action, special bool, args ...string) *RouteImpl {
return &RouteImpl{fname, path, action, special, args, []Runnable{}, nil}
}
func View(fname string, path string, args ...string) *RouteImpl {
func View(fname, path string, args ...string) *RouteImpl {
return route(fname, path, false, false, args...)
}
func MView(fname string, path string, args ...string) *RouteImpl {
func MView(fname, path string, args ...string) *RouteImpl {
route := route(fname, path, false, false, args...)
if !route.hasBefore("SuperModOnly", "AdminOnly") {
route.Before("MemberOnly")
@ -93,7 +94,7 @@ func MView(fname string, path string, args ...string) *RouteImpl {
return route
}
func MemberView(fname string, path string, args ...string) *RouteImpl {
func MemberView(fname, path string, args ...string) *RouteImpl {
route := route(fname, path, false, false, args...)
if !route.hasBefore("SuperModOnly", "AdminOnly") {
route.Before("MemberOnly")
@ -101,7 +102,7 @@ func MemberView(fname string, path string, args ...string) *RouteImpl {
return route
}
func ModView(fname string, path string, args ...string) *RouteImpl {
func ModView(fname, path string, args ...string) *RouteImpl {
route := route(fname, path, false, false, args...)
if !route.hasBefore("AdminOnly") {
route.Before("SuperModOnly")
@ -109,7 +110,7 @@ func ModView(fname string, path string, args ...string) *RouteImpl {
return route
}
func Action(fname string, path string, args ...string) *RouteImpl {
func Action(fname, path string, args ...string) *RouteImpl {
route := route(fname, path, true, false, args...)
route.Before("NoSessionMismatch")
if !route.hasBefore("SuperModOnly", "AdminOnly") {
@ -118,11 +119,11 @@ func Action(fname string, path string, args ...string) *RouteImpl {
return route
}
func AnonAction(fname string, path string, args ...string) *RouteImpl {
func AnonAction(fname, path string, args ...string) *RouteImpl {
return route(fname, path, true, false, args...).Before("ParseForm")
}
func Special(fname string, path string, args ...string) *RouteImpl {
func Special(fname, path string, args ...string) *RouteImpl {
return route(fname, path, false, true, args...).LitBefore("req.URL.Path += extraData")
}
@ -131,7 +132,7 @@ type uploadAction struct {
Route *RouteImpl
}
func UploadAction(fname string, path string, args ...string) *uploadAction {
func UploadAction(fname, path string, args ...string) *uploadAction {
route := route(fname, path, true, false, args...)
if !route.hasBefore("SuperModOnly", "AdminOnly") {
route.Before("MemberOnly")
@ -154,6 +155,6 @@ type RouteSet struct {
Items []*RouteImpl
}
func Set(name string, path string, routes ...*RouteImpl) RouteSet {
func Set(name, path string, routes ...*RouteImpl) RouteSet {
return RouteSet{name, path, routes}
}

6
routes/stubs.go
View File

@ -7,6 +7,12 @@ func UploadedFile() {
}
func BadRoute() {
}
func Favicon() {
}
// TODO: Temporary stub for handling route group errors
func Error() {
}
// Real implementation is in router_gen/main.go, this is just a stub to map the analytics onto
func HTTPSRedirect() {

1
schema/mssql/query_viewchunks.sql
View File

@ -1,5 +1,6 @@
CREATE TABLE [viewchunks] (
[count] int DEFAULT 0 not null,
[avg] int DEFAULT 0 not null,
[createdAt] datetime not null,
[route] nvarchar (200) not null
);

1
schema/mysql/query_viewchunks.sql
View File

@ -1,5 +1,6 @@
CREATE TABLE `viewchunks` (
`count` int DEFAULT 0 not null,
`avg` int DEFAULT 0 not null,
`createdAt` datetime not null,
`route` varchar(200) not null
);

1
schema/pgsql/query_viewchunks.sql
View File

@ -1,5 +1,6 @@
CREATE TABLE "viewchunks" (
`count` int DEFAULT 0 not null,
`avg` int DEFAULT 0 not null,
`createdAt` timestamp not null,
`route` varchar (200) not null
);