gosora/routes/panel/analytics.go

package panel

import (
	"database/sql"
	"errors"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	c "github.com/Azareal/Gosora/common"
	p "github.com/Azareal/Gosora/common/phrases"
	qgen "github.com/Azareal/Gosora/query_gen"
)

// TODO: Move this to another file, probably common/pages.go
type AnalyticsTimeRange struct {
	Quantity   int
	Unit       string
	Slices     int
	SliceWidth int
	Range      string
}

func analyticsTimeRange(rawTimeRange string) (*AnalyticsTimeRange, error) {
	tRange := &AnalyticsTimeRange{
		Quantity:   6,
		Unit:       "hour",
		Slices:     12,
		SliceWidth: 60 * 30,
		Range:      "six-hours",
	}

	switch rawTimeRange {
	// This might be pushing it, we might want to come up with a more efficient scheme for dealing with large timeframes like this
	case "one-year":
		tRange.Quantity = 12
		tRange.Unit = "month"
		tRange.Slices = 12
		tRange.SliceWidth = 60 * 60 * 24 * 30
		tRange.Range = "one-year"
	case "three-months":
		tRange.Quantity = 90
		tRange.Unit = "day"
		tRange.Slices = 30
		tRange.SliceWidth = 60 * 60 * 24 * 3
		tRange.Range = "three-months"
	case "one-month":
		tRange.Quantity = 30
		tRange.Unit = "day"
		tRange.Slices = 30
		tRange.SliceWidth = 60 * 60 * 24
		tRange.Range = "one-month"
	case "one-week":
		tRange.Quantity = 7
		tRange.Unit = "day"
		tRange.Slices = 14
		tRange.SliceWidth = 60 * 60 * 12
		tRange.Range = "one-week"
	case "two-days": // Two days is experimental
		tRange.Quantity = 2
		tRange.Unit = "day"
		tRange.Slices = 24
		tRange.SliceWidth = 60 * 60 * 2
		tRange.Range = "two-days"
	case "one-day":
		tRange.Quantity = 1
		tRange.Unit = "day"
		tRange.Slices = 24
		tRange.SliceWidth = 60 * 60
		tRange.Range = "one-day"
	case "twelve-hours":
		tRange.Quantity = 12
		tRange.Slices = 24
		tRange.Range = "twelve-hours"
	case "six-hours", "":
	default:
		return tRange, errors.New("Unknown time range")
	}
	return tRange, nil
}

func analyticsTimeRangeToLabelList(timeRange *AnalyticsTimeRange) (revLabelList []int64, labelList []int64, viewMap map[int64]int64) {
	viewMap = make(map[int64]int64)
	currentTime := time.Now().Unix()
	for i := 1; i <= timeRange.Slices; i++ {
		label := currentTime - int64(i*timeRange.SliceWidth)
		revLabelList = append(revLabelList, label)
		viewMap[label] = 0
	}
	for _, value := range revLabelList {
		labelList = append(labelList, value)
	}
	return revLabelList, labelList, viewMap
}

func analyticsRowsToViewMap(rows *sql.Rows, labelList []int64, viewMap map[int64]int64) (map[int64]int64, error) {
	defer rows.Close()
	for rows.Next() {
		var count int64
		var createdAt time.Time
		err := rows.Scan(&count, &createdAt)
		if err != nil {
			return viewMap, err
		}
		unixCreatedAt := createdAt.Unix()
		// TODO: Bulk log this
		if c.Dev.SuperDebug {
			log.Print("count: ", count)
			log.Print("createdAt: ", createdAt)
			log.Print("unixCreatedAt: ", unixCreatedAt)
		}
		for _, value := range labelList {
			if unixCreatedAt > value {
				viewMap[value] += count
				break
			}
		}
	}
	return viewMap, rows.Err()
}

type pAvg struct {
	Avg int64
	Tot int64
}

func analyticsRowsToAverageMap(rows *sql.Rows, labelList []int64, avgMap map[int64]int64) (map[int64]int64, error) {
	defer rows.Close()
	for rows.Next() {
		var count int64
		var createdAt time.Time
		err := rows.Scan(&count, &createdAt)
		if err != nil {
			return avgMap, err
		}
		unixCreatedAt := createdAt.Unix()
		// TODO: Bulk log this
		if c.Dev.SuperDebug {
			log.Print("count: ", count)
			log.Print("createdAt: ", createdAt)
			log.Print("unixCreatedAt: ", unixCreatedAt)
		}
		pAvgMap := make(map[int64]pAvg)
		for _, value := range labelList {
			if unixCreatedAt > value {
				prev := pAvgMap[value]
				prev.Avg += count
				prev.Tot++
				pAvgMap[value] = prev
				break
			}
		}
		for key, pAvg := range pAvgMap {
			avgMap[key] = pAvg.Avg / pAvg.Tot
		}
	}
	return avgMap, rows.Err()
}

func analyticsRowsToAverageMap2(rows *sql.Rows, labelList []int64, avgMap map[int64]int64, typ int) (map[int64]int64, error) {
	defer rows.Close()
	for rows.Next() {
		var stack, heap int64
		var createdAt time.Time
		err := rows.Scan(&stack, &heap, &createdAt)
		if err != nil {
			return avgMap, err
		}
		unixCreatedAt := createdAt.Unix()
		// TODO: Bulk log this
		if c.Dev.SuperDebug {
			log.Print("stack: ", stack)
			log.Print("heap: ", heap)
			log.Print("createdAt: ", createdAt)
			log.Print("unixCreatedAt: ", unixCreatedAt)
		}
		if typ == 1 {
			heap = 0
		} else if typ == 2 {
			stack = 0
		}
		pAvgMap := make(map[int64]pAvg)
		for _, value := range labelList {
			if unixCreatedAt > value {
				prev := pAvgMap[value]
				prev.Avg += stack + heap
				prev.Tot++
				pAvgMap[value] = prev
				break
			}
		}
		for key, pAvg := range pAvgMap {
			avgMap[key] = pAvg.Avg / pAvg.Tot
		}
	}
	return avgMap, rows.Err()
}

func analyticsRowsToAverageMap3(rows *sql.Rows, labelList []int64, avgMap map[int64]int64, typ int) (map[int64]int64, error) {
	defer rows.Close()
	for rows.Next() {
		var low, high, avg int64
		var createdAt time.Time
		err := rows.Scan(&low, &high, &avg, &createdAt)
		if err != nil {
			return avgMap, err
		}
		unixCreatedAt := createdAt.Unix()
		// TODO: Bulk log this
		if c.Dev.SuperDebug {
			log.Print("low: ", low)
			log.Print("high: ", high)
			log.Print("avg: ", avg)
			log.Print("createdAt: ", createdAt)
			log.Print("unixCreatedAt: ", unixCreatedAt)
		}
		var dat int64
		switch typ {
		case 0:
			dat = low
		case 1:
			dat = high
		default:
			dat = avg
		}
		pAvgMap := make(map[int64]pAvg)
		for _, value := range labelList {
			if unixCreatedAt > value {
				prev := pAvgMap[value]
				prev.Avg += dat
				prev.Tot++
				pAvgMap[value] = prev
				break
			}
		}
		for key, pAvg := range pAvgMap {
			avgMap[key] = pAvg.Avg / pAvg.Tot
		}
	}
	return avgMap, rows.Err()
}

func PreAnalyticsDetail(w http.ResponseWriter, r *http.Request, user *c.User) (*c.BasePanelPage, c.RouteError) {
	bp, ferr := buildBasePage(w, r, user, "analytics", "analytics")
	if ferr != nil {
		return nil, ferr
	}
	bp.AddSheet("chartist/chartist.min.css")
	bp.AddScript("chartist/chartist.min.js")
	bp.AddScriptAsync("analytics.js")
	bp.LooseCSP = true
	return bp, nil
}

func AnalyticsViews(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)

	c.DebugLog("in panel.AnalyticsViews")
	// TODO: Add some sort of analytics store / iterator?
	rows, err := qgen.NewAcc().Select("viewchunks").Columns("count,createdAt").Where("route=''").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	viewList := make([]int64, len(revLabelList))
	viewItems := make([]c.PanelAnalyticsItem, len(revLabelList))
	for i, value := range revLabelList {
		viewList[i] = viewMap[value]
		viewItems[i] = c.PanelAnalyticsItem{Time: value, Count: viewMap[value]}
	}

	graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)
	var ttime string
	if timeRange.Range == "six-hours" || timeRange.Range == "twelve-hours" || timeRange.Range == "one-day" {
		ttime = "time"
	}

	pi := c.PanelAnalyticsStd{graph, viewItems, timeRange.Range, timeRange.Unit, ttime}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_views", pi})
}

func AnalyticsRouteViews(w http.ResponseWriter, r *http.Request, user c.User, route string) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)

	c.DebugLog("in panel.AnalyticsRouteViews")
	// TODO: Validate the route is valid
	rows, err := qgen.NewAcc().Select("viewchunks").Columns("count,createdAt").Where("route=?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(route)
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	var viewList []int64
	var viewItems []c.PanelAnalyticsItem
	for _, value := range revLabelList {
		viewList = append(viewList, viewMap[value])
		viewItems = append(viewItems, c.PanelAnalyticsItem{Time: value, Count: viewMap[value]})
	}
	graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)

	pi := c.PanelAnalyticsRoutePage{basePage, c.SanitiseSingleLine(route), graph, viewItems, timeRange.Range}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_route_views", pi})
}

func AnalyticsAgentViews(w http.ResponseWriter, r *http.Request, user c.User, agent string) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)
	// ? Only allow valid agents? The problem with this is that agents wind up getting renamed and it would take a migration to get them all up to snuff
	agent = c.SanitiseSingleLine(agent)

	c.DebugLog("in panel.AnalyticsAgentViews")
	// TODO: Verify the agent is valid
	rows, err := qgen.NewAcc().Select("viewchunks_agents").Columns("count,createdAt").Where("browser=?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(agent)
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	var viewList []int64
	for _, value := range revLabelList {
		viewList = append(viewList, viewMap[value])
	}
	graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)

	friendlyAgent, ok := p.GetUserAgentPhrase(agent)
	if !ok {
		friendlyAgent = agent
	}

	pi := c.PanelAnalyticsAgentPage{basePage, agent, friendlyAgent, graph, timeRange.Range}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_agent_views", pi})
}

func AnalyticsForumViews(w http.ResponseWriter, r *http.Request, user c.User, sfid string) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)

	fid, err := strconv.Atoi(sfid)
	if err != nil {
		return c.LocalError("Invalid integer", w, r, user)
	}

	c.DebugLog("in panel.AnalyticsForumViews")
	// TODO: Verify the agent is valid
	rows, err := qgen.NewAcc().Select("viewchunks_forums").Columns("count,createdAt").Where("forum=?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(fid)
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	var viewList []int64
	for _, value := range revLabelList {
		viewList = append(viewList, viewMap[value])
	}
	graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)

	forum, err := c.Forums.Get(fid)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	pi := c.PanelAnalyticsAgentPage{basePage, sfid, forum.Name, graph, timeRange.Range}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_forum_views", pi})
}

func AnalyticsSystemViews(w http.ResponseWriter, r *http.Request, user c.User, system string) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)
	system = c.SanitiseSingleLine(system)

	c.DebugLog("in panel.AnalyticsSystemViews")
	// TODO: Verify the OS name is valid
	rows, err := qgen.NewAcc().Select("viewchunks_systems").Columns("count,createdAt").Where("system=?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(system)
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	var viewList []int64
	for _, value := range revLabelList {
		viewList = append(viewList, viewMap[value])
	}
	graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)

	friendlySystem, ok := p.GetOSPhrase(system)
	if !ok {
		friendlySystem = system
	}

	pi := c.PanelAnalyticsAgentPage{basePage, system, friendlySystem, graph, timeRange.Range}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_system_views", pi})
}

func AnalyticsLanguageViews(w http.ResponseWriter, r *http.Request, user c.User, lang string) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)
	lang = c.SanitiseSingleLine(lang)

	c.DebugLog("in panel.AnalyticsLanguageViews")
	// TODO: Verify the language code is valid
	rows, err := qgen.NewAcc().Select("viewchunks_langs").Columns("count,createdAt").Where("lang=?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(lang)
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}

	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	var viewList []int64
	for _, value := range revLabelList {
		viewList = append(viewList, viewMap[value])
	}
	graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)

	friendlyLang, ok := p.GetHumanLangPhrase(lang)
	if !ok {
		friendlyLang = lang
	}

	pi := c.PanelAnalyticsAgentPage{basePage, lang, friendlyLang, graph, timeRange.Range}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_lang_views", pi})
}

func AnalyticsReferrerViews(w http.ResponseWriter, r *http.Request, user c.User, domain string) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)

	c.DebugLog("in panel.AnalyticsReferrerViews")
	// TODO: Verify the agent is valid
	rows, err := qgen.NewAcc().Select("viewchunks_referrers").Columns("count,createdAt").Where("domain=?").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query(domain)
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	var viewList []int64
	for _, value := range revLabelList {
		viewList = append(viewList, viewMap[value])
	}
	graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)
	pi := c.PanelAnalyticsAgentPage{basePage, c.SanitiseSingleLine(domain), "", graph, timeRange.Range}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_referrer_views", pi})
}

func AnalyticsTopics(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)

	c.DebugLog("in panel.AnalyticsTopics")
	rows, err := qgen.NewAcc().Select("topicchunks").Columns("count,createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	var viewList []int64
	var viewItems []c.PanelAnalyticsItem
	for _, value := range revLabelList {
		viewList = append(viewList, viewMap[value])
		viewItems = append(viewItems, c.PanelAnalyticsItem{Time: value, Count: viewMap[value]})
	}
	graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)
	pi := c.PanelAnalyticsStd{graph, viewItems, timeRange.Range, timeRange.Unit, "time"}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_topics", pi})
}

func AnalyticsPosts(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)

	c.DebugLog("in panel.AnalyticsPosts")
	rows, err := qgen.NewAcc().Select("postchunks").Columns("count,createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	viewMap, err = analyticsRowsToViewMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	var viewList []int64
	var viewItems []c.PanelAnalyticsItem
	for _, value := range revLabelList {
		viewList = append(viewList, viewMap[value])
		viewItems = append(viewItems, c.PanelAnalyticsItem{Time: value, Count: viewMap[value]})
	}
	graph := c.PanelTimeGraph{Series: [][]int64{viewList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)
	pi := c.PanelAnalyticsStd{graph, viewItems, timeRange.Range, timeRange.Unit, "time"}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_posts", pi})
}

func AnalyticsMemory(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, avgMap := analyticsTimeRangeToLabelList(timeRange)

	c.DebugLog("in panel.AnalyticsMemory")
	rows, err := qgen.NewAcc().Select("memchunks").Columns("count,createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	avgMap, err = analyticsRowsToAverageMap(rows, labelList, avgMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	// TODO: Adjust for the missing chunks in week and month
	var avgList []int64
	var avgItems []c.PanelAnalyticsItemUnit
	for _, value := range revLabelList {
		avgList = append(avgList, avgMap[value])
		cv, cu := c.ConvertByteUnit(float64(avgMap[value]))
		avgItems = append(avgItems, c.PanelAnalyticsItemUnit{Time: value, Unit: cu, Count: int64(cv)})
	}
	graph := c.PanelTimeGraph{Series: [][]int64{avgList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)
	pi := c.PanelAnalyticsStdUnit{graph, avgItems, timeRange.Range, timeRange.Unit, "time"}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_memory", pi})
}

// TODO: Show stack and heap memory separately on the chart
func AnalyticsActiveMemory(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, avgMap := analyticsTimeRangeToLabelList(timeRange)

	c.DebugLog("in panel.AnalyticsActiveMemory")
	rows, err := qgen.NewAcc().Select("memchunks").Columns("stack,heap,createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}

	var typ int
	switch r.FormValue("mtype") {
	case "1":
		typ = 1
	case "2":
		typ = 2
	default:
		typ = 0
	}
	avgMap, err = analyticsRowsToAverageMap2(rows, labelList, avgMap, typ)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	// TODO: Adjust for the missing chunks in week and month
	var avgList []int64
	var avgItems []c.PanelAnalyticsItemUnit
	for _, value := range revLabelList {
		avgList = append(avgList, avgMap[value])
		cv, cu := c.ConvertByteUnit(float64(avgMap[value]))
		avgItems = append(avgItems, c.PanelAnalyticsItemUnit{Time: value, Unit: cu, Count: int64(cv)})
	}
	graph := c.PanelTimeGraph{Series: [][]int64{avgList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)
	pi := c.PanelAnalyticsActiveMemory{graph, avgItems, timeRange.Range, timeRange.Unit, "time", typ}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_active_memory", pi})
}

func AnalyticsPerf(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, avgMap := analyticsTimeRangeToLabelList(timeRange)

	c.DebugLog("in panel.AnalyticsPerf")
	rows, err := qgen.NewAcc().Select("perfchunks").Columns("low,high,avg,createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}

	var typ int
	switch r.FormValue("type") {
	case "0":
		typ = 0
	case "1":
		typ = 1
	default:
		typ = 2
	}
	avgMap, err = analyticsRowsToAverageMap3(rows, labelList, avgMap, typ)
	if err != nil {
		return c.InternalError(err, w, r)
	}

	// TODO: Adjust for the missing chunks in week and month
	var avgList []int64
	var avgItems []c.PanelAnalyticsItemUnit
	for _, value := range revLabelList {
		avgList = append(avgList, avgMap[value])
		cv, cu := c.ConvertByteUnit(float64(avgMap[value]))
		avgItems = append(avgItems, c.PanelAnalyticsItemUnit{Time: value, Unit: cu, Count: int64(cv)})
	}
	graph := c.PanelTimeGraph{Series: [][]int64{avgList}, Labels: labelList}
	c.DebugLogf("graph: %+v\n", graph)
	pi := c.PanelAnalyticsPerf{graph, avgItems, timeRange.Range, timeRange.Unit, "time", typ}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_performance", pi})
}

func analyticsRowsToRefMap(rows *sql.Rows) (map[string]int, error) {
	nameMap := make(map[string]int)
	defer rows.Close()
	for rows.Next() {
		var count int
		var name string
		err := rows.Scan(&count, &name)
		if err != nil {
			return nameMap, err
		}
		// TODO: Bulk log this
		if c.Dev.SuperDebug {
			log.Print("count: ", count)
			log.Print("name: ", name)
		}
		nameMap[name] += count
	}
	return nameMap, rows.Err()
}

func analyticsRowsToDuoMap(rows *sql.Rows, labelList []int64, viewMap map[int64]int64) (map[string]map[int64]int64, map[string]int, error) {
	vMap := make(map[string]map[int64]int64)
	nameMap := make(map[string]int)
	defer rows.Close()
	for rows.Next() {
		var count int64
		var name string
		var createdAt time.Time
		err := rows.Scan(&count, &name, &createdAt)
		if err != nil {
			return vMap, nameMap, err
		}

		// TODO: Bulk log this
		unixCreatedAt := createdAt.Unix()
		if c.Dev.SuperDebug {
			log.Print("count: ", count)
			log.Print("name: ", name)
			log.Print("createdAt: ", createdAt)
			log.Print("unixCreatedAt: ", unixCreatedAt)
		}

		vvMap, ok := vMap[name]
		if !ok {
			vvMap = make(map[int64]int64)
			for key, val := range viewMap {
				vvMap[key] = val
			}
			vMap[name] = vvMap
		}
		for _, value := range labelList {
			if unixCreatedAt > value {
				vvMap[value] += count
				break
			}
		}
		nameMap[name] += int(count)
	}
	return vMap, nameMap, rows.Err()
}

type OVItem struct {
	name    string
	count   int
	viewMap map[int64]int64
}

func analyticsVMapToOVList(vMap map[string]map[int64]int64) (ovList []OVItem) {
	// Order the map
	for name, viewMap := range vMap {
		var totcount int
		for _, count := range viewMap {
			totcount += int(count)
		}
		ovList = append(ovList, OVItem{name, totcount, viewMap})
	}

	// Use bubble sort for now as there shouldn't be too many items
	for i := 0; i < len(ovList)-1; i++ {
		for j := 0; j < len(ovList)-1; j++ {
			if ovList[j].count > ovList[j+1].count {
				temp := ovList[j]
				ovList[j] = ovList[j+1]
				ovList[j+1] = temp
			}
		}
	}

	// Invert the direction
	var tOVList []OVItem
	for i := len(ovList) - 1; i >= 0; i-- {
		tOVList = append(tOVList, ovList[i])
	}
	return tOVList
}

func AnalyticsForums(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	basePage.AddScript("chartist/chartist-plugin-legend.min.js")
	basePage.AddSheet("chartist/chartist-plugin-legend.css")

	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)

	rows, err := qgen.NewAcc().Select("viewchunks_forums").Columns("count,forum,createdAt").Where("forum!=''").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	vMap, forumMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}
	ovList := analyticsVMapToOVList(vMap)

	var vList [][]int64
	var legendList []string
	var i int
	for _, ovitem := range ovList {
		var viewList []int64
		for _, value := range revLabelList {
			viewList = append(viewList, ovitem.viewMap[value])
		}
		vList = append(vList, viewList)
		fid, err := strconv.Atoi(ovitem.name)
		if err != nil {
			return c.InternalError(err, w, r)
		}
		var lName string
		forum, err := c.Forums.Get(fid)
		if err == sql.ErrNoRows {
			// TODO: Localise this
			lName = "Deleted Forum"
		} else if err != nil {
			return c.InternalError(err, w, r)
		} else {
			lName = forum.Name
		}
		legendList = append(legendList, lName)
		if i >= 6 {
			break
		}
		i++
	}
	graph := c.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList}
	c.DebugLogf("graph: %+v\n", graph)

	// TODO: Sort this slice
	var forumItems []c.PanelAnalyticsAgentsItem
	for sfid, count := range forumMap {
		fid, err := strconv.Atoi(sfid)
		if err != nil {
			return c.InternalError(err, w, r)
		}
		var lName string
		forum, err := c.Forums.Get(fid)
		if err == sql.ErrNoRows {
			// TODO: Localise this
			lName = "Deleted Forum"
		} else if err != nil {
			return c.InternalError(err, w, r)
		} else {
			lName = forum.Name
		}
		forumItems = append(forumItems, c.PanelAnalyticsAgentsItem{
			Agent:         sfid,
			FriendlyAgent: lName,
			Count:         count,
		})
	}

	pi := c.PanelAnalyticsDuoPage{basePage, forumItems, graph, timeRange.Range}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_forums", pi})
}

func AnalyticsRoutes(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	basePage.AddScript("chartist/chartist-plugin-legend.min.js")
	basePage.AddSheet("chartist/chartist-plugin-legend.css")

	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)

	rows, err := qgen.NewAcc().Select("viewchunks").Columns("count,route,createdAt").Where("route!=''").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	vMap, routeMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}
	ovList := analyticsVMapToOVList(vMap)

	ex := strings.Split(r.FormValue("ex"), ",")
	inEx := func(name string) bool {
		for _, e := range ex {
			if e == name {
				return true
			}
		}
		return false
	}

	var vList [][]int64
	var legendList []string
	var i int
	for _, ovitem := range ovList {
		if inEx(ovitem.name) {
			continue
		}
		var viewList []int64
		for _, value := range revLabelList {
			viewList = append(viewList, ovitem.viewMap[value])
		}
		vList = append(vList, viewList)
		legendList = append(legendList, ovitem.name)
		if i >= 6 {
			break
		}
		i++
	}
	graph := c.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList}
	c.DebugLogf("graph: %+v\n", graph)

	// TODO: Sort this slice
	var routeItems []c.PanelAnalyticsRoutesItem
	for route, count := range routeMap {
		if inEx(route) {
			continue
		}
		routeItems = append(routeItems, c.PanelAnalyticsRoutesItem{
			Route: route,
			Count: count,
		})
	}

	pi := c.PanelAnalyticsRoutesPage{basePage, routeItems, graph, timeRange.Range}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_routes", pi})
}

// Trialling multi-series charts
func AnalyticsAgents(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	basePage.AddScript("chartist/chartist-plugin-legend.min.js")
	basePage.AddSheet("chartist/chartist-plugin-legend.css")

	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)

	rows, err := qgen.NewAcc().Select("viewchunks_agents").Columns("count,browser,createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	vMap, agentMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}
	ovList := analyticsVMapToOVList(vMap)

	ex := strings.Split(r.FormValue("ex"), ",")
	inEx := func(name string) bool {
		for _, e := range ex {
			if e == name {
				return true
			}
		}
		return false
	}

	var vList [][]int64
	var legendList []string
	var i int
	for _, ovitem := range ovList {
		if inEx(ovitem.name) {
			continue
		}
		lName, ok := p.GetUserAgentPhrase(ovitem.name)
		if !ok {
			lName = ovitem.name
		}
		if inEx(lName) {
			continue
		}
		var viewList []int64
		for _, value := range revLabelList {
			viewList = append(viewList, ovitem.viewMap[value])
		}
		vList = append(vList, viewList)
		legendList = append(legendList, lName)
		if i >= 6 {
			break
		}
		i++
	}
	graph := c.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList}
	c.DebugLogf("graph: %+v\n", graph)

	// TODO: Sort this slice
	var agentItems []c.PanelAnalyticsAgentsItem
	for agent, count := range agentMap {
		if inEx(agent) {
			continue
		}
		aAgent, ok := p.GetUserAgentPhrase(agent)
		if !ok {
			aAgent = agent
		}
		if inEx(aAgent) {
			continue
		}
		agentItems = append(agentItems, c.PanelAnalyticsAgentsItem{
			Agent:         agent,
			FriendlyAgent: aAgent,
			Count:         count,
		})
	}

	pi := c.PanelAnalyticsDuoPage{basePage, agentItems, graph, timeRange.Range}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_agents", pi})
}

func AnalyticsSystems(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	basePage.AddScript("chartist/chartist-plugin-legend.min.js")
	basePage.AddSheet("chartist/chartist-plugin-legend.css")

	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)

	rows, err := qgen.NewAcc().Select("viewchunks_systems").Columns("count,system,createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	vMap, osMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}
	ovList := analyticsVMapToOVList(vMap)

	var vList [][]int64
	var legendList []string
	var i int
	for _, ovitem := range ovList {
		var viewList []int64
		for _, value := range revLabelList {
			viewList = append(viewList, ovitem.viewMap[value])
		}
		vList = append(vList, viewList)
		lName, ok := p.GetOSPhrase(ovitem.name)
		if !ok {
			lName = ovitem.name
		}
		legendList = append(legendList, lName)
		if i >= 6 {
			break
		}
		i++
	}
	graph := c.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList}
	c.DebugLogf("graph: %+v\n", graph)

	// TODO: Sort this slice
	var systemItems []c.PanelAnalyticsAgentsItem
	for system, count := range osMap {
		sSystem, ok := p.GetOSPhrase(system)
		if !ok {
			sSystem = system
		}
		systemItems = append(systemItems, c.PanelAnalyticsAgentsItem{
			Agent:         system,
			FriendlyAgent: sSystem,
			Count:         count,
		})
	}

	pi := c.PanelAnalyticsDuoPage{basePage, systemItems, graph, timeRange.Range}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_systems", pi})
}

func AnalyticsLanguages(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := PreAnalyticsDetail(w, r, &user)
	if ferr != nil {
		return ferr
	}
	basePage.AddScript("chartist/chartist-plugin-legend.min.js")
	basePage.AddSheet("chartist/chartist-plugin-legend.css")

	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}
	revLabelList, labelList, viewMap := analyticsTimeRangeToLabelList(timeRange)

	rows, err := qgen.NewAcc().Select("viewchunks_langs").Columns("count,lang,createdAt").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	vMap, langMap, err := analyticsRowsToDuoMap(rows, labelList, viewMap)
	if err != nil {
		return c.InternalError(err, w, r)
	}
	ovList := analyticsVMapToOVList(vMap)

	ex := strings.Split(r.FormValue("ex"), ",")
	inEx := func(name string) bool {
		for _, e := range ex {
			if e == name {
				return true
			}
		}
		return false
	}

	var vList [][]int64
	var legendList []string
	var i int
	for _, ovitem := range ovList {
		if inEx(ovitem.name) {
			continue
		}
		lName, ok := p.GetHumanLangPhrase(ovitem.name)
		if !ok {
			lName = ovitem.name
		}
		if inEx(lName) {
			continue
		}
		var viewList []int64
		for _, value := range revLabelList {
			viewList = append(viewList, ovitem.viewMap[value])
		}
		vList = append(vList, viewList)
		legendList = append(legendList, lName)
		if i >= 6 {
			break
		}
		i++
	}
	graph := c.PanelTimeGraph{Series: vList, Labels: labelList, Legends: legendList}
	c.DebugLogf("graph: %+v\n", graph)

	// TODO: Can we de-duplicate these analytics functions further?
	// TODO: Sort this slice
	var langItems []c.PanelAnalyticsAgentsItem
	for lang, count := range langMap {
		if inEx(lang) {
			continue
		}
		lLang, ok := p.GetHumanLangPhrase(lang)
		if !ok {
			lLang = lang
		}
		if inEx(lLang) {
			continue
		}
		langItems = append(langItems, c.PanelAnalyticsAgentsItem{
			Agent:         lang,
			FriendlyAgent: lLang,
			Count:         count,
		})
	}

	pi := c.PanelAnalyticsDuoPage{basePage, langItems, graph, timeRange.Range}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_langs", pi})
}

func AnalyticsReferrers(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
	basePage, ferr := buildBasePage(w, r, &user, "analytics", "analytics")
	if ferr != nil {
		return ferr
	}
	timeRange, err := analyticsTimeRange(r.FormValue("timeRange"))
	if err != nil {
		return c.LocalError(err.Error(), w, r, user)
	}

	rows, err := qgen.NewAcc().Select("viewchunks_referrers").Columns("count,domain").DateCutoff("createdAt", timeRange.Quantity, timeRange.Unit).Query()
	if err != nil && err != sql.ErrNoRows {
		return c.InternalError(err, w, r)
	}
	refMap, err := analyticsRowsToRefMap(rows)
	if err != nil {
		return c.InternalError(err, w, r)
	}
	showSpam := r.FormValue("spam") == "1"

	isSpammy := func(domain string) bool {
		for _, substr := range c.SpammyDomainBits {
			if strings.Contains(domain, substr) {
				return true
			}
		}
		return false
	}

	// TODO: Sort this slice
	var refItems []c.PanelAnalyticsAgentsItem
	for domain, count := range refMap {
		sdomain := c.SanitiseSingleLine(domain)
		if !showSpam && isSpammy(sdomain) {
			continue
		}
		refItems = append(refItems, c.PanelAnalyticsAgentsItem{
			Agent: sdomain,
			Count: count,
		})
	}

	pi := c.PanelAnalyticsReferrersPage{basePage, refItems, timeRange.Range, showSpam}
	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_analytics_right", "analytics", "panel_analytics_referrers", pi})
}