From df6e268a0619ec3423a2cc1fe0e862f3defe2c68 Mon Sep 17 00:00:00 2001 From: Azareal Date: Sun, 13 Oct 2019 08:49:08 +1000 Subject: [PATCH] Speed up template compilation by not building the same parse trees multiple times. Compile panel_debug. Add the ability to print int8, int16, int32, uint, uint8, uint16, uint32 and uint64 in compiled templates. Add the ability to pass string nodes to subtemplates in the template compiler. Fix bunit in the template compiler. Shorten some things. --- common/template_init.go | 45 +++++--- common/templates/templates.go | 124 ++++++++++++++------- routes/panel/debug.go | 6 +- templates/panel_user_edit.html | 4 +- themes/nox/overrides/panel_inner_menu.html | 2 +- 5 files changed, 120 insertions(+), 61 deletions(-) diff --git a/common/template_init.go b/common/template_init.go index 888c67b0..ae2568ed 100644 --- a/common/template_init.go +++ b/common/template_init.go @@ -6,14 +6,16 @@ import ( "io" "log" "path/filepath" + "runtime" "strconv" "strings" "sync" "time" "github.com/Azareal/Gosora/common/alerts" - "github.com/Azareal/Gosora/common/phrases" + p "github.com/Azareal/Gosora/common/phrases" "github.com/Azareal/Gosora/common/templates" + "github.com/Azareal/Gosora/query_gen" ) var Ctemplates []string // TODO: Use this to filter out top level templates we don't need @@ -184,8 +186,7 @@ func CompileTemplates() error { log.Print("Compiling the default templates") var wg sync.WaitGroup - err := compileTemplates(&wg, c, "") - if err != nil { + if err := compileTemplates(&wg, c, ""); err != nil { return err } oroots := c.GetOverridenRoots() @@ -198,7 +199,7 @@ func CompileTemplates() error { c.SetPerThemeTmpls(tmpls) log.Print("theme: ", theme) log.Printf("perThemeTmpls: %+v\n", tmpls) - err = compileTemplates(&wg, c, theme) + err := compileTemplates(&wg, c, theme) if err != nil { return err } @@ -207,30 +208,30 @@ func CompileTemplates() error { return nil } -func compileCommons(c *tmpl.CTemplateSet, header *Header, header2 *Header, forumList []Forum, out TItemHold) error { +func compileCommons(c *tmpl.CTemplateSet, head *Header, head2 *Header, forumList []Forum, o TItemHold) error { // TODO: Add support for interface{}s _, user2, user3 := tmplInitUsers() now := time.Now() // Convienience function to save a line here and there htitle := func(name string) *Header { - header.Title = name - return header + head.Title = name + return head } /*htitle2 := func(name string) *Header { - header2.Title = name - return header2 + head2.Title = name + return head2 }*/ var topicsList []*TopicsRow topicsList = append(topicsList, &TopicsRow{1, "topic-title", "Topic Title", "The topic content.", 1, false, false, now, now, user3.ID, 1, 1, "", "127.0.0.1", 1, 0, 1, 1, 0, "classname", 0, "", &user2, "", 0, &user3, "General", "/forum/general.2", nil}) topicListPage := TopicListPage{htitle("Topic List"), topicsList, forumList, Config.DefaultForum, TopicListSort{"lastupdated", false}, Paginator{[]int{1}, 1, 1}} - out.Add("topics", "c.TopicListPage", topicListPage) + o.Add("topics", "c.TopicListPage", topicListPage) forumItem := BlankForum(1, "general-forum.1", "General Forum", "Where the general stuff happens", true, "all", 0, "", 0) forumPage := ForumPage{htitle("General Forum"), topicsList, forumItem, Paginator{[]int{1}, 1, 1}} - out.Add("forum", "c.ForumPage", forumPage) - out.Add("forums", "c.ForumsPage", ForumsPage{htitle("Forum List"), forumList}) + o.Add("forum", "c.ForumPage", forumPage) + o.Add("forums", "c.ForumsPage", ForumsPage{htitle("Forum List"), forumList}) poll := Poll{ID: 1, Type: 0, Options: map[int]string{0: "Nothing", 1: "Something"}, Results: map[int]int{0: 5, 1: 2}, QuickOptions: []PollOption{ PollOption{0, "Nothing"}, @@ -247,8 +248,8 @@ func compileCommons(c *tmpl.CTemplateSet, header *Header, header2 *Header, forum replyList = append(replyList, ru) tpage := TopicPage{htitle("Topic Name"), replyList, topic, &Forum{ID: 1, Name: "Hahaha"}, poll, Paginator{[]int{1}, 1, 1}} tpage.Forum.Link = BuildForumURL(NameToSlug(tpage.Forum.Name), tpage.Forum.ID) - out.Add("topic", "c.TopicPage", tpage) - out.Add("topic_alt", "c.TopicPage", tpage) + o.Add("topic", "c.TopicPage", tpage) + o.Add("topic_alt", "c.TopicPage", tpage) return nil } @@ -362,6 +363,16 @@ func compileTemplates(wg *sync.WaitGroup, c *tmpl.CTemplateSet, themeName string t.AddStd("panel", "c.Panel", Panel{basePage, "panel_dashboard_right", "", "panel_dashboard", inter}) ges := []GridElement{GridElement{"","", "", 1, "grid_istat", "", "", ""}} t.AddStd("panel_dashboard", "c.DashGrids", DashGrids{ges,ges}) + + goVersion := runtime.Version() + dbVersion := qgen.Builder.DbVersion() + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + debugCache := DebugPageCache{1, 1, 1, 2, 2, 2, true} + debugDatabase := DebugPageDatabase{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1} + debugDisk := DebugPageDisk{1,1,1,1,1,1} + dpage := PanelDebugPage{basePage, goVersion, dbVersion, "0s", 1, qgen.Builder.GetAdapter().GetName(), 1, 1, memStats, debugCache, debugDatabase, debugDisk} + t.AddStd("panel_debug", "c.PanelDebugPage", dpage) //t.AddStd("panel_analytics", "c.PanelAnalytics", Panel{basePage, "panel_dashboard_right","panel_dashboard", inter}) writeTemplate := func(name string, content interface{}) { @@ -732,7 +743,7 @@ func initDefaultTmplFuncMap() { panic("phraseNameInt is not a string") } // TODO: Log non-existent phrases? - return template.HTML(phrases.GetTmplPhrase(phraseName)) + return template.HTML(p.GetTmplPhrase(phraseName)) } // TODO: Implement this in the template generator too @@ -743,7 +754,7 @@ func initDefaultTmplFuncMap() { } // TODO: Log non-existent phrases? // TODO: Optimise TmplPhrasef so we don't use slow Sprintf there - return template.HTML(phrases.GetTmplPhrasef(phraseName, args...)) + return template.HTML(p.GetTmplPhrasef(phraseName, args...)) } fmap["level"] = func(levelInt interface{}) interface{} { @@ -751,7 +762,7 @@ func initDefaultTmplFuncMap() { if !ok { panic("levelInt is not an integer") } - return template.HTML(phrases.GetLevelPhrase(level)) + return template.HTML(p.GetLevelPhrase(level)) } fmap["bunit"] = func(byteInt interface{}) interface{} { diff --git a/common/templates/templates.go b/common/templates/templates.go index f04993dd..47ed277d 100644 --- a/common/templates/templates.go +++ b/common/templates/templates.go @@ -218,18 +218,15 @@ import "errors" if !c.config.SkipInitBlock { stub += "// nolint\nfunc init() {\n" - if !c.config.SkipHandles && c.themeName == "" { stub += "\tc.Template_" + fname + "_handle = Template_" + fname + "\n" stub += "\tc.Ctemplates = append(c.Ctemplates,\"" + fname + "\")\n" } - if !c.config.SkipTmplPtrMap { stub += "tmpl := Template_" + fname + "\n" stub += "\tc.TmplPtrMap[\"" + fname + "\"] = &tmpl\n" stub += "\tc.TmplPtrMap[\"o_" + fname + "\"] = tmpl\n" } - stub += "}\n\n" } @@ -287,8 +284,7 @@ func (c *CTemplateSet) compile(name string, content string, expects string, expe if r := recover(); r != nil { fmt.Println(r) debug.PrintStack() - err := c.loggerf.Sync() - if err != nil { + if err := c.loggerf.Sync(); err != nil { fmt.Println(err) } log.Fatal("") @@ -311,13 +307,14 @@ func (c *CTemplateSet) compile(name string, content string, expects string, expe c.localDispStructIndex = 0 c.stats = make(map[string]int) - tree := parse.New(name, c.funcMap) - treeSet := make(map[string]*parse.Tree) - tree, err = tree.Parse(content, "{{", "}}", treeSet, c.funcMap) + //tree := parse.New(name, c.funcMap) + //treeSet := make(map[string]*parse.Tree) + treeSet, err := parse.Parse(name, content, "{{", "}}", c.funcMap) if err != nil { return "", err } c.detail(name) + c.detailf("treeSet: %+v\n", treeSet) fname := strings.TrimSuffix(name, filepath.Ext(name)) if c.themeName != "" { @@ -374,8 +371,21 @@ func (c *CTemplateSet) compile(name string, content string, expects string, expe TemplateName: fname, OutBuf: &outBuf, } - c.templateList = map[string]*parse.Tree{fname: tree} - c.detail(c.templateList) + + c.templateList = map[string]*parse.Tree{} + for nname, tree := range treeSet { + if name == nname { + c.templateList[fname] = tree + } else { + if !strings.HasPrefix(nname, ".html") { + c.templateList[nname] = tree + } else { + c.templateList[strings.TrimSuffix(nname, ".html")] = tree + } + } + } + c.detailf("c.templateList: %+v\n", c.templateList) + c.localVars = make(map[string]map[string]VarItemReflect) c.localVars[fname] = make(map[string]VarItemReflect) c.localVars[fname]["."] = VarItemReflect{".", con.VarHolder, con.HoldReflect} @@ -392,8 +402,14 @@ func (c *CTemplateSet) compile(name string, content string, expects string, expe //} //c.detailf("c: %+v\n", c) + c.detailf("name: %+v\n", name) + c.detailf("fname: %+v\n", fname) startIndex := con.StartTemplate("") - c.rootIterate(c.templateList[fname], con) + ttree := c.templateList[fname] + if ttree == nil { + panic("ttree is nil") + } + c.rootIterate(ttree, con) con.EndTemplate("") c.afterTemplate(con, startIndex) //c.templateFragmentCount[fname] = c.fragmentCursor[fname] + 1 @@ -574,6 +590,10 @@ w.Write([]byte(`, " + ", -1) func (c *CTemplateSet) rootIterate(tree *parse.Tree, con CContext) { c.dumpCall("rootIterate", tree, con) + if tree.Root == nil { + c.detailf("tree: %+v\n", tree) + panic("tree root node is empty") + } c.detail(tree.Root) for _, node := range tree.Root.Nodes { c.detail("Node:", node.String()) @@ -1195,8 +1215,9 @@ ArgLoop: } leftParam, _ := c.compileIfVarSub(con, leftOperand) out = "{\nbyteFloat, unit := c.ConvertByteUnit(float64(" + leftParam + "))\n" - out += "w.Write(fmt.Sprintf(\"%.1f\", byteFloat) + unit)\n" + out += "w.Write(StringToBytes(fmt.Sprintf(\"%.1f\", byteFloat) + unit))\n}\n" literal = true + c.importMap["fmt"] = "fmt" break ArgLoop case "abstime": // TODO: Implement level literals @@ -1601,9 +1622,18 @@ func (c *CTemplateSet) compileVarSub(con CContext, varname string, val reflect.V if c.guestOnly && base == "StringToBytes("+con.RootHolder+".CurrentUser.Session))" { return } + case reflect.Int8, reflect.Int16, reflect.Int32: + c.importMap["strconv"] = "strconv" + base = "StringToBytes(strconv.FormatInt(int64(" + varname + "), 10))" case reflect.Int64: c.importMap["strconv"] = "strconv" base = "StringToBytes(strconv.FormatInt(" + varname + ", 10))" + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: + c.importMap["strconv"] = "strconv" + base = "StringToBytes(strconv.FormatUint(uint64(" + varname + "), 10))" + case reflect.Uint64: + c.importMap["strconv"] = "strconv" + base = "StringToBytes(strconv.FormatUint(" + varname + ", 10))" case reflect.Struct: // TODO: Avoid clashing with other packages which have structs named Time if val.Type().Name() == "Time" { @@ -1638,19 +1668,6 @@ func (c *CTemplateSet) compileSubTemplate(pcon CContext, node *parse.TemplateNod defer c.retCall("compileSubTemplate") c.detail("Template Node: ", node.Name) - // TODO: Cascade errors back up the tree to the caller? - content, err := c.loadTemplate(c.fileDir, node.Name) - if err != nil { - c.logger.Fatal(err) - } - - tree := parse.New(node.Name, c.funcMap) - var treeSet = make(map[string]*parse.Tree) - tree, err = tree.Parse(content, "{{", "}}", treeSet, c.funcMap) - if err != nil { - c.logger.Fatal(err) - } - fname := strings.TrimSuffix(node.Name, filepath.Ext(node.Name)) if c.themeName != "" { _, ok := c.perThemeTmpls[fname] @@ -1666,6 +1683,36 @@ func (c *CTemplateSet) compileSubTemplate(pcon CContext, node *parse.TemplateNod fname += "_member" } + _, ok := c.templateList[fname] + if !ok { + // TODO: Cascade errors back up the tree to the caller? + content, err := c.loadTemplate(c.fileDir, node.Name) + if err != nil { + c.logger.Fatal(err) + } + + //tree := parse.New(node.Name, c.funcMap) + //treeSet := make(map[string]*parse.Tree) + treeSet, err := parse.Parse(node.Name, content, "{{", "}}", c.funcMap) + if err != nil { + c.logger.Fatal(err) + } + c.detailf("treeSet: %+v\n", treeSet) + + for nname, tree := range treeSet { + if node.Name == nname { + c.templateList[fname] = tree + } else { + if !strings.HasPrefix(nname, ".html") { + c.templateList[nname] = tree + } else { + c.templateList[strings.TrimSuffix(nname, ".html")] = tree + } + } + } + c.detailf("c.templateList: %+v\n", c.templateList) + } + con := pcon con.VarHolder = "tmpl_" + fname + "_vars" con.TemplateName = fname @@ -1675,7 +1722,6 @@ func (c *CTemplateSet) compileSubTemplate(pcon CContext, node *parse.TemplateNod case *parse.FieldNode: // TODO: Incomplete but it should cover the basics cur := pcon.HoldReflect - var varBit string if cur.Kind() == reflect.Interface { cur = cur.Elem() @@ -1707,6 +1753,11 @@ func (c *CTemplateSet) compileSubTemplate(pcon CContext, node *parse.TemplateNod } con.VarHolder = pcon.VarHolder + varBit con.HoldReflect = cur + case *parse.StringNode: + //con.VarHolder = pcon.VarHolder + //con.HoldReflect = pcon.HoldReflect + con.VarHolder = p.Quoted + con.HoldReflect = reflect.ValueOf(p.Quoted) case *parse.DotNode: con.VarHolder = pcon.VarHolder con.HoldReflect = pcon.HoldReflect @@ -1730,7 +1781,7 @@ func (c *CTemplateSet) compileSubTemplate(pcon CContext, node *parse.TemplateNod } } - c.templateList[fname] = tree + //c.templateList[fname] = tree subtree := c.templateList[fname] c.detail("subtree.Root", subtree.Root) c.localVars[fname] = make(map[string]VarItemReflect) @@ -1746,9 +1797,7 @@ func (c *CTemplateSet) compileSubTemplate(pcon CContext, node *parse.TemplateNod c.rootIterate(subtree, con) con.EndTemplate(endBit) //c.templateFragmentCount[fname] = c.fragmentCursor[fname] + 1 - - _, ok := c.fragOnce[fname] - if !ok { + if _, ok := c.fragOnce[fname]; !ok { c.fragOnce[fname] = true } @@ -1785,11 +1834,11 @@ func (c *CTemplateSet) compileSubTemplate(pcon CContext, node *parse.TemplateNod func (c *CTemplateSet) loadTemplate(fileDir string, name string) (content string, err error) { c.dumpCall("loadTemplate", fileDir, name) - c.detail("c.themeName: ", c.themeName) if c.themeName != "" { - c.detail("per-theme override: ", "./themes/"+c.themeName+"/overrides/"+name) - res, err := ioutil.ReadFile("./themes/" + c.themeName + "/overrides/" + name) + t := "./themes/" + c.themeName + "/overrides/" + name + c.detail("per-theme override: ", true) + res, err := ioutil.ReadFile(t) if err == nil { content = string(res) if c.config.Minify { @@ -1820,11 +1869,10 @@ func (c *CTemplateSet) afterTemplate(con CContext, startIndex int) { c.dumpCall("afterTemplate", con, startIndex) defer c.retCall("afterTemplate") - var loopDepth = 0 + loopDepth := 0 var outBuf = *con.OutBuf - var varcounts = make(map[string]int) - var loopStart = startIndex - + varcounts := make(map[string]int) + loopStart := startIndex if outBuf[startIndex].Type == "startloop" && (len(outBuf) > startIndex+1) { loopStart++ } @@ -1852,7 +1900,7 @@ func (c *CTemplateSet) afterTemplate(con CContext, startIndex int) { var varstr string var i int - var varmap = make(map[string]int) + varmap := make(map[string]int) for name, count := range varcounts { if count > 1 { varstr += "var cached_var_" + strconv.Itoa(i) + " = " + name + "\n" diff --git a/routes/panel/debug.go b/routes/panel/debug.go index c9946513..5428abfd 100644 --- a/routes/panel/debug.go +++ b/routes/panel/debug.go @@ -18,10 +18,10 @@ func Debug(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { goVersion := runtime.Version() dbVersion := qgen.Builder.DbVersion() - var uptime string upDuration := time.Since(c.StartTime) hours := int(upDuration.Hours()) minutes := int(upDuration.Minutes()) + var uptime string if hours > 24 { days := hours / 24 hours -= days * 24 @@ -63,7 +63,7 @@ func Debug(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { debugCache := c.DebugPageCache{tlen, ulen, rlen, tcap, ucap, rcap, topicListThawed} var fErr error - var count = func(tbl string) int { + count := func(tbl string) int { if fErr != nil { return 0 } @@ -96,7 +96,7 @@ func Debug(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError { debugDatabase := c.DebugPageDatabase{c.Topics.Count(),c.Users.Count(),c.Rstore.Count(),c.Prstore.Count(),c.Activity.Count(),c.Likes.Count(),attachs,polls,loginLogs,regLogs,modLogs,adminLogs,views,viewsAgents,viewsForums,viewsLangs,viewsReferrers,viewsSystems,postChunks,topicChunks} - var dirSize = func(path string) int { + dirSize := func(path string) int { if fErr != nil { return 0 } diff --git a/templates/panel_user_edit.html b/templates/panel_user_edit.html index 0f64bc72..03ff22c4 100644 --- a/templates/panel_user_edit.html +++ b/templates/panel_user_edit.html @@ -19,7 +19,7 @@
{{lang "panel_user_name"}}
-
+
{{if .CurrentUser.Perms.EditUserPassword}}
{{lang "panel_user_password"}}
@@ -34,7 +34,7 @@
{{lang "panel_user_group"}}
{{end}} diff --git a/themes/nox/overrides/panel_inner_menu.html b/themes/nox/overrides/panel_inner_menu.html index e31d47f8..11342ab2 100644 --- a/themes/nox/overrides/panel_inner_menu.html +++ b/themes/nox/overrides/panel_inner_menu.html @@ -94,4 +94,4 @@ {{if .CurrentUser.IsAdmin}}
{{lang "panel_menu_debug"}}
{{end}} - + \ No newline at end of file