gosora/common/files.go

package common

import (
	"bytes"
	"compress/gzip"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"fmt"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	tmpl "git.tuxpa.in/a/gosora/tmpl_client"
	"github.com/andybalholm/brotli"
)

//type SFileList map[string]*SFile
//type SFileListShort map[string]*SFile

var StaticFiles = SFileList{"/s/", make(map[string]*SFile), make(map[string]*SFile)}

//var StaticFilesShort SFileList = make(map[string]*SFile)
var staticFileMutex sync.RWMutex

// ? Is it efficient to have two maps for this?
type SFileList struct {
    Prefix string
    Long   map[string]*SFile
    Short  map[string]*SFile
}

type SFile struct {
    // TODO: Move these to the end?
    Data     []byte
    GzipData []byte
    BrData   []byte

    Sha256 string
    Sha256I string
    OName  string
    Pos    int64

    Length        int64
    StrLength     string
    GzipLength    int64
    StrGzipLength string
    BrLength      int64
    StrBrLength   string

    Mimetype         string
    Info             os.FileInfo
    FormattedModTime string
}

type CSSData struct {
    Phrases map[string]string
}

func (l SFileList) JSTmplInit() error {
    DebugLog("Initialising the client side templates")
    return filepath.Walk("./tmpl_client", func(path string, f os.FileInfo, err error) error {
        if f.IsDir() || strings.HasSuffix(path, "tmpl_list.go") || strings.HasSuffix(path, "stub.go") {
            return nil
        }
        path = strings.Replace(path, "\\", "/", -1)
        DebugLog("Processing client template " + path)
        data, err := ioutil.ReadFile(path)
        if err != nil {
            return err
        }

        path = strings.TrimPrefix(path, "tmpl_client/")
        tmplName := strings.TrimSuffix(path, ".jgo")
        shortName := strings.TrimPrefix(tmplName, "tmpl_")

        replace := func(data []byte, replaceThis, withThis string) []byte {
            return bytes.Replace(data, []byte(replaceThis), []byte(withThis), -1)
        }
        rep := func(replaceThis, withThis string) {
            data = replace(data, replaceThis, withThis)
        }

        startIndex, hasFunc := skipAllUntilCharsExist(data, 0, []byte("if(tmplInits===undefined)"))
        if !hasFunc {
            return errors.New("no init map found")
        }
        data = data[startIndex-len([]byte("if(tmplInits===undefined)")):]
        rep("// nolint", "")
        //rep("func ", "function ")
        rep("func ", "function ")
        rep(" error {\n", " {\nlet o=\"\"\n")
        funcIndex, hasFunc := skipAllUntilCharsExist(data, 0, []byte("function Tmpl_"))
        if !hasFunc {
            return errors.New("no template function found")
        }
        spaceIndex, hasSpace := skipUntilIfExists(data, funcIndex, ' ')
        if !hasSpace {
            return errors.New("no spaces found after the template function name")
        }
        endBrace, hasBrace := skipUntilIfExists(data, spaceIndex, ')')
        if !hasBrace {
            return errors.New("no right brace found after the template function name")
        }
        fmt.Println("spaceIndex: ", spaceIndex)
        fmt.Println("endBrace: ", endBrace)
        fmt.Println("string(data[spaceIndex:endBrace]): ", string(data[spaceIndex:endBrace]))

        preLen := len(data)
        rep(string(data[spaceIndex:endBrace]), "")
        rep("))\n", "  \n")
        endBrace -= preLen - len(data) // Offset it as we've deleted portions
        fmt.Println("new endBrace: ", endBrace)
        fmt.Println("data: ", string(data))

        /*showPos := func(data []byte, index int) (out string) {
            out = "["
            for j, char := range data {
                if index == j {
                    out += "[" + string(char) + "] "
                } else {
                    out += string(char) + " "
                }
            }
            return out + "]"
        }*/

        // ? Can we just use a regex? I'm thinking of going more efficient, or just outright rolling wasm, this is a temp hack in a place where performance doesn't particularly matter
        each := func(phrase string, h func(index int)) {
            //fmt.Println("find each '" + phrase + "'")
            index := endBrace
            if index < 0 {
                panic("index under zero: " + strconv.Itoa(index))
            }
            var foundIt bool
            for {
                //fmt.Println("in index: ", index)
                //fmt.Println("pos: ", showPos(data, index))
                index, foundIt = skipAllUntilCharsExist(data, index, []byte(phrase))
                if !foundIt {
                    break
                }
                h(index)
            }
        }
        each("strconv.Itoa(", func(index int) {
            braceAt, hasEndBrace := skipUntilIfExistsOrLine(data, index, ')')
            if hasEndBrace {
                data[braceAt] = ' ' // Blank it
            }
        })
        each("[]byte(", func(index int) {
            braceAt, hasEndBrace := skipUntilIfExistsOrLine(data, index, ')')
            if hasEndBrace {
                data[braceAt] = ' ' // Blank it
            }
        })
        each("StringToBytes(", func(index int) {
            braceAt, hasEndBrace := skipUntilIfExistsOrLine(data, index, ')')
            if hasEndBrace {
                data[braceAt] = ' ' // Blank it
            }
        })
        each("w.Write(", func(index int) {
            braceAt, hasEndBrace := skipUntilIfExistsOrLine(data, index, ')')
            if hasEndBrace {
                data[braceAt] = ' ' // Blank it
            }
        })
        each("RelativeTime(", func(index int) {
            braceAt, _ := skipUntilIfExistsOrLine(data, index, 10)
            if data[braceAt-1] == ' ' {
                data[braceAt-1] = ' ' // Blank it
            }
        })
        each("if ", func(index int) {
            //fmt.Println("if index: ", index)
            braceAt, hasBrace := skipUntilIfExistsOrLine(data, index, '{')
            if hasBrace {
                if data[braceAt-1] != ' ' {
                    panic("couldn't find space before brace, found ' " + string(data[braceAt-1]) + "' instead")
                }
                data[braceAt-1] = ')' // Drop a brace here to satisfy JS
            }
        })
        each("for _, item := range ", func(index int) {
            //fmt.Println("for index: ", index)
            braceAt, hasBrace := skipUntilIfExists(data, index, '{')
            if hasBrace {
                if data[braceAt-1] != ' ' {
                    panic("couldn't find space before brace, found ' " + string(data[braceAt-1]) + "' instead")
                }
                data[braceAt-1] = ')' // Drop a brace here to satisfy JS
            }
        })
        rep("for _, item := range ", "for(item of ")
        rep("w.Write([]byte(", "o += ")
        rep("w.Write(StringToBytes(", "o += ")
        rep("w.Write(", "o += ")
        rep("+= c.", "+= ")
        rep("strconv.Itoa(", "")
        rep("strconv.FormatInt(", "")
        rep("    c.", "")
        rep("phrases.", "")
        rep(", 10;", "")

        //rep("var plist = GetTmplPhrasesBytes("+shortName+"_tmpl_phrase_id)", "const plist = tmplPhrases[\""+tmplName+"\"];")
        //rep("//var plist = GetTmplPhrasesBytes("+shortName+"_tmpl_phrase_id)", "const "+shortName+"_phrase_arr = tmplPhrases[\""+tmplName+"\"];")
        rep("//var plist = GetTmplPhrasesBytes("+shortName+"_tmpl_phrase_id)", "const pl=tmplPhrases[\""+tmplName+"\"];")
        rep(shortName+"_phrase_arr", "pl")
        rep(shortName+"_phrase", "pl")
        rep("tmpl_"+shortName+"_vars", "t_v")

        rep("var c_v_", "let c_v_")
        rep(`t_vars, ok := tmpl_i.`, `/*`)
        rep("[]byte(", "")
        rep("StringToBytes(", "")
        rep("RelativeTime(t_v.", "t_v.Relative")
        // TODO: Format dates properly on the client side
        rep(".Format(\"2006-01-02 15:04:05\"", "")
        rep(", 10", "")
        rep("if ", "if(")
        rep("return nil", "return o")
        rep(" )", ")")
        rep(" \n", "\n")
        rep("\n", ";\n")
        rep("{;", "{")
        rep("};", "}")
        rep("[;", "[")
        rep(",;", ",")
        rep("=;", "=")
        rep(`,
    });
}`, "\n\t];")
        rep(`=
}`, "=[]")
        rep("o += ", "o+=")
        rep(shortName+"_frags[", "fr[")
        rep("function Tmpl_"+shortName+"(t_v) {", "var Tmpl_"+shortName+"=(t_v)=>{")

        fragset := tmpl.GetFrag(shortName)
        if fragset != nil {
            //sfrags := []byte("let " + shortName + "_frags=[\n")
            sfrags := []byte("{const fr=[")
            for i, frags := range fragset {
                //sfrags = append(sfrags, []byte(shortName+"_frags.push(`"+string(frags)+"`);\n")...)
                //sfrags = append(sfrags, []byte("`"+string(frags)+"`,\n")...)
                if i == 0 {
                    sfrags = append(sfrags, []byte("`"+string(frags)+"`")...)
                } else {
                    sfrags = append(sfrags, []byte(",`"+string(frags)+"`")...)
                }
            }
            //sfrags = append(sfrags, []byte("];\n")...)
            sfrags = append(sfrags, []byte("];")...)
            data = append(sfrags, data...)
        }
        rep("\n;", "\n")
        rep(";;", ";")

        data = append(data, '}')
        for name, _ := range Themes {
            if strings.HasSuffix(shortName, "_"+name) {
                data = append(data, "var Tmpl_"+strings.TrimSuffix(shortName, "_"+name)+"=Tmpl_"+shortName+";"...)
                break
            }
        }

        path = tmplName + ".js"
        DebugLog("js path: ", path)
        ext := filepath.Ext("/tmpl_client/" + path)

        brData, err := CompressBytesBrotli(data)
        if err != nil {
            return err
        }
        // Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
        if len(brData) >= (len(data) + 110) {
            brData = nil
        } else {
            diff := len(data) - len(brData)
            if diff <= len(data)/100 {
                brData = nil
            }
        }

        gzipData, err := CompressBytesGzip(data)
        if err != nil {
            return err
        }
        // Don't use Gzip if we get meagre gains from it as it takes longer to process the responses
        if len(gzipData) >= (len(data) + 120) {
            gzipData = nil
        } else {
            diff := len(data) - len(gzipData)
            if diff <= len(data)/100 {
                gzipData = nil
            }
        }

        // Get a checksum for CSPs and cache busting
        hasher := sha256.New()
        hasher.Write(data)
        sum := hasher.Sum(nil)
        checksum := hex.EncodeToString(sum)
        integrity := base64.StdEncoding.EncodeToString(sum)

        l.Set(l.Prefix+path, &SFile{data, gzipData, brData, checksum, integrity, l.Prefix + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})

        DebugLogf("Added the '%s' static file.", path)
        return nil
    })
}

func (l SFileList) Init() error {
    return filepath.Walk("./public", func(path string, f os.FileInfo, err error) error {
        if f.IsDir() {
            return nil
        }

        path = strings.Replace(path, "\\", "/", -1)
        data, err := ioutil.ReadFile(path)
        if err != nil {
            return err
        }
        path = strings.TrimPrefix(path, "public/")
        ext := filepath.Ext("/public/" + path)
        if ext == ".js" {
            data = bytes.Replace(data, []byte("\r"), []byte(""), -1)
        }
        mimetype := mime.TypeByExtension(ext)

        // Get a checksum for CSPs and cache busting
        hasher := sha256.New()
        hasher.Write(data)
        sum := hasher.Sum(nil)
        checksum := hex.EncodeToString(sum)
        integrity := base64.StdEncoding.EncodeToString(sum)

        // Avoid double-compressing images
        var gzipData, brData []byte
        if mimetype != "image/jpeg" && mimetype != "image/png" && mimetype != "image/gif" {
            brData, err = CompressBytesBrotli(data)
            if err != nil {
                return err
            }
            // Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
            if len(brData) >= (len(data) + 130) {
                brData = nil
            } else {
                diff := len(data) - len(brData)
                if diff <= len(data)/100 {
                    brData = nil
                }
            }

            gzipData, err = CompressBytesGzip(data)
            if err != nil {
                return err
            }
            // Don't use Gzip if we get meagre gains from it as it takes longer to process the responses
            if len(gzipData) >= (len(data) + 150) {
                gzipData = nil
            } else {
                diff := len(data) - len(gzipData)
                if diff <= len(data)/100 {
                    gzipData = nil
                }
            }
        }

        l.Set(l.Prefix+path, &SFile{data, gzipData, brData, checksum, integrity, l.Prefix + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mimetype, f, f.ModTime().UTC().Format(http.TimeFormat)})

        DebugLogf("Added the '%s' static file.", path)
        return nil
    })
}

func (l SFileList) Add(path, prefix string) error {
    data, err := ioutil.ReadFile(path)
    if err != nil {
        return err
    }
    fi, err := os.Open(path)
    if err != nil {
        return err
    }
    f, err := fi.Stat()
    if err != nil {
        return err
    }

    ext := filepath.Ext(path)
    path = strings.TrimPrefix(path, prefix)

    brData, err := CompressBytesBrotli(data)
    if err != nil {
        return err
    }
    // Don't use Brotli if we get meagre gains from it as it takes longer to process the responses
    if len(brData) >= (len(data) + 130) {
        brData = nil
    } else {
        diff := len(data) - len(brData)
        if diff <= len(data)/100 {
            brData = nil
        }
    }

    gzipData, err := CompressBytesGzip(data)
    if err != nil {
        return err
    }
    // Don't use Gzip if we get meagre gains from it as it takes longer to process the responses
    if len(gzipData) >= (len(data) + 150) {
        gzipData = nil
    } else {
        diff := len(data) - len(gzipData)
        if diff <= len(data)/100 {
            gzipData = nil
        }
    }

    // Get a checksum for CSPs and cache busting
    hasher := sha256.New()
    hasher.Write(data)
    sum := hasher.Sum(nil)
        checksum := hex.EncodeToString(sum)
        integrity := base64.StdEncoding.EncodeToString(sum)

    l.Set(l.Prefix+path, &SFile{data, gzipData, brData, checksum, integrity, l.Prefix + path + "?h=" + checksum, 0, int64(len(data)), strconv.Itoa(len(data)), int64(len(gzipData)), strconv.Itoa(len(gzipData)), int64(len(brData)), strconv.Itoa(len(brData)), mime.TypeByExtension(ext), f, f.ModTime().UTC().Format(http.TimeFormat)})

    DebugLogf("Added the '%s' static file", path)
    return nil
}

func (l SFileList) Get(path string) (file *SFile, exists bool) {
    staticFileMutex.RLock()
    defer staticFileMutex.RUnlock()
    file, exists = l.Long[path]
    return file, exists
}

// fetch without /s/ to avoid allocing in pages.go
func (l SFileList) GetShort(name string) (file *SFile, exists bool) {
    staticFileMutex.RLock()
    defer staticFileMutex.RUnlock()
    file, exists = l.Short[name]
    return file, exists
}

func (l SFileList) Set(name string, data *SFile) {
    staticFileMutex.Lock()
    defer staticFileMutex.Unlock()
    // TODO: Propagate errors back up
    uurl, err := url.Parse(name)
    if err != nil {
        return
    }
    l.Long[uurl.Path] = data
    l.Short[strings.TrimPrefix(strings.TrimPrefix(name, l.Prefix), "/")] = data
}

var gzipBestCompress sync.Pool

func CompressBytesGzip(in []byte) (b []byte, err error) {
    var buf bytes.Buffer
    ii := gzipBestCompress.Get()
    var gz *gzip.Writer
    if ii == nil {
        gz, err = gzip.NewWriterLevel(&buf, gzip.BestCompression)
        if err != nil {
            return nil, err
        }
    } else {
        gz = ii.(*gzip.Writer)
        gz.Reset(&buf)
    }
    _, err = gz.Write(in)
    if err != nil {
        return nil, err
    }
    err = gz.Close()
    if err != nil {
        return nil, err
    }
    gzipBestCompress.Put(gz)
    return buf.Bytes(), nil
}

func CompressBytesBrotli(in []byte) ([]byte, error) {
    var buff bytes.Buffer
    br := brotli.NewWriterLevel(&buff, brotli.BestCompression)
    _, err := br.Write(in)
    if err != nil {
        return nil, err
    }
    err = br.Close()
    if err != nil {
        return nil, err
    }
    return buff.Bytes(), nil
}