From 0a628f7201260fe6b9bb41a4f1c2bd26b60bd33c Mon Sep 17 00:00:00 2001 From: Azareal Date: Sat, 28 Jul 2018 22:52:23 +1000 Subject: [PATCH] PNG and JPG avatars are now encoded as JPG images leading to a dramatic drop in the amount of bandwidth used. Did some work on image thumbnailing, but our dependencies are acting up delaying this from being released. Fixed the positions of the topic list bits for Nox on mobile. Removed APNG as an accepted image format, as we don't currently have a good way of optimising these images. Added a comment regarding the constant time compare for sessions. Added a warning about putting Gosora in www folders. Noavatars can now take a width parameters. Added a bit of missing validation for the avatar uploader. Refactored the multiple file detector for the avatar uploader. Added a Run method to accDeleteBuilder. Added an EachInt method to AccSelectBuilder. Added a Run method to accInsertBuilder. Added the users_avatar_queue table, you will need to run the patcher / update script. You might also want to update the Noavatar field in your config.json file with the new one. --- README.md | 4 +- common/auth.go | 1 + common/common.go | 2 +- common/reply.go | 1 + common/site.go | 2 +- common/template_init.go | 25 +++++--- common/thumbnailer.go | 70 +++++++++++++++++++++-- common/topic.go | 3 + common/user.go | 66 +++++++++++++++------ common/user_store.go | 12 ++-- config/config_example.json | 2 +- extend/guilds/lib/guilds.go | 4 +- gen_tables.go | 21 +++---- install/install.go | 3 +- main.go | 57 ++++++++++++++++++ patcher/patches.go | 17 ++++++ query_gen/lib/acc_builders.go | 64 ++++++++++++++++++--- query_gen/lib/accumulator.go | 1 + query_gen/tables.go | 11 ++++ routes/account.go | 38 ++++++++---- routes/profile.go | 6 +- routes/topic.go | 6 +- schema/mssql/query_users_avatar_queue.sql | 4 ++ schema/mysql/query_users_avatar_queue.sql | 4 ++ schema/pgsql/query_users_avatar_queue.sql | 4 ++ schema/schema.json | 2 +- themes/nox/public/main.css | 6 ++ 27 files changed, 356 insertions(+), 80 deletions(-) create mode 100644 schema/mssql/query_users_avatar_queue.sql create mode 100644 schema/mysql/query_users_avatar_queue.sql create mode 100644 schema/pgsql/query_users_avatar_queue.sql diff --git a/README.md b/README.md index 7cc0070b..d0512f36 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,9 @@ On Windows, you might want to try the [GosoraBootstrapper](https://github.com/Az *Linux* -First, you will need to jump to the place where you want to put the code, we will use `/home/gosora` here, but if you want to use something else, then you'll have to modify the service file with your own path. +First, you will need to jump to the place where you want to put the code, we will use `/home/gosora` here, but if you want to use something else, then you'll have to modify the service file with your own path (but *never* in a folder where the files are automatically served by a webserver). + +If you place it in `/www/`, `/public_html/` or any similar folder, then there's a chance that your server might be compromised. You can navigate to it by typing the following six commands into the console and hitting enter: diff --git a/common/auth.go b/common/auth.go index a2394874..aa0c4c48 100644 --- a/common/auth.go +++ b/common/auth.go @@ -250,6 +250,7 @@ func (auth *DefaultAuth) SessionCheck(w http.ResponseWriter, r *http.Request) (u return &GuestUser, true } + // We need to do a constant time compare, otherwise someone might be able to deduce the session character by character based on how long it takes to do the comparison. Change this at your own peril. if user.Session == "" || subtle.ConstantTimeCompare([]byte(session), []byte(user.Session)) != 1 { return &GuestUser, false } diff --git a/common/common.go b/common/common.go index fbb37c59..523859db 100644 --- a/common/common.go +++ b/common/common.go @@ -43,7 +43,7 @@ type StringList []string // ? - Should we allow users to upload .php or .go files? It could cause security issues. We could store them with a mangled extension to render them inert // TODO: Let admins manage this from the Control Panel var AllowedFileExts = StringList{ - "png", "jpg", "jpeg", "svg", "bmp", "gif", "tif", "webp", "apng", // images + "png", "jpg", "jpeg", "svg", "bmp", "gif", "tif", "webp", /*"apng",*/ // images "txt", "xml", "json", "yaml", "toml", "ini", "md", "html", "rtf", "js", "py", "rb", "css", "scss", "less", "eqcss", "pcss", "java", "ts", "cs", "c", "cc", "cpp", "cxx", "C", "c++", "h", "hh", "hpp", "hxx", "h++", "rs", "rlib", "htaccess", "gitignore", // text diff --git a/common/reply.go b/common/reply.go index 763e44b5..d8c53518 100644 --- a/common/reply.go +++ b/common/reply.go @@ -29,6 +29,7 @@ type ReplyUser struct { LastEdit int LastEditBy int Avatar string + MicroAvatar string ClassName string ContentLines int Tag string diff --git a/common/site.go b/common/site.go index 9d90d12d..4a075092 100644 --- a/common/site.go +++ b/common/site.go @@ -173,7 +173,7 @@ func ProcessConfig() (err error) { if Config.MaxUsernameLength == 0 { Config.MaxUsernameLength = 100 } - GuestUser.Avatar = BuildAvatar(0, "") + GuestUser.Avatar, GuestUser.MicroAvatar = BuildAvatar(0, "") if Config.HashAlgo != "" { // TODO: Set the alternate hash algo, e.g. argon2 diff --git a/common/template_init.go b/common/template_init.go index 2c1df3d6..31de634c 100644 --- a/common/template_init.go +++ b/common/template_init.go @@ -123,10 +123,15 @@ var Template_ip_search_handle = func(pi IPSearchPage, w io.Writer) error { } func tmplInitUsers() (User, User, User) { - user := User{62, BuildProfileURL("fake-user", 62), "Fake User", "compiler@localhost", 0, false, false, false, false, false, false, GuestPerms, make(map[string]bool), "", false, BuildAvatar(62, ""), "", "", "", "", 0, 0, 0, "0.0.0.0.0", 0} + avatar, microAvatar := BuildAvatar(62, "") + user := User{62, BuildProfileURL("fake-user", 62), "Fake User", "compiler@localhost", 0, false, false, false, false, false, false, GuestPerms, make(map[string]bool), "", false, "", avatar, microAvatar, "", "", "", "", 0, 0, 0, "0.0.0.0.0", 0} + // TODO: Do a more accurate level calculation for this? - user2 := User{1, BuildProfileURL("admin-alice", 1), "Admin Alice", "alice@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, BuildAvatar(1, ""), "", "", "", "", 58, 1000, 0, "127.0.0.1", 0} - user3 := User{2, BuildProfileURL("admin-fred", 62), "Admin Fred", "fred@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, BuildAvatar(2, ""), "", "", "", "", 42, 900, 0, "::1", 0} + avatar, microAvatar = BuildAvatar(1, "") + user2 := User{1, BuildProfileURL("admin-alice", 1), "Admin Alice", "alice@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, "", avatar, microAvatar, "", "", "", "", 58, 1000, 0, "127.0.0.1", 0} + + avatar, microAvatar = BuildAvatar(2, "") + user3 := User{2, BuildProfileURL("admin-fred", 62), "Admin Fred", "fred@localhost", 1, true, true, true, true, false, false, AllPerms, make(map[string]bool), "", true, "", avatar, microAvatar, "", "", "", "", 42, 900, 0, "::1", 0} return user, user2, user3 } @@ -183,9 +188,12 @@ func CompileTemplates() error { PollOption{0, "Nothing"}, PollOption{1, "Something"}, }, VoteCount: 7} - topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, "", 0, "", "", "", "", "", 58, false} + avatar, microAvatar := BuildAvatar(62, "") + topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false} var replyList []ReplyUser - replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, "", "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""}) + // TODO: Do we want the UID on this to be 0? + avatar, microAvatar = BuildAvatar(0, "") + replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""}) var varList = make(map[string]tmpl.VarItem) header.Title = "Topic Name" @@ -367,9 +375,12 @@ func CompileJSTemplates() error { PollOption{0, "Nothing"}, PollOption{1, "Something"}, }, VoteCount: 7} - topic := TopicUser{1, "blah", "Blah", "Hey there!", 0, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, "", 0, "", "", "", "", "", 58, false} + avatar, microAvatar := BuildAvatar(62, "") + topic := TopicUser{1, "blah", "Blah", "Hey there!", 62, false, false, now, RelativeTime(now), now, RelativeTime(now), 0, "", "127.0.0.1", 0, 1, "classname", poll.ID, "weird-data", BuildProfileURL("fake-user", 62), "Fake User", Config.DefaultGroup, avatar, microAvatar, 0, "", "", "", "", "", 58, false} var replyList []ReplyUser - replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, "", "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""}) + // TODO: Do we really want the UID here to be zero? + avatar, microAvatar = BuildAvatar(0, "") + replyList = append(replyList, ReplyUser{0, 0, "Yo!", "Yo!", 0, "alice", "Alice", Config.DefaultGroup, now, RelativeTime(now), 0, 0, avatar, microAvatar, "", 0, "", "", "", "", 0, "127.0.0.1", false, 1, "", ""}) varList = make(map[string]tmpl.VarItem) header.Title = "Topic Name" diff --git a/common/thumbnailer.go b/common/thumbnailer.go index 1f9081fb..a9d22bd9 100644 --- a/common/thumbnailer.go +++ b/common/thumbnailer.go @@ -1,28 +1,86 @@ package common +import ( + "image" + _ "image/gif" + "image/jpeg" + _ "image/png" + "os" +) + var Thumbnailer ThumbnailerInt type ThumbnailerInt interface { + Resize(inPath string, tmpPath string, outPath string, width int) error } type RezThumbnailer struct { } -func (thumb *RezThumbnailer) Resize(path string, width int) error { +func (thumb *RezThumbnailer) Resize(inPath string, tmpPath string, outPath string, width int) error { // TODO: Sniff the aspect ratio of the image and calculate the dest height accordingly, bug make sure it isn't excessively high return nil } -func (thumb *RezThumbnailer) resize(path string, width int, height int) error { +func (thumb *RezThumbnailer) resize(inPath string, outPath string, width int, height int) error { return nil } +// ! Note: CaireThumbnailer can't handle gifs, so we'll have to either cap their sizes or have another resizer deal with them +type CaireThumbnailer struct { +} + +func NewCaireThumbnailer() *CaireThumbnailer { + return &CaireThumbnailer{} +} + +func precodeImage(inPath string, tmpPath string) error { + imageFile, err := os.Open(inPath) + if err != nil { + return err + } + defer imageFile.Close() + + img, _, err := image.Decode(imageFile) + if err != nil { + return err + } + + outFile, err := os.Create(tmpPath) + if err != nil { + return err + } + defer outFile.Close() + + return jpeg.Encode(outFile, img, nil) +} + +func (thumb *CaireThumbnailer) Resize(inPath string, tmpPath string, outPath string, width int) error { + err := precodeImage(inPath, tmpPath) + if err != nil { + return err + } + return nil + + // TODO: Caire doesn't work. Try something else. Or get them to fix the index out of range. We get enough wins from re-encoding as jpeg anyway + /*imageFile, err := os.Open(tmpPath) + if err != nil { + return err + } + defer imageFile.Close() + + outFile, err := os.Create(outPath) + if err != nil { + return err + } + defer outFile.Close() + + p := &caire.Processor{NewWidth: width, Scale: true} + return p.Process(imageFile, outFile)*/ +} + /* type LilliputThumbnailer struct { -} - -type ResizeThumbnailer struct { - } */ diff --git a/common/topic.go b/common/topic.go index 5d180c47..fb1af831 100644 --- a/common/topic.go +++ b/common/topic.go @@ -70,6 +70,7 @@ type TopicUser struct { CreatedByName string Group int Avatar string + MicroAvatar string ContentLines int ContentHTML string Tag string @@ -366,6 +367,7 @@ func GetTopicUser(tid int) (TopicUser, error) { tu := TopicUser{ID: tid} err := topicStmts.getTopicUser.QueryRow(tid).Scan(&tu.Title, &tu.Content, &tu.CreatedBy, &tu.CreatedAt, &tu.IsClosed, &tu.Sticky, &tu.ParentID, &tu.IPAddress, &tu.PostCount, &tu.LikeCount, &tu.Poll, &tu.CreatedByName, &tu.Avatar, &tu.Group, &tu.URLPrefix, &tu.URLName, &tu.Level) + tu.Avatar, tu.MicroAvatar = BuildAvatar(tu.CreatedBy, tu.Avatar) tu.Link = BuildTopicURL(NameToSlug(tu.Title), tu.ID) tu.UserLink = BuildProfileURL(NameToSlug(tu.CreatedByName), tu.CreatedBy) tu.Tag = Groups.DirtyGet(tu.Group).Tag @@ -383,6 +385,7 @@ func copyTopicToTopicUser(topic *Topic, user *User) (tu TopicUser) { tu.CreatedByName = user.Name tu.Group = user.Group tu.Avatar = user.Avatar + tu.MicroAvatar = user.MicroAvatar tu.URLPrefix = user.URLPrefix tu.URLName = user.URLName tu.Level = user.Level diff --git a/common/user.go b/common/user.go index 37e0a46c..7e07e068 100644 --- a/common/user.go +++ b/common/user.go @@ -14,6 +14,7 @@ import ( "time" "../query_gen/lib" + "github.com/go-sql-driver/mysql" ) // TODO: Replace any literals with this @@ -40,17 +41,19 @@ type User struct { PluginPerms map[string]bool Session string //AuthToken string - Loggedin bool - Avatar string - Message string - URLPrefix string // Move this to another table? Create a user lite? - URLName string - Tag string - Level int - Score int - Liked int - LastIP string // ! This part of the UserCache data might fall out of date - TempGroup int + Loggedin bool + RawAvatar string + Avatar string + MicroAvatar string + Message string + URLPrefix string // Move this to another table? Create a user lite? + URLName string + Tag string + Level int + Score int + Liked int + LastIP string // ! This part of the UserCache data might fall out of date + TempGroup int } func (user *User) WebSockets() *WsJSONUser { @@ -96,6 +99,8 @@ type UserStmts struct { updateLastIP *sql.Stmt setPassword *sql.Stmt + + scheduleAvatarResize *sql.Stmt } var userStmts UserStmts @@ -123,13 +128,15 @@ func init() { updateLastIP: acc.SimpleUpdate("users", "last_ip = ?", where), setPassword: acc.Update("users").Set("password = ?, salt = ?").Where(where).Prepare(), + + scheduleAvatarResize: acc.Insert("users_avatar_queue").Columns("uid").Fields("?").Prepare(), } return acc.FirstError() }) } func (user *User) Init() { - user.Avatar = BuildAvatar(user.ID, user.Avatar) + user.Avatar, user.MicroAvatar = BuildAvatar(user.ID, user.RawAvatar) user.Link = BuildProfileURL(NameToSlug(user.Name), user.ID) user.Tag = Groups.DirtyGet(user.Group).Tag user.InitPerms() @@ -268,6 +275,23 @@ func (user *User) ChangeAvatar(avatar string) (err error) { return user.bindStmt(userStmts.setAvatar, avatar) } +// TODO: Abstract this with an interface so we can scale this with an actual dedicated queue in a real cluster +func (user *User) ScheduleAvatarResize() (err error) { + _, err = userStmts.scheduleAvatarResize.Exec(user.ID) + if err != nil { + // TODO: Do a more generic check so that we're not as tied to MySQL + me, ok := err.(*mysql.MySQLError) + if !ok { + return err + } + // If it's just telling us that the item already exists in the database, then we can ignore it, as it doesn't matter if it's this call or another which schedules the item in the queue + if me.Number != 1062 { + return err + } + } + return nil +} + func (user *User) ChangeGroup(group int) (err error) { return user.bindStmt(userStmts.changeGroup, group) } @@ -381,15 +405,25 @@ func (user *User) InitPerms() { } } +func buildNoavatar(uid int, width int) string { + return strings.Replace(strings.Replace(Config.Noavatar, "{id}", strconv.Itoa(uid), 1), "{width}", strconv.Itoa(width), 1) +} + // ? Make this part of *User? -func BuildAvatar(uid int, avatar string) string { +// TODO: Write tests for this +func BuildAvatar(uid int, avatar string) (normalAvatar string, microAvatar string) { if avatar != "" { if avatar[0] == '.' { - return "/uploads/avatar_" + strconv.Itoa(uid) + avatar + if avatar[1] == '.' { + normalAvatar = "/uploads/avatar_" + strconv.Itoa(uid) + "_tmp" + avatar[1:] + return normalAvatar, normalAvatar + } + normalAvatar = "/uploads/avatar_" + strconv.Itoa(uid) + avatar + return normalAvatar, normalAvatar } - return avatar + return avatar, avatar } - return strings.Replace(Config.Noavatar, "{id}", strconv.Itoa(uid), 1) + return buildNoavatar(uid, 200), buildNoavatar(uid, 48) } // TODO: Move this to *User diff --git a/common/user_store.go b/common/user_store.go index fae83ac7..6d8cd8a2 100644 --- a/common/user_store.go +++ b/common/user_store.go @@ -68,7 +68,7 @@ func (mus *DefaultUserStore) DirtyGet(id int) *User { } user = &User{ID: id, Loggedin: true} - err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) + err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) user.Init() if err == nil { @@ -86,7 +86,7 @@ func (mus *DefaultUserStore) Get(id int) (*User, error) { } user = &User{ID: id, Loggedin: true} - err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) + err = mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) user.Init() if err == nil { @@ -106,7 +106,7 @@ func (store *DefaultUserStore) GetOffset(offset int, perPage int) (users []*User for rows.Next() { user := &User{Loggedin: true} - err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) + err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) if err != nil { return nil, err } @@ -162,7 +162,7 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro for rows.Next() { user := &User{Loggedin: true} - err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) + err := rows.Scan(&user.ID, &user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) if err != nil { return list, err } @@ -203,7 +203,7 @@ func (mus *DefaultUserStore) BulkGetMap(ids []int) (list map[int]*User, err erro func (mus *DefaultUserStore) BypassGet(id int) (*User, error) { user := &User{ID: id, Loggedin: true} - err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) + err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) user.Init() return user, err @@ -211,7 +211,7 @@ func (mus *DefaultUserStore) BypassGet(id int) (*User, error) { func (mus *DefaultUserStore) Reload(id int) error { user := &User{ID: id, Loggedin: true} - err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.Avatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) + err := mus.get.QueryRow(id).Scan(&user.Name, &user.Group, &user.Active, &user.IsSuperAdmin, &user.Session, &user.Email, &user.RawAvatar, &user.Message, &user.URLPrefix, &user.URLName, &user.Level, &user.Score, &user.Liked, &user.LastIP, &user.TempGroup) if err != nil { mus.cache.Remove(id) return err diff --git a/config/config_example.json b/config/config_example.json index d0e7a48c..36fe9b56 100644 --- a/config/config_example.json +++ b/config/config_example.json @@ -30,7 +30,7 @@ "MinifyTemplates":true, "BuildSlugs":true, "ServerCount":1, - "Noavatar":"https://api.adorable.io/avatars/285/{id}@{site_url}.png", + "Noavatar":"https://api.adorable.io/avatars/{width}/{id}@{site_url}.png", "ItemsPerPage":25 }, "Database": { diff --git a/extend/guilds/lib/guilds.go b/extend/guilds/lib/guilds.go index 9c285286..8d301c0f 100644 --- a/extend/guilds/lib/guilds.go +++ b/extend/guilds/lib/guilds.go @@ -329,12 +329,12 @@ func RouteMemberList(w http.ResponseWriter, r *http.Request, user common.User) c var guildMembers []Member for rows.Next() { guildMember := Member{PostCount: 0} - err := rows.Scan(&guildMember.User.ID, &guildMember.Rank, &guildMember.PostCount, &guildMember.JoinedAt, &guildMember.User.Name, &guildMember.User.Avatar) + err := rows.Scan(&guildMember.User.ID, &guildMember.Rank, &guildMember.PostCount, &guildMember.JoinedAt, &guildMember.User.Name, &guildMember.User.RawAvatar) if err != nil { return common.InternalError(err, w, r) } guildMember.Link = common.BuildProfileURL(common.NameToSlug(guildMember.User.Name), guildMember.User.ID) - guildMember.User.Avatar = common.BuildAvatar(guildMember.User.ID, guildMember.User.Avatar) + guildMember.User.Avatar, guildMember.User.MicroAvatar = common.BuildAvatar(guildMember.User.ID, guildMember.User.RawAvatar) guildMember.JoinedAt, _ = common.RelativeTimeFromString(guildMember.JoinedAt) if guildItem.Owner == guildMember.User.ID { guildMember.RankString = "Owner" diff --git a/gen_tables.go b/gen_tables.go index bc71a247..cfa11187 100644 --- a/gen_tables.go +++ b/gen_tables.go @@ -2,21 +2,22 @@ package main var dbTablePrimaryKeys = map[string]string{ - "users_groups":"gid", - "attachments":"attachID", - "users_replies":"rid", - "menu_items":"miid", - "pages":"pid", - "polls":"pollID", - "activity_stream":"asid", "users_groups_scheduler":"uid", "replies":"rid", + "attachments":"attachID", + "revisions":"reviseID", + "polls":"pollID", + "users_replies":"rid", + "activity_stream":"asid", + "users":"uid", + "pages":"pid", "word_filters":"wfid", "menus":"mid", "registration_logs":"rlid", - "users":"uid", - "users_2fa_keys":"uid", "forums":"fid", + "users_2fa_keys":"uid", + "users_avatar_queue":"uid", "topics":"tid", - "revisions":"reviseID", + "users_groups":"gid", + "menu_items":"miid", } diff --git a/install/install.go b/install/install.go index 2a73a777..17448fc0 100644 --- a/install/install.go +++ b/install/install.go @@ -128,7 +128,7 @@ func main() { "MinifyTemplates":true, "BuildSlugs":true, "ServerCount":1, - "Noavatar":"https://api.adorable.io/avatars/285/{id}@{site_url}.png", + "Noavatar":"https://api.adorable.io/avatars/{width}/{id}@{site_url}.png", "ItemsPerPage":25 }, "Database": { @@ -152,7 +152,6 @@ func main() { } }`) - //"Noavatar": "https://api.adorable.io/avatars/{width}/{id}@{site_url}.png" Maybe allow this sort of syntax? fmt.Println("Opening the configuration file") configFile, err := os.Create("./config/config.json") if err != nil { diff --git a/main.go b/main.go index 7f75a2c1..a5e48e5e 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( "net/http" "os" "os/signal" + "strconv" "strings" "sync/atomic" "syscall" @@ -158,6 +159,8 @@ func afterDBInit() (err error) { if err != nil { return errors.WithStack(err) } + // TODO: Let the admin choose other thumbnailers, maybe ones defined in plugins + common.Thumbnailer = common.NewCaireThumbnailer() log.Print("Initialising the view counters") counters.GlobalViewCounter, err = counters.NewGlobalViewCounter(acc) @@ -309,6 +312,7 @@ func main() { return nil } + // TODO: Expand this to more types of files var err error for { select { @@ -357,6 +361,58 @@ func main() { } } + // Thumbnailer goroutine, we only want one image being thumbnailed at a time, otherwise they might wind up consuming all the CPU time and leave no resources left to service the actual requests + // TODO: Could we expand this to attachments and other things too? + thumbChan := make(chan bool) + go func() { + acc := qgen.Builder.Accumulator() + for { + // Put this goroutine to sleep until we have work to do + <-thumbChan + + // TODO: Use a real queue + err := acc.Select("users_avatar_queue").Columns("uid").Limit("0,5").EachInt(func(uid int) error { + //log.Print("uid: ", uid) + // TODO: Do a bulk user fetch instead? + user, err := common.Users.Get(uid) + if err != nil { + return errors.WithStack(err) + } + //log.Print("user.RawAvatar: ", user.RawAvatar) + + // Has the avatar been removed or already been processed by the thumbnailer? + if len(user.RawAvatar) < 2 || user.RawAvatar[1] == '.' { + _, _ = acc.Delete("users_avatar_queue").Where("uid = ?").Run(uid) + return nil + } + // This means it's an external image, they aren't currently implemented, but this is here for when they are + if user.RawAvatar[0] != '.' { + return nil + } + /*if user.RawAvatar == ".gif" { + return nil + }*/ + if user.RawAvatar != ".png" && user.RawAvatar != "jpg" && user.RawAvatar != "jpeg" && user.RawAvatar != "gif" { + return nil + } + + err = common.Thumbnailer.Resize("./uploads/avatar_"+strconv.Itoa(user.ID)+user.RawAvatar, "./uploads/avatar_"+strconv.Itoa(user.ID)+"_tmp"+user.RawAvatar, "./uploads/avatar_"+strconv.Itoa(user.ID)+"_w48"+user.RawAvatar, 48) + if err != nil { + return errors.WithStack(err) + } + + err = user.ChangeAvatar("." + user.RawAvatar) + if err != nil { + return errors.WithStack(err) + } + _, err = acc.Delete("users_avatar_queue").Where("uid = ?").Run(uid) + return errors.WithStack(err) + }) + if err != nil { + common.LogError(err) + } + } + }() // TODO: Write tests for these // Run this goroutine once every half second halfSecondTicker := time.NewTicker(time.Second / 2) @@ -402,6 +458,7 @@ func main() { continue } runHook("before_second_tick") + go func() { thumbChan <- true }() runTasks(common.ScheduledSecondTasks) // TODO: Stop hard-coding this diff --git a/patcher/patches.go b/patcher/patches.go index d6dea396..e1a4538a 100644 --- a/patcher/patches.go +++ b/patcher/patches.go @@ -15,6 +15,7 @@ func init() { addPatch(4, patch4) addPatch(5, patch5) addPatch(6, patch6) + addPatch(7, patch7) } func patch0(scanner *bufio.Scanner) (err error) { @@ -519,3 +520,19 @@ func patch6(scanner *bufio.Scanner) error { return nil } + +func patch7(scanner *bufio.Scanner) error { + err := execStmt(qgen.Builder.CreateTable("users_avatar_queue", "", "", + []qgen.DBTableColumn{ + qgen.DBTableColumn{"uid", "int", 0, false, false, ""}, // TODO: Make this a foreign key + }, + []qgen.DBTableKey{ + qgen.DBTableKey{"uid", "primary"}, + }, + )) + if err != nil { + return err + } + + return nil +} diff --git a/query_gen/lib/acc_builders.go b/query_gen/lib/acc_builders.go index 47ea86e1..59a1c9fd 100644 --- a/query_gen/lib/acc_builders.go +++ b/query_gen/lib/acc_builders.go @@ -12,16 +12,31 @@ type accDeleteBuilder struct { build *Accumulator } -func (delete *accDeleteBuilder) Where(where string) *accDeleteBuilder { - if delete.where != "" { - delete.where += " AND " +func (builder *accDeleteBuilder) Where(where string) *accDeleteBuilder { + if builder.where != "" { + builder.where += " AND " } - delete.where += where - return delete + builder.where += where + return builder } -func (delete *accDeleteBuilder) Prepare() *sql.Stmt { - return delete.build.SimpleDelete(delete.table, delete.where) +func (builder *accDeleteBuilder) Prepare() *sql.Stmt { + return builder.build.SimpleDelete(builder.table, builder.where) +} + +func (builder *accDeleteBuilder) Run(args ...interface{}) (int, error) { + stmt := builder.Prepare() + if stmt == nil { + return 0, builder.build.FirstError() + } + + res, err := stmt.Exec(args...) + if err != nil { + return 0, err + } + + lastID, err := res.LastInsertId() + return int(lastID), err } type accUpdateBuilder struct { @@ -169,6 +184,26 @@ func (selectItem *AccSelectBuilder) Each(handle func(*sql.Rows) error) error { } return rows.Err() } +func (selectItem *AccSelectBuilder) EachInt(handle func(int) error) error { + rows, err := selectItem.Query() + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var theInt int + err = rows.Scan(&theInt) + if err != nil { + return err + } + err = handle(theInt) + if err != nil { + return err + } + } + return rows.Err() +} type accInsertBuilder struct { table string @@ -200,6 +235,21 @@ func (insert *accInsertBuilder) Exec(args ...interface{}) (res sql.Result, err e return res, insert.build.FirstError() } +func (builder *accInsertBuilder) Run(args ...interface{}) (int, error) { + stmt := builder.Prepare() + if stmt == nil { + return 0, builder.build.FirstError() + } + + res, err := stmt.Exec(args...) + if err != nil { + return 0, err + } + + lastID, err := res.LastInsertId() + return int(lastID), err +} + type accCountBuilder struct { table string where string diff --git a/query_gen/lib/accumulator.go b/query_gen/lib/accumulator.go index f9c21c91..787b9e42 100644 --- a/query_gen/lib/accumulator.go +++ b/query_gen/lib/accumulator.go @@ -45,6 +45,7 @@ func (build *Accumulator) recordError(err error) { } func (build *Accumulator) prepare(res string, err error) *sql.Stmt { + // TODO: Can we make this less noisy on debug mode? if LogPrepares { log.Print("res: ", res) } diff --git a/query_gen/tables.go b/query_gen/tables.go index 0ca59026..3ebd0112 100644 --- a/query_gen/tables.go +++ b/query_gen/tables.go @@ -127,6 +127,17 @@ func createTables(adapter qgen.Adapter) error { }, ) + // TODO: Can we use a piece of software dedicated to persistent queues for this rather than relying on the database for it? + qgen.Install.CreateTable("users_avatar_queue", "", "", + []qgen.DBTableColumn{ + qgen.DBTableColumn{"uid", "int", 0, false, false, ""}, // TODO: Make this a foreign key + }, + []qgen.DBTableKey{ + qgen.DBTableKey{"uid", "primary"}, + }, + ) + + // TODO: Should we add a users prefix to this table to fit the "unofficial convention"? qgen.Install.CreateTable("emails", "", "", []qgen.DBTableColumn{ qgen.DBTableColumn{"email", "varchar", 200, false, false, ""}, diff --git a/routes/account.go b/routes/account.go index 43b86835..966ec79d 100644 --- a/routes/account.go +++ b/routes/account.go @@ -447,7 +447,22 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common return ferr } - var filename, ext string + // We don't want multiple files + // TODO: Are we doing this correctly? + filenameMap := make(map[string]bool) + for _, fheaders := range r.MultipartForm.File { + for _, hdr := range fheaders { + if hdr.Filename == "" { + continue + } + filenameMap[hdr.Filename] = true + } + } + if len(filenameMap) > 1 { + return common.LocalError("You may only upload one avatar", w, r, user) + } + + var ext string for _, fheaders := range r.MultipartForm.File { for _, hdr := range fheaders { if hdr.Filename == "" { @@ -459,17 +474,6 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common } defer infile.Close() - // We don't want multiple files - // TODO: Check the length of r.MultipartForm.File and error rather than doing this x.x - if filename != "" { - if filename != hdr.Filename { - os.Remove("./uploads/avatar_" + strconv.Itoa(user.ID) + "." + ext) - return common.LocalError("You may only upload one avatar", w, r, user) - } - } else { - filename = hdr.Filename - } - if ext == "" { extarr := strings.Split(hdr.Filename, ".") if len(extarr) < 2 { @@ -484,8 +488,13 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common } ext = reg.ReplaceAllString(ext, "") ext = strings.ToLower(ext) + + if !common.ImageFileExts.Contains(ext) { + return common.LocalError("You can only use an image for your avatar", w, r, user) + } } + // TODO: Centralise this string, so we don't have to change it in two different places when it changes outfile, err := os.Create("./uploads/avatar_" + strconv.Itoa(user.ID) + "." + ext) if err != nil { return common.LocalError("Upload failed [File Creation Failed]", w, r, user) @@ -506,6 +515,11 @@ func AccountEditAvatarSubmit(w http.ResponseWriter, r *http.Request, user common if err != nil { return common.InternalError(err, w, r) } + // TODO: Only schedule a resize if the avatar isn't tiny + err = user.ScheduleAvatarResize() + if err != nil { + return common.InternalError(err, w, r) + } http.Redirect(w, r, "/user/edit/?avatar_updated=1", http.StatusSeeOther) return nil } diff --git a/routes/profile.go b/routes/profile.go index d6a989a1..7ca29a59 100644 --- a/routes/profile.go +++ b/routes/profile.go @@ -37,7 +37,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo var err error var replyCreatedAt time.Time - var replyContent, replyCreatedByName, replyRelativeCreatedAt, replyAvatar, replyTag, replyClassName string + var replyContent, replyCreatedByName, replyRelativeCreatedAt, replyAvatar, replyMicroAvatar, replyTag, replyClassName string var rid, replyCreatedBy, replyLastEdit, replyLastEditBy, replyLines, replyGroup int var replyList []common.ReplyUser @@ -93,7 +93,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo } else { replyClassName = "" } - replyAvatar = common.BuildAvatar(replyCreatedBy, replyAvatar) + replyAvatar, replyMicroAvatar = common.BuildAvatar(replyCreatedBy, replyAvatar) if group.Tag != "" { replyTag = group.Tag @@ -109,7 +109,7 @@ func ViewProfile(w http.ResponseWriter, r *http.Request, user common.User) commo // TODO: Add a hook here - replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyRelativeCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, "", ""}) + replyList = append(replyList, common.ReplyUser{rid, puser.ID, replyContent, common.ParseMessage(replyContent, 0, ""), replyCreatedBy, common.BuildProfileURL(common.NameToSlug(replyCreatedByName), replyCreatedBy), replyCreatedByName, replyGroup, replyCreatedAt, replyRelativeCreatedAt, replyLastEdit, replyLastEditBy, replyAvatar, replyMicroAvatar, replyClassName, replyLines, replyTag, "", "", "", 0, "", replyLiked, replyLikeCount, "", ""}) } err = rows.Err() if err != nil { diff --git a/routes/topic.go b/routes/topic.go index 599c893b..073232ca 100644 --- a/routes/topic.go +++ b/routes/topic.go @@ -89,9 +89,6 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit } topic.RelativeCreatedAt = common.RelativeTime(topic.CreatedAt) - // TODO: Make a function for this? Build a more sophisticated noavatar handling system? - topic.Avatar = common.BuildAvatar(topic.CreatedBy, topic.Avatar) - var poll common.Poll if topic.Poll != 0 { pPoll, err := common.Polls.Get(topic.Poll) @@ -129,6 +126,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit } defer rows.Close() + // TODO: Factor the user fields out and embed a user struct instead replyItem := common.ReplyUser{ClassName: ""} for rows.Next() { err := rows.Scan(&replyItem.ID, &replyItem.Content, &replyItem.CreatedBy, &replyItem.CreatedAt, &replyItem.LastEdit, &replyItem.LastEditBy, &replyItem.Avatar, &replyItem.CreatedByName, &replyItem.Group, &replyItem.URLPrefix, &replyItem.URLName, &replyItem.Level, &replyItem.IPAddress, &replyItem.LikeCount, &replyItem.ActionType) @@ -153,7 +151,7 @@ func ViewTopic(w http.ResponseWriter, r *http.Request, user common.User, urlBit } // TODO: Make a function for this? Build a more sophisticated noavatar handling system? Do bulk user loads and let the common.UserStore initialise this? - replyItem.Avatar = common.BuildAvatar(replyItem.CreatedBy, replyItem.Avatar) + replyItem.Avatar, replyItem.MicroAvatar = common.BuildAvatar(replyItem.CreatedBy, replyItem.Avatar) replyItem.Tag = postGroup.Tag replyItem.RelativeCreatedAt = common.RelativeTime(replyItem.CreatedAt) diff --git a/schema/mssql/query_users_avatar_queue.sql b/schema/mssql/query_users_avatar_queue.sql new file mode 100644 index 00000000..4e294fdf --- /dev/null +++ b/schema/mssql/query_users_avatar_queue.sql @@ -0,0 +1,4 @@ +CREATE TABLE [users_avatar_queue] ( + [uid] int not null, + primary key([uid]) +); \ No newline at end of file diff --git a/schema/mysql/query_users_avatar_queue.sql b/schema/mysql/query_users_avatar_queue.sql new file mode 100644 index 00000000..bd8eaf90 --- /dev/null +++ b/schema/mysql/query_users_avatar_queue.sql @@ -0,0 +1,4 @@ +CREATE TABLE `users_avatar_queue` ( + `uid` int not null, + primary key(`uid`) +); \ No newline at end of file diff --git a/schema/pgsql/query_users_avatar_queue.sql b/schema/pgsql/query_users_avatar_queue.sql new file mode 100644 index 00000000..bd8eaf90 --- /dev/null +++ b/schema/pgsql/query_users_avatar_queue.sql @@ -0,0 +1,4 @@ +CREATE TABLE `users_avatar_queue` ( + `uid` int not null, + primary key(`uid`) +); \ No newline at end of file diff --git a/schema/schema.json b/schema/schema.json index 1169c29d..ca0fa0d6 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -1,5 +1,5 @@ { - "DBVersion":"7", + "DBVersion":"8", "DynamicFileVersion":"0", "MinGoVersion":"1.10", "MinVersion":"" diff --git a/themes/nox/public/main.css b/themes/nox/public/main.css index a3957dd8..07ef414d 100644 --- a/themes/nox/public/main.css +++ b/themes/nox/public/main.css @@ -696,6 +696,12 @@ button, .formbutton { .topic_list .topic_middle { display: none; } + .topic_left, .topic_right, .topic_middle { + width: 50%; + } + .topic_right_inside .lastName, .topic_left .rowtopic { + margin-top: -4px; + } .topic_left img, .topic_right img { height: 32px; width: 32px;