More conversations progress, still not usable yet.

Flush the debug page incrementally.
Eliminate a tiny bit of code in PreparseMessage.
Shorten some of the pointer capture variables in the poll store.
Tweak the advanced installation instructions for EasyJSON to make it more repeatable.

Added the ConvoKey config setting.
Added /.git/ disk usage to the debug page, if it exists.
Added a UserStore.BulkGetMap multi-user test.

common/conversations.go

@@ -1,37 +1,173 @@
 package common
 
-import "database/sql"
+import (
-import "github.com/Azareal/Gosora/query_gen"
+	"io"
+	"time"
+	"database/sql"
+	"encoding/hex"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+
+	qgen "github.com/Azareal/Gosora/query_gen"
+)
 
 /*
 conversations
 conversations_posts
 */
 
+var ConvoPostProcess ConvoPostProcessor = NewDefaultConvoPostProcessor()
+
+type ConvoPostProcessor interface {
+	OnLoad(co *ConversationPost) (*ConversationPost, error)
+	OnSave(co *ConversationPost) (*ConversationPost, error)
+}
+
+type DefaultConvoPostProcessor struct {
+}
+
+func NewDefaultConvoPostProcessor() *DefaultConvoPostProcessor {
+	return &DefaultConvoPostProcessor{}
+}
+
+func (pr *DefaultConvoPostProcessor) OnLoad(co *ConversationPost) (*ConversationPost, error) {
+	return co, nil
+}
+
+func (pr *DefaultConvoPostProcessor) OnSave(co *ConversationPost) (*ConversationPost, error) {
+	return co, nil
+}
+
+type AesConvoPostProcessor struct {
+}
+
+func NewAesConvoPostProcessor() *AesConvoPostProcessor {
+	return &AesConvoPostProcessor{}
+}
+
+func (pr *AesConvoPostProcessor) OnLoad(co *ConversationPost) (*ConversationPost, error) {
+	if co.Post != "aes" {
+		return co, nil
+	}
+	key, _ := hex.DecodeString(Config.ConvoKey)
+
+	ciphertext, err := hex.DecodeString(co.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
+	nonceSize := aesgcm.NonceSize()
+	if len(ciphertext) < nonceSize {
+		return nil, err
+	}
+
+	nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:]
+	plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	lco := *co
+	lco.Body = string(plaintext)
+	return &lco, nil
+}
+
+func (pr *AesConvoPostProcessor) OnSave(co *ConversationPost) (*ConversationPost, error) {
+	key, _ := hex.DecodeString(Config.ConvoKey)
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	nonce := make([]byte, 12)
+	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+		return nil, err
+	}
+
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+	ciphertext := aesgcm.Seal(nil, nonce, []byte(co.Body), nil)
+
+	lco := *co
+	lco.Body = hex.EncodeToString(ciphertext)
+	lco.Post = "aes"
+	return &lco, nil
+}
+
 var convoStmts ConvoStmts
 
 type ConvoStmts struct {
+	getPosts *sql.Stmt
 	edit   *sql.Stmt
 	create *sql.Stmt
+
+	editPost *sql.Stmt
+	createPost *sql.Stmt
 }
 
-func init() {
+/*func init() {
-	/*DbInits.Add(func(acc *qgen.Accumulator) error {
+	DbInits.Add(func(acc *qgen.Accumulator) error {
 		convoStmts = ConvoStmts{
-			edit: acc.Update("conversations").Set("participants = ?").Where("cid = ?").Prepare(),
+			getPosts: acc.Select("conversations_posts").Columns("pid, body, post").Where("cid = ?").Prepare(),
-			create: acc.Insert("conversations").Columns("participants").Fields("?").Prepare(),
+			edit: acc.Update("conversations").Set("participants = ?, lastReplyAt = ?").Where("cid = ?").Prepare(),
+			create: acc.Insert("conversations").Columns("participants, createdAt, lastReplyAt").Fields("?,UTC_TIMESTAMP(),UTC_TIMESTAMP()").Prepare(),
+
+			editPost: acc.Update("conversations_posts").Set("body = ?").Where("cid = ?").Prepare(),
+			createPost: acc.Insert("conversations_posts").Columns("body").Fields("?").Prepare(),
 		}
 		return acc.FirstError()
-	})*/
+	})
-}
+}*/
 
 type Conversation struct {
 	ID           int
 	Participants string
+	CreatedAt time.Time
+	LastReplyAt time.Time
+}
+
+func (co *Conversation) Posts(offset int) (posts []*ConversationPost, err error) {
+	rows, err := convoStmts.getPosts.Query(co.ID, offset, Config.ItemsPerPage)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		convo := &ConversationPost{CID: co.ID}
+		err := rows.Scan(&convo.ID, &convo.Body, &convo.Post)
+		if err != nil {
+			return nil, err
+		}
+		convo, err = ConvoPostProcess.OnLoad(convo)
+		if err != nil {
+			return nil, err
+		}
+		posts = append(posts, convo)
+	}
+	err = rows.Err()
+	if err != nil {
+		return nil, err
+	}
+	
+	return posts, err
 }
 
 func (co *Conversation) Update() error {
-	_, err := convoStmts.edit.Exec(co.Participants, co.ID)
+	_, err := convoStmts.edit.Exec(co.Participants, co.CreatedAt, co.LastReplyAt, co.ID)
 	return err
 }
 
@@ -46,6 +182,35 @@ func (co *Conversation) Create() (int, error) {
 }
 
 type ConversationPost struct {
+	ID   int
+	CID int
+	Body string
+	Post string // aes, ''
+}
+
+func (co *ConversationPost) Update() error {
+	lco, err := ConvoPostProcess.OnSave(co)
+	if err != nil {
+		return err
+	}
+	//GetHookTable().VhookNoRet("convo_post_update", lco)
+	_, err = convoStmts.editPost.Exec(lco.Body, lco.ID)
+	return err
+}
+
+func (co *ConversationPost) Create() (int, error) {
+	lco, err := ConvoPostProcess.OnSave(co)
+	if err != nil {
+		return 0, err
+	}
+	//GetHookTable().VhookNoRet("convo_post_create", lco)
+	res, err := convoStmts.createPost.Exec(lco.Body)
+	if err != nil {
+		return 0, err
+	}
+
+	lastID, err := res.LastInsertId()
+	return int(lastID), err
 }
 
 type ConversationStore interface {
@@ -62,15 +227,15 @@ type DefaultConversationStore struct {
 
 func NewDefaultConversationStore(acc *qgen.Accumulator) (*DefaultConversationStore, error) {
 	return &DefaultConversationStore{
-		get: acc.Select("conversations").Columns("participants").Where("cid = ?").Prepare(),
+		get:    acc.Select("conversations").Columns("participants, createdAt, lastReplyAt").Where("cid = ?").Prepare(),
 		delete: acc.Delete("conversations").Where("cid = ?").Prepare(),
 		count:  acc.Count("conversations").Prepare(),
 	}, acc.FirstError()
 }
 
 func (s *DefaultConversationStore) Get(id int) (*Conversation, error) {
-	convo := &Conversation{ID:id}
+	convo := &Conversation{ID: id}
-	err := s.get.QueryRow(id).Scan(&convo.Participants)
+	err := s.get.QueryRow(id).Scan(&convo.Participants, &convo.CreatedAt, &convo.LastReplyAt)
 	return nil, err
 }
 

common/pages.go

@@ -642,6 +642,7 @@ type DebugPageDisk struct {
 	Avatars int
 	Logs int
 	Backups int
+	Git int
 }
 
 type PanelDebugPage struct {

common/parser.go

@@ -338,12 +338,13 @@ func PreparseMessage(msg string) string {
 			tags := allowedTags[char]
 			if len(tags) == 0 {
 				//fmt.Println("couldn't find char in allowedTags")
+				msg += "&"
 				if closeTag {
 					//msg += "</"
-					msg += "&"
+					//msg += "&"
 					i -= 5
 				} else {
-					msg += "&"
+					//msg += "&"
 					i -= 4
 				}
 				continue

common/poll_store.go

@@ -90,38 +90,38 @@ func NewDefaultPollStore(cache PollCache) (*DefaultPollStore, error) {
 	}, acc.FirstError()
 }
 
-func (store *DefaultPollStore) Exists(id int) bool {
+func (s *DefaultPollStore) Exists(id int) bool {
-	err := store.exists.QueryRow(id).Scan(&id)
+	err := s.exists.QueryRow(id).Scan(&id)
 	if err != nil && err != ErrNoRows {
 		LogError(err)
 	}
 	return err != ErrNoRows
 }
 
-func (store *DefaultPollStore) Get(id int) (*Poll, error) {
+func (s *DefaultPollStore) Get(id int) (*Poll, error) {
-	poll, err := store.cache.Get(id)
+	poll, err := s.cache.Get(id)
 	if err == nil {
 		return poll, nil
 	}
 
 	poll = &Poll{ID: id}
 	var optionTxt []byte
-	err = store.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount)
+	err = s.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount)
 	if err != nil {
 		return nil, err
 	}
 
 	err = json.Unmarshal(optionTxt, &poll.Options)
 	if err == nil {
-		poll.QuickOptions = store.unpackOptionsMap(poll.Options)
+		poll.QuickOptions = s.unpackOptionsMap(poll.Options)
-		store.cache.Set(poll)
+		s.cache.Set(poll)
 	}
 	return poll, err
 }
 
 // TODO: Optimise the query to avoid preparing it on the spot? Maybe, use knowledge of the most common IN() parameter counts?
 // TODO: ID of 0 should always error?
-func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err error) {
+func (s *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err error) {
 	var idCount = len(ids)
 	list = make(map[int]*Poll)
 	if idCount == 0 {
@@ -129,7 +129,7 @@ func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err er
 	}
 
 	var stillHere []int
-	sliceList := store.cache.BulkGet(ids)
+	sliceList := s.cache.BulkGet(ids)
 	for i, sliceItem := range sliceList {
 		if sliceItem != nil {
 			list[sliceItem.ID] = sliceItem
@@ -170,8 +170,8 @@ func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err er
 		if err != nil {
 			return list, err
 		}
-		poll.QuickOptions = store.unpackOptionsMap(poll.Options)
+		poll.QuickOptions = s.unpackOptionsMap(poll.Options)
-		store.cache.Set(poll)
+		s.cache.Set(poll)
 
 		list[poll.ID] = poll
 	}
@@ -205,27 +205,27 @@ func (store *DefaultPollStore) BulkGetMap(ids []int) (list map[int]*Poll, err er
 	return list, err
 }
 
-func (store *DefaultPollStore) Reload(id int) error {
+func (s *DefaultPollStore) Reload(id int) error {
 	poll := &Poll{ID: id}
 	var optionTxt []byte
-	err := store.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount)
+	err := s.get.QueryRow(id).Scan(&poll.ParentID, &poll.ParentTable, &poll.Type, &optionTxt, &poll.VoteCount)
 	if err != nil {
-		store.cache.Remove(id)
+		s.cache.Remove(id)
 		return err
 	}
 
 	err = json.Unmarshal(optionTxt, &poll.Options)
 	if err != nil {
-		store.cache.Remove(id)
+		s.cache.Remove(id)
 		return err
 	}
 
-	poll.QuickOptions = store.unpackOptionsMap(poll.Options)
+	poll.QuickOptions = s.unpackOptionsMap(poll.Options)
-	_ = store.cache.Set(poll)
+	_ = s.cache.Set(poll)
 	return nil
 }
 
-func (store *DefaultPollStore) unpackOptionsMap(rawOptions map[int]string) []PollOption {
+func (s *DefaultPollStore) unpackOptionsMap(rawOptions map[int]string) []PollOption {
 	options := make([]PollOption, len(rawOptions))
 	for id, option := range rawOptions {
 		options[id] = PollOption{id, option}
@@ -234,27 +234,27 @@ func (store *DefaultPollStore) unpackOptionsMap(rawOptions map[int]string) []Pol
 }
 
 // TODO: Use a transaction for this?
-func (store *DefaultPollStore) CastVote(optionIndex int, pollID int, uid int, ipaddress string) error {
+func (s *DefaultPollStore) CastVote(optionIndex int, pollID int, uid int, ipaddress string) error {
-	_, err := store.addVote.Exec(pollID, uid, optionIndex, ipaddress)
+	_, err := s.addVote.Exec(pollID, uid, optionIndex, ipaddress)
 	if err != nil {
 		return err
 	}
-	_, err = store.incrementVoteCount.Exec(pollID)
+	_, err = s.incrementVoteCount.Exec(pollID)
 	if err != nil {
 		return err
 	}
-	_, err = store.incrementVoteCountForOption.Exec(optionIndex, pollID)
+	_, err = s.incrementVoteCountForOption.Exec(optionIndex, pollID)
 	return err
 }
 
 // TODO: Use a transaction for this
-func (store *DefaultPollStore) Create(parent Pollable, pollType int, pollOptions map[int]string) (id int, err error) {
+func (s *DefaultPollStore) Create(parent Pollable, pollType int, pollOptions map[int]string) (id int, err error) {
 	pollOptionsTxt, err := json.Marshal(pollOptions)
 	if err != nil {
 		return 0, err
 	}
 
-	res, err := store.createPoll.Exec(parent.GetID(), parent.GetTable(), pollType, pollOptionsTxt)
+	res, err := s.createPoll.Exec(parent.GetID(), parent.GetTable(), pollType, pollOptionsTxt)
 	if err != nil {
 		return 0, err
 	}
@@ -265,23 +265,24 @@ func (store *DefaultPollStore) Create(parent Pollable, pollType int, pollOptions
 	}
 
 	for i := 0; i < len(pollOptions); i++ {
-		_, err := store.createPollOption.Exec(lastID, i)
+		_, err := s.createPollOption.Exec(lastID, i)
 		if err != nil {
 			return 0, err
 		}
 	}
+
 	return int(lastID), parent.SetPoll(int(lastID)) // TODO: Delete the poll (and options) if SetPoll fails
 }
 
-func (store *DefaultPollStore) SetCache(cache PollCache) {
+func (s *DefaultPollStore) SetCache(cache PollCache) {
-	store.cache = cache
+	s.cache = cache
 }
 
 // TODO: We're temporarily doing this so that you can do ucache != nil in getTopicUser. Refactor it.
-func (store *DefaultPollStore) GetCache() PollCache {
+func (s *DefaultPollStore) GetCache() PollCache {
-	_, ok := store.cache.(*NullPollCache)
+	_, ok := s.cache.(*NullPollCache)
 	if ok {
 		return nil
 	}
-	return store.cache
+	return s.cache
 }

common/site.go

@@ -62,6 +62,7 @@ type config struct {
 	SslPrivkey   string
 	SslFullchain string
 	HashAlgo     string // Defaults to bcrypt, and in the future, possibly something stronger
+	ConvoKey string
 
 	MaxRequestSizeStr  string
 	MaxRequestSize     int

docs/installation.md

@@ -146,7 +146,7 @@ go build "./cmd/install"
 
 install.exe
 
-go get github.com/mailru/easyjson/...
+go get -u github.com/mailru/easyjson/...
 
 easyjson -pkg common
 

misc_test.go

@@ -184,6 +184,31 @@ func userStoreTest(t *testing.T, newUserID int) {
 		expect(t, user.ID == newUserID, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID))
 	}
 
+	userList, _ = c.Users.BulkGetMap([]int{1,uid})
+	expect(t, len(userList) == 2, fmt.Sprintf("Returned map should have two results, not %d", len(userList)))
+
+	if ucache != nil {
+		expectIntToBeX(t, ucache.Length(), 2, "User cache length should be 2, not %d")
+		user, err = ucache.Get(1)
+		recordMustExist(t, err, "Couldn't find UID #%d in the cache", 1)
+		expect(t, user.ID == 1, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID))
+		user, err = ucache.Get(newUserID)
+		recordMustExist(t, err, "Couldn't find UID #%d in the cache", newUserID)
+		expect(t, user.ID == newUserID, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID))
+		ucache.Flush()
+	}
+
+	user, err = c.Users.Get(newUserID)
+	recordMustExist(t, err, "Couldn't find UID #%d", newUserID)
+	expectUser(user, newUserID, "Sam", 5, false, false, false, false)
+
+	if ucache != nil {
+		expectIntToBeX(t, ucache.Length(), 1, "User cache length should be 1, not %d")
+		user, err = ucache.Get(newUserID)
+		recordMustExist(t, err, "Couldn't find UID #%d in the cache", newUserID)
+		expect(t, user.ID == newUserID, fmt.Sprintf("user.ID does not match the requested UID. Got '%d' instead.", user.ID))
+	}
+
 	err = user.Activate()
 	expectNilErr(t, err)
 	expectIntToBeX(t, user.Group, 5, "Sam should still be in group 5 in this copy")

routes/panel/debug.go

@@ -113,8 +113,9 @@ func Debug(w http.ResponseWriter, r *http.Request, user c.User) c.RouteError {
 	if fErr != nil {
 		return c.InternalError(fErr,w,r)
 	}
+	gitSize, _ := c.DirSize("./.git")
 
-	debugDisk := c.DebugPageDisk{staticSize,attachSize,uploadsSize,logsSize,backupsSize}
+	debugDisk := c.DebugPageDisk{staticSize,attachSize,uploadsSize,logsSize,backupsSize,gitSize}
 
 	pi := c.PanelDebugPage{basePage, goVersion, dbVersion, uptime, openConnCount, qgen.Builder.GetAdapter().GetName(), goroutines, cpus, memStats, debugCache, debugDatabase, debugDisk}
 	return renderTemplate("panel", w, r, basePage.Header, c.Panel{basePage, "panel_dashboard_right", "debug_page", "panel_debug", pi})

templates/panel_debug.html

@@ -32,7 +32,7 @@
 
 <div class="colstack_item colstack_head colstack_sub_head">
 	<div class="rowitem"><h2>Memory Statistics</h2></div>
-</div>
+</div>{{flush}}
 <div id="panel_debug" class="colstack_grid">
 	<div class="grid_item grid_stat grid_stat_head"><span>Sys</span></div>
 	<div class="grid_item grid_stat grid_stat_head"><span>HeapSys</span></div>
@@ -92,7 +92,7 @@
 </div>
 <div class="colstack_item colstack_head colstack_sub_head">
 	<div class="rowitem"><h2>Database</h2></div>
-</div>
+</div>{{flush}}
 <div id="panel_debug" class="colstack_grid">
 	<div class="grid_item grid_stat grid_stat_head"><span>Topics</span></div>
 	<div class="grid_item grid_stat grid_stat_head"><span>Users</span></div>
@@ -167,7 +167,7 @@
 </div>
 <div class="colstack_item colstack_head colstack_sub_head">
 	<div class="rowitem"><h2>Disk</h2></div>
-</div>
+</div>{{flush}}
 <div id="panel_debug" class="colstack_grid">
 	<div class="grid_item grid_stat grid_stat_head"><span>Static Files</span></div>
 	<div class="grid_item grid_stat grid_stat_head"><span>Attachments</span></div>
@@ -180,9 +180,9 @@
 
 	<div class="grid_item grid_stat grid_stat_head"><span>Log Files</span></div>
 	<div class="grid_item grid_stat grid_stat_head"><span>Backups</span></div>
-	<div class="grid_item grid_stat grid_stat_head"><span>???</span></div>
+	<div class="grid_item grid_stat grid_stat_head"><span>Git</span></div>
 
 	<div class="grid_item grid_stat"><span>{{bunit .Disk.Logs}}</span></div>
 	<div class="grid_item grid_stat"><span>{{bunit .Disk.Backups}}</span></div>
-	<div class="grid_item grid_stat"><span>?</span></div>
+	<div class="grid_item grid_stat"><span>{{bunit .Disk.Git}}</span></div>
 </div>