diff --git a/common/common.go b/common/common.go index 2d7bbc30..fbb37c59 100644 --- a/common/common.go +++ b/common/common.go @@ -28,6 +28,7 @@ var TmplPtrMap = make(map[string]interface{}) var JSTokenBox atomic.Value // TODO: Move this and some of these other globals somewhere else var SessionSigningKeyBox atomic.Value // For MFA to avoid hitting the database unneccesarily var OldSessionSigningKeyBox atomic.Value // Just in case we've signed with a key that's about to go stale so we don't annoy the user too much +var IsDBDown int32 = 0 // 0 = false, 1 = true. this is value which should be manipulated with package atomic for representing whether the database is down so we don't spam the log with lots of redundant errors // ErrNoRows is an alias of sql.ErrNoRows, just in case we end up with non-database/sql datastores var ErrNoRows = sql.ErrNoRows diff --git a/common/errors.go b/common/errors.go index a6b795f4..4475860a 100644 --- a/common/errors.go +++ b/common/errors.go @@ -124,6 +124,13 @@ func InternalErrorJS(err error, w http.ResponseWriter, r *http.Request) RouteErr return HandledRouteError() } +// When the task system detects if the database is down, some database errors might lip by this +func DatabaseError(w http.ResponseWriter, r *http.Request) RouteError { + pi := ErrorPage{errorHeader(w, GuestUser, "Internal Server Error"), "A problem has occurred in the system."} + handleErrorTemplate(w, r, pi) + return HandledRouteError() +} + var xmlInternalError = []byte(` A problem has occured`) diff --git a/gen_router.go b/gen_router.go index 6c8c356b..e7e461ed 100644 --- a/gen_router.go +++ b/gen_router.go @@ -7,6 +7,7 @@ import ( "strings" "strconv" "sync" + "sync/atomic" "errors" "os" "net/http" @@ -701,6 +702,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { routes.StaticFile(w, req) return } + if atomic.LoadInt32(&common.IsDBDown) == 1 { + common.DatabaseError(w, req) + return + } if common.Dev.SuperDebug { router.requestLogger.Print("before PreRoute") } diff --git a/main.go b/main.go index 6c2c9868..91ef110b 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( "os" "os/signal" "strings" + "sync/atomic" "syscall" "time" @@ -346,6 +347,7 @@ func main() { } log.Print("Initialising the task system") + // TODO: Name the tasks so we can figure out which one it was when something goes wrong? Or maybe toss it up WithStack down there? var runTasks = func(tasks []func() error) { for _, task := range tasks { if task() != nil { @@ -361,6 +363,24 @@ func main() { fifteenMinuteTicker := time.NewTicker(15 * time.Minute) hourTicker := time.NewTicker(time.Hour) go func() { + var startTick = func() (abort bool) { + var isDBDown = atomic.LoadInt32(&common.IsDBDown) + err := db.Ping() + if err != nil { + // TODO: There's a bit of a race here, but it doesn't matter if this error appears multiple times in the logs as it's capped at three times, we just want to cut it down 99% of the time + if isDBDown == 0 { + common.LogWarning(err) + common.LogWarning(errors.New("The database is down")) + } + atomic.StoreInt32(&common.IsDBDown, 1) + return true + } + if isDBDown == 1 { + log.Print("The database is back") + } + atomic.StoreInt32(&common.IsDBDown, 0) + return false + } var runHook = func(name string) { err := common.RunTaskHook(name) if err != nil { @@ -370,10 +390,16 @@ func main() { for { select { case <-halfSecondTicker.C: + if startTick() { + continue + } runHook("before_half_second_tick") runTasks(common.ScheduledHalfSecondTasks) runHook("after_half_second_tick") case <-secondTicker.C: + if startTick() { + continue + } runHook("before_second_tick") runTasks(common.ScheduledSecondTasks) @@ -397,6 +423,9 @@ func main() { // TODO: Rescan the static files for changes runHook("after_second_tick") case <-fifteenMinuteTicker.C: + if startTick() { + continue + } runHook("before_fifteen_minute_tick") runTasks(common.ScheduledFifteenMinuteTasks) @@ -404,6 +433,9 @@ func main() { // TODO: Publish scheduled posts. runHook("after_fifteen_minute_tick") case <-hourTicker.C: + if startTick() { + continue + } runHook("before_hour_tick") jsToken, err := common.GenerateSafeString(80) diff --git a/router_gen/main.go b/router_gen/main.go index 37d3f4ea..5f2476eb 100644 --- a/router_gen/main.go +++ b/router_gen/main.go @@ -225,6 +225,7 @@ import ( "strings" "strconv" "sync" + "sync/atomic" "errors" "os" "net/http" @@ -481,6 +482,10 @@ func (router *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { routes.StaticFile(w, req) return } + if atomic.LoadInt32(&common.IsDBDown) == 1 { + common.DatabaseError(w, req) + return + } if common.Dev.SuperDebug { router.requestLogger.Print("before PreRoute") }