Open platform to collect and prioritize feedback
enhancement: output logs to SQL (#432)
Guilherme Oenning47602114ff02
diff --git a/app/cmd/migrate.go b/app/cmd/migrate.goindex 4969d2313..6f123ac69 100644--- a/app/cmd/migrate.go+++ b/app/cmd/migrate.go@@ -3,13 +3,18 @@ package cmdimport ("github.com/getfider/fider/app/pkg/dbx""github.com/getfider/fider/app/pkg/log"+ "github.com/getfider/fider/app/pkg/log/database"+ "github.com/getfider/fider/app/pkg/uuid")// RunMigrate run all pending migrations on current DATABASE_URL// Returns an exitcode, 0 for OK and 1 for ERRORfunc RunMigrate() int {- logger := log.NewConsoleLogger("MIGRATE")- db := dbx.NewWithLogger(logger)+ db := dbx.New()+ logger := database.NewLogger("MIGRATE", db)+ logger.SetProperty(log.PropertyKeyContextID, uuid.NewV4().String())+ db.SetLogger(logger)+defer db.Close()err := db.Migrate("/migrations")if err != nil {diff --git a/app/cmd/routes.go b/app/cmd/routes.goindex 9322229fa..c4e9bf208 100644--- a/app/cmd/routes.go+++ b/app/cmd/routes.go@@ -12,7 +12,7 @@ import ()func routes(r *web.Engine) *web.Engine {- r.Worker().Use(middlewares.WorkerSetup(r.Worker().Logger()))+ r.Worker().Use(middlewares.WorkerSetup())r.Use(middlewares.Secure())r.Use(middlewares.Compress())@@ -25,7 +25,7 @@ func routes(r *web.Engine) *web.Engine {assets.Static("/assets/*filepath", "dist")}- r.Use(middlewares.WebSetup(r.Logger()))+ r.Use(middlewares.WebSetup())noTenant := r.Group(){diff --git a/app/cmd/server.go b/app/cmd/server.goindex b1e8352dc..26415fcc7 100644--- a/app/cmd/server.go+++ b/app/cmd/server.go@@ -5,6 +5,7 @@ import ("fmt""os""os/signal"+ "runtime""runtime/pprof""syscall"@@ -22,9 +23,6 @@ func RunServer(settings *models.SystemSettings) int {return exit}- fmt.Printf("Application is starting...\n")- fmt.Printf("GO_ENV: %s\n", env.Current())-e := routes(web.New(settings))go e.Start(":" + env.GetEnvOrDefault("PORT", "3000"))@@ -45,17 +43,19 @@ func listenSignals(e *web.Engine, settings *models.SystemSettings) int {}return 0case syscall.SIGUSR1:- e.Logger().Infof("SIGUSR1 received")- e.Logger().Infof("Dumping process status")+ e.Logger().Info("SIGUSR1 received")+ e.Logger().Info("Dumping process status")buf := new(bytes.Buffer)- buf.WriteString(fmt.Sprintf("Version: %s\n", settings.Version))- buf.WriteString(fmt.Sprintf("BuildTime: %s\n", settings.BuildTime))- buf.WriteString(fmt.Sprintf("Compiler: %s\n", settings.Compiler))- buf.WriteString(fmt.Sprintf("Environment: %s\n", settings.Environment))- buf.WriteString(fmt.Sprintf("Worker Queue: %d\n", e.Worker().Length()))pprof.Lookup("goroutine").WriteTo(buf, 1)pprof.Lookup("heap").WriteTo(buf, 1)- e.Logger().Infof("%s", buf.String())+ buf.WriteString("\n")+ buf.WriteString(fmt.Sprintf("# FIDER v%s\n", settings.Version))+ buf.WriteString(fmt.Sprintf("# BuildTime: %s\n", settings.BuildTime))+ buf.WriteString(fmt.Sprintf("# Compiler: %s\n", settings.Compiler))+ buf.WriteString(fmt.Sprintf("# Environment: %s\n", settings.Environment))+ buf.WriteString(fmt.Sprintf("# Worker Queue: %d\n", e.Worker().Length()))+ buf.WriteString(fmt.Sprintf("# Num Goroutines: %d\n", runtime.NumGoroutine()))+ e.Logger().Info(buf.String())}}}diff --git a/app/handlers/images.go b/app/handlers/images.goindex 822036a7d..939669fa4 100644--- a/app/handlers/images.go+++ b/app/handlers/images.go@@ -9,6 +9,7 @@ import ("strings""github.com/getfider/fider/app/pkg/img"+ "github.com/getfider/fider/app/pkg/log""github.com/getfider/fider/app/pkg/md5""github.com/getfider/fider/app/pkg/web"@@ -27,7 +28,9 @@ func Avatar() web.HandlerFunc {if err == nil && user.Tenant.ID == c.Tenant().ID {if user.Email != "" {url := fmt.Sprintf("https://www.gravatar.com/avatar/%s?s=%d&d=404", md5.Hash(user.Email), size)- c.Logger().Debugf("Requesting gravatar: %s", url)+ c.Logger().Debugf("Requesting gravatar: @{GravatarURL}", log.Props{+ "GravatarURL": url,+ })resp, err := http.Get(url)if err == nil {defer resp.Body.Close()diff --git a/app/middlewares/domain.go b/app/middlewares/domain.goindex 634c56cae..2303cbf50 100644--- a/app/middlewares/domain.go+++ b/app/middlewares/domain.go@@ -8,6 +8,7 @@ import ("github.com/getfider/fider/app""github.com/getfider/fider/app/pkg/env""github.com/getfider/fider/app/pkg/errors"+ "github.com/getfider/fider/app/pkg/log""github.com/getfider/fider/app/pkg/web")@@ -65,7 +66,9 @@ func MultiTenant() web.MiddlewareFunc {}if errors.Cause(err) == app.ErrNotFound {- c.Logger().Debugf("Tenant not found for '%s'.", hostname)+ c.Logger().Debugf("Tenant not found for '@{Hostname}'.", log.Props{+ "Hostname": hostname,+ })return c.NotFound()}diff --git a/app/middlewares/setup.go b/app/middlewares/setup.goindex 19e7c0047..5836b6b92 100644--- a/app/middlewares/setup.go+++ b/app/middlewares/setup.go@@ -8,7 +8,6 @@ import ("github.com/getfider/fider/app/pkg/worker""github.com/getfider/fider/app"- "github.com/getfider/fider/app/pkg/dbx""github.com/getfider/fider/app/pkg/oauth""github.com/getfider/fider/app/pkg/web""github.com/getfider/fider/app/storage/postgres"@@ -24,28 +23,34 @@ func Noop() web.MiddlewareFunc {}//WorkerSetup current context with some services-func WorkerSetup(logger log.Logger) worker.MiddlewareFunc {- db := dbx.NewWithLogger(logger)- emailer := app.NewEmailer(logger)+func WorkerSetup() worker.MiddlewareFunc {return func(next worker.Job) worker.Job {return func(c *worker.Context) (err error) {start := time.Now()- c.Logger().Debugf("Task '%s' started on worker '%s'", log.Magenta(c.TaskName()), log.Magenta(c.WorkerID()))+ c.Logger().Debugf("Task '@{TaskName:magenta}' started on worker '@{WorkerID:magenta}'", log.Props{+ "TaskName": c.TaskName(),+ "WorkerID": c.WorkerID(),+ })- trx, err := db.Begin()+ trx, err := c.Database().Begin()if err != nil {err = c.Failure(err)- c.Logger().Debugf("Task '%s' finished in %s", log.Magenta(c.TaskName()), log.Magenta(time.Since(start).String()))+ c.Logger().Debugf("Task '@{TaskName:magenta}' finished in @{ElapsedMs:magenta}ms", log.Props{+ "TaskName": c.TaskName(),+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })return err}+ trx.SetLogger(c.Logger())+c.SetServices(&app.Services{Tenants: postgres.NewTenantStorage(trx),Users: postgres.NewUserStorage(trx),Ideas: postgres.NewIdeaStorage(trx),Tags: postgres.NewTagStorage(trx),Notifications: postgres.NewNotificationStorage(trx),- Emailer: emailer,+ Emailer: app.NewEmailer(c.Logger()),})//In case it panics somewhere@@ -57,49 +62,68 @@ func WorkerSetup(logger log.Logger) worker.MiddlewareFunc {}c.Failure(err)trx.Rollback()- c.Logger().Debugf("Task '%s' panicked in %s (rolled back)", log.Magenta(c.TaskName()), log.Magenta(time.Since(start).String()))+ c.Logger().Debugf("Task '@{TaskName:magenta}' panicked in @{ElapsedMs:magenta}ms (rolled back)", log.Props{+ "TaskName": c.TaskName(),+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })}}()//Execute the chainif err = next(c); err != nil {trx.Rollback()- c.Logger().Debugf("Task '%s' finished in %s (rolled back)", log.Magenta(c.TaskName()), log.Magenta(time.Since(start).String()))+ c.Logger().Debugf("Task '@{TaskName:magenta}' finished in @{ElapsedMs:magenta}ms (rolled back)", log.Props{+ "TaskName": c.TaskName(),+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })return err}//No errors, so try to commit itif err = trx.Commit(); err != nil {- c.Logger().Errorf("Failed to commit request with: %s", err.Error())- c.Logger().Debugf("Task '%s' finished in %s (rolled back)", log.Magenta(c.TaskName()), log.Magenta(time.Since(start).String()))+ c.Logger().Errorf("Failed to commit request with: @{Error}", log.Props{+ "Error": err.Error(),+ })+ c.Logger().Debugf("Task '@{TaskName:magenta}' finished in @{ElapsedMs:magenta}ms (rolled back)", log.Props{+ "TaskName": c.TaskName(),+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })return err}//Still no errors, everything is fine!- c.Logger().Debugf("Task '%s' finished in %s (committed)", log.Magenta(c.TaskName()), log.Magenta(time.Since(start).String()))+ c.Logger().Debugf("Task '@{TaskName:magenta}' finished in @{ElapsedMs:magenta}ms (committed)", log.Props{+ "TaskName": c.TaskName(),+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })return nil}}}//WebSetup current context with some services-func WebSetup(logger log.Logger) web.MiddlewareFunc {- db := dbx.NewWithLogger(logger)- emailer := app.NewEmailer(logger)+func WebSetup() web.MiddlewareFunc {return func(next web.HandlerFunc) web.HandlerFunc {return func(c web.Context) error {- path := log.Magenta(c.Request.Method + " " + c.Request.URL.RequestURI())-start := time.Now()- c.Logger().Debugf("%s started", path)+ c.Logger().Debugf("@{HttpMethod:magenta} @{RequestURI:magenta} started", log.Props{+ "HttpMethod": c.Request.Method,+ "RequestURI": c.Request.URL.RequestURI(),+ })- trx, err := db.Begin()+ trx, err := c.Engine().Database().Begin()if err != nil {err = c.Failure(err)- c.Logger().Debugf("%s finished in %s", path, log.Magenta(time.Since(start).String()))+ c.Logger().Debugf("@{HttpMethod:magenta} @{RequestURI:magenta} finished in @{ElapsedMs:magenta}ms", log.Props{+ "HttpMethod": c.Request.Method,+ "RequestURI": c.Request.URL.RequestURI(),+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })return err}+ trx.SetLogger(c.Logger())+c.SetActiveTransaction(trx)c.SetServices(&app.Services{Tenants: postgres.NewTenantStorage(trx),@@ -108,7 +132,7 @@ func WebSetup(logger log.Logger) web.MiddlewareFunc {Ideas: postgres.NewIdeaStorage(trx),Tags: postgres.NewTagStorage(trx),Notifications: postgres.NewNotificationStorage(trx),- Emailer: emailer,+ Emailer: app.NewEmailer(c.Logger()),})//In case it panics somewhere@@ -120,26 +144,44 @@ func WebSetup(logger log.Logger) web.MiddlewareFunc {}c.Failure(err)c.Rollback()- c.Logger().Debugf("%s panicked in %s (rolled back)", path, log.Magenta(time.Since(start).String()))+ c.Logger().Debugf("@{HttpMethod:magenta} @{RequestURI:magenta} panicked in @{ElapsedMs:magenta}ms (rolled back)", log.Props{+ "HttpMethod": c.Request.Method,+ "RequestURI": c.Request.URL.RequestURI(),+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })}}()//Execute the chainif err := next(c); err != nil {c.Rollback()- c.Logger().Debugf("%s finished in %s (rolled back)", path, log.Magenta(time.Since(start).String()))+ c.Logger().Debugf("@{HttpMethod:magenta} @{RequestURI:magenta} finished in @{ElapsedMs:magenta}ms (rolled back)", log.Props{+ "HttpMethod": c.Request.Method,+ "RequestURI": c.Request.URL.RequestURI(),+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })return err}//No errors, so try to commit itif err := c.Commit(); err != nil {- c.Logger().Errorf("Failed to commit request with: %s", err.Error())- c.Logger().Debugf("%s finished in %s (rolled back)", path, log.Magenta(time.Since(start).String()))+ c.Logger().Errorf("Failed to commit request with: @{Error}", log.Props{+ "Error": err.Error(),+ })+ c.Logger().Debugf("@{HttpMethod:magenta} @{RequestURI:magenta} finished in @{ElapsedMs:magenta}ms (rolled back)", log.Props{+ "HttpMethod": c.Request.Method,+ "RequestURI": c.Request.URL.RequestURI(),+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })return err}//Still no errors, everything is fine!- c.Logger().Debugf("%s finished in %s (committed)", path, log.Magenta(time.Since(start).String()))+ c.Logger().Debugf("@{HttpMethod:magenta} @{RequestURI:magenta} finished in @{ElapsedMs:magenta}ms (committed)", log.Props{+ "HttpMethod": c.Request.Method,+ "RequestURI": c.Request.URL.RequestURI(),+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })return nil}}diff --git a/app/middlewares/setup_test.go b/app/middlewares/setup_test.goindex 22a55c07e..f03cdba34 100644--- a/app/middlewares/setup_test.go+++ b/app/middlewares/setup_test.go@@ -7,7 +7,8 @@ import ("github.com/getfider/fider/app/middlewares". "github.com/getfider/fider/app/pkg/assert"- "github.com/getfider/fider/app/pkg/log"+ "github.com/getfider/fider/app/pkg/dbx"+ "github.com/getfider/fider/app/pkg/log/noop""github.com/getfider/fider/app/pkg/mock""github.com/getfider/fider/app/pkg/web""github.com/getfider/fider/app/pkg/worker"@@ -17,7 +18,7 @@ func TestWebSetup(t *testing.T) {RegisterT(t)server, _ := mock.NewServer()- server.Use(middlewares.WebSetup(log.NewNoopLogger()))+ server.Use(middlewares.WebSetup())status, _ := server.Execute(func(c web.Context) error {Expect(c.ActiveTransaction()).IsNotNil()return c.NoContent(http.StatusOK)@@ -30,7 +31,7 @@ func TestWebSetup_Failure(t *testing.T) {RegisterT(t)server, _ := mock.NewServer()- server.Use(middlewares.WebSetup(log.NewNoopLogger()))+ server.Use(middlewares.WebSetup())status, _ := server.Execute(func(c web.Context) error {return c.Failure(errors.New("Something went wrong..."))})@@ -42,7 +43,7 @@ func TestWebSetup_Panic(t *testing.T) {RegisterT(t)server, _ := mock.NewServer()- server.Use(middlewares.WebSetup(log.NewNoopLogger()))+ server.Use(middlewares.WebSetup())status, _ := server.Execute(func(c web.Context) error {panic("Boom!")})@@ -54,7 +55,7 @@ func TestWebSetup_NotQueueTask_OnFailure(t *testing.T) {RegisterT(t)server, _ := mock.NewServer()- server.Use(middlewares.WebSetup(log.NewNoopLogger()))+ server.Use(middlewares.WebSetup())status, _ := server.Execute(func(c web.Context) error {c.Enqueue(mock.NewNoopTask())return c.Failure(errors.New("Something went wrong..."))@@ -68,7 +69,7 @@ func TestWebSetup_QueueTask_OnSuccess(t *testing.T) {RegisterT(t)server, _ := mock.NewServer()- server.Use(middlewares.WebSetup(log.NewNoopLogger()))+ server.Use(middlewares.WebSetup())status, _ := server.Execute(func(c web.Context) error {c.Enqueue(mock.NewNoopTask())return c.Ok(web.Map{})@@ -81,8 +82,8 @@ func TestWebSetup_QueueTask_OnSuccess(t *testing.T) {func TestWorkerSetup(t *testing.T) {RegisterT(t)- c := worker.NewContext("0", "Any Task", log.NewNoopLogger())- mw := middlewares.WorkerSetup(log.NewNoopLogger())+ c := worker.NewContext("0", "Any Task", dbx.New(), noop.NewLogger())+ mw := middlewares.WorkerSetup()err := mw(func(c *worker.Context) error {Expect(c.Services()).IsNotNil()return nil@@ -93,8 +94,8 @@ func TestWorkerSetup(t *testing.T) {func TestWorkerSetup_Failure(t *testing.T) {RegisterT(t)- c := worker.NewContext("0", "Any Task", log.NewNoopLogger())- mw := middlewares.WorkerSetup(log.NewNoopLogger())+ c := worker.NewContext("0", "Any Task", dbx.New(), noop.NewLogger())+ mw := middlewares.WorkerSetup()err := mw(func(c *worker.Context) error {Expect(c.Services()).IsNotNil()return errors.New("Not Found")diff --git a/app/pkg/dbx/dbx.go b/app/pkg/dbx/dbx.goindex 9230e2ada..91e72a76b 100644--- a/app/pkg/dbx/dbx.go+++ b/app/pkg/dbx/dbx.go@@ -12,6 +12,7 @@ import ("github.com/getfider/fider/app/pkg/env""github.com/getfider/fider/app/pkg/errors""github.com/getfider/fider/app/pkg/log"+ "github.com/getfider/fider/app/pkg/log/noop"//required_ "github.com/lib/pq"@@ -19,7 +20,7 @@ import (// New creates a new Database instance without loggingfunc New() *Database {- return NewWithLogger(&log.NoopLogger{})+ return NewWithLogger(noop.NewLogger())}// NewWithLogger creates a new Database instance with logging or panic@@ -41,6 +42,11 @@ type Database struct {mapper *RowMapper}+// SetLogger replaces current database Logger+func (db *Database) SetLogger(logger log.Logger) {+ db.logger = logger+}+// Begin returns a new SQL transactionfunc (db *Database) Begin() (*Trx, error) {tx, err := db.conn.Begin()@@ -86,13 +92,27 @@ type Trx struct {var formatter = strings.NewReplacer("\t", "", "\n", " ")+// SetLogger replaces current transaction Logger+func (trx *Trx) SetLogger(logger log.Logger) {+ trx.logger = logger+}++// NoLogs disable logs from this transaction+func (trx *Trx) NoLogs() {+ trx.logger.SetLevel(log.NONE)+}+// Execute given SQL commandfunc (trx *Trx) Execute(command string, args ...interface{}) (int64, error) {if trx.logger.IsEnabled(log.DEBUG) {command = formatter.Replace(command)start := time.Now()defer func() {- trx.logger.Debugf("%s %v in %s", log.Yellow(command), log.Blue(args), log.Magenta(time.Since(start).String()))+ trx.logger.Debugf("@{Command:yellow} @{Args:blue} executed in @{ElapsedMs:magenta}ms", log.Props{+ "Command": command,+ "Args": args,+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })}()}@@ -111,7 +131,11 @@ func (trx *Trx) Scalar(data interface{}, command string, args ...interface{}) ercommand = formatter.Replace(command)start := time.Now()defer func() {- trx.logger.Debugf("%s %v in %s", log.Yellow(command), log.Blue(args), log.Magenta(time.Since(start).String()))+ trx.logger.Debugf("@{Command:yellow} @{Args:blue} executed in @{ElapsedMs:magenta}ms", log.Props{+ "Command": command,+ "Args": args,+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })}()}@@ -132,7 +156,11 @@ func (trx *Trx) Get(data interface{}, command string, args ...interface{}) errorcommand = formatter.Replace(command)start := time.Now()defer func() {- trx.logger.Debugf("%s %v in %s", log.Yellow(command), log.Blue(args), log.Magenta(time.Since(start).String()))+ trx.logger.Debugf("@{Command:yellow} @{Args:blue} executed in @{ElapsedMs:magenta}ms", log.Props{+ "Command": command,+ "Args": args,+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })}()}@@ -160,7 +188,11 @@ func (trx *Trx) QueryIntArray(command string, args ...interface{}) ([]int, errorcommand = formatter.Replace(command)start := time.Now()defer func() {- trx.logger.Debugf("%s %v in %s", log.Yellow(command), log.Blue(args), log.Magenta(time.Since(start).String()))+ trx.logger.Debugf("@{Command:yellow} @{Args:blue} executed in @{ElapsedMs:magenta}ms", log.Props{+ "Command": command,+ "Args": args,+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })}()}@@ -188,7 +220,11 @@ func (trx *Trx) Exists(command string, args ...interface{}) (bool, error) {command = formatter.Replace(command)start := time.Now()defer func() {- trx.logger.Debugf("%s %v in %s", log.Yellow(command), log.Blue(args), log.Magenta(time.Since(start).String()))+ trx.logger.Debugf("@{Command:yellow} @{Args:blue} executed in @{ElapsedMs:magenta}ms", log.Props{+ "Command": command,+ "Args": args,+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })}()}@@ -207,7 +243,11 @@ func (trx *Trx) Count(command string, args ...interface{}) (int, error) {command = formatter.Replace(command)start := time.Now()defer func() {- trx.logger.Debugf("%s %v in %s", log.Yellow(command), log.Blue(args), log.Magenta(time.Since(start).String()))+ trx.logger.Debugf("@{Command:yellow} @{Args:blue} executed in @{ElapsedMs:magenta}ms", log.Props{+ "Command": command,+ "Args": args,+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })}()}@@ -230,7 +270,11 @@ func (trx *Trx) Select(data interface{}, command string, args ...interface{}) ercommand = formatter.Replace(command)start := time.Now()defer func() {- trx.logger.Debugf("%s %v in %s", log.Yellow(command), log.Blue(args), log.Magenta(time.Since(start).String()))+ trx.logger.Debugf("@{Command:yellow} @{Args:blue} executed in @{ElapsedMs:magenta}ms", log.Props{+ "Command": command,+ "Args": args,+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })}()}diff --git a/app/pkg/dbx/migrate.go b/app/pkg/dbx/migrate.goindex ee27ccfd1..9e84bd7a9 100644--- a/app/pkg/dbx/migrate.go+++ b/app/pkg/dbx/migrate.go@@ -11,6 +11,7 @@ import ("github.com/getfider/fider/app/pkg/env""github.com/getfider/fider/app/pkg/errors"+ "github.com/getfider/fider/app/pkg/log")// ErrNoChanges means that the migration process didn't change execute any file@@ -18,7 +19,7 @@ var ErrNoChanges = stdErrors.New("nothing to migrate.")// Migrate the database to latest versionfunc (db *Database) Migrate(path string) error {- db.logger.Infof("Running migrations...")+ db.logger.Info("Running migrations...")dir, err := os.Open(env.Path(path))if err != nil {return errors.Wrap(err, "failed to open dir '%s'", path)@@ -41,19 +42,28 @@ func (db *Database) Migrate(path string) error {}}sort.Ints(versions)- db.logger.Infof("Found total of %d migration files.", len(versions))++ db.logger.Infof("Found total of @{Total} migration files.", log.Props{+ "Total": len(versions),+ })lastVersion, err := db.getLastMigration()if err != nil {return errors.Wrap(err, "failed to get last migration record")}- db.logger.Infof("Current version is %d", lastVersion)++ db.logger.Infof("Current version is @{Version}", log.Props{+ "Version": lastVersion,+ })// Apply all migrationsfor _, version := range versions {if version > lastVersion {fileName := versionFiles[version]- db.logger.Infof("Running Version: %d (%s)", version, fileName)+ db.logger.Infof("Running Version: @{Version} (@{FileName})", log.Props{+ "Version": version,+ "FileName": fileName,+ })err := db.runMigration(version, path, fileName)if err != nil {return errors.Wrap(err, "failed to run migration '%s'", fileName)@@ -61,7 +71,7 @@ func (db *Database) Migrate(path string) error {}}- db.logger.Infof("Migrations finished with success.")+ db.logger.Info("Migrations finished with success.")return nil}diff --git a/app/pkg/dbx/setup.sql b/app/pkg/dbx/setup.sqlindex 9ca4068cb..ac9659de9 100644--- a/app/pkg/dbx/setup.sql+++ b/app/pkg/dbx/setup.sql@@ -1,3 +1,4 @@+TRUNCATE TABLE logs RESTART IDENTITY CASCADE;TRUNCATE TABLE tenants RESTART IDENTITY CASCADE;INSERT INTO tenants (name, subdomain, created_on, cname, invitation, welcome_message, status, is_private, custom_css)diff --git a/app/pkg/email/mailgun/mailgun.go b/app/pkg/email/mailgun/mailgun.goindex 6bb3577e9..9b30fcde1 100644--- a/app/pkg/email/mailgun/mailgun.go+++ b/app/pkg/email/mailgun/mailgun.go@@ -69,7 +69,10 @@ func (s *Sender) BatchSend(tenant *models.Tenant, templateName string, params emform.Add("to", fmt.Sprintf("%s <%s>", r.Name, r.Address))recipientVariables[r.Address] = r.Params} else {- s.logger.Warnf("Skipping email to '%s <%s>' due to whitelist.", r.Name, r.Address)+ s.logger.Warnf("Skipping email to '@{Name} <@{Address}>' due to whitelist.", log.Props{+ "Name": r.Name,+ "Address": r.Address,+ })}}}@@ -89,9 +92,15 @@ func (s *Sender) BatchSend(tenant *models.Tenant, templateName string, params em}if isBatch {- s.logger.Debugf("Sending email to %d recipients with template %s.", len(recipientVariables), templateName)+ s.logger.Debugf("Sending email to @{CountRecipients} recipients with template @{TemplateName}.", log.Props{+ "CountRecipients": len(recipientVariables),+ "TemplateName": templateName,+ })} else {- s.logger.Debugf("Sending email to %s with template %s.", to[0].Address, templateName)+ s.logger.Debugf("Sending email to @{Address} with template @{TemplateName}.", log.Props{+ "Address": to[0].Address,+ "TemplateName": templateName,+ })}url := fmt.Sprintf(baseURL, s.domain)@@ -109,6 +118,8 @@ func (s *Sender) BatchSend(tenant *models.Tenant, templateName string, params em}defer resp.Body.Close()- s.logger.Debugf("Email sent with response code %d.", resp.StatusCode)+ s.logger.Debugf("Email sent with response code @{StatusCode}.", log.Props{+ "StatusCode": resp.StatusCode,+ })return nil}diff --git a/app/pkg/email/smtp/smtp.go b/app/pkg/email/smtp/smtp.goindex a2c4be1d4..4f45626cf 100644--- a/app/pkg/email/smtp/smtp.go+++ b/app/pkg/email/smtp/smtp.go@@ -38,11 +38,18 @@ func (s *Sender) Send(tenant *models.Tenant, templateName string, params email.P}if !email.CanSendTo(to.Address) {- s.logger.Warnf("Skipping email to %s <%s> due to whitelist.", to.Name, to.Address)+ s.logger.Warnf("Skipping email to '@{Name} <@{Address}>' due to whitelist.", log.Props{+ "Name": to.Name,+ "Address": to.Address,+ })return nil}- s.logger.Debugf("Sending email to %s with template %s and params %s.", to.Address, templateName, to.Params)+ s.logger.Debugf("Sending email to @{Address} with template @{TemplateName} and params @{Params}.", log.Props{+ "Address": to.Address,+ "TemplateName": templateName,+ "Params": to.Params,+ })message := email.RenderMessage(templateName, params.Merge(to.Params))headers := make(map[string]string)@@ -64,7 +71,7 @@ func (s *Sender) Send(tenant *models.Tenant, templateName string, params email.Pif err != nil {return errors.Wrap(err, "failed to send email with template %s", templateName)}- s.logger.Debugf("Email sent.")+ s.logger.Debug("Email sent.")return nil}diff --git a/app/pkg/log/colors.go b/app/pkg/log/colors.goindex 32b445420..913911477 100644--- a/app/pkg/log/colors.go+++ b/app/pkg/log/colors.go@@ -1,6 +1,9 @@package log-import "fmt"+import (+ "fmt"+ "strings"+)var (colorRed = "\033[31m"@@ -13,6 +16,28 @@ var (resetAll = "\033[0m")+// Color return content with given color+func Color(colorName string, content interface{}) string {+ switch strings.ToUpper(colorName) {+ case "RED":+ return Red(content)+ case "GREEN":+ return Green(content)+ case "YELLOW":+ return Yellow(content)+ case "BLUE":+ return Blue(content)+ case "MAGENTA":+ return Magenta(content)+ case "BOLD":+ return Bold(content)+ case "REVERSE":+ return Reverse(content)+ default:+ return fmt.Sprintf("%v", content)+ }+}+// Red return content with red colorfunc Red(content interface{}) string {return fmt.Sprintf("%s%v%s", colorRed, content, resetAll)diff --git a/app/pkg/log/console/logger.go b/app/pkg/log/console/logger.gonew file mode 100644index 000000000..f8032f25a--- /dev/null+++ b/app/pkg/log/console/logger.go@@ -0,0 +1,131 @@+package console++import (+ "fmt"+ stdLog "log"+ "os"+ "strings"+ "time"++ "github.com/getfider/fider/app/pkg/env"+ "github.com/getfider/fider/app/pkg/log"+)++// Logger output messages to console+type Logger struct {+ logger *stdLog.Logger+ level log.Level+ tag string+ props log.Props+}++// NewLogger creates a new Logger+func NewLogger(tag string) *Logger {+ level := strings.ToUpper(env.GetEnvOrDefault("LOG_LEVEL", ""))+ return &Logger{+ tag: tag,+ logger: stdLog.New(os.Stdout, "", 0),+ level: log.ParseLevel(level),+ props: make(log.Props, 0),+ }+}++// SetLevel increases/decreases current log level+func (l *Logger) SetLevel(level log.Level) {+ l.level = level+}++// IsEnabled returns true if given level is enabled+func (l *Logger) IsEnabled(level log.Level) bool {+ return level >= l.level+}++// Debug logs a DEBUG message+func (l *Logger) Debug(message string) {+ l.log(log.DEBUG, message, nil)+}++// Debugf logs a DEBUG message+func (l *Logger) Debugf(message string, props log.Props) {+ l.log(log.DEBUG, message, props)+}++// Info logs a INFO message+func (l *Logger) Info(message string) {+ l.log(log.INFO, message, nil)+}++// Infof logs a INFO message+func (l *Logger) Infof(message string, props log.Props) {+ l.log(log.INFO, message, props)+}++// Warn logs a WARN message+func (l *Logger) Warn(message string) {+ l.log(log.WARN, message, nil)+}++// Warnf logs a WARN message+func (l *Logger) Warnf(message string, props log.Props) {+ l.log(log.WARN, message, props)+}++// Errorf logs a ERROR message+func (l *Logger) Errorf(message string, props log.Props) {+ l.log(log.ERROR, message, props)+}++// Error logs a ERROR message+func (l *Logger) Error(err error) {+ if err != nil {+ l.log(log.ERROR, err.Error(), nil)+ } else {+ l.log(log.ERROR, "nil", nil)+ }+}++// Write writes len(p) bytes from p to the underlying data stream.+func (l *Logger) Write(p []byte) (int, error) {+ l.Debug(fmt.Sprintf("%s", p))+ return len(p), nil+}++// New returns a copy of current logger with empty context+func (l *Logger) New() log.Logger {+ return NewLogger(l.tag)+}++// SetProperty with given key:value into current logger context+func (l *Logger) SetProperty(key string, value interface{}) {+ l.props[key] = value+}++func (l *Logger) log(level log.Level, message string, props log.Props) {+ props = l.props.Merge(props)+ if !l.IsEnabled(level) {+ return+ }++ message = log.Parse(message, props, true)+ contextID := l.props[log.PropertyKeyContextID]+ l.logger.Printf("%s [%s] [%s] [%s] %s\n", colorizeLevel(level), time.Now().Format(time.RFC3339), l.tag, contextID, message)+}++func colorizeLevel(level log.Level) string {+ var color func(interface{}) string++ switch level {+ case log.DEBUG:+ color = log.Magenta+ case log.INFO:+ color = log.Blue+ case log.WARN:+ color = log.Yellow+ case log.ERROR:+ color = log.Red+ default:+ color = log.Red+ }++ return log.Bold(color(level.String()))+}diff --git a/app/pkg/log/database/logger.go b/app/pkg/log/database/logger.gonew file mode 100644index 000000000..1f8f6a402--- /dev/null+++ b/app/pkg/log/database/logger.go@@ -0,0 +1,158 @@+package database++import (+ "fmt"+ "strings"+ "time"++ "github.com/getfider/fider/app/pkg/errors"+ "github.com/getfider/fider/app/pkg/log/console"++ "github.com/getfider/fider/app/pkg/dbx"+ "github.com/getfider/fider/app/pkg/env"+ "github.com/getfider/fider/app/pkg/log"+)++// Logger writes logs to a SQL database+type Logger struct {+ db *dbx.Database+ console log.Logger+ level log.Level+ tag string+ props log.Props+}++// NewLogger creates a new Logger+func NewLogger(tag string, db *dbx.Database) *Logger {+ level := strings.ToUpper(env.GetEnvOrDefault("LOG_LEVEL", ""))+ return &Logger{+ tag: tag,+ db: db,+ console: console.NewLogger(tag),+ level: log.ParseLevel(level),+ props: make(log.Props, 0),+ }+}++// SetLevel increases/decreases current log level+func (l *Logger) SetLevel(level log.Level) {+ l.level = level+ l.console.SetLevel(level)+}++// IsEnabled returns true if given level is enabled+func (l *Logger) IsEnabled(level log.Level) bool {+ return level >= l.level+}++// Debug logs a DEBUG message+func (l *Logger) Debug(message string) {+ l.console.Debug(message)+ l.log(log.DEBUG, message, nil)+}++// Debugf logs a DEBUG message+func (l *Logger) Debugf(message string, props log.Props) {+ l.console.Debugf(message, props)+ l.log(log.DEBUG, message, props)+}++// Info logs a INFO message+func (l *Logger) Info(message string) {+ l.console.Info(message)+ l.log(log.INFO, message, nil)+}++// Infof logs a INFO message+func (l *Logger) Infof(message string, props log.Props) {+ l.console.Infof(message, props)+ l.log(log.INFO, message, props)+}++// Warn logs a WARN message+func (l *Logger) Warn(message string) {+ l.console.Warn(message)+ l.log(log.WARN, message, nil)+}++// Warnf logs a WARN message+func (l *Logger) Warnf(message string, props log.Props) {+ l.console.Warnf(message, props)+ l.log(log.WARN, message, props)+}++// Errorf logs a ERROR message+func (l *Logger) Errorf(message string, props log.Props) {+ l.console.Errorf(message, props)+ l.log(log.ERROR, message, props)+}++// Error logs a ERROR message+func (l *Logger) Error(err error) {+ if err != nil {+ l.log(log.ERROR, err.Error(), nil)+ } else {+ l.log(log.ERROR, "nil", nil)+ }+ l.console.Error(err)+}++// Write writes len(p) bytes from p to the underlying data stream.+func (l *Logger) Write(p []byte) (int, error) {+ l.console.Write(p)+ l.Debug(fmt.Sprintf("%s", p))+ return len(p), nil+}++// New returns a copy of current logger with empty context+func (l *Logger) New() log.Logger {+ return NewLogger(l.tag, l.db)+}++// SetProperty with given key:value into current logger context+func (l *Logger) SetProperty(key string, value interface{}) {+ l.props[key] = value+ l.console.SetProperty(key, value)+}++func (l *Logger) log(level log.Level, message string, props log.Props) {+ if !l.IsEnabled(level) {+ return+ }++ write := func() {+ props = l.props.Merge(props)+ trx, err := l.db.Begin()+ if err != nil {+ l.console.Error(errors.Wrap(err, "failed to open transaction"))+ return+ }+ trx.NoLogs()++ message = log.Parse(message, props, false)++ _, err = trx.Execute(+ "INSERT INTO logs (tag, level, text, created_on, properties) VALUES ($1, $2, $3, $4, $5)",+ l.tag, level.String(), message, time.Now(), props,+ )++ if err != nil {+ l.console.Error(errors.Wrap(err, "failed to insert log"))+ err = trx.Rollback()+ if err != nil {+ l.console.Error(errors.Wrap(err, "failed to rollback transaction"))+ }+ } else {+ err = trx.Commit()+ if err != nil {+ l.console.Error(errors.Wrap(err, "failed to commit transaction"))+ }+ }+ }++ if env.IsTest() {+ write()+ } else {+ go write()+ }+}diff --git a/app/pkg/log/database/logger_test.go b/app/pkg/log/database/logger_test.gonew file mode 100644index 000000000..b17f4ce40--- /dev/null+++ b/app/pkg/log/database/logger_test.go@@ -0,0 +1,41 @@+package database_test++import (+ "testing"++ . "github.com/getfider/fider/app/pkg/assert"+ "github.com/getfider/fider/app/pkg/dbx"+ "github.com/getfider/fider/app/pkg/log"+ "github.com/getfider/fider/app/pkg/log/database"+)++type logEntry struct {+ Tag string `db:"tag"`+ Level string `db:"level"`+ Text string `db:"text"`+}++func TestLog(t *testing.T) {+ RegisterT(t)+ db := dbx.New()+ defer db.Close()+ trx, _ := db.Begin()+ trx.Execute("DELETE FROM logs WHERE properties->>'ContextID' = 'MyContextID'")+ defer trx.Commit()++ logger := database.NewLogger("TEST", db)+ logger.SetLevel(log.INFO)+ logger.SetProperty(log.PropertyKeyContextID, "MyContextID")+ db.SetLogger(logger)++ logger.Infof("2 + 2 is @{Result}", log.Props{"Result": 4})+ logger.Debugf("2 + 2 is @{Result}", log.Props{"Result": 4})++ var entries []*logEntry+ err := trx.Select(&entries, "SELECT tag, level, text FROM logs WHERE properties->>'ContextID' = 'MyContextID'")+ Expect(err).IsNil()+ Expect(entries).HasLen(1)+ Expect(entries[0].Tag).Equals("TEST")+ Expect(entries[0].Level).Equals("INFO")+ Expect(entries[0].Text).Equals("2 + 2 is 4")+}diff --git a/app/pkg/log/log.go b/app/pkg/log/log.gonew file mode 100644index 000000000..03b5dce61--- /dev/null+++ b/app/pkg/log/log.go@@ -0,0 +1,137 @@+package log++import (+ "database/sql/driver"+ "encoding/json"+ "fmt"+ "regexp"+ "strings"+)++// Level defines all possible log levels+type Level uint8++// Logger defines the logging interface.+type Logger interface {+ SetLevel(level Level)+ SetProperty(key string, value interface{})+ Debug(message string)+ Debugf(message string, props Props)+ Info(message string)+ Infof(message string, props Props)+ Warn(format string)+ Warnf(message string, props Props)+ Error(err error)+ Errorf(message string, props Props)+ IsEnabled(level Level) bool+ Write(p []byte) (int, error)+ New() Logger+}++const (+ // PropertyKeyContextID is the context id of current logger+ PropertyKeyContextID = "ContextID"+ // PropertyKeyUserID is the user id of current logger+ PropertyKeyUserID = "UserID"+ // PropertyKeyTenantID is the tenant id of current logger+ PropertyKeyTenantID = "TenantID"+)++const (+ // DEBUG for verbose logs+ DEBUG Level = iota + 1+ // INFO for WARN+ERROR+INFO logs+ INFO+ // WARN for WARN+ERROR logs+ WARN+ // ERROR for ERROR only logs+ ERROR+ // NONE is used to disable logs+ NONE+)++// ParseLevel returns a log.Level based on input string+func ParseLevel(level string) Level {+ switch level {+ case "DEBUG":+ return DEBUG+ case "WARN":+ return WARN+ case "ERROR":+ return ERROR+ default:+ return INFO+ }+}++func (l Level) String() string {+ switch l {+ case DEBUG:+ return "DEBUG"+ case INFO:+ return "INFO"+ case WARN:+ return "WARN"+ case ERROR:+ return "ERROR"+ case NONE:+ return "NONE"+ default:+ return "???"+ }+}++// Props is a map of key:value+type Props map[string]interface{}++// Value converts props into a database value+func (p Props) Value() (driver.Value, error) {+ j, err := json.Marshal(p)+ return j, err+}++// Merge current props with given props+func (p Props) Merge(props Props) Props {+ new := Props{}+ if p != nil {+ for k, v := range p {+ new[k] = v+ }+ }+ if props != nil {+ for k, v := range props {+ new[k] = v+ }+ }+ return new+}++var placeholderFinder = regexp.MustCompile("@{.*?}")++// Parse is used to merge props into format and return a text message+func Parse(format string, props Props, colorize bool) string {+ if props == nil || len(props) == 0 {+ return format+ }++ for {+ indexes := placeholderFinder.FindSubmatchIndex([]byte(format))+ if len(indexes) == 0 {+ return format+ }++ ph := format[indexes[0]:indexes[1]]+ phContent := ph[2 : len(ph)-1]+ phSeparatorIdx := strings.Index(phContent, ":")+ value := props[phContent]+ if phSeparatorIdx >= 0 {+ phName := phContent[:phSeparatorIdx]+ phColor := phContent[phSeparatorIdx+1:]+ value = props[phName]+ if colorize {+ value = Color(phColor, value)+ }+ }+ format = fmt.Sprintf("%s%v%s", format[:indexes[0]], value, format[indexes[1]:])+ }+}diff --git a/app/pkg/log/log_test.go b/app/pkg/log/log_test.gonew file mode 100644index 000000000..22e1922c8--- /dev/null+++ b/app/pkg/log/log_test.go@@ -0,0 +1,97 @@+package log_test++import (+ "testing"++ . "github.com/getfider/fider/app/pkg/assert"+ "github.com/getfider/fider/app/pkg/log"+)++func TestLevel_Parse(t *testing.T) {+ RegisterT(t)++ Expect(log.ParseLevel("DEBUG")).Equals(log.DEBUG)+ Expect(log.ParseLevel("INFO")).Equals(log.INFO)+ Expect(log.ParseLevel("WARN")).Equals(log.WARN)+ Expect(log.ParseLevel("ERROR")).Equals(log.ERROR)+ Expect(log.ParseLevel("NONE")).Equals(log.INFO)+ Expect(log.ParseLevel("")).Equals(log.INFO)+}++func TestLevel_ToString(t *testing.T) {+ RegisterT(t)++ Expect(log.DEBUG.String()).Equals("DEBUG")+ Expect(log.INFO.String()).Equals("INFO")+ Expect(log.WARN.String()).Equals("WARN")+ Expect(log.ERROR.String()).Equals("ERROR")+ Expect(log.DEBUG.String()).Equals("DEBUG")+ Expect(log.Level(88).String()).Equals("???")+}++func TestPropsMerge(t *testing.T) {+ RegisterT(t)++ Expect(log.Props{+ "Name": "John",+ }.Merge(nil)).Equals(log.Props{"Name": "John"})++ Expect(log.Props{+ "Name": "John",+ }.Merge(log.Props{+ "Name": "Maria",+ })).Equals(log.Props{"Name": "Maria"})++ Expect(log.Props(nil).Merge(log.Props{+ "Name": "Maria",+ })).Equals(log.Props{"Name": "Maria"})++ Expect(log.Props{+ "Name": "John",+ "DOB": "05/02/1998",+ }.Merge(log.Props{+ "Name": "Maria",+ "Age": 66,+ })).Equals(log.Props{+ "Age": 66,+ "DOB": "05/02/1998",+ "Name": "Maria",+ })+}++func TestParseText(t *testing.T) {+ RegisterT(t)++ var testCases = []struct {+ format string+ props log.Props+ colorize bool+ expected string+ }{+ {"Hello World", nil, true, "Hello World"},+ {"Hello World", log.Props{}, true, "Hello World"},+ {"Hello @{Name}", log.Props{+ "Name": "John",+ }, true, "Hello John"},+ {"My name is @{Name} and I'm @{Age} years old", log.Props{+ "Name": "John",+ "Age": 55,+ }, true, "My name is John and I'm 55 years old"},+ {"Hello @{Name}", nil, true, "Hello @{Name}"},+ {"Hello @{Name}", log.Props{"Age": 55}, true, "Hello <nil>"},+ {"Hello @{Name:blue}", log.Props{+ "Name": "John",+ }, true, "Hello \033[34mJohn\033[0m"},+ {"Hello @{Name:undefined}", log.Props{+ "Name": "John",+ }, true, "Hello John"},+ {"Hello @{Name:blue}", log.Props{+ "Name": "John",+ }, false, "Hello John"},+ }++ for _, testCase := range testCases {+ text := log.Parse(testCase.format, testCase.props, testCase.colorize)+ Expect(text).Equals(testCase.expected)+ }+}diff --git a/app/pkg/log/logger.go b/app/pkg/log/logger.godeleted file mode 100644index 3334995cd..000000000--- a/app/pkg/log/logger.go+++ /dev/null@@ -1,135 +0,0 @@-package log--import (- "fmt"- stdLog "log"- "os"- "strings"- "time"-- "github.com/getfider/fider/app/pkg/env"-)--// Level defines all possible log levels-type Level uint8--const (- // DEBUG for verbose logs- DEBUG Level = iota + 1- // INFO for WARN+ERROR+INFO logs- INFO- // WARN for WARN+ERROR logs- WARN- // ERROR for ERROR only logs- ERROR-)--// Logger defines the logging interface.-type Logger interface {- SetLevel(level Level)- Debugf(format string, args ...interface{})- Infof(format string, args ...interface{})- Warnf(format string, args ...interface{})- Errorf(format string, args ...interface{})- Error(err error)- IsEnabled(level Level) bool- Write(p []byte) (int, error)-}--// ConsoleLogger output messages to console-type ConsoleLogger struct {- logger *stdLog.Logger- level Level- tag string-}--// NewConsoleLogger creates a new ConsoleLogger-func NewConsoleLogger(tag string) *ConsoleLogger {- logger := &ConsoleLogger{tag: tag, logger: stdLog.New(os.Stdout, "", 0)}- level := strings.ToUpper(env.GetEnvOrDefault("LOG_LEVEL", ""))-- switch level {- case "DEBUG":- logger.SetLevel(DEBUG)- case "WARN":- logger.SetLevel(WARN)- case "ERROR":- logger.SetLevel(ERROR)- default:- logger.SetLevel(INFO)- }- return logger-}--// SetLevel increases/decreases current log level-func (l *ConsoleLogger) SetLevel(level Level) {- l.level = level-}--// IsEnabled returns true if given level is enabled-func (l *ConsoleLogger) IsEnabled(level Level) bool {- return level >= l.level-}--// Debugf logs a DEBUG message-func (l *ConsoleLogger) Debugf(format string, args ...interface{}) {- l.log(DEBUG, format, args...)-}--// Infof logs a INFO message-func (l *ConsoleLogger) Infof(format string, args ...interface{}) {- l.log(INFO, format, args...)-}--// Warnf logs a WARN message-func (l *ConsoleLogger) Warnf(format string, args ...interface{}) {- l.log(WARN, format, args...)-}--// Errorf logs a ERROR message-func (l *ConsoleLogger) Errorf(format string, args ...interface{}) {- l.log(ERROR, format, args...)-}--// Error logs a ERROR message-func (l *ConsoleLogger) Error(err error) {- if err != nil {- l.log(ERROR, err.Error())- } else {- l.log(ERROR, "nil")- }-}--// Write writes len(p) bytes from p to the underlying data stream.-func (l *ConsoleLogger) Write(p []byte) (int, error) {- l.Debugf("%s", p)- return len(p), nil-}--func (l *ConsoleLogger) log(level Level, format string, args ...interface{}) {- if level >= l.level {- message := ""- if format == "" {- message = fmt.Sprint(args...)- } else {- message = fmt.Sprintf(format, args...)- }-- l.logger.Printf("%s [%s] [%s] %s\n", levelString(level), time.Now().Format(time.RFC3339), l.tag, message)- }-}--func levelString(level Level) string {- switch level {- case DEBUG:- return Bold(Magenta("DEBUG"))- case INFO:- return Bold(Blue("INFO"))- case WARN:- return Bold(Yellow("WARN"))- case ERROR:- return Bold(Red("ERROR"))- default:- return Bold(Red("???"))- }-}diff --git a/app/pkg/log/noop.go b/app/pkg/log/noop.godeleted file mode 100644index f734bf316..000000000--- a/app/pkg/log/noop.go+++ /dev/null@@ -1,47 +0,0 @@-package log--// NoopLogger doesn't log anything-type NoopLogger struct {-}--// NewNoopLogger creates a new NoopLogger-func NewNoopLogger() *NoopLogger {- return &NoopLogger{}-}--// SetLevel increases/decreases current log level-func (l *NoopLogger) SetLevel(level Level) {-}--// IsEnabled returns true if given level is enabled-func (l *NoopLogger) IsEnabled(level Level) bool {- return true-}--// Debugf logs a DEBUG message-func (l *NoopLogger) Debugf(format string, args ...interface{}) {-}--// Infof logs a INFO message-func (l *NoopLogger) Infof(format string, args ...interface{}) {-}--// Warnf logs a WARN message-func (l *NoopLogger) Warnf(format string, args ...interface{}) {-}--// Errorf logs a ERROR message-func (l *NoopLogger) Errorf(format string, args ...interface{}) {-}--// Error logs a ERROR message-func (l *NoopLogger) Error(err error) {-}--// Write writes len(p) bytes from p to the underlying data stream.-func (l *NoopLogger) Write(p []byte) (int, error) {- return 0, nil-}--func (l *NoopLogger) log(level Level, format string, args ...interface{}) {-}diff --git a/app/pkg/log/noop/logger.go b/app/pkg/log/noop/logger.gonew file mode 100644index 000000000..bef398919--- /dev/null+++ b/app/pkg/log/noop/logger.go@@ -0,0 +1,70 @@+package noop++import "github.com/getfider/fider/app/pkg/log"++// Logger doesn't log anything+type Logger struct {+}++// NewLogger creates a new Logger+func NewLogger() *Logger {+ return &Logger{}+}++// SetLevel increases/decreases current log level+func (l *Logger) SetLevel(level log.Level) {+}++// IsEnabled returns true if given level is enabled+func (l *Logger) IsEnabled(level log.Level) bool {+ return true+}++// Debug logs a DEBUG message+func (l *Logger) Debug(message string) {+}++// Debugf logs a DEBUG message+func (l *Logger) Debugf(message string, props log.Props) {+}++// Info logs a INFO message+func (l *Logger) Info(message string) {+}++// Infof logs a INFO message+func (l *Logger) Infof(message string, props log.Props) {+}++// Warn logs a WARN message+func (l *Logger) Warn(message string) {+}++// Warnf logs a WARN message+func (l *Logger) Warnf(message string, props log.Props) {+}++// Error logs a ERROR message+func (l *Logger) Error(err error) {+}++// Errorf logs a ERROR message+func (l *Logger) Errorf(message string, props log.Props) {+}++// Write writes len(p) bytes from p to the underlying data stream.+func (l *Logger) Write(p []byte) (int, error) {+ return 0, nil+}++// New returns a copy of current logger with empty context+func (l *Logger) New() log.Logger {+ return NewLogger()+}++// SetProperty with given key:value into current logger context+func (l *Logger) SetProperty(key string, value interface{}) {+}++func (l *Logger) log(level log.Level, format string, args ...interface{}) {+}diff --git a/app/pkg/mock/server.go b/app/pkg/mock/server.goindex c43551a3b..f0a9d77b7 100644--- a/app/pkg/mock/server.go+++ b/app/pkg/mock/server.go@@ -6,7 +6,6 @@ import ("net/http""net/http/httptest""net/url"- "strings""github.com/getfider/fider/app""github.com/getfider/fider/app/middlewares"@@ -107,7 +106,7 @@ func (s *Server) ExecuteAsJSON(handler web.HandlerFunc) (int, *jsonq.Query) {// ExecutePost executes given handler as POST and return responsefunc (s *Server) ExecutePost(handler web.HandlerFunc, body string) (int, *httptest.ResponseRecorder) {s.context.Request.Method = "POST"- s.context.Request.Body = ioutil.NopCloser(strings.NewReader(body))+ s.context.Request.Body = bodys.context.Request.ContentLength = int64(len(body))s.context.Request.SetHeader("Content-Type", web.UTF8JSONContentType)diff --git a/app/pkg/mock/worker.go b/app/pkg/mock/worker.goindex 2e7c7f9b8..3edb602ff 100644--- a/app/pkg/mock/worker.go+++ b/app/pkg/mock/worker.go@@ -3,7 +3,7 @@ package mockimport ("github.com/getfider/fider/app""github.com/getfider/fider/app/models"- "github.com/getfider/fider/app/pkg/log"+ "github.com/getfider/fider/app/pkg/log/noop""github.com/getfider/fider/app/pkg/worker")@@ -34,7 +34,7 @@ func (w *Worker) AsUser(user *models.User) *Worker {// Execute given task with current contextfunc (w *Worker) Execute(task worker.Task) error {- context := worker.NewContext("0", task.Name, log.NewNoopLogger())+ context := worker.NewContext("0", task.Name, nil, noop.NewLogger())context.SetServices(w.services)context.SetUser(w.user)context.SetTenant(w.tenant)diff --git a/app/pkg/web/binder.go b/app/pkg/web/binder.goindex 2abbb02e5..87d81869e 100644--- a/app/pkg/web/binder.go+++ b/app/pkg/web/binder.go@@ -2,7 +2,7 @@ package webimport ("encoding/json"- "errors"+ stdErrors "errors""net/http""reflect""strconv"@@ -11,7 +11,7 @@ import (var (//ErrContentTypeNotAllowed is used when POSTing a body that is not json- ErrContentTypeNotAllowed = errors.New("Only Content-Type application/json is allowed")+ ErrContentTypeNotAllowed = stdErrors.New("Only Content-Type application/json is allowed")intType = reflect.TypeOf(0)stringType = reflect.TypeOf(""))@@ -38,7 +38,7 @@ func (b *DefaultBinder) Bind(target interface{}, c *Context) error {return ErrContentTypeNotAllowed}- if err := json.NewDecoder(c.Request.Body).Decode(target); err != nil {+ if err := json.Unmarshal([]byte(c.Request.Body), target); err != nil {return err}}diff --git a/app/pkg/web/context.go b/app/pkg/web/context.goindex b0cad7507..57611bc2f 100644--- a/app/pkg/web/context.go+++ b/app/pkg/web/context.go@@ -141,9 +141,7 @@ func (ctx *Context) Tenant() *models.Tenant {//SetTenant update HTTP context with current tenantfunc (ctx *Context) SetTenant(tenant *models.Tenant) {if tenant != nil {- ctx.Logger().Debugf("Current tenant: %v (ID: %v)", tenant.Name, tenant.ID)- } else {- ctx.Logger().Debugf("Current tenant: nil")+ ctx.logger.SetProperty(log.PropertyKeyTenantID, tenant.ID)}if ctx.Services() != nil {ctx.Services().SetCurrentTenant(tenant)@@ -212,19 +210,10 @@ func (ctx *Context) Failure(err error) error {return ctx.NotFound()}- tenant := "undefined"- if ctx.Tenant() != nil {- tenant = fmt.Sprintf("%s (%d)", ctx.Tenant().Name, ctx.Tenant().ID)- }-- user := "not signed in"- if ctx.User() != nil {- user = fmt.Sprintf("%s (%d)", ctx.User().Name, ctx.User().ID)- }+ ctx.Logger().Errorf(err.Error(), log.Props{+ "Body": ctx.Request.Body,+ })- url := ctx.Request.URL.String()- message := fmt.Sprintf("URL: %s\nTenant: %s\nUser: %s\n%s", url, tenant, user, err.Error())- ctx.Logger().Errorf(log.Red(message))ctx.Render(http.StatusInternalServerError, "500.html", Props{Title: "Shoot! Well, this is unexpected…",Description: "An error has occurred and we're working to fix the problem!",@@ -294,9 +283,7 @@ func (ctx *Context) User() *models.User {//SetUser update HTTP context with current userfunc (ctx *Context) SetUser(user *models.User) {if user != nil {- ctx.Logger().Debugf("Logged as: %v [%v] (ID: %v)", user.Name, user.Email, user.ID)- } else {- ctx.Logger().Debugf("Logged as: nil")+ ctx.logger.SetProperty(log.PropertyKeyUserID, user.ID)}if ctx.Services() != nil {ctx.Services().SetCurrentUser(user)diff --git a/app/pkg/web/context_test.go b/app/pkg/web/context_test.goindex bc9c2535c..916dc1520 100644--- a/app/pkg/web/context_test.go+++ b/app/pkg/web/context_test.go@@ -50,7 +50,7 @@ func TestContextID(t *testing.T) {ctx := newGetContext("http://demo.test.fider.io:3000", nil)Expect(ctx.ContextID()).IsNotEmpty()- Expect(ctx.ContextID()).HasLen(32)+ Expect(ctx.ContextID()).HasLen(36)}func TestBaseURL(t *testing.T) {diff --git a/app/pkg/web/engine.go b/app/pkg/web/engine.goindex d969f090d..6acbd8de0 100644--- a/app/pkg/web/engine.go+++ b/app/pkg/web/engine.go@@ -8,13 +8,14 @@ import ("os""runtime""strconv"- "strings""time""github.com/getfider/fider/app/models"+ "github.com/getfider/fider/app/pkg/dbx""github.com/getfider/fider/app/pkg/env""github.com/getfider/fider/app/pkg/errors""github.com/getfider/fider/app/pkg/log"+ "github.com/getfider/fider/app/pkg/log/database""github.com/getfider/fider/app/pkg/uuid""github.com/getfider/fider/app/pkg/worker""github.com/julienschmidt/httprouter"@@ -55,6 +56,7 @@ type Engine struct {mux *httprouter.Routerlogger log.Loggerrenderer *Renderer+ db *dbx.Databasebinder *DefaultBindermiddlewares []MiddlewareFuncworker worker.Worker@@ -63,14 +65,21 @@ type Engine struct {//New creates a new Enginefunc New(settings *models.SystemSettings) *Engine {- logger := log.NewConsoleLogger("WEB")+ db := dbx.New()+ logger := database.NewLogger("WEB", db)+ logger.SetProperty(log.PropertyKeyContextID, uuid.NewV4().String())++ bgLogger := database.NewLogger("BGW", db)+ bgLogger.SetProperty(log.PropertyKeyContextID, uuid.NewV4().String())+router := &Engine{mux: httprouter.New(),+ db: db,logger: logger,renderer: NewRenderer(settings, logger),binder: NewDefaultBinder(),middlewares: make([]MiddlewareFunc, 0),- worker: worker.New(),+ worker: worker.New(db, bgLogger),}router.mux.NotFound = ¬FoundHandler{router}@@ -79,6 +88,11 @@ func New(settings *models.SystemSettings) *Engine {//Start the server.func (e *Engine) Start(address string) {+ e.logger.Info("Application is starting")+ e.logger.Infof("GO_ENV: @{Env}", log.Props{+ "Env": env.Current(),+ })+certFile := env.GetEnvOrDefault("SSL_CERT", "")keyFile := env.GetEnvOrDefault("SSL_CERT_KEY", "")autoSSL := env.GetEnvOrDefault("SSL_AUTO", "")@@ -111,14 +125,14 @@ func (e *Engine) Start(address string) {}e.server.TLSConfig.GetCertificate = certManager.GetCertificate- e.logger.Infof("https (auto ssl) server started on %s", address)+ e.logger.Infof("https (auto ssl) server started on @{Address}", log.Props{"Address": address})go certManager.StartHTTPServer()err = e.server.ListenAndServeTLS("", "")} else if certFile == "" && keyFile == "" {- e.logger.Infof("http server started on %s", address)+ e.logger.Infof("http server started on @{Address}", log.Props{"Address": address})err = e.server.ListenAndServe()} else {- e.logger.Infof("https server started on %s", address)+ e.logger.Infof("https server started on @{Address}", log.Props{"Address": address})err = e.server.ListenAndServeTLS(certFilePath, keyFilePath)}@@ -132,29 +146,34 @@ func (e *Engine) Stop() error {ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)defer cancel()- e.logger.Infof("server is shutting down")+ e.logger.Info("server is shutting down")if err := e.server.Shutdown(ctx); err != nil {return errors.Wrap(err, "failed to shutdown server")}- e.logger.Infof("server has shutdown")+ e.logger.Info("server has shutdown")- e.logger.Infof("worker is shutting down")+ e.logger.Info("worker is shutting down")if err := e.worker.Shutdown(ctx); err != nil {return errors.Wrap(err, "failed to shutdown worker")}- e.logger.Infof("worker has shutdown")+ e.logger.Info("worker has shutdown")return nil}//NewContext creates and return a new contextfunc (e *Engine) NewContext(res http.ResponseWriter, req *http.Request, params StringMap) Context {+ contextID := uuid.NewV4().String()+ request := WrapRequest(req)+ ctxLogger := e.logger.New()+ ctxLogger.SetProperty(log.PropertyKeyContextID, contextID)+return Context{- id: strings.Replace(uuid.NewV4().String(), "-", "", 4),+ id: contextID,Response: res,- Request: WrapRequest(req),+ Request: request,engine: e,- logger: e.logger,+ logger: ctxLogger,params: params,worker: e.worker,}@@ -165,6 +184,11 @@ func (e *Engine) Logger() log.Logger {return e.logger}+//Database returns current database+func (e *Engine) Database() *dbx.Database {+ return e.db+}+//Worker returns current worker referencfunc (e *Engine) Worker() worker.Worker {return e.workerdiff --git a/app/pkg/web/engine_test.go b/app/pkg/web/engine_test.goindex 3ccf91ffc..5e85ce0c3 100644--- a/app/pkg/web/engine_test.go+++ b/app/pkg/web/engine_test.go@@ -3,7 +3,6 @@ package web_testimport ("net/http""testing"- "time""github.com/getfider/fider/app/models"@@ -23,16 +22,20 @@ func TestEngine_StartRequestStop(t *testing.T) {}go w.Start(":8080")- time.Sleep(time.Second)+ Expect(func() error {+ _, err := http.Get("http://127.0.0.1:8080/hello")+ return err+ }).EventuallyEquals(nil)+resp, err := http.Get("http://127.0.0.1:8080/hello")Expect(err).IsNil()- resp.Body.Close()Expect(resp.StatusCode).Equals(http.StatusOK)+ resp.Body.Close()resp, err = http.Get("http://127.0.0.1:8080/world")Expect(err).IsNil()- resp.Body.Close()Expect(resp.StatusCode).Equals(http.StatusNotFound)+ resp.Body.Close()err = w.Stop()Expect(err).IsNil()diff --git a/app/pkg/web/request.go b/app/pkg/web/request.goindex f40d5cc49..78917a636 100644--- a/app/pkg/web/request.go+++ b/app/pkg/web/request.go@@ -1,8 +1,7 @@package webimport (- "fmt"- "io"+ "io/ioutil""net/http""net/url"@@ -14,7 +13,7 @@ type Request struct {instance *http.RequestMethod stringContentLength int64- Body io.ReadCloser+ Body stringURL *url.URL}@@ -33,14 +32,22 @@ func WrapRequest(request *http.Request) Request {fullURL := protocol + "://" + host + request.RequestURIu, err := url.Parse(fullURL)if err != nil {- panic(errors.New(fmt.Sprintf("Failed to parse url '%s'", fullURL)))+ panic(errors.Wrap(err, "Failed to parse url '%s'", fullURL))+ }++ var bodyBytes []byte+ if request.ContentLength > 0 {+ bodyBytes, err = ioutil.ReadAll(request.Body)+ if err != nil {+ panic(errors.Wrap(err, "failed to read body").Error())+ }}return Request{instance: request,Method: request.Method,ContentLength: request.ContentLength,- Body: request.Body,+ Body: string(bodyBytes),URL: u,}}diff --git a/app/pkg/worker/context.go b/app/pkg/worker/context.goindex 38ff9e6eb..41542ee1d 100644--- a/app/pkg/worker/context.go+++ b/app/pkg/worker/context.go@@ -1,12 +1,12 @@package workerimport (- "fmt"-"github.com/getfider/fider/app""github.com/getfider/fider/app/models"+ "github.com/getfider/fider/app/pkg/dbx""github.com/getfider/fider/app/pkg/errors""github.com/getfider/fider/app/pkg/log"+ "github.com/getfider/fider/app/pkg/uuid")//Context holds references to services available for jobs@@ -15,17 +15,23 @@ type Context struct {taskName stringservices *app.Serviceslogger log.Logger+ db *dbx.DatabasebaseURL stringuser *models.Usertenant *models.Tenant}//NewContext creates a new context-func NewContext(workerID, taskName string, logger log.Logger) *Context {+func NewContext(workerID, taskName string, db *dbx.Database, logger log.Logger) *Context {+ ctxLogger := logger.New()+ contextID := uuid.NewV4().String()+ ctxLogger.SetProperty(log.PropertyKeyContextID, contextID)+return &Context{workerID: workerID,taskName: taskName,- logger: logger,+ db: db,+ logger: ctxLogger,}}@@ -37,12 +43,18 @@ func (c *Context) SetBaseURL(baseURL string) {//SetUser on contextfunc (c *Context) SetUser(user *models.User) {c.user = user+ if user != nil {+ c.logger.SetProperty(log.PropertyKeyUserID, user.ID)+ }c.services.SetCurrentUser(user)}//SetTenant on contextfunc (c *Context) SetTenant(tenant *models.Tenant) {c.tenant = tenant+ if tenant != nil {+ c.logger.SetProperty(log.PropertyKeyTenantID, tenant.ID)+ }c.services.SetCurrentTenant(tenant)}@@ -86,21 +98,14 @@ func (c *Context) Logger() log.Logger {return c.logger}+//Database from current context+func (c *Context) Database() *dbx.Database {+ return c.db+}+//Failure logs details of errorfunc (c *Context) Failure(err error) error {err = errors.StackN(err, 1)-- tenant := "undefined"- if c.Tenant() != nil {- tenant = fmt.Sprintf("%s (%d)", c.Tenant().Name, c.Tenant().ID)- }-- user := "not signed in"- if c.User() != nil {- user = fmt.Sprintf("%s (%d)", c.User().Name, c.User().ID)- }-- message := fmt.Sprintf("Task: %s\nTenant: %s\nUser: %s\n%s", c.taskName, tenant, user, err.Error())- c.logger.Errorf(log.Red(message))+ c.logger.Error(err)return err}diff --git a/app/pkg/worker/worker.go b/app/pkg/worker/worker.goindex 6fe1b3914..1d7f5609b 100644--- a/app/pkg/worker/worker.go+++ b/app/pkg/worker/worker.go@@ -6,6 +6,7 @@ import ("sync""time"+ "github.com/getfider/fider/app/pkg/dbx""github.com/getfider/fider/app/pkg/log")@@ -33,6 +34,7 @@ type Worker interface {//BackgroundWorker is a worker that runs tasks on backgroundtype BackgroundWorker struct {+ db *dbx.Databaselogger log.Loggerqueue chan Tasklen int64@@ -43,9 +45,10 @@ type BackgroundWorker struct {var maxQueueSize = 100//New creates a new BackgroundWorker-func New() *BackgroundWorker {+func New(db *dbx.Database, logger log.Logger) *BackgroundWorker {return &BackgroundWorker{- logger: log.NewConsoleLogger("BGW"),+ db: db,+ logger: logger,queue: make(chan Task, maxQueueSize),middleware: func(next Job) Job {return next@@ -55,14 +58,12 @@ func New() *BackgroundWorker {//Run initializes the worker loopfunc (w *BackgroundWorker) Run(id string) {- w.logger.Infof("Starting worker %s.", log.Magenta(id))+ w.logger.Infof("Starting worker @{WorkerID:magenta}.", log.Props{+ "WorkerID": id,+ })for task := range w.queue {- c := &Context{- workerID: id,- taskName: task.Name,- logger: w.logger,- }+ c := NewContext(id, task.Name, w.db, w.logger)w.middleware(task.Job)(c)w.Lock()@@ -81,7 +82,11 @@ func (w *BackgroundWorker) Shutdown(ctx context.Context) error {if count == 0 {return nil}- w.logger.Infof("waiting for work queue: %d", count)++ w.logger.Infof("Waiting for work queue: @{Count}", log.Props{+ "Count": count,+ })+select {case <-ctx.Done():return errors.New("timeout waiting for worker queue")diff --git a/app/pkg/worker/worker_test.go b/app/pkg/worker/worker_test.goindex fa2625fbd..7a80e7880 100644--- a/app/pkg/worker/worker_test.go+++ b/app/pkg/worker/worker_test.go@@ -6,6 +6,8 @@ import ("testing""time"+ "github.com/getfider/fider/app/pkg/log/noop"+"github.com/getfider/fider/app/pkg/worker". "github.com/getfider/fider/app/pkg/assert"@@ -24,7 +26,7 @@ func TestBackgroundWorker(t *testing.T) {var finished boolmu := &sync.RWMutex{}- w := worker.New()+ w := worker.New(nil, noop.NewLogger())w.Enqueue(worker.Task{Name: "Do Something",Job: func(c *worker.Context) error {@@ -50,7 +52,7 @@ func TestBackgroundWorker_ShutdownWhenEmpty(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second)defer cancel()- w := worker.New()+ w := worker.New(nil, noop.NewLogger())Expect(w.Shutdown(ctx)).IsNil()}@@ -60,7 +62,7 @@ func TestBackgroundWorker_ShutdownWithStuckTasks(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second)defer cancel()- w := worker.New()+ w := worker.New(nil, noop.NewLogger())w.Enqueue(dummyTask)Expect(w.Shutdown(ctx)).IsNotNil()}@@ -71,7 +73,7 @@ func TestBackgroundWorker_ShutdownWithRunningTasks(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second)defer cancel()- w := worker.New()+ w := worker.New(nil, noop.NewLogger())w.Enqueue(dummyTask)go w.Run("worker-1")Expect(w.Shutdown(ctx)).IsNil()diff --git a/migrations/201806191904_create_logs.sql b/migrations/201806191904_create_logs.sqlnew file mode 100644index 000000000..91e5a36cd--- /dev/null+++ b/migrations/201806191904_create_logs.sql@@ -0,0 +1,9 @@+create table if not exists logs (+ id serial not null,+ tag varchar(50) not null,+ level varchar(50) not null,+ text text not null,+ properties jsonb null,+ created_on timestamptz not null default now(),+ primary key (id)+);\ No newline at end of file