Open platform to collect and prioritize feedback
enhancement: graceful shutdown and dump process status upon request (#373)
Guilherme Oenning41278c90795b
diff --git a/app/cmd/server.go b/app/cmd/server.goindex bff83c1d6..89ca3a795 100644--- a/app/cmd/server.go+++ b/app/cmd/server.go@@ -1,10 +1,16 @@package cmdimport (+ "bytes""fmt"+ "os"+ "os/signal"+ "runtime/pprof"+ "syscall""github.com/getfider/fider/app/models""github.com/getfider/fider/app/pkg/env"+ "github.com/getfider/fider/app/pkg/errors""github.com/getfider/fider/app/pkg/web")@@ -12,11 +18,39 @@ import (//Returns an exitcode, 0 for OK and 1 for ERRORfunc RunServer(settings *models.SystemSettings) int {+ e := routes(web.New(settings))fmt.Printf("Application is starting...\n")fmt.Printf("GO_ENV: %s\n", env.Current())- e := routes(web.New(settings))- e.Start(":" + env.GetEnvOrDefault("PORT", "3000"))+ go e.Start(":" + env.GetEnvOrDefault("PORT", "3000"))+ return listenSignals(e, settings)+}- return 0+func listenSignals(e *web.Engine, settings *models.SystemSettings) int {+ signals := make(chan os.Signal, 1)+ signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1)+ for {+ s := <-signals+ switch s {+ case syscall.SIGINT, syscall.SIGTERM:+ err := e.Stop()+ if err != nil {+ e.Logger().Error(errors.Wrap(err, "failed to stop fider"))+ return 1+ }+ return 0+ case syscall.SIGUSR1:+ e.Logger().Infof("SIGUSR1 received")+ e.Logger().Infof("Dumping process status")+ buf := new(bytes.Buffer)+ buf.WriteString(fmt.Sprintf("Version: %s\n", settings.Version))+ buf.WriteString(fmt.Sprintf("BuildTime: %s\n", settings.BuildTime))+ buf.WriteString(fmt.Sprintf("Compiler: %s\n", settings.Compiler))+ buf.WriteString(fmt.Sprintf("Environment: %s\n", settings.Environment))+ buf.WriteString(fmt.Sprintf("Worker Queue: %d\n", e.Worker().Length()))+ pprof.Lookup("goroutine").WriteTo(buf, 1)+ pprof.Lookup("heap").WriteTo(buf, 1)+ e.Logger().Infof("%s", buf.String())+ }+ }}diff --git a/app/middlewares/setup_test.go b/app/middlewares/setup_test.goindex 5dd8db3af..22a55c07e 100644--- a/app/middlewares/setup_test.go+++ b/app/middlewares/setup_test.go@@ -61,7 +61,7 @@ func TestWebSetup_NotQueueTask_OnFailure(t *testing.T) {})Expect(status).Equals(http.StatusInternalServerError)- Expect(server.Engine().Worker().Length()).Equals(0)+ Expect(server.Engine().Worker().Length()).Equals(int64(0))}func TestWebSetup_QueueTask_OnSuccess(t *testing.T) {@@ -75,7 +75,7 @@ func TestWebSetup_QueueTask_OnSuccess(t *testing.T) {})Expect(status).Equals(http.StatusOK)- Expect(server.Engine().Worker().Length()).Equals(1)+ Expect(server.Engine().Worker().Length()).Equals(int64(1))}func TestWorkerSetup(t *testing.T) {diff --git a/app/pkg/assert/assert.go b/app/pkg/assert/assert.goindex 4a52cb05f..cbdae1f8b 100644--- a/app/pkg/assert/assert.go+++ b/app/pkg/assert/assert.go@@ -17,14 +17,14 @@ func mustBeFunction(v interface{}) {}}-//Assertions is the entry point for the assersions-type Assertions struct {- t *testing.T-}+func describe(v interface{}) string {+ value := reflect.ValueOf(v)++ if v == nil {+ return "[nil] nil"+ }-//New creates a new Assertions entry point for further reuse-func New(t *testing.T) *Assertions {- return &Assertions{t}+ return fmt.Sprintf("[%s] %v", value.Type(), v)}//Fail the current test case with given message@@ -37,14 +37,17 @@ func Fail(msg string, args ...interface{}) {var currentT *testing.T+//RegisterT saves current testing.T for further usage by Expectfunc RegisterT(t *testing.T) {currentT = t}+//AnyAssertions is used to assert any kind of valuetype AnyAssertions struct {actual interface{}}+//Expect starts new assertions on given valuefunc Expect(actual interface{}) *AnyAssertions {if currentT == nil {panic("Did you forget to call RegisterT(t)?")@@ -54,29 +57,32 @@ func Expect(actual interface{}) *AnyAssertions {}}+//Equals asserts that actual value equals expected valuefunc (a *AnyAssertions) Equals(expected interface{}) bool {if reflect.DeepEqual(expected, a.actual) {return true}- err := fmt.Errorf("Equals assertion failed. \n Expected: \n\t\t %v \n Actual: \n\t\t %v", expected, a.actual)+ err := fmt.Errorf("Equals assertion failed. \n Expected: \n\t\t %s \n Actual: \n\t\t %s", describe(expected), describe(a.actual))currentT.Error(errors.StackN(err, 1))return false}+//ContainsSubstring asserts that actual value contains given substringfunc (a *AnyAssertions) ContainsSubstring(substr string) bool {if strings.Contains(a.actual.(string), substr) {return true}- err := fmt.Errorf("ContainsSubstring assertion failed. \n Substring: \n\t\t %s \n Actual: \n\t\t %v", substr, a.actual)+ err := fmt.Errorf("ContainsSubstring assertion failed. \n Substring: \n\t\t %s \n Actual: \n\t\t %s", substr, describe(a.actual))currentT.Error(errors.StackN(err, 1))return false}-func (a *AnyAssertions) NotEquals(expected interface{}) bool {- if !reflect.DeepEqual(expected, a.actual) {+//NotEquals asserts that actual value is different than given value+func (a *AnyAssertions) NotEquals(other interface{}) bool {+ if !reflect.DeepEqual(other, a.actual) {return true}- err := fmt.Errorf("NotEquals assertion failed. \n Expected: \n\t\t %v \n Actual: \n\t\t %v", expected, a.actual)+ err := fmt.Errorf("NotEquals assertion failed. \n Other: \n\t\t %s \n Actual: \n\t\t %s", describe(other), describe(a.actual))currentT.Error(errors.StackN(err, 1))return false}@@ -164,7 +170,7 @@ func (a *AnyAssertions) EventuallyEquals(expected interface{}) bool {}select {case <-ctx.Done():- err := fmt.Errorf("func.EventuallyEquals assertion failed. \n Expected: \n\t\t %s \n Actual: \n\t\t %s", expected, a.actual)+ err := fmt.Errorf("func.EventuallyEquals assertion failed. \n Expected: \n\t\t %s \n Actual: \n\t\t %s", describe(expected), describe(a.actual))currentT.Error(errors.StackN(err, 1))return falsecase <-ticker.C:diff --git a/app/pkg/web/engine.go b/app/pkg/web/engine.goindex 2498396a9..38e6eee44 100644--- a/app/pkg/web/engine.go+++ b/app/pkg/web/engine.go@@ -1,6 +1,7 @@package webimport (+ "context""crypto/tls""fmt"stdLog "log"@@ -58,6 +59,7 @@ type Engine struct {binder *DefaultBindermiddlewares []MiddlewareFuncworker worker.Worker+ server *http.Server}//New creates a new Engine@@ -76,13 +78,13 @@ func New(settings *models.SystemSettings) *Engine {return router}-//Start an HTTP server.+//Start the server.func (e *Engine) Start(address string) {certFile := env.GetEnvOrDefault("SSL_CERT", "")keyFile := env.GetEnvOrDefault("SSL_CERT_KEY", "")autoSSL := env.GetEnvOrDefault("SSL_AUTO", "")- server := &http.Server{+ e.server = &http.Server{ReadTimeout: 5 * time.Second,WriteTimeout: 10 * time.Second,IdleTimeout: 120 * time.Second,@@ -105,26 +107,46 @@ func (e *Engine) Start(address string) {panic(errors.Wrap(err, "failed to initialize CertificateManager"))}- server.TLSConfig = &tls.Config{+ e.server.TLSConfig = &tls.Config{GetCertificate: certManager.GetCertificate,}e.logger.Infof("https (auto ssl) server started on %s", address)go certManager.StartHTTPServer()- err = server.ListenAndServeTLS("", "")+ err = e.server.ListenAndServeTLS("", "")} else if certFile == "" && keyFile == "" {e.logger.Infof("http server started on %s", address)- err = server.ListenAndServe()+ err = e.server.ListenAndServe()} else {e.logger.Infof("https server started on %s", address)- err = server.ListenAndServeTLS(certFile, keyFile)+ err = e.server.ListenAndServeTLS(certFile, keyFile)}- if err != nil {+ if err != nil && err != http.ErrServerClosed {panic(errors.Wrap(err, "failed to start server"))}}+//Stop the server.+func (e *Engine) Stop() error {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ e.logger.Infof("server is shutting down")+ if err := e.server.Shutdown(ctx); err != nil {+ return errors.Wrap(err, "failed to shutdown server")+ }+ e.logger.Infof("server has shutdown")++ e.logger.Infof("worker is shutting down")+ if err := e.worker.Shutdown(ctx); err != nil {+ return errors.Wrap(err, "failed to shutdown worker")+ }+ e.logger.Infof("worker has shutdown")++ return nil+}+//NewContext creates and return a new contextfunc (e *Engine) NewContext(res http.ResponseWriter, req *http.Request, params StringMap) Context {return Context{diff --git a/app/pkg/web/engine_test.go b/app/pkg/web/engine_test.gonew file mode 100644index 000000000..3ccf91ffc--- /dev/null+++ b/app/pkg/web/engine_test.go@@ -0,0 +1,43 @@+package web_test++import (+ "net/http"+ "testing"+ "time"++ "github.com/getfider/fider/app/models"++ "github.com/getfider/fider/app/pkg/web"++ . "github.com/getfider/fider/app/pkg/assert"+)++func TestEngine_StartRequestStop(t *testing.T) {+ RegisterT(t)+ w := web.New(&models.SystemSettings{})+ group := w.Group()+ {+ group.Get("/hello", func(c web.Context) error {+ return c.Ok(web.Map{})+ })+ }++ go w.Start(":8080")+ time.Sleep(time.Second)+ resp, err := http.Get("http://127.0.0.1:8080/hello")+ Expect(err).IsNil()+ resp.Body.Close()+ Expect(resp.StatusCode).Equals(http.StatusOK)++ resp, err = http.Get("http://127.0.0.1:8080/world")+ Expect(err).IsNil()+ resp.Body.Close()+ Expect(resp.StatusCode).Equals(http.StatusNotFound)++ err = w.Stop()+ Expect(err).IsNil()++ resp, err = http.Get("http://127.0.0.1:8080/hello")+ Expect(err).IsNotNil()+ Expect(resp).IsNil()+}diff --git a/app/pkg/worker/worker.go b/app/pkg/worker/worker.goindex b89af2782..c2e8cdb1c 100644--- a/app/pkg/worker/worker.go+++ b/app/pkg/worker/worker.go@@ -1,6 +1,11 @@package workerimport (+ "context"+ "errors"+ "sync/atomic"+ "time"+"github.com/getfider/fider/app/pkg/log")@@ -22,13 +27,15 @@ type Worker interface {Enqueue(task Task)Logger() log.LoggerUse(middleware MiddlewareFunc)- Length() int+ Length() int64+ Shutdown(ctx context.Context) error}//BackgroundWorker is a worker that runs tasks on backgroundtype BackgroundWorker struct {logger log.Loggerqueue chan Task+ len int64middleware MiddlewareFunc}@@ -57,11 +64,34 @@ func (w *BackgroundWorker) Run(id string) {}w.middleware(task.Job)(c)+ atomic.AddInt64(&w.len, -1)+ }+}++//Shutdown current worker+func (w *BackgroundWorker) Shutdown(ctx context.Context) error {+ if w.Length() > 0 {+ ticker := time.NewTicker(500 * time.Millisecond)+ defer ticker.Stop()+ for {+ count := w.Length()+ if count == 0 {+ return nil+ }+ w.logger.Infof("waiting for work queue: %d", count)+ select {+ case <-ctx.Done():+ return errors.New("timeout waiting for worker queue")+ case <-ticker.C:+ }+ }}+ return nil}//Enqueue a task on current workerfunc (w *BackgroundWorker) Enqueue(task Task) {+ atomic.AddInt64(&w.len, 1)w.queue <- task}@@ -71,8 +101,8 @@ func (w *BackgroundWorker) Logger() log.Logger {}//Length from current queue length-func (w *BackgroundWorker) Length() int {- return len(w.queue)+func (w *BackgroundWorker) Length() int64 {+ return w.len}//Use this to inject worker dependenciesdiff --git a/app/pkg/worker/worker_test.go b/app/pkg/worker/worker_test.goindex fdea65795..26b284b60 100644--- a/app/pkg/worker/worker_test.go+++ b/app/pkg/worker/worker_test.go@@ -1,13 +1,22 @@package worker_testimport (+ "context""testing"+ "time""github.com/getfider/fider/app/pkg/worker". "github.com/getfider/fider/app/pkg/assert")+var dummyTask = worker.Task{+ Name: "Do Something",+ Job: func(c *worker.Context) error {+ return nil+ },+}+func TestBackgroundWorker(t *testing.T) {RegisterT(t)@@ -21,9 +30,42 @@ func TestBackgroundWorker(t *testing.T) {return nil},})- Expect(w.Length()).Equals(1)+ Expect(w.Length()).Equals(int64(1))go w.Run("worker-1")Expect(func() bool {return finished}).EventuallyEquals(true)}++func TestBackgroundWorker_ShutdownWhenEmpty(t *testing.T) {+ RegisterT(t)++ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()++ w := worker.New()+ Expect(w.Shutdown(ctx)).IsNil()+}++func TestBackgroundWorker_ShutdownWithStuckTasks(t *testing.T) {+ RegisterT(t)++ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()++ w := worker.New()+ w.Enqueue(dummyTask)+ Expect(w.Shutdown(ctx)).IsNotNil()+}++func TestBackgroundWorker_ShutdownWithRunningTasks(t *testing.T) {+ RegisterT(t)++ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()++ w := worker.New()+ w.Enqueue(dummyTask)+ go w.Run("worker-1")+ Expect(w.Shutdown(ctx)).IsNil()+}