Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

enhancement: graceful shutdown and dump process status upon request (#373)

Guilherme Oenning41278c90795b
diff --git a/app/cmd/server.go b/app/cmd/server.go
index bff83c1d6..89ca3a795 100644
--- a/app/cmd/server.go
+++ b/app/cmd/server.go
@@ -1,10 +1,16 @@
package cmd
import (
+ "bytes"
"fmt"
+ "os"
+ "os/signal"
+ "runtime/pprof"
+ "syscall"
"github.com/getfider/fider/app/models"
"github.com/getfider/fider/app/pkg/env"
+ "github.com/getfider/fider/app/pkg/errors"
"github.com/getfider/fider/app/pkg/web"
)
@@ -12,11 +18,39 @@ import (
//Returns an exitcode, 0 for OK and 1 for ERROR
func RunServer(settings *models.SystemSettings) int {
+ e := routes(web.New(settings))
fmt.Printf("Application is starting...\n")
fmt.Printf("GO_ENV: %s\n", env.Current())
- e := routes(web.New(settings))
- e.Start(":" + env.GetEnvOrDefault("PORT", "3000"))
+ go e.Start(":" + env.GetEnvOrDefault("PORT", "3000"))
+ return listenSignals(e, settings)
+}
- return 0
+func listenSignals(e *web.Engine, settings *models.SystemSettings) int {
+ signals := make(chan os.Signal, 1)
+ signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1)
+ for {
+ s := <-signals
+ switch s {
+ case syscall.SIGINT, syscall.SIGTERM:
+ err := e.Stop()
+ if err != nil {
+ e.Logger().Error(errors.Wrap(err, "failed to stop fider"))
+ return 1
+ }
+ return 0
+ case syscall.SIGUSR1:
+ e.Logger().Infof("SIGUSR1 received")
+ e.Logger().Infof("Dumping process status")
+ buf := new(bytes.Buffer)
+ buf.WriteString(fmt.Sprintf("Version: %s\n", settings.Version))
+ buf.WriteString(fmt.Sprintf("BuildTime: %s\n", settings.BuildTime))
+ buf.WriteString(fmt.Sprintf("Compiler: %s\n", settings.Compiler))
+ buf.WriteString(fmt.Sprintf("Environment: %s\n", settings.Environment))
+ buf.WriteString(fmt.Sprintf("Worker Queue: %d\n", e.Worker().Length()))
+ pprof.Lookup("goroutine").WriteTo(buf, 1)
+ pprof.Lookup("heap").WriteTo(buf, 1)
+ e.Logger().Infof("%s", buf.String())
+ }
+ }
}
diff --git a/app/middlewares/setup_test.go b/app/middlewares/setup_test.go
index 5dd8db3af..22a55c07e 100644
--- a/app/middlewares/setup_test.go
+++ b/app/middlewares/setup_test.go
@@ -61,7 +61,7 @@ func TestWebSetup_NotQueueTask_OnFailure(t *testing.T) {
})
Expect(status).Equals(http.StatusInternalServerError)
- Expect(server.Engine().Worker().Length()).Equals(0)
+ Expect(server.Engine().Worker().Length()).Equals(int64(0))
}
func TestWebSetup_QueueTask_OnSuccess(t *testing.T) {
@@ -75,7 +75,7 @@ func TestWebSetup_QueueTask_OnSuccess(t *testing.T) {
})
Expect(status).Equals(http.StatusOK)
- Expect(server.Engine().Worker().Length()).Equals(1)
+ Expect(server.Engine().Worker().Length()).Equals(int64(1))
}
func TestWorkerSetup(t *testing.T) {
diff --git a/app/pkg/assert/assert.go b/app/pkg/assert/assert.go
index 4a52cb05f..cbdae1f8b 100644
--- a/app/pkg/assert/assert.go
+++ b/app/pkg/assert/assert.go
@@ -17,14 +17,14 @@ func mustBeFunction(v interface{}) {
}
}
-//Assertions is the entry point for the assersions
-type Assertions struct {
- t *testing.T
-}
+func describe(v interface{}) string {
+ value := reflect.ValueOf(v)
+
+ if v == nil {
+ return "[nil] nil"
+ }
-//New creates a new Assertions entry point for further reuse
-func New(t *testing.T) *Assertions {
- return &Assertions{t}
+ return fmt.Sprintf("[%s] %v", value.Type(), v)
}
//Fail the current test case with given message
@@ -37,14 +37,17 @@ func Fail(msg string, args ...interface{}) {
var currentT *testing.T
+//RegisterT saves current testing.T for further usage by Expect
func RegisterT(t *testing.T) {
currentT = t
}
+//AnyAssertions is used to assert any kind of value
type AnyAssertions struct {
actual interface{}
}
+//Expect starts new assertions on given value
func Expect(actual interface{}) *AnyAssertions {
if currentT == nil {
panic("Did you forget to call RegisterT(t)?")
@@ -54,29 +57,32 @@ func Expect(actual interface{}) *AnyAssertions {
}
}
+//Equals asserts that actual value equals expected value
func (a *AnyAssertions) Equals(expected interface{}) bool {
if reflect.DeepEqual(expected, a.actual) {
return true
}
- err := fmt.Errorf("Equals assertion failed. \n Expected: \n\t\t %v \n Actual: \n\t\t %v", expected, a.actual)
+ err := fmt.Errorf("Equals assertion failed. \n Expected: \n\t\t %s \n Actual: \n\t\t %s", describe(expected), describe(a.actual))
currentT.Error(errors.StackN(err, 1))
return false
}
+//ContainsSubstring asserts that actual value contains given substring
func (a *AnyAssertions) ContainsSubstring(substr string) bool {
if strings.Contains(a.actual.(string), substr) {
return true
}
- err := fmt.Errorf("ContainsSubstring assertion failed. \n Substring: \n\t\t %s \n Actual: \n\t\t %v", substr, a.actual)
+ err := fmt.Errorf("ContainsSubstring assertion failed. \n Substring: \n\t\t %s \n Actual: \n\t\t %s", substr, describe(a.actual))
currentT.Error(errors.StackN(err, 1))
return false
}
-func (a *AnyAssertions) NotEquals(expected interface{}) bool {
- if !reflect.DeepEqual(expected, a.actual) {
+//NotEquals asserts that actual value is different than given value
+func (a *AnyAssertions) NotEquals(other interface{}) bool {
+ if !reflect.DeepEqual(other, a.actual) {
return true
}
- err := fmt.Errorf("NotEquals assertion failed. \n Expected: \n\t\t %v \n Actual: \n\t\t %v", expected, a.actual)
+ err := fmt.Errorf("NotEquals assertion failed. \n Other: \n\t\t %s \n Actual: \n\t\t %s", describe(other), describe(a.actual))
currentT.Error(errors.StackN(err, 1))
return false
}
@@ -164,7 +170,7 @@ func (a *AnyAssertions) EventuallyEquals(expected interface{}) bool {
}
select {
case <-ctx.Done():
- err := fmt.Errorf("func.EventuallyEquals assertion failed. \n Expected: \n\t\t %s \n Actual: \n\t\t %s", expected, a.actual)
+ err := fmt.Errorf("func.EventuallyEquals assertion failed. \n Expected: \n\t\t %s \n Actual: \n\t\t %s", describe(expected), describe(a.actual))
currentT.Error(errors.StackN(err, 1))
return false
case <-ticker.C:
diff --git a/app/pkg/web/engine.go b/app/pkg/web/engine.go
index 2498396a9..38e6eee44 100644
--- a/app/pkg/web/engine.go
+++ b/app/pkg/web/engine.go
@@ -1,6 +1,7 @@
package web
import (
+ "context"
"crypto/tls"
"fmt"
stdLog "log"
@@ -58,6 +59,7 @@ type Engine struct {
binder *DefaultBinder
middlewares []MiddlewareFunc
worker worker.Worker
+ server *http.Server
}
//New creates a new Engine
@@ -76,13 +78,13 @@ func New(settings *models.SystemSettings) *Engine {
return router
}
-//Start an HTTP server.
+//Start the server.
func (e *Engine) Start(address string) {
certFile := env.GetEnvOrDefault("SSL_CERT", "")
keyFile := env.GetEnvOrDefault("SSL_CERT_KEY", "")
autoSSL := env.GetEnvOrDefault("SSL_AUTO", "")
- server := &http.Server{
+ e.server = &http.Server{
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 120 * time.Second,
@@ -105,26 +107,46 @@ func (e *Engine) Start(address string) {
panic(errors.Wrap(err, "failed to initialize CertificateManager"))
}
- server.TLSConfig = &tls.Config{
+ e.server.TLSConfig = &tls.Config{
GetCertificate: certManager.GetCertificate,
}
e.logger.Infof("https (auto ssl) server started on %s", address)
go certManager.StartHTTPServer()
- err = server.ListenAndServeTLS("", "")
+ err = e.server.ListenAndServeTLS("", "")
} else if certFile == "" && keyFile == "" {
e.logger.Infof("http server started on %s", address)
- err = server.ListenAndServe()
+ err = e.server.ListenAndServe()
} else {
e.logger.Infof("https server started on %s", address)
- err = server.ListenAndServeTLS(certFile, keyFile)
+ err = e.server.ListenAndServeTLS(certFile, keyFile)
}
- if err != nil {
+ if err != nil && err != http.ErrServerClosed {
panic(errors.Wrap(err, "failed to start server"))
}
}
+//Stop the server.
+func (e *Engine) Stop() error {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ e.logger.Infof("server is shutting down")
+ if err := e.server.Shutdown(ctx); err != nil {
+ return errors.Wrap(err, "failed to shutdown server")
+ }
+ e.logger.Infof("server has shutdown")
+
+ e.logger.Infof("worker is shutting down")
+ if err := e.worker.Shutdown(ctx); err != nil {
+ return errors.Wrap(err, "failed to shutdown worker")
+ }
+ e.logger.Infof("worker has shutdown")
+
+ return nil
+}
+
//NewContext creates and return a new context
func (e *Engine) NewContext(res http.ResponseWriter, req *http.Request, params StringMap) Context {
return Context{
diff --git a/app/pkg/web/engine_test.go b/app/pkg/web/engine_test.go
new file mode 100644
index 000000000..3ccf91ffc
--- /dev/null
+++ b/app/pkg/web/engine_test.go
@@ -0,0 +1,43 @@
+package web_test
+
+import (
+ "net/http"
+ "testing"
+ "time"
+
+ "github.com/getfider/fider/app/models"
+
+ "github.com/getfider/fider/app/pkg/web"
+
+ . "github.com/getfider/fider/app/pkg/assert"
+)
+
+func TestEngine_StartRequestStop(t *testing.T) {
+ RegisterT(t)
+ w := web.New(&models.SystemSettings{})
+ group := w.Group()
+ {
+ group.Get("/hello", func(c web.Context) error {
+ return c.Ok(web.Map{})
+ })
+ }
+
+ go w.Start(":8080")
+ time.Sleep(time.Second)
+ resp, err := http.Get("http://127.0.0.1:8080/hello")
+ Expect(err).IsNil()
+ resp.Body.Close()
+ Expect(resp.StatusCode).Equals(http.StatusOK)
+
+ resp, err = http.Get("http://127.0.0.1:8080/world")
+ Expect(err).IsNil()
+ resp.Body.Close()
+ Expect(resp.StatusCode).Equals(http.StatusNotFound)
+
+ err = w.Stop()
+ Expect(err).IsNil()
+
+ resp, err = http.Get("http://127.0.0.1:8080/hello")
+ Expect(err).IsNotNil()
+ Expect(resp).IsNil()
+}
diff --git a/app/pkg/worker/worker.go b/app/pkg/worker/worker.go
index b89af2782..c2e8cdb1c 100644
--- a/app/pkg/worker/worker.go
+++ b/app/pkg/worker/worker.go
@@ -1,6 +1,11 @@
package worker
import (
+ "context"
+ "errors"
+ "sync/atomic"
+ "time"
+
"github.com/getfider/fider/app/pkg/log"
)
@@ -22,13 +27,15 @@ type Worker interface {
Enqueue(task Task)
Logger() log.Logger
Use(middleware MiddlewareFunc)
- Length() int
+ Length() int64
+ Shutdown(ctx context.Context) error
}
//BackgroundWorker is a worker that runs tasks on background
type BackgroundWorker struct {
logger log.Logger
queue chan Task
+ len int64
middleware MiddlewareFunc
}
@@ -57,11 +64,34 @@ func (w *BackgroundWorker) Run(id string) {
}
w.middleware(task.Job)(c)
+ atomic.AddInt64(&w.len, -1)
+ }
+}
+
+//Shutdown current worker
+func (w *BackgroundWorker) Shutdown(ctx context.Context) error {
+ if w.Length() > 0 {
+ ticker := time.NewTicker(500 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ count := w.Length()
+ if count == 0 {
+ return nil
+ }
+ w.logger.Infof("waiting for work queue: %d", count)
+ select {
+ case <-ctx.Done():
+ return errors.New("timeout waiting for worker queue")
+ case <-ticker.C:
+ }
+ }
}
+ return nil
}
//Enqueue a task on current worker
func (w *BackgroundWorker) Enqueue(task Task) {
+ atomic.AddInt64(&w.len, 1)
w.queue <- task
}
@@ -71,8 +101,8 @@ func (w *BackgroundWorker) Logger() log.Logger {
}
//Length from current queue length
-func (w *BackgroundWorker) Length() int {
- return len(w.queue)
+func (w *BackgroundWorker) Length() int64 {
+ return w.len
}
//Use this to inject worker dependencies
diff --git a/app/pkg/worker/worker_test.go b/app/pkg/worker/worker_test.go
index fdea65795..26b284b60 100644
--- a/app/pkg/worker/worker_test.go
+++ b/app/pkg/worker/worker_test.go
@@ -1,13 +1,22 @@
package worker_test
import (
+ "context"
"testing"
+ "time"
"github.com/getfider/fider/app/pkg/worker"
. "github.com/getfider/fider/app/pkg/assert"
)
+var dummyTask = worker.Task{
+ Name: "Do Something",
+ Job: func(c *worker.Context) error {
+ return nil
+ },
+}
+
func TestBackgroundWorker(t *testing.T) {
RegisterT(t)
@@ -21,9 +30,42 @@ func TestBackgroundWorker(t *testing.T) {
return nil
},
})
- Expect(w.Length()).Equals(1)
+ Expect(w.Length()).Equals(int64(1))
go w.Run("worker-1")
Expect(func() bool {
return finished
}).EventuallyEquals(true)
}
+
+func TestBackgroundWorker_ShutdownWhenEmpty(t *testing.T) {
+ RegisterT(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ w := worker.New()
+ Expect(w.Shutdown(ctx)).IsNil()
+}
+
+func TestBackgroundWorker_ShutdownWithStuckTasks(t *testing.T) {
+ RegisterT(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ w := worker.New()
+ w.Enqueue(dummyTask)
+ Expect(w.Shutdown(ctx)).IsNotNil()
+}
+
+func TestBackgroundWorker_ShutdownWithRunningTasks(t *testing.T) {
+ RegisterT(t)
+
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ w := worker.New()
+ w.Enqueue(dummyTask)
+ go w.Run("worker-1")
+ Expect(w.Shutdown(ctx)).IsNil()
+}