The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
better systemd notifications
narrow-adventurebbf64d5c046d
diff --git a/backend/cmd/run.go b/backend/cmd/run.goindex be09d6de..f3058733 100644--- a/backend/cmd/run.go+++ b/backend/cmd/run.go@@ -35,7 +35,6 @@ import ("github.com/tracewayapp/traceway/backend/app/synthetics""github.com/tracewayapp/traceway/backend/static"- "github.com/coreos/go-systemd/v22/daemon""github.com/gin-gonic/gin""github.com/joho/godotenv"traceway "go.tracewayapp.com"@@ -325,7 +324,7 @@ func Run(opts ...Option) {}(listener)}- notifySystemd()+ notifySystemd(ctx)config.Logln("Starting server on " + listeners[0].Addr().String())serveHTTP(router, listeners[0])}@@ -505,25 +504,6 @@ func parsePositiveInt(s string, def int) int {return v}-func notifySystemd() {- sent, err := daemon.SdNotify(false, daemon.SdNotifyReady)- if err != nil {- config.Logf("Failed to notify systemd: %v", err)- } else if sent {- config.Logln("Notified systemd that service is ready")- }-- go func() {- defer traceway.Recover()-- ticker := time.NewTicker(15 * time.Second)- defer ticker.Stop()- for range ticker.C {- daemon.SdNotify(false, daemon.SdNotifyWatchdog)- }- }()-}-func mustSubFS(fsys fs.FS, dir string) fs.FS {sub, err := fs.Sub(fsys, dir)if err != nil {diff --git a/backend/cmd/systemd.go b/backend/cmd/systemd.gonew file mode 100644index 00000000..8470ba3b--- /dev/null+++ b/backend/cmd/systemd.go@@ -0,0 +1,77 @@+package cmd++import (+ "context"+ "fmt"+ "time"++ "github.com/coreos/go-systemd/v22/daemon"+ "github.com/tracewayapp/traceway/backend/app/config"+ traceway "go.tracewayapp.com"+)++func notifySystemd(ctx context.Context) {+ sent, err := daemon.SdNotify(false, daemon.SdNotifyReady)+ if err != nil {+ reportSystemdError(fmt.Errorf("notify systemd that service is ready: %w", err))+ } else if sent {+ config.Logln("Notified systemd that service is ready")+ }++ interval, err := systemdWatchdogInterval()+ if err != nil {+ reportSystemdError(fmt.Errorf("read systemd watchdog configuration: %w", err))+ return+ }+ if interval == 0 {+ return+ }+ config.Logf("Systemd watchdog heartbeat interval: %s", interval)++ go func() {+ defer traceway.Recover()+ runSystemdWatchdog(ctx, interval, daemon.SdNotify)+ }()+}++func systemdWatchdogInterval() (time.Duration, error) {+ timeout, err := daemon.SdWatchdogEnabled(false)+ if err != nil || timeout == 0 {+ return 0, err+ }+ // Keep headroom for scheduling delays under load, even with a generous watchdog timeout.+ return min(timeout/2, 5*time.Second), nil+}++func runSystemdWatchdog(ctx context.Context, interval time.Duration, notify func(bool, string) (bool, error)) {+ ticker := time.NewTicker(interval)+ defer ticker.Stop()+ failed := false+ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ sent, err := notify(false, daemon.SdNotifyWatchdog)+ if err == nil && !sent {+ err = fmt.Errorf("NOTIFY_SOCKET is unset")+ }+ if err != nil {+ if !failed {+ reportSystemdError(fmt.Errorf("send systemd watchdog heartbeat: %w", err))+ }+ failed = true+ continue+ }+ if failed {+ config.Logln("Systemd watchdog heartbeat delivery recovered")+ }+ failed = false+ }+ }+}++func reportSystemdError(err error) {+ config.Logf("Systemd notification failed: %v", err)+ traceway.CaptureException(err)+}diff --git a/backend/cmd/systemd_test.go b/backend/cmd/systemd_test.gonew file mode 100644index 00000000..97e0178e--- /dev/null+++ b/backend/cmd/systemd_test.go@@ -0,0 +1,79 @@+package cmd++import (+ "context"+ "errors"+ "os"+ "strconv"+ "sync/atomic"+ "testing"+ "testing/synctest"+ "time"++ "github.com/coreos/go-systemd/v22/daemon"+)++func TestSystemdWatchdogInterval(t *testing.T) {+ for _, tt := range []struct {+ name string+ usec string+ pid string+ want time.Duration+ wantErr bool+ }{+ {name: "disabled"},+ {name: "production timeout", usec: "20000000", want: 5 * time.Second},+ {name: "short timeout", usec: "4000000", want: 2 * time.Second},+ {name: "long timeout", usec: "120000000", want: 5 * time.Second},+ {name: "matching pid", usec: "20000000", pid: strconv.Itoa(os.Getpid()), want: 5 * time.Second},+ {name: "other pid", usec: "20000000", pid: strconv.Itoa(os.Getpid() + 1)},+ {name: "invalid timeout", usec: "invalid", wantErr: true},+ {name: "zero timeout", usec: "0", wantErr: true},+ } {+ t.Run(tt.name, func(t *testing.T) {+ t.Setenv("WATCHDOG_USEC", tt.usec)+ t.Setenv("WATCHDOG_PID", tt.pid)+ got, err := systemdWatchdogInterval()+ if got != tt.want || (err != nil) != tt.wantErr {+ t.Fatalf("systemdWatchdogInterval() = (%s, %v), want (%s, error=%v)", got, err, tt.want, tt.wantErr)+ }+ })+ }+}++func TestSystemdWatchdogRetriesAndStopsOnCancellation(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ var calls atomic.Int64+ go runSystemdWatchdog(ctx, 5*time.Second, func(unset bool, state string) (bool, error) {+ if unset || state != daemon.SdNotifyWatchdog {+ t.Errorf("notify(%v, %q), want (false, %q)", unset, state, daemon.SdNotifyWatchdog)+ }+ switch calls.Add(1) {+ case 1:+ return false, errors.New("temporary send failure")+ case 2:+ return false, nil+ default:+ return true, nil+ }+ })+ synctest.Wait()++ for want := int64(1); want <= 4; want++ {+ time.Sleep(5 * time.Second)+ synctest.Wait()+ if got := calls.Load(); got != want {+ t.Fatalf("heartbeat calls after %s = %d, want %d", time.Duration(want)*5*time.Second, got, want)+ }+ }+ cancel()+ synctest.Wait()+ time.Sleep(20 * time.Second)+ synctest.Wait()+ if got := calls.Load(); got != 4 {+ t.Fatalf("heartbeats continued after cancellation: %d calls", got)+ }+ })+}