Skip to content

The only tool you need to know what is happening and how to fix it.

better systemd notifications

narrow-adventurebbf64d5c046d
diff --git a/backend/cmd/run.go b/backend/cmd/run.go
index be09d6de..f3058733 100644
--- a/backend/cmd/run.go
+++ b/backend/cmd/run.go
@@ -35,7 +35,6 @@ import (
"github.com/tracewayapp/traceway/backend/app/synthetics"
"github.com/tracewayapp/traceway/backend/static"
- "github.com/coreos/go-systemd/v22/daemon"
"github.com/gin-gonic/gin"
"github.com/joho/godotenv"
traceway "go.tracewayapp.com"
@@ -325,7 +324,7 @@ func Run(opts ...Option) {
}(listener)
}
- notifySystemd()
+ notifySystemd(ctx)
config.Logln("Starting server on " + listeners[0].Addr().String())
serveHTTP(router, listeners[0])
}
@@ -505,25 +504,6 @@ func parsePositiveInt(s string, def int) int {
return v
}
-func notifySystemd() {
- sent, err := daemon.SdNotify(false, daemon.SdNotifyReady)
- if err != nil {
- config.Logf("Failed to notify systemd: %v", err)
- } else if sent {
- config.Logln("Notified systemd that service is ready")
- }
-
- go func() {
- defer traceway.Recover()
-
- ticker := time.NewTicker(15 * time.Second)
- defer ticker.Stop()
- for range ticker.C {
- daemon.SdNotify(false, daemon.SdNotifyWatchdog)
- }
- }()
-}
-
func mustSubFS(fsys fs.FS, dir string) fs.FS {
sub, err := fs.Sub(fsys, dir)
if err != nil {
diff --git a/backend/cmd/systemd.go b/backend/cmd/systemd.go
new file mode 100644
index 00000000..8470ba3b
--- /dev/null
+++ b/backend/cmd/systemd.go
@@ -0,0 +1,77 @@
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/coreos/go-systemd/v22/daemon"
+ "github.com/tracewayapp/traceway/backend/app/config"
+ traceway "go.tracewayapp.com"
+)
+
+func notifySystemd(ctx context.Context) {
+ sent, err := daemon.SdNotify(false, daemon.SdNotifyReady)
+ if err != nil {
+ reportSystemdError(fmt.Errorf("notify systemd that service is ready: %w", err))
+ } else if sent {
+ config.Logln("Notified systemd that service is ready")
+ }
+
+ interval, err := systemdWatchdogInterval()
+ if err != nil {
+ reportSystemdError(fmt.Errorf("read systemd watchdog configuration: %w", err))
+ return
+ }
+ if interval == 0 {
+ return
+ }
+ config.Logf("Systemd watchdog heartbeat interval: %s", interval)
+
+ go func() {
+ defer traceway.Recover()
+ runSystemdWatchdog(ctx, interval, daemon.SdNotify)
+ }()
+}
+
+func systemdWatchdogInterval() (time.Duration, error) {
+ timeout, err := daemon.SdWatchdogEnabled(false)
+ if err != nil || timeout == 0 {
+ return 0, err
+ }
+ // Keep headroom for scheduling delays under load, even with a generous watchdog timeout.
+ return min(timeout/2, 5*time.Second), nil
+}
+
+func runSystemdWatchdog(ctx context.Context, interval time.Duration, notify func(bool, string) (bool, error)) {
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+ failed := false
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ sent, err := notify(false, daemon.SdNotifyWatchdog)
+ if err == nil && !sent {
+ err = fmt.Errorf("NOTIFY_SOCKET is unset")
+ }
+ if err != nil {
+ if !failed {
+ reportSystemdError(fmt.Errorf("send systemd watchdog heartbeat: %w", err))
+ }
+ failed = true
+ continue
+ }
+ if failed {
+ config.Logln("Systemd watchdog heartbeat delivery recovered")
+ }
+ failed = false
+ }
+ }
+}
+
+func reportSystemdError(err error) {
+ config.Logf("Systemd notification failed: %v", err)
+ traceway.CaptureException(err)
+}
diff --git a/backend/cmd/systemd_test.go b/backend/cmd/systemd_test.go
new file mode 100644
index 00000000..97e0178e
--- /dev/null
+++ b/backend/cmd/systemd_test.go
@@ -0,0 +1,79 @@
+package cmd
+
+import (
+ "context"
+ "errors"
+ "os"
+ "strconv"
+ "sync/atomic"
+ "testing"
+ "testing/synctest"
+ "time"
+
+ "github.com/coreos/go-systemd/v22/daemon"
+)
+
+func TestSystemdWatchdogInterval(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ usec string
+ pid string
+ want time.Duration
+ wantErr bool
+ }{
+ {name: "disabled"},
+ {name: "production timeout", usec: "20000000", want: 5 * time.Second},
+ {name: "short timeout", usec: "4000000", want: 2 * time.Second},
+ {name: "long timeout", usec: "120000000", want: 5 * time.Second},
+ {name: "matching pid", usec: "20000000", pid: strconv.Itoa(os.Getpid()), want: 5 * time.Second},
+ {name: "other pid", usec: "20000000", pid: strconv.Itoa(os.Getpid() + 1)},
+ {name: "invalid timeout", usec: "invalid", wantErr: true},
+ {name: "zero timeout", usec: "0", wantErr: true},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Setenv("WATCHDOG_USEC", tt.usec)
+ t.Setenv("WATCHDOG_PID", tt.pid)
+ got, err := systemdWatchdogInterval()
+ if got != tt.want || (err != nil) != tt.wantErr {
+ t.Fatalf("systemdWatchdogInterval() = (%s, %v), want (%s, error=%v)", got, err, tt.want, tt.wantErr)
+ }
+ })
+ }
+}
+
+func TestSystemdWatchdogRetriesAndStopsOnCancellation(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ var calls atomic.Int64
+ go runSystemdWatchdog(ctx, 5*time.Second, func(unset bool, state string) (bool, error) {
+ if unset || state != daemon.SdNotifyWatchdog {
+ t.Errorf("notify(%v, %q), want (false, %q)", unset, state, daemon.SdNotifyWatchdog)
+ }
+ switch calls.Add(1) {
+ case 1:
+ return false, errors.New("temporary send failure")
+ case 2:
+ return false, nil
+ default:
+ return true, nil
+ }
+ })
+ synctest.Wait()
+
+ for want := int64(1); want <= 4; want++ {
+ time.Sleep(5 * time.Second)
+ synctest.Wait()
+ if got := calls.Load(); got != want {
+ t.Fatalf("heartbeat calls after %s = %d, want %d", time.Duration(want)*5*time.Second, got, want)
+ }
+ }
+ cancel()
+ synctest.Wait()
+ time.Sleep(20 * time.Second)
+ synctest.Wait()
+ if got := calls.Load(); got != 4 {
+ t.Fatalf("heartbeats continued after cancellation: %d calls", got)
+ }
+ })
+}