Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix(websocket): aggregate hub overload warnings (#3240)
Kyle Mendell56b3af9f3587
diff --git a/backend/pkg/libarcane/ws/hub.go b/backend/pkg/libarcane/ws/hub.goindex da560b08c..4aa8abe53 100644--- a/backend/pkg/libarcane/ws/hub.go+++ b/backend/pkg/libarcane/ws/hub.go@@ -4,8 +4,12 @@ import ("context""log/slog""sync"+ "sync/atomic"+ "time")+const broadcastDropWarningInterval = 5 * time.Second+type Hub struct {mu sync.RWMutexclients map[*Client]struct{}@@ -15,6 +19,8 @@ type Hub struct {onFirst func()onActive func()onEmpty func()+ dropped atomic.Uint64+ lastWarned atomic.Int64}func NewHub(buffer int) *Hub {@@ -99,7 +105,13 @@ func (h *Hub) Broadcast(msg []byte) {default:// prevent global stall if hub buffer fills// This indicates the hub is not processing messages fast enough- slog.Warn("websocket hub broadcast buffer full; dropping message")+ h.dropped.Add(1)++ now := time.Now().UnixNano()+ lastWarned := h.lastWarned.Load()+ if now-lastWarned >= broadcastDropWarningInterval.Nanoseconds() && h.lastWarned.CompareAndSwap(lastWarned, now) {+ slog.Warn("websocket hub broadcast buffer full; dropping messages", "dropped_count", h.dropped.Swap(0))+ }}}diff --git a/backend/pkg/libarcane/ws/hub_test.go b/backend/pkg/libarcane/ws/hub_test.goindex 7be11cc56..56a9d0110 100644--- a/backend/pkg/libarcane/ws/hub_test.go+++ b/backend/pkg/libarcane/ws/hub_test.go@@ -1,7 +1,9 @@package wsimport (+ "bytes""context"+ "log/slog""net/http""net/http/httptest""strings"@@ -322,6 +324,44 @@ func TestHub_BroadcastBufferFull(t *testing.T) {go h.Run(ctx)}+func TestHub_BroadcastOverloadAggregatesDropWarning(t *testing.T) {+ var logBuffer bytes.Buffer+ previousLogger := slog.Default()+ slog.SetDefault(slog.New(slog.NewTextHandler(&logBuffer, &slog.HandlerOptions{Level: slog.LevelWarn})))+ t.Cleanup(func() { slog.SetDefault(previousLogger) })++ h := NewHub(1)+ h.broadcast <- []byte("fill")+ h.lastWarned.Store(time.Now().UnixNano())++ const dropCount = 1000+ var wg sync.WaitGroup+ for range dropCount {+ wg.Add(1)+ go func() {+ defer wg.Done()+ h.Broadcast([]byte("overload"))+ }()+ }+ wg.Wait()++ assert.Equal(t, uint64(dropCount), h.dropped.Load())+ assert.Empty(t, logBuffer.String())++ h.lastWarned.Store(time.Now().Add(-broadcastDropWarningInterval).UnixNano())+ h.Broadcast([]byte("trigger aggregate warning"))++ logLines := strings.Split(strings.TrimSpace(logBuffer.String()), "\n")+ require.Len(t, logLines, 1)+ assert.Contains(t, logLines[0], "websocket hub broadcast buffer full; dropping messages")+ assert.Contains(t, logLines[0], "dropped_count=1001")+ assert.Zero(t, h.dropped.Load())++ h.Broadcast([]byte("still overloaded"))+ assert.Len(t, strings.Split(strings.TrimSpace(logBuffer.String()), "\n"), 1)+ assert.Equal(t, uint64(1), h.dropped.Load())+}+func TestHub_ConcurrentOperations(t *testing.T) {h := NewHub(100)ctx := t.Context()