Skip to content

Modern Docker Management, Designed for Everyone

fix(websocket): aggregate hub overload warnings (#3240)

Kyle Mendell56b3af9f3587
diff --git a/backend/pkg/libarcane/ws/hub.go b/backend/pkg/libarcane/ws/hub.go
index da560b08c..4aa8abe53 100644
--- a/backend/pkg/libarcane/ws/hub.go
+++ b/backend/pkg/libarcane/ws/hub.go
@@ -4,8 +4,12 @@ import (
"context"
"log/slog"
"sync"
+ "sync/atomic"
+ "time"
)
+const broadcastDropWarningInterval = 5 * time.Second
+
type Hub struct {
mu sync.RWMutex
clients map[*Client]struct{}
@@ -15,6 +19,8 @@ type Hub struct {
onFirst func()
onActive func()
onEmpty func()
+ dropped atomic.Uint64
+ lastWarned atomic.Int64
}
func NewHub(buffer int) *Hub {
@@ -99,7 +105,13 @@ func (h *Hub) Broadcast(msg []byte) {
default:
// prevent global stall if hub buffer fills
// This indicates the hub is not processing messages fast enough
- slog.Warn("websocket hub broadcast buffer full; dropping message")
+ h.dropped.Add(1)
+
+ now := time.Now().UnixNano()
+ lastWarned := h.lastWarned.Load()
+ if now-lastWarned >= broadcastDropWarningInterval.Nanoseconds() && h.lastWarned.CompareAndSwap(lastWarned, now) {
+ slog.Warn("websocket hub broadcast buffer full; dropping messages", "dropped_count", h.dropped.Swap(0))
+ }
}
}
diff --git a/backend/pkg/libarcane/ws/hub_test.go b/backend/pkg/libarcane/ws/hub_test.go
index 7be11cc56..56a9d0110 100644
--- a/backend/pkg/libarcane/ws/hub_test.go
+++ b/backend/pkg/libarcane/ws/hub_test.go
@@ -1,7 +1,9 @@
package ws
import (
+ "bytes"
"context"
+ "log/slog"
"net/http"
"net/http/httptest"
"strings"
@@ -322,6 +324,44 @@ func TestHub_BroadcastBufferFull(t *testing.T) {
go h.Run(ctx)
}
+func TestHub_BroadcastOverloadAggregatesDropWarning(t *testing.T) {
+ var logBuffer bytes.Buffer
+ previousLogger := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logBuffer, &slog.HandlerOptions{Level: slog.LevelWarn})))
+ t.Cleanup(func() { slog.SetDefault(previousLogger) })
+
+ h := NewHub(1)
+ h.broadcast <- []byte("fill")
+ h.lastWarned.Store(time.Now().UnixNano())
+
+ const dropCount = 1000
+ var wg sync.WaitGroup
+ for range dropCount {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ h.Broadcast([]byte("overload"))
+ }()
+ }
+ wg.Wait()
+
+ assert.Equal(t, uint64(dropCount), h.dropped.Load())
+ assert.Empty(t, logBuffer.String())
+
+ h.lastWarned.Store(time.Now().Add(-broadcastDropWarningInterval).UnixNano())
+ h.Broadcast([]byte("trigger aggregate warning"))
+
+ logLines := strings.Split(strings.TrimSpace(logBuffer.String()), "\n")
+ require.Len(t, logLines, 1)
+ assert.Contains(t, logLines[0], "websocket hub broadcast buffer full; dropping messages")
+ assert.Contains(t, logLines[0], "dropped_count=1001")
+ assert.Zero(t, h.dropped.Load())
+
+ h.Broadcast([]byte("still overloaded"))
+ assert.Len(t, strings.Split(strings.TrimSpace(logBuffer.String()), "\n"), 1)
+ assert.Equal(t, uint64(1), h.dropped.Load())
+}
+
func TestHub_ConcurrentOperations(t *testing.T) {
h := NewHub(100)
ctx := t.Context()