Skip to content

Modern Docker Management, Designed for Everyone

fix: abnormal cpu load climbing over time (#1652)

Kyle Mendell6ce778618cd3
diff --git a/backend/internal/api/diagnostics_handler.go b/backend/internal/api/diagnostics_handler.go
new file mode 100644
index 000000000..f820d3d21
--- /dev/null
+++ b/backend/internal/api/diagnostics_handler.go
@@ -0,0 +1,46 @@
+package api
+
+import (
+ "net/http"
+ "runtime"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/internal/middleware"
+ ws "github.com/getarcaneapp/arcane/backend/internal/utils/ws"
+ "github.com/gin-gonic/gin"
+)
+
+type DiagnosticsHandler struct {
+ wsMetrics *WebSocketMetrics
+}
+
+func RegisterDiagnosticsRoutes(group *gin.RouterGroup, authMiddleware *middleware.AuthMiddleware, wsMetrics *WebSocketMetrics) {
+ h := &DiagnosticsHandler{wsMetrics: wsMetrics}
+
+ diagnostics := group.Group("/diagnostics")
+ diagnostics.Use(authMiddleware.Add())
+ {
+ diagnostics.GET("/ws", h.WebSocketDiagnostics)
+ }
+}
+
+func (h *DiagnosticsHandler) WebSocketDiagnostics(c *gin.Context) {
+ isAdmin, _ := c.Get("userIsAdmin")
+ if admin, ok := isAdmin.(bool); !ok || !admin {
+ c.JSON(http.StatusForbidden, gin.H{"error": "Admin access required"})
+ return
+ }
+
+ metrics := h.wsMetrics.Snapshot()
+ connections := h.wsMetrics.Connections()
+
+ c.JSON(http.StatusOK, gin.H{
+ "timestamp": time.Now().UTC().Format(time.RFC3339Nano),
+ "goroutines": runtime.NumGoroutine(),
+ "wsWorkerGoroutine": ws.CountWorkerGoroutines(),
+ "gomaxprocs": runtime.GOMAXPROCS(0),
+ "goVersion": runtime.Version(),
+ "activeConnections": metrics,
+ "connections": connections,
+ })
+}
diff --git a/backend/internal/api/ws_handler.go b/backend/internal/api/ws_handler.go
index 424d86461..5e5a77230 100644
--- a/backend/internal/api/ws_handler.go
+++ b/backend/internal/api/ws_handler.go
@@ -25,6 +25,7 @@ import (
"github.com/getarcaneapp/arcane/backend/internal/utils/docker"
httputil "github.com/getarcaneapp/arcane/backend/internal/utils/http"
ws "github.com/getarcaneapp/arcane/backend/internal/utils/ws"
+ systemtypes "github.com/getarcaneapp/arcane/types/system"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/shirou/gopsutil/v4/cpu"
@@ -37,31 +38,113 @@ const (
gpuCacheDuration = 30 * time.Second
)
-// GPUStats represents statistics for a single GPU
-type GPUStats struct {
- Name string `json:"name"`
- Index int `json:"index"`
- MemoryUsed float64 `json:"memoryUsed"`
- MemoryTotal float64 `json:"memoryTotal"`
+// amdGPUSysfsPath is the base path for AMD GPU sysfs entries
+const amdGPUSysfsPath = "/sys/class/drm"
+
+// ============================================================================
+// WebSocket Metrics
+// ============================================================================
+
+// WebSocketMetrics tracks active WebSocket connections and their counts.
+type WebSocketMetrics struct {
+ projectLogsActive atomic.Int64
+ containerLogsActive atomic.Int64
+ containerStats atomic.Int64
+ containerExec atomic.Int64
+ systemStats atomic.Int64
+ seq atomic.Uint64
+ mu sync.RWMutex
+ connections map[string]systemtypes.WebSocketConnectionInfo
}
-// SystemStats represents system resource statistics for WebSocket streaming.
-type SystemStats struct {
- CPUUsage float64 `json:"cpuUsage"`
- MemoryUsage uint64 `json:"memoryUsage"`
- MemoryTotal uint64 `json:"memoryTotal"`
- DiskUsage uint64 `json:"diskUsage,omitempty"`
- DiskTotal uint64 `json:"diskTotal,omitempty"`
- CPUCount int `json:"cpuCount"`
- Architecture string `json:"architecture"`
- Platform string `json:"platform"`
- Hostname string `json:"hostname,omitempty"`
- GPUCount int `json:"gpuCount"`
- GPUs []GPUStats `json:"gpus,omitempty"`
+// NewWebSocketMetrics creates a new WebSocketMetrics instance.
+func NewWebSocketMetrics() *WebSocketMetrics {
+ return &WebSocketMetrics{
+ connections: make(map[string]systemtypes.WebSocketConnectionInfo),
+ }
}
-// amdGPUSysfsPath is the base path for AMD GPU sysfs entries
-const amdGPUSysfsPath = "/sys/class/drm"
+// Snapshot returns a point-in-time copy of the active connection counts.
+func (m *WebSocketMetrics) Snapshot() systemtypes.WebSocketMetricsSnapshot {
+ return systemtypes.WebSocketMetricsSnapshot{
+ ProjectLogsActive: m.projectLogsActive.Load(),
+ ContainerLogsActive: m.containerLogsActive.Load(),
+ ContainerStats: m.containerStats.Load(),
+ ContainerExec: m.containerExec.Load(),
+ SystemStats: m.systemStats.Load(),
+ }
+}
+
+// Connections returns a snapshot of all tracked WebSocket connections.
+func (m *WebSocketMetrics) Connections() []systemtypes.WebSocketConnectionInfo {
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ result := make([]systemtypes.WebSocketConnectionInfo, 0, len(m.connections))
+ for _, info := range m.connections {
+ result = append(result, info)
+ }
+ return result
+}
+
+// RegisterConnection adds a connection to the tracker and increments the
+// appropriate kind counter. Returns the assigned connection ID.
+func (m *WebSocketMetrics) RegisterConnection(info systemtypes.WebSocketConnectionInfo) string {
+ if info.ID == "" {
+ info.ID = "ws-" + strconv.FormatUint(m.seq.Add(1), 10)
+ }
+ if info.StartedAt.IsZero() {
+ info.StartedAt = time.Now().UTC()
+ }
+ m.mu.Lock()
+ m.connections[info.ID] = info
+ m.mu.Unlock()
+ m.applyDelta(info.Kind, 1)
+ return info.ID
+}
+
+// UnregisterConnection removes a connection from the tracker and decrements
+// the appropriate kind counter.
+func (m *WebSocketMetrics) UnregisterConnection(id string) {
+ if id == "" {
+ return
+ }
+ var info systemtypes.WebSocketConnectionInfo
+ m.mu.Lock()
+ if existing, ok := m.connections[id]; ok {
+ info = existing
+ delete(m.connections, id)
+ }
+ m.mu.Unlock()
+ if info.Kind != "" {
+ m.applyDelta(info.Kind, -1)
+ }
+}
+
+func (m *WebSocketMetrics) applyDelta(kind string, delta int64) {
+ switch kind {
+ case systemtypes.WSKindProjectLogs:
+ m.projectLogsActive.Add(delta)
+ case systemtypes.WSKindContainerLogs:
+ m.containerLogsActive.Add(delta)
+ case systemtypes.WSKindContainerStats:
+ m.containerStats.Add(delta)
+ case systemtypes.WSKindContainerExec:
+ m.containerExec.Add(delta)
+ case systemtypes.WSKindSystemStats:
+ m.systemStats.Add(delta)
+ }
+}
+
+var defaultWebSocketMetrics = NewWebSocketMetrics()
+
+// DefaultWebSocketMetrics returns the package-level WebSocketMetrics singleton.
+func DefaultWebSocketMetrics() *WebSocketMetrics {
+ return defaultWebSocketMetrics
+}
+
+// ============================================================================
+// WebSocket Handler
+// ============================================================================
// WebSocketHandler consolidates all WebSocket and streaming endpoints.
// REST endpoints are handled by Huma handlers.
@@ -70,6 +153,7 @@ type WebSocketHandler struct {
containerService *services.ContainerService
systemService *services.SystemService
wsUpgrader websocket.Upgrader
+ wsMetrics *WebSocketMetrics
activeConnections sync.Map
cpuCache struct {
sync.RWMutex
@@ -101,6 +185,26 @@ type wsLogStream struct {
seq atomic.Uint64
}
+func getContextUserIDInternal(c *gin.Context) string {
+ if val, ok := c.Get("userID"); ok {
+ if userID, ok := val.(string); ok {
+ return userID
+ }
+ }
+ return ""
+}
+
+func buildWSConnectionInfoInternal(c *gin.Context, kind, resourceID string) systemtypes.WebSocketConnectionInfo {
+ return systemtypes.WebSocketConnectionInfo{
+ Kind: kind,
+ EnvID: c.Param("id"),
+ ResourceID: resourceID,
+ ClientIP: c.ClientIP(),
+ UserID: getContextUserIDInternal(c),
+ UserAgent: c.GetHeader("User-Agent"),
+ }
+}
+
func NewWebSocketHandler(
group *gin.RouterGroup,
projectService *services.ProjectService,
@@ -113,6 +217,7 @@ func NewWebSocketHandler(
projectService: projectService,
containerService: containerService,
systemService: systemService,
+ wsMetrics: defaultWebSocketMetrics,
gpuMonitoringEnabled: cfg.GPUMonitoringEnabled,
gpuType: cfg.GPUType,
wsUpgrader: websocket.Upgrader{
@@ -177,14 +282,17 @@ func (h *WebSocketHandler) ProjectLogs(c *gin.Context) {
return
}
- hub := h.startProjectLogHub(projectID, format, batched, follow, tail, since, timestamps)
+ connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindProjectLogs, projectID))
+ hub := h.startProjectLogHub(projectID, format, batched, follow, tail, since, timestamps, func() {
+ h.wsMetrics.UnregisterConnection(connID)
+ })
// WebSocket connections use context.Background() because they are long-lived and should not
// be tied to the HTTP request context. Cleanup is handled via the hub's OnEmpty callback
// which triggers when all clients disconnect.
ws.ServeClient(context.Background(), hub, conn)
}
-func (h *WebSocketHandler) startProjectLogHub(projectID, format string, batched, follow bool, tail, since string, timestamps bool) *ws.Hub {
+func (h *WebSocketHandler) startProjectLogHub(projectID, format string, batched, follow bool, tail, since string, timestamps bool, onEmptyHook func()) *ws.Hub {
ls := &wsLogStream{
hub: ws.NewHub(1024),
format: format,
@@ -194,6 +302,9 @@ func (h *WebSocketHandler) startProjectLogHub(projectID, format string, batched,
ls.cancel = cancel
ls.hub.SetOnEmpty(func() {
+ if onEmptyHook != nil {
+ onEmptyHook()
+ }
slog.Debug("client disconnected, cleaning up project log hub", "projectID", projectID)
cancel()
})
@@ -289,14 +400,17 @@ func (h *WebSocketHandler) ContainerLogs(c *gin.Context) {
return
}
- hub := h.startContainerLogHub(containerID, format, batched, follow, tail, since, timestamps)
+ connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindContainerLogs, containerID))
+ hub := h.startContainerLogHub(containerID, format, batched, follow, tail, since, timestamps, func() {
+ h.wsMetrics.UnregisterConnection(connID)
+ })
// WebSocket connections use context.Background() because they are long-lived and should not
// be tied to the HTTP request context. Cleanup is handled via the hub's OnEmpty callback
// which triggers when all clients disconnect.
ws.ServeClient(context.Background(), hub, conn)
}
-func (h *WebSocketHandler) startContainerLogHub(containerID, format string, batched, follow bool, tail, since string, timestamps bool) *ws.Hub {
+func (h *WebSocketHandler) startContainerLogHub(containerID, format string, batched, follow bool, tail, since string, timestamps bool, onEmptyHook func()) *ws.Hub {
ls := &wsLogStream{
hub: ws.NewHub(1024),
format: format,
@@ -306,6 +420,9 @@ func (h *WebSocketHandler) startContainerLogHub(containerID, format string, batc
ls.cancel = cancel
ls.hub.SetOnEmpty(func() {
+ if onEmptyHook != nil {
+ onEmptyHook()
+ }
slog.Debug("client disconnected, cleaning up container log hub", "containerID", containerID)
cancel()
})
@@ -369,19 +486,25 @@ func (h *WebSocketHandler) ContainerStats(c *gin.Context) {
return
}
- hub := h.startContainerStatsHub(containerID)
+ connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindContainerStats, containerID))
+ hub := h.startContainerStatsHub(containerID, func() {
+ h.wsMetrics.UnregisterConnection(connID)
+ })
// WebSocket connections use context.Background() because they are long-lived and should not
// be tied to the HTTP request context. Cleanup is handled via the hub's OnEmpty callback
// which triggers when all clients disconnect.
ws.ServeClient(context.Background(), hub, conn)
}
-func (h *WebSocketHandler) startContainerStatsHub(containerID string) *ws.Hub {
+func (h *WebSocketHandler) startContainerStatsHub(containerID string, onEmptyHook func()) *ws.Hub {
hub := ws.NewHub(64)
ctx, cancel := context.WithCancel(context.Background())
hub.SetOnEmpty(func() {
+ if onEmptyHook != nil {
+ onEmptyHook()
+ }
slog.Debug("client disconnected, cleaning up container stats hub", "containerID", containerID)
cancel()
})
@@ -435,6 +558,8 @@ func (h *WebSocketHandler) ContainerExec(c *gin.Context) {
if err != nil {
return
}
+ connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindContainerExec, containerID))
+ defer h.wsMetrics.UnregisterConnection(connID)
defer conn.Close()
ctx, cancel := context.WithCancel(c.Request.Context())
@@ -583,7 +708,7 @@ func (h *WebSocketHandler) startCPUSampler(ctx context.Context, ticker *time.Tic
}
// collectSystemStats gathers all system statistics.
-func (h *WebSocketHandler) collectSystemStats(ctx context.Context) SystemStats {
+func (h *WebSocketHandler) collectSystemStats(ctx context.Context) systemtypes.SystemStats {
h.cpuCache.RLock()
cpuUsage := h.cpuCache.value
h.cpuCache.RUnlock()
@@ -595,7 +720,7 @@ func (h *WebSocketHandler) collectSystemStats(ctx context.Context) SystemStats {
hostname := h.getHostname()
gpuStats, gpuCount := h.getGPUInfo(ctx)
- return SystemStats{
+ return systemtypes.SystemStats{
CPUUsage: cpuUsage,
MemoryUsage: memUsed,
MemoryTotal: memTotal,
@@ -675,7 +800,7 @@ func (h *WebSocketHandler) getHostname() string {
}
// getGPUInfo returns GPU statistics if monitoring is enabled.
-func (h *WebSocketHandler) getGPUInfo(ctx context.Context) ([]GPUStats, int) {
+func (h *WebSocketHandler) getGPUInfo(ctx context.Context) ([]systemtypes.GPUStats, int) {
if !h.gpuMonitoringEnabled {
return nil, 0
}
@@ -720,6 +845,8 @@ func (h *WebSocketHandler) SystemStats(c *gin.Context) {
if err != nil {
return
}
+ connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindSystemStats, ""))
+ defer h.wsMetrics.UnregisterConnection(connID)
defer conn.Close()
interval, _ := httputil.GetIntQueryParam(c, "interval", false)
@@ -727,8 +854,23 @@ func (h *WebSocketHandler) SystemStats(c *gin.Context) {
interval = 2
}
+ const (
+ statsPongWait = 60 * time.Second
+ statsPingWriteWait = 1 * time.Second
+ )
+ statsPingPeriod := statsPongWait * 9 / 10
+
+ conn.SetReadLimit(512)
+ _ = conn.SetReadDeadline(time.Now().Add(statsPongWait))
+ conn.SetPongHandler(func(string) error {
+ _ = conn.SetReadDeadline(time.Now().Add(statsPongWait))
+ return nil
+ })
+
ticker := time.NewTicker(time.Duration(interval) * time.Second)
defer ticker.Stop()
+ pingTicker := time.NewTicker(statsPingPeriod)
+ defer pingTicker.Stop()
cpuUpdateTicker := time.NewTicker(1 * time.Second)
defer cpuUpdateTicker.Stop()
@@ -736,6 +878,8 @@ func (h *WebSocketHandler) SystemStats(c *gin.Context) {
ctx, cancel := context.WithCancel(c.Request.Context())
defer cancel()
+ go h.readSystemStatsPumpInternal(ctx, cancel, conn)
+
h.startCPUSampler(ctx, cpuUpdateTicker)
send := func() error {
@@ -752,12 +896,33 @@ func (h *WebSocketHandler) SystemStats(c *gin.Context) {
for {
select {
- case <-c.Request.Context().Done():
+ case <-ctx.Done():
return
case <-ticker.C:
if err := send(); err != nil {
return
}
+ case <-pingTicker.C:
+ _ = conn.SetWriteDeadline(time.Now().Add(statsPingWriteWait))
+ if err := conn.WriteMessage(websocket.PingMessage, nil); err != nil {
+ return
+ }
+ }
+ }
+}
+
+// readSystemStatsPumpInternal is the single reader for the SystemStats websocket.
+// Do not add additional readers for this connection.
+func (h *WebSocketHandler) readSystemStatsPumpInternal(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn) {
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ if _, _, err := conn.ReadMessage(); err != nil {
+ cancel()
+ return
+ }
}
}
}
@@ -792,7 +957,7 @@ func (h *WebSocketHandler) getDiskUsagePath(ctx context.Context) string {
// ============================================================================
// getGPUStats collects and returns GPU statistics for all available GPUs
-func (h *WebSocketHandler) getGPUStats(ctx context.Context) ([]GPUStats, error) {
+func (h *WebSocketHandler) getGPUStats(ctx context.Context) ([]systemtypes.GPUStats, error) {
h.detectionMutex.Lock()
done := h.detectionDone
h.detectionMutex.Unlock()
@@ -934,7 +1099,7 @@ func (h *WebSocketHandler) detectGPUs(ctx context.Context) error {
}
// getNvidiaStats collects NVIDIA GPU statistics using nvidia-smi
-func (h *WebSocketHandler) getNvidiaStats(ctx context.Context) ([]GPUStats, error) {
+func (h *WebSocketHandler) getNvidiaStats(ctx context.Context) ([]systemtypes.GPUStats, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
@@ -957,7 +1122,7 @@ func (h *WebSocketHandler) getNvidiaStats(ctx context.Context) ([]GPUStats, erro
return nil, fmt.Errorf("failed to parse nvidia-smi output: %w", err)
}
- var stats []GPUStats
+ var stats []systemtypes.GPUStats
for _, record := range records {
if len(record) < 4 {
continue
@@ -983,7 +1148,7 @@ func (h *WebSocketHandler) getNvidiaStats(ctx context.Context) ([]GPUStats, erro
continue
}
- stats = append(stats, GPUStats{
+ stats = append(stats, systemtypes.GPUStats{
Name: name,
Index: index,
MemoryUsed: memUsed * 1024 * 1024,
@@ -1000,8 +1165,8 @@ func (h *WebSocketHandler) getNvidiaStats(ctx context.Context) ([]GPUStats, erro
}
// getAMDStats collects AMD GPU statistics using sysfs
-func (h *WebSocketHandler) getAMDStats(ctx context.Context) ([]GPUStats, error) {
- var stats []GPUStats
+func (h *WebSocketHandler) getAMDStats(ctx context.Context) ([]systemtypes.GPUStats, error) {
+ var stats []systemtypes.GPUStats
// Find AMD GPU cards by looking for mem_info_vram_total in /sys/class/drm/card*/device/
entries, err := os.ReadDir(amdGPUSysfsPath)
@@ -1040,7 +1205,7 @@ func (h *WebSocketHandler) getAMDStats(ctx context.Context) ([]GPUStats, error)
continue
}
- stats = append(stats, GPUStats{
+ stats = append(stats, systemtypes.GPUStats{
Name: fmt.Sprintf("AMD GPU %d", index),
Index: index,
MemoryUsed: float64(memUsedBytes),
@@ -1095,8 +1260,8 @@ func hasAMDGPUInternal() bool {
}
// getIntelStats collects Intel GPU statistics using intel_gpu_top
-func (h *WebSocketHandler) getIntelStats(ctx context.Context) ([]GPUStats, error) {
- stats := []GPUStats{
+func (h *WebSocketHandler) getIntelStats(ctx context.Context) ([]systemtypes.GPUStats, error) {
+ stats := []systemtypes.GPUStats{
{
Name: "Intel GPU",
Index: 0,
diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.go
index 809db81fb..f9f6bc4b5 100644
--- a/backend/internal/bootstrap/router_bootstrap.go
+++ b/backend/internal/bootstrap/router_bootstrap.go
@@ -151,6 +151,8 @@ func setupRouter(ctx context.Context, cfg *config.Config, appServices *Services)
Config: cfg,
})
+ api.RegisterDiagnosticsRoutes(apiGroup, authMiddleware, api.DefaultWebSocketMetrics()) //nolint:contextcheck
+
// Remaining Gin handlers (WebSocket/streaming)
api.NewWebSocketHandler(apiGroup, appServices.Project, appServices.Container, appServices.System, authMiddleware, cfg) //nolint:contextcheck
diff --git a/backend/internal/services/container_service.go b/backend/internal/services/container_service.go
index fba591ea2..5b9d56ec1 100644
--- a/backend/internal/services/container_service.go
+++ b/backend/internal/services/container_service.go
@@ -273,23 +273,25 @@ func (s *ContainerService) StreamStats(ctx context.Context, containerID string,
decoder := json.NewDecoder(stats.Body)
for {
- select {
- case <-ctx.Done():
- return ctx.Err()
- default:
- var statsData interface{}
- if err := decoder.Decode(&statsData); err != nil {
- if err == io.EOF {
- return nil
- }
- return fmt.Errorf("failed to decode stats: %w", err)
- }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
- select {
- case statsChan <- statsData:
- case <-ctx.Done():
+ var statsData interface{}
+ if err := decoder.Decode(&statsData); err != nil {
+ if ctx.Err() != nil {
return ctx.Err()
}
+ if err == io.EOF {
+ return nil
+ }
+ return fmt.Errorf("failed to decode stats: %w", err)
+ }
+
+ select {
+ case statsChan <- statsData:
+ case <-ctx.Done():
+ return ctx.Err()
}
}
}
diff --git a/backend/internal/utils/fs/watcher.go b/backend/internal/utils/fs/watcher.go
index 49d6f1fa1..22e5e743f 100644
--- a/backend/internal/utils/fs/watcher.go
+++ b/backend/internal/utils/fs/watcher.go
@@ -5,6 +5,7 @@ import (
"log/slog"
"os"
"path/filepath"
+ "runtime"
"strings"
"time"
@@ -78,10 +79,12 @@ func (fw *Watcher) Stop() error {
func (fw *Watcher) watchLoop(ctx context.Context) {
defer close(fw.stoppedCh)
- debounceTimer := time.NewTimer(0)
+ debounceTimer := time.NewTimer(fw.debounce)
if !debounceTimer.Stop() {
<-debounceTimer.C
}
+ debouncePending := false
+ lastGoroutineLog := time.Time{}
for {
select {
@@ -90,11 +93,12 @@ func (fw *Watcher) watchLoop(ctx context.Context) {
case <-fw.stopCh:
return
case event, ok := <-fw.watcher.Events:
- if !ok {
+ if fw.processEventInternal(ctx, event, ok, debounceTimer, &debouncePending) {
return
}
- if fw.shouldHandleEvent(event) {
- fw.handleEvent(ctx, event, debounceTimer)
+ case <-debounceTimer.C:
+ if fw.fireDebounceInternal(ctx, &debouncePending, &lastGoroutineLog) {
+ continue
}
case err, ok := <-fw.watcher.Errors:
if !ok {
@@ -105,7 +109,45 @@ func (fw *Watcher) watchLoop(ctx context.Context) {
}
}
-func (fw *Watcher) handleEvent(ctx context.Context, event fsnotify.Event, debounceTimer *time.Timer) {
+// processEventInternal handles one fs event; returns true if the watch loop should exit.
+func (fw *Watcher) processEventInternal(ctx context.Context, event fsnotify.Event, ok bool, debounceTimer *time.Timer, debouncePending *bool) bool {
+ if !ok {
+ return true
+ }
+ if !fw.shouldHandleEvent(event) {
+ return false
+ }
+ fw.handleEvent(ctx, event)
+ if !debounceTimer.Stop() {
+ select {
+ case <-debounceTimer.C:
+ default:
+ }
+ }
+ debounceTimer.Reset(fw.debounce)
+ *debouncePending = true
+ return false
+}
+
+// fireDebounceInternal runs the debounced onChange callback; returns true if nothing was pending (caller should continue).
+func (fw *Watcher) fireDebounceInternal(ctx context.Context, debouncePending *bool, lastGoroutineLog *time.Time) bool {
+ if !*debouncePending {
+ return true
+ }
+ *debouncePending = false
+ if time.Since(*lastGoroutineLog) > 30*time.Second {
+ slog.DebugContext(ctx, "Filesystem watcher debounce triggered",
+ "path", fw.watchedPath,
+ "goroutines", runtime.NumGoroutine())
+ *lastGoroutineLog = time.Now()
+ }
+ if fw.onChange != nil {
+ go fw.onChange(ctx)
+ }
+ return false
+}
+
+func (fw *Watcher) handleEvent(ctx context.Context, event fsnotify.Event) {
if event.Has(fsnotify.Create) {
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
if fw.shouldWatchDir(event.Name) {
@@ -121,40 +163,6 @@ func (fw *Watcher) handleEvent(ctx context.Context, event fsnotify.Event, deboun
slog.DebugContext(ctx, "Filesystem change detected",
"path", event.Name,
"operation", event.Op.String())
-
- // Reset debounce timer
- if !debounceTimer.Stop() {
- select {
- case <-debounceTimer.C:
- default:
- }
- }
- debounceTimer.Reset(fw.debounce)
-
- // Start debounce handler if not already running
- select {
- case <-debounceTimer.C:
- // Timer expired, trigger sync
- if fw.onChange != nil {
- go fw.onChange(ctx)
- }
- default:
- // Timer still running, will trigger later
- go fw.handleDebounce(ctx, debounceTimer)
- }
-}
-
-func (fw *Watcher) handleDebounce(ctx context.Context, timer *time.Timer) {
- select {
- case <-ctx.Done():
- return
- case <-fw.stopCh:
- return
- case <-timer.C:
- if fw.onChange != nil {
- fw.onChange(ctx)
- }
- }
}
func (fw *Watcher) shouldHandleEvent(event fsnotify.Event) bool {
diff --git a/backend/internal/utils/ws/benchmark_test.go b/backend/internal/utils/ws/benchmark_test.go
new file mode 100644
index 000000000..757fdc190
--- /dev/null
+++ b/backend/internal/utils/ws/benchmark_test.go
@@ -0,0 +1,333 @@
+package ws
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/gorilla/websocket"
+)
+
+// benchWSServer creates a test WebSocket server for benchmarks.
+// The server keeps connections open until the returned cleanup is called.
+func benchWSServer(b *testing.B) (url string, cleanup func()) {
+ b.Helper()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ go func() {
+ // Drain reads until context cancelled
+ for {
+ if _, _, err := conn.ReadMessage(); err != nil {
+ return
+ }
+ }
+ }()
+ <-ctx.Done()
+ conn.Close()
+ }))
+
+ wsURL := "ws" + strings.TrimPrefix(server.URL, "http")
+ return wsURL, func() {
+ cancel()
+ server.Close()
+ }
+}
+
+// benchHub creates a hub with N connected clients whose send channels are drained.
+// Returns the hub, a context cancel func, and a cleanup func.
+func benchHub(b *testing.B, numClients int, hubBuf int) (*Hub, context.CancelFunc, func()) {
+ b.Helper()
+
+ h := NewHub(hubBuf)
+ ctx, cancel := context.WithCancel(context.Background())
+ go h.Run(ctx)
+
+ wsURL, serverCleanup := benchWSServer(b)
+
+ var conns []*websocket.Conn
+ for range numClients {
+ conn, resp, err := websocket.DefaultDialer.Dial(wsURL, nil)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if resp != nil {
+ resp.Body.Close()
+ }
+ conns = append(conns, conn)
+
+ c := NewClient(conn, 4096)
+ h.register <- c
+
+ // Drain send channel to simulate fast client
+ go func() {
+ for range c.send {
+ }
+ }()
+ }
+
+ // Wait for all clients to register
+ for h.ClientCount() < numClients {
+ time.Sleep(time.Millisecond)
+ }
+
+ return h, cancel, func() {
+ cancel()
+ for _, c := range conns {
+ c.Close()
+ }
+ serverCleanup()
+ }
+}
+
+// ============================================================================
+// Hub Benchmarks
+// ============================================================================
+
+// BenchmarkHub_Broadcast measures hub broadcast throughput with varying client counts.
+func BenchmarkHub_Broadcast(b *testing.B) {
+ for _, numClients := range []int{1, 10, 50, 100} {
+ b.Run(fmt.Sprintf("clients_%d", numClients), func(b *testing.B) {
+ h, _, cleanup := benchHub(b, numClients, 4096)
+ defer cleanup()
+
+ msg := []byte(`{"level":"info","message":"benchmark log message","timestamp":"2024-01-15T10:30:45Z"}`)
+
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for b.Loop() {
+ h.Broadcast(msg)
+ }
+ })
+ }
+}
+
+// BenchmarkHub_BroadcastMessageSizes measures broadcast performance at different message sizes.
+func BenchmarkHub_BroadcastMessageSizes(b *testing.B) {
+ for _, size := range []int{64, 256, 1024, 4096, 16384} {
+ b.Run(fmt.Sprintf("bytes_%d", size), func(b *testing.B) {
+ h, _, cleanup := benchHub(b, 10, 4096)
+ defer cleanup()
+
+ msg := make([]byte, size)
+ for i := range msg {
+ msg[i] = 'A' + byte(i%26)
+ }
+
+ b.ReportAllocs()
+ b.SetBytes(int64(size))
+ b.ResetTimer()
+
+ for b.Loop() {
+ h.Broadcast(msg)
+ }
+ })
+ }
+}
+
+// ============================================================================
+// Parse Benchmarks
+// ============================================================================
+
+// BenchmarkNormalizeContainerLine measures log line parsing performance.
+func BenchmarkNormalizeContainerLine(b *testing.B) {
+ cases := map[string]string{
+ "plain": "hello world this is a normal log message",
+ "stderr_prefix": "[STDERR] error occurred in module X",
+ "with_timestamp": "2024-01-15T10:30:45.123456789Z application started successfully on port 8080",
+ "stderr_timestamp": "[STDERR] 2024-06-01T12:00:00.000Z critical error in database connection",
+ "long_line": strings.Repeat("abcdefghij", 100),
+ "with_trailing_crlf": "message with trailing chars\r\n",
+ }
+
+ for name, input := range cases {
+ b.Run(name, func(b *testing.B) {
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ NormalizeContainerLine(input)
+ }
+ })
+ }
+}
+
+// BenchmarkNormalizeProjectLine measures project log line parsing performance.
+func BenchmarkNormalizeProjectLine(b *testing.B) {
+ cases := map[string]string{
+ "with_service": "web | Starting server on port 8080",
+ "without_service": "plain log line without service",
+ "complex": "2024-01-15T10:30:45.123Z api-gateway | Request received from 192.168.1.1",
+ }
+
+ for name, input := range cases {
+ b.Run(name, func(b *testing.B) {
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ NormalizeProjectLine(input)
+ }
+ })
+ }
+}
+
+// ============================================================================
+// JSON Marshaling Benchmarks
+// ============================================================================
+
+// BenchmarkLogMessage_Marshal measures JSON marshaling cost for a single log message.
+func BenchmarkLogMessage_Marshal(b *testing.B) {
+ msg := LogMessage{
+ Seq: 42,
+ Level: "stdout",
+ Message: "2024-01-15 10:30:45.123 INFO [main] Application started successfully on port 8080",
+ Timestamp: "2024-01-15T10:30:45.123456789Z",
+ Service: "web-api",
+ ContainerID: "abc123def456",
+ }
+
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for b.Loop() {
+ if _, err := json.Marshal(msg); err != nil {
+ b.Fatal(err)
+ }
+ }
+}
+
+// BenchmarkLogMessageBatch_Marshal measures batch JSON marshaling cost.
+func BenchmarkLogMessageBatch_Marshal(b *testing.B) {
+ for _, batchSize := range []int{1, 10, 50, 100} {
+ b.Run(fmt.Sprintf("batch_%d", batchSize), func(b *testing.B) {
+ batch := make([]LogMessage, batchSize)
+ for i := range batch {
+ batch[i] = LogMessage{
+ Seq: uint64(i), //nolint:gosec // range index is non-negative
+ Level: "stdout",
+ Message: fmt.Sprintf("log message number %d from service", i),
+ Timestamp: "2024-01-15T10:30:45.123456789Z",
+ Service: "web-api",
+ }
+ }
+
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for b.Loop() {
+ if _, err := json.Marshal(batch); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
+
+// ============================================================================
+// Bridge Throughput Benchmarks
+// ============================================================================
+
+// BenchmarkForwardLogJSONBatched_Throughput measures end-to-end batched forwarding throughput.
+func BenchmarkForwardLogJSONBatched_Throughput(b *testing.B) {
+ for _, batchSize := range []int{1, 10, 50} {
+ b.Run(fmt.Sprintf("maxBatch_%d", batchSize), func(b *testing.B) {
+ h, _, cleanup := benchHub(b, 1, 8192)
+ defer cleanup()
+
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ ctx := context.Background()
+ logs := make(chan LogMessage, b.N)
+ for range b.N {
+ logs <- LogMessage{
+ Seq: 1,
+ Level: "stdout",
+ Message: "benchmark log message for throughput testing",
+ Timestamp: "2024-01-15T10:30:45.123456789Z",
+ Service: "api",
+ }
+ }
+ close(logs)
+
+ ForwardLogJSONBatched(ctx, h, logs, batchSize, 10*time.Millisecond)
+ })
+ }
+}
+
+// BenchmarkForwardLines_Throughput measures plain text line forwarding throughput.
+func BenchmarkForwardLines_Throughput(b *testing.B) {
+ h, _, cleanup := benchHub(b, 1, 8192)
+ defer cleanup()
+
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ ctx := context.Background()
+ lines := make(chan string, b.N)
+ for range b.N {
+ lines <- "2024-01-15T10:30:45Z this is a benchmark log line for testing throughput performance"
+ }
+ close(lines)
+
+ ForwardLines(ctx, h, lines)
+}
+
+// ============================================================================
+// Memory Benchmarks
+// ============================================================================
+
+// BenchmarkHub_MemoryPerClient measures memory overhead per connected client.
+func BenchmarkHub_MemoryPerClient(b *testing.B) {
+ for _, sendBuf := range []int{16, 64, 256} {
+ b.Run(fmt.Sprintf("sendBuffer_%d", sendBuf), func(b *testing.B) {
+ h := NewHub(100)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ wsURL, serverCleanup := benchWSServer(b)
+ defer serverCleanup()
+
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for b.Loop() {
+ conn, resp, err := websocket.DefaultDialer.Dial(wsURL, nil)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if resp != nil {
+ resp.Body.Close()
+ }
+ c := NewClient(conn, sendBuf)
+ h.register <- c
+ go func() {
+ for range c.send {
+ }
+ }()
+ }
+ })
+ }
+}
+
+// BenchmarkNowRFC3339 measures timestamp generation performance.
+func BenchmarkNowRFC3339(b *testing.B) {
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for b.Loop() {
+ NowRFC3339()
+ }
+}
diff --git a/backend/internal/utils/ws/bridge_test.go b/backend/internal/utils/ws/bridge_test.go
new file mode 100644
index 000000000..519694fef
--- /dev/null
+++ b/backend/internal/utils/ws/bridge_test.go
@@ -0,0 +1,307 @@
+package ws
+
+import (
+ "context"
+ "encoding/json"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// registerCollector registers a client on the hub and returns its send channel.
+// Must be called before broadcasting so the client is present to receive messages.
+func registerCollector(t *testing.T, h *Hub, bufSize int) *Client {
+ t.Helper()
+ _, sc, cleanup := newTestWSPair(t)
+ t.Cleanup(cleanup)
+
+ c := NewClient(sc, bufSize)
+ h.register <- c
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() >= 1
+ }, time.Second, 5*time.Millisecond)
+
+ return c
+}
+
+// drainClient reads count messages from the client's send channel.
+func drainClient(t *testing.T, c *Client, count int, timeout time.Duration) [][]byte {
+ t.Helper()
+ var msgs [][]byte
+ deadline := time.After(timeout)
+ for range count {
+ select {
+ case msg := <-c.send:
+ msgs = append(msgs, msg)
+ case <-deadline:
+ t.Fatalf("timed out waiting for messages, got %d/%d", len(msgs), count)
+ }
+ }
+ return msgs
+}
+
+func TestForwardLines(t *testing.T) {
+ h := NewHub(100)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ // Register collector BEFORE starting forwarder
+ c := registerCollector(t, h, 256)
+
+ lines := make(chan string, 5)
+ lines <- "line 1"
+ lines <- "line 2"
+ lines <- "line 3"
+ close(lines)
+
+ go ForwardLines(ctx, h, lines)
+
+ msgs := drainClient(t, c, 3, 2*time.Second)
+ assert.Equal(t, "line 1", string(msgs[0]))
+ assert.Equal(t, "line 2", string(msgs[1]))
+ assert.Equal(t, "line 3", string(msgs[2]))
+}
+
+func TestForwardLines_ContextCancellation(t *testing.T) {
+ h := NewHub(100)
+ hubCtx, hubCancel := context.WithCancel(context.Background())
+ defer hubCancel()
+ go h.Run(hubCtx)
+
+ lines := make(chan string)
+ forwardCtx, forwardCancel := context.WithCancel(context.Background())
+
+ done := make(chan struct{})
+ go func() {
+ ForwardLines(forwardCtx, h, lines)
+ close(done)
+ }()
+
+ forwardCancel()
+
+ select {
+ case <-done:
+ // ForwardLines exited as expected
+ case <-time.After(time.Second):
+ t.Fatal("ForwardLines did not exit after context cancellation")
+ }
+}
+
+func TestForwardLogJSON(t *testing.T) {
+ h := NewHub(100)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ c := registerCollector(t, h, 256)
+
+ logs := make(chan LogMessage, 3)
+ logs <- LogMessage{Seq: 1, Level: "stdout", Message: "hello", Timestamp: "2024-01-15T10:30:45Z"}
+ logs <- LogMessage{Seq: 2, Level: "stderr", Message: "error", Timestamp: "2024-01-15T10:30:46Z"}
+ close(logs)
+
+ go ForwardLogJSON(ctx, h, logs)
+
+ msgs := drainClient(t, c, 2, 2*time.Second)
+
+ var m1, m2 LogMessage
+ require.NoError(t, json.Unmarshal(msgs[0], &m1))
+ require.NoError(t, json.Unmarshal(msgs[1], &m2))
+
+ assert.Equal(t, uint64(1), m1.Seq)
+ assert.Equal(t, "hello", m1.Message)
+ assert.Equal(t, uint64(2), m2.Seq)
+ assert.Equal(t, "error", m2.Message)
+}
+
+func TestForwardLogJSON_FillsEmptyTimestamp(t *testing.T) {
+ h := NewHub(100)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ c := registerCollector(t, h, 256)
+
+ logs := make(chan LogMessage, 1)
+ logs <- LogMessage{Seq: 1, Message: "no timestamp"}
+ close(logs)
+
+ go ForwardLogJSON(ctx, h, logs)
+
+ msgs := drainClient(t, c, 1, 2*time.Second)
+
+ var m LogMessage
+ require.NoError(t, json.Unmarshal(msgs[0], &m))
+ assert.NotEmpty(t, m.Timestamp, "timestamp should be auto-filled")
+
+ _, err := time.Parse(time.RFC3339Nano, m.Timestamp)
+ assert.NoError(t, err)
+}
+
+func TestForwardLogJSONBatched(t *testing.T) {
+ h := NewHub(100)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ c := registerCollector(t, h, 256)
+
+ logs := make(chan LogMessage, 10)
+ for i := range 5 {
+ logs <- LogMessage{
+ Seq: uint64(i + 1), //nolint:gosec // range index is non-negative
+ Message: "msg",
+ Timestamp: "2024-01-15T10:30:45Z",
+ }
+ }
+ close(logs)
+
+ go ForwardLogJSONBatched(ctx, h, logs, 5, 100*time.Millisecond)
+
+ msgs := drainClient(t, c, 1, 2*time.Second)
+
+ var batch []LogMessage
+ require.NoError(t, json.Unmarshal(msgs[0], &batch))
+ assert.Len(t, batch, 5, "all messages should be in a single batch")
+}
+
+func TestForwardLogJSONBatched_FlushesOnInterval(t *testing.T) {
+ h := NewHub(100)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ c := registerCollector(t, h, 256)
+
+ logs := make(chan LogMessage, 10)
+ logs <- LogMessage{Seq: 1, Message: "a", Timestamp: "2024-01-15T10:30:45Z"}
+ logs <- LogMessage{Seq: 2, Message: "b", Timestamp: "2024-01-15T10:30:46Z"}
+
+ go ForwardLogJSONBatched(ctx, h, logs, 10, 50*time.Millisecond)
+
+ msgs := drainClient(t, c, 1, 2*time.Second)
+
+ var batch []LogMessage
+ require.NoError(t, json.Unmarshal(msgs[0], &batch))
+ assert.Len(t, batch, 2)
+}
+
+func TestForwardLogJSONBatched_MultipleBatches(t *testing.T) {
+ h := NewHub(100)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ c := registerCollector(t, h, 256)
+
+ logs := make(chan LogMessage, 10)
+ for i := range 6 {
+ logs <- LogMessage{
+ Seq: uint64(i + 1), //nolint:gosec // range index is non-negative
+ Message: "msg",
+ Timestamp: "2024-01-15T10:30:45Z",
+ }
+ }
+ close(logs)
+
+ go ForwardLogJSONBatched(ctx, h, logs, 3, time.Second)
+
+ msgs := drainClient(t, c, 2, 2*time.Second)
+
+ var batch1, batch2 []LogMessage
+ require.NoError(t, json.Unmarshal(msgs[0], &batch1))
+ require.NoError(t, json.Unmarshal(msgs[1], &batch2))
+ assert.Len(t, batch1, 3)
+ assert.Len(t, batch2, 3)
+}
+
+func TestForwardLogJSONBatched_DelegatesToUnbatchedWhenMaxBatchOne(t *testing.T) {
+ h := NewHub(100)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ c := registerCollector(t, h, 256)
+
+ logs := make(chan LogMessage, 3)
+ logs <- LogMessage{Seq: 1, Message: "single", Timestamp: "2024-01-15T10:30:45Z"}
+ logs <- LogMessage{Seq: 2, Message: "items", Timestamp: "2024-01-15T10:30:46Z"}
+ close(logs)
+
+ go ForwardLogJSONBatched(ctx, h, logs, 1, time.Second)
+
+ msgs := drainClient(t, c, 2, 2*time.Second)
+
+ // Each message should be a single object, not an array
+ var m1 LogMessage
+ require.NoError(t, json.Unmarshal(msgs[0], &m1))
+ assert.Equal(t, "single", m1.Message)
+}
+
+func TestForwardLogJSONBatched_FlushesOnChannelClose(t *testing.T) {
+ h := NewHub(100)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ c := registerCollector(t, h, 256)
+
+ logs := make(chan LogMessage, 10)
+ // Send 2 messages but maxBatch is 100 — should flush on channel close
+ logs <- LogMessage{Seq: 1, Message: "flush on close", Timestamp: "2024-01-15T10:30:45Z"}
+ logs <- LogMessage{Seq: 2, Message: "also flushed", Timestamp: "2024-01-15T10:30:46Z"}
+ close(logs)
+
+ go ForwardLogJSONBatched(ctx, h, logs, 100, time.Hour)
+
+ msgs := drainClient(t, c, 1, 2*time.Second)
+
+ var batch []LogMessage
+ require.NoError(t, json.Unmarshal(msgs[0], &batch))
+ assert.Len(t, batch, 2)
+ assert.Equal(t, "flush on close", batch[0].Message)
+ assert.Equal(t, "also flushed", batch[1].Message)
+}
+
+func TestForwardLogJSONBatched_FlushesOnContextCancel(t *testing.T) {
+ h := NewHub(100)
+ hubCtx, hubCancel := context.WithCancel(context.Background())
+ defer hubCancel()
+ go h.Run(hubCtx)
+
+ c := registerCollector(t, h, 256)
+
+ // Use an unbuffered channel so we can control exactly when messages are sent
+ logs := make(chan LogMessage)
+ forwardCtx, forwardCancel := context.WithCancel(context.Background())
+
+ done := make(chan struct{})
+ go func() {
+ ForwardLogJSONBatched(forwardCtx, h, logs, 100, time.Hour)
+ close(done)
+ }()
+
+ // Send one message
+ logs <- LogMessage{Seq: 1, Message: "flush on cancel", Timestamp: "2024-01-15T10:30:45Z"}
+
+ // Give the forwarder time to buffer it
+ time.Sleep(50 * time.Millisecond)
+ forwardCancel()
+
+ select {
+ case <-done:
+ case <-time.After(2 * time.Second):
+ t.Fatal("ForwardLogJSONBatched did not exit after context cancellation")
+ }
+
+ // The buffered message should have been flushed
+ msgs := drainClient(t, c, 1, 2*time.Second)
+ var batch []LogMessage
+ require.NoError(t, json.Unmarshal(msgs[0], &batch))
+ assert.Len(t, batch, 1)
+ assert.Equal(t, "flush on cancel", batch[0].Message)
+}
diff --git a/backend/internal/utils/ws/client.go b/backend/internal/utils/ws/client.go
index 6b5c3a5f6..23e27ea09 100644
--- a/backend/internal/utils/ws/client.go
+++ b/backend/internal/utils/ws/client.go
@@ -11,9 +11,10 @@ import (
)
const (
- writeWait = 10 * time.Second
- pongWait = 60 * time.Second
- pingPeriod = pongWait * 9 / 10
+ writeWait = 10 * time.Second
+ pingWriteWait = 1 * time.Second // shorter so writePump doesn't block long on dead conns
+ pongWait = 60 * time.Second
+ pingPeriod = pongWait * 9 / 10
maxMessageSize = 64 * 1024
clientSendBuffer = 256
@@ -80,10 +81,11 @@ func (c *Client) readPump(ctx context.Context, hub *Hub) {
}
func (c *Client) writePump(ctx context.Context, hub *Hub) {
- ticker := time.NewTicker(pingPeriod)
- // Stop ticker and ensure client is removed from hub without writing to hub.unregister.
+ // Timer instead of Ticker: no drain-after-Stop, and we can reset after each ping
+ // so shutdown sees ctx.Done() without waiting for an extra tick.
+ pingTimer := time.NewTimer(pingPeriod)
defer func() {
- ticker.Stop()
+ pingTimer.Stop()
c.safeRemove(hub)
}()
@@ -94,7 +96,6 @@ func (c *Client) writePump(ctx context.Context, hub *Hub) {
case msg, ok := <-c.send:
_ = c.conn.SetWriteDeadline(time.Now().Add(writeWait))
if !ok {
- // Hub closed the channel.
_ = c.conn.WriteMessage(websocket.CloseMessage, []byte{})
return
}
@@ -104,11 +105,17 @@ func (c *Client) writePump(ctx context.Context, hub *Hub) {
}
return
}
- case <-ticker.C:
- _ = c.conn.SetWriteDeadline(time.Now().Add(writeWait))
+ case <-pingTimer.C:
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ }
+ _ = c.conn.SetWriteDeadline(time.Now().Add(pingWriteWait))
if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil {
return
}
+ pingTimer.Reset(pingPeriod)
}
}
}
diff --git a/backend/internal/utils/ws/client_test.go b/backend/internal/utils/ws/client_test.go
new file mode 100644
index 000000000..7a1bdfb53
--- /dev/null
+++ b/backend/internal/utils/ws/client_test.go
@@ -0,0 +1,252 @@
+package ws
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/gorilla/websocket"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNewClient(t *testing.T) {
+ _, serverConn, cleanup := newTestWSPair(t)
+ defer cleanup()
+
+ c := NewClient(serverConn, 64)
+ require.NotNil(t, c)
+ assert.NotNil(t, c.conn)
+ assert.NotNil(t, c.send)
+ assert.Equal(t, 64, cap(c.send))
+}
+
+func TestServeClient_ReceivesBroadcast(t *testing.T) {
+ h := NewHub(10)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ // Set up a test WS server that upgrades and serves the client
+ serverReady := make(chan struct{})
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ close(serverReady)
+ ServeClient(ctx, h, conn)
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+ defer clientConn.Close()
+
+ <-serverReady
+
+ // Wait for registration
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ // Broadcast a message
+ h.Broadcast([]byte("test message"))
+
+ // Client should receive it
+ _ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, msg, err := clientConn.ReadMessage()
+ require.NoError(t, err)
+ assert.Equal(t, "test message", string(msg))
+}
+
+func TestServeClient_ClientDisconnect(t *testing.T) {
+ h := NewHub(10)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ServeClient(ctx, h, conn)
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ // Close the client connection
+ clientConn.Close()
+
+ // The hub should eventually remove the client
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 0
+ }, 5*time.Second, 50*time.Millisecond)
+}
+
+func TestServeClient_ContextCancellation(t *testing.T) {
+ h := NewHub(10)
+ hubCtx, hubCancel := context.WithCancel(context.Background())
+ defer hubCancel()
+ go h.Run(hubCtx)
+
+ clientCtx, clientCancel := context.WithCancel(context.Background())
+
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ServeClient(clientCtx, h, conn)
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+ defer clientConn.Close()
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ // Cancel the client context
+ clientCancel()
+
+ // Client should be removed from hub
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 0
+ }, 5*time.Second, 50*time.Millisecond)
+}
+
+func TestIsExpectedCloseError(t *testing.T) {
+ tests := []struct {
+ name string
+ err error
+ expected bool
+ }{
+ {
+ name: "nil error",
+ err: nil,
+ expected: true,
+ },
+ {
+ name: "normal closure",
+ err: &websocket.CloseError{Code: websocket.CloseNormalClosure},
+ expected: true,
+ },
+ {
+ name: "going away",
+ err: &websocket.CloseError{Code: websocket.CloseGoingAway},
+ expected: true,
+ },
+ {
+ name: "no status received",
+ err: &websocket.CloseError{Code: websocket.CloseNoStatusReceived},
+ expected: true,
+ },
+ {
+ name: "use of closed network connection",
+ err: errors.New("read tcp: use of closed network connection"),
+ expected: true,
+ },
+ {
+ name: "connection reset by peer",
+ err: errors.New("read: connection reset by peer"),
+ expected: true,
+ },
+ {
+ name: "broken pipe",
+ err: errors.New("write: broken pipe"),
+ expected: true,
+ },
+ {
+ name: "unexpected error",
+ err: errors.New("some unexpected error"),
+ expected: false,
+ },
+ {
+ name: "protocol error",
+ err: &websocket.CloseError{Code: websocket.CloseProtocolError},
+ expected: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := isExpectedCloseError(tt.err)
+ assert.Equal(t, tt.expected, result)
+ })
+ }
+}
+
+func TestServeClient_MultipleMessages(t *testing.T) {
+ h := NewHub(100)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ serverReady := make(chan struct{})
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ close(serverReady)
+ ServeClient(ctx, h, conn)
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+ defer clientConn.Close()
+
+ <-serverReady
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ // Send multiple messages and verify order
+ messages := []string{"first", "second", "third", "fourth", "fifth"}
+ for _, m := range messages {
+ h.Broadcast([]byte(m))
+ }
+
+ _ = clientConn.SetReadDeadline(time.Now().Add(5 * time.Second))
+ for _, expected := range messages {
+ _, msg, err := clientConn.ReadMessage()
+ require.NoError(t, err)
+ assert.Equal(t, expected, string(msg))
+ }
+}
diff --git a/backend/internal/utils/ws/cpu_regression_test.go b/backend/internal/utils/ws/cpu_regression_test.go
new file mode 100644
index 000000000..1906d7b3f
--- /dev/null
+++ b/backend/internal/utils/ws/cpu_regression_test.go
@@ -0,0 +1,391 @@
+package ws
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "runtime"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/gorilla/websocket"
+ "github.com/stretchr/testify/require"
+)
+
+// startStatsHubPipeline starts the stats producer and JSON broadcaster goroutines
+// for benchmarks/tests that need a container-stats-style hub. Caller must run hub.Run and ServeClient.
+func startStatsHubPipeline(ctx context.Context, hub *Hub) {
+ statsChan := make(chan interface{}, 64)
+ go func() {
+ defer close(statsChan)
+ ticker := time.NewTicker(100 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ select {
+ case <-ctx.Done():
+ return
+ case statsChan <- map[string]float64{"cpu": 10.0, "memory": 1024.0}:
+ default:
+ }
+ }
+ }
+ }()
+ go func() {
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case stats, ok := <-statsChan:
+ if !ok {
+ return
+ }
+ if data, err := json.Marshal(stats); err == nil {
+ hub.Broadcast(data)
+ }
+ }
+ }
+ }()
+}
+
+// ============================================================================
+// CPU Regression Benchmarks
+//
+// These benchmarks simulate the exact usage patterns from the bug report:
+// https://github.com/getarcaneapp/arcane/issues/XXXX
+//
+// The bug: CPU load climbs permanently after using Arcane, especially after
+// reloading the container page 20-30 times. Root cause is goroutine leaks
+// from Hub+streaming pipelines not being properly cleaned up.
+//
+// These benchmarks measure:
+// 1. Goroutine count growth over repeated connect/disconnect cycles
+// 2. Per-cycle overhead to detect if cleanup is happening
+// 3. Memory growth to catch leaked resources
+// ============================================================================
+
+// BenchmarkCPU_PageReloadSimulation simulates the exact bug scenario:
+// repeatedly connecting and disconnecting to a container stats WebSocket
+// (like reloading the container page). Each connection creates a full pipeline.
+//
+// Run with: go test -bench=BenchmarkCPU_PageReloadSimulation -benchtime=30x -benchmem
+//
+// A healthy result shows constant goroutine count and stable allocations.
+// A regression shows goroutine count growing linearly with iterations.
+func BenchmarkCPU_PageReloadSimulation(b *testing.B) {
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ctx, cancel := context.WithCancel(r.Context())
+ hub := NewHub(64)
+ hub.SetOnEmpty(func() { cancel() })
+ go hub.Run(ctx)
+ startStatsHubPipeline(ctx, hub)
+ ServeClient(ctx, hub, conn)
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+
+ runtime.GC()
+ time.Sleep(100 * time.Millisecond)
+ baselineGoroutines := runtime.NumGoroutine()
+
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for i := range b.N {
+ conn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if resp != nil {
+ resp.Body.Close()
+ }
+
+ // Read one message to exercise the pipeline
+ _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, _, _ = conn.ReadMessage()
+ conn.Close()
+
+ // Brief settle for cleanup
+ time.Sleep(50 * time.Millisecond)
+
+ // Report goroutine growth periodically
+ if (i+1)%10 == 0 || i == b.N-1 {
+ current := runtime.NumGoroutine()
+ b.ReportMetric(float64(current-baselineGoroutines), "goroutine_delta")
+ }
+ }
+
+ b.StopTimer()
+
+ // Final goroutine check
+ time.Sleep(500 * time.Millisecond)
+ runtime.GC()
+ time.Sleep(200 * time.Millisecond)
+ finalGoroutines := runtime.NumGoroutine()
+ b.ReportMetric(float64(finalGoroutines-baselineGoroutines), "final_goroutine_leak")
+}
+
+// BenchmarkCPU_ContainerLogReloadSimulation simulates reloading a container
+// logs page with JSON batched output (the most complex pipeline).
+func BenchmarkCPU_ContainerLogReloadSimulation(b *testing.B) {
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ctx, cancel := context.WithCancel(r.Context())
+ hub := NewHub(1024)
+ hub.SetOnEmpty(func() { cancel() })
+ go hub.Run(ctx)
+
+ lines := make(chan string, 256)
+ go func() {
+ defer close(lines)
+ ticker := time.NewTicker(20 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ lines <- "2024-01-15T10:30:45.123Z INFO container log message"
+ }
+ }
+ }()
+
+ msgs := make(chan LogMessage, 256)
+ go func() {
+ defer close(msgs)
+ seq := uint64(0)
+ for line := range lines {
+ level, msg, ts := NormalizeContainerLine(line)
+ seq++
+ if ts == "" {
+ ts = NowRFC3339()
+ }
+ msgs <- LogMessage{Seq: seq, Level: level, Message: msg, Timestamp: ts}
+ }
+ }()
+
+ go ForwardLogJSONBatched(ctx, hub, msgs, 50, 400*time.Millisecond)
+
+ ServeClient(ctx, hub, conn)
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+
+ runtime.GC()
+ time.Sleep(100 * time.Millisecond)
+ baselineGoroutines := runtime.NumGoroutine()
+
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for i := range b.N {
+ conn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if resp != nil {
+ resp.Body.Close()
+ }
+
+ _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, _, _ = conn.ReadMessage()
+ conn.Close()
+ time.Sleep(50 * time.Millisecond)
+ if (i+1)%10 == 0 || i == b.N-1 {
+ current := runtime.NumGoroutine()
+ b.ReportMetric(float64(current-baselineGoroutines), "goroutine_delta")
+ }
+ }
+ b.StopTimer()
+ time.Sleep(500 * time.Millisecond)
+ runtime.GC()
+ time.Sleep(200 * time.Millisecond)
+ finalGoroutines := runtime.NumGoroutine()
+ b.ReportMetric(float64(finalGoroutines-baselineGoroutines), "final_goroutine_leak")
+}
+
+// BenchmarkCPU_GoroutineScaling measures how goroutine count scales with
+// concurrent connections. This catches leaks that only manifest under load.
+func BenchmarkCPU_GoroutineScaling(b *testing.B) {
+ for _, concurrency := range []int{1, 5, 10, 20} {
+ b.Run(fmt.Sprintf("concurrent_%d", concurrency), func(b *testing.B) {
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ctx, cancel := context.WithCancel(r.Context())
+ hub := NewHub(64)
+ hub.SetOnEmpty(func() { cancel() })
+ go hub.Run(ctx)
+ startStatsHubPipeline(ctx, hub)
+ ServeClient(ctx, hub, conn)
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+
+ runtime.GC()
+ time.Sleep(100 * time.Millisecond)
+ baselineGoroutines := runtime.NumGoroutine()
+
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for range b.N {
+ // Open `concurrency` connections simultaneously
+ conns := make([]*websocket.Conn, concurrency)
+ for j := range concurrency {
+ conn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if resp != nil {
+ resp.Body.Close()
+ }
+ conns[j] = conn
+ }
+
+ for _, c := range conns {
+ _ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, _, _ = c.ReadMessage()
+ }
+
+ // Close all
+ for _, conn := range conns {
+ conn.Close()
+ }
+
+ time.Sleep(100 * time.Millisecond)
+ }
+
+ b.StopTimer()
+ time.Sleep(1 * time.Second)
+ runtime.GC()
+ time.Sleep(200 * time.Millisecond)
+ finalGoroutines := runtime.NumGoroutine()
+ b.ReportMetric(float64(finalGoroutines-baselineGoroutines), "final_goroutine_leak")
+ })
+ }
+}
+
+// BenchmarkCPU_SustainedStreaming measures CPU overhead of maintaining
+// an active streaming connection over time. This catches CPU drift from
+// inefficient polling or tight loops.
+func BenchmarkCPU_SustainedStreaming(b *testing.B) {
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ctx, cancel := context.WithCancel(r.Context())
+ hub := NewHub(1024)
+ hub.SetOnEmpty(func() { cancel() })
+ go hub.Run(ctx)
+ startStatsHubPipeline(ctx, hub)
+ ServeClient(ctx, hub, conn)
+ }))
+ defer server.Close()
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ conn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if resp != nil {
+ resp.Body.Close()
+ }
+ for range 10 {
+ _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ if _, _, err := conn.ReadMessage(); err != nil {
+ break
+ }
+ }
+ conn.Close()
+ time.Sleep(50 * time.Millisecond)
+ }
+}
+
+// TestCPU_GoroutineCountReport is not a benchmark but a diagnostic test
+// that outputs detailed goroutine counts at each stage. Run with -v to see output.
+func TestCPU_GoroutineCountReport(t *testing.T) {
+ runtime.GC()
+ time.Sleep(100 * time.Millisecond)
+ baseline := runtime.NumGoroutine()
+ t.Logf("baseline goroutines: %d", baseline)
+
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ctx, cancel := context.WithCancel(r.Context())
+ hub := NewHub(64)
+ hub.SetOnEmpty(func() { cancel() })
+ go hub.Run(ctx)
+ startStatsHubPipeline(ctx, hub)
+ ServeClient(ctx, hub, conn)
+ }))
+ defer server.Close()
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+
+ peakGoroutines := baseline
+ for i := range 30 {
+ conn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+ _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, _, _ = conn.ReadMessage()
+ conn.Close()
+ time.Sleep(50 * time.Millisecond)
+ current := runtime.NumGoroutine()
+ if current > peakGoroutines {
+ peakGoroutines = current
+ }
+ if (i+1)%5 == 0 {
+ t.Logf("after %d cycles: goroutines=%d (delta=%d, peak=%d)",
+ i+1, current, current-baseline, peakGoroutines)
+ }
+ }
+
+ // Wait for final cleanup
+ time.Sleep(2 * time.Second)
+ runtime.GC()
+ time.Sleep(200 * time.Millisecond)
+ final := runtime.NumGoroutine()
+
+ t.Logf("FINAL: baseline=%d, peak=%d, final=%d, leaked=%d",
+ baseline, peakGoroutines, final, final-baseline)
+
+ // The key assertion: after all connections are closed, goroutine count
+ // should be close to baseline. A leak would show final >> baseline.
+ require.LessOrEqual(t, final-baseline, 5,
+ "goroutine count should return near baseline after all connections close")
+}
diff --git a/backend/internal/utils/ws/diagnostics.go b/backend/internal/utils/ws/diagnostics.go
new file mode 100644
index 000000000..36b717f56
--- /dev/null
+++ b/backend/internal/utils/ws/diagnostics.go
@@ -0,0 +1,74 @@
+package ws
+
+import (
+ "runtime"
+ "strings"
+ "sync"
+ "time"
+)
+
+const wsPkgPath = "internal/utils/ws"
+
+const workerGoroutineCountTTL = 5 * time.Second
+
+var workerGoroutineCountCache struct {
+ sync.Mutex
+ value int
+ at time.Time
+}
+
+// CountWorkerGoroutines returns a best-effort count of websocket worker goroutines
+// belonging to this package. Intended for diagnostics endpoints only.
+func CountWorkerGoroutines() int {
+ workerGoroutineCountCache.Lock()
+ if !workerGoroutineCountCache.at.IsZero() && time.Since(workerGoroutineCountCache.at) < workerGoroutineCountTTL {
+ value := workerGoroutineCountCache.value
+ workerGoroutineCountCache.Unlock()
+ return value
+ }
+ workerGoroutineCountCache.Unlock()
+
+ count := countWorkerGoroutinesInternal()
+
+ workerGoroutineCountCache.Lock()
+ workerGoroutineCountCache.value = count
+ workerGoroutineCountCache.at = time.Now()
+ workerGoroutineCountCache.Unlock()
+
+ return count
+}
+
+func countWorkerGoroutinesInternal() int {
+ buf := make([]byte, 1<<20)
+ for {
+ n := runtime.Stack(buf, true)
+ if n < len(buf) {
+ buf = buf[:n]
+ break
+ }
+ if len(buf) >= 8<<20 {
+ buf = buf[:n]
+ break
+ }
+ buf = make([]byte, len(buf)*2)
+ }
+
+ s := string(buf)
+ blocks := strings.Split(s, "\n\n")
+ count := 0
+ for _, block := range blocks {
+ if block == "" || !strings.Contains(block, wsPkgPath) {
+ continue
+ }
+ if strings.Contains(block, ".Run(") ||
+ strings.Contains(block, "readPump") ||
+ strings.Contains(block, "writePump") ||
+ strings.Contains(block, "ForwardLines") ||
+ strings.Contains(block, "ForwardLogJSON") ||
+ strings.Contains(block, "ForwardLogJSONBatched") {
+ count++
+ }
+ }
+
+ return count
+}
diff --git a/backend/internal/utils/ws/goroutine_leak_test.go b/backend/internal/utils/ws/goroutine_leak_test.go
new file mode 100644
index 000000000..c957620fc
--- /dev/null
+++ b/backend/internal/utils/ws/goroutine_leak_test.go
@@ -0,0 +1,643 @@
+package ws
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "runtime"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/gorilla/websocket"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// goroutineCount returns the current number of goroutines with a brief settling delay.
+func goroutineCount() int {
+ // Allow goroutines from prior tests to finish
+ runtime.GC()
+ time.Sleep(50 * time.Millisecond)
+ return runtime.NumGoroutine()
+}
+
+// pkgPath is the ws package path for stack matching (avoid counting other packages).
+const pkgPath = "internal/utils/ws"
+
+// countWsWorkerGoroutines returns the number of goroutines that are running ws package
+// workers (Hub.Run, readPump, writePump, ForwardLines, ForwardLogJSONBatched, or
+// test-style producer/broadcaster loops). Used to assert zero leak of our code only,
+// ignoring httptest/net/http goroutines.
+func countWsWorkerGoroutines() int {
+ buf := make([]byte, 1<<20)
+ n := runtime.Stack(buf, true)
+ s := string(buf[:n])
+ count := 0
+ blocks := strings.Split(s, "\n\n")
+ for _, block := range blocks {
+ if block == "" || !strings.Contains(block, pkgPath) {
+ continue
+ }
+ // Match our long-lived workers only (not the test runner).
+ if strings.Contains(block, ".Run(") ||
+ strings.Contains(block, "readPump") ||
+ strings.Contains(block, "writePump") ||
+ strings.Contains(block, "ForwardLines") ||
+ strings.Contains(block, "ForwardLogJSONBatched") ||
+ strings.Contains(block, "statsChan") ||
+ strings.Contains(block, "hub.Broadcast") {
+ count++
+ }
+ }
+ return count
+}
+
+// waitForWsWorkerGoroutines waits until the ws worker goroutine count is <= target or timeout.
+func waitForWsWorkerGoroutines(t *testing.T, target int, timeout time.Duration) int {
+ t.Helper()
+ deadline := time.After(timeout)
+ for {
+ select {
+ case <-deadline:
+ return countWsWorkerGoroutines()
+ default:
+ runtime.GC()
+ time.Sleep(20 * time.Millisecond)
+ n := countWsWorkerGoroutines()
+ if n <= target {
+ return n
+ }
+ }
+ }
+}
+
+// waitForGoroutineCount waits for goroutine count to reach target (±tolerance).
+func waitForGoroutineCount(t *testing.T, target, tolerance int, timeout time.Duration) int {
+ t.Helper()
+ deadline := time.After(timeout)
+ for {
+ select {
+ case <-deadline:
+ actual := runtime.NumGoroutine()
+ return actual
+ default:
+ runtime.GC()
+ actual := runtime.NumGoroutine()
+ if actual <= target+tolerance {
+ return actual
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+ }
+}
+
+// ============================================================================
+// Core goroutine leak tests: these replicate the pattern from ws_handler.go
+// where each page view creates a Hub + streaming goroutines + ServeClient,
+// and cleanup depends on the OnEmpty callback firing when the client disconnects.
+// ============================================================================
+
+// TestLeak_HubRunExitsOnContextCancel verifies that Hub.Run goroutine exits
+// when the context is cancelled (simulates cleanup path).
+func TestLeak_HubRunExitsOnContextCancel(t *testing.T) {
+ baseline := goroutineCount()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ h := NewHub(64)
+ go h.Run(ctx)
+
+ // Verify Run is active
+ assert.Eventually(t, func() bool {
+ return runtime.NumGoroutine() > baseline
+ }, time.Second, 10*time.Millisecond)
+
+ cancel()
+
+ actual := waitForGoroutineCount(t, baseline, 2, 3*time.Second)
+ assert.LessOrEqual(t, actual, baseline+2,
+ "Hub.Run goroutine should exit after context cancel; goroutine delta: %d", actual-baseline)
+}
+
+// TestLeak_ServeClientFullLifecycle verifies zero goroutine leaks from the full
+// ServeClient lifecycle: register -> receive messages -> client disconnect -> cleanup.
+// This is the core pattern for every WebSocket page in Arcane.
+func TestLeak_ServeClientFullLifecycle(t *testing.T) {
+ baseline := goroutineCount()
+
+ h := NewHub(64)
+ ctx, cancel := context.WithCancel(context.Background())
+ go h.Run(ctx)
+
+ // Set up WebSocket server that calls ServeClient (mirrors ws_handler.go pattern)
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ServeClient(ctx, h, conn)
+ }))
+ defer server.Close()
+
+ // Connect a client
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ // The client is connected; now we should have extra goroutines:
+ // Hub.Run + readPump + writePump = 3 goroutines above the httptest server goroutines
+ midpoint := runtime.NumGoroutine()
+ assert.Greater(t, midpoint, baseline, "should have active goroutines while connected")
+
+ // Client disconnects (simulates browser close/reload)
+ clientConn.Close()
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 0
+ }, 3*time.Second, 50*time.Millisecond, "client should be removed from hub after close")
+
+ // Cancel context to stop Hub.Run (simulates OnEmpty calling cancel())
+ cancel()
+
+ actual := waitForGoroutineCount(t, baseline, 3, 5*time.Second)
+ assert.LessOrEqual(t, actual, baseline+3,
+ "all goroutines should exit after client disconnect + cancel; delta: %d", actual-baseline)
+}
+
+// TestLeak_OnEmptyCallbackCancelsContext verifies the OnEmpty -> cancel() pattern
+// used in ws_handler.go actually cleans up all goroutines when the last client leaves.
+func TestLeak_OnEmptyCallbackCancelsContext(t *testing.T) {
+ baseline := goroutineCount()
+
+ h := NewHub(64)
+ ctx, cancel := context.WithCancel(context.Background())
+
+ h.SetOnEmpty(func() {
+ cancel() // This is what ws_handler.go does
+ })
+
+ go h.Run(ctx)
+
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ServeClient(ctx, h, conn)
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ // Close client -> triggers safeRemove -> hub.remove -> OnEmpty -> cancel()
+ clientConn.Close()
+
+ // Hub.Run should exit because OnEmpty called cancel()
+ actual := waitForGoroutineCount(t, baseline, 3, 5*time.Second)
+ assert.LessOrEqual(t, actual, baseline+3,
+ "OnEmpty -> cancel() should clean up Hub.Run + pumps; delta: %d", actual-baseline)
+}
+
+// TestLeak_RepeatedConnectDisconnect simulates the exact bug scenario:
+// rapidly connecting and disconnecting (like reloading a container page 20+ times).
+// Each cycle creates a new Hub + ServeClient (matching the ws_handler.go pattern).
+// Goroutine count should remain bounded.
+func TestLeak_RepeatedConnectDisconnect(t *testing.T) {
+ baseline := goroutineCount()
+
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ctx, cancel := context.WithCancel(r.Context())
+ hub := NewHub(64)
+ hub.SetOnEmpty(func() { cancel() })
+ go hub.Run(ctx)
+ ServeClient(ctx, hub, conn)
+ }))
+ defer server.Close()
+
+ const iterations = 100
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+
+ for i := range iterations {
+ conn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err, "iteration %d", i)
+ if resp != nil {
+ resp.Body.Close()
+ }
+
+ // Brief moment to let registration happen
+ time.Sleep(10 * time.Millisecond)
+
+ // Close (simulates page reload)
+ conn.Close()
+
+ // Brief moment for cleanup
+ time.Sleep(20 * time.Millisecond)
+ }
+
+ // Wait for all ws worker goroutines (Hub.Run, readPump, writePump) to exit.
+ wsWorkers := waitForWsWorkerGoroutines(t, 0, 5*time.Second)
+ assert.Equal(t, 0, wsWorkers,
+ "after %d connect/disconnect cycles, ws worker goroutines must be 0; got %d",
+ iterations, wsWorkers)
+
+ // Close the test server so its listener goroutine exits before we measure.
+ server.Close()
+
+ actual := waitForGoroutineCount(t, baseline, 0, 3*time.Second)
+ leaked := actual - baseline
+ assert.Equal(t, 0, leaked,
+ "after %d connect/disconnect cycles, global goroutine delta must be 0; "+
+ "baseline=%d, final=%d", iterations, baseline, actual)
+ t.Logf("goroutine leak check: baseline=%d, final=%d, delta=%d, ws_workers=%d (after %d cycles)",
+ baseline, actual, leaked, wsWorkers, iterations)
+}
+
+// TestLeak_HubWithForwardLinesLifecycle tests the full pipeline:
+// Hub.Run + ForwardLines + ServeClient, all cleaned up when client disconnects.
+// This mirrors startContainerLogHub() with format="text".
+func TestLeak_HubWithForwardLinesLifecycle(t *testing.T) {
+ baseline := goroutineCount()
+
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { //nolint:contextcheck // intentional: context must outlive the HTTP request for WebSocket streaming
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ hub := NewHub(1024)
+ hub.SetOnEmpty(func() { cancel() })
+ go hub.Run(ctx) //nolint:contextcheck // intentional: context must outlive the HTTP request for WebSocket streaming
+
+ // Simulate a streaming data source (like Docker container logs)
+ lines := make(chan string, 256)
+ go func() {
+ defer close(lines)
+ ticker := time.NewTicker(50 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ lines <- "log line from container"
+ }
+ }
+ }()
+
+ go ForwardLines(ctx, hub, lines) //nolint:contextcheck // intentional: context must outlive the HTTP request
+
+ ServeClient(ctx, hub, conn) //nolint:contextcheck // intentional: context must outlive the HTTP request
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+
+ // Read a few messages to confirm the pipeline works
+ _ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, _, err = clientConn.ReadMessage()
+ require.NoError(t, err)
+
+ // Close client -> OnEmpty -> cancel() -> all goroutines should exit
+ clientConn.Close()
+
+ actual := waitForGoroutineCount(t, baseline, 3, 5*time.Second)
+ assert.LessOrEqual(t, actual, baseline+3,
+ "Hub+ForwardLines+streaming goroutines should all exit; delta: %d", actual-baseline)
+}
+
+// TestLeak_HubWithForwardLogJSONBatchedLifecycle tests the batched JSON pipeline:
+// Hub.Run + stream producer + normalizer + ForwardLogJSONBatched + ServeClient.
+// This mirrors startContainerLogHub() with format="json" and batched=true.
+func TestLeak_HubWithForwardLogJSONBatchedLifecycle(t *testing.T) {
+ baseline := goroutineCount()
+
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { //nolint:contextcheck // intentional: context must outlive the HTTP request for WebSocket streaming
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ hub := NewHub(1024)
+ hub.SetOnEmpty(func() { cancel() })
+ go hub.Run(ctx) //nolint:contextcheck // intentional: context must outlive the HTTP request
+
+ // Simulate streaming source
+ lines := make(chan string, 256)
+ go func() {
+ defer close(lines)
+ ticker := time.NewTicker(50 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ lines <- "2024-01-15T10:30:45.123Z container log line"
+ }
+ }
+ }()
+
+ // Normalizer goroutine (matches ws_handler.go pattern)
+ msgs := make(chan LogMessage, 256)
+ go func() {
+ defer close(msgs)
+ seq := uint64(0)
+ for line := range lines {
+ level, msg, ts := NormalizeContainerLine(line)
+ seq++
+ if ts == "" {
+ ts = NowRFC3339()
+ }
+ msgs <- LogMessage{
+ Seq: seq,
+ Level: level,
+ Message: msg,
+ Timestamp: ts,
+ }
+ }
+ }()
+
+ go ForwardLogJSONBatched(ctx, hub, msgs, 50, 400*time.Millisecond) //nolint:contextcheck // intentional: context must outlive the HTTP request
+
+ ServeClient(ctx, hub, conn) //nolint:contextcheck // intentional: context must outlive the HTTP request
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+
+ // Read a message to confirm the full pipeline works
+ _ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, raw, err := clientConn.ReadMessage()
+ require.NoError(t, err)
+
+ // Should be a JSON array (batched)
+ var batch []LogMessage
+ require.NoError(t, json.Unmarshal(raw, &batch))
+ assert.NotEmpty(t, batch)
+
+ // Disconnect
+ clientConn.Close()
+
+ actual := waitForGoroutineCount(t, baseline, 3, 5*time.Second)
+ assert.LessOrEqual(t, actual, baseline+3,
+ "full batched JSON pipeline goroutines should all exit; delta: %d", actual-baseline)
+}
+
+// startStatsHubForTest starts Hub.Run plus stats producer and JSON broadcaster;
+// caller must call ServeClient(ctx, hub, conn). Used by container-stats leak tests.
+func startStatsHubForTest(ctx context.Context, hub *Hub) {
+ statsChan := make(chan interface{}, 64)
+ go func() {
+ defer close(statsChan)
+ ticker := time.NewTicker(100 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ select {
+ case <-ctx.Done():
+ return
+ case statsChan <- map[string]interface{}{"cpu_percent": 12.5, "memory": 1024000}:
+ default:
+ }
+ }
+ }
+ }()
+ go func() {
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case stats, ok := <-statsChan:
+ if !ok {
+ return
+ }
+ if b, err := json.Marshal(stats); err == nil {
+ hub.Broadcast(b)
+ }
+ }
+ }
+ }()
+}
+
+// startStatsHubRepeatedTest starts stats producer + broadcaster with 50ms ticker (for repeated cycles test).
+func startStatsHubRepeatedTest(ctx context.Context, hub *Hub) {
+ statsChan := make(chan interface{}, 64)
+ go func() {
+ defer close(statsChan)
+ ticker := time.NewTicker(50 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ select {
+ case <-ctx.Done():
+ return
+ case statsChan <- map[string]float64{"cpu": 10.0}:
+ default:
+ }
+ }
+ }
+ }()
+ go func() {
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case stats, ok := <-statsChan:
+ if !ok {
+ return
+ }
+ if b, err := json.Marshal(stats); err == nil {
+ hub.Broadcast(b)
+ }
+ }
+ }
+ }()
+}
+
+// TestLeak_ContainerStatsHubPattern tests the container stats streaming pattern:
+// Hub.Run + stats producer + JSON broadcaster + ServeClient.
+func TestLeak_ContainerStatsHubPattern(t *testing.T) {
+ baseline := goroutineCount()
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { //nolint:contextcheck // intentional: context must outlive the HTTP request for WebSocket streaming
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ hub := NewHub(64)
+ hub.SetOnEmpty(func() { cancel() })
+ go hub.Run(ctx) //nolint:contextcheck // intentional: context must outlive the HTTP request
+ startStatsHubForTest(ctx, hub) //nolint:contextcheck // intentional: context must outlive the HTTP request
+ ServeClient(ctx, hub, conn) //nolint:contextcheck // intentional: context must outlive the HTTP request
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+ _ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, raw, err := clientConn.ReadMessage()
+ require.NoError(t, err)
+ var stats map[string]interface{}
+ require.NoError(t, json.Unmarshal(raw, &stats))
+ clientConn.Close()
+
+ actual := waitForGoroutineCount(t, baseline, 3, 5*time.Second)
+ assert.LessOrEqual(t, actual, baseline+3,
+ "container stats hub goroutines should all exit; delta: %d", actual-baseline)
+}
+
+// TestLeak_RepeatedStatsHubCycles simulates the exact bug report:
+// "Go to the container page. Reload the page 20/30 times."
+func TestLeak_RepeatedStatsHubCycles(t *testing.T) {
+ baseline := goroutineCount()
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ctx, cancel := context.WithCancel(r.Context())
+ hub := NewHub(64)
+ hub.SetOnEmpty(func() { cancel() })
+ go hub.Run(ctx)
+ startStatsHubRepeatedTest(ctx, hub)
+ ServeClient(ctx, hub, conn)
+ }))
+ defer server.Close()
+
+ const iterations = 100
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ for i := range iterations {
+ conn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err, "iteration %d", i)
+ if resp != nil {
+ resp.Body.Close()
+ }
+
+ _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, _, _ = conn.ReadMessage()
+ conn.Close()
+ time.Sleep(30 * time.Millisecond)
+ }
+
+ // Wait for all ws worker goroutines to exit (no blocking on full statsChan).
+ wsWorkers := waitForWsWorkerGoroutines(t, 0, 5*time.Second)
+ assert.Equal(t, 0, wsWorkers,
+ "after %d stats hub cycles, ws worker goroutines must be 0; got %d",
+ iterations, wsWorkers)
+
+ // Close the test server so its listener goroutine exits before we measure.
+ server.Close()
+
+ actual := waitForGoroutineCount(t, baseline, 0, 3*time.Second)
+ leaked := actual - baseline
+ assert.Equal(t, 0, leaked,
+ "after %d stats hub cycles, global goroutine delta must be 0; "+
+ "baseline=%d, final=%d", iterations, baseline, actual)
+ t.Logf("stats hub leak check: baseline=%d, final=%d, delta=%d, ws_workers=%d (after %d cycles)",
+ baseline, actual, leaked, wsWorkers, iterations)
+}
+
+// TestLeak_MultipleClientsOnSameHub verifies that multiple clients connecting
+// to the same hub don't leak goroutines when they all disconnect.
+func TestLeak_MultipleClientsOnSameHub(t *testing.T) {
+ baseline := goroutineCount()
+
+ h := NewHub(64)
+ ctx, cancel := context.WithCancel(context.Background())
+
+ h.SetOnEmpty(func() {
+ cancel()
+ })
+
+ go h.Run(ctx)
+
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ ServeClient(ctx, h, conn)
+ }))
+ defer server.Close()
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ const numClients = 10
+ var clients []*websocket.Conn
+
+ for range numClients {
+ conn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+ clients = append(clients, conn)
+ }
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == numClients
+ }, 2*time.Second, 10*time.Millisecond)
+
+ // Close all clients
+ for _, c := range clients {
+ c.Close()
+ }
+
+ // OnEmpty should fire after last client leaves, cancelling context
+ actual := waitForGoroutineCount(t, baseline, 3, 5*time.Second)
+ assert.LessOrEqual(t, actual, baseline+3,
+ "all client goroutines + Hub.Run should exit; delta: %d", actual-baseline)
+}
diff --git a/backend/internal/utils/ws/hub.go b/backend/internal/utils/ws/hub.go
index 4409cfddd..ac72523e3 100644
--- a/backend/internal/utils/ws/hub.go
+++ b/backend/internal/utils/ws/hub.go
@@ -3,7 +3,6 @@ package ws
import (
"context"
"log/slog"
- "runtime"
"sync"
)
@@ -49,23 +48,8 @@ func (h *Hub) Run(ctx context.Context) {
h.clients[c] = struct{}{}
h.mu.Unlock()
case c := <-h.unregister:
+ // remove() handles the onEmpty callback when the last client disconnects.
h.remove(c)
- if h.ClientCount() == 0 {
- // Run onEmpty callback in a separate goroutine to avoid blocking the hub
- go func() {
- // Yield to allow other goroutines to run
- runtime.Gosched()
-
- h.mu.RLock()
- empty := len(h.clients) == 0
- onEmpty := h.onEmpty
- h.mu.RUnlock()
-
- if empty && onEmpty != nil {
- onEmpty()
- }
- }()
- }
case msg := <-h.broadcast:
h.mu.RLock()
var slowClients []*Client
@@ -99,12 +83,24 @@ func (h *Hub) Broadcast(msg []byte) {
func (h *Hub) remove(c *Client) {
h.mu.Lock()
- if _, ok := h.clients[c]; ok {
+ _, exists := h.clients[c]
+ if exists {
delete(h.clients, c)
close(c.send)
_ = c.conn.Close()
}
+ // Capture onEmpty under the lock so we can call it outside.
+ var onEmpty func()
+ if exists && len(h.clients) == 0 && h.onEmpty != nil {
+ onEmpty = h.onEmpty
+ }
h.mu.Unlock()
+
+ // Call outside the lock to avoid holding the mutex during cancel().
+ if onEmpty != nil {
+ slog.Debug("websocket hub empty after client removal, triggering cleanup")
+ onEmpty()
+ }
}
func (h *Hub) closeAll() {
diff --git a/backend/internal/utils/ws/hub_test.go b/backend/internal/utils/ws/hub_test.go
new file mode 100644
index 000000000..4f4732fcd
--- /dev/null
+++ b/backend/internal/utils/ws/hub_test.go
@@ -0,0 +1,376 @@
+package ws
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/gorilla/websocket"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// newTestWSPair creates a connected client/server WebSocket pair using httptest.
+func newTestWSPair(t *testing.T) (clientConn *websocket.Conn, serverConn *websocket.Conn, cleanup func()) {
+ t.Helper()
+ serverReady := make(chan *websocket.Conn, 1)
+
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ serverReady <- conn
+ }))
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ cc, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+
+ sc := <-serverReady
+
+ return cc, sc, func() {
+ cc.Close()
+ sc.Close()
+ server.Close()
+ }
+}
+
+func TestNewHub(t *testing.T) {
+ h := NewHub(10)
+ require.NotNil(t, h)
+ assert.Equal(t, 0, h.ClientCount())
+}
+
+func TestHub_RegisterAndClientCount(t *testing.T) {
+ h := NewHub(10)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ _, serverConn, cleanup := newTestWSPair(t)
+ defer cleanup()
+
+ c := NewClient(serverConn, 16)
+ h.register <- c
+
+ // Wait for registration to process
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+}
+
+func TestHub_UnregisterClient(t *testing.T) {
+ h := NewHub(10)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ _, serverConn, cleanup := newTestWSPair(t)
+ defer cleanup()
+
+ c := NewClient(serverConn, 16)
+ h.register <- c
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ h.unregister <- c
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 0
+ }, time.Second, 5*time.Millisecond)
+}
+
+func TestHub_Broadcast(t *testing.T) {
+ h := NewHub(10)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ _, serverConn, cleanup := newTestWSPair(t)
+ defer cleanup()
+
+ c := NewClient(serverConn, 16)
+ h.register <- c
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ msg := []byte("hello world")
+ h.Broadcast(msg)
+
+ // Message should appear on the client's send channel
+ select {
+ case received := <-c.send:
+ assert.Equal(t, msg, received)
+ case <-time.After(time.Second):
+ t.Fatal("timed out waiting for broadcast message")
+ }
+}
+
+func TestHub_BroadcastToMultipleClients(t *testing.T) {
+ h := NewHub(10)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ const numClients = 5
+ clients := make([]*Client, numClients)
+ cleanups := make([]func(), numClients)
+
+ for i := range numClients {
+ _, sc, cleanup := newTestWSPair(t)
+ cleanups[i] = cleanup
+ clients[i] = NewClient(sc, 16)
+ h.register <- clients[i]
+ }
+ defer func() {
+ for _, fn := range cleanups {
+ fn()
+ }
+ }()
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == numClients
+ }, time.Second, 5*time.Millisecond)
+
+ msg := []byte("broadcast to all")
+ h.Broadcast(msg)
+
+ for i, c := range clients {
+ select {
+ case received := <-c.send:
+ assert.Equal(t, msg, received, "client %d did not receive expected message", i)
+ case <-time.After(time.Second):
+ t.Fatalf("client %d timed out waiting for broadcast message", i)
+ }
+ }
+}
+
+func TestHub_BackpressureDropsSlowClient(t *testing.T) {
+ h := NewHub(10)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ // Create a client with a tiny send buffer so it fills up fast
+ _, serverConn, cleanup := newTestWSPair(t)
+ defer cleanup()
+
+ c := NewClient(serverConn, 1)
+ h.register <- c
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ // Fill the client's send buffer
+ c.send <- []byte("fill")
+
+ // Now broadcast — the client's channel is full, so it should get dropped
+ h.Broadcast([]byte("overflow"))
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 0
+ }, time.Second, 5*time.Millisecond)
+}
+
+func TestHub_OnEmptyCallback(t *testing.T) {
+ h := NewHub(10)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ var called atomic.Bool
+ h.SetOnEmpty(func() {
+ called.Store(true)
+ })
+
+ _, serverConn, cleanup := newTestWSPair(t)
+ defer cleanup()
+
+ c := NewClient(serverConn, 16)
+ h.register <- c
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ h.unregister <- c
+
+ require.Eventually(t, called.Load, time.Second, 5*time.Millisecond, "onEmpty callback was not called")
+}
+
+func TestHub_OnEmptyCalledOnlyOnce(t *testing.T) {
+ h := NewHub(10)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ var callCount atomic.Int32
+ h.SetOnEmpty(func() {
+ callCount.Add(1)
+ })
+
+ // Register two clients
+ _, sc1, cleanup1 := newTestWSPair(t)
+ defer cleanup1()
+ _, sc2, cleanup2 := newTestWSPair(t)
+ defer cleanup2()
+
+ c1 := NewClient(sc1, 16)
+ c2 := NewClient(sc2, 16)
+ h.register <- c1
+ h.register <- c2
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 2
+ }, time.Second, 5*time.Millisecond)
+
+ // Unregister first — should NOT trigger onEmpty
+ h.unregister <- c1
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ time.Sleep(50 * time.Millisecond)
+ assert.Equal(t, int32(0), callCount.Load(), "onEmpty should not fire while clients remain")
+
+ // Unregister last — should trigger onEmpty exactly once
+ h.unregister <- c2
+ require.Eventually(t, func() bool {
+ return callCount.Load() == 1
+ }, time.Second, 5*time.Millisecond)
+}
+
+func TestHub_ContextCancellation(t *testing.T) {
+ h := NewHub(10)
+ ctx, cancel := context.WithCancel(context.Background())
+
+ done := make(chan struct{})
+ go func() {
+ h.Run(ctx)
+ close(done)
+ }()
+
+ _, serverConn, cleanup := newTestWSPair(t)
+ defer cleanup()
+
+ c := NewClient(serverConn, 16)
+ h.register <- c
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ cancel()
+
+ select {
+ case <-done:
+ // Hub exited as expected
+ case <-time.After(2 * time.Second):
+ t.Fatal("Hub.Run did not exit after context cancellation")
+ }
+
+ // After closeAll, clients map should be empty
+ assert.Equal(t, 0, h.ClientCount())
+}
+
+func TestHub_BroadcastBufferFull(t *testing.T) {
+ // Hub with a buffer of 1
+ h := NewHub(1)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ // Don't run the hub — the broadcast channel will fill up
+ h.broadcast <- []byte("fill")
+
+ // This should not block (drops the message)
+ h.Broadcast([]byte("should be dropped"))
+
+ // Verify no panic or deadlock occurred
+ go h.Run(ctx)
+}
+
+func TestHub_ConcurrentOperations(t *testing.T) {
+ h := NewHub(100)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ const goroutines = 10
+ var wg sync.WaitGroup
+
+ // Create all pairs in the test goroutine (newTestWSPair uses t).
+ pairs := make([]struct {
+ sc *websocket.Conn
+ cleanup func()
+ }, goroutines)
+ for i := range goroutines {
+ _, pairs[i].sc, pairs[i].cleanup = newTestWSPair(t)
+ defer pairs[i].cleanup()
+ }
+
+ for i := range goroutines {
+ wg.Add(1)
+ go func(idx int) {
+ defer wg.Done()
+ sc := pairs[idx].sc
+ c := NewClient(sc, 16)
+ h.register <- c
+
+ time.Sleep(10 * time.Millisecond)
+ h.Broadcast([]byte("concurrent message"))
+ time.Sleep(10 * time.Millisecond)
+ h.unregister <- c
+ }(i)
+ }
+
+ wg.Wait()
+
+ // All clients should be gone
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 0
+ }, 2*time.Second, 10*time.Millisecond)
+}
+
+func TestHub_DoubleUnregister(t *testing.T) {
+ h := NewHub(10)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go h.Run(ctx)
+
+ _, serverConn, cleanup := newTestWSPair(t)
+ defer cleanup()
+
+ c := NewClient(serverConn, 16)
+ h.register <- c
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 1
+ }, time.Second, 5*time.Millisecond)
+
+ // Unregister twice — should not panic
+ h.unregister <- c
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 0
+ }, time.Second, 5*time.Millisecond)
+
+ // Second unregister should be harmless
+ h.unregister <- c
+ time.Sleep(50 * time.Millisecond)
+ assert.Equal(t, 0, h.ClientCount())
+}
diff --git a/backend/internal/utils/ws/parse_test.go b/backend/internal/utils/ws/parse_test.go
new file mode 100644
index 000000000..8a3208e79
--- /dev/null
+++ b/backend/internal/utils/ws/parse_test.go
@@ -0,0 +1,211 @@
+package ws
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNormalizeContainerLine(t *testing.T) {
+ tests := []struct {
+ name string
+ raw string
+ expectedLevel string
+ expectedMsg string
+ expectedTS string
+ }{
+ {
+ name: "plain stdout line",
+ raw: "hello world",
+ expectedLevel: "stdout",
+ expectedMsg: "hello world",
+ expectedTS: "",
+ },
+ {
+ name: "stderr prefix",
+ raw: "[STDERR] error occurred",
+ expectedLevel: "stderr",
+ expectedMsg: "error occurred",
+ expectedTS: "",
+ },
+ {
+ name: "stderr colon prefix",
+ raw: "stderr:error message",
+ expectedLevel: "stderr",
+ expectedMsg: "error message",
+ expectedTS: "",
+ },
+ {
+ name: "stdout colon prefix",
+ raw: "stdout:normal message",
+ expectedLevel: "stdout",
+ expectedMsg: "normal message",
+ expectedTS: "",
+ },
+ {
+ name: "trailing newline stripped",
+ raw: "message with newline\n",
+ expectedLevel: "stdout",
+ expectedMsg: "message with newline",
+ expectedTS: "",
+ },
+ {
+ name: "trailing carriage return and newline",
+ raw: "message with crlf\r\n",
+ expectedLevel: "stdout",
+ expectedMsg: "message with crlf",
+ expectedTS: "",
+ },
+ {
+ name: "multiple trailing newlines",
+ raw: "multiple newlines\n\n\n",
+ expectedLevel: "stdout",
+ expectedMsg: "multiple newlines",
+ expectedTS: "",
+ },
+ {
+ name: "Docker timestamp RFC3339Nano",
+ raw: "2024-01-15T10:30:45.123456789Z hello from container",
+ expectedLevel: "stdout",
+ expectedMsg: "hello from container",
+ expectedTS: "2024-01-15T10:30:45.123456789Z",
+ },
+ {
+ name: "Docker timestamp RFC3339",
+ raw: "2024-01-15T10:30:45Z hello from container",
+ expectedLevel: "stdout",
+ expectedMsg: "hello from container",
+ expectedTS: "2024-01-15T10:30:45Z",
+ },
+ {
+ name: "stderr with Docker timestamp",
+ raw: "[STDERR] 2024-06-01T12:00:00.000Z critical error",
+ expectedLevel: "stderr",
+ expectedMsg: "critical error",
+ expectedTS: "2024-06-01T12:00:00Z",
+ },
+ {
+ name: "empty line",
+ raw: "",
+ expectedLevel: "stdout",
+ expectedMsg: "",
+ expectedTS: "",
+ },
+ {
+ name: "only whitespace",
+ raw: " \n\r\n",
+ expectedLevel: "stdout",
+ expectedMsg: "",
+ expectedTS: "",
+ },
+ {
+ name: "short line not mistaken for timestamp",
+ raw: "short",
+ expectedLevel: "stdout",
+ expectedMsg: "short",
+ expectedTS: "",
+ },
+ {
+ name: "numeric start but not timestamp",
+ raw: "123 this is not a timestamp but starts with numbers and is long enough to check",
+ expectedLevel: "stdout",
+ expectedMsg: "123 this is not a timestamp but starts with numbers and is long enough to check",
+ expectedTS: "",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ level, msg, ts := NormalizeContainerLine(tt.raw)
+ assert.Equal(t, tt.expectedLevel, level, "level mismatch")
+ assert.Equal(t, tt.expectedMsg, msg, "message mismatch")
+ assert.Equal(t, tt.expectedTS, ts, "timestamp mismatch")
+ })
+ }
+}
+
+func TestNormalizeProjectLine(t *testing.T) {
+ tests := []struct {
+ name string
+ raw string
+ expectedLevel string
+ expectedService string
+ expectedMsg string
+ expectedTS string
+ }{
+ {
+ name: "service pipe pattern",
+ raw: "web | Starting server on port 8080",
+ expectedLevel: "stdout",
+ expectedService: "web",
+ expectedMsg: "Starting server on port 8080",
+ expectedTS: "",
+ },
+ {
+ name: "no service pattern",
+ raw: "plain log line without service",
+ expectedLevel: "stdout",
+ expectedService: "",
+ expectedMsg: "plain log line without service",
+ expectedTS: "",
+ },
+ {
+ name: "stderr with service",
+ raw: "[STDERR] database | Connection refused",
+ expectedLevel: "stderr",
+ expectedService: "database",
+ expectedMsg: "Connection refused",
+ expectedTS: "",
+ },
+ {
+ name: "service with timestamp",
+ raw: "2024-01-15T10:30:45.123Z api | Request received",
+ expectedLevel: "stdout",
+ expectedService: "api",
+ expectedMsg: "Request received",
+ expectedTS: "2024-01-15T10:30:45.123Z",
+ },
+ {
+ name: "pipe in message without service format",
+ raw: "this | has | multiple | pipes",
+ expectedLevel: "stdout",
+ expectedService: "this",
+ expectedMsg: "has | multiple | pipes",
+ expectedTS: "",
+ },
+ {
+ name: "service with whitespace padding",
+ raw: " redis | PING received",
+ expectedLevel: "stdout",
+ expectedService: "redis",
+ expectedMsg: "PING received",
+ expectedTS: "",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ level, service, msg, ts := NormalizeProjectLine(tt.raw)
+ assert.Equal(t, tt.expectedLevel, level, "level mismatch")
+ assert.Equal(t, tt.expectedService, service, "service mismatch")
+ assert.Equal(t, tt.expectedMsg, msg, "message mismatch")
+ assert.Equal(t, tt.expectedTS, ts, "timestamp mismatch")
+ })
+ }
+}
+
+func TestNowRFC3339(t *testing.T) {
+ before := time.Now().UTC()
+ result := NowRFC3339()
+ after := time.Now().UTC()
+
+ parsed, err := time.Parse(time.RFC3339Nano, result)
+ require.NoError(t, err, "NowRFC3339 should return a valid RFC3339Nano string")
+
+ assert.False(t, parsed.Before(before.Add(-time.Millisecond)),
+ "timestamp should not be before the call")
+ assert.False(t, parsed.After(after.Add(time.Millisecond)),
+ "timestamp should not be after the call")
+}
diff --git a/backend/internal/utils/ws/proxy_test.go b/backend/internal/utils/ws/proxy_test.go
new file mode 100644
index 000000000..a1d27b302
--- /dev/null
+++ b/backend/internal/utils/ws/proxy_test.go
@@ -0,0 +1,219 @@
+package ws
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/gorilla/websocket"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestProxyHTTP_BidirectionalMessages(t *testing.T) {
+ // 1. Create a "remote" WebSocket server that echoes messages
+ remoteServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ up := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ conn, err := up.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ defer conn.Close()
+
+ for {
+ mt, msg, err := conn.ReadMessage()
+ if err != nil {
+ return
+ }
+ // Echo with prefix
+ if err := conn.WriteMessage(mt, append([]byte("echo:"), msg...)); err != nil {
+ return
+ }
+ }
+ }))
+ defer remoteServer.Close()
+
+ remoteWS := "ws" + strings.TrimPrefix(remoteServer.URL, "http")
+
+ // 2. Create a "proxy" server that uses ProxyHTTP
+ proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = ProxyHTTP(w, r, remoteWS, nil)
+ }))
+ defer proxyServer.Close()
+
+ // 3. Connect a client to the proxy
+ proxyURL := "ws" + strings.TrimPrefix(proxyServer.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(proxyURL, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+ defer clientConn.Close()
+
+ // 4. Send messages and verify they get proxied and echoed
+ testMessages := []string{"hello", "world", "test123"}
+ for _, msg := range testMessages {
+ err := clientConn.WriteMessage(websocket.TextMessage, []byte(msg))
+ require.NoError(t, err)
+
+ _ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, received, err := clientConn.ReadMessage()
+ require.NoError(t, err)
+ assert.Equal(t, "echo:"+msg, string(received))
+ }
+}
+
+func TestProxyHTTP_RemoteClose(t *testing.T) {
+ // Remote server that closes immediately after upgrade
+ remoteServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ up := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ conn, err := up.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ // Close immediately
+ _ = conn.WriteMessage(websocket.CloseMessage,
+ websocket.FormatCloseMessage(websocket.CloseNormalClosure, "bye"))
+ conn.Close()
+ }))
+ defer remoteServer.Close()
+
+ remoteWS := "ws" + strings.TrimPrefix(remoteServer.URL, "http")
+
+ proxyDone := make(chan error, 1)
+ proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ proxyDone <- ProxyHTTP(w, r, remoteWS, nil)
+ }))
+ defer proxyServer.Close()
+
+ proxyURL := "ws" + strings.TrimPrefix(proxyServer.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(proxyURL, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+ defer clientConn.Close()
+
+ // The proxy should complete (not hang)
+ select {
+ case <-proxyDone:
+ // OK
+ case <-time.After(5 * time.Second):
+ t.Fatal("ProxyHTTP did not return after remote closed")
+ }
+}
+
+func TestProxyHTTP_InvalidRemoteURL(t *testing.T) {
+ proxyDone := make(chan error, 1)
+ proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ proxyDone <- ProxyHTTP(w, r, "ws://127.0.0.1:1", nil)
+ }))
+ defer proxyServer.Close()
+
+ proxyURL := "ws" + strings.TrimPrefix(proxyServer.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(proxyURL, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+ defer clientConn.Close()
+
+ select {
+ case err := <-proxyDone:
+ require.Error(t, err, "ProxyHTTP should return error when remote is unreachable")
+ case <-time.After(50 * time.Second):
+ t.Fatal("ProxyHTTP did not return after failed dial")
+ }
+}
+
+func TestProxyHTTP_BinaryMessages(t *testing.T) {
+ // Remote server that echoes binary messages
+ remoteServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ up := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ conn, err := up.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ defer conn.Close()
+
+ for {
+ mt, msg, err := conn.ReadMessage()
+ if err != nil {
+ return
+ }
+ if err := conn.WriteMessage(mt, msg); err != nil {
+ return
+ }
+ }
+ }))
+ defer remoteServer.Close()
+
+ remoteWS := "ws" + strings.TrimPrefix(remoteServer.URL, "http")
+
+ proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = ProxyHTTP(w, r, remoteWS, nil)
+ }))
+ defer proxyServer.Close()
+
+ proxyURL := "ws" + strings.TrimPrefix(proxyServer.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(proxyURL, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+ defer clientConn.Close()
+
+ binaryData := []byte{0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD}
+ err = clientConn.WriteMessage(websocket.BinaryMessage, binaryData)
+ require.NoError(t, err)
+
+ _ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ mt, received, err := clientConn.ReadMessage()
+ require.NoError(t, err)
+ assert.Equal(t, websocket.BinaryMessage, mt)
+ assert.Equal(t, binaryData, received)
+}
+
+func TestProxyHTTP_HeadersForwarded(t *testing.T) {
+ headersCh := make(chan http.Header, 1)
+ remoteServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ headersCh <- r.Header.Clone()
+ up := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ conn, err := up.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ conn.Close()
+ }))
+ defer remoteServer.Close()
+
+ remoteWS := "ws" + strings.TrimPrefix(remoteServer.URL, "http")
+
+ customHeaders := http.Header{
+ "X-Custom-Header": {"test-value"},
+ "Authorization": {"Bearer token123"},
+ }
+
+ proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = ProxyHTTP(w, r, remoteWS, customHeaders)
+ }))
+ defer proxyServer.Close()
+
+ proxyURL := "ws" + strings.TrimPrefix(proxyServer.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(proxyURL, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ resp.Body.Close()
+ }
+ defer clientConn.Close()
+
+ select {
+ case receivedHeaders := <-headersCh:
+ assert.Equal(t, "test-value", receivedHeaders.Get("X-Custom-Header"))
+ assert.Equal(t, "Bearer token123", receivedHeaders.Get("Authorization"))
+ case <-time.After(2 * time.Second):
+ t.Fatal("timed out waiting for headers")
+ }
+}
diff --git a/cli/internal/client/client.go b/cli/internal/client/client.go
index a6b135a6c..f83bfe1cd 100644
--- a/cli/internal/client/client.go
+++ b/cli/internal/client/client.go
@@ -69,13 +69,11 @@ func New(cfg *types.Config) (*Client, error) {
}
return &Client{
- baseURL: cfg.ServerURL,
- apiKey: cfg.APIKey,
- jwtToken: cfg.JWTToken,
- envID: envID,
- httpClient: &http.Client{
- Timeout: defaultTimeout,
- },
+ baseURL: cfg.ServerURL,
+ apiKey: cfg.APIKey,
+ jwtToken: cfg.JWTToken,
+ envID: envID,
+ httpClient: newHTTPClientInternal(),
}, nil
}
@@ -92,14 +90,26 @@ func NewUnauthenticated(cfg *types.Config) (*Client, error) {
}
return &Client{
- baseURL: cfg.ServerURL,
- envID: envID,
- httpClient: &http.Client{
- Timeout: defaultTimeout,
- },
+ baseURL: cfg.ServerURL,
+ envID: envID,
+ httpClient: newHTTPClientInternal(),
}, nil
}
+func newHTTPClientInternal() *http.Client {
+ transport, ok := http.DefaultTransport.(*http.Transport)
+ if ok {
+ return &http.Client{
+ Timeout: defaultTimeout,
+ Transport: transport.Clone(),
+ }
+ }
+
+ return &http.Client{
+ Timeout: defaultTimeout,
+ }
+}
+
// NewFromConfig loads the CLI configuration from disk and creates a new client.
// This is the recommended way to create a client in CLI commands.
// It returns an error if the configuration cannot be loaded or is invalid.
diff --git a/types/system/stats.go b/types/system/stats.go
new file mode 100644
index 000000000..0759e5ba3
--- /dev/null
+++ b/types/system/stats.go
@@ -0,0 +1,61 @@
+package system
+
+// GPUStats represents resource statistics for a single GPU.
+type GPUStats struct {
+ // Name is the GPU model or identifier.
+ //
+ // Required: true
+ Name string `json:"name"`
+ // Index is the zero-based GPU index.
+ //
+ // Required: true
+ Index int `json:"index"`
+ // MemoryUsed is the GPU memory currently used, in bytes.
+ //
+ // Required: true
+ MemoryUsed float64 `json:"memoryUsed"`
+ // MemoryTotal is the total GPU memory available, in bytes.
+ //
+ // Required: true
+ MemoryTotal float64 `json:"memoryTotal"`
+}
+
+// SystemStats represents system resource statistics for WebSocket streaming.
+type SystemStats struct {
+ // CPUUsage is the total CPU usage percentage.
+ //
+ // Required: true
+ CPUUsage float64 `json:"cpuUsage"`
+ // MemoryUsage is the used system memory, in bytes.
+ //
+ // Required: true
+ MemoryUsage uint64 `json:"memoryUsage"`
+ // MemoryTotal is the total system memory, in bytes.
+ //
+ // Required: true
+ MemoryTotal uint64 `json:"memoryTotal"`
+ // DiskUsage is the used disk space, in bytes.
+ DiskUsage uint64 `json:"diskUsage,omitempty"`
+ // DiskTotal is the total disk space, in bytes.
+ DiskTotal uint64 `json:"diskTotal,omitempty"`
+ // CPUCount is the number of CPUs available to the system.
+ //
+ // Required: true
+ CPUCount int `json:"cpuCount"`
+ // Architecture is the CPU architecture (e.g., amd64).
+ //
+ // Required: true
+ Architecture string `json:"architecture"`
+ // Platform is the operating system platform (e.g., linux).
+ //
+ // Required: true
+ Platform string `json:"platform"`
+ // Hostname is the system hostname.
+ Hostname string `json:"hostname,omitempty"`
+ // GPUCount is the number of GPUs detected.
+ //
+ // Required: true
+ GPUCount int `json:"gpuCount"`
+ // GPUs contains per-GPU resource statistics.
+ GPUs []GPUStats `json:"gpus,omitempty"`
+}
diff --git a/types/system/websocket.go b/types/system/websocket.go
new file mode 100644
index 000000000..ea41d414d
--- /dev/null
+++ b/types/system/websocket.go
@@ -0,0 +1,53 @@
+package system
+
+import "time"
+
+// WebSocket connection kind constants.
+const (
+ WSKindProjectLogs = "project_logs"
+ WSKindContainerLogs = "container_logs"
+ WSKindContainerStats = "container_stats"
+ WSKindContainerExec = "container_exec"
+ WSKindSystemStats = "system_stats"
+)
+
+// WebSocketConnectionInfo describes a single active WebSocket connection.
+type WebSocketConnectionInfo struct {
+ // ID is the unique identifier for the connection.
+ //
+ // Required: true
+ ID string `json:"id"`
+ // Kind is the type of WebSocket connection (e.g. project_logs, container_stats).
+ //
+ // Required: true
+ Kind string `json:"kind"`
+ // EnvID is the environment the connection belongs to.
+ EnvID string `json:"envId,omitempty"`
+ // ResourceID is the ID of the resource being streamed (container, project, etc.).
+ ResourceID string `json:"resourceId,omitempty"`
+ // ClientIP is the remote address of the client.
+ ClientIP string `json:"clientIp,omitempty"`
+ // UserID is the authenticated user who opened the connection.
+ UserID string `json:"userId,omitempty"`
+ // UserAgent is the HTTP User-Agent header from the client.
+ UserAgent string `json:"userAgent,omitempty"`
+ // StartedAt is when the connection was established.
+ //
+ // Required: true
+ StartedAt time.Time `json:"startedAt"`
+}
+
+// WebSocketMetricsSnapshot is a point-in-time copy of active WebSocket
+// connection counts, broken down by kind.
+type WebSocketMetricsSnapshot struct {
+ // ProjectLogsActive is the number of active project-log streams.
+ ProjectLogsActive int64 `json:"projectLogsActive"`
+ // ContainerLogsActive is the number of active container-log streams.
+ ContainerLogsActive int64 `json:"containerLogsActive"`
+ // ContainerStats is the number of active container-stats streams.
+ ContainerStats int64 `json:"containerStats"`
+ // ContainerExec is the number of active container-exec sessions.
+ ContainerExec int64 `json:"containerExec"`
+ // SystemStats is the number of active system-stats streams.
+ SystemStats int64 `json:"systemStats"`
+}