Skip to content

Modern Docker Management, Designed for Everyone

perf: share websocket streams, cache settings reads, and parallelize auto-heal inspections (#1957)

Kyle Mendellf6b5b4c1a06c
diff --git a/backend/internal/api/ws_handler.go b/backend/internal/api/ws_handler.go
index 0df4b0c45..28c871053 100644
--- a/backend/internal/api/ws_handler.go
+++ b/backend/internal/api/ws_handler.go
@@ -37,6 +37,7 @@ import (
const (
gpuCacheDuration = 30 * time.Second
+ cgroupCacheTTL = 30 * time.Second
)
// amdGPUSysfsPath is the base path for AMD GPU sysfs entries
@@ -156,11 +157,34 @@ type WebSocketHandler struct {
wsUpgrader websocket.Upgrader
wsMetrics *WebSocketMetrics
activeConnections sync.Map
+ logStreamsMu sync.Mutex
+ logStreams map[string]*wsLogStream
cpuCache struct {
sync.RWMutex
value float64
timestamp time.Time
}
+ systemStaticInfo struct {
+ once sync.Once
+ cpuCount int
+ hostname string
+ }
+ systemStatsSampler struct {
+ stateMu sync.RWMutex
+ latest systemtypes.SystemStats
+ timestamp time.Time
+ lifecycleMu sync.Mutex
+ clients int
+ cancel context.CancelFunc
+ ready chan struct{}
+ running bool
+ }
+ cgroupCache struct {
+ sync.RWMutex
+ value *docker.CgroupLimits
+ detected bool
+ timestamp time.Time
+ }
diskUsagePathCache struct {
sync.RWMutex
value string
@@ -177,13 +201,22 @@ type WebSocketHandler struct {
detectionMutex sync.Mutex
gpuMonitoringEnabled bool
gpuType string
+ projectLogStreamer func(ctx context.Context, projectID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error
+ containerLogStreamer func(ctx context.Context, containerID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error
+ systemStatsCollector func(ctx context.Context) systemtypes.SystemStats
+ cpuUsageReader func(interval time.Duration) (float64, bool)
+ cgroupLimitsDetector func() (*docker.CgroupLimits, error)
}
type wsLogStream struct {
- hub *ws.Hub
- cancel context.CancelFunc
- format string
- seq atomic.Uint64
+ hub *ws.Hub
+ cancel context.CancelFunc
+ firstSubscriber chan struct{}
+ format string
+ key string
+ refs int
+ done bool
+ seq atomic.Uint64
}
func getContextUserIDInternal(c *gin.Context) string {
@@ -206,6 +239,98 @@ func buildWSConnectionInfoInternal(c *gin.Context, kind, resourceID string) syst
}
}
+func buildLogStreamKeyInternal(envID, kind, resourceID, format string, batched, follow bool, tail, since string, timestamps bool) string {
+ return strings.Join([]string{
+ envID,
+ kind,
+ resourceID,
+ format,
+ strconv.FormatBool(batched),
+ strconv.FormatBool(follow),
+ tail,
+ since,
+ strconv.FormatBool(timestamps),
+ }, "|")
+}
+
+func (h *WebSocketHandler) streamProjectLogsInternal(ctx context.Context, projectID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {
+ if h.projectLogStreamer != nil {
+ return h.projectLogStreamer(ctx, projectID, logsChan, follow, tail, since, timestamps)
+ }
+ return h.projectService.StreamProjectLogs(ctx, projectID, logsChan, follow, tail, since, timestamps)
+}
+
+func (h *WebSocketHandler) streamContainerLogsInternal(ctx context.Context, containerID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {
+ if h.containerLogStreamer != nil {
+ return h.containerLogStreamer(ctx, containerID, logsChan, follow, tail, since, timestamps)
+ }
+ return h.containerService.StreamLogs(ctx, containerID, logsChan, follow, tail, since, timestamps)
+}
+
+func (h *WebSocketHandler) getOrCreateLogStreamInternal(key string, create func(onEmpty func(*wsLogStream)) *wsLogStream) *wsLogStream {
+ h.logStreamsMu.Lock()
+ defer h.logStreamsMu.Unlock()
+
+ if stream, ok := h.logStreams[key]; ok {
+ if !stream.done {
+ stream.refs++
+ return stream
+ }
+ }
+
+ stream := create(func(stream *wsLogStream) {
+ h.markLogStreamDoneInternal(key, stream)
+ })
+ stream.key = key
+ stream.refs = 1
+ h.logStreams[key] = stream
+ return stream
+}
+
+func takeLogStreamCancelInternal(stream *wsLogStream) context.CancelFunc {
+ cancel := stream.cancel
+ stream.cancel = nil
+ return cancel
+}
+
+func (h *WebSocketHandler) releaseLogStreamInternal(key string, stream *wsLogStream) {
+ var cancel context.CancelFunc
+
+ h.logStreamsMu.Lock()
+ if stream.refs > 0 {
+ stream.refs--
+ }
+ if stream.refs == 0 {
+ if current, ok := h.logStreams[key]; ok && current == stream {
+ delete(h.logStreams, key)
+ }
+ cancel = takeLogStreamCancelInternal(stream)
+ }
+ h.logStreamsMu.Unlock()
+
+ if cancel != nil {
+ cancel()
+ }
+}
+
+func (h *WebSocketHandler) markLogStreamDoneInternal(key string, stream *wsLogStream) {
+ var cancel context.CancelFunc
+
+ h.logStreamsMu.Lock()
+ stream.done = true
+ if stream.refs == 0 {
+ if current, ok := h.logStreams[key]; ok && current == stream {
+ delete(h.logStreams, key)
+ }
+ cancel = takeLogStreamCancelInternal(stream)
+ }
+ h.logStreamsMu.Unlock()
+
+ if cancel != nil {
+ cancel()
+ }
+}
+
func NewWebSocketHandler(
group *gin.RouterGroup,
projectService *services.ProjectService,
@@ -219,6 +344,7 @@ func NewWebSocketHandler(
containerService: containerService,
systemService: systemService,
wsMetrics: defaultWebSocketMetrics,
+ logStreams: make(map[string]*wsLogStream),
gpuMonitoringEnabled: cfg.GPUMonitoringEnabled,
gpuType: cfg.GPUType,
wsUpgrader: websocket.Upgrader{
@@ -228,7 +354,6 @@ func NewWebSocketHandler(
EnableCompression: true,
},
}
-
wsGroup := group.Group("/environments/:id/ws")
wsGroup.Use(authMiddleware.WithAdminNotRequired().Add())
{
@@ -283,101 +408,162 @@ func (h *WebSocketHandler) ProjectLogs(c *gin.Context) {
return
}
- connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindProjectLogs, projectID))
- hub := h.startProjectLogHub(projectID, format, batched, follow, tail, since, timestamps, func() {
- h.wsMetrics.UnregisterConnection(connID)
+ streamKey := buildLogStreamKeyInternal(c.Param("id"), systemtypes.WSKindProjectLogs, projectID, format, batched, follow, tail, since, timestamps)
+ stream := h.getOrCreateLogStreamInternal(streamKey, func(onEmpty func(*wsLogStream)) *wsLogStream {
+ return h.startProjectLogHub(streamKey, projectID, format, batched, follow, tail, since, timestamps, onEmpty)
})
+ connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindProjectLogs, projectID))
// WebSocket connections use context.Background() because they are long-lived and should not
// be tied to the HTTP request context. Cleanup is handled via the hub's OnEmpty callback
// which triggers when all clients disconnect.
- ws.ServeClient(context.Background(), hub, conn)
+ ws.ServeClientWithOnRemove(context.Background(), stream.hub, conn, func() {
+ h.wsMetrics.UnregisterConnection(connID)
+ h.releaseLogStreamInternal(streamKey, stream)
+ })
}
-func (h *WebSocketHandler) startProjectLogHub(projectID, format string, batched, follow bool, tail, since string, timestamps bool, onEmptyHook func()) *ws.Hub {
+func newWSLogStreamInternal(key, format string) (*wsLogStream, context.Context) {
ls := &wsLogStream{
- hub: ws.NewHub(1024),
- format: format,
+ hub: ws.NewHub(1024),
+ firstSubscriber: make(chan struct{}),
+ format: format,
+ key: key,
}
+ ls.hub.SetOnFirstClient(func() {
+ close(ls.firstSubscriber)
+ })
ctx, cancel := context.WithCancel(context.Background()) //nolint:gosec // cancel is intentionally retained and invoked by the hub OnEmpty callback.
ls.cancel = cancel
- ls.hub.SetOnEmpty(func() {
- if onEmptyHook != nil {
- onEmptyHook()
- }
- slog.Debug("client disconnected, cleaning up project log hub", "projectID", projectID)
- cancel()
- })
-
go ls.hub.Run(ctx)
+ return ls, ctx
+}
+
+func (h *WebSocketHandler) startProjectLogSourceInternal(ctx context.Context, key, projectID, format string, follow bool, tail, since string, timestamps bool, ls *wsLogStream) <-chan string {
lines := make(chan string, 256)
- go func(ctx context.Context) {
+
+ go func() {
defer close(lines)
- if err := h.projectService.StreamProjectLogs(ctx, projectID, lines, follow, tail, since, timestamps); err != nil {
+ if !waitForLogStreamSubscriberInternal(ctx, ls.firstSubscriber) {
+ return
+ }
+
+ if err := h.streamProjectLogsInternal(ctx, projectID, lines, follow, tail, since, timestamps); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return
}
- slog.Warn("project log stream failed", "projectID", projectID, "error", err)
+ h.broadcastProjectLogStreamErrorInternal(projectID, format, err, ls)
+ h.markLogStreamDoneInternal(key, ls)
+ return
+ }
- if format == "json" {
- msg := ws.LogMessage{
- Seq: ls.seq.Add(1),
- Level: "error",
- Message: "Failed to stream project logs: " + err.Error(),
- Service: "arcane",
- Timestamp: ws.NowRFC3339(),
- }
- if b, merr := json.Marshal(msg); merr == nil {
- ls.hub.Broadcast(b)
- }
- return
- }
+ if ctx.Err() == nil {
+ h.markLogStreamDoneInternal(key, ls)
+ }
+ }()
- ls.hub.Broadcast([]byte("Failed to stream project logs: " + err.Error()))
+ return lines
+}
+
+func waitForLogStreamSubscriberInternal(ctx context.Context, firstSubscriber <-chan struct{}) bool {
+ for {
+ select {
+ case <-ctx.Done():
+ return false
+ case <-firstSubscriber:
+ return true
}
- }(ctx)
+ }
+}
+
+func (h *WebSocketHandler) broadcastProjectLogStreamErrorInternal(projectID, format string, err error, ls *wsLogStream) {
+ slog.Warn("project log stream failed", "projectID", projectID, "error", err)
if format == "json" {
- msgs := make(chan ws.LogMessage, 256)
- go func() {
- defer close(msgs)
- for line := range lines {
- level, service, msg, ts := ws.NormalizeProjectLine(line)
- seq := ls.seq.Add(1)
- timestamp := ts
- if timestamp == "" {
- timestamp = ws.NowRFC3339()
- }
- msgs <- ws.LogMessage{
- Seq: seq,
- Level: level,
- Message: msg,
- Service: service,
- Timestamp: timestamp,
- }
- }
- }()
- if batched {
- go ws.ForwardLogJSONBatched(ctx, ls.hub, msgs, 50, 400*time.Millisecond)
- } else {
- go ws.ForwardLogJSON(ctx, ls.hub, msgs)
+ msg := ws.LogMessage{
+ Seq: ls.seq.Add(1),
+ Level: "error",
+ Message: "Failed to stream project logs: " + err.Error(),
+ Service: "arcane",
+ Timestamp: ws.NowRFC3339(),
}
- } else {
- cleanChan := make(chan string, 256)
- go func() {
- defer close(cleanChan)
- for line := range lines {
- _, _, msg, _ := ws.NormalizeProjectLine(line)
- cleanChan <- msg
+ if b, marshalErr := json.Marshal(msg); marshalErr == nil {
+ ls.hub.Broadcast(b)
+ }
+ return
+ }
+
+ ls.hub.Broadcast([]byte("Failed to stream project logs: " + err.Error()))
+}
+
+func startProjectLogForwardersInternal(ctx context.Context, format string, batched bool, lines <-chan string, ls *wsLogStream) {
+ if format == "json" {
+ startProjectJSONForwarderInternal(ctx, batched, lines, ls)
+ return
+ }
+
+ startProjectTextForwarderInternal(ctx, lines, ls)
+}
+
+func startProjectJSONForwarderInternal(ctx context.Context, batched bool, lines <-chan string, ls *wsLogStream) {
+ msgs := make(chan ws.LogMessage, 256)
+ go func() {
+ defer close(msgs)
+ for line := range lines {
+ level, service, msg, ts := ws.NormalizeProjectLine(line)
+ seq := ls.seq.Add(1)
+ timestamp := ts
+ if timestamp == "" {
+ timestamp = ws.NowRFC3339()
}
- }()
- go ws.ForwardLines(ctx, ls.hub, cleanChan)
+ msgs <- ws.LogMessage{
+ Seq: seq,
+ Level: level,
+ Message: msg,
+ Service: service,
+ Timestamp: timestamp,
+ }
+ }
+ }()
+
+ if batched {
+ go ws.ForwardLogJSONBatched(ctx, ls.hub, msgs, 50, 400*time.Millisecond)
+ return
}
- return ls.hub
+ go ws.ForwardLogJSON(ctx, ls.hub, msgs)
+}
+
+func startProjectTextForwarderInternal(ctx context.Context, lines <-chan string, ls *wsLogStream) {
+ cleanChan := make(chan string, 256)
+ go func() {
+ defer close(cleanChan)
+ for line := range lines {
+ _, _, msg, _ := ws.NormalizeProjectLine(line)
+ cleanChan <- msg
+ }
+ }()
+
+ go ws.ForwardLines(ctx, ls.hub, cleanChan)
+}
+
+func (h *WebSocketHandler) startProjectLogHub(key, projectID, format string, batched, follow bool, tail, since string, timestamps bool, onEmptyHook func(*wsLogStream)) *wsLogStream {
+ ls, ctx := newWSLogStreamInternal(key, format)
+
+ ls.hub.SetOnEmpty(func() {
+ if onEmptyHook != nil {
+ onEmptyHook(ls)
+ }
+ slog.Debug("client disconnected, cleaning up project log hub", "projectID", projectID)
+ })
+
+ lines := h.startProjectLogSourceInternal(ctx, key, projectID, format, follow, tail, since, timestamps, ls)
+ startProjectLogForwardersInternal(ctx, format, batched, lines, ls)
+
+ return ls
}
// ============================================================================
@@ -423,45 +609,49 @@ func (h *WebSocketHandler) ContainerLogs(c *gin.Context) {
return
}
- connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindContainerLogs, containerID))
- hub := h.startContainerLogHub(containerID, format, batched, follow, tail, since, timestamps, func() {
- h.wsMetrics.UnregisterConnection(connID)
+ streamKey := buildLogStreamKeyInternal(c.Param("id"), systemtypes.WSKindContainerLogs, containerID, format, batched, follow, tail, since, timestamps)
+ stream := h.getOrCreateLogStreamInternal(streamKey, func(onEmpty func(*wsLogStream)) *wsLogStream {
+ return h.startContainerLogHub(streamKey, containerID, format, batched, follow, tail, since, timestamps, onEmpty)
})
+ connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindContainerLogs, containerID))
// WebSocket connections use context.Background() because they are long-lived and should not
// be tied to the HTTP request context. Cleanup is handled via the hub's OnEmpty callback
// which triggers when all clients disconnect.
- ws.ServeClient(context.Background(), hub, conn)
+ ws.ServeClientWithOnRemove(context.Background(), stream.hub, conn, func() {
+ h.wsMetrics.UnregisterConnection(connID)
+ h.releaseLogStreamInternal(streamKey, stream)
+ })
}
-func (h *WebSocketHandler) startContainerLogHub(containerID, format string, batched, follow bool, tail, since string, timestamps bool, onEmptyHook func()) *ws.Hub {
- ls := &wsLogStream{
- hub: ws.NewHub(1024),
- format: format,
- }
-
- ctx, cancel := context.WithCancel(context.Background()) //nolint:gosec // cancel is intentionally retained and invoked by the hub OnEmpty callback.
- ls.cancel = cancel
+func (h *WebSocketHandler) startContainerLogHub(key, containerID, format string, batched, follow bool, tail, since string, timestamps bool, onEmptyHook func(*wsLogStream)) *wsLogStream {
+ ls, ctx := newWSLogStreamInternal(key, format)
ls.hub.SetOnEmpty(func() {
if onEmptyHook != nil {
- onEmptyHook()
+ onEmptyHook(ls)
}
slog.Debug("client disconnected, cleaning up container log hub", "containerID", containerID)
- cancel()
})
- go ls.hub.Run(ctx)
-
lines := make(chan string, 256)
go func(ctx context.Context) {
defer close(lines)
- if err := h.containerService.StreamLogs(ctx, containerID, lines, follow, tail, since, timestamps); err != nil {
+ if !waitForLogStreamSubscriberInternal(ctx, ls.firstSubscriber) {
+ return
+ }
+
+ if err := h.streamContainerLogsInternal(ctx, containerID, lines, follow, tail, since, timestamps); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return
}
- slog.Warn("container log stream failed", "containerID", containerID, "error", err)
- broadcastContainerLogStreamErrorInternal(ls, err)
+ h.broadcastContainerLogStreamErrorInternal(containerID, format, err, ls)
+ h.markLogStreamDoneInternal(key, ls)
+ return
+ }
+
+ if ctx.Err() == nil {
+ h.markLogStreamDoneInternal(key, ls)
}
}(ctx)
@@ -493,29 +683,27 @@ func (h *WebSocketHandler) startContainerLogHub(containerID, format string, batc
go ws.ForwardLines(ctx, ls.hub, lines)
}
- return ls.hub
+ return ls
}
-func broadcastContainerLogStreamErrorInternal(stream *wsLogStream, err error) {
- if stream == nil || stream.hub == nil || err == nil {
- return
- }
+func (h *WebSocketHandler) broadcastContainerLogStreamErrorInternal(containerID, format string, err error, ls *wsLogStream) {
+ slog.Warn("container log stream failed", "containerID", containerID, "error", err)
- message := "Failed to stream container logs: " + err.Error()
- if stream.format == "json" {
+ if format == "json" {
msg := ws.LogMessage{
- Seq: stream.seq.Add(1),
+ Seq: ls.seq.Add(1),
Level: "error",
- Message: message,
+ Message: "Failed to stream container logs: " + err.Error(),
+ Service: "arcane",
Timestamp: ws.NowRFC3339(),
}
if b, marshalErr := json.Marshal(msg); marshalErr == nil {
- stream.hub.Broadcast(b)
+ ls.hub.Broadcast(b)
}
return
}
- stream.hub.Broadcast([]byte(message))
+ ls.hub.Broadcast([]byte("Failed to stream container logs: " + err.Error()))
}
// ContainerStats streams container stats over WebSocket.
@@ -740,23 +928,115 @@ func (h *WebSocketHandler) releaseRateLimit(clientIP string, count *int32) {
}
}
-// startCPUSampler starts a background goroutine that samples CPU usage.
-func (h *WebSocketHandler) startCPUSampler(ctx context.Context, ticker *time.Ticker) {
- go func(ctx context.Context) {
- for {
- select {
- case <-ctx.Done():
- return
- case <-ticker.C:
- if vals, err := cpu.Percent(0, false); err == nil && len(vals) > 0 {
- h.cpuCache.Lock()
- h.cpuCache.value = vals[0]
- h.cpuCache.timestamp = time.Now()
- h.cpuCache.Unlock()
- }
- }
+func (h *WebSocketHandler) acquireSystemStatsSamplerInternal(ctx context.Context) bool {
+ h.systemStatsSampler.lifecycleMu.Lock()
+
+ h.systemStatsSampler.clients++
+ if h.systemStatsSampler.running {
+ ready := h.systemStatsSampler.ready
+ h.systemStatsSampler.lifecycleMu.Unlock()
+ return waitForSystemStatsSamplerReadyInternal(ctx, ready)
+ }
+
+ samplerCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) //nolint:gosec // cancel is intentionally retained in sampler state and invoked when the last subscriber disconnects.
+ ready := make(chan struct{})
+ h.systemStatsSampler.cancel = cancel
+ h.systemStatsSampler.ready = ready
+ h.systemStatsSampler.running = true
+ h.systemStatsSampler.lifecycleMu.Unlock()
+
+ go func() {
+ closeReady := sync.OnceFunc(func() {
+ close(ready)
+ })
+ if !h.initializeCPUCacheCtx(samplerCtx) {
+ closeReady()
+ return
}
- }(ctx)
+ if samplerCtx.Err() != nil {
+ closeReady()
+ return
+ }
+
+ h.storeSystemStatsSnapshotInternal(h.collectSystemStatsSnapshotInternal(samplerCtx))
+ closeReady()
+ if samplerCtx.Err() != nil {
+ return
+ }
+
+ h.runSystemStatsSamplerInternal(samplerCtx)
+ }()
+
+ return waitForSystemStatsSamplerReadyInternal(ctx, ready)
+}
+
+func waitForSystemStatsSamplerReadyInternal(ctx context.Context, ready <-chan struct{}) bool {
+ if ready == nil {
+ return true
+ }
+
+ select {
+ case <-ctx.Done():
+ return false
+ case <-ready:
+ return ctx.Err() == nil
+ }
+}
+
+func (h *WebSocketHandler) releaseSystemStatsSamplerInternal() {
+ var cancel context.CancelFunc
+
+ h.systemStatsSampler.lifecycleMu.Lock()
+ if h.systemStatsSampler.clients > 0 {
+ h.systemStatsSampler.clients--
+ }
+ if h.systemStatsSampler.clients == 0 && h.systemStatsSampler.running {
+ cancel = h.systemStatsSampler.cancel
+ h.systemStatsSampler.cancel = nil
+ h.systemStatsSampler.ready = nil
+ h.systemStatsSampler.running = false
+ }
+ h.systemStatsSampler.lifecycleMu.Unlock()
+
+ if cancel != nil {
+ cancel()
+ }
+}
+
+func (h *WebSocketHandler) runSystemStatsSamplerInternal(ctx context.Context) {
+ ticker := time.NewTicker(time.Second)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ h.updateCPUCacheInternal(0)
+ h.storeSystemStatsSnapshotInternal(h.collectSystemStatsSnapshotInternal(ctx))
+ }
+ }
+}
+
+func (h *WebSocketHandler) storeSystemStatsSnapshotInternal(stats systemtypes.SystemStats) {
+ h.systemStatsSampler.stateMu.Lock()
+ h.systemStatsSampler.latest = stats
+ h.systemStatsSampler.timestamp = time.Now()
+ h.systemStatsSampler.stateMu.Unlock()
+}
+
+func (h *WebSocketHandler) latestSystemStatsSnapshotInternal() systemtypes.SystemStats {
+ h.systemStatsSampler.stateMu.RLock()
+ stats := h.systemStatsSampler.latest
+ h.systemStatsSampler.stateMu.RUnlock()
+ return stats
+}
+
+func (h *WebSocketHandler) collectSystemStatsSnapshotInternal(ctx context.Context) systemtypes.SystemStats {
+ if h.systemStatsCollector != nil {
+ return h.systemStatsCollector(ctx)
+ }
+ return h.collectSystemStats(ctx)
}
// collectSystemStats gathers all system statistics.
@@ -789,11 +1069,26 @@ func (h *WebSocketHandler) collectSystemStats(ctx context.Context) systemtypes.S
// getCPUCount returns the number of CPUs.
func (h *WebSocketHandler) getCPUCount() int {
- cpuCount, err := cpu.Counts(true)
- if err != nil {
- return runtime.NumCPU()
- }
- return cpuCount
+ h.initSystemStaticInfoInternal()
+ return h.systemStaticInfo.cpuCount
+}
+
+func (h *WebSocketHandler) initSystemStaticInfoInternal() {
+ h.systemStaticInfo.once.Do(func() {
+ cpuCount, err := cpu.Counts(true)
+ if err != nil {
+ cpuCount = runtime.NumCPU()
+ }
+
+ hostInfo, _ := host.Info()
+ hostname := ""
+ if hostInfo != nil {
+ hostname = hostInfo.Hostname
+ }
+
+ h.systemStaticInfo.cpuCount = cpuCount
+ h.systemStaticInfo.hostname = hostname
+ })
}
// getMemoryInfo returns memory usage and total.
@@ -807,8 +1102,8 @@ func (h *WebSocketHandler) getMemoryInfo() (uint64, uint64) {
// applyCgroupLimits applies cgroup limits when running in a container.
func (h *WebSocketHandler) applyCgroupLimits(cpuCount int, memUsed, memTotal uint64) (int, uint64, uint64) {
- cgroupLimits, err := docker.DetectCgroupLimits()
- if err != nil {
+ cgroupLimits := h.getCachedCgroupLimitsInternal()
+ if cgroupLimits == nil {
return cpuCount, memUsed, memTotal
}
@@ -844,11 +1139,8 @@ func (h *WebSocketHandler) getDiskInfo(ctx context.Context) (uint64, uint64) {
// getHostname returns the system hostname.
func (h *WebSocketHandler) getHostname() string {
- hostInfo, _ := host.Info()
- if hostInfo == nil {
- return ""
- }
- return hostInfo.Hostname
+ h.initSystemStaticInfoInternal()
+ return h.systemStaticInfo.hostname
}
// getGPUInfo returns GPU statistics if monitoring is enabled.
@@ -863,16 +1155,102 @@ func (h *WebSocketHandler) getGPUInfo(ctx context.Context) ([]systemtypes.GPUSta
return gpuData, len(gpuData)
}
-// initializeCPUCache performs initial CPU sampling.
-func (h *WebSocketHandler) initializeCPUCache() {
- if vals, err := cpu.Percent(time.Second, false); err == nil && len(vals) > 0 {
- h.cpuCache.Lock()
- h.cpuCache.value = vals[0]
- h.cpuCache.timestamp = time.Now()
- h.cpuCache.Unlock()
+// initializeCPUCacheCtx performs initial CPU sampling and returns early if the sampler is canceled.
+func (h *WebSocketHandler) initializeCPUCacheCtx(ctx context.Context) bool {
+ result := make(chan float64, 1)
+
+ go func() {
+ if val, ok := h.readCPUUsageInternal(time.Second); ok {
+ result <- val
+ }
+ close(result)
+ }()
+
+ select {
+ case <-ctx.Done():
+ return false
+ case val, ok := <-result:
+ if !ok || ctx.Err() != nil {
+ return false
+ }
+ h.storeCPUCacheValueInternal(val)
+ return true
+ }
+}
+
+func (h *WebSocketHandler) updateCPUCacheInternal(interval time.Duration) {
+ if val, ok := h.readCPUUsageInternal(interval); ok {
+ h.storeCPUCacheValueInternal(val)
}
}
+func (h *WebSocketHandler) readCPUUsageInternal(interval time.Duration) (float64, bool) {
+ if h.cpuUsageReader != nil {
+ return h.cpuUsageReader(interval)
+ }
+
+ return defaultReadCPUUsageInternal(interval)
+}
+
+var defaultReadCPUUsageInternal = func(interval time.Duration) (float64, bool) {
+ if vals, err := cpu.Percent(interval, false); err == nil && len(vals) > 0 {
+ return vals[0], true
+ }
+
+ return 0, false
+}
+
+func (h *WebSocketHandler) storeCPUCacheValueInternal(value float64) {
+ h.cpuCache.Lock()
+ h.cpuCache.value = value
+ h.cpuCache.timestamp = time.Now()
+ h.cpuCache.Unlock()
+}
+
+func (h *WebSocketHandler) getCachedCgroupLimitsInternal() *docker.CgroupLimits {
+ h.cgroupCache.RLock()
+ value := h.cgroupCache.value
+ detected := h.cgroupCache.detected
+ fresh := time.Since(h.cgroupCache.timestamp) < cgroupCacheTTL
+ h.cgroupCache.RUnlock()
+ if fresh {
+ if detected {
+ return value
+ }
+ return nil
+ }
+
+ h.cgroupCache.Lock()
+ defer h.cgroupCache.Unlock()
+
+ if time.Since(h.cgroupCache.timestamp) < cgroupCacheTTL {
+ if h.cgroupCache.detected {
+ return h.cgroupCache.value
+ }
+ return nil
+ }
+
+ detect := h.cgroupLimitsDetector
+ if detect == nil {
+ detect = docker.DetectCgroupLimits
+ }
+
+ limits, err := detect()
+ h.cgroupCache.timestamp = time.Now()
+ if err != nil {
+ h.cgroupCache.value = nil
+ h.cgroupCache.detected = false
+ } else {
+ h.cgroupCache.value = limits
+ h.cgroupCache.detected = true
+ }
+ if !h.cgroupCache.detected {
+ return nil
+ }
+
+ return h.cgroupCache.value
+}
+
// SystemStats streams system stats over WebSocket.
//
// @Summary Get system stats via WebSocket
@@ -924,24 +1302,22 @@ func (h *WebSocketHandler) SystemStats(c *gin.Context) {
pingTicker := time.NewTicker(statsPingPeriod)
defer pingTicker.Stop()
- cpuUpdateTicker := time.NewTicker(1 * time.Second)
- defer cpuUpdateTicker.Stop()
-
ctx, cancel := context.WithCancel(c.Request.Context())
defer cancel()
+ if !h.acquireSystemStatsSamplerInternal(ctx) {
+ h.releaseSystemStatsSamplerInternal()
+ return
+ }
+ defer h.releaseSystemStatsSamplerInternal()
go h.readSystemStatsPumpInternal(ctx, cancel, conn)
- h.startCPUSampler(ctx, cpuUpdateTicker)
-
send := func() error {
- stats := h.collectSystemStats(ctx)
+ stats := h.latestSystemStatsSnapshotInternal()
_ = conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
return conn.WriteJSON(stats)
}
- h.initializeCPUCache()
-
if err := send(); err != nil {
return
}
diff --git a/backend/internal/api/ws_handler_test.go b/backend/internal/api/ws_handler_test.go
index a7822800f..5519f2240 100644
--- a/backend/internal/api/ws_handler_test.go
+++ b/backend/internal/api/ws_handler_test.go
@@ -1,15 +1,21 @@
package api
import (
+ "context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
+ "sync"
+ "sync/atomic"
"testing"
"time"
+ docker "github.com/getarcaneapp/arcane/backend/pkg/dockerutil"
ws "github.com/getarcaneapp/arcane/backend/pkg/libarcane/ws"
+ systemtypes "github.com/getarcaneapp/arcane/types/system"
+ "github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/stretchr/testify/require"
)
@@ -29,7 +35,8 @@ func TestBroadcastContainerLogStreamErrorInternal_JSON(t *testing.T) {
format: "json",
}
- broadcastContainerLogStreamErrorInternal(stream, errors.New("boom"))
+ handler := newTestWebSocketHandler()
+ handler.broadcastContainerLogStreamErrorInternal("container-1", "json", errors.New("boom"), stream)
_ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
_, payload, err := clientConn.ReadMessage()
@@ -58,7 +65,8 @@ func TestBroadcastContainerLogStreamErrorInternal_Text(t *testing.T) {
format: "text",
}
- broadcastContainerLogStreamErrorInternal(stream, errors.New("boom"))
+ handler := newTestWebSocketHandler()
+ handler.broadcastContainerLogStreamErrorInternal("container-1", "text", errors.New("boom"), stream)
_ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
_, payload, err := clientConn.ReadMessage()
@@ -94,3 +102,523 @@ func newTestWSPairInternal(t *testing.T) (clientConn *websocket.Conn, serverConn
server.Close()
}
}
+
+func newTestWebSocketHandler() *WebSocketHandler {
+ return &WebSocketHandler{
+ wsMetrics: NewWebSocketMetrics(),
+ logStreams: make(map[string]*wsLogStream),
+ wsUpgrader: websocket.Upgrader{
+ CheckOrigin: func(r *http.Request) bool { return true },
+ },
+ }
+}
+
+func dialWebSocket(t *testing.T, serverURL, path string) *websocket.Conn {
+ t.Helper()
+
+ wsURL := "ws" + strings.TrimPrefix(serverURL, "http") + path
+ conn, resp, err := websocket.DefaultDialer.Dial(wsURL, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ _ = resp.Body.Close()
+ }
+
+ return conn
+}
+
+func TestWebSocketHandler_ProjectLogs_SharedStreamPerTarget(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ handler := newTestWebSocketHandler()
+ var starts atomic.Int32
+ var cancels atomic.Int32
+
+ handler.projectLogStreamer = func(ctx context.Context, projectID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {
+ starts.Add(1)
+ ticker := time.NewTicker(20 * time.Millisecond)
+ defer ticker.Stop()
+ defer cancels.Add(1)
+
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-ticker.C:
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case logsChan <- "api | shared project log":
+ }
+ }
+ }
+ }
+
+ router := gin.New()
+ router.GET("/api/environments/:id/ws/projects/:projectId/logs", handler.ProjectLogs)
+ server := httptest.NewServer(router)
+ defer server.Close()
+
+ conn1 := dialWebSocket(t, server.URL, "/api/environments/0/ws/projects/project-1/logs")
+ conn2 := dialWebSocket(t, server.URL, "/api/environments/0/ws/projects/project-1/logs")
+
+ _ = conn1.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, _, err := conn1.ReadMessage()
+ require.NoError(t, err)
+
+ _ = conn2.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, _, err = conn2.ReadMessage()
+ require.NoError(t, err)
+
+ require.Eventually(t, func() bool {
+ return starts.Load() == 1
+ }, 2*time.Second, 20*time.Millisecond)
+
+ require.Eventually(t, func() bool {
+ handler.logStreamsMu.Lock()
+ defer handler.logStreamsMu.Unlock()
+ return len(handler.logStreams) == 1
+ }, time.Second, 20*time.Millisecond)
+
+ require.NoError(t, conn1.Close())
+
+ require.Eventually(t, func() bool {
+ handler.logStreamsMu.Lock()
+ defer handler.logStreamsMu.Unlock()
+ return len(handler.logStreams) == 1
+ }, 2*time.Second, 10*time.Millisecond)
+
+ handler.logStreamsMu.Lock()
+ activeAfterFirstClose := len(handler.logStreams)
+ handler.logStreamsMu.Unlock()
+ require.Equal(t, 1, activeAfterFirstClose)
+ require.Equal(t, int32(0), cancels.Load())
+
+ require.NoError(t, conn2.Close())
+
+ require.Eventually(t, func() bool {
+ handler.logStreamsMu.Lock()
+ defer handler.logStreamsMu.Unlock()
+ return len(handler.logStreams) == 0
+ }, 2*time.Second, 20*time.Millisecond)
+ require.Eventually(t, func() bool {
+ return cancels.Load() == 1
+ }, 2*time.Second, 20*time.Millisecond)
+}
+
+func TestWebSocketHandler_ProjectLogs_CompletedSourceStartsFreshStream(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ handler := newTestWebSocketHandler()
+ var starts atomic.Int32
+ firstDone := make(chan struct{})
+
+ handler.projectLogStreamer = func(ctx context.Context, projectID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {
+ call := starts.Add(1)
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case logsChan <- "api | finite project log":
+ }
+ if call == 1 {
+ close(firstDone)
+ }
+ return nil
+ }
+
+ router := gin.New()
+ router.GET("/api/environments/:id/ws/projects/:projectId/logs", handler.ProjectLogs)
+ server := httptest.NewServer(router)
+ defer server.Close()
+
+ path := "/api/environments/0/ws/projects/project-1/logs?follow=false"
+ conn1 := dialWebSocket(t, server.URL, path)
+ defer conn1.Close()
+
+ _ = conn1.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, msg1, err := conn1.ReadMessage()
+ require.NoError(t, err)
+ require.Equal(t, "finite project log", string(msg1))
+
+ select {
+ case <-firstDone:
+ case <-time.After(2 * time.Second):
+ t.Fatal("first finite log stream did not complete")
+ }
+
+ conn2 := dialWebSocket(t, server.URL, path)
+ defer conn2.Close()
+
+ _ = conn2.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, msg2, err := conn2.ReadMessage()
+ require.NoError(t, err)
+ require.Equal(t, "finite project log", string(msg2))
+
+ require.Eventually(t, func() bool {
+ return starts.Load() == 2
+ }, 2*time.Second, 20*time.Millisecond)
+}
+
+func TestWebSocketHandler_ContainerLogs_BroadcastsStreamErrors(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ handler := newTestWebSocketHandler()
+ handler.containerLogStreamer = func(ctx context.Context, containerID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case logsChan <- "api | container log":
+ }
+ return errors.New("stream failed")
+ }
+
+ router := gin.New()
+ router.GET("/api/environments/:id/ws/containers/:containerId/logs", handler.ContainerLogs)
+ server := httptest.NewServer(router)
+ defer server.Close()
+
+ conn := dialWebSocket(t, server.URL, "/api/environments/0/ws/containers/container-1/logs")
+ defer conn.Close()
+
+ var got []string
+
+ _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, msg, err := conn.ReadMessage()
+ require.NoError(t, err)
+ got = append(got, string(msg))
+
+ _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, msg, err = conn.ReadMessage()
+ require.NoError(t, err)
+ got = append(got, string(msg))
+
+ require.ElementsMatch(t, []string{
+ "api | container log",
+ "Failed to stream container logs: stream failed",
+ }, got)
+}
+
+func TestWebSocketHandler_ContainerLogs_ErrorStartsFreshStreamForNewSubscribers(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ handler := newTestWebSocketHandler()
+ var starts atomic.Int32
+ handler.containerLogStreamer = func(ctx context.Context, containerID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {
+ starts.Add(1)
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case logsChan <- "api | container log":
+ }
+ return errors.New("stream failed")
+ }
+
+ router := gin.New()
+ router.GET("/api/environments/:id/ws/containers/:containerId/logs", handler.ContainerLogs)
+ server := httptest.NewServer(router)
+ defer server.Close()
+
+ path := "/api/environments/0/ws/containers/container-1/logs"
+ conn1 := dialWebSocket(t, server.URL, path)
+ defer conn1.Close()
+
+ got1 := make([]string, 0, 2)
+ for range 2 {
+ _ = conn1.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, msg, err := conn1.ReadMessage()
+ require.NoError(t, err)
+ got1 = append(got1, string(msg))
+ }
+ require.ElementsMatch(t, []string{
+ "api | container log",
+ "Failed to stream container logs: stream failed",
+ }, got1)
+
+ conn2 := dialWebSocket(t, server.URL, path)
+ defer conn2.Close()
+
+ got2 := make([]string, 0, 2)
+ for range 2 {
+ _ = conn2.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, msg, err := conn2.ReadMessage()
+ require.NoError(t, err)
+ got2 = append(got2, string(msg))
+ }
+ require.ElementsMatch(t, []string{
+ "api | container log",
+ "Failed to stream container logs: stream failed",
+ }, got2)
+
+ require.Eventually(t, func() bool {
+ return starts.Load() == 2
+ }, 2*time.Second, 20*time.Millisecond)
+}
+
+func TestWebSocketHandler_SystemStats_UsesSharedSampler(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ handler := newTestWebSocketHandler()
+ var collects atomic.Int32
+
+ handler.systemStatsCollector = func(ctx context.Context) systemtypes.SystemStats {
+ n := collects.Add(1)
+ return systemtypes.SystemStats{
+ CPUUsage: float64(n),
+ }
+ }
+
+ router := gin.New()
+ router.GET("/api/environments/:id/ws/system/stats", handler.SystemStats)
+ server := httptest.NewServer(router)
+ defer server.Close()
+
+ conn1 := dialWebSocket(t, server.URL, "/api/environments/0/ws/system/stats?interval=1")
+ conn2 := dialWebSocket(t, server.URL, "/api/environments/0/ws/system/stats?interval=1")
+
+ _ = conn1.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, _, err := conn1.ReadMessage()
+ require.NoError(t, err)
+
+ _ = conn2.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, _, err = conn2.ReadMessage()
+ require.NoError(t, err)
+
+ require.Eventually(t, func() bool {
+ return collects.Load() >= 1
+ }, 2*time.Second, 50*time.Millisecond)
+
+ require.NoError(t, conn1.Close())
+ require.NoError(t, conn2.Close())
+
+ require.Eventually(t, func() bool {
+ handler.systemStatsSampler.lifecycleMu.Lock()
+ defer handler.systemStatsSampler.lifecycleMu.Unlock()
+ return !handler.systemStatsSampler.running && handler.systemStatsSampler.clients == 0
+ }, 2*time.Second, 20*time.Millisecond)
+
+ stoppedAt := collects.Load()
+ require.Never(t, func() bool {
+ return collects.Load() != stoppedAt
+ }, 1200*time.Millisecond, 100*time.Millisecond)
+}
+
+func TestWebSocketHandler_AcquireSystemStatsSampler_WaitsForInitialSnapshot(t *testing.T) {
+ handler := newTestWebSocketHandler()
+ handler.systemStatsCollector = func(ctx context.Context) systemtypes.SystemStats {
+ return systemtypes.SystemStats{CPUUsage: 42}
+ }
+
+ firstDone := make(chan struct{})
+ go func() {
+ handler.acquireSystemStatsSamplerInternal(context.Background())
+ close(firstDone)
+ }()
+
+ require.Eventually(t, func() bool {
+ handler.systemStatsSampler.lifecycleMu.Lock()
+ defer handler.systemStatsSampler.lifecycleMu.Unlock()
+ return handler.systemStatsSampler.running && handler.systemStatsSampler.ready != nil
+ }, 500*time.Millisecond, 10*time.Millisecond)
+
+ secondDone := make(chan struct{})
+ go func() {
+ handler.acquireSystemStatsSamplerInternal(context.Background())
+ close(secondDone)
+ }()
+
+ require.Never(t, func() bool {
+ select {
+ case <-secondDone:
+ return true
+ default:
+ return false
+ }
+ }, 200*time.Millisecond, 20*time.Millisecond)
+
+ select {
+ case <-firstDone:
+ case <-time.After(2 * time.Second):
+ t.Fatal("first sampler acquisition did not finish")
+ }
+
+ select {
+ case <-secondDone:
+ case <-time.After(2 * time.Second):
+ t.Fatal("second sampler acquisition did not wait for readiness")
+ }
+
+ stats := handler.latestSystemStatsSnapshotInternal()
+ require.Equal(t, 42.0, stats.CPUUsage)
+
+ handler.releaseSystemStatsSamplerInternal()
+ handler.releaseSystemStatsSamplerInternal()
+
+ require.Eventually(t, func() bool {
+ handler.systemStatsSampler.lifecycleMu.Lock()
+ defer handler.systemStatsSampler.lifecycleMu.Unlock()
+ return !handler.systemStatsSampler.running && handler.systemStatsSampler.clients == 0 && handler.systemStatsSampler.ready == nil
+ }, 2*time.Second, 20*time.Millisecond)
+}
+
+func TestWebSocketHandler_AcquireSystemStatsSampler_StopsWaitingWhenCallerCancels(t *testing.T) {
+ handler := newTestWebSocketHandler()
+
+ readyToCancel := make(chan struct{})
+ handler.systemStatsCollector = func(ctx context.Context) systemtypes.SystemStats {
+ return systemtypes.SystemStats{CPUUsage: 42}
+ }
+
+ handler.cpuUsageReader = func(interval time.Duration) (float64, bool) {
+ close(readyToCancel)
+ time.Sleep(200 * time.Millisecond)
+ return 42, true
+ }
+
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan bool, 1)
+ go func() {
+ done <- handler.acquireSystemStatsSamplerInternal(ctx)
+ }()
+
+ select {
+ case <-readyToCancel:
+ case <-time.After(time.Second):
+ t.Fatal("sampler did not start CPU initialization")
+ }
+
+ cancel()
+
+ select {
+ case ok := <-done:
+ require.False(t, ok)
+ case <-time.After(100 * time.Millisecond):
+ t.Fatal("sampler acquisition did not stop waiting after cancellation")
+ }
+
+ handler.releaseSystemStatsSamplerInternal()
+
+ require.Eventually(t, func() bool {
+ handler.systemStatsSampler.lifecycleMu.Lock()
+ defer handler.systemStatsSampler.lifecycleMu.Unlock()
+ return !handler.systemStatsSampler.running && handler.systemStatsSampler.clients == 0
+ }, time.Second, 20*time.Millisecond)
+}
+
+func TestWebSocketHandler_LogStream_ReplacesDoneStreamAndCleansStaleRefs(t *testing.T) {
+ handler := newTestWebSocketHandler()
+ key := "env|project|resource|text|false|true|100||false"
+
+ var staleCancels atomic.Int32
+ stale := &wsLogStream{
+ hub: ws.NewHub(1),
+ cancel: func() { staleCancels.Add(1) },
+ refs: 2,
+ done: true,
+ }
+ handler.logStreams[key] = stale
+
+ var freshCancels atomic.Int32
+ fresh := handler.getOrCreateLogStreamInternal(key, func(onEmpty func(*wsLogStream)) *wsLogStream {
+ return &wsLogStream{
+ hub: ws.NewHub(1),
+ cancel: func() { freshCancels.Add(1) },
+ }
+ })
+
+ require.NotSame(t, stale, fresh)
+ handler.logStreamsMu.Lock()
+ require.Same(t, fresh, handler.logStreams[key])
+ handler.logStreamsMu.Unlock()
+
+ handler.markLogStreamDoneInternal(key, stale)
+ handler.logStreamsMu.Lock()
+ require.Same(t, fresh, handler.logStreams[key])
+ handler.logStreamsMu.Unlock()
+
+ handler.releaseLogStreamInternal(key, stale)
+ require.Equal(t, 1, stale.refs)
+ handler.logStreamsMu.Lock()
+ require.Same(t, fresh, handler.logStreams[key])
+ handler.logStreamsMu.Unlock()
+
+ handler.releaseLogStreamInternal(key, stale)
+ require.Equal(t, int32(1), staleCancels.Load())
+ handler.logStreamsMu.Lock()
+ require.Same(t, fresh, handler.logStreams[key])
+ handler.logStreamsMu.Unlock()
+
+ handler.releaseLogStreamInternal(key, fresh)
+ require.Equal(t, int32(1), freshCancels.Load())
+ handler.logStreamsMu.Lock()
+ _, ok := handler.logStreams[key]
+ handler.logStreamsMu.Unlock()
+ require.False(t, ok)
+}
+
+func TestWebSocketHandler_LogStream_CancelsOnlyOnce(t *testing.T) {
+ handler := newTestWebSocketHandler()
+ key := "env|project|resource|text|false|true|100||false"
+
+ var cancels atomic.Int32
+ stream := &wsLogStream{
+ hub: ws.NewHub(1),
+ cancel: func() { cancels.Add(1) },
+ refs: 1,
+ }
+ handler.logStreams[key] = stream
+
+ handler.markLogStreamDoneInternal(key, stream)
+ handler.releaseLogStreamInternal(key, stream)
+ handler.markLogStreamDoneInternal(key, stream)
+
+ require.Equal(t, int32(1), cancels.Load())
+}
+
+func TestWebSocketHandler_GetCachedCgroupLimitsInternal_DeduplicatesRefresh(t *testing.T) {
+ handler := newTestWebSocketHandler()
+ handler.cgroupCache.timestamp = time.Now().Add(-2 * cgroupCacheTTL)
+
+ var calls atomic.Int32
+ start := make(chan struct{})
+ release := make(chan struct{})
+ handler.cgroupLimitsDetector = func() (*docker.CgroupLimits, error) {
+ calls.Add(1)
+ close(start)
+ <-release
+ return &docker.CgroupLimits{CPUCount: 2}, nil
+ }
+
+ const goroutines = 8
+ results := make(chan *docker.CgroupLimits, goroutines)
+ ready := make(chan struct{})
+ var entered sync.WaitGroup
+ entered.Add(goroutines)
+ var wg sync.WaitGroup
+ for range goroutines {
+ wg.Go(func() {
+ <-ready
+ entered.Done()
+ results <- handler.getCachedCgroupLimitsInternal()
+ })
+ }
+
+ close(ready)
+ entered.Wait()
+
+ select {
+ case <-start:
+ case <-time.After(2 * time.Second):
+ t.Fatal("detector was not called")
+ }
+
+ require.Equal(t, int32(1), calls.Load())
+
+ close(release)
+ wg.Wait()
+ close(results)
+
+ for result := range results {
+ require.NotNil(t, result)
+ require.Equal(t, 2, result.CPUCount)
+ }
+ require.Equal(t, int32(1), calls.Load())
+}
diff --git a/backend/internal/models/settings.go b/backend/internal/models/settings.go
index 350e17633..21a0eaaee 100644
--- a/backend/internal/models/settings.go
+++ b/backend/internal/models/settings.go
@@ -5,9 +5,9 @@ import (
"errors"
"fmt"
"reflect"
- "slices"
"strconv"
"strings"
+ "sync"
"time"
)
@@ -21,6 +21,21 @@ type SettingVariable struct {
Value string
}
+type settingFieldMeta struct {
+ index int
+ key string
+ attrs string
+ isPublic bool
+ isSensitive bool
+ isLocal bool
+}
+
+var settingsFieldCache struct {
+ once sync.Once
+ ordered []settingFieldMeta
+ byKey map[string]settingFieldMeta
+}
+
// IsTrue returns true if the value is a truthy string
func (s SettingVariable) IsTrue() bool {
ok, _ := strconv.ParseBool(s.Value)
@@ -154,29 +169,63 @@ func (SettingVariable) TableName() string {
return "settings"
}
-func (s *Settings) ToSettingVariableSlice(showAll bool, redactSensitiveValues bool) []SettingVariable {
- cfgValue := reflect.ValueOf(s).Elem()
- cfgType := cfgValue.Type()
-
- var res []SettingVariable
-
- for i := 0; i < cfgType.NumField(); i++ {
- field := cfgType.Field(i)
+func buildSettingsFieldCacheInternal() {
+ rt := reflect.TypeFor[Settings]()
+ ordered := make([]settingFieldMeta, 0, rt.NumField())
+ byKey := make(map[string]settingFieldMeta, rt.NumField())
+ for i := 0; i < rt.NumField(); i++ {
+ field := rt.Field(i)
key, attrs, _ := strings.Cut(field.Tag.Get("key"), ",")
if key == "" {
continue
}
- if !showAll && !strings.Contains(attrs, "public") {
+ meta := settingFieldMeta{
+ index: i,
+ key: key,
+ attrs: attrs,
+ isPublic: strings.Contains(attrs, "public"),
+ isSensitive: strings.Contains(attrs, "sensitive"),
+ isLocal: strings.Contains(attrs, "local"),
+ }
+ ordered = append(ordered, meta)
+ byKey[key] = meta
+ }
+
+ settingsFieldCache.ordered = ordered
+ settingsFieldCache.byKey = byKey
+}
+
+func getSettingsFieldCacheInternal() ([]settingFieldMeta, map[string]settingFieldMeta) {
+ settingsFieldCache.once.Do(buildSettingsFieldCacheInternal)
+ return settingsFieldCache.ordered, settingsFieldCache.byKey
+}
+
+func (s *Settings) Clone() *Settings {
+ if s == nil {
+ return &Settings{}
+ }
+
+ clone := *s
+ return &clone
+}
+
+func (s *Settings) ToSettingVariableSlice(showAll bool, redactSensitiveValues bool) []SettingVariable {
+ cfgValue := reflect.ValueOf(s).Elem()
+ fields, _ := getSettingsFieldCacheInternal()
+
+ res := make([]SettingVariable, 0, len(fields))
+ for _, field := range fields {
+ if !showAll && !field.isPublic {
continue
}
- value := cfgValue.Field(i).FieldByName("Value").String()
- value = redactSettingValue(key, value, attrs, redactSensitiveValues)
+ value := cfgValue.Field(field.index).FieldByName("Value").String()
+ value = redactSettingValue(field.key, value, field.attrs, redactSensitiveValues)
settingVariable := SettingVariable{
- Key: key,
+ Key: field.key,
Value: value,
}
res = append(res, settingVariable)
@@ -187,65 +236,47 @@ func (s *Settings) ToSettingVariableSlice(showAll bool, redactSensitiveValues bo
func (s *Settings) FieldByKey(key string) (defaultValue string, isPublic bool, isSensitive bool, err error) {
rv := reflect.ValueOf(s).Elem()
- rt := rv.Type()
-
- for i := 0; i < rt.NumField(); i++ {
- tagValue := strings.Split(rt.Field(i).Tag.Get("key"), ",")
- keyFromTag := tagValue[0]
- isPublic = slices.Contains(tagValue, "public")
- isSensitive = slices.Contains(tagValue, "sensitive")
+ _, byKey := getSettingsFieldCacheInternal()
- if keyFromTag != key {
- continue
- }
-
- valueField := rv.Field(i).FieldByName("Value")
- return valueField.String(), isPublic, isSensitive, nil
+ field, ok := byKey[key]
+ if !ok {
+ return "", false, false, SettingKeyNotFoundError{field: key}
}
- return "", false, false, SettingKeyNotFoundError{field: key}
+ valueField := rv.Field(field.index).FieldByName("Value")
+ return valueField.String(), field.isPublic, field.isSensitive, nil
}
func (s *Settings) IsLocalSetting(key string) bool {
- rt := reflect.TypeFor[Settings]()
-
- for field := range rt.Fields() {
- tagValue := strings.Split(field.Tag.Get("key"), ",")
- keyFromTag := tagValue[0]
-
- if keyFromTag == key {
- return slices.Contains(tagValue, "local")
- }
+ _, byKey := getSettingsFieldCacheInternal()
+ field, ok := byKey[key]
+ if !ok {
+ return false
}
- return false
+ return field.isLocal
}
func (s *Settings) UpdateField(key string, value string, noSensitive bool) error {
rv := reflect.ValueOf(s).Elem()
- rt := rv.Type()
-
- for i := 0; i < rt.NumField(); i++ {
- tagValue, attrs, _ := strings.Cut(rt.Field(i).Tag.Get("key"), ",")
- if tagValue != key {
- continue
- }
+ _, byKey := getSettingsFieldCacheInternal()
- // If the field is sensitive and noSensitive is true, we skip that
- if noSensitive && strings.Contains(attrs, "sensitive") {
- return SettingSensitiveForbiddenError{field: key}
- }
+ field, ok := byKey[key]
+ if !ok {
+ return SettingKeyNotFoundError{field: key}
+ }
- valueField := rv.Field(i).FieldByName("Value")
- if !valueField.CanSet() {
- return fmt.Errorf("field Value in SettingVariable is not settable for config key '%s'", key)
- }
+ if noSensitive && field.isSensitive {
+ return SettingSensitiveForbiddenError{field: key}
+ }
- valueField.SetString(value)
- return nil
+ valueField := rv.Field(field.index).FieldByName("Value")
+ if !valueField.CanSet() {
+ return fmt.Errorf("field Value in SettingVariable is not settable for config key '%s'", key)
}
- return SettingKeyNotFoundError{field: key}
+ valueField.SetString(value)
+ return nil
}
// helper keeps redaction logic in one place; behavior unchanged
diff --git a/backend/internal/services/settings_service.go b/backend/internal/services/settings_service.go
index a609d1a45..27b10eb60 100644
--- a/backend/internal/services/settings_service.go
+++ b/backend/internal/services/settings_service.go
@@ -57,10 +57,6 @@ func NewSettingsService(ctx context.Context, db *database.DB) (*SettingsService,
return nil, fmt.Errorf("failed to setup instance ID: %w", err)
}
- if err = svc.LoadDatabaseSettings(ctx); err != nil {
- return nil, fmt.Errorf("failed to reload settings after instance ID setup: %w", err)
- }
-
return svc, nil
}
@@ -84,6 +80,14 @@ func (s *SettingsService) LoadDatabaseSettings(ctx context.Context) (err error)
return nil
}
+func (s *SettingsService) refreshSettingsCacheInternal(ctx context.Context) error {
+ if err := s.LoadDatabaseSettings(ctx); err != nil {
+ return fmt.Errorf("failed to refresh settings cache: %w", err)
+ }
+
+ return nil
+}
+
func (s *SettingsService) getDefaultSettings() *models.Settings {
return &models.Settings{
ProjectsDirectory: models.SettingVariable{Value: "/app/data/projects"},
@@ -326,28 +330,8 @@ func (s *SettingsService) isEnvOverrideActiveInternal(key string) bool {
}
func (s *SettingsService) GetSettings(ctx context.Context) (*models.Settings, error) {
- var settingVars []models.SettingVariable
- err := s.db.WithContext(ctx).Find(&settingVars).Error
- if err != nil {
- return nil, err
- }
-
- settings := &models.Settings{}
-
- for _, sv := range settingVars {
- if err := settings.UpdateField(sv.Key, sv.Value, false); err != nil {
- var notFoundErr models.SettingKeyNotFoundError
- if !errors.As(err, &notFoundErr) {
- return nil, fmt.Errorf("failed to load setting %s: %w", sv.Key, err)
- }
- }
- }
-
- // Apply environment variable overrides for fields tagged with "envOverride".
- // This keeps behavior consistent with the cached settings path (LoadDatabaseSettingsInternal)
- // and allows env vars like OIDC_MERGE_ACCOUNTS to affect runtime behavior.
+ settings := s.GetSettingsConfig().Clone()
s.applyEnvOverrides(ctx, settings)
-
return settings, nil
}
@@ -459,6 +443,14 @@ func (s *SettingsService) migrateOidcKeyNames(ctx context.Context) error {
}
func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string) error {
+ if err := s.updateSettingValueNoRefreshInternal(ctx, key, value); err != nil {
+ return err
+ }
+
+ return s.refreshSettingsCacheInternal(ctx)
+}
+
+func (s *SettingsService) updateSettingValueNoRefreshInternal(ctx context.Context, key, value string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
settingVar := &models.SettingVariable{
Key: key,
@@ -470,10 +462,8 @@ func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string)
func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.Update) ([]models.SettingVariable, error) {
defaultCfg := s.getDefaultSettings()
- cfg, err := s.GetSettings(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to load current settings: %w", err)
- }
+ cfg := s.GetSettingsConfig().Clone()
+ s.applyEnvOverrides(ctx, cfg)
valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedAutoHeal, changedTimeouts, err := s.prepareUpdateValues(updates, cfg, defaultCfg)
if err != nil {
@@ -488,13 +478,10 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.U
return nil, err
}
- // Reload and store settings BEFORE calling callbacks so they read updated values
- settings, err := s.GetSettings(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to retrieve updated settings: %w", err)
+ if err := s.refreshSettingsCacheInternal(ctx); err != nil {
+ return nil, err
}
-
- s.config.Store(settings)
+ settings := s.GetSettingsConfig()
// Now call callbacks after in-memory config is updated
if changedPolling && s.OnImagePollingSettingsChanged != nil {
@@ -667,7 +654,7 @@ func (s *SettingsService) handleOidcConfigUpdate(ctx context.Context, updates se
return fmt.Errorf("failed to marshal merged OIDC config: %w", err)
}
- if err := s.UpdateSetting(ctx, "authOidcConfig", string(mergedBytes)); err != nil {
+ if err := s.updateSettingValueNoRefreshInternal(ctx, "authOidcConfig", string(mergedBytes)); err != nil {
return fmt.Errorf("failed to update authOidcConfig: %w", err)
}
}
@@ -688,7 +675,7 @@ func (s *SettingsService) handleOidcConfigUpdate(ctx context.Context, updates se
}
}
- if err := s.UpdateSetting(ctx, "oidcClientSecret", secret); err != nil {
+ if err := s.updateSettingValueNoRefreshInternal(ctx, "oidcClientSecret", secret); err != nil {
return fmt.Errorf("failed to update oidcClientSecret: %w", err)
}
}
@@ -916,34 +903,15 @@ func (s *SettingsService) GetStringSetting(ctx context.Context, key, defaultValu
}
func (s *SettingsService) SetBoolSetting(ctx context.Context, key string, value bool) error {
- if err := s.UpdateSetting(ctx, key, fmt.Sprintf("%t", value)); err != nil {
- return err
- }
- // Rebuild a fresh snapshot instead of mutating current pointer (avoids races)
- if err := s.LoadDatabaseSettings(ctx); err != nil {
- return fmt.Errorf("failed to refresh settings cache: %w", err)
- }
- return nil
+ return s.UpdateSetting(ctx, key, fmt.Sprintf("%t", value))
}
func (s *SettingsService) SetIntSetting(ctx context.Context, key string, value int) error {
- if err := s.UpdateSetting(ctx, key, fmt.Sprintf("%d", value)); err != nil {
- return err
- }
- if err := s.LoadDatabaseSettings(ctx); err != nil {
- return fmt.Errorf("failed to refresh settings cache: %w", err)
- }
- return nil
+ return s.UpdateSetting(ctx, key, fmt.Sprintf("%d", value))
}
func (s *SettingsService) SetStringSetting(ctx context.Context, key, value string) error {
- if err := s.UpdateSetting(ctx, key, value); err != nil {
- return err
- }
- if err := s.LoadDatabaseSettings(ctx); err != nil {
- return fmt.Errorf("failed to refresh settings cache: %w", err)
- }
- return nil
+ return s.UpdateSetting(ctx, key, value)
}
func (s *SettingsService) EnsureEncryptionKey(ctx context.Context) (string, error) {
@@ -1041,10 +1009,6 @@ func (s *SettingsService) NormalizeProjectsDirectory(ctx context.Context, projec
return fmt.Errorf("failed to update projectsDirectory: %w", err)
}
- if err := s.LoadDatabaseSettings(ctx); err != nil {
- return fmt.Errorf("failed to reload settings after normalization: %w", err)
- }
-
slog.InfoContext(ctx, "Successfully normalized projects directory")
} else {
slog.DebugContext(ctx, "Projects directory already normalized or custom, skipping", "value", projectsDirSetting.Value)
@@ -1091,10 +1055,6 @@ func (s *SettingsService) NormalizeBuildsDirectory(ctx context.Context) error {
return fmt.Errorf("failed to update buildsDirectory: %w", err)
}
- if err := s.LoadDatabaseSettings(ctx); err != nil {
- return fmt.Errorf("failed to reload settings after normalization: %w", err)
- }
-
slog.InfoContext(ctx, "Successfully normalized builds directory")
} else {
slog.DebugContext(ctx, "Builds directory already normalized or custom, skipping", "value", buildsDirSetting.Value)
diff --git a/backend/internal/services/settings_service_test.go b/backend/internal/services/settings_service_test.go
index 252f6f5bc..e0c775c0e 100644
--- a/backend/internal/services/settings_service_test.go
+++ b/backend/internal/services/settings_service_test.go
@@ -91,6 +91,22 @@ func TestSettingsService_GetSettings_UnknownKeysIgnored(t *testing.T) {
require.NoError(t, err)
}
+func TestSettingsService_GetSettings_UsesCachedSnapshotWithoutDatabase(t *testing.T) {
+ ctx := context.Background()
+ db := setupSettingsTestDB(t)
+ svc, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+
+ require.NoError(t, svc.SetStringSetting(ctx, "baseServerUrl", "http://cached"))
+
+ // GetSettings should clone the in-memory snapshot and not touch the database.
+ svc.db = nil
+
+ settings, err := svc.GetSettings(ctx)
+ require.NoError(t, err)
+ require.Equal(t, "http://cached", settings.BaseServerURL.Value)
+}
+
func TestSettingsService_PruneUnknownSettings_RemovesStaleKeys(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
@@ -295,6 +311,51 @@ func TestSettingsService_UpdateSetting(t *testing.T) {
require.Equal(t, "all", sv.Value)
}
+func TestSettingsService_UpdateSetting_RefreshesCachedSnapshot(t *testing.T) {
+ ctx := context.Background()
+ db := setupSettingsTestDB(t)
+ svc, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+
+ require.Equal(t, "http://localhost", svc.GetSettingsConfig().BaseServerURL.Value)
+ require.NoError(t, svc.UpdateSetting(ctx, "baseServerUrl", "https://arcane.test"))
+
+ require.Equal(t, "https://arcane.test", svc.GetSettingsConfig().BaseServerURL.Value)
+
+ settings, err := svc.GetSettings(ctx)
+ require.NoError(t, err)
+ require.Equal(t, "https://arcane.test", settings.BaseServerURL.Value)
+}
+
+func BenchmarkSettingsService_GetSettings(b *testing.B) {
+ ctx := context.Background()
+ db, err := gorm.Open(glsqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ b.Fatal(err)
+ }
+ if err := db.AutoMigrate(&models.SettingVariable{}); err != nil {
+ b.Fatal(err)
+ }
+ settingsDB := &database.DB{DB: db}
+ svc, err := NewSettingsService(ctx, settingsDB)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for b.Loop() {
+ settings, err := svc.GetSettings(ctx)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if settings == nil {
+ b.Fatal("settings should not be nil")
+ }
+ }
+}
+
func TestSettingsService_EnsureEncryptionKey(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
@@ -422,6 +483,31 @@ func TestSettingsService_UpdateSettings_RefreshesCache(t *testing.T) {
require.True(t, found, "projectsDirectory setting not found in cached list")
}
+func TestSettingsService_UpdateSettings_ReturnsEnvOverriddenValues(t *testing.T) {
+ t.Setenv("DOCKER_HOST", "tcp://env-docker:2375")
+
+ ctx := context.Background()
+ db := setupSettingsTestDB(t)
+ svc, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+ require.NoError(t, svc.EnsureDefaultSettings(ctx))
+
+ newDir := "custom/projects2"
+ settingsList, err := svc.UpdateSettings(ctx, settings.Update{
+ ProjectsDirectory: &newDir,
+ })
+ require.NoError(t, err)
+
+ found := false
+ for _, sv := range settingsList {
+ if sv.Key == "dockerHost" {
+ found = true
+ require.Equal(t, "tcp://env-docker:2375", sv.Value)
+ }
+ }
+ require.True(t, found, "dockerHost setting not found in update response")
+}
+
func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
diff --git a/backend/pkg/libarcane/ws/client.go b/backend/pkg/libarcane/ws/client.go
index 23e27ea09..4a4504c4e 100644
--- a/backend/pkg/libarcane/ws/client.go
+++ b/backend/pkg/libarcane/ws/client.go
@@ -22,9 +22,10 @@ const (
// Client represents a single WebSocket connection.
type Client struct {
- conn *websocket.Conn
- send chan []byte
- once sync.Once
+ conn *websocket.Conn
+ send chan []byte
+ once sync.Once
+ onRemove func()
}
func NewClient(conn *websocket.Conn, sendBuffer int) *Client {
@@ -39,7 +40,12 @@ func NewClient(conn *websocket.Conn, sendBuffer int) *Client {
// ServeClient registers the client with the hub and starts read/write pumps.
// Caller is responsible for creating/closing the websocket.Conn.
func ServeClient(ctx context.Context, hub *Hub, conn *websocket.Conn) {
+ ServeClientWithOnRemove(ctx, hub, conn, nil)
+}
+
+func ServeClientWithOnRemove(ctx context.Context, hub *Hub, conn *websocket.Conn, onRemove func()) {
c := NewClient(conn, clientSendBuffer)
+ c.onRemove = onRemove
hub.register <- c
go c.writePump(ctx, hub)
@@ -48,6 +54,9 @@ func ServeClient(ctx context.Context, hub *Hub, conn *websocket.Conn) {
func (c *Client) safeRemove(hub *Hub) {
c.once.Do(func() {
+ if c.onRemove != nil {
+ c.onRemove()
+ }
hub.remove(c)
})
}
diff --git a/backend/pkg/libarcane/ws/hub.go b/backend/pkg/libarcane/ws/hub.go
index 6a2e656e3..50ce0cf91 100644
--- a/backend/pkg/libarcane/ws/hub.go
+++ b/backend/pkg/libarcane/ws/hub.go
@@ -12,6 +12,7 @@ type Hub struct {
register chan *Client
unregister chan *Client
broadcast chan []byte
+ onFirst func()
onEmpty func()
}
@@ -24,6 +25,12 @@ func NewHub(buffer int) *Hub {
}
}
+func (h *Hub) SetOnFirstClient(fn func()) {
+ h.mu.Lock()
+ h.onFirst = fn
+ h.mu.Unlock()
+}
+
func (h *Hub) SetOnEmpty(fn func()) {
h.mu.Lock()
h.onEmpty = fn
@@ -38,9 +45,17 @@ func (h *Hub) Run(ctx context.Context) {
case <-ctx.Done():
return
case c := <-h.register:
+ var onFirst func()
h.mu.Lock()
h.clients[c] = struct{}{}
+ if len(h.clients) == 1 && h.onFirst != nil {
+ onFirst = h.onFirst
+ h.onFirst = nil
+ }
h.mu.Unlock()
+ if onFirst != nil {
+ onFirst()
+ }
case c := <-h.unregister:
// remove() handles the onEmpty callback when the last client disconnects.
h.remove(c)
diff --git a/backend/pkg/libarcane/ws/hub_client_count_test.go b/backend/pkg/libarcane/ws/hub_client_count_test.go
deleted file mode 100644
index b610d0027..000000000
--- a/backend/pkg/libarcane/ws/hub_client_count_test.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package ws
-
-// ClientCount is test-only and intentionally compiled from *_test.go to avoid
-// deadcode in production builds.
-func (h *Hub) ClientCount() int {
- h.mu.RLock()
- defer h.mu.RUnlock()
- return len(h.clients)
-}
diff --git a/backend/pkg/libarcane/ws/hub_test.go b/backend/pkg/libarcane/ws/hub_test.go
index 0d1132240..7be11cc56 100644
--- a/backend/pkg/libarcane/ws/hub_test.go
+++ b/backend/pkg/libarcane/ws/hub_test.go
@@ -207,6 +207,32 @@ func TestHub_OnEmptyCallback(t *testing.T) {
require.Eventually(t, called.Load, time.Second, 5*time.Millisecond, "onEmpty callback was not called")
}
+func TestHub_OnFirstClientCallback(t *testing.T) {
+ h := NewHub(10)
+ ctx := t.Context()
+ go h.Run(ctx)
+
+ var callCount atomic.Int32
+ h.SetOnFirstClient(func() {
+ callCount.Add(1)
+ })
+
+ _, sc1, cleanup1 := newTestWSPair(t)
+ defer cleanup1()
+ _, sc2, cleanup2 := newTestWSPair(t)
+ defer cleanup2()
+
+ h.register <- NewClient(sc1, 16)
+ h.register <- NewClient(sc2, 16)
+
+ require.Eventually(t, func() bool {
+ return h.ClientCount() == 2
+ }, time.Second, 5*time.Millisecond)
+ require.Eventually(t, func() bool {
+ return callCount.Load() == 1
+ }, time.Second, 5*time.Millisecond)
+}
+
func TestHub_OnEmptyCalledOnlyOnce(t *testing.T) {
h := NewHub(10)
ctx := t.Context()
diff --git a/backend/pkg/libarcane/ws/hub_test_helper_test.go b/backend/pkg/libarcane/ws/hub_test_helper_test.go
new file mode 100644
index 000000000..d7e7347e4
--- /dev/null
+++ b/backend/pkg/libarcane/ws/hub_test_helper_test.go
@@ -0,0 +1,7 @@
+package ws
+
+func (h *Hub) ClientCount() int {
+ h.mu.RLock()
+ defer h.mu.RUnlock()
+ return len(h.clients)
+}
diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.go
index 05433384c..aa1ed2429 100644
--- a/backend/pkg/scheduler/auto_heal_job.go
+++ b/backend/pkg/scheduler/auto_heal_job.go
@@ -13,9 +13,11 @@ import (
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
"github.com/robfig/cron/v3"
+ "golang.org/x/sync/errgroup"
)
const AutoHealJobName = "auto-heal"
+const autoHealInspectConcurrency = 4
// restartRecord tracks restart timestamps for a single container.
type restartRecord struct {
@@ -30,6 +32,11 @@ type AutoHealJob struct {
mu sync.Mutex
restarts map[string]*restartRecord
+
+ getDockerClient func() (*client.Client, error)
+ listContainers func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error)
+ inspectContainer func(ctx context.Context, dockerClient *client.Client, containerID string) (container.InspectResponse, error)
+ restartContainer func(ctx context.Context, dockerClient *client.Client, containerID string) error
}
func NewAutoHealJob(
@@ -73,78 +80,107 @@ func (j *AutoHealJob) Run(ctx context.Context) {
return
}
- dockerClient, err := j.dockerClientService.GetClient(ctx)
+ dockerClient, err := j.getDockerClientInternal(ctx)
if err != nil {
slog.ErrorContext(ctx, "auto-heal failed to get Docker client", "error", err)
return
}
- containerList, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})
+ containerList, err := j.listContainersInternal(ctx, dockerClient)
if err != nil {
slog.ErrorContext(ctx, "auto-heal failed to list containers", "error", err)
return
}
- containers := containerList.Items
+ containers := containerList
excludedContainers := j.parseExcludedContainers(ctx)
maxRestarts := j.settingsService.GetIntSetting(ctx, "autoHealMaxRestarts", 5)
restartWindowMinutes := j.settingsService.GetIntSetting(ctx, "autoHealRestartWindow", 30)
restartWindow := time.Duration(restartWindowMinutes) * time.Minute
+ candidates := j.filterCandidatesInternal(containers, excludedContainers)
+
+ g, groupCtx := errgroup.WithContext(ctx)
+ g.SetLimit(autoHealInspectConcurrency)
+
+ for _, candidate := range candidates {
+ g.Go(func() error {
+ j.processCandidateInternal(groupCtx, dockerClient, candidate, maxRestarts, restartWindow, restartWindowMinutes)
+ return nil
+ })
+ }
+
+ _ = g.Wait()
+}
+
+func (j *AutoHealJob) filterCandidatesInternal(containers []container.Summary, excludedContainers map[string]struct{}) []container.Summary {
+ candidates := make([]container.Summary, 0, len(containers))
for _, c := range containers {
- // Skip Arcane internal containers
if libarcane.IsInternalContainer(c.Labels) {
continue
}
containerName := j.getContainerName(c.Names)
-
- // Skip excluded containers
if j.isExcluded(containerName, excludedContainers) {
continue
}
- // Inspect to get health status
- inspect, err := libarcane.ContainerInspectWithCompatibility(ctx, dockerClient, c.ID, client.ContainerInspectOptions{})
- if err != nil {
- slog.WarnContext(ctx, "auto-heal failed to inspect container", "container", containerName, "error", err)
- continue
- }
- containerInspect := inspect.Container
+ candidates = append(candidates, c)
+ }
- // Skip if no healthcheck configured or not unhealthy
- if containerInspect.State == nil || containerInspect.State.Health == nil {
- continue
- }
- if containerInspect.State.Health.Status != container.Unhealthy {
- continue
- }
+ return candidates
+}
- // Check restart-loop protection
- if !j.canRestart(c.ID, maxRestarts, restartWindow) {
- slog.WarnContext(ctx, "auto-heal restart-loop protection: skipping container",
- "container", containerName,
- "max_restarts", maxRestarts,
- "window_minutes", restartWindowMinutes,
- )
- continue
- }
+func (j *AutoHealJob) processCandidateInternal(
+ ctx context.Context,
+ dockerClient *client.Client,
+ candidate container.Summary,
+ maxRestarts int,
+ restartWindow time.Duration,
+ restartWindowMinutes int,
+) {
+ containerID := candidate.ID
+ containerName := j.getContainerName(candidate.Names)
+
+ inspect, err := j.inspectContainerInternal(ctx, dockerClient, containerID)
+ if err != nil {
+ slog.WarnContext(ctx, "auto-heal failed to inspect container", "container", containerName, "error", err)
+ return
+ }
- // Restart the container
- slog.InfoContext(ctx, "auto-heal restarting unhealthy container", "container", containerName, "container_id", c.ID)
- if _, err := dockerClient.ContainerRestart(ctx, c.ID, client.ContainerRestartOptions{}); err != nil {
- slog.ErrorContext(ctx, "auto-heal failed to restart container", "container", containerName, "error", err)
- continue
- }
+ if inspect.State == nil || inspect.State.Health == nil {
+ return
+ }
+ if inspect.State.Health.Status != container.Unhealthy {
+ return
+ }
+
+ if !j.canRestart(containerID, maxRestarts, restartWindow) {
+ slog.WarnContext(ctx, "auto-heal restart-loop protection: skipping container",
+ "container", containerName,
+ "max_restarts", maxRestarts,
+ "window_minutes", restartWindowMinutes,
+ )
+ return
+ }
- // Record the restart
- j.recordRestart(c.ID)
+ if err := j.restartContainerInternal(ctx, dockerClient, containerID); err != nil {
+ slog.ErrorContext(ctx, "auto-heal failed to restart container", "container", containerName, "error", err)
+ return
+ }
+
+ j.recordRestart(containerID)
+ j.postRestartActionsInternal(ctx, containerID, containerName)
- // Log event
+ slog.InfoContext(ctx, "auto-heal restarted unhealthy container", "container", containerName, "container_id", containerID)
+}
+
+func (j *AutoHealJob) postRestartActionsInternal(ctx context.Context, containerID, containerName string) {
+ if j.eventService != nil {
if err := j.eventService.LogContainerEvent(
ctx,
models.EventTypeContainerRestart,
- c.ID,
+ containerID,
containerName,
"", // no user - system action
"system",
@@ -153,13 +189,12 @@ func (j *AutoHealJob) Run(ctx context.Context) {
); err != nil {
slog.WarnContext(ctx, "auto-heal failed to log event", "container", containerName, "error", err)
}
+ }
- // Send notification
- if err := j.notificationService.SendAutoHealNotification(ctx, containerName, c.ID); err != nil {
+ if j.notificationService != nil {
+ if err := j.notificationService.SendAutoHealNotification(ctx, containerName, containerID); err != nil {
slog.WarnContext(ctx, "auto-heal failed to send notification", "container", containerName, "error", err)
}
-
- slog.InfoContext(ctx, "auto-heal successfully restarted container", "container", containerName)
}
}
@@ -238,6 +273,49 @@ func (j *AutoHealJob) isExcluded(name string, excluded map[string]struct{}) bool
return ok
}
+func (j *AutoHealJob) inspectContainerInternal(ctx context.Context, dockerClient *client.Client, containerID string) (container.InspectResponse, error) {
+ if j.inspectContainer != nil {
+ return j.inspectContainer(ctx, dockerClient, containerID)
+ }
+
+ inspect, err := libarcane.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ if err != nil {
+ return container.InspectResponse{}, err
+ }
+
+ return inspect.Container, nil
+}
+
+func (j *AutoHealJob) restartContainerInternal(ctx context.Context, dockerClient *client.Client, containerID string) error {
+ if j.restartContainer != nil {
+ return j.restartContainer(ctx, dockerClient, containerID)
+ }
+
+ _, err := dockerClient.ContainerRestart(ctx, containerID, client.ContainerRestartOptions{})
+ return err
+}
+
+func (j *AutoHealJob) getDockerClientInternal(ctx context.Context) (*client.Client, error) {
+ if j.getDockerClient != nil {
+ return j.getDockerClient()
+ }
+
+ return j.dockerClientService.GetClient(ctx)
+}
+
+func (j *AutoHealJob) listContainersInternal(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {
+ if j.listContainers != nil {
+ return j.listContainers(ctx, dockerClient)
+ }
+
+ containerList, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})
+ if err != nil {
+ return nil, err
+ }
+
+ return containerList.Items, nil
+}
+
// ResetRestartTracking clears all restart records (exported for testing).
func (j *AutoHealJob) ResetRestartTracking() {
j.mu.Lock()
diff --git a/backend/pkg/scheduler/auto_heal_job_test.go b/backend/pkg/scheduler/auto_heal_job_test.go
index ec3222862..16afdffe9 100644
--- a/backend/pkg/scheduler/auto_heal_job_test.go
+++ b/backend/pkg/scheduler/auto_heal_job_test.go
@@ -1,10 +1,19 @@
package scheduler
import (
+ "context"
+ "sync/atomic"
"testing"
"time"
+ "github.com/getarcaneapp/arcane/backend/internal/database"
+ "github.com/getarcaneapp/arcane/backend/internal/models"
+ "github.com/getarcaneapp/arcane/backend/internal/services"
+ glsqlite "github.com/glebarez/sqlite"
+ "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/client"
"github.com/stretchr/testify/require"
+ "gorm.io/gorm"
)
func newTestAutoHealJob() *AutoHealJob {
@@ -112,3 +121,66 @@ func TestAutoHeal_ResetRestartTracking(t *testing.T) {
// Should be allowed again
require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))
}
+
+func TestAutoHeal_Run_UsesBoundedConcurrency(t *testing.T) {
+ ctx := context.Background()
+
+ db, err := gorm.Open(glsqlite.Open(":memory:"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))
+
+ settingsSvc, err := services.NewSettingsService(ctx, &database.DB{DB: db})
+ require.NoError(t, err)
+ require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoHealEnabled", true))
+ require.NoError(t, settingsSvc.SetStringSetting(ctx, "autoHealExcludedContainers", "skip-me"))
+
+ job := NewAutoHealJob(nil, settingsSvc, nil, nil)
+ job.getDockerClient = func() (*client.Client, error) { return nil, nil }
+ job.listContainers = func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {
+ return []container.Summary{
+ {ID: "c1", Names: []string{"/one"}},
+ {ID: "c2", Names: []string{"/two"}},
+ {ID: "c3", Names: []string{"/three"}},
+ {ID: "c4", Names: []string{"/four"}},
+ {ID: "c5", Names: []string{"/five"}},
+ {ID: "c6", Names: []string{"/six"}},
+ {ID: "skip", Names: []string{"/skip-me"}},
+ }, nil
+ }
+
+ var current int32
+ var maxConcurrent int32
+ var restarts int32
+
+ job.inspectContainer = func(ctx context.Context, dockerClient *client.Client, containerID string) (container.InspectResponse, error) {
+ active := atomic.AddInt32(&current, 1)
+ for {
+ maxSeen := atomic.LoadInt32(&maxConcurrent)
+ if active <= maxSeen || atomic.CompareAndSwapInt32(&maxConcurrent, maxSeen, active) {
+ break
+ }
+ }
+ defer atomic.AddInt32(&current, -1)
+
+ time.Sleep(40 * time.Millisecond)
+
+ switch containerID {
+ case "c5":
+ return container.InspectResponse{State: &container.State{}}, nil
+ case "c6":
+ return container.InspectResponse{State: &container.State{Health: &container.Health{Status: container.Healthy}}}, nil
+ default:
+ return container.InspectResponse{State: &container.State{Health: &container.Health{Status: container.Unhealthy}}}, nil
+ }
+ }
+ job.restartContainer = func(ctx context.Context, dockerClient *client.Client, containerID string) error {
+ atomic.AddInt32(&restarts, 1)
+ return nil
+ }
+
+ job.Run(ctx)
+
+ require.Greater(t, atomic.LoadInt32(&maxConcurrent), int32(1))
+ require.LessOrEqual(t, atomic.LoadInt32(&maxConcurrent), int32(autoHealInspectConcurrency))
+ require.Equal(t, int32(4), atomic.LoadInt32(&restarts))
+}