Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
perf: share websocket streams, cache settings reads, and parallelize auto-heal inspections (#1957)
Kyle Mendellf6b5b4c1a06c
diff --git a/backend/internal/api/ws_handler.go b/backend/internal/api/ws_handler.goindex 0df4b0c45..28c871053 100644--- a/backend/internal/api/ws_handler.go+++ b/backend/internal/api/ws_handler.go@@ -37,6 +37,7 @@ import (const (gpuCacheDuration = 30 * time.Second+ cgroupCacheTTL = 30 * time.Second)// amdGPUSysfsPath is the base path for AMD GPU sysfs entries@@ -156,11 +157,34 @@ type WebSocketHandler struct {wsUpgrader websocket.UpgraderwsMetrics *WebSocketMetricsactiveConnections sync.Map+ logStreamsMu sync.Mutex+ logStreams map[string]*wsLogStreamcpuCache struct {sync.RWMutexvalue float64timestamp time.Time}+ systemStaticInfo struct {+ once sync.Once+ cpuCount int+ hostname string+ }+ systemStatsSampler struct {+ stateMu sync.RWMutex+ latest systemtypes.SystemStats+ timestamp time.Time+ lifecycleMu sync.Mutex+ clients int+ cancel context.CancelFunc+ ready chan struct{}+ running bool+ }+ cgroupCache struct {+ sync.RWMutex+ value *docker.CgroupLimits+ detected bool+ timestamp time.Time+ }diskUsagePathCache struct {sync.RWMutexvalue string@@ -177,13 +201,22 @@ type WebSocketHandler struct {detectionMutex sync.MutexgpuMonitoringEnabled boolgpuType string+ projectLogStreamer func(ctx context.Context, projectID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error+ containerLogStreamer func(ctx context.Context, containerID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error+ systemStatsCollector func(ctx context.Context) systemtypes.SystemStats+ cpuUsageReader func(interval time.Duration) (float64, bool)+ cgroupLimitsDetector func() (*docker.CgroupLimits, error)}type wsLogStream struct {- hub *ws.Hub- cancel context.CancelFunc- format string- seq atomic.Uint64+ hub *ws.Hub+ cancel context.CancelFunc+ firstSubscriber chan struct{}+ format string+ key string+ refs int+ done bool+ seq atomic.Uint64}func getContextUserIDInternal(c *gin.Context) string {@@ -206,6 +239,98 @@ func buildWSConnectionInfoInternal(c *gin.Context, kind, resourceID string) syst}}+func buildLogStreamKeyInternal(envID, kind, resourceID, format string, batched, follow bool, tail, since string, timestamps bool) string {+ return strings.Join([]string{+ envID,+ kind,+ resourceID,+ format,+ strconv.FormatBool(batched),+ strconv.FormatBool(follow),+ tail,+ since,+ strconv.FormatBool(timestamps),+ }, "|")+}++func (h *WebSocketHandler) streamProjectLogsInternal(ctx context.Context, projectID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {+ if h.projectLogStreamer != nil {+ return h.projectLogStreamer(ctx, projectID, logsChan, follow, tail, since, timestamps)+ }+ return h.projectService.StreamProjectLogs(ctx, projectID, logsChan, follow, tail, since, timestamps)+}++func (h *WebSocketHandler) streamContainerLogsInternal(ctx context.Context, containerID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {+ if h.containerLogStreamer != nil {+ return h.containerLogStreamer(ctx, containerID, logsChan, follow, tail, since, timestamps)+ }+ return h.containerService.StreamLogs(ctx, containerID, logsChan, follow, tail, since, timestamps)+}++func (h *WebSocketHandler) getOrCreateLogStreamInternal(key string, create func(onEmpty func(*wsLogStream)) *wsLogStream) *wsLogStream {+ h.logStreamsMu.Lock()+ defer h.logStreamsMu.Unlock()++ if stream, ok := h.logStreams[key]; ok {+ if !stream.done {+ stream.refs+++ return stream+ }+ }++ stream := create(func(stream *wsLogStream) {+ h.markLogStreamDoneInternal(key, stream)+ })+ stream.key = key+ stream.refs = 1+ h.logStreams[key] = stream+ return stream+}++func takeLogStreamCancelInternal(stream *wsLogStream) context.CancelFunc {+ cancel := stream.cancel+ stream.cancel = nil+ return cancel+}++func (h *WebSocketHandler) releaseLogStreamInternal(key string, stream *wsLogStream) {+ var cancel context.CancelFunc++ h.logStreamsMu.Lock()+ if stream.refs > 0 {+ stream.refs--+ }+ if stream.refs == 0 {+ if current, ok := h.logStreams[key]; ok && current == stream {+ delete(h.logStreams, key)+ }+ cancel = takeLogStreamCancelInternal(stream)+ }+ h.logStreamsMu.Unlock()++ if cancel != nil {+ cancel()+ }+}++func (h *WebSocketHandler) markLogStreamDoneInternal(key string, stream *wsLogStream) {+ var cancel context.CancelFunc++ h.logStreamsMu.Lock()+ stream.done = true+ if stream.refs == 0 {+ if current, ok := h.logStreams[key]; ok && current == stream {+ delete(h.logStreams, key)+ }+ cancel = takeLogStreamCancelInternal(stream)+ }+ h.logStreamsMu.Unlock()++ if cancel != nil {+ cancel()+ }+}+func NewWebSocketHandler(group *gin.RouterGroup,projectService *services.ProjectService,@@ -219,6 +344,7 @@ func NewWebSocketHandler(containerService: containerService,systemService: systemService,wsMetrics: defaultWebSocketMetrics,+ logStreams: make(map[string]*wsLogStream),gpuMonitoringEnabled: cfg.GPUMonitoringEnabled,gpuType: cfg.GPUType,wsUpgrader: websocket.Upgrader{@@ -228,7 +354,6 @@ func NewWebSocketHandler(EnableCompression: true,},}-wsGroup := group.Group("/environments/:id/ws")wsGroup.Use(authMiddleware.WithAdminNotRequired().Add()){@@ -283,101 +408,162 @@ func (h *WebSocketHandler) ProjectLogs(c *gin.Context) {return}- connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindProjectLogs, projectID))- hub := h.startProjectLogHub(projectID, format, batched, follow, tail, since, timestamps, func() {- h.wsMetrics.UnregisterConnection(connID)+ streamKey := buildLogStreamKeyInternal(c.Param("id"), systemtypes.WSKindProjectLogs, projectID, format, batched, follow, tail, since, timestamps)+ stream := h.getOrCreateLogStreamInternal(streamKey, func(onEmpty func(*wsLogStream)) *wsLogStream {+ return h.startProjectLogHub(streamKey, projectID, format, batched, follow, tail, since, timestamps, onEmpty)})+ connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindProjectLogs, projectID))// WebSocket connections use context.Background() because they are long-lived and should not// be tied to the HTTP request context. Cleanup is handled via the hub's OnEmpty callback// which triggers when all clients disconnect.- ws.ServeClient(context.Background(), hub, conn)+ ws.ServeClientWithOnRemove(context.Background(), stream.hub, conn, func() {+ h.wsMetrics.UnregisterConnection(connID)+ h.releaseLogStreamInternal(streamKey, stream)+ })}-func (h *WebSocketHandler) startProjectLogHub(projectID, format string, batched, follow bool, tail, since string, timestamps bool, onEmptyHook func()) *ws.Hub {+func newWSLogStreamInternal(key, format string) (*wsLogStream, context.Context) {ls := &wsLogStream{- hub: ws.NewHub(1024),- format: format,+ hub: ws.NewHub(1024),+ firstSubscriber: make(chan struct{}),+ format: format,+ key: key,}+ ls.hub.SetOnFirstClient(func() {+ close(ls.firstSubscriber)+ })ctx, cancel := context.WithCancel(context.Background()) //nolint:gosec // cancel is intentionally retained and invoked by the hub OnEmpty callback.ls.cancel = cancel- ls.hub.SetOnEmpty(func() {- if onEmptyHook != nil {- onEmptyHook()- }- slog.Debug("client disconnected, cleaning up project log hub", "projectID", projectID)- cancel()- })-go ls.hub.Run(ctx)+ return ls, ctx+}++func (h *WebSocketHandler) startProjectLogSourceInternal(ctx context.Context, key, projectID, format string, follow bool, tail, since string, timestamps bool, ls *wsLogStream) <-chan string {lines := make(chan string, 256)- go func(ctx context.Context) {++ go func() {defer close(lines)- if err := h.projectService.StreamProjectLogs(ctx, projectID, lines, follow, tail, since, timestamps); err != nil {+ if !waitForLogStreamSubscriberInternal(ctx, ls.firstSubscriber) {+ return+ }++ if err := h.streamProjectLogsInternal(ctx, projectID, lines, follow, tail, since, timestamps); err != nil {if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {return}- slog.Warn("project log stream failed", "projectID", projectID, "error", err)+ h.broadcastProjectLogStreamErrorInternal(projectID, format, err, ls)+ h.markLogStreamDoneInternal(key, ls)+ return+ }- if format == "json" {- msg := ws.LogMessage{- Seq: ls.seq.Add(1),- Level: "error",- Message: "Failed to stream project logs: " + err.Error(),- Service: "arcane",- Timestamp: ws.NowRFC3339(),- }- if b, merr := json.Marshal(msg); merr == nil {- ls.hub.Broadcast(b)- }- return- }+ if ctx.Err() == nil {+ h.markLogStreamDoneInternal(key, ls)+ }+ }()- ls.hub.Broadcast([]byte("Failed to stream project logs: " + err.Error()))+ return lines+}++func waitForLogStreamSubscriberInternal(ctx context.Context, firstSubscriber <-chan struct{}) bool {+ for {+ select {+ case <-ctx.Done():+ return false+ case <-firstSubscriber:+ return true}- }(ctx)+ }+}++func (h *WebSocketHandler) broadcastProjectLogStreamErrorInternal(projectID, format string, err error, ls *wsLogStream) {+ slog.Warn("project log stream failed", "projectID", projectID, "error", err)if format == "json" {- msgs := make(chan ws.LogMessage, 256)- go func() {- defer close(msgs)- for line := range lines {- level, service, msg, ts := ws.NormalizeProjectLine(line)- seq := ls.seq.Add(1)- timestamp := ts- if timestamp == "" {- timestamp = ws.NowRFC3339()- }- msgs <- ws.LogMessage{- Seq: seq,- Level: level,- Message: msg,- Service: service,- Timestamp: timestamp,- }- }- }()- if batched {- go ws.ForwardLogJSONBatched(ctx, ls.hub, msgs, 50, 400*time.Millisecond)- } else {- go ws.ForwardLogJSON(ctx, ls.hub, msgs)+ msg := ws.LogMessage{+ Seq: ls.seq.Add(1),+ Level: "error",+ Message: "Failed to stream project logs: " + err.Error(),+ Service: "arcane",+ Timestamp: ws.NowRFC3339(),}- } else {- cleanChan := make(chan string, 256)- go func() {- defer close(cleanChan)- for line := range lines {- _, _, msg, _ := ws.NormalizeProjectLine(line)- cleanChan <- msg+ if b, marshalErr := json.Marshal(msg); marshalErr == nil {+ ls.hub.Broadcast(b)+ }+ return+ }++ ls.hub.Broadcast([]byte("Failed to stream project logs: " + err.Error()))+}++func startProjectLogForwardersInternal(ctx context.Context, format string, batched bool, lines <-chan string, ls *wsLogStream) {+ if format == "json" {+ startProjectJSONForwarderInternal(ctx, batched, lines, ls)+ return+ }++ startProjectTextForwarderInternal(ctx, lines, ls)+}++func startProjectJSONForwarderInternal(ctx context.Context, batched bool, lines <-chan string, ls *wsLogStream) {+ msgs := make(chan ws.LogMessage, 256)+ go func() {+ defer close(msgs)+ for line := range lines {+ level, service, msg, ts := ws.NormalizeProjectLine(line)+ seq := ls.seq.Add(1)+ timestamp := ts+ if timestamp == "" {+ timestamp = ws.NowRFC3339()}- }()- go ws.ForwardLines(ctx, ls.hub, cleanChan)+ msgs <- ws.LogMessage{+ Seq: seq,+ Level: level,+ Message: msg,+ Service: service,+ Timestamp: timestamp,+ }+ }+ }()++ if batched {+ go ws.ForwardLogJSONBatched(ctx, ls.hub, msgs, 50, 400*time.Millisecond)+ return}- return ls.hub+ go ws.ForwardLogJSON(ctx, ls.hub, msgs)+}++func startProjectTextForwarderInternal(ctx context.Context, lines <-chan string, ls *wsLogStream) {+ cleanChan := make(chan string, 256)+ go func() {+ defer close(cleanChan)+ for line := range lines {+ _, _, msg, _ := ws.NormalizeProjectLine(line)+ cleanChan <- msg+ }+ }()++ go ws.ForwardLines(ctx, ls.hub, cleanChan)+}++func (h *WebSocketHandler) startProjectLogHub(key, projectID, format string, batched, follow bool, tail, since string, timestamps bool, onEmptyHook func(*wsLogStream)) *wsLogStream {+ ls, ctx := newWSLogStreamInternal(key, format)++ ls.hub.SetOnEmpty(func() {+ if onEmptyHook != nil {+ onEmptyHook(ls)+ }+ slog.Debug("client disconnected, cleaning up project log hub", "projectID", projectID)+ })++ lines := h.startProjectLogSourceInternal(ctx, key, projectID, format, follow, tail, since, timestamps, ls)+ startProjectLogForwardersInternal(ctx, format, batched, lines, ls)++ return ls}// ============================================================================@@ -423,45 +609,49 @@ func (h *WebSocketHandler) ContainerLogs(c *gin.Context) {return}- connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindContainerLogs, containerID))- hub := h.startContainerLogHub(containerID, format, batched, follow, tail, since, timestamps, func() {- h.wsMetrics.UnregisterConnection(connID)+ streamKey := buildLogStreamKeyInternal(c.Param("id"), systemtypes.WSKindContainerLogs, containerID, format, batched, follow, tail, since, timestamps)+ stream := h.getOrCreateLogStreamInternal(streamKey, func(onEmpty func(*wsLogStream)) *wsLogStream {+ return h.startContainerLogHub(streamKey, containerID, format, batched, follow, tail, since, timestamps, onEmpty)})+ connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindContainerLogs, containerID))// WebSocket connections use context.Background() because they are long-lived and should not// be tied to the HTTP request context. Cleanup is handled via the hub's OnEmpty callback// which triggers when all clients disconnect.- ws.ServeClient(context.Background(), hub, conn)+ ws.ServeClientWithOnRemove(context.Background(), stream.hub, conn, func() {+ h.wsMetrics.UnregisterConnection(connID)+ h.releaseLogStreamInternal(streamKey, stream)+ })}-func (h *WebSocketHandler) startContainerLogHub(containerID, format string, batched, follow bool, tail, since string, timestamps bool, onEmptyHook func()) *ws.Hub {- ls := &wsLogStream{- hub: ws.NewHub(1024),- format: format,- }-- ctx, cancel := context.WithCancel(context.Background()) //nolint:gosec // cancel is intentionally retained and invoked by the hub OnEmpty callback.- ls.cancel = cancel+func (h *WebSocketHandler) startContainerLogHub(key, containerID, format string, batched, follow bool, tail, since string, timestamps bool, onEmptyHook func(*wsLogStream)) *wsLogStream {+ ls, ctx := newWSLogStreamInternal(key, format)ls.hub.SetOnEmpty(func() {if onEmptyHook != nil {- onEmptyHook()+ onEmptyHook(ls)}slog.Debug("client disconnected, cleaning up container log hub", "containerID", containerID)- cancel()})- go ls.hub.Run(ctx)-lines := make(chan string, 256)go func(ctx context.Context) {defer close(lines)- if err := h.containerService.StreamLogs(ctx, containerID, lines, follow, tail, since, timestamps); err != nil {+ if !waitForLogStreamSubscriberInternal(ctx, ls.firstSubscriber) {+ return+ }++ if err := h.streamContainerLogsInternal(ctx, containerID, lines, follow, tail, since, timestamps); err != nil {if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {return}- slog.Warn("container log stream failed", "containerID", containerID, "error", err)- broadcastContainerLogStreamErrorInternal(ls, err)+ h.broadcastContainerLogStreamErrorInternal(containerID, format, err, ls)+ h.markLogStreamDoneInternal(key, ls)+ return+ }++ if ctx.Err() == nil {+ h.markLogStreamDoneInternal(key, ls)}}(ctx)@@ -493,29 +683,27 @@ func (h *WebSocketHandler) startContainerLogHub(containerID, format string, batcgo ws.ForwardLines(ctx, ls.hub, lines)}- return ls.hub+ return ls}-func broadcastContainerLogStreamErrorInternal(stream *wsLogStream, err error) {- if stream == nil || stream.hub == nil || err == nil {- return- }+func (h *WebSocketHandler) broadcastContainerLogStreamErrorInternal(containerID, format string, err error, ls *wsLogStream) {+ slog.Warn("container log stream failed", "containerID", containerID, "error", err)- message := "Failed to stream container logs: " + err.Error()- if stream.format == "json" {+ if format == "json" {msg := ws.LogMessage{- Seq: stream.seq.Add(1),+ Seq: ls.seq.Add(1),Level: "error",- Message: message,+ Message: "Failed to stream container logs: " + err.Error(),+ Service: "arcane",Timestamp: ws.NowRFC3339(),}if b, marshalErr := json.Marshal(msg); marshalErr == nil {- stream.hub.Broadcast(b)+ ls.hub.Broadcast(b)}return}- stream.hub.Broadcast([]byte(message))+ ls.hub.Broadcast([]byte("Failed to stream container logs: " + err.Error()))}// ContainerStats streams container stats over WebSocket.@@ -740,23 +928,115 @@ func (h *WebSocketHandler) releaseRateLimit(clientIP string, count *int32) {}}-// startCPUSampler starts a background goroutine that samples CPU usage.-func (h *WebSocketHandler) startCPUSampler(ctx context.Context, ticker *time.Ticker) {- go func(ctx context.Context) {- for {- select {- case <-ctx.Done():- return- case <-ticker.C:- if vals, err := cpu.Percent(0, false); err == nil && len(vals) > 0 {- h.cpuCache.Lock()- h.cpuCache.value = vals[0]- h.cpuCache.timestamp = time.Now()- h.cpuCache.Unlock()- }- }+func (h *WebSocketHandler) acquireSystemStatsSamplerInternal(ctx context.Context) bool {+ h.systemStatsSampler.lifecycleMu.Lock()++ h.systemStatsSampler.clients+++ if h.systemStatsSampler.running {+ ready := h.systemStatsSampler.ready+ h.systemStatsSampler.lifecycleMu.Unlock()+ return waitForSystemStatsSamplerReadyInternal(ctx, ready)+ }++ samplerCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) //nolint:gosec // cancel is intentionally retained in sampler state and invoked when the last subscriber disconnects.+ ready := make(chan struct{})+ h.systemStatsSampler.cancel = cancel+ h.systemStatsSampler.ready = ready+ h.systemStatsSampler.running = true+ h.systemStatsSampler.lifecycleMu.Unlock()++ go func() {+ closeReady := sync.OnceFunc(func() {+ close(ready)+ })+ if !h.initializeCPUCacheCtx(samplerCtx) {+ closeReady()+ return}- }(ctx)+ if samplerCtx.Err() != nil {+ closeReady()+ return+ }++ h.storeSystemStatsSnapshotInternal(h.collectSystemStatsSnapshotInternal(samplerCtx))+ closeReady()+ if samplerCtx.Err() != nil {+ return+ }++ h.runSystemStatsSamplerInternal(samplerCtx)+ }()++ return waitForSystemStatsSamplerReadyInternal(ctx, ready)+}++func waitForSystemStatsSamplerReadyInternal(ctx context.Context, ready <-chan struct{}) bool {+ if ready == nil {+ return true+ }++ select {+ case <-ctx.Done():+ return false+ case <-ready:+ return ctx.Err() == nil+ }+}++func (h *WebSocketHandler) releaseSystemStatsSamplerInternal() {+ var cancel context.CancelFunc++ h.systemStatsSampler.lifecycleMu.Lock()+ if h.systemStatsSampler.clients > 0 {+ h.systemStatsSampler.clients--+ }+ if h.systemStatsSampler.clients == 0 && h.systemStatsSampler.running {+ cancel = h.systemStatsSampler.cancel+ h.systemStatsSampler.cancel = nil+ h.systemStatsSampler.ready = nil+ h.systemStatsSampler.running = false+ }+ h.systemStatsSampler.lifecycleMu.Unlock()++ if cancel != nil {+ cancel()+ }+}++func (h *WebSocketHandler) runSystemStatsSamplerInternal(ctx context.Context) {+ ticker := time.NewTicker(time.Second)+ defer ticker.Stop()++ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ h.updateCPUCacheInternal(0)+ h.storeSystemStatsSnapshotInternal(h.collectSystemStatsSnapshotInternal(ctx))+ }+ }+}++func (h *WebSocketHandler) storeSystemStatsSnapshotInternal(stats systemtypes.SystemStats) {+ h.systemStatsSampler.stateMu.Lock()+ h.systemStatsSampler.latest = stats+ h.systemStatsSampler.timestamp = time.Now()+ h.systemStatsSampler.stateMu.Unlock()+}++func (h *WebSocketHandler) latestSystemStatsSnapshotInternal() systemtypes.SystemStats {+ h.systemStatsSampler.stateMu.RLock()+ stats := h.systemStatsSampler.latest+ h.systemStatsSampler.stateMu.RUnlock()+ return stats+}++func (h *WebSocketHandler) collectSystemStatsSnapshotInternal(ctx context.Context) systemtypes.SystemStats {+ if h.systemStatsCollector != nil {+ return h.systemStatsCollector(ctx)+ }+ return h.collectSystemStats(ctx)}// collectSystemStats gathers all system statistics.@@ -789,11 +1069,26 @@ func (h *WebSocketHandler) collectSystemStats(ctx context.Context) systemtypes.S// getCPUCount returns the number of CPUs.func (h *WebSocketHandler) getCPUCount() int {- cpuCount, err := cpu.Counts(true)- if err != nil {- return runtime.NumCPU()- }- return cpuCount+ h.initSystemStaticInfoInternal()+ return h.systemStaticInfo.cpuCount+}++func (h *WebSocketHandler) initSystemStaticInfoInternal() {+ h.systemStaticInfo.once.Do(func() {+ cpuCount, err := cpu.Counts(true)+ if err != nil {+ cpuCount = runtime.NumCPU()+ }++ hostInfo, _ := host.Info()+ hostname := ""+ if hostInfo != nil {+ hostname = hostInfo.Hostname+ }++ h.systemStaticInfo.cpuCount = cpuCount+ h.systemStaticInfo.hostname = hostname+ })}// getMemoryInfo returns memory usage and total.@@ -807,8 +1102,8 @@ func (h *WebSocketHandler) getMemoryInfo() (uint64, uint64) {// applyCgroupLimits applies cgroup limits when running in a container.func (h *WebSocketHandler) applyCgroupLimits(cpuCount int, memUsed, memTotal uint64) (int, uint64, uint64) {- cgroupLimits, err := docker.DetectCgroupLimits()- if err != nil {+ cgroupLimits := h.getCachedCgroupLimitsInternal()+ if cgroupLimits == nil {return cpuCount, memUsed, memTotal}@@ -844,11 +1139,8 @@ func (h *WebSocketHandler) getDiskInfo(ctx context.Context) (uint64, uint64) {// getHostname returns the system hostname.func (h *WebSocketHandler) getHostname() string {- hostInfo, _ := host.Info()- if hostInfo == nil {- return ""- }- return hostInfo.Hostname+ h.initSystemStaticInfoInternal()+ return h.systemStaticInfo.hostname}// getGPUInfo returns GPU statistics if monitoring is enabled.@@ -863,16 +1155,102 @@ func (h *WebSocketHandler) getGPUInfo(ctx context.Context) ([]systemtypes.GPUStareturn gpuData, len(gpuData)}-// initializeCPUCache performs initial CPU sampling.-func (h *WebSocketHandler) initializeCPUCache() {- if vals, err := cpu.Percent(time.Second, false); err == nil && len(vals) > 0 {- h.cpuCache.Lock()- h.cpuCache.value = vals[0]- h.cpuCache.timestamp = time.Now()- h.cpuCache.Unlock()+// initializeCPUCacheCtx performs initial CPU sampling and returns early if the sampler is canceled.+func (h *WebSocketHandler) initializeCPUCacheCtx(ctx context.Context) bool {+ result := make(chan float64, 1)++ go func() {+ if val, ok := h.readCPUUsageInternal(time.Second); ok {+ result <- val+ }+ close(result)+ }()++ select {+ case <-ctx.Done():+ return false+ case val, ok := <-result:+ if !ok || ctx.Err() != nil {+ return false+ }+ h.storeCPUCacheValueInternal(val)+ return true+ }+}++func (h *WebSocketHandler) updateCPUCacheInternal(interval time.Duration) {+ if val, ok := h.readCPUUsageInternal(interval); ok {+ h.storeCPUCacheValueInternal(val)}}+func (h *WebSocketHandler) readCPUUsageInternal(interval time.Duration) (float64, bool) {+ if h.cpuUsageReader != nil {+ return h.cpuUsageReader(interval)+ }++ return defaultReadCPUUsageInternal(interval)+}++var defaultReadCPUUsageInternal = func(interval time.Duration) (float64, bool) {+ if vals, err := cpu.Percent(interval, false); err == nil && len(vals) > 0 {+ return vals[0], true+ }++ return 0, false+}++func (h *WebSocketHandler) storeCPUCacheValueInternal(value float64) {+ h.cpuCache.Lock()+ h.cpuCache.value = value+ h.cpuCache.timestamp = time.Now()+ h.cpuCache.Unlock()+}++func (h *WebSocketHandler) getCachedCgroupLimitsInternal() *docker.CgroupLimits {+ h.cgroupCache.RLock()+ value := h.cgroupCache.value+ detected := h.cgroupCache.detected+ fresh := time.Since(h.cgroupCache.timestamp) < cgroupCacheTTL+ h.cgroupCache.RUnlock()+ if fresh {+ if detected {+ return value+ }+ return nil+ }++ h.cgroupCache.Lock()+ defer h.cgroupCache.Unlock()++ if time.Since(h.cgroupCache.timestamp) < cgroupCacheTTL {+ if h.cgroupCache.detected {+ return h.cgroupCache.value+ }+ return nil+ }++ detect := h.cgroupLimitsDetector+ if detect == nil {+ detect = docker.DetectCgroupLimits+ }++ limits, err := detect()+ h.cgroupCache.timestamp = time.Now()+ if err != nil {+ h.cgroupCache.value = nil+ h.cgroupCache.detected = false+ } else {+ h.cgroupCache.value = limits+ h.cgroupCache.detected = true+ }+ if !h.cgroupCache.detected {+ return nil+ }++ return h.cgroupCache.value+}+// SystemStats streams system stats over WebSocket.//// @Summary Get system stats via WebSocket@@ -924,24 +1302,22 @@ func (h *WebSocketHandler) SystemStats(c *gin.Context) {pingTicker := time.NewTicker(statsPingPeriod)defer pingTicker.Stop()- cpuUpdateTicker := time.NewTicker(1 * time.Second)- defer cpuUpdateTicker.Stop()-ctx, cancel := context.WithCancel(c.Request.Context())defer cancel()+ if !h.acquireSystemStatsSamplerInternal(ctx) {+ h.releaseSystemStatsSamplerInternal()+ return+ }+ defer h.releaseSystemStatsSamplerInternal()go h.readSystemStatsPumpInternal(ctx, cancel, conn)- h.startCPUSampler(ctx, cpuUpdateTicker)-send := func() error {- stats := h.collectSystemStats(ctx)+ stats := h.latestSystemStatsSnapshotInternal()_ = conn.SetWriteDeadline(time.Now().Add(10 * time.Second))return conn.WriteJSON(stats)}- h.initializeCPUCache()-if err := send(); err != nil {return}diff --git a/backend/internal/api/ws_handler_test.go b/backend/internal/api/ws_handler_test.goindex a7822800f..5519f2240 100644--- a/backend/internal/api/ws_handler_test.go+++ b/backend/internal/api/ws_handler_test.go@@ -1,15 +1,21 @@package apiimport (+ "context""encoding/json""errors""net/http""net/http/httptest""strings"+ "sync"+ "sync/atomic""testing""time"+ docker "github.com/getarcaneapp/arcane/backend/pkg/dockerutil"ws "github.com/getarcaneapp/arcane/backend/pkg/libarcane/ws"+ systemtypes "github.com/getarcaneapp/arcane/types/system"+ "github.com/gin-gonic/gin""github.com/gorilla/websocket""github.com/stretchr/testify/require")@@ -29,7 +35,8 @@ func TestBroadcastContainerLogStreamErrorInternal_JSON(t *testing.T) {format: "json",}- broadcastContainerLogStreamErrorInternal(stream, errors.New("boom"))+ handler := newTestWebSocketHandler()+ handler.broadcastContainerLogStreamErrorInternal("container-1", "json", errors.New("boom"), stream)_ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))_, payload, err := clientConn.ReadMessage()@@ -58,7 +65,8 @@ func TestBroadcastContainerLogStreamErrorInternal_Text(t *testing.T) {format: "text",}- broadcastContainerLogStreamErrorInternal(stream, errors.New("boom"))+ handler := newTestWebSocketHandler()+ handler.broadcastContainerLogStreamErrorInternal("container-1", "text", errors.New("boom"), stream)_ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))_, payload, err := clientConn.ReadMessage()@@ -94,3 +102,523 @@ func newTestWSPairInternal(t *testing.T) (clientConn *websocket.Conn, serverConnserver.Close()}}++func newTestWebSocketHandler() *WebSocketHandler {+ return &WebSocketHandler{+ wsMetrics: NewWebSocketMetrics(),+ logStreams: make(map[string]*wsLogStream),+ wsUpgrader: websocket.Upgrader{+ CheckOrigin: func(r *http.Request) bool { return true },+ },+ }+}++func dialWebSocket(t *testing.T, serverURL, path string) *websocket.Conn {+ t.Helper()++ wsURL := "ws" + strings.TrimPrefix(serverURL, "http") + path+ conn, resp, err := websocket.DefaultDialer.Dial(wsURL, nil)+ require.NoError(t, err)+ if resp != nil {+ _ = resp.Body.Close()+ }++ return conn+}++func TestWebSocketHandler_ProjectLogs_SharedStreamPerTarget(t *testing.T) {+ gin.SetMode(gin.TestMode)++ handler := newTestWebSocketHandler()+ var starts atomic.Int32+ var cancels atomic.Int32++ handler.projectLogStreamer = func(ctx context.Context, projectID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {+ starts.Add(1)+ ticker := time.NewTicker(20 * time.Millisecond)+ defer ticker.Stop()+ defer cancels.Add(1)++ for {+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-ticker.C:+ select {+ case <-ctx.Done():+ return ctx.Err()+ case logsChan <- "api | shared project log":+ }+ }+ }+ }++ router := gin.New()+ router.GET("/api/environments/:id/ws/projects/:projectId/logs", handler.ProjectLogs)+ server := httptest.NewServer(router)+ defer server.Close()++ conn1 := dialWebSocket(t, server.URL, "/api/environments/0/ws/projects/project-1/logs")+ conn2 := dialWebSocket(t, server.URL, "/api/environments/0/ws/projects/project-1/logs")++ _ = conn1.SetReadDeadline(time.Now().Add(2 * time.Second))+ _, _, err := conn1.ReadMessage()+ require.NoError(t, err)++ _ = conn2.SetReadDeadline(time.Now().Add(2 * time.Second))+ _, _, err = conn2.ReadMessage()+ require.NoError(t, err)++ require.Eventually(t, func() bool {+ return starts.Load() == 1+ }, 2*time.Second, 20*time.Millisecond)++ require.Eventually(t, func() bool {+ handler.logStreamsMu.Lock()+ defer handler.logStreamsMu.Unlock()+ return len(handler.logStreams) == 1+ }, time.Second, 20*time.Millisecond)++ require.NoError(t, conn1.Close())++ require.Eventually(t, func() bool {+ handler.logStreamsMu.Lock()+ defer handler.logStreamsMu.Unlock()+ return len(handler.logStreams) == 1+ }, 2*time.Second, 10*time.Millisecond)++ handler.logStreamsMu.Lock()+ activeAfterFirstClose := len(handler.logStreams)+ handler.logStreamsMu.Unlock()+ require.Equal(t, 1, activeAfterFirstClose)+ require.Equal(t, int32(0), cancels.Load())++ require.NoError(t, conn2.Close())++ require.Eventually(t, func() bool {+ handler.logStreamsMu.Lock()+ defer handler.logStreamsMu.Unlock()+ return len(handler.logStreams) == 0+ }, 2*time.Second, 20*time.Millisecond)+ require.Eventually(t, func() bool {+ return cancels.Load() == 1+ }, 2*time.Second, 20*time.Millisecond)+}++func TestWebSocketHandler_ProjectLogs_CompletedSourceStartsFreshStream(t *testing.T) {+ gin.SetMode(gin.TestMode)++ handler := newTestWebSocketHandler()+ var starts atomic.Int32+ firstDone := make(chan struct{})++ handler.projectLogStreamer = func(ctx context.Context, projectID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {+ call := starts.Add(1)+ select {+ case <-ctx.Done():+ return ctx.Err()+ case logsChan <- "api | finite project log":+ }+ if call == 1 {+ close(firstDone)+ }+ return nil+ }++ router := gin.New()+ router.GET("/api/environments/:id/ws/projects/:projectId/logs", handler.ProjectLogs)+ server := httptest.NewServer(router)+ defer server.Close()++ path := "/api/environments/0/ws/projects/project-1/logs?follow=false"+ conn1 := dialWebSocket(t, server.URL, path)+ defer conn1.Close()++ _ = conn1.SetReadDeadline(time.Now().Add(2 * time.Second))+ _, msg1, err := conn1.ReadMessage()+ require.NoError(t, err)+ require.Equal(t, "finite project log", string(msg1))++ select {+ case <-firstDone:+ case <-time.After(2 * time.Second):+ t.Fatal("first finite log stream did not complete")+ }++ conn2 := dialWebSocket(t, server.URL, path)+ defer conn2.Close()++ _ = conn2.SetReadDeadline(time.Now().Add(2 * time.Second))+ _, msg2, err := conn2.ReadMessage()+ require.NoError(t, err)+ require.Equal(t, "finite project log", string(msg2))++ require.Eventually(t, func() bool {+ return starts.Load() == 2+ }, 2*time.Second, 20*time.Millisecond)+}++func TestWebSocketHandler_ContainerLogs_BroadcastsStreamErrors(t *testing.T) {+ gin.SetMode(gin.TestMode)++ handler := newTestWebSocketHandler()+ handler.containerLogStreamer = func(ctx context.Context, containerID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {+ select {+ case <-ctx.Done():+ return ctx.Err()+ case logsChan <- "api | container log":+ }+ return errors.New("stream failed")+ }++ router := gin.New()+ router.GET("/api/environments/:id/ws/containers/:containerId/logs", handler.ContainerLogs)+ server := httptest.NewServer(router)+ defer server.Close()++ conn := dialWebSocket(t, server.URL, "/api/environments/0/ws/containers/container-1/logs")+ defer conn.Close()++ var got []string++ _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))+ _, msg, err := conn.ReadMessage()+ require.NoError(t, err)+ got = append(got, string(msg))++ _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))+ _, msg, err = conn.ReadMessage()+ require.NoError(t, err)+ got = append(got, string(msg))++ require.ElementsMatch(t, []string{+ "api | container log",+ "Failed to stream container logs: stream failed",+ }, got)+}++func TestWebSocketHandler_ContainerLogs_ErrorStartsFreshStreamForNewSubscribers(t *testing.T) {+ gin.SetMode(gin.TestMode)++ handler := newTestWebSocketHandler()+ var starts atomic.Int32+ handler.containerLogStreamer = func(ctx context.Context, containerID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {+ starts.Add(1)+ select {+ case <-ctx.Done():+ return ctx.Err()+ case logsChan <- "api | container log":+ }+ return errors.New("stream failed")+ }++ router := gin.New()+ router.GET("/api/environments/:id/ws/containers/:containerId/logs", handler.ContainerLogs)+ server := httptest.NewServer(router)+ defer server.Close()++ path := "/api/environments/0/ws/containers/container-1/logs"+ conn1 := dialWebSocket(t, server.URL, path)+ defer conn1.Close()++ got1 := make([]string, 0, 2)+ for range 2 {+ _ = conn1.SetReadDeadline(time.Now().Add(2 * time.Second))+ _, msg, err := conn1.ReadMessage()+ require.NoError(t, err)+ got1 = append(got1, string(msg))+ }+ require.ElementsMatch(t, []string{+ "api | container log",+ "Failed to stream container logs: stream failed",+ }, got1)++ conn2 := dialWebSocket(t, server.URL, path)+ defer conn2.Close()++ got2 := make([]string, 0, 2)+ for range 2 {+ _ = conn2.SetReadDeadline(time.Now().Add(2 * time.Second))+ _, msg, err := conn2.ReadMessage()+ require.NoError(t, err)+ got2 = append(got2, string(msg))+ }+ require.ElementsMatch(t, []string{+ "api | container log",+ "Failed to stream container logs: stream failed",+ }, got2)++ require.Eventually(t, func() bool {+ return starts.Load() == 2+ }, 2*time.Second, 20*time.Millisecond)+}++func TestWebSocketHandler_SystemStats_UsesSharedSampler(t *testing.T) {+ gin.SetMode(gin.TestMode)++ handler := newTestWebSocketHandler()+ var collects atomic.Int32++ handler.systemStatsCollector = func(ctx context.Context) systemtypes.SystemStats {+ n := collects.Add(1)+ return systemtypes.SystemStats{+ CPUUsage: float64(n),+ }+ }++ router := gin.New()+ router.GET("/api/environments/:id/ws/system/stats", handler.SystemStats)+ server := httptest.NewServer(router)+ defer server.Close()++ conn1 := dialWebSocket(t, server.URL, "/api/environments/0/ws/system/stats?interval=1")+ conn2 := dialWebSocket(t, server.URL, "/api/environments/0/ws/system/stats?interval=1")++ _ = conn1.SetReadDeadline(time.Now().Add(2 * time.Second))+ _, _, err := conn1.ReadMessage()+ require.NoError(t, err)++ _ = conn2.SetReadDeadline(time.Now().Add(2 * time.Second))+ _, _, err = conn2.ReadMessage()+ require.NoError(t, err)++ require.Eventually(t, func() bool {+ return collects.Load() >= 1+ }, 2*time.Second, 50*time.Millisecond)++ require.NoError(t, conn1.Close())+ require.NoError(t, conn2.Close())++ require.Eventually(t, func() bool {+ handler.systemStatsSampler.lifecycleMu.Lock()+ defer handler.systemStatsSampler.lifecycleMu.Unlock()+ return !handler.systemStatsSampler.running && handler.systemStatsSampler.clients == 0+ }, 2*time.Second, 20*time.Millisecond)++ stoppedAt := collects.Load()+ require.Never(t, func() bool {+ return collects.Load() != stoppedAt+ }, 1200*time.Millisecond, 100*time.Millisecond)+}++func TestWebSocketHandler_AcquireSystemStatsSampler_WaitsForInitialSnapshot(t *testing.T) {+ handler := newTestWebSocketHandler()+ handler.systemStatsCollector = func(ctx context.Context) systemtypes.SystemStats {+ return systemtypes.SystemStats{CPUUsage: 42}+ }++ firstDone := make(chan struct{})+ go func() {+ handler.acquireSystemStatsSamplerInternal(context.Background())+ close(firstDone)+ }()++ require.Eventually(t, func() bool {+ handler.systemStatsSampler.lifecycleMu.Lock()+ defer handler.systemStatsSampler.lifecycleMu.Unlock()+ return handler.systemStatsSampler.running && handler.systemStatsSampler.ready != nil+ }, 500*time.Millisecond, 10*time.Millisecond)++ secondDone := make(chan struct{})+ go func() {+ handler.acquireSystemStatsSamplerInternal(context.Background())+ close(secondDone)+ }()++ require.Never(t, func() bool {+ select {+ case <-secondDone:+ return true+ default:+ return false+ }+ }, 200*time.Millisecond, 20*time.Millisecond)++ select {+ case <-firstDone:+ case <-time.After(2 * time.Second):+ t.Fatal("first sampler acquisition did not finish")+ }++ select {+ case <-secondDone:+ case <-time.After(2 * time.Second):+ t.Fatal("second sampler acquisition did not wait for readiness")+ }++ stats := handler.latestSystemStatsSnapshotInternal()+ require.Equal(t, 42.0, stats.CPUUsage)++ handler.releaseSystemStatsSamplerInternal()+ handler.releaseSystemStatsSamplerInternal()++ require.Eventually(t, func() bool {+ handler.systemStatsSampler.lifecycleMu.Lock()+ defer handler.systemStatsSampler.lifecycleMu.Unlock()+ return !handler.systemStatsSampler.running && handler.systemStatsSampler.clients == 0 && handler.systemStatsSampler.ready == nil+ }, 2*time.Second, 20*time.Millisecond)+}++func TestWebSocketHandler_AcquireSystemStatsSampler_StopsWaitingWhenCallerCancels(t *testing.T) {+ handler := newTestWebSocketHandler()++ readyToCancel := make(chan struct{})+ handler.systemStatsCollector = func(ctx context.Context) systemtypes.SystemStats {+ return systemtypes.SystemStats{CPUUsage: 42}+ }++ handler.cpuUsageReader = func(interval time.Duration) (float64, bool) {+ close(readyToCancel)+ time.Sleep(200 * time.Millisecond)+ return 42, true+ }++ ctx, cancel := context.WithCancel(context.Background())+ done := make(chan bool, 1)+ go func() {+ done <- handler.acquireSystemStatsSamplerInternal(ctx)+ }()++ select {+ case <-readyToCancel:+ case <-time.After(time.Second):+ t.Fatal("sampler did not start CPU initialization")+ }++ cancel()++ select {+ case ok := <-done:+ require.False(t, ok)+ case <-time.After(100 * time.Millisecond):+ t.Fatal("sampler acquisition did not stop waiting after cancellation")+ }++ handler.releaseSystemStatsSamplerInternal()++ require.Eventually(t, func() bool {+ handler.systemStatsSampler.lifecycleMu.Lock()+ defer handler.systemStatsSampler.lifecycleMu.Unlock()+ return !handler.systemStatsSampler.running && handler.systemStatsSampler.clients == 0+ }, time.Second, 20*time.Millisecond)+}++func TestWebSocketHandler_LogStream_ReplacesDoneStreamAndCleansStaleRefs(t *testing.T) {+ handler := newTestWebSocketHandler()+ key := "env|project|resource|text|false|true|100||false"++ var staleCancels atomic.Int32+ stale := &wsLogStream{+ hub: ws.NewHub(1),+ cancel: func() { staleCancels.Add(1) },+ refs: 2,+ done: true,+ }+ handler.logStreams[key] = stale++ var freshCancels atomic.Int32+ fresh := handler.getOrCreateLogStreamInternal(key, func(onEmpty func(*wsLogStream)) *wsLogStream {+ return &wsLogStream{+ hub: ws.NewHub(1),+ cancel: func() { freshCancels.Add(1) },+ }+ })++ require.NotSame(t, stale, fresh)+ handler.logStreamsMu.Lock()+ require.Same(t, fresh, handler.logStreams[key])+ handler.logStreamsMu.Unlock()++ handler.markLogStreamDoneInternal(key, stale)+ handler.logStreamsMu.Lock()+ require.Same(t, fresh, handler.logStreams[key])+ handler.logStreamsMu.Unlock()++ handler.releaseLogStreamInternal(key, stale)+ require.Equal(t, 1, stale.refs)+ handler.logStreamsMu.Lock()+ require.Same(t, fresh, handler.logStreams[key])+ handler.logStreamsMu.Unlock()++ handler.releaseLogStreamInternal(key, stale)+ require.Equal(t, int32(1), staleCancels.Load())+ handler.logStreamsMu.Lock()+ require.Same(t, fresh, handler.logStreams[key])+ handler.logStreamsMu.Unlock()++ handler.releaseLogStreamInternal(key, fresh)+ require.Equal(t, int32(1), freshCancels.Load())+ handler.logStreamsMu.Lock()+ _, ok := handler.logStreams[key]+ handler.logStreamsMu.Unlock()+ require.False(t, ok)+}++func TestWebSocketHandler_LogStream_CancelsOnlyOnce(t *testing.T) {+ handler := newTestWebSocketHandler()+ key := "env|project|resource|text|false|true|100||false"++ var cancels atomic.Int32+ stream := &wsLogStream{+ hub: ws.NewHub(1),+ cancel: func() { cancels.Add(1) },+ refs: 1,+ }+ handler.logStreams[key] = stream++ handler.markLogStreamDoneInternal(key, stream)+ handler.releaseLogStreamInternal(key, stream)+ handler.markLogStreamDoneInternal(key, stream)++ require.Equal(t, int32(1), cancels.Load())+}++func TestWebSocketHandler_GetCachedCgroupLimitsInternal_DeduplicatesRefresh(t *testing.T) {+ handler := newTestWebSocketHandler()+ handler.cgroupCache.timestamp = time.Now().Add(-2 * cgroupCacheTTL)++ var calls atomic.Int32+ start := make(chan struct{})+ release := make(chan struct{})+ handler.cgroupLimitsDetector = func() (*docker.CgroupLimits, error) {+ calls.Add(1)+ close(start)+ <-release+ return &docker.CgroupLimits{CPUCount: 2}, nil+ }++ const goroutines = 8+ results := make(chan *docker.CgroupLimits, goroutines)+ ready := make(chan struct{})+ var entered sync.WaitGroup+ entered.Add(goroutines)+ var wg sync.WaitGroup+ for range goroutines {+ wg.Go(func() {+ <-ready+ entered.Done()+ results <- handler.getCachedCgroupLimitsInternal()+ })+ }++ close(ready)+ entered.Wait()++ select {+ case <-start:+ case <-time.After(2 * time.Second):+ t.Fatal("detector was not called")+ }++ require.Equal(t, int32(1), calls.Load())++ close(release)+ wg.Wait()+ close(results)++ for result := range results {+ require.NotNil(t, result)+ require.Equal(t, 2, result.CPUCount)+ }+ require.Equal(t, int32(1), calls.Load())+}diff --git a/backend/internal/models/settings.go b/backend/internal/models/settings.goindex 350e17633..21a0eaaee 100644--- a/backend/internal/models/settings.go+++ b/backend/internal/models/settings.go@@ -5,9 +5,9 @@ import ("errors""fmt""reflect"- "slices""strconv""strings"+ "sync""time")@@ -21,6 +21,21 @@ type SettingVariable struct {Value string}+type settingFieldMeta struct {+ index int+ key string+ attrs string+ isPublic bool+ isSensitive bool+ isLocal bool+}++var settingsFieldCache struct {+ once sync.Once+ ordered []settingFieldMeta+ byKey map[string]settingFieldMeta+}+// IsTrue returns true if the value is a truthy stringfunc (s SettingVariable) IsTrue() bool {ok, _ := strconv.ParseBool(s.Value)@@ -154,29 +169,63 @@ func (SettingVariable) TableName() string {return "settings"}-func (s *Settings) ToSettingVariableSlice(showAll bool, redactSensitiveValues bool) []SettingVariable {- cfgValue := reflect.ValueOf(s).Elem()- cfgType := cfgValue.Type()-- var res []SettingVariable-- for i := 0; i < cfgType.NumField(); i++ {- field := cfgType.Field(i)+func buildSettingsFieldCacheInternal() {+ rt := reflect.TypeFor[Settings]()+ ordered := make([]settingFieldMeta, 0, rt.NumField())+ byKey := make(map[string]settingFieldMeta, rt.NumField())+ for i := 0; i < rt.NumField(); i++ {+ field := rt.Field(i)key, attrs, _ := strings.Cut(field.Tag.Get("key"), ",")if key == "" {continue}- if !showAll && !strings.Contains(attrs, "public") {+ meta := settingFieldMeta{+ index: i,+ key: key,+ attrs: attrs,+ isPublic: strings.Contains(attrs, "public"),+ isSensitive: strings.Contains(attrs, "sensitive"),+ isLocal: strings.Contains(attrs, "local"),+ }+ ordered = append(ordered, meta)+ byKey[key] = meta+ }++ settingsFieldCache.ordered = ordered+ settingsFieldCache.byKey = byKey+}++func getSettingsFieldCacheInternal() ([]settingFieldMeta, map[string]settingFieldMeta) {+ settingsFieldCache.once.Do(buildSettingsFieldCacheInternal)+ return settingsFieldCache.ordered, settingsFieldCache.byKey+}++func (s *Settings) Clone() *Settings {+ if s == nil {+ return &Settings{}+ }++ clone := *s+ return &clone+}++func (s *Settings) ToSettingVariableSlice(showAll bool, redactSensitiveValues bool) []SettingVariable {+ cfgValue := reflect.ValueOf(s).Elem()+ fields, _ := getSettingsFieldCacheInternal()++ res := make([]SettingVariable, 0, len(fields))+ for _, field := range fields {+ if !showAll && !field.isPublic {continue}- value := cfgValue.Field(i).FieldByName("Value").String()- value = redactSettingValue(key, value, attrs, redactSensitiveValues)+ value := cfgValue.Field(field.index).FieldByName("Value").String()+ value = redactSettingValue(field.key, value, field.attrs, redactSensitiveValues)settingVariable := SettingVariable{- Key: key,+ Key: field.key,Value: value,}res = append(res, settingVariable)@@ -187,65 +236,47 @@ func (s *Settings) ToSettingVariableSlice(showAll bool, redactSensitiveValues bofunc (s *Settings) FieldByKey(key string) (defaultValue string, isPublic bool, isSensitive bool, err error) {rv := reflect.ValueOf(s).Elem()- rt := rv.Type()-- for i := 0; i < rt.NumField(); i++ {- tagValue := strings.Split(rt.Field(i).Tag.Get("key"), ",")- keyFromTag := tagValue[0]- isPublic = slices.Contains(tagValue, "public")- isSensitive = slices.Contains(tagValue, "sensitive")+ _, byKey := getSettingsFieldCacheInternal()- if keyFromTag != key {- continue- }-- valueField := rv.Field(i).FieldByName("Value")- return valueField.String(), isPublic, isSensitive, nil+ field, ok := byKey[key]+ if !ok {+ return "", false, false, SettingKeyNotFoundError{field: key}}- return "", false, false, SettingKeyNotFoundError{field: key}+ valueField := rv.Field(field.index).FieldByName("Value")+ return valueField.String(), field.isPublic, field.isSensitive, nil}func (s *Settings) IsLocalSetting(key string) bool {- rt := reflect.TypeFor[Settings]()-- for field := range rt.Fields() {- tagValue := strings.Split(field.Tag.Get("key"), ",")- keyFromTag := tagValue[0]-- if keyFromTag == key {- return slices.Contains(tagValue, "local")- }+ _, byKey := getSettingsFieldCacheInternal()+ field, ok := byKey[key]+ if !ok {+ return false}- return false+ return field.isLocal}func (s *Settings) UpdateField(key string, value string, noSensitive bool) error {rv := reflect.ValueOf(s).Elem()- rt := rv.Type()-- for i := 0; i < rt.NumField(); i++ {- tagValue, attrs, _ := strings.Cut(rt.Field(i).Tag.Get("key"), ",")- if tagValue != key {- continue- }+ _, byKey := getSettingsFieldCacheInternal()- // If the field is sensitive and noSensitive is true, we skip that- if noSensitive && strings.Contains(attrs, "sensitive") {- return SettingSensitiveForbiddenError{field: key}- }+ field, ok := byKey[key]+ if !ok {+ return SettingKeyNotFoundError{field: key}+ }- valueField := rv.Field(i).FieldByName("Value")- if !valueField.CanSet() {- return fmt.Errorf("field Value in SettingVariable is not settable for config key '%s'", key)- }+ if noSensitive && field.isSensitive {+ return SettingSensitiveForbiddenError{field: key}+ }- valueField.SetString(value)- return nil+ valueField := rv.Field(field.index).FieldByName("Value")+ if !valueField.CanSet() {+ return fmt.Errorf("field Value in SettingVariable is not settable for config key '%s'", key)}- return SettingKeyNotFoundError{field: key}+ valueField.SetString(value)+ return nil}// helper keeps redaction logic in one place; behavior unchangeddiff --git a/backend/internal/services/settings_service.go b/backend/internal/services/settings_service.goindex a609d1a45..27b10eb60 100644--- a/backend/internal/services/settings_service.go+++ b/backend/internal/services/settings_service.go@@ -57,10 +57,6 @@ func NewSettingsService(ctx context.Context, db *database.DB) (*SettingsService,return nil, fmt.Errorf("failed to setup instance ID: %w", err)}- if err = svc.LoadDatabaseSettings(ctx); err != nil {- return nil, fmt.Errorf("failed to reload settings after instance ID setup: %w", err)- }-return svc, nil}@@ -84,6 +80,14 @@ func (s *SettingsService) LoadDatabaseSettings(ctx context.Context) (err error)return nil}+func (s *SettingsService) refreshSettingsCacheInternal(ctx context.Context) error {+ if err := s.LoadDatabaseSettings(ctx); err != nil {+ return fmt.Errorf("failed to refresh settings cache: %w", err)+ }++ return nil+}+func (s *SettingsService) getDefaultSettings() *models.Settings {return &models.Settings{ProjectsDirectory: models.SettingVariable{Value: "/app/data/projects"},@@ -326,28 +330,8 @@ func (s *SettingsService) isEnvOverrideActiveInternal(key string) bool {}func (s *SettingsService) GetSettings(ctx context.Context) (*models.Settings, error) {- var settingVars []models.SettingVariable- err := s.db.WithContext(ctx).Find(&settingVars).Error- if err != nil {- return nil, err- }-- settings := &models.Settings{}-- for _, sv := range settingVars {- if err := settings.UpdateField(sv.Key, sv.Value, false); err != nil {- var notFoundErr models.SettingKeyNotFoundError- if !errors.As(err, ¬FoundErr) {- return nil, fmt.Errorf("failed to load setting %s: %w", sv.Key, err)- }- }- }-- // Apply environment variable overrides for fields tagged with "envOverride".- // This keeps behavior consistent with the cached settings path (LoadDatabaseSettingsInternal)- // and allows env vars like OIDC_MERGE_ACCOUNTS to affect runtime behavior.+ settings := s.GetSettingsConfig().Clone()s.applyEnvOverrides(ctx, settings)-return settings, nil}@@ -459,6 +443,14 @@ func (s *SettingsService) migrateOidcKeyNames(ctx context.Context) error {}func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string) error {+ if err := s.updateSettingValueNoRefreshInternal(ctx, key, value); err != nil {+ return err+ }++ return s.refreshSettingsCacheInternal(ctx)+}++func (s *SettingsService) updateSettingValueNoRefreshInternal(ctx context.Context, key, value string) error {return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {settingVar := &models.SettingVariable{Key: key,@@ -470,10 +462,8 @@ func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string)func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.Update) ([]models.SettingVariable, error) {defaultCfg := s.getDefaultSettings()- cfg, err := s.GetSettings(ctx)- if err != nil {- return nil, fmt.Errorf("failed to load current settings: %w", err)- }+ cfg := s.GetSettingsConfig().Clone()+ s.applyEnvOverrides(ctx, cfg)valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedAutoHeal, changedTimeouts, err := s.prepareUpdateValues(updates, cfg, defaultCfg)if err != nil {@@ -488,13 +478,10 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.Ureturn nil, err}- // Reload and store settings BEFORE calling callbacks so they read updated values- settings, err := s.GetSettings(ctx)- if err != nil {- return nil, fmt.Errorf("failed to retrieve updated settings: %w", err)+ if err := s.refreshSettingsCacheInternal(ctx); err != nil {+ return nil, err}-- s.config.Store(settings)+ settings := s.GetSettingsConfig()// Now call callbacks after in-memory config is updatedif changedPolling && s.OnImagePollingSettingsChanged != nil {@@ -667,7 +654,7 @@ func (s *SettingsService) handleOidcConfigUpdate(ctx context.Context, updates sereturn fmt.Errorf("failed to marshal merged OIDC config: %w", err)}- if err := s.UpdateSetting(ctx, "authOidcConfig", string(mergedBytes)); err != nil {+ if err := s.updateSettingValueNoRefreshInternal(ctx, "authOidcConfig", string(mergedBytes)); err != nil {return fmt.Errorf("failed to update authOidcConfig: %w", err)}}@@ -688,7 +675,7 @@ func (s *SettingsService) handleOidcConfigUpdate(ctx context.Context, updates se}}- if err := s.UpdateSetting(ctx, "oidcClientSecret", secret); err != nil {+ if err := s.updateSettingValueNoRefreshInternal(ctx, "oidcClientSecret", secret); err != nil {return fmt.Errorf("failed to update oidcClientSecret: %w", err)}}@@ -916,34 +903,15 @@ func (s *SettingsService) GetStringSetting(ctx context.Context, key, defaultValu}func (s *SettingsService) SetBoolSetting(ctx context.Context, key string, value bool) error {- if err := s.UpdateSetting(ctx, key, fmt.Sprintf("%t", value)); err != nil {- return err- }- // Rebuild a fresh snapshot instead of mutating current pointer (avoids races)- if err := s.LoadDatabaseSettings(ctx); err != nil {- return fmt.Errorf("failed to refresh settings cache: %w", err)- }- return nil+ return s.UpdateSetting(ctx, key, fmt.Sprintf("%t", value))}func (s *SettingsService) SetIntSetting(ctx context.Context, key string, value int) error {- if err := s.UpdateSetting(ctx, key, fmt.Sprintf("%d", value)); err != nil {- return err- }- if err := s.LoadDatabaseSettings(ctx); err != nil {- return fmt.Errorf("failed to refresh settings cache: %w", err)- }- return nil+ return s.UpdateSetting(ctx, key, fmt.Sprintf("%d", value))}func (s *SettingsService) SetStringSetting(ctx context.Context, key, value string) error {- if err := s.UpdateSetting(ctx, key, value); err != nil {- return err- }- if err := s.LoadDatabaseSettings(ctx); err != nil {- return fmt.Errorf("failed to refresh settings cache: %w", err)- }- return nil+ return s.UpdateSetting(ctx, key, value)}func (s *SettingsService) EnsureEncryptionKey(ctx context.Context) (string, error) {@@ -1041,10 +1009,6 @@ func (s *SettingsService) NormalizeProjectsDirectory(ctx context.Context, projecreturn fmt.Errorf("failed to update projectsDirectory: %w", err)}- if err := s.LoadDatabaseSettings(ctx); err != nil {- return fmt.Errorf("failed to reload settings after normalization: %w", err)- }-slog.InfoContext(ctx, "Successfully normalized projects directory")} else {slog.DebugContext(ctx, "Projects directory already normalized or custom, skipping", "value", projectsDirSetting.Value)@@ -1091,10 +1055,6 @@ func (s *SettingsService) NormalizeBuildsDirectory(ctx context.Context) error {return fmt.Errorf("failed to update buildsDirectory: %w", err)}- if err := s.LoadDatabaseSettings(ctx); err != nil {- return fmt.Errorf("failed to reload settings after normalization: %w", err)- }-slog.InfoContext(ctx, "Successfully normalized builds directory")} else {slog.DebugContext(ctx, "Builds directory already normalized or custom, skipping", "value", buildsDirSetting.Value)diff --git a/backend/internal/services/settings_service_test.go b/backend/internal/services/settings_service_test.goindex 252f6f5bc..e0c775c0e 100644--- a/backend/internal/services/settings_service_test.go+++ b/backend/internal/services/settings_service_test.go@@ -91,6 +91,22 @@ func TestSettingsService_GetSettings_UnknownKeysIgnored(t *testing.T) {require.NoError(t, err)}+func TestSettingsService_GetSettings_UsesCachedSnapshotWithoutDatabase(t *testing.T) {+ ctx := context.Background()+ db := setupSettingsTestDB(t)+ svc, err := NewSettingsService(ctx, db)+ require.NoError(t, err)++ require.NoError(t, svc.SetStringSetting(ctx, "baseServerUrl", "http://cached"))++ // GetSettings should clone the in-memory snapshot and not touch the database.+ svc.db = nil++ settings, err := svc.GetSettings(ctx)+ require.NoError(t, err)+ require.Equal(t, "http://cached", settings.BaseServerURL.Value)+}+func TestSettingsService_PruneUnknownSettings_RemovesStaleKeys(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)@@ -295,6 +311,51 @@ func TestSettingsService_UpdateSetting(t *testing.T) {require.Equal(t, "all", sv.Value)}+func TestSettingsService_UpdateSetting_RefreshesCachedSnapshot(t *testing.T) {+ ctx := context.Background()+ db := setupSettingsTestDB(t)+ svc, err := NewSettingsService(ctx, db)+ require.NoError(t, err)++ require.Equal(t, "http://localhost", svc.GetSettingsConfig().BaseServerURL.Value)+ require.NoError(t, svc.UpdateSetting(ctx, "baseServerUrl", "https://arcane.test"))++ require.Equal(t, "https://arcane.test", svc.GetSettingsConfig().BaseServerURL.Value)++ settings, err := svc.GetSettings(ctx)+ require.NoError(t, err)+ require.Equal(t, "https://arcane.test", settings.BaseServerURL.Value)+}++func BenchmarkSettingsService_GetSettings(b *testing.B) {+ ctx := context.Background()+ db, err := gorm.Open(glsqlite.Open(":memory:"), &gorm.Config{})+ if err != nil {+ b.Fatal(err)+ }+ if err := db.AutoMigrate(&models.SettingVariable{}); err != nil {+ b.Fatal(err)+ }+ settingsDB := &database.DB{DB: db}+ svc, err := NewSettingsService(ctx, settingsDB)+ if err != nil {+ b.Fatal(err)+ }++ b.ReportAllocs()+ b.ResetTimer()++ for b.Loop() {+ settings, err := svc.GetSettings(ctx)+ if err != nil {+ b.Fatal(err)+ }+ if settings == nil {+ b.Fatal("settings should not be nil")+ }+ }+}+func TestSettingsService_EnsureEncryptionKey(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)@@ -422,6 +483,31 @@ func TestSettingsService_UpdateSettings_RefreshesCache(t *testing.T) {require.True(t, found, "projectsDirectory setting not found in cached list")}+func TestSettingsService_UpdateSettings_ReturnsEnvOverriddenValues(t *testing.T) {+ t.Setenv("DOCKER_HOST", "tcp://env-docker:2375")++ ctx := context.Background()+ db := setupSettingsTestDB(t)+ svc, err := NewSettingsService(ctx, db)+ require.NoError(t, err)+ require.NoError(t, svc.EnsureDefaultSettings(ctx))++ newDir := "custom/projects2"+ settingsList, err := svc.UpdateSettings(ctx, settings.Update{+ ProjectsDirectory: &newDir,+ })+ require.NoError(t, err)++ found := false+ for _, sv := range settingsList {+ if sv.Key == "dockerHost" {+ found = true+ require.Equal(t, "tcp://env-docker:2375", sv.Value)+ }+ }+ require.True(t, found, "dockerHost setting not found in update response")+}+func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)diff --git a/backend/pkg/libarcane/ws/client.go b/backend/pkg/libarcane/ws/client.goindex 23e27ea09..4a4504c4e 100644--- a/backend/pkg/libarcane/ws/client.go+++ b/backend/pkg/libarcane/ws/client.go@@ -22,9 +22,10 @@ const (// Client represents a single WebSocket connection.type Client struct {- conn *websocket.Conn- send chan []byte- once sync.Once+ conn *websocket.Conn+ send chan []byte+ once sync.Once+ onRemove func()}func NewClient(conn *websocket.Conn, sendBuffer int) *Client {@@ -39,7 +40,12 @@ func NewClient(conn *websocket.Conn, sendBuffer int) *Client {// ServeClient registers the client with the hub and starts read/write pumps.// Caller is responsible for creating/closing the websocket.Conn.func ServeClient(ctx context.Context, hub *Hub, conn *websocket.Conn) {+ ServeClientWithOnRemove(ctx, hub, conn, nil)+}++func ServeClientWithOnRemove(ctx context.Context, hub *Hub, conn *websocket.Conn, onRemove func()) {c := NewClient(conn, clientSendBuffer)+ c.onRemove = onRemovehub.register <- cgo c.writePump(ctx, hub)@@ -48,6 +54,9 @@ func ServeClient(ctx context.Context, hub *Hub, conn *websocket.Conn) {func (c *Client) safeRemove(hub *Hub) {c.once.Do(func() {+ if c.onRemove != nil {+ c.onRemove()+ }hub.remove(c)})}diff --git a/backend/pkg/libarcane/ws/hub.go b/backend/pkg/libarcane/ws/hub.goindex 6a2e656e3..50ce0cf91 100644--- a/backend/pkg/libarcane/ws/hub.go+++ b/backend/pkg/libarcane/ws/hub.go@@ -12,6 +12,7 @@ type Hub struct {register chan *Clientunregister chan *Clientbroadcast chan []byte+ onFirst func()onEmpty func()}@@ -24,6 +25,12 @@ func NewHub(buffer int) *Hub {}}+func (h *Hub) SetOnFirstClient(fn func()) {+ h.mu.Lock()+ h.onFirst = fn+ h.mu.Unlock()+}+func (h *Hub) SetOnEmpty(fn func()) {h.mu.Lock()h.onEmpty = fn@@ -38,9 +45,17 @@ func (h *Hub) Run(ctx context.Context) {case <-ctx.Done():returncase c := <-h.register:+ var onFirst func()h.mu.Lock()h.clients[c] = struct{}{}+ if len(h.clients) == 1 && h.onFirst != nil {+ onFirst = h.onFirst+ h.onFirst = nil+ }h.mu.Unlock()+ if onFirst != nil {+ onFirst()+ }case c := <-h.unregister:// remove() handles the onEmpty callback when the last client disconnects.h.remove(c)diff --git a/backend/pkg/libarcane/ws/hub_client_count_test.go b/backend/pkg/libarcane/ws/hub_client_count_test.godeleted file mode 100644index b610d0027..000000000--- a/backend/pkg/libarcane/ws/hub_client_count_test.go+++ /dev/null@@ -1,9 +0,0 @@-package ws--// ClientCount is test-only and intentionally compiled from *_test.go to avoid-// deadcode in production builds.-func (h *Hub) ClientCount() int {- h.mu.RLock()- defer h.mu.RUnlock()- return len(h.clients)-}diff --git a/backend/pkg/libarcane/ws/hub_test.go b/backend/pkg/libarcane/ws/hub_test.goindex 0d1132240..7be11cc56 100644--- a/backend/pkg/libarcane/ws/hub_test.go+++ b/backend/pkg/libarcane/ws/hub_test.go@@ -207,6 +207,32 @@ func TestHub_OnEmptyCallback(t *testing.T) {require.Eventually(t, called.Load, time.Second, 5*time.Millisecond, "onEmpty callback was not called")}+func TestHub_OnFirstClientCallback(t *testing.T) {+ h := NewHub(10)+ ctx := t.Context()+ go h.Run(ctx)++ var callCount atomic.Int32+ h.SetOnFirstClient(func() {+ callCount.Add(1)+ })++ _, sc1, cleanup1 := newTestWSPair(t)+ defer cleanup1()+ _, sc2, cleanup2 := newTestWSPair(t)+ defer cleanup2()++ h.register <- NewClient(sc1, 16)+ h.register <- NewClient(sc2, 16)++ require.Eventually(t, func() bool {+ return h.ClientCount() == 2+ }, time.Second, 5*time.Millisecond)+ require.Eventually(t, func() bool {+ return callCount.Load() == 1+ }, time.Second, 5*time.Millisecond)+}+func TestHub_OnEmptyCalledOnlyOnce(t *testing.T) {h := NewHub(10)ctx := t.Context()diff --git a/backend/pkg/libarcane/ws/hub_test_helper_test.go b/backend/pkg/libarcane/ws/hub_test_helper_test.gonew file mode 100644index 000000000..d7e7347e4--- /dev/null+++ b/backend/pkg/libarcane/ws/hub_test_helper_test.go@@ -0,0 +1,7 @@+package ws++func (h *Hub) ClientCount() int {+ h.mu.RLock()+ defer h.mu.RUnlock()+ return len(h.clients)+}diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.goindex 05433384c..aa1ed2429 100644--- a/backend/pkg/scheduler/auto_heal_job.go+++ b/backend/pkg/scheduler/auto_heal_job.go@@ -13,9 +13,11 @@ import ("github.com/moby/moby/api/types/container""github.com/moby/moby/client""github.com/robfig/cron/v3"+ "golang.org/x/sync/errgroup")const AutoHealJobName = "auto-heal"+const autoHealInspectConcurrency = 4// restartRecord tracks restart timestamps for a single container.type restartRecord struct {@@ -30,6 +32,11 @@ type AutoHealJob struct {mu sync.Mutexrestarts map[string]*restartRecord++ getDockerClient func() (*client.Client, error)+ listContainers func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error)+ inspectContainer func(ctx context.Context, dockerClient *client.Client, containerID string) (container.InspectResponse, error)+ restartContainer func(ctx context.Context, dockerClient *client.Client, containerID string) error}func NewAutoHealJob(@@ -73,78 +80,107 @@ func (j *AutoHealJob) Run(ctx context.Context) {return}- dockerClient, err := j.dockerClientService.GetClient(ctx)+ dockerClient, err := j.getDockerClientInternal(ctx)if err != nil {slog.ErrorContext(ctx, "auto-heal failed to get Docker client", "error", err)return}- containerList, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})+ containerList, err := j.listContainersInternal(ctx, dockerClient)if err != nil {slog.ErrorContext(ctx, "auto-heal failed to list containers", "error", err)return}- containers := containerList.Items+ containers := containerListexcludedContainers := j.parseExcludedContainers(ctx)maxRestarts := j.settingsService.GetIntSetting(ctx, "autoHealMaxRestarts", 5)restartWindowMinutes := j.settingsService.GetIntSetting(ctx, "autoHealRestartWindow", 30)restartWindow := time.Duration(restartWindowMinutes) * time.Minute+ candidates := j.filterCandidatesInternal(containers, excludedContainers)++ g, groupCtx := errgroup.WithContext(ctx)+ g.SetLimit(autoHealInspectConcurrency)++ for _, candidate := range candidates {+ g.Go(func() error {+ j.processCandidateInternal(groupCtx, dockerClient, candidate, maxRestarts, restartWindow, restartWindowMinutes)+ return nil+ })+ }++ _ = g.Wait()+}++func (j *AutoHealJob) filterCandidatesInternal(containers []container.Summary, excludedContainers map[string]struct{}) []container.Summary {+ candidates := make([]container.Summary, 0, len(containers))for _, c := range containers {- // Skip Arcane internal containersif libarcane.IsInternalContainer(c.Labels) {continue}containerName := j.getContainerName(c.Names)-- // Skip excluded containersif j.isExcluded(containerName, excludedContainers) {continue}- // Inspect to get health status- inspect, err := libarcane.ContainerInspectWithCompatibility(ctx, dockerClient, c.ID, client.ContainerInspectOptions{})- if err != nil {- slog.WarnContext(ctx, "auto-heal failed to inspect container", "container", containerName, "error", err)- continue- }- containerInspect := inspect.Container+ candidates = append(candidates, c)+ }- // Skip if no healthcheck configured or not unhealthy- if containerInspect.State == nil || containerInspect.State.Health == nil {- continue- }- if containerInspect.State.Health.Status != container.Unhealthy {- continue- }+ return candidates+}- // Check restart-loop protection- if !j.canRestart(c.ID, maxRestarts, restartWindow) {- slog.WarnContext(ctx, "auto-heal restart-loop protection: skipping container",- "container", containerName,- "max_restarts", maxRestarts,- "window_minutes", restartWindowMinutes,- )- continue- }+func (j *AutoHealJob) processCandidateInternal(+ ctx context.Context,+ dockerClient *client.Client,+ candidate container.Summary,+ maxRestarts int,+ restartWindow time.Duration,+ restartWindowMinutes int,+) {+ containerID := candidate.ID+ containerName := j.getContainerName(candidate.Names)++ inspect, err := j.inspectContainerInternal(ctx, dockerClient, containerID)+ if err != nil {+ slog.WarnContext(ctx, "auto-heal failed to inspect container", "container", containerName, "error", err)+ return+ }- // Restart the container- slog.InfoContext(ctx, "auto-heal restarting unhealthy container", "container", containerName, "container_id", c.ID)- if _, err := dockerClient.ContainerRestart(ctx, c.ID, client.ContainerRestartOptions{}); err != nil {- slog.ErrorContext(ctx, "auto-heal failed to restart container", "container", containerName, "error", err)- continue- }+ if inspect.State == nil || inspect.State.Health == nil {+ return+ }+ if inspect.State.Health.Status != container.Unhealthy {+ return+ }++ if !j.canRestart(containerID, maxRestarts, restartWindow) {+ slog.WarnContext(ctx, "auto-heal restart-loop protection: skipping container",+ "container", containerName,+ "max_restarts", maxRestarts,+ "window_minutes", restartWindowMinutes,+ )+ return+ }- // Record the restart- j.recordRestart(c.ID)+ if err := j.restartContainerInternal(ctx, dockerClient, containerID); err != nil {+ slog.ErrorContext(ctx, "auto-heal failed to restart container", "container", containerName, "error", err)+ return+ }++ j.recordRestart(containerID)+ j.postRestartActionsInternal(ctx, containerID, containerName)- // Log event+ slog.InfoContext(ctx, "auto-heal restarted unhealthy container", "container", containerName, "container_id", containerID)+}++func (j *AutoHealJob) postRestartActionsInternal(ctx context.Context, containerID, containerName string) {+ if j.eventService != nil {if err := j.eventService.LogContainerEvent(ctx,models.EventTypeContainerRestart,- c.ID,+ containerID,containerName,"", // no user - system action"system",@@ -153,13 +189,12 @@ func (j *AutoHealJob) Run(ctx context.Context) {); err != nil {slog.WarnContext(ctx, "auto-heal failed to log event", "container", containerName, "error", err)}+ }- // Send notification- if err := j.notificationService.SendAutoHealNotification(ctx, containerName, c.ID); err != nil {+ if j.notificationService != nil {+ if err := j.notificationService.SendAutoHealNotification(ctx, containerName, containerID); err != nil {slog.WarnContext(ctx, "auto-heal failed to send notification", "container", containerName, "error", err)}-- slog.InfoContext(ctx, "auto-heal successfully restarted container", "container", containerName)}}@@ -238,6 +273,49 @@ func (j *AutoHealJob) isExcluded(name string, excluded map[string]struct{}) boolreturn ok}+func (j *AutoHealJob) inspectContainerInternal(ctx context.Context, dockerClient *client.Client, containerID string) (container.InspectResponse, error) {+ if j.inspectContainer != nil {+ return j.inspectContainer(ctx, dockerClient, containerID)+ }++ inspect, err := libarcane.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ if err != nil {+ return container.InspectResponse{}, err+ }++ return inspect.Container, nil+}++func (j *AutoHealJob) restartContainerInternal(ctx context.Context, dockerClient *client.Client, containerID string) error {+ if j.restartContainer != nil {+ return j.restartContainer(ctx, dockerClient, containerID)+ }++ _, err := dockerClient.ContainerRestart(ctx, containerID, client.ContainerRestartOptions{})+ return err+}++func (j *AutoHealJob) getDockerClientInternal(ctx context.Context) (*client.Client, error) {+ if j.getDockerClient != nil {+ return j.getDockerClient()+ }++ return j.dockerClientService.GetClient(ctx)+}++func (j *AutoHealJob) listContainersInternal(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {+ if j.listContainers != nil {+ return j.listContainers(ctx, dockerClient)+ }++ containerList, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})+ if err != nil {+ return nil, err+ }++ return containerList.Items, nil+}+// ResetRestartTracking clears all restart records (exported for testing).func (j *AutoHealJob) ResetRestartTracking() {j.mu.Lock()diff --git a/backend/pkg/scheduler/auto_heal_job_test.go b/backend/pkg/scheduler/auto_heal_job_test.goindex ec3222862..16afdffe9 100644--- a/backend/pkg/scheduler/auto_heal_job_test.go+++ b/backend/pkg/scheduler/auto_heal_job_test.go@@ -1,10 +1,19 @@package schedulerimport (+ "context"+ "sync/atomic""testing""time"+ "github.com/getarcaneapp/arcane/backend/internal/database"+ "github.com/getarcaneapp/arcane/backend/internal/models"+ "github.com/getarcaneapp/arcane/backend/internal/services"+ glsqlite "github.com/glebarez/sqlite"+ "github.com/moby/moby/api/types/container"+ "github.com/moby/moby/client""github.com/stretchr/testify/require"+ "gorm.io/gorm")func newTestAutoHealJob() *AutoHealJob {@@ -112,3 +121,66 @@ func TestAutoHeal_ResetRestartTracking(t *testing.T) {// Should be allowed againrequire.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))}++func TestAutoHeal_Run_UsesBoundedConcurrency(t *testing.T) {+ ctx := context.Background()++ db, err := gorm.Open(glsqlite.Open(":memory:"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))++ settingsSvc, err := services.NewSettingsService(ctx, &database.DB{DB: db})+ require.NoError(t, err)+ require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoHealEnabled", true))+ require.NoError(t, settingsSvc.SetStringSetting(ctx, "autoHealExcludedContainers", "skip-me"))++ job := NewAutoHealJob(nil, settingsSvc, nil, nil)+ job.getDockerClient = func() (*client.Client, error) { return nil, nil }+ job.listContainers = func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {+ return []container.Summary{+ {ID: "c1", Names: []string{"/one"}},+ {ID: "c2", Names: []string{"/two"}},+ {ID: "c3", Names: []string{"/three"}},+ {ID: "c4", Names: []string{"/four"}},+ {ID: "c5", Names: []string{"/five"}},+ {ID: "c6", Names: []string{"/six"}},+ {ID: "skip", Names: []string{"/skip-me"}},+ }, nil+ }++ var current int32+ var maxConcurrent int32+ var restarts int32++ job.inspectContainer = func(ctx context.Context, dockerClient *client.Client, containerID string) (container.InspectResponse, error) {+ active := atomic.AddInt32(¤t, 1)+ for {+ maxSeen := atomic.LoadInt32(&maxConcurrent)+ if active <= maxSeen || atomic.CompareAndSwapInt32(&maxConcurrent, maxSeen, active) {+ break+ }+ }+ defer atomic.AddInt32(¤t, -1)++ time.Sleep(40 * time.Millisecond)++ switch containerID {+ case "c5":+ return container.InspectResponse{State: &container.State{}}, nil+ case "c6":+ return container.InspectResponse{State: &container.State{Health: &container.Health{Status: container.Healthy}}}, nil+ default:+ return container.InspectResponse{State: &container.State{Health: &container.Health{Status: container.Unhealthy}}}, nil+ }+ }+ job.restartContainer = func(ctx context.Context, dockerClient *client.Client, containerID string) error {+ atomic.AddInt32(&restarts, 1)+ return nil+ }++ job.Run(ctx)++ require.Greater(t, atomic.LoadInt32(&maxConcurrent), int32(1))+ require.LessOrEqual(t, atomic.LoadInt32(&maxConcurrent), int32(autoHealInspectConcurrency))+ require.Equal(t, int32(4), atomic.LoadInt32(&restarts))+}