Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
perf: cache GPU stats and pace the system-stats sampler to subscribers (#3609)
Kyle Mendell850dfd257c4d
diff --git a/backend/api/ws/container_stats.go b/backend/api/ws/container_stats.gonew file mode 100644index 000000000..89c92c37c--- /dev/null+++ b/backend/api/ws/container_stats.go@@ -0,0 +1,170 @@+package ws++import (+ "context"+ "encoding/json/v2"+ "log/slog"+ "net/http"+ "strings"+ "sync"+ "time"++ "emperror.dev/errors"+ "github.com/labstack/echo/v5"++ wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"+ systemtypes "github.com/getarcaneapp/arcane/types/v2/system"+)++// ============================================================================+// Container Stats WebSocket Endpoint+// ============================================================================++// ContainerStats streams container stats over WebSocket.+//+// @Summary Get container stats via WebSocket+// @Description Stream container resource statistics over WebSocket connection+// @Tags WebSocket+// @Param id path string true "Environment ID"+// @Param containerId path string true "Container ID"+// @Router /api/environments/{id}/ws/containers/{containerId}/stats [get]+func (h *WebSocketHandler) ContainerStats(c *echo.Context) error {+ containerID := c.Param("containerId")+ if strings.TrimSpace(containerID) == "" {+ return wsErrorJSONInternal(c, http.StatusBadRequest, "Container ID is required")+ }++ conn, onRemove, ok := h.acceptWSInternal(c, systemtypes.WSKindContainerStats, containerID)+ if !ok {+ return nil+ }++ // A reconnect can land exactly on the 5s idle-teardown boundary and load a+ // hub whose Run has already exited. Drop that hub and retry once so the+ // client gets a live producer instead of a silent socket.+ // WebSocket connections use context.Background() because they are long-lived and should not+ // be tied to the HTTP request context. Cleanup is handled by the shared hub when it idles.+ for attempt := range 2 {+ hub := h.getOrCreateContainerStatsHubInternal(containerID)+ if wshub.ServeClientWithOnRemove(context.Background(), hub, conn, onRemove) {+ return nil+ }+ h.containerStatsHubs.CompareAndDelete(containerID, hub)+ slog.DebugContext(c.Request().Context(), "container stats hub stopped before client registration; retrying",+ "containerID", containerID, "attempt", attempt+1)+ }++ slog.WarnContext(c.Request().Context(), "failed to register container stats client", "containerID", containerID)+ _ = conn.CloseNow()+ onRemove()+ return nil+}++func (h *WebSocketHandler) getOrCreateContainerStatsHubInternal(containerID string) *wshub.Hub {+ if existing, ok := h.containerStatsHubs.Load(containerID); ok {+ if hub, ok := existing.(*wshub.Hub); ok {+ return hub+ }+ }++ hub := wshub.NewHub(64)+ actual, loaded := h.containerStatsHubs.LoadOrStore(containerID, hub)+ if loaded {+ if existingHub, ok := actual.(*wshub.Hub); ok {+ return existingHub+ }+ // type assertion failure is impossible in practice, but avoid running+ // an unregistered hub if it somehow occurs+ return hub+ }++ h.runContainerStatsHubInternal(containerID, hub)+ return hub+}++// containerStatsErrorDrainGrace is how long a failed stats hub stays alive after+// broadcasting its error frame, so clients receive it before the hub shuts down.+const containerStatsErrorDrainGrace = 2 * time.Second++func (h *WebSocketHandler) runContainerStatsHubInternal(containerID string, hub *wshub.Hub) {+ ctx, cancel := context.WithCancel(context.Background())+ var cleanupTimer *time.Timer+ var cleanupTimerMu sync.Mutex++ hub.SetOnEmpty(func() {+ cleanupTimerMu.Lock()+ if cleanupTimer != nil {+ cleanupTimer.Stop()+ }+ var timer *time.Timer+ timer = time.AfterFunc(5*time.Second, func() {+ cleanupTimerMu.Lock()+ defer cleanupTimerMu.Unlock()+ if cleanupTimer != timer {+ return+ }+ if existing, ok := h.containerStatsHubs.Load(containerID); ok && existing == hub {+ h.containerStatsHubs.Delete(containerID)+ }+ slog.Debug("container stats hub idle, cleaning up upstream stream", "containerID", containerID)+ cleanupTimer = nil+ cancel()+ })+ cleanupTimer = timer+ cleanupTimerMu.Unlock()+ })+ hub.SetOnActive(func() {+ cleanupTimerMu.Lock()+ if cleanupTimer != nil {+ cleanupTimer.Stop()+ cleanupTimer = nil+ }+ cleanupTimerMu.Unlock()+ })++ go hub.Run(ctx)++ statsChan := make(chan any, 64)+ go func(ctx context.Context) {+ defer close(statsChan)++ err := h.containerService.StreamStats(ctx, containerID, statsChan)+ if err == nil || errors.Is(err, context.Canceled) || ctx.Err() != nil {+ return+ }++ // The producer died but the hub stayed cached, so clients kept an open+ // socket that would never emit another sample. Tell them why, then drop+ // the hub so the next connect rebuilds a producer instead of attaching+ // to this dead one.+ slog.Warn("container stats stream failed", "containerID", containerID, "error", err)+ if b, marshalErr := json.Marshal(map[string]any{+ "error": "Failed to stream container stats: " + err.Error(),+ "timestamp": wshub.NowRFC3339(),+ }); marshalErr == nil {+ hub.Broadcast(b)+ }+ h.containerStatsHubs.CompareAndDelete(containerID, hub)++ // Cancelling immediately would race hub.Run between ctx.Done and the+ // queued error frame, dropping the very message clients need. Give the+ // broadcast a moment to drain, then tear the hub down.+ time.AfterFunc(containerStatsErrorDrainGrace, cancel)+ }(ctx)++ go func() {+ for {+ select {+ case <-ctx.Done():+ return+ case stats, ok := <-statsChan:+ if !ok {+ return+ }+ if b, err := json.Marshal(stats); err == nil {+ hub.Broadcast(b)+ }+ }+ }+ }()+}diff --git a/backend/api/ws/exec.go b/backend/api/ws/exec.gonew file mode 100644index 000000000..28625bc42--- /dev/null+++ b/backend/api/ws/exec.go@@ -0,0 +1,146 @@+package ws++import (+ "cmp"+ "context"+ "io"+ "log/slog"+ "net/http"+ "strings"+ "sync"+ "time"++ "emperror.dev/errors"+ "github.com/coder/websocket"+ "github.com/labstack/echo/v5"++ systemtypes "github.com/getarcaneapp/arcane/types/v2/system"+)++// ============================================================================+// Container Exec WebSocket Endpoint+// ============================================================================++// ContainerExec provides interactive terminal access to a container.+//+// @Summary Execute command in container via WebSocket+// @Description Interactive terminal access to a container over WebSocket+// @Tags WebSocket+// @Param id path string true "Environment ID"+// @Param containerId path string true "Container ID"+// @Param shell query string false "Shell to execute" default(/bin/sh)+// @Router /api/environments/{id}/ws/containers/{containerId}/terminal [get]+func (h *WebSocketHandler) ContainerExec(c *echo.Context) error {+ containerID := c.Param("containerId")+ if strings.TrimSpace(containerID) == "" {+ return wsErrorJSONInternal(c, http.StatusBadRequest, "Container ID is required")+ }++ shell := cmp.Or(c.QueryParam("shell"), "/bin/sh")++ conn, unregister, ok := h.acceptWSInternal(c, systemtypes.WSKindContainerExec, containerID)+ if !ok {+ return nil+ }+ defer unregister()+ defer func() {+ if err := conn.CloseNow(); err != nil {+ slog.Debug("Failed to close container exec websocket connection", "containerID", containerID, "error", err)+ }+ }()++ // Allow large terminal pastes; coder/websocket's default limit is 32KB.+ conn.SetReadLimit(1 << 20)++ ctx, cancel := context.WithCancel(c.Request().Context())+ defer cancel()++ // The pong is serviced by the concurrent stdin reader.+ go keepWSConnAliveInternal(ctx, cancel, conn, 54*time.Second)++ h.runContainerExecInternal(ctx, cancel, conn, containerID, shell)+ return nil+}++func (h *WebSocketHandler) runContainerExecInternal(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, containerID, shell string) {+ // Create exec instance+ execID, err := h.containerService.CreateExec(ctx, containerID, []string{shell})+ if err != nil {+ h.writeExecErrorInternal(ctx, conn, errors.WithMessage(err, "Error creating exec"))+ return+ }++ // Attach to exec+ execSession, err := h.containerService.AttachExec(ctx, containerID, execID)+ if err != nil {+ h.writeExecErrorInternal(ctx, conn, errors.WithMessage(err, "Error attaching to exec"))+ return+ }+ // Cleanup must proceed even if the parent ctx is canceled, and must also+ // run on cancellation while the stdout pipe is blocked in a read — closing+ // the session is what unblocks it.+ cleanup := sync.OnceFunc(func() {+ slog.Debug("Cleaning up exec session", "execID", execID, "containerID", containerID, "contextErr", ctx.Err())+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cleanupCancel()+ if err := execSession.Close(cleanupCtx); err != nil { //nolint:contextcheck+ slog.Warn("Failed to clean up exec session", "execID", execID, "error", err)+ }+ })+ defer cleanup()+ go func() {+ <-ctx.Done()+ cleanup()+ }()++ done := make(chan struct{})+ go h.pipeExecOutputInternal(ctx, conn, execSession.Stdout(), execID, containerID, done)+ go h.pipeExecInputInternal(ctx, cancel, conn, execSession.Stdin(), execID, containerID)++ <-done+}++func (h *WebSocketHandler) writeExecErrorInternal(ctx context.Context, conn *websocket.Conn, err error) {+ wctx, cancel := context.WithTimeout(ctx, 10*time.Second)+ defer cancel()+ _ = conn.Write(wctx, websocket.MessageText, []byte(err.Error()+"\r\n"))+}++func (h *WebSocketHandler) pipeExecOutputInternal(ctx context.Context, conn *websocket.Conn, stdout io.Reader, execID, containerID string, done chan<- struct{}) {+ defer close(done)+ buf := make([]byte, 4096)+ for {+ select {+ case <-ctx.Done():+ return+ default:+ }++ n, err := stdout.Read(buf)+ if err != nil {+ slog.Debug("Exec stdout read error", "execID", execID, "containerID", containerID, "error", err)+ return+ }+ if n > 0 {+ if err := conn.Write(ctx, websocket.MessageBinary, buf[:n]); err != nil {+ slog.Debug("Exec websocket write error", "execID", execID, "containerID", containerID, "error", err)+ return+ }+ }+ }+}++func (h *WebSocketHandler) pipeExecInputInternal(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, stdin io.Writer, execID, containerID string) {+ for {+ _, data, err := conn.Read(ctx)+ if err != nil {+ slog.Debug("Exec websocket read error", "execID", execID, "containerID", containerID, "error", err)+ cancel()+ return+ }+ if _, err := stdin.Write(data); err != nil {+ slog.Debug("Exec stdin write error", "execID", execID, "containerID", containerID, "error", err)+ return+ }+ }+}diff --git a/backend/api/ws/handler.go b/backend/api/ws/handler.goindex 9fd28d264..449c6a409 100644--- a/backend/api/ws/handler.go+++ b/backend/api/ws/handler.go@@ -2,47 +2,32 @@ package wsimport ("context"- "encoding/json/v2"- "io""log/slog""net/http"- "runtime"- "strconv"- "strings""sync""sync/atomic""time"- "github.com/getarcaneapp/arcane/backend/v2/internal/auth"- "github.com/getarcaneapp/arcane/backend/v2/internal/diagnostics"- "github.com/getarcaneapp/arcane/backend/v2/internal/project"- "github.com/getarcaneapp/arcane/backend/v2/internal/swarm"-- "github.com/getarcaneapp/arcane/backend/v2/internal/container"-- "emperror.dev/errors"-"github.com/coder/websocket""github.com/labstack/echo/v5""github.com/samber/hot"- "github.com/shirou/gopsutil/v4/cpu"- "github.com/shirou/gopsutil/v4/disk"- "github.com/shirou/gopsutil/v4/host"- "github.com/shirou/gopsutil/v4/mem"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/auth""github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/container"+ "github.com/getarcaneapp/arcane/backend/v2/internal/diagnostics""github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/internal/project"+ "github.com/getarcaneapp/arcane/backend/v2/internal/swarm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/system"systemlib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/system"wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"httputil "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"systemtypes "github.com/getarcaneapp/arcane/types/v2/system"-- "github.com/getarcaneapp/arcane/backend/v2/internal/system""go.getarcane.app/sys/cgroup")-const cgroupCacheTTL = 30 * time.Second-var defaultWebSocketMetrics = wshub.NewWebSocketMetrics()// ============================================================================@@ -62,26 +47,29 @@ type WebSocketHandler struct {activeConnections sync.MaplogStreamsMu sync.MutexlogStreams map[string]*wsLogStream- cpuCache struct {- sync.RWMutex-- value float64- timestamp time.Time- }- systemStaticInfo struct {+ cpuCache actors.Snapshot[float64]+ systemStaticInfo struct {once sync.OncecpuCount inthostname string}systemStatsSampler struct {- stateMu sync.RWMutex- latest systemtypes.SystemStats- timestamp time.Time+ latest actors.Snapshot[systemtypes.SystemStats]lifecycleMu sync.Mutexclients int- cancel context.CancelFunc- ready chan struct{}- running bool+ // intervals counts subscribers per requested interval; the sampler+ // ticks at the smallest one (in effectiveInterval) rather than a+ // fixed 1Hz, so samples are never produced faster than the fastest+ // consumer reads them.+ intervals map[time.Duration]int+ effectiveInterval atomic.Int64+ cancel context.CancelFunc+ ready chan struct{}+ // wake nudges the running sampler when effectiveInterval changes,+ // so a newly joined faster subscriber doesn't wait out a slower+ // tick already in flight.+ wake chan struct{}+ running bool}containerStatsHubs sync.MapcgroupCache *cgroup.Cache@@ -94,129 +82,6 @@ type WebSocketHandler struct {cpuUsageReader func(interval time.Duration) (float64, bool)}-type wsLogStream struct {- hub *wshub.Hub- cancel context.CancelFunc- firstSubscriber chan struct{}- format string- key string- refs int- done bool- seq atomic.Uint64-}--func getContextUserIDInternal(c *echo.Context) string {- if val := c.Get("userID"); val != nil {- if userID, ok := val.(string); ok {- return userID- }- }- return ""-}--func buildWSConnectionInfoInternal(c *echo.Context, kind, resourceID string) systemtypes.WebSocketConnectionInfo {- return systemtypes.WebSocketConnectionInfo{- Kind: kind,- EnvID: c.Param("id"),- ResourceID: resourceID,- ClientIP: c.RealIP(),- UserID: getContextUserIDInternal(c),- UserAgent: c.Request().Header.Get("User-Agent"),- }-}--func buildLogStreamKeyInternal(envID, kind, resourceID, format string, batched, follow bool, tail, since string, timestamps bool) string {- return strings.Join([]string{- envID,- kind,- resourceID,- format,- strconv.FormatBool(batched),- strconv.FormatBool(follow),- tail,- since,- strconv.FormatBool(timestamps),- }, "|")-}--func (h *WebSocketHandler) streamProjectLogsInternal(ctx context.Context, projectID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {- if h.projectLogStreamer != nil {- return h.projectLogStreamer(ctx, projectID, logsChan, follow, tail, since, timestamps)- }- return h.projectService.StreamProjectLogs(ctx, projectID, logsChan, follow, tail, since, timestamps)-}--func (h *WebSocketHandler) streamContainerLogsInternal(ctx context.Context, containerID string, logsChan chan<- string, follow bool, tail, since string, timestamps bool) error {- if h.containerLogStreamer != nil {- return h.containerLogStreamer(ctx, containerID, logsChan, follow, tail, since, timestamps)- }- return h.containerService.StreamLogs(ctx, containerID, logsChan, follow, tail, since, timestamps)-}--func (h *WebSocketHandler) getOrCreateLogStreamInternal(key string, create func(onEmpty func(*wsLogStream)) *wsLogStream) *wsLogStream {- h.logStreamsMu.Lock()- defer h.logStreamsMu.Unlock()-- if stream, ok := h.logStreams[key]; ok {- if !stream.done {- stream.refs++- return stream- }- }-- stream := create(func(stream *wsLogStream) {- h.markLogStreamDoneInternal(key, stream)- })- stream.key = key- stream.refs = 1- h.logStreams[key] = stream- return stream-}--func takeLogStreamCancelInternal(stream *wsLogStream) context.CancelFunc {- cancel := stream.cancel- stream.cancel = nil- return cancel-}--func (h *WebSocketHandler) releaseLogStreamInternal(key string, stream *wsLogStream) {- var cancel context.CancelFunc-- h.logStreamsMu.Lock()- if stream.refs > 0 {- stream.refs--- }- if stream.refs == 0 {- if current, ok := h.logStreams[key]; ok && current == stream {- delete(h.logStreams, key)- }- cancel = takeLogStreamCancelInternal(stream)- }- h.logStreamsMu.Unlock()-- if cancel != nil {- cancel()- }-}--func (h *WebSocketHandler) markLogStreamDoneInternal(key string, stream *wsLogStream) {- var cancel context.CancelFunc-- h.logStreamsMu.Lock()- stream.done = true- if stream.refs == 0 {- if current, ok := h.logStreams[key]; ok && current == stream {- delete(h.logStreams, key)- }- cancel = takeLogStreamCancelInternal(stream)- }- h.logStreamsMu.Unlock()-- if cancel != nil {- cancel()- }-}-func NewWebSocketHandler(group *echo.Group,projectService *project.ProjectService,@@ -249,598 +114,43 @@ func NewWebSocketHandler(handler.registerDiagnosticsRoutesInternal(group, authMiddleware)}-// ============================================================================-// Shared Log Stream Helpers-// ============================================================================--// logStreamParams holds the standard query parameters shared by every WS log endpoint.-type logStreamParams struct {- tail string- since string- format string- follow bool- timestamps bool- batched bool-}--func parseLogStreamParamsInternal(c *echo.Context) logStreamParams {- req := c.Request()- tail, _ := httputil.GetQueryParam(req, "tail", false)- if tail == "" {- tail = "100"- }- since, _ := httputil.GetQueryParam(req, "since", false)- format, _ := httputil.GetQueryParam(req, "format", false)- if format == "" {- format = "text"- }- return logStreamParams{- follow: queryParamWithDefaultInternal(c, "follow", "true") == "true",- tail: tail,- since: since,- timestamps: queryParamWithDefaultInternal(c, "timestamps", "false") == "true",- format: format,- batched: queryParamWithDefaultInternal(c, "batched", "false") == "true",- }-}--func queryParamWithDefaultInternal(c *echo.Context, key, def string) string {- if v := c.QueryParam(key); v != "" {- return v+func buildWSConnectionInfoInternal(c *echo.Context, kind, resourceID string) systemtypes.WebSocketConnectionInfo {+ userID, _ := c.Get("userID").(string)+ return systemtypes.WebSocketConnectionInfo{+ Kind: kind,+ EnvID: c.Param("id"),+ ResourceID: resourceID,+ ClientIP: c.RealIP(),+ UserID: userID,+ UserAgent: c.Request().Header.Get("User-Agent"),}- return def}-// serveLogStreamInternal is the shared scaffold for all three WS log endpoints (project, container, service).-// It performs upgrade, builds the stream key, gets-or-creates the multiplexing hub, registers metrics,-// and serves the client. The caller-supplied hubBuilder constructs the underlying *wsLogStream-// when no hub already exists for streamKey.-func (h *WebSocketHandler) serveLogStreamInternal(- c *echo.Context,- kind, resourceID string,- params logStreamParams,- hubBuilder func(streamKey string, onEmpty func(*wsLogStream)) *wsLogStream,-) {+// acceptWSInternal upgrades the request to a WebSocket and registers it with+// the metrics tracker. The returned unregister func must be called exactly+// once when the connection ends.+func (h *WebSocketHandler) acceptWSInternal(c *echo.Context, kind, resourceID string) (*websocket.Conn, func(), bool) {conn, err := wshub.Accept(c.Response(), c.Request(), h.checkWSOrigin)if err != nil {- return+ slog.DebugContext(c.Request().Context(), "websocket accept failed", "kind", kind, "resourceID", resourceID, "error", err)+ return nil, nil, false}-- streamKey := buildLogStreamKeyInternal(c.Param("id"), kind, resourceID, params.format, params.batched, params.follow, params.tail, params.since, params.timestamps)- stream := h.getOrCreateLogStreamInternal(streamKey, func(onEmpty func(*wsLogStream)) *wsLogStream {- return hubBuilder(streamKey, onEmpty)- })connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, kind, resourceID))- release := func() {- h.wsMetrics.UnregisterConnection(connID)- h.releaseLogStreamInternal(streamKey, stream)- }- // WebSocket connections use context.Background() because they are long-lived and should not- // be tied to the HTTP request context. Cleanup is handled via the hub's OnEmpty callback- // which triggers when all clients disconnect.- if !wshub.ServeClientWithOnRemove(context.Background(), stream.hub, conn, release) {- // The stream refcount normally keeps this hub alive, so a stopped hub- // here means it was torn down out from under us; drop our reference- // rather than leaking the connection and the metrics entry.- slog.Debug("log stream hub stopped before client registration", "streamKey", streamKey)- _ = conn.CloseNow()- release()- }-}--// broadcastLogStreamErrorInternal emits an error message to every client of a log stream.-// resourceLabel is the human-readable noun used in slog/error text (e.g. "project log stream").-// errorPrefix is the user-facing message prefix (e.g. "Failed to stream project logs: ").-func broadcastLogStreamErrorInternal(resourceLabel, errorPrefix string, resourceID string, format string, err error, ls *wsLogStream) {- slog.Warn(resourceLabel+" failed", "resourceID", resourceID, "error", err)-- if format == "json" {- msg := wshub.LogMessage{- Seq: ls.seq.Add(1),- Level: "error",- Message: errorPrefix + err.Error(),- Service: "arcane",- Timestamp: wshub.NowRFC3339(),- }- if b, marshalErr := json.Marshal(msg); marshalErr == nil {- ls.hub.Broadcast(b)- }- return- }-- ls.hub.Broadcast([]byte(errorPrefix + err.Error()))-}--// ============================================================================-// Project WebSocket/Streaming Endpoints-// ============================================================================--// ProjectLogs streams project logs over WebSocket.-//-// @Summary Get project logs via WebSocket-// @Description Stream project logs over WebSocket connection-// @Tags WebSocket-// @Param id path string true "Environment ID"-// @Param projectId path string true "Project ID"-// @Param follow query bool false "Follow log output" default(true)-// @Param tail query string false "Number of lines to show from the end" default(100)-// @Param since query string false "Show logs since timestamp"-// @Param timestamps query bool false "Show timestamps" default(false)-// @Param format query string false "Output format (text or json)" default(text)-// @Param batched query bool false "Batch log messages" default(false)-// @Router /api/environments/{id}/ws/projects/{projectId}/logs [get]-func (h *WebSocketHandler) ProjectLogs(c *echo.Context) error {- projectID := c.Param("projectId")- if strings.TrimSpace(projectID) == "" {- return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Project ID is required"})- }-- params := parseLogStreamParamsInternal(c)- h.serveLogStreamInternal(c, systemtypes.WSKindProjectLogs, projectID, params, func(streamKey string, onEmpty func(*wsLogStream)) *wsLogStream {- return h.startLogHubInternal(- streamKey,- projectID,- "project",- params,- h.streamProjectLogsInternal,- normalizeProjectLogMessageInternal,- normalizeProjectLogTextInternal,- onEmpty,- )- })- return nil-}--func newWSLogStreamInternal(key, format string) (*wsLogStream, context.Context) {- ls := &wsLogStream{- hub: wshub.NewHub(1024),- firstSubscriber: make(chan struct{}),- format: format,- key: key,- }- ls.hub.SetOnFirstClient(func() {- close(ls.firstSubscriber)- })-- ctx, cancel := context.WithCancel(context.Background()) //nolint:gosec // cancel is intentionally retained and invoked by the hub OnEmpty callback.- ls.cancel = cancel-- go ls.hub.Run(ctx)-- return ls, ctx-}--func waitForLogStreamSubscriberInternal(ctx context.Context, firstSubscriber <-chan struct{}) bool {- for {- select {- case <-ctx.Done():- return false- case <-firstSubscriber:- return true- }- }-}--func normalizeProjectLogMessageInternal(line string) wshub.LogMessage {- level, service, message, timestamp := wshub.NormalizeProjectLine(line)- return wshub.LogMessage{- Level: level,- Message: message,- Service: service,- Timestamp: timestamp,- }-}--func normalizeContainerLogMessageInternal(line string) wshub.LogMessage {- level, message, timestamp := wshub.NormalizeContainerLine(line)- return wshub.LogMessage{- Level: level,- Message: message,- Timestamp: timestamp,- }-}--func normalizeProjectLogTextInternal(line string) string {- _, _, message, _ := wshub.NormalizeProjectLine(line)- return message-}--func (h *WebSocketHandler) startLogHubInternal(- key, resourceID, label string,- params logStreamParams,- stream func(context.Context, string, chan<- string, bool, string, string, bool) error,- normalizeJSON func(string) wshub.LogMessage,- normalizeText func(string) string,- onEmptyHook func(*wsLogStream),-) *wsLogStream {- ls, ctx := newWSLogStreamInternal(key, params.format)-- ls.hub.SetOnEmpty(func() {- if onEmptyHook != nil {- onEmptyHook(ls)- }- slog.Debug("client disconnected, cleaning up "+label+" log hub", label+"ID", resourceID)- })-- lines := h.startLogSourceInternal(ctx, key, resourceID, label, params, stream, ls)- startLogForwardersInternal(ctx, ls, lines, params, normalizeJSON, normalizeText)-- return ls-}--func (h *WebSocketHandler) startLogSourceInternal(- ctx context.Context,- key, resourceID, label string,- params logStreamParams,- stream func(context.Context, string, chan<- string, bool, string, string, bool) error,- ls *wsLogStream,-) <-chan string {- lines := make(chan string, 256)- go func() {- defer close(lines)- if !waitForLogStreamSubscriberInternal(ctx, ls.firstSubscriber) {- return- }-- if err := stream(ctx, resourceID, lines, params.follow, params.tail, params.since, params.timestamps); err != nil {- if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {- return- }-- h.markLogStreamDoneInternal(key, ls)- broadcastLogStreamErrorInternal(label+" log stream", "Failed to stream "+label+" logs: ", resourceID, params.format, err, ls)- return- }-- if ctx.Err() == nil {- h.markLogStreamDoneInternal(key, ls)- }- }()-- return lines-}--func startLogForwardersInternal(- ctx context.Context,- ls *wsLogStream,- lines <-chan string,- params logStreamParams,- normalizeJSON func(string) wshub.LogMessage,- normalizeText func(string) string,-) {- if params.format == "json" {- messages := mapLogLinesInternal(ctx, lines, func(line string) wshub.LogMessage {- message := normalizeJSON(line)- message.Seq = ls.seq.Add(1)- if message.Timestamp == "" {- message.Timestamp = wshub.NowRFC3339()- }- return message- })-- if params.batched {- go wshub.ForwardLogJSONBatched(ctx, ls.hub, messages, 50, 400*time.Millisecond)- } else {- go wshub.ForwardLogJSON(ctx, ls.hub, messages)- }-- return- }-- textLines := lines- if normalizeText != nil {- textLines = mapLogLinesInternal(ctx, lines, normalizeText)- }- go wshub.ForwardLines(ctx, ls.hub, textLines)-}--func mapLogLinesInternal[T any](ctx context.Context, lines <-chan string, transform func(string) T) <-chan T {- mapped := make(chan T, 256)- go func() {- defer close(mapped)- for {- select {- case <-ctx.Done():- return- case line, ok := <-lines:- if !ok {- return- }-- select {- case <-ctx.Done():- return- case mapped <- transform(line):- }- }- }- }()-- return mapped-}--// ============================================================================-// Container WebSocket Endpoints-// ============================================================================--// ContainerLogs streams container logs over WebSocket.-//-// @Summary Get container logs via WebSocket-// @Description Stream container logs over WebSocket connection-// @Tags WebSocket-// @Param id path string true "Environment ID"-// @Param containerId path string true "Container ID"-// @Param follow query bool false "Follow log output" default(true)-// @Param tail query string false "Number of lines to show from the end" default(100)-// @Param since query string false "Show logs since timestamp"-// @Param timestamps query bool false "Show timestamps" default(false)-// @Param format query string false "Output format (text or json)" default(text)-// @Param batched query bool false "Batch log messages" default(false)-// @Router /api/environments/{id}/ws/containers/{containerId}/logs [get]-func (h *WebSocketHandler) ContainerLogs(c *echo.Context) error {- containerID := c.Param("containerId")- if strings.TrimSpace(containerID) == "" {- return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Container ID is required"})- }-- params := parseLogStreamParamsInternal(c)- h.serveLogStreamInternal(c, systemtypes.WSKindContainerLogs, containerID, params, func(streamKey string, onEmpty func(*wsLogStream)) *wsLogStream {- return h.startLogHubInternal(- streamKey,- containerID,- "container",- params,- h.streamContainerLogsInternal,- normalizeContainerLogMessageInternal,- nil,- onEmpty,- )- })- return nil-}--// ============================================================================-// Swarm Service WebSocket/Streaming Endpoints-// ============================================================================--// ServiceLogs streams swarm service logs over WebSocket.-//-// @Summary Get swarm service logs via WebSocket-// @Description Stream swarm service logs over WebSocket connection-// @Tags WebSocket-// @Param id path string true "Environment ID"-// @Param serviceId path string true "Service ID"-// @Param follow query bool false "Follow log output" default(true)-// @Param tail query string false "Number of lines to show from the end" default(100)-// @Param since query string false "Show logs since timestamp"-// @Param timestamps query bool false "Show timestamps" default(false)-// @Param format query string false "Output format (text or json)" default(text)-// @Param batched query bool false "Batch log messages" default(false)-// @Router /api/environments/{id}/ws/swarm/services/{serviceId}/logs [get]-func (h *WebSocketHandler) ServiceLogs(c *echo.Context) error {- serviceID := c.Param("serviceId")- if strings.TrimSpace(serviceID) == "" {- return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Service ID is required"})- }-- params := parseLogStreamParamsInternal(c)- h.serveLogStreamInternal(c, systemtypes.WSKindServiceLogs, serviceID, params, func(streamKey string, onEmpty func(*wsLogStream)) *wsLogStream {- return h.startLogHubInternal(- streamKey,- serviceID,- "service",- params,- h.swarmService.StreamServiceLogs,- normalizeContainerLogMessageInternal,- nil,- onEmpty,- )- })- return nil-}--// ContainerStats streams container stats over WebSocket.-//-// @Summary Get container stats via WebSocket-// @Description Stream container resource statistics over WebSocket connection-// @Tags WebSocket-// @Param id path string true "Environment ID"-// @Param containerId path string true "Container ID"-// @Router /api/environments/{id}/ws/containers/{containerId}/stats [get]-func (h *WebSocketHandler) ContainerStats(c *echo.Context) error {- containerID := c.Param("containerId")- if strings.TrimSpace(containerID) == "" {- return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Container ID is required"})- }-- conn, err := wshub.Accept(c.Response(), c.Request(), h.checkWSOrigin)- if err != nil {- slog.DebugContext(c.Request().Context(), "Failed to upgrade WebSocket for container stats", "containerID", containerID, "error", err)- return nil- }-- connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindContainerStats, containerID))- onRemove := func() {- h.wsMetrics.UnregisterConnection(connID)- }-- // A reconnect can land exactly on the 5s idle-teardown boundary and load a- // hub whose Run has already exited. Drop that hub and retry once so the- // client gets a live producer instead of a silent socket.- // WebSocket connections use context.Background() because they are long-lived and should not- // be tied to the HTTP request context. Cleanup is handled by the shared hub when it idles.- for attempt := range 2 {- hub := h.getOrCreateContainerStatsHubInternal(containerID)- if wshub.ServeClientWithOnRemove(context.Background(), hub, conn, onRemove) {- return nil- }- h.containerStatsHubs.CompareAndDelete(containerID, hub)- slog.DebugContext(c.Request().Context(), "container stats hub stopped before client registration; retrying",- "containerID", containerID, "attempt", attempt+1)- }-- slog.WarnContext(c.Request().Context(), "failed to register container stats client", "containerID", containerID)- _ = conn.CloseNow()- onRemove()- return nil-}--func (h *WebSocketHandler) getOrCreateContainerStatsHubInternal(containerID string) *wshub.Hub {- if existing, ok := h.containerStatsHubs.Load(containerID); ok {- if hub, ok := existing.(*wshub.Hub); ok {- return hub- }- }-- hub := wshub.NewHub(64)- actual, loaded := h.containerStatsHubs.LoadOrStore(containerID, hub)- if loaded {- if existingHub, ok := actual.(*wshub.Hub); ok {- return existingHub- }- // type assertion failure is impossible in practice, but avoid running- // an unregistered hub if it somehow occurs- return hub- }-- h.runContainerStatsHubInternal(containerID, hub)- return hub+ return conn, func() { h.wsMetrics.UnregisterConnection(connID) }, true}-// containerStatsErrorDrainGrace is how long a failed stats hub stays alive after-// broadcasting its error frame, so clients receive it before the hub shuts down.-const containerStatsErrorDrainGrace = 2 * time.Second--func (h *WebSocketHandler) runContainerStatsHubInternal(containerID string, hub *wshub.Hub) {- ctx, cancel := context.WithCancel(context.Background())- var cleanupTimer *time.Timer- var cleanupTimerMu sync.Mutex-- hub.SetOnEmpty(func() {- cleanupTimerMu.Lock()- if cleanupTimer != nil {- cleanupTimer.Stop()- }- var timer *time.Timer- timer = time.AfterFunc(5*time.Second, func() {- cleanupTimerMu.Lock()- defer cleanupTimerMu.Unlock()- if cleanupTimer != timer {- return- }- if existing, ok := h.containerStatsHubs.Load(containerID); ok && existing == hub {- h.containerStatsHubs.Delete(containerID)- }- slog.Debug("container stats hub idle, cleaning up upstream stream", "containerID", containerID)- cleanupTimer = nil- cancel()- })- cleanupTimer = timer- cleanupTimerMu.Unlock()- })- hub.SetOnActive(func() {- cleanupTimerMu.Lock()- if cleanupTimer != nil {- cleanupTimer.Stop()- cleanupTimer = nil- }- cleanupTimerMu.Unlock()- })-- go hub.Run(ctx)-- statsChan := make(chan any, 64)- go func(ctx context.Context) {- defer close(statsChan)-- err := h.containerService.StreamStats(ctx, containerID, statsChan)- if err == nil || errors.Is(err, context.Canceled) || ctx.Err() != nil {- return- }-- // The producer died but the hub stayed cached, so clients kept an open- // socket that would never emit another sample. Tell them why, then drop- // the hub so the next connect rebuilds a producer instead of attaching- // to this dead one.- slog.Warn("container stats stream failed", "containerID", containerID, "error", err)- if b, marshalErr := json.Marshal(map[string]any{- "error": "Failed to stream container stats: " + err.Error(),- "timestamp": wshub.NowRFC3339(),- }); marshalErr == nil {- hub.Broadcast(b)- }- h.containerStatsHubs.CompareAndDelete(containerID, hub)-- // Cancelling immediately would race hub.Run between ctx.Done and the- // queued error frame, dropping the very message clients need. Give the- // broadcast a moment to drain, then tear the hub down.- time.AfterFunc(containerStatsErrorDrainGrace, cancel)- }(ctx)-- go func() {- for {- select {- case <-ctx.Done():- return- case stats, ok := <-statsChan:- if !ok {- return- }- if b, err := json.Marshal(stats); err == nil {- hub.Broadcast(b)- }- }- }- }()-}--// ContainerExec provides interactive terminal access to a container.-//-// @Summary Execute command in container via WebSocket-// @Description Interactive terminal access to a container over WebSocket-// @Tags WebSocket-// @Param id path string true "Environment ID"-// @Param containerId path string true "Container ID"-// @Param shell query string false "Shell to execute" default(/bin/sh)-// @Router /api/environments/{id}/ws/containers/{containerId}/terminal [get]-func (h *WebSocketHandler) ContainerExec(c *echo.Context) error {- containerID := c.Param("containerId")- if strings.TrimSpace(containerID) == "" {- return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Container ID is required"})- }-- shell := queryParamWithDefaultInternal(c, "shell", "/bin/sh")-- conn, err := wshub.Accept(c.Response(), c.Request(), h.checkWSOrigin)- if err != nil {- return nil- }- connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindContainerExec, containerID))- defer h.wsMetrics.UnregisterConnection(connID)- defer func() {- if err := conn.CloseNow(); err != nil {- slog.Debug("Failed to close container exec websocket connection", "containerID", containerID, "error", err)- }- }()-- // Allow large terminal pastes; coder/websocket's default limit is 32KB.- conn.SetReadLimit(1 << 20)-- ctx, cancel := context.WithCancel(c.Request().Context())- defer cancel()-- go h.pingExecConnInternal(ctx, cancel, conn, 54*time.Second)-- h.runContainerExecInternal(ctx, cancel, conn, containerID, shell)- return nil+// wsErrorJSONInternal replies with the {"success":false,"error":...} body+// shared by every WS endpoint's pre-upgrade error path.+func wsErrorJSONInternal(c *echo.Context, status int, msg string) error {+ return c.JSON(status, map[string]any{"success": false, "error": msg})}-// pingExecConnInternal keeps the exec websocket alive. Ping round-trips (the-// pong is serviced by the concurrent stdin reader) and is safe concurrently-// with the exec output writer. A failed ping means the client is gone, so it-// cancels the exec session — a silently-dead client would otherwise keep the-// session open until an output write fails.-func (h *WebSocketHandler) pingExecConnInternal(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, period time.Duration) {+// keepWSConnAliveInternal pings the peer every period. Ping round-trips (the+// pong must be serviced by the connection's concurrent reader) and is safe+// alongside a concurrent writer. A failed ping means the client is gone, so+// it cancels the connection's context — a silently-dead client would+// otherwise keep the session open until a write fails.+func keepWSConnAliveInternal(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, period time.Duration) {ticker := time.NewTicker(period)defer ticker.Stop()for {@@ -858,569 +168,3 @@ func (h *WebSocketHandler) pingExecConnInternal(ctx context.Context, cancel cont}}}--func (h *WebSocketHandler) runContainerExecInternal(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, containerID, shell string) {- // Create exec instance- execID, err := h.containerService.CreateExec(ctx, containerID, []string{shell})- if err != nil {- h.writeExecErrorInternal(ctx, conn, errors.WithMessage(err, "Error creating exec"))- return- }-- // Attach to exec- execSession, err := h.containerService.AttachExec(ctx, containerID, execID)- if err != nil {- h.writeExecErrorInternal(ctx, conn, errors.WithMessage(err, "Error attaching to exec"))- return- }- cleanup := h.execCleanupFuncInternal(ctx, execSession, execID, containerID)- defer cleanup()- h.watchExecContextInternal(ctx, execID, containerID, cleanup)-- done := make(chan struct{})- go h.pipeExecOutputInternal(ctx, conn, execSession.Stdout(), execID, containerID, done)- go h.pipeExecInputInternal(ctx, cancel, conn, execSession.Stdin(), execID, containerID)-- <-done-}--func (h *WebSocketHandler) writeExecErrorInternal(ctx context.Context, conn *websocket.Conn, err error) {- wctx, cancel := context.WithTimeout(ctx, 10*time.Second)- defer cancel()- _ = conn.Write(wctx, websocket.MessageText, []byte(err.Error()+"\r\n"))-}--func (h *WebSocketHandler) execCleanupFuncInternal(ctx context.Context, execSession *container.ExecSession, execID, containerID string) func() {- return func() {- slog.Debug("Cleaning up exec session", "execID", execID, "containerID", containerID, "contextErr", ctx.Err())- // Cleanup must proceed even if parent ctx is canceled.- cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second)- defer cleanupCancel()- if err := execSession.Close(cleanupCtx); err != nil { //nolint:contextcheck- slog.Warn("Failed to clean up exec session", "execID", execID, "error", err)- }- }-}--func (h *WebSocketHandler) watchExecContextInternal(ctx context.Context, execID, containerID string, cleanup func()) {- go func() {- <-ctx.Done()- slog.Debug("Exec context cancelled", "execID", execID, "containerID", containerID)- cleanup()- }()-}--func (h *WebSocketHandler) pipeExecOutputInternal(ctx context.Context, conn *websocket.Conn, stdout io.Reader, execID, containerID string, done chan<- struct{}) {- defer close(done)- buf := make([]byte, 4096)- for {- select {- case <-ctx.Done():- return- default:- }-- n, err := stdout.Read(buf)- if err != nil {- slog.Debug("Exec stdout read error", "execID", execID, "containerID", containerID, "error", err)- return- }- if n > 0 {- if err := conn.Write(ctx, websocket.MessageBinary, buf[:n]); err != nil {- slog.Debug("Exec websocket write error", "execID", execID, "containerID", containerID, "error", err)- return- }- }- }-}--func (h *WebSocketHandler) pipeExecInputInternal(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn, stdin io.Writer, execID, containerID string) {- for {- _, data, err := conn.Read(ctx)- if err != nil {- slog.Debug("Exec websocket read error", "execID", execID, "containerID", containerID, "error", err)- cancel()- return- }- if _, err := stdin.Write(data); err != nil {- slog.Debug("Exec stdin write error", "execID", execID, "containerID", containerID, "error", err)- return- }- }-}--// ============================================================================-// System WebSocket Endpoints-// ============================================================================--// checkRateLimitInternal checks and applies rate limiting for WebSocket connections.-// Returns the counter and whether the connection should be allowed.-func (h *WebSocketHandler) checkRateLimitInternal(clientIP string) (*int32, bool) {- connCount, _ := h.activeConnections.LoadOrStore(clientIP, new(int32))- count, ok := connCount.(*int32)- if !ok {- return nil, false- }-- currentCount := atomic.AddInt32(count, 1)- if currentCount > 5 {- atomic.AddInt32(count, -1)- return nil, false- }- return count, true-}--// releaseRateLimitInternal decrements the connection counter and cleans up if needed.-func (h *WebSocketHandler) releaseRateLimitInternal(clientIP string, count *int32) {- newCount := atomic.AddInt32(count, -1)- if newCount <= 0 {- // CompareAndDelete, not Delete: a plain delete removed whatever counter- // was stored for this IP, which after a racing reconnect is a live one —- // wiping its count and letting that IP exceed the limit.- h.activeConnections.CompareAndDelete(clientIP, count)- }-}--func (h *WebSocketHandler) acquireSystemStatsSamplerInternal(ctx context.Context) bool {- h.systemStatsSampler.lifecycleMu.Lock()-- h.systemStatsSampler.clients++- if h.systemStatsSampler.running {- ready := h.systemStatsSampler.ready- h.systemStatsSampler.lifecycleMu.Unlock()- return waitForSystemStatsSamplerReadyInternal(ctx, ready)- }-- samplerCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))- ready := make(chan struct{})- h.systemStatsSampler.cancel = cancel- h.systemStatsSampler.ready = ready- h.systemStatsSampler.running = true- h.systemStatsSampler.lifecycleMu.Unlock()-- go func() {- closeReady := sync.OnceFunc(func() {- close(ready)- })- if !h.initializeCPUCacheCtx(samplerCtx) {- closeReady()- return- }- if samplerCtx.Err() != nil {- closeReady()- return- }-- h.storeSystemStatsSnapshotInternal(h.collectSystemStatsSnapshotInternal(samplerCtx))- closeReady()- if samplerCtx.Err() != nil {- return- }-- h.runSystemStatsSamplerInternal(samplerCtx)- }()-- return waitForSystemStatsSamplerReadyInternal(ctx, ready)-}--func waitForSystemStatsSamplerReadyInternal(ctx context.Context, ready <-chan struct{}) bool {- if ready == nil {- return true- }-- select {- case <-ctx.Done():- return false- case <-ready:- return ctx.Err() == nil- }-}--func (h *WebSocketHandler) releaseSystemStatsSamplerInternal() {- var cancel context.CancelFunc-- h.systemStatsSampler.lifecycleMu.Lock()- if h.systemStatsSampler.clients > 0 {- h.systemStatsSampler.clients--- }- if h.systemStatsSampler.clients == 0 && h.systemStatsSampler.running {- cancel = h.systemStatsSampler.cancel- h.systemStatsSampler.cancel = nil- h.systemStatsSampler.ready = nil- h.systemStatsSampler.running = false- }- h.systemStatsSampler.lifecycleMu.Unlock()-- if cancel != nil {- cancel()- }-}--func (h *WebSocketHandler) runSystemStatsSamplerInternal(ctx context.Context) {- ticker := time.NewTicker(time.Second)- defer ticker.Stop()-- for {- select {- case <-ctx.Done():- return- case <-ticker.C:- h.updateCPUCacheInternal(0)- h.storeSystemStatsSnapshotInternal(h.collectSystemStatsSnapshotInternal(ctx))- }- }-}--func (h *WebSocketHandler) storeSystemStatsSnapshotInternal(stats systemtypes.SystemStats) {- h.systemStatsSampler.stateMu.Lock()- h.systemStatsSampler.latest = stats- h.systemStatsSampler.timestamp = time.Now()- h.systemStatsSampler.stateMu.Unlock()-}--func (h *WebSocketHandler) latestSystemStatsSnapshotInternal() systemtypes.SystemStats {- h.systemStatsSampler.stateMu.RLock()- stats := h.systemStatsSampler.latest- h.systemStatsSampler.stateMu.RUnlock()- return stats-}--func (h *WebSocketHandler) collectSystemStatsSnapshotInternal(ctx context.Context) systemtypes.SystemStats {- if h.systemStatsCollector != nil {- return h.systemStatsCollector(ctx)- }- return h.collectSystemStats(ctx)-}--// collectSystemStats gathers all system statistics.-func (h *WebSocketHandler) collectSystemStats(ctx context.Context) systemtypes.SystemStats {- h.cpuCache.RLock()- cpuUsage := h.cpuCache.value- h.cpuCache.RUnlock()-- cpuCount := h.getCPUCount()- memUsed, memTotal := h.getMemoryInfo()- cpuCount, memUsed, memTotal = h.applyCgroupLimits(cpuCount, memUsed, memTotal)- diskUsed, diskTotal := h.getDiskInfo(ctx)- hostname := h.getHostname()- gpuStats, gpuCount := h.getGPUInfo(ctx)-- return systemtypes.SystemStats{- CPUUsage: cpuUsage,- MemoryUsage: memUsed,- MemoryTotal: memTotal,- DiskUsage: diskUsed,- DiskTotal: diskTotal,- CPUCount: cpuCount,- Architecture: runtime.GOARCH,- Platform: runtime.GOOS,- Hostname: hostname,- GPUCount: gpuCount,- GPUs: gpuStats,- }-}--// getCPUCount returns the number of CPUs.-func (h *WebSocketHandler) getCPUCount() int {- h.initSystemStaticInfoInternal()- return h.systemStaticInfo.cpuCount-}--func (h *WebSocketHandler) initSystemStaticInfoInternal() {- h.systemStaticInfo.once.Do(func() {- cpuCount, err := cpu.Counts(true)- if err != nil {- cpuCount = runtime.NumCPU()- }-- hostInfo, _ := host.Info()- hostname := ""- if hostInfo != nil {- hostname = hostInfo.Hostname- }-- h.systemStaticInfo.cpuCount = cpuCount- h.systemStaticInfo.hostname = hostname- })-}--// getMemoryInfo returns memory usage and total.-func (h *WebSocketHandler) getMemoryInfo() (uint64, uint64) {- memInfo, _ := mem.VirtualMemory()- if memInfo == nil {- return 0, 0- }- // gopsutil counts ZFS ARC as used memory (the kernel excludes it from- // MemAvailable). Treat the reclaimable portion as cache, matching- // btop/htop, so the dashboard does not over-report usage on ZFS hosts.- used := memInfo.Used- if arc := cgroup.ZFSARCReclaimable(); arc > 0 {- used -= min(used, arc)- }- return used, memInfo.Total-}--// applyCgroupLimits applies cgroup limits when running in an LXC (or similar)-// container where the limits represent the real hardware budget.-//-// It is intentionally a no-op inside Docker: Docker's --cpus / --memory flags-// set artificial cgroup constraints that are unrelated to the host totals we-// want to display. gopsutil already reads the correct host values there (via-// the bind-mounted /proc). Applying cgroup limits on top would produce the-// "#2343 regression" where the dashboard shows "512 MB RAM" while the host-// has 32 GB (#1110).-//-// In LXC the situation is the opposite: gopsutil reads the host's /proc-// (which shows the physical machine's RAM/CPU) rather than the slice of-// resources actually allocated to the LXC guest. The cgroup limits ARE the-// correct numbers to show.-func (h *WebSocketHandler) applyCgroupLimits(cpuCount int, memUsed, memTotal uint64) (int, uint64, uint64) {- if cgroup.IsDockerContainer() {- return cpuCount, memUsed, memTotal- }- cpuCount = applyCPUAffinityLimitInternal(cpuCount)- cgroupLimits := h.getCachedCgroupLimitsInternal()- if cgroupLimits == nil {- return cpuCount, memUsed, memTotal- }-- if limit := cgroupLimits.MemoryLimit; limit > 0 {- limitUint := uint64(limit)- if memTotal == 0 || limitUint < memTotal {- memTotal = limitUint- if cgroupLimits.MemoryUsage > 0 {- memUsed = uint64(cgroupLimits.MemoryUsage)- }- }- }- if cgroupLimits.CPUCount > 0 && (cpuCount == 0 || cgroupLimits.CPUCount < cpuCount) {- cpuCount = cgroupLimits.CPUCount- }- return cpuCount, memUsed, memTotal-}--// applyCPUAffinityLimitInternal caps the CPU count at the scheduler affinity-// mask size. In an unprivileged LXC guest the cgroup cpu quota is usually-// unset at the guest root (the limit lives host-side), so the affinity mask is-// the only signal of the cores actually assigned — including gap bindings like-// CPU1+CPU7 — while /proc/cpuinfo shows the whole host (#3161). runtime.NumCPU-// reads that mask at process start. Not applied inside Docker, matching-// applyCgroupLimits' host-totals convention there.-func applyCPUAffinityLimitInternal(cpuCount int) int {- if n := runtime.NumCPU(); n > 0 && (cpuCount == 0 || n < cpuCount) {- return n- }- return cpuCount-}--// getDiskInfo returns disk usage and total.-func (h *WebSocketHandler) getDiskInfo(ctx context.Context) (uint64, uint64) {- diskUsagePath := h.getDiskUsagePath(ctx)- diskInfo, err := disk.Usage(diskUsagePath)- if err != nil || diskInfo == nil || diskInfo.Total == 0 {- if diskUsagePath != "/" {- diskInfo, _ = disk.Usage("/")- }- }- if diskInfo == nil {- return 0, 0- }- return diskInfo.Used, diskInfo.Total-}--// getHostname returns the system hostname.-func (h *WebSocketHandler) getHostname() string {- h.initSystemStaticInfoInternal()- return h.systemStaticInfo.hostname-}--// getGPUInfo returns GPU statistics if monitoring is enabled.-func (h *WebSocketHandler) getGPUInfo(ctx context.Context) ([]systemtypes.GPUStats, int) {- if h.gpuMonitor == nil || !h.gpuMonitor.Enabled() {- return nil, 0- }- gpuData, err := h.gpuMonitor.Stats(ctx)- if err != nil {- return nil, 0- }- return gpuData, len(gpuData)-}--// initializeCPUCacheCtx performs initial CPU sampling and returns early if the sampler is canceled.-func (h *WebSocketHandler) initializeCPUCacheCtx(ctx context.Context) bool {- result := make(chan float64, 1)-- go func() {- if val, ok := h.readCPUUsageInternal(time.Second); ok {- result <- val- }- close(result)- }()-- select {- case <-ctx.Done():- return false- case val, ok := <-result:- if !ok || ctx.Err() != nil {- return false- }- h.storeCPUCacheValueInternal(val)- return true- }-}--func (h *WebSocketHandler) updateCPUCacheInternal(interval time.Duration) {- if val, ok := h.readCPUUsageInternal(interval); ok {- h.storeCPUCacheValueInternal(val)- }-}--func (h *WebSocketHandler) readCPUUsageInternal(interval time.Duration) (float64, bool) {- if h.cpuUsageReader != nil {- return h.cpuUsageReader(interval)- }-- return defaultReadCPUUsageInternal(interval)-}--var defaultReadCPUUsageInternal = func(interval time.Duration) (float64, bool) {- if vals, err := cpu.Percent(interval, false); err == nil && len(vals) > 0 {- return vals[0], true- }-- return 0, false-}--func (h *WebSocketHandler) storeCPUCacheValueInternal(value float64) {- h.cpuCache.Lock()- h.cpuCache.value = value- h.cpuCache.timestamp = time.Now()- h.cpuCache.Unlock()-}--func (h *WebSocketHandler) getCachedCgroupLimitsInternal() *cgroup.Limits {- if h.cgroupCache == nil {- return nil- }- return h.cgroupCache.Get()-}--// SystemStats streams system stats over WebSocket.-//-// @Summary Get system stats via WebSocket-// @Description Stream system resource statistics over WebSocket connection-// @Tags WebSocket-// @Param id path string true "Environment ID"-// @Router /api/environments/{id}/ws/system/stats [get]-func (h *WebSocketHandler) SystemStats(c *echo.Context) error {- clientIP := c.RealIP()-- count, allowed := h.checkRateLimitInternal(clientIP)- if !allowed {- return c.JSON(http.StatusTooManyRequests, map[string]any{- "success": false,- "error": "Too many concurrent stats connections from this IP",- })- }- defer h.releaseRateLimitInternal(clientIP, count)-- conn, err := wshub.Accept(c.Response(), c.Request(), h.checkWSOrigin)- if err != nil {- return nil- }- connID := h.wsMetrics.RegisterConnection(buildWSConnectionInfoInternal(c, systemtypes.WSKindSystemStats, ""))- defer h.wsMetrics.UnregisterConnection(connID)- defer func() {- if err := conn.CloseNow(); err != nil {- slog.Debug("Failed to close system stats websocket connection", "clientIP", clientIP, "error", err)- }- }()-- interval, _ := httputil.GetIntQueryParam(c.Request(), "interval", false)- if interval <= 0 {- interval = 2- }-- const statsPingPeriod = 54 * time.Second-- conn.SetReadLimit(512)-- ticker := time.NewTicker(time.Duration(interval) * time.Second)- defer ticker.Stop()- pingTicker := time.NewTicker(statsPingPeriod)- defer pingTicker.Stop()-- ctx, cancel := context.WithCancel(c.Request().Context())- defer cancel()- if !h.acquireSystemStatsSamplerInternal(ctx) {- h.releaseSystemStatsSamplerInternal()- return nil- }- defer h.releaseSystemStatsSamplerInternal()-- go h.readSystemStatsPumpInternal(ctx, cancel, conn)-- send := func() error {- stats := h.latestSystemStatsSnapshotInternal()- b, err := json.Marshal(stats)- if err != nil {- return err- }- wctx, cancel := context.WithTimeout(ctx, 10*time.Second)- defer cancel()- return conn.Write(wctx, websocket.MessageText, b)- }-- if err := send(); err != nil {- return nil- }-- for {- select {- case <-ctx.Done():- return nil- case <-ticker.C:- if err := send(); err != nil {- return nil- }- case <-pingTicker.C:- // Ping round-trips; the pong is serviced by readSystemStatsPumpInternal.- pctx, pcancel := context.WithTimeout(ctx, 10*time.Second)- err := conn.Ping(pctx)- pcancel()- if err != nil {- return nil- }- }- }-}--// readSystemStatsPumpInternal is the single reader for the SystemStats websocket.-// Do not add additional readers for this connection.-func (h *WebSocketHandler) readSystemStatsPumpInternal(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn) {- for {- if _, _, err := conn.Read(ctx); err != nil {- cancel()- return- }- }-}--func (h *WebSocketHandler) getDiskUsagePath(ctx context.Context) string {- if h.diskUsagePathCache == nil {- h.diskUsagePathCache = hot.NewHotCache[struct{}, string](hot.LRU, 1).- WithTTL(5 * time.Minute).- Build()- }- path, found, err := h.diskUsagePathCache.GetWithLoaders(struct{}{}, func(_ []struct{}) (map[struct{}]string, error) {- path := "/"- if h.systemService != nil {- path = h.systemService.GetDiskUsagePath(ctx)- }- return map[struct{}]string{{}: path}, nil- })- if err != nil || !found {- return "/"- }- return path-}diff --git a/backend/api/ws/handler_test.go b/backend/api/ws/handler_test.godeleted file mode 100644index 5d0ce4098..000000000--- a/backend/api/ws/handler_test.go+++ /dev/null@@ -1,20 +0,0 @@-package ws--import (- "runtime"- "testing"-- "github.com/stretchr/testify/require"-)--func TestApplyCPUAffinityLimitInternal(t *testing.T) {- affinity := runtime.NumCPU()-- // A host-wide count larger than the affinity mask (LXC without lxcfs,- // #3161) is capped at the mask size; a smaller cgroup-derived count wins.- require.Equal(t, affinity, applyCPUAffinityLimitInternal(affinity+8))- require.Equal(t, affinity, applyCPUAffinityLimitInternal(0))- if affinity > 1 {- require.Equal(t, 1, applyCPUAffinityLimitInternal(1))- }-}diff --git a/backend/api/ws/log_streams.go b/backend/api/ws/log_streams.gonew file mode 100644index 000000000..ed4c0a48e--- /dev/null+++ b/backend/api/ws/log_streams.go@@ -0,0 +1,482 @@+package ws++import (+ "cmp"+ "context"+ "encoding/json/v2"+ "log/slog"+ "net/http"+ "strconv"+ "strings"+ "sync/atomic"+ "time"++ "emperror.dev/errors"+ "github.com/labstack/echo/v5"++ wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"+ systemtypes "github.com/getarcaneapp/arcane/types/v2/system"+)++// ============================================================================+// Shared Log Stream Helpers+// ============================================================================++type wsLogStream struct {+ hub *wshub.Hub+ cancel context.CancelFunc+ firstSubscriber chan struct{}+ format string+ key string+ refs int+ done bool+ seq atomic.Uint64+}++func buildLogStreamKeyInternal(envID, kind, resourceID, format string, batched, follow bool, tail, since string, timestamps bool) string {+ return strings.Join([]string{+ envID,+ kind,+ resourceID,+ format,+ strconv.FormatBool(batched),+ strconv.FormatBool(follow),+ tail,+ since,+ strconv.FormatBool(timestamps),+ }, "|")+}++func (h *WebSocketHandler) getOrCreateLogStreamInternal(key string, create func(onEmpty func(*wsLogStream)) *wsLogStream) *wsLogStream {+ h.logStreamsMu.Lock()+ defer h.logStreamsMu.Unlock()++ if stream, ok := h.logStreams[key]; ok {+ if !stream.done {+ stream.refs+++ return stream+ }+ }++ stream := create(func(stream *wsLogStream) {+ h.markLogStreamDoneInternal(key, stream)+ })+ stream.key = key+ stream.refs = 1+ h.logStreams[key] = stream+ return stream+}++func takeLogStreamCancelInternal(stream *wsLogStream) context.CancelFunc {+ cancel := stream.cancel+ stream.cancel = nil+ return cancel+}++func (h *WebSocketHandler) releaseLogStreamInternal(key string, stream *wsLogStream) {+ var cancel context.CancelFunc++ h.logStreamsMu.Lock()+ if stream.refs > 0 {+ stream.refs--+ }+ if stream.refs == 0 {+ if current, ok := h.logStreams[key]; ok && current == stream {+ delete(h.logStreams, key)+ }+ cancel = takeLogStreamCancelInternal(stream)+ }+ h.logStreamsMu.Unlock()++ if cancel != nil {+ cancel()+ }+}++func (h *WebSocketHandler) markLogStreamDoneInternal(key string, stream *wsLogStream) {+ var cancel context.CancelFunc++ h.logStreamsMu.Lock()+ stream.done = true+ if stream.refs == 0 {+ if current, ok := h.logStreams[key]; ok && current == stream {+ delete(h.logStreams, key)+ }+ cancel = takeLogStreamCancelInternal(stream)+ }+ h.logStreamsMu.Unlock()++ if cancel != nil {+ cancel()+ }+}++// logStreamParams holds the standard query parameters shared by every WS log endpoint.+type logStreamParams struct {+ tail string+ since string+ format string+ follow bool+ timestamps bool+ batched bool+}++func parseLogStreamParamsInternal(c *echo.Context) logStreamParams {+ return logStreamParams{+ follow: cmp.Or(c.QueryParam("follow"), "true") == "true",+ tail: cmp.Or(c.QueryParam("tail"), "100"),+ since: c.QueryParam("since"),+ timestamps: c.QueryParam("timestamps") == "true",+ format: cmp.Or(c.QueryParam("format"), "text"),+ batched: c.QueryParam("batched") == "true",+ }+}++// serveLogStreamInternal is the shared scaffold for all three WS log endpoints (project, container, service).+// It performs upgrade, builds the stream key, gets-or-creates the multiplexing hub, registers metrics,+// and serves the client. The caller-supplied hubBuilder constructs the underlying *wsLogStream+// when no hub already exists for streamKey.+func (h *WebSocketHandler) serveLogStreamInternal(+ c *echo.Context,+ kind, resourceID string,+ params logStreamParams,+ hubBuilder func(streamKey string, onEmpty func(*wsLogStream)) *wsLogStream,+) {+ conn, unregister, ok := h.acceptWSInternal(c, kind, resourceID)+ if !ok {+ return+ }++ streamKey := buildLogStreamKeyInternal(c.Param("id"), kind, resourceID, params.format, params.batched, params.follow, params.tail, params.since, params.timestamps)+ stream := h.getOrCreateLogStreamInternal(streamKey, func(onEmpty func(*wsLogStream)) *wsLogStream {+ return hubBuilder(streamKey, onEmpty)+ })+ release := func() {+ unregister()+ h.releaseLogStreamInternal(streamKey, stream)+ }+ // WebSocket connections use context.Background() because they are long-lived and should not+ // be tied to the HTTP request context. Cleanup is handled via the hub's OnEmpty callback+ // which triggers when all clients disconnect.+ if !wshub.ServeClientWithOnRemove(context.Background(), stream.hub, conn, release) {+ // The stream refcount normally keeps this hub alive, so a stopped hub+ // here means it was torn down out from under us; drop our reference+ // rather than leaking the connection and the metrics entry.+ slog.Debug("log stream hub stopped before client registration", "streamKey", streamKey)+ _ = conn.CloseNow()+ release()+ }+}++// broadcastLogStreamErrorInternal emits an error message to every client of a log stream.+// resourceLabel is the human-readable noun used in slog/error text (e.g. "project log stream").+// errorPrefix is the user-facing message prefix (e.g. "Failed to stream project logs: ").+func broadcastLogStreamErrorInternal(resourceLabel, errorPrefix string, resourceID string, format string, err error, ls *wsLogStream) {+ slog.Warn(resourceLabel+" failed", "resourceID", resourceID, "error", err)++ if format == "json" {+ msg := wshub.LogMessage{+ Seq: ls.seq.Add(1),+ Level: "error",+ Message: errorPrefix + err.Error(),+ Service: "arcane",+ Timestamp: wshub.NowRFC3339(),+ }+ if b, marshalErr := json.Marshal(msg); marshalErr == nil {+ ls.hub.Broadcast(b)+ }+ return+ }++ ls.hub.Broadcast([]byte(errorPrefix + err.Error()))+}++// ============================================================================+// Project WebSocket/Streaming Endpoints+// ============================================================================++// ProjectLogs streams project logs over WebSocket.+//+// @Summary Get project logs via WebSocket+// @Description Stream project logs over WebSocket connection+// @Tags WebSocket+// @Param id path string true "Environment ID"+// @Param projectId path string true "Project ID"+// @Param follow query bool false "Follow log output" default(true)+// @Param tail query string false "Number of lines to show from the end" default(100)+// @Param since query string false "Show logs since timestamp"+// @Param timestamps query bool false "Show timestamps" default(false)+// @Param format query string false "Output format (text or json)" default(text)+// @Param batched query bool false "Batch log messages" default(false)+// @Router /api/environments/{id}/ws/projects/{projectId}/logs [get]+func (h *WebSocketHandler) ProjectLogs(c *echo.Context) error {+ projectID := c.Param("projectId")+ if strings.TrimSpace(projectID) == "" {+ return wsErrorJSONInternal(c, http.StatusBadRequest, "Project ID is required")+ }++ streamLogs := h.projectLogStreamer+ if streamLogs == nil {+ streamLogs = h.projectService.StreamProjectLogs+ }++ params := parseLogStreamParamsInternal(c)+ h.serveLogStreamInternal(c, systemtypes.WSKindProjectLogs, projectID, params, func(streamKey string, onEmpty func(*wsLogStream)) *wsLogStream {+ return h.startLogHubInternal(+ streamKey,+ projectID,+ "project",+ params,+ streamLogs,+ normalizeProjectLogMessageInternal,+ normalizeProjectLogTextInternal,+ onEmpty,+ )+ })+ return nil+}++func newWSLogStreamInternal(key, format string) (*wsLogStream, context.Context) {+ ls := &wsLogStream{+ hub: wshub.NewHub(1024),+ firstSubscriber: make(chan struct{}),+ format: format,+ key: key,+ }+ ls.hub.SetOnFirstClient(func() {+ close(ls.firstSubscriber)+ })++ ctx, cancel := context.WithCancel(context.Background()) //nolint:gosec // cancel is intentionally retained and invoked by the hub OnEmpty callback.+ ls.cancel = cancel++ go ls.hub.Run(ctx)++ return ls, ctx+}++func normalizeProjectLogMessageInternal(line string) wshub.LogMessage {+ level, service, message, timestamp := wshub.NormalizeProjectLine(line)+ return wshub.LogMessage{+ Level: level,+ Message: message,+ Service: service,+ Timestamp: timestamp,+ }+}++func normalizeContainerLogMessageInternal(line string) wshub.LogMessage {+ level, message, timestamp := wshub.NormalizeContainerLine(line)+ return wshub.LogMessage{+ Level: level,+ Message: message,+ Timestamp: timestamp,+ }+}++func normalizeProjectLogTextInternal(line string) string {+ _, _, message, _ := wshub.NormalizeProjectLine(line)+ return message+}++func (h *WebSocketHandler) startLogHubInternal(+ key, resourceID, label string,+ params logStreamParams,+ stream func(context.Context, string, chan<- string, bool, string, string, bool) error,+ normalizeJSON func(string) wshub.LogMessage,+ normalizeText func(string) string,+ onEmptyHook func(*wsLogStream),+) *wsLogStream {+ ls, ctx := newWSLogStreamInternal(key, params.format)++ ls.hub.SetOnEmpty(func() {+ if onEmptyHook != nil {+ onEmptyHook(ls)+ }+ slog.Debug("client disconnected, cleaning up "+label+" log hub", label+"ID", resourceID)+ })++ lines := h.startLogSourceInternal(ctx, key, resourceID, label, params, stream, ls)+ startLogForwardersInternal(ctx, ls, lines, params, normalizeJSON, normalizeText)++ return ls+}++func (h *WebSocketHandler) startLogSourceInternal(+ ctx context.Context,+ key, resourceID, label string,+ params logStreamParams,+ stream func(context.Context, string, chan<- string, bool, string, string, bool) error,+ ls *wsLogStream,+) <-chan string {+ lines := make(chan string, 256)+ go func() {+ defer close(lines)+ // The Docker log stream is only started once someone is listening.+ select {+ case <-ctx.Done():+ return+ case <-ls.firstSubscriber:+ }++ if err := stream(ctx, resourceID, lines, params.follow, params.tail, params.since, params.timestamps); err != nil {+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {+ return+ }++ h.markLogStreamDoneInternal(key, ls)+ broadcastLogStreamErrorInternal(label+" log stream", "Failed to stream "+label+" logs: ", resourceID, params.format, err, ls)+ return+ }++ if ctx.Err() == nil {+ h.markLogStreamDoneInternal(key, ls)+ }+ }()++ return lines+}++func startLogForwardersInternal(+ ctx context.Context,+ ls *wsLogStream,+ lines <-chan string,+ params logStreamParams,+ normalizeJSON func(string) wshub.LogMessage,+ normalizeText func(string) string,+) {+ if params.format == "json" {+ messages := mapLogLinesInternal(ctx, lines, func(line string) wshub.LogMessage {+ message := normalizeJSON(line)+ message.Seq = ls.seq.Add(1)+ if message.Timestamp == "" {+ message.Timestamp = wshub.NowRFC3339()+ }+ return message+ })++ if params.batched {+ go wshub.ForwardLogJSONBatched(ctx, ls.hub, messages, 50, 400*time.Millisecond)+ } else {+ go wshub.ForwardLogJSON(ctx, ls.hub, messages)+ }++ return+ }++ textLines := lines+ if normalizeText != nil {+ textLines = mapLogLinesInternal(ctx, lines, normalizeText)+ }+ go wshub.ForwardLines(ctx, ls.hub, textLines)+}++func mapLogLinesInternal[T any](ctx context.Context, lines <-chan string, transform func(string) T) <-chan T {+ mapped := make(chan T, 256)+ go func() {+ defer close(mapped)+ for {+ select {+ case <-ctx.Done():+ return+ case line, ok := <-lines:+ if !ok {+ return+ }++ select {+ case <-ctx.Done():+ return+ case mapped <- transform(line):+ }+ }+ }+ }()++ return mapped+}++// ============================================================================+// Container WebSocket Endpoints+// ============================================================================++// ContainerLogs streams container logs over WebSocket.+//+// @Summary Get container logs via WebSocket+// @Description Stream container logs over WebSocket connection+// @Tags WebSocket+// @Param id path string true "Environment ID"+// @Param containerId path string true "Container ID"+// @Param follow query bool false "Follow log output" default(true)+// @Param tail query string false "Number of lines to show from the end" default(100)+// @Param since query string false "Show logs since timestamp"+// @Param timestamps query bool false "Show timestamps" default(false)+// @Param format query string false "Output format (text or json)" default(text)+// @Param batched query bool false "Batch log messages" default(false)+// @Router /api/environments/{id}/ws/containers/{containerId}/logs [get]+func (h *WebSocketHandler) ContainerLogs(c *echo.Context) error {+ containerID := c.Param("containerId")+ if strings.TrimSpace(containerID) == "" {+ return wsErrorJSONInternal(c, http.StatusBadRequest, "Container ID is required")+ }++ streamLogs := h.containerLogStreamer+ if streamLogs == nil {+ streamLogs = h.containerService.StreamLogs+ }++ params := parseLogStreamParamsInternal(c)+ h.serveLogStreamInternal(c, systemtypes.WSKindContainerLogs, containerID, params, func(streamKey string, onEmpty func(*wsLogStream)) *wsLogStream {+ return h.startLogHubInternal(+ streamKey,+ containerID,+ "container",+ params,+ streamLogs,+ normalizeContainerLogMessageInternal,+ nil,+ onEmpty,+ )+ })+ return nil+}++// ============================================================================+// Swarm Service WebSocket/Streaming Endpoints+// ============================================================================++// ServiceLogs streams swarm service logs over WebSocket.+//+// @Summary Get swarm service logs via WebSocket+// @Description Stream swarm service logs over WebSocket connection+// @Tags WebSocket+// @Param id path string true "Environment ID"+// @Param serviceId path string true "Service ID"+// @Param follow query bool false "Follow log output" default(true)+// @Param tail query string false "Number of lines to show from the end" default(100)+// @Param since query string false "Show logs since timestamp"+// @Param timestamps query bool false "Show timestamps" default(false)+// @Param format query string false "Output format (text or json)" default(text)+// @Param batched query bool false "Batch log messages" default(false)+// @Router /api/environments/{id}/ws/swarm/services/{serviceId}/logs [get]+func (h *WebSocketHandler) ServiceLogs(c *echo.Context) error {+ serviceID := c.Param("serviceId")+ if strings.TrimSpace(serviceID) == "" {+ return wsErrorJSONInternal(c, http.StatusBadRequest, "Service ID is required")+ }++ params := parseLogStreamParamsInternal(c)+ h.serveLogStreamInternal(c, systemtypes.WSKindServiceLogs, serviceID, params, func(streamKey string, onEmpty func(*wsLogStream)) *wsLogStream {+ return h.startLogHubInternal(+ streamKey,+ serviceID,+ "service",+ params,+ h.swarmService.StreamServiceLogs,+ normalizeContainerLogMessageInternal,+ nil,+ onEmpty,+ )+ })+ return nil+}diff --git a/backend/api/ws/system_stats.go b/backend/api/ws/system_stats.gonew file mode 100644index 000000000..9ca50737f--- /dev/null+++ b/backend/api/ws/system_stats.go@@ -0,0 +1,527 @@+package ws++import (+ "context"+ "encoding/json/v2"+ "log/slog"+ "net/http"+ "runtime"+ "sync"+ "sync/atomic"+ "time"++ "github.com/coder/websocket"+ "github.com/labstack/echo/v5"+ "github.com/samber/hot"+ "github.com/shirou/gopsutil/v4/cpu"+ "github.com/shirou/gopsutil/v4/disk"+ "github.com/shirou/gopsutil/v4/host"+ "github.com/shirou/gopsutil/v4/mem"++ httputil "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"+ systemtypes "github.com/getarcaneapp/arcane/types/v2/system"+ "go.getarcane.app/sys/cgroup"+)++const (+ cgroupCacheTTL = 30 * time.Second++ // systemStatsSamplerDefaultInterval matches the stats endpoint's default+ // subscriber interval; the shared sampler never ticks faster than the+ // fastest active subscriber.+ systemStatsSamplerDefaultInterval = 2 * time.Second+)++// ============================================================================+// System WebSocket Endpoints+// ============================================================================++// checkRateLimitInternal checks and applies rate limiting for WebSocket connections.+// Returns the counter and whether the connection should be allowed.+func (h *WebSocketHandler) checkRateLimitInternal(clientIP string) (*int32, bool) {+ connCount, _ := h.activeConnections.LoadOrStore(clientIP, new(int32))+ count, ok := connCount.(*int32)+ if !ok {+ return nil, false+ }++ currentCount := atomic.AddInt32(count, 1)+ if currentCount > 5 {+ atomic.AddInt32(count, -1)+ return nil, false+ }+ return count, true+}++// releaseRateLimitInternal decrements the connection counter and cleans up if needed.+func (h *WebSocketHandler) releaseRateLimitInternal(clientIP string, count *int32) {+ newCount := atomic.AddInt32(count, -1)+ if newCount <= 0 {+ // CompareAndDelete, not Delete: a plain delete removed whatever counter+ // was stored for this IP, which after a racing reconnect is a live one —+ // wiping its count and letting that IP exceed the limit.+ h.activeConnections.CompareAndDelete(clientIP, count)+ }+}++func (h *WebSocketHandler) acquireSystemStatsSamplerInternal(ctx context.Context, interval time.Duration) bool {+ h.systemStatsSampler.lifecycleMu.Lock()++ h.systemStatsSampler.clients+++ if h.systemStatsSampler.intervals == nil {+ h.systemStatsSampler.intervals = map[time.Duration]int{}+ }+ h.systemStatsSampler.intervals[interval]+++ h.refreshSystemStatsSamplerIntervalLockedInternal()+ if h.systemStatsSampler.running {+ ready := h.systemStatsSampler.ready+ h.systemStatsSampler.lifecycleMu.Unlock()+ return waitForSystemStatsSamplerReadyInternal(ctx, ready)+ }++ samplerCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))+ ready := make(chan struct{})+ wake := make(chan struct{}, 1)+ h.systemStatsSampler.cancel = cancel+ h.systemStatsSampler.ready = ready+ h.systemStatsSampler.wake = wake+ h.systemStatsSampler.running = true+ h.systemStatsSampler.lifecycleMu.Unlock()++ collect := h.systemStatsCollector+ if collect == nil {+ collect = h.collectSystemStats+ }++ go func() {+ closeReady := sync.OnceFunc(func() {+ close(ready)+ })+ if !h.initializeCPUCacheCtx(samplerCtx) {+ closeReady()+ return+ }+ if samplerCtx.Err() != nil {+ closeReady()+ return+ }++ h.systemStatsSampler.latest.Store(collect(samplerCtx))+ closeReady()+ if samplerCtx.Err() != nil {+ return+ }++ h.runSystemStatsSamplerInternal(samplerCtx, wake, collect)+ }()++ return waitForSystemStatsSamplerReadyInternal(ctx, ready)+}++func waitForSystemStatsSamplerReadyInternal(ctx context.Context, ready <-chan struct{}) bool {+ if ready == nil {+ return true+ }++ select {+ case <-ctx.Done():+ return false+ case <-ready:+ return ctx.Err() == nil+ }+}++func (h *WebSocketHandler) releaseSystemStatsSamplerInternal(interval time.Duration) {+ var cancel context.CancelFunc++ h.systemStatsSampler.lifecycleMu.Lock()+ if h.systemStatsSampler.clients > 0 {+ h.systemStatsSampler.clients--+ }+ if remaining := h.systemStatsSampler.intervals[interval]; remaining > 1 {+ h.systemStatsSampler.intervals[interval] = remaining - 1+ } else {+ delete(h.systemStatsSampler.intervals, interval)+ }+ h.refreshSystemStatsSamplerIntervalLockedInternal()+ if h.systemStatsSampler.clients == 0 && h.systemStatsSampler.running {+ cancel = h.systemStatsSampler.cancel+ h.systemStatsSampler.cancel = nil+ h.systemStatsSampler.ready = nil+ h.systemStatsSampler.wake = nil+ h.systemStatsSampler.running = false+ }+ h.systemStatsSampler.lifecycleMu.Unlock()++ if cancel != nil {+ cancel()+ }+}++// refreshSystemStatsSamplerIntervalLockedInternal recomputes the sampler's+// effective interval — the smallest one any subscriber requested — and must+// be called with lifecycleMu held.+func (h *WebSocketHandler) refreshSystemStatsSamplerIntervalLockedInternal() {+ effective := time.Duration(0)+ for interval := range h.systemStatsSampler.intervals {+ if effective == 0 || interval < effective {+ effective = interval+ }+ }+ if effective <= 0 {+ effective = systemStatsSamplerDefaultInterval+ }+ previous := time.Duration(h.systemStatsSampler.effectiveInterval.Swap(int64(effective)))+ if previous != effective && h.systemStatsSampler.wake != nil {+ select {+ case h.systemStatsSampler.wake <- struct{}{}:+ default:+ }+ }+}++func (h *WebSocketHandler) systemStatsSamplerIntervalInternal() time.Duration {+ if interval := time.Duration(h.systemStatsSampler.effectiveInterval.Load()); interval > 0 {+ return interval+ }+ return systemStatsSamplerDefaultInterval+}++func (h *WebSocketHandler) runSystemStatsSamplerInternal(ctx context.Context, wake <-chan struct{}, collect func(context.Context) systemtypes.SystemStats) {+ interval := h.systemStatsSamplerIntervalInternal()+ ticker := time.NewTicker(interval)+ defer ticker.Stop()++ // Subscribers come and go while the sampler runs; re-arm the ticker+ // whenever the fastest requested interval changed.+ rearm := func() {+ if next := h.systemStatsSamplerIntervalInternal(); next != interval {+ interval = next+ ticker.Reset(interval)+ }+ }++ for {+ select {+ case <-ctx.Done():+ return+ case <-wake:+ rearm()+ case <-ticker.C:+ h.updateCPUCacheInternal(0)+ h.systemStatsSampler.latest.Store(collect(ctx))+ rearm()+ }+ }+}++// collectSystemStats gathers all system statistics.+func (h *WebSocketHandler) collectSystemStats(ctx context.Context) systemtypes.SystemStats {+ cpuUsage, _ := h.cpuCache.Load()++ cpuCount := h.getCPUCount()+ memUsed, memTotal := h.getMemoryInfo()+ cpuCount, memUsed, memTotal = h.applyCgroupLimits(cpuCount, memUsed, memTotal)+ diskUsed, diskTotal := h.getDiskInfo(ctx)+ hostname := h.getHostname()+ gpuStats, gpuCount := h.getGPUInfo(ctx)++ return systemtypes.SystemStats{+ CPUUsage: cpuUsage,+ MemoryUsage: memUsed,+ MemoryTotal: memTotal,+ DiskUsage: diskUsed,+ DiskTotal: diskTotal,+ CPUCount: cpuCount,+ Architecture: runtime.GOARCH,+ Platform: runtime.GOOS,+ Hostname: hostname,+ GPUCount: gpuCount,+ GPUs: gpuStats,+ }+}++// getCPUCount returns the number of CPUs.+func (h *WebSocketHandler) getCPUCount() int {+ h.initSystemStaticInfoInternal()+ return h.systemStaticInfo.cpuCount+}++func (h *WebSocketHandler) initSystemStaticInfoInternal() {+ h.systemStaticInfo.once.Do(func() {+ cpuCount, err := cpu.Counts(true)+ if err != nil {+ cpuCount = runtime.NumCPU()+ }++ hostInfo, _ := host.Info()+ hostname := ""+ if hostInfo != nil {+ hostname = hostInfo.Hostname+ }++ h.systemStaticInfo.cpuCount = cpuCount+ h.systemStaticInfo.hostname = hostname+ })+}++// getMemoryInfo returns memory usage and total.+func (h *WebSocketHandler) getMemoryInfo() (uint64, uint64) {+ memInfo, _ := mem.VirtualMemory()+ if memInfo == nil {+ return 0, 0+ }+ // gopsutil counts ZFS ARC as used memory (the kernel excludes it from+ // MemAvailable). Treat the reclaimable portion as cache, matching+ // btop/htop, so the dashboard does not over-report usage on ZFS hosts.+ used := memInfo.Used+ if arc := cgroup.ZFSARCReclaimable(); arc > 0 {+ used -= min(used, arc)+ }+ return used, memInfo.Total+}++// applyCgroupLimits applies cgroup limits when running in an LXC (or similar)+// container where the limits represent the real hardware budget.+//+// It is intentionally a no-op inside Docker: Docker's --cpus / --memory flags+// set artificial cgroup constraints that are unrelated to the host totals we+// want to display. gopsutil already reads the correct host values there (via+// the bind-mounted /proc). Applying cgroup limits on top would produce the+// "#2343 regression" where the dashboard shows "512 MB RAM" while the host+// has 32 GB (#1110).+//+// In LXC the situation is the opposite: gopsutil reads the host's /proc+// (which shows the physical machine's RAM/CPU) rather than the slice of+// resources actually allocated to the LXC guest. The cgroup limits ARE the+// correct numbers to show.+func (h *WebSocketHandler) applyCgroupLimits(cpuCount int, memUsed, memTotal uint64) (int, uint64, uint64) {+ if cgroup.IsDockerContainer() {+ return cpuCount, memUsed, memTotal+ }+ cpuCount = applyCPUAffinityLimitInternal(cpuCount)+ cgroupLimits := h.getCachedCgroupLimitsInternal()+ if cgroupLimits == nil {+ return cpuCount, memUsed, memTotal+ }++ if limit := cgroupLimits.MemoryLimit; limit > 0 {+ limitUint := uint64(limit)+ if memTotal == 0 || limitUint < memTotal {+ memTotal = limitUint+ if cgroupLimits.MemoryUsage > 0 {+ memUsed = uint64(cgroupLimits.MemoryUsage)+ }+ }+ }+ if cgroupLimits.CPUCount > 0 && (cpuCount == 0 || cgroupLimits.CPUCount < cpuCount) {+ cpuCount = cgroupLimits.CPUCount+ }+ return cpuCount, memUsed, memTotal+}++// applyCPUAffinityLimitInternal caps the CPU count at the scheduler affinity+// mask size. In an unprivileged LXC guest the cgroup cpu quota is usually+// unset at the guest root (the limit lives host-side), so the affinity mask is+// the only signal of the cores actually assigned — including gap bindings like+// CPU1+CPU7 — while /proc/cpuinfo shows the whole host (#3161). runtime.NumCPU+// reads that mask at process start. Not applied inside Docker, matching+// applyCgroupLimits' host-totals convention there.+func applyCPUAffinityLimitInternal(cpuCount int) int {+ if n := runtime.NumCPU(); n > 0 && (cpuCount == 0 || n < cpuCount) {+ return n+ }+ return cpuCount+}++// getDiskInfo returns disk usage and total.+func (h *WebSocketHandler) getDiskInfo(ctx context.Context) (uint64, uint64) {+ diskUsagePath := h.getDiskUsagePath(ctx)+ diskInfo, err := disk.Usage(diskUsagePath)+ if err != nil || diskInfo == nil || diskInfo.Total == 0 {+ if diskUsagePath != "/" {+ diskInfo, _ = disk.Usage("/")+ }+ }+ if diskInfo == nil {+ return 0, 0+ }+ return diskInfo.Used, diskInfo.Total+}++// getHostname returns the system hostname.+func (h *WebSocketHandler) getHostname() string {+ h.initSystemStaticInfoInternal()+ return h.systemStaticInfo.hostname+}++// getGPUInfo returns GPU statistics if monitoring is enabled.+func (h *WebSocketHandler) getGPUInfo(ctx context.Context) ([]systemtypes.GPUStats, int) {+ if h.gpuMonitor == nil || !h.gpuMonitor.Enabled() {+ return nil, 0+ }+ gpuData, err := h.gpuMonitor.Stats(ctx)+ if err != nil {+ return nil, 0+ }+ return gpuData, len(gpuData)+}++// initializeCPUCacheCtx performs initial CPU sampling and returns early if the sampler is canceled.+func (h *WebSocketHandler) initializeCPUCacheCtx(ctx context.Context) bool {+ result := make(chan float64, 1)++ go func() {+ if val, ok := h.readCPUUsageInternal(time.Second); ok {+ result <- val+ }+ close(result)+ }()++ select {+ case <-ctx.Done():+ return false+ case val, ok := <-result:+ if !ok || ctx.Err() != nil {+ return false+ }+ h.cpuCache.Store(val)+ return true+ }+}++func (h *WebSocketHandler) updateCPUCacheInternal(interval time.Duration) {+ if val, ok := h.readCPUUsageInternal(interval); ok {+ h.cpuCache.Store(val)+ }+}++func (h *WebSocketHandler) readCPUUsageInternal(interval time.Duration) (float64, bool) {+ if h.cpuUsageReader != nil {+ return h.cpuUsageReader(interval)+ }++ return defaultReadCPUUsageInternal(interval)+}++func defaultReadCPUUsageInternal(interval time.Duration) (float64, bool) {+ if vals, err := cpu.Percent(interval, false); err == nil && len(vals) > 0 {+ return vals[0], true+ }++ return 0, false+}++func (h *WebSocketHandler) getCachedCgroupLimitsInternal() *cgroup.Limits {+ if h.cgroupCache == nil {+ return nil+ }+ return h.cgroupCache.Get()+}++// SystemStats streams system stats over WebSocket.+//+// @Summary Get system stats via WebSocket+// @Description Stream system resource statistics over WebSocket connection+// @Tags WebSocket+// @Param id path string true "Environment ID"+// @Router /api/environments/{id}/ws/system/stats [get]+func (h *WebSocketHandler) SystemStats(c *echo.Context) error {+ clientIP := c.RealIP()++ count, allowed := h.checkRateLimitInternal(clientIP)+ if !allowed {+ return wsErrorJSONInternal(c, http.StatusTooManyRequests, "Too many concurrent stats connections from this IP")+ }+ defer h.releaseRateLimitInternal(clientIP, count)++ conn, unregister, ok := h.acceptWSInternal(c, systemtypes.WSKindSystemStats, "")+ if !ok {+ return nil+ }+ defer unregister()+ defer func() {+ if err := conn.CloseNow(); err != nil {+ slog.Debug("Failed to close system stats websocket connection", "clientIP", clientIP, "error", err)+ }+ }()++ interval, _ := httputil.GetIntQueryParam(c.Request(), "interval", false)+ if interval <= 0 {+ interval = 2+ }+ sampleInterval := time.Duration(interval) * time.Second++ conn.SetReadLimit(512)++ ticker := time.NewTicker(sampleInterval)+ defer ticker.Stop()++ ctx, cancel := context.WithCancel(c.Request().Context())+ defer cancel()+ if !h.acquireSystemStatsSamplerInternal(ctx, sampleInterval) {+ h.releaseSystemStatsSamplerInternal(sampleInterval)+ return nil+ }+ defer h.releaseSystemStatsSamplerInternal(sampleInterval)++ go h.readSystemStatsPumpInternal(ctx, cancel, conn)+ // The pong is serviced by readSystemStatsPumpInternal.+ go keepWSConnAliveInternal(ctx, cancel, conn, 54*time.Second)++ send := func() error {+ stats, _ := h.systemStatsSampler.latest.Load()+ b, err := json.Marshal(stats)+ if err != nil {+ return err+ }+ wctx, cancel := context.WithTimeout(ctx, 10*time.Second)+ defer cancel()+ return conn.Write(wctx, websocket.MessageText, b)+ }++ if err := send(); err != nil {+ return nil+ }++ for {+ select {+ case <-ctx.Done():+ return nil+ case <-ticker.C:+ if err := send(); err != nil {+ return nil+ }+ }+ }+}++// readSystemStatsPumpInternal is the single reader for the SystemStats websocket.+// Do not add additional readers for this connection.+func (h *WebSocketHandler) readSystemStatsPumpInternal(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn) {+ for {+ if _, _, err := conn.Read(ctx); err != nil {+ cancel()+ return+ }+ }+}++func (h *WebSocketHandler) getDiskUsagePath(ctx context.Context) string {+ if h.diskUsagePathCache == nil {+ h.diskUsagePathCache = hot.NewHotCache[struct{}, string](hot.LRU, 1).+ WithTTL(5 * time.Minute).+ Build()+ }+ path, found, err := h.diskUsagePathCache.GetWithLoaders(struct{}{}, func(_ []struct{}) (map[struct{}]string, error) {+ path := "/"+ if h.systemService != nil {+ path = h.systemService.GetDiskUsagePath(ctx)+ }+ return map[struct{}]string{{}: path}, nil+ })+ if err != nil || !found {+ return "/"+ }+ return path+}diff --git a/backend/api/ws/system_stats_test.go b/backend/api/ws/system_stats_test.gonew file mode 100644index 000000000..fa105efe8--- /dev/null+++ b/backend/api/ws/system_stats_test.go@@ -0,0 +1,104 @@+package ws++import (+ "context"+ "runtime"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ systemtypes "github.com/getarcaneapp/arcane/types/v2/system"+)++func TestApplyCPUAffinityLimitInternal(t *testing.T) {+ affinity := runtime.NumCPU()++ // A host-wide count larger than the affinity mask (LXC without lxcfs,+ // #3161) is capped at the mask size; a smaller cgroup-derived count wins.+ require.Equal(t, affinity, applyCPUAffinityLimitInternal(affinity+8))+ require.Equal(t, affinity, applyCPUAffinityLimitInternal(0))+ if affinity > 1 {+ require.Equal(t, 1, applyCPUAffinityLimitInternal(1))+ }+}++func TestSystemStatsSamplerEffectiveInterval(t *testing.T) {+ tests := []struct {+ name string+ intervals map[time.Duration]int+ want time.Duration+ }{+ {+ name: "no subscribers falls back to the default",+ want: systemStatsSamplerDefaultInterval,+ },+ {+ // The sampler must not tick faster than anyone reads.+ name: "single slow subscriber sets the pace",+ intervals: map[time.Duration]int{5 * time.Second: 1},+ want: 5 * time.Second,+ },+ {+ name: "fastest of several subscribers wins",+ intervals: map[time.Duration]int{5 * time.Second: 1, 2 * time.Second: 1},+ want: 2 * time.Second,+ },+ {+ // Mirrors releasing the fast subscriber from the case above.+ name: "remaining subscriber's pace after a release",+ intervals: map[time.Duration]int{5 * time.Second: 1},+ want: 5 * time.Second,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ h := &WebSocketHandler{}+ h.systemStatsSampler.intervals = tt.intervals+ h.refreshSystemStatsSamplerIntervalLockedInternal()+ require.Equal(t, tt.want, h.systemStatsSamplerIntervalInternal())+ })+ }+}++// TestSystemStatsSamplerRearmsOnFasterSubscriber verifies that a faster+// subscriber joining a running sampler re-arms the ticker immediately+// instead of waiting out the slower tick already in flight (which left+// the fast subscriber rereading a stale snapshot for up to the old+// interval).+func TestSystemStatsSamplerRearmsOnFasterSubscriber(t *testing.T) {+ collections := make(chan struct{}, 16)+ h := &WebSocketHandler{+ cpuUsageReader: func(time.Duration) (float64, bool) { return 0, true },+ systemStatsCollector: func(context.Context) systemtypes.SystemStats {+ select {+ case collections <- struct{}{}:+ default:+ }+ return systemtypes.SystemStats{}+ },+ }++ ctx := context.Background()+ slow := time.Hour+ require.True(t, h.acquireSystemStatsSamplerInternal(ctx, slow))+ defer h.releaseSystemStatsSamplerInternal(slow)++ // Drain the ready-gate collection; the ticker is now armed for an hour.+ select {+ case <-collections:+ case <-time.After(5 * time.Second):+ t.Fatal("sampler never took its initial snapshot")+ }++ fast := 10 * time.Millisecond+ require.True(t, h.acquireSystemStatsSamplerInternal(ctx, fast))+ defer h.releaseSystemStatsSamplerInternal(fast)++ select {+ case <-collections:+ case <-time.After(5 * time.Second):+ t.Fatal("sampler did not re-arm to the faster subscriber's interval")+ }+}diff --git a/backend/pkg/libarcane/system/gpu.go b/backend/pkg/libarcane/system/gpu.goindex cc23126b9..369a69c6e 100644--- a/backend/pkg/libarcane/system/gpu.go+++ b/backend/pkg/libarcane/system/gpu.go@@ -11,6 +11,7 @@ import ("strconv""strings""sync"+ "sync/atomic""time""emperror.dev/errors"@@ -25,6 +26,11 @@ const (// gpuDetectionTTL bounds how long a successful detection result is reused before re-detecting.gpuDetectionTTL = 30 * time.Second++ // gpuStatsTTL bounds how often the vendor tool is invoked: reading VRAM+ // stats forks a process (nvidia-smi, intel_gpu_top), which is far too+ // expensive to repeat on every sampler tick.+ gpuStatsTTL = 3 * time.Second)// GPUMonitor probes for an attached GPU (NVIDIA / AMD / Intel) and reports VRAM usage.@@ -37,9 +43,20 @@ type GPUMonitor struct {detectionCache *hot.HotCache[struct{}, gpuDetection]detectionDone bool+ // statsCache holds the last vendor-tool result (success or failure) for+ // gpuStatsTTL so concurrent consumers and fast sampler ticks share one+ // process fork instead of each launching their own.+ statsCache atomic.Pointer[gpuStatsSnapshotInternal]+enabled bool}+type gpuStatsSnapshotInternal struct {+ stats []systemtypes.GPUStats+ err error+ sampledAt time.Time+}+type gpuDetection struct {detected boolgpuType string@@ -64,11 +81,23 @@ func (m *GPUMonitor) Enabled() bool { return m.enabled }// Stats returns per-GPU VRAM stats. Returns (nil, 0, nil) when monitoring is disabled// or no GPU is detected; vendor-specific errors are propagated otherwise.+// Results (including failures) are cached for gpuStatsTTL, so callers may+// receive stats up to that many seconds old.func (m *GPUMonitor) Stats(ctx context.Context) ([]systemtypes.GPUStats, error) {if !m.enabled {return nil, nil}+ if snapshot := m.statsCache.Load(); snapshot != nil && time.Since(snapshot.sampledAt) < gpuStatsTTL {+ return snapshot.stats, snapshot.err+ }++ stats, err := m.statsUncachedInternal(ctx)+ m.statsCache.Store(&gpuStatsSnapshotInternal{stats: stats, err: err, sampledAt: time.Now()})+ return stats, err+}++func (m *GPUMonitor) statsUncachedInternal(ctx context.Context) ([]systemtypes.GPUStats, error) {m.detectionMu.Lock()done := m.detectionDonem.detectionMu.Unlock()diff --git a/backend/pkg/libarcane/system/gpu_test.go b/backend/pkg/libarcane/system/gpu_test.gonew file mode 100644index 000000000..3704afe33--- /dev/null+++ b/backend/pkg/libarcane/system/gpu_test.go@@ -0,0 +1,40 @@+//go:build !windows++package system++import (+ "context"+ "os"+ "path/filepath"+ "testing"++ "github.com/stretchr/testify/require"+)++// TestGPUMonitorStatsCachedWithinTTL verifies that repeated Stats calls+// within gpuStatsTTL share one vendor-tool fork instead of launching+// nvidia-smi per call.+func TestGPUMonitorStatsCachedWithinTTL(t *testing.T) {+ binDir := t.TempDir()+ countFile := filepath.Join(binDir, "calls")++ script := "#!/bin/sh\necho run >> " + countFile + "\necho '0, Fake GPU, 512, 8192'\n"+ require.NoError(t, os.WriteFile(filepath.Join(binDir, "nvidia-smi"), []byte(script), 0o755))+ t.Setenv("PATH", binDir)++ monitor := NewGPUMonitor(true, "nvidia")++ stats, err := monitor.Stats(context.Background())+ require.NoError(t, err)+ require.Len(t, stats, 1)+ require.Equal(t, "Fake GPU", stats[0].Name)+ require.EqualValues(t, 512*1024*1024, stats[0].MemoryUsed)++ again, err := monitor.Stats(context.Background())+ require.NoError(t, err)+ require.Equal(t, stats, again)++ invocations, err := os.ReadFile(countFile)+ require.NoError(t, err)+ require.Equal(t, "run\n", string(invocations), "nvidia-smi should have been forked exactly once within the TTL")+}