Skip to content

Modern Docker Management, Designed for Everyone

11.9 KB
Raw Blame
package ws

import (
	"cmp"
	"context"
	"encoding/json/v2"
	"log/slog"
	"net/http"
	"runtime"
	"sync"
	"time"

	"github.com/coder/websocket"
	systemtypes "github.com/getarcaneapp/arcane/types/v2/system"
	"github.com/labstack/echo/v5"
	"github.com/shirou/gopsutil/v4/cpu"
	"github.com/shirou/gopsutil/v4/disk"
	"github.com/shirou/gopsutil/v4/host"
	"github.com/shirou/gopsutil/v4/mem"
	kit "go.getarcane.app/kit/pkg"
	"go.getarcane.app/sys/cgroup"

	wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
	httputil "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
)

const (
	cgroupCacheTTL = 30 * time.Second

	// systemStatsSamplerDefaultInterval matches the stats endpoint's default
	// subscriber interval; the shared sampler never ticks faster than the
	// fastest active subscriber.
	systemStatsSamplerDefaultInterval = 2 * time.Second
)

// ============================================================================
// System WebSocket Endpoints
// ============================================================================

func (h *WebSocketHandler) acquireSystemStatsSampler(ctx context.Context, interval time.Duration) bool {
	h.systemStatsSampler.lifecycleMu.Lock()

	h.systemStatsSampler.clients++
	if h.systemStatsSampler.intervals == nil {
		h.systemStatsSampler.intervals = map[time.Duration]int{}
	}
	h.systemStatsSampler.intervals[interval]++
	h.refreshSystemStatsSamplerIntervalLocked()
	if h.systemStatsSampler.running {
		ready := h.systemStatsSampler.ready
		h.systemStatsSampler.lifecycleMu.Unlock()
		return waitForSystemStatsSamplerReady(ctx, ready)
	}

	samplerCtx, cancel := context.WithCancel(h.ctx)
	ready := make(chan struct{})
	wake := make(chan struct{}, 1)
	h.systemStatsSampler.cancel = cancel
	h.systemStatsSampler.ready = ready
	h.systemStatsSampler.wake = wake
	h.systemStatsSampler.running = true
	h.systemStatsSampler.lifecycleMu.Unlock()

	collect := h.systemStatsCollector
	if collect == nil {
		collect = h.collectSystemStats
	}

	go func() {
		closeReady := sync.OnceFunc(func() {
			close(ready)
		})
		if !h.initializeCPUCache(samplerCtx) {
			closeReady()
			return
		}
		if samplerCtx.Err() != nil {
			closeReady()
			return
		}

		h.systemStatsSampler.latest.Store(collect(samplerCtx))
		closeReady()
		if samplerCtx.Err() != nil {
			return
		}

		h.runSystemStatsSampler(samplerCtx, wake, collect)
	}()

	return waitForSystemStatsSamplerReady(ctx, ready)
}

func waitForSystemStatsSamplerReady(ctx context.Context, ready <-chan struct{}) bool {
	if ready == nil {
		return true
	}

	select {
	case <-ctx.Done():
		return false
	case <-ready:
		return ctx.Err() == nil
	}
}

func (h *WebSocketHandler) releaseSystemStatsSampler(interval time.Duration) {
	var cancel context.CancelFunc

	h.systemStatsSampler.lifecycleMu.Lock()
	if h.systemStatsSampler.clients > 0 {
		h.systemStatsSampler.clients--
	}
	if remaining := h.systemStatsSampler.intervals[interval]; remaining > 1 {
		h.systemStatsSampler.intervals[interval] = remaining - 1
	} else {
		delete(h.systemStatsSampler.intervals, interval)
	}
	h.refreshSystemStatsSamplerIntervalLocked()
	if h.systemStatsSampler.clients == 0 && h.systemStatsSampler.running {
		cancel = h.systemStatsSampler.cancel
		h.systemStatsSampler.cancel = nil
		h.systemStatsSampler.ready = nil
		h.systemStatsSampler.wake = nil
		h.systemStatsSampler.running = false
	}
	h.systemStatsSampler.lifecycleMu.Unlock()

	if cancel != nil {
		cancel()
	}
}

// refreshSystemStatsSamplerIntervalLocked recomputes the sampler's
// effective interval — the smallest one any subscriber requested — and must
// be called with lifecycleMu held.
func (h *WebSocketHandler) refreshSystemStatsSamplerIntervalLocked() {
	effective := time.Duration(0)
	for interval := range h.systemStatsSampler.intervals {
		if effective == 0 || interval < effective {
			effective = interval
		}
	}
	if effective <= 0 {
		effective = systemStatsSamplerDefaultInterval
	}
	previous := time.Duration(h.systemStatsSampler.effectiveInterval.Swap(int64(effective)))
	if previous != effective && h.systemStatsSampler.wake != nil {
		select {
		case h.systemStatsSampler.wake <- struct{}{}:
		default:
		}
	}
}

func (h *WebSocketHandler) systemStatsSamplerInterval() time.Duration {
	if interval := time.Duration(h.systemStatsSampler.effectiveInterval.Load()); interval > 0 {
		return interval
	}
	return systemStatsSamplerDefaultInterval
}

func (h *WebSocketHandler) runSystemStatsSampler(ctx context.Context, wake <-chan struct{}, collect func(context.Context) systemtypes.SystemStats) {
	interval := h.systemStatsSamplerInterval()
	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	// Subscribers come and go while the sampler runs; re-arm the ticker
	// whenever the fastest requested interval changed.
	rearm := func() {
		if next := h.systemStatsSamplerInterval(); next != interval {
			interval = next
			ticker.Reset(interval)
		}
	}

	for {
		select {
		case <-ctx.Done():
			return
		case <-wake:
			rearm()
		case <-ticker.C:
			if val, ok := h.readCPUUsage(0); ok {
				h.cpuCache.Store(val)
			}
			h.systemStatsSampler.latest.Store(collect(ctx))
			rearm()
		}
	}
}

// collectSystemStats gathers all system statistics.
func (h *WebSocketHandler) collectSystemStats(ctx context.Context) systemtypes.SystemStats {
	cpuUsage, _ := h.cpuCache.Load()

	h.systemStaticInfo.once.Do(func() {
		cpuCount, err := cpu.Counts(true)
		if err != nil {
			cpuCount = runtime.NumCPU()
		}
		hostInfo, _ := host.Info()
		h.systemStaticInfo.cpuCount = cpuCount
		h.systemStaticInfo.hostname = kit.FromPtr(hostInfo).Hostname
	})

	var memUsed, memTotal uint64
	if memInfo, _ := mem.VirtualMemory(); memInfo != nil {
		// gopsutil counts ZFS ARC as used memory; treat the reclaimable portion as cache, matching btop/htop.
		memUsed = memInfo.Used - min(memInfo.Used, cgroup.ZFSARCReclaimable())
		memTotal = memInfo.Total
	}
	cpuCount, memUsed, memTotal := h.applyCgroupLimits(ctx, h.systemStaticInfo.cpuCount, memUsed, memTotal)

	diskUsagePath, found, err := h.diskUsagePathCache.GetWithLoaders(struct{}{}, func(_ []struct{}) (map[struct{}]string, error) {
		path := "/"
		if h.systemService != nil {
			path = h.systemService.GetDiskUsagePath(ctx)
		}
		return map[struct{}]string{{}: path}, nil
	})
	diskUsagePath = kit.Ternary(err != nil || !found, "/", diskUsagePath)
	diskInfo, err := disk.Usage(diskUsagePath)
	if (err != nil || diskInfo == nil || diskInfo.Total == 0) && diskUsagePath != "/" {
		diskInfo, _ = disk.Usage("/")
	}

	var gpuStats []systemtypes.GPUStats
	if h.gpuMonitor != nil && h.gpuMonitor.Enabled() {
		if stats, statsErr := h.gpuMonitor.Stats(ctx); statsErr == nil {
			gpuStats = stats
		}
	}

	return systemtypes.SystemStats{
		CPUUsage:     cpuUsage,
		MemoryUsage:  memUsed,
		MemoryTotal:  memTotal,
		DiskUsage:    kit.FromPtr(diskInfo).Used,
		DiskTotal:    kit.FromPtr(diskInfo).Total,
		CPUCount:     cpuCount,
		Architecture: runtime.GOARCH,
		Platform:     runtime.GOOS,
		Hostname:     h.systemStaticInfo.hostname,
		GPUCount:     len(gpuStats),
		GPUs:         gpuStats,
	}
}

// applyCgroupLimits uses guest resources without applying Docker's
// per-container limits to host statistics.
func (h *WebSocketHandler) applyCgroupLimits(ctx context.Context, cpuCount int, memUsed, memTotal uint64) (int, uint64, uint64) {
	if cgroup.IsDockerContainer() {
		if h.systemService != nil {
			if used, total, ok := h.systemService.GetDockerHostMemory(ctx); ok {
				return cpuCount, used, total
			}
		}
		return cpuCount, memUsed, memTotal
	}
	cpuCount = applyCPUAffinityLimit(cpuCount)
	cgroupLimits := h.cgroupCache.Get()
	if cgroupLimits == nil {
		return cpuCount, memUsed, memTotal
	}

	if limit := cgroupLimits.MemoryLimit; limit > 0 {
		limitUint := uint64(limit)
		if memTotal == 0 || limitUint < memTotal {
			memTotal = limitUint
			if cgroupLimits.MemoryUsage > 0 {
				memUsed = uint64(cgroupLimits.MemoryUsage)
			}
		}
	}
	if cgroupLimits.CPUCount > 0 && (cpuCount == 0 || cgroupLimits.CPUCount < cpuCount) {
		cpuCount = cgroupLimits.CPUCount
	}
	return cpuCount, memUsed, memTotal
}

// applyCPUAffinityLimit caps the CPU count at the scheduler affinity
// mask size. In an unprivileged LXC guest the cgroup cpu quota is usually
// unset at the guest root (the limit lives host-side), so the affinity mask is
// the only signal of the cores actually assigned — including gap bindings like
// CPU1+CPU7 — while /proc/cpuinfo shows the whole host (#3161). runtime.NumCPU
// reads that mask at process start. Not applied inside Docker, matching
// applyCgroupLimits's host-totals convention there.
func applyCPUAffinityLimit(cpuCount int) int {
	if n := runtime.NumCPU(); n > 0 && (cpuCount == 0 || n < cpuCount) {
		return n
	}
	return cpuCount
}

// initializeCPUCache permits unavailable CPU samples but stops on cancellation.
func (h *WebSocketHandler) initializeCPUCache(ctx context.Context) bool {
	result := make(chan float64, 1)

	go func() {
		if val, ok := h.readCPUUsage(time.Second); ok {
			result <- val
		}
		close(result)
	}()

	select {
	case <-ctx.Done():
		return false
	case val, ok := <-result:
		if ctx.Err() != nil {
			return false
		}
		if ok {
			h.cpuCache.Store(val)
		}
		return true
	}
}

func (h *WebSocketHandler) readCPUUsage(interval time.Duration) (float64, bool) {
	if h.cpuUsageReader != nil {
		return h.cpuUsageReader(interval)
	}
	if vals, err := cpu.Percent(interval, false); err == nil && len(vals) > 0 {
		return vals[0], true
	}
	return 0, false
}

// SystemStats streams system stats over WebSocket.
//
//	@Summary		Get system stats via WebSocket
//	@Description	Stream system resource statistics over WebSocket connection
//	@Tags			WebSocket
//	@Param			id	path	string	true	"Environment ID"
//	@Router			/api/environments/{id}/ws/system/stats [get]
func (h *WebSocketHandler) SystemStats(c *echo.Context) error {
	userID, _ := c.Get("userID").(string)
	clientKey := cmp.Or(userID, c.RealIP())

	h.activeConnectionsMu.Lock()
	if h.activeConnections[clientKey] >= 5 {
		h.activeConnectionsMu.Unlock()
		return c.JSON(http.StatusTooManyRequests, map[string]any{"success": false, "error": "Too many concurrent stats connections for this client"})
	}
	if h.activeConnections == nil {
		h.activeConnections = make(map[string]int)
	}
	h.activeConnections[clientKey]++
	h.activeConnectionsMu.Unlock()
	defer func() {
		h.activeConnectionsMu.Lock()
		defer h.activeConnectionsMu.Unlock()
		if count := h.activeConnections[clientKey]; count > 1 {
			h.activeConnections[clientKey] = count - 1
		} else {
			delete(h.activeConnections, clientKey)
		}
	}()

	conn, unregister, ok := h.acceptWS(c, systemtypes.WSKindSystemStats, "")
	if !ok {
		return nil
	}
	defer unregister()
	defer func() {
		if err := conn.CloseNow(); err != nil {
			slog.DebugContext(c.Request().Context(), "Failed to close system stats websocket connection", "clientKey", clientKey, "error", err)
		}
	}()

	interval, _ := httputil.GetIntQueryParam(c.Request(), "interval", false)
	sampleInterval := time.Duration(min(cmp.Or(max(interval, 0), 2), 60)) * time.Second

	conn.SetReadLimit(512)

	ticker := time.NewTicker(sampleInterval)
	defer ticker.Stop()

	ctx, cancel := context.WithCancel(c.Request().Context())
	defer cancel()
	if !h.acquireSystemStatsSampler(ctx, sampleInterval) {
		h.releaseSystemStatsSampler(sampleInterval)
		return nil
	}
	defer h.releaseSystemStatsSampler(sampleInterval)

	// Single reader for this connection; it also services the keepalive pong. Do not add others.
	go func() {
		for {
			if _, _, err := conn.Read(ctx); err != nil {
				cancel()
				return
			}
		}
	}()
	go wshub.KeepAlive(ctx, cancel, conn)

	send := func() error {
		stats, _ := h.systemStatsSampler.latest.Load()
		b, err := json.Marshal(stats)
		if err != nil {
			return err
		}
		wctx, localCancel := context.WithTimeout(ctx, 10*time.Second)
		defer localCancel()
		return conn.Write(wctx, websocket.MessageText, b)
	}

	if err := send(); err != nil {
		return nil
	}

	for {
		select {
		case <-ctx.Done():
			return nil
		case <-ticker.C:
			if err := send(); err != nil {
				return nil
			}
		}
	}
}