Skip to content

Modern Docker Management, Designed for Everyone

feat: add gpu monitoring (#799)

Timothy Duongd4c622c858cd
diff --git a/.gitignore b/.gitignore
index 18d382736..8ed9c8976 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,10 @@
node_modules
+# fork-timothy
+TIMOTHY-README.MD
+gpu_service.go
+compose.gpu-amd.yaml
+compose.gpu-intel.yaml
+compose.gpu-nvidia.yaml
# Output
.output
@@ -37,4 +43,4 @@ backend/build-errors.log
frontend/bun.lock
tests/bun.lock
tests/package-lock.json
-/.idea/**
+/.idea/**
\ No newline at end of file
diff --git a/backend/internal/api/system_handler.go b/backend/internal/api/system_handler.go
index de1879d65..11a5944dd 100644
--- a/backend/internal/api/system_handler.go
+++ b/backend/internal/api/system_handler.go
@@ -1,11 +1,18 @@
package api
import (
+ "bytes"
"context"
+ "encoding/csv"
+ "encoding/json"
"errors"
+ "fmt"
"log/slog"
"net/http"
+ "os/exec"
"runtime"
+ "strconv"
+ "strings"
"sync"
"sync/atomic"
"time"
@@ -26,6 +33,18 @@ import (
"github.com/shirou/gopsutil/v4/mem"
)
+const (
+ gpuCacheDuration = 30 * time.Second
+)
+
+// GPUStats represents statistics for a single GPU
+type GPUStats struct {
+ Name string `json:"name"`
+ Index int `json:"index"`
+ MemoryUsed float64 `json:"memoryUsed"` // in bytes
+ MemoryTotal float64 `json:"memoryTotal"` // in bytes
+}
+
type SystemHandler struct {
dockerService *services.DockerClientService
systemService *services.SystemService
@@ -42,6 +61,15 @@ type SystemHandler struct {
value string
timestamp time.Time
}
+ gpuDetectionCache struct {
+ sync.RWMutex
+ detected bool
+ timestamp time.Time
+ gpuType string // "nvidia", "amd", "intel", "jetson", or ""
+ toolPath string
+ }
+ detectionDone bool
+ detectionMutex sync.Mutex
}
func NewSystemHandler(group *gin.RouterGroup, dockerService *services.DockerClientService, systemService *services.SystemService, upgradeService *services.SystemUpgradeService, authMiddleware *middleware.AuthMiddleware, cfg *config.Config) {
@@ -73,15 +101,17 @@ func NewSystemHandler(group *gin.RouterGroup, dockerService *services.DockerClie
}
type SystemStats struct {
- CPUUsage float64 `json:"cpuUsage"`
- MemoryUsage uint64 `json:"memoryUsage"`
- MemoryTotal uint64 `json:"memoryTotal"`
- DiskUsage uint64 `json:"diskUsage,omitempty"`
- DiskTotal uint64 `json:"diskTotal,omitempty"`
- CPUCount int `json:"cpuCount"`
- Architecture string `json:"architecture"`
- Platform string `json:"platform"`
- Hostname string `json:"hostname,omitempty"`
+ CPUUsage float64 `json:"cpuUsage"`
+ MemoryUsage uint64 `json:"memoryUsage"`
+ MemoryTotal uint64 `json:"memoryTotal"`
+ DiskUsage uint64 `json:"diskUsage,omitempty"`
+ DiskTotal uint64 `json:"diskTotal,omitempty"`
+ CPUCount int `json:"cpuCount"`
+ Architecture string `json:"architecture"`
+ Platform string `json:"platform"`
+ Hostname string `json:"hostname,omitempty"`
+ GPUCount int `json:"gpuCount"`
+ GPUs []GPUStats `json:"gpus,omitempty"`
}
func (h *SystemHandler) Health(c *gin.Context) {
@@ -400,6 +430,14 @@ func (h *SystemHandler) Stats(c *gin.Context) {
hostname = hostInfo.Hostname
}
+ // Collect GPU stats (non-blocking, fails gracefully)
+ var gpuStats []GPUStats
+ var gpuCount int
+ if gpuData, err := h.getGPUStats(ctx); err == nil {
+ gpuStats = gpuData
+ gpuCount = len(gpuData)
+ }
+
stats := SystemStats{
CPUUsage: cpuUsage,
MemoryUsage: memUsed,
@@ -410,6 +448,8 @@ func (h *SystemHandler) Stats(c *gin.Context) {
Architecture: runtime.GOARCH,
Platform: runtime.GOOS,
Hostname: hostname,
+ GPUCount: gpuCount,
+ GPUs: gpuStats,
}
_ = conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
@@ -530,3 +570,361 @@ func (h *SystemHandler) TriggerUpgrade(c *gin.Context) {
"success": true,
})
}
+
+// getGPUStats collects and returns GPU statistics for all available GPUs
+func (h *SystemHandler) getGPUStats(ctx context.Context) ([]GPUStats, error) {
+ // Check if we need to detect GPUs
+ h.detectionMutex.Lock()
+ done := h.detectionDone
+ h.detectionMutex.Unlock()
+
+ if !done {
+ if err := h.detectGPUs(ctx); err != nil {
+ return nil, err
+ }
+ }
+
+ // Check cache
+ h.gpuDetectionCache.RLock()
+ if h.gpuDetectionCache.detected && time.Since(h.gpuDetectionCache.timestamp) < gpuCacheDuration {
+ gpuType := h.gpuDetectionCache.gpuType
+ h.gpuDetectionCache.RUnlock()
+
+ // Collect stats based on GPU type
+ switch gpuType {
+ case "nvidia":
+ return h.getNvidiaStats(ctx)
+ case "amd":
+ return h.getAMDStats(ctx)
+ case "intel":
+ return h.getIntelStats(ctx)
+ case "jetson":
+ return h.getJetsonStats(ctx)
+ }
+ }
+ h.gpuDetectionCache.RUnlock()
+
+ // Re-detect if cache expired
+ if err := h.detectGPUs(ctx); err != nil {
+ return nil, err
+ }
+
+ // Try again after detection
+ h.gpuDetectionCache.RLock()
+ gpuType := h.gpuDetectionCache.gpuType
+ h.gpuDetectionCache.RUnlock()
+
+ switch gpuType {
+ case "nvidia":
+ return h.getNvidiaStats(ctx)
+ case "amd":
+ return h.getAMDStats(ctx)
+ case "intel":
+ return h.getIntelStats(ctx)
+ case "jetson":
+ return h.getJetsonStats(ctx)
+ default:
+ return nil, fmt.Errorf("no supported GPU found")
+ }
+}
+
+// detectGPUs detects available GPU management tools
+func (h *SystemHandler) detectGPUs(ctx context.Context) error {
+ h.detectionMutex.Lock()
+ defer h.detectionMutex.Unlock()
+
+ slog.DebugContext(ctx, "Starting GPU detection")
+
+ // Check for NVIDIA
+ if path, err := exec.LookPath("nvidia-smi"); err == nil {
+ h.gpuDetectionCache.Lock()
+ h.gpuDetectionCache.detected = true
+ h.gpuDetectionCache.gpuType = "nvidia"
+ h.gpuDetectionCache.toolPath = path
+ h.gpuDetectionCache.timestamp = time.Now()
+ h.gpuDetectionCache.Unlock()
+ h.detectionDone = true
+ slog.InfoContext(ctx, "NVIDIA GPU detected", slog.String("tool", "nvidia-smi"), slog.String("path", path))
+ return nil
+ }
+
+ // Check for AMD ROCm
+ if path, err := exec.LookPath("rocm-smi"); err == nil {
+ h.gpuDetectionCache.Lock()
+ h.gpuDetectionCache.detected = true
+ h.gpuDetectionCache.gpuType = "amd"
+ h.gpuDetectionCache.toolPath = path
+ h.gpuDetectionCache.timestamp = time.Now()
+ h.gpuDetectionCache.Unlock()
+ h.detectionDone = true
+ slog.InfoContext(ctx, "AMD GPU detected", slog.String("tool", "rocm-smi"), slog.String("path", path))
+ return nil
+ }
+
+ // Check for NVIDIA Jetson
+ if path, err := exec.LookPath("tegrastats"); err == nil {
+ h.gpuDetectionCache.Lock()
+ h.gpuDetectionCache.detected = true
+ h.gpuDetectionCache.gpuType = "jetson"
+ h.gpuDetectionCache.toolPath = path
+ h.gpuDetectionCache.timestamp = time.Now()
+ h.gpuDetectionCache.Unlock()
+ h.detectionDone = true
+ slog.InfoContext(ctx, "NVIDIA Jetson detected", slog.String("tool", "tegrastats"), slog.String("path", path))
+ return nil
+ }
+
+ // Check for Intel GPU
+ if path, err := exec.LookPath("intel_gpu_top"); err == nil {
+ h.gpuDetectionCache.Lock()
+ h.gpuDetectionCache.detected = true
+ h.gpuDetectionCache.gpuType = "intel"
+ h.gpuDetectionCache.toolPath = path
+ h.gpuDetectionCache.timestamp = time.Now()
+ h.gpuDetectionCache.Unlock()
+ h.detectionDone = true
+ slog.InfoContext(ctx, "Intel GPU detected", slog.String("tool", "intel_gpu_top"), slog.String("path", path))
+ return nil
+ }
+
+ h.detectionDone = true
+ slog.DebugContext(ctx, "No GPU detected on this system")
+ return fmt.Errorf("no supported GPU found")
+}
+
+// getNvidiaStats collects NVIDIA GPU statistics using nvidia-smi
+func (h *SystemHandler) getNvidiaStats(ctx context.Context) ([]GPUStats, error) {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+
+ // Query: index, name, memory.used, memory.total
+ cmd := exec.CommandContext(ctx, "nvidia-smi",
+ "--query-gpu=index,name,memory.used,memory.total",
+ "--format=csv,noheader,nounits")
+
+ output, err := cmd.Output()
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to execute nvidia-smi", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("nvidia-smi execution failed: %w", err)
+ }
+
+ return h.parseNvidiaOutput(ctx, output)
+}
+
+// parseNvidiaOutput parses CSV output from nvidia-smi
+func (h *SystemHandler) parseNvidiaOutput(ctx context.Context, output []byte) ([]GPUStats, error) {
+ reader := csv.NewReader(bytes.NewReader(output))
+ reader.TrimLeadingSpace = true
+
+ records, err := reader.ReadAll()
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to parse nvidia-smi CSV output", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("failed to parse nvidia-smi output: %w", err)
+ }
+
+ var stats []GPUStats
+ for _, record := range records {
+ if len(record) < 4 {
+ continue
+ }
+
+ index, err := strconv.Atoi(strings.TrimSpace(record[0]))
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to parse GPU index", slog.String("value", record[0]))
+ continue
+ }
+
+ name := strings.TrimSpace(record[1])
+
+ memUsed, err := strconv.ParseFloat(strings.TrimSpace(record[2]), 64)
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to parse memory used", slog.String("value", record[2]))
+ continue
+ }
+
+ memTotal, err := strconv.ParseFloat(strings.TrimSpace(record[3]), 64)
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to parse memory total", slog.String("value", record[3]))
+ continue
+ }
+
+ // nvidia-smi returns MiB (mebibytes), convert to bytes (1 MiB = 1024*1024 bytes)
+ stats = append(stats, GPUStats{
+ Name: name,
+ Index: index,
+ MemoryUsed: memUsed * 1024 * 1024,
+ MemoryTotal: memTotal * 1024 * 1024,
+ })
+ }
+
+ if len(stats) == 0 {
+ return nil, fmt.Errorf("no GPU data parsed from nvidia-smi")
+ }
+
+ slog.DebugContext(ctx, "Collected NVIDIA GPU stats", slog.Int("gpu_count", len(stats)))
+ return stats, nil
+}
+
+// getAMDStats collects AMD GPU statistics using rocm-smi
+func (h *SystemHandler) getAMDStats(ctx context.Context) ([]GPUStats, error) {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+
+ cmd := exec.CommandContext(ctx, "rocm-smi", "--showmeminfo", "vram", "--json")
+ output, err := cmd.Output()
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to execute rocm-smi", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("rocm-smi execution failed: %w", err)
+ }
+
+ return h.parseROCmOutput(ctx, output)
+}
+
+// ROCmSMIOutput represents the JSON structure from rocm-smi
+type ROCmSMIOutput map[string]ROCmGPUInfo
+
+type ROCmGPUInfo struct {
+ VRAMUsed string `json:"VRAM Total Used Memory (B)"`
+ VRAMTotal string `json:"VRAM Total Memory (B)"`
+}
+
+// parseROCmOutput parses JSON output from rocm-smi
+func (h *SystemHandler) parseROCmOutput(ctx context.Context, output []byte) ([]GPUStats, error) {
+ var rocmData ROCmSMIOutput
+ if err := json.Unmarshal(output, &rocmData); err != nil {
+ slog.WarnContext(ctx, "Failed to parse rocm-smi JSON output", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("failed to parse rocm-smi output: %w", err)
+ }
+
+ var stats []GPUStats
+ index := 0
+ for gpuID, info := range rocmData {
+ // Parse memory used (in bytes)
+ memUsedBytes, err := strconv.ParseFloat(info.VRAMUsed, 64)
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to parse AMD memory used", slog.String("gpu", gpuID), slog.String("value", info.VRAMUsed))
+ continue
+ }
+
+ // Parse memory total (in bytes)
+ memTotalBytes, err := strconv.ParseFloat(info.VRAMTotal, 64)
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to parse AMD memory total", slog.String("gpu", gpuID), slog.String("value", info.VRAMTotal))
+ continue
+ }
+
+ // ROCm already returns bytes, use directly
+ stats = append(stats, GPUStats{
+ Name: fmt.Sprintf("AMD GPU %s", gpuID),
+ Index: index,
+ MemoryUsed: memUsedBytes,
+ MemoryTotal: memTotalBytes,
+ })
+ index++
+ }
+
+ if len(stats) == 0 {
+ return nil, fmt.Errorf("no GPU data parsed from rocm-smi")
+ }
+
+ slog.DebugContext(ctx, "Collected AMD GPU stats", slog.Int("gpu_count", len(stats)))
+ return stats, nil
+}
+
+// getIntelStats collects Intel GPU statistics using intel_gpu_top
+func (h *SystemHandler) getIntelStats(ctx context.Context) ([]GPUStats, error) {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+
+ // intel_gpu_top requires running with -o - for single sample JSON output
+ cmd := exec.CommandContext(ctx, "intel_gpu_top", "-J", "-o", "-")
+ output, err := cmd.Output()
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to execute intel_gpu_top", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("intel_gpu_top execution failed: %w", err)
+ }
+
+ return h.parseIntelOutput(ctx, output)
+}
+
+// parseIntelOutput parses JSON output from intel_gpu_top
+func (h *SystemHandler) parseIntelOutput(ctx context.Context, output []byte) ([]GPUStats, error) {
+ // intel_gpu_top doesn't provide straightforward total memory info
+ // This is a simplified implementation - for MVP we'll return basic info
+
+ // For now, return a placeholder indicating Intel GPU detected
+ // A more complete implementation would parse /sys/class/drm/card*/device/mem_info_vram_total
+ stats := []GPUStats{
+ {
+ Name: "Intel GPU",
+ Index: 0,
+ MemoryUsed: 0,
+ MemoryTotal: 0,
+ },
+ }
+
+ slog.DebugContext(ctx, "Intel GPU detected but detailed stats not yet implemented")
+ return stats, nil
+}
+
+// getJetsonStats collects NVIDIA Jetson statistics using tegrastats
+func (h *SystemHandler) getJetsonStats(ctx context.Context) ([]GPUStats, error) {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+
+ // tegrastats outputs continuous stream, we'll use --interval for single sample
+ cmd := exec.CommandContext(ctx, "tegrastats", "--interval", "1000")
+ output, err := cmd.Output()
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to execute tegrastats", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("tegrastats execution failed: %w", err)
+ }
+
+ return h.parseJetsonOutput(ctx, output)
+}
+
+// parseJetsonOutput parses text output from tegrastats
+func (h *SystemHandler) parseJetsonOutput(ctx context.Context, output []byte) ([]GPUStats, error) {
+ // tegrastats output format: RAM 1234/5678MB (lfb 1x2MB) ...
+ // This is a simplified parser for MVP
+
+ lines := strings.Split(string(output), "\n")
+ if len(lines) == 0 {
+ return nil, fmt.Errorf("no output from tegrastats")
+ }
+
+ // Parse first line for RAM info (Jetson uses unified memory)
+ // Example: "RAM 1234/5678MB"
+ for _, line := range lines {
+ if strings.Contains(line, "RAM") {
+ // Simple parsing - extract RAM usage
+ parts := strings.Fields(line)
+ for i, part := range parts {
+ if part == "RAM" && i+1 < len(parts) {
+ memPart := parts[i+1]
+ if strings.Contains(memPart, "/") && strings.HasSuffix(memPart, "MB") {
+ memPart = strings.TrimSuffix(memPart, "MB")
+ memValues := strings.Split(memPart, "/")
+ if len(memValues) == 2 {
+ used, _ := strconv.ParseFloat(memValues[0], 64)
+ total, _ := strconv.ParseFloat(memValues[1], 64)
+
+ // tegrastats returns MB, convert to bytes
+ return []GPUStats{
+ {
+ Name: "NVIDIA Jetson",
+ Index: 0,
+ MemoryUsed: used * 1024 * 1024,
+ MemoryTotal: total * 1024 * 1024,
+ },
+ }, nil
+ }
+ }
+ }
+ }
+ }
+ }
+
+ slog.DebugContext(ctx, "NVIDIA Jetson detected but could not parse memory stats")
+ return nil, fmt.Errorf("failed to parse tegrastats output")
+}
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 14456ad6a..5aaaed58d 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -41,10 +41,14 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
FROM alpine:3 AS runner
ARG VERSION="dev"
ARG REVISION="unknown"
-RUN apk upgrade && apk --no-cache add ca-certificates curl tzdata
+RUN apk upgrade && apk --no-cache add ca-certificates curl tzdata gcompat
ENV GIN_MODE=release
ENV PORT=3552
ENV ENVIRONMENT=production
+# Multi-distro, multi-arch library path for GPU support
+# Includes paths for: NVIDIA runtime mounts, Debian/Ubuntu (AMD64/ARM64), RHEL/CentOS/Fedora, Arch, and standard fallbacks
+# Covers: NVIDIA GPUs (all distros/architectures), future AMD ROCm support, and Intel GPU support
+ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/opt/rocm/lib:/usr/lib/x86_64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib64:/usr/lib:/lib64:/lib
WORKDIR /app
RUN mkdir -p /app/data
COPY --from=backend-builder /build/arcane .
diff --git a/docker/Dockerfile-agent b/docker/Dockerfile-agent
index 82ad890c3..52a583f6c 100644
--- a/docker/Dockerfile-agent
+++ b/docker/Dockerfile-agent
@@ -29,7 +29,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
./cmd/main.go
FROM alpine:3 AS agent
-RUN apk upgrade && apk --no-cache add ca-certificates tzdata curl
+RUN apk upgrade && apk --no-cache add ca-certificates tzdata curl gcompat
WORKDIR /app
RUN mkdir -p /app/data
@@ -39,6 +39,11 @@ COPY --from=agent-builder /out/arcane-agent /usr/local/bin/arcane-agent
ARG VERSION="dev"
ARG REVISION="unknown"
+# Multi-distro, multi-arch library path for GPU support
+# Includes paths for: NVIDIA runtime mounts, Debian/Ubuntu (AMD64/ARM64), RHEL/CentOS/Fedora, Arch, and standard fallbacks
+# Covers: NVIDIA GPUs (all distros/architectures), future AMD ROCm support, and Intel GPU support
+ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/opt/rocm/lib:/usr/lib/x86_64-linux-gnu:/usr/lib/aarch64-linux-gnu:/usr/lib64:/usr/lib:/lib64:/lib
+
ENV ENVIRONMENT=production \
AGENT_MODE=true \
GIN_MODE=release \
diff --git a/docker/Dockerfile-agent-static b/docker/Dockerfile-agent-static
index 8f4f257ea..b095f365d 100644
--- a/docker/Dockerfile-agent-static
+++ b/docker/Dockerfile-agent-static
@@ -3,7 +3,7 @@ FROM alpine:3
ARG TARGETARCH
WORKDIR /app
-RUN apk upgrade && apk --no-cache add ca-certificates tzdata curl
+RUN apk upgrade && apk --no-cache add ca-certificates tzdata curl gcompat
RUN mkdir -p /app/data
COPY ./backend/.bin/arcane-agent-linux-${TARGETARCH} ./arcane-agent
diff --git a/docker/Dockerfile-static b/docker/Dockerfile-static
index 77f9efe88..3baea4b9a 100644
--- a/docker/Dockerfile-static
+++ b/docker/Dockerfile-static
@@ -3,7 +3,7 @@ FROM alpine:3
ARG TARGETARCH
WORKDIR /app
-RUN apk upgrade && apk --no-cache add ca-certificates tzdata curl
+RUN apk upgrade && apk --no-cache add ca-certificates tzdata curl gcompat
RUN mkdir -p /app/data
COPY ./backend/.bin/arcane-linux-${TARGETARCH} ./arcane
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 195585ef0..2f1a132ff 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -343,6 +343,10 @@
"dashboard_meter_disk_free": "Free",
"dashboard_meter_disk_monitoring": "Monitoring: {path}",
"dashboard_meter_disk_total": "Total",
+ "dashboard_meter_gpu": "GPU Usage",
+ "dashboard_meter_gpu_desc": "GPU memory utilization",
+ "dashboard_meter_gpu_device": "device",
+ "dashboard_meter_gpu_devices": "devices",
"_comment_containers": "=== CONTAINERS ===",
"containers_title": "Containers",
diff --git a/frontend/src/lib/components/gpu-meter.svelte b/frontend/src/lib/components/gpu-meter.svelte
new file mode 100644
index 000000000..4fa1a40a1
--- /dev/null
+++ b/frontend/src/lib/components/gpu-meter.svelte
@@ -0,0 +1,80 @@
+<script lang="ts">
+ import * as Card from '$lib/components/ui/card';
+ import { Progress } from '$lib/components/ui/progress/index.js';
+ import GpuIcon from '@lucide/svelte/icons/gpu';
+ import { m } from '$lib/paraglide/messages';
+ import bytes from 'bytes';
+ import type { GPUStats } from '$lib/types/system-stats.type';
+
+ interface Props {
+ gpus?: GPUStats[];
+ loading?: boolean;
+ }
+
+ let { gpus, loading = false }: Props = $props();
+
+ function formatBytes(bytesValue: number): string {
+ return bytes.format(bytesValue, { unitSeparator: ' ' }) ?? '-';
+ }
+
+ function getPercentage(used: number, total: number): number {
+ if (total === 0) return 0;
+ return Math.min(100, (used / total) * 100);
+ }
+</script>
+
+<Card.Root>
+ {#snippet children()}
+ <Card.Header icon={GpuIcon} iconVariant="primary" compact {loading}>
+ {#snippet children()}
+ <div class="min-w-0 flex-1">
+ <div class="text-foreground text-sm font-semibold">{m.dashboard_meter_gpu()}</div>
+ {#if gpus && gpus.length > 0}
+ <div class="text-muted-foreground text-xs">
+ {gpus.length} {gpus.length === 1 ? m.dashboard_meter_gpu_device() : m.dashboard_meter_gpu_devices()}
+ </div>
+ {/if}
+ </div>
+ {/snippet}
+ </Card.Header>
+
+ <Card.Content class="flex flex-col justify-center p-3">
+ {#if loading}
+ <div class="w-full space-y-3">
+ <div class="bg-muted h-16 w-full animate-pulse rounded"></div>
+ </div>
+ {:else if !gpus || gpus.length === 0}
+ <div class="text-muted-foreground text-center text-xs">
+ {m.common_na()}
+ </div>
+ {:else}
+ <div class="w-full space-y-3">
+ {#each gpus as gpu}
+ <div class="space-y-1.5">
+ <div class="flex items-center justify-between">
+ <span class="text-foreground text-xs font-medium">{gpu.name}</span>
+ <span class="text-muted-foreground text-[10px] font-mono">
+ GPU {gpu.index}
+ </span>
+ </div>
+ <div class="text-center">
+ <div class="text-foreground text-sm font-bold">
+ {formatBytes(gpu.memoryUsed)}
+ </div>
+ </div>
+ <Progress value={getPercentage(gpu.memoryUsed, gpu.memoryTotal)} max={100} class="h-1.5" />
+ <div class="flex items-center justify-between text-xs">
+ <span class="text-muted-foreground font-medium">
+ {getPercentage(gpu.memoryUsed, gpu.memoryTotal).toFixed(1)}%
+ </span>
+ <span class="text-muted-foreground/70 font-mono text-[10px]">
+ {formatBytes(gpu.memoryUsed)} / {formatBytes(gpu.memoryTotal)}
+ </span>
+ </div>
+ </div>
+ {/each}
+ </div>
+ {/if}
+ </Card.Content>
+ {/snippet}
+</Card.Root>
diff --git a/frontend/src/lib/types/system-stats.type.ts b/frontend/src/lib/types/system-stats.type.ts
index 178d3a1f5..b6809c8cd 100644
--- a/frontend/src/lib/types/system-stats.type.ts
+++ b/frontend/src/lib/types/system-stats.type.ts
@@ -8,4 +8,13 @@ export interface SystemStats {
architecture: string;
platform: string;
hostname?: string;
+ gpuCount: number;
+ gpus?: GPUStats[];
+}
+
+export interface GPUStats {
+ name: string;
+ index: number;
+ memoryUsed: number; // in MB
+ memoryTotal: number; // in MB
}
diff --git a/frontend/src/routes/dashboard/+page.svelte b/frontend/src/routes/dashboard/+page.svelte
index 714531530..da44cb84b 100644
--- a/frontend/src/routes/dashboard/+page.svelte
+++ b/frontend/src/routes/dashboard/+page.svelte
@@ -12,6 +12,7 @@
import type { ReconnectingWebSocket } from '$lib/utils/ws';
import MeterMetric from '$lib/components/meter-metric.svelte';
import DiskMeter from '$lib/components/disk-meter.svelte';
+ import GpuMeter from '$lib/components/gpu-meter.svelte';
import QuickActions from '$lib/components/quick-actions.svelte';
import DockerOverview from '$lib/components/docker-overview.svelte';
import type { SystemStats } from '$lib/types/system-stats.type';
@@ -65,6 +66,7 @@
cpu: [] as Array<{ date: Date; value: number }>,
memory: [] as Array<{ date: Date; value: number }>,
disk: [] as Array<{ date: Date; value: number }>,
+ gpu: [] as Array<{ date: Date; value: number }>,
containers: [] as Array<{ date: Date; value: number }>
});
@@ -100,6 +102,23 @@
}
}
+ if (stats.gpus && stats.gpus.length > 0) {
+ // Track average GPU memory usage percentage across all GPUs
+ // Filter out GPUs with zero total memory to avoid invalid calculations
+ const validGpus = stats.gpus.filter(gpu => gpu.memoryTotal > 0);
+
+ if (validGpus.length > 0) {
+ const totalGpuPercent = validGpus.reduce((sum, gpu) => {
+ return sum + (gpu.memoryUsed / gpu.memoryTotal) * 100;
+ }, 0);
+ const avgGpuPercent = totalGpuPercent / validGpus.length;
+ historicalData.gpu.push({ date: now, value: avgGpuPercent });
+ if (historicalData.gpu.length > maxPoints) {
+ historicalData.gpu = historicalData.gpu.slice(-maxPoints);
+ }
+ }
+ }
+
historicalData.containers.push({ date: now, value: dockerInfo!.containersRunning });
if (historicalData.containers.length > maxPoints) {
historicalData.containers = historicalData.containers.slice(-maxPoints);
@@ -137,6 +156,7 @@
cpu: [],
memory: [],
disk: [],
+ gpu: [],
containers: []
};
}
@@ -297,7 +317,7 @@
{/if}
</div>
<div class="space-y-3">
- <div class="grid grid-cols-2 gap-3 sm:grid-cols-2 lg:grid-cols-3">
+ <div class="grid grid-cols-2 gap-3 sm:grid-cols-2 {currentStats?.gpuCount && currentStats.gpuCount > 0 ? 'lg:grid-cols-4' : 'lg:grid-cols-3'}">
<MeterMetric
title={m.dashboard_meter_cpu()}
icon={CpuIcon}
@@ -335,6 +355,13 @@
loading={isLoading.loadingStats || !hasInitialStatsLoaded}
class="col-span-2 sm:col-span-1"
/>
+
+ {#if currentStats?.gpuCount && currentStats.gpuCount > 0}
+ <GpuMeter
+ gpus={currentStats?.gpus}
+ loading={isLoading.loadingStats || !hasInitialStatsLoaded}
+ />
+ {/if}
</div>
<DockerOverview