Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
feat: gpu monitoring environment flags (#1034)
Timothy Duonga843a5cb9b38
diff --git a/.gitignore b/.gitignoreindex efa3afe39..f7de0d676 100644--- a/.gitignore+++ b/.gitignore@@ -1,10 +1,4 @@node_modules-# fork-timothy-TIMOTHY-README.MD-gpu_service.go-compose.gpu-amd.yaml-compose.gpu-intel.yaml-compose.gpu-nvidia.yaml# Output.outputdiff --git a/backend/internal/api/system_handler.go b/backend/internal/api/system_handler.goindex fc7fdbca6..824f14472 100644--- a/backend/internal/api/system_handler.go+++ b/backend/internal/api/system_handler.go@@ -66,18 +66,22 @@ type SystemHandler struct {sync.RWMutexdetected booltimestamp time.Time- gpuType string // "nvidia", "amd", "intel", "jetson", or ""+ gpuType string // "nvidia", "amd", "intel", or ""toolPath string}- detectionDone bool- detectionMutex sync.Mutex+ detectionDone bool+ detectionMutex sync.Mutex+ gpuMonitoringEnabled bool+ gpuType string}func NewSystemHandler(group *gin.RouterGroup, dockerService *services.DockerClientService, systemService *services.SystemService, upgradeService *services.SystemUpgradeService, authMiddleware *middleware.AuthMiddleware, cfg *config.Config) {handler := &SystemHandler{- dockerService: dockerService,- systemService: systemService,- upgradeService: upgradeService,+ dockerService: dockerService,+ systemService: systemService,+ upgradeService: upgradeService,+ gpuMonitoringEnabled: cfg.GPUMonitoringEnabled,+ gpuType: cfg.GPUType,sysWsUpgrader: websocket.Upgrader{CheckOrigin: httputil.ValidateWebSocketOrigin(cfg.AppUrl),},@@ -442,9 +446,11 @@ func (h *SystemHandler) Stats(c *gin.Context) {var gpuStats []GPUStatsvar gpuCount int- if gpuData, err := h.getGPUStats(ctx); err == nil {- gpuStats = gpuData- gpuCount = len(gpuData)+ if h.gpuMonitoringEnabled {+ if gpuData, err := h.getGPUStats(ctx); err == nil {+ gpuStats = gpuData+ gpuCount = len(gpuData)+ }}stats := SystemStats{@@ -607,8 +613,6 @@ func (h *SystemHandler) getGPUStats(ctx context.Context) ([]GPUStats, error) {return h.getAMDStats(ctx)case "intel":return h.getIntelStats(ctx)- case "jetson":- return h.getJetsonStats(ctx)}}h.gpuDetectionCache.RUnlock()@@ -630,8 +634,6 @@ func (h *SystemHandler) getGPUStats(ctx context.Context) ([]GPUStats, error) {return h.getAMDStats(ctx)case "intel":return h.getIntelStats(ctx)- case "jetson":- return h.getJetsonStats(ctx)default:return nil, fmt.Errorf("no supported GPU found")}@@ -642,6 +644,58 @@ func (h *SystemHandler) detectGPUs(ctx context.Context) error {h.detectionMutex.Lock()defer h.detectionMutex.Unlock()+ // If GPU type is explicitly specified, skip detection and use that type+ if h.gpuType != "" && h.gpuType != "auto" {+ switch h.gpuType {+ case "nvidia":+ if path, err := exec.LookPath("nvidia-smi"); err == nil {+ h.gpuDetectionCache.Lock()+ h.gpuDetectionCache.detected = true+ h.gpuDetectionCache.gpuType = "nvidia"+ h.gpuDetectionCache.toolPath = path+ h.gpuDetectionCache.timestamp = time.Now()+ h.gpuDetectionCache.Unlock()+ h.detectionDone = true+ slog.InfoContext(ctx, "Using configured GPU type", slog.String("type", "nvidia"))+ return nil+ }+ return fmt.Errorf("nvidia-smi not found but GPU_TYPE set to nvidia")++ case "amd":+ if path, err := exec.LookPath("rocm-smi"); err == nil {+ h.gpuDetectionCache.Lock()+ h.gpuDetectionCache.detected = true+ h.gpuDetectionCache.gpuType = "amd"+ h.gpuDetectionCache.toolPath = path+ h.gpuDetectionCache.timestamp = time.Now()+ h.gpuDetectionCache.Unlock()+ h.detectionDone = true+ slog.InfoContext(ctx, "Using configured GPU type", slog.String("type", "amd"))+ return nil+ }+ return fmt.Errorf("rocm-smi not found but GPU_TYPE set to amd")++ case "intel":+ if path, err := exec.LookPath("intel_gpu_top"); err == nil {+ h.gpuDetectionCache.Lock()+ h.gpuDetectionCache.detected = true+ h.gpuDetectionCache.gpuType = "intel"+ h.gpuDetectionCache.toolPath = path+ h.gpuDetectionCache.timestamp = time.Now()+ h.gpuDetectionCache.Unlock()+ h.detectionDone = true+ slog.InfoContext(ctx, "Using configured GPU type", slog.String("type", "intel"))+ return nil+ }+ return fmt.Errorf("intel_gpu_top not found but GPU_TYPE set to intel")++ default:+ slog.WarnContext(ctx, "Invalid GPU_TYPE specified, falling back to auto-detection", slog.String("gpu_type", h.gpuType))+ // Fall through to auto-detection+ }+ }++ // Auto-detection: Check in order (nvidia → amd → intel)// Check for NVIDIAif path, err := exec.LookPath("nvidia-smi"); err == nil {h.gpuDetectionCache.Lock()@@ -668,19 +722,6 @@ func (h *SystemHandler) detectGPUs(ctx context.Context) error {return nil}- // Check for NVIDIA Jetson- if path, err := exec.LookPath("tegrastats"); err == nil {- h.gpuDetectionCache.Lock()- h.gpuDetectionCache.detected = true- h.gpuDetectionCache.gpuType = "jetson"- h.gpuDetectionCache.toolPath = path- h.gpuDetectionCache.timestamp = time.Now()- h.gpuDetectionCache.Unlock()- h.detectionDone = true- slog.InfoContext(ctx, "NVIDIA Jetson detected", "tool", "tegrastats", "path", path)- return nil- }-// Check for Intel GPUif path, err := exec.LookPath("intel_gpu_top"); err == nil {h.gpuDetectionCache.Lock()@@ -872,65 +913,3 @@ func (h *SystemHandler) parseIntelOutput(ctx context.Context, output []byte) ([]slog.DebugContext(ctx, "Intel GPU detected but detailed stats not yet implemented")return stats, nil}--// getJetsonStats collects NVIDIA Jetson statistics using tegrastats-func (h *SystemHandler) getJetsonStats(ctx context.Context) ([]GPUStats, error) {- ctx, cancel := context.WithTimeout(ctx, 3*time.Second)- defer cancel()-- // tegrastats outputs continuous stream, we'll use --interval for single sample- cmd := exec.CommandContext(ctx, "tegrastats", "--interval", "1000")- output, err := cmd.Output()- if err != nil {- slog.WarnContext(ctx, "Failed to execute tegrastats", "error", err)- return nil, fmt.Errorf("tegrastats execution failed: %w", err)- }-- return h.parseJetsonOutput(ctx, output)-}--// parseJetsonOutput parses text output from tegrastats-func (h *SystemHandler) parseJetsonOutput(ctx context.Context, output []byte) ([]GPUStats, error) {- // tegrastats output format: RAM 1234/5678MB (lfb 1x2MB) ...- // This is a simplified parser for MVP-- lines := strings.Split(string(output), "\n")- if len(lines) == 0 {- return nil, fmt.Errorf("no output from tegrastats")- }-- // Parse first line for RAM info (Jetson uses unified memory)- // Example: "RAM 1234/5678MB"- for _, line := range lines {- if strings.Contains(line, "RAM") {- // Simple parsing - extract RAM usage- parts := strings.Fields(line)- for i, part := range parts {- if part == "RAM" && i+1 < len(parts) {- memPart := parts[i+1]- if strings.Contains(memPart, "/") && strings.HasSuffix(memPart, "MB") {- memPart = strings.TrimSuffix(memPart, "MB")- memValues := strings.Split(memPart, "/")- if len(memValues) == 2 {- used, _ := strconv.ParseFloat(memValues[0], 64)- total, _ := strconv.ParseFloat(memValues[1], 64)-- // tegrastats returns MB, convert to bytes- return []GPUStats{- {- Name: "NVIDIA Jetson",- Index: 0,- MemoryUsed: used * 1024 * 1024,- MemoryTotal: total * 1024 * 1024,- },- }, nil- }- }- }- }- }- }-- slog.DebugContext(ctx, "NVIDIA Jetson detected but could not parse memory stats")- return nil, fmt.Errorf("failed to parse tegrastats output")-}diff --git a/backend/internal/config/config.go b/backend/internal/config/config.goindex 33dc4b279..b92767a0b 100644--- a/backend/internal/config/config.go+++ b/backend/internal/config/config.go@@ -40,6 +40,8 @@ type Config struct {UpdateCheckDisabled boolUIConfigurationDisabled boolAnalyticsDisabled bool+ GPUMonitoringEnabled bool+ GPUType string}func Load() *Config {@@ -68,6 +70,8 @@ func Load() *Config {UpdateCheckDisabled: getBoolEnvOrDefault("UPDATE_CHECK_DISABLED", false),UIConfigurationDisabled: getBoolEnvOrDefault("UI_CONFIGURATION_DISABLED", false),AnalyticsDisabled: getBoolEnvOrDefault("ANALYTICS_DISABLED", false),+ GPUMonitoringEnabled: getBoolEnvOrDefault("GPU_MONITORING_ENABLED", false),+ GPUType: getEnvOrDefault("GPU_TYPE", "auto"),}}