Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

78.1 KB
Raw Blame
package api

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"net"
	"net/http"
	"os"
	"os/user"
	"regexp"
	"runtime"
	"runtime/debug"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	alertconfig "github.com/rcourtman/pulse-go-rewrite/internal/alerts/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/internal/updates"
	"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"github.com/rs/zerolog/log"
	"github.com/shirou/gopsutil/v4/process"
	"golang.org/x/crypto/ssh"
	"golang.org/x/sync/singleflight"
)

// DiagnosticsInfo contains comprehensive diagnostic information
type DiagnosticsInfo struct {
	Version      string                  `json:"version"`
	Runtime      string                  `json:"runtime"`
	Uptime       float64                 `json:"uptime"`
	Nodes        []NodeDiagnostic        `json:"nodes"`
	PBS          []PBSDiagnostic         `json:"pbs"`
	System       SystemDiagnostic        `json:"system"`
	MetricsStore *MetricsStoreDiagnostic `json:"metricsStore,omitempty"`
	Discovery    *DiscoveryDiagnostic    `json:"discovery,omitempty"`
	APITokens    *APITokenDiagnostic     `json:"apiTokens,omitempty"`
	DockerAgents *DockerAgentDiagnostic  `json:"dockerAgents,omitempty"`
	Alerts       *AlertsDiagnostic       `json:"alerts,omitempty"`
	AIChat       *AIChatDiagnostic       `json:"aiChat,omitempty"`
	Errors       []string                `json:"errors"`
	// NodeSnapshots captures the raw memory payload and derived usage Pulse last observed per node.
	NodeSnapshots []monitoring.NodeMemorySnapshot `json:"nodeSnapshots"`
	// GuestSnapshots captures recent per-guest memory breakdowns (VM/LXC) with the raw Proxmox fields.
	GuestSnapshots []monitoring.GuestMemorySnapshot `json:"guestSnapshots"`
	// MemorySources summarizes how many nodes currently rely on each memory source per instance.
	MemorySources []MemorySourceStat `json:"memorySources"`
	// MemorySourceBreakdown captures node and guest source quality, trust, and fallback reasons.
	MemorySourceBreakdown []MemorySourceBreakdown `json:"memorySourceBreakdown"`
}

func EmptyDiagnosticsInfo() DiagnosticsInfo {
	return DiagnosticsInfo{}.NormalizeCollections()
}

func (d DiagnosticsInfo) NormalizeCollections() DiagnosticsInfo {
	if d.Nodes == nil {
		d.Nodes = []NodeDiagnostic{}
	}
	if d.PBS == nil {
		d.PBS = []PBSDiagnostic{}
	}
	if d.Errors == nil {
		d.Errors = []string{}
	}
	if d.NodeSnapshots == nil {
		d.NodeSnapshots = []monitoring.NodeMemorySnapshot{}
	}
	if d.GuestSnapshots == nil {
		d.GuestSnapshots = []monitoring.GuestMemorySnapshot{}
	}
	if d.MemorySources == nil {
		d.MemorySources = []MemorySourceStat{}
	}
	if d.MemorySourceBreakdown == nil {
		d.MemorySourceBreakdown = []MemorySourceBreakdown{}
	}
	for i := range d.Nodes {
		d.Nodes[i] = d.Nodes[i].NormalizeCollections()
	}
	for i := range d.MemorySourceBreakdown {
		d.MemorySourceBreakdown[i] = d.MemorySourceBreakdown[i].NormalizeCollections()
	}
	if d.MetricsStore != nil {
		normalized := d.MetricsStore.NormalizeCollections()
		d.MetricsStore = &normalized
	}
	if d.Discovery != nil {
		normalized := d.Discovery.NormalizeCollections()
		d.Discovery = &normalized
	}
	if d.APITokens != nil {
		normalized := d.APITokens.NormalizeCollections()
		d.APITokens = &normalized
	}
	if d.DockerAgents != nil {
		normalized := d.DockerAgents.NormalizeCollections()
		d.DockerAgents = &normalized
	}
	if d.Alerts != nil {
		normalized := d.Alerts.NormalizeCollections()
		d.Alerts = &normalized
	}
	if d.AIChat != nil {
		normalized := d.AIChat.NormalizeCollections()
		d.AIChat = &normalized
	}
	return d
}

// DiscoveryDiagnostic summarizes discovery configuration and recent activity.
type DiscoveryDiagnostic struct {
	Enabled             bool                   `json:"enabled"`
	ConfiguredSubnet    string                 `json:"configuredSubnet,omitempty"`
	ActiveSubnet        string                 `json:"activeSubnet,omitempty"`
	EnvironmentOverride string                 `json:"environmentOverride,omitempty"`
	SubnetAllowlist     []string               `json:"subnetAllowlist"`
	SubnetBlocklist     []string               `json:"subnetBlocklist"`
	Scanning            bool                   `json:"scanning"`
	ScanInterval        string                 `json:"scanInterval,omitempty"`
	LastScanStartedAt   string                 `json:"lastScanStartedAt,omitempty"`
	LastResultTimestamp string                 `json:"lastResultTimestamp,omitempty"`
	LastResultServers   int                    `json:"lastResultServers,omitempty"`
	LastResultErrors    int                    `json:"lastResultErrors,omitempty"`
	History             []DiscoveryHistoryItem `json:"history"`
}

func (d DiscoveryDiagnostic) NormalizeCollections() DiscoveryDiagnostic {
	if d.SubnetAllowlist == nil {
		d.SubnetAllowlist = []string{}
	}
	if d.SubnetBlocklist == nil {
		d.SubnetBlocklist = []string{}
	}
	if d.History == nil {
		d.History = []DiscoveryHistoryItem{}
	}
	return d
}

// DiscoveryHistoryItem summarizes the outcome of a recent discovery scan.
type DiscoveryHistoryItem struct {
	StartedAt       string `json:"startedAt"`
	CompletedAt     string `json:"completedAt"`
	Duration        string `json:"duration"`
	DurationMs      int64  `json:"durationMs"`
	Subnet          string `json:"subnet"`
	ServerCount     int    `json:"serverCount"`
	ErrorCount      int    `json:"errorCount"`
	BlocklistLength int    `json:"blocklistLength"`
	Status          string `json:"status"`
}

// MemorySourceStat aggregates memory-source usage per instance.
type MemorySourceStat struct {
	Instance    string `json:"instance"`
	Source      string `json:"source"`
	NodeCount   int    `json:"nodeCount"`
	LastUpdated string `json:"lastUpdated"`
	Fallback    bool   `json:"fallback"`
	Trust       string `json:"trust,omitempty"`
}

// MemorySourceBreakdown captures diagnostics for node and guest memory source selection.
type MemorySourceBreakdown struct {
	Instance        string   `json:"instance"`
	Scope           string   `json:"scope"`
	Source          string   `json:"source"`
	Count           int      `json:"count"`
	LastUpdated     string   `json:"lastUpdated"`
	Fallback        bool     `json:"fallback"`
	Trust           string   `json:"trust"`
	FallbackReasons []string `json:"fallbackReasons"`
}

func (b MemorySourceBreakdown) NormalizeCollections() MemorySourceBreakdown {
	if b.FallbackReasons == nil {
		b.FallbackReasons = []string{}
	}
	return b
}

// MetricsStoreDiagnostic summarizes metrics store health and data availability.
type MetricsStoreDiagnostic struct {
	Enabled     bool     `json:"enabled"`
	Status      string   `json:"status"`
	DBSize      int64    `json:"dbSize,omitempty"`
	RawCount    int64    `json:"rawCount,omitempty"`
	MinuteCount int64    `json:"minuteCount,omitempty"`
	HourlyCount int64    `json:"hourlyCount,omitempty"`
	DailyCount  int64    `json:"dailyCount,omitempty"`
	TotalPoints int64    `json:"totalPoints,omitempty"`
	BufferSize  int      `json:"bufferSize,omitempty"`
	Notes       []string `json:"notes"`
	Error       string   `json:"error,omitempty"`
}

func (d MetricsStoreDiagnostic) NormalizeCollections() MetricsStoreDiagnostic {
	if d.Notes == nil {
		d.Notes = []string{}
	}
	return d
}

func isFallbackMemorySource(source string) bool {
	return monitoring.MemorySourceIsFallback(source)
}

func classifyMemorySourceTrust(source string) string {
	return monitoring.MemorySourceTrust(source)
}

func buildMemorySourceDiagnostics(snapshots monitoring.DiagnosticSnapshotSet) ([]MemorySourceStat, []MemorySourceBreakdown) {
	type nodeAgg struct {
		stat   MemorySourceStat
		latest time.Time
	}
	type breakdownAgg struct {
		stat            MemorySourceBreakdown
		latest          time.Time
		fallbackReasons map[string]struct{}
	}

	nodeStats := make(map[string]*nodeAgg)
	breakdown := make(map[string]*breakdownAgg)

	appendBreakdown := func(instance, scope, source string, retrievedAt time.Time, fallbackReason string) {
		source = monitoring.CanonicalMemorySource(source)
		if source == "" {
			source = "unknown"
		}
		if strings.TrimSpace(fallbackReason) == "" {
			fallbackReason = monitoring.MemorySourceFallbackReason(source)
		}

		key := fmt.Sprintf("%s|%s|%s", instance, scope, source)
		entry, ok := breakdown[key]
		if !ok {
			trust := classifyMemorySourceTrust(source)
			entry = &breakdownAgg{
				stat: MemorySourceBreakdown{
					Instance: instance,
					Scope:    scope,
					Source:   source,
					Fallback: isFallbackMemorySource(source),
					Trust:    trust,
				},
				fallbackReasons: make(map[string]struct{}),
			}
			breakdown[key] = entry
		}

		entry.stat.Count++
		if fallbackReason = strings.TrimSpace(fallbackReason); fallbackReason != "" {
			entry.fallbackReasons[fallbackReason] = struct{}{}
		}
		if retrievedAt.After(entry.latest) {
			entry.latest = retrievedAt
		}
	}

	for _, snap := range snapshots.Nodes {
		source := monitoring.CanonicalMemorySource(snap.MemorySource)
		if source == "" {
			source = "unknown"
		}

		key := fmt.Sprintf("%s|%s", snap.Instance, source)
		entry, ok := nodeStats[key]
		if !ok {
			trust := classifyMemorySourceTrust(source)
			entry = &nodeAgg{
				stat: MemorySourceStat{
					Instance: snap.Instance,
					Source:   source,
					Fallback: isFallbackMemorySource(source),
					Trust:    trust,
				},
			}
			nodeStats[key] = entry
		}

		entry.stat.NodeCount++
		if snap.RetrievedAt.After(entry.latest) {
			entry.latest = snap.RetrievedAt
		}

		appendBreakdown(snap.Instance, "node", source, snap.RetrievedAt, snap.FallbackReason)
	}

	for _, snap := range snapshots.Guests {
		appendBreakdown(snap.Instance, "guest", snap.MemorySource, snap.RetrievedAt, snap.FallbackReason)
	}

	nodeSourceStats := make([]MemorySourceStat, 0, len(nodeStats))
	for _, entry := range nodeStats {
		if !entry.latest.IsZero() {
			entry.stat.LastUpdated = entry.latest.UTC().Format(time.RFC3339)
		}
		nodeSourceStats = append(nodeSourceStats, entry.stat)
	}
	sort.Slice(nodeSourceStats, func(i, j int) bool {
		if nodeSourceStats[i].Instance == nodeSourceStats[j].Instance {
			return nodeSourceStats[i].Source < nodeSourceStats[j].Source
		}
		return nodeSourceStats[i].Instance < nodeSourceStats[j].Instance
	})

	breakdownStats := make([]MemorySourceBreakdown, 0, len(breakdown))
	for _, entry := range breakdown {
		if !entry.latest.IsZero() {
			entry.stat.LastUpdated = entry.latest.UTC().Format(time.RFC3339)
		}
		if len(entry.fallbackReasons) > 0 {
			entry.stat.FallbackReasons = make([]string, 0, len(entry.fallbackReasons))
			for reason := range entry.fallbackReasons {
				entry.stat.FallbackReasons = append(entry.stat.FallbackReasons, reason)
			}
			sort.Strings(entry.stat.FallbackReasons)
		}
		breakdownStats = append(breakdownStats, entry.stat)
	}
	sort.Slice(breakdownStats, func(i, j int) bool {
		if breakdownStats[i].Instance == breakdownStats[j].Instance {
			if breakdownStats[i].Scope == breakdownStats[j].Scope {
				return breakdownStats[i].Source < breakdownStats[j].Source
			}
			return breakdownStats[i].Scope < breakdownStats[j].Scope
		}
		return breakdownStats[i].Instance < breakdownStats[j].Instance
	})

	return nodeSourceStats, breakdownStats
}

func buildMetricsStoreDiagnostic(monitor *monitoring.Monitor) *MetricsStoreDiagnostic {
	if monitor == nil {
		return &MetricsStoreDiagnostic{
			Enabled: false,
			Status:  "unavailable",
			Error:   "monitor not initialized",
		}
	}

	store := monitor.GetMetricsStore()
	if store == nil {
		return &MetricsStoreDiagnostic{
			Enabled: false,
			Status:  "unavailable",
			Error:   "metrics store not initialized",
		}
	}

	stats := store.GetStats()
	total := stats.RawCount + stats.MinuteCount + stats.HourlyCount + stats.DailyCount
	status := "healthy"
	notes := []string{}

	switch {
	case total == 0 && stats.BufferSize > 0:
		status = "buffering"
		notes = append(notes, "Metrics are buffered but not yet flushed")
	case total == 0:
		status = "empty"
		notes = append(notes, "No historical metrics written yet")
	}

	return &MetricsStoreDiagnostic{
		Enabled:     true,
		Status:      status,
		DBSize:      stats.DBSize,
		RawCount:    stats.RawCount,
		MinuteCount: stats.MinuteCount,
		HourlyCount: stats.HourlyCount,
		DailyCount:  stats.DailyCount,
		TotalPoints: total,
		BufferSize:  stats.BufferSize,
		Notes:       notes,
	}
}

const diagnosticsCacheTTL = 45 * time.Second

type cachedDiagnosticsEntry struct {
	diag     DiagnosticsInfo
	cachedAt time.Time
}

var (
	diagnosticsMetricsOnce sync.Once

	diagnosticsCacheMu sync.RWMutex
	diagnosticsCache   = map[string]cachedDiagnosticsEntry{}

	// diagnosticsComputeGroup dedupes concurrent cache-miss compute calls
	// keyed by scope. Without this, N concurrent requests on cache miss
	// each pay the full computeDiagnostics cost; with it, only the first
	// goroutine runs and the rest wait on the shared result.
	diagnosticsComputeGroup singleflight.Group

	diagnosticsCacheHits = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "pulse",
		Subsystem: "diagnostics",
		Name:      "cache_hits_total",
		Help:      "Total number of diagnostics cache hits.",
	})

	diagnosticsCacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "pulse",
		Subsystem: "diagnostics",
		Name:      "cache_misses_total",
		Help:      "Total number of diagnostics cache misses.",
	})

	diagnosticsRefreshDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
		Namespace: "pulse",
		Subsystem: "diagnostics",
		Name:      "refresh_duration_seconds",
		Help:      "Duration of diagnostics refresh operations in seconds.",
		Buckets:   []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30},
	})
)

func diagnosticsScopeKey(ctx context.Context) string {
	if orgID := strings.TrimSpace(GetOrgID(ctx)); orgID != "" {
		return orgID
	}
	return "__default__"
}

// NodeDiagnostic contains diagnostic info for a Proxmox node
type NodeDiagnostic struct {
	ID              string             `json:"id"`
	Name            string             `json:"name"`
	Host            string             `json:"host"`
	Type            string             `json:"type"`
	AuthMethod      string             `json:"authMethod"`
	Connected       bool               `json:"connected"`
	Error           string             `json:"error,omitempty"`
	ErrorKind       string             `json:"errorKind,omitempty"`
	Troubleshooting string             `json:"troubleshooting,omitempty"`
	Details         *NodeDetails       `json:"details,omitempty"`
	LastPoll        string             `json:"lastPoll,omitempty"`
	ClusterInfo     *ClusterInfo       `json:"clusterInfo,omitempty"`
	VMDiskCheck     *VMDiskCheckResult `json:"vmDiskCheck,omitempty"`
	PhysicalDisks   *PhysicalDiskCheck `json:"physicalDisks,omitempty"`
}

func (d NodeDiagnostic) NormalizeCollections() NodeDiagnostic {
	if d.VMDiskCheck != nil {
		normalized := d.VMDiskCheck.NormalizeCollections()
		d.VMDiskCheck = &normalized
	}
	if d.PhysicalDisks != nil {
		normalized := d.PhysicalDisks.NormalizeCollections()
		d.PhysicalDisks = &normalized
	}
	return d
}

// NodeDetails contains node-specific details
type NodeDetails struct {
	NodeCount int    `json:"node_count,omitempty"`
	Version   string `json:"version,omitempty"`
}

// VMDiskCheckResult contains VM disk monitoring diagnostic results
type VMDiskCheckResult struct {
	VMsFound         int                `json:"vmsFound"`
	VMsWithAgent     int                `json:"vmsWithAgent"`
	VMsWithDiskData  int                `json:"vmsWithDiskData"`
	TestVMID         int                `json:"testVMID,omitempty"`
	TestVMName       string             `json:"testVMName,omitempty"`
	TestResult       string             `json:"testResult,omitempty"`
	Permissions      []string           `json:"permissions"`
	Recommendations  []string           `json:"recommendations"`
	ProblematicVMs   []VMDiskIssue      `json:"problematicVMs"`
	FilesystemsFound []FilesystemDetail `json:"filesystemsFound"`
}

func (r VMDiskCheckResult) NormalizeCollections() VMDiskCheckResult {
	if r.Permissions == nil {
		r.Permissions = []string{}
	}
	if r.Recommendations == nil {
		r.Recommendations = []string{}
	}
	if r.ProblematicVMs == nil {
		r.ProblematicVMs = []VMDiskIssue{}
	}
	if r.FilesystemsFound == nil {
		r.FilesystemsFound = []FilesystemDetail{}
	}
	return r
}

type VMDiskIssue struct {
	VMID   int    `json:"vmid"`
	Name   string `json:"name"`
	Status string `json:"status"`
	Issue  string `json:"issue"`
}

type FilesystemDetail struct {
	Mountpoint string `json:"mountpoint"`
	Type       string `json:"type"`
	Total      uint64 `json:"total"`
	Used       uint64 `json:"used"`
	Filtered   bool   `json:"filtered"`
	Reason     string `json:"reason,omitempty"`
}

// PhysicalDiskCheck contains diagnostic results for physical disk detection
type PhysicalDiskCheck struct {
	NodesChecked    int              `json:"nodesChecked"`
	NodesWithDisks  int              `json:"nodesWithDisks"`
	TotalDisks      int              `json:"totalDisks"`
	NodeResults     []NodeDiskResult `json:"nodeResults"`
	TestResult      string           `json:"testResult,omitempty"`
	Recommendations []string         `json:"recommendations"`
}

func (c PhysicalDiskCheck) NormalizeCollections() PhysicalDiskCheck {
	if c.NodeResults == nil {
		c.NodeResults = []NodeDiskResult{}
	}
	if c.Recommendations == nil {
		c.Recommendations = []string{}
	}
	for i := range c.NodeResults {
		c.NodeResults[i] = c.NodeResults[i].NormalizeCollections()
	}
	return c
}

type NodeDiskResult struct {
	NodeName    string   `json:"nodeName"`
	DiskCount   int      `json:"diskCount"`
	Error       string   `json:"error,omitempty"`
	DiskDevices []string `json:"diskDevices"`
	APIResponse string   `json:"apiResponse,omitempty"`
}

func (r NodeDiskResult) NormalizeCollections() NodeDiskResult {
	if r.DiskDevices == nil {
		r.DiskDevices = []string{}
	}
	return r
}

// ClusterInfo contains cluster information
type ClusterInfo struct {
	Nodes int `json:"nodes"`
}

// PBSDiagnostic contains diagnostic info for a PBS instance
type PBSDiagnostic struct {
	ID              string             `json:"id"`
	Name            string             `json:"name"`
	Host            string             `json:"host"`
	AuthMethod      string             `json:"authMethod"`
	Connected       bool               `json:"connected"`
	State           ConnectionState    `json:"state"`
	StateReason     string             `json:"stateReason,omitempty"`
	LastSeen        *time.Time         `json:"lastSeen,omitempty"`
	LastError       *ConnectionError   `json:"lastError,omitempty"`
	Error           string             `json:"error,omitempty"`
	ErrorKind       string             `json:"errorKind,omitempty"`
	Troubleshooting string             `json:"troubleshooting,omitempty"`
	Details         *PBSDetails        `json:"details,omitempty"`
	Probe           PBSProbeDiagnostic `json:"probe"`
}

// PBSProbeDiagnostic keeps the direct diagnostics request separate from the
// canonical monitor result. A successful one-off probe is useful recovery
// evidence, but it cannot overwrite the last completed runtime poll.
type PBSProbeDiagnostic struct {
	Connected       bool        `json:"connected"`
	Error           string      `json:"error,omitempty"`
	ErrorKind       string      `json:"errorKind,omitempty"`
	Troubleshooting string      `json:"troubleshooting,omitempty"`
	Details         *PBSDetails `json:"details,omitempty"`
}

// PBSDetails contains PBS-specific details
type PBSDetails struct {
	Version string `json:"version,omitempty"`
}

// SystemDiagnostic contains system-level diagnostic info
type SystemDiagnostic struct {
	OS                string `json:"os"`
	Arch              string `json:"arch"`
	GoVersion         string `json:"goVersion"`
	NumCPU            int    `json:"numCPU"`
	NumGoroutine      int    `json:"numGoroutine"`
	MemoryMB          uint64 `json:"memoryMB"`
	ProcessRSSMB      uint64 `json:"processRssMB,omitempty"`
	HeapAllocMB       uint64 `json:"heapAllocMB,omitempty"`
	HeapInUseMB       uint64 `json:"heapInUseMB,omitempty"`
	HeapIdleMB        uint64 `json:"heapIdleMB,omitempty"`
	HeapReleasedMB    uint64 `json:"heapReleasedMB,omitempty"`
	RuntimeRetainedMB uint64 `json:"runtimeRetainedMB,omitempty"`
	GCMemoryLimitMB   uint64 `json:"gcMemoryLimitMB,omitempty"`
}

const bytesPerMiB = uint64(1024 * 1024)

func bytesToMiB(value uint64) uint64 {
	return value / bytesPerMiB
}

func currentProcessRSS() uint64 {
	p, err := process.NewProcess(int32(os.Getpid()))
	if err != nil {
		return 0
	}
	info, err := p.MemoryInfo()
	if err != nil || info == nil {
		return 0
	}
	return info.RSS
}

func buildSystemDiagnostic(memStats runtime.MemStats, processRSS uint64, gcMemoryLimit int64) SystemDiagnostic {
	runtimeRetained := memStats.Sys
	if memStats.HeapReleased < runtimeRetained {
		runtimeRetained -= memStats.HeapReleased
	} else {
		runtimeRetained = 0
	}

	diagnostic := SystemDiagnostic{
		OS:                runtime.GOOS,
		Arch:              runtime.GOARCH,
		GoVersion:         runtime.Version(),
		NumCPU:            runtime.NumCPU(),
		NumGoroutine:      runtime.NumGoroutine(),
		MemoryMB:          bytesToMiB(memStats.Alloc), // Backward-compatible live-heap field.
		ProcessRSSMB:      bytesToMiB(processRSS),
		HeapAllocMB:       bytesToMiB(memStats.Alloc),
		HeapInUseMB:       bytesToMiB(memStats.HeapInuse),
		HeapIdleMB:        bytesToMiB(memStats.HeapIdle),
		HeapReleasedMB:    bytesToMiB(memStats.HeapReleased),
		RuntimeRetainedMB: bytesToMiB(runtimeRetained),
	}
	if gcMemoryLimit > 0 && gcMemoryLimit < math.MaxInt64 {
		diagnostic.GCMemoryLimitMB = bytesToMiB(uint64(gcMemoryLimit))
	}
	return diagnostic
}

// APITokenDiagnostic reports on the state of the multi-token authentication system.
type APITokenDiagnostic struct {
	Enabled             bool              `json:"enabled"`
	TokenCount          int               `json:"tokenCount"`
	RecommendTokenSetup bool              `json:"recommendTokenSetup"`
	UnusedTokenCount    int               `json:"unusedTokenCount,omitempty"`
	Notes               []string          `json:"notes"`
	Tokens              []APITokenSummary `json:"tokens"`
	Usage               []APITokenUsage   `json:"usage"`
}

func (d APITokenDiagnostic) NormalizeCollections() APITokenDiagnostic {
	if d.Notes == nil {
		d.Notes = []string{}
	}
	if d.Tokens == nil {
		d.Tokens = []APITokenSummary{}
	}
	if d.Usage == nil {
		d.Usage = []APITokenUsage{}
	}
	for i := range d.Usage {
		d.Usage[i] = d.Usage[i].NormalizeCollections()
	}
	return d
}

// APITokenSummary provides sanitized token metadata for diagnostics display.
type APITokenSummary struct {
	ID         string `json:"id"`
	Name       string `json:"name"`
	Hint       string `json:"hint,omitempty"`
	CreatedAt  string `json:"createdAt,omitempty"`
	LastUsedAt string `json:"lastUsedAt,omitempty"`
	Source     string `json:"source,omitempty"`
}

// APITokenUsage summarises how tokens are consumed by connected agents.
type APITokenUsage struct {
	TokenID    string   `json:"tokenId"`
	AgentCount int      `json:"agentCount"`
	Agents     []string `json:"agents"`
}

func (u APITokenUsage) NormalizeCollections() APITokenUsage {
	if u.Agents == nil {
		u.Agents = []string{}
	}
	return u
}

// DockerAgentDiagnostic summarizes adoption of the Docker / Podman module command system.
type DockerAgentDiagnostic struct {
	AgentsTotal               int                    `json:"agentsTotal"`
	AgentsOnline              int                    `json:"agentsOnline"`
	AgentsReportingVersion    int                    `json:"agentsReportingVersion"`
	AgentsWithTokenBinding    int                    `json:"agentsWithTokenBinding"`
	AgentsWithoutTokenBinding int                    `json:"agentsWithoutTokenBinding"`
	AgentsWithoutVersion      int                    `json:"agentsWithoutVersion,omitempty"`
	AgentsOutdatedVersion     int                    `json:"agentsOutdatedVersion,omitempty"`
	AgentsWithStaleCommand    int                    `json:"agentsWithStaleCommand,omitempty"`
	AgentsPendingUninstall    int                    `json:"agentsPendingUninstall,omitempty"`
	AgentsNeedingAttention    int                    `json:"agentsNeedingAttention"`
	RecommendedAgentVersion   string                 `json:"recommendedAgentVersion,omitempty"`
	Attention                 []DockerAgentAttention `json:"attention"`
	Notes                     []string               `json:"notes"`
}

func (d DockerAgentDiagnostic) NormalizeCollections() DockerAgentDiagnostic {
	if d.Attention == nil {
		d.Attention = []DockerAgentAttention{}
	}
	if d.Notes == nil {
		d.Notes = []string{}
	}
	for i := range d.Attention {
		d.Attention[i] = d.Attention[i].NormalizeCollections()
	}
	return d
}

// DockerAgentAttention captures an individual agent that requires user action.
type DockerAgentAttention struct {
	AgentID      string   `json:"agentId"`
	Name         string   `json:"name"`
	Status       string   `json:"status"`
	AgentVersion string   `json:"agentVersion,omitempty"`
	TokenHint    string   `json:"tokenHint,omitempty"`
	LastSeen     string   `json:"lastSeen,omitempty"`
	Issues       []string `json:"issues"`
}

func (a DockerAgentAttention) NormalizeCollections() DockerAgentAttention {
	if a.Issues == nil {
		a.Issues = []string{}
	}
	return a
}

// AlertsDiagnostic summarises alert configuration migration state.
type AlertsDiagnostic struct {
	MissingCooldown       bool                       `json:"missingCooldown"`
	MissingGroupingWindow bool                       `json:"missingGroupingWindow"`
	Notes                 []string                   `json:"notes"`
	Overrides             []AlertsOverrideDiagnostic `json:"overrides,omitempty"`
}

// AlertsOverrideDiagnostic captures one threshold override entry so support
// triage can verify the persisted key matches the runtime resource ID
// without asking users to cat their alerts config from inside the container.
type AlertsOverrideDiagnostic struct {
	Key                 string             `json:"key"`
	Disabled            bool               `json:"disabled,omitempty"`
	DisableConnectivity bool               `json:"disableConnectivity,omitempty"`
	Thresholds          map[string]float64 `json:"thresholds,omitempty"`
}

func (d AlertsDiagnostic) NormalizeCollections() AlertsDiagnostic {
	if d.Notes == nil {
		d.Notes = []string{}
	}
	return d
}

// AIChatDiagnostic reports on the native Pulse Assistant runtime status.
type AIChatDiagnostic struct {
	Enabled                   bool     `json:"enabled"`
	Running                   bool     `json:"running"`
	Healthy                   bool     `json:"healthy"`
	Port                      int      `json:"port,omitempty"`
	URL                       string   `json:"url,omitempty"`
	Model                     string   `json:"model,omitempty"`
	AssistantRuntimeConnected bool     `json:"assistantRuntimeConnected"`
	Notes                     []string `json:"notes"`
}

func (d AIChatDiagnostic) NormalizeCollections() AIChatDiagnostic {
	if d.Notes == nil {
		d.Notes = []string{}
	}
	return d
}

// handleDiagnostics returns comprehensive diagnostic information
func (r *Router) handleDiagnostics(w http.ResponseWriter, req *http.Request) {
	diagnosticsMetricsOnce.Do(func() {
		prometheus.MustRegister(diagnosticsCacheHits, diagnosticsCacheMisses, diagnosticsRefreshDuration)
	})

	now := time.Now()
	scopeKey := diagnosticsScopeKey(req.Context())

	diagnosticsCacheMu.RLock()
	cachedEntry, ok := diagnosticsCache[scopeKey]
	diagnosticsCacheMu.RUnlock()

	if ok && !cachedEntry.cachedAt.IsZero() && now.Sub(cachedEntry.cachedAt) <= diagnosticsCacheTTL {
		diagnosticsCacheHits.Inc()
		writeDiagnosticsResponse(w, r.diagnosticsForCaller(req, cachedEntry.diag), cachedEntry.cachedAt)
		return
	}

	diagnosticsCacheMisses.Inc()

	ctx, cancel := context.WithTimeout(req.Context(), 30*time.Second)
	defer cancel()

	type diagResult struct {
		diag     DiagnosticsInfo
		cachedAt time.Time
	}

	v, _, _ := diagnosticsComputeGroup.Do(scopeKey, func() (any, error) {
		// Re-check cache inside the singleflight barrier: an earlier
		// caller may have populated it between our miss above and us
		// acquiring the singleflight slot.
		diagnosticsCacheMu.RLock()
		entry, ok := diagnosticsCache[scopeKey]
		diagnosticsCacheMu.RUnlock()
		if ok && !entry.cachedAt.IsZero() && time.Since(entry.cachedAt) <= diagnosticsCacheTTL {
			return diagResult{diag: entry.diag, cachedAt: entry.cachedAt}, nil
		}

		start := time.Now()
		fresh := r.computeDiagnostics(ctx)
		diagnosticsRefreshDuration.Observe(time.Since(start).Seconds())

		diagnosticsCacheMu.Lock()
		cachedAt := time.Now()
		diagnosticsCache[scopeKey] = cachedDiagnosticsEntry{
			diag:     fresh,
			cachedAt: cachedAt,
		}
		diagnosticsCacheMu.Unlock()

		return diagResult{diag: fresh, cachedAt: cachedAt}, nil
	})

	result := v.(diagResult)
	writeDiagnosticsResponse(w, r.diagnosticsForCaller(req, result.diag), result.cachedAt)
}

// diagnosticsForCaller adapts a cached diagnostics result to the caller. The
// cache holds one result per organization scope, built for the instance, and
// is shared by every caller, so per-caller filtering happens here, after the
// cache. The API token section lists credentials, so an API-token caller sees
// only the tokens it may manage (see callerTokenMayManage); instance-wide
// tokens and browser sessions get the full section.
func (r *Router) diagnosticsForCaller(req *http.Request, diag DiagnosticsInfo) DiagnosticsInfo {
	caller := getAPITokenRecordFromRequest(req)
	if caller == nil || tokenHasInstanceWideAuthority(caller) || diag.APITokens == nil {
		return diag
	}
	view := diag
	view.APITokens = buildAPITokenDiagnostic(r.config, r.monitor, caller)
	return view
}

func writeDiagnosticsResponse(w http.ResponseWriter, diag DiagnosticsInfo, cachedAt time.Time) {
	diag = diag.NormalizeCollections()
	w.Header().Set("Content-Type", "application/json")
	if !cachedAt.IsZero() {
		w.Header().Set("X-Diagnostics-Cached-At", cachedAt.UTC().Format(time.RFC3339))
	}
	if err := json.NewEncoder(w).Encode(diag); err != nil {
		log.Error().Err(err).Msg("Failed to encode diagnostics")
		http.Error(w, "Failed to generate diagnostics", http.StatusInternalServerError)
	}
}

// diagnosticsMonitorConnectionStatus looks up the live monitor connection state
// for a node by its connection-status key (e.g. "pve-<name>"/"pbs-<name>").
func diagnosticsMonitorConnectionStatus(m *monitoring.Monitor, key string) (bool, bool) {
	if m == nil || strings.TrimSpace(key) == "" {
		return false, false
	}
	statuses := m.GetConnectionStatuses()
	connected, ok := statuses[key]
	return connected, ok
}

// mergeDiagnosticsConnection prevents a transient diagnostics-probe failure from
// reporting an instance as disconnected when the long-running monitor connection
// still reports it connected.
func mergeDiagnosticsConnection(probeConnected bool, probeError string, monitorConnected bool, hasMonitorStatus bool) (bool, string) {
	if probeConnected {
		return true, probeError
	}
	if !hasMonitorStatus || !monitorConnected {
		return false, probeError
	}
	if strings.TrimSpace(probeError) == "" {
		return true, ""
	}
	return true, "Live diagnostics probe failed, but the monitor still reports this instance connected: " + probeError
}

type diagnosticsConnectionFailure struct {
	kind            string
	message         string
	troubleshooting string
}

func diagnosticsAuthMethod(user, password, tokenName, tokenValue string) string {
	if tokenName != "" && tokenValue != "" {
		return "api_token"
	}
	if user != "" && password != "" {
		return "username_password"
	}
	return "none"
}

func missingStoredSecretDiagnosticsFailure() diagnosticsConnectionFailure {
	return diagnosticsConnectionFailure{
		kind:            "no_stored_secret",
		message:         "Pulse has no stored credentials for this node",
		troubleshooting: "Add or rotate this node's API token or credentials in Settings.",
	}
}

// The scheduler's retained LastError is text, unlike a live probe's typed
// error. Recognise the existing producer's explicit status markers there,
// not arbitrary numbers in a host, token ID or quoted response body. The
// first marker wins; a gateway's body cannot replace its outer status.
var diagnosticsAPIStatusPattern = regexp.MustCompile(`(?i)(?:^|:\s+)(?:API error\s+|authentication failed \(status\s+)([1-5][0-9]{2})\b`)

func diagnosticsAPIStatus(err error) (int, bool) {
	if status, ok := proxmox.APIErrorStatus(err); ok {
		return status, true
	}
	if status, ok := pbs.HTTPStatus(err); ok {
		return status, true
	}
	if err == nil {
		return 0, false
	}
	match := diagnosticsAPIStatusPattern.FindStringSubmatch(strings.TrimSpace(err.Error()))
	if len(match) != 2 {
		return 0, false
	}
	status, err := strconv.Atoi(match[1])
	return status, err == nil
}

func diagnosticsAuthenticationFailure() diagnosticsConnectionFailure {
	return diagnosticsConnectionFailure{
		kind:            "proxmox_auth_rejected",
		message:         "Proxmox rejected the stored API token or credentials",
		troubleshooting: "Check the configured user, realm and token ID and whether the credential has expired or been revoked. Keep secrets private; replace a credential only when its invalidity is confirmed, not as a permissions test.",
	}
}

func diagnosticsPermissionFailure() diagnosticsConnectionFailure {
	return diagnosticsConnectionFailure{
		kind:            "proxmox_permission_denied",
		message:         "Proxmox denied access to the requested resource",
		troubleshooting: "Check the owning user's and token's ACLs, including inheritance, for the required read privilege on the intended resource path. A privilege-separated token is limited by both. Keep intentional access boundaries; do not rotate the token, disable privilege separation or grant Administrator to silence a denial.",
	}
}

func diagnosticsTimeoutFailure() diagnosticsConnectionFailure {
	return diagnosticsConnectionFailure{
		kind:            "connection_timeout",
		message:         "Connection timed out",
		troubleshooting: "Check network latency, API responsiveness, and slow or unreachable backend storage from the Pulse server.",
	}
}

func classifyDiagnosticsConnectionFailure(err error, fallbackMessage string) diagnosticsConnectionFailure {
	raw := ""
	if err != nil {
		raw = strings.TrimSpace(err.Error())
	}
	lower := strings.ToLower(raw)

	// A stopped handshake is not evidence that a certificate is wrong. Check
	// typed causes before text, which also contains the requested URL and can
	// mention "certificate" or "TLS" in an otherwise ordinary endpoint name.
	if errors.Is(err, context.Canceled) {
		return diagnosticsConnectionFailure{
			kind:            "connection_failed",
			message:         "Connection check was cancelled",
			troubleshooting: "The diagnostic request was cancelled; no certificate-trust failure was established. Keep the saved TLS settings.",
		}
	}
	var networkError net.Error
	if errors.Is(err, context.DeadlineExceeded) ||
		(errors.As(err, &networkError) && networkError.Timeout()) {
		return diagnosticsTimeoutFailure()
	}

	// Observe wire status before inspecting text: an HTTP error body can quote
	// a different status, certificate failure, timeout or credential diagnosis.
	if status, ok := diagnosticsAPIStatus(err); ok {
		switch status {
		case http.StatusUnauthorized:
			return diagnosticsAuthenticationFailure()
		case http.StatusForbidden:
			return diagnosticsPermissionFailure()
		default:
			return diagnosticsConnectionFailure{
				kind:            "proxmox_api_error",
				message:         fmt.Sprintf("Proxmox API returned HTTP %d", status),
				troubleshooting: "Review the platform or proxy logs privately for the failed request. This response alone does not establish invalid credentials or a missing permission. Keep saved credentials, access boundaries and TLS verification.",
			}
		}
	}

	if strings.Contains(lower, "context deadline exceeded") ||
		strings.Contains(lower, "client.timeout exceeded") ||
		strings.Contains(lower, "i/o timeout") ||
		strings.Contains(lower, "tls handshake timeout") ||
		strings.Contains(lower, "tls: handshake timeout") ||
		strings.Contains(lower, "timeout awaiting response headers") ||
		strings.Contains(lower, "connection timed out") {
		return diagnosticsTimeoutFailure()
	}

	if strings.Contains(lower, "invalid") && strings.Contains(lower, "host") {
		return diagnosticsConnectionFailure{
			kind:            "invalid_host_url",
			message:         "Stored node URL is invalid",
			troubleshooting: "Check the node host URL in Settings, including the scheme and port.",
		}
	}
	if strings.Contains(lower, "connection refused") {
		return diagnosticsConnectionFailure{
			kind:            "connection_refused",
			message:         "TCP connection refused by host",
			troubleshooting: "Check the node host URL and port, then confirm the Proxmox API service is listening and reachable from Pulse.",
		}
	}
	if strings.Contains(lower, "no route to host") ||
		strings.Contains(lower, "network is unreachable") ||
		strings.Contains(lower, "host is down") {
		return diagnosticsConnectionFailure{
			kind:            "network_unreachable",
			message:         "Network route to host is unavailable",
			troubleshooting: "Check DNS, routing, firewall rules, and the host URL from the Pulse server.",
		}
	}
	if strings.Contains(lower, "no such host") ||
		strings.Contains(lower, "temporary failure in name resolution") ||
		strings.Contains(lower, "name or service not known") {
		return diagnosticsConnectionFailure{
			kind:            "dns_resolution_failed",
			message:         "Node hostname could not be resolved",
			troubleshooting: "Check the node hostname in Settings and verify DNS resolution from the Pulse server.",
		}
	}
	if strings.Contains(lower, "certificate") ||
		strings.Contains(lower, "x509") ||
		strings.Contains(lower, "tls:") ||
		strings.Contains(lower, "fingerprint") {
		return diagnosticsConnectionFailure{
			kind:            "tls_certificate_mismatch",
			message:         "TLS certificate mismatch or untrusted certificate",
			troubleshooting: "Check the configured endpoint's certificate hostname, expiry and trusted chain. If a fingerprint is needed or has changed, independently verify the certificate's identity and add or update the saved fingerprint only after that verification. Do not disable verification or switch to HTTP to diagnose the failure.",
		}
	}
	if strings.Contains(lower, "forbidden") || strings.Contains(lower, "permission denied") {
		return diagnosticsPermissionFailure()
	}
	if strings.Contains(lower, "authentication") ||
		strings.Contains(lower, "unauthorized") ||
		strings.Contains(lower, "invalid credentials") ||
		strings.Contains(lower, "invalid token") {
		return diagnosticsAuthenticationFailure()
	}
	if fallbackMessage == "" {
		fallbackMessage = "Connection check failed"
	}
	return diagnosticsConnectionFailure{
		kind:            "connection_failed",
		message:         fallbackMessage,
		troubleshooting: "Review Pulse logs for the full transport error, then verify the endpoint from the Pulse server.",
	}
}

func applyNodeDiagnosticFailure(diag *NodeDiagnostic, failure diagnosticsConnectionFailure) {
	diag.Connected = false
	diag.Error = failure.message
	diag.ErrorKind = failure.kind
	diag.Troubleshooting = failure.troubleshooting
}

func applyPBSDiagnosticFailure(diag *PBSDiagnostic, failure diagnosticsConnectionFailure) {
	diag.Probe.Connected = false
	diag.Probe.Error = failure.message
	diag.Probe.ErrorKind = failure.kind
	diag.Probe.Troubleshooting = failure.troubleshooting
}

func applyPBSConnectionDiagnostic(diag *PBSDiagnostic, connection Connection) {
	if diag == nil {
		return
	}
	diag.State = connection.State
	diag.StateReason = connection.StateReason
	diag.LastSeen = connection.LastSeen
	diag.LastError = connection.LastError
	diag.Connected = connection.State == ConnectionStateActive
	if diag.Connected {
		return
	}

	diag.Error = connection.StateReason
	if connection.LastError != nil {
		failure := classifyDiagnosticsConnectionFailure(errors.New(connection.LastError.Message), connection.StateReason)
		diag.Error = failure.message
		diag.ErrorKind = failure.kind
		diag.Troubleshooting = failure.troubleshooting
	}
}

// Carry bounded API-rejection guidance into the existing Errors Detected
// consumer, not only the downloadable record. Use positional references so
// the shareable error summary introduces no host, username or token identity.
// PBS monitor and live-check evidence stay separate, even when they disagree.
func appendDiagnosticsAPIRejections(diag *DiagnosticsInfo) {
	appendFailure := func(label, kind, message, guidance string) {
		switch kind {
		case "proxmox_auth_rejected", "proxmox_permission_denied", "proxmox_api_error":
			diag.Errors = append(diag.Errors, label+": "+message+". "+guidance)
		}
	}
	for i, node := range diag.Nodes {
		appendFailure(fmt.Sprintf("PVE connection %d check", i+1), node.ErrorKind, node.Error, node.Troubleshooting)
	}
	for i, node := range diag.PBS {
		appendFailure(fmt.Sprintf("PBS connection %d monitor", i+1), node.ErrorKind, node.Error, node.Troubleshooting)
		appendFailure(fmt.Sprintf("PBS connection %d live check", i+1), node.Probe.ErrorKind, node.Probe.Error, node.Probe.Troubleshooting)
	}
}

func (r *Router) computeDiagnostics(ctx context.Context) DiagnosticsInfo {
	diag := EmptyDiagnosticsInfo()

	// Version info
	if versionInfo, err := updates.GetCurrentVersion(); err == nil {
		diag.Version = versionInfo.Version
		diag.Runtime = versionInfo.Runtime
	} else {
		diag.Version = "unknown"
		diag.Runtime = "go"
	}

	// Uptime
	diag.Uptime = time.Since(r.monitor.GetStartTime()).Seconds()

	// System info
	var memStats runtime.MemStats
	runtime.ReadMemStats(&memStats)
	diag.System = buildSystemDiagnostic(memStats, currentProcessRSS(), debug.SetMemoryLimit(-1))

	diag.APITokens = buildAPITokenDiagnostic(r.config, r.monitor, nil)
	diag.MetricsStore = buildMetricsStoreDiagnostic(r.monitor)

	// Test each configured node
	for _, node := range r.config.PVEInstances {
		nodeDiag := NodeDiagnostic{
			ID:   node.Name,
			Name: node.Name,
			Host: node.Host,
			Type: "pve",
		}
		// GetConnectionStatuses keys by name, falling back to host when the
		// configured name is empty; mirror that so the merge is not a silent
		// no-op for unnamed instances.
		pveKeyName := node.Name
		if strings.TrimSpace(pveKeyName) == "" {
			pveKeyName = node.Host
		}
		monitorConnected, hasMonitorStatus := diagnosticsMonitorConnectionStatus(r.monitor, "pve-"+pveKeyName)

		// Determine auth method (sanitized - don't expose actual values)
		nodeDiag.AuthMethod = diagnosticsAuthMethod(node.User, node.Password, node.TokenName, node.TokenValue)
		if nodeDiag.AuthMethod == "none" {
			applyNodeDiagnosticFailure(&nodeDiag, missingStoredSecretDiagnosticsFailure())
			diag.Nodes = append(diag.Nodes, nodeDiag)
			continue
		}

		// Test connection
		testCfg := proxmox.ClientConfig{
			Host:        node.Host,
			User:        node.User,
			Password:    node.Password,
			TokenName:   node.TokenName,
			TokenValue:  node.TokenValue,
			Fingerprint: node.Fingerprint,
			VerifySSL:   node.VerifySSL,
		}

		client, err := proxmox.NewClient(testCfg)
		if err != nil {
			failure := classifyDiagnosticsConnectionFailure(err, "Failed to initialize connection")
			applyNodeDiagnosticFailure(&nodeDiag, failure)
			log.Error().Err(err).Str("node", node.Name).Msg("Diagnostics: Proxmox client init failed")
			nodeDiag.Connected, nodeDiag.Error = mergeDiagnosticsConnection(nodeDiag.Connected, nodeDiag.Error, monitorConnected, hasMonitorStatus)
		} else {
			nodes, err := client.GetNodes(ctx)
			if err != nil {
				failure := classifyDiagnosticsConnectionFailure(err, "Failed to connect to Proxmox API")
				applyNodeDiagnosticFailure(&nodeDiag, failure)
				log.Error().Err(err).Str("node", node.Name).Msg("Diagnostics: Proxmox API connection failed")
				nodeDiag.Connected, nodeDiag.Error = mergeDiagnosticsConnection(nodeDiag.Connected, nodeDiag.Error, monitorConnected, hasMonitorStatus)
			} else {
				nodeDiag.Connected = true

				if len(nodes) > 0 {
					nodeDiag.Details = &NodeDetails{
						NodeCount: len(nodes),
					}

					if status, err := client.GetNodeStatus(ctx, nodes[0].Node); err == nil && status != nil {
						if status.PVEVersion != "" {
							nodeDiag.Details.Version = status.PVEVersion
						}
					}
				}

				if clusterStatus, err := client.GetClusterStatus(ctx); err == nil {
					nodeDiag.ClusterInfo = &ClusterInfo{Nodes: len(clusterStatus)}
				} else {
					log.Debug().Str("node", node.Name).Msg("Cluster status not available (likely standalone node)")
					nodeDiag.ClusterInfo = &ClusterInfo{Nodes: 1}
				}

				nodeDiag.VMDiskCheck = r.checkVMDiskMonitoring(ctx, client, node.Name)
				nodeDiag.PhysicalDisks = r.checkPhysicalDisks(ctx, client, node.Name)
			}
		}

		diag.Nodes = append(diag.Nodes, nodeDiag)
	}

	pbsConnections := make(map[string]Connection)
	for _, connection := range buildConnections(buildAggregatorInputsWithRuntimeSources(
		ctx,
		r.config,
		r.persistence,
		r.monitor,
		aggregatorRuntimeSources{},
	)) {
		if connection.Type == ConnectionTypePBS {
			pbsConnections[connection.Name] = connection
		}
	}

	// Test PBS instances. Connected/state use the canonical scheduler ledger;
	// Probe records this request's direct check as separate evidence.
	for _, pbsNode := range r.config.PBSInstances {
		pbsDiag := PBSDiagnostic{
			ID:         pbsNode.Name,
			Name:       pbsNode.Name,
			Host:       pbsNode.Host,
			AuthMethod: diagnosticsAuthMethod(pbsNode.User, pbsNode.Password, pbsNode.TokenName, pbsNode.TokenValue),
		}
		if connection, ok := pbsConnections[pbsNode.Name]; ok {
			applyPBSConnectionDiagnostic(&pbsDiag, connection)
		}
		if pbsDiag.AuthMethod == "none" {
			applyPBSDiagnosticFailure(&pbsDiag, missingStoredSecretDiagnosticsFailure())
			diag.PBS = append(diag.PBS, pbsDiag)
			continue
		}

		testCfg := pbs.ClientConfig{
			Host:        pbsNode.Host,
			User:        pbsNode.User,
			Password:    pbsNode.Password,
			TokenName:   pbsNode.TokenName,
			TokenValue:  pbsNode.TokenValue,
			Fingerprint: pbsNode.Fingerprint,
			VerifySSL:   pbsNode.VerifySSL,
		}

		client, err := pbs.NewClient(testCfg)
		if err != nil {
			failure := classifyDiagnosticsConnectionFailure(err, "Failed to initialize connection")
			applyPBSDiagnosticFailure(&pbsDiag, failure)
			log.Error().Err(err).Str("pbs", pbsNode.Name).Msg("Diagnostics: PBS client init failed")
		} else {
			if version, err := client.GetVersion(ctx); err != nil {
				failure := classifyDiagnosticsConnectionFailure(err, "Connection established but version check failed")
				applyPBSDiagnosticFailure(&pbsDiag, failure)
				log.Error().Err(err).Str("pbs", pbsNode.Name).Msg("Diagnostics: PBS version check failed")
			} else {
				pbsDiag.Probe.Connected = true
				pbsDiag.Details = &PBSDetails{Version: version.Version}
				pbsDiag.Probe.Details = pbsDiag.Details
			}
		}

		diag.PBS = append(diag.PBS, pbsDiag)
	}

	diag.DockerAgents = buildDockerAgentDiagnostic(r.monitor, diag.Version)
	diag.Alerts = buildAlertsDiagnostic(r.monitor)
	diag.AIChat = buildAIChatDiagnostic(r.config, r.aiHandler)

	diag.Discovery = buildDiscoveryDiagnostic(r.config, r.monitor)

	if r.monitor != nil {
		snapshots := r.monitor.GetDiagnosticSnapshots()
		if len(snapshots.Nodes) > 0 {
			diag.NodeSnapshots = snapshots.Nodes
		}
		if len(snapshots.Guests) > 0 {
			diag.GuestSnapshots = snapshots.Guests
		}
		diag.MemorySources, diag.MemorySourceBreakdown = buildMemorySourceDiagnostics(snapshots)
	}

	appendDiagnosticsAPIRejections(&diag)
	return diag.NormalizeCollections()
}

func copyStringSlice(values []string) []string {
	if len(values) == 0 {
		return []string{}
	}
	return append([]string(nil), values...)
}

func buildDiscoveryDiagnostic(cfg *config.Config, monitor *monitoring.Monitor) *DiscoveryDiagnostic {
	if cfg == nil {
		return nil
	}

	discovery := &DiscoveryDiagnostic{
		Enabled:             cfg.DiscoveryEnabled,
		ConfiguredSubnet:    strings.TrimSpace(cfg.DiscoverySubnet),
		EnvironmentOverride: strings.TrimSpace(cfg.Discovery.EnvironmentOverride),
		SubnetAllowlist:     copyStringSlice(cfg.Discovery.SubnetAllowlist),
		SubnetBlocklist:     copyStringSlice(cfg.Discovery.SubnetBlocklist),
	}

	if discovery.ConfiguredSubnet == "" {
		discovery.ConfiguredSubnet = "auto"
	}
	if discovery.SubnetAllowlist == nil {
		discovery.SubnetAllowlist = []string{}
	}
	if discovery.SubnetBlocklist == nil {
		discovery.SubnetBlocklist = []string{}
	}

	if monitor != nil {
		if svc := monitor.GetDiscoveryService(); svc != nil {
			status := svc.GetStatus()

			if val, ok := status["subnet"].(string); ok {
				discovery.ActiveSubnet = val
			}
			if val, ok := status["is_scanning"].(bool); ok {
				discovery.Scanning = val
			}
			if val, ok := status["interval"].(string); ok {
				discovery.ScanInterval = val
			}
			if val, ok := status["last_scan"].(time.Time); ok && !val.IsZero() {
				discovery.LastScanStartedAt = val.UTC().Format(time.RFC3339)
			}

			if result, updated := svc.GetCachedResult(); result != nil {
				discovery.LastResultServers = len(result.Servers)
				discovery.LastResultErrors = len(result.StructuredErrors)
				if !updated.IsZero() {
					discovery.LastResultTimestamp = updated.UTC().Format(time.RFC3339)
				}
			}

			history := svc.GetHistory(10)
			if len(history) > 0 {
				items := make([]DiscoveryHistoryItem, 0, len(history))
				for _, entry := range history {
					item := DiscoveryHistoryItem{
						StartedAt:       entry.StartedAt().UTC().Format(time.RFC3339),
						CompletedAt:     entry.CompletedAt().UTC().Format(time.RFC3339),
						Duration:        entry.Duration().Truncate(time.Millisecond).String(),
						DurationMs:      entry.Duration().Milliseconds(),
						Subnet:          entry.Subnet(),
						ServerCount:     entry.ServerCount(),
						ErrorCount:      entry.ErrorCount(),
						BlocklistLength: entry.BlocklistLength(),
						Status:          entry.Status(),
					}
					items = append(items, item)
				}
				discovery.History = items
			}
		}
	}

	return discovery
}

// buildAPITokenDiagnostic summarises the token inventory. A nil caller sees
// every token; an API-token caller sees only the tokens callerTokenMayManage
// allows, and the counts, notes and usage are computed over that view. It
// reads a snapshot taken under config.Mu, so callers must not hold that lock.
func buildAPITokenDiagnostic(cfg *config.Config, monitor *monitoring.Monitor, caller *config.APITokenRecord) *APITokenDiagnostic {
	if cfg == nil {
		return nil
	}

	inventory := snapshotAPITokenRecords(cfg)
	records := make([]config.APITokenRecord, 0, len(inventory))
	visibleIDs := make(map[string]struct{}, len(inventory))
	for i := range inventory {
		record := &inventory[i]
		if !callerTokenMayManage(caller, record) {
			continue
		}
		records = append(records, *record)
		visibleIDs[record.ID] = struct{}{}
	}

	diag := &APITokenDiagnostic{
		Enabled:    len(inventory) > 0,
		TokenCount: len(records),
	}

	appendNote := func(note string) {
		if note == "" || contains(diag.Notes, note) {
			return
		}
		diag.Notes = append(diag.Notes, note)
	}

	diag.RecommendTokenSetup = len(inventory) == 0

	if diag.RecommendTokenSetup {
		appendNote("No API tokens are configured. Open Settings → Security to generate dedicated tokens for each automation or agent.")
	}

	tokens := make([]APITokenSummary, 0, len(records))
	unusedCount := 0
	for _, record := range records {
		summary := APITokenSummary{
			ID:   record.ID,
			Name: record.Name,
		}

		if !record.CreatedAt.IsZero() {
			summary.CreatedAt = record.CreatedAt.UTC().Format(time.RFC3339)
		}

		if record.LastUsedAt != nil && !record.LastUsedAt.IsZero() {
			summary.LastUsedAt = record.LastUsedAt.UTC().Format(time.RFC3339)
		} else {
			unusedCount++
		}

		switch {
		case record.Prefix != "" && record.Suffix != "":
			summary.Hint = fmt.Sprintf("%s…%s", record.Prefix, record.Suffix)
		case record.Prefix != "":
			summary.Hint = record.Prefix + "…"
		case record.Suffix != "":
			summary.Hint = "…" + record.Suffix
		}

		summary.Source = "user"

		tokens = append(tokens, summary)
	}

	diag.Tokens = tokens
	diag.UnusedTokenCount = unusedCount

	if len(records) > 0 {
		if unusedCount == len(records) {
			appendNote("Configured API tokens have not been used yet. Update your agents or automations to switch to the new tokens.")
		} else if unusedCount > 0 {
			appendNote(fmt.Sprintf("%d API token(s) have never been used. Remove unused tokens or update the corresponding agents.", unusedCount))
		}
	}

	tokenUsage := make(map[string][]string)
	if monitor != nil {
		if readState := monitor.GetUnifiedReadStateOrSnapshot(); readState != nil {
			for _, host := range readState.DockerHosts() {
				name := preferredDockerHostName(host)
				if strings.TrimSpace(host.TokenID()) == "" {
					continue
				}
				tokenID := strings.TrimSpace(host.TokenID())
				if _, visible := visibleIDs[tokenID]; !visible {
					continue
				}
				tokenUsage[tokenID] = append(tokenUsage[tokenID], name)
			}
		}
	}

	if len(tokenUsage) > 0 {
		keys := make([]string, 0, len(tokenUsage))
		for tokenID := range tokenUsage {
			keys = append(keys, tokenID)
		}
		sort.Strings(keys)

		diag.Usage = make([]APITokenUsage, 0, len(keys))
		for _, tokenID := range keys {
			agents := tokenUsage[tokenID]
			sort.Strings(agents)
			diag.Usage = append(diag.Usage, APITokenUsage{
				TokenID:    tokenID,
				AgentCount: len(agents),
				Agents:     agents,
			})
		}
	}

	return diag
}

func buildDockerAgentDiagnostic(m *monitoring.Monitor, serverVersion string) *DockerAgentDiagnostic {
	if m == nil {
		return nil
	}

	readState := m.GetUnifiedReadStateOrSnapshot()
	if readState == nil {
		return nil
	}
	hosts := readState.DockerHosts()
	diag := &DockerAgentDiagnostic{
		AgentsTotal:             len(hosts),
		RecommendedAgentVersion: normalizeVersionLabel(serverVersion),
	}

	appendNote := func(note string) {
		if note == "" || contains(diag.Notes, note) {
			return
		}
		diag.Notes = append(diag.Notes, note)
	}

	if len(hosts) == 0 {
		appendNote("No Docker / Podman modules have reported in yet. Use Settings → Infrastructure to install pulse-agent with Docker / Podman monitoring and unlock remote commands.")
		return diag
	}

	var (
		serverVer        *updates.Version
		recommendedLabel = diag.RecommendedAgentVersion
	)
	if serverVersion != "" {
		if parsed, err := updates.ParseVersion(serverVersion); err == nil {
			serverVer = parsed
			recommendedLabel = normalizeVersionLabel(parsed.String())
			diag.RecommendedAgentVersion = recommendedLabel
		}
	}

	now := time.Now().UTC()
	legacyTokenHosts := 0
	for _, host := range hosts {
		status := strings.ToLower(strings.TrimSpace(string(host.Status())))
		if status == "online" {
			diag.AgentsOnline++
		}
		versionStr := strings.TrimSpace(host.AgentVersion())
		if versionStr != "" {
			diag.AgentsReportingVersion++
		} else {
			diag.AgentsWithoutVersion++
		}

		if strings.TrimSpace(host.TokenID()) != "" {
			diag.AgentsWithTokenBinding++
		} else {
			legacyTokenHosts++
		}

		issues := make([]string, 0, 4)

		if status != "online" && status != "" {
			issues = append(issues, fmt.Sprintf("Docker / Podman module reports status %q.", status))
		}

		if versionStr == "" {
			issues = append(issues, "Agent has not reported a version (pre v4.24). Reinstall using Settings → Infrastructure.")
		} else if serverVer != nil {
			if agentVer, err := updates.ParseVersion(versionStr); err == nil {
				if agentVer.Compare(serverVer) < 0 {
					diag.AgentsOutdatedVersion++
					issues = append(issues, fmt.Sprintf("Agent version %s lags behind the recommended %s. Re-run the installer to update.", normalizeVersionLabel(versionStr), recommendedLabel))
				}
			} else {
				issues = append(issues, fmt.Sprintf("Unrecognized agent version string %q. Reinstall to ensure command support.", versionStr))
			}
		}

		if strings.TrimSpace(host.TokenID()) == "" {
			issues = append(issues, "Docker / Podman module is still using the shared API token. Generate a dedicated token in Settings → Security and rerun the installer.")
		}

		if !host.LastSeen().IsZero() && now.Sub(host.LastSeen().UTC()) > 10*time.Minute {
			issues = append(issues, fmt.Sprintf("No heartbeat since %s. Verify the agent container is running.", host.LastSeen().UTC().Format(time.RFC3339)))
		}

		if command := host.Command(); command != nil {
			cmdStatus := strings.ToLower(strings.TrimSpace(command.Status))
			switch cmdStatus {
			case monitoring.DockerCommandStatusQueued, monitoring.DockerCommandStatusDispatched, monitoring.DockerCommandStatusAcknowledged:
				message := fmt.Sprintf("Command %s is still in progress.", cmdStatus)
				if !command.UpdatedAt.IsZero() && now.Sub(command.UpdatedAt.UTC()) > 15*time.Minute {
					diag.AgentsWithStaleCommand++
					message = fmt.Sprintf("Command %s has been pending since %s; consider allowing re-enrolment.", cmdStatus, command.UpdatedAt.UTC().Format(time.RFC3339))
				}
				issues = append(issues, message)
			}
		}

		if host.PendingUninstall() {
			diag.AgentsPendingUninstall++
			issues = append(issues, "Docker / Podman module is pending uninstall; confirm pulse-agent stopped reporting Docker / Podman telemetry or clear the flag.")
		}

		if len(issues) == 0 {
			continue
		}
		hostID := host.HostSourceID()
		if hostID == "" {
			hostID = host.ID()
		}

		diag.Attention = append(diag.Attention, DockerAgentAttention{
			AgentID:      hostID,
			Name:         preferredDockerHostName(host),
			Status:       string(host.Status()),
			AgentVersion: versionStr,
			TokenHint:    host.TokenHint(),
			LastSeen:     formatTimeMaybe(host.LastSeen()),
			Issues:       issues,
		})
	}

	diag.AgentsWithoutTokenBinding = legacyTokenHosts
	diag.AgentsNeedingAttention = len(diag.Attention)

	if legacyTokenHosts > 0 {
		appendNote(fmt.Sprintf("%s still %s on the shared API token. Migrate each module to a dedicated token via Settings → Security and rerun the installer.", dockerPodmanAgentCount(legacyTokenHosts), pluralVerb(legacyTokenHosts, "relies", "rely")))
	}
	if diag.AgentsOutdatedVersion > 0 {
		appendNote(fmt.Sprintf("%s %s out of date. Re-run the installer from Settings → Infrastructure to upgrade.", dockerPodmanAgentCount(diag.AgentsOutdatedVersion), pluralVerb(diag.AgentsOutdatedVersion, "is", "are")))
	}
	if diag.AgentsWithoutVersion > 0 {
		appendNote(fmt.Sprintf("%s %s not reported an agent version yet. Reinstall from Settings → Infrastructure to enable the new command system.", dockerPodmanAgentCount(diag.AgentsWithoutVersion), pluralVerb(diag.AgentsWithoutVersion, "has", "have")))
	}
	if diag.AgentsWithStaleCommand > 0 {
		appendNote(fmt.Sprintf("%s %s stuck. Use the 'Allow reconnect' action in Settings → Infrastructure to reset.", dockerPodmanAgentCommandCount(diag.AgentsWithStaleCommand), pluralVerb(diag.AgentsWithStaleCommand, "appears", "appear")))
	}
	if diag.AgentsPendingUninstall > 0 {
		appendNote(fmt.Sprintf("%s %s pending uninstall. Confirm the uninstall or clear the flag from Settings → Infrastructure.", dockerPodmanAgentCount(diag.AgentsPendingUninstall), pluralVerb(diag.AgentsPendingUninstall, "is", "are")))
	}
	if diag.AgentsNeedingAttention == 0 {
		appendNote("All Docker / Podman modules are reporting with dedicated tokens and the expected version.")
	}

	return diag
}

func dockerPodmanAgentCount(count int) string {
	if count == 1 {
		return "1 Docker / Podman module"
	}
	return fmt.Sprintf("%d Docker / Podman modules", count)
}

func dockerPodmanAgentCommandCount(count int) string {
	if count == 1 {
		return "1 Docker / Podman module command"
	}
	return fmt.Sprintf("%d Docker / Podman module commands", count)
}

func pluralVerb(count int, singular, plural string) string {
	if count == 1 {
		return singular
	}
	return plural
}

func buildAlertsDiagnostic(m *monitoring.Monitor) *AlertsDiagnostic {
	if m == nil {
		return nil
	}

	manager := m.GetAlertManager()
	if manager == nil {
		return nil
	}

	config := manager.GetConfig()
	diag := &AlertsDiagnostic{}

	appendNote := func(note string) {
		if note == "" || contains(diag.Notes, note) {
			return
		}
		diag.Notes = append(diag.Notes, note)
	}

	if config.Schedule.Cooldown <= 0 {
		diag.MissingCooldown = true
		appendNote("Alert cooldown is not configured. Set a cooldown under Alerts → Schedule to prevent alert storms.")
	}
	if config.Schedule.Grouping.Window <= 0 {
		diag.MissingGroupingWindow = true
		appendNote("Alert grouping window is disabled. Configure a grouping window to bundle related alerts.")
	}

	overrideKeys := make([]string, 0, len(config.Overrides))
	for key := range config.Overrides {
		overrideKeys = append(overrideKeys, key)
	}
	sort.Strings(overrideKeys)
	for _, key := range overrideKeys {
		diag.Overrides = append(diag.Overrides, summarizeAlertOverride(key, config.Overrides[key]))
	}

	return diag
}

func summarizeAlertOverride(key string, th alertconfig.ThresholdConfig) AlertsOverrideDiagnostic {
	out := AlertsOverrideDiagnostic{
		Key:                 key,
		Disabled:            th.Disabled,
		DisableConnectivity: th.DisableConnectivity,
	}

	thresholds := map[string]float64{}
	addThreshold := func(name string, t *alertconfig.HysteresisThreshold) {
		if t == nil {
			return
		}
		thresholds[name] = t.Trigger
	}
	addThreshold("cpu", th.CPU)
	addThreshold("memory", th.Memory)
	addThreshold("disk", th.Disk)
	addThreshold("diskRead", th.DiskRead)
	addThreshold("diskWrite", th.DiskWrite)
	addThreshold("networkIn", th.NetworkIn)
	addThreshold("networkOut", th.NetworkOut)
	addThreshold("usage", th.Usage)
	addThreshold("temperature", th.Temperature)
	addThreshold("diskTemperature", th.DiskTemperature)

	if len(thresholds) > 0 {
		out.Thresholds = thresholds
	}
	return out
}

func fingerprintPublicKey(pub string) (string, error) {
	pub = strings.TrimSpace(pub)
	if pub == "" {
		return "", fmt.Errorf("empty public key")
	}
	key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pub))
	if err != nil {
		return "", err
	}
	return ssh.FingerprintSHA256(key), nil
}

func resolveUserName(uid uint32) string {
	uidStr := strconv.FormatUint(uint64(uid), 10)
	if usr, err := user.LookupId(uidStr); err == nil && usr.Username != "" {
		return usr.Username
	}
	return "uid:" + uidStr
}

func resolveGroupName(gid uint32) string {
	gidStr := strconv.FormatUint(uint64(gid), 10)
	if grp, err := user.LookupGroupId(gidStr); err == nil && grp != nil && grp.Name != "" {
		return grp.Name
	}
	return "gid:" + gidStr
}

func countLegacySSHKeys(dir string) (int, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return 0, nil
		}
		return 0, err
	}
	count := 0
	for _, entry := range entries {
		if entry.IsDir() {
			continue
		}
		name := entry.Name()
		if strings.HasPrefix(name, "id_") {
			count++
		}
	}
	return count, nil
}

func preferredDockerHostName(host *unifiedresources.DockerHostView) string {
	if host == nil {
		return ""
	}
	if name := strings.TrimSpace(host.Name()); name != "" {
		return name
	}
	if name := strings.TrimSpace(host.Hostname()); name != "" {
		return name
	}
	if name := strings.TrimSpace(host.AgentID()); name != "" {
		return name
	}
	if name := strings.TrimSpace(host.HostSourceID()); name != "" {
		return name
	}
	return host.ID()
}

func formatTimeMaybe(t time.Time) string {
	if t.IsZero() {
		return ""
	}
	return t.UTC().Format(time.RFC3339)
}

func normalizeVersionLabel(raw string) string {
	value := strings.TrimSpace(raw)
	if value == "" {
		return ""
	}
	if strings.HasPrefix(value, "v") {
		return value
	}
	first := value[0]
	if first < '0' || first > '9' {
		return value
	}
	return "v" + value
}

// checkVMDiskMonitoring performs diagnostic checks for VM disk monitoring
func (r *Router) checkVMDiskMonitoring(ctx context.Context, client *proxmox.Client, _ string) *VMDiskCheckResult {
	result := &VMDiskCheckResult{
		Recommendations: []string{},
		Permissions:     []string{},
	}

	// Get all nodes to check
	nodes, err := client.GetNodes(ctx)
	if err != nil {
		log.Error().Err(err).Msg("VM disk check: failed to get nodes")
		result.TestResult = "Failed to get nodes"
		return result
	}

	if len(nodes) == 0 {
		result.TestResult = "No nodes found"
		return result
	}

	// Fetch VMs once per node and keep lookup map
	nodeVMMap := make(map[string][]proxmox.VM)
	var allVMs []proxmox.VM
	for _, node := range nodes {
		vmCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
		vms, err := client.GetVMs(vmCtx, node.Node)
		cancel()
		if err != nil {
			log.Debug().Err(err).Str("node", node.Node).Msg("Failed to get VMs from node")
			continue
		}
		nodeVMMap[node.Node] = vms
		allVMs = append(allVMs, vms...)
	}

	result.VMsFound = len(allVMs)
	vms := allVMs

	if len(vms) == 0 {
		result.TestResult = "No VMs found to test"
		result.Recommendations = append(result.Recommendations, "Create a test VM to verify disk monitoring")
		return result
	}

	// Check VMs for agent and disk data
	var testVM *proxmox.VM
	var testVMNode string
	result.ProblematicVMs = []VMDiskIssue{}
	for i := range vms {
		vm := vms[i]
		if vm.Template == 0 && vm.Status == "running" {
			vmNode := strings.TrimSpace(vm.Node)
			if vmNode == "" {
				continue
			}

			// Check if agent is configured
			statusCtx, statusCancel := context.WithTimeout(ctx, 10*time.Second)
			vmStatus, err := client.GetVMStatus(statusCtx, vmNode, vm.VMID)
			statusCancel()
			if err != nil {
				log.Error().Err(err).Int("vmid", vm.VMID).Msg("VM disk check: failed to get VM status")
				result.ProblematicVMs = append(result.ProblematicVMs, VMDiskIssue{
					VMID:   vm.VMID,
					Name:   vm.Name,
					Status: vm.Status,
					Issue:  "Failed to get VM status",
				})
			} else if vmStatus != nil && vmStatus.Agent.Value > 0 {
				result.VMsWithAgent++

				// Try to get filesystem info
				fsCtx, fsCancel := context.WithTimeout(ctx, 10*time.Second)
				fsInfo, err := client.GetVMFSInfo(fsCtx, vmNode, vm.VMID)
				fsCancel()
				if err != nil {
					result.ProblematicVMs = append(result.ProblematicVMs, VMDiskIssue{
						VMID:   vm.VMID,
						Name:   vm.Name,
						Status: vm.Status,
						Issue:  "Agent enabled but failed to get filesystem info",
					})
					if testVM == nil {
						testVM = &vms[i]
						testVMNode = vmNode
					}
				} else if len(fsInfo) == 0 {
					result.ProblematicVMs = append(result.ProblematicVMs, VMDiskIssue{
						VMID:   vm.VMID,
						Name:   vm.Name,
						Status: vm.Status,
						Issue:  "Agent returned no filesystem info",
					})
					if testVM == nil {
						testVM = &vms[i]
						testVMNode = vmNode
					}
				} else {
					// Check if we get usable disk data
					hasUsableFS := false
					for _, fs := range fsInfo {
						if fs.Type != "tmpfs" && fs.Type != "devtmpfs" &&
							!strings.HasPrefix(fs.Mountpoint, "/dev") &&
							!strings.HasPrefix(fs.Mountpoint, "/proc") &&
							!strings.HasPrefix(fs.Mountpoint, "/sys") &&
							fs.TotalBytes > 0 {
							hasUsableFS = true
							break
						}
					}

					if hasUsableFS {
						result.VMsWithDiskData++
					} else {
						result.ProblematicVMs = append(result.ProblematicVMs, VMDiskIssue{
							VMID:   vm.VMID,
							Name:   vm.Name,
							Status: vm.Status,
							Issue:  fmt.Sprintf("Agent returned %d filesystems but none are usable for disk metrics", len(fsInfo)),
						})
					}

					if testVM == nil {
						testVM = &vms[i]
						testVMNode = vmNode
					}
				}
			} else if vmStatus != nil {
				// Agent not enabled
				result.ProblematicVMs = append(result.ProblematicVMs, VMDiskIssue{
					VMID:   vm.VMID,
					Name:   vm.Name,
					Status: vm.Status,
					Issue:  "Guest agent not enabled in VM configuration",
				})
			}
		}
	}

	// Perform detailed test on one VM
	if testVM != nil {
		result.TestVMID = testVM.VMID
		result.TestVMName = testVM.Name

		// Check VM status for agent
		statusCtx, statusCancel := context.WithTimeout(ctx, 10*time.Second)
		vmStatus, err := client.GetVMStatus(statusCtx, testVMNode, testVM.VMID)
		statusCancel()
		if err != nil {
			errStr := err.Error()
			log.Error().Err(err).Int("vmid", testVM.VMID).Msg("VM disk check: failed to get VM status for test VM")
			result.TestResult = "Failed to get VM status"
			if errors.Is(err, context.DeadlineExceeded) || strings.Contains(errStr, "context deadline exceeded") {
				result.Recommendations = append(result.Recommendations,
					"VM status request timed out; check network connectivity to the node",
					"If this persists, increase the diagnostics timeout or reduce VM load during checks",
				)
			} else if strings.Contains(errStr, "403") || strings.Contains(errStr, "401") {
				result.Recommendations = append(result.Recommendations,
					"Ensure API token has PVEAuditor role for baseline access",
					"Add VM.GuestAgent.Audit (PVE 9) or VM.Monitor (PVE 8) privileges; Pulse setup adds these via the PulseMonitor role",
					"Include Sys.Audit when available for Ceph metrics",
				)
			} else {
				result.Recommendations = append(result.Recommendations,
					"Verify the node is reachable and API token is valid",
				)
			}
		} else if vmStatus == nil || vmStatus.Agent.Value == 0 {
			result.TestResult = "Guest agent not enabled in VM configuration"
			result.Recommendations = append(result.Recommendations,
				"Enable QEMU Guest Agent in VM Options",
				"Install qemu-guest-agent package in the VM")
		} else {
			// Try to get filesystem info
			fsCtx, fsCancel := context.WithTimeout(ctx, 10*time.Second)
			fsInfo, err := client.GetVMFSInfo(fsCtx, testVMNode, testVM.VMID)
			fsCancel()
			if err != nil {
				errStr := err.Error()
				if strings.Contains(errStr, "500") || strings.Contains(errStr, "not running") {
					result.TestResult = "Guest agent not running inside VM"
					result.Recommendations = append(result.Recommendations,
						"SSH into VM and run: systemctl status qemu-guest-agent",
						"If not installed: apt install qemu-guest-agent",
						"If installed but not running: systemctl start qemu-guest-agent")
				} else if strings.Contains(errStr, "403") || strings.Contains(errStr, "401") {
					result.TestResult = "Permission denied accessing guest agent"
					result.Recommendations = append(result.Recommendations,
						"Ensure API token has PVEAuditor role for baseline access",
						"Add VM.GuestAgent.Audit (PVE 9) or VM.Monitor (PVE 8) privileges; Pulse setup adds these via the PulseMonitor role",
						"Include Sys.Audit when available for Ceph metrics")
				} else if errors.Is(err, context.DeadlineExceeded) || strings.Contains(errStr, "context deadline exceeded") {
					result.TestResult = "Guest agent request timed out"
					result.Recommendations = append(result.Recommendations,
						"Ensure the VM responds to guest agent queries promptly",
						"Consider increasing the diagnostics timeout if the environment is large",
					)
				} else {
					log.Error().Err(err).Int("vmid", testVM.VMID).Msg("VM disk check: failed to get guest agent data")
					result.TestResult = "Failed to get guest agent data"
				}
			} else if len(fsInfo) == 0 {
				result.TestResult = "Guest agent returned no filesystem info"
				result.Recommendations = append(result.Recommendations,
					"Guest agent may need restart inside VM",
					"Check VM has mounted filesystems")
			} else {
				// Calculate disk usage from filesystem info
				var totalBytes, usedBytes uint64
				result.FilesystemsFound = []FilesystemDetail{}

				for _, fs := range fsInfo {
					fsDetail := FilesystemDetail{
						Mountpoint: fs.Mountpoint,
						Type:       fs.Type,
						Total:      fs.TotalBytes,
						Used:       fs.UsedBytes,
					}

					// Check if this filesystem should be filtered
					if fs.Type == "tmpfs" || fs.Type == "devtmpfs" {
						fsDetail.Filtered = true
						fsDetail.Reason = "Special filesystem type"
					} else if strings.HasPrefix(fs.Mountpoint, "/dev") ||
						strings.HasPrefix(fs.Mountpoint, "/proc") ||
						strings.HasPrefix(fs.Mountpoint, "/sys") ||
						strings.HasPrefix(fs.Mountpoint, "/run") ||
						fs.Mountpoint == "/boot/efi" {
						fsDetail.Filtered = true
						fsDetail.Reason = "System mount point"
					} else if fs.TotalBytes == 0 {
						fsDetail.Filtered = true
						fsDetail.Reason = "Zero total bytes"
					} else {
						// This filesystem counts toward disk usage
						totalBytes += fs.TotalBytes
						usedBytes += fs.UsedBytes
					}

					result.FilesystemsFound = append(result.FilesystemsFound, fsDetail)
				}

				if totalBytes > 0 {
					percent := float64(usedBytes) / float64(totalBytes) * 100
					result.TestResult = fmt.Sprintf("SUCCESS: Guest agent working! Disk usage: %.1f%% (%d/%d bytes)",
						percent, usedBytes, totalBytes)
				} else {
					result.TestResult = fmt.Sprintf("Guest agent returned %d filesystems but no usable disk data (all filtered out)", len(fsInfo))
				}
			}
		}
	} else {
		result.TestResult = "No running VMs found to test"
		result.Recommendations = append(result.Recommendations, "Start a VM to test disk monitoring")
	}

	// Add general recommendations based on results
	if result.VMsWithAgent > 0 && result.VMsWithDiskData == 0 {
		result.Recommendations = append(result.Recommendations,
			"Guest agent is configured but not providing disk data",
			"Check guest agent is running inside VMs",
			"Verify API token permissions")
	}

	return result
}

// checkPhysicalDisks performs diagnostic checks for physical disk detection
func (r *Router) checkPhysicalDisks(ctx context.Context, client *proxmox.Client, _ string) *PhysicalDiskCheck {
	result := &PhysicalDiskCheck{
		Recommendations: []string{},
		NodeResults:     []NodeDiskResult{},
	}

	// Get all nodes
	nodes, err := client.GetNodes(ctx)
	if err != nil {
		log.Error().Err(err).Msg("Physical disk check: failed to get nodes")
		result.TestResult = "Failed to get nodes"
		return result
	}

	result.NodesChecked = len(nodes)

	// Check each node for physical disks
	for _, node := range nodes {
		nodeResult := NodeDiskResult{
			NodeName: node.Node,
		}

		// Skip offline nodes
		if node.Status != "online" {
			nodeResult.Error = "Node is offline"
			result.NodeResults = append(result.NodeResults, nodeResult)
			continue
		}

		// Try to get disk list
		diskCtx, diskCancel := context.WithTimeout(ctx, 10*time.Second)
		disks, err := client.GetDisks(diskCtx, node.Node)
		diskCancel()
		if err != nil {
			errStr := err.Error()
			log.Error().Err(err).Str("node", node.Node).Msg("Physical disk check: failed to get disks")
			nodeResult.Error = "Failed to query disk information"

			// Provide specific recommendations based on error
			if strings.Contains(errStr, "401") || strings.Contains(errStr, "403") {
				nodeResult.APIResponse = "Permission denied"
				if !contains(result.Recommendations, "Check API token has sufficient permissions for disk monitoring") {
					result.Recommendations = append(result.Recommendations,
						"Check API token has sufficient permissions for disk monitoring",
						"Token needs at least PVEAuditor role on the node")
				}
			} else if errors.Is(err, context.DeadlineExceeded) || strings.Contains(errStr, "context deadline exceeded") {
				nodeResult.APIResponse = "Timeout"
				if !contains(result.Recommendations, "Disk query timed out; verify node connectivity and load") {
					result.Recommendations = append(result.Recommendations,
						"Disk query timed out; verify node connectivity and load",
						"Increase diagnostics timeout if nodes are slow to respond")
				}
			} else if strings.Contains(errStr, "404") || strings.Contains(errStr, "501") {
				nodeResult.APIResponse = "Endpoint not available"
				if !contains(result.Recommendations, "Node may be running older Proxmox version without disk API support") {
					result.Recommendations = append(result.Recommendations,
						"Node may be running older Proxmox version without disk API support",
						"Check if node is running on non-standard hardware (Raspberry Pi, etc)")
				}
			} else {
				nodeResult.APIResponse = "API error"
			}
		} else {
			nodeResult.DiskCount = len(disks)
			if len(disks) > 0 {
				result.NodesWithDisks++
				result.TotalDisks += len(disks)

				// List disk devices
				for _, disk := range disks {
					nodeResult.DiskDevices = append(nodeResult.DiskDevices, disk.DevPath)
				}
			} else {
				nodeResult.APIResponse = "Empty response (no traditional disks found)"
				// This could be normal for SD card/USB based systems
				if !contains(result.Recommendations, "Some nodes returned no disks - may be using SD cards or USB storage") {
					result.Recommendations = append(result.Recommendations,
						"Some nodes returned no disks - may be using SD cards or USB storage",
						"Proxmox disk API only returns SATA/NVMe/SAS disks, not SD cards")
				}
			}
		}

		result.NodeResults = append(result.NodeResults, nodeResult)
	}

	// Generate summary
	if result.NodesChecked == 0 {
		result.TestResult = "No nodes found to check"
	} else if result.NodesWithDisks == 0 {
		result.TestResult = fmt.Sprintf("Checked %d nodes, none returned physical disks", result.NodesChecked)
	} else {
		result.TestResult = fmt.Sprintf("Found %d disks across %d of %d nodes",
			result.TotalDisks, result.NodesWithDisks, result.NodesChecked)
	}

	return result
}

// Helper function to check if slice contains string
func contains(slice []string, str string) bool {
	for _, s := range slice {
		if s == str {
			return true
		}
	}
	return false
}

func containsFold(slice []string, candidate string) bool {
	target := strings.ToLower(strings.TrimSpace(candidate))
	if target == "" {
		return false
	}

	for _, s := range slice {
		if strings.ToLower(strings.TrimSpace(s)) == target {
			return true
		}
	}
	return false
}

func interfaceToStringSlice(value interface{}) []string {
	switch v := value.(type) {
	case []string:
		out := make([]string, len(v))
		copy(out, v)
		return out
	case []interface{}:
		result := make([]string, 0, len(v))
		for _, item := range v {
			if str, ok := item.(string); ok {
				result = append(result, str)
			}
		}
		return result
	default:
		return nil
	}
}

func buildAIChatDiagnostic(cfg *config.Config, aiHandler *AIHandler) *AIChatDiagnostic {
	if cfg == nil {
		return nil
	}

	diag := &AIChatDiagnostic{
		Enabled: false,
		Notes:   []string{},
	}

	// Calculate enabled state based on AI config
	// NOTE: aiHandler might be nil during early startup
	if aiHandler != nil {
		ctx := context.Background()
		aiCfg := aiHandler.GetAIConfig(ctx)
		if aiCfg != nil {
			diag.Enabled = aiCfg.Enabled
			diag.Model = aiCfg.GetChatModel()
		}

		svc := aiHandler.GetService(ctx)
		if svc != nil {
			diag.Running = svc.IsRunning()
			diag.Healthy = svc.IsRunning() // Consolidate for now

			// Get connection details
			baseURL := svc.GetBaseURL()
			if baseURL != "" {
				diag.URL = baseURL
				// Parse port from URL
				if parts := strings.Split(baseURL, ":"); len(parts) > 2 {
					if port, err := strconv.Atoi(parts[2]); err == nil {
						diag.Port = port
					}
				}
			}

			// Native Assistant diagnostics report runtime availability, not MCP transport.
			diag.AssistantRuntimeConnected = diag.Running

			if !diag.Running && diag.Enabled {
				diag.Notes = append(diag.Notes, "Pulse Assistant service is enabled but not running")
			}
		} else if diag.Enabled {
			diag.Notes = append(diag.Notes, "Pulse Assistant service is nil")
		}
	} else {
		diag.Notes = append(diag.Notes, "Pulse Assistant handler not initialized")
	}

	return diag
}