Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"net/netip"
"sort"
"strconv"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/fleethealth"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/platformsupport"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/internal/updates"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
internalauth "github.com/rcourtman/pulse-go-rewrite/pkg/auth"
"github.com/rs/zerolog/log"
)
const (
AgentFleetDiagnosticsSchemaVersion = 1
AgentFleetStatusHealthy = "healthy"
AgentFleetStatusWarning = "warning"
AgentFleetStatusCritical = "critical"
AgentFleetStatusRemoved = "removed"
AgentFleetReasonUpdateDisabled = "agent_update_disabled"
AgentFleetReasonUpdateFailed = "agent_update_failed"
AgentFleetReasonUpdateStateUnknown = "agent_update_state_unknown"
AgentFleetReasonUpdatePlatformUnknown = "agent_update_platform_unknown"
AgentFleetReasonUpdateStateUnverified = "agent_update_installer_state_unverified"
AgentFleetReasonModuleFailed = "agent_module_failed"
AgentFleetReasonModuleDegraded = "agent_module_degraded"
AgentFleetReasonPrivilegeHelperDegraded = "agent_privilege_helper_degraded"
AgentFleetReasonCredentialMissing = "agent_credential_missing"
AgentFleetReasonCredentialExpired = "agent_credential_expired"
AgentFleetReasonCredentialUnlisted = "agent_credential_registry_stale"
AgentFleetReasonExecScopeMissing = "agent_exec_scope_missing"
AgentFleetReasonExecScopeExcess = "agent_exec_scope_excess"
AgentFleetReasonCredentialScopeExcess = "agent_credential_scope_excess"
AgentFleetReasonDuplicateInstallation = "duplicate_host_agent_installation"
AgentFleetActionAllowReenroll = "allow_reenroll"
AgentFleetActionCopyUpgradeCommand = "copy_upgrade_command"
AgentFleetActionRepairAuthentication = "repair_authentication"
AgentFleetRepairModeHandoff = "handoff"
)
type AgentFleetDiagnostics struct {
SchemaVersion int `json:"schemaVersion"`
GeneratedAt int64 `json:"generatedAt"`
ServerVersion string `json:"serverVersion,omitempty"`
AgentUpdateTargetVersion string `json:"agentUpdateTargetVersion,omitempty"`
Summary AgentFleetDiagnosticSummary `json:"summary"`
Agents []AgentFleetAgentDiagnostic `json:"agents"`
}
type AgentFleetDiagnosticSummary struct {
Total int `json:"total"`
Healthy int `json:"healthy"`
Warning int `json:"warning"`
Critical int `json:"critical"`
Removed int `json:"removed"`
}
type AgentFleetAgentDiagnostic struct {
RowKey string `json:"rowKey"`
ID string `json:"id"`
AgentID string `json:"agentId,omitempty"`
ConnectionID string `json:"connectionId,omitempty"`
Name string `json:"name"`
Hostname string `json:"hostname,omitempty"`
Platform string `json:"platform,omitempty"`
OSName string `json:"osName,omitempty"`
OSVersion string `json:"osVersion,omitempty"`
KernelVersion string `json:"kernelVersion,omitempty"`
Architecture string `json:"architecture,omitempty"`
MachineIDFingerprint string `json:"machineIdFingerprint,omitempty"`
ReportIP string `json:"reportIp,omitempty"`
InterfaceAddresses []string `json:"interfaceAddresses,omitempty"`
Types []string `json:"types"`
Status string `json:"status"`
RawStatus string `json:"rawStatus,omitempty"`
LastSeen int64 `json:"lastSeen,omitempty"`
IntervalSeconds int `json:"intervalSeconds,omitempty"`
Version string `json:"version,omitempty"`
ProfileID string `json:"profileId,omitempty"`
ProfileName string `json:"profileName,omitempty"`
ProfileVersion int `json:"profileVersion,omitempty"`
DeployedProfileVersion int `json:"deployedProfileVersion,omitempty"`
AgentUpdate *AgentFleetDiagnosticUpdate `json:"agentUpdate,omitempty"`
AgentModules []AgentFleetDiagnosticModule `json:"agentModules,omitempty"`
Privilege *AgentFleetDiagnosticPrivilege `json:"privilege,omitempty"`
Reasons []AgentFleetDiagnosticReason `json:"reasons"`
RepairActions []AgentFleetDiagnosticRepair `json:"repairActions,omitempty"`
}
type AgentFleetDiagnosticUpdate struct {
State string `json:"state"`
AutoUpdate bool `json:"autoUpdate"`
UpdatedFrom string `json:"updatedFrom,omitempty"`
AvailableVersion string `json:"availableVersion,omitempty"`
LastCheckedAt *time.Time `json:"lastCheckedAt,omitempty"`
LastAttemptAt *time.Time `json:"lastAttemptAt,omitempty"`
LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
LastError string `json:"lastError,omitempty"`
}
type AgentFleetDiagnosticModule struct {
Name string `json:"name"`
Enabled bool `json:"enabled"`
State string `json:"state"`
LastError string `json:"lastError,omitempty"`
UpdatedAt time.Time `json:"updatedAt"`
}
// AgentFleetDiagnosticPrivilege combines the collector-reported privilege
// profile with server-observed action-runner connection state. It is surfaced
// descriptively beside modules and update state. It is deliberately not a
// Reason: a least-privilege install or a disconnected optional runner must
// never degrade monitoring health by itself.
type AgentFleetDiagnosticPrivilege struct {
RunningAsRoot bool `json:"runningAsRoot"`
ServiceUser string `json:"serviceUser,omitempty"`
CommandAuthority string `json:"commandAuthority,omitempty"`
CredentialKnown bool `json:"credentialKnown,omitempty"`
CredentialExec bool `json:"credentialExec,omitempty"`
TypedHelper bool `json:"typedHelper,omitempty"`
SmartctlHelper bool `json:"smartctlHelper,omitempty"`
PctHelper bool `json:"pctHelper,omitempty"`
ActionRunnerCredentialIssued bool `json:"actionRunnerCredentialIssued,omitempty"`
ActionRunnerCredentialActive bool `json:"actionRunnerCredentialActive,omitempty"`
ActionRunnerRuntimeRole string `json:"actionRunnerRuntimeRole,omitempty"`
ActionRunnerCapability string `json:"actionRunnerCapability,omitempty"`
ActionRunnerBindingVersion string `json:"actionRunnerBindingVersion,omitempty"`
ActionRunnerConnected bool `json:"actionRunnerConnected,omitempty"`
ActionRunnerVersion string `json:"actionRunnerVersion,omitempty"`
ActionRunnerConnectedAt int64 `json:"actionRunnerConnectedAt,omitempty"`
ActionRunnerReceiptProtocol int `json:"actionRunnerReceiptProtocol,omitempty"`
ActionRunnerPreflightProtocol int `json:"actionRunnerPreflightProtocol,omitempty"`
ActionRunnerDockerObservationProtocol int `json:"actionRunnerDockerObservationProtocol,omitempty"`
}
type AgentFleetDiagnosticReason struct {
Code string `json:"code"`
Severity string `json:"severity"`
Message string `json:"message"`
Evidence []string `json:"evidence,omitempty"`
}
type AgentFleetDiagnosticRepair struct {
Code string `json:"code"`
Label string `json:"label"`
Description string `json:"description"`
Supported bool `json:"supported"`
Mode string `json:"mode"`
Platform string `json:"platform,omitempty"`
Scope string `json:"scope,omitempty"`
}
type agentFleetSubject struct {
rowKey string
id string
agentID string
name string
hostname string
types map[string]struct{}
rawStatus string
lastSeen time.Time
intervalSeconds int
version string
tokenID string
host *models.Host
docker *models.DockerHost
kubernetes *models.KubernetesCluster
removed bool
removedAt time.Time
removedPlatform string // last-known reported platform retained on the removed record
}
type agentFleetTokenInventory struct {
known bool
active map[string]config.APITokenRecord
expired map[string]struct{}
}
// GetAgentFleetDiagnostics preserves the original call contract for callers
// where the server version is also the agent update target.
func (m *Monitor) GetAgentFleetDiagnostics(serverVersion string, now time.Time) AgentFleetDiagnostics {
return m.GetAgentFleetDiagnosticsForTarget(serverVersion, serverVersion, now)
}
// GetAgentFleetDiagnosticsForTarget returns a read-only fleet health view
// derived from reported state, the canonical agent update target, and existing
// profile deployment state.
func (m *Monitor) GetAgentFleetDiagnosticsForTarget(serverVersion, agentUpdateTargetVersion string, now time.Time) AgentFleetDiagnostics {
if now.IsZero() {
now = time.Now().UTC()
}
now = now.UTC()
out := AgentFleetDiagnostics{
SchemaVersion: AgentFleetDiagnosticsSchemaVersion,
GeneratedAt: now.UnixMilli(),
ServerVersion: strings.TrimSpace(serverVersion),
AgentUpdateTargetVersion: strings.TrimSpace(agentUpdateTargetVersion),
}
if m == nil || m.state == nil {
return out
}
state := m.GetState()
tokenInventory := m.agentFleetTokenInventory(now)
profiles, assignments, deployments := m.agentFleetProfileState()
profileByID := mapProfilesByID(profiles)
assignmentByAgent := mapAssignmentsByAgent(assignments)
deploymentByAgentProfile := mapDeploymentsByAgentProfile(deployments)
subjects := buildAgentFleetSubjects(state)
for i := range subjects {
diagnostic := diagnoseAgentFleetSubject(subjects[i], state, out.AgentUpdateTargetVersion, now, tokenInventory, profileByID, assignmentByAgent, deploymentByAgentProfile)
out.Agents = append(out.Agents, diagnostic)
}
sort.Slice(out.Agents, func(i, j int) bool {
if out.Agents[i].Status != out.Agents[j].Status {
return agentFleetStatusRank(out.Agents[i].Status) > agentFleetStatusRank(out.Agents[j].Status)
}
return strings.ToLower(out.Agents[i].Name) < strings.ToLower(out.Agents[j].Name)
})
out.Summary.Total = len(out.Agents)
for _, agent := range out.Agents {
switch agent.Status {
case AgentFleetStatusCritical:
out.Summary.Critical++
case AgentFleetStatusWarning:
out.Summary.Warning++
case AgentFleetStatusRemoved:
out.Summary.Removed++
default:
out.Summary.Healthy++
}
}
return out
}
func (m *Monitor) agentFleetTokenInventory(now time.Time) agentFleetTokenInventory {
if m == nil || m.config == nil || mock.IsMockEnabled() {
return agentFleetTokenInventory{}
}
inventory := agentFleetTokenInventory{
known: true,
active: make(map[string]config.APITokenRecord),
expired: make(map[string]struct{}),
}
for _, record := range m.APITokenSnapshot() {
id := strings.TrimSpace(record.ID)
if id == "" {
continue
}
if record.ExpiresAt != nil && !record.ExpiresAt.After(now) {
inventory.expired[id] = struct{}{}
continue
}
inventory.active[id] = record
}
return inventory
}
func (m *Monitor) agentFleetProfileState() ([]models.AgentProfile, []models.AgentProfileAssignment, []models.ProfileDeploymentStatus) {
if m == nil || m.persistence == nil {
return nil, nil, nil
}
profiles, assignments := m.getAgentProfileCache()
deployments, err := m.persistence.LoadProfileDeploymentStatus()
if err != nil {
log.Warn().Err(err).Msg("Failed to load agent profile deployment status for fleet diagnostics")
}
return profiles, assignments, deployments
}
func buildAgentFleetSubjects(state models.StateSnapshot) []agentFleetSubject {
subjectsByID := make(map[string]*agentFleetSubject)
order := make([]string, 0, len(state.Hosts)+len(state.DockerHosts)+len(state.KubernetesClusters))
getOrCreate := func(id string) *agentFleetSubject {
id = strings.TrimSpace(id)
if existing := subjectsByID[id]; existing != nil {
return existing
}
subject := &agentFleetSubject{
rowKey: "agent-" + id,
id: id,
types: map[string]struct{}{},
}
subjectsByID[id] = subject
order = append(order, id)
return subject
}
for i := range state.Hosts {
host := state.Hosts[i]
if strings.TrimSpace(host.ID) == "" {
continue
}
subject := getOrCreate(host.ID)
subject.host = &host
subject.types["host"] = struct{}{}
subject.agentID = firstNonEmpty(subject.agentID, host.ID)
subject.name = firstNonEmpty(host.DisplayName, host.Hostname, host.ID)
subject.hostname = firstNonEmpty(host.Hostname, subject.hostname)
subject.rawStatus = firstNonEmpty(host.Status, subject.rawStatus)
subject.lastSeen = newestTime(subject.lastSeen, host.LastSeen)
subject.intervalSeconds = maxInt(subject.intervalSeconds, host.IntervalSeconds)
subject.version = firstNonEmpty(host.AgentVersion, subject.version)
subject.tokenID = firstNonEmpty(host.TokenID, subject.tokenID)
}
for i := range state.DockerHosts {
docker := state.DockerHosts[i]
if strings.TrimSpace(docker.ID) == "" || docker.Hidden {
continue
}
subject := getOrCreate(docker.ID)
subject.docker = &docker
subject.types["docker"] = struct{}{}
subject.agentID = firstNonEmpty(subject.agentID, docker.AgentID, docker.ID)
subject.name = firstNonEmpty(docker.CustomDisplayName, docker.DisplayName, docker.Hostname, docker.ID, subject.name)
subject.hostname = firstNonEmpty(docker.Hostname, subject.hostname)
subject.rawStatus = firstNonEmpty(docker.Status, subject.rawStatus)
subject.lastSeen = newestTime(subject.lastSeen, docker.LastSeen)
subject.intervalSeconds = maxInt(subject.intervalSeconds, docker.IntervalSeconds)
subject.version = firstNonEmpty(subject.version, docker.AgentVersion, docker.DockerVersion)
subject.tokenID = firstNonEmpty(subject.tokenID, docker.TokenID)
}
for i := range state.KubernetesClusters {
cluster := state.KubernetesClusters[i]
if strings.TrimSpace(cluster.ID) == "" || cluster.Hidden {
continue
}
id := "k8s:" + cluster.ID
subject := &agentFleetSubject{
rowKey: "k8s-" + cluster.ID,
id: cluster.ID,
agentID: firstNonEmpty(cluster.AgentID, cluster.ID),
name: firstNonEmpty(cluster.CustomDisplayName, cluster.DisplayName, cluster.Name, cluster.ID),
types: map[string]struct{}{"kubernetes": {}},
rawStatus: cluster.Status,
lastSeen: cluster.LastSeen,
intervalSeconds: cluster.IntervalSeconds,
version: firstNonEmpty(cluster.AgentVersion, cluster.Version),
tokenID: cluster.TokenID,
kubernetes: &cluster,
}
subjectsByID[id] = subject
order = append(order, id)
}
for i := range state.RemovedDockerHosts {
removed := state.RemovedDockerHosts[i]
subjectsByID["removed-docker:"+removed.ID] = &agentFleetSubject{
rowKey: "removed-docker-" + removed.ID,
id: removed.ID,
name: firstNonEmpty(removed.DisplayName, removed.Hostname, removed.ID),
hostname: removed.Hostname,
types: map[string]struct{}{"docker": {}},
removed: true,
removedAt: removed.RemovedAt,
removedPlatform: removed.Platform,
}
order = append(order, "removed-docker:"+removed.ID)
}
for i := range state.RemovedHostAgents {
removed := state.RemovedHostAgents[i]
subjectsByID["removed-host:"+removed.ID] = &agentFleetSubject{
rowKey: "removed-host-" + removed.ID,
id: removed.ID,
name: firstNonEmpty(removed.DisplayName, removed.Hostname, removed.ID),
hostname: removed.Hostname,
types: map[string]struct{}{"host": {}},
removed: true,
removedAt: removed.RemovedAt,
removedPlatform: removed.Platform,
}
order = append(order, "removed-host:"+removed.ID)
}
for i := range state.RemovedKubernetesClusters {
removed := state.RemovedKubernetesClusters[i]
subjectsByID["removed-k8s:"+removed.ID] = &agentFleetSubject{
rowKey: "removed-k8s-" + removed.ID,
id: removed.ID,
name: firstNonEmpty(removed.DisplayName, removed.Name, removed.ID),
types: map[string]struct{}{"kubernetes": {}},
removed: true,
removedAt: removed.RemovedAt,
}
order = append(order, "removed-k8s:"+removed.ID)
}
subjects := make([]agentFleetSubject, 0, len(order))
for _, key := range order {
if subject := subjectsByID[key]; subject != nil {
subjects = append(subjects, *subject)
}
}
return subjects
}
func diagnoseAgentFleetSubject(
subject agentFleetSubject,
state models.StateSnapshot,
serverVersion string,
now time.Time,
tokenInventory agentFleetTokenInventory,
profileByID map[string]models.AgentProfile,
assignmentByAgent map[string]models.AgentProfileAssignment,
deploymentByAgentProfile map[string]models.ProfileDeploymentStatus,
) AgentFleetAgentDiagnostic {
identity := agentFleetIdentityForSubject(subject)
result := AgentFleetAgentDiagnostic{
RowKey: subject.rowKey,
ID: subject.id,
AgentID: subject.agentID,
ConnectionID: identity.connectionID,
Name: firstNonEmpty(subject.name, subject.hostname, subject.id),
Hostname: subject.hostname,
Platform: identity.platform,
OSName: identity.osName,
OSVersion: identity.osVersion,
KernelVersion: identity.kernelVersion,
Architecture: identity.architecture,
MachineIDFingerprint: identity.machineIDFingerprint,
ReportIP: identity.reportIP,
InterfaceAddresses: identity.interfaceAddresses,
Types: sortedAgentTypes(subject.types),
RawStatus: subject.rawStatus,
IntervalSeconds: subject.intervalSeconds,
Version: subject.version,
AgentUpdate: agentFleetUpdateForSubject(subject),
AgentModules: agentFleetModulesForSubject(subject),
Privilege: agentFleetPrivilegeForSubject(subject, tokenInventory),
}
if !subject.lastSeen.IsZero() {
result.LastSeen = subject.lastSeen.UnixMilli()
}
if subject.removed {
result.Status = AgentFleetStatusRemoved
if !subject.removedAt.IsZero() {
result.LastSeen = subject.removedAt.UnixMilli()
}
result.Reasons = append(result.Reasons, AgentFleetDiagnosticReason{
Code: "agent_removed_blocked",
Severity: AgentFleetStatusWarning,
Message: "This agent is intentionally removed and blocked from re-enrolling until an admin allows it.",
Evidence: []string{
fmt.Sprintf("Removed at %s", subject.removedAt.UTC().Format(time.RFC3339)),
},
})
result.RepairActions = append(result.RepairActions, AgentFleetDiagnosticRepair{
Code: AgentFleetActionAllowReenroll,
Label: "Allow re-enroll",
Description: "Uses the existing allow re-enroll action for removed agents.",
Supported: true,
Mode: AgentFleetRepairModeHandoff,
Scope: "settings:write",
})
return result
}
result.Reasons = append(result.Reasons, diagnoseAgentConnectivity(subject, now)...)
result.Reasons = append(result.Reasons, diagnoseAgentCredential(subject, tokenInventory, now)...)
result.Reasons = append(result.Reasons, diagnoseAgentVersion(subject, serverVersion)...)
result.Reasons = append(result.Reasons, diagnoseAgentUpdate(subject, serverVersion)...)
result.Reasons = append(result.Reasons, diagnoseAgentModules(subject)...)
result.Reasons = append(result.Reasons, diagnoseDuplicateHostAgentInstallation(subject, state)...)
result.Reasons = append(result.Reasons, diagnoseAgentIdentitySplit(subject, state)...)
assignment, hasAssignment := findAgentAssignment(subject, assignmentByAgent)
if hasAssignment {
result.ProfileID = assignment.ProfileID
profile, hasProfile := profileByID[assignment.ProfileID]
if hasProfile {
result.ProfileName = profile.Name
result.ProfileVersion = expectedProfileVersion(profile, assignment)
result.Reasons = append(result.Reasons, diagnoseProfileDeployment(subject, assignment, profile, deploymentByAgentProfile)...)
result.Reasons = append(result.Reasons, diagnoseProfileCapabilityDrift(subject, state, profile)...)
} else {
result.Reasons = append(result.Reasons, AgentFleetDiagnosticReason{
Code: "profile_missing",
Severity: AgentFleetStatusWarning,
Message: "A profile is assigned to this agent, but that profile no longer exists.",
Evidence: []string{
"Assigned profile ID: " + assignment.ProfileID,
},
})
}
if deployment, ok := deploymentByAgentProfile[deploymentKey(assignment.AgentID, assignment.ProfileID)]; ok {
result.DeployedProfileVersion = deployment.DeployedVersion
}
}
if diagnosticHasAnyReason(
result.Reasons,
AgentFleetReasonCredentialMissing,
AgentFleetReasonCredentialExpired,
AgentFleetReasonCredentialScopeExcess,
AgentFleetReasonExecScopeMissing,
) {
platform, supported := safeAgentUpdatePlatform(subject)
if platform == platformsupport.RuntimePlatformFreeBSD || diagnosticHasAnyReason(result.Reasons, AgentFleetReasonDuplicateInstallation) {
supported = false
}
result.RepairActions = append(result.RepairActions, AgentFleetDiagnosticRepair{
Code: AgentFleetActionRepairAuthentication,
Label: "Repair authentication",
Description: "Generate a fresh scoped agent credential and run the host-local repair command for this installation.",
Supported: supported,
Mode: AgentFleetRepairModeHandoff,
Platform: platform,
Scope: "settings:write + local_admin_shell",
})
}
for _, reason := range result.Reasons {
if reason.Code == "agent_version_stale" {
platform, supported := safeAgentUpdatePlatform(subject)
if !supported {
result.Reasons = append(result.Reasons, AgentFleetDiagnosticReason{
Code: AgentFleetReasonUpdatePlatformUnknown,
Severity: AgentFleetStatusWarning,
Message: "Pulse cannot safely choose an agent installer command because the reported platform is missing or unsupported.",
})
} else if platform == platformsupport.RuntimePlatformFreeBSD {
supported = false
result.Reasons = append(result.Reasons, AgentFleetDiagnosticReason{
Code: AgentFleetReasonUpdateStateUnverified,
Severity: AgentFleetStatusWarning,
Message: "Pulse cannot verify the saved FreeBSD or pfSense installer state required for a safe in-place update.",
})
}
if diagnosticHasAnyReason(result.Reasons, AgentFleetReasonDuplicateInstallation) {
supported = false
}
result.RepairActions = append(result.RepairActions, AgentFleetDiagnosticRepair{
Code: AgentFleetActionCopyUpgradeCommand,
Label: "Copy upgrade command",
Description: "Uses the existing installer command from Settings -> Agents; no remote command is queued.",
Supported: supported,
Mode: AgentFleetRepairModeHandoff,
Platform: platform,
Scope: "local_admin_shell",
})
break
}
}
result.Status = diagnosticStatusFromReasons(result.Reasons)
return result
}
func diagnosticHasAnyReason(reasons []AgentFleetDiagnosticReason, codes ...string) bool {
for _, reason := range reasons {
for _, code := range codes {
if reason.Code == code {
return true
}
}
}
return false
}
func diagnoseAgentCredential(subject agentFleetSubject, inventory agentFleetTokenInventory, now time.Time) []AgentFleetDiagnosticReason {
tokenID := strings.TrimSpace(subject.tokenID)
if subject.removed || !inventory.known || tokenID == "" {
return nil
}
if record, ok := inventory.active[tokenID]; ok {
reasons := make([]AgentFleetDiagnosticReason, 0, 3)
role := strings.TrimSpace(record.Metadata[internalauth.RuntimeRoleMetadataKey])
if unexpected := internalauth.UnexpectedScopes(role, record.Scopes); len(unexpected) > 0 {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: AgentFleetReasonCredentialScopeExcess,
Severity: AgentFleetStatusCritical,
Message: "This collector credential carries authority outside its runtime role allowlist.",
Evidence: []string{
"Credential runtime role: " + role,
"Unexpected scopes: " + strings.Join(unexpected, ", "),
},
})
}
if subject.host != nil && subject.host.CommandsEnabled && !record.HasScope(config.ScopeAgentExec) {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: AgentFleetReasonExecScopeMissing,
Severity: AgentFleetStatusCritical,
Message: "The agent is configured to accept Pulse commands, but its credential does not grant command execution.",
Evidence: []string{
"Agent command execution: enabled",
"Credential scope: agent:exec missing",
},
})
}
if subject.host != nil && subject.host.AgentPrivilege != nil {
authority := strings.TrimSpace(subject.host.AgentPrivilege.CommandAuthority)
hasExec := record.HasScope(config.ScopeAgentExec)
switch {
case authority == "monitoring-only" && hasExec:
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: AgentFleetReasonExecScopeExcess,
Severity: AgentFleetStatusWarning,
Message: "This monitoring-only runtime still uses a credential that grants command execution.",
Evidence: []string{"Local command authority: monitoring-only", "Credential scope: agent:exec present"},
})
case authority == "command-capable" && !hasExec && !subject.host.CommandsEnabled:
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: AgentFleetReasonExecScopeMissing,
Severity: AgentFleetStatusWarning,
Message: "This runtime was installed as command-capable, but its credential cannot re-enable commands.",
Evidence: []string{"Local command authority: command-capable", "Credential scope: agent:exec missing"},
})
}
}
return reasons
}
if _, ok := inventory.expired[tokenID]; ok {
return []AgentFleetDiagnosticReason{{
Code: AgentFleetReasonCredentialExpired,
Severity: AgentFleetStatusCritical,
Message: "The credential last used by this agent has expired and can no longer authenticate.",
Evidence: []string{"Credential status: expired", "Credential id: " + tokenID},
}}
}
// The row itself proves whether the credential still authenticates. If it
// was used moments ago, a missing inventory entry means the server's token
// registry view is stale, not that the agent lost access (#1730), and the
// repair-authentication loop cannot fix that. Only a same-token row can
// vouch, so a fresh sibling row on a different credential never rescues a
// genuinely revoked one.
if lastUsed := agentFleetCredentialLastUsed(subject, tokenID); lastUsed != nil &&
now.Sub(*lastUsed) <= agentFleetCredentialFreshnessWindow(subject) {
return []AgentFleetDiagnosticReason{{
Code: AgentFleetReasonCredentialUnlisted,
Severity: AgentFleetStatusWarning,
Message: "The credential last used by this agent authenticated recently but is missing from this server's token registry view.",
Evidence: []string{
"Credential status: authenticating but unlisted",
"Credential id: " + tokenID,
"Credential last authenticated: " + lastUsed.UTC().Format(time.RFC3339),
},
}}
}
// The judged id is the token record UUID this host row last reported
// with, not secret material. Without it an operator cannot tell which
// credential the verdict is about, so a host that re-enrolled onto a
// fresh token but left a stale row behind reads as a live outage (#1730).
return []AgentFleetDiagnosticReason{{
Code: AgentFleetReasonCredentialMissing,
Severity: AgentFleetStatusCritical,
Message: "The credential last used by this agent no longer exists on this Pulse server.",
Evidence: []string{"Credential status: missing or revoked", "Credential id: " + tokenID},
}}
}
// agentFleetCredentialLastUsed returns the newest time the judged credential
// authenticated according to the subject's own rows. Only rows that reported
// with the same token id count.
func agentFleetCredentialLastUsed(subject agentFleetSubject, tokenID string) *time.Time {
var newest *time.Time
consider := func(rowTokenID string, lastUsed *time.Time) {
if strings.TrimSpace(rowTokenID) != tokenID || lastUsed == nil || lastUsed.IsZero() {
return
}
if newest == nil || lastUsed.After(*newest) {
t := *lastUsed
newest = &t
}
}
if subject.host != nil {
consider(subject.host.TokenID, subject.host.TokenLastUsedAt)
}
if subject.docker != nil {
consider(subject.docker.TokenID, subject.docker.TokenLastUsedAt)
}
if subject.kubernetes != nil {
consider(subject.kubernetes.TokenID, subject.kubernetes.TokenLastUsedAt)
}
return newest
}
// agentFleetCredentialFreshnessWindow bounds how recent an authentication has
// to be before it outweighs a missing token registry entry. Slow-reporting
// agents get a window scaled to their interval so a normal gap between
// reports never re-opens the false outage.
func agentFleetCredentialFreshnessWindow(subject agentFleetSubject) time.Duration {
window := 10 * time.Minute
if interval := time.Duration(subject.intervalSeconds) * time.Second; interval*3 > window {
window = interval * 3
}
return window
}
func diagnoseDuplicateHostAgentInstallation(subject agentFleetSubject, state models.StateSnapshot) []AgentFleetDiagnosticReason {
if subject.removed || subject.host == nil {
return nil
}
machineID := strings.TrimSpace(subject.host.MachineID)
hostname := strings.TrimSpace(subject.host.Hostname)
if machineID == "" || hostname == "" {
return nil
}
for _, peer := range state.Hosts {
if strings.TrimSpace(peer.ID) == strings.TrimSpace(subject.host.ID) ||
strings.TrimSpace(peer.MachineID) != machineID ||
!unifiedresources.HostnamesEquivalent(peer.Hostname, hostname) {
continue
}
evidence := []string{
"Peer agent ID: " + strings.TrimSpace(peer.ID),
"Peer hostname: " + strings.TrimSpace(peer.Hostname),
}
if version := strings.TrimSpace(peer.AgentVersion); version != "" {
evidence = append(evidence, "Peer version: "+version)
}
if !peer.LastSeen.IsZero() {
evidence = append(evidence, "Peer last seen: "+peer.LastSeen.UTC().Format(time.RFC3339))
}
return []AgentFleetDiagnosticReason{{
Code: AgentFleetReasonDuplicateInstallation,
Severity: AgentFleetStatusCritical,
Message: "Multiple host-installed Pulse Agents are reporting from the same machine. A generic update command cannot safely choose which local service to change.",
Evidence: evidence,
}}
}
return nil
}
func diagnoseAgentConnectivity(subject agentFleetSubject, now time.Time) []AgentFleetDiagnosticReason {
if subject.kubernetes == nil && subject.host == nil && subject.docker == nil {
return nil
}
reasons := []AgentFleetDiagnosticReason{}
interval := subject.intervalSeconds
if interval <= 0 {
interval = 30
}
staleThreshold := fleethealth.AgentStaleThreshold(subject.intervalSeconds)
switch fleethealth.DeriveAgentLiveness(subject.lastSeen, now, subject.intervalSeconds) {
case fleethealth.AgentLivenessPending:
return append(reasons, AgentFleetDiagnosticReason{
Code: "agent_never_reported",
Severity: AgentFleetStatusCritical,
Message: "This agent has no last-seen timestamp, so Pulse cannot confirm it is reporting.",
})
case fleethealth.AgentLivenessStale:
age := now.Sub(subject.lastSeen)
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: "agent_disconnected",
Severity: AgentFleetStatusCritical,
Message: fmt.Sprintf("No report has arrived for %s. Pulse marks an agent stale after %s without a report.", formatFleetDuration(age), formatFleetDuration(staleThreshold)),
Evidence: []string{
"Last seen: " + subject.lastSeen.UTC().Format(time.RFC3339),
fmt.Sprintf("Expected report interval: %ds", interval),
},
})
}
if subject.rawStatus != "" && !agentFleetStatusIsOnline(subject.rawStatus) {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: "agent_status_not_online",
Severity: AgentFleetStatusWarning,
Message: fmt.Sprintf("The agent's most recent report described it as %q rather than online.", subject.rawStatus),
})
}
return reasons
}
func diagnoseAgentVersion(subject agentFleetSubject, targetVersion string) []AgentFleetDiagnosticReason {
if subject.removed {
return nil
}
agentVersion := strings.TrimSpace(subject.version)
if agentVersion == "" {
return []AgentFleetDiagnosticReason{{
Code: "agent_version_missing",
Severity: AgentFleetStatusWarning,
Message: "The agent did not report a version, so Pulse cannot verify update health.",
}}
}
targetVersion = strings.TrimSpace(targetVersion)
if targetVersion == "" || strings.EqualFold(targetVersion, "dev") {
return nil
}
if _, err := updates.ParseVersion(targetVersion); err != nil {
return nil
}
if _, err := updates.ParseVersion(agentVersion); err != nil {
return []AgentFleetDiagnosticReason{{
Code: "agent_version_unparseable",
Severity: AgentFleetStatusWarning,
Message: fmt.Sprintf("The agent reported version %q, which cannot be compared with update target %q.", agentVersion, targetVersion),
}}
}
if fleethealth.DeriveAgentVersionDrift(agentVersion, targetVersion) == fleethealth.AgentVersionBehind {
return []AgentFleetDiagnosticReason{{
Code: "agent_version_stale",
Severity: AgentFleetStatusWarning,
Message: fmt.Sprintf("Agent version %s is older than the agent update target %s.", agentVersion, targetVersion),
Evidence: []string{
"Agent version: " + agentVersion,
"Agent update target: " + targetVersion,
},
}}
}
return nil
}
func diagnoseAgentUpdate(subject agentFleetSubject, targetVersion string) []AgentFleetDiagnosticReason {
if subject.host == nil || subject.host.AgentUpdate == nil {
return nil
}
update := subject.host.AgentUpdate
state := strings.ToLower(strings.TrimSpace(update.State))
evidence := updateStatusEvidence(update)
switch state {
case "", "idle", "checking", "update-available", "updating":
return nil
case "disabled":
if fleethealth.DeriveAgentVersionDrift(subject.version, targetVersion) != fleethealth.AgentVersionBehind {
return nil
}
return []AgentFleetDiagnosticReason{{
Code: AgentFleetReasonUpdateDisabled,
Severity: AgentFleetStatusWarning,
Message: "The agent is behind the update target and its automatic updater is disabled.",
Evidence: evidence,
}}
case "error":
return []AgentFleetDiagnosticReason{{
Code: AgentFleetReasonUpdateFailed,
Severity: AgentFleetStatusWarning,
Message: "The agent's most recent self-update check or attempt failed.",
Evidence: evidence,
}}
default:
return []AgentFleetDiagnosticReason{{
Code: AgentFleetReasonUpdateStateUnknown,
Severity: AgentFleetStatusWarning,
Message: fmt.Sprintf("The agent reported an unrecognized updater state %q.", update.State),
Evidence: evidence,
}}
}
}
func diagnoseAgentModules(subject agentFleetSubject) []AgentFleetDiagnosticReason {
reasons := make([]AgentFleetDiagnosticReason, 0)
for _, module := range agentFleetModuleStatuses(subject) {
if !module.Enabled {
continue
}
name := strings.TrimSpace(module.Name)
state := strings.ToLower(strings.TrimSpace(module.State))
if state == "running" {
continue
}
severity := AgentFleetStatusWarning
code := AgentFleetReasonModuleDegraded
message := fmt.Sprintf("Enabled agent module %q is not running.", name)
if name == agentshost.ModuleNameTypedPrivilegeHelper {
code = AgentFleetReasonPrivilegeHelperDegraded
message = "The typed privilege helper is degraded; affected privileged telemetry was omitted without widening collector privilege."
}
if state == "error" || state == "failed" {
severity = AgentFleetStatusCritical
code = AgentFleetReasonModuleFailed
message = fmt.Sprintf("Enabled agent module %q failed.", name)
if name == agentshost.ModuleNameTypedPrivilegeHelper {
code = AgentFleetReasonPrivilegeHelperDegraded
message = "The typed privilege helper failed; privileged telemetry is unavailable without widening collector privilege."
}
}
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: code,
Severity: severity,
Message: message,
Evidence: moduleStatusEvidence(module),
})
}
return reasons
}
type agentFleetIdentityEvidence struct {
connectionID string
platform string
osName string
osVersion string
kernelVersion string
architecture string
machineIDFingerprint string
reportIP string
interfaceAddresses []string
}
func agentFleetIdentityForSubject(subject agentFleetSubject) agentFleetIdentityEvidence {
identity := agentFleetIdentityEvidence{}
var machineID string
var interfaces []models.HostNetworkInterface
if subject.host != nil {
host := subject.host
identity.connectionID = fleethealth.AgentConnectionID(host.ID)
identity.platform, _ = safeAgentUpdatePlatform(subject)
if identity.platform == "" {
identity.platform = platformsupport.NormalizeAgentReportedPlatform(host.Platform)
}
identity.osName = strings.TrimSpace(host.OSName)
identity.osVersion = strings.TrimSpace(host.OSVersion)
identity.kernelVersion = strings.TrimSpace(host.KernelVersion)
identity.architecture = strings.TrimSpace(host.Architecture)
identity.reportIP = safeDiagnosticIPAddress(host.ReportIP)
machineID = host.MachineID
interfaces = append(interfaces, host.NetworkInterfaces...)
}
if subject.docker != nil {
docker := subject.docker
if identity.platform == "" {
identity.platform, _ = safeAgentUpdatePlatform(subject)
if identity.platform == "" {
identity.platform = platformsupport.NormalizeAgentReportedPlatform(docker.OS)
}
}
identity.osName = firstNonEmpty(identity.osName, docker.OS)
identity.kernelVersion = firstNonEmpty(identity.kernelVersion, docker.KernelVersion)
identity.architecture = firstNonEmpty(identity.architecture, docker.Architecture)
machineID = firstNonEmpty(machineID, docker.MachineID)
interfaces = append(interfaces, docker.NetworkInterfaces...)
}
if identity.platform == "" && strings.TrimSpace(subject.removedPlatform) != "" {
// Removed subjects have no live host/docker record; resolve the
// platform retained on the removed record through the same strict
// normalization so downstream commands never scope from a guess.
identity.platform, _ = safeAgentUpdatePlatform(subject)
}
identity.machineIDFingerprint = machineIDFingerprint(machineID)
identity.interfaceAddresses = safeDiagnosticInterfaceAddresses(interfaces)
return identity
}
func agentFleetUpdateForSubject(subject agentFleetSubject) *AgentFleetDiagnosticUpdate {
if subject.host == nil || subject.host.AgentUpdate == nil {
return nil
}
update := subject.host.AgentUpdate
return &AgentFleetDiagnosticUpdate{
State: strings.TrimSpace(update.State),
AutoUpdate: update.AutoUpdate,
UpdatedFrom: strings.TrimSpace(update.UpdatedFrom),
AvailableVersion: strings.TrimSpace(update.AvailableVersion),
LastCheckedAt: cloneFleetDiagnosticTime(update.LastCheckedAt),
LastAttemptAt: cloneFleetDiagnosticTime(update.LastAttemptAt),
LastSuccessAt: cloneFleetDiagnosticTime(update.LastSuccessAt),
LastError: safeDiagnosticError(update.LastError),
}
}
func agentFleetPrivilegeForSubject(subject agentFleetSubject, inventory agentFleetTokenInventory) *AgentFleetDiagnosticPrivilege {
if subject.host == nil || subject.host.AgentPrivilege == nil {
return nil
}
privilege := subject.host.AgentPrivilege
result := &AgentFleetDiagnosticPrivilege{
RunningAsRoot: privilege.RunningAsRoot,
ServiceUser: strings.TrimSpace(privilege.ServiceUser),
CommandAuthority: strings.TrimSpace(privilege.CommandAuthority),
TypedHelper: privilege.TypedHelper,
SmartctlHelper: privilege.SmartctlHelper,
PctHelper: privilege.PctHelper,
}
if record, ok := inventory.active[strings.TrimSpace(subject.tokenID)]; ok {
result.CredentialKnown = true
result.CredentialExec = record.HasScope(config.ScopeAgentExec)
}
return result
}
func agentFleetModulesForSubject(subject agentFleetSubject) []AgentFleetDiagnosticModule {
statuses := agentFleetModuleStatuses(subject)
if len(statuses) == 0 {
return nil
}
modules := make([]AgentFleetDiagnosticModule, 0, len(statuses))
for _, module := range statuses {
modules = append(modules, AgentFleetDiagnosticModule{
Name: strings.TrimSpace(module.Name),
Enabled: module.Enabled,
State: strings.TrimSpace(module.State),
LastError: safeDiagnosticError(module.LastError),
UpdatedAt: module.UpdatedAt.UTC(),
})
}
sort.Slice(modules, func(i, j int) bool {
return strings.ToLower(modules[i].Name) < strings.ToLower(modules[j].Name)
})
return modules
}
func agentFleetModuleStatuses(subject agentFleetSubject) []models.AgentModuleStatus {
byName := make(map[string]models.AgentModuleStatus)
merge := func(values []models.AgentModuleStatus) {
for _, module := range values {
name := strings.TrimSpace(module.Name)
if name == "" {
continue
}
current, ok := byName[name]
if !ok || module.UpdatedAt.After(current.UpdatedAt) {
module.Name = name
byName[name] = module
}
}
}
if subject.host != nil {
merge(subject.host.AgentModules)
}
if subject.docker != nil {
merge(subject.docker.AgentModules)
}
if len(byName) == 0 {
return nil
}
names := make([]string, 0, len(byName))
for name := range byName {
names = append(names, name)
}
sort.Strings(names)
result := make([]models.AgentModuleStatus, 0, len(names))
for _, name := range names {
result = append(result, byName[name])
}
return result
}
func safeAgentUpdatePlatform(subject agentFleetSubject) (string, bool) {
values := make([]string, 0, 3)
if subject.host != nil {
values = append(values, subject.host.Platform, subject.host.OSName)
}
if subject.docker != nil {
values = append(values, subject.docker.OS)
}
values = append(values, subject.removedPlatform)
for _, value := range values {
appliance := strings.ToLower(strings.TrimSpace(value))
if strings.Contains(appliance, "pfsense") || strings.Contains(appliance, "opnsense") {
return platformsupport.RuntimePlatformFreeBSD, true
}
normalized := platformsupport.NormalizeAgentReportedPlatform(value)
switch normalized {
case platformsupport.RuntimePlatformWindows,
platformsupport.RuntimePlatformMacOS,
platformsupport.RuntimePlatformFreeBSD,
platformsupport.RuntimePlatformLinux:
return normalized, true
}
if runtimePlatform, ok := platformsupport.ResolveAgentRuntimePlatform(normalized); ok {
return runtimePlatform, true
}
}
return "", false
}
func machineIDFingerprint(machineID string) string {
machineID = strings.TrimSpace(machineID)
if machineID == "" {
return ""
}
digest := sha256.Sum256([]byte(machineID))
return "sha256:" + hex.EncodeToString(digest[:8])
}
func safeDiagnosticInterfaceAddresses(interfaces []models.HostNetworkInterface) []string {
const maxAddresses = 32
set := make(map[string]struct{})
for _, iface := range interfaces {
for _, value := range iface.Addresses {
if normalized := safeDiagnosticIPAddress(value); normalized != "" {
set[normalized] = struct{}{}
}
}
}
addresses := make([]string, 0, len(set))
for value := range set {
addresses = append(addresses, value)
}
sort.Strings(addresses)
if len(addresses) > maxAddresses {
addresses = addresses[:maxAddresses]
}
return addresses
}
func safeDiagnosticIPAddress(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
if address, err := netip.ParseAddr(value); err == nil {
return address.String()
}
if prefix, err := netip.ParsePrefix(value); err == nil {
return prefix.String()
}
return ""
}
func updateStatusEvidence(update *models.AgentUpdateStatus) []string {
if update == nil {
return nil
}
evidence := nonEmptyStrings("Updater state: " + strings.TrimSpace(update.State))
if update.LastCheckedAt != nil {
evidence = append(evidence, "Last checked: "+update.LastCheckedAt.UTC().Format(time.RFC3339))
}
if update.LastAttemptAt != nil {
evidence = append(evidence, "Last attempted: "+update.LastAttemptAt.UTC().Format(time.RFC3339))
}
if lastError := safeDiagnosticError(update.LastError); lastError != "" {
evidence = append(evidence, "Last error: "+lastError)
}
return evidence
}
func moduleStatusEvidence(module models.AgentModuleStatus) []string {
evidence := nonEmptyStrings(
"Module: "+strings.TrimSpace(module.Name),
"Module state: "+strings.TrimSpace(module.State),
)
if !module.UpdatedAt.IsZero() {
evidence = append(evidence, "Updated at: "+module.UpdatedAt.UTC().Format(time.RFC3339))
}
if lastError := safeDiagnosticError(module.LastError); lastError != "" {
evidence = append(evidence, "Last error: "+lastError)
}
return evidence
}
func safeDiagnosticError(value string) string {
const maxErrorLength = 512
value = strings.TrimSpace(unifiedresources.RedactAuditText(value))
runes := []rune(value)
if len(runes) > maxErrorLength {
value = string(runes[:maxErrorLength]) + "..."
}
return value
}
func cloneFleetDiagnosticTime(value *time.Time) *time.Time {
if value == nil || value.IsZero() {
return nil
}
cloned := value.UTC()
return &cloned
}
func diagnoseAgentIdentitySplit(subject agentFleetSubject, state models.StateSnapshot) []AgentFleetDiagnosticReason {
if subject.hostname == "" || subject.removed {
return nil
}
if subject.host != nil && subject.docker == nil {
if docker, ok := findLikelyDockerPeer(subject, state.DockerHosts); ok {
return []AgentFleetDiagnosticReason{identitySplitReason("Docker", docker.ID, docker.AgentID, docker.TokenID)}
}
}
if subject.docker != nil && subject.host == nil {
if host, ok := findLikelyHostPeer(subject, state.Hosts); ok {
return []AgentFleetDiagnosticReason{identitySplitReason("Host", host.ID, host.ID, host.TokenID)}
}
}
return nil
}
func diagnoseProfileDeployment(
subject agentFleetSubject,
assignment models.AgentProfileAssignment,
profile models.AgentProfile,
deploymentByAgentProfile map[string]models.ProfileDeploymentStatus,
) []AgentFleetDiagnosticReason {
expectedVersion := expectedProfileVersion(profile, assignment)
if expectedVersion <= 0 {
expectedVersion = profile.Version
}
deployment, ok := deploymentByAgentProfile[deploymentKey(assignment.AgentID, assignment.ProfileID)]
if !ok {
// ProfileDeploymentStatus predates the applied-config fingerprint carried
// by current Unified Agent reports. Absence is therefore not failure
// evidence: config convergence is owned by the desired-versus-applied
// fingerprint contract projected through /api/connections. Retain real
// legacy acknowledgements below so explicit failed, pending, and stale
// deployments remain useful diagnostics.
return nil
}
reasons := []AgentFleetDiagnosticReason{}
if deployment.ProfileID != assignment.ProfileID {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: "profile_deployment_mismatch",
Severity: AgentFleetStatusWarning,
Message: "The agent acknowledged a different profile than the one currently assigned.",
Evidence: []string{
"Assigned profile ID: " + assignment.ProfileID,
"Deployed profile ID: " + deployment.ProfileID,
},
})
}
if deployment.DeploymentStatus == "failed" {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: "profile_deployment_failed",
Severity: AgentFleetStatusCritical,
Message: "The assigned profile failed to deploy to this agent.",
Evidence: nonEmptyStrings(deployment.ErrorMessage),
})
} else if deployment.DeploymentStatus != "" && deployment.DeploymentStatus != "deployed" {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: "profile_deployment_pending",
Severity: AgentFleetStatusWarning,
Message: fmt.Sprintf("The assigned profile is still marked %q for this agent.", deployment.DeploymentStatus),
})
}
if expectedVersion > 0 && deployment.DeployedVersion > 0 && deployment.DeployedVersion < expectedVersion {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: "profile_version_drift",
Severity: AgentFleetStatusWarning,
Message: fmt.Sprintf("The agent has profile version %d, but version %d is assigned.", deployment.DeployedVersion, expectedVersion),
Evidence: []string{
fmt.Sprintf("Profile: %s", firstNonEmpty(profile.Name, profile.ID)),
fmt.Sprintf("Last deployed at: %s", deployment.LastDeployedAt.UTC().Format(time.RFC3339)),
},
})
}
_ = subject
return reasons
}
func diagnoseProfileCapabilityDrift(subject agentFleetSubject, state models.StateSnapshot, profile models.AgentProfile) []AgentFleetDiagnosticReason {
reasons := []AgentFleetDiagnosticReason{}
if enabled, ok := configBool(profile.Config, "enable_docker"); ok && enabled {
if subject.docker == nil {
if _, split := findLikelyDockerPeer(subject, state.DockerHosts); split {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: "docker_profile_identity_split",
Severity: AgentFleetStatusWarning,
Message: "The assigned profile enables Docker monitoring, but Docker telemetry is reporting under a separate agent identity.",
Evidence: []string{
"Profile key enable_docker=true",
"Likely same host matched by hostname or token, not by agent ID",
},
})
} else {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: "docker_expected_missing",
Severity: AgentFleetStatusCritical,
Message: "The assigned profile enables Docker monitoring, but no Docker host telemetry is reporting for this agent.",
Evidence: []string{
"Profile key enable_docker=true",
"No matching Docker host record by agent ID, host ID, hostname, or token",
"Local causes can include missing Docker socket access, installing on the wrong host, or Docker mode being disabled",
},
})
}
}
}
if enabled, ok := configBool(profile.Config, "enable_kubernetes"); ok && enabled && subject.kubernetes == nil && !hasType(subject.types, "kubernetes") {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: "kubernetes_expected_missing",
Severity: AgentFleetStatusWarning,
Message: "The assigned profile enables Kubernetes monitoring, but no Kubernetes cluster telemetry is reporting for this agent.",
Evidence: []string{"Profile key enable_kubernetes=true"},
})
}
proxmoxType, _ := configString(profile.Config, "proxmox_type")
pbsProfile := strings.EqualFold(proxmoxType, "pbs")
pbsHost := !strings.EqualFold(proxmoxType, "pve") &&
subject.host != nil &&
hostMatchesPBSInstance(*subject.host, state.PBSInstances)
if enabled, ok := configBool(profile.Config, "enable_proxmox"); ok &&
enabled &&
!pbsProfile &&
!pbsHost &&
subject.host != nil &&
subject.host.LinkedNodeID == "" {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: "proxmox_profile_unlinked",
Severity: AgentFleetStatusWarning,
Message: "The assigned profile enables Proxmox mode, but this host agent is not linked to a Proxmox node.",
Evidence: []string{"Profile key enable_proxmox=true"},
})
}
if enabled, ok := configBool(profile.Config, "enable_host"); ok && !enabled && subject.host != nil {
reasons = append(reasons, AgentFleetDiagnosticReason{
Code: "host_profile_not_applied",
Severity: AgentFleetStatusWarning,
Message: "The assigned profile disables host monitoring, but host telemetry is still reporting.",
Evidence: []string{"Profile key enable_host=false"},
})
}
return reasons
}
func identitySplitReason(peerType, peerID, peerAgentID, peerTokenID string) AgentFleetDiagnosticReason {
evidence := []string{"Peer type: " + peerType, "Peer ID: " + peerID}
if peerAgentID != "" {
evidence = append(evidence, "Peer agent ID: "+peerAgentID)
}
if peerTokenID != "" {
evidence = append(evidence, "Peer shares the reporting token binding")
}
return AgentFleetDiagnosticReason{
Code: "agent_identity_split",
Severity: AgentFleetStatusWarning,
Message: "Host and workload telemetry appear to belong to the same machine but are reporting as separate agent identities.",
Evidence: evidence,
}
}
func findLikelyDockerPeer(subject agentFleetSubject, dockerHosts []models.DockerHost) (models.DockerHost, bool) {
for _, docker := range dockerHosts {
if docker.Hidden || docker.ID == subject.id {
continue
}
if sameAgentIdentity(subject, docker.ID, docker.AgentID, docker.Hostname, docker.TokenID) {
return docker, true
}
}
return models.DockerHost{}, false
}
func findLikelyHostPeer(subject agentFleetSubject, hosts []models.Host) (models.Host, bool) {
for _, host := range hosts {
if host.ID == subject.id {
continue
}
if sameAgentIdentity(subject, host.ID, host.ID, host.Hostname, host.TokenID) {
return host, true
}
}
return models.Host{}, false
}
func sameAgentIdentity(subject agentFleetSubject, id, agentID, hostname, tokenID string) bool {
if subject.agentID != "" && (subject.agentID == agentID || subject.agentID == id) {
return true
}
if subject.tokenID != "" && tokenID != "" && subject.tokenID == tokenID {
return true
}
if subject.hostname != "" && strings.EqualFold(subject.hostname, hostname) {
return true
}
return false
}
func mapProfilesByID(profiles []models.AgentProfile) map[string]models.AgentProfile {
result := make(map[string]models.AgentProfile, len(profiles))
for _, profile := range profiles {
result[profile.ID] = profile
}
return result
}
func mapAssignmentsByAgent(assignments []models.AgentProfileAssignment) map[string]models.AgentProfileAssignment {
result := make(map[string]models.AgentProfileAssignment, len(assignments))
for _, assignment := range assignments {
result[assignment.AgentID] = assignment
}
return result
}
func mapDeploymentsByAgentProfile(deployments []models.ProfileDeploymentStatus) map[string]models.ProfileDeploymentStatus {
result := make(map[string]models.ProfileDeploymentStatus, len(deployments))
for _, deployment := range deployments {
result[deploymentKey(deployment.AgentID, deployment.ProfileID)] = deployment
}
return result
}
func findAgentAssignment(subject agentFleetSubject, assignmentByAgent map[string]models.AgentProfileAssignment) (models.AgentProfileAssignment, bool) {
for _, id := range uniqueNonEmptyStrings(subject.agentID, subject.id) {
if assignment, ok := assignmentByAgent[id]; ok {
return assignment, true
}
}
return models.AgentProfileAssignment{}, false
}
func deploymentKey(agentID, profileID string) string {
return strings.TrimSpace(agentID) + "\x00" + strings.TrimSpace(profileID)
}
func expectedProfileVersion(profile models.AgentProfile, assignment models.AgentProfileAssignment) int {
if assignment.ProfileVersion > 0 {
return assignment.ProfileVersion
}
return profile.Version
}
func configBool(config models.AgentConfigMap, key string) (bool, bool) {
raw, ok := config[key]
if !ok {
return false, false
}
switch value := raw.(type) {
case bool:
return value, true
case string:
parsed, err := strconv.ParseBool(strings.TrimSpace(value))
return parsed, err == nil
default:
return false, false
}
}
func configString(config models.AgentConfigMap, key string) (string, bool) {
raw, ok := config[key]
if !ok {
return "", false
}
value, ok := raw.(string)
if !ok {
return "", false
}
return strings.TrimSpace(value), true
}
func hostMatchesPBSInstance(host models.Host, instances []models.PBSInstance) bool {
hostName := unifiedresources.NormalizeHostname(host.Hostname)
reportedIPs := make(map[string]struct{})
if reportIP := unifiedresources.NormalizeIP(host.ReportIP); reportIP != "" {
reportedIPs[reportIP] = struct{}{}
}
for _, network := range host.NetworkInterfaces {
for _, address := range network.Addresses {
if ip := unifiedresources.NormalizeIP(address); ip != "" {
reportedIPs[ip] = struct{}{}
}
}
}
for _, instance := range instances {
endpointHost := extractHostname(instance.Host)
for _, candidate := range []string{instance.Name, endpointHost} {
if hostName != "" && unifiedresources.NormalizeHostname(candidate) == hostName {
return true
}
}
if endpointIP := unifiedresources.NormalizeIP(endpointHost); endpointIP != "" {
if _, ok := reportedIPs[endpointIP]; ok {
return true
}
}
}
return false
}
func sortedAgentTypes(types map[string]struct{}) []string {
order := []string{"host", "docker", "kubernetes"}
out := make([]string, 0, len(types))
for _, candidate := range order {
if _, ok := types[candidate]; ok {
out = append(out, candidate)
}
}
return out
}
func diagnosticStatusFromReasons(reasons []AgentFleetDiagnosticReason) string {
status := AgentFleetStatusHealthy
for _, reason := range reasons {
if agentFleetStatusRank(reason.Severity) > agentFleetStatusRank(status) {
status = reason.Severity
}
}
return status
}
func agentFleetStatusRank(status string) int {
switch status {
case AgentFleetStatusCritical:
return 3
case AgentFleetStatusWarning:
return 2
case AgentFleetStatusRemoved:
return 1
default:
return 0
}
}
func agentFleetStatusIsOnline(status string) bool {
switch strings.ToLower(strings.TrimSpace(status)) {
case "online", "running", "healthy":
return true
default:
return false
}
}
func hasType(types map[string]struct{}, kind string) bool {
_, ok := types[kind]
return ok
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if trimmed := strings.TrimSpace(value); trimmed != "" {
return trimmed
}
}
return ""
}
func newestTime(a, b time.Time) time.Time {
if a.IsZero() || b.After(a) {
return b
}
return a
}
func maxInt(a, b int) int {
if b > a {
return b
}
return a
}
func nonEmptyStrings(values ...string) []string {
out := make([]string, 0, len(values))
for _, value := range values {
if trimmed := strings.TrimSpace(value); trimmed != "" {
out = append(out, trimmed)
}
}
return out
}
// formatFleetDuration renders a duration the way a person would say it
// ("45s", "5m", "10m 2s", "1h 3m", "2d 4h") instead of Go's "5m0s" form.
func formatFleetDuration(duration time.Duration) string {
if duration < 0 {
duration = 0
}
duration = duration.Round(time.Second)
days := int(duration.Hours()) / 24
hours := int(duration.Hours()) % 24
minutes := int(duration.Minutes()) % 60
seconds := int(duration.Seconds()) % 60
switch {
case days > 0:
if hours == 0 {
return fmt.Sprintf("%dd", days)
}
return fmt.Sprintf("%dd %dh", days, hours)
case duration >= time.Hour:
if minutes == 0 {
return fmt.Sprintf("%dh", hours)
}
return fmt.Sprintf("%dh %dm", hours, minutes)
case duration >= time.Minute:
if seconds == 0 {
return fmt.Sprintf("%dm", minutes)
}
return fmt.Sprintf("%dm %ds", minutes, seconds)
default:
return fmt.Sprintf("%ds", seconds)
}
}