Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Name demo Patrol findings by their unified resource
rcourtmanb45b5562fa95
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.mdindex 66544d0517..905fec6cc4 100644--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md@@ -7111,6 +7111,30 @@ a capacity finding about `/backup` must not hide under the `/data` alerta stamped finding to its alert by `mirrors_alert_id`, which is the attentionitem's operational record ID.+Demo findings carry the unified resource ID too. `synthesizeDemoPatrolFindings`+reads mock values from the legacy snapshot and resolves the reference an+alert on each record carries through Patrol's alert reference resolver, the+way Inbox subjects resolve (`AttentionSubjectResourceID`), so a demo finding+names the resource an Inbox item about the same record names. App containers+resolve through the container views by host and container ID, because a+Docker container alert reference binds only a complete container ID and mock+containers carry short ones. Without a read state, or while it holds none of+the record kinds the synthesizer reads (guests, storage, Docker hosts and+containers, PBS, PMG), the resolver derives a registry from the snapshot, so a+first cycle that races the ingest does not keep source IDs for the life of the+process; a record a saved operator link folds into another resource keeps that+derived ID until restart. Once the read state holds such records it is the+only authority: a+record it lacks or refuses (an operator link can fold a Docker host into a+guest), or one that binds another type of resource, keeps its source ID rather+than an ID from a registry without the operator's links. Guests are typed `vm`+or `system-container`. Until 24+October 2026 every demo finding carried the snapshot's source ID, so in mock+and demo mode an Inbox item never joined a demo finding on its resource, the+resource-scoped findings list and resource intelligence never listed one, and+an offline Docker host's finding never folded under its `docker:<host>` alert+(`TestPatrolService_RunDemoPatrolCycle_NamesUnifiedResources`).+Flapping is owned by the existing storm throttler rather than a parallelmechanism. `FindingsStore.recordTransitionLifecycleLocked` routes everyregressed, resolved, and auto_resolved transition throughdiff --git a/internal/ai/demo.go b/internal/ai/demo.goindex c2e562af0c..e806d70777 100644--- a/internal/ai/demo.go+++ b/internal/ai/demo.go@@ -5,8 +5,10 @@ import ("strings""time"+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts""github.com/rcourtman/pulse-go-rewrite/internal/mockmode""github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources""github.com/rs/zerolog/log")@@ -326,15 +328,76 @@ func (p *PatrolService) backfillDemoRunHistory(template PatrolRunRecord, now timreturn true}+// demoResourceResolver names the snapshot records the demo synthesizer reads+// by the unified resources real Patrol findings name. The Inbox, resource+// drawers, the resource-scoped findings list and the alert-mirror reconcile+// all join a finding to its resource by that ID, so a demo finding stamped+// with a snapshot source ID joined none of them.+type demoResourceResolver struct {+ readState unifiedresources.ReadState+ subjects unifiedresources.AlertResourceReferenceResolver+}++// newDemoResourceResolver resolves against Patrol's read state. Without one,+// or while it holds none of the record kinds the synthesizer reads (a demo+// cycle can start before the registry's first ingest), it derives one from+// the snapshot, as the real producers do: the findings store keeps a+// finding's resource for the life of the process, so a first cycle must not+// settle for source IDs. The derived IDs match what the registry assigns on+// ingest unless a saved operator link folds the record into another+// resource; such a finding keeps the derived ID until restart. Once the+// registry holds such records it is the only authority, so a record it lacks+// or refuses (an operator link can fold a Docker host into a guest) never+// takes an ID from a registry without the operator's links.+func newDemoResourceResolver(snap patrolRuntimeState) *demoResourceResolver {+ if snap.readState == nil || demoReadStateLacksRecords(snap.readState) {+ snap = snap.withDerivedProviders()+ }+ return &demoResourceResolver{readState: snap.readState, subjects: patrolAlertReferenceResolver(snap)}+}++func demoReadStateLacksRecords(rs unifiedresources.ReadState) bool {+ return len(rs.VMs()) == 0 && len(rs.Containers()) == 0 && len(rs.StoragePools()) == 0 &&+ len(rs.DockerHosts()) == 0 && len(rs.DockerContainers()) == 0 &&+ len(rs.PBSInstances()) == 0 && len(rs.PMGInstances()) == 0+}++// unifiedID resolves the reference an alert on the record carries the way+// the Inbox resolves an alert's subject (AttentionSubjectResourceID), so a+// demo finding names the resource an Inbox item about the same record names.+// A reference that binds nothing, or binds another type of resource, keeps+// the record's source ID.+func (r *demoResourceResolver) unifiedID(reference, sourceID string, resourceType unifiedresources.ResourceType) string {+ binding, _ := r.subjects.ResolveAlertResourceReference(reference)+ if binding.ResourceID == "" || binding.Component || unifiedresources.CanonicalResourceType(binding.ResourceType) != resourceType {+ return sourceID+ }+ return binding.ResourceID+}++// containerID resolves an app container through the container views, by+// host and container ID: a Docker container alert reference binds only a+// complete container ID, and mock containers carry short ones.+func (r *demoResourceResolver) containerID(hostID, containerID string) string {+ host, container := strings.TrimSpace(hostID), strings.TrimSpace(containerID)+ for _, view := range r.readState.DockerContainers() {+ if view != nil && view.HostSourceID() == host && view.ContainerID() == container {+ return view.ID()+ }+ }+ return containerID+}+// synthesizeDemoPatrolFindings derives demo findings from the live mock-// state. Every finding references a resource that exists in the demo dataset-// and quotes its actual observed values, so the patrol surface stays-// consistent with what the rest of the demo UI shows. The demo scenario data-// pins a set of deliberate anomalies (an offline Docker edge host, a degraded-// PBS instance, unhealthy containers), which gives the surface a stable core-// of findings across cycles.+// state. Every finding names a resource that exists in the demo dataset by+// its unified resource ID and quotes its actual observed values, so the+// patrol surface stays consistent with what the rest of the demo UI shows.+// The demo scenario data pins a set of deliberate anomalies (an offline+// Docker edge host, a degraded PBS instance, unhealthy containers), which+// gives the surface a stable core of findings across cycles.func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*Finding {var findings []*Finding+ resources := newDemoResourceResolver(snap)// Offline Docker host: the demo scenario keeps one edge host offline.for _, host := range snap.DockerHosts {@@ -347,7 +410,7 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*FinKey: fmt.Sprintf("docker:%s:offline", host.Hostname),Severity: FindingSeverityCritical,Category: FindingCategoryReliability,- ResourceID: host.ID,+ ResourceID: resources.unifiedID(alerts.DockerHostResourceID(host.ID), host.ID, unifiedresources.ResourceTypeAgent),ResourceName: name,ResourceType: "docker-host",Node: host.Hostname,@@ -394,7 +457,7 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*FinKey: fmt.Sprintf("storage:%s:capacity", topStorage.Name),Severity: severity,Category: FindingCategoryCapacity,- ResourceID: topStorage.ID,+ ResourceID: resources.unifiedID(topStorage.ID, topStorage.ID, unifiedresources.ResourceTypeStorage),ResourceName: topStorage.Name,ResourceType: "storage",Node: topStorage.Node,@@ -423,7 +486,7 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*FinKey: fmt.Sprintf("pbs:%s:connection", pbs.Name),Severity: FindingSeverityWarning,Category: FindingCategoryBackup,- ResourceID: pbs.ID,+ ResourceID: resources.unifiedID(pbs.ID, pbs.ID, unifiedresources.ResourceTypePBS),ResourceName: pbs.Name,ResourceType: "pbs",Node: pbs.Name,@@ -471,7 +534,7 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*FinKey: fmt.Sprintf("docker:%s/%s:health", host.Hostname, c.Name),Severity: FindingSeverityWarning,Category: FindingCategoryReliability,- ResourceID: c.ID,+ ResourceID: resources.containerID(host.ID, c.ID),ResourceName: c.Name,ResourceType: "app-container",Node: host.Hostname,@@ -505,7 +568,7 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*FinKey: fmt.Sprintf("pmg:%s:connection", pmg.Name),Severity: FindingSeverityWarning,Category: FindingCategoryReliability,- ResourceID: pmg.ID,+ ResourceID: resources.unifiedID(pmg.ID, pmg.ID, unifiedresources.ResourceTypePMG),ResourceName: pmg.Name,ResourceType: "pmg",Node: pmg.Name,@@ -525,26 +588,27 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*Fin// Highest-memory running guest, only when genuinely high.type guestRef struct {- id, name, node, kind string- usage float64- used, total int64+ id, name, node, label string+ resourceType unifiedresources.ResourceType+ usage float64+ used, total int64}var topGuest *guestRef- consider := func(id, name, node, kind string, status string, mem models.Memory) {+ consider := func(id, name, node, label string, resourceType unifiedresources.ResourceType, status string, mem models.Memory) {if !strings.EqualFold(strings.TrimSpace(status), "running") || mem.Total <= 0 {return}if topGuest == nil || mem.Usage > topGuest.usage {- topGuest = &guestRef{id: id, name: name, node: node, kind: kind, usage: mem.Usage, used: mem.Used, total: mem.Total}+ topGuest = &guestRef{id: id, name: name, node: node, label: label, resourceType: resourceType, usage: mem.Usage, used: mem.Used, total: mem.Total}}}for i := range snap.VMs {vm := &snap.VMs[i]- consider(vm.ID, vm.Name, vm.Node, "VM", vm.Status, vm.Memory)+ consider(vm.ID, vm.Name, vm.Node, "VM", unifiedresources.ResourceTypeVM, vm.Status, vm.Memory)}for i := range snap.Containers {ct := &snap.Containers[i]- consider(ct.ID, ct.Name, ct.Node, "Container", ct.Status, ct.Memory)+ consider(ct.ID, ct.Name, ct.Node, "Container", unifiedresources.ResourceTypeSystemContainer, ct.Status, ct.Memory)}if topGuest != nil && topGuest.usage >= 85 {findings = append(findings, &Finding{@@ -552,14 +616,14 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*FinKey: fmt.Sprintf("guest:%s:memory", topGuest.id),Severity: FindingSeverityWarning,Category: FindingCategoryPerformance,- ResourceID: topGuest.id,+ ResourceID: resources.unifiedID(topGuest.id, topGuest.id, topGuest.resourceType),ResourceName: topGuest.name,- ResourceType: strings.ToLower(topGuest.kind),+ ResourceType: string(topGuest.resourceType),Node: topGuest.node,- Title: fmt.Sprintf("%s %q memory usage at %.0f%%", topGuest.kind, topGuest.name, topGuest.usage),+ Title: fmt.Sprintf("%s %q memory usage at %.0f%%", topGuest.label, topGuest.name, topGuest.usage),Description: fmt.Sprintf("%s %s on %s is using %s of its %s allocation. Sustained usage at this level risks swapping and degraded performance under load.",- topGuest.kind, topGuest.name, topGuest.node, formatDemoBytes(topGuest.used), formatDemoBytes(topGuest.total)),+ topGuest.label, topGuest.name, topGuest.node, formatDemoBytes(topGuest.used), formatDemoBytes(topGuest.total)),Recommendation: "Increase the memory allocation if the host has capacity, or check the workload for a leak " +"and review application cache settings to reduce steady-state usage.",Evidence: fmt.Sprintf("memory=%.1f%% used=%s total=%s", topGuest.usage, formatDemoBytes(topGuest.used), formatDemoBytes(topGuest.total)),diff --git a/internal/ai/demo_test.go b/internal/ai/demo_test.goindex b7890d99ca..774c178a1a 100644--- a/internal/ai/demo_test.go+++ b/internal/ai/demo_test.go@@ -5,8 +5,10 @@ import ("testing""time"+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts""github.com/rcourtman/pulse-go-rewrite/internal/mockruntime""github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources")func TestIsDemoMode(t *testing.T) {@@ -140,6 +142,140 @@ func keysOfDemoFindings(m map[string]*Finding) []string {return keys}+// Demo findings name the unified resource, as real Patrol findings do. The+// Inbox names an alert's subject by the unified resource its reference+// resolves to, resource drawers ask for a resource's findings by that ID, and+// the alert-mirror reconcile folds a finding under its alert through it. A+// demo finding stamped with the snapshot's source ID joined none of them.+func TestPatrolService_RunDemoPatrolCycle_NamesUnifiedResources(t *testing.T) {+ original := mockruntime.IsEnabled()+ t.Cleanup(func() { mockruntime.SetEnabled(original) })+ mockruntime.SetEnabled(true)++ now := time.Now()+ snapshot := demoTestSnapshot(now)+ // Mock-shaped Docker IDs: a host alert reference ("docker:<host ID>")+ // cannot carry a slash, and mock container IDs are short.+ snapshot.DockerHosts[0].ID = "edge-01-mock"+ snapshot.DockerHosts[1].ID = "core-01-mock"+ snapshot.DockerHosts[1].Containers[0].ID = "core-01-5666a65b2b52"+ // The busiest guest is an LXC container.+ snapshot.Containers = []models.Container{+ {ID: "pve1-ct-102", VMID: 102, Name: "dns-01", Node: "pve1", Status: "running", Type: "lxc",+ Memory: models.Memory{Total: 2 << 30, Used: 1946 << 20, Usage: 95.0}},+ }+ offlineAlert := models.Alert{ID: "docker:edge-01-mock-connectivity", Type: "docker-host-offline",+ ResourceID: alerts.DockerHostResourceID("edge-01-mock"), StartTime: now.Add(-time.Hour)}+ usageAlert := models.Alert{ID: "pve1-local-zfs-usage", Type: "usage", ResourceID: "pve1-local-zfs",+ StartTime: now.Add(-time.Hour)}+ snapshot.ActiveAlerts = []models.Alert{offlineAlert, usageAlert}++ registry := unifiedresources.NewRegistry(nil)+ registry.IngestSnapshot(snapshot)+ service := NewPatrolService(nil, demoSnapshotProvider{snapshot: snapshot})+ service.SetReadState(registry)+ service.runDemoPatrolCycle(TriggerReasonStartup)++ containerResourceID := ""+ for _, resource := range registry.List() {+ if resource.Type == unifiedresources.ResourceTypeAppContainer && resource.Name == "uptime-kuma" {+ containerResourceID = resource.ID+ }+ }+ if containerResourceID == "" {+ t.Fatal("expected the registry to hold the uptime-kuma app container")+ }++ cases := []struct {+ findingID string+ resourceID string+ resourceType string+ }{+ // An alert on each resource carries these references.+ {"demo-docker-host-offline-edge-01", AttentionSubjectResourceID(registry, offlineAlert.ResourceID), "docker-host"},+ {"demo-storage-capacity-pve1-local-zfs", AttentionSubjectResourceID(registry, usageAlert.ResourceID), "storage"},+ {"demo-pbs-degraded-dr-vault", AttentionSubjectResourceID(registry, "pbs/dr-vault"), "pbs"},+ {"demo-guest-memory-pve1-ct-102", AttentionSubjectResourceID(registry, "pve1-ct-102"), "system-container"},+ // A short container ID binds no alert reference, so name the resource directly.+ {"demo-docker-container-core-01-uptime-kuma", containerResourceID, "app-container"},+ }+ for _, tc := range cases {+ finding := service.findings.Get(tc.findingID)+ if finding == nil {+ t.Fatalf("expected finding %s", tc.findingID)+ }+ if _, ok := registry.Get(tc.resourceID); !ok {+ t.Fatalf("%s: expected %q to be a unified resource", tc.findingID, tc.resourceID)+ }+ if finding.ResourceID != tc.resourceID {+ t.Fatalf("%s: resource ID = %q, want unified %q", tc.findingID, finding.ResourceID, tc.resourceID)+ }+ if finding.ResourceType != tc.resourceType {+ t.Fatalf("%s: resource type = %q, want %q", tc.findingID, finding.ResourceType, tc.resourceType)+ }+ onResource := service.GetFindingsForResource(tc.resourceID)+ if len(onResource) != 1 || onResource[0].ID != tc.findingID {+ t.Fatalf("%s: findings for resource %q = %v", tc.findingID, tc.resourceID, onResource)+ }+ }++ // Both findings restate an active alert, so they fold under it.+ for findingID, alertID := range map[string]string{+ "demo-docker-host-offline-edge-01": offlineAlert.ID,+ "demo-storage-capacity-pve1-local-zfs": usageAlert.ID,+ } {+ if got := service.findings.Get(findingID).MirrorsAlertID; got != alertID {+ t.Fatalf("%s: mirrors alert %q, want %q", findingID, got, alertID)+ }+ }++ // Without a read state the views are derived from the same snapshot.+ for _, derived := range synthesizeDemoPatrolFindings(newPatrolRuntimeState(snapshot), now) {+ if want := service.findings.Get(derived.ID); want == nil || derived.ResourceID != want.ResourceID {+ t.Fatalf("%s: derived resource ID %q differs from the read state's", derived.ID, derived.ResourceID)+ }+ }++ // A registry that has not ingested the snapshot yet must not leave the+ // first cycle's findings on source IDs: the store keeps a finding's+ // resource for as long as the process runs.+ early := NewPatrolService(nil, demoSnapshotProvider{snapshot: snapshot})+ early.SetReadState(unifiedresources.NewRegistry(nil))+ early.runDemoPatrolCycle(TriggerReasonStartup)+ for _, tc := range cases {+ finding := early.findings.Get(tc.findingID)+ if finding == nil {+ t.Fatalf("expected finding %s before the registry ingests", tc.findingID)+ }+ if finding.ResourceID != tc.resourceID {+ t.Fatalf("%s: resource ID before the registry ingests = %q, want %q", tc.findingID, finding.ResourceID, tc.resourceID)+ }+ }++ // Once the registry holds resources it is the only authority. A record it+ // lacks or refuses (an operator link can fold a Docker host into a guest)+ // keeps its source ID instead of an ID from a registry without those links.+ partial := snapshot+ partial.DockerHosts = nil+ partialRegistry := unifiedresources.NewRegistry(nil)+ partialRegistry.IngestSnapshot(partial)+ lagging := NewPatrolService(nil, demoSnapshotProvider{snapshot: snapshot})+ lagging.SetReadState(partialRegistry)+ lagging.runDemoPatrolCycle(TriggerReasonStartup)+ for findingID, want := range map[string]string{+ "demo-docker-host-offline-edge-01": "edge-01-mock",+ "demo-storage-capacity-pve1-local-zfs": AttentionSubjectResourceID(registry, usageAlert.ResourceID),+ } {+ finding := lagging.findings.Get(findingID)+ if finding == nil {+ t.Fatalf("expected finding %s with a registry lacking Docker hosts", findingID)+ }+ if finding.ResourceID != want {+ t.Fatalf("%s: resource ID with a registry lacking Docker hosts = %q, want %q", findingID, finding.ResourceID, want)+ }+ }+}+func TestPatrolService_RunDemoPatrolCycle_ResolvesStaleDemoFindings(t *testing.T) {original := mockruntime.IsEnabled()t.Cleanup(func() { mockruntime.SetEnabled(original) })