Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Name demo Patrol findings by their unified resource

rcourtmanb45b5562fa95
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md
index 66544d0517..905fec6cc4 100644
--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md
+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md
@@ -7111,6 +7111,30 @@ a capacity finding about `/backup` must not hide under the `/data` alert
a stamped finding to its alert by `mirrors_alert_id`, which is the attention
item's operational record ID.
+Demo findings carry the unified resource ID too. `synthesizeDemoPatrolFindings`
+reads mock values from the legacy snapshot and resolves the reference an
+alert on each record carries through Patrol's alert reference resolver, the
+way Inbox subjects resolve (`AttentionSubjectResourceID`), so a demo finding
+names the resource an Inbox item about the same record names. App containers
+resolve through the container views by host and container ID, because a
+Docker container alert reference binds only a complete container ID and mock
+containers carry short ones. Without a read state, or while it holds none of
+the record kinds the synthesizer reads (guests, storage, Docker hosts and
+containers, PBS, PMG), the resolver derives a registry from the snapshot, so a
+first cycle that races the ingest does not keep source IDs for the life of the
+process; a record a saved operator link folds into another resource keeps that
+derived ID until restart. Once the read state holds such records it is the
+only authority: a
+record it lacks or refuses (an operator link can fold a Docker host into a
+guest), or one that binds another type of resource, keeps its source ID rather
+than an ID from a registry without the operator's links. Guests are typed `vm`
+or `system-container`. Until 24
+October 2026 every demo finding carried the snapshot's source ID, so in mock
+and demo mode an Inbox item never joined a demo finding on its resource, the
+resource-scoped findings list and resource intelligence never listed one, and
+an offline Docker host's finding never folded under its `docker:<host>` alert
+(`TestPatrolService_RunDemoPatrolCycle_NamesUnifiedResources`).
+
Flapping is owned by the existing storm throttler rather than a parallel
mechanism. `FindingsStore.recordTransitionLifecycleLocked` routes every
regressed, resolved, and auto_resolved transition through
diff --git a/internal/ai/demo.go b/internal/ai/demo.go
index c2e562af0c..e806d70777 100644
--- a/internal/ai/demo.go
+++ b/internal/ai/demo.go
@@ -5,8 +5,10 @@ import (
"strings"
"time"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/mockmode"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
@@ -326,15 +328,76 @@ func (p *PatrolService) backfillDemoRunHistory(template PatrolRunRecord, now tim
return true
}
+// demoResourceResolver names the snapshot records the demo synthesizer reads
+// by the unified resources real Patrol findings name. The Inbox, resource
+// drawers, the resource-scoped findings list and the alert-mirror reconcile
+// all join a finding to its resource by that ID, so a demo finding stamped
+// with a snapshot source ID joined none of them.
+type demoResourceResolver struct {
+ readState unifiedresources.ReadState
+ subjects unifiedresources.AlertResourceReferenceResolver
+}
+
+// newDemoResourceResolver resolves against Patrol's read state. Without one,
+// or while it holds none of the record kinds the synthesizer reads (a demo
+// cycle can start before the registry's first ingest), it derives one from
+// the snapshot, as the real producers do: the findings store keeps a
+// finding's resource for the life of the process, so a first cycle must not
+// settle for source IDs. The derived IDs match what the registry assigns on
+// ingest unless a saved operator link folds the record into another
+// resource; such a finding keeps the derived ID until restart. Once the
+// registry holds such records it is the only authority, so a record it lacks
+// or refuses (an operator link can fold a Docker host into a guest) never
+// takes an ID from a registry without the operator's links.
+func newDemoResourceResolver(snap patrolRuntimeState) *demoResourceResolver {
+ if snap.readState == nil || demoReadStateLacksRecords(snap.readState) {
+ snap = snap.withDerivedProviders()
+ }
+ return &demoResourceResolver{readState: snap.readState, subjects: patrolAlertReferenceResolver(snap)}
+}
+
+func demoReadStateLacksRecords(rs unifiedresources.ReadState) bool {
+ return len(rs.VMs()) == 0 && len(rs.Containers()) == 0 && len(rs.StoragePools()) == 0 &&
+ len(rs.DockerHosts()) == 0 && len(rs.DockerContainers()) == 0 &&
+ len(rs.PBSInstances()) == 0 && len(rs.PMGInstances()) == 0
+}
+
+// unifiedID resolves the reference an alert on the record carries the way
+// the Inbox resolves an alert's subject (AttentionSubjectResourceID), so a
+// demo finding names the resource an Inbox item about the same record names.
+// A reference that binds nothing, or binds another type of resource, keeps
+// the record's source ID.
+func (r *demoResourceResolver) unifiedID(reference, sourceID string, resourceType unifiedresources.ResourceType) string {
+ binding, _ := r.subjects.ResolveAlertResourceReference(reference)
+ if binding.ResourceID == "" || binding.Component || unifiedresources.CanonicalResourceType(binding.ResourceType) != resourceType {
+ return sourceID
+ }
+ return binding.ResourceID
+}
+
+// containerID resolves an app container through the container views, by
+// host and container ID: a Docker container alert reference binds only a
+// complete container ID, and mock containers carry short ones.
+func (r *demoResourceResolver) containerID(hostID, containerID string) string {
+ host, container := strings.TrimSpace(hostID), strings.TrimSpace(containerID)
+ for _, view := range r.readState.DockerContainers() {
+ if view != nil && view.HostSourceID() == host && view.ContainerID() == container {
+ return view.ID()
+ }
+ }
+ return containerID
+}
+
// synthesizeDemoPatrolFindings derives demo findings from the live mock
-// state. Every finding references a resource that exists in the demo dataset
-// and quotes its actual observed values, so the patrol surface stays
-// consistent with what the rest of the demo UI shows. The demo scenario data
-// pins a set of deliberate anomalies (an offline Docker edge host, a degraded
-// PBS instance, unhealthy containers), which gives the surface a stable core
-// of findings across cycles.
+// state. Every finding names a resource that exists in the demo dataset by
+// its unified resource ID and quotes its actual observed values, so the
+// patrol surface stays consistent with what the rest of the demo UI shows.
+// The demo scenario data pins a set of deliberate anomalies (an offline
+// Docker edge host, a degraded PBS instance, unhealthy containers), which
+// gives the surface a stable core of findings across cycles.
func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*Finding {
var findings []*Finding
+ resources := newDemoResourceResolver(snap)
// Offline Docker host: the demo scenario keeps one edge host offline.
for _, host := range snap.DockerHosts {
@@ -347,7 +410,7 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*Fin
Key: fmt.Sprintf("docker:%s:offline", host.Hostname),
Severity: FindingSeverityCritical,
Category: FindingCategoryReliability,
- ResourceID: host.ID,
+ ResourceID: resources.unifiedID(alerts.DockerHostResourceID(host.ID), host.ID, unifiedresources.ResourceTypeAgent),
ResourceName: name,
ResourceType: "docker-host",
Node: host.Hostname,
@@ -394,7 +457,7 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*Fin
Key: fmt.Sprintf("storage:%s:capacity", topStorage.Name),
Severity: severity,
Category: FindingCategoryCapacity,
- ResourceID: topStorage.ID,
+ ResourceID: resources.unifiedID(topStorage.ID, topStorage.ID, unifiedresources.ResourceTypeStorage),
ResourceName: topStorage.Name,
ResourceType: "storage",
Node: topStorage.Node,
@@ -423,7 +486,7 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*Fin
Key: fmt.Sprintf("pbs:%s:connection", pbs.Name),
Severity: FindingSeverityWarning,
Category: FindingCategoryBackup,
- ResourceID: pbs.ID,
+ ResourceID: resources.unifiedID(pbs.ID, pbs.ID, unifiedresources.ResourceTypePBS),
ResourceName: pbs.Name,
ResourceType: "pbs",
Node: pbs.Name,
@@ -471,7 +534,7 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*Fin
Key: fmt.Sprintf("docker:%s/%s:health", host.Hostname, c.Name),
Severity: FindingSeverityWarning,
Category: FindingCategoryReliability,
- ResourceID: c.ID,
+ ResourceID: resources.containerID(host.ID, c.ID),
ResourceName: c.Name,
ResourceType: "app-container",
Node: host.Hostname,
@@ -505,7 +568,7 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*Fin
Key: fmt.Sprintf("pmg:%s:connection", pmg.Name),
Severity: FindingSeverityWarning,
Category: FindingCategoryReliability,
- ResourceID: pmg.ID,
+ ResourceID: resources.unifiedID(pmg.ID, pmg.ID, unifiedresources.ResourceTypePMG),
ResourceName: pmg.Name,
ResourceType: "pmg",
Node: pmg.Name,
@@ -525,26 +588,27 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*Fin
// Highest-memory running guest, only when genuinely high.
type guestRef struct {
- id, name, node, kind string
- usage float64
- used, total int64
+ id, name, node, label string
+ resourceType unifiedresources.ResourceType
+ usage float64
+ used, total int64
}
var topGuest *guestRef
- consider := func(id, name, node, kind string, status string, mem models.Memory) {
+ consider := func(id, name, node, label string, resourceType unifiedresources.ResourceType, status string, mem models.Memory) {
if !strings.EqualFold(strings.TrimSpace(status), "running") || mem.Total <= 0 {
return
}
if topGuest == nil || mem.Usage > topGuest.usage {
- topGuest = &guestRef{id: id, name: name, node: node, kind: kind, usage: mem.Usage, used: mem.Used, total: mem.Total}
+ topGuest = &guestRef{id: id, name: name, node: node, label: label, resourceType: resourceType, usage: mem.Usage, used: mem.Used, total: mem.Total}
}
}
for i := range snap.VMs {
vm := &snap.VMs[i]
- consider(vm.ID, vm.Name, vm.Node, "VM", vm.Status, vm.Memory)
+ consider(vm.ID, vm.Name, vm.Node, "VM", unifiedresources.ResourceTypeVM, vm.Status, vm.Memory)
}
for i := range snap.Containers {
ct := &snap.Containers[i]
- consider(ct.ID, ct.Name, ct.Node, "Container", ct.Status, ct.Memory)
+ consider(ct.ID, ct.Name, ct.Node, "Container", unifiedresources.ResourceTypeSystemContainer, ct.Status, ct.Memory)
}
if topGuest != nil && topGuest.usage >= 85 {
findings = append(findings, &Finding{
@@ -552,14 +616,14 @@ func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*Fin
Key: fmt.Sprintf("guest:%s:memory", topGuest.id),
Severity: FindingSeverityWarning,
Category: FindingCategoryPerformance,
- ResourceID: topGuest.id,
+ ResourceID: resources.unifiedID(topGuest.id, topGuest.id, topGuest.resourceType),
ResourceName: topGuest.name,
- ResourceType: strings.ToLower(topGuest.kind),
+ ResourceType: string(topGuest.resourceType),
Node: topGuest.node,
- Title: fmt.Sprintf("%s %q memory usage at %.0f%%", topGuest.kind, topGuest.name, topGuest.usage),
+ Title: fmt.Sprintf("%s %q memory usage at %.0f%%", topGuest.label, topGuest.name, topGuest.usage),
Description: fmt.Sprintf(
"%s %s on %s is using %s of its %s allocation. Sustained usage at this level risks swapping and degraded performance under load.",
- topGuest.kind, topGuest.name, topGuest.node, formatDemoBytes(topGuest.used), formatDemoBytes(topGuest.total)),
+ topGuest.label, topGuest.name, topGuest.node, formatDemoBytes(topGuest.used), formatDemoBytes(topGuest.total)),
Recommendation: "Increase the memory allocation if the host has capacity, or check the workload for a leak " +
"and review application cache settings to reduce steady-state usage.",
Evidence: fmt.Sprintf("memory=%.1f%% used=%s total=%s", topGuest.usage, formatDemoBytes(topGuest.used), formatDemoBytes(topGuest.total)),
diff --git a/internal/ai/demo_test.go b/internal/ai/demo_test.go
index b7890d99ca..774c178a1a 100644
--- a/internal/ai/demo_test.go
+++ b/internal/ai/demo_test.go
@@ -5,8 +5,10 @@ import (
"testing"
"time"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/mockruntime"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func TestIsDemoMode(t *testing.T) {
@@ -140,6 +142,140 @@ func keysOfDemoFindings(m map[string]*Finding) []string {
return keys
}
+// Demo findings name the unified resource, as real Patrol findings do. The
+// Inbox names an alert's subject by the unified resource its reference
+// resolves to, resource drawers ask for a resource's findings by that ID, and
+// the alert-mirror reconcile folds a finding under its alert through it. A
+// demo finding stamped with the snapshot's source ID joined none of them.
+func TestPatrolService_RunDemoPatrolCycle_NamesUnifiedResources(t *testing.T) {
+ original := mockruntime.IsEnabled()
+ t.Cleanup(func() { mockruntime.SetEnabled(original) })
+ mockruntime.SetEnabled(true)
+
+ now := time.Now()
+ snapshot := demoTestSnapshot(now)
+ // Mock-shaped Docker IDs: a host alert reference ("docker:<host ID>")
+ // cannot carry a slash, and mock container IDs are short.
+ snapshot.DockerHosts[0].ID = "edge-01-mock"
+ snapshot.DockerHosts[1].ID = "core-01-mock"
+ snapshot.DockerHosts[1].Containers[0].ID = "core-01-5666a65b2b52"
+ // The busiest guest is an LXC container.
+ snapshot.Containers = []models.Container{
+ {ID: "pve1-ct-102", VMID: 102, Name: "dns-01", Node: "pve1", Status: "running", Type: "lxc",
+ Memory: models.Memory{Total: 2 << 30, Used: 1946 << 20, Usage: 95.0}},
+ }
+ offlineAlert := models.Alert{ID: "docker:edge-01-mock-connectivity", Type: "docker-host-offline",
+ ResourceID: alerts.DockerHostResourceID("edge-01-mock"), StartTime: now.Add(-time.Hour)}
+ usageAlert := models.Alert{ID: "pve1-local-zfs-usage", Type: "usage", ResourceID: "pve1-local-zfs",
+ StartTime: now.Add(-time.Hour)}
+ snapshot.ActiveAlerts = []models.Alert{offlineAlert, usageAlert}
+
+ registry := unifiedresources.NewRegistry(nil)
+ registry.IngestSnapshot(snapshot)
+ service := NewPatrolService(nil, demoSnapshotProvider{snapshot: snapshot})
+ service.SetReadState(registry)
+ service.runDemoPatrolCycle(TriggerReasonStartup)
+
+ containerResourceID := ""
+ for _, resource := range registry.List() {
+ if resource.Type == unifiedresources.ResourceTypeAppContainer && resource.Name == "uptime-kuma" {
+ containerResourceID = resource.ID
+ }
+ }
+ if containerResourceID == "" {
+ t.Fatal("expected the registry to hold the uptime-kuma app container")
+ }
+
+ cases := []struct {
+ findingID string
+ resourceID string
+ resourceType string
+ }{
+ // An alert on each resource carries these references.
+ {"demo-docker-host-offline-edge-01", AttentionSubjectResourceID(registry, offlineAlert.ResourceID), "docker-host"},
+ {"demo-storage-capacity-pve1-local-zfs", AttentionSubjectResourceID(registry, usageAlert.ResourceID), "storage"},
+ {"demo-pbs-degraded-dr-vault", AttentionSubjectResourceID(registry, "pbs/dr-vault"), "pbs"},
+ {"demo-guest-memory-pve1-ct-102", AttentionSubjectResourceID(registry, "pve1-ct-102"), "system-container"},
+ // A short container ID binds no alert reference, so name the resource directly.
+ {"demo-docker-container-core-01-uptime-kuma", containerResourceID, "app-container"},
+ }
+ for _, tc := range cases {
+ finding := service.findings.Get(tc.findingID)
+ if finding == nil {
+ t.Fatalf("expected finding %s", tc.findingID)
+ }
+ if _, ok := registry.Get(tc.resourceID); !ok {
+ t.Fatalf("%s: expected %q to be a unified resource", tc.findingID, tc.resourceID)
+ }
+ if finding.ResourceID != tc.resourceID {
+ t.Fatalf("%s: resource ID = %q, want unified %q", tc.findingID, finding.ResourceID, tc.resourceID)
+ }
+ if finding.ResourceType != tc.resourceType {
+ t.Fatalf("%s: resource type = %q, want %q", tc.findingID, finding.ResourceType, tc.resourceType)
+ }
+ onResource := service.GetFindingsForResource(tc.resourceID)
+ if len(onResource) != 1 || onResource[0].ID != tc.findingID {
+ t.Fatalf("%s: findings for resource %q = %v", tc.findingID, tc.resourceID, onResource)
+ }
+ }
+
+ // Both findings restate an active alert, so they fold under it.
+ for findingID, alertID := range map[string]string{
+ "demo-docker-host-offline-edge-01": offlineAlert.ID,
+ "demo-storage-capacity-pve1-local-zfs": usageAlert.ID,
+ } {
+ if got := service.findings.Get(findingID).MirrorsAlertID; got != alertID {
+ t.Fatalf("%s: mirrors alert %q, want %q", findingID, got, alertID)
+ }
+ }
+
+ // Without a read state the views are derived from the same snapshot.
+ for _, derived := range synthesizeDemoPatrolFindings(newPatrolRuntimeState(snapshot), now) {
+ if want := service.findings.Get(derived.ID); want == nil || derived.ResourceID != want.ResourceID {
+ t.Fatalf("%s: derived resource ID %q differs from the read state's", derived.ID, derived.ResourceID)
+ }
+ }
+
+ // A registry that has not ingested the snapshot yet must not leave the
+ // first cycle's findings on source IDs: the store keeps a finding's
+ // resource for as long as the process runs.
+ early := NewPatrolService(nil, demoSnapshotProvider{snapshot: snapshot})
+ early.SetReadState(unifiedresources.NewRegistry(nil))
+ early.runDemoPatrolCycle(TriggerReasonStartup)
+ for _, tc := range cases {
+ finding := early.findings.Get(tc.findingID)
+ if finding == nil {
+ t.Fatalf("expected finding %s before the registry ingests", tc.findingID)
+ }
+ if finding.ResourceID != tc.resourceID {
+ t.Fatalf("%s: resource ID before the registry ingests = %q, want %q", tc.findingID, finding.ResourceID, tc.resourceID)
+ }
+ }
+
+ // Once the registry holds resources it is the only authority. A record it
+ // lacks or refuses (an operator link can fold a Docker host into a guest)
+ // keeps its source ID instead of an ID from a registry without those links.
+ partial := snapshot
+ partial.DockerHosts = nil
+ partialRegistry := unifiedresources.NewRegistry(nil)
+ partialRegistry.IngestSnapshot(partial)
+ lagging := NewPatrolService(nil, demoSnapshotProvider{snapshot: snapshot})
+ lagging.SetReadState(partialRegistry)
+ lagging.runDemoPatrolCycle(TriggerReasonStartup)
+ for findingID, want := range map[string]string{
+ "demo-docker-host-offline-edge-01": "edge-01-mock",
+ "demo-storage-capacity-pve1-local-zfs": AttentionSubjectResourceID(registry, usageAlert.ResourceID),
+ } {
+ finding := lagging.findings.Get(findingID)
+ if finding == nil {
+ t.Fatalf("expected finding %s with a registry lacking Docker hosts", findingID)
+ }
+ if finding.ResourceID != want {
+ t.Fatalf("%s: resource ID with a registry lacking Docker hosts = %q, want %q", findingID, finding.ResourceID, want)
+ }
+ }
+}
+
func TestPatrolService_RunDemoPatrolCycle_ResolvesStaleDemoFindings(t *testing.T) {
original := mockruntime.IsEnabled()
t.Cleanup(func() { mockruntime.SetEnabled(original) })