Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Stop Patrol resolving Docker alerts the snapshot still shows
rcourtmanbee3a94ce65e
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.mdindex 19b7ef5ffa..accf9a6fca 100644--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md@@ -4192,13 +4192,38 @@ and cost-gated in the keep direction. Local state never resolves analert: only a parsed model RESOLVE verdict (or operator action) clearsone. But an aged active alert whose trigger condition demonstrablystill holds in the current runtime snapshot — a metric alert whose-looked-up current value is still at/above its positive threshold, or-an offline-family alert whose node/agent/VM/system-container resource-is present and still not online (`alertStillFiringState`) — must be-skipped without a billed model call, because a correct review could-only answer KEEP. Every uncertain case (unknown alert type, missing-resource, unmapped metric, non-positive threshold, resource types the-online check does not model) still goes to the model. The remaining+looked-up current value is still at/above its positive threshold, an+offline-family alert whose node/agent/VM/system-container resource+is present and still not online, or a Docker container alert whose+container still reports what raised it (a health result other than+healthy, starting or none while running; a runtime state the alert+subsystem's state alert covers; exited or dead with OOM-killed set)+(`alertStillFiringState`) — must be skipped without a billed model+call, because a correct review could only answer KEEP and a wrong+RESOLVE is raised again by the alert engine on the resource's next+report, so every Patrol pass would flap the alert and its+notifications. A restart-loop alert whose container is caught in+Docker's restart backoff is kept the same way, on weaker evidence: the+snapshot carries no restart window, so this shows restarting, not the+alert subsystem's count, and that subsystem clears the alert itself+once its window empties. A Docker container alert is+judged against the container its `docker:<host>/<container>` resource+ID names, never a same-named container on another host. Every+uncertain case (unknown alert type, missing resource, unmapped metric,+non-positive threshold, resource types the online check does not+model, a restart loop on a running container) still goes to the+model. The review context for a Docker container carries its health,+restart count and OOM kill (yes, no or unknown), and the prompt makes a+condition the shown state does not cover a KEEP: normal readings for+other metrics are not recovery. When the context lookup has no row for+an alert's subject but current inventory still holds it or its owner+by durable identity (`ResolveAlertResourceReference`: a ZFS pool's+storage, a guest's backup, a Swarm service), the context says the+resource or its owner is present rather than not found, so a lookup+gap is never offered to the model as removal. Owner presence does not+prove a child (a pool device, a host's disk) survives; an alert the+alert engine stops observing is cleared by its stale-alert sweep after+24 hours. The remainingcandidates are reviewed in batched `QuickAnalysis` calls of at most`patrolAlertReviewBatchSize` alerts with numbered per-alert verdictlines; verdict parsing is fail-safe (unparseable or out-of-rangediff --git a/internal/ai/patrol_ai.go b/internal/ai/patrol_ai.goindex f7df72d704..75340bead6 100644--- a/internal/ai/patrol_ai.go+++ b/internal/ai/patrol_ai.go@@ -1865,9 +1865,14 @@ type patrolDockerHostRow struct {type patrolAppContainerRow struct {id, name, status string- health string- restartCount int- cpu, memory float64+ // alertResourceID is the container's ID in the alert subsystem+ // ("docker:<host>/<container ID>"), so an alert resolves to this+ // container rather than a same-named one on another host.+ alertResourceID string+ health string+ restartCount int+ oomKilled *bool+ cpu, memory float64}type patrolStoragePoolRow struct {@@ -2189,13 +2194,15 @@ func patrolAppContainerRows(snap patrolRuntimeState, scopedSet map[string]bool)continue}rows = append(rows, patrolAppContainerRow{- id: cv.ID(),- name: cv.Name(),- status: strings.TrimSpace(cv.ContainerState()),- health: strings.TrimSpace(cv.Health()),- restartCount: cv.RestartCount(),- cpu: cv.CPUPercent(),- memory: cv.MemoryPercent(),+ id: cv.ID(),+ name: cv.Name(),+ status: strings.TrimSpace(cv.ContainerState()),+ alertResourceID: dockerAlertScopeAlias(cv.HostSourceID(), cv.ContainerID(), cv.Name()),+ health: strings.TrimSpace(cv.Health()),+ restartCount: cv.RestartCount(),+ oomKilled: cv.OOMKilled(),+ cpu: cv.CPUPercent(),+ memory: cv.MemoryPercent(),})}return rows@@ -2212,13 +2219,15 @@ func patrolAppContainerRows(snap patrolRuntimeState, scopedSet map[string]bool)continue}rows = append(rows, patrolAppContainerRow{- id: container.ID,- name: container.Name,- status: container.State,- health: container.Health,- restartCount: container.RestartCount,- cpu: container.CPUPercent,- memory: container.MemoryPercent,+ id: container.ID,+ name: container.Name,+ status: container.State,+ alertResourceID: dockerAlertScopeAlias(host.ID, container.ID, container.Name),+ health: container.Health,+ restartCount: container.RestartCount,+ oomKilled: container.OOMKilled,+ cpu: container.CPUPercent,+ memory: container.MemoryPercent,})}}diff --git a/internal/ai/patrol_alerts_review_test.go b/internal/ai/patrol_alerts_review_test.goindex 4a7bdf44ca..a672b4eb71 100644--- a/internal/ai/patrol_alerts_review_test.go+++ b/internal/ai/patrol_alerts_review_test.go@@ -2,9 +2,11 @@ package aiimport ("context"+ "strings""testing""time"+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts""github.com/rcourtman/pulse-go-rewrite/internal/config""github.com/rcourtman/pulse-go-rewrite/internal/models""github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"@@ -226,3 +228,191 @@ func TestReviewAndResolveAlerts_BatchedModelReviewResolves(t *testing.T) {t.Fatalf("expected only the model-resolved alert to clear, got %v", resolver.clears)}}++// dockerAlertReviewStates returns the same Docker hosts as a legacy snapshot+// and as canonical read state, the form a production Patrol run reads.+func dockerAlertReviewStates(hosts ...models.DockerHost) map[string]patrolRuntimeState {+ snapshot := models.StateSnapshot{DockerHosts: hosts}+ registry := unifiedresources.NewRegistry(nil)+ registry.IngestSnapshot(snapshot)+ return map[string]patrolRuntimeState{+ "snapshot": newPatrolRuntimeState(snapshot),+ "read state": {readState: registry},+ }+}++// dockerContainerAlert builds an alert the way the alert subsystem raises+// it, keyed by its "docker:<host>/<container>" resource ID.+func dockerContainerAlert(alertType, hostID string, container models.DockerContainer) AlertInfo {+ return AlertInfo{+ ID: alertType + "-" + hostID + "-" + container.ID,+ Type: alertType,+ ResourceID: alerts.DockerContainerResourceID(hostID, container.ID, container.Name),+ ResourceName: container.Name,+ ResourceType: "app-container",+ StartTime: time.Now().Add(-30 * time.Minute),+ }+}++func TestAlertStillFiring_DockerContainerConditions(t *testing.T) {+ oom, noOOM := true, false+ cases := []struct {+ name string+ alertType string+ container models.DockerContainer+ want bool+ }{+ {"unhealthy while running", "docker-container-health", models.DockerContainer{State: "running", Health: "unhealthy"}, true},+ {"unhealthy with no reported state", "docker-container-health", models.DockerContainer{Health: "unhealthy"}, true},+ {"healthy again", "docker-container-health", models.DockerContainer{State: "running", Health: "healthy"}, false},+ {"health check starting", "docker-container-health", models.DockerContainer{State: "running", Health: "starting"}, false},+ {"health check removed", "docker-container-health", models.DockerContainer{State: "running"}, false},+ {"stopped with stale unhealthy result", "docker-container-health", models.DockerContainer{State: "exited", Health: "unhealthy"}, false},+ {"still exited", "docker-container-state", models.DockerContainer{State: "exited"}, true},+ {"still paused", "docker-container-state", models.DockerContainer{State: "paused"}, true},+ {"running again", "docker-container-state", models.DockerContainer{State: "running"}, false},+ {"state not reported", "docker-container-state", models.DockerContainer{}, false},+ {"still OOM-killed", "docker-container-oom-kill", models.DockerContainer{State: "exited", OOMKilled: &oom}, true},+ {"exited without OOM", "docker-container-oom-kill", models.DockerContainer{State: "exited", OOMKilled: &noOOM}, false},+ {"OOM unknown", "docker-container-oom-kill", models.DockerContainer{State: "exited"}, false},+ {"restarted after OOM", "docker-container-oom-kill", models.DockerContainer{State: "running", OOMKilled: &oom}, false},+ {"waiting to restart", "docker-container-restart-loop", models.DockerContainer{State: "restarting"}, true},+ {"running between restarts", "docker-container-restart-loop", models.DockerContainer{State: "running"}, false},+ }++ ps := NewPatrolService(nil, nil)+ for _, tc := range cases {+ tc.container.ID, tc.container.Name = "c0ffee", "web"+ alert := dockerContainerAlert(tc.alertType, "docker-host-a", tc.container)+ states := dockerAlertReviewStates(models.DockerHost{ID: "docker-host-a", Hostname: "dockhost-a", Containers: []models.DockerContainer{tc.container}})+ for form, snap := range states {+ if got := ps.alertStillFiringState(alert, snap); got != tc.want {+ t.Errorf("%s (%s): still firing = %t, want %t", tc.name, form, got, tc.want)+ }+ }+ }++ // A container missing from the snapshot may have been removed: that is+ // the model's call, not the gate's.+ gone := dockerContainerAlert("docker-container-health", "docker-host-a", models.DockerContainer{ID: "gone", Name: "web"})+ for form, snap := range dockerAlertReviewStates(models.DockerHost{ID: "docker-host-a", Hostname: "dockhost-a"}) {+ if ps.alertStillFiringState(gone, snap) {+ t.Errorf("missing container (%s): expected model review, gate claimed still firing", form)+ }+ }+}++// Two hosts run a container named "web". Each alert is judged against its+// own container, never the same-named one on the other host.+func TestAlertStillFiring_DockerContainerUsesAlertIdentity(t *testing.T) {+ sick := models.DockerContainer{ID: "aaa111", Name: "web", State: "running", Health: "unhealthy"}+ fine := models.DockerContainer{ID: "bbb222", Name: "web", State: "running", Health: "healthy"}+ states := dockerAlertReviewStates(+ models.DockerHost{ID: "docker-host-b", Hostname: "dockhost-b", Containers: []models.DockerContainer{fine}},+ models.DockerHost{ID: "docker-host-a", Hostname: "dockhost-a", Containers: []models.DockerContainer{sick}},+ )++ ps := NewPatrolService(nil, nil)+ sickAlert := dockerContainerAlert("docker-container-health", "docker-host-a", sick)+ fineAlert := dockerContainerAlert("docker-container-health", "docker-host-b", fine)+ for form, snap := range states {+ if !ps.alertStillFiringState(sickAlert, snap) {+ t.Errorf("%s: unhealthy container's alert was judged against its healthy namesake", form)+ }+ if ps.alertStillFiringState(fineAlert, snap) {+ t.Errorf("%s: healthy container's alert was judged against its unhealthy namesake", form)+ }+ if got := ps.getResourceCurrentStateState(fineAlert, snap); !strings.Contains(got, "Health: healthy") {+ t.Errorf("%s: review context for the healthy container = %q", form, got)+ }+ }+}++// The live run on 24 Oct 2026: the review resolved a Docker health alert+// from CPU and memory ("no unhealthy indicator present") while Docker still+// reported the container unhealthy, and the alert engine raised it again on+// the agent's next report. An unhealthy container keeps its alert without a+// model call, whatever the model would have answered.+func TestReviewAndResolveAlerts_UnhealthyContainerKeepsHealthAlert(t *testing.T) {+ container := models.DockerContainer{ID: "5eed", Name: "pulse-live-web", State: "running", Health: "unhealthy", CPUPercent: 0.1, MemoryPercent: 1.2}+ for form, snap := range dockerAlertReviewStates(models.DockerHost{ID: "docker-host-a", Hostname: "dockhost-a", Containers: []models.DockerContainer{container}}) {+ t.Run(form, func(t *testing.T) {+ ps := NewPatrolService(nil, nil)+ resolver := &stubAlertResolver{alerts: []AlertInfo{dockerContainerAlert("docker-container-health", "docker-host-a", container)}}+ ps.alertResolver = resolver+ provider := &mockPatrolProvider{response: "ALERT 1: RESOLVE: Container is running with normal CPU/Memory, no unhealthy indicator present"}+ ps.aiService = &Service{provider: provider, cfg: &config.AIConfig{Enabled: true, PatrolModel: "mock:model"}}++ if resolved := ps.reviewAndResolveAlertsState(context.Background(), snap, true, ""); resolved != 0 {+ t.Fatalf("expected the unhealthy container's alert to stay active, resolved %d", resolved)+ }+ if provider.calls != 0 || len(resolver.clears) != 0 {+ t.Fatalf("expected no model call and no clear, got calls=%d clears=%v", provider.calls, resolver.clears)+ }+ })+ }+}++// Once Docker reports the container healthy, the alert goes to the model+// with that health in its context, and a RESOLVE verdict clears it.+func TestReviewAndResolveAlerts_RecoveredContainerHealthAlertResolves(t *testing.T) {+ container := models.DockerContainer{ID: "5eed", Name: "pulse-live-web", State: "running", Health: "healthy"}+ for form, snap := range dockerAlertReviewStates(models.DockerHost{ID: "docker-host-a", Hostname: "dockhost-a", Containers: []models.DockerContainer{container}}) {+ t.Run(form, func(t *testing.T) {+ ps := NewPatrolService(nil, nil)+ alert := dockerContainerAlert("docker-container-health", "docker-host-a", container)+ resolver := &stubAlertResolver{alerts: []AlertInfo{alert}}+ ps.alertResolver = resolver+ provider := &mockPatrolProvider{response: "ALERT 1: RESOLVE: health check passing again"}+ ps.aiService = &Service{provider: provider, cfg: &config.AIConfig{Enabled: true, PatrolModel: "mock:model"}}++ if resolved := ps.reviewAndResolveAlertsState(context.Background(), snap, true, ""); resolved != 1 {+ t.Fatalf("expected the recovered container's alert to resolve, resolved %d", resolved)+ }+ if provider.calls != 1 || len(resolver.clears) != 1 || resolver.clears[0] != alert.ID {+ t.Fatalf("expected one model call clearing %s, got calls=%d clears=%v", alert.ID, provider.calls, resolver.clears)+ }+ prompt := provider.lastReq.Messages[1].Content+ if !strings.Contains(prompt, "Docker container 'pulse-live-web'") || !strings.Contains(prompt, "Health: healthy") {+ t.Fatalf("expected the review context to carry the container's health, got %q", prompt)+ }+ if !strings.Contains(prompt, "If the current state does not cover the condition the alert is about, answer KEEP") {+ t.Fatalf("expected the review prompt to treat missing evidence as KEEP, got %q", prompt)+ }+ })+ }+}++// A ZFS pool alert names "<storage>/zfs-pool:<pool>", which no storage row+// matches. The storage is still in inventory, so the review must not be told+// the resource may have been removed, which invites a RESOLVE while the pool+// still reports its errors. A storage that is really gone keeps that wording.+func TestGetResourceCurrentState_PresentChildSubjectIsNotRemoved(t *testing.T) {+ storage := models.Storage{+ ID: "pve1-node1-local-zfs", Name: "local-zfs", Node: "node1", Instance: "pve1", Type: "zfspool",+ Status: "available", Total: 100, Used: 40, Free: 60, Usage: 40,+ ZFSPool: &models.ZFSPool{Name: "rpool", State: "ONLINE", ChecksumErrors: 12},+ }+ vm := models.VM{ID: "pve1-node1-100", VMID: 100, Name: "web", Node: "node1", Instance: "pve1", Type: "qemu", Status: "running"}+ snapshot := models.StateSnapshot{Storage: []models.Storage{storage}, VMs: []models.VM{vm}}+ registry := unifiedresources.NewRegistry(nil)+ registry.IngestSnapshot(snapshot)++ ps := NewPatrolService(nil, nil)+ present := AlertInfo{ID: "zfs-errors", Type: "zfs-pool-errors", ResourceID: storage.ID + "/zfs-pool:rpool", ResourceName: "local-zfs (rpool)", ResourceType: "storage"}+ // A backup-age alert names its guest by alerts.BuildGuestKey.+ backup := AlertInfo{ID: "backup-age", Type: "backup-age", ResourceID: alerts.BuildGuestKey("pve1", "node1", 100), ResourceName: "web backup", ResourceType: "vm"}+ gone := present+ gone.ResourceID = "pve1-node1-retired/zfs-pool:rpool"+ for form, snap := range map[string]patrolRuntimeState{"snapshot": newPatrolRuntimeState(snapshot), "read state": {readState: registry}} {+ if got := ps.getResourceCurrentStateState(present, snap); !strings.Contains(got, "Present in current inventory") {+ t.Errorf("%s: present pool's storage described as %q", form, got)+ }+ if got := ps.getResourceCurrentStateState(backup, snap); !strings.Contains(got, "Present in current inventory") {+ t.Errorf("%s: backed-up guest described as %q", form, got)+ }+ if got := ps.getResourceCurrentStateState(gone, snap); got != "Storage not found in current state (may have been removed)" {+ t.Errorf("%s: missing storage described as %q", form, got)+ }+ }+}diff --git a/internal/ai/patrol_run.go b/internal/ai/patrol_run.goindex e34dafe56f..ed487911f8 100644--- a/internal/ai/patrol_run.go+++ b/internal/ai/patrol_run.go@@ -2425,10 +2425,12 @@ func (p *PatrolService) GetCurrentStreamOutput() (string, string) {//// Two structural cost controls run before any model call:// - alerts whose trigger condition demonstrably still holds in the current-// snapshot (metric still at/above threshold, resource still offline) are-// skipped — a correct review could only answer KEEP, so the question is-// not worth a billed call. Resolution stays model-owned: Pulse never-// locally decides that an issue HAS cleared.+// snapshot (metric still at/above threshold, resource still offline,+// Docker container still unhealthy, stopped or OOM-killed) are skipped —+// a correct review could only answer KEEP, and a wrong RESOLVE would be+// raised again by the alert engine on the resource's next report. The+// question is not worth a billed call. Resolution stays model-owned:+// Pulse never locally decides that an issue HAS cleared.// - the remaining candidates are reviewed in batched model calls instead// of one call per alert.func (p *PatrolService) reviewAndResolveAlertsState(ctx context.Context, state patrolRuntimeState, llmAllowed bool, executionID string) int {@@ -2453,6 +2455,10 @@ func (p *PatrolService) reviewAndResolveAlertsState(ctx context.Context, state pif aiSvc == nil || !aiSvc.IsEnabled() {return 0}+ if state.alertResourceResolver == nil {+ // Built once: each review context may consult it.+ state.alertResourceResolver = patrolAlertReferenceResolver(state)+ }// Only review alerts that have been active for at least 10 minutes// This avoids thrashing on transient alerts@@ -2512,12 +2518,15 @@ func (p *PatrolService) reviewAndResolveAlertsState(ctx context.Context, state p// alertStillFiringState reports whether the current runtime snapshot still// demonstrably shows the alert's trigger condition — a metric alert whose-// current value is still at or above its threshold, or an offline alert-// whose resource is still not online. For these alerts a correct model-// review can only answer KEEP, so Patrol skips the billed call and keeps-// the alert. The gate never decides that an issue HAS cleared: anything-// uncertain (unknown alert type, missing resource, unmapped metric)-// returns false so the model still owns the resolution decision.+// current value is still at or above its threshold, an offline alert+// whose resource is still not online, or a Docker container alert whose+// container still reports the state that raised it (for a restart loop,+// a container caught in Docker's restart backoff). For these alerts a+// correct model review can only answer KEEP, so Patrol skips the billed+// call and keeps the alert. The gate never decides that an issue HAS+// cleared: anything uncertain (unknown alert type, missing resource,+// unmapped metric) returns false so the model still owns the resolution+// decision.func (p *PatrolService) alertStillFiringState(alert AlertInfo, snap patrolRuntimeState) bool {switch alert.Type {case "cpu", "memory", "usage", "disk":@@ -2540,6 +2549,55 @@ func (p *PatrolService) alertStillFiringState(alert AlertInfo, snap patrolRuntimdefault:return false}+ case "docker-container-health", "docker-container-state", "docker-container-oom-kill", "docker-container-restart-loop":+ resource := lookupPatrolAlertResourceState(alert, snap)+ if !resource.found || resource.resourceType != "app-container" {+ return false+ }+ return dockerContainerAlertStillFiring(alert.Type, resource)+ default:+ return false+ }+}++// dockerContainerAlertingStates are the runtime states the alert subsystem+// raises docker-container-state for (checkDockerContainerState).+var dockerContainerAlertingStates = map[string]bool{+ "created": true, "restarting": true, "removing": true, "paused": true,+ "exited": true, "dead": true, "unknown": true,+}++// dockerContainerAlertStillFiring mirrors the alert subsystem's Docker+// container health, state and OOM checks (internal/alerts/docker.go)+// against the container's current report. It answers only in the keep+// direction: a container that may have recovered returns false and goes+// to the model.+func dockerContainerAlertStillFiring(alertType string, container patrolAlertResourceState) bool {+ state := strings.ToLower(strings.TrimSpace(container.status))+ switch alertType {+ case "docker-container-health":+ // Health is live evidence only while the container runs; a+ // stopped container's last health result belongs to the state+ // alert, which the alert subsystem raises instead.+ if state != "" && state != "running" {+ return false+ }+ switch strings.ToLower(strings.TrimSpace(container.health)) {+ case "", "none", "healthy", "starting":+ return false+ }+ return true+ case "docker-container-state":+ return dockerContainerAlertingStates[state]+ case "docker-container-oom-kill":+ return (state == "exited" || state == "dead") && container.oomKilled != nil && *container.oomKilled+ case "docker-container-restart-loop":+ // The alert subsystem counts restarts over a window the snapshot+ // does not carry, so this is not its check. A container caught in+ // Docker's restart backoff is still restarting, though; keeping+ // its alert defers to the alert subsystem, which clears it once+ // the window empties.+ return state == "restarting"default:return false}@@ -2551,6 +2609,9 @@ type patrolAlertResourceState struct {platform stringname stringstatus string+ health string+ restartCount int+ oomKilled *boolcpu float64memory float64guestMemory *patrolGuestMemoryReading@@ -2610,15 +2671,37 @@ func lookupPatrolAlertResourceState(alert AlertInfo, snap patrolRuntimeState) pareturn patrolAlertResourceState{}}+// patrolAlertSubjectInInventory reports whether current inventory holds the+// alert's subject, or the resource that owns it, by durable identity.+func patrolAlertSubjectInInventory(alert AlertInfo, snap patrolRuntimeState) bool {+ if strings.TrimSpace(alert.ResourceID) == "" {+ return false+ }+ binding, _ := patrolAlertReferenceResolver(snap).ResolveAlertResourceReference(alert.ResourceID)+ return binding.ResourceID != ""+}+func patrolLookupAppContainerAlertResourceState(alert AlertInfo, snap patrolRuntimeState) (patrolAlertResourceState, bool) {- for _, container := range patrolAppContainerRows(snap, nil) {- if !patrolAlertNameMatches(alert, container.id, container.name) {+ rows := patrolAppContainerRows(snap, nil)+ // An alert subsystem container ID ("docker:<host>/<container>") names+ // one container on one host, so it alone decides: a display-name match+ // could judge the alert against a same-named container elsewhere.+ exact := strings.HasPrefix(alert.ResourceID, "docker:")+ for _, container := range rows {+ if exact {+ if container.alertResourceID != alert.ResourceID {+ continue+ }+ } else if !patrolAlertNameMatches(alert, container.id, container.name) {continue}return patrolAlertResourceState{resourceType: "app-container",name: container.name,status: container.status,+ health: container.health,+ restartCount: container.restartCount,+ oomKilled: container.oomKilled,cpu: container.cpu,memory: container.memory,found: true,@@ -2856,7 +2939,9 @@ func (p *PatrolService) reviewAlertBatchState(ctx context.Context, batch []Alert// buildAlertReviewPromptState formats the batched alert-review prompt: one// numbered section per alert with its trigger details and current resource-// state, plus strict per-line verdict output instructions.+// state, plus strict per-line verdict output instructions. The current+// state covers only some alert conditions, so the prompt says that a+// condition it does not show is a KEEP, not a recovery.func (p *PatrolService) buildAlertReviewPromptState(batch []AlertInfo, snap patrolRuntimeState) string {var sb strings.Buildersb.WriteString("Review these active alerts and determine, for each, if it should be auto-resolved based on current state.\n")@@ -2881,6 +2966,7 @@ CURRENT STATE OF THIS RESOURCE:}sb.WriteString(`For each alert, should it be RESOLVED because the underlying issue is fixed?+Answer RESOLVE only when the current state shown for that alert directly shows its condition has cleared (for example the metric is back under its threshold, the resource is online again, or the reported health is healthy), or shows the resource no longer exists. If the current state does not cover the condition the alert is about, answer KEEP: normal readings for other metrics are not evidence that it is fixed.Respond with EXACTLY one line per alert, in order, and nothing else:ALERT <number>: RESOLVE: <brief reason>or@@ -2921,6 +3007,13 @@ func patrolAlertResolveReason(raw string) string {func (p *PatrolService) getResourceCurrentStateState(alert AlertInfo, snap patrolRuntimeState) string {resource := lookupPatrolAlertResourceState(alert, snap)if !resource.found {+ // The lookup reads only some alert subjects. One the canonical+ // inventory still holds, or whose owner it holds (a ZFS pool's+ // storage, a guest's backup, a Swarm service), is not missing, and+ // calling it missing invites the model to resolve it as removed.+ if patrolAlertSubjectInInventory(alert, snap) {+ return "Present in current inventory (this alert's resource or the resource that owns it); this review has no current reading of the alert's condition"+ }switch patrolAlertLookupType(alert) {case "storage":return "Storage not found in current state (may have been removed)"@@ -2958,13 +3051,36 @@ func (p *PatrolService) getResourceCurrentStateState(alert AlertInfo, snap patroreturn fmt.Sprintf("Container '%s': CPU %.1f%%, Memory %s, Status: %s",resource.name, resource.cpu, resource.guestMemory.display(), resource.status)case "app-container":- return fmt.Sprintf("Docker container '%s': CPU %.1f%%, Memory %.1f%%, State: %s",- resource.name, resource.cpu, resource.memory, resource.status)+ return fmt.Sprintf("Docker container '%s': CPU %.1f%%, Memory %.1f%%, State: %s, Health: %s, Restart count: %d, OOM-killed: %s",+ resource.name, resource.cpu, resource.memory, resource.status,+ dockerContainerHealthText(resource.health), resource.restartCount, dockerContainerOOMText(resource.oomKilled))default:return "Resource state unknown"}}+// dockerContainerHealthText names the container's reported health check+// result, so a health alert's review sees the reading that raised it.+func dockerContainerHealthText(health string) string {+ if health = strings.TrimSpace(health); health == "" {+ return "none"+ }+ return health+}++// dockerContainerOOMText keeps an explicit "no" apart from an agent that+// does not report OOM kills, which proves nothing either way.+func dockerContainerOOMText(oomKilled *bool) string {+ switch {+ case oomKilled == nil:+ return "unknown"+ case *oomKilled:+ return "yes"+ default:+ return "no"+ }+}+// TriggerPatrolForAlert triggers an immediate patrol for a specific alertfunc (p *PatrolService) TriggerPatrolForAlert(alert *alerts.Alert) {if alert == nil {