Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Rank a confirmed availability outage first in the summary check
rcourtmanade799307a54
diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.mdindex 53e49325b6..5a4ebb001d 100644--- a/docs/release-control/v6/internal/subsystems/unified-resources.md+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md@@ -3443,6 +3443,20 @@ It owns probe status, incidents, history, evidence, and the outgoing `checks`relationship. The matched resource carries every correlated check in the canonical`availabilityChecks` facet, keyed by saved target id, while `availability`remains an additive singular compatibility summary selected from that set.+The summary is the worst check: a confirmed outage first, then any other+observed failure, then an unchecked, ambiguous or unresolved check, then a+passing one, ties going to the first saved target id. A confirmed outage is an+enabled check whose observed failures reach its failure threshold, with an+unavailable aggregate when one is reported and an outcome that is not+indeterminate. That is the gate the availability poller applies before it+raises `availability_unreachable`, so a probe agent that stops reporting keeps+its old failure count but confirms nothing. `availabilityOutageConfirmed` in+`internal/unifiedresources/availability.go` holds that rule for both the+ranking and `EvaluateResourceHealth`, which reads the summary. A confirmed+outage on any attached check therefore makes the resource's health critical+even when a check that sorts earlier has failed below its threshold, with the+`availability_failed` reason unless a critical alert explains the verdict+first.Adding a second explicit or unambiguously correlated check must retain bothsource-owned endpoint rows, project both facets onto the same resource, andemit one `checks` relationship per target from the check to that resource.diff --git a/internal/unifiedresources/availability.go b/internal/unifiedresources/availability.goindex 35620d04da..87c5d9ac21 100644--- a/internal/unifiedresources/availability.go+++ b/internal/unifiedresources/availability.go@@ -99,6 +99,10 @@ func availabilityCheckKey(check AvailabilityData) string {return fmt.Sprintf("endpoint:%s:%s:%d:%s", protocol, address, check.Port, path)}+// primaryAvailabilityCheck selects the singular compatibility summary: the+// worst check, ties going to the first by key. Health and row presentation+// read only the summary, so a confirmed outage on any check must outrank a+// failure that has not yet reached its threshold.func primaryAvailabilityCheck(checks []AvailabilityData) *AvailabilityData {if len(checks) == 0 {return nil@@ -113,13 +117,39 @@ func primaryAvailabilityCheck(checks []AvailabilityData) *AvailabilityData {}func availabilityCheckPriority(check AvailabilityData) int {- if check.LastChecked != nil && !check.Available {+ if availabilityOutageConfirmed(check) {return 0}+ if check.LastChecked != nil && !check.Available {+ return 1+ }if check.LastChecked == nil ||check.CorrelationState == AvailabilityCorrelationAmbiguous ||check.CorrelationState == AvailabilityCorrelationUnresolved {- return 1+ return 2}- return 2+ return 3+}++// availabilityOutageConfirmed reports whether an enabled check has failed at+// least its failure threshold in a row, by the same gate the availability+// poller uses before raising availability_unreachable: an observed failure+// whose aggregate, when present, is unavailable. A probe agent that stops+// reporting keeps its failure count but reads indeterminate with an unknown+// aggregate, and confirms nothing.+func availabilityOutageConfirmed(check AvailabilityData) bool {+ if !check.Enabled || check.Available || check.LastChecked == nil {+ return false+ }+ if state := strings.TrimSpace(check.AggregateState); state != "" && !strings.EqualFold(state, "unavailable") {+ return false+ }+ if strings.EqualFold(strings.TrimSpace(check.ProbeOutcome), "indeterminate") {+ return false+ }+ threshold := check.FailureThreshold+ if threshold <= 0 {+ threshold = 1+ }+ return check.ConsecutiveFailures >= threshold}diff --git a/internal/unifiedresources/availability_link_test.go b/internal/unifiedresources/availability_link_test.goindex 1daaa01432..fcb21d0289 100644--- a/internal/unifiedresources/availability_link_test.go+++ b/internal/unifiedresources/availability_link_test.go@@ -403,6 +403,133 @@ func TestAvailabilityKeepsEveryConfiguredCheckWithMultipleServicesOnOneHost(t *t}}+func TestAvailabilitySummaryPrefersConfirmedOutageOverEarlierUnconfirmedFailure(t *testing.T) {+ rr := NewRegistry(nil)+ hostID := ingestAgentFixture(t, rr, "host-1", "machine-1")+ now := time.Now().UTC()+ check := func(targetID, address string, facet AvailabilityData) IngestRecord {+ checkedAt := now+ facet.LinkedResourceID = hostID+ facet.Address = address+ facet.Protocol = "https"+ facet.Enabled = true+ facet.LastChecked = &checkedAt+ facet.FailureThreshold = 3+ return availabilityProbeRecord(targetID, address, &facet)+ }++ // probe-a sorts first by target id and has failed once, below its+ // threshold. probe-b is past its threshold and owns the outage incident.+ unconfirmed := check("probe-a", "203.0.113.10", AvailabilityData{+ AggregateState: "unavailable",+ ConsecutiveFailures: 1,+ ApplicationFailureCode: "status_mismatch",+ })+ unconfirmed.Resource.Status = StatusWarning+ confirmed := check("probe-b", "203.0.113.11", AvailabilityData{+ AggregateState: "unavailable",+ ConsecutiveFailures: 3,+ })+ confirmed.Resource.Status = StatusOffline+ confirmed.Resource.Incidents = []ResourceIncident{{+ Provider: string(SourceAvailability),+ NativeID: "probe-b",+ Code: "availability_unreachable",+ }}+ rr.IngestRecords(SourceAvailability, []IngestRecord{unconfirmed, confirmed})++ host, ok := rr.Get(hostID)+ if !ok || host == nil {+ t.Fatalf("host %q missing after ingest", hostID)+ }+ if len(AvailabilityChecksForResource(*host)) != 2 {+ t.Fatalf("availability checks = %+v, want both attached checks", host.AvailabilityChecks)+ }+ if host.Availability == nil || host.Availability.TargetID != "probe-b" {+ t.Fatalf("availability summary = %+v, want the confirmed outage on probe-b", host.Availability)+ }+ health := EvaluateResourceHealth(*host, nil, now)+ if health.Verdict != HealthCritical || len(health.Reasons) == 0 ||+ health.Reasons[0].Code != "availability_failed" || health.Reasons[0].Detail != "" {+ t.Fatalf("host health = %+v, want availability_failed from probe-b", health)+ }++ // Once probe-b recovers, the unconfirmed failure is the worst check left:+ // it is the summary again, and it does not make the host an outage.+ rr.IngestRecords(SourceAvailability, []IngestRecord{+ check("probe-b", "203.0.113.11", AvailabilityData{+ AggregateState: "healthy",+ Available: true,+ }),+ })+ host, _ = rr.Get(hostID)+ if host.Availability == nil || host.Availability.TargetID != "probe-a" {+ t.Fatalf("availability summary = %+v, want the remaining failure on probe-a", host.Availability)+ }+ health = EvaluateResourceHealth(*host, nil, now)+ for _, reason := range health.Reasons {+ if reason.Code == "availability_failed" {+ t.Fatalf("host health = %+v, want no outage below the failure threshold", health)+ }+ }+}++func TestAvailabilitySummaryDoesNotConfirmOutageFromSilentProbeAgent(t *testing.T) {+ rr := NewRegistry(nil)+ hostID := ingestAgentFixture(t, rr, "host-1", "machine-1")+ now := time.Now().UTC()+ checkedAt := now+ unconfirmed := availabilityProbeRecord("probe-a", "203.0.113.10", &AvailabilityData{+ LinkedResourceID: hostID,+ Address: "203.0.113.10",+ Protocol: "https",+ Enabled: true,+ AggregateState: "unavailable",+ ProbeOutcome: "unreachable",+ LastChecked: &checkedAt,+ ConsecutiveFailures: 1,+ FailureThreshold: 3,+ })+ unconfirmed.Resource.Status = StatusWarning++ // The poller's read of a single-location probe whose agent stopped+ // reporting: the old failure count survives, but the outcome is+ // indeterminate, the aggregate unknown, and no incident is raised.+ lastReport := now.Add(-time.Hour)+ silent := availabilityProbeRecord("probe-b", "203.0.113.11", &AvailabilityData{+ LinkedResourceID: hostID,+ Address: "203.0.113.11",+ Protocol: "https",+ ProbeAgentID: "agent-probe-1",+ Enabled: true,+ AggregateState: "unknown",+ ProbeOutcome: "indeterminate",+ ExpectedLocations: 1,+ LastChecked: &lastReport,+ ConsecutiveFailures: 5,+ FailureThreshold: 3,+ LastError: "no recent report from probe agent",+ })+ silent.Resource.Status = StatusWarning+ rr.IngestRecords(SourceAvailability, []IngestRecord{unconfirmed, silent})++ host, ok := rr.Get(hostID)+ if !ok || host == nil {+ t.Fatalf("host %q missing after ingest", hostID)+ }+ if host.Availability == nil || host.Availability.TargetID != "probe-a" {+ t.Fatalf("availability summary = %+v, want probe-a: a silent probe confirms no outage", host.Availability)+ }+ for _, resource := range []Resource{*host, availabilityEndpointByTarget(t, rr, "probe-b")} {+ health := EvaluateResourceHealth(resource, nil, now)+ for _, reason := range health.Reasons {+ if reason.Code == "availability_failed" {+ t.Fatalf("%s health = %+v, want no outage from a silent probe agent", resource.ID, health)+ }+ }+ }+}+func TestAvailabilityEditReplacesEndpointAndMovesProjection(t *testing.T) {rr := NewRegistry(nil)hostA := ingestAgentFixture(t, rr, "host-a", "machine-a")diff --git a/internal/unifiedresources/health.go b/internal/unifiedresources/health.goindex fabf9c7bd0..ef74dc69ea 100644--- a/internal/unifiedresources/health.go+++ b/internal/unifiedresources/health.go@@ -187,15 +187,10 @@ func healthAlertSeverity(level string) int {}}+// availabilityFailureConfirmed reads the compatibility summary, which ranks a+// confirmed outage on any attached check first.func availabilityFailureConfirmed(resource Resource) bool {- if resource.Availability == nil || !resource.Availability.Enabled || resource.Availability.Available {- return false- }- threshold := resource.Availability.FailureThreshold- if threshold <= 0 {- threshold = 1- }- return resource.Availability.ConsecutiveFailures >= threshold+ return resource.Availability != nil && availabilityOutageConfirmed(*resource.Availability)}func availabilityFailureDetail(resource Resource) string {diff --git a/internal/unifiedresources/health_test.go b/internal/unifiedresources/health_test.goindex c51de693e4..696733d0cb 100644--- a/internal/unifiedresources/health_test.go+++ b/internal/unifiedresources/health_test.go@@ -31,7 +31,7 @@ func TestEvaluateResourceHealthPrecedenceAndVerdicts(t *testing.T) {},{name: "confirmed availability failure is critical",- resource: Resource{ID: "check-1", Type: ResourceTypeNetworkEndpoint, Status: StatusOnline, Availability: &AvailabilityData{Enabled: true, Available: false, ConsecutiveFailures: 3, FailureThreshold: 3}},+ resource: Resource{ID: "check-1", Type: ResourceTypeNetworkEndpoint, Status: StatusOnline, Availability: &AvailabilityData{Enabled: true, Available: false, LastChecked: &now, ConsecutiveFailures: 3, FailureThreshold: 3}},want: HealthCritical, code: "availability_failed",},{