Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Rank a confirmed availability outage first in the summary check

rcourtmanade799307a54
diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md
index 53e49325b6..5a4ebb001d 100644
--- a/docs/release-control/v6/internal/subsystems/unified-resources.md
+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md
@@ -3443,6 +3443,20 @@ It owns probe status, incidents, history, evidence, and the outgoing `checks`
relationship. The matched resource carries every correlated check in the canonical
`availabilityChecks` facet, keyed by saved target id, while `availability`
remains an additive singular compatibility summary selected from that set.
+The summary is the worst check: a confirmed outage first, then any other
+observed failure, then an unchecked, ambiguous or unresolved check, then a
+passing one, ties going to the first saved target id. A confirmed outage is an
+enabled check whose observed failures reach its failure threshold, with an
+unavailable aggregate when one is reported and an outcome that is not
+indeterminate. That is the gate the availability poller applies before it
+raises `availability_unreachable`, so a probe agent that stops reporting keeps
+its old failure count but confirms nothing. `availabilityOutageConfirmed` in
+`internal/unifiedresources/availability.go` holds that rule for both the
+ranking and `EvaluateResourceHealth`, which reads the summary. A confirmed
+outage on any attached check therefore makes the resource's health critical
+even when a check that sorts earlier has failed below its threshold, with the
+`availability_failed` reason unless a critical alert explains the verdict
+first.
Adding a second explicit or unambiguously correlated check must retain both
source-owned endpoint rows, project both facets onto the same resource, and
emit one `checks` relationship per target from the check to that resource.
diff --git a/internal/unifiedresources/availability.go b/internal/unifiedresources/availability.go
index 35620d04da..87c5d9ac21 100644
--- a/internal/unifiedresources/availability.go
+++ b/internal/unifiedresources/availability.go
@@ -99,6 +99,10 @@ func availabilityCheckKey(check AvailabilityData) string {
return fmt.Sprintf("endpoint:%s:%s:%d:%s", protocol, address, check.Port, path)
}
+// primaryAvailabilityCheck selects the singular compatibility summary: the
+// worst check, ties going to the first by key. Health and row presentation
+// read only the summary, so a confirmed outage on any check must outrank a
+// failure that has not yet reached its threshold.
func primaryAvailabilityCheck(checks []AvailabilityData) *AvailabilityData {
if len(checks) == 0 {
return nil
@@ -113,13 +117,39 @@ func primaryAvailabilityCheck(checks []AvailabilityData) *AvailabilityData {
}
func availabilityCheckPriority(check AvailabilityData) int {
- if check.LastChecked != nil && !check.Available {
+ if availabilityOutageConfirmed(check) {
return 0
}
+ if check.LastChecked != nil && !check.Available {
+ return 1
+ }
if check.LastChecked == nil ||
check.CorrelationState == AvailabilityCorrelationAmbiguous ||
check.CorrelationState == AvailabilityCorrelationUnresolved {
- return 1
+ return 2
}
- return 2
+ return 3
+}
+
+// availabilityOutageConfirmed reports whether an enabled check has failed at
+// least its failure threshold in a row, by the same gate the availability
+// poller uses before raising availability_unreachable: an observed failure
+// whose aggregate, when present, is unavailable. A probe agent that stops
+// reporting keeps its failure count but reads indeterminate with an unknown
+// aggregate, and confirms nothing.
+func availabilityOutageConfirmed(check AvailabilityData) bool {
+ if !check.Enabled || check.Available || check.LastChecked == nil {
+ return false
+ }
+ if state := strings.TrimSpace(check.AggregateState); state != "" && !strings.EqualFold(state, "unavailable") {
+ return false
+ }
+ if strings.EqualFold(strings.TrimSpace(check.ProbeOutcome), "indeterminate") {
+ return false
+ }
+ threshold := check.FailureThreshold
+ if threshold <= 0 {
+ threshold = 1
+ }
+ return check.ConsecutiveFailures >= threshold
}
diff --git a/internal/unifiedresources/availability_link_test.go b/internal/unifiedresources/availability_link_test.go
index 1daaa01432..fcb21d0289 100644
--- a/internal/unifiedresources/availability_link_test.go
+++ b/internal/unifiedresources/availability_link_test.go
@@ -403,6 +403,133 @@ func TestAvailabilityKeepsEveryConfiguredCheckWithMultipleServicesOnOneHost(t *t
}
}
+func TestAvailabilitySummaryPrefersConfirmedOutageOverEarlierUnconfirmedFailure(t *testing.T) {
+ rr := NewRegistry(nil)
+ hostID := ingestAgentFixture(t, rr, "host-1", "machine-1")
+ now := time.Now().UTC()
+ check := func(targetID, address string, facet AvailabilityData) IngestRecord {
+ checkedAt := now
+ facet.LinkedResourceID = hostID
+ facet.Address = address
+ facet.Protocol = "https"
+ facet.Enabled = true
+ facet.LastChecked = &checkedAt
+ facet.FailureThreshold = 3
+ return availabilityProbeRecord(targetID, address, &facet)
+ }
+
+ // probe-a sorts first by target id and has failed once, below its
+ // threshold. probe-b is past its threshold and owns the outage incident.
+ unconfirmed := check("probe-a", "203.0.113.10", AvailabilityData{
+ AggregateState: "unavailable",
+ ConsecutiveFailures: 1,
+ ApplicationFailureCode: "status_mismatch",
+ })
+ unconfirmed.Resource.Status = StatusWarning
+ confirmed := check("probe-b", "203.0.113.11", AvailabilityData{
+ AggregateState: "unavailable",
+ ConsecutiveFailures: 3,
+ })
+ confirmed.Resource.Status = StatusOffline
+ confirmed.Resource.Incidents = []ResourceIncident{{
+ Provider: string(SourceAvailability),
+ NativeID: "probe-b",
+ Code: "availability_unreachable",
+ }}
+ rr.IngestRecords(SourceAvailability, []IngestRecord{unconfirmed, confirmed})
+
+ host, ok := rr.Get(hostID)
+ if !ok || host == nil {
+ t.Fatalf("host %q missing after ingest", hostID)
+ }
+ if len(AvailabilityChecksForResource(*host)) != 2 {
+ t.Fatalf("availability checks = %+v, want both attached checks", host.AvailabilityChecks)
+ }
+ if host.Availability == nil || host.Availability.TargetID != "probe-b" {
+ t.Fatalf("availability summary = %+v, want the confirmed outage on probe-b", host.Availability)
+ }
+ health := EvaluateResourceHealth(*host, nil, now)
+ if health.Verdict != HealthCritical || len(health.Reasons) == 0 ||
+ health.Reasons[0].Code != "availability_failed" || health.Reasons[0].Detail != "" {
+ t.Fatalf("host health = %+v, want availability_failed from probe-b", health)
+ }
+
+ // Once probe-b recovers, the unconfirmed failure is the worst check left:
+ // it is the summary again, and it does not make the host an outage.
+ rr.IngestRecords(SourceAvailability, []IngestRecord{
+ check("probe-b", "203.0.113.11", AvailabilityData{
+ AggregateState: "healthy",
+ Available: true,
+ }),
+ })
+ host, _ = rr.Get(hostID)
+ if host.Availability == nil || host.Availability.TargetID != "probe-a" {
+ t.Fatalf("availability summary = %+v, want the remaining failure on probe-a", host.Availability)
+ }
+ health = EvaluateResourceHealth(*host, nil, now)
+ for _, reason := range health.Reasons {
+ if reason.Code == "availability_failed" {
+ t.Fatalf("host health = %+v, want no outage below the failure threshold", health)
+ }
+ }
+}
+
+func TestAvailabilitySummaryDoesNotConfirmOutageFromSilentProbeAgent(t *testing.T) {
+ rr := NewRegistry(nil)
+ hostID := ingestAgentFixture(t, rr, "host-1", "machine-1")
+ now := time.Now().UTC()
+ checkedAt := now
+ unconfirmed := availabilityProbeRecord("probe-a", "203.0.113.10", &AvailabilityData{
+ LinkedResourceID: hostID,
+ Address: "203.0.113.10",
+ Protocol: "https",
+ Enabled: true,
+ AggregateState: "unavailable",
+ ProbeOutcome: "unreachable",
+ LastChecked: &checkedAt,
+ ConsecutiveFailures: 1,
+ FailureThreshold: 3,
+ })
+ unconfirmed.Resource.Status = StatusWarning
+
+ // The poller's read of a single-location probe whose agent stopped
+ // reporting: the old failure count survives, but the outcome is
+ // indeterminate, the aggregate unknown, and no incident is raised.
+ lastReport := now.Add(-time.Hour)
+ silent := availabilityProbeRecord("probe-b", "203.0.113.11", &AvailabilityData{
+ LinkedResourceID: hostID,
+ Address: "203.0.113.11",
+ Protocol: "https",
+ ProbeAgentID: "agent-probe-1",
+ Enabled: true,
+ AggregateState: "unknown",
+ ProbeOutcome: "indeterminate",
+ ExpectedLocations: 1,
+ LastChecked: &lastReport,
+ ConsecutiveFailures: 5,
+ FailureThreshold: 3,
+ LastError: "no recent report from probe agent",
+ })
+ silent.Resource.Status = StatusWarning
+ rr.IngestRecords(SourceAvailability, []IngestRecord{unconfirmed, silent})
+
+ host, ok := rr.Get(hostID)
+ if !ok || host == nil {
+ t.Fatalf("host %q missing after ingest", hostID)
+ }
+ if host.Availability == nil || host.Availability.TargetID != "probe-a" {
+ t.Fatalf("availability summary = %+v, want probe-a: a silent probe confirms no outage", host.Availability)
+ }
+ for _, resource := range []Resource{*host, availabilityEndpointByTarget(t, rr, "probe-b")} {
+ health := EvaluateResourceHealth(resource, nil, now)
+ for _, reason := range health.Reasons {
+ if reason.Code == "availability_failed" {
+ t.Fatalf("%s health = %+v, want no outage from a silent probe agent", resource.ID, health)
+ }
+ }
+ }
+}
+
func TestAvailabilityEditReplacesEndpointAndMovesProjection(t *testing.T) {
rr := NewRegistry(nil)
hostA := ingestAgentFixture(t, rr, "host-a", "machine-a")
diff --git a/internal/unifiedresources/health.go b/internal/unifiedresources/health.go
index fabf9c7bd0..ef74dc69ea 100644
--- a/internal/unifiedresources/health.go
+++ b/internal/unifiedresources/health.go
@@ -187,15 +187,10 @@ func healthAlertSeverity(level string) int {
}
}
+// availabilityFailureConfirmed reads the compatibility summary, which ranks a
+// confirmed outage on any attached check first.
func availabilityFailureConfirmed(resource Resource) bool {
- if resource.Availability == nil || !resource.Availability.Enabled || resource.Availability.Available {
- return false
- }
- threshold := resource.Availability.FailureThreshold
- if threshold <= 0 {
- threshold = 1
- }
- return resource.Availability.ConsecutiveFailures >= threshold
+ return resource.Availability != nil && availabilityOutageConfirmed(*resource.Availability)
}
func availabilityFailureDetail(resource Resource) string {
diff --git a/internal/unifiedresources/health_test.go b/internal/unifiedresources/health_test.go
index c51de693e4..696733d0cb 100644
--- a/internal/unifiedresources/health_test.go
+++ b/internal/unifiedresources/health_test.go
@@ -31,7 +31,7 @@ func TestEvaluateResourceHealthPrecedenceAndVerdicts(t *testing.T) {
},
{
name: "confirmed availability failure is critical",
- resource: Resource{ID: "check-1", Type: ResourceTypeNetworkEndpoint, Status: StatusOnline, Availability: &AvailabilityData{Enabled: true, Available: false, ConsecutiveFailures: 3, FailureThreshold: 3}},
+ resource: Resource{ID: "check-1", Type: ResourceTypeNetworkEndpoint, Status: StatusOnline, Availability: &AvailabilityData{Enabled: true, Available: false, LastChecked: &now, ConsecutiveFailures: 3, FailureThreshold: 3}},
want: HealthCritical, code: "availability_failed",
},
{