Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep undated TrueNAS replication failures visible

pulse-triage[bot]4ea721e5e6f6
diff --git a/docs/TRUENAS.md b/docs/TRUENAS.md
index ae9086f5c0..5e08d4cff3 100644
--- a/docs/TRUENAS.md
+++ b/docs/TRUENAS.md
@@ -103,7 +103,10 @@ record in TrueNAS:
releases could show collection time as **Completed**; those undated records
are corrected on upgrade without deleting snapshot history. To judge backup
age or retention, confirm the native snapshot's creation time. Replication
- with no known last-run time is **Unknown**, not a newly completed copy.
+ reporting success with no known last-run time is **Unknown**, not a newly
+ completed copy. **Failed**, **Warning** and **Running** remain visible even
+ without that time; missing dates do not clear a reported problem. Failed
+ and warning events remain in **Attention**, without an invented timestamp.
3. **Check the destination without starting work.** Use the provider's existing
received-snapshot and task records. Confirm the same source snapshot, target
and successful transfer result. Missing **Verified**, **Size** or timestamp
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 697c57035e..15f007d1c6 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -619,11 +619,26 @@ Provider incident nativeSeverity is evidence, not a storage recovery result or i
TrueNAS replication state `FINISHED` is successful completion evidence when a
last-run timestamp exists, matching the provider success event. Explicit error
-text still takes precedence; a missing last-run timestamp remains unknown.
+text still takes precedence. Missing or zero last-run dates cannot establish a
+successful copy: success becomes unknown, but explicit failed, warning and
+running outcomes survive without start/completion timestamps. Collection time
+must not fill those dates. Undated task IDs remain stable across polls and
+separate across connections; dated native outcomes/times are unchanged. Failed
+and warning events remain in the Protection Attention filter even when their
+time is unavailable. Normal ingest corrects the same undated task row; no
+history is deleted and no native task or restore is run.
Persisted recovery rollups must retain this outcome and success time rather
than classify a completed replication as unknown. This does not imply restore
verification or resolve provider INFO alert lifecycle behaviour.
+Verification: `TestFromTrueNASSnapshot_ReplicationTimeAndStateEvidence` covers
+nil/zero/native dates, error precedence and known-state retention;
+`TestFromTrueNASSnapshot_UndatedReplicationIdentity` covers repeated polls,
+connection isolation and non-mutation. `TrueNASProtectionTable.test.tsx` checks
+undated attention/filter/detail behaviour; `truenasProtectionEvidenceDocs.test.ts`
+checks the shipped guidance. Consuming browser proof must use actual mapped
+payloads at desktop and narrow viewports, not a hand-written outcome fixture.
+
Unraid `StorageMeta.numDisks` is optional source evidence: zero denotes an explicitly empty parity array, not proof that pools are protected or that a recovery point exists. Suppressing the no-parity warning for zero must not erase disk-failure evidence or grant storage/recovery authority.
A manifestless PBS snapshot never becomes a successful recovery point and
diff --git a/frontend-modern/public/docs/TRUENAS.md b/frontend-modern/public/docs/TRUENAS.md
index ae9086f5c0..5e08d4cff3 100644
--- a/frontend-modern/public/docs/TRUENAS.md
+++ b/frontend-modern/public/docs/TRUENAS.md
@@ -103,7 +103,10 @@ record in TrueNAS:
releases could show collection time as **Completed**; those undated records
are corrected on upgrade without deleting snapshot history. To judge backup
age or retention, confirm the native snapshot's creation time. Replication
- with no known last-run time is **Unknown**, not a newly completed copy.
+ reporting success with no known last-run time is **Unknown**, not a newly
+ completed copy. **Failed**, **Warning** and **Running** remain visible even
+ without that time; missing dates do not clear a reported problem. Failed
+ and warning events remain in **Attention**, without an invented timestamp.
3. **Check the destination without starting work.** Use the provider's existing
received-snapshot and task records. Confirm the same source snapshot, target
and successful transfer result. Missing **Verified**, **Size** or timestamp
diff --git a/frontend-modern/src/features/docs/__tests__/truenasProtectionEvidenceDocs.test.ts b/frontend-modern/src/features/docs/__tests__/truenasProtectionEvidenceDocs.test.ts
index 8473ed51f5..549df59a01 100644
--- a/frontend-modern/src/features/docs/__tests__/truenasProtectionEvidenceDocs.test.ts
+++ b/frontend-modern/src/features/docs/__tests__/truenasProtectionEvidenceDocs.test.ts
@@ -62,7 +62,10 @@ describe('TrueNAS protection evidence help', () => {
expect(text).toContain('corrected on upgrade without deleting snapshot history');
expect(text).toContain("confirm the native snapshot's creation time");
expect(text).toContain('Running remains incomplete even if a timestamp is present');
- expect(text).toContain('Replication with no known last-run time is Unknown');
+ expect(text).toContain('Replication reporting success with no known last-run time is Unknown');
+ expect(text).toContain('Failed, Warning and Running remain visible even without that time');
+ expect(text).toContain('missing dates do not clear a reported problem');
+ expect(text).toContain('Failed and warning events remain in Attention, without an invented timestamp');
});
it('uses current detail labels to compare the same source and destination without active work', () => {
diff --git a/frontend-modern/src/features/truenas/__tests__/TrueNASProtectionTable.test.tsx b/frontend-modern/src/features/truenas/__tests__/TrueNASProtectionTable.test.tsx
index 7252749280..f9fd85e2f9 100644
--- a/frontend-modern/src/features/truenas/__tests__/TrueNASProtectionTable.test.tsx
+++ b/frontend-modern/src/features/truenas/__tests__/TrueNASProtectionTable.test.tsx
@@ -73,6 +73,57 @@ describe('TrueNASProtectionTable', () => {
expect(document.querySelectorAll('[data-truenas-protection-row]')).toHaveLength(3);
});
+ it('keeps undated replication problems in Attention without inventing completion dates', async () => {
+ renderInRouter(() => (
+ <TrueNASProtectionTable
+ points={(['failed', 'warning', 'running', 'unknown'] as const).map((outcome) =>
+ makeRecoveryPoint({
+ id: outcome,
+ kind: 'backup',
+ mode: 'remote',
+ outcome,
+ startedAt: undefined,
+ completedAt: undefined,
+ details: {
+ taskId: `task-${outcome}`,
+ taskName: `replicate-${outcome}`,
+ sourceDatasets: [`tank/${outcome}`],
+ targetDataset: `vault/${outcome}`,
+ lastState: outcome.toUpperCase(),
+ },
+ }),
+ )}
+ emptyIcon={<span />}
+ emptyTitle="No protection"
+ emptyDescription="No protection"
+ />
+ ));
+
+ await fireEvent.click(screen.getByRole('button', { name: 'Attention, 2' }));
+ const rows = document.querySelectorAll('[data-truenas-protection-row]');
+ expect(rows).toHaveLength(2);
+ for (const outcome of ['failed', 'warning']) {
+ const row = document.querySelector(`[data-truenas-protection-row="${outcome}"]`)!;
+ expect(row).toHaveAttribute('data-truenas-protection-outcome', outcome);
+ expect(row.querySelectorAll('td')[3]).toHaveTextContent('-');
+ }
+
+ await fireEvent.click(document.querySelector('[data-truenas-protection-row="failed"]')!);
+ const detail = within(screen.getByTestId('truenas-protection-detail'));
+ expect(detail.getByText('task-failed')).toBeInTheDocument();
+ expect(detail.getByText('FAILED')).toBeInTheDocument();
+ expect(detail.queryByText('Started', { exact: true })).not.toBeInTheDocument();
+ expect(detail.queryByText('Completed', { exact: true })).not.toBeInTheDocument();
+ await fireEvent.click(detail.getByRole('button', { name: /^Collapse .* details$/ }));
+
+ await fireEvent.click(screen.getByRole('button', { name: 'Running, 1' }));
+ expect(document.querySelectorAll('[data-truenas-protection-row]')).toHaveLength(1);
+ expect(document.querySelector('[data-truenas-protection-row="running"]')).not.toBeNull();
+ await fireEvent.click(screen.getByRole('button', { name: 'Unknown, 1' }));
+ expect(document.querySelectorAll('[data-truenas-protection-row]')).toHaveLength(1);
+ expect(document.querySelector('[data-truenas-protection-row="unknown"]')).not.toBeNull();
+ });
+
it('opens inline table details for a TrueNAS replication recovery point', async () => {
const replication = makeRecoveryPoint({
id: 'replicate-tank-apps',
diff --git a/internal/recovery/mapper/truenas/mapper.go b/internal/recovery/mapper/truenas/mapper.go
index d365c94664..c6cf624572 100644
--- a/internal/recovery/mapper/truenas/mapper.go
+++ b/internal/recovery/mapper/truenas/mapper.go
@@ -145,11 +145,15 @@ func FromTrueNASSnapshot(connectionID string, snapshot *truenas.FixtureSnapshot)
}
completedAt := task.LastRun
+ if completedAt != nil && completedAt.IsZero() {
+ completedAt = nil
+ }
startedAt := completedAt
outcome := outcomeFromTrueNASReplication(task.LastState, task.LastError)
- if completedAt == nil {
- // If we don't know when it last ran, treat it as unknown rather than "success".
+ if completedAt == nil && outcome == recovery.OutcomeSuccess {
+ // Missing time cannot establish a completed copy. It does not
+ // erase an explicit failure, warning or running state, though.
outcome = recovery.OutcomeUnknown
}
diff --git a/internal/recovery/mapper/truenas/mapper_test.go b/internal/recovery/mapper/truenas/mapper_test.go
index 6250dadc9f..051d58367e 100644
--- a/internal/recovery/mapper/truenas/mapper_test.go
+++ b/internal/recovery/mapper/truenas/mapper_test.go
@@ -129,12 +129,102 @@ func TestFromTrueNASSnapshot_ReplicationTaskNilLastRun(t *testing.T) {
t.Fatalf("expected 1 point, got %d", len(result))
}
- // When LastRun is nil, outcome should be Unknown
+ // Undated success cannot establish a completed copy.
if result[0].Outcome != recovery.OutcomeUnknown {
t.Errorf("Outcome = %v, want %v (nil LastRun)", result[0].Outcome, recovery.OutcomeUnknown)
}
}
+func TestFromTrueNASSnapshot_ReplicationTimeAndStateEvidence(t *testing.T) {
+ zero := time.Time{}
+ native := time.Date(2026, 10, 1, 12, 30, 0, 123000000, time.FixedZone("native", 3600))
+ for _, date := range []struct {
+ name string
+ time *time.Time
+ }{
+ {"missing", nil},
+ {"zero", &zero},
+ {"native", &native},
+ } {
+ for _, state := range []struct {
+ name string
+ state string
+ errText string
+ want recovery.Outcome
+ }{
+ {"success", "SUCCESS", "", recovery.OutcomeSuccess},
+ {"finished", "FINISHED", "", recovery.OutcomeSuccess},
+ {"failed", "FAILED", "", recovery.OutcomeFailed},
+ {"aborted", "ABORTED", "", recovery.OutcomeFailed},
+ {"warning", "WARNING", "", recovery.OutcomeWarning},
+ {"partial", "PARTIAL", "", recovery.OutcomeWarning},
+ {"running", "RUNNING", "", recovery.OutcomeRunning},
+ {"active", "ACTIVE", "", recovery.OutcomeRunning},
+ {"unknown", "UNRECOGNISED", "", recovery.OutcomeUnknown},
+ {"absent", "", "", recovery.OutcomeUnknown},
+ {"error overrides success", "SUCCESS", "fixture target unavailable", recovery.OutcomeFailed},
+ {"error without state", "", "fixture target unavailable", recovery.OutcomeFailed},
+ } {
+ t.Run(date.name+"/"+state.name, func(t *testing.T) {
+ snapshot := &truenas.FixtureSnapshot{ReplicationTasks: []truenas.ReplicationTask{{
+ ID: "7", Name: "offsite", LastRun: date.time, LastState: state.state, LastError: state.errText,
+ }}}
+ point := FromTrueNASSnapshot("nas-a", snapshot)[0]
+ want := state.want
+ if date.time == nil || date.time.IsZero() {
+ if want == recovery.OutcomeSuccess {
+ want = recovery.OutcomeUnknown
+ }
+ if point.StartedAt != nil || point.CompletedAt != nil {
+ t.Errorf("unavailable date gained timestamps: %+v", point)
+ }
+ } else if point.StartedAt == nil || point.CompletedAt == nil ||
+ !point.StartedAt.Equal(native) || !point.CompletedAt.Equal(native) {
+ t.Errorf("native date changed: %+v", point)
+ }
+ if point.Outcome != want {
+ t.Errorf("Outcome = %q, want %q", point.Outcome, want)
+ }
+ if point.Details["lastState"] != state.state || point.Details["lastError"] != state.errText {
+ t.Errorf("provider evidence changed: %+v", point.Details)
+ }
+ wantID := idgen.StableID("truenas", "nas-a", "replication-task", "7", "offsite", idgen.TimeKey(date.time, date.time))
+ if point.ID != wantID {
+ t.Errorf("task identity changed: %s, want %s", point.ID, wantID)
+ }
+ })
+ }
+ }
+}
+
+func TestFromTrueNASSnapshot_UndatedReplicationIdentity(t *testing.T) {
+ zero := time.Time{}
+ snapshot := &truenas.FixtureSnapshot{ReplicationTasks: []truenas.ReplicationTask{{
+ ID: "7", Name: "offsite", SourceDatasets: []string{"tank/apps"}, TargetDataset: "vault/apps",
+ }}}
+ wantID := idgen.StableID("truenas", "nas-a", "replication-task", "7", "offsite", "")
+ for _, date := range []*time.Time{nil, &zero} {
+ for i, state := range []string{"FAILED", "WARNING", "RUNNING", "SUCCESS"} {
+ snapshot.CollectedAt = time.Date(2026, 10, 1, 12+i, 0, 0, 0, time.UTC)
+ snapshot.ReplicationTasks[0].LastRun = date
+ snapshot.ReplicationTasks[0].LastState = state
+ point := FromTrueNASSnapshot("nas-a", snapshot)[0]
+ if point.ID != wantID || point.StartedAt != nil || point.CompletedAt != nil {
+ t.Fatalf("collection/state changed undated identity/time: %+v", point)
+ }
+ if point.SubjectRef.ID != "tank/apps" || point.RepositoryRef.ID != "vault/apps" {
+ t.Fatalf("lost source/target: %+v", point)
+ }
+ if other := FromTrueNASSnapshot("nas-b", snapshot)[0]; other.ID == point.ID {
+ t.Fatal("connections share a task ID")
+ }
+ if snapshot.ReplicationTasks[0].LastRun != date || !zero.IsZero() {
+ t.Fatal("mapping mutated the provider's last-run time")
+ }
+ }
+ }
+}
+
func TestFromTrueNASSnapshot_Truncation(t *testing.T) {
snapshot := &truenas.FixtureSnapshot{
System: truenas.SystemInfo{Hostname: "truenas-main"},
diff --git a/internal/recovery/store/store_test.go b/internal/recovery/store/store_test.go
index 4b6c02c3f2..3815c41f24 100644
--- a/internal/recovery/store/store_test.go
+++ b/internal/recovery/store/store_test.go
@@ -40,6 +40,81 @@ func TestNormalizeLimitAndPageUseSharedListBounds(t *testing.T) {
}
}
+func TestStore_UndatedTrueNASReplicationRetainsState(t *testing.T) {
+ t.Parallel()
+ zero := time.Time{}
+ for _, date := range []struct {
+ name string
+ time *time.Time
+ }{{"missing", nil}, {"zero", &zero}} {
+ t.Run(date.name, func(t *testing.T) {
+ ctx := context.Background()
+ dbPath := filepath.Join(t.TempDir(), "recovery.db")
+ s, err := Open(dbPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = s.Close() })
+ snapshot := &truenas.FixtureSnapshot{ReplicationTasks: []truenas.ReplicationTask{{
+ ID: "7", Name: "offsite", LastRun: date.time, LastState: "FAILED",
+ SourceDatasets: []string{"tank/apps"}, TargetDataset: "vault/apps",
+ }}}
+ mapped := truenasmapper.FromTrueNASSnapshot("nas-a", snapshot)
+ // A former ingest erased the state but retained its source evidence.
+ legacy := mapped[0]
+ legacy.Outcome = recovery.OutcomeUnknown
+ if err := s.UpsertPoints(ctx, []recovery.RecoveryPoint{legacy}); err != nil {
+ t.Fatal(err)
+ }
+ other := truenasmapper.FromTrueNASSnapshot("nas-b", snapshot)[0]
+ other.Outcome = recovery.OutcomeUnknown
+ if err := s.UpsertPoints(ctx, []recovery.RecoveryPoint{other}); err != nil {
+ t.Fatal(err)
+ }
+ for i, state := range []struct {
+ native string
+ want recovery.Outcome
+ }{{"FAILED", recovery.OutcomeFailed}, {"WARNING", recovery.OutcomeWarning},
+ {"RUNNING", recovery.OutcomeRunning}, {"SUCCESS", recovery.OutcomeUnknown}} {
+ snapshot.ReplicationTasks[0].LastState = state.native
+ snapshot.CollectedAt = time.Date(2026, 10, 1, 12+i, 0, 0, 0, time.UTC)
+ if err := s.UpsertPoints(ctx, truenasmapper.FromTrueNASSnapshot("nas-a", snapshot)); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.Close(); err != nil {
+ t.Fatal(err)
+ }
+ s, err = Open(dbPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ points, total, err := s.ListPoints(ctx, recovery.ListPointsOptions{Provider: recovery.ProviderTrueNAS})
+ if err != nil || total != 2 || len(points) != 2 {
+ t.Fatalf("ListPoints = %d/%d points, %v; want two connection-scoped rows", len(points), total, err)
+ }
+ byID := map[string]recovery.RecoveryPoint{}
+ for _, p := range points {
+ byID[p.ID] = p
+ }
+ p := byID[legacy.ID]
+ if p.Outcome != state.want || p.StartedAt != nil || p.CompletedAt != nil {
+ t.Fatalf("%s lost state or gained dates after reopen: %+v", state.native, p)
+ }
+ if p.SubjectRef.ID != "tank/apps" || p.RepositoryRef.ID != "vault/apps" || p.Details["lastState"] != state.native {
+ t.Fatalf("task source/target/evidence changed: %+v", p)
+ }
+ if byID[other.ID].Outcome != recovery.OutcomeUnknown {
+ t.Fatal("ingest crossed connection scope")
+ }
+ filtered, _, err := s.ListPoints(ctx, recovery.ListPointsOptions{Provider: recovery.ProviderTrueNAS, Outcome: state.want})
+ if err != nil || len(filtered) == 0 {
+ t.Fatalf("outcome filter lost undated %s: %+v, %v", state.want, filtered, err)
+ }
+ }
+ })
+ }
+}
+
func TestStore_UpsertAndList(t *testing.T) {
t.Parallel()