Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep undated TrueNAS replication failures visible
pulse-triage[bot]4ea721e5e6f6
diff --git a/docs/TRUENAS.md b/docs/TRUENAS.mdindex ae9086f5c0..5e08d4cff3 100644--- a/docs/TRUENAS.md+++ b/docs/TRUENAS.md@@ -103,7 +103,10 @@ record in TrueNAS:releases could show collection time as **Completed**; those undated recordsare corrected on upgrade without deleting snapshot history. To judge backupage or retention, confirm the native snapshot's creation time. Replication- with no known last-run time is **Unknown**, not a newly completed copy.+ reporting success with no known last-run time is **Unknown**, not a newly+ completed copy. **Failed**, **Warning** and **Running** remain visible even+ without that time; missing dates do not clear a reported problem. Failed+ and warning events remain in **Attention**, without an invented timestamp.3. **Check the destination without starting work.** Use the provider's existingreceived-snapshot and task records. Confirm the same source snapshot, targetand successful transfer result. Missing **Verified**, **Size** or timestampdiff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 697c57035e..15f007d1c6 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -619,11 +619,26 @@ Provider incident nativeSeverity is evidence, not a storage recovery result or iTrueNAS replication state `FINISHED` is successful completion evidence when alast-run timestamp exists, matching the provider success event. Explicit error-text still takes precedence; a missing last-run timestamp remains unknown.+text still takes precedence. Missing or zero last-run dates cannot establish a+successful copy: success becomes unknown, but explicit failed, warning and+running outcomes survive without start/completion timestamps. Collection time+must not fill those dates. Undated task IDs remain stable across polls and+separate across connections; dated native outcomes/times are unchanged. Failed+and warning events remain in the Protection Attention filter even when their+time is unavailable. Normal ingest corrects the same undated task row; no+history is deleted and no native task or restore is run.Persisted recovery rollups must retain this outcome and success time ratherthan classify a completed replication as unknown. This does not imply restoreverification or resolve provider INFO alert lifecycle behaviour.+Verification: `TestFromTrueNASSnapshot_ReplicationTimeAndStateEvidence` covers+nil/zero/native dates, error precedence and known-state retention;+`TestFromTrueNASSnapshot_UndatedReplicationIdentity` covers repeated polls,+connection isolation and non-mutation. `TrueNASProtectionTable.test.tsx` checks+undated attention/filter/detail behaviour; `truenasProtectionEvidenceDocs.test.ts`+checks the shipped guidance. Consuming browser proof must use actual mapped+payloads at desktop and narrow viewports, not a hand-written outcome fixture.+Unraid `StorageMeta.numDisks` is optional source evidence: zero denotes an explicitly empty parity array, not proof that pools are protected or that a recovery point exists. Suppressing the no-parity warning for zero must not erase disk-failure evidence or grant storage/recovery authority.A manifestless PBS snapshot never becomes a successful recovery point anddiff --git a/frontend-modern/public/docs/TRUENAS.md b/frontend-modern/public/docs/TRUENAS.mdindex ae9086f5c0..5e08d4cff3 100644--- a/frontend-modern/public/docs/TRUENAS.md+++ b/frontend-modern/public/docs/TRUENAS.md@@ -103,7 +103,10 @@ record in TrueNAS:releases could show collection time as **Completed**; those undated recordsare corrected on upgrade without deleting snapshot history. To judge backupage or retention, confirm the native snapshot's creation time. Replication- with no known last-run time is **Unknown**, not a newly completed copy.+ reporting success with no known last-run time is **Unknown**, not a newly+ completed copy. **Failed**, **Warning** and **Running** remain visible even+ without that time; missing dates do not clear a reported problem. Failed+ and warning events remain in **Attention**, without an invented timestamp.3. **Check the destination without starting work.** Use the provider's existingreceived-snapshot and task records. Confirm the same source snapshot, targetand successful transfer result. Missing **Verified**, **Size** or timestampdiff --git a/frontend-modern/src/features/docs/__tests__/truenasProtectionEvidenceDocs.test.ts b/frontend-modern/src/features/docs/__tests__/truenasProtectionEvidenceDocs.test.tsindex 8473ed51f5..549df59a01 100644--- a/frontend-modern/src/features/docs/__tests__/truenasProtectionEvidenceDocs.test.ts+++ b/frontend-modern/src/features/docs/__tests__/truenasProtectionEvidenceDocs.test.ts@@ -62,7 +62,10 @@ describe('TrueNAS protection evidence help', () => {expect(text).toContain('corrected on upgrade without deleting snapshot history');expect(text).toContain("confirm the native snapshot's creation time");expect(text).toContain('Running remains incomplete even if a timestamp is present');- expect(text).toContain('Replication with no known last-run time is Unknown');+ expect(text).toContain('Replication reporting success with no known last-run time is Unknown');+ expect(text).toContain('Failed, Warning and Running remain visible even without that time');+ expect(text).toContain('missing dates do not clear a reported problem');+ expect(text).toContain('Failed and warning events remain in Attention, without an invented timestamp');});it('uses current detail labels to compare the same source and destination without active work', () => {diff --git a/frontend-modern/src/features/truenas/__tests__/TrueNASProtectionTable.test.tsx b/frontend-modern/src/features/truenas/__tests__/TrueNASProtectionTable.test.tsxindex 7252749280..f9fd85e2f9 100644--- a/frontend-modern/src/features/truenas/__tests__/TrueNASProtectionTable.test.tsx+++ b/frontend-modern/src/features/truenas/__tests__/TrueNASProtectionTable.test.tsx@@ -73,6 +73,57 @@ describe('TrueNASProtectionTable', () => {expect(document.querySelectorAll('[data-truenas-protection-row]')).toHaveLength(3);});+ it('keeps undated replication problems in Attention without inventing completion dates', async () => {+ renderInRouter(() => (+ <TrueNASProtectionTable+ points={(['failed', 'warning', 'running', 'unknown'] as const).map((outcome) =>+ makeRecoveryPoint({+ id: outcome,+ kind: 'backup',+ mode: 'remote',+ outcome,+ startedAt: undefined,+ completedAt: undefined,+ details: {+ taskId: `task-${outcome}`,+ taskName: `replicate-${outcome}`,+ sourceDatasets: [`tank/${outcome}`],+ targetDataset: `vault/${outcome}`,+ lastState: outcome.toUpperCase(),+ },+ }),+ )}+ emptyIcon={<span />}+ emptyTitle="No protection"+ emptyDescription="No protection"+ />+ ));++ await fireEvent.click(screen.getByRole('button', { name: 'Attention, 2' }));+ const rows = document.querySelectorAll('[data-truenas-protection-row]');+ expect(rows).toHaveLength(2);+ for (const outcome of ['failed', 'warning']) {+ const row = document.querySelector(`[data-truenas-protection-row="${outcome}"]`)!;+ expect(row).toHaveAttribute('data-truenas-protection-outcome', outcome);+ expect(row.querySelectorAll('td')[3]).toHaveTextContent('-');+ }++ await fireEvent.click(document.querySelector('[data-truenas-protection-row="failed"]')!);+ const detail = within(screen.getByTestId('truenas-protection-detail'));+ expect(detail.getByText('task-failed')).toBeInTheDocument();+ expect(detail.getByText('FAILED')).toBeInTheDocument();+ expect(detail.queryByText('Started', { exact: true })).not.toBeInTheDocument();+ expect(detail.queryByText('Completed', { exact: true })).not.toBeInTheDocument();+ await fireEvent.click(detail.getByRole('button', { name: /^Collapse .* details$/ }));++ await fireEvent.click(screen.getByRole('button', { name: 'Running, 1' }));+ expect(document.querySelectorAll('[data-truenas-protection-row]')).toHaveLength(1);+ expect(document.querySelector('[data-truenas-protection-row="running"]')).not.toBeNull();+ await fireEvent.click(screen.getByRole('button', { name: 'Unknown, 1' }));+ expect(document.querySelectorAll('[data-truenas-protection-row]')).toHaveLength(1);+ expect(document.querySelector('[data-truenas-protection-row="unknown"]')).not.toBeNull();+ });+it('opens inline table details for a TrueNAS replication recovery point', async () => {const replication = makeRecoveryPoint({id: 'replicate-tank-apps',diff --git a/internal/recovery/mapper/truenas/mapper.go b/internal/recovery/mapper/truenas/mapper.goindex d365c94664..c6cf624572 100644--- a/internal/recovery/mapper/truenas/mapper.go+++ b/internal/recovery/mapper/truenas/mapper.go@@ -145,11 +145,15 @@ func FromTrueNASSnapshot(connectionID string, snapshot *truenas.FixtureSnapshot)}completedAt := task.LastRun+ if completedAt != nil && completedAt.IsZero() {+ completedAt = nil+ }startedAt := completedAtoutcome := outcomeFromTrueNASReplication(task.LastState, task.LastError)- if completedAt == nil {- // If we don't know when it last ran, treat it as unknown rather than "success".+ if completedAt == nil && outcome == recovery.OutcomeSuccess {+ // Missing time cannot establish a completed copy. It does not+ // erase an explicit failure, warning or running state, though.outcome = recovery.OutcomeUnknown}diff --git a/internal/recovery/mapper/truenas/mapper_test.go b/internal/recovery/mapper/truenas/mapper_test.goindex 6250dadc9f..051d58367e 100644--- a/internal/recovery/mapper/truenas/mapper_test.go+++ b/internal/recovery/mapper/truenas/mapper_test.go@@ -129,12 +129,102 @@ func TestFromTrueNASSnapshot_ReplicationTaskNilLastRun(t *testing.T) {t.Fatalf("expected 1 point, got %d", len(result))}- // When LastRun is nil, outcome should be Unknown+ // Undated success cannot establish a completed copy.if result[0].Outcome != recovery.OutcomeUnknown {t.Errorf("Outcome = %v, want %v (nil LastRun)", result[0].Outcome, recovery.OutcomeUnknown)}}+func TestFromTrueNASSnapshot_ReplicationTimeAndStateEvidence(t *testing.T) {+ zero := time.Time{}+ native := time.Date(2026, 10, 1, 12, 30, 0, 123000000, time.FixedZone("native", 3600))+ for _, date := range []struct {+ name string+ time *time.Time+ }{+ {"missing", nil},+ {"zero", &zero},+ {"native", &native},+ } {+ for _, state := range []struct {+ name string+ state string+ errText string+ want recovery.Outcome+ }{+ {"success", "SUCCESS", "", recovery.OutcomeSuccess},+ {"finished", "FINISHED", "", recovery.OutcomeSuccess},+ {"failed", "FAILED", "", recovery.OutcomeFailed},+ {"aborted", "ABORTED", "", recovery.OutcomeFailed},+ {"warning", "WARNING", "", recovery.OutcomeWarning},+ {"partial", "PARTIAL", "", recovery.OutcomeWarning},+ {"running", "RUNNING", "", recovery.OutcomeRunning},+ {"active", "ACTIVE", "", recovery.OutcomeRunning},+ {"unknown", "UNRECOGNISED", "", recovery.OutcomeUnknown},+ {"absent", "", "", recovery.OutcomeUnknown},+ {"error overrides success", "SUCCESS", "fixture target unavailable", recovery.OutcomeFailed},+ {"error without state", "", "fixture target unavailable", recovery.OutcomeFailed},+ } {+ t.Run(date.name+"/"+state.name, func(t *testing.T) {+ snapshot := &truenas.FixtureSnapshot{ReplicationTasks: []truenas.ReplicationTask{{+ ID: "7", Name: "offsite", LastRun: date.time, LastState: state.state, LastError: state.errText,+ }}}+ point := FromTrueNASSnapshot("nas-a", snapshot)[0]+ want := state.want+ if date.time == nil || date.time.IsZero() {+ if want == recovery.OutcomeSuccess {+ want = recovery.OutcomeUnknown+ }+ if point.StartedAt != nil || point.CompletedAt != nil {+ t.Errorf("unavailable date gained timestamps: %+v", point)+ }+ } else if point.StartedAt == nil || point.CompletedAt == nil ||+ !point.StartedAt.Equal(native) || !point.CompletedAt.Equal(native) {+ t.Errorf("native date changed: %+v", point)+ }+ if point.Outcome != want {+ t.Errorf("Outcome = %q, want %q", point.Outcome, want)+ }+ if point.Details["lastState"] != state.state || point.Details["lastError"] != state.errText {+ t.Errorf("provider evidence changed: %+v", point.Details)+ }+ wantID := idgen.StableID("truenas", "nas-a", "replication-task", "7", "offsite", idgen.TimeKey(date.time, date.time))+ if point.ID != wantID {+ t.Errorf("task identity changed: %s, want %s", point.ID, wantID)+ }+ })+ }+ }+}++func TestFromTrueNASSnapshot_UndatedReplicationIdentity(t *testing.T) {+ zero := time.Time{}+ snapshot := &truenas.FixtureSnapshot{ReplicationTasks: []truenas.ReplicationTask{{+ ID: "7", Name: "offsite", SourceDatasets: []string{"tank/apps"}, TargetDataset: "vault/apps",+ }}}+ wantID := idgen.StableID("truenas", "nas-a", "replication-task", "7", "offsite", "")+ for _, date := range []*time.Time{nil, &zero} {+ for i, state := range []string{"FAILED", "WARNING", "RUNNING", "SUCCESS"} {+ snapshot.CollectedAt = time.Date(2026, 10, 1, 12+i, 0, 0, 0, time.UTC)+ snapshot.ReplicationTasks[0].LastRun = date+ snapshot.ReplicationTasks[0].LastState = state+ point := FromTrueNASSnapshot("nas-a", snapshot)[0]+ if point.ID != wantID || point.StartedAt != nil || point.CompletedAt != nil {+ t.Fatalf("collection/state changed undated identity/time: %+v", point)+ }+ if point.SubjectRef.ID != "tank/apps" || point.RepositoryRef.ID != "vault/apps" {+ t.Fatalf("lost source/target: %+v", point)+ }+ if other := FromTrueNASSnapshot("nas-b", snapshot)[0]; other.ID == point.ID {+ t.Fatal("connections share a task ID")+ }+ if snapshot.ReplicationTasks[0].LastRun != date || !zero.IsZero() {+ t.Fatal("mapping mutated the provider's last-run time")+ }+ }+ }+}+func TestFromTrueNASSnapshot_Truncation(t *testing.T) {snapshot := &truenas.FixtureSnapshot{System: truenas.SystemInfo{Hostname: "truenas-main"},diff --git a/internal/recovery/store/store_test.go b/internal/recovery/store/store_test.goindex 4b6c02c3f2..3815c41f24 100644--- a/internal/recovery/store/store_test.go+++ b/internal/recovery/store/store_test.go@@ -40,6 +40,81 @@ func TestNormalizeLimitAndPageUseSharedListBounds(t *testing.T) {}}+func TestStore_UndatedTrueNASReplicationRetainsState(t *testing.T) {+ t.Parallel()+ zero := time.Time{}+ for _, date := range []struct {+ name string+ time *time.Time+ }{{"missing", nil}, {"zero", &zero}} {+ t.Run(date.name, func(t *testing.T) {+ ctx := context.Background()+ dbPath := filepath.Join(t.TempDir(), "recovery.db")+ s, err := Open(dbPath)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = s.Close() })+ snapshot := &truenas.FixtureSnapshot{ReplicationTasks: []truenas.ReplicationTask{{+ ID: "7", Name: "offsite", LastRun: date.time, LastState: "FAILED",+ SourceDatasets: []string{"tank/apps"}, TargetDataset: "vault/apps",+ }}}+ mapped := truenasmapper.FromTrueNASSnapshot("nas-a", snapshot)+ // A former ingest erased the state but retained its source evidence.+ legacy := mapped[0]+ legacy.Outcome = recovery.OutcomeUnknown+ if err := s.UpsertPoints(ctx, []recovery.RecoveryPoint{legacy}); err != nil {+ t.Fatal(err)+ }+ other := truenasmapper.FromTrueNASSnapshot("nas-b", snapshot)[0]+ other.Outcome = recovery.OutcomeUnknown+ if err := s.UpsertPoints(ctx, []recovery.RecoveryPoint{other}); err != nil {+ t.Fatal(err)+ }+ for i, state := range []struct {+ native string+ want recovery.Outcome+ }{{"FAILED", recovery.OutcomeFailed}, {"WARNING", recovery.OutcomeWarning},+ {"RUNNING", recovery.OutcomeRunning}, {"SUCCESS", recovery.OutcomeUnknown}} {+ snapshot.ReplicationTasks[0].LastState = state.native+ snapshot.CollectedAt = time.Date(2026, 10, 1, 12+i, 0, 0, 0, time.UTC)+ if err := s.UpsertPoints(ctx, truenasmapper.FromTrueNASSnapshot("nas-a", snapshot)); err != nil {+ t.Fatal(err)+ }+ if err := s.Close(); err != nil {+ t.Fatal(err)+ }+ s, err = Open(dbPath)+ if err != nil {+ t.Fatal(err)+ }+ points, total, err := s.ListPoints(ctx, recovery.ListPointsOptions{Provider: recovery.ProviderTrueNAS})+ if err != nil || total != 2 || len(points) != 2 {+ t.Fatalf("ListPoints = %d/%d points, %v; want two connection-scoped rows", len(points), total, err)+ }+ byID := map[string]recovery.RecoveryPoint{}+ for _, p := range points {+ byID[p.ID] = p+ }+ p := byID[legacy.ID]+ if p.Outcome != state.want || p.StartedAt != nil || p.CompletedAt != nil {+ t.Fatalf("%s lost state or gained dates after reopen: %+v", state.native, p)+ }+ if p.SubjectRef.ID != "tank/apps" || p.RepositoryRef.ID != "vault/apps" || p.Details["lastState"] != state.native {+ t.Fatalf("task source/target/evidence changed: %+v", p)+ }+ if byID[other.ID].Outcome != recovery.OutcomeUnknown {+ t.Fatal("ingest crossed connection scope")+ }+ filtered, _, err := s.ListPoints(ctx, recovery.ListPointsOptions{Provider: recovery.ProviderTrueNAS, Outcome: state.want})+ if err != nil || len(filtered) == 0 {+ t.Fatalf("outcome filter lost undated %s: %+v, %v", state.want, filtered, err)+ }+ }+ })+ }+}+func TestStore_UpsertAndList(t *testing.T) {t.Parallel()