Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Allow explicit recovery of retained guest metadata after failed close
pulse-triage[bot]5e5106904816
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.mdindex 6743cf3006..0e9db6c03d 100644--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md@@ -7534,7 +7534,9 @@ metadata close. Failure retains the monitor, cancellation/done handles andlifecycle guard: lazy recreation cannot start a second writer in the samedirectory. `BeginTenantDeletion` returns that failure to the organisationhandler, which keeps persistence and offboarding callbacks untouched. Only-successful shutdown allows directory deletion. Other tenants keep their existing+successful shutdown allows directory deletion. A later explicit shutdown attempt+may persist retained metadata after a completed write failure is repaired;+admission stays sealed and no in-flight attempt is replayed. Other tenants keep their existingindependent runtime. Store drain does not establish independent guest thaw,workload liveness or containment in installed software.diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 40cf4ace67..354ad8d5e2 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -4605,7 +4605,10 @@ persistence failure remains an error. Monitor Stop logs that failure instead ofclaiming a clean stop; tenant removal retains the runtime owner and lifecycleguard, and organisation deletion refuses to remove its directory. An unfinishedmonitoring loop likewise prevents deletion. Shutdown does not cancel an admitted-filesystem write or silently discard metadata to obtain a timely result.+filesystem write or silently discard metadata to obtain a timely result. A later+explicit close may retry a completed failed snapshot after filesystem repair;+admission stays sealed, successful close is idempotent, and no close retries+itself or overlaps a still-running attempt.`internal/config/guest_writer_control_test.go` exercises blocked snapshots at25/100 updates and caller-copy ownership. `guest_metadata_writer_test.go` coversdiff --git a/internal/config/guest_metadata.go b/internal/config/guest_metadata.goindex d68e281060..cba3f71834 100644--- a/internal/config/guest_metadata.go+++ b/internal/config/guest_metadata.go@@ -42,9 +42,13 @@ type GuestMetadataStore struct {lastWriteErr errorwriterDone chan struct{}closed atomic.Bool- closeOnce sync.Once- closeDone chan struct{}- closeErr error+ closeMu sync.Mutex+ closeAttempt *guestMetadataCloseAttempt+}++type guestMetadataCloseAttempt struct {+ done chan struct{}+ err error}var ErrGuestMetadataStoreClosed = errors.New("guest metadata store is closed")@@ -177,27 +181,49 @@ func (s *GuestMetadataStore) Close(timeout time.Duration) error {}// Seal before waiting for a synchronous mutation's I/O-held memory lock.// One owned close observer, not one goroutine per waiter, covers both that- // mutation and the background writer without extending the caller's budget.- s.closeOnce.Do(func() {- s.closed.Store(true)+ // mutation and the background writer within the caller's unchanged budget.+ s.closed.Store(true)+ s.closeMu.Lock()+ attempt := s.closeAttempt+ retry := false+ if attempt != nil {+ select {+ case <-attempt.done:+ // A later explicit shutdown/offboarding attempt may flush retained+ // state after the filesystem is repaired. Never reopen admission,+ // retry an in-flight write or spin-retry a failed close on its own.+ if attempt.err != nil {+ attempt = nil+ retry = true+ }+ default:+ }+ }+ if attempt == nil {+ attempt = &guestMetadataCloseAttempt{done: make(chan struct{})}+ s.closeAttempt = attemptgo func() {- s.mu.RLock()+ s.mu.Lock()+ if retry && s.lastWriteErr != nil && s.writerDone == nil {+ s.queueSaveLocked()+ }done := s.writerDone- s.mu.RUnlock()+ s.mu.Unlock()if done != nil {<-done}s.mu.RLock()- s.closeErr = s.lastWriteErr+ attempt.err = s.lastWriteErrs.mu.RUnlock()- close(s.closeDone)+ close(attempt.done)}()- })+ }+ s.closeMu.Unlock()timer := time.NewTimer(timeout)defer timer.Stop()select {- case <-s.closeDone:- return s.closeErr+ case <-attempt.done:+ return attempt.errcase <-timer.C:return fmt.Errorf("guest metadata writes did not drain within %s", timeout)}@@ -223,10 +249,9 @@ func cloneGuestMetadata(meta *GuestMetadata) *GuestMetadata {// NewGuestMetadataStore creates a new metadata storefunc NewGuestMetadataStore(dataPath string, fs FileSystem) *GuestMetadataStore {store := &GuestMetadataStore{- metadata: make(map[string]*GuestMetadata),- dataPath: dataPath,- fs: fs,- closeDone: make(chan struct{}),+ metadata: make(map[string]*GuestMetadata),+ dataPath: dataPath,+ fs: fs,}if store.fs == nil {diff --git a/internal/config/guest_metadata_writer_test.go b/internal/config/guest_metadata_writer_test.goindex d71b685f2d..f4c78ce497 100644--- a/internal/config/guest_metadata_writer_test.go+++ b/internal/config/guest_metadata_writer_test.go@@ -359,3 +359,38 @@ func TestGuestMetadataWriterRealFilesystemTenantIsolation(t *testing.T) {t.Fatal("tenants share identity")}}++// Retention after a failed close is not a permanent offboarding dead end.+// A later explicit close may flush retained state after the filesystem recovers,+// without accepting new mutations or overlapping the original writer.+func TestGuestMetadataWriterExplicitCloseRetryPersistsRetainedState(t *testing.T) {+ fs := &guestWriterFaultFS{FileSystem: defaultFileSystem{}}+ fs.failWrite.Store(true)+ store := NewGuestMetadataStore(t.TempDir(), fs)+ if err := store.SetAsync("retained", &GuestMetadata{LastKnownName: "retained"}); err != nil {+ t.Fatal(err)+ }+ if err := store.Close(time.Second); err == nil {+ t.Fatal("first failed close reported success")+ }+ if fs.writes.Load() != 1 {+ t.Fatalf("failed close retried without an explicit later call: %d", fs.writes.Load())+ }+ if err := store.SetAsync("late", &GuestMetadata{}); !errors.Is(err, ErrGuestMetadataStoreClosed) {+ t.Fatalf("failed close reopened admission: %v", err)+ }+ fs.failWrite.Store(false)+ if err := store.Close(time.Second); err != nil {+ t.Fatal(err)+ }+ if err := store.Close(time.Second); err != nil {+ t.Fatal(err)+ }+ if fs.writes.Load() != 2 {+ t.Fatalf("explicit recovery or successful idempotent close wrote unexpected snapshots: %d", fs.writes.Load())+ }+ reloaded := NewGuestMetadataStore(store.dataPath, nil)+ if reloaded.Get("retained") == nil || reloaded.Get("late") != nil {+ t.Fatal("explicit close recovery lost retained state or wrote a late admission")+ }+}