Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Allow explicit recovery of retained guest metadata after failed close

pulse-triage[bot]5e5106904816
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index 6743cf3006..0e9db6c03d 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -7534,7 +7534,9 @@ metadata close. Failure retains the monitor, cancellation/done handles and
lifecycle guard: lazy recreation cannot start a second writer in the same
directory. `BeginTenantDeletion` returns that failure to the organisation
handler, which keeps persistence and offboarding callbacks untouched. Only
-successful shutdown allows directory deletion. Other tenants keep their existing
+successful shutdown allows directory deletion. A later explicit shutdown attempt
+may persist retained metadata after a completed write failure is repaired;
+admission stays sealed and no in-flight attempt is replayed. Other tenants keep their existing
independent runtime. Store drain does not establish independent guest thaw,
workload liveness or containment in installed software.
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 40cf4ace67..354ad8d5e2 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -4605,7 +4605,10 @@ persistence failure remains an error. Monitor Stop logs that failure instead of
claiming a clean stop; tenant removal retains the runtime owner and lifecycle
guard, and organisation deletion refuses to remove its directory. An unfinished
monitoring loop likewise prevents deletion. Shutdown does not cancel an admitted
-filesystem write or silently discard metadata to obtain a timely result.
+filesystem write or silently discard metadata to obtain a timely result. A later
+explicit close may retry a completed failed snapshot after filesystem repair;
+admission stays sealed, successful close is idempotent, and no close retries
+itself or overlaps a still-running attempt.
`internal/config/guest_writer_control_test.go` exercises blocked snapshots at
25/100 updates and caller-copy ownership. `guest_metadata_writer_test.go` covers
diff --git a/internal/config/guest_metadata.go b/internal/config/guest_metadata.go
index d68e281060..cba3f71834 100644
--- a/internal/config/guest_metadata.go
+++ b/internal/config/guest_metadata.go
@@ -42,9 +42,13 @@ type GuestMetadataStore struct {
lastWriteErr error
writerDone chan struct{}
closed atomic.Bool
- closeOnce sync.Once
- closeDone chan struct{}
- closeErr error
+ closeMu sync.Mutex
+ closeAttempt *guestMetadataCloseAttempt
+}
+
+type guestMetadataCloseAttempt struct {
+ done chan struct{}
+ err error
}
var ErrGuestMetadataStoreClosed = errors.New("guest metadata store is closed")
@@ -177,27 +181,49 @@ func (s *GuestMetadataStore) Close(timeout time.Duration) error {
}
// Seal before waiting for a synchronous mutation's I/O-held memory lock.
// One owned close observer, not one goroutine per waiter, covers both that
- // mutation and the background writer without extending the caller's budget.
- s.closeOnce.Do(func() {
- s.closed.Store(true)
+ // mutation and the background writer within the caller's unchanged budget.
+ s.closed.Store(true)
+ s.closeMu.Lock()
+ attempt := s.closeAttempt
+ retry := false
+ if attempt != nil {
+ select {
+ case <-attempt.done:
+ // A later explicit shutdown/offboarding attempt may flush retained
+ // state after the filesystem is repaired. Never reopen admission,
+ // retry an in-flight write or spin-retry a failed close on its own.
+ if attempt.err != nil {
+ attempt = nil
+ retry = true
+ }
+ default:
+ }
+ }
+ if attempt == nil {
+ attempt = &guestMetadataCloseAttempt{done: make(chan struct{})}
+ s.closeAttempt = attempt
go func() {
- s.mu.RLock()
+ s.mu.Lock()
+ if retry && s.lastWriteErr != nil && s.writerDone == nil {
+ s.queueSaveLocked()
+ }
done := s.writerDone
- s.mu.RUnlock()
+ s.mu.Unlock()
if done != nil {
<-done
}
s.mu.RLock()
- s.closeErr = s.lastWriteErr
+ attempt.err = s.lastWriteErr
s.mu.RUnlock()
- close(s.closeDone)
+ close(attempt.done)
}()
- })
+ }
+ s.closeMu.Unlock()
timer := time.NewTimer(timeout)
defer timer.Stop()
select {
- case <-s.closeDone:
- return s.closeErr
+ case <-attempt.done:
+ return attempt.err
case <-timer.C:
return fmt.Errorf("guest metadata writes did not drain within %s", timeout)
}
@@ -223,10 +249,9 @@ func cloneGuestMetadata(meta *GuestMetadata) *GuestMetadata {
// NewGuestMetadataStore creates a new metadata store
func NewGuestMetadataStore(dataPath string, fs FileSystem) *GuestMetadataStore {
store := &GuestMetadataStore{
- metadata: make(map[string]*GuestMetadata),
- dataPath: dataPath,
- fs: fs,
- closeDone: make(chan struct{}),
+ metadata: make(map[string]*GuestMetadata),
+ dataPath: dataPath,
+ fs: fs,
}
if store.fs == nil {
diff --git a/internal/config/guest_metadata_writer_test.go b/internal/config/guest_metadata_writer_test.go
index d71b685f2d..f4c78ce497 100644
--- a/internal/config/guest_metadata_writer_test.go
+++ b/internal/config/guest_metadata_writer_test.go
@@ -359,3 +359,38 @@ func TestGuestMetadataWriterRealFilesystemTenantIsolation(t *testing.T) {
t.Fatal("tenants share identity")
}
}
+
+// Retention after a failed close is not a permanent offboarding dead end.
+// A later explicit close may flush retained state after the filesystem recovers,
+// without accepting new mutations or overlapping the original writer.
+func TestGuestMetadataWriterExplicitCloseRetryPersistsRetainedState(t *testing.T) {
+ fs := &guestWriterFaultFS{FileSystem: defaultFileSystem{}}
+ fs.failWrite.Store(true)
+ store := NewGuestMetadataStore(t.TempDir(), fs)
+ if err := store.SetAsync("retained", &GuestMetadata{LastKnownName: "retained"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.Close(time.Second); err == nil {
+ t.Fatal("first failed close reported success")
+ }
+ if fs.writes.Load() != 1 {
+ t.Fatalf("failed close retried without an explicit later call: %d", fs.writes.Load())
+ }
+ if err := store.SetAsync("late", &GuestMetadata{}); !errors.Is(err, ErrGuestMetadataStoreClosed) {
+ t.Fatalf("failed close reopened admission: %v", err)
+ }
+ fs.failWrite.Store(false)
+ if err := store.Close(time.Second); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.Close(time.Second); err != nil {
+ t.Fatal(err)
+ }
+ if fs.writes.Load() != 2 {
+ t.Fatalf("explicit recovery or successful idempotent close wrote unexpected snapshots: %d", fs.writes.Load())
+ }
+ reloaded := NewGuestMetadataStore(store.dataPath, nil)
+ if reloaded.Get("retained") == nil || reloaded.Get("late") != nil {
+ t.Fatal("explicit close recovery lost retained state or wrote a late admission")
+ }
+}