Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Settle succession lock groups in application order
rcourtmand70c60e5c472
diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.mdindex 6c1f8cc5a4..e2efb80393 100644--- a/docs/release-control/v6/internal/subsystems/unified-resources.md+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md@@ -3599,7 +3599,8 @@ records itself done only once it has converged, does not run again after that,so a lock lifted afterwards stays lifted, and a failure is logged and retriedat the next start rather than keeping the store closed. Sharing after asuccession batch waits until every predecessor's row in the batch has moved-and follows each member through the batch's re-keys, so a block-only row made+and follows each group's members through the re-keys applied after it, in+application order, so a block-only row madefor one predecessor cannot take the place of another's whole row and an ID alater succession in the batch re-keyed away is not given a row. The link readsthat settle a lock take only the columns that decide a pair and treat a time@@ -3617,8 +3618,8 @@ the operator-state API resolves a folded ID to, and a restart, a rebuild or are-declared succession does not put it back. The copies the sharing wrote onfolded members cannot be reached through that API while the pair is linked, sothey share again at the component's next link, unlink or succession. Unlinking-the pair removes no carried lock from either side; once unlinked, each member-can be edited or cleared on its own.+the pair removes no carried lock from either side; once no other link still+joins them, each member can be edited or cleared on its own.The sharing is bounded by when it runs, and the contract does not claim more.A lock written to a linked ID after the link was recorded reaches the otherdiff --git a/internal/unifiedresources/canonical_id_succession.go b/internal/unifiedresources/canonical_id_succession.goindex 60ac83f43b..38d6b21e2b 100644--- a/internal/unifiedresources/canonical_id_succession.go+++ b/internal/unifiedresources/canonical_id_succession.go@@ -84,8 +84,8 @@ func (s *SQLiteResourceStore) ApplyCanonicalIDSuccessions(successions []Canonicaapplied := make([]CanonicalIDSuccession, 0, len(pending))var carried []carriedLock- var groups [][]string- rekeyed := make(map[string]string, len(pending))+ var groups []savedGroup+ var rekeys []rekeyedIDfor _, succession := range pending {oldID := succession.OldCanonicalIDnewID := succession.NewCanonicalID@@ -139,8 +139,8 @@ func (s *SQLiteResourceStore) ApplyCanonicalIDSuccessions(successions []Canonicaif _, err := tx.Exec(`DELETE FROM resource_identities WHERE canonical_id = ?`, oldID); err != nil {return fmt.Errorf("delete superseded identity pin %q: %w", oldID, err)}- groups = append(groups, groupAfterRekey(members, oldID, newID))- rekeyed[oldID] = newID+ groups = append(groups, savedGroup{index: len(rekeys), members: groupAfterRekey(members, oldID, newID)})+ rekeys = append(rekeys, rekeyedID{oldID: oldID, newID: newID})applied = append(applied, succession)}@@ -152,7 +152,7 @@ func (s *SQLiteResourceStore) ApplyCanonicalIDSuccessions(successions []Canonica// the re-key of a later predecessor into the same successor.sharedAt := time.Now().UTC()for _, group := range groups {- shared, err := shareRemediationLockAmongSQL(tx, settledGroup(group, rekeyed), sharedAt)+ shared, err := shareRemediationLockAmongSQL(tx, settledGroup(group, rekeys), sharedAt)if err != nil {return fmt.Errorf("share remediation lock across linked resources %v: %w", group, err)}@@ -230,8 +230,8 @@ func (m *MemoryStore) ApplyCanonicalIDSuccessions(successions []CanonicalIDSuccem.mu.Lock()defer m.mu.Unlock()- var groups [][]string- rekeyed := make(map[string]string, len(successions))+ var groups []savedGroup+ var rekeys []rekeyedIDfor _, succession := range successions {oldID := CanonicalResourceID(succession.OldCanonicalID)newID := CanonicalResourceID(succession.NewCanonicalID)@@ -280,13 +280,13 @@ func (m *MemoryStore) ApplyCanonicalIDSuccessions(successions []CanonicalIDSuccem.canonicalSuccessions = make(map[string]string)}m.canonicalSuccessions[oldID] = newID- groups = append(groups, groupAfterRekey(members, oldID, newID))- rekeyed[oldID] = newID+ groups = append(groups, savedGroup{index: len(rekeys), members: groupAfterRekey(members, oldID, newID)})+ rekeys = append(rekeys, rekeyedID{oldID: oldID, newID: newID})}// See the SQLite store: sharing waits for the whole batch.sharedAt := time.Now().UTC()for _, group := range groups {- m.shareRemediationLockAmongLocked(settledGroup(group, rekeyed), sharedAt)+ m.shareRemediationLockAmongLocked(settledGroup(group, rekeys), sharedAt)}return nil}diff --git a/internal/unifiedresources/registry_test.go b/internal/unifiedresources/registry_test.goindex a7118ef715..599346178b 100644--- a/internal/unifiedresources/registry_test.go+++ b/internal/unifiedresources/registry_test.go@@ -8461,3 +8461,75 @@ func TestSQLiteStoreRestoresLocksAcrossRecordedSuccessionChains(t *testing.T) {t.Fatalf("restoring the lock rewrote the successor's own settings: %+v", got)}}++// A succession batch is applied in the order it is declared, and each group of+// linked IDs it saved follows only the re-keys applied after it. A batch+// declared against its direction, and one that returns an ID to its start, both+// leave the lock on the IDs that survive and give no row to an ID re-keyed away.+func TestSuccessionBatchOrderKeepsLinkedRemediationLock(t *testing.T) {+ stores := map[string]func(t *testing.T) ResourceStore{+ "memory": func(t *testing.T) ResourceStore { return NewMemoryStore() },+ "sqlite": func(t *testing.T) ResourceStore {+ store, err := NewSQLiteResourceStore(t.TempDir(), "default")+ if err != nil {+ t.Fatalf("NewSQLiteResourceStore: %v", err)+ }+ t.Cleanup(func() { _ = store.Close() })+ return store+ },+ }+ now := time.Now().UTC()+ locked := func(t *testing.T, store ResourceStore, id string) bool {+ t.Helper()+ state, found, err := store.GetResourceOperatorState(id)+ if err != nil {+ t.Fatalf("read operator state of %s: %v", id, err)+ }+ return found && state.BlocksRemediation()+ }+ for name, newStore := range stores {+ t.Run(name+"/declared against its direction", func(t *testing.T) {+ store := newStore(t)+ if err := store.AddLink(ResourceLink{ResourceA: "rev-a", ResourceB: "rev-b", PrimaryID: "rev-b"}); err != nil {+ t.Fatalf("link: %v", err)+ }+ if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: "rev-a", NeverAutoRemediate: true, SetAt: now}); err != nil {+ t.Fatalf("lock: %v", err)+ }+ if err := store.(canonicalIDSuccessor).ApplyCanonicalIDSuccessions([]CanonicalIDSuccession{+ {OldCanonicalID: "rev-b", NewCanonicalID: "rev-c"},+ {OldCanonicalID: "rev-a", NewCanonicalID: "rev-b"},+ }); err != nil {+ t.Fatalf("ApplyCanonicalIDSuccessions: %v", err)+ }+ for _, id := range []string{"rev-b", "rev-c"} {+ if !locked(t, store, id) {+ t.Fatalf("%s lost the lock when the batch was declared against its direction", id)+ }+ }+ })+ t.Run(name+"/returns an ID to its start", func(t *testing.T) {+ store := newStore(t)+ if err := store.AddLink(ResourceLink{ResourceA: "loop-a", ResourceB: "loop-x", PrimaryID: "loop-x"}); err != nil {+ t.Fatalf("link: %v", err)+ }+ if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: "loop-a", NeverAutoRemediate: true, SetAt: now}); err != nil {+ t.Fatalf("lock: %v", err)+ }+ if err := store.(canonicalIDSuccessor).ApplyCanonicalIDSuccessions([]CanonicalIDSuccession{+ {OldCanonicalID: "loop-a", NewCanonicalID: "loop-b"},+ {OldCanonicalID: "loop-b", NewCanonicalID: "loop-a"},+ }); err != nil {+ t.Fatalf("ApplyCanonicalIDSuccessions: %v", err)+ }+ for _, id := range []string{"loop-a", "loop-x"} {+ if !locked(t, store, id) {+ t.Fatalf("%s lost the lock across a batch that returned to its start", id)+ }+ }+ if state, found, err := store.GetResourceOperatorState("loop-b"); err != nil || found {+ t.Fatalf("the re-keyed intermediate ID was given a row: found=%v err=%v state=%+v", found, err, state)+ }+ })+ }+}diff --git a/internal/unifiedresources/remediation_lock_carry.go b/internal/unifiedresources/remediation_lock_carry.goindex d38e968ebe..109d8f0aeb 100644--- a/internal/unifiedresources/remediation_lock_carry.go+++ b/internal/unifiedresources/remediation_lock_carry.go@@ -282,28 +282,33 @@ func flattenComponents(components [][]string) []string {return out}-// resolveThroughRekeys follows id through a batch's re-keys (a -> b, then-// b -> c) to the ID that holds it once the whole batch has applied.-func resolveThroughRekeys(id string, rekeyed map[string]string) string {- for hops := 0; hops <= len(rekeyed); hops++ {- next, moved := rekeyed[id]- if !moved {- return id- }- id = next- }- return id+// rekeyedID is one applied succession of a batch.+type rekeyedID struct {+ oldID string+ newID string}-// settledGroup maps a succession's saved sharing group through the batch's-// re-keys, so a member a later succession in the batch re-keyed away is-// replaced by its successor instead of being given a row of its own.-func settledGroup(group []string, rekeyed map[string]string) []string {- resolved := make([]string, 0, len(group))- for _, id := range group {- resolved = append(resolved, resolveThroughRekeys(id, rekeyed))+// savedGroup is the sharing group of the succession at position index in a+// batch's application order.+type savedGroup struct {+ index int+ members []string+}++// settledGroup maps a saved group through the re-keys applied AFTER it, in+// application order, so a member a later succession in the batch re-keyed away+// is replaced by its successor instead of being given a row of its own. The+// re-keys before the group's own succession are already reflected in it.+func settledGroup(group savedGroup, rekeys []rekeyedID) []string {+ members := append([]string(nil), group.members...)+ for _, rekey := range rekeys[group.index+1:] {+ for i, id := range members {+ if id == rekey.oldID {+ members[i] = rekey.newID+ }+ }}- return flattenComponents([][]string{resolved})+ return flattenComponents([][]string{members})}// groupAfterRekey is the set a succession's pre-re-key members form once the