Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Settle succession lock groups in application order

rcourtmand70c60e5c472
diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md
index 6c1f8cc5a4..e2efb80393 100644
--- a/docs/release-control/v6/internal/subsystems/unified-resources.md
+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md
@@ -3599,7 +3599,8 @@ records itself done only once it has converged, does not run again after that,
so a lock lifted afterwards stays lifted, and a failure is logged and retried
at the next start rather than keeping the store closed. Sharing after a
succession batch waits until every predecessor's row in the batch has moved
-and follows each member through the batch's re-keys, so a block-only row made
+and follows each group's members through the re-keys applied after it, in
+application order, so a block-only row made
for one predecessor cannot take the place of another's whole row and an ID a
later succession in the batch re-keyed away is not given a row. The link reads
that settle a lock take only the columns that decide a pair and treat a time
@@ -3617,8 +3618,8 @@ the operator-state API resolves a folded ID to, and a restart, a rebuild or a
re-declared succession does not put it back. The copies the sharing wrote on
folded members cannot be reached through that API while the pair is linked, so
they share again at the component's next link, unlink or succession. Unlinking
-the pair removes no carried lock from either side; once unlinked, each member
-can be edited or cleared on its own.
+the pair removes no carried lock from either side; once no other link still
+joins them, each member can be edited or cleared on its own.
The sharing is bounded by when it runs, and the contract does not claim more.
A lock written to a linked ID after the link was recorded reaches the other
diff --git a/internal/unifiedresources/canonical_id_succession.go b/internal/unifiedresources/canonical_id_succession.go
index 60ac83f43b..38d6b21e2b 100644
--- a/internal/unifiedresources/canonical_id_succession.go
+++ b/internal/unifiedresources/canonical_id_succession.go
@@ -84,8 +84,8 @@ func (s *SQLiteResourceStore) ApplyCanonicalIDSuccessions(successions []Canonica
applied := make([]CanonicalIDSuccession, 0, len(pending))
var carried []carriedLock
- var groups [][]string
- rekeyed := make(map[string]string, len(pending))
+ var groups []savedGroup
+ var rekeys []rekeyedID
for _, succession := range pending {
oldID := succession.OldCanonicalID
newID := succession.NewCanonicalID
@@ -139,8 +139,8 @@ func (s *SQLiteResourceStore) ApplyCanonicalIDSuccessions(successions []Canonica
if _, err := tx.Exec(`DELETE FROM resource_identities WHERE canonical_id = ?`, oldID); err != nil {
return fmt.Errorf("delete superseded identity pin %q: %w", oldID, err)
}
- groups = append(groups, groupAfterRekey(members, oldID, newID))
- rekeyed[oldID] = newID
+ groups = append(groups, savedGroup{index: len(rekeys), members: groupAfterRekey(members, oldID, newID)})
+ rekeys = append(rekeys, rekeyedID{oldID: oldID, newID: newID})
applied = append(applied, succession)
}
@@ -152,7 +152,7 @@ func (s *SQLiteResourceStore) ApplyCanonicalIDSuccessions(successions []Canonica
// the re-key of a later predecessor into the same successor.
sharedAt := time.Now().UTC()
for _, group := range groups {
- shared, err := shareRemediationLockAmongSQL(tx, settledGroup(group, rekeyed), sharedAt)
+ shared, err := shareRemediationLockAmongSQL(tx, settledGroup(group, rekeys), sharedAt)
if err != nil {
return fmt.Errorf("share remediation lock across linked resources %v: %w", group, err)
}
@@ -230,8 +230,8 @@ func (m *MemoryStore) ApplyCanonicalIDSuccessions(successions []CanonicalIDSucce
m.mu.Lock()
defer m.mu.Unlock()
- var groups [][]string
- rekeyed := make(map[string]string, len(successions))
+ var groups []savedGroup
+ var rekeys []rekeyedID
for _, succession := range successions {
oldID := CanonicalResourceID(succession.OldCanonicalID)
newID := CanonicalResourceID(succession.NewCanonicalID)
@@ -280,13 +280,13 @@ func (m *MemoryStore) ApplyCanonicalIDSuccessions(successions []CanonicalIDSucce
m.canonicalSuccessions = make(map[string]string)
}
m.canonicalSuccessions[oldID] = newID
- groups = append(groups, groupAfterRekey(members, oldID, newID))
- rekeyed[oldID] = newID
+ groups = append(groups, savedGroup{index: len(rekeys), members: groupAfterRekey(members, oldID, newID)})
+ rekeys = append(rekeys, rekeyedID{oldID: oldID, newID: newID})
}
// See the SQLite store: sharing waits for the whole batch.
sharedAt := time.Now().UTC()
for _, group := range groups {
- m.shareRemediationLockAmongLocked(settledGroup(group, rekeyed), sharedAt)
+ m.shareRemediationLockAmongLocked(settledGroup(group, rekeys), sharedAt)
}
return nil
}
diff --git a/internal/unifiedresources/registry_test.go b/internal/unifiedresources/registry_test.go
index a7118ef715..599346178b 100644
--- a/internal/unifiedresources/registry_test.go
+++ b/internal/unifiedresources/registry_test.go
@@ -8461,3 +8461,75 @@ func TestSQLiteStoreRestoresLocksAcrossRecordedSuccessionChains(t *testing.T) {
t.Fatalf("restoring the lock rewrote the successor's own settings: %+v", got)
}
}
+
+// A succession batch is applied in the order it is declared, and each group of
+// linked IDs it saved follows only the re-keys applied after it. A batch
+// declared against its direction, and one that returns an ID to its start, both
+// leave the lock on the IDs that survive and give no row to an ID re-keyed away.
+func TestSuccessionBatchOrderKeepsLinkedRemediationLock(t *testing.T) {
+ stores := map[string]func(t *testing.T) ResourceStore{
+ "memory": func(t *testing.T) ResourceStore { return NewMemoryStore() },
+ "sqlite": func(t *testing.T) ResourceStore {
+ store, err := NewSQLiteResourceStore(t.TempDir(), "default")
+ if err != nil {
+ t.Fatalf("NewSQLiteResourceStore: %v", err)
+ }
+ t.Cleanup(func() { _ = store.Close() })
+ return store
+ },
+ }
+ now := time.Now().UTC()
+ locked := func(t *testing.T, store ResourceStore, id string) bool {
+ t.Helper()
+ state, found, err := store.GetResourceOperatorState(id)
+ if err != nil {
+ t.Fatalf("read operator state of %s: %v", id, err)
+ }
+ return found && state.BlocksRemediation()
+ }
+ for name, newStore := range stores {
+ t.Run(name+"/declared against its direction", func(t *testing.T) {
+ store := newStore(t)
+ if err := store.AddLink(ResourceLink{ResourceA: "rev-a", ResourceB: "rev-b", PrimaryID: "rev-b"}); err != nil {
+ t.Fatalf("link: %v", err)
+ }
+ if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: "rev-a", NeverAutoRemediate: true, SetAt: now}); err != nil {
+ t.Fatalf("lock: %v", err)
+ }
+ if err := store.(canonicalIDSuccessor).ApplyCanonicalIDSuccessions([]CanonicalIDSuccession{
+ {OldCanonicalID: "rev-b", NewCanonicalID: "rev-c"},
+ {OldCanonicalID: "rev-a", NewCanonicalID: "rev-b"},
+ }); err != nil {
+ t.Fatalf("ApplyCanonicalIDSuccessions: %v", err)
+ }
+ for _, id := range []string{"rev-b", "rev-c"} {
+ if !locked(t, store, id) {
+ t.Fatalf("%s lost the lock when the batch was declared against its direction", id)
+ }
+ }
+ })
+ t.Run(name+"/returns an ID to its start", func(t *testing.T) {
+ store := newStore(t)
+ if err := store.AddLink(ResourceLink{ResourceA: "loop-a", ResourceB: "loop-x", PrimaryID: "loop-x"}); err != nil {
+ t.Fatalf("link: %v", err)
+ }
+ if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: "loop-a", NeverAutoRemediate: true, SetAt: now}); err != nil {
+ t.Fatalf("lock: %v", err)
+ }
+ if err := store.(canonicalIDSuccessor).ApplyCanonicalIDSuccessions([]CanonicalIDSuccession{
+ {OldCanonicalID: "loop-a", NewCanonicalID: "loop-b"},
+ {OldCanonicalID: "loop-b", NewCanonicalID: "loop-a"},
+ }); err != nil {
+ t.Fatalf("ApplyCanonicalIDSuccessions: %v", err)
+ }
+ for _, id := range []string{"loop-a", "loop-x"} {
+ if !locked(t, store, id) {
+ t.Fatalf("%s lost the lock across a batch that returned to its start", id)
+ }
+ }
+ if state, found, err := store.GetResourceOperatorState("loop-b"); err != nil || found {
+ t.Fatalf("the re-keyed intermediate ID was given a row: found=%v err=%v state=%+v", found, err, state)
+ }
+ })
+ }
+}
diff --git a/internal/unifiedresources/remediation_lock_carry.go b/internal/unifiedresources/remediation_lock_carry.go
index d38e968ebe..109d8f0aeb 100644
--- a/internal/unifiedresources/remediation_lock_carry.go
+++ b/internal/unifiedresources/remediation_lock_carry.go
@@ -282,28 +282,33 @@ func flattenComponents(components [][]string) []string {
return out
}
-// resolveThroughRekeys follows id through a batch's re-keys (a -> b, then
-// b -> c) to the ID that holds it once the whole batch has applied.
-func resolveThroughRekeys(id string, rekeyed map[string]string) string {
- for hops := 0; hops <= len(rekeyed); hops++ {
- next, moved := rekeyed[id]
- if !moved {
- return id
- }
- id = next
- }
- return id
+// rekeyedID is one applied succession of a batch.
+type rekeyedID struct {
+ oldID string
+ newID string
}
-// settledGroup maps a succession's saved sharing group through the batch's
-// re-keys, so a member a later succession in the batch re-keyed away is
-// replaced by its successor instead of being given a row of its own.
-func settledGroup(group []string, rekeyed map[string]string) []string {
- resolved := make([]string, 0, len(group))
- for _, id := range group {
- resolved = append(resolved, resolveThroughRekeys(id, rekeyed))
+// savedGroup is the sharing group of the succession at position index in a
+// batch's application order.
+type savedGroup struct {
+ index int
+ members []string
+}
+
+// settledGroup maps a saved group through the re-keys applied AFTER it, in
+// application order, so a member a later succession in the batch re-keyed away
+// is replaced by its successor instead of being given a row of its own. The
+// re-keys before the group's own succession are already reflected in it.
+func settledGroup(group savedGroup, rekeys []rekeyedID) []string {
+ members := append([]string(nil), group.members...)
+ for _, rekey := range rekeys[group.index+1:] {
+ for i, id := range members {
+ if id == rekey.oldID {
+ members[i] = rekey.newID
+ }
+ }
}
- return flattenComponents([][]string{resolved})
+ return flattenComponents([][]string{members})
}
// groupAfterRekey is the set a succession's pre-re-key members form once the