Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

master: replicate the recovery-wait growth unfreeze through raft (#12165)

Chris Lueec2452e30db
diff --git a/weed/server/raft_server.go b/weed/server/raft_server.go
index f274a821d2..ae81d30ada 100644
--- a/weed/server/raft_server.go
+++ b/weed/server/raft_server.go
@@ -60,9 +60,10 @@ func (s StateMachine) Save() ([]byte, error) {
MaxVolumeId: s.topo.GetMaxVolumeId(),
TopologyId: s.topo.GetTopologyId(),
}
- if s.topo.IsGrowthFrozen() {
+ if frozen, frozenAt := s.topo.GrowthFreezeState(); frozen {
state.GrowthFrozen = new(bool)
*state.GrowthFrozen = true
+ state.GrowthFrozenAt = frozenAt
}
glog.V(1).Infof("Save raft state %+v", state)
return json.Marshal(state)
@@ -81,7 +82,7 @@ func (s StateMachine) Recovery(data []byte) error {
glog.V(0).Infof("Recovered TopologyId: %s", state.TopologyId)
}
if state.GrowthFrozen != nil {
- s.topo.SetGrowthFrozen(*state.GrowthFrozen)
+ s.topo.SetGrowthFrozen(*state.GrowthFrozen, state.GrowthFrozenAt, state.GrowthUnfreezeFor)
}
if err := s.topo.PersistMaxVolumeId(state.MaxVolumeId); err != nil {
glog.Warningf("persist max volume id %d: %v", state.MaxVolumeId, err)
@@ -106,7 +107,7 @@ func (s *StateMachine) Apply(l *hashicorpRaft.Log) interface{} {
}
}
if state.GrowthFrozen != nil {
- s.topo.SetGrowthFrozen(*state.GrowthFrozen)
+ s.topo.SetGrowthFrozen(*state.GrowthFrozen, state.GrowthFrozenAt, state.GrowthUnfreezeFor)
}
if err := s.topo.PersistMaxVolumeId(state.MaxVolumeId); err != nil {
glog.Warningf("persist max volume id %d: %v", state.MaxVolumeId, err)
@@ -117,11 +118,12 @@ func (s *StateMachine) Apply(l *hashicorpRaft.Log) interface{} {
}
func (s *StateMachine) Snapshot() (hashicorpRaft.FSMSnapshot, error) {
- frozen := s.topo.IsGrowthFrozen()
+ frozen, frozenAt := s.topo.GrowthFreezeState()
state := &topology.MaxVolumeIdCommand{
- MaxVolumeId: s.topo.GetMaxVolumeId(),
- TopologyId: s.topo.GetTopologyId(),
- GrowthFrozen: &frozen,
+ MaxVolumeId: s.topo.GetMaxVolumeId(),
+ TopologyId: s.topo.GetTopologyId(),
+ GrowthFrozen: &frozen,
+ GrowthFrozenAt: frozenAt,
}
return state, nil
}
diff --git a/weed/topology/cluster_commands.go b/weed/topology/cluster_commands.go
index a4d7527863..c12528a2db 100644
--- a/weed/topology/cluster_commands.go
+++ b/weed/topology/cluster_commands.go
@@ -16,6 +16,17 @@ type MaxVolumeIdCommand struct {
// GrowthFrozen replicates the volume-growth freeze through raft so a
// leader change does not silently lift it. nil means no change.
GrowthFrozen *bool `json:"growthFrozen,omitempty"`
+ // GrowthFrozenAt pins the freeze instant so every master measures the
+ // same recovery window and can tell one freeze generation from the next.
+ // Zero on entries written before the field existed.
+ GrowthFrozenAt int64 `json:"growthFrozenAt,omitempty"`
+ // GrowthUnfreezeFor binds GrowthFrozen=false to the freeze generation
+ // the submitter observed: the lift applies only while that exact
+ // generation is armed, so a stale unfreeze cannot erase a freeze that
+ // reached the log first. Zero lifts unconditionally, for entries
+ // written before the binding existed; -1 binds the ageless generation
+ // left by freeze entries written before instants existed.
+ GrowthUnfreezeFor int64 `json:"growthUnfreezeFor,omitempty"`
}
func NewMaxVolumeIdCommand(value needle.VolumeId, topologyId string) *MaxVolumeIdCommand {
@@ -47,7 +58,7 @@ func (c *MaxVolumeIdCommand) Apply(server raft.Server) (interface{}, error) {
}
}
if c.GrowthFrozen != nil {
- topo.SetGrowthFrozen(*c.GrowthFrozen)
+ topo.SetGrowthFrozen(*c.GrowthFrozen, c.GrowthFrozenAt, c.GrowthUnfreezeFor)
}
if err := topo.PersistMaxVolumeId(c.MaxVolumeId); err != nil {
glog.Warningf("persist max volume id %d: %v", c.MaxVolumeId, err)
diff --git a/weed/topology/growth_freeze_test.go b/weed/topology/growth_freeze_test.go
index 65e6e5812c..2404c809f2 100644
--- a/weed/topology/growth_freeze_test.go
+++ b/weed/topology/growth_freeze_test.go
@@ -1,10 +1,36 @@
package topology
import (
+ "errors"
"testing"
"time"
+
+ "github.com/seaweedfs/raft"
)
+// appliedRaftServer answers as leader and records each command Do applies.
+type appliedRaftServer struct {
+ raft.Server
+ applied chan raft.Command
+}
+
+func (appliedRaftServer) State() string { return raft.Leader }
+
+func (s appliedRaftServer) Do(c raft.Command) (interface{}, error) {
+ s.applied <- c
+ return nil, nil
+}
+
+type failingRaftServer struct {
+ raft.Server
+}
+
+func (failingRaftServer) State() string { return raft.Leader }
+
+func (failingRaftServer) Do(raft.Command) (interface{}, error) {
+ return nil, errors.New("not the leader")
+}
+
func TestGrowthFreezeOnTopologyMismatch(t *testing.T) {
topo := NewTopology("topo", nil, 128, 3, false)
topo.SetGrowthFreezeConfig(true, 0)
@@ -58,6 +84,120 @@ func TestGrowthFreezeAutoRecovery(t *testing.T) {
}
}
+func TestGrowthFreezeAutoRecoveryReplicates(t *testing.T) {
+ applied := make(chan raft.Command, 2)
+ topo := NewTopology("topo", nil, 128, 3, false)
+ topo.RaftServer = appliedRaftServer{applied: applied}
+ topo.SetGrowthFreezeConfig(true, 50*time.Millisecond)
+ topo.SetTopologyId("cluster-a")
+
+ topo.CheckReportedTopologyId("cluster-b")
+ if !topo.IsGrowthFrozen() {
+ t.Fatal("mismatching topology id should freeze growth")
+ }
+ cmd := (<-applied).(*MaxVolumeIdCommand)
+ if cmd.GrowthFrozen == nil || !*cmd.GrowthFrozen {
+ t.Fatal("the freeze should be replicated")
+ }
+
+ time.Sleep(60 * time.Millisecond)
+ if topo.IsGrowthFrozen() {
+ t.Fatal("growth should auto-resume past the recovery wait")
+ }
+ select {
+ case c := <-applied:
+ lift := c.(*MaxVolumeIdCommand)
+ if lift.GrowthFrozen == nil || *lift.GrowthFrozen {
+ t.Fatal("the auto unfreeze should be replicated like volume.growth.resume")
+ }
+ if lift.GrowthUnfreezeFor != cmd.GrowthFrozenAt {
+ t.Fatal("the unfreeze should be bound to the freeze instant it lifts")
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("auto unfreeze was not replicated; a restart would re-apply the freeze")
+ }
+}
+
+// A repeat mismatch while already frozen must keep the generation the
+// replicated freeze carries: re-stamping locally would bind every later
+// lift to a generation followers never saw, and the freeze would return on
+// replay.
+func TestRepeatMismatchKeepsFreezeGeneration(t *testing.T) {
+ topo := NewTopology("topo", nil, 128, 3, false)
+ topo.SetGrowthFreezeConfig(true, 0)
+ topo.SetTopologyId("cluster-a")
+
+ topo.CheckReportedTopologyId("cluster-b")
+ gen := topo.growthFrozenState.Load()
+ if gen == 0 {
+ t.Fatal("mismatching topology id should freeze growth")
+ }
+ topo.CheckReportedTopologyId("cluster-c")
+ if topo.growthFrozenState.Load() != gen {
+ t.Fatal("a repeat mismatch must not re-stamp the armed freeze")
+ }
+}
+
+// Snapshots persist the flag and the generation together, so the pair must
+// come from one read: a lifted freeze reports (false, 0) and an armed one
+// reports its generation.
+func TestGrowthFreezeStateConsistentPair(t *testing.T) {
+ topo := NewTopology("topo", nil, 128, 3, false)
+ topo.SetGrowthFreezeConfig(true, 0)
+
+ topo.SetGrowthFrozen(true, time.Now().UnixNano(), 0)
+ armed := topo.growthFrozenState.Load()
+ if frozen, at := topo.GrowthFreezeState(); !frozen || at != armed {
+ t.Fatalf("armed freeze should report (true, %d), got (%v, %d)", armed, frozen, at)
+ }
+ topo.ResumeVolumeGrowth()
+ if frozen, at := topo.GrowthFreezeState(); frozen || at != 0 {
+ t.Fatalf("lifted freeze should report (false, 0), got (%v, %d)", frozen, at)
+ }
+}
+
+func TestReplicateGrowthUnfreezeSkipsRefreeze(t *testing.T) {
+ applied := make(chan raft.Command, 1)
+ topo := NewTopology("topo", nil, 128, 3, false)
+ topo.RaftServer = appliedRaftServer{applied: applied}
+
+ topo.SetGrowthFrozen(true, 1000, 0)
+ topo.SetGrowthFrozen(true, 2000, 0)
+ topo.replicateGrowthUnfreeze(1000)
+ select {
+ case <-applied:
+ t.Fatal("a stale unfreeze must not lift a freeze that landed after it")
+ default:
+ }
+ if topo.growthFrozenState.Load() != 2000 {
+ t.Fatal("the newer freeze must stay armed")
+ }
+}
+
+func TestReplicateGrowthUnfreezeRearmsOnFailure(t *testing.T) {
+ topo := NewTopology("topo", nil, 128, 3, false)
+ topo.RaftServer = failingRaftServer{}
+
+ topo.SetGrowthFrozen(true, 1000, 0)
+ topo.growthFrozenState.CompareAndSwap(1000, 0) // the local lift
+ topo.replicateGrowthUnfreeze(1000)
+ if topo.growthFrozenState.Load() != 1000 {
+ t.Fatal("a failed replication must re-arm the freeze so a later read retries the lift")
+ }
+}
+
+func TestReplicateGrowthUnfreezeDoesNotRearmAfterResume(t *testing.T) {
+ topo := NewTopology("topo", nil, 128, 3, false)
+
+ topo.SetGrowthFrozen(true, 1000, 0)
+ topo.SetGrowthFrozen(true, 2000, 0)
+ topo.ResumeVolumeGrowth()
+ topo.replicateGrowthUnfreeze(1000)
+ if topo.growthFrozenState.Load() != 0 {
+ t.Fatal("a superseded lift must not re-arm the freeze an operator resumed")
+ }
+}
+
func TestCheckReportedTopologyIdBeforeTopologyIdKnown(t *testing.T) {
topo := NewTopology("topo", nil, 128, 3, false)
topo.SetGrowthFreezeConfig(true, 0)
@@ -118,17 +258,57 @@ func TestGrowthFreezeCommandApply(t *testing.T) {
topo.SetGrowthFreezeConfig(true, 0)
frozen := true
- topo.SetGrowthFrozen(frozen)
+ topo.SetGrowthFrozen(frozen, 0, 0)
if !topo.IsGrowthFrozen() {
t.Fatal("replicated freeze should set the flag")
}
- unfrozen := false
- topo.SetGrowthFrozen(unfrozen)
+ topo.SetGrowthFrozen(false, 0, 0)
if topo.IsGrowthFrozen() {
t.Fatal("replicated unfreeze should clear the flag")
}
}
+// Freeze entries written before the instant field existed replay as their
+// own stable generation: a lift bound to it still applies, a lift bound to a
+// different generation cannot clear it, and the recovery window reads as
+// already elapsed — a replay-time stamp would shadow every committed
+// unfreeze and re-apply the freeze on each restart.
+func TestLegacyFreezeReplayStaysAgeless(t *testing.T) {
+ topo := NewTopology("topo", nil, 128, 3, false)
+ topo.SetGrowthFreezeConfig(true, time.Minute)
+
+ topo.SetGrowthFrozen(true, 0, 0)
+ topo.SetGrowthFrozen(false, 0, 1234)
+ if topo.growthFrozenState.Load() != -1 {
+ t.Fatal("a lift bound to a different generation must not clear the ageless freeze")
+ }
+ topo.SetGrowthFrozen(false, 0, -1)
+ if topo.growthFrozenState.Load() != 0 {
+ t.Fatal("a lift bound to the ageless generation should clear it")
+ }
+
+ topo.SetGrowthFrozen(true, 0, 0)
+ if topo.IsGrowthFrozen() {
+ t.Fatal("an ageless freeze should read its recovery window as elapsed")
+ }
+}
+
+// A lift bound to a freeze instant must not erase a newer freeze, so a stale
+// unfreeze cannot overwrite a freeze that committed first but applied later.
+func TestGrowthUnfreezeBoundToFreezeInstant(t *testing.T) {
+ topo := NewTopology("topo", nil, 128, 3, false)
+
+ topo.SetGrowthFrozen(true, 1000, 0)
+ topo.SetGrowthFrozen(false, 0, 999)
+ if !topo.IsGrowthFrozen() {
+ t.Fatal("a lift for an older freeze instant must not clear a newer freeze")
+ }
+ topo.SetGrowthFrozen(false, 0, 1000)
+ if topo.IsGrowthFrozen() {
+ t.Fatal("a lift bound to the current freeze instant should clear it")
+ }
+}
+
func TestAllocationDelay(t *testing.T) {
topo := NewTopology("topo", nil, 128, 3, false)
diff --git a/weed/topology/topology.go b/weed/topology/topology.go
index 177c66a63d..5559240385 100644
--- a/weed/topology/topology.go
+++ b/weed/topology/topology.go
@@ -92,8 +92,14 @@ type Topology struct {
lastPersistedTopologyId atomic.Value
lastPersistedGrowthFrozen atomic.Bool
- growthFrozen atomic.Bool
- growthFrozenAt atomic.Int64
+ // growthFrozenState packs the freeze flag and its generation into one
+ // value: 0 = unfrozen, -1 = frozen without a recorded instant (entries
+ // written before the field existed), >0 = frozen at that instant. Every
+ // transition is a single atomic swap or compare-and-swap, so a lift can
+ // only clear the exact generation it was bound to — a stale unfreeze
+ // cannot erase a newer freeze, a resume cannot lift a generation it did
+ // not observe, and stamps need not be monotonic across masters.
+ growthFrozenState atomic.Int64
metaReadOnly atomic.Bool
freezeOnMismatch atomic.Bool
stateLossRecoveryWait atomic.Int64
@@ -122,6 +128,11 @@ type Topology struct {
reportedTopologyIdsLock sync.Mutex
pendingReportedTopologyIds map[string]struct{}
+ // growthFreezeApplyLock serializes replicated freeze writes so the log
+ // order of concurrent mismatch reports cannot land an older freeze
+ // instant after a newer one. Unfreezes skip it: the state machine
+ // applies a lift only against the freeze generation it targeted.
+ growthFreezeApplyLock sync.Mutex
// quarantinedVolumeIds are (node, vid) pairs force-read-only because the
// vid collided across layouts. Kept separate from the reported flag so
@@ -1591,15 +1602,22 @@ func (t *Topology) CheckReportedTopologyId(reported string) {
return
}
t.reportedTopologyIdsLock.Lock()
- defer t.reportedTopologyIdsLock.Unlock()
- t.checkReportedTopologyIdLocked(reported)
+ freeze := t.checkReportedTopologyIdLocked(reported)
+ t.reportedTopologyIdsLock.Unlock()
+ if freeze {
+ // The decision landed under the lock; the raft wait must not, or a
+ // slow apply would stall every later report — including one that
+ // needs to trigger the next freeze.
+ t.applyGrowthFreeze()
+ }
}
// checkReportedTopologyIdLocked runs the mismatch check inside
// reportedTopologyIdsLock, the same critical section the init gate drains
// under: a report received before the gate opens is either stashed or frozen
-// before the gate can release.
-func (t *Topology) checkReportedTopologyIdLocked(reported string) {
+// before the gate can release. It returns whether the freeze must be
+// replicated, which the caller does after releasing the lock.
+func (t *Topology) checkReportedTopologyIdLocked(reported string) (freeze bool) {
own := t.GetTopologyId()
if own == "" {
// The local topology id is still being minted. Keep the report so a
@@ -1609,82 +1627,175 @@ func (t *Topology) checkReportedTopologyIdLocked(reported string) {
t.pendingReportedTopologyIds = make(map[string]struct{})
}
t.pendingReportedTopologyIds[reported] = struct{}{}
- return
+ return false
}
if reported == own {
- return
+ return false
}
- t.growthFrozenAt.Store(time.Now().UnixNano())
- if t.growthFrozen.CompareAndSwap(false, true) {
+ // Only the first mismatch stamps a generation: a re-stamp never reaches
+ // the log, so bound lifts would target a generation followers never saw.
+ if t.growthFrozenState.CompareAndSwap(0, time.Now().UnixNano()) {
stats.MasterVolumeGrowthFrozen.Set(1)
glog.Errorf("volume growth frozen: a volume server reports topology id %s but this cluster is %s; the master state was rebuilt. Verify the cluster inventory, then run: volume.growth.resume", reported, own)
- t.applyGrowthFreeze(true)
+ return true
}
+ return false
}
func (t *Topology) IsGrowthFrozen() bool {
- if !t.growthFrozen.Load() {
- return false
+ frozen, _ := t.GrowthFreezeState()
+ return frozen
+}
+
+// GrowthFreezeState returns the freeze flag and its generation from one
+// load, so snapshots persisting both cannot split the pair across a lift.
+func (t *Topology) GrowthFreezeState() (frozen bool, frozenAt int64) {
+ state := t.growthFrozenState.Load()
+ if state == 0 {
+ return false, 0
}
if wait := t.stateLossRecoveryWait.Load(); wait > 0 {
- if time.Duration(time.Now().UnixNano()-t.growthFrozenAt.Load()) > time.Duration(wait) {
- if t.growthFrozen.CompareAndSwap(true, false) {
+ // A legacy generation carries no instant; its window counts as
+ // already elapsed.
+ if state < 0 || time.Duration(time.Now().UnixNano()-state) > time.Duration(wait) {
+ if t.growthFrozenState.CompareAndSwap(state, 0) {
stats.MasterVolumeGrowthFrozen.Set(0)
glog.V(0).Infof("volume growth unfrozen after %v", time.Duration(wait))
+ go t.replicateGrowthUnfreeze(state)
+ return false, 0
}
- return false
+ state = t.growthFrozenState.Load()
}
}
- return true
+ return state != 0, state
+}
+
+// replicateGrowthUnfreeze pushes the recovery-wait lift through raft like
+// ResumeVolumeGrowth, so a restart does not replay the freeze. The command
+// carries the generation it lifts: a newer freeze — even one committed but
+// not yet applied when this node won an election — makes the apply a no-op
+// instead of being erased. If the write never lands, the generation is
+// re-armed so a later IsGrowthFrozen retries the lift.
+func (t *Topology) replicateGrowthUnfreeze(frozenAt int64) {
+ for attempt := 0; attempt < 3; attempt++ {
+ if attempt > 0 {
+ time.Sleep(time.Duration(attempt) * time.Second)
+ }
+ // The caller's lift already cleared the flag, so unfrozen is the
+ // state to expect; anything else armed is a newer generation this
+ // lift must not touch — and falling through to the re-arm would
+ // undo a resume that followed it.
+ if s := t.growthFrozenState.Load(); s != 0 && s != frozenAt {
+ return
+ }
+ if err := t.applyGrowthUnfreeze(frozenAt); err == nil {
+ return
+ }
+ }
+ if frozenAt != 0 && t.growthFrozenState.CompareAndSwap(0, frozenAt) {
+ stats.MasterVolumeGrowthFrozen.Set(1)
+ glog.Warningf("volume growth unfreeze not replicated; freeze re-armed for retry")
+ }
}
func (t *Topology) ResumeVolumeGrowth() bool {
- if !t.growthFrozen.CompareAndSwap(true, false) {
- return false
+ for {
+ // The compare binds the resume to the generation it observed: a
+ // freeze landing between the load and the swap just moves to the
+ // next iteration, which lifts the generation actually armed.
+ frozenAt := t.growthFrozenState.Load()
+ if frozenAt == 0 {
+ return false
+ }
+ if t.growthFrozenState.CompareAndSwap(frozenAt, 0) {
+ stats.MasterVolumeGrowthFrozen.Set(0)
+ glog.V(0).Infof("volume growth resumed by operator")
+ t.applyGrowthUnfreeze(frozenAt)
+ return true
+ }
}
- stats.MasterVolumeGrowthFrozen.Set(0)
- glog.V(0).Infof("volume growth resumed by operator")
- t.applyGrowthFreeze(false)
- return true
}
-// SetGrowthFrozen applies a replicated freeze state on every master. The
-// timestamp lands before the flag so a reader can never see frozen with a
-// zero time and misjudge the auto-resume window.
-func (t *Topology) SetGrowthFrozen(frozen bool) {
+// SetGrowthFrozen applies a replicated freeze transition on every master. A
+// positive frozenAt pins the instant so all nodes share one recovery
+// window; a non-positive one — entries written before the field existed —
+// becomes the ageless generation, a stable identity a bound lift can still
+// match and the recovery check reads as already elapsed. An unfreeze bound
+// to unfreezeFor applies only while that exact generation is armed, so a
+// stale lift cannot erase a newer freeze; zero lifts unconditionally, for
+// entries written before the binding existed.
+func (t *Topology) SetGrowthFrozen(frozen bool, frozenAt int64, unfreezeFor int64) {
if frozen {
- t.growthFrozenAt.Store(time.Now().UnixNano())
- if t.growthFrozen.CompareAndSwap(false, true) {
+ gen := frozenAt
+ if gen == 0 {
+ gen = -1
+ }
+ if t.growthFrozenState.Swap(gen) == 0 {
stats.MasterVolumeGrowthFrozen.Set(1)
glog.V(0).Infof("volume growth freeze applied from raft")
}
return
}
- if t.growthFrozen.CompareAndSwap(true, false) {
+ if unfreezeFor == 0 {
+ if t.growthFrozenState.Swap(0) != 0 {
+ stats.MasterVolumeGrowthFrozen.Set(0)
+ }
+ return
+ }
+ if t.growthFrozenState.CompareAndSwap(unfreezeFor, 0) {
stats.MasterVolumeGrowthFrozen.Set(0)
}
}
-// applyGrowthFreeze replicates the freeze flag so followers keep it across a
-// leader change. No raft server (single-master test topologies) just sets it.
-func (t *Topology) applyGrowthFreeze(frozen bool) {
+// applyGrowthFreeze replicates the current freeze flag and instant so
+// followers keep both across a leader change.
+func (t *Topology) applyGrowthFreeze() error {
+ t.growthFreezeApplyLock.Lock()
+ defer t.growthFreezeApplyLock.Unlock()
cmd := NewMaxVolumeIdCommand(t.GetMaxVolumeId(), t.GetTopologyId())
+ frozen := true
cmd.GrowthFrozen = &frozen
+ cmd.GrowthFrozenAt = t.growthFrozenState.Load()
+ return t.applyGrowthFreezeCommand(cmd)
+}
+
+// applyGrowthUnfreeze replicates a lift bound to the freeze instant the
+// caller observed.
+func (t *Topology) applyGrowthUnfreeze(frozenAt int64) error {
+ cmd := NewMaxVolumeIdCommand(t.GetMaxVolumeId(), t.GetTopologyId())
+ frozen := false
+ cmd.GrowthFrozen = &frozen
+ cmd.GrowthUnfreezeFor = frozenAt
+ return t.applyGrowthFreezeCommand(cmd)
+}
+
+// applyGrowthFreezeCommand sends the transition through raft. With no raft
+// server (single-master test topologies) it applies locally.
+func (t *Topology) applyGrowthFreezeCommand(cmd *MaxVolumeIdCommand) error {
+ frozen := cmd.GrowthFrozen != nil && *cmd.GrowthFrozen
t.RaftServerAccessLock.RLock()
defer t.RaftServerAccessLock.RUnlock()
if t.RaftServer != nil {
if _, err := t.RaftServer.Do(cmd); err != nil {
glog.Warningf("apply growth freeze %v: %v", frozen, err)
+ return err
}
- } else if t.HashicorpRaft != nil {
- if b, err := json.Marshal(cmd); err != nil {
+ return nil
+ }
+ if t.HashicorpRaft != nil {
+ b, err := json.Marshal(cmd)
+ if err != nil {
glog.Warningf("marshal growth freeze %v: %v", frozen, err)
- } else if future := t.HashicorpRaft.Apply(b, time.Second); future.Error() != nil {
+ return err
+ }
+ if future := t.HashicorpRaft.Apply(b, time.Second); future.Error() != nil {
glog.Warningf("apply growth freeze %v: %v", frozen, future.Error())
+ return future.Error()
}
- } else {
- t.SetGrowthFrozen(frozen)
+ return nil
}
+ t.SetGrowthFrozen(*cmd.GrowthFrozen, cmd.GrowthFrozenAt, cmd.GrowthUnfreezeFor)
+ return nil
}
// MarkFreshMint records that this boot minted the TopologyId rather than
@@ -1713,13 +1824,17 @@ func (t *Topology) SetTopologyInitPending(pending bool) {
// a mismatching report checked here, or still in flight under the same
// lock, lands its freeze before any allocation can observe the open gate.
t.reportedTopologyIdsLock.Lock()
- defer t.reportedTopologyIdsLock.Unlock()
reported := t.pendingReportedTopologyIds
t.pendingReportedTopologyIds = nil
+ freeze := false
for id := range reported {
- t.checkReportedTopologyIdLocked(id)
+ freeze = t.checkReportedTopologyIdLocked(id) || freeze
}
t.topologyInitPending.Store(false)
+ t.reportedTopologyIdsLock.Unlock()
+ if freeze {
+ t.applyGrowthFreeze()
+ }
}
// ReleaseAllocationOnFirstReport opts the topology into lifting the post-mint
@@ -1879,7 +1994,7 @@ func (t *Topology) SetMaxVolumeIdFile(file string) {
}
if cmd.GrowthFrozen != nil {
t.lastPersistedGrowthFrozen.Store(*cmd.GrowthFrozen)
- t.SetGrowthFrozen(*cmd.GrowthFrozen)
+ t.SetGrowthFrozen(*cmd.GrowthFrozen, cmd.GrowthFrozenAt, 0)
}
}
@@ -1888,7 +2003,7 @@ func (t *Topology) SetMaxVolumeIdFile(file string) {
// recovery, so follower mirrors stay current too.
func (t *Topology) PersistMaxVolumeId(vid needle.VolumeId) error {
topoId := t.GetTopologyId()
- frozen := t.IsGrowthFrozen()
+ frozen, frozenAt := t.GrowthFreezeState()
mirrored := func() bool {
return t.lastPersistedMaxVolumeId.Load() >= uint32(vid) && t.lastPersistedTopologyId.Load() == topoId && t.lastPersistedGrowthFrozen.Load() == frozen
}
@@ -1908,6 +2023,9 @@ func (t *Topology) PersistMaxVolumeId(vid needle.VolumeId) error {
}
cmd := NewMaxVolumeIdCommand(vid, topoId)
cmd.GrowthFrozen = &frozen
+ if frozen {
+ cmd.GrowthFrozenAt = frozenAt
+ }
b, err := json.Marshal(cmd)
if err != nil {
return err