Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

admin/maintenance: reload in-flight tasks on startup instead of discarding them (#9857)

Chris Lu8a4fdf06c048
diff --git a/weed/admin/maintenance/maintenance_queue.go b/weed/admin/maintenance/maintenance_queue.go
index 1cec742b65..8af9c8f909 100644
--- a/weed/admin/maintenance/maintenance_queue.go
+++ b/weed/admin/maintenance/maintenance_queue.go
@@ -34,16 +34,66 @@ func (mq *MaintenanceQueue) SetPersistence(persistence TaskPersistence) {
glog.V(1).Infof("Maintenance queue configured with task persistence")
}
-// LoadTasksFromPersistence is called on startup. Previous task states are NOT loaded
-// into memory — the maintenance scanner will re-detect current needs from the live
-// cluster state. Stale task files from previous runs are deleted from disk.
+// LoadTasksFromPersistence is called on startup to restore in-flight tasks
+// across an admin restart. Non-terminal tasks (pending/assigned/in_progress)
+// are re-queued as pending — the worker that held an in-flight task is gone
+// after a restart, and maintenance tasks are idempotent, so re-running a
+// partially-done one is safe. Terminal task files are deleted. Anything missed
+// is still re-detected by the scanner from live cluster state.
func (mq *MaintenanceQueue) LoadTasksFromPersistence() error {
- if mq.persistence != nil {
- if err := mq.persistence.DeleteAllTaskStates(); err != nil {
- glog.Warningf("Failed to clean up old task files: %v", err)
+ if mq.persistence == nil {
+ glog.Infof("Task queue initialized (no persistence configured)")
+ return nil
+ }
+ tasks, err := mq.persistence.LoadAllTaskStates()
+ if err != nil {
+ glog.Warningf("Failed to load persisted task states: %v; starting with an empty queue", err)
+ return nil
+ }
+
+ var terminal []string
+ var restored []*MaintenanceTask
+ mq.mutex.Lock()
+ requeued := 0
+ for _, task := range tasks {
+ if task == nil {
+ continue
+ }
+ switch task.Status {
+ case TaskStatusPending, TaskStatusAssigned, TaskStatusInProgress:
+ task.Status = TaskStatusPending
+ task.WorkerID = ""
+ task.Progress = 0
+ mq.tasks[task.ID] = task
+ mq.pendingTasks = append(mq.pendingTasks, task)
+ restored = append(restored, snapshotTask(task))
+ requeued++
+ default:
+ terminal = append(terminal, task.ID)
}
}
- glog.Infof("Task queue initialized (previous tasks will be re-detected by scanner)")
+ sort.Slice(mq.pendingTasks, func(i, j int) bool {
+ if mq.pendingTasks[i].Priority != mq.pendingTasks[j].Priority {
+ return mq.pendingTasks[i].Priority > mq.pendingTasks[j].Priority
+ }
+ return mq.pendingTasks[i].ScheduledAt.Before(mq.pendingTasks[j].ScheduledAt)
+ })
+ mq.mutex.Unlock()
+
+ // ActiveTopology is in-memory and starts empty, so restored tasks must be
+ // re-synced or GetNextTask's AssignTask would reject them as unknown and
+ // they'd sit pending forever. Done outside the lock, like the retry path.
+ if mq.integration != nil {
+ for _, task := range restored {
+ mq.integration.SyncTask(task)
+ }
+ }
+
+ // Delete stale terminal files outside the lock to avoid blocking on disk I/O.
+ for _, id := range terminal {
+ mq.deleteTaskState(id)
+ }
+ glog.Infof("Task queue initialized: re-queued %d in-flight task(s) from persistence; scanner will re-detect the rest", requeued)
return nil
}
diff --git a/weed/admin/maintenance/maintenance_queue_test.go b/weed/admin/maintenance/maintenance_queue_test.go
index bd8fe6839d..7a1328919c 100644
--- a/weed/admin/maintenance/maintenance_queue_test.go
+++ b/weed/admin/maintenance/maintenance_queue_test.go
@@ -697,8 +697,7 @@ func (m *MockPersistence) DeleteAllTaskStates() error
func (m *MockPersistence) CleanupCompletedTasks() error { return nil }
func (m *MockPersistence) SaveTaskPolicy(taskType string, policy *TaskPolicy) error { return nil }
-func TestMaintenanceQueue_LoadTasksStartsEmpty(t *testing.T) {
- // Setup
+func TestMaintenanceQueue_LoadRequeuesInFlightTasks(t *testing.T) {
policy := &MaintenancePolicy{
TaskPolicies: map[string]*worker_pb.TaskPolicy{
"balance": {MaxConcurrent: 1},
@@ -706,24 +705,30 @@ func TestMaintenanceQueue_LoadTasksStartsEmpty(t *testing.T) {
}
mq := NewMaintenanceQueue(policy)
- // Setup mock persistence with tasks — these should NOT be loaded
- mockTask := &MaintenanceTask{
- ID: "old_task_123",
- Type: "balance",
- Status: TaskStatusPending,
- }
- mq.SetPersistence(&MockPersistence{tasks: []*MaintenanceTask{mockTask}})
+ // Non-terminal tasks must be re-queued across a restart; an in-progress
+ // task is reset to pending (its worker is gone). Terminal tasks are dropped.
+ mq.SetPersistence(&MockPersistence{tasks: []*MaintenanceTask{
+ {ID: "pending_1", Type: "balance", Status: TaskStatusPending},
+ {ID: "inprogress_1", Type: "balance", Status: TaskStatusInProgress, WorkerID: "gone", Progress: 42},
+ {ID: "done_1", Type: "balance", Status: TaskStatusCompleted},
+ }})
- // LoadTasksFromPersistence should be a no-op — scanner will re-detect
- err := mq.LoadTasksFromPersistence()
- if err != nil {
+ if err := mq.LoadTasksFromPersistence(); err != nil {
t.Fatalf("LoadTasksFromPersistence failed: %v", err)
}
- // Queue should be empty — tasks will be re-detected by scanner
- stats := mq.GetStats()
- if stats.TotalTasks != 0 {
- t.Errorf("Expected 0 tasks after startup, got %d", stats.TotalTasks)
+ if stats := mq.GetStats(); stats.TotalTasks != 2 {
+ t.Errorf("expected 2 re-queued tasks, got %d", stats.TotalTasks)
+ }
+ ip := mq.tasks["inprogress_1"]
+ if ip == nil {
+ t.Fatal("in-progress task was not re-queued")
+ }
+ if ip.Status != TaskStatusPending || ip.WorkerID != "" || ip.Progress != 0 {
+ t.Errorf("in-progress task not reset to pending: status=%q worker=%q progress=%v", ip.Status, ip.WorkerID, ip.Progress)
+ }
+ if mq.tasks["done_1"] != nil {
+ t.Error("completed task should not be re-queued")
}
}