Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

fix(workspace): never report a failed or interrupted restore as ready (#129)

Robert0622eb28d2337f09
diff --git a/internal/engine/workspace_restore.go b/internal/engine/workspace_restore.go
index 0cf620ee..a23203b8 100644
--- a/internal/engine/workspace_restore.go
+++ b/internal/engine/workspace_restore.go
@@ -26,14 +26,18 @@ func (e *Engine) WorkspaceRestoreStatus(id string) (domain.WorkspaceRestoreStatu
return e.WorkspaceRestoreStatus(workspaceRunID(run))
}
state := domain.WorkspaceRestoreStatus{Status: "idle", StartedAt: meta.RestoreStarted}
- if meta.RestoreStatus == "restoring" && meta.LeaseUntil != nil && meta.LeaseUntil.After(time.Now()) {
+ // A restore clears the cleanup mark when it binds the workspace, before its
+ // remaining work finishes. Its recorded outcome therefore takes precedence
+ // over the binding: a failed or interrupted restore is never "ready".
+ switch {
+ case meta.RestoreStatus == "restoring" && meta.LeaseUntil != nil && meta.LeaseUntil.After(time.Now()):
state.Status = "restoring"
- } else if run.WorkspaceCleanedAt.IsZero() || meta.SavedAt == nil {
- state.Status = "ready"
- } else if meta.RestoreStatus == "failed" {
+ case meta.RestoreStatus == "failed":
state.Status, state.Error = "failed", meta.RestoreError
- } else if meta.RestoreStatus == "restoring" {
- state.Status, state.Error = "failed", "工作区恢复已中断,请重新发起重跑"
+ case meta.RestoreStatus == "restoring":
+ state.Status, state.Error = "failed", repo.WorkspaceRestoreInterrupted
+ case run.WorkspaceCleanedAt.IsZero() || meta.SavedAt == nil:
+ state.Status = "ready"
}
return state, nil
}
diff --git a/internal/engine/workspace_restore_test.go b/internal/engine/workspace_restore_test.go
index 12a5ad96..b9b46ea6 100644
--- a/internal/engine/workspace_restore_test.go
+++ b/internal/engine/workspace_restore_test.go
@@ -277,6 +277,105 @@ func TestWorkspaceRestoreExpiredLeaseFencesOldCompletion(t *testing.T) {
}
}
+func TestWorkspaceRestoreOutcomeOutranksClearedBinding(t *testing.T) {
+ t.Parallel()
+ for _, backend := range []string{"sqlite", "mysql"} {
+ for _, scenario := range []string{"expired-after-binding", "failed-after-binding"} {
+ t.Run(backend+"/"+scenario, func(t *testing.T) {
+ worker, controller, id := remoteWorkspaceRestoreFixture(t, backend)
+ now := time.Now().UTC()
+ token, err := worker.repo.BeginWorkspaceRestore(id, now)
+ if err != nil || token == "" {
+ t.Fatalf("restore lease not acquired: %q %v", token, err)
+ }
+ // The restore binds the workspace, which clears the cleanup mark,
+ // before the rest of its work has finished.
+ if err = worker.repo.RestoreCheckpointWorkspaceBinding(id, token, worker.workspaceHost, worker.sandboxes.Root(), now); err != nil {
+ t.Fatal(err)
+ }
+ wantError := repo.WorkspaceRestoreInterrupted
+ if scenario == "expired-after-binding" {
+ // The process exits before recording an outcome.
+ if err = worker.repo.DB.Model(&repo.WorkspaceCheckpoint{}).Where("run_id = ?", id).
+ Update("lease_until", now.Add(-time.Minute)).Error; err != nil {
+ t.Fatal(err)
+ }
+ } else {
+ wantError = "post-binding failure"
+ if err = worker.repo.FinishWorkspaceRestore(id, token, now, wantError); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if run, _ := worker.repo.RunState(id); !run.WorkspaceCleanedAt.IsZero() {
+ t.Fatal("fixture binding did not clear the cleanup mark")
+ }
+ state, err := controller.WorkspaceRestoreStatus(id)
+ if err != nil || state.Status != "failed" || state.Error != wantError {
+ t.Fatalf("unfinished restore reported as %+v %v", state, err)
+ }
+ // The failure stays retryable although the workspace is bound.
+ if state, err = controller.StartWorkspaceRestore(id); err != nil || state.Status == "failed" {
+ t.Fatalf("retry was not started: %+v %v", state, err)
+ }
+ if state = awaitWorkspaceRestore(t, controller, id); state.Status != "ready" || state.Error != "" {
+ t.Fatalf("retry did not finish the restore: %+v", state)
+ }
+ if meta, _ := worker.repo.WorkspaceCheckpoint(id); meta.RestoreStatus != "ready" || meta.LeaseToken != "" {
+ t.Fatalf("retry left restore state %q lease=%q", meta.RestoreStatus, meta.LeaseToken)
+ }
+ })
+ }
+ }
+}
+
+func TestWorkspaceUseSupersedesEarlierRestoreOutcome(t *testing.T) {
+ t.Parallel()
+ for _, backend := range []string{"sqlite", "mysql"} {
+ for _, scenario := range []string{"failed", "interrupted"} {
+ t.Run(backend+"/"+scenario, func(t *testing.T) {
+ worker, controller, id := remoteWorkspaceRestoreFixture(t, backend)
+ now := time.Now().UTC()
+ var old string
+ var err error
+ if scenario == "failed" {
+ old, err = worker.repo.BeginWorkspaceRestore(id, now)
+ if err == nil {
+ err = worker.repo.FinishWorkspaceRestore(id, old, now, "runner disconnected")
+ }
+ } else {
+ old, err = worker.repo.BeginWorkspaceRestore(id, now.Add(-20*time.Minute))
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ // An ordinary workspace use takes the lease and binds the workspace.
+ token, err := worker.repo.BeginWorkspaceUse(id, now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if scenario == "interrupted" {
+ meta, _ := worker.repo.WorkspaceCheckpoint(id)
+ if meta.RestoreStatus != "failed" || meta.RestoreError != repo.WorkspaceRestoreInterrupted {
+ t.Fatalf("taken-over restore not recorded as interrupted: %q %q", meta.RestoreStatus, meta.RestoreError)
+ }
+ if err = worker.repo.FinishWorkspaceRestore(id, old, now, ""); !errors.Is(err, repo.ErrWorkspaceInUse) {
+ t.Fatalf("fenced restore recorded an outcome: %v", err)
+ }
+ }
+ if err = worker.repo.RestoreCheckpointWorkspaceBinding(id, token, worker.workspaceHost, worker.sandboxes.Root(), now); err != nil {
+ t.Fatal(err)
+ }
+ if state, err := controller.WorkspaceRestoreStatus(id); err != nil || state.Status != "ready" || state.Error != "" {
+ t.Fatalf("bound workspace still reports the earlier restore: %+v %v", state, err)
+ }
+ if token, err = worker.repo.BeginWorkspaceRestore(id, now.Add(time.Hour)); err != nil || token != "" {
+ t.Fatalf("ready workspace started another restore: %q %v", token, err)
+ }
+ })
+ }
+ }
+}
+
func TestLegacyWorkspaceProbeConfirmsMissingRoot(t *testing.T) {
t.Parallel()
for _, scenario := range []string{"runner-missing", "runner-file-missing", "worker-missing", "worker-file-missing"} {
diff --git a/internal/repo/workspace_checkpoints.go b/internal/repo/workspace_checkpoints.go
index 37b84dcc..24f6781f 100644
--- a/internal/repo/workspace_checkpoints.go
+++ b/internal/repo/workspace_checkpoints.go
@@ -77,19 +77,18 @@ func (r *RunsRepo) beginWorkspaceLease(runID string, now time.Time, touch, resto
if err != nil {
return err
}
- if restoring && run.WorkspaceCleaned == nil {
+ var checkpoint WorkspaceCheckpoint
+ if err := tx.Omit("archive").Where("run_id = ?", runID).Take(&checkpoint).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+ return err
+ }
+ // An unfinished outcome on a bound workspace is retried, never assumed.
+ unfinished := checkpoint.RestoreStatus == "failed" || checkpoint.RestoreStatus == "restoring"
+ if restoring && run.WorkspaceCleaned == nil && !unfinished {
token = ""
return nil
}
- if !touch {
- var checkpoint WorkspaceCheckpoint
- if err := tx.Omit("archive").Where("run_id = ?", runID).Take(&checkpoint).Error; err == nil {
- if checkpoint.SaveRetryAt != nil && now.Before(*checkpoint.SaveRetryAt) {
- return ErrWorkspaceCheckpointBackoff
- }
- } else if !errors.Is(err, gorm.ErrRecordNotFound) {
- return err
- }
+ if !touch && checkpoint.SaveRetryAt != nil && now.Before(*checkpoint.SaveRetryAt) {
+ return ErrWorkspaceCheckpointBackoff
}
until := now.Add(15 * time.Minute)
lastUsed := now
@@ -106,6 +105,11 @@ func (r *RunsRepo) beginWorkspaceLease(runID string, now time.Time, touch, resto
}
if restoring {
values["restore_status"], values["restore_started"], values["restore_error"] = "restoring", now, ""
+ } else if checkpoint.RestoreStatus == "restoring" {
+ // Taking over an expired restore lease fences that restore out, so
+ // record it as interrupted. A later binding under this lease can
+ // then tell a stale outcome from a restore still in progress.
+ values["restore_status"], values["restore_error"] = "failed", WorkspaceRestoreInterrupted
}
result := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND (lease_until IS NULL OR lease_until < ?)", runID, now).Updates(values)
if result.Error != nil {
@@ -232,6 +236,13 @@ func (r *RunsRepo) RestoreCheckpointWorkspaceBinding(runID, token, host, root st
if count != 1 {
return ErrWorkspaceInUse
}
+ // A restore lease keeps "restoring" until FinishWorkspaceRestore records
+ // its outcome. Any other lease that binds the workspace supersedes an
+ // earlier failed or interrupted restore.
+ if err := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND lease_token = ? AND restore_status = ?", runID, token, "failed").
+ Updates(map[string]any{"restore_status": "ready", "restore_error": ""}).Error; err != nil {
+ return err
+ }
values := map[string]any{"workspace_cleaned_at": nil, "revision": gorm.Expr("revision + 1")}
if host != "" {
values["workspace_host"] = host
diff --git a/internal/repo/workspace_restore.go b/internal/repo/workspace_restore.go
index 53362796..662fdd60 100644
--- a/internal/repo/workspace_restore.go
+++ b/internal/repo/workspace_restore.go
@@ -6,8 +6,15 @@ import (
"gorm.io/gorm"
)
+// WorkspaceRestoreInterrupted is reported for a restore whose lease expired
+// before it recorded an outcome, e.g. because its process exited.
+const WorkspaceRestoreInterrupted = "工作区恢复已中断,请重新发起重跑"
+
// BeginWorkspaceRestore publishes the background operation and acquires its
// cross-instance lease in the same run-row transaction as other workspace uses.
+// A workspace that is already bound needs no restore, unless the last restore
+// failed or was interrupted after binding it; that one is retried so its
+// outcome can become ready.
func (r *RunsRepo) BeginWorkspaceRestore(runID string, now time.Time) (string, error) {
return r.beginWorkspaceLease(runID, now.UTC(), true, true)
}