Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.
JavaScript agent-orchestrationagentic-workflowai-agentsai-codingclaude-codecode-reviewcodebuddycodexcoding-agentscursordeveloper-toolshuman-on-the-loopllmmcpmulti-agentsdlcself-hostedsopworkflow-automationworkflow-engine
fix(workspace): never report a failed or interrupted restore as ready (#129)
Robert0622eb28d2337f09
diff --git a/internal/engine/workspace_restore.go b/internal/engine/workspace_restore.goindex 0cf620ee..a23203b8 100644--- a/internal/engine/workspace_restore.go+++ b/internal/engine/workspace_restore.go@@ -26,14 +26,18 @@ func (e *Engine) WorkspaceRestoreStatus(id string) (domain.WorkspaceRestoreStatureturn e.WorkspaceRestoreStatus(workspaceRunID(run))}state := domain.WorkspaceRestoreStatus{Status: "idle", StartedAt: meta.RestoreStarted}- if meta.RestoreStatus == "restoring" && meta.LeaseUntil != nil && meta.LeaseUntil.After(time.Now()) {+ // A restore clears the cleanup mark when it binds the workspace, before its+ // remaining work finishes. Its recorded outcome therefore takes precedence+ // over the binding: a failed or interrupted restore is never "ready".+ switch {+ case meta.RestoreStatus == "restoring" && meta.LeaseUntil != nil && meta.LeaseUntil.After(time.Now()):state.Status = "restoring"- } else if run.WorkspaceCleanedAt.IsZero() || meta.SavedAt == nil {- state.Status = "ready"- } else if meta.RestoreStatus == "failed" {+ case meta.RestoreStatus == "failed":state.Status, state.Error = "failed", meta.RestoreError- } else if meta.RestoreStatus == "restoring" {- state.Status, state.Error = "failed", "工作区恢复已中断,请重新发起重跑"+ case meta.RestoreStatus == "restoring":+ state.Status, state.Error = "failed", repo.WorkspaceRestoreInterrupted+ case run.WorkspaceCleanedAt.IsZero() || meta.SavedAt == nil:+ state.Status = "ready"}return state, nil}diff --git a/internal/engine/workspace_restore_test.go b/internal/engine/workspace_restore_test.goindex 12a5ad96..b9b46ea6 100644--- a/internal/engine/workspace_restore_test.go+++ b/internal/engine/workspace_restore_test.go@@ -277,6 +277,105 @@ func TestWorkspaceRestoreExpiredLeaseFencesOldCompletion(t *testing.T) {}}+func TestWorkspaceRestoreOutcomeOutranksClearedBinding(t *testing.T) {+ t.Parallel()+ for _, backend := range []string{"sqlite", "mysql"} {+ for _, scenario := range []string{"expired-after-binding", "failed-after-binding"} {+ t.Run(backend+"/"+scenario, func(t *testing.T) {+ worker, controller, id := remoteWorkspaceRestoreFixture(t, backend)+ now := time.Now().UTC()+ token, err := worker.repo.BeginWorkspaceRestore(id, now)+ if err != nil || token == "" {+ t.Fatalf("restore lease not acquired: %q %v", token, err)+ }+ // The restore binds the workspace, which clears the cleanup mark,+ // before the rest of its work has finished.+ if err = worker.repo.RestoreCheckpointWorkspaceBinding(id, token, worker.workspaceHost, worker.sandboxes.Root(), now); err != nil {+ t.Fatal(err)+ }+ wantError := repo.WorkspaceRestoreInterrupted+ if scenario == "expired-after-binding" {+ // The process exits before recording an outcome.+ if err = worker.repo.DB.Model(&repo.WorkspaceCheckpoint{}).Where("run_id = ?", id).+ Update("lease_until", now.Add(-time.Minute)).Error; err != nil {+ t.Fatal(err)+ }+ } else {+ wantError = "post-binding failure"+ if err = worker.repo.FinishWorkspaceRestore(id, token, now, wantError); err != nil {+ t.Fatal(err)+ }+ }+ if run, _ := worker.repo.RunState(id); !run.WorkspaceCleanedAt.IsZero() {+ t.Fatal("fixture binding did not clear the cleanup mark")+ }+ state, err := controller.WorkspaceRestoreStatus(id)+ if err != nil || state.Status != "failed" || state.Error != wantError {+ t.Fatalf("unfinished restore reported as %+v %v", state, err)+ }+ // The failure stays retryable although the workspace is bound.+ if state, err = controller.StartWorkspaceRestore(id); err != nil || state.Status == "failed" {+ t.Fatalf("retry was not started: %+v %v", state, err)+ }+ if state = awaitWorkspaceRestore(t, controller, id); state.Status != "ready" || state.Error != "" {+ t.Fatalf("retry did not finish the restore: %+v", state)+ }+ if meta, _ := worker.repo.WorkspaceCheckpoint(id); meta.RestoreStatus != "ready" || meta.LeaseToken != "" {+ t.Fatalf("retry left restore state %q lease=%q", meta.RestoreStatus, meta.LeaseToken)+ }+ })+ }+ }+}++func TestWorkspaceUseSupersedesEarlierRestoreOutcome(t *testing.T) {+ t.Parallel()+ for _, backend := range []string{"sqlite", "mysql"} {+ for _, scenario := range []string{"failed", "interrupted"} {+ t.Run(backend+"/"+scenario, func(t *testing.T) {+ worker, controller, id := remoteWorkspaceRestoreFixture(t, backend)+ now := time.Now().UTC()+ var old string+ var err error+ if scenario == "failed" {+ old, err = worker.repo.BeginWorkspaceRestore(id, now)+ if err == nil {+ err = worker.repo.FinishWorkspaceRestore(id, old, now, "runner disconnected")+ }+ } else {+ old, err = worker.repo.BeginWorkspaceRestore(id, now.Add(-20*time.Minute))+ }+ if err != nil {+ t.Fatal(err)+ }+ // An ordinary workspace use takes the lease and binds the workspace.+ token, err := worker.repo.BeginWorkspaceUse(id, now)+ if err != nil {+ t.Fatal(err)+ }+ if scenario == "interrupted" {+ meta, _ := worker.repo.WorkspaceCheckpoint(id)+ if meta.RestoreStatus != "failed" || meta.RestoreError != repo.WorkspaceRestoreInterrupted {+ t.Fatalf("taken-over restore not recorded as interrupted: %q %q", meta.RestoreStatus, meta.RestoreError)+ }+ if err = worker.repo.FinishWorkspaceRestore(id, old, now, ""); !errors.Is(err, repo.ErrWorkspaceInUse) {+ t.Fatalf("fenced restore recorded an outcome: %v", err)+ }+ }+ if err = worker.repo.RestoreCheckpointWorkspaceBinding(id, token, worker.workspaceHost, worker.sandboxes.Root(), now); err != nil {+ t.Fatal(err)+ }+ if state, err := controller.WorkspaceRestoreStatus(id); err != nil || state.Status != "ready" || state.Error != "" {+ t.Fatalf("bound workspace still reports the earlier restore: %+v %v", state, err)+ }+ if token, err = worker.repo.BeginWorkspaceRestore(id, now.Add(time.Hour)); err != nil || token != "" {+ t.Fatalf("ready workspace started another restore: %q %v", token, err)+ }+ })+ }+ }+}+func TestLegacyWorkspaceProbeConfirmsMissingRoot(t *testing.T) {t.Parallel()for _, scenario := range []string{"runner-missing", "runner-file-missing", "worker-missing", "worker-file-missing"} {diff --git a/internal/repo/workspace_checkpoints.go b/internal/repo/workspace_checkpoints.goindex 37b84dcc..24f6781f 100644--- a/internal/repo/workspace_checkpoints.go+++ b/internal/repo/workspace_checkpoints.go@@ -77,19 +77,18 @@ func (r *RunsRepo) beginWorkspaceLease(runID string, now time.Time, touch, restoif err != nil {return err}- if restoring && run.WorkspaceCleaned == nil {+ var checkpoint WorkspaceCheckpoint+ if err := tx.Omit("archive").Where("run_id = ?", runID).Take(&checkpoint).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {+ return err+ }+ // An unfinished outcome on a bound workspace is retried, never assumed.+ unfinished := checkpoint.RestoreStatus == "failed" || checkpoint.RestoreStatus == "restoring"+ if restoring && run.WorkspaceCleaned == nil && !unfinished {token = ""return nil}- if !touch {- var checkpoint WorkspaceCheckpoint- if err := tx.Omit("archive").Where("run_id = ?", runID).Take(&checkpoint).Error; err == nil {- if checkpoint.SaveRetryAt != nil && now.Before(*checkpoint.SaveRetryAt) {- return ErrWorkspaceCheckpointBackoff- }- } else if !errors.Is(err, gorm.ErrRecordNotFound) {- return err- }+ if !touch && checkpoint.SaveRetryAt != nil && now.Before(*checkpoint.SaveRetryAt) {+ return ErrWorkspaceCheckpointBackoff}until := now.Add(15 * time.Minute)lastUsed := now@@ -106,6 +105,11 @@ func (r *RunsRepo) beginWorkspaceLease(runID string, now time.Time, touch, resto}if restoring {values["restore_status"], values["restore_started"], values["restore_error"] = "restoring", now, ""+ } else if checkpoint.RestoreStatus == "restoring" {+ // Taking over an expired restore lease fences that restore out, so+ // record it as interrupted. A later binding under this lease can+ // then tell a stale outcome from a restore still in progress.+ values["restore_status"], values["restore_error"] = "failed", WorkspaceRestoreInterrupted}result := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND (lease_until IS NULL OR lease_until < ?)", runID, now).Updates(values)if result.Error != nil {@@ -232,6 +236,13 @@ func (r *RunsRepo) RestoreCheckpointWorkspaceBinding(runID, token, host, root stif count != 1 {return ErrWorkspaceInUse}+ // A restore lease keeps "restoring" until FinishWorkspaceRestore records+ // its outcome. Any other lease that binds the workspace supersedes an+ // earlier failed or interrupted restore.+ if err := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND lease_token = ? AND restore_status = ?", runID, token, "failed").+ Updates(map[string]any{"restore_status": "ready", "restore_error": ""}).Error; err != nil {+ return err+ }values := map[string]any{"workspace_cleaned_at": nil, "revision": gorm.Expr("revision + 1")}if host != "" {values["workspace_host"] = hostdiff --git a/internal/repo/workspace_restore.go b/internal/repo/workspace_restore.goindex 53362796..662fdd60 100644--- a/internal/repo/workspace_restore.go+++ b/internal/repo/workspace_restore.go@@ -6,8 +6,15 @@ import ("gorm.io/gorm")+// WorkspaceRestoreInterrupted is reported for a restore whose lease expired+// before it recorded an outcome, e.g. because its process exited.+const WorkspaceRestoreInterrupted = "工作区恢复已中断,请重新发起重跑"+// BeginWorkspaceRestore publishes the background operation and acquires its// cross-instance lease in the same run-row transaction as other workspace uses.+// A workspace that is already bound needs no restore, unless the last restore+// failed or was interrupted after binding it; that one is retried so its+// outcome can become ready.func (r *RunsRepo) BeginWorkspaceRestore(runID string, now time.Time) (string, error) {return r.beginWorkspaceLease(runID, now.UTC(), true, true)}