Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

fix(workspace): fail fast when a checkpoint operation meets an open terminal (#130)

Robert06221f072c2608f7
diff --git a/internal/engine/workspace_checkpoint_stream.go b/internal/engine/workspace_checkpoint_stream.go
index b450cab9..4b48a6a2 100644
--- a/internal/engine/workspace_checkpoint_stream.go
+++ b/internal/engine/workspace_checkpoint_stream.go
@@ -67,12 +67,20 @@ func (e *Engine) closeCheckpointTransfer(ctx context.Context, run domain.Run, id
_, _ = e.requestWorkspaceCheckpoint(ctx, run, domain.RunnerWorkspaceRequest{RunID: run.ID, WorkspaceLayout: run.ExecutionContext.RunnerWorkspaceLayout, Operation: domain.RunnerWorkspaceOperationCheckpointClose, CheckpointTransfer: &domain.WorkspaceCheckpointTransfer{ID: id}})
}
+// legacyCheckpointTerminalActive is the message runners sent before checkpoint
+// responses carried WorkspaceErrorTerminalActive.
+const legacyCheckpointTerminalActive = "workspace terminal is still active"
+
func workspaceCheckpointResponseError(response domain.RunnerWorkspaceResponse) error {
if response.Error == "" && response.Status < 400 {
return nil
}
- // Older runners have no code field. Preserve their execution fence too.
- if runnerWorkspaceBusy(response) || response.Code == domain.WorkspaceErrorTerminalActive || response.Error == "workspace terminal is still active" {
+ // An open terminal is an execution fence that waiting cannot clear: fail
+ // fast as workspace preparation does, never as retryable lease contention.
+ if response.Code == domain.WorkspaceErrorTerminalActive || response.Error == legacyCheckpointTerminalActive {
+ return fmt.Errorf("%w(%s)", errWorkspaceTerminalActive, response.Error)
+ }
+ if runnerWorkspaceBusy(response) {
return repo.ErrWorkspaceInUse
}
if response.Code == domain.WorkspaceErrorMissing || response.Status == 404 {
diff --git a/internal/engine/workspace_checkpoint_terminal_test.go b/internal/engine/workspace_checkpoint_terminal_test.go
new file mode 100644
index 00000000..ebc0d2f0
--- /dev/null
+++ b/internal/engine/workspace_checkpoint_terminal_test.go
@@ -0,0 +1,145 @@
+package engine
+
+import (
+ "context"
+ "errors"
+ "os"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "agentflow/internal/domain"
+ "agentflow/internal/repo"
+)
+
+// checkpointTerminalResponses are what current and older runners send while a
+// workspace terminal is open during a checkpoint operation.
+var checkpointTerminalResponses = map[string]domain.RunnerWorkspaceResponse{
+ "coded": {Status: 409, Code: domain.WorkspaceErrorTerminalActive, Error: "workspace terminal is still active"},
+ "legacy": {Status: 409, Error: "workspace terminal is still active"},
+}
+
+func requireTerminalActive(t *testing.T, err error) {
+ t.Helper()
+ if !errors.Is(err, errWorkspaceTerminalActive) || errors.Is(err, repo.ErrWorkspaceInUse) {
+ t.Fatalf("open terminal must fail fast instead of waiting as lease contention: %v", err)
+ }
+}
+
+func TestWorkspaceCheckpointResponseErrorSeparatesTerminalFromContention(t *testing.T) {
+ for name, response := range checkpointTerminalResponses {
+ t.Run(name, func(t *testing.T) {
+ requireTerminalActive(t, workspaceCheckpointResponseError(response))
+ })
+ }
+ for _, busy := range []domain.RunnerWorkspaceResponse{
+ {Status: 409, Code: domain.WorkspaceErrorBusy, Error: "workspace is still in use"},
+ {Status: 409, Error: "workspace is still in use"},
+ } {
+ if err := workspaceCheckpointResponseError(busy); !errors.Is(err, repo.ErrWorkspaceInUse) {
+ t.Fatalf("lease contention must stay retryable: %+v -> %v", busy, err)
+ }
+ }
+ if err := workspaceCheckpointResponseError(domain.RunnerWorkspaceResponse{Status: 404, Code: domain.WorkspaceErrorMissing, Error: "missing"}); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("missing workspace classification changed: %v", err)
+ }
+}
+
+// terminalRestoreFixture is a reclaimed runner workspace whose runner reports
+// an open terminal for the operations named in terminalOps.
+func terminalRestoreFixture(t *testing.T, response domain.RunnerWorkspaceResponse, terminalOps ...string) (*Engine, domain.Run, *atomic.Int32) {
+ t.Helper()
+ worker, controller, id := remoteWorkspaceRestoreFixture(t, "sqlite")
+ run, _ := worker.repo.RunState(id)
+ run.ExecutionContext.ExecutionMode, run.ExecutionContext.ExecutionRunnerID = domain.ExecutionModeRunner, "terminal-runner"
+ if err := worker.repo.PutRun(run); err != nil {
+ t.Fatal(err)
+ }
+ run, _ = worker.repo.RunState(id)
+ var terminalCalls atomic.Int32
+ controller.WorkspaceCheckpointTransport = func(_ context.Context, _ domain.Run, req domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {
+ for _, op := range terminalOps {
+ if req.Operation == op {
+ terminalCalls.Add(1)
+ return response, nil
+ }
+ }
+ switch req.Operation {
+ case domain.RunnerWorkspaceOperationCheckpointStatus:
+ return domain.RunnerWorkspaceResponse{Status: 404, Code: domain.WorkspaceErrorMissing, Error: "missing"}, nil
+ case domain.RunnerWorkspaceOperationCheckpointClose:
+ return domain.RunnerWorkspaceResponse{Status: 200}, nil
+ }
+ return domain.RunnerWorkspaceResponse{}, errors.New("unexpected operation " + req.Operation)
+ }
+ return controller, run, &terminalCalls
+}
+
+func TestCheckpointExportFailsFastWhileTerminalIsOpen(t *testing.T) {
+ t.Parallel()
+ for name, response := range checkpointTerminalResponses {
+ t.Run(name, func(t *testing.T) {
+ e, run, calls := terminalRestoreFixture(t, response, domain.RunnerWorkspaceOperationCheckpointExport)
+ requireTerminalActive(t, e.saveConversationWorkspace(context.Background(), run, ""))
+ if calls.Load() != 1 {
+ t.Fatalf("export retried an open terminal %d times", calls.Load())
+ }
+ })
+ }
+}
+
+func TestCheckpointProbeFailsFastWhileTerminalIsOpen(t *testing.T) {
+ t.Parallel()
+ for name, response := range checkpointTerminalResponses {
+ t.Run(name, func(t *testing.T) {
+ // Runners without the cheap status probe answer with an export.
+ e, run, calls := terminalRestoreFixture(t, response, domain.RunnerWorkspaceOperationCheckpointExport)
+ base := e.WorkspaceCheckpointTransport
+ restores := 0
+ e.WorkspaceCheckpointTransport = func(ctx context.Context, r domain.Run, req domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {
+ switch req.Operation {
+ case domain.RunnerWorkspaceOperationCheckpointStatus:
+ return domain.RunnerWorkspaceResponse{Status: 400, Error: "unsupported operation"}, nil
+ case domain.RunnerWorkspaceOperationRestore:
+ restores++
+ }
+ return base(ctx, r, req)
+ }
+ requireTerminalActive(t, e.restoreOrReuseRemoteConversationCheckpoint(context.Background(), run))
+ if calls.Load() != 1 || restores != 0 {
+ t.Fatalf("probe retried or restored over an open terminal: probes=%d restores=%d", calls.Load(), restores)
+ }
+ })
+ }
+}
+
+func TestCheckpointRestoreFailsFastWhileTerminalIsOpen(t *testing.T) {
+ t.Parallel()
+ for name, response := range checkpointTerminalResponses {
+ t.Run(name+"/execution", func(t *testing.T) {
+ e, run, calls := terminalRestoreFixture(t, response, domain.RunnerWorkspaceOperationRestore)
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ busy := false
+ err := e.restoreCheckpointForExecution(ctx, run, func() { busy = true })
+ requireTerminalActive(t, err)
+ if busy || calls.Load() != 1 {
+ t.Fatalf("node start waited on an open terminal: busy=%v restores=%d", busy, calls.Load())
+ }
+ })
+ t.Run(name+"/background", func(t *testing.T) {
+ e, run, calls := terminalRestoreFixture(t, response, domain.RunnerWorkspaceOperationRestore)
+ if _, err := e.StartWorkspaceRestore(run.ID); err != nil {
+ t.Fatal(err)
+ }
+ state := awaitWorkspaceRestore(t, e, run.ID)
+ if state.Status != "failed" || !strings.Contains(state.Error, errWorkspaceTerminalActive.Error()) || calls.Load() != 1 {
+ t.Fatalf("background restore hid or retried an open terminal: %+v restores=%d", state, calls.Load())
+ }
+ if meta, _ := e.repo.WorkspaceCheckpoint(run.ID); meta.LeaseToken != "" {
+ t.Fatal("failed restore kept its workspace lease")
+ }
+ })
+ }
+}
diff --git a/internal/runner/workspace.go b/internal/runner/workspace.go
index 5a41dae7..4942f5e7 100644
--- a/internal/runner/workspace.go
+++ b/internal/runner/workspace.go
@@ -126,7 +126,7 @@ func (d *Daemon) readRunnerWorkspace(ctx context.Context, request domain.RunnerW
if domain.IsWorkspaceCheckpointTransferOperation(request.Operation) {
if request.Operation == domain.RunnerWorkspaceOperationCheckpointExport || request.Operation == domain.RunnerWorkspaceOperationCheckpointCommit {
if d.workspaceTerminals.HasRunScope(request.RunID) {
- return domain.RunnerWorkspaceResponse{Status: 409, Error: "workspace terminal is still active"}
+ return workspaceTerminalActiveResponse("workspace terminal is still active")
}
gate := d.requestGate(request)
if !gate.TryLock() {
@@ -137,7 +137,7 @@ func (d *Daemon) readRunnerWorkspace(ctx context.Context, request domain.RunnerW
return d.checkpointTransfers.Handle(ctx, root, request)
}
if (request.Operation == domain.RunnerWorkspaceOperationRestore || request.Operation == domain.RunnerWorkspaceOperationCheckpoint) && d.workspaceTerminals.HasRunScope(request.RunID) {
- return domain.RunnerWorkspaceResponse{Status: 409, Error: "workspace terminal is still active"}
+ return workspaceTerminalActiveResponse("workspace terminal is still active")
}
if request.Operation == domain.RunnerWorkspaceOperationRestore {
gate := d.requestGate(request)
diff --git a/internal/runner/workspace_checkpoint_test.go b/internal/runner/workspace_checkpoint_test.go
index 714b0a1d..e0d3c14f 100644
--- a/internal/runner/workspace_checkpoint_test.go
+++ b/internal/runner/workspace_checkpoint_test.go
@@ -34,13 +34,18 @@ func TestRunnerWorkspaceCheckpointRestoreAndActiveTerminalProtection(t *testing.
if err != nil {
t.Fatal(err)
}
- if blocked := d.readRunnerWorkspace(context.Background(), req); blocked.Status != 409 {
- t.Fatal("active terminal captured")
+ // The code lets the control plane fail fast; the message stays readable by
+ // older control planes that only match the text.
+ terminalActive := func(r domain.RunnerWorkspaceResponse) bool {
+ return r.Status == 409 && r.Code == domain.WorkspaceErrorTerminalActive && r.Error == "workspace terminal is still active"
+ }
+ if blocked := d.readRunnerWorkspace(context.Background(), req); !terminalActive(blocked) {
+ t.Fatalf("active terminal captured: %+v", blocked)
}
chunkRequest := req
chunkRequest.Operation = domain.RunnerWorkspaceOperationCheckpointExport
- if blocked := d.readRunnerWorkspace(context.Background(), chunkRequest); blocked.Status != 409 {
- t.Fatal("chunked capture ignored active terminal")
+ if blocked := d.readRunnerWorkspace(context.Background(), chunkRequest); !terminalActive(blocked) {
+ t.Fatalf("chunked capture ignored active terminal: %+v", blocked)
}
if _, err = d.workspaceTerminals.Handle(workspace, "checkpoint:1", domain.WorkspaceTerminalRequest{Operation: "close", ID: opened.ID}, nil); err != nil {
t.Fatal(err)