Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.
JavaScript agent-orchestrationagentic-workflowai-agentsai-codingclaude-codecode-reviewcodebuddycodexcoding-agentscursordeveloper-toolshuman-on-the-loopllmmcpmulti-agentsdlcself-hostedsopworkflow-automationworkflow-engine
fix(workspace): fail fast when a checkpoint operation meets an open terminal (#130)
Robert06221f072c2608f7
diff --git a/internal/engine/workspace_checkpoint_stream.go b/internal/engine/workspace_checkpoint_stream.goindex b450cab9..4b48a6a2 100644--- a/internal/engine/workspace_checkpoint_stream.go+++ b/internal/engine/workspace_checkpoint_stream.go@@ -67,12 +67,20 @@ func (e *Engine) closeCheckpointTransfer(ctx context.Context, run domain.Run, id_, _ = e.requestWorkspaceCheckpoint(ctx, run, domain.RunnerWorkspaceRequest{RunID: run.ID, WorkspaceLayout: run.ExecutionContext.RunnerWorkspaceLayout, Operation: domain.RunnerWorkspaceOperationCheckpointClose, CheckpointTransfer: &domain.WorkspaceCheckpointTransfer{ID: id}})}+// legacyCheckpointTerminalActive is the message runners sent before checkpoint+// responses carried WorkspaceErrorTerminalActive.+const legacyCheckpointTerminalActive = "workspace terminal is still active"+func workspaceCheckpointResponseError(response domain.RunnerWorkspaceResponse) error {if response.Error == "" && response.Status < 400 {return nil}- // Older runners have no code field. Preserve their execution fence too.- if runnerWorkspaceBusy(response) || response.Code == domain.WorkspaceErrorTerminalActive || response.Error == "workspace terminal is still active" {+ // An open terminal is an execution fence that waiting cannot clear: fail+ // fast as workspace preparation does, never as retryable lease contention.+ if response.Code == domain.WorkspaceErrorTerminalActive || response.Error == legacyCheckpointTerminalActive {+ return fmt.Errorf("%w(%s)", errWorkspaceTerminalActive, response.Error)+ }+ if runnerWorkspaceBusy(response) {return repo.ErrWorkspaceInUse}if response.Code == domain.WorkspaceErrorMissing || response.Status == 404 {diff --git a/internal/engine/workspace_checkpoint_terminal_test.go b/internal/engine/workspace_checkpoint_terminal_test.gonew file mode 100644index 00000000..ebc0d2f0--- /dev/null+++ b/internal/engine/workspace_checkpoint_terminal_test.go@@ -0,0 +1,145 @@+package engine++import (+ "context"+ "errors"+ "os"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "agentflow/internal/domain"+ "agentflow/internal/repo"+)++// checkpointTerminalResponses are what current and older runners send while a+// workspace terminal is open during a checkpoint operation.+var checkpointTerminalResponses = map[string]domain.RunnerWorkspaceResponse{+ "coded": {Status: 409, Code: domain.WorkspaceErrorTerminalActive, Error: "workspace terminal is still active"},+ "legacy": {Status: 409, Error: "workspace terminal is still active"},+}++func requireTerminalActive(t *testing.T, err error) {+ t.Helper()+ if !errors.Is(err, errWorkspaceTerminalActive) || errors.Is(err, repo.ErrWorkspaceInUse) {+ t.Fatalf("open terminal must fail fast instead of waiting as lease contention: %v", err)+ }+}++func TestWorkspaceCheckpointResponseErrorSeparatesTerminalFromContention(t *testing.T) {+ for name, response := range checkpointTerminalResponses {+ t.Run(name, func(t *testing.T) {+ requireTerminalActive(t, workspaceCheckpointResponseError(response))+ })+ }+ for _, busy := range []domain.RunnerWorkspaceResponse{+ {Status: 409, Code: domain.WorkspaceErrorBusy, Error: "workspace is still in use"},+ {Status: 409, Error: "workspace is still in use"},+ } {+ if err := workspaceCheckpointResponseError(busy); !errors.Is(err, repo.ErrWorkspaceInUse) {+ t.Fatalf("lease contention must stay retryable: %+v -> %v", busy, err)+ }+ }+ if err := workspaceCheckpointResponseError(domain.RunnerWorkspaceResponse{Status: 404, Code: domain.WorkspaceErrorMissing, Error: "missing"}); !errors.Is(err, os.ErrNotExist) {+ t.Fatalf("missing workspace classification changed: %v", err)+ }+}++// terminalRestoreFixture is a reclaimed runner workspace whose runner reports+// an open terminal for the operations named in terminalOps.+func terminalRestoreFixture(t *testing.T, response domain.RunnerWorkspaceResponse, terminalOps ...string) (*Engine, domain.Run, *atomic.Int32) {+ t.Helper()+ worker, controller, id := remoteWorkspaceRestoreFixture(t, "sqlite")+ run, _ := worker.repo.RunState(id)+ run.ExecutionContext.ExecutionMode, run.ExecutionContext.ExecutionRunnerID = domain.ExecutionModeRunner, "terminal-runner"+ if err := worker.repo.PutRun(run); err != nil {+ t.Fatal(err)+ }+ run, _ = worker.repo.RunState(id)+ var terminalCalls atomic.Int32+ controller.WorkspaceCheckpointTransport = func(_ context.Context, _ domain.Run, req domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {+ for _, op := range terminalOps {+ if req.Operation == op {+ terminalCalls.Add(1)+ return response, nil+ }+ }+ switch req.Operation {+ case domain.RunnerWorkspaceOperationCheckpointStatus:+ return domain.RunnerWorkspaceResponse{Status: 404, Code: domain.WorkspaceErrorMissing, Error: "missing"}, nil+ case domain.RunnerWorkspaceOperationCheckpointClose:+ return domain.RunnerWorkspaceResponse{Status: 200}, nil+ }+ return domain.RunnerWorkspaceResponse{}, errors.New("unexpected operation " + req.Operation)+ }+ return controller, run, &terminalCalls+}++func TestCheckpointExportFailsFastWhileTerminalIsOpen(t *testing.T) {+ t.Parallel()+ for name, response := range checkpointTerminalResponses {+ t.Run(name, func(t *testing.T) {+ e, run, calls := terminalRestoreFixture(t, response, domain.RunnerWorkspaceOperationCheckpointExport)+ requireTerminalActive(t, e.saveConversationWorkspace(context.Background(), run, ""))+ if calls.Load() != 1 {+ t.Fatalf("export retried an open terminal %d times", calls.Load())+ }+ })+ }+}++func TestCheckpointProbeFailsFastWhileTerminalIsOpen(t *testing.T) {+ t.Parallel()+ for name, response := range checkpointTerminalResponses {+ t.Run(name, func(t *testing.T) {+ // Runners without the cheap status probe answer with an export.+ e, run, calls := terminalRestoreFixture(t, response, domain.RunnerWorkspaceOperationCheckpointExport)+ base := e.WorkspaceCheckpointTransport+ restores := 0+ e.WorkspaceCheckpointTransport = func(ctx context.Context, r domain.Run, req domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {+ switch req.Operation {+ case domain.RunnerWorkspaceOperationCheckpointStatus:+ return domain.RunnerWorkspaceResponse{Status: 400, Error: "unsupported operation"}, nil+ case domain.RunnerWorkspaceOperationRestore:+ restores+++ }+ return base(ctx, r, req)+ }+ requireTerminalActive(t, e.restoreOrReuseRemoteConversationCheckpoint(context.Background(), run))+ if calls.Load() != 1 || restores != 0 {+ t.Fatalf("probe retried or restored over an open terminal: probes=%d restores=%d", calls.Load(), restores)+ }+ })+ }+}++func TestCheckpointRestoreFailsFastWhileTerminalIsOpen(t *testing.T) {+ t.Parallel()+ for name, response := range checkpointTerminalResponses {+ t.Run(name+"/execution", func(t *testing.T) {+ e, run, calls := terminalRestoreFixture(t, response, domain.RunnerWorkspaceOperationRestore)+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ busy := false+ err := e.restoreCheckpointForExecution(ctx, run, func() { busy = true })+ requireTerminalActive(t, err)+ if busy || calls.Load() != 1 {+ t.Fatalf("node start waited on an open terminal: busy=%v restores=%d", busy, calls.Load())+ }+ })+ t.Run(name+"/background", func(t *testing.T) {+ e, run, calls := terminalRestoreFixture(t, response, domain.RunnerWorkspaceOperationRestore)+ if _, err := e.StartWorkspaceRestore(run.ID); err != nil {+ t.Fatal(err)+ }+ state := awaitWorkspaceRestore(t, e, run.ID)+ if state.Status != "failed" || !strings.Contains(state.Error, errWorkspaceTerminalActive.Error()) || calls.Load() != 1 {+ t.Fatalf("background restore hid or retried an open terminal: %+v restores=%d", state, calls.Load())+ }+ if meta, _ := e.repo.WorkspaceCheckpoint(run.ID); meta.LeaseToken != "" {+ t.Fatal("failed restore kept its workspace lease")+ }+ })+ }+}diff --git a/internal/runner/workspace.go b/internal/runner/workspace.goindex 5a41dae7..4942f5e7 100644--- a/internal/runner/workspace.go+++ b/internal/runner/workspace.go@@ -126,7 +126,7 @@ func (d *Daemon) readRunnerWorkspace(ctx context.Context, request domain.RunnerWif domain.IsWorkspaceCheckpointTransferOperation(request.Operation) {if request.Operation == domain.RunnerWorkspaceOperationCheckpointExport || request.Operation == domain.RunnerWorkspaceOperationCheckpointCommit {if d.workspaceTerminals.HasRunScope(request.RunID) {- return domain.RunnerWorkspaceResponse{Status: 409, Error: "workspace terminal is still active"}+ return workspaceTerminalActiveResponse("workspace terminal is still active")}gate := d.requestGate(request)if !gate.TryLock() {@@ -137,7 +137,7 @@ func (d *Daemon) readRunnerWorkspace(ctx context.Context, request domain.RunnerWreturn d.checkpointTransfers.Handle(ctx, root, request)}if (request.Operation == domain.RunnerWorkspaceOperationRestore || request.Operation == domain.RunnerWorkspaceOperationCheckpoint) && d.workspaceTerminals.HasRunScope(request.RunID) {- return domain.RunnerWorkspaceResponse{Status: 409, Error: "workspace terminal is still active"}+ return workspaceTerminalActiveResponse("workspace terminal is still active")}if request.Operation == domain.RunnerWorkspaceOperationRestore {gate := d.requestGate(request)diff --git a/internal/runner/workspace_checkpoint_test.go b/internal/runner/workspace_checkpoint_test.goindex 714b0a1d..e0d3c14f 100644--- a/internal/runner/workspace_checkpoint_test.go+++ b/internal/runner/workspace_checkpoint_test.go@@ -34,13 +34,18 @@ func TestRunnerWorkspaceCheckpointRestoreAndActiveTerminalProtection(t *testing.if err != nil {t.Fatal(err)}- if blocked := d.readRunnerWorkspace(context.Background(), req); blocked.Status != 409 {- t.Fatal("active terminal captured")+ // The code lets the control plane fail fast; the message stays readable by+ // older control planes that only match the text.+ terminalActive := func(r domain.RunnerWorkspaceResponse) bool {+ return r.Status == 409 && r.Code == domain.WorkspaceErrorTerminalActive && r.Error == "workspace terminal is still active"+ }+ if blocked := d.readRunnerWorkspace(context.Background(), req); !terminalActive(blocked) {+ t.Fatalf("active terminal captured: %+v", blocked)}chunkRequest := reqchunkRequest.Operation = domain.RunnerWorkspaceOperationCheckpointExport- if blocked := d.readRunnerWorkspace(context.Background(), chunkRequest); blocked.Status != 409 {- t.Fatal("chunked capture ignored active terminal")+ if blocked := d.readRunnerWorkspace(context.Background(), chunkRequest); !terminalActive(blocked) {+ t.Fatalf("chunked capture ignored active terminal: %+v", blocked)}if _, err = d.workspaceTerminals.Handle(workspace, "checkpoint:1", domain.WorkspaceTerminalRequest{Operation: "close", ID: opened.ID}, nil); err != nil {t.Fatal(err)