Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.
JavaScript agent-orchestrationagentic-workflowai-agentsai-codingclaude-codecode-reviewcodebuddycodexcoding-agentscursordeveloper-toolshuman-on-the-loopllmmcpmulti-agentsdlcself-hostedsopworkflow-automationworkflow-engine
package engine
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"agentflow/internal/domain"
"agentflow/internal/repo"
"agentflow/internal/runtime"
)
func awaitWorkspaceRestore(t *testing.T, e *Engine, id string) domain.WorkspaceRestoreStatus {
t.Helper()
deadline := time.Now().Add(30 * time.Second)
for time.Now().Before(deadline) {
state, err := e.WorkspaceRestoreStatus(id)
if err != nil {
t.Fatal(err)
}
if state.Status != "restoring" {
return state
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("background workspace restore did not settle")
return domain.WorkspaceRestoreStatus{}
}
func TestBackgroundWorkspaceRestoreSharesLeaseAcrossAPIInstances(t *testing.T) {
t.Parallel()
for _, backend := range []string{"sqlite", "mysql"} {
t.Run(backend, func(t *testing.T) {
worker, controller, id := remoteWorkspaceRestoreFixture(t, backend)
before, _ := worker.repo.WorkspaceCheckpoint(id)
entered, release := make(chan struct{}), make(chan struct{})
var once sync.Once
var restores atomic.Int32
serveWorkspaceRestoreWorker(t, worker, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
r.Body = io.NopCloser(bytes.NewReader(body))
var request domain.RunnerWorkspaceRequest
_ = json.Unmarshal(body, &request)
if request.Operation == domain.RunnerWorkspaceOperationRestore {
if restores.Add(1) == 1 {
close(entered)
}
<-release
}
worker.HandleWorkspaceCheckpoint(w, r)
}))
t.Cleanup(func() { once.Do(func() { close(release) }); awaitWorkspaceRestore(t, controller, id) })
state, err := controller.StartWorkspaceRestore(id)
if err != nil || state.Status != "restoring" {
t.Fatalf("restore not acknowledged promptly: %+v %v", state, err)
}
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("restore RPC did not start")
}
other := New(worker.repo, runtime.NewManager(t.TempDir()))
for i := 0; i < 3; i++ {
state, err = other.StartWorkspaceRestore(id)
if err != nil || state.Status != "restoring" || state.StartedAt == nil {
t.Fatalf("other API did not reuse the operation: %+v %v", state, err)
}
}
if _, err = other.BeginConversationWorkspaceUse(id); !errors.Is(err, repo.ErrWorkspaceInUse) {
t.Fatalf("background restore lost its workspace lease: %v", err)
}
once.Do(func() { close(release) })
if state = awaitWorkspaceRestore(t, other, id); state.Status != "ready" {
t.Fatalf("restore failed: %+v", state)
}
after, _ := worker.repo.WorkspaceCheckpoint(id)
run, _ := worker.repo.RunState(id)
if restores.Load() != 1 || !run.WorkspaceCleanedAt.IsZero() || after.LeaseToken != "" || after.LeaseUntil != nil {
t.Fatalf("restore was duplicated or not acknowledged: count=%d run=%+v meta=%+v", restores.Load(), run, after)
}
if !after.SavedAt.Equal(*before.SavedAt) || run.Status != "failed" || run.Nodes["prepare"].Attempt != 1 {
t.Fatal("restoration resaved the archive or changed SOP execution")
}
if data, err := os.ReadFile(filepath.Join(worker.RunWorkspace(id), "README.md")); err != nil || string(data) != "saved changes" {
t.Fatalf("restored files lost: %q %v", data, err)
}
})
}
}
func TestRemoteWorkspaceProbeDoesNotRestoreOnBusyOrTransportFailure(t *testing.T) {
t.Parallel()
for _, scenario := range []string{"busy", "legacy-busy", "offline", "invalid-directory"} {
t.Run(scenario, func(t *testing.T) {
worker, controller, id := remoteWorkspaceRestoreFixture(t, "sqlite")
run, _ := worker.repo.RunState(id)
run.ExecutionContext.ExecutionMode = domain.ExecutionModeRunner
run.ExecutionContext.ExecutionRunnerID = "probe-runner"
calls := 0
controller.WorkspaceCheckpointTransport = func(_ context.Context, _ domain.Run, req domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {
calls++
if req.Operation != domain.RunnerWorkspaceOperationCheckpointStatus && req.Operation != domain.RunnerWorkspaceOperationCheckpointExport {
t.Fatalf("unexpected upload/restore on %s: %s", scenario, req.Operation)
}
switch scenario {
case "offline":
return domain.RunnerWorkspaceResponse{}, errors.New("runner disconnected")
case "legacy-busy":
if req.Operation == domain.RunnerWorkspaceOperationCheckpointStatus {
return domain.RunnerWorkspaceResponse{Status: 400, Error: "unsupported operation"}, nil
}
return domain.RunnerWorkspaceResponse{Status: 409, Error: "workspace is still in use"}, nil
case "invalid-directory":
return domain.RunnerWorkspaceResponse{Status: 409, Error: "workspace is not a directory"}, nil
default:
return domain.RunnerWorkspaceResponse{Status: 409, Code: domain.WorkspaceErrorBusy, Error: "workspace is still in use"}, nil
}
}
err := controller.restoreOrReuseRemoteConversationCheckpoint(context.Background(), run)
if err == nil || strings.Contains(scenario, "busy") && !errors.Is(err, repo.ErrWorkspaceInUse) {
t.Fatalf("probe failure lost: %v", err)
}
if calls != 1 && (scenario != "legacy-busy" || calls != 2) {
t.Fatalf("unexpected probe retries: %d", calls)
}
})
}
}
func TestBackgroundWorkspaceRestoreFailureIsVisibleAndExplicitlyRetryable(t *testing.T) {
t.Parallel()
worker, controller, id := remoteWorkspaceRestoreFixture(t, "sqlite")
run, _ := worker.repo.RunState(id)
run.ExecutionContext.ExecutionMode, run.ExecutionContext.ExecutionRunnerID = domain.ExecutionModeRunner, "restore-runner"
if err := worker.repo.PutRun(run); err != nil {
t.Fatal(err)
}
controller.WorkspaceCheckpointTransport = func(context.Context, domain.Run, domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {
return domain.RunnerWorkspaceResponse{}, errors.New("runner disconnected")
}
if _, err := controller.StartWorkspaceRestore(id); err != nil {
t.Fatal(err)
}
state := awaitWorkspaceRestore(t, controller, id)
if state.Status != "failed" || !strings.Contains(state.Error, "runner disconnected") {
t.Fatalf("background error hidden: %+v", state)
}
meta, _ := worker.repo.WorkspaceCheckpoint(id)
if meta.LeaseToken != "" || meta.Error != "" {
t.Fatal("restore failure retained the lease or invalidated the saved snapshot")
}
controller.WorkspaceCheckpointTransport = func(_ context.Context, _ domain.Run, req domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {
if req.Operation == domain.RunnerWorkspaceOperationCheckpointStatus {
return domain.RunnerWorkspaceResponse{Status: 404, Code: domain.WorkspaceErrorMissing, Error: "missing"}, nil
}
if req.Operation != domain.RunnerWorkspaceOperationRestore {
return domain.RunnerWorkspaceResponse{}, errors.New("unexpected restore request: " + req.Operation)
}
return domain.RunnerWorkspaceResponse{Status: 200}, nil
}
if _, err := controller.StartWorkspaceRestore(id); err != nil {
t.Fatal(err)
}
if state = awaitWorkspaceRestore(t, controller, id); state.Status != "ready" {
t.Fatalf("explicit retry failed: %+v", state)
}
}
func TestCheckpointClosePreservesSharedWorkspaceScope(t *testing.T) {
t.Parallel()
e, id := checkpointFixture(t)
run, _ := e.repo.RunState(id)
run.ExecutionContext.ExecutionMode, run.ExecutionContext.ExecutionRunnerID = domain.ExecutionModeRunner, "scope-runner"
run.ExecutionContext.RunnerWorkspaceLayout = domain.RunnerWorkspaceLayoutShared
called := false
e.WorkspaceCheckpointTransport = func(ctx context.Context, _ domain.Run, req domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {
called = true
if req.WorkspaceLayout != domain.RunnerWorkspaceLayoutShared || req.Operation != domain.RunnerWorkspaceOperationCheckpointClose || ctx.Err() != nil {
t.Fatalf("transfer close lost scope or inherited cancellation: %+v %v", req, ctx.Err())
}
return domain.RunnerWorkspaceResponse{Status: 200}, nil
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
e.closeCheckpointTransfer(ctx, run, "transfer")
if !called {
t.Fatal("transfer cleanup skipped")
}
}
func TestBackgroundWorkspaceRestoreWaitsForOlderRunnerRPC(t *testing.T) {
t.Parallel()
worker, controller, id := remoteWorkspaceRestoreFixture(t, "sqlite")
run, _ := worker.repo.RunState(id)
run.ExecutionContext.ExecutionMode, run.ExecutionContext.ExecutionRunnerID = domain.ExecutionModeRunner, "older-rpc-runner"
if err := worker.repo.PutRun(run); err != nil {
t.Fatal(err)
}
var busy atomic.Bool
busy.Store(true)
probed := make(chan struct{}, 1)
controller.WorkspaceCheckpointTransport = func(_ context.Context, _ domain.Run, req domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {
if req.Operation != domain.RunnerWorkspaceOperationCheckpointStatus {
return domain.RunnerWorkspaceResponse{}, errors.New("must not re-upload while an older restore finishes")
}
if busy.Load() {
select {
case probed <- struct{}{}:
default:
}
return domain.RunnerWorkspaceResponse{Status: 409, Code: domain.WorkspaceErrorBusy, Error: "workspace is still in use"}, nil
}
return runtime.WorkspaceCheckpointStatus(worker.RunWorkspace(id)), nil
}
if _, err := controller.StartWorkspaceRestore(id); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { busy.Store(false); awaitWorkspaceRestore(t, controller, id) })
select {
case <-probed:
case <-time.After(5 * time.Second):
t.Fatal("older RPC was not probed")
}
if err := os.MkdirAll(worker.RunWorkspace(id), 0700); err != nil {
t.Fatal(err)
}
path := filepath.Join(worker.RunWorkspace(id), "newer.md")
if err := os.WriteFile(path, []byte("surviving changes"), 0600); err != nil {
t.Fatal(err)
}
busy.Store(false)
if state := awaitWorkspaceRestore(t, controller, id); state.Status != "ready" {
t.Fatalf("normal restore contention failed: %+v", state)
}
if data, err := os.ReadFile(path); err != nil || string(data) != "surviving changes" {
t.Fatalf("older RPC result overwritten: %q %v", data, err)
}
}
func TestWorkspaceRestoreExpiredLeaseFencesOldCompletion(t *testing.T) {
t.Parallel()
for _, backend := range []string{"sqlite", "mysql"} {
t.Run(backend, func(t *testing.T) {
worker, controller, id := remoteWorkspaceRestoreFixture(t, backend)
old, err := worker.repo.BeginWorkspaceRestore(id, time.Now().Add(-20*time.Minute))
if err != nil {
t.Fatal(err)
}
if state, err := controller.WorkspaceRestoreStatus(id); err != nil || state.Status != "failed" {
t.Fatalf("interrupted restore stayed busy forever: %+v %v", state, err)
}
current, err := worker.repo.BeginWorkspaceRestore(id, time.Now().UTC())
if err != nil {
t.Fatal(err)
}
if err := worker.repo.FinishWorkspaceRestore(id, old, time.Now().UTC(), "late error"); !errors.Is(err, repo.ErrWorkspaceInUse) {
t.Fatalf("old completion changed successor state: %v", err)
}
meta, _ := worker.repo.WorkspaceCheckpoint(id)
if meta.LeaseToken != current || meta.RestoreStatus != "restoring" || meta.RestoreError != "" {
t.Fatal("stale owner released a new restore lease")
}
if err := worker.repo.FinishWorkspaceRestore(id, current, time.Now().UTC(), "interrupted"); err != nil {
t.Fatal(err)
}
})
}
}
func TestWorkspaceRestoreOutcomeOutranksClearedBinding(t *testing.T) {
t.Parallel()
for _, backend := range []string{"sqlite", "mysql"} {
for _, scenario := range []string{"expired-after-binding", "failed-after-binding"} {
t.Run(backend+"/"+scenario, func(t *testing.T) {
worker, controller, id := remoteWorkspaceRestoreFixture(t, backend)
now := time.Now().UTC()
token, err := worker.repo.BeginWorkspaceRestore(id, now)
if err != nil || token == "" {
t.Fatalf("restore lease not acquired: %q %v", token, err)
}
// The restore binds the workspace, which clears the cleanup mark,
// before the rest of its work has finished.
if err = worker.repo.RestoreCheckpointWorkspaceBinding(id, token, worker.workspaceHost, worker.sandboxes.Root(), now); err != nil {
t.Fatal(err)
}
wantError := repo.WorkspaceRestoreInterrupted
if scenario == "expired-after-binding" {
// The process exits before recording an outcome.
if err = worker.repo.DB.Model(&repo.WorkspaceCheckpoint{}).Where("run_id = ?", id).
Update("lease_until", now.Add(-time.Minute)).Error; err != nil {
t.Fatal(err)
}
} else {
wantError = "post-binding failure"
if err = worker.repo.FinishWorkspaceRestore(id, token, now, wantError); err != nil {
t.Fatal(err)
}
}
if run, _ := worker.repo.RunState(id); !run.WorkspaceCleanedAt.IsZero() {
t.Fatal("fixture binding did not clear the cleanup mark")
}
state, err := controller.WorkspaceRestoreStatus(id)
if err != nil || state.Status != "failed" || state.Error != wantError {
t.Fatalf("unfinished restore reported as %+v %v", state, err)
}
// The failure stays retryable although the workspace is bound.
if state, err = controller.StartWorkspaceRestore(id); err != nil || state.Status == "failed" {
t.Fatalf("retry was not started: %+v %v", state, err)
}
if state = awaitWorkspaceRestore(t, controller, id); state.Status != "ready" || state.Error != "" {
t.Fatalf("retry did not finish the restore: %+v", state)
}
if meta, _ := worker.repo.WorkspaceCheckpoint(id); meta.RestoreStatus != "ready" || meta.LeaseToken != "" {
t.Fatalf("retry left restore state %q lease=%q", meta.RestoreStatus, meta.LeaseToken)
}
})
}
}
}
func TestWorkspaceUseSupersedesEarlierRestoreOutcome(t *testing.T) {
t.Parallel()
for _, backend := range []string{"sqlite", "mysql"} {
for _, scenario := range []string{"failed", "interrupted"} {
t.Run(backend+"/"+scenario, func(t *testing.T) {
worker, controller, id := remoteWorkspaceRestoreFixture(t, backend)
now := time.Now().UTC()
var old string
var err error
if scenario == "failed" {
old, err = worker.repo.BeginWorkspaceRestore(id, now)
if err == nil {
err = worker.repo.FinishWorkspaceRestore(id, old, now, "runner disconnected")
}
} else {
old, err = worker.repo.BeginWorkspaceRestore(id, now.Add(-20*time.Minute))
}
if err != nil {
t.Fatal(err)
}
// An ordinary workspace use takes the lease and binds the workspace.
token, err := worker.repo.BeginWorkspaceUse(id, now)
if err != nil {
t.Fatal(err)
}
if scenario == "interrupted" {
meta, _ := worker.repo.WorkspaceCheckpoint(id)
if meta.RestoreStatus != "failed" || meta.RestoreError != repo.WorkspaceRestoreInterrupted {
t.Fatalf("taken-over restore not recorded as interrupted: %q %q", meta.RestoreStatus, meta.RestoreError)
}
if err = worker.repo.FinishWorkspaceRestore(id, old, now, ""); !errors.Is(err, repo.ErrWorkspaceInUse) {
t.Fatalf("fenced restore recorded an outcome: %v", err)
}
}
if err = worker.repo.RestoreCheckpointWorkspaceBinding(id, token, worker.workspaceHost, worker.sandboxes.Root(), now); err != nil {
t.Fatal(err)
}
if state, err := controller.WorkspaceRestoreStatus(id); err != nil || state.Status != "ready" || state.Error != "" {
t.Fatalf("bound workspace still reports the earlier restore: %+v %v", state, err)
}
if token, err = worker.repo.BeginWorkspaceRestore(id, now.Add(time.Hour)); err != nil || token != "" {
t.Fatalf("ready workspace started another restore: %q %v", token, err)
}
})
}
}
}
func TestLegacyWorkspaceProbeConfirmsMissingRoot(t *testing.T) {
t.Parallel()
for _, scenario := range []string{"runner-missing", "runner-file-missing", "worker-missing", "worker-file-missing"} {
t.Run(scenario, func(t *testing.T) {
worker, controller, id := remoteWorkspaceRestoreFixture(t, "sqlite")
run, _ := worker.repo.RunState(id)
if strings.HasPrefix(scenario, "runner-") {
run.ExecutionContext.ExecutionMode, run.ExecutionContext.ExecutionRunnerID = domain.ExecutionModeRunner, "legacy-runner"
}
missingFile := strings.Contains(scenario, "file-missing")
root := worker.RunWorkspace(id)
missing := root
if missingFile {
missing = filepath.Join(root, "README.md")
}
restores := 0
controller.WorkspaceCheckpointTransport = func(_ context.Context, _ domain.Run, req domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {
switch req.Operation {
case domain.RunnerWorkspaceOperationCheckpointStatus:
return domain.RunnerWorkspaceResponse{Status: 400, Error: "unsupported operation"}, nil
case domain.RunnerWorkspaceOperationCheckpointExport:
return domain.RunnerWorkspaceResponse{Status: 409, Error: "lstat " + missing + ": no such file or directory"}, nil
case domain.RunnerWorkspaceOperationInfo:
if missingFile {
return domain.RunnerWorkspaceResponse{Status: 200}, nil
}
return domain.RunnerWorkspaceResponse{Status: 404, Error: "stat " + root + ": no such file or directory"}, nil
case domain.RunnerWorkspaceOperationRestore:
restores++
return domain.RunnerWorkspaceResponse{Status: 200}, nil
default:
return domain.RunnerWorkspaceResponse{}, errors.New("unexpected legacy operation")
}
}
if strings.HasPrefix(scenario, "worker-") {
// Exercise a legacy Worker's HTTP response without Runner routing.
serveWorkspaceRestoreWorker(t, worker, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var req domain.RunnerWorkspaceRequest
_ = json.NewDecoder(r.Body).Decode(&req)
response, err := controller.WorkspaceCheckpointTransport(r.Context(), run, req)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
_ = json.NewEncoder(w).Encode(response)
}))
}
err := controller.restoreOrReuseRemoteConversationCheckpoint(context.Background(), run)
if missingFile && (err == nil || restores != 0) || !missingFile && (err != nil || restores != 1) {
t.Fatalf("legacy probe confused root and file errors: restores=%d err=%v", restores, err)
}
})
}
}