Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.
JavaScript agent-orchestrationagentic-workflowai-agentsai-codingclaude-codecode-reviewcodebuddycodexcoding-agentscursordeveloper-toolshuman-on-the-loopllmmcpmulti-agentsdlcself-hostedsopworkflow-automationworkflow-engine
package engine
import (
"path/filepath"
"testing"
"time"
"agentflow/internal/domain"
"agentflow/internal/repo"
"agentflow/internal/runtime"
)
func parallelTestEngine(t *testing.T, wf domain.WorkflowDefinition) (*Engine, *repo.Store) {
t.Helper()
s, err := repo.OpenTest()
if err != nil {
t.Fatal(err)
}
t.Cleanup(s.Close)
if err := s.PutWorkflow(wf); err != nil {
t.Fatal(err)
}
return New(s, runtime.NewManager(filepath.Join(t.TempDir(), "workspaces"))), s
}
func scriptNode(id, script string) domain.WorkflowNode {
return domain.WorkflowNode{ID: id, Name: id, Type: "script", ScriptLanguage: "shell", Script: script, MaxAttempts: 1}
}
func edge(source, target string) domain.WorkflowEdge {
return domain.WorkflowEdge{ID: source + "-" + target, Source: source, Target: target, When: "success"}
}
func nodeEvents(r domain.Run, nodeID, eventType string) int {
n := 0
for _, event := range r.Events {
if event.NodeID == nodeID && event.Type == eventType {
n++
}
}
return n
}
func overlaps(a, b *domain.NodeRun) bool {
return a.StartedAt.Before(b.FinishedAt) && b.StartedAt.Before(a.FinishedAt)
}
// Two sleeping branches run at the same time, and a fast branch's successor
// starts without waiting for its slow sibling (no wave barrier).
func TestParallelBranchesExecuteConcurrently(t *testing.T) {
t.Parallel()
wf := domain.WorkflowDefinition{ID: "parallel-overlap", Name: "parallel overlap", SandboxProfileID: "local",
Nodes: []domain.WorkflowNode{
scriptNode("start", "true"),
scriptNode("left", "sleep 1.5"),
scriptNode("right", "sleep 1.5"),
scriptNode("fast", "true"),
scriptNode("after-fast", "true"),
scriptNode("join", "true"),
},
Edges: []domain.WorkflowEdge{
edge("start", "left"), edge("start", "right"), edge("start", "fast"),
edge("fast", "after-fast"),
edge("left", "join"), edge("right", "join"), edge("after-fast", "join"),
},
}
e, s := parallelTestEngine(t, wf)
run, err := e.Start(wf.ID, "")
if err != nil {
t.Fatal(err)
}
waitFor(t, s, run.ID, "succeeded")
got, _ := s.Run(run.ID)
left, right, afterFast, join := got.Nodes["left"], got.Nodes["right"], got.Nodes["after-fast"], got.Nodes["join"]
if !overlaps(left, right) {
t.Fatalf("parallel branches did not overlap: left %s..%s right %s..%s", left.StartedAt, left.FinishedAt, right.StartedAt, right.FinishedAt)
}
if !afterFast.StartedAt.Before(left.FinishedAt) || !afterFast.StartedAt.Before(right.FinishedAt) {
t.Fatalf("fast branch successor waited for slow siblings: after-fast started %s, left finished %s, right finished %s", afterFast.StartedAt, left.FinishedAt, right.FinishedAt)
}
if join.StartedAt.Before(left.FinishedAt) || join.StartedAt.Before(right.FinishedAt) {
t.Fatalf("join started before its dependencies finished")
}
}
// A failed branch must not settle the run while a sibling is still running:
// the sibling finishes and its successor still runs before the run fails.
func TestSchedulerWaitsForRunningSiblingBeforeSettling(t *testing.T) {
t.Parallel()
wf := domain.WorkflowDefinition{ID: "parallel-fail-wait", Name: "parallel fail wait", SandboxProfileID: "local",
Nodes: []domain.WorkflowNode{
scriptNode("start", "true"),
scriptNode("broken", "exit 3"),
scriptNode("slow", "sleep 1"),
scriptNode("after-slow", "true"),
},
Edges: []domain.WorkflowEdge{edge("start", "broken"), edge("start", "slow"), edge("slow", "after-slow")},
}
e, s := parallelTestEngine(t, wf)
run, err := e.Start(wf.ID, "")
if err != nil {
t.Fatal(err)
}
waitFor(t, s, run.ID, "failed")
got, _ := s.Run(run.ID)
if got.Nodes["broken"].Status != "failed" {
t.Fatalf("broken = %+v", got.Nodes["broken"])
}
if got.Nodes["slow"].Status != "succeeded" || got.Nodes["after-slow"].Status != "succeeded" {
t.Fatalf("scheduler settled under a running sibling: slow=%s after-slow=%s", got.Nodes["slow"].Status, got.Nodes["after-slow"].Status)
}
}
// A branch waiting for approval keeps the run "running" while its sibling
// works; the run only waits once nothing is in flight.
func TestWaitingBranchDoesNotStopRunningSibling(t *testing.T) {
t.Parallel()
wf := domain.WorkflowDefinition{ID: "parallel-wait-sibling", Name: "parallel wait sibling", SandboxProfileID: "local",
Nodes: []domain.WorkflowNode{
scriptNode("start", "true"),
{ID: "gate", Name: "gate", Type: "approval"},
scriptNode("slow", "sleep 1"),
scriptNode("after-slow", "true"),
},
Edges: []domain.WorkflowEdge{edge("start", "gate"), edge("start", "slow"), edge("slow", "after-slow")},
}
e, s := parallelTestEngine(t, wf)
run, err := e.Start(wf.ID, "")
if err != nil {
t.Fatal(err)
}
mid := waitRun(t, s, run.ID, 30*time.Second, func(r domain.Run) bool {
return r.Nodes["gate"] != nil && r.Nodes["gate"].Status == "waiting_approval" && r.Nodes["slow"] != nil && r.Nodes["slow"].Status == "running"
})
if mid.Status != "running" {
t.Fatalf("run status with a running sibling = %s", mid.Status)
}
waitFor(t, s, run.ID, "waiting")
got, _ := s.Run(run.ID)
if got.Nodes["after-slow"].Status != "succeeded" {
t.Fatalf("after-slow = %s", got.Nodes["after-slow"].Status)
}
if err := e.Approve(run.ID, "gate", ""); err != nil {
t.Fatal(err)
}
waitFor(t, s, run.ID, "succeeded")
}
// Rework from one branch resets a sibling that is still running. The stale
// execution is cancelled, never commits, and the sibling runs exactly once
// more as a new attempt.
func TestReworkResetsRunningSiblingWithoutDoubleCommit(t *testing.T) {
t.Parallel()
wf := domain.WorkflowDefinition{ID: "parallel-rework", Name: "parallel rework", SandboxProfileID: "local",
Nodes: []domain.WorkflowNode{
scriptNode("start", "true"),
{ID: "gate", Name: "gate", Type: "approval", RejectTo: "start", MaxReworks: 3},
scriptNode("slow", "sleep 2"),
},
Edges: []domain.WorkflowEdge{edge("start", "gate"), edge("start", "slow")},
}
e, s := parallelTestEngine(t, wf)
run, err := e.Start(wf.ID, "")
if err != nil {
t.Fatal(err)
}
waitRun(t, s, run.ID, 30*time.Second, func(r domain.Run) bool {
return r.Nodes["gate"] != nil && r.Nodes["gate"].Status == "waiting_approval" && r.Nodes["slow"] != nil && r.Nodes["slow"].Status == "running"
})
if err := e.Reject(run.ID, domain.RejectRequest{NodeID: "gate", TargetID: "start", Reason: "redo"}); err != nil {
t.Fatal(err)
}
waitFor(t, s, run.ID, "waiting")
got, _ := s.Run(run.ID)
if got.Nodes["slow"].Status != "succeeded" {
t.Fatalf("slow = %+v", got.Nodes["slow"])
}
if n := nodeEvents(got, "slow", "node.succeeded"); n != 1 {
t.Fatalf("slow committed %d successes; the reset execution must not commit", n)
}
if got.Nodes["slow"].Attempt != 2 {
t.Fatalf("slow attempt = %d, want 2 (one abandoned, one rerun)", got.Nodes["slow"].Attempt)
}
if n := nodeEvents(got, "start", "node.succeeded"); n != 2 {
t.Fatalf("start succeeded %d times, want 2", n)
}
if err := e.Approve(run.ID, "gate", ""); err != nil {
t.Fatal(err)
}
waitFor(t, s, run.ID, "succeeded")
}
// Pausing parallel branches and continuing at once resumes both; no node is
// committed twice and the run completes.
func TestPauseThenImmediateResumeParallelBranches(t *testing.T) {
t.Parallel()
wf := domain.WorkflowDefinition{ID: "parallel-pause", Name: "parallel pause", SandboxProfileID: "local",
Nodes: []domain.WorkflowNode{
scriptNode("start", "true"),
scriptNode("left", "sleep 1"),
scriptNode("right", "sleep 1"),
scriptNode("join", "true"),
},
Edges: []domain.WorkflowEdge{edge("start", "left"), edge("start", "right"), edge("left", "join"), edge("right", "join")},
}
e, s := parallelTestEngine(t, wf)
run, err := e.Start(wf.ID, "")
if err != nil {
t.Fatal(err)
}
waitRun(t, s, run.ID, 30*time.Second, func(r domain.Run) bool {
return r.Nodes["left"] != nil && r.Nodes["left"].Status == "running" && r.Nodes["right"] != nil && r.Nodes["right"].Status == "running"
})
if err := e.Pause(run.ID); err != nil {
t.Fatal(err)
}
waitFor(t, s, run.ID, "waiting")
if err := e.ResumePaused(run.ID, ""); err != nil {
t.Fatal(err)
}
waitFor(t, s, run.ID, "succeeded")
got, _ := s.Run(run.ID)
for _, id := range []string{"left", "right", "join"} {
if n := nodeEvents(got, id, "node.succeeded"); n != 1 {
t.Fatalf("%s committed %d successes", id, n)
}
}
}
func TestKeyedLocksSerializeOnlySameKey(t *testing.T) {
t.Parallel()
var locks keyedLocks
releaseA := locks.lock("a")
acquiredB := make(chan struct{})
go func() {
release := locks.lock("b")
close(acquiredB)
release()
}()
select {
case <-acquiredB:
case <-time.After(2 * time.Second):
t.Fatal("different key blocked")
}
acquiredA := make(chan struct{})
go func() {
release := locks.lock("a")
close(acquiredA)
release()
}()
select {
case <-acquiredA:
t.Fatal("same key not serialized")
case <-time.After(100 * time.Millisecond):
}
releaseA()
select {
case <-acquiredA:
case <-time.After(2 * time.Second):
t.Fatal("same key never released")
}
locks.mu.Lock()
defer locks.mu.Unlock()
if len(locks.locks) != 0 {
t.Fatalf("lock entries leaked: %d", len(locks.locks))
}
}
func TestNodeControlKeyIsolatesChildSOP(t *testing.T) {
t.Parallel()
parent := domain.Run{ID: "run-parent"}
child := domain.Run{ID: "run-child", ExecutionContext: domain.TelemetryContext{WorkspaceRunID: "run-parent"}}
if nodeControlKey(parent, "dev") != "dev" {
t.Fatal("owner keys must stay unchanged")
}
if gateResultPath(nodeControlKey(parent, "dev"), 1) == gateResultPath(nodeControlKey(child, "dev"), 1) {
t.Fatal("child SOP gate result path collides with the parent's")
}
artifact := domain.Artifact{NodeID: "dev", Name: "spec", MediaType: "text/markdown"}
if artifactInputRelPath(parent, artifact) == artifactInputRelPath(child, artifact) {
t.Fatal("child SOP input path collides with the parent's")
}
}
// Cancelling during a parallel stage stops both branches. Replaying one of
// them must resume the other too, and the run must settle instead of staying
// "running" with a cancelled branch nobody schedules.
func TestRetryAfterCancelResumesCancelledParallelSibling(t *testing.T) {
t.Parallel()
wf := domain.WorkflowDefinition{ID: "parallel-cancel-retry", Name: "parallel cancel retry", SandboxProfileID: "local",
Nodes: []domain.WorkflowNode{
scriptNode("start", "true"),
scriptNode("left", "sleep 1"),
scriptNode("right", "sleep 1"),
scriptNode("join", "true"),
},
Edges: []domain.WorkflowEdge{edge("start", "left"), edge("start", "right"), edge("left", "join"), edge("right", "join")},
}
e, s := parallelTestEngine(t, wf)
run, err := e.Start(wf.ID, "")
if err != nil {
t.Fatal(err)
}
waitRun(t, s, run.ID, 30*time.Second, func(r domain.Run) bool {
return r.Nodes["left"] != nil && r.Nodes["left"].Status == "running" && r.Nodes["right"] != nil && r.Nodes["right"].Status == "running"
})
if err := e.Cancel(run.ID); err != nil {
t.Fatal(err)
}
cancelled := waitRun(t, s, run.ID, 30*time.Second, func(r domain.Run) bool { return r.Status == "cancelled" && !r.HasActiveNode() })
if cancelled.Nodes["left"].Status != "cancelled" || cancelled.Nodes["right"].Status != "cancelled" {
t.Fatalf("cancel left left=%s right=%s", cancelled.Nodes["left"].Status, cancelled.Nodes["right"].Status)
}
if err := e.Retry(run.ID, "left", "", ""); err != nil {
t.Fatal(err)
}
waitFor(t, s, run.ID, "succeeded")
got, _ := s.Run(run.ID)
for _, id := range []string{"left", "right", "join"} {
if got.Nodes[id].Status != "succeeded" {
t.Fatalf("%s = %s", id, got.Nodes[id].Status)
}
}
if n := nodeEvents(got, "start", "node.succeeded"); n != 1 {
t.Fatalf("start reran %d times; replaying a branch must not rerun its upstream", n)
}
}
// A run left "running" with a cancelled node (state written before the
// revive fix, or by another path) settles instead of spinning forever.
func TestSchedulerSettlesRunWithCancelledNode(t *testing.T) {
t.Parallel()
wf := domain.WorkflowDefinition{ID: "parallel-cancelled-settle", Name: "parallel cancelled settle", SandboxProfileID: "local",
Nodes: []domain.WorkflowNode{scriptNode("left", "true"), scriptNode("right", "true")},
}
e, s := parallelTestEngine(t, wf)
now := time.Now().UTC()
run := domain.Run{ID: "run-cancelled-settle", WorkflowID: wf.ID, WorkflowSnapshot: wf, Status: "running", CreatedAt: now, UpdatedAt: now,
Nodes: map[string]*domain.NodeRun{
"left": {NodeID: "left", Status: "pending"},
"right": {NodeID: "right", Status: "cancelled", Attempt: 1},
}}
if err := s.PutRun(run); err != nil {
t.Fatal(err)
}
if _, err := e.acquireLease(run); err != nil {
t.Fatal(err)
}
e.resume(run.ID)
waitFor(t, s, run.ID, "failed")
got, _ := s.Run(run.ID)
if got.Nodes["left"].Status != "succeeded" || got.Nodes["right"].Status != "cancelled" {
t.Fatalf("left=%s right=%s", got.Nodes["left"].Status, got.Nodes["right"].Status)
}
}