Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.
JavaScript agent-orchestrationagentic-workflowai-agentsai-codingclaude-codecode-reviewcodebuddycodexcoding-agentscursordeveloper-toolshuman-on-the-loopllmmcpmulti-agentsdlcself-hostedsopworkflow-automationworkflow-engine
// Real SOP executions in the deployment's sandbox using script nodes (no model
// cost): approval with rework, workspace browsing, artifacts, logs, telemetry,
// SSE, retry, manual pass, pause/resume, decline, cancel, child SOPs, delete
// and run timing.
import { test, before, after } from 'node:test';
import assert from 'node:assert/strict';
import { api, get, post, del, list, cleanup, ensureProject, waitRun, nodeOf, nodeStatus } from '../lib/api.mjs';
import { cfg } from '../lib/env.mjs';
import { createWorkflow, script, approval, sopNode, startRun, repoSkip, childSOPHandoffScenario } from '../lib/fixtures.mjs';
before(ensureProject);
after(cleanup);
const logsText = async id => (await api('GET', `/api/runs/${id}/logs.txt`, undefined, { raw: true })).text;
function checkTiming(run) {
// Present once the run-timing change is deployed; older builds omit it.
if (!run.timing) return;
const { executionMs, wallMs, waitingMs } = run.timing;
assert.ok(executionMs >= 0 && wallMs >= 0, `bad timing ${JSON.stringify(run.timing)}`);
assert.ok(executionMs <= wallMs + 1000, `execution ${executionMs}ms exceeds wall ${wallMs}ms`);
return { executionMs, wallMs, waitingMs };
}
test('approval gate: reject reworks upstream in the same workspace, approve completes', async () => {
const wf = await createWorkflow('approval', [
script('prep', 'echo "attempt-$(date +%s%N)" >> notes.txt && mkdir -p out && printf "# E2E result\\n\\nok\\n" > out/result.md && echo PREP_DONE', {
artifactOutputs: [{ name: 'result', path: 'out/result.md', mediaType: 'text/markdown', required: true }],
}),
approval('gate', 'prep'),
script('finish', 'test -f notes.txt && echo "LINES=$(wc -l < notes.txt)" && echo FINISH_DONE'),
]);
const { id } = await startRun(wf, 'e2e 审批流');
let run = await waitRun(id, r => nodeStatus(r, 'gate') === 'waiting_approval', { label: 'gate waiting' });
assert.equal(run.status, 'waiting');
assert.equal(nodeStatus(run, 'prep'), 'succeeded');
await test.test('workspace is browsable while waiting', async () => {
const tree = await get(`/api/runs/${id}/workspace/tree`);
assert.match(JSON.stringify(tree), /notes\.txt/);
const file = await api('GET', `/api/runs/${id}/workspace/file?path=notes.txt`, undefined, { raw: true });
assert.match(file.text, /attempt-/);
const hits = await get(`/api/runs/${id}/workspace/search?q=attempt`);
assert.match(JSON.stringify(hits), /notes\.txt/);
await api('GET', `/api/runs/${id}/workspace/file?path=../../etc/passwd`, undefined, { expect: [400, 403, 404] });
await get(`/api/runs/${id}/workspace`);
});
await test.test('wrong node cannot be approved', async () => {
await api('POST', `/api/runs/${id}/approve`, { nodeId: 'prep' }, { expect: [400, 409, 500] });
});
await post(`/api/runs/${id}/reject`, { nodeId: 'gate', targetId: 'prep', reason: 'e2e 打回:再写一行' });
run = await waitRun(id, r => nodeStatus(r, 'gate') === 'waiting_approval' && (nodeOf(r, 'prep').attempt >= 2 || nodeOf(r, 'prep').reworkCount >= 1), { label: 'rework back at gate' });
await post(`/api/runs/${id}/approve`, { nodeId: 'gate', context: 'e2e 批准' });
run = await waitRun(id, r => r.status === 'succeeded', { label: 'succeeded' });
assert.equal(nodeStatus(run, 'finish'), 'succeeded');
const logs = await logsText(id);
assert.match(logs, /FINISH_DONE/);
assert.match(logs, /LINES=2/, 'rework must reuse the workspace so notes.txt has two lines');
const artifacts = run.artifacts || [];
const result = artifacts.find(a => a.name === 'result');
assert.ok(result, `declared artifact missing: ${JSON.stringify(artifacts.map(a => a.name))}`);
assert.equal(result.nodeId, 'prep');
const content = await api('GET', `/api/runs/${id}/artifacts/${encodeURIComponent(result.id)}`, undefined, { raw: true });
assert.match(content.text, /E2E result/);
await api('GET', `/api/runs/${id}/artifacts/artifact-e2e-missing`, undefined, { expect: 404 });
const structured = await get(`/api/runs/${id}/logs`);
assert.ok((structured.logs || []).length > 0);
const nodeLogs = await get(`/api/runs/${id}/logs?nodeId=finish`);
assert.ok((nodeLogs.logs || []).every(l => !l.context?.nodeId || l.context.nodeId === 'finish'));
const telemetry = await get(`/api/runs/${id}/telemetry`);
assert.ok((telemetry.events || []).length > 0);
const timing = checkTiming(run);
if (timing) assert.ok(timing.waitingMs > 0, 'time parked at the approval gate must count as waiting');
const summary = list(await get(`/api/runs?projectId=${cfg.projectId}`)).find(r => r.id === id);
assert.equal(summary?.status, 'succeeded');
});
test('failed script: retry from the node succeeds on the second attempt', {
todo: 'POST /api/runs/:id/retry tears the failed attempt\'s sandbox down inside the request (docker rm -f of the agentflow-run-*-scripts sidecar, up to dockerTeardownTimeout=60s); under nested Docker that rm hangs, so the call outlives the 60s client timeout',
}, async () => {
const wf = await createWorkflow('retry', [
script('flaky', 'if [ -f .e2e-retry-ok ]; then echo RETRY_OK; else touch .e2e-retry-ok; echo FIRST_FAIL >&2; exit 3; fi'),
script('after', 'echo AFTER_OK'),
]);
const { id } = await startRun(wf);
let run = await waitRun(id, r => r.status === 'failed', { label: 'failed' });
assert.equal(nodeStatus(run, 'flaky'), 'failed');
assert.equal(nodeStatus(run, 'after'), 'pending');
assert.match(nodeOf(run, 'flaky').error || '', /exit status 3/);
await post(`/api/runs/${id}/retry`, { nodeId: 'flaky', reason: 'e2e 重试' });
run = await waitRun(id, r => r.status === 'succeeded', { label: 'succeeded after retry' });
assert.equal(nodeOf(run, 'flaky').attempt, 2);
assert.match(await logsText(id), /RETRY_OK[\s\S]*AFTER_OK|AFTER_OK[\s\S]*RETRY_OK/);
});
test('failed script: manual pass skips it and continues downstream', async () => {
const wf = await createWorkflow('pass', [script('broken', 'exit 7'), script('after', 'echo AFTER_PASS')]);
const { id } = await startRun(wf);
await waitRun(id, r => r.status === 'failed', { label: 'failed' });
await api('POST', `/api/runs/${id}/pass`, { nodeId: 'after' }, { expect: [400, 409, 500] });
await post(`/api/runs/${id}/pass`, { nodeId: 'broken', context: 'e2e 人工通过' });
const run = await waitRun(id, r => r.status === 'succeeded', { label: 'succeeded after pass' });
assert.equal(nodeOf(run, 'broken').attempt, 1, 'pass must not re-execute the node');
assert.match(await logsText(id), /AFTER_PASS/);
});
test('pause interrupts a running script, resume re-runs it to success', async () => {
const wf = await createWorkflow('pause', [script('slow', 'sleep 12; echo SLOW_DONE'), script('next', 'echo NEXT_DONE')]);
const { id } = await startRun(wf);
await waitRun(id, r => nodeStatus(r, 'slow') === 'running', { label: 'slow running' });
await post(`/api/runs/${id}/pause`, {});
let run = await waitRun(id, r => r.status === 'waiting' && r.pauseRequested, { label: 'paused', timeoutMs: 60_000 });
assert.equal(nodeStatus(run, 'slow'), 'waiting_approval');
await post(`/api/runs/${id}/resume`, { context: 'e2e 继续' });
run = await waitRun(id, r => r.status === 'succeeded', { label: 'succeeded after resume' });
assert.equal(nodeOf(run, 'slow').attempt, 2);
assert.equal(nodeStatus(run, 'next'), 'succeeded');
checkTiming(run);
});
test('paused script: declining the retry fails the run', async () => {
const wf = await createWorkflow('decline', [script('slow', 'sleep 60')]);
const { id } = await startRun(wf);
await waitRun(id, r => nodeStatus(r, 'slow') === 'running', { label: 'slow running' });
await post(`/api/runs/${id}/pause`, {});
await waitRun(id, r => r.status === 'waiting' && nodeStatus(r, 'slow') === 'waiting_approval', { label: 'paused', timeoutMs: 60_000 });
await post(`/api/runs/${id}/decline-retry`, { nodeId: 'slow' });
const run = await waitRun(id, r => r.status === 'failed', { label: 'failed after decline', timeoutMs: 30_000 });
assert.equal(nodeStatus(run, 'slow'), 'failed');
});
test('cancel stops a running script and the run can then be deleted', async () => {
const wf = await createWorkflow('cancel', [script('forever', 'sleep 600')]);
const { id } = await startRun(wf);
await waitRun(id, r => nodeStatus(r, 'forever') === 'running', { label: 'running' });
await api('DELETE', `/api/runs/${id}`, undefined, { expect: [400, 409] });
await post(`/api/runs/${id}/cancel`, {});
const run = await waitRun(id, r => r.status === 'cancelled', { label: 'cancelled', timeoutMs: 60_000 });
assert.notEqual(nodeStatus(run, 'forever'), 'running');
await api('POST', `/api/runs/${id}/cancel`, {}, { expect: [200, 204, 400, 409, 500] });
await del(`/api/runs/${id}`, { expect: [200, 204] });
await api('GET', `/api/runs/${id}`, undefined, { expect: 404 });
// Deleting twice reports the engine's "run not found" (400) rather than 404.
await api('DELETE', `/api/runs/${id}`, undefined, { expect: [400, 404] });
});
test('SOP node runs a child SOP and continues the parent after it', async () => {
const child = await createWorkflow('child', [script('c1', 'echo CHILD_RAN > child.txt && echo CHILD_DONE')]);
const parent = await createWorkflow('parent', [
script('p1', 'echo PARENT_START'),
sopNode('sub', child),
script('p2', 'echo PARENT_DONE'),
]);
const { id } = await startRun(parent);
const run = await waitRun(id, r => r.status === 'succeeded', { label: 'parent succeeded', timeoutMs: 300_000 });
assert.equal(nodeStatus(run, 'sub'), 'succeeded');
const childRuns = list(await get('/api/runs')).filter(r => r.workflowId === child);
assert.ok(childRuns.length >= 1, 'child run not recorded');
assert.ok(childRuns.every(r => r.status === 'succeeded'), `child runs: ${childRuns.map(r => r.status)}`);
assert.match(await logsText(id), /PARENT_DONE/);
});
// 子 SOP 和父流程共用一个检出目录:代码改动双向可见,节点不提交、不推送工作分支。
test('repository-bound child SOP hands code back through the shared checkout', { skip: repoSkip, timeout: 700_000 }, async () => {
const { run } = await childSOPHandoffScenario({ label: 'repo-sop', waitRun });
const logs = await logsText(run.id);
assert.match(logs, /PARENT_SAW_CHILD/);
});
test('parallel branches run together and a fast successor does not wait for them', async () => {
const start = script('start', 'echo START_DONE');
const left = script('left', 'sleep 2; echo LEFT_DONE');
const right = script('right', 'sleep 2; echo RIGHT_DONE');
const fast = script('fast', 'echo FAST_DONE');
const after = script('after-fast', 'echo AFTER_FAST_DONE');
const join = script('join', 'echo JOIN_DONE');
const edge = (source, target) => ({ id: `${source}-${target}`, source, target, when: 'success' });
const wf = await createWorkflow('parallel', [start, left, right, fast, after, join], {
edges: [
edge('start', 'left'), edge('start', 'right'), edge('start', 'fast'),
edge('fast', 'after-fast'),
edge('left', 'join'), edge('right', 'join'), edge('after-fast', 'join'),
],
});
const { id } = await startRun(wf);
const run = await waitRun(id, r => r.status === 'succeeded', { label: 'parallel succeeded' });
const at = value => Date.parse(value);
const leftN = nodeOf(run, 'left');
const rightN = nodeOf(run, 'right');
const afterN = nodeOf(run, 'after-fast');
const joinN = nodeOf(run, 'join');
assert.ok(at(leftN.startedAt) < at(rightN.finishedAt) && at(rightN.startedAt) < at(leftN.finishedAt),
`branches did not overlap left ${leftN.startedAt}..${leftN.finishedAt} right ${rightN.startedAt}..${rightN.finishedAt}`);
assert.ok(at(afterN.startedAt) < at(leftN.finishedAt) && at(afterN.startedAt) < at(rightN.finishedAt),
`fast successor waited for slow siblings: after ${afterN.startedAt} left ${leftN.finishedAt} right ${rightN.finishedAt}`);
assert.ok(at(joinN.startedAt) >= at(leftN.finishedAt) && at(joinN.startedAt) >= at(rightN.finishedAt), 'join started before its dependencies finished');
});
test('retry after cancelling a parallel stage restores the cancelled sibling', async () => {
const start = script('start', 'echo START_DONE');
const left = script('left', 'sleep 12; echo LEFT_DONE');
const right = script('right', 'sleep 12; echo RIGHT_DONE');
const join = script('join', 'echo JOIN_DONE');
const edge = (source, target) => ({ id: `${source}-${target}`, source, target, when: 'success' });
const wf = await createWorkflow('parallel-cancel', [start, left, right, join], {
edges: [edge('start', 'left'), edge('start', 'right'), edge('left', 'join'), edge('right', 'join')],
});
const { id } = await startRun(wf);
await waitRun(id, r => nodeStatus(r, 'left') === 'running' && nodeStatus(r, 'right') === 'running', { label: 'both branches running', timeoutMs: 60_000 });
await post(`/api/runs/${id}/cancel`, {});
const cancelled = await waitRun(id, r => r.status === 'cancelled', { label: 'cancelled', timeoutMs: 60_000 });
assert.equal(nodeStatus(cancelled, 'left'), 'cancelled');
assert.equal(nodeStatus(cancelled, 'right'), 'cancelled');
await post(`/api/runs/${id}/retry`, { nodeId: 'left', reason: 'e2e restore parallel branch' });
const run = await waitRun(id, r => r.status === 'succeeded', { label: 'succeeded after parallel retry' });
assert.equal(nodeStatus(run, 'right'), 'succeeded');
assert.equal(nodeStatus(run, 'join'), 'succeeded');
assert.equal(nodeOf(run, 'start').attempt, 1, 'replaying one branch must not rerun its upstream');
});
test('run stream emits server-sent events', async () => {
const wf = await createWorkflow('stream', [script('s', 'for i in 1 2 3; do echo tick-$i; sleep 1; done')]);
const { id } = await startRun(wf);
const ctrl = new AbortController();
const timer = setTimeout(() => ctrl.abort(), 20_000);
const res = await fetch(`${cfg.base}/api/runs/${id}/stream`, { headers: { Authorization: `Bearer ${cfg.token}`, Accept: 'text/event-stream' }, signal: ctrl.signal });
assert.equal(res.status, 200);
assert.match(res.headers.get('content-type') || '', /event-stream/);
const reader = res.body.getReader();
let text = '';
try {
while (!/data:/.test(text)) {
const { value, done } = await reader.read();
if (done) break;
text += new TextDecoder().decode(value);
}
} finally {
clearTimeout(timer);
ctrl.abort();
}
assert.match(text, /data:/);
await waitRun(id, r => r.status === 'succeeded', { label: 'succeeded' });
});
test('starting a run validates its input', async () => {
await api('POST', '/api/runs', { workflowId: 'e2e-no-such-sop', projectId: cfg.projectId }, { expect: [400, 404, 409, 500] });
await api('POST', '/api/runs', { workflowId: 'generic-backend-delivery', projectId: 'e2e-no-such-project' }, { expect: [400, 404, 409, 500] });
const project = await get(`/api/projects/${cfg.projectId}`);
// A fresh database (local.sh) has no built-in templates to try this with.
const template = await api('GET', '/api/workflows/generic-backend-delivery', undefined, { expect: [200, 404], raw: true });
if (!project.defaultRepositoryId && template.status === 200 && template.data?.generic) {
const run = await startRun('generic-backend-delivery', 'Check repository-free admission');
assert.deepEqual(run.repositories || [], []);
await post(`/api/runs/${run.id}/cancel`, {});
}
});
test('unknown run actions answer 404', async () => {
for (const path of ['approve', 'retry', 'cancel', 'pause', 'resume']) {
await api('POST', `/api/runs/run-e2e-missing/${path}`, { nodeId: 'x' }, { expect: [400, 404, 409, 500] });
}
});