Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

feat: run parallel SOP branches together (#144)

crispinhu276f79c7a6ce22
diff --git a/docs/architecture.md b/docs/architecture.md
index b3844381..25177109 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -7,7 +7,7 @@ The platform keeps the upstream Go module name `agentflow`, package boundaries a
| `cmd/agentflow` | API, worker or combined process |
| `cmd/agentflow-runner` | Personal and pool execution machines |
| `cmd/agentflow-mcp`, `cmd/agentflow-gate-mcp` | Public MCP and structured gate results |
-| `internal/engine` | SOP graph, leases, scheduling, sandbox lifecycle and delivery |
+| `internal/engine` | SOP graph, leases, scheduling, sandbox lifecycle and delivery. Ready branches execute together in the run workspace; rerunning one branch after a cancel restores cancelled siblings |
| `internal/repo` | MySQL persistence, one repository per business domain |
| `internal/httpapi` | HTTP/MCP handlers, one handler per business domain |
| `internal/runtime`, `internal/runner` | Executor protocols, credentials, isolation and artifacts |
diff --git a/e2e/api/30-runs.test.mjs b/e2e/api/30-runs.test.mjs
index 9a582aa8..78c58c9e 100644
--- a/e2e/api/30-runs.test.mjs
+++ b/e2e/api/30-runs.test.mjs
@@ -176,6 +176,57 @@ test('repository-bound child SOP hands code back through the shared checkout', {
assert.match(logs, /PARENT_SAW_CHILD/);
});
+test('parallel branches run together and a fast successor does not wait for them', async () => {
+ const start = script('start', 'echo START_DONE');
+ const left = script('left', 'sleep 2; echo LEFT_DONE');
+ const right = script('right', 'sleep 2; echo RIGHT_DONE');
+ const fast = script('fast', 'echo FAST_DONE');
+ const after = script('after-fast', 'echo AFTER_FAST_DONE');
+ const join = script('join', 'echo JOIN_DONE');
+ const edge = (source, target) => ({ id: `${source}-${target}`, source, target, when: 'success' });
+ const wf = await createWorkflow('parallel', [start, left, right, fast, after, join], {
+ edges: [
+ edge('start', 'left'), edge('start', 'right'), edge('start', 'fast'),
+ edge('fast', 'after-fast'),
+ edge('left', 'join'), edge('right', 'join'), edge('after-fast', 'join'),
+ ],
+ });
+ const { id } = await startRun(wf);
+ const run = await waitRun(id, r => r.status === 'succeeded', { label: 'parallel succeeded' });
+ const at = value => Date.parse(value);
+ const leftN = nodeOf(run, 'left');
+ const rightN = nodeOf(run, 'right');
+ const afterN = nodeOf(run, 'after-fast');
+ const joinN = nodeOf(run, 'join');
+ assert.ok(at(leftN.startedAt) < at(rightN.finishedAt) && at(rightN.startedAt) < at(leftN.finishedAt),
+ `branches did not overlap left ${leftN.startedAt}..${leftN.finishedAt} right ${rightN.startedAt}..${rightN.finishedAt}`);
+ assert.ok(at(afterN.startedAt) < at(leftN.finishedAt) && at(afterN.startedAt) < at(rightN.finishedAt),
+ `fast successor waited for slow siblings: after ${afterN.startedAt} left ${leftN.finishedAt} right ${rightN.finishedAt}`);
+ assert.ok(at(joinN.startedAt) >= at(leftN.finishedAt) && at(joinN.startedAt) >= at(rightN.finishedAt), 'join started before its dependencies finished');
+});
+
+test('retry after cancelling a parallel stage restores the cancelled sibling', async () => {
+ const start = script('start', 'echo START_DONE');
+ const left = script('left', 'sleep 12; echo LEFT_DONE');
+ const right = script('right', 'sleep 12; echo RIGHT_DONE');
+ const join = script('join', 'echo JOIN_DONE');
+ const edge = (source, target) => ({ id: `${source}-${target}`, source, target, when: 'success' });
+ const wf = await createWorkflow('parallel-cancel', [start, left, right, join], {
+ edges: [edge('start', 'left'), edge('start', 'right'), edge('left', 'join'), edge('right', 'join')],
+ });
+ const { id } = await startRun(wf);
+ await waitRun(id, r => nodeStatus(r, 'left') === 'running' && nodeStatus(r, 'right') === 'running', { label: 'both branches running', timeoutMs: 60_000 });
+ await post(`/api/runs/${id}/cancel`, {});
+ const cancelled = await waitRun(id, r => r.status === 'cancelled', { label: 'cancelled', timeoutMs: 60_000 });
+ assert.equal(nodeStatus(cancelled, 'left'), 'cancelled');
+ assert.equal(nodeStatus(cancelled, 'right'), 'cancelled');
+ await post(`/api/runs/${id}/retry`, { nodeId: 'left', reason: 'e2e restore parallel branch' });
+ const run = await waitRun(id, r => r.status === 'succeeded', { label: 'succeeded after parallel retry' });
+ assert.equal(nodeStatus(run, 'right'), 'succeeded');
+ assert.equal(nodeStatus(run, 'join'), 'succeeded');
+ assert.equal(nodeOf(run, 'start').attempt, 1, 'replaying one branch must not rerun its upstream');
+});
+
test('run stream emits server-sent events', async () => {
const wf = await createWorkflow('stream', [script('s', 'for i in 1 2 3; do echo tick-$i; sleep 1; done')]);
const { id } = await startRun(wf);
diff --git a/internal/engine/artifacts.go b/internal/engine/artifacts.go
index f7bf0964..fc364e69 100644
--- a/internal/engine/artifacts.go
+++ b/internal/engine/artifacts.go
@@ -12,11 +12,16 @@ import (
"path/filepath"
"sort"
"strings"
+ "sync/atomic"
"time"
)
const maxArtifactBytes = 1 << 20
+// artifactSeq keeps IDs unique when parallel branches publish in the same
+// clock tick.
+var artifactSeq atomic.Uint64
+
func materializeInputs(workspace string, wf domain.WorkflowDefinition, node domain.WorkflowNode, run domain.Run) (string, error) {
sources := map[string]bool{}
for _, edge := range wf.Edges {
@@ -48,7 +53,7 @@ func materializeInputs(workspace string, wf domain.WorkflowDefinition, node doma
fmt.Fprintf(&manifest, "## %s / repository-patch (%s)\n\nNo full diff is stored. The run workspace is the source of truth.\n\n%s\n\n", artifact.NodeID, artifact.RepositoryDir, summary)
continue
}
- nodeSegment := safePathSegment(artifact.NodeID)
+ nodeSegment := safePathSegment(nodeControlKey(run, artifact.NodeID))
controlSegments := []string{"inputs", nodeSegment}
if artifact.RepositoryDir != "" {
controlSegments = append(controlSegments, safePathSegment(artifact.RepositoryDir))
@@ -58,7 +63,7 @@ func materializeInputs(workspace string, wf domain.WorkflowDefinition, node doma
return "", err
}
fileName := safePathSegment(artifact.Name) + extensionFor(artifact.MediaType)
- rel := artifactInputRelPath(artifact)
+ rel := artifactInputRelPath(run, artifact)
target := filepath.Join(inputDir, fileName)
if info, err := os.Lstat(target); err == nil {
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
@@ -88,8 +93,8 @@ func materializeInputs(workspace string, wf domain.WorkflowDefinition, node doma
// artifactInputRelPath is where an upstream artifact is mounted for the
// consuming node, relative to the run workspace.
-func artifactInputRelPath(artifact domain.Artifact) string {
- segments := []string{".agentflow", "inputs", safePathSegment(artifact.NodeID)}
+func artifactInputRelPath(run domain.Run, artifact domain.Artifact) string {
+ segments := []string{".agentflow", "inputs", safePathSegment(nodeControlKey(run, artifact.NodeID))}
if artifact.RepositoryDir != "" {
segments = append(segments, safePathSegment(artifact.RepositoryDir))
}
@@ -406,7 +411,7 @@ func artifactPromptBody(artifact domain.Artifact) string {
func newArtifact(runID, nodeID string, attempt int, name, path, mediaType string, content []byte) domain.Artifact {
sum := sha256.Sum256(content)
now := time.Now().UTC()
- return domain.Artifact{ID: fmt.Sprintf("artifact-%d", now.UnixNano()), RunID: runID, NodeID: nodeID, Attempt: attempt, Name: name, Path: path, MediaType: mediaType, Size: int64(len(content)), SHA256: fmt.Sprintf("%x", sum), Content: string(content), CreatedAt: now}
+ return domain.Artifact{ID: fmt.Sprintf("artifact-%d-%d", now.UnixNano(), artifactSeq.Add(1)), RunID: runID, NodeID: nodeID, Attempt: attempt, Name: name, Path: path, MediaType: mediaType, Size: int64(len(content)), SHA256: fmt.Sprintf("%x", sum), Content: string(content), CreatedAt: now}
}
func appendArtifact(run *domain.Run, nodeID string, attempt int, name, path, mediaType string, content []byte) {
diff --git a/internal/engine/cloud_pool.go b/internal/engine/cloud_pool.go
index 3827bf87..e4c99954 100644
--- a/internal/engine/cloud_pool.go
+++ b/internal/engine/cloud_pool.go
@@ -140,6 +140,13 @@ func isPoolRun(run domain.Run) bool {
// machine to return before continuing. Runs on a user's own machine are left
// alone.
func (e *Engine) bindPoolRunner(ctx context.Context, run domain.Run, nodeID string) (domain.Run, error) {
+ // Parallel first nodes bind together: the first picks the machine, the
+ // others re-read the run and follow it.
+ release := e.lockPoolBinding(run.ID)
+ defer release()
+ if fresh, ok := e.repo.RunState(run.ID); ok {
+ run = fresh
+ }
ec := run.ExecutionContext
convert := ec.ExecutionMode == domain.ExecutionModeCloud && e.cloudPoolRequired(ec.ProjectID)
if convert && hasCodeWorkspace(run) && hasWorkspaceProgress(run, nodeID) {
diff --git a/internal/engine/control_recovery_cases_test.go b/internal/engine/control_recovery_cases_test.go
index 77458617..c2029b1b 100644
--- a/internal/engine/control_recovery_cases_test.go
+++ b/internal/engine/control_recovery_cases_test.go
@@ -158,7 +158,7 @@ func TestRunRecoveryAndSchedulerBookkeeping(t *testing.T) {
if e.ActiveRunCount() != 1 || !e.isRunSchedulerActive("bookkeeping") {
t.Fatal("active scheduler missing")
}
- unlock := e.lockRunExec("bookkeeping")
+ unlock := e.lockNodeExec("bookkeeping", "node")
unlock()
e.clearRunning("bookkeeping")
if e.ActiveRunCount() != 0 {
diff --git a/internal/engine/engine.go b/internal/engine/engine.go
index b5205a86..fba61b33 100644
--- a/internal/engine/engine.go
+++ b/internal/engine/engine.go
@@ -32,7 +32,9 @@ type Engine struct {
// per run, read where the engine used to inspect its own workspace copy.
workspaceFacts map[string]domain.RunnerWorkspaceFacts
cloudPool CloudRunnerPool
- runExec map[string]*sync.Mutex
+ execLocks keyedLocks
+ nodeExecMu sync.Mutex
+ nodeExecs map[string]map[string]nodeExecution
subMu sync.Mutex
subs map[string]map[chan domain.Run]struct{}
runnerTaskMu sync.Mutex
diff --git a/internal/engine/execution_cleanup_cases_test.go b/internal/engine/execution_cleanup_cases_test.go
index 915e21a1..8a533422 100644
--- a/internal/engine/execution_cleanup_cases_test.go
+++ b/internal/engine/execution_cleanup_cases_test.go
@@ -335,6 +335,6 @@ func TestInterruptedRunRecoverySettlesControlAndCompletedGraphs(t *testing.T) {
t.Fatal("claim running idempotence")
}
e.clearRunning("claimed")
- unlock := e.lockRunExec("run")
+ unlock := e.lockNodeExec("run", "node")
unlock()
}
diff --git a/internal/engine/human_context.go b/internal/engine/human_context.go
index c094f17c..35b4defc 100644
--- a/internal/engine/human_context.go
+++ b/internal/engine/human_context.go
@@ -52,7 +52,30 @@ func writeHumanContextFile(workspace, note string) error {
if note == "" || strings.TrimSpace(workspace) == "" {
return nil
}
- return os.WriteFile(filepath.Join(workspace, "human-context.md"), []byte("# 人工补充上下文\n\n"+note+"\n"), 0o644)
+ // Replace through a rename: a parallel branch's CLI may be reading the
+ // file and must never see it truncated.
+ target := filepath.Join(workspace, "human-context.md")
+ content := []byte("# 人工补充上下文\n\n" + note + "\n")
+ if existing, err := os.ReadFile(target); err == nil && string(existing) == string(content) {
+ return nil
+ }
+ temp, err := os.CreateTemp(workspace, ".human-context-*.md")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(temp.Name())
+ if _, err := temp.Write(content); err != nil {
+ temp.Close()
+ return err
+ }
+ if err := temp.Chmod(0o644); err != nil {
+ temp.Close()
+ return err
+ }
+ if err := temp.Close(); err != nil {
+ return err
+ }
+ return os.Rename(temp.Name(), target)
}
func safePathSegment(value string) string {
diff --git a/internal/engine/node_execute.go b/internal/engine/node_execute.go
index 660ac152..f70dd451 100644
--- a/internal/engine/node_execute.go
+++ b/internal/engine/node_execute.go
@@ -50,11 +50,28 @@ func (e *Engine) execute(ctx context.Context, runID string) {
cancel()
return
}
+ e.reconcileNodeExecutions(runID)
}
}
}()
+ // Branches run as soon as their dependencies succeed; no wave waits for
+ // its slowest member. inflight is shared with the node goroutines, which
+ // remove themselves and wake the loop when they finish.
+ var (
+ inflightMu sync.Mutex
+ inflight = map[string]bool{}
+ nodesWG sync.WaitGroup
+ wake = make(chan struct{}, 1)
+ )
+ // stopNodes cancels and drains in-flight nodes before the run settles, so a
+ // terminal or paused status is never written under a running branch.
+ stopNodes := func() {
+ cancel()
+ nodesWG.Wait()
+ }
defer func() {
cancel()
+ nodesWG.Wait()
<-heartbeatDone
_ = e.repo.ReleaseRunLease(runID, e.instanceID, generation)
e.mu.Lock()
@@ -64,6 +81,7 @@ func (e *Engine) execute(ctx context.Context, runID string) {
for {
select {
case <-ctx.Done():
+ nodesWG.Wait()
e.finishInterruptedAt(runID, generation)
return
default:
@@ -73,10 +91,12 @@ func (e *Engine) execute(ctx context.Context, runID string) {
return
}
if r.CancelRequested {
+ stopNodes()
e.finishAt(runID, generation, "cancelled", "流水线已取消")
return
}
if r.PauseRequested {
+ stopNodes()
e.pauseAt(runID, generation, "运行已由用户暂停")
return
}
@@ -84,7 +104,26 @@ func (e *Engine) execute(ctx context.Context, runID string) {
return
}
wf := workflowForRun(e.repo, r)
- ready := readyNodes(wf, r)
+ e.reconcileNodeExecutions(runID)
+ inflightMu.Lock()
+ var ready []domain.WorkflowNode
+ for _, node := range readyNodes(wf, r) {
+ if !inflight[node.ID] {
+ ready = append(ready, node)
+ }
+ }
+ running := len(inflight)
+ inflightMu.Unlock()
+ if len(ready) == 0 && running > 0 {
+ // A sibling is still running: its result may unblock more nodes, and
+ // a failed or waiting branch must not settle the run under it.
+ select {
+ case <-wake:
+ case <-ctx.Done():
+ case <-time.After(500 * time.Millisecond):
+ }
+ continue
+ }
if len(ready) == 0 {
if allSucceeded(wf, r) {
rework, err := e.integrateScheduledRun(runID)
@@ -114,16 +153,35 @@ func (e *Engine) execute(ctx context.Context, runID string) {
e.finishAt(runID, generation, "failed", pipelineFailedMessage)
return
}
+ if node, ok := cancelledNode(wf, r); ok {
+ // Nothing reschedules a cancelled node by itself; waiting here
+ // would leave the run "running" forever.
+ e.finishAt(runID, generation, "failed", "节点「"+node.Name+"」已取消,流水线无法继续;请从该节点重跑")
+ return
+ }
time.Sleep(80 * time.Millisecond)
continue
}
- var wg sync.WaitGroup
for _, node := range ready {
node := node
- wg.Add(1)
- go func() { defer wg.Done(); e.executeNode(ctx, runID, wf, node) }()
+ inflightMu.Lock()
+ inflight[node.ID] = true
+ inflightMu.Unlock()
+ nodesWG.Add(1)
+ go func() {
+ defer nodesWG.Done()
+ defer func() {
+ inflightMu.Lock()
+ delete(inflight, node.ID)
+ inflightMu.Unlock()
+ select {
+ case wake <- struct{}{}:
+ default:
+ }
+ }()
+ e.executeNode(ctx, runID, wf, node)
+ }()
}
- wg.Wait()
}
}
@@ -259,7 +317,9 @@ func (e *Engine) executeNode(ctx context.Context, runID string, wf domain.Workfl
if !ok || !e.ownsRun(currentLease) {
return
}
- unlock := e.lockRunExec(runID)
+ // Sibling branches run concurrently; only another execution of this same
+ // node (a resume, rework or takeover racing a winding-down attempt) waits.
+ unlock := e.lockNodeExec(runID, node.ID)
defer unlock()
executionGeneration := currentLease.Generation
executionAttempt := 0
@@ -288,6 +348,9 @@ func (e *Engine) executeNode(ctx context.Context, runID string, wf domain.Workfl
if !started {
return
}
+ ctx, cancelNode := context.WithCancel(ctx)
+ defer cancelNode()
+ defer e.registerNodeExecution(runID, node.ID, executionAttempt, cancelNode)()
failNode := func(err error, allowReject bool) {
if err == nil {
return
@@ -510,6 +573,12 @@ func (e *Engine) executeNode(ctx context.Context, runID string, wf domain.Workfl
failNode(preparationErr, nodeRejects(node))
return
}
+ // The environment is ready and this node's inputs are in place; parallel
+ // branches execute from here on without holding the environment lock.
+ preparing = false
+ sb.nodeMu.Unlock()
+ sb.activeExecutions.Add(1)
+ defer sb.activeExecutions.Add(-1)
var output string
defaultFailure := "fail"
if nodeRejects(node) {
@@ -607,7 +676,12 @@ func (e *Engine) executeNode(ctx context.Context, runID string, wf domain.Workfl
maxIterations = max(1, agent.Loop.MaxIterations)
loopOnExhausted = agent.Loop.OnExhausted
}
+ agentLockOwner := workspaceRunID(current)
runOnce = func(feedback string) (string, error) {
+ // Same-role nodes share the Agent's control directories in this
+ // checkout, so their CLI invocations take turns.
+ releaseAgent := e.lockAgentInvocation(agentLockOwner, agent.ID)
+ defer releaseAgent()
if reported != nil {
reported.reset()
}
@@ -626,7 +700,7 @@ func (e *Engine) executeNode(ctx context.Context, runID string, wf domain.Workfl
resumeSessionID = strings.TrimSpace(nr.SessionID)
}
}
- invocationID := fmt.Sprintf("%s-%d-%d", node.ID, attempt, invocationSeq)
+ invocationID := fmt.Sprintf("%s-%d-%d", nodeControlKey(current, node.ID), attempt, invocationSeq)
rendered := renderPromptWithStore(e.repo, wf, effectiveNode, current)
hadCatalog := strings.Contains(rendered, "{{artifacts}}")
prompt := strings.ReplaceAll(rendered, "{{artifacts}}", artifactContext)
@@ -659,7 +733,7 @@ func (e *Engine) executeNode(ctx context.Context, runID string, wf domain.Workfl
}
options := runtime.InvocationOptions{RunID: runID, NodeID: node.ID, ResumeSessionID: resumeSessionID, Repositories: current.RepositoryContexts()}
if requiresSubmit {
- options.GateResultPath = gateResultPath(node.ID, attempt)
+ options.GateResultPath = gateResultPath(nodeControlKey(current, node.ID), attempt)
}
ex := ex
if cb := sb.credentialSnapshot().CodeBuddy; ex.Kind == "codebuddy" && cb.Sponsored {
@@ -700,7 +774,7 @@ func (e *Engine) executeNode(ctx context.Context, runID string, wf domain.Workfl
lastIteration := 0
submitPath := ""
if requiresSubmit {
- submitPath = gateResultPath(node.ID, executionAttempt)
+ submitPath = gateResultPath(nodeControlKey(current, node.ID), executionAttempt)
}
for iteration := 1; iteration <= maxIterations; iteration++ {
lastIteration = iteration
@@ -881,6 +955,9 @@ func (e *Engine) executeNode(ctx context.Context, runID string, wf domain.Workfl
if isMaxTurnsErr(err) {
e.updateRun(runID, func(r *domain.Run) {
nr := r.Nodes[node.ID]
+ if !shouldCommitNodeOutcome(*r, nr, executionGeneration, executionAttempt, false) {
+ return
+ }
nr.Output, nr.Error, nr.Status, nr.Activity, nr.FinishedAt = output, err.Error(), "waiting_approval", "对话轮次用尽,等待确认是否重试", time.Now().UTC()
e.event(r, node.ID, "agent.turns.exhausted", "对话轮次已用尽,请确认是否重试本节点")
})
@@ -889,6 +966,9 @@ func (e *Engine) executeNode(ctx context.Context, runID string, wf domain.Workfl
if failedAction == "human" {
e.updateRun(runID, func(r *domain.Run) {
nr := r.Nodes[node.ID]
+ if !shouldCommitNodeOutcome(*r, nr, executionGeneration, executionAttempt, false) {
+ return
+ }
nr.Output, nr.Error, nr.Status, nr.Activity, nr.FinishedAt = output, err.Error(), "waiting_approval", "等待人工确认", time.Now().UTC()
message := "Agent 请求人工确认"
if nr.GateResult != nil && strings.TrimSpace(nr.GateResult.ActionRequired) != "" {
@@ -903,6 +983,9 @@ func (e *Engine) executeNode(ctx context.Context, runID string, wf domain.Workfl
if maxIterations > 1 && loopOnExhausted == "human" {
e.updateRun(runID, func(r *domain.Run) {
nr := r.Nodes[node.ID]
+ if !shouldCommitNodeOutcome(*r, nr, executionGeneration, executionAttempt, false) {
+ return
+ }
nr.Output, nr.Error, nr.Status, nr.FinishedAt = output, err.Error(), "waiting_approval", time.Now().UTC()
e.event(r, node.ID, "agent.loop.exhausted", "Agent 自修正次数已耗尽,转人工判断:"+err.Error())
})
diff --git a/internal/engine/node_outcome.go b/internal/engine/node_outcome.go
index 22e5e5d0..5395c315 100644
--- a/internal/engine/node_outcome.go
+++ b/internal/engine/node_outcome.go
@@ -28,15 +28,17 @@ func (e *Engine) nodeFailed(runID string, node domain.WorkflowNode, err error, a
}
// shouldCommitNodeOutcome is the exclusive-execute write fence for one node.
-// Generation mismatch means another instance took the lease. A successful
-// outcome may still land after retries bumped Attempt, so a live agent that
-// passed gates is not dropped. Failures stay attempt-scoped and never
-// overwrite succeeded or waiting_approval.
+// Generation mismatch means another instance took the lease. Either outcome
+// needs the node still in flight: with parallel branches a rework or retry can
+// reset a running sibling to pending, and its late result must neither
+// overwrite the reset nor let it be scheduled twice. A successful outcome may
+// still land after retries bumped Attempt, so a live agent that passed gates
+// is not dropped for an attempt mismatch. Failures stay attempt-scoped.
func shouldCommitNodeOutcome(r domain.Run, nr *domain.NodeRun, generation, attempt int, success bool) bool {
if r.Generation != generation || nr == nil {
return false
}
- if nr.Status == "succeeded" || nr.Status == "waiting_approval" {
+ if nr.Status != "running" && nr.Status != "pending_input" {
return false
}
if success {
diff --git a/internal/engine/node_outputs.go b/internal/engine/node_outputs.go
index 17eaeffd..71d9d24b 100644
--- a/internal/engine/node_outputs.go
+++ b/internal/engine/node_outputs.go
@@ -181,7 +181,7 @@ func runnerArtifactInputs(wf domain.WorkflowDefinition, node domain.WorkflowNode
manifest.WriteString("## " + artifact.NodeID + " / repository-patch (" + artifact.RepositoryDir + ")\n\nNo full diff is stored. The run workspace is the source of truth.\n\n" + repositoryPatchSummary(artifact) + "\n\n")
continue
}
- rel := artifactInputRelPath(artifact)
+ rel := artifactInputRelPath(run, artifact)
files = append(files, domain.RunnerInputFile{Path: rel, Content: artifact.Content})
writeInputManifestEntry(&manifest, artifact, rel)
}
diff --git a/internal/engine/parallel_branches_test.go b/internal/engine/parallel_branches_test.go
new file mode 100644
index 00000000..67c3db0f
--- /dev/null
+++ b/internal/engine/parallel_branches_test.go
@@ -0,0 +1,355 @@
+package engine
+
+import (
+ "path/filepath"
+ "testing"
+ "time"
+
+ "agentflow/internal/domain"
+ "agentflow/internal/repo"
+ "agentflow/internal/runtime"
+)
+
+func parallelTestEngine(t *testing.T, wf domain.WorkflowDefinition) (*Engine, *repo.Store) {
+ t.Helper()
+ s, err := repo.OpenTest()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(s.Close)
+ if err := s.PutWorkflow(wf); err != nil {
+ t.Fatal(err)
+ }
+ return New(s, runtime.NewManager(filepath.Join(t.TempDir(), "workspaces"))), s
+}
+
+func scriptNode(id, script string) domain.WorkflowNode {
+ return domain.WorkflowNode{ID: id, Name: id, Type: "script", ScriptLanguage: "shell", Script: script, MaxAttempts: 1}
+}
+
+func edge(source, target string) domain.WorkflowEdge {
+ return domain.WorkflowEdge{ID: source + "-" + target, Source: source, Target: target, When: "success"}
+}
+
+func nodeEvents(r domain.Run, nodeID, eventType string) int {
+ n := 0
+ for _, event := range r.Events {
+ if event.NodeID == nodeID && event.Type == eventType {
+ n++
+ }
+ }
+ return n
+}
+
+func overlaps(a, b *domain.NodeRun) bool {
+ return a.StartedAt.Before(b.FinishedAt) && b.StartedAt.Before(a.FinishedAt)
+}
+
+// Two sleeping branches run at the same time, and a fast branch's successor
+// starts without waiting for its slow sibling (no wave barrier).
+func TestParallelBranchesExecuteConcurrently(t *testing.T) {
+ t.Parallel()
+ wf := domain.WorkflowDefinition{ID: "parallel-overlap", Name: "parallel overlap", SandboxProfileID: "local",
+ Nodes: []domain.WorkflowNode{
+ scriptNode("start", "true"),
+ scriptNode("left", "sleep 1.5"),
+ scriptNode("right", "sleep 1.5"),
+ scriptNode("fast", "true"),
+ scriptNode("after-fast", "true"),
+ scriptNode("join", "true"),
+ },
+ Edges: []domain.WorkflowEdge{
+ edge("start", "left"), edge("start", "right"), edge("start", "fast"),
+ edge("fast", "after-fast"),
+ edge("left", "join"), edge("right", "join"), edge("after-fast", "join"),
+ },
+ }
+ e, s := parallelTestEngine(t, wf)
+ run, err := e.Start(wf.ID, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ waitFor(t, s, run.ID, "succeeded")
+ got, _ := s.Run(run.ID)
+ left, right, afterFast, join := got.Nodes["left"], got.Nodes["right"], got.Nodes["after-fast"], got.Nodes["join"]
+ if !overlaps(left, right) {
+ t.Fatalf("parallel branches did not overlap: left %s..%s right %s..%s", left.StartedAt, left.FinishedAt, right.StartedAt, right.FinishedAt)
+ }
+ if !afterFast.StartedAt.Before(left.FinishedAt) || !afterFast.StartedAt.Before(right.FinishedAt) {
+ t.Fatalf("fast branch successor waited for slow siblings: after-fast started %s, left finished %s, right finished %s", afterFast.StartedAt, left.FinishedAt, right.FinishedAt)
+ }
+ if join.StartedAt.Before(left.FinishedAt) || join.StartedAt.Before(right.FinishedAt) {
+ t.Fatalf("join started before its dependencies finished")
+ }
+}
+
+// A failed branch must not settle the run while a sibling is still running:
+// the sibling finishes and its successor still runs before the run fails.
+func TestSchedulerWaitsForRunningSiblingBeforeSettling(t *testing.T) {
+ t.Parallel()
+ wf := domain.WorkflowDefinition{ID: "parallel-fail-wait", Name: "parallel fail wait", SandboxProfileID: "local",
+ Nodes: []domain.WorkflowNode{
+ scriptNode("start", "true"),
+ scriptNode("broken", "exit 3"),
+ scriptNode("slow", "sleep 1"),
+ scriptNode("after-slow", "true"),
+ },
+ Edges: []domain.WorkflowEdge{edge("start", "broken"), edge("start", "slow"), edge("slow", "after-slow")},
+ }
+ e, s := parallelTestEngine(t, wf)
+ run, err := e.Start(wf.ID, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ waitFor(t, s, run.ID, "failed")
+ got, _ := s.Run(run.ID)
+ if got.Nodes["broken"].Status != "failed" {
+ t.Fatalf("broken = %+v", got.Nodes["broken"])
+ }
+ if got.Nodes["slow"].Status != "succeeded" || got.Nodes["after-slow"].Status != "succeeded" {
+ t.Fatalf("scheduler settled under a running sibling: slow=%s after-slow=%s", got.Nodes["slow"].Status, got.Nodes["after-slow"].Status)
+ }
+}
+
+// A branch waiting for approval keeps the run "running" while its sibling
+// works; the run only waits once nothing is in flight.
+func TestWaitingBranchDoesNotStopRunningSibling(t *testing.T) {
+ t.Parallel()
+ wf := domain.WorkflowDefinition{ID: "parallel-wait-sibling", Name: "parallel wait sibling", SandboxProfileID: "local",
+ Nodes: []domain.WorkflowNode{
+ scriptNode("start", "true"),
+ {ID: "gate", Name: "gate", Type: "approval"},
+ scriptNode("slow", "sleep 1"),
+ scriptNode("after-slow", "true"),
+ },
+ Edges: []domain.WorkflowEdge{edge("start", "gate"), edge("start", "slow"), edge("slow", "after-slow")},
+ }
+ e, s := parallelTestEngine(t, wf)
+ run, err := e.Start(wf.ID, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ mid := waitRun(t, s, run.ID, 30*time.Second, func(r domain.Run) bool {
+ return r.Nodes["gate"] != nil && r.Nodes["gate"].Status == "waiting_approval" && r.Nodes["slow"] != nil && r.Nodes["slow"].Status == "running"
+ })
+ if mid.Status != "running" {
+ t.Fatalf("run status with a running sibling = %s", mid.Status)
+ }
+ waitFor(t, s, run.ID, "waiting")
+ got, _ := s.Run(run.ID)
+ if got.Nodes["after-slow"].Status != "succeeded" {
+ t.Fatalf("after-slow = %s", got.Nodes["after-slow"].Status)
+ }
+ if err := e.Approve(run.ID, "gate", ""); err != nil {
+ t.Fatal(err)
+ }
+ waitFor(t, s, run.ID, "succeeded")
+}
+
+// Rework from one branch resets a sibling that is still running. The stale
+// execution is cancelled, never commits, and the sibling runs exactly once
+// more as a new attempt.
+func TestReworkResetsRunningSiblingWithoutDoubleCommit(t *testing.T) {
+ t.Parallel()
+ wf := domain.WorkflowDefinition{ID: "parallel-rework", Name: "parallel rework", SandboxProfileID: "local",
+ Nodes: []domain.WorkflowNode{
+ scriptNode("start", "true"),
+ {ID: "gate", Name: "gate", Type: "approval", RejectTo: "start", MaxReworks: 3},
+ scriptNode("slow", "sleep 2"),
+ },
+ Edges: []domain.WorkflowEdge{edge("start", "gate"), edge("start", "slow")},
+ }
+ e, s := parallelTestEngine(t, wf)
+ run, err := e.Start(wf.ID, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ waitRun(t, s, run.ID, 30*time.Second, func(r domain.Run) bool {
+ return r.Nodes["gate"] != nil && r.Nodes["gate"].Status == "waiting_approval" && r.Nodes["slow"] != nil && r.Nodes["slow"].Status == "running"
+ })
+ if err := e.Reject(run.ID, domain.RejectRequest{NodeID: "gate", TargetID: "start", Reason: "redo"}); err != nil {
+ t.Fatal(err)
+ }
+ waitFor(t, s, run.ID, "waiting")
+ got, _ := s.Run(run.ID)
+ if got.Nodes["slow"].Status != "succeeded" {
+ t.Fatalf("slow = %+v", got.Nodes["slow"])
+ }
+ if n := nodeEvents(got, "slow", "node.succeeded"); n != 1 {
+ t.Fatalf("slow committed %d successes; the reset execution must not commit", n)
+ }
+ if got.Nodes["slow"].Attempt != 2 {
+ t.Fatalf("slow attempt = %d, want 2 (one abandoned, one rerun)", got.Nodes["slow"].Attempt)
+ }
+ if n := nodeEvents(got, "start", "node.succeeded"); n != 2 {
+ t.Fatalf("start succeeded %d times, want 2", n)
+ }
+ if err := e.Approve(run.ID, "gate", ""); err != nil {
+ t.Fatal(err)
+ }
+ waitFor(t, s, run.ID, "succeeded")
+}
+
+// Pausing parallel branches and continuing at once resumes both; no node is
+// committed twice and the run completes.
+func TestPauseThenImmediateResumeParallelBranches(t *testing.T) {
+ t.Parallel()
+ wf := domain.WorkflowDefinition{ID: "parallel-pause", Name: "parallel pause", SandboxProfileID: "local",
+ Nodes: []domain.WorkflowNode{
+ scriptNode("start", "true"),
+ scriptNode("left", "sleep 1"),
+ scriptNode("right", "sleep 1"),
+ scriptNode("join", "true"),
+ },
+ Edges: []domain.WorkflowEdge{edge("start", "left"), edge("start", "right"), edge("left", "join"), edge("right", "join")},
+ }
+ e, s := parallelTestEngine(t, wf)
+ run, err := e.Start(wf.ID, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ waitRun(t, s, run.ID, 30*time.Second, func(r domain.Run) bool {
+ return r.Nodes["left"] != nil && r.Nodes["left"].Status == "running" && r.Nodes["right"] != nil && r.Nodes["right"].Status == "running"
+ })
+ if err := e.Pause(run.ID); err != nil {
+ t.Fatal(err)
+ }
+ waitFor(t, s, run.ID, "waiting")
+ if err := e.ResumePaused(run.ID, ""); err != nil {
+ t.Fatal(err)
+ }
+ waitFor(t, s, run.ID, "succeeded")
+ got, _ := s.Run(run.ID)
+ for _, id := range []string{"left", "right", "join"} {
+ if n := nodeEvents(got, id, "node.succeeded"); n != 1 {
+ t.Fatalf("%s committed %d successes", id, n)
+ }
+ }
+}
+
+func TestKeyedLocksSerializeOnlySameKey(t *testing.T) {
+ t.Parallel()
+ var locks keyedLocks
+ releaseA := locks.lock("a")
+ acquiredB := make(chan struct{})
+ go func() {
+ release := locks.lock("b")
+ close(acquiredB)
+ release()
+ }()
+ select {
+ case <-acquiredB:
+ case <-time.After(2 * time.Second):
+ t.Fatal("different key blocked")
+ }
+ acquiredA := make(chan struct{})
+ go func() {
+ release := locks.lock("a")
+ close(acquiredA)
+ release()
+ }()
+ select {
+ case <-acquiredA:
+ t.Fatal("same key not serialized")
+ case <-time.After(100 * time.Millisecond):
+ }
+ releaseA()
+ select {
+ case <-acquiredA:
+ case <-time.After(2 * time.Second):
+ t.Fatal("same key never released")
+ }
+ locks.mu.Lock()
+ defer locks.mu.Unlock()
+ if len(locks.locks) != 0 {
+ t.Fatalf("lock entries leaked: %d", len(locks.locks))
+ }
+}
+
+func TestNodeControlKeyIsolatesChildSOP(t *testing.T) {
+ t.Parallel()
+ parent := domain.Run{ID: "run-parent"}
+ child := domain.Run{ID: "run-child", ExecutionContext: domain.TelemetryContext{WorkspaceRunID: "run-parent"}}
+ if nodeControlKey(parent, "dev") != "dev" {
+ t.Fatal("owner keys must stay unchanged")
+ }
+ if gateResultPath(nodeControlKey(parent, "dev"), 1) == gateResultPath(nodeControlKey(child, "dev"), 1) {
+ t.Fatal("child SOP gate result path collides with the parent's")
+ }
+ artifact := domain.Artifact{NodeID: "dev", Name: "spec", MediaType: "text/markdown"}
+ if artifactInputRelPath(parent, artifact) == artifactInputRelPath(child, artifact) {
+ t.Fatal("child SOP input path collides with the parent's")
+ }
+}
+
+// Cancelling during a parallel stage stops both branches. Replaying one of
+// them must resume the other too, and the run must settle instead of staying
+// "running" with a cancelled branch nobody schedules.
+func TestRetryAfterCancelResumesCancelledParallelSibling(t *testing.T) {
+ t.Parallel()
+ wf := domain.WorkflowDefinition{ID: "parallel-cancel-retry", Name: "parallel cancel retry", SandboxProfileID: "local",
+ Nodes: []domain.WorkflowNode{
+ scriptNode("start", "true"),
+ scriptNode("left", "sleep 1"),
+ scriptNode("right", "sleep 1"),
+ scriptNode("join", "true"),
+ },
+ Edges: []domain.WorkflowEdge{edge("start", "left"), edge("start", "right"), edge("left", "join"), edge("right", "join")},
+ }
+ e, s := parallelTestEngine(t, wf)
+ run, err := e.Start(wf.ID, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ waitRun(t, s, run.ID, 30*time.Second, func(r domain.Run) bool {
+ return r.Nodes["left"] != nil && r.Nodes["left"].Status == "running" && r.Nodes["right"] != nil && r.Nodes["right"].Status == "running"
+ })
+ if err := e.Cancel(run.ID); err != nil {
+ t.Fatal(err)
+ }
+ cancelled := waitRun(t, s, run.ID, 30*time.Second, func(r domain.Run) bool { return r.Status == "cancelled" && !hasRunningNode(r) })
+ if cancelled.Nodes["left"].Status != "cancelled" || cancelled.Nodes["right"].Status != "cancelled" {
+ t.Fatalf("cancel left left=%s right=%s", cancelled.Nodes["left"].Status, cancelled.Nodes["right"].Status)
+ }
+ if err := e.Retry(run.ID, "left", "", ""); err != nil {
+ t.Fatal(err)
+ }
+ waitFor(t, s, run.ID, "succeeded")
+ got, _ := s.Run(run.ID)
+ for _, id := range []string{"left", "right", "join"} {
+ if got.Nodes[id].Status != "succeeded" {
+ t.Fatalf("%s = %s", id, got.Nodes[id].Status)
+ }
+ }
+ if n := nodeEvents(got, "start", "node.succeeded"); n != 1 {
+ t.Fatalf("start reran %d times; replaying a branch must not rerun its upstream", n)
+ }
+}
+
+// A run left "running" with a cancelled node (state written before the
+// revive fix, or by another path) settles instead of spinning forever.
+func TestSchedulerSettlesRunWithCancelledNode(t *testing.T) {
+ t.Parallel()
+ wf := domain.WorkflowDefinition{ID: "parallel-cancelled-settle", Name: "parallel cancelled settle", SandboxProfileID: "local",
+ Nodes: []domain.WorkflowNode{scriptNode("left", "true"), scriptNode("right", "true")},
+ }
+ e, s := parallelTestEngine(t, wf)
+ now := time.Now().UTC()
+ run := domain.Run{ID: "run-cancelled-settle", WorkflowID: wf.ID, WorkflowSnapshot: wf, Status: "running", CreatedAt: now, UpdatedAt: now,
+ Nodes: map[string]*domain.NodeRun{
+ "left": {NodeID: "left", Status: "pending"},
+ "right": {NodeID: "right", Status: "cancelled", Attempt: 1},
+ }}
+ if err := s.PutRun(run); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := e.acquireLease(run); err != nil {
+ t.Fatal(err)
+ }
+ e.resume(run.ID)
+ waitFor(t, s, run.ID, "failed")
+ got, _ := s.Run(run.ID)
+ if got.Nodes["left"].Status != "succeeded" || got.Nodes["right"].Status != "cancelled" {
+ t.Fatalf("left=%s right=%s", got.Nodes["left"].Status, got.Nodes["right"].Status)
+ }
+}
diff --git a/internal/engine/parallel_exec.go b/internal/engine/parallel_exec.go
new file mode 100644
index 00000000..59c92384
--- /dev/null
+++ b/internal/engine/parallel_exec.go
@@ -0,0 +1,157 @@
+package engine
+
+import (
+ "context"
+ "sync"
+
+ "agentflow/internal/domain"
+)
+
+// Parallel SOP branches run as independent executeNode goroutines. Nothing
+// serializes a whole run any more; what must not overlap is locked by a
+// narrower key:
+//
+// - one execution per (run, node): a paused-then-resumed run, a rework or a
+// lease takeover may start a node again while its previous execution is
+// still winding down; the new one waits instead of interleaving;
+// - one CLI invocation per (workspace owner, agent): an Agent's control
+// directories (.agentflow/<agent>, config/<agent>, private home, gate MCP
+// config) are keyed by its ID, so two nodes of the same role in one
+// checkout (including a child SOP in its parent's checkout) take turns;
+// - one pool binding per run, so concurrent first nodes pick one machine.
+//
+// Everything else (preparation, checkout, input materialization) stays under
+// runEnvironment.nodeMu, which a node now holds only while preparing.
+
+type keyedLocks struct {
+ mu sync.Mutex
+ locks map[string]*keyedLock
+}
+
+type keyedLock struct {
+ mu sync.Mutex
+ refs int
+}
+
+// lock blocks until key is free and returns its release. Entries are
+// reference counted so finished runs do not leave mutexes behind.
+func (k *keyedLocks) lock(key string) func() {
+ k.mu.Lock()
+ if k.locks == nil {
+ k.locks = map[string]*keyedLock{}
+ }
+ entry := k.locks[key]
+ if entry == nil {
+ entry = &keyedLock{}
+ k.locks[key] = entry
+ }
+ entry.refs++
+ k.mu.Unlock()
+ entry.mu.Lock()
+ var once sync.Once
+ return func() {
+ once.Do(func() {
+ entry.mu.Unlock()
+ k.mu.Lock()
+ entry.refs--
+ if entry.refs == 0 {
+ delete(k.locks, key)
+ }
+ k.mu.Unlock()
+ })
+ }
+}
+
+func (e *Engine) lockNodeExec(runID, nodeID string) func() {
+ return e.execLocks.lock("node\x00" + runID + "\x00" + nodeID)
+}
+
+func (e *Engine) lockAgentInvocation(workspaceOwner, agentID string) func() {
+ return e.execLocks.lock("agent\x00" + workspaceOwner + "\x00" + agentID)
+}
+
+func (e *Engine) lockPoolBinding(runID string) func() {
+ return e.execLocks.lock("pool\x00" + runID)
+}
+
+// nodeExecution is a started node attempt this instance is running. Its
+// cancel stops only that node, so a rework that resets a running sibling
+// does not wait for it to finish on its own.
+type nodeExecution struct {
+ attempt int
+ cancel context.CancelFunc
+}
+
+func (e *Engine) registerNodeExecution(runID, nodeID string, attempt int, cancel context.CancelFunc) func() {
+ e.nodeExecMu.Lock()
+ if e.nodeExecs == nil {
+ e.nodeExecs = map[string]map[string]nodeExecution{}
+ }
+ if e.nodeExecs[runID] == nil {
+ e.nodeExecs[runID] = map[string]nodeExecution{}
+ }
+ e.nodeExecs[runID][nodeID] = nodeExecution{attempt: attempt, cancel: cancel}
+ e.nodeExecMu.Unlock()
+ return func() {
+ e.nodeExecMu.Lock()
+ if current, ok := e.nodeExecs[runID][nodeID]; ok && current.attempt == attempt {
+ delete(e.nodeExecs[runID], nodeID)
+ if len(e.nodeExecs[runID]) == 0 {
+ delete(e.nodeExecs, runID)
+ }
+ }
+ e.nodeExecMu.Unlock()
+ }
+}
+
+// nodeExecutionLive reports whether the persisted node still belongs to the
+// execution that started it: same attempt and still in flight.
+func nodeExecutionLive(nr *domain.NodeRun, attempt int) bool {
+ return nr != nil && nr.Attempt == attempt && (nr.Status == "running" || nr.Status == "pending_input")
+}
+
+// reconcileNodeExecutions cancels executions whose node was reset or
+// re-attempted by a control decision (rework, retry, refresh), possibly made
+// on another instance. Their outcome would be discarded anyway; cancelling
+// frees the node lock so the new attempt can start.
+//
+// The registry is copied before the run is read: an execution registers only
+// after committing its running status, so a state read that follows the copy
+// is never older than any copied registration. Reading first would let a node
+// that started in between look stale and be cancelled with status "running"
+// left behind.
+func (e *Engine) reconcileNodeExecutions(runID string) {
+ e.nodeExecMu.Lock()
+ registered := make(map[string]nodeExecution, len(e.nodeExecs[runID]))
+ for nodeID, execution := range e.nodeExecs[runID] {
+ registered[nodeID] = execution
+ }
+ e.nodeExecMu.Unlock()
+ if len(registered) == 0 {
+ return
+ }
+ r, ok := e.repo.RunState(runID)
+ if !ok {
+ return
+ }
+ for nodeID, execution := range registered {
+ if !nodeExecutionLive(r.Nodes[nodeID], execution.attempt) {
+ execution.cancel()
+ }
+ }
+}
+
+// cancelNodeExecutions stops this instance's executions of the given nodes.
+func (e *Engine) cancelNodeExecutions(runID string, nodeIDs map[string]bool) {
+ e.nodeExecMu.Lock()
+ var stale []context.CancelFunc
+ for nodeID, execution := range e.nodeExecs[runID] {
+ if nodeIDs[nodeID] {
+ stale = append(stale, execution.cancel)
+ }
+ }
+ e.nodeExecMu.Unlock()
+ for _, cancel := range stale {
+ cancel()
+ }
+}
diff --git a/internal/engine/recovery_test.go b/internal/engine/recovery_test.go
index 9f9e30d1..feb7b898 100644
--- a/internal/engine/recovery_test.go
+++ b/internal/engine/recovery_test.go
@@ -50,7 +50,9 @@ func TestResumeInterruptedRunsResumesOrphansAndSparesGatedRuns(t *testing.T) {
// Inspect recovery before the resumed scheduler starts the next node.
// An empty workflow could finish between ResumeInterruptedRuns and Run.
- unlock := e.lockRunExec(orphan.ID)
+ // Node execution is per node now; holding "a" keeps the scheduler from
+ // moving that pending node before the assertions below.
+ unlock := e.lockNodeExec(orphan.ID, "a")
defer unlock()
resumed := e.ResumeInterruptedRuns()
if len(resumed) != 1 || resumed[0] != "run-orphan" {
diff --git a/internal/engine/run_environment.go b/internal/engine/run_environment.go
index 159f8c57..de10891d 100644
--- a/internal/engine/run_environment.go
+++ b/internal/engine/run_environment.go
@@ -15,8 +15,9 @@ import (
)
// runEnvironment owns the single sandbox/workspace assigned to a workflow run.
-// SOP nodes serialize complete executions. Group tasks hold nodeMu only while
-// preparing the shared environment and use independent invocation configuration.
+// SOP nodes and group tasks hold nodeMu only while preparing the shared
+// environment (checkout, preparation scripts, inputs); parallel branches then
+// execute concurrently and are counted in activeExecutions.
type runEnvironment struct {
sandbox runtime.Sandbox
nodeMu sync.Mutex
diff --git a/internal/engine/run_ops.go b/internal/engine/run_ops.go
index 2c374728..c082f1b4 100644
--- a/internal/engine/run_ops.go
+++ b/internal/engine/run_ops.go
@@ -315,9 +315,17 @@ func (e *Engine) Reject(runID string, req domain.RejectRequest, expected ...Node
reset[target] = true
overlayLatestNodeBindings(e.repo, &r, reset)
var stale []string
+ interrupted := map[string]bool{}
for id := range reset {
if nr := r.Nodes[id]; nr != nil {
stale = append(stale, nr.ChildRunID)
+ if nr.Status == "running" || nr.Status == "pending_input" {
+ // A parallel sibling still running inside the reset range: its
+ // attempt is abandoned, and the rerun must count as a new attempt
+ // rather than continue the interrupted one.
+ interrupted[id] = true
+ nr.Activity = ""
+ }
nr.Status = "pending"
nr.Error = ""
nr.Output = ""
@@ -335,7 +343,11 @@ func (e *Engine) Reject(runID string, req domain.RejectRequest, expected ...Node
appendControlTelemetry(&r, req.NodeID, "gate.rejected", fmt.Sprintf("打回到 %s:%s", target, req.Reason))
r.Status = "running"
e.cancelLiveChildRuns(stale)
- return e.persistControlDecision(runID, r, expected...)
+ if err := e.persistControlDecision(runID, r, expected...); err != nil {
+ return err
+ }
+ e.cancelNodeExecutions(runID, interrupted)
+ return nil
}
// Retry reopens a stopped workflow from a selected
@@ -541,6 +553,23 @@ func (e *Engine) retryContext(ctx context.Context, runID, nodeID, reason, humanC
}
e.sandboxes.DiscardRunContainer(context.Background(), runID)
}
+ // Parallel branches stopped together with this one by a cancel are not
+ // failures of their own: resume them as well, or nothing would schedule
+ // them again and the run would never settle. This happens after the
+ // whole-SOP check above so reviving siblings never discards the checkout.
+ var revived []string
+ for _, node := range r.WorkflowSnapshot.Nodes {
+ if reset[node.ID] {
+ continue
+ }
+ if nr := r.Nodes[node.ID]; nr != nil && nr.Status == "cancelled" {
+ revived = append(revived, node.Name)
+ reset[node.ID] = true
+ for id := range descendants(r.WorkflowSnapshot, node.ID) {
+ reset[id] = true
+ }
+ }
+ }
overlayLatestNodeBindings(e.repo, &r, reset)
for _, node := range r.WorkflowSnapshot.Nodes {
if node.ID == nodeID && isScriptNode(node) {
@@ -575,6 +604,9 @@ func (e *Engine) retryContext(ctx context.Context, runID, nodeID, reason, humanC
retryLabel = "从指定节点重跑"
}
e.event(&r, nodeID, "node.retry_requested", retryLabel+":"+note)
+ if len(revived) > 0 {
+ e.event(&r, "", "node.retry_requested", "一并恢复被取消的并行分支:"+strings.Join(revived, "、"))
+ }
appendControlTelemetry(&r, nodeID, "node.retry_requested", retryLabel+":"+strings.TrimSpace(reason))
r.CancelRequested = false
r.PauseRequested = false
diff --git a/internal/engine/run_recovery.go b/internal/engine/run_recovery.go
index 29eae386..5b9a31bf 100644
--- a/internal/engine/run_recovery.go
+++ b/internal/engine/run_recovery.go
@@ -8,7 +8,6 @@ import (
"fmt"
"log"
"strings"
- "sync"
"time"
)
@@ -306,21 +305,6 @@ func (e *Engine) claimRunning(runID string, cancel context.CancelFunc) bool {
return true
}
-func (e *Engine) lockRunExec(runID string) func() {
- e.runEnvMu.Lock()
- if e.runExec == nil {
- e.runExec = map[string]*sync.Mutex{}
- }
- mu := e.runExec[runID]
- if mu == nil {
- mu = &sync.Mutex{}
- e.runExec[runID] = mu
- }
- e.runEnvMu.Unlock()
- mu.Lock()
- return mu.Unlock
-}
-
func (e *Engine) clearRunning(runID string) {
e.mu.Lock()
delete(e.running, runID)
diff --git a/internal/engine/state_helpers_cases_test.go b/internal/engine/state_helpers_cases_test.go
index 9a45cd97..c5e430fb 100644
--- a/internal/engine/state_helpers_cases_test.go
+++ b/internal/engine/state_helpers_cases_test.go
@@ -170,11 +170,11 @@ func TestArtifactFilesValidationCollectionAndRecovery(t *testing.T) {
artifact := newArtifact(run.ID, "up", 1, "note", "note.md", "text/markdown", []byte("body"))
artifact.RepositoryDir = "repo"
- if !strings.Contains(artifactInputRelPath(artifact), "repo") {
- t.Fatal(artifactInputRelPath(artifact))
+ if !strings.Contains(artifactInputRelPath(run, artifact), "repo") {
+ t.Fatal(artifactInputRelPath(run, artifact))
}
var manifest strings.Builder
- writeInputManifestEntry(&manifest, artifact, artifactInputRelPath(artifact))
+ writeInputManifestEntry(&manifest, artifact, artifactInputRelPath(run, artifact))
if !strings.Contains(manifest.String(), artifact.SHA256) {
t.Fatal(manifest.String())
}
diff --git a/internal/engine/workflow_graph.go b/internal/engine/workflow_graph.go
index f001ca9d..2f756334 100644
--- a/internal/engine/workflow_graph.go
+++ b/internal/engine/workflow_graph.go
@@ -133,6 +133,17 @@ func hasWaiting(wf domain.WorkflowDefinition, r domain.Run) bool {
return false
}
+// cancelledNode returns a node stopped by a cancel that nothing will run
+// again unless an operator resumes it.
+func cancelledNode(wf domain.WorkflowDefinition, r domain.Run) (domain.WorkflowNode, bool) {
+ for _, definition := range wf.Nodes {
+ if n := r.Nodes[definition.ID]; n != nil && n.Status == "cancelled" {
+ return definition, true
+ }
+ }
+ return domain.WorkflowNode{}, false
+}
+
func hasFailed(wf domain.WorkflowDefinition, r domain.Run) bool {
for _, definition := range wf.Nodes {
if n := r.Nodes[definition.ID]; n != nil && n.Status == "failed" {
diff --git a/internal/engine/workspace_owner.go b/internal/engine/workspace_owner.go
index 1e1ac63f..5172e7b4 100644
--- a/internal/engine/workspace_owner.go
+++ b/internal/engine/workspace_owner.go
@@ -25,6 +25,18 @@ func sharesWorkspace(r domain.Run) bool {
return workspaceRunID(r) != r.ID
}
+// nodeControlKey names a node's own control paths in the checkout (gate
+// results, materialized inputs, runtime invocation directories). A child SOP
+// runs in its parent's checkout, concurrently with the parent's other
+// branches, and its workflow may reuse the parent's node IDs; its keys carry
+// the child run so neither side clears or reads the other's files.
+func nodeControlKey(r domain.Run, nodeID string) string {
+ if !sharesWorkspace(r) {
+ return nodeID
+ }
+ return r.ID + "--" + nodeID
+}
+
// workspaceBind returns the (host, root) holding r's workspace. A child SOP
// follows the run that owns the checkout once that run has bound one.
func (e *Engine) workspaceBind(r domain.Run) (string, string) {
diff --git a/internal/engine/workspace_preparation.go b/internal/engine/workspace_preparation.go
index d05796a1..9e12310b 100644
--- a/internal/engine/workspace_preparation.go
+++ b/internal/engine/workspace_preparation.go
@@ -430,10 +430,10 @@ func (e *Engine) recordNodeWorkspacePreparation(ctx context.Context, run domain.
}
return nil
}
- if run.IsWorkspace() {
- env.nodeMu.Lock()
- defer env.nodeMu.Unlock()
- }
+ // The receipt is read-modify-written; parallel branches no longer hold
+ // nodeMu while executing, so take it for the record itself.
+ env.nodeMu.Lock()
+ defer env.nodeMu.Unlock()
_, err := runtime.RecordWorkspacePreparation(ctx, env.sandbox, runtime.WorkspacePreparationWorkflow(run.WorkflowSnapshot), run.RepositoryContexts(), node.ID, output)
return err
}
diff --git a/internal/gitwork/patch.go b/internal/gitwork/patch.go
index 08d32c1c..9614a63d 100644
--- a/internal/gitwork/patch.go
+++ b/internal/gitwork/patch.go
@@ -9,6 +9,7 @@ import (
"sort"
"strconv"
"strings"
+ "sync"
)
// MaxPatchBytes bounds one repository-patch inventory; it matches the
@@ -106,6 +107,10 @@ func RecordBaseline(workspace, baseBranch string) error {
// and SOP deliverable markdown are excluded. Untracked source files are
// intent-added so they appear in the list. No recorded baseline yields nil.
func PatchInventory(workspace string) ([]byte, error) {
+ // Parallel branches finishing together inventory the same checkout;
+ // StageUntracked's git add -N would race on .git/index.lock.
+ unlock := lockPatchWorkspace(workspace)
+ defer unlock()
baseline, err := os.ReadFile(BaselinePath(workspace))
if err != nil {
if os.IsNotExist(err) {
@@ -328,6 +333,19 @@ func ParseNameStatusNumstat(nameStatus, numstat []byte) []PatchFileSummary {
return out
}
+var patchWorkspaceLocks sync.Map // cleaned absolute path -> *sync.Mutex
+
+func lockPatchWorkspace(workspace string) func() {
+ key := filepath.Clean(workspace)
+ if abs, err := filepath.Abs(key); err == nil {
+ key = abs
+ }
+ value, _ := patchWorkspaceLocks.LoadOrStore(key, &sync.Mutex{})
+ mu := value.(*sync.Mutex)
+ mu.Lock()
+ return mu.Unlock
+}
+
func isObjectID(s string) bool {
if len(s) != 40 && len(s) != 64 {
return false
diff --git a/internal/observability/hooks.go b/internal/observability/hooks.go
index b4f55884..0eee9a11 100644
--- a/internal/observability/hooks.go
+++ b/internal/observability/hooks.go
@@ -48,7 +48,7 @@ func WriteCodeBuddyHookBridgeAt(workspace, relativeRoot string) error {
return err
}
script := filepath.Join(root, "hook.py")
- if err := os.WriteFile(script, []byte(hookPython), 0o755); err != nil {
+ if err := writeFileIfChanged(script, []byte(hookPython), 0o755); err != nil {
return err
}
cmd := codeBuddyHookCommand(filepath.Join(relativeRoot, "hook.py"))
@@ -68,7 +68,36 @@ func WriteCodeBuddyHookBridgeAt(workspace, relativeRoot string) error {
if err != nil {
return err
}
- return os.WriteFile(filepath.Join(workspace, filepath.Dir(relativeRoot), "codebuddy-settings.json"), raw, 0o600)
+ return writeFileIfChanged(filepath.Join(workspace, filepath.Dir(relativeRoot), "codebuddy-settings.json"), raw, 0o600)
+}
+
+// writeFileIfChanged leaves an identical file untouched and otherwise
+// replaces it through a rename: a CLI of a parallel branch may be executing
+// hook.py or reading its settings at this moment and must never observe a
+// truncated file.
+func writeFileIfChanged(path string, data []byte, mode os.FileMode) error {
+ if info, err := os.Lstat(path); err == nil && info.Mode().IsRegular() && info.Mode().Perm() == mode.Perm() {
+ if existing, err := os.ReadFile(path); err == nil && string(existing) == string(data) {
+ return nil
+ }
+ }
+ temp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".tmp-")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(temp.Name())
+ if _, err := temp.Write(data); err != nil {
+ temp.Close()
+ return err
+ }
+ if err := temp.Chmod(mode); err != nil {
+ temp.Close()
+ return err
+ }
+ if err := temp.Close(); err != nil {
+ return err
+ }
+ return os.Rename(temp.Name(), path)
}
// codeBuddyHookCommand returns a PreToolUse-safe shell command that recreates
diff --git a/internal/runner/outputs.go b/internal/runner/outputs.go
index 28622bad..976ec231 100644
--- a/internal/runner/outputs.go
+++ b/internal/runner/outputs.go
@@ -7,6 +7,7 @@ import (
"path"
"path/filepath"
"strings"
+ "sync"
"agentflow/internal/domain"
"agentflow/internal/gitwork"
@@ -20,6 +21,10 @@ func prepareInvocationWorkspace(workspace string, inv *domain.RunnerInvocation)
if inv == nil {
return nil
}
+ // Parallel branches consuming the same upstream artifact write the same
+ // input file; chmod/write/chmod must not interleave.
+ unlock := lockInvocationWorkspace(workspace)
+ defer unlock()
if _, err := runtime.RestoreMissingWorkspaceArtifacts(workspace, inv.ArchivedFiles); err != nil {
return fmt.Errorf("restore archived outputs: %w", err)
}
@@ -39,6 +44,19 @@ func prepareInvocationWorkspace(workspace string, inv *domain.RunnerInvocation)
return nil
}
+var invocationWorkspaceLocks sync.Map // cleaned absolute workspace -> *sync.Mutex
+
+func lockInvocationWorkspace(workspace string) func() {
+ key := filepath.Clean(workspace)
+ if abs, err := filepath.Abs(key); err == nil {
+ key = abs
+ }
+ value, _ := invocationWorkspaceLocks.LoadOrStore(key, &sync.Mutex{})
+ mu := value.(*sync.Mutex)
+ mu.Lock()
+ return mu.Unlock
+}
+
func writeInputFile(workspace string, input domain.RunnerInputFile) error {
if !domain.SafeWorkspaceRelPath(input.Path) || !strings.HasPrefix(input.Path, ".agentflow/inputs/") {
return fmt.Errorf("unsafe input path %q", input.Path)
diff --git a/internal/runtime/git_isolation.go b/internal/runtime/git_isolation.go
index da2a393e..18799872 100644
--- a/internal/runtime/git_isolation.go
+++ b/internal/runtime/git_isolation.go
@@ -11,6 +11,7 @@ import (
stdruntime "runtime"
"strconv"
"strings"
+ "sync"
)
const projectAgentGitExcludeBlock = `# AgentFlow managed sandbox: repository Agent configuration
@@ -73,7 +74,13 @@ func EnsureAgentFlowGitExcludes(workspace string) error {
return appendGitExcludeBlock(workspace, agentFlowGitExcludeBlock, "AgentFlow runtime excludes")
}
+var gitExcludeMu sync.Mutex
+
func appendGitExcludeBlock(workspace, block, label string) error {
+ // Parallel branches prepare the same checkout; check-then-append must not
+ // interleave or the block is appended twice.
+ gitExcludeMu.Lock()
+ defer gitExcludeMu.Unlock()
root, excludePath, err := openWorkspacePath(workspace, filepath.Join(workspace, ".git", "info", "exclude"))
if err != nil {
return err
diff --git a/internal/runtime/hooks_test.go b/internal/runtime/hooks_test.go
index 8ed24687..4090fc6d 100644
--- a/internal/runtime/hooks_test.go
+++ b/internal/runtime/hooks_test.go
@@ -64,7 +64,7 @@ func TestAgentHooksCompileForAllCLIsAndStayInvocationScoped(t *testing.T) {
t.Fatalf("Claude settings not attached: %v", compiled.Args)
}
case "codebuddy":
- settings := filepath.Join(workspace, ".agentflow", "hooks", "codebuddy-settings.json")
+ settings := filepath.Join(workspace, ".agentflow", "config", "reviewer", "codebuddy-settings.json")
assertHookSettings(t, settings, "PreToolUse", "python3 '.agentflow/config/reviewer/hooks/audit-agentflow.py'")
case "codex":
joined := strings.Join(compiled.Args, " ")
diff --git a/internal/runtime/materialize.go b/internal/runtime/materialize.go
index eb007b6a..ef20eb37 100644
--- a/internal/runtime/materialize.go
+++ b/internal/runtime/materialize.go
@@ -40,6 +40,11 @@ func materializeInvocationForDriver(workspace, invocationID string, ex domain.Ex
if invocationID == "" {
return compiledConfig{}, fmt.Errorf("empty invocation id")
}
+ // Parallel branches materialize different Agents into one checkout; the
+ // shared rules/skills/mcp catalogs and git excludes are written one
+ // invocation at a time.
+ unlock := lockWorkspace(workspace)
+ defer unlock()
agentID := safeID(a.ID)
if agentID == "" {
return compiledConfig{}, fmt.Errorf("agent id is required for runtime isolation")
@@ -150,14 +155,15 @@ func materializeInvocationForDriver(workspace, invocationID string, ex domain.Ex
}
var skillPaths []string
for _, item := range selectedSkills {
- dir, err := resetControlDirectory(workspace, "skills", item.id)
- if err != nil {
+ // Another Agent running in parallel may be reading this skill; an
+ // unchanged skill is left in place instead of being recreated.
+ item := item
+ if _, err := refreshSharedControlDirectory(workspace, func(dir string) error {
+ return writeSkillFiles(dir, item, assets.SkillFiles[item.skill.ID])
+ }, "skills", item.id); err != nil {
return compiled, err
}
skillPaths = append(skillPaths, item.root)
- if err := writeSkillFiles(dir, item, assets.SkillFiles[item.skill.ID]); err != nil {
- return compiled, err
- }
if err := replaceRelativeSymlink(filepath.Join(bindingRoot, "skills", item.id), filepath.Join("..", "..", "..", "skills", item.id)); err != nil {
return compiled, err
}
diff --git a/internal/runtime/provider_codebuddy.go b/internal/runtime/provider_codebuddy.go
index 02d1958e..3d49264c 100644
--- a/internal/runtime/provider_codebuddy.go
+++ b/internal/runtime/provider_codebuddy.go
@@ -16,22 +16,31 @@ func (codeBuddyProvider) Materialize(ctx *providerMaterializeContext) error {
if err := materializeClaudePlugin(ctx); err != nil {
return err
}
- if _, err := resetControlDirectory(ctx.Workspace, "hooks", "codebuddy"); err != nil {
+ // The hook bridge is static and shared by every CodeBuddy Agent in the
+ // checkout, and a parallel branch may be running it: keep the directory
+ // and rewrite only changed files. The settings, which merge this Agent's
+ // own hooks, are per Agent so parallel roles do not overwrite each other.
+ if _, err := EnsureControlDirectory(ctx.Workspace, "hooks", "codebuddy"); err != nil {
return err
}
- if err := writeManagedFile(ctx.Workspace, filepath.Join(ctx.Workspace, ".agentflow", "hooks", "codebuddy-settings.json"), nil, 0o600); err != nil {
+ if err := observability.WriteCodeBuddyHookBridgeAt(ctx.Workspace, filepath.Join(".agentflow", "hooks", "codebuddy")); err != nil {
return err
}
- if err := observability.WriteCodeBuddyHookBridgeAt(ctx.Workspace, filepath.Join(".agentflow", "hooks", "codebuddy")); err != nil {
+ base, err := readWorkspaceFile(ctx.Workspace, filepath.Join(ctx.Workspace, ".agentflow", "hooks", "codebuddy-settings.json"))
+ if err != nil {
+ return err
+ }
+ settingsPath := filepath.Join(ctx.ConfigRoot, "codebuddy-settings.json")
+ if err := writeManagedFile(ctx.Workspace, settingsPath, base, 0o600); err != nil {
return err
}
- if err := mergeHookSettings(ctx.Workspace, filepath.Join(ctx.Workspace, ".agentflow", "hooks", "codebuddy-settings.json"), ctx.Compiled.Hooks); err != nil {
+ if err := mergeHookSettings(ctx.Workspace, settingsPath, ctx.Compiled.Hooks); err != nil {
return err
}
if err := replaceRelativeSymlink(filepath.Join(ctx.AgentRoot, "hooks"), filepath.Join("..", "hooks", "codebuddy")); err != nil {
return err
}
- if err := replaceRelativeSymlink(filepath.Join(ctx.AgentRoot, "codebuddy-settings.json"), filepath.Join("..", "hooks", "codebuddy-settings.json")); err != nil {
+ if err := replaceRelativeSymlink(filepath.Join(ctx.AgentRoot, "codebuddy-settings.json"), filepath.Join("..", "config", ctx.AgentID, "codebuddy-settings.json")); err != nil {
return err
}
ctx.Compiled.Args = append(ctx.Compiled.Args,
diff --git a/internal/runtime/sandbox_docker.go b/internal/runtime/sandbox_docker.go
index 8eadd7ff..308206bc 100644
--- a/internal/runtime/sandbox_docker.go
+++ b/internal/runtime/sandbox_docker.go
@@ -27,6 +27,10 @@ type dockerSandbox struct {
mu sync.Mutex
browserMu sync.Mutex
started bool
+ // execGate lets parallel branches exec concurrently (read side) while a
+ // cwd-recovery Recreate (write side) waits for them instead of docker
+ // rm-ing the container under a sibling's running command.
+ execGate sync.RWMutex
}
func (s *dockerSandbox) ID() string { return s.id }
@@ -111,12 +115,19 @@ func (s *dockerSandbox) Exec(ctx context.Context, command CommandSpec, sink Even
if err := s.ensureStarted(ctx); err != nil {
return "", err
}
+ s.execGate.RLock()
out, err := s.execOnce(ctx, command, sink)
+ s.execGate.RUnlock()
if err != nil && isDockerCwdUnsafe(out, err) {
- if recErr := s.Recreate(ctx); recErr != nil {
+ s.execGate.Lock()
+ recErr := s.Recreate(ctx)
+ s.execGate.Unlock()
+ if recErr != nil {
return out, fmt.Errorf("current working directory is outside of container mount namespace: %w", err)
}
+ s.execGate.RLock()
out, err = s.execOnce(ctx, command, sink)
+ s.execGate.RUnlock()
if err != nil && isDockerCwdUnsafe(out, err) {
return out, fmt.Errorf("current working directory is outside of container mount namespace: %w", err)
}
diff --git a/internal/runtime/sandbox_manager.go b/internal/runtime/sandbox_manager.go
index 0653250c..62058b7e 100644
--- a/internal/runtime/sandbox_manager.go
+++ b/internal/runtime/sandbox_manager.go
@@ -422,12 +422,14 @@ func (s *managedSandbox) Exec(ctx context.Context, command CommandSpec, sink Eve
return "", ctx.Err()
}
defer func() { <-s.slots }()
- s.execMu.Lock()
- defer s.execMu.Unlock()
s.credentialsMu.RLock()
credentials := s.credentials
s.credentialsMu.RUnlock()
+ // Binding writes the credential helpers into the workspace; parallel
+ // branches then run their commands concurrently.
+ s.execMu.Lock()
bound, err := bindCommand(s.Sandbox, s.workspace, command, credentials)
+ s.execMu.Unlock()
if err != nil {
return "", err
}
diff --git a/internal/runtime/workspace_locks.go b/internal/runtime/workspace_locks.go
new file mode 100644
index 00000000..d6bbfccb
--- /dev/null
+++ b/internal/runtime/workspace_locks.go
@@ -0,0 +1,155 @@
+package runtime
+
+import (
+ "bytes"
+ "crypto/rand"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "sync"
+)
+
+// Parallel SOP branches materialize and execute in one shared checkout. The
+// helpers here keep the few workspace-wide control writes from interleaving;
+// per-Agent directories are serialized by the engine (one invocation per
+// Agent and checkout), so only state shared across Agents needs care.
+
+var workspaceLocks sync.Map // cleaned workspace path -> *sync.Mutex
+
+func lockWorkspace(workspace string) func() {
+ key := filepath.Clean(workspace)
+ if abs, err := filepath.Abs(key); err == nil {
+ key = abs
+ }
+ value, _ := workspaceLocks.LoadOrStore(key, &sync.Mutex{})
+ mu := value.(*sync.Mutex)
+ mu.Lock()
+ return mu.Unlock
+}
+
+// refreshSharedControlDirectory rebuilds a control directory that concurrent
+// invocations of different Agents read (skills/<id>). The new content is built
+// beside the live directory and swapped in only when it differs, so a running
+// CLI never sees an unchanged skill disappear underneath it.
+func refreshSharedControlDirectory(workspace string, build func(dir string) error, elements ...string) (string, error) {
+ if len(elements) == 0 {
+ return "", fmt.Errorf("missing AgentFlow control directory")
+ }
+ name := elements[len(elements)-1]
+ if name == "" || name == "." || name == ".." || filepath.Base(name) != name {
+ return "", fmt.Errorf("invalid AgentFlow control path segment %q", name)
+ }
+ parent, err := EnsureControlDirectory(workspace, elements[:len(elements)-1]...)
+ if err != nil {
+ return "", err
+ }
+ target := filepath.Join(parent, name)
+ if info, err := os.Lstat(target); err == nil && !info.IsDir() {
+ return "", fmt.Errorf("managed path is not a real directory: %s", target)
+ } else if err != nil && !os.IsNotExist(err) {
+ return "", err
+ }
+ staging := filepath.Join(parent, "."+name+".new-"+rand.Text())
+ if err := os.Mkdir(staging, 0o700); err != nil {
+ return "", err
+ }
+ defer os.RemoveAll(staging)
+ if err := build(staging); err != nil {
+ return "", err
+ }
+ if same, err := sameDirectoryTree(staging, target); err == nil && same {
+ return target, nil
+ }
+ retired := ""
+ if _, err := os.Lstat(target); err == nil {
+ retired = filepath.Join(parent, "."+name+".old-"+rand.Text())
+ if err := os.Rename(target, retired); err != nil {
+ return "", err
+ }
+ }
+ if err := os.Rename(staging, target); err != nil {
+ if retired != "" {
+ _ = os.Rename(retired, target)
+ }
+ return "", err
+ }
+ if retired != "" {
+ _ = os.RemoveAll(retired)
+ }
+ return target, nil
+}
+
+// sameDirectoryTree compares names, kinds, permission bits, file contents and
+// symlink targets of two trees.
+func sameDirectoryTree(a, b string) (bool, error) {
+ collect := func(root string) (map[string]fs.FileInfo, error) {
+ entries := map[string]fs.FileInfo{}
+ err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ rel, err := filepath.Rel(root, path)
+ if err != nil {
+ return err
+ }
+ info, err := os.Lstat(path)
+ if err != nil {
+ return err
+ }
+ entries[rel] = info
+ return nil
+ })
+ return entries, err
+ }
+ left, err := collect(a)
+ if err != nil {
+ return false, err
+ }
+ right, err := collect(b)
+ if err != nil {
+ return false, err
+ }
+ if len(left) != len(right) {
+ return false, nil
+ }
+ for rel, li := range left {
+ ri, ok := right[rel]
+ if !ok || li.Mode().Type() != ri.Mode().Type() {
+ return false, nil
+ }
+ if rel != "." && li.Mode().Perm() != ri.Mode().Perm() {
+ return false, nil
+ }
+ switch {
+ case li.Mode().IsRegular():
+ if li.Size() != ri.Size() {
+ return false, nil
+ }
+ lb, err := os.ReadFile(filepath.Join(a, rel))
+ if err != nil {
+ return false, err
+ }
+ rb, err := os.ReadFile(filepath.Join(b, rel))
+ if err != nil {
+ return false, err
+ }
+ if !bytes.Equal(lb, rb) {
+ return false, nil
+ }
+ case li.Mode()&os.ModeSymlink != 0:
+ lt, err := os.Readlink(filepath.Join(a, rel))
+ if err != nil {
+ return false, err
+ }
+ rt, err := os.Readlink(filepath.Join(b, rel))
+ if err != nil {
+ return false, err
+ }
+ if lt != rt {
+ return false, nil
+ }
+ }
+ }
+ return true, nil
+}
diff --git a/internal/runtime/workspace_locks_test.go b/internal/runtime/workspace_locks_test.go
new file mode 100644
index 00000000..0b77785d
--- /dev/null
+++ b/internal/runtime/workspace_locks_test.go
@@ -0,0 +1,49 @@
+package runtime
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestRefreshSharedControlDirectoryKeepsUnchangedTree(t *testing.T) {
+ workspace := t.TempDir()
+ write := func(body string) func(string) error {
+ return func(dir string) error {
+ return os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte(body), 0o644)
+ }
+ }
+ target, err := refreshSharedControlDirectory(workspace, write("v1"), "skills", "s1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ before, err := os.Stat(target)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // A parallel Agent materializing the same skill must not recreate it.
+ if _, err := refreshSharedControlDirectory(workspace, write("v1"), "skills", "s1"); err != nil {
+ t.Fatal(err)
+ }
+ after, err := os.Stat(target)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !os.SameFile(before, after) {
+ t.Fatal("unchanged skill directory was recreated")
+ }
+ if _, err := refreshSharedControlDirectory(workspace, write("v2"), "skills", "s1"); err != nil {
+ t.Fatal(err)
+ }
+ raw, err := os.ReadFile(filepath.Join(target, "SKILL.md"))
+ if err != nil || string(raw) != "v2" {
+ t.Fatalf("changed skill not swapped in: %q %v", raw, err)
+ }
+ entries, err := os.ReadDir(filepath.Dir(target))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(entries) != 1 {
+ t.Fatalf("staging directories left behind: %v", entries)
+ }
+}