Skip to content

"Vibe-Trading: Your Personal Trading Agent"

feat(swarm): replay failed/cancelled runs keeping completed task artifacts

cgycorey09f34004ef29
diff --git a/agent/mcp_server.py b/agent/mcp_server.py
index cab8473a7..c423083d1 100644
--- a/agent/mcp_server.py
+++ b/agent/mcp_server.py
@@ -2684,7 +2684,7 @@ def reap_stale_runs() -> str:
@mcp.tool
-def retry_run(run_id: str) -> str:
+def retry_run(run_id: str, resume: bool = False) -> str:
"""Retry a failed, stale, or cancelled swarm run.
Re-launches a brand-new run with the same preset and variables as the
@@ -2692,8 +2692,17 @@ def retry_run(run_id: str) -> str:
spotting a ``failed`` or stale run via ``list_runs``. A still-``running``
run cannot be retried — cancel or reap it first.
+ With ``resume=True`` (replay), completed upstream tasks and their artifacts
+ are carried into the new run as-is and only the failed/cancelled subgraph
+ re-executes — completed independent branches are not re-run. ``resume``
+ only applies to a ``failed`` or ``cancelled`` run; ``resume=True`` for any
+ other status is refused with an error (a plain retry without ``resume``
+ remains available for any non-running run).
+
Args:
run_id: ID of the run to retry (from ``list_runs`` / ``get_swarm_status``).
+ resume: Replay the failed/cancelled subgraph instead of re-running the
+ whole preset. Default ``False`` preserves the existing full re-run.
Returns:
JSON payload for the newly created run (``run_id`` / ``status`` /
@@ -2719,6 +2728,17 @@ def retry_run(run_id: str) -> str:
{"status": "error", "error": "Cannot retry a running run. Cancel or reap it first."},
ensure_ascii=False,
)
+ if resume and reconciled.status not in (RunStatus.failed, RunStatus.cancelled):
+ return json.dumps(
+ {
+ "status": "error",
+ "error": (
+ f"Cannot resume a run in status '{reconciled.status.value}'; "
+ "resume only applies to failed or cancelled runs."
+ ),
+ },
+ ensure_ascii=False,
+ )
agent_config = load_swarm_agent_config()
runtime = SwarmRuntime(store=store, agent_config=agent_config)
@@ -2727,6 +2747,7 @@ def retry_run(run_id: str) -> str:
reconciled.preset_name,
reconciled.user_vars or {},
include_shell_tools=_include_shell_tools,
+ resume_from=reconciled if resume else None,
)
except FileNotFoundError as exc:
return json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False)
diff --git a/agent/src/api/swarm_routes.py b/agent/src/api/swarm_routes.py
index ce65d611e..932813c8d 100644
--- a/agent/src/api/swarm_routes.py
+++ b/agent/src/api/swarm_routes.py
@@ -221,10 +221,13 @@ def register_swarm_routes(
return {"status": "cancelled"}
@app.post("/swarm/runs/{run_id}/retry", dependencies=[Depends(require_auth)])
- async def retry_swarm_run(run_id: str, http_request: Request):
+ async def retry_swarm_run(run_id: str, http_request: Request, resume: bool = Query(False)):
"""Retry a failed, stale, or cancelled swarm run.
Creates a new run with the same preset and user_vars as the original.
+ ``resume=true`` replays: completed upstream tasks and their artifacts
+ are carried into the new run and only the failed/cancelled subgraph
+ re-executes.
"""
_host_validate_path_param(run_id, "run_id")
runtime = _get_swarm_runtime()
@@ -241,12 +244,21 @@ def register_swarm_routes(
raise HTTPException(
status_code=409, detail="Cannot retry a running run. Cancel it first."
)
+ if resume and reconciled.status not in (RunStatus.failed, RunStatus.cancelled):
+ raise HTTPException(
+ status_code=409,
+ detail=(
+ f"Cannot resume a run in status '{reconciled.status.value}'; "
+ "resume only applies to failed or cancelled runs."
+ ),
+ )
try:
new_run = runtime.start_run(
reconciled.preset_name,
reconciled.user_vars or {},
include_shell_tools=_host_shell_tools_enabled_for_request(http_request),
+ resume_from=reconciled if resume else None,
)
return {
"id": new_run.id,
diff --git a/agent/src/swarm/runtime.py b/agent/src/swarm/runtime.py
index 4189f75e3..21c7593be 100644
--- a/agent/src/swarm/runtime.py
+++ b/agent/src/swarm/runtime.py
@@ -8,6 +8,7 @@ with cancellation and event callback support.
from __future__ import annotations
import logging
+import shutil
import threading
from concurrent.futures import (
Future,
@@ -50,6 +51,159 @@ from src.swarm.worker import agent_artifact_dir, clear_agent_artifacts, run_work
logger = logging.getLogger(__name__)
+def _rehome_artifact_paths(
+ artifact_paths: list[str],
+ source_run_dir: Path,
+ target_run_dir: Path,
+) -> list[str]:
+ """Copy a kept task's run-relative artifacts into *target_run_dir*.
+
+ Workers persist artifacts as POSIX paths relative to the run directory
+ (``artifacts/<agent>/<file>``; see ``worker._collect_artifacts``). A
+ resumed run must not reference the old run's directory — the old run
+ stays as a record and may be deleted later.
+
+ The stored artifact list is on-disk state that may be stale or tampered,
+ so every entry is validated before use:
+ - Only paths under the ``artifacts/`` subtree are accepted, resolved to a
+ real path on BOTH sides: ``artifacts/../run.json`` normalizes into the
+ run root (control-file clobber) and a symlinked component such as
+ ``artifacts/analyst -> tasks`` redirects the copy onto task files. The
+ resolved source and resolved destination must each land inside BOTH the
+ run directory and its ``artifacts/`` subtree — the run-root bound stops a
+ symlinked ``artifacts/`` dir (e.g. ``artifacts -> /outside``) from
+ redefining the safe boundary, the subtree bound stops redirection onto
+ control/task files. Anything else (``run.json``, ``events.jsonl``,
+ ``tasks/*.json``, absolute paths, ``..``, symlink redirection) is a
+ control-file or path-traversal vector and is dropped — re-homing a kept
+ artifact must never clobber the new run's own state files or write
+ outside the run.
+ - Missing files are dropped rather than re-homed.
+
+ Returns:
+ New run-relative artifact paths, preserving the input order.
+ """
+ rehomed: list[str] = []
+ source_root = source_run_dir.resolve()
+ target_root = target_run_dir.resolve()
+ source_artifacts = (source_root / "artifacts").resolve()
+ target_artifacts = (target_root / "artifacts").resolve()
+ for raw in artifact_paths or []:
+ rel = Path(raw)
+ parts = rel.parts
+ if rel.is_absolute() or not parts or len(parts) < 2 or parts[0] != "artifacts":
+ logger.warning("Resume: dropping non-artifact path %r", raw)
+ continue
+ if ".." in parts:
+ logger.warning(
+ "Resume: dropping artifact path escaping artifacts/: %r", raw
+ )
+ continue
+ src = source_run_dir / rel
+ try:
+ resolved_src = src.resolve()
+ if not (
+ resolved_src.is_relative_to(source_root)
+ and resolved_src.is_relative_to(source_artifacts)
+ ):
+ logger.warning(
+ "Resume: dropping artifact escaping source run: %r", raw
+ )
+ continue
+ except (OSError, RuntimeError, ValueError):
+ continue
+ if not src.is_file():
+ continue
+ dst = target_run_dir / rel
+ try:
+ # Validate containment BEFORE mkdir: creating parent dirs through a
+ # symlinked component (e.g. artifacts -> /outside) would already
+ # write outside the run dir.
+ resolved_dst = dst.resolve(strict=False)
+ if not (
+ resolved_dst.is_relative_to(target_root)
+ and resolved_dst.is_relative_to(target_artifacts)
+ ):
+ logger.warning(
+ "Resume: dropping artifact whose destination escapes the run dir: %r",
+ raw,
+ )
+ continue
+ dst.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(src, dst)
+ except OSError:
+ logger.warning("Resume: failed to re-home artifact %s", rel, exc_info=True)
+ continue
+ rehomed.append(rel.as_posix())
+ return rehomed
+
+
+def _agent_spec_changed(prev: SwarmAgentSpec, new: SwarmAgentSpec) -> bool:
+ """Return whether an agent's defining configuration differs between runs.
+
+ Only the fields that determine what/with-what the agent produces are
+ compared: role, system prompt, tool/skill whitelists, model choice,
+ retry budget, and timeout. A kept task result is only valid if the agent
+ that produced it still means the same thing.
+ """
+ return (
+ prev.role != new.role
+ or prev.system_prompt != new.system_prompt
+ or prev.tools != new.tools
+ or prev.skills != new.skills
+ or prev.model_name != new.model_name
+ or prev.max_retries != new.max_retries
+ or prev.max_iterations != new.max_iterations
+ or prev.timeout_seconds != new.timeout_seconds
+ )
+
+
+def _task_definition_changed(
+ prev: SwarmTask,
+ new: SwarmTask,
+ prev_agents: dict[str, SwarmAgentSpec],
+ new_agents: dict[str, SwarmAgentSpec],
+ prev_run: SwarmRun,
+ new_run: SwarmRun,
+) -> bool:
+ """Return whether a task's semantic definition differs between runs.
+
+ Compares both the fields that define *what the task is* — the agent that
+ executes it, the prompt template, and the DAG wiring (``depends_on`` and
+ ``input_from``) — and the agent spec that executes it (see
+ ``_agent_spec_changed``). Runtime state (status, summary, blocked_by,
+ completed_at, …) is deliberately excluded.
+
+ When neither the old nor the new agent pins ``model_name`` (bundled presets
+ leave it unset), the task runs on the run-level default captured in
+ ``SwarmRun.model`` / ``.provider`` — so a global model/provider swap
+ between the original run and the replay also invalidates kept results.
+ Per-agent overrides are compared by ``_agent_spec_changed`` already.
+
+ A replayed run must keep a completed task only when the task, its agent,
+ and the effective model that produced it still mean the same thing —
+ otherwise the kept result is stale with respect to the current run.
+ """
+ if (
+ prev.agent_id != new.agent_id
+ or prev.prompt_template != new.prompt_template
+ or prev.depends_on != new.depends_on
+ or prev.input_from != new.input_from
+ ):
+ return True
+ prev_agent = prev_agents.get(prev.agent_id)
+ new_agent = new_agents.get(new.agent_id)
+ if prev_agent is None or new_agent is None:
+ # Agent vanished from or appeared in the preset — treat as changed.
+ return prev_agent is not new_agent
+ if _agent_spec_changed(prev_agent, new_agent):
+ return True
+ if prev_agent.model_name is None and new_agent.model_name is None:
+ # Both defer to the run-level default model/provider.
+ return prev_run.model != new_run.model or prev_run.provider != new_run.provider
+ return False
+
+
class SwarmRuntime:
"""Swarm DAG orchestration engine.
@@ -93,6 +247,7 @@ class SwarmRuntime:
user_vars: dict[str, str],
live_callback: Callable | None = None,
include_shell_tools: bool = False,
+ resume_from: SwarmRun | None = None,
) -> SwarmRun:
"""Start a swarm run. Returns immediately, execution happens in background.
@@ -101,6 +256,10 @@ class SwarmRuntime:
user_vars: User-provided variables for prompt templates.
live_callback: Optional callback invoked for each event in real-time.
include_shell_tools: Whether workers may register shell tools.
+ resume_from: Optional prior ``failed`` / ``cancelled`` run to resume.
+ Its completed tasks (and their artifacts) are carried into the
+ new run as-is; every other task re-executes. ``None`` (default)
+ starts a fully fresh run.
Returns:
The created SwarmRun instance (status=pending initially).
@@ -150,7 +309,7 @@ class SwarmRuntime:
thread = threading.Thread(
target=self._execute_run,
- args=(run, cancel_event, include_shell_tools),
+ args=(run, cancel_event, include_shell_tools, resume_from),
name=f"swarm-{run.id}",
daemon=True,
)
@@ -224,6 +383,7 @@ class SwarmRuntime:
run: SwarmRun,
cancel_event: threading.Event,
include_shell_tools: bool = False,
+ resume_from: SwarmRun | None = None,
) -> None:
"""Core orchestration loop (runs in background thread).
@@ -241,6 +401,8 @@ class SwarmRuntime:
run: SwarmRun to execute.
cancel_event: Threading event for cancellation signalling.
include_shell_tools: Whether workers may register shell tools.
+ resume_from: Optional prior run whose completed tasks are kept
+ (see :meth:`_apply_resume_overlay`). ``None`` executes fresh.
"""
run_id = run.id
run_dir = self._store.run_dir(run_id)
@@ -275,6 +437,17 @@ class SwarmRuntime:
all_succeeded = True
try:
+ # The overlay runs inside the try so a failure here finalizes the
+ # run (run_error event + failed status) instead of leaving a
+ # zombie "running" run on disk. run.tasks is re-synced from the
+ # store afterwards: _cancel_remaining_tasks and the final snapshot
+ # read in-memory task state, which must see kept tasks as
+ # completed — otherwise a cancellation in the pre-layer window
+ # would relabel them cancelled.
+ if resume_from is not None:
+ self._apply_resume_overlay(run, run_dir, task_store, resume_from, task_summaries)
+ run.tasks = task_store.load_all()
+
for layer_idx, layer_task_ids in enumerate(layers):
# Check cancellation between layers
if cancel_event.is_set():
@@ -362,10 +535,15 @@ class SwarmRuntime:
# therefore not present in layer_results — they were already
# marked TaskStatus.blocked in _execute_layer and emitted
# task_blocked. Account for them in run-level status so the
- # run is marked failed, not silently completed.
+ # run is marked failed, not silently completed. Tasks kept
+ # completed by a resume overlay are equally absent from
+ # layer_results but are not failures.
for tid in layer_task_ids:
- if tid not in layer_results:
- all_succeeded = False
+ if tid in layer_results:
+ continue
+ if task_store.load_task(tid).status == TaskStatus.completed:
+ continue
+ all_succeeded = False
# Snapshot run.json at the layer boundary so list_runs and any
# client that reads run.json directly sees fresh task statuses
@@ -420,6 +598,143 @@ class SwarmRuntime:
except Exception:
logger.warning("Layer-boundary run.json sync failed", exc_info=True)
+ def _apply_resume_overlay(
+ self,
+ run: SwarmRun,
+ run_dir: Path,
+ task_store: TaskStore,
+ resume_from: SwarmRun,
+ task_summaries: dict[str, str],
+ ) -> None:
+ """Carry completed tasks from *resume_from* into *run* before execution.
+
+ DAG semantics guarantee a completed task never depends on a failed one,
+ so "keep everything completed, reset everything else" is the core of
+ the failed/cancelled-subgraph re-execution #1157 asks for. Two
+ exceptions, both enforced here:
+ - A completed task is NOT kept when its definition changed since the
+ original run — the task's own defining fields (agent, prompt
+ template, DAG wiring) or the agent spec that executes it (role,
+ system prompt, tools, skills, model, retries, timeout) — because a
+ kept result must still mean the same thing.
+ - A completed task is NOT kept when any of its transitive upstreams
+ re-executes, because its kept summary was produced from the upstream's
+ old output and would be stale relative to the fresh upstream output
+ flowing into it (and unchanged dependents cascade the same way).
+ Kept tasks are re-marked ``completed`` with their original
+ summary/artifacts (artifacts re-homed into this run's directory so the
+ new run is self-contained), kept ids are resolved out of downstream
+ ``blocked_by`` edges, and their summaries are seeded into
+ ``task_summaries`` so ``input_from`` context flows to re-executed
+ dependents. Tasks with no counterpart in the original run (added to the
+ preset since) run fresh.
+
+ Degradation: a wholly missing or unreadable prior task store falls back
+ to a fully fresh run (logged, never fatal). Individual tasks absent
+ from an otherwise-readable store (e.g. preset evolution that removed
+ them) simply have no kept counterpart and re-execute — that is not a
+ store failure, so it does not trigger the fresh-run fallback.
+ """
+ prev_run_dir = self._store.run_dir(resume_from.id)
+ if not prev_run_dir.exists():
+ logger.warning(
+ "Resume: run %s directory missing — replaying fresh",
+ resume_from.id,
+ )
+ return
+ try:
+ prev_store = TaskStore(prev_run_dir)
+ prev_by_id = {task.id: task for task in prev_store.load_all()}
+ except Exception:
+ logger.warning(
+ "Resume: could not read task store of run %s — replaying fresh",
+ resume_from.id,
+ exc_info=True,
+ )
+ return
+
+ prev_agents = {a.id: a for a in resume_from.agents}
+ new_agents = {a.id: a for a in run.agents}
+
+ # Phase 1: candidates = completed tasks whose own definition is unchanged.
+ to_keep: set[str] = set()
+ for task in run.tasks:
+ prev = prev_by_id.get(task.id)
+ if prev is None or prev.status != TaskStatus.completed:
+ continue
+ if _task_definition_changed(prev, task, prev_agents, new_agents, resume_from, run):
+ logger.info(
+ "Resume: task %s definition changed since run %s — re-executing",
+ task.id,
+ resume_from.id,
+ )
+ continue
+ to_keep.add(task.id)
+
+ # Phase 2: a kept result is only valid if every transitive upstream is
+ # also kept — a dependent whose upstream re-executes must itself
+ # re-execute, or its kept summary would be stale with respect to the
+ # fresh upstream output flowing into it. Iterate to fixpoint (DAGs are
+ # small; worst case is quadratic).
+ invalidated = True
+ while invalidated:
+ invalidated = False
+ for task in run.tasks:
+ if task.id not in to_keep:
+ continue
+ for dep in task.depends_on:
+ if dep not in to_keep:
+ to_keep.discard(task.id)
+ logger.info(
+ "Resume: task %s invalidated — upstream %s re-executes",
+ task.id,
+ dep,
+ )
+ invalidated = True
+ break
+
+ # Phase 3: apply the keep for the survivors.
+ kept = 0
+ for task in run.tasks:
+ if task.id not in to_keep:
+ continue
+ prev = prev_by_id[task.id]
+ artifact_paths = _rehome_artifact_paths(prev.artifacts, prev_run_dir, run_dir)
+ task_store.update_status(
+ task.id,
+ TaskStatus.completed,
+ summary=prev.summary,
+ completed_at=prev.completed_at,
+ artifacts=artifact_paths,
+ worker_iterations=prev.worker_iterations,
+ blocked_by=[],
+ )
+ resolve_dependencies(run_dir / "tasks", task.id)
+ # Mirror _execute_run's live seeding (unconditional key): a kept
+ # task must contribute its input_from key to re-executed dependents
+ # even when the stored summary is empty or None.
+ task_summaries[task.id] = prev.summary or ""
+ kept += 1
+ self._emit_event(
+ run.id,
+ self._make_event(
+ "task_resumed",
+ agent_id=task.agent_id,
+ task_id=task.id,
+ data={
+ "source_run_id": resume_from.id,
+ "completed_at": prev.completed_at,
+ },
+ ),
+ )
+ if kept:
+ logger.info(
+ "Run %s resumed from %s: kept %d completed task(s)",
+ run.id,
+ resume_from.id,
+ kept,
+ )
+
def _prefetch_grounding_data(self, run: SwarmRun) -> None:
"""Fetch run-level grounding data without blocking ``start_run``."""
symbols = grounding.extract_symbols_from_user_vars(run.user_vars)
@@ -520,6 +835,13 @@ class SwarmRuntime:
for tid in layer_task_ids:
task = task_store.load_task(tid)
+ # Replay support: a task kept completed by the resume overlay
+ # must not be re-dispatched — its summary and artifacts were
+ # carried into this run and its summary seeded into
+ # task_summaries before layers started.
+ if task.status == TaskStatus.completed:
+ continue
+
# Dependency-aware gating: without this check, a failed upstream
# silently produces an empty task_summaries entry (the worker
# upstream loop below only copies summaries that exist) and the
diff --git a/agent/tests/test_swarm_replay.py b/agent/tests/test_swarm_replay.py
new file mode 100644
index 000000000..05a87bffd
--- /dev/null
+++ b/agent/tests/test_swarm_replay.py
@@ -0,0 +1,706 @@
+"""Tests for swarm run replay (#1157): resuming a failed/cancelled run keeps
+completed upstream tasks (with re-homed artifacts) and re-executes only the
+failed/cancelled subgraph.
+
+The overlay is exercised through the real orchestration loop
+(``_execute_run``) with a fake worker, mirroring ``test_swarm_dag_gating.py``.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+
+import pytest
+
+import src.swarm.runtime as rt
+from src.swarm.models import (
+ RunStatus,
+ SwarmAgentSpec,
+ SwarmRun,
+ SwarmTask,
+ TaskStatus,
+ WorkerResult,
+)
+from src.swarm.store import SwarmStore
+from src.swarm.task_store import TaskStore
+
+
+def _make_dag_run(run_id: str, status: RunStatus) -> SwarmRun:
+ """A three-task DAG: t-a and t-b (layer 1), t-c consuming both (layer 2)."""
+ agents = [
+ SwarmAgentSpec(id="analyst", role="Analyst", system_prompt="x", max_retries=0),
+ SwarmAgentSpec(id="scout", role="Scout", system_prompt="x", max_retries=0),
+ SwarmAgentSpec(id="pm", role="PM", system_prompt="x", max_retries=0),
+ ]
+ tasks = [
+ SwarmTask(id="t-a", agent_id="analyst", prompt_template="do a"),
+ SwarmTask(id="t-b", agent_id="scout", prompt_template="do b"),
+ SwarmTask(
+ id="t-c",
+ agent_id="pm",
+ prompt_template="do c",
+ depends_on=["t-a", "t-b"],
+ blocked_by=["t-a", "t-b"],
+ input_from={"a": "t-a", "b": "t-b"},
+ ),
+ ]
+ run = SwarmRun(
+ id=run_id,
+ preset_name="demo",
+ created_at="2026-08-20T09:00:00+00:00",
+ agents=agents,
+ tasks=tasks,
+ )
+ run.status = status
+ return run
+
+
+def _seed_original_run(store: SwarmStore, run: SwarmRun) -> None:
+ """Persist a failed/cancelled run: t-a completed (with an artifact),
+ t-b failed, t-c blocked."""
+ store.create_run(run)
+ task_store = TaskStore(store.run_dir(run.id))
+ for task in run.tasks:
+ task_store.save_task(task)
+ task_store.update_status(
+ "t-a",
+ TaskStatus.completed,
+ summary="AAPL BUY 5 conviction",
+ completed_at="2026-08-20T09:01:00+00:00",
+ artifacts=["artifacts/analyst/report.md", "../outside.md"],
+ worker_iterations=3,
+ )
+ artifact_file = store.run_dir(run.id) / "artifacts" / "analyst" / "report.md"
+ artifact_file.parent.mkdir(parents=True, exist_ok=True)
+ artifact_file.write_text("# kept report", encoding="utf-8")
+ task_store.update_status(
+ "t-b",
+ TaskStatus.failed,
+ error="mock data rejected",
+ completed_at="2026-08-20T09:01:30+00:00",
+ )
+ task_store.update_status(
+ "t-c",
+ TaskStatus.blocked,
+ error="Blocked: upstream not completed (t-b=failed)",
+ blocked_by=["t-b"],
+ )
+
+
+def workers_succeed(monkeypatch):
+ """Every worker invocation succeeds; record (task_id, upstream) calls."""
+
+ def fake_worker(agent_spec, task, upstream_summaries=None, **kwargs):
+ workers_succeed.calls.append((task.id, dict(upstream_summaries or {})))
+ return WorkerResult(
+ status="completed",
+ summary=f"fresh-{task.id}",
+ artifact_paths=[],
+ iterations=1,
+ input_tokens=0,
+ output_tokens=0,
+ )
+
+ workers_succeed.calls = []
+ monkeypatch.setattr(rt, "run_worker", fake_worker)
+ return workers_succeed
+
+
[email protected]("orig_status", [RunStatus.failed, RunStatus.cancelled])
+def test_replay_keeps_completed_upstream_and_reruns_subgraph(
+ tmp_path, workers_succeed, orig_status
+):
+ """t-a is carried over untouched; t-b and t-c re-execute; t-c sees the
+ kept summary as upstream context."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", orig_status)
+ _seed_original_run(store, original)
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ reloaded = store.load_run("r-new")
+ assert reloaded is not None
+ assert reloaded.status == RunStatus.completed
+
+ by_id = {t.id: t for t in reloaded.tasks}
+ assert by_id["t-a"].status == TaskStatus.completed
+ assert by_id["t-a"].summary == "AAPL BUY 5 conviction", (
+ "kept task must preserve the original summary, not re-execute"
+ )
+ # Only the contained artifact is re-homed into the new run; the escaped
+ # path is dropped.
+ assert by_id["t-a"].artifacts == ["artifacts/analyst/report.md"]
+ rehomed = store.run_dir("r-new") / "artifacts" / "analyst" / "report.md"
+ assert rehomed.is_file(), "kept artifact must be copied into the new run dir"
+ assert (store.run_dir("r-orig") / "artifacts" / "analyst" / "report.md").is_file(), (
+ "the original run must stay untouched as a record"
+ )
+ assert by_id["t-b"].status == TaskStatus.completed
+ assert by_id["t-b"].summary == "fresh-t-b", "failed task must re-execute"
+ assert by_id["t-c"].status == TaskStatus.completed
+ assert by_id["t-c"].summary == "fresh-t-c"
+
+ # t-a was never dispatched; t-b and t-c ran, in DAG order.
+ called = [tid for tid, _ in workers_succeed.calls]
+ assert called == ["t-b", "t-c"], f"unexpected worker calls: {called}"
+ t_c_upstream = dict(workers_succeed.calls[1][1])
+ assert t_c_upstream.get("a") == "AAPL BUY 5 conviction", (
+ "re-executed dependent must receive the kept task's summary"
+ )
+ assert t_c_upstream.get("b") == "fresh-t-b"
+
+
+def test_replay_emits_task_resumed_event(tmp_path, workers_succeed):
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ _seed_original_run(store, original)
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ events_file = tmp_path / "r-new" / "events.jsonl"
+ events = [json.loads(line) for line in events_file.read_text().splitlines() if line.strip()]
+ resumed = [e for e in events if e.get("type") == "task_resumed"]
+ assert len(resumed) == 1
+ assert resumed[0]["task_id"] == "t-a"
+ assert resumed[0]["data"]["source_run_id"] == "r-orig"
+ assert resumed[0].get("agent_id") == "analyst"
+
+
+def test_replay_missing_prior_task_store_runs_fresh(tmp_path, workers_succeed):
+ """A resume_from run whose task files are gone degrades to a full re-run."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ store.create_run(original) # run.json only — no task files
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ reloaded = store.load_run("r-new")
+ assert reloaded is not None
+ assert reloaded.status == RunStatus.completed
+ by_id = {t.id: t for t in reloaded.tasks}
+ assert by_id["t-a"].summary == "fresh-t-a", (
+ "no kept tasks => every task re-executes"
+ )
+ assert [tid for tid, _ in workers_succeed.calls] == ["t-a", "t-b", "t-c"]
+
+
+def test_replay_seeds_empty_kept_summary(tmp_path, workers_succeed):
+ """A kept task with an empty summary still seeds its input_from key, so a
+ re-executed dependent sees an empty-string upstream (matching live runs)."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ store.create_run(original)
+ task_store = TaskStore(store.run_dir(original.id))
+ for task in original.tasks:
+ task_store.save_task(task)
+ task_store.update_status(
+ "t-a", TaskStatus.completed, summary="", completed_at="2026-08-20T09:01:00+00:00"
+ )
+ task_store.update_status("t-b", TaskStatus.failed, error="mock data rejected")
+ task_store.update_status("t-c", TaskStatus.blocked, error="upstream failed")
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ t_c_upstream = dict(workers_succeed.calls[-1][1])
+ assert "a" in t_c_upstream, "empty kept summary must still seed the input_from key"
+ assert t_c_upstream["a"] == ""
+
+
+def test_replay_does_not_keep_changed_task_definition(tmp_path, workers_succeed):
+ """A completed task whose definition changed since the original run must be
+ re-executed, not kept — a kept result must still mean the same thing."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ _seed_original_run(store, original)
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ # Redefine t-a's prompt (definition changed) — everything else identical.
+ for task in fresh.tasks:
+ if task.id == "t-a":
+ task.prompt_template = "do a, but differently"
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ by_id = {t.id: t for t in store.load_run("r-new").tasks}
+ assert by_id["t-a"].status == TaskStatus.completed
+ assert by_id["t-a"].summary == "fresh-t-a", (
+ "changed task definition must re-execute, not keep the old result"
+ )
+ assert [tid for tid, _ in workers_succeed.calls] == ["t-a", "t-b", "t-c"]
+
+
+def test_replay_rehomes_only_artifacts_subtree(tmp_path, workers_succeed):
+ """Control-file paths stored as artifacts (run.json, tasks/*, absolute) must
+ be dropped: they must never clobber the new run's own state files."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ store.create_run(original)
+ task_store = TaskStore(store.run_dir(original.id))
+ for task in original.tasks:
+ task_store.save_task(task)
+ task_store.update_status(
+ "t-a",
+ TaskStatus.completed,
+ summary="KEPT",
+ completed_at="2026-08-20T09:01:00+00:00",
+ artifacts=["run.json", "tasks/task-t-b.json", "/etc/passwd", "artifacts/analyst/report.md"],
+ )
+ report = store.run_dir(original.id) / "artifacts" / "analyst" / "report.md"
+ report.parent.mkdir(parents=True, exist_ok=True)
+ report.write_text("# kept report", encoding="utf-8")
+ task_store.update_status("t-b", TaskStatus.failed, error="mock data rejected")
+ task_store.update_status("t-c", TaskStatus.blocked, error="upstream failed")
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ by_id = {t.id: t for t in store.load_run("r-new").tasks}
+ assert by_id["t-a"].artifacts == ["artifacts/analyst/report.md"], (
+ f"control-file paths must be dropped, got {by_id['t-a'].artifacts}"
+ )
+ # The new run's own state file still parses as the new run's state.
+ reloaded = store.load_run("r-new")
+ assert reloaded is not None and reloaded.id == "r-new"
+ assert reloaded.status == RunStatus.completed
+
+
+def test_replay_artifact_destination_symlink_escape_blocked(tmp_path, workers_succeed):
+ """A symlink planted in the new run's artifact tree must not redirect a
+ re-homed artifact outside the run directory."""
+ outside = tmp_path / "outside"
+ outside.mkdir()
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ _seed_original_run(store, original)
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ store.create_run(fresh)
+ # Plant a symlink where the re-home would create the agent artifact dir.
+ (store.run_dir("r-new") / "artifacts" / "analyst").symlink_to(
+ outside, target_is_directory=True
+ )
+
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ by_id = {t.id: t for t in store.load_run("r-new").tasks}
+ assert by_id["t-a"].artifacts == [], (
+ "artifact re-home must refuse a destination that escapes the run dir"
+ )
+ assert list(outside.iterdir()) == [], "nothing may be written outside the run dir"
+
+
+def test_replay_cancel_before_first_layer_keeps_completed_tasks(tmp_path, workers_succeed):
+ """A cancellation signalled before layer 0 must not relabel kept tasks as
+ cancelled — the in-memory task list is synced from the store after the
+ overlay, so _cancel_remaining_tasks sees kept tasks as completed."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ _seed_original_run(store, original)
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ store.create_run(fresh)
+ cancel_event = threading.Event()
+ cancel_event.set() # cancel BEFORE the first layer's cancel check
+ runtime._execute_run(fresh, cancel_event, resume_from=original)
+
+ by_id = {t.id: t for t in store.load_run("r-new").tasks}
+ assert by_id["t-a"].status == TaskStatus.completed, (
+ f"kept task must survive cancellation, got {by_id['t-a'].status}"
+ )
+ assert by_id["t-a"].summary == "AAPL BUY 5 conviction"
+ assert by_id["t-b"].status == TaskStatus.cancelled
+ assert by_id["t-c"].status == TaskStatus.cancelled
+
+
+def test_replay_keeps_completed_task_in_later_layer(tmp_path, workers_succeed):
+ """Completed tasks in later layers are kept when their upstreams are kept;
+ only the failed task re-executes."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ agents = [
+ SwarmAgentSpec(id="analyst", role="Analyst", system_prompt="x", max_retries=0),
+ SwarmAgentSpec(id="pm", role="PM", system_prompt="x", max_retries=0),
+ ]
+ original = SwarmRun(
+ id="r-orig",
+ preset_name="demo",
+ created_at="2026-08-20T09:00:00+00:00",
+ agents=agents,
+ tasks=[
+ SwarmTask(id="t-x", agent_id="analyst", prompt_template="do x"),
+ SwarmTask(
+ id="t-y",
+ agent_id="pm",
+ prompt_template="do y",
+ depends_on=["t-x"],
+ blocked_by=["t-x"],
+ input_from={"x": "t-x"},
+ ),
+ SwarmTask(
+ id="t-z",
+ agent_id="pm",
+ prompt_template="do z",
+ depends_on=["t-y"],
+ blocked_by=["t-y"],
+ input_from={"y": "t-y"},
+ ),
+ ],
+ )
+ original.status = RunStatus.failed
+ store.create_run(original)
+ task_store = TaskStore(store.run_dir(original.id))
+ for task in original.tasks:
+ task_store.save_task(task)
+ # DAG-gated shape: t-x and t-y completed, the run failed at the last task.
+ task_store.update_status(
+ "t-x",
+ TaskStatus.completed,
+ summary="KEPT-X",
+ completed_at="2026-08-20T09:01:00+00:00",
+ artifacts=[],
+ )
+ task_store.update_status(
+ "t-y",
+ TaskStatus.completed,
+ summary="KEPT-LAYER2",
+ completed_at="2026-08-20T09:02:00+00:00",
+ artifacts=[],
+ )
+ task_store.update_status("t-z", TaskStatus.failed, error="mock data rejected")
+
+ fresh = original.model_copy(deep=True)
+ fresh.id = "r-new"
+ fresh.status = RunStatus.pending
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ reloaded = store.load_run("r-new")
+ by_id = {t.id: t for t in reloaded.tasks}
+ assert reloaded.status == RunStatus.completed
+ assert by_id["t-x"].summary == "KEPT-X"
+ assert by_id["t-y"].status == TaskStatus.completed
+ assert by_id["t-y"].summary == "KEPT-LAYER2", "completed later-layer task must be kept"
+ assert by_id["t-z"].summary == "fresh-t-z"
+ t_z_upstream = dict(workers_succeed.calls[-1][1])
+ assert t_z_upstream.get("y") == "KEPT-LAYER2"
+ # Only t-z ran; t-x and t-y were never dispatched.
+ assert [tid for tid, _ in workers_succeed.calls] == ["t-z"]
+
+
+def test_replay_rejects_dotdot_artifact_clobber(tmp_path, workers_succeed):
+ """artifacts/../run.json normalizes into the run root and must be dropped —
+ it would otherwise overwrite the new run's control file."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ store.create_run(original)
+ task_store = TaskStore(store.run_dir(original.id))
+ for task in original.tasks:
+ task_store.save_task(task)
+ task_store.update_status(
+ "t-a",
+ TaskStatus.completed,
+ summary="KEPT",
+ completed_at="2026-08-20T09:01:00+00:00",
+ artifacts=["artifacts/../run.json"],
+ )
+ task_store.update_status("t-b", TaskStatus.failed, error="mock data rejected")
+ task_store.update_status("t-c", TaskStatus.blocked, error="upstream failed")
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ by_id = {t.id: t for t in store.load_run("r-new").tasks}
+ assert by_id["t-a"].artifacts == [], (
+ "dotdot artifact path must be dropped, not clobber the run's control file"
+ )
+ reloaded = store.load_run("r-new")
+ assert reloaded.id == "r-new", "new run's control file must not be overwritten"
+ assert reloaded.status == RunStatus.completed
+
+
+def test_replay_source_symlink_artifact_dropped(tmp_path, workers_succeed):
+ """A symlinked component in the SOURCE artifact tree (artifacts/analyst ->
+ tasks) must not redirect a re-homed artifact onto task files."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ store.create_run(original)
+ task_store = TaskStore(store.run_dir(original.id))
+ for task in original.tasks:
+ task_store.save_task(task)
+ task_store.update_status(
+ "t-a",
+ TaskStatus.completed,
+ summary="KEPT",
+ completed_at="2026-08-20T09:01:00+00:00",
+ artifacts=["artifacts/analyst/task-t-a.json"],
+ )
+ task_store.update_status("t-b", TaskStatus.failed, error="mock data rejected")
+ task_store.update_status("t-c", TaskStatus.blocked, error="upstream failed")
+ # Plant an internal symlink: artifacts/analyst -> tasks.
+ run_dir = store.run_dir(original.id)
+ analyst = run_dir / "artifacts" / "analyst"
+ analyst.parent.mkdir(parents=True, exist_ok=True)
+ analyst.symlink_to(run_dir / "tasks", target_is_directory=True)
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ by_id = {t.id: t for t in store.load_run("r-new").tasks}
+ assert by_id["t-a"].artifacts == [], (
+ "artifact resolving outside the source artifacts subtree must be dropped"
+ )
+ assert {t.id for t in store.load_run("r-new").tasks} == {"t-a", "t-b", "t-c"}, (
+ "new run's task files must remain intact"
+ )
+
+
+def test_replay_does_not_keep_changed_agent_definition(tmp_path, workers_succeed):
+ """A completed task must not be kept when the agent that executes it changed
+ (here its system prompt) even though the task fields are identical."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ _seed_original_run(store, original)
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ for agent in fresh.agents:
+ if agent.id == "analyst":
+ agent.system_prompt = "a completely different analyst"
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ by_id = {t.id: t for t in store.load_run("r-new").tasks}
+ assert by_id["t-a"].status == TaskStatus.completed
+ assert by_id["t-a"].summary == "fresh-t-a", (
+ "changed agent definition must re-execute, not keep the old result"
+ )
+ assert [tid for tid, _ in workers_succeed.calls] == ["t-a", "t-b", "t-c"]
+
+
+def test_replay_reexecutes_dependents_of_reexecuted_upstream(tmp_path, workers_succeed):
+ """When a kept-candidate's upstream re-executes (definition changed), the
+ dependent must also re-execute — its kept summary would be stale relative
+ to the fresh upstream output flowing into it."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ agents = [
+ SwarmAgentSpec(id="analyst", role="Analyst", system_prompt="x", max_retries=0),
+ SwarmAgentSpec(id="pm", role="PM", system_prompt="x", max_retries=0),
+ ]
+ original = SwarmRun(
+ id="r-orig",
+ preset_name="demo",
+ created_at="2026-08-20T09:00:00+00:00",
+ agents=agents,
+ tasks=[
+ SwarmTask(id="t-x", agent_id="analyst", prompt_template="do x"),
+ SwarmTask(
+ id="t-y",
+ agent_id="pm",
+ prompt_template="do y",
+ depends_on=["t-x"],
+ blocked_by=["t-x"],
+ input_from={"x": "t-x"},
+ ),
+ SwarmTask(
+ id="t-z",
+ agent_id="pm",
+ prompt_template="do z",
+ depends_on=["t-y"],
+ blocked_by=["t-y"],
+ input_from={"y": "t-y"},
+ ),
+ ],
+ )
+ original.status = RunStatus.failed
+ store.create_run(original)
+ task_store = TaskStore(store.run_dir(original.id))
+ for task in original.tasks:
+ task_store.save_task(task)
+ task_store.update_status(
+ "t-x", TaskStatus.completed, summary="OLD-X",
+ completed_at="2026-08-20T09:01:00+00:00", artifacts=[],
+ )
+ task_store.update_status(
+ "t-y", TaskStatus.completed, summary="OLD-Y",
+ completed_at="2026-08-20T09:02:00+00:00", artifacts=[],
+ )
+ task_store.update_status("t-z", TaskStatus.failed, error="mock data rejected")
+
+ fresh = original.model_copy(deep=True)
+ fresh.id = "r-new"
+ fresh.status = RunStatus.pending
+ for task in fresh.tasks:
+ if task.id == "t-x":
+ task.prompt_template = "do x, but differently"
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ reloaded = store.load_run("r-new")
+ by_id = {t.id: t for t in reloaded.tasks}
+ assert reloaded.status == RunStatus.completed
+ assert by_id["t-x"].summary == "fresh-t-x", "changed upstream must re-execute"
+ assert by_id["t-y"].summary == "fresh-t-y", (
+ "dependent of a re-executed upstream must re-execute, not keep a stale summary"
+ )
+ assert by_id["t-z"].summary == "fresh-t-z"
+ assert [tid for tid, _ in workers_succeed.calls] == ["t-x", "t-y", "t-z"]
+
+
+def test_replay_destination_artifacts_dir_symlink_escape_blocked(tmp_path, workers_succeed):
+ """A symlink replacing the NEW run's whole artifacts/ dir must not let a
+ re-homed artifact write outside the run — the containment check is against
+ both the run root and the artifacts subtree, so the symlink cannot redefine
+ the safe boundary."""
+ outside = tmp_path / "outside"
+ outside.mkdir()
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ _seed_original_run(store, original)
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ store.create_run(fresh)
+ # Replace the new run's real artifacts/ dir with a symlink to an external
+ # dir (as a tampering process with run-dir access could before the overlay).
+ artifacts_rd = store.run_dir("r-new") / "artifacts"
+ artifacts_rd.rmdir()
+ artifacts_rd.symlink_to(outside, target_is_directory=True)
+
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ by_id = {t.id: t for t in store.load_run("r-new").tasks}
+ assert by_id["t-a"].artifacts == [], (
+ "artifact re-home must refuse a destination outside the run dir"
+ )
+ assert list(outside.iterdir()) == [], "nothing may be written outside the run dir"
+
+
+def test_replay_does_not_keep_run_level_model_change(tmp_path, workers_succeed):
+ """A global model/provider swap between runs must invalidate kept results
+ when agents use the run-level default (model_name=None) — the effective
+ model that produced a kept result must still mean the same thing."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ original.model = "deepseek-v1"
+ original.provider = "openai"
+ _seed_original_run(store, original)
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ fresh.model = "deepseek-v2" # global default changed; agents unchanged
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ by_id = {t.id: t for t in store.load_run("r-new").tasks}
+ assert by_id["t-a"].status == TaskStatus.completed
+ assert by_id["t-a"].summary == "fresh-t-a", (
+ "run-level model change must invalidate kept results produced under the old default"
+ )
+ assert [tid for tid, _ in workers_succeed.calls] == ["t-a", "t-b", "t-c"]
+
+
+def test_replay_seeds_none_summary_key_for_downstream(tmp_path, workers_succeed):
+ """A kept task whose stored summary is None must still contribute its
+ input_from key to re-executed dependents — the live run path seeds the key
+ unconditionally, so a silent None must not behave differently."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ agents = [
+ SwarmAgentSpec(id="analyst", role="Analyst", system_prompt="x", max_retries=0),
+ SwarmAgentSpec(id="pm", role="PM", system_prompt="x", max_retries=0),
+ ]
+ original = SwarmRun(
+ id="r-orig",
+ preset_name="demo",
+ created_at="2026-08-20T09:00:00+00:00",
+ agents=agents,
+ tasks=[
+ SwarmTask(id="t-a", agent_id="analyst", prompt_template="do a"),
+ SwarmTask(
+ id="t-b",
+ agent_id="pm",
+ prompt_template="do b",
+ depends_on=["t-a"],
+ blocked_by=["t-a"],
+ input_from={"a": "t-a"},
+ ),
+ ],
+ )
+ original.status = RunStatus.failed
+ store.create_run(original)
+ task_store = TaskStore(store.run_dir(original.id))
+ for task in original.tasks:
+ task_store.save_task(task)
+ # t-a completed but its stored summary is None (legacy/corrupt old state).
+ task_store.update_status(
+ "t-a",
+ TaskStatus.completed,
+ summary=None,
+ completed_at="2026-08-20T09:01:00+00:00",
+ artifacts=[],
+ )
+ task_store.update_status("t-b", TaskStatus.failed, error="mock data rejected")
+
+ fresh = original.model_copy(deep=True)
+ fresh.id = "r-new"
+ fresh.status = RunStatus.pending
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ reloaded = store.load_run("r-new")
+ by_id = {t.id: t for t in reloaded.tasks}
+ assert reloaded.status == RunStatus.completed
+ assert by_id["t-a"].summary is None, "kept task keeps its (None) summary"
+ assert by_id["t-b"].status == TaskStatus.completed
+ t_b_upstream = dict(workers_succeed.calls[-1][1])
+ assert "a" in t_b_upstream, (
+ "None-summary kept task must still contribute its input_from key"
+ )
+
+
+def test_replay_partially_missing_store_keeps_present_completed(tmp_path, workers_succeed):
+ """A task file missing from an otherwise-readable store does not trigger the
+ fresh-run fallback: present completed tasks are kept, missing ones re-execute
+ (documented partial-resume contract, distinct from whole-store degradation)."""
+ store = SwarmStore(base_dir=tmp_path)
+ runtime = rt.SwarmRuntime(store=store)
+ original = _make_dag_run("r-orig", RunStatus.failed)
+ _seed_original_run(store, original)
+ # Delete t-b's task file: it exists in the run model but not the store.
+ (store.run_dir("r-orig") / "tasks" / "task-t-b.json").unlink()
+
+ fresh = _make_dag_run("r-new", RunStatus.pending)
+ store.create_run(fresh)
+ runtime._execute_run(fresh, threading.Event(), resume_from=original)
+
+ reloaded = store.load_run("r-new")
+ by_id = {t.id: t for t in reloaded.tasks}
+ assert reloaded.status == RunStatus.completed
+ assert by_id["t-a"].summary == "AAPL BUY 5 conviction", (
+ "present completed task with a readable file must still be kept"
+ )
+ assert by_id["t-b"].summary == "fresh-t-b", "missing task file must re-execute"
diff --git a/agent/tests/test_swarm_retry.py b/agent/tests/test_swarm_retry.py
index b982e4741..65f47533b 100644
--- a/agent/tests/test_swarm_retry.py
+++ b/agent/tests/test_swarm_retry.py
@@ -93,3 +93,148 @@ def test_retry_run_relaunches_failed_run_with_same_preset(tmp_path, monkeypatch)
# A fresh run id is returned, not the original.
assert payload["run_id"] == "r-retry"
assert payload["status"] == "running"
+
+
+def test_retry_run_default_resume_passes_no_resume_from(tmp_path, monkeypatch):
+ """Default retry stays a full re-run: no resume_from handed to the runtime."""
+ store = SwarmStore(base_dir=tmp_path)
+ original = _make_run("r-failed", RunStatus.failed)
+ store.create_run(original)
+ monkeypatch.setattr(mcp_server, "_get_swarm_store", lambda: store)
+
+ captured: dict[str, object] = {}
+
+ def fake_start_run(self, preset_name, variables, **kwargs):
+ captured["resume_from"] = kwargs.get("resume_from")
+ new = _make_run("r-retry", RunStatus.running)
+ self._store.create_run(new)
+ return new
+
+ monkeypatch.setattr(rt.SwarmRuntime, "start_run", fake_start_run)
+
+ json.loads(mcp_server.retry_run("r-failed"))
+
+ assert captured["resume_from"] is None
+
+
+def test_retry_run_resume_true_passes_reconciled_run(tmp_path, monkeypatch):
+ """resume=True replays: the reconciled original run is handed to the runtime."""
+ store = SwarmStore(base_dir=tmp_path)
+ original = _make_run("r-failed", RunStatus.failed)
+ store.create_run(original)
+ monkeypatch.setattr(mcp_server, "_get_swarm_store", lambda: store)
+
+ captured: dict[str, object] = {}
+
+ def fake_start_run(self, preset_name, variables, **kwargs):
+ captured["resume_from"] = kwargs.get("resume_from")
+ new = _make_run("r-retry", RunStatus.running)
+ self._store.create_run(new)
+ return new
+
+ monkeypatch.setattr(rt.SwarmRuntime, "start_run", fake_start_run)
+
+ json.loads(mcp_server.retry_run("r-failed", resume=True))
+
+ resume_from = captured["resume_from"]
+ assert resume_from is not None
+ assert resume_from.id == "r-failed"
+ assert resume_from.preset_name == "demo"
+
+
+def test_retry_run_resume_rejects_completed_run(tmp_path, monkeypatch):
+ """resume=True is refused for a completed run (failed/cancelled only)."""
+ store = SwarmStore(base_dir=tmp_path)
+ original = _make_run("r-completed", RunStatus.completed)
+ store.create_run(original)
+ monkeypatch.setattr(mcp_server, "_get_swarm_store", lambda: store)
+
+ payload = json.loads(mcp_server.retry_run("r-completed", resume=True))
+
+ assert payload["status"] == "error"
+ assert "failed or cancelled" in payload["error"]
+
+ # Plain retry of a completed run stays allowed (backward compatible).
+ captured: dict[str, object] = {}
+
+ def fake_start_run(self, preset_name, variables, **kwargs):
+ captured["resume_from"] = kwargs.get("resume_from")
+ new = _make_run("r-retry", RunStatus.running)
+ self._store.create_run(new)
+ return new
+
+ monkeypatch.setattr(rt.SwarmRuntime, "start_run", fake_start_run)
+
+ json.loads(mcp_server.retry_run("r-completed"))
+ assert captured["resume_from"] is None
+
+
+def test_http_retry_route_resume_rejects_completed_run(tmp_path, monkeypatch):
+ """POST /swarm/runs/{id}/retry?resume=true on a completed run -> 409."""
+ import api_server
+ from fastapi.testclient import TestClient
+
+ from src.api import swarm_routes
+
+ monkeypatch.delenv("API_AUTH_KEY", raising=False)
+ monkeypatch.setattr(api_server, "_API_KEY", "")
+ client = TestClient(api_server.app, client=("127.0.0.1", 50000))
+
+ store = SwarmStore(base_dir=tmp_path)
+ original = _make_run("r-http-done", RunStatus.completed)
+ store.create_run(original)
+
+ monkeypatch.setattr(swarm_routes, "_get_swarm_runtime", lambda: type("R", (), {"_store": store})())
+ monkeypatch.setattr(
+ api_server, "_shell_tools_enabled_for_request", lambda request: False
+ )
+
+ resp = client.post("/swarm/runs/r-http-done/retry?resume=true")
+ assert resp.status_code == 409
+ assert "failed or cancelled" in resp.json()["detail"]
+
+
+def test_http_retry_route_resume_query_plumbs_resume_from(tmp_path, monkeypatch):
+ """POST /swarm/runs/{id}/retry?resume=true hands the reconciled run to
+ start_run as resume_from; without the query it stays a full re-run."""
+ import api_server
+ from fastapi.testclient import TestClient
+
+ from src.api import swarm_routes
+
+ monkeypatch.delenv("API_AUTH_KEY", raising=False)
+ monkeypatch.setattr(api_server, "_API_KEY", "")
+ client = TestClient(api_server.app, client=("127.0.0.1", 50000))
+
+ store = SwarmStore(base_dir=tmp_path)
+ original = _make_run("r-http", RunStatus.failed)
+ store.create_run(original)
+
+ captured: dict[str, object] = {}
+ call_count = 0
+
+ class _FakeRuntime:
+ _store = store
+
+ def start_run(self, preset_name, variables, **kwargs):
+ nonlocal call_count
+ call_count += 1
+ captured["resume_from"] = kwargs.get("resume_from")
+ new = _make_run(f"r-http-retry{call_count}", RunStatus.running)
+ self._store.create_run(new)
+ return new
+
+ monkeypatch.setattr(swarm_routes, "_get_swarm_runtime", lambda: _FakeRuntime())
+ monkeypatch.setattr(
+ api_server, "_shell_tools_enabled_for_request", lambda request: False
+ )
+
+ resp = client.post("/swarm/runs/r-http/retry?resume=true")
+ assert resp.status_code == 200
+ resume_from = captured["resume_from"]
+ assert resume_from is not None
+ assert resume_from.id == "r-http"
+
+ resp = client.post("/swarm/runs/r-http/retry")
+ assert resp.status_code == 200
+ assert captured["resume_from"] is None