"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
feat(swarm): replay failed/cancelled runs keeping completed task artifacts
cgycorey09f34004ef29
diff --git a/agent/mcp_server.py b/agent/mcp_server.pyindex cab8473a7..c423083d1 100644--- a/agent/mcp_server.py+++ b/agent/mcp_server.py@@ -2684,7 +2684,7 @@ def reap_stale_runs() -> str:@mcp.tool-def retry_run(run_id: str) -> str:+def retry_run(run_id: str, resume: bool = False) -> str:"""Retry a failed, stale, or cancelled swarm run.Re-launches a brand-new run with the same preset and variables as the@@ -2692,8 +2692,17 @@ def retry_run(run_id: str) -> str:spotting a ``failed`` or stale run via ``list_runs``. A still-``running``run cannot be retried — cancel or reap it first.+ With ``resume=True`` (replay), completed upstream tasks and their artifacts+ are carried into the new run as-is and only the failed/cancelled subgraph+ re-executes — completed independent branches are not re-run. ``resume``+ only applies to a ``failed`` or ``cancelled`` run; ``resume=True`` for any+ other status is refused with an error (a plain retry without ``resume``+ remains available for any non-running run).+Args:run_id: ID of the run to retry (from ``list_runs`` / ``get_swarm_status``).+ resume: Replay the failed/cancelled subgraph instead of re-running the+ whole preset. Default ``False`` preserves the existing full re-run.Returns:JSON payload for the newly created run (``run_id`` / ``status`` /@@ -2719,6 +2728,17 @@ def retry_run(run_id: str) -> str:{"status": "error", "error": "Cannot retry a running run. Cancel or reap it first."},ensure_ascii=False,)+ if resume and reconciled.status not in (RunStatus.failed, RunStatus.cancelled):+ return json.dumps(+ {+ "status": "error",+ "error": (+ f"Cannot resume a run in status '{reconciled.status.value}'; "+ "resume only applies to failed or cancelled runs."+ ),+ },+ ensure_ascii=False,+ )agent_config = load_swarm_agent_config()runtime = SwarmRuntime(store=store, agent_config=agent_config)@@ -2727,6 +2747,7 @@ def retry_run(run_id: str) -> str:reconciled.preset_name,reconciled.user_vars or {},include_shell_tools=_include_shell_tools,+ resume_from=reconciled if resume else None,)except FileNotFoundError as exc:return json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False)diff --git a/agent/src/api/swarm_routes.py b/agent/src/api/swarm_routes.pyindex ce65d611e..932813c8d 100644--- a/agent/src/api/swarm_routes.py+++ b/agent/src/api/swarm_routes.py@@ -221,10 +221,13 @@ def register_swarm_routes(return {"status": "cancelled"}@app.post("/swarm/runs/{run_id}/retry", dependencies=[Depends(require_auth)])- async def retry_swarm_run(run_id: str, http_request: Request):+ async def retry_swarm_run(run_id: str, http_request: Request, resume: bool = Query(False)):"""Retry a failed, stale, or cancelled swarm run.Creates a new run with the same preset and user_vars as the original.+ ``resume=true`` replays: completed upstream tasks and their artifacts+ are carried into the new run and only the failed/cancelled subgraph+ re-executes."""_host_validate_path_param(run_id, "run_id")runtime = _get_swarm_runtime()@@ -241,12 +244,21 @@ def register_swarm_routes(raise HTTPException(status_code=409, detail="Cannot retry a running run. Cancel it first.")+ if resume and reconciled.status not in (RunStatus.failed, RunStatus.cancelled):+ raise HTTPException(+ status_code=409,+ detail=(+ f"Cannot resume a run in status '{reconciled.status.value}'; "+ "resume only applies to failed or cancelled runs."+ ),+ )try:new_run = runtime.start_run(reconciled.preset_name,reconciled.user_vars or {},include_shell_tools=_host_shell_tools_enabled_for_request(http_request),+ resume_from=reconciled if resume else None,)return {"id": new_run.id,diff --git a/agent/src/swarm/runtime.py b/agent/src/swarm/runtime.pyindex 4189f75e3..21c7593be 100644--- a/agent/src/swarm/runtime.py+++ b/agent/src/swarm/runtime.py@@ -8,6 +8,7 @@ with cancellation and event callback support.from __future__ import annotationsimport logging+import shutilimport threadingfrom concurrent.futures import (Future,@@ -50,6 +51,159 @@ from src.swarm.worker import agent_artifact_dir, clear_agent_artifacts, run_worklogger = logging.getLogger(__name__)+def _rehome_artifact_paths(+ artifact_paths: list[str],+ source_run_dir: Path,+ target_run_dir: Path,+) -> list[str]:+ """Copy a kept task's run-relative artifacts into *target_run_dir*.++ Workers persist artifacts as POSIX paths relative to the run directory+ (``artifacts/<agent>/<file>``; see ``worker._collect_artifacts``). A+ resumed run must not reference the old run's directory — the old run+ stays as a record and may be deleted later.++ The stored artifact list is on-disk state that may be stale or tampered,+ so every entry is validated before use:+ - Only paths under the ``artifacts/`` subtree are accepted, resolved to a+ real path on BOTH sides: ``artifacts/../run.json`` normalizes into the+ run root (control-file clobber) and a symlinked component such as+ ``artifacts/analyst -> tasks`` redirects the copy onto task files. The+ resolved source and resolved destination must each land inside BOTH the+ run directory and its ``artifacts/`` subtree — the run-root bound stops a+ symlinked ``artifacts/`` dir (e.g. ``artifacts -> /outside``) from+ redefining the safe boundary, the subtree bound stops redirection onto+ control/task files. Anything else (``run.json``, ``events.jsonl``,+ ``tasks/*.json``, absolute paths, ``..``, symlink redirection) is a+ control-file or path-traversal vector and is dropped — re-homing a kept+ artifact must never clobber the new run's own state files or write+ outside the run.+ - Missing files are dropped rather than re-homed.++ Returns:+ New run-relative artifact paths, preserving the input order.+ """+ rehomed: list[str] = []+ source_root = source_run_dir.resolve()+ target_root = target_run_dir.resolve()+ source_artifacts = (source_root / "artifacts").resolve()+ target_artifacts = (target_root / "artifacts").resolve()+ for raw in artifact_paths or []:+ rel = Path(raw)+ parts = rel.parts+ if rel.is_absolute() or not parts or len(parts) < 2 or parts[0] != "artifacts":+ logger.warning("Resume: dropping non-artifact path %r", raw)+ continue+ if ".." in parts:+ logger.warning(+ "Resume: dropping artifact path escaping artifacts/: %r", raw+ )+ continue+ src = source_run_dir / rel+ try:+ resolved_src = src.resolve()+ if not (+ resolved_src.is_relative_to(source_root)+ and resolved_src.is_relative_to(source_artifacts)+ ):+ logger.warning(+ "Resume: dropping artifact escaping source run: %r", raw+ )+ continue+ except (OSError, RuntimeError, ValueError):+ continue+ if not src.is_file():+ continue+ dst = target_run_dir / rel+ try:+ # Validate containment BEFORE mkdir: creating parent dirs through a+ # symlinked component (e.g. artifacts -> /outside) would already+ # write outside the run dir.+ resolved_dst = dst.resolve(strict=False)+ if not (+ resolved_dst.is_relative_to(target_root)+ and resolved_dst.is_relative_to(target_artifacts)+ ):+ logger.warning(+ "Resume: dropping artifact whose destination escapes the run dir: %r",+ raw,+ )+ continue+ dst.parent.mkdir(parents=True, exist_ok=True)+ shutil.copy2(src, dst)+ except OSError:+ logger.warning("Resume: failed to re-home artifact %s", rel, exc_info=True)+ continue+ rehomed.append(rel.as_posix())+ return rehomed+++def _agent_spec_changed(prev: SwarmAgentSpec, new: SwarmAgentSpec) -> bool:+ """Return whether an agent's defining configuration differs between runs.++ Only the fields that determine what/with-what the agent produces are+ compared: role, system prompt, tool/skill whitelists, model choice,+ retry budget, and timeout. A kept task result is only valid if the agent+ that produced it still means the same thing.+ """+ return (+ prev.role != new.role+ or prev.system_prompt != new.system_prompt+ or prev.tools != new.tools+ or prev.skills != new.skills+ or prev.model_name != new.model_name+ or prev.max_retries != new.max_retries+ or prev.max_iterations != new.max_iterations+ or prev.timeout_seconds != new.timeout_seconds+ )+++def _task_definition_changed(+ prev: SwarmTask,+ new: SwarmTask,+ prev_agents: dict[str, SwarmAgentSpec],+ new_agents: dict[str, SwarmAgentSpec],+ prev_run: SwarmRun,+ new_run: SwarmRun,+) -> bool:+ """Return whether a task's semantic definition differs between runs.++ Compares both the fields that define *what the task is* — the agent that+ executes it, the prompt template, and the DAG wiring (``depends_on`` and+ ``input_from``) — and the agent spec that executes it (see+ ``_agent_spec_changed``). Runtime state (status, summary, blocked_by,+ completed_at, …) is deliberately excluded.++ When neither the old nor the new agent pins ``model_name`` (bundled presets+ leave it unset), the task runs on the run-level default captured in+ ``SwarmRun.model`` / ``.provider`` — so a global model/provider swap+ between the original run and the replay also invalidates kept results.+ Per-agent overrides are compared by ``_agent_spec_changed`` already.++ A replayed run must keep a completed task only when the task, its agent,+ and the effective model that produced it still mean the same thing —+ otherwise the kept result is stale with respect to the current run.+ """+ if (+ prev.agent_id != new.agent_id+ or prev.prompt_template != new.prompt_template+ or prev.depends_on != new.depends_on+ or prev.input_from != new.input_from+ ):+ return True+ prev_agent = prev_agents.get(prev.agent_id)+ new_agent = new_agents.get(new.agent_id)+ if prev_agent is None or new_agent is None:+ # Agent vanished from or appeared in the preset — treat as changed.+ return prev_agent is not new_agent+ if _agent_spec_changed(prev_agent, new_agent):+ return True+ if prev_agent.model_name is None and new_agent.model_name is None:+ # Both defer to the run-level default model/provider.+ return prev_run.model != new_run.model or prev_run.provider != new_run.provider+ return False++class SwarmRuntime:"""Swarm DAG orchestration engine.@@ -93,6 +247,7 @@ class SwarmRuntime:user_vars: dict[str, str],live_callback: Callable | None = None,include_shell_tools: bool = False,+ resume_from: SwarmRun | None = None,) -> SwarmRun:"""Start a swarm run. Returns immediately, execution happens in background.@@ -101,6 +256,10 @@ class SwarmRuntime:user_vars: User-provided variables for prompt templates.live_callback: Optional callback invoked for each event in real-time.include_shell_tools: Whether workers may register shell tools.+ resume_from: Optional prior ``failed`` / ``cancelled`` run to resume.+ Its completed tasks (and their artifacts) are carried into the+ new run as-is; every other task re-executes. ``None`` (default)+ starts a fully fresh run.Returns:The created SwarmRun instance (status=pending initially).@@ -150,7 +309,7 @@ class SwarmRuntime:thread = threading.Thread(target=self._execute_run,- args=(run, cancel_event, include_shell_tools),+ args=(run, cancel_event, include_shell_tools, resume_from),name=f"swarm-{run.id}",daemon=True,)@@ -224,6 +383,7 @@ class SwarmRuntime:run: SwarmRun,cancel_event: threading.Event,include_shell_tools: bool = False,+ resume_from: SwarmRun | None = None,) -> None:"""Core orchestration loop (runs in background thread).@@ -241,6 +401,8 @@ class SwarmRuntime:run: SwarmRun to execute.cancel_event: Threading event for cancellation signalling.include_shell_tools: Whether workers may register shell tools.+ resume_from: Optional prior run whose completed tasks are kept+ (see :meth:`_apply_resume_overlay`). ``None`` executes fresh."""run_id = run.idrun_dir = self._store.run_dir(run_id)@@ -275,6 +437,17 @@ class SwarmRuntime:all_succeeded = Truetry:+ # The overlay runs inside the try so a failure here finalizes the+ # run (run_error event + failed status) instead of leaving a+ # zombie "running" run on disk. run.tasks is re-synced from the+ # store afterwards: _cancel_remaining_tasks and the final snapshot+ # read in-memory task state, which must see kept tasks as+ # completed — otherwise a cancellation in the pre-layer window+ # would relabel them cancelled.+ if resume_from is not None:+ self._apply_resume_overlay(run, run_dir, task_store, resume_from, task_summaries)+ run.tasks = task_store.load_all()+for layer_idx, layer_task_ids in enumerate(layers):# Check cancellation between layersif cancel_event.is_set():@@ -362,10 +535,15 @@ class SwarmRuntime:# therefore not present in layer_results — they were already# marked TaskStatus.blocked in _execute_layer and emitted# task_blocked. Account for them in run-level status so the- # run is marked failed, not silently completed.+ # run is marked failed, not silently completed. Tasks kept+ # completed by a resume overlay are equally absent from+ # layer_results but are not failures.for tid in layer_task_ids:- if tid not in layer_results:- all_succeeded = False+ if tid in layer_results:+ continue+ if task_store.load_task(tid).status == TaskStatus.completed:+ continue+ all_succeeded = False# Snapshot run.json at the layer boundary so list_runs and any# client that reads run.json directly sees fresh task statuses@@ -420,6 +598,143 @@ class SwarmRuntime:except Exception:logger.warning("Layer-boundary run.json sync failed", exc_info=True)+ def _apply_resume_overlay(+ self,+ run: SwarmRun,+ run_dir: Path,+ task_store: TaskStore,+ resume_from: SwarmRun,+ task_summaries: dict[str, str],+ ) -> None:+ """Carry completed tasks from *resume_from* into *run* before execution.++ DAG semantics guarantee a completed task never depends on a failed one,+ so "keep everything completed, reset everything else" is the core of+ the failed/cancelled-subgraph re-execution #1157 asks for. Two+ exceptions, both enforced here:+ - A completed task is NOT kept when its definition changed since the+ original run — the task's own defining fields (agent, prompt+ template, DAG wiring) or the agent spec that executes it (role,+ system prompt, tools, skills, model, retries, timeout) — because a+ kept result must still mean the same thing.+ - A completed task is NOT kept when any of its transitive upstreams+ re-executes, because its kept summary was produced from the upstream's+ old output and would be stale relative to the fresh upstream output+ flowing into it (and unchanged dependents cascade the same way).+ Kept tasks are re-marked ``completed`` with their original+ summary/artifacts (artifacts re-homed into this run's directory so the+ new run is self-contained), kept ids are resolved out of downstream+ ``blocked_by`` edges, and their summaries are seeded into+ ``task_summaries`` so ``input_from`` context flows to re-executed+ dependents. Tasks with no counterpart in the original run (added to the+ preset since) run fresh.++ Degradation: a wholly missing or unreadable prior task store falls back+ to a fully fresh run (logged, never fatal). Individual tasks absent+ from an otherwise-readable store (e.g. preset evolution that removed+ them) simply have no kept counterpart and re-execute — that is not a+ store failure, so it does not trigger the fresh-run fallback.+ """+ prev_run_dir = self._store.run_dir(resume_from.id)+ if not prev_run_dir.exists():+ logger.warning(+ "Resume: run %s directory missing — replaying fresh",+ resume_from.id,+ )+ return+ try:+ prev_store = TaskStore(prev_run_dir)+ prev_by_id = {task.id: task for task in prev_store.load_all()}+ except Exception:+ logger.warning(+ "Resume: could not read task store of run %s — replaying fresh",+ resume_from.id,+ exc_info=True,+ )+ return++ prev_agents = {a.id: a for a in resume_from.agents}+ new_agents = {a.id: a for a in run.agents}++ # Phase 1: candidates = completed tasks whose own definition is unchanged.+ to_keep: set[str] = set()+ for task in run.tasks:+ prev = prev_by_id.get(task.id)+ if prev is None or prev.status != TaskStatus.completed:+ continue+ if _task_definition_changed(prev, task, prev_agents, new_agents, resume_from, run):+ logger.info(+ "Resume: task %s definition changed since run %s — re-executing",+ task.id,+ resume_from.id,+ )+ continue+ to_keep.add(task.id)++ # Phase 2: a kept result is only valid if every transitive upstream is+ # also kept — a dependent whose upstream re-executes must itself+ # re-execute, or its kept summary would be stale with respect to the+ # fresh upstream output flowing into it. Iterate to fixpoint (DAGs are+ # small; worst case is quadratic).+ invalidated = True+ while invalidated:+ invalidated = False+ for task in run.tasks:+ if task.id not in to_keep:+ continue+ for dep in task.depends_on:+ if dep not in to_keep:+ to_keep.discard(task.id)+ logger.info(+ "Resume: task %s invalidated — upstream %s re-executes",+ task.id,+ dep,+ )+ invalidated = True+ break++ # Phase 3: apply the keep for the survivors.+ kept = 0+ for task in run.tasks:+ if task.id not in to_keep:+ continue+ prev = prev_by_id[task.id]+ artifact_paths = _rehome_artifact_paths(prev.artifacts, prev_run_dir, run_dir)+ task_store.update_status(+ task.id,+ TaskStatus.completed,+ summary=prev.summary,+ completed_at=prev.completed_at,+ artifacts=artifact_paths,+ worker_iterations=prev.worker_iterations,+ blocked_by=[],+ )+ resolve_dependencies(run_dir / "tasks", task.id)+ # Mirror _execute_run's live seeding (unconditional key): a kept+ # task must contribute its input_from key to re-executed dependents+ # even when the stored summary is empty or None.+ task_summaries[task.id] = prev.summary or ""+ kept += 1+ self._emit_event(+ run.id,+ self._make_event(+ "task_resumed",+ agent_id=task.agent_id,+ task_id=task.id,+ data={+ "source_run_id": resume_from.id,+ "completed_at": prev.completed_at,+ },+ ),+ )+ if kept:+ logger.info(+ "Run %s resumed from %s: kept %d completed task(s)",+ run.id,+ resume_from.id,+ kept,+ )+def _prefetch_grounding_data(self, run: SwarmRun) -> None:"""Fetch run-level grounding data without blocking ``start_run``."""symbols = grounding.extract_symbols_from_user_vars(run.user_vars)@@ -520,6 +835,13 @@ class SwarmRuntime:for tid in layer_task_ids:task = task_store.load_task(tid)+ # Replay support: a task kept completed by the resume overlay+ # must not be re-dispatched — its summary and artifacts were+ # carried into this run and its summary seeded into+ # task_summaries before layers started.+ if task.status == TaskStatus.completed:+ continue+# Dependency-aware gating: without this check, a failed upstream# silently produces an empty task_summaries entry (the worker# upstream loop below only copies summaries that exist) and thediff --git a/agent/tests/test_swarm_replay.py b/agent/tests/test_swarm_replay.pynew file mode 100644index 000000000..05a87bffd--- /dev/null+++ b/agent/tests/test_swarm_replay.py@@ -0,0 +1,706 @@+"""Tests for swarm run replay (#1157): resuming a failed/cancelled run keeps+completed upstream tasks (with re-homed artifacts) and re-executes only the+failed/cancelled subgraph.++The overlay is exercised through the real orchestration loop+(``_execute_run``) with a fake worker, mirroring ``test_swarm_dag_gating.py``.+"""++from __future__ import annotations++import json+import threading++import pytest++import src.swarm.runtime as rt+from src.swarm.models import (+ RunStatus,+ SwarmAgentSpec,+ SwarmRun,+ SwarmTask,+ TaskStatus,+ WorkerResult,+)+from src.swarm.store import SwarmStore+from src.swarm.task_store import TaskStore+++def _make_dag_run(run_id: str, status: RunStatus) -> SwarmRun:+ """A three-task DAG: t-a and t-b (layer 1), t-c consuming both (layer 2)."""+ agents = [+ SwarmAgentSpec(id="analyst", role="Analyst", system_prompt="x", max_retries=0),+ SwarmAgentSpec(id="scout", role="Scout", system_prompt="x", max_retries=0),+ SwarmAgentSpec(id="pm", role="PM", system_prompt="x", max_retries=0),+ ]+ tasks = [+ SwarmTask(id="t-a", agent_id="analyst", prompt_template="do a"),+ SwarmTask(id="t-b", agent_id="scout", prompt_template="do b"),+ SwarmTask(+ id="t-c",+ agent_id="pm",+ prompt_template="do c",+ depends_on=["t-a", "t-b"],+ blocked_by=["t-a", "t-b"],+ input_from={"a": "t-a", "b": "t-b"},+ ),+ ]+ run = SwarmRun(+ id=run_id,+ preset_name="demo",+ created_at="2026-08-20T09:00:00+00:00",+ agents=agents,+ tasks=tasks,+ )+ run.status = status+ return run+++def _seed_original_run(store: SwarmStore, run: SwarmRun) -> None:+ """Persist a failed/cancelled run: t-a completed (with an artifact),+ t-b failed, t-c blocked."""+ store.create_run(run)+ task_store = TaskStore(store.run_dir(run.id))+ for task in run.tasks:+ task_store.save_task(task)+ task_store.update_status(+ "t-a",+ TaskStatus.completed,+ summary="AAPL BUY 5 conviction",+ completed_at="2026-08-20T09:01:00+00:00",+ artifacts=["artifacts/analyst/report.md", "../outside.md"],+ worker_iterations=3,+ )+ artifact_file = store.run_dir(run.id) / "artifacts" / "analyst" / "report.md"+ artifact_file.parent.mkdir(parents=True, exist_ok=True)+ artifact_file.write_text("# kept report", encoding="utf-8")+ task_store.update_status(+ "t-b",+ TaskStatus.failed,+ error="mock data rejected",+ completed_at="2026-08-20T09:01:30+00:00",+ )+ task_store.update_status(+ "t-c",+ TaskStatus.blocked,+ error="Blocked: upstream not completed (t-b=failed)",+ blocked_by=["t-b"],+ )+++def workers_succeed(monkeypatch):+ """Every worker invocation succeeds; record (task_id, upstream) calls."""++ def fake_worker(agent_spec, task, upstream_summaries=None, **kwargs):+ workers_succeed.calls.append((task.id, dict(upstream_summaries or {})))+ return WorkerResult(+ status="completed",+ summary=f"fresh-{task.id}",+ artifact_paths=[],+ iterations=1,+ input_tokens=0,+ output_tokens=0,+ )++ workers_succeed.calls = []+ monkeypatch.setattr(rt, "run_worker", fake_worker)+ return workers_succeed++[email protected]("orig_status", [RunStatus.failed, RunStatus.cancelled])+def test_replay_keeps_completed_upstream_and_reruns_subgraph(+ tmp_path, workers_succeed, orig_status+):+ """t-a is carried over untouched; t-b and t-c re-execute; t-c sees the+ kept summary as upstream context."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", orig_status)+ _seed_original_run(store, original)++ fresh = _make_dag_run("r-new", RunStatus.pending)+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ reloaded = store.load_run("r-new")+ assert reloaded is not None+ assert reloaded.status == RunStatus.completed++ by_id = {t.id: t for t in reloaded.tasks}+ assert by_id["t-a"].status == TaskStatus.completed+ assert by_id["t-a"].summary == "AAPL BUY 5 conviction", (+ "kept task must preserve the original summary, not re-execute"+ )+ # Only the contained artifact is re-homed into the new run; the escaped+ # path is dropped.+ assert by_id["t-a"].artifacts == ["artifacts/analyst/report.md"]+ rehomed = store.run_dir("r-new") / "artifacts" / "analyst" / "report.md"+ assert rehomed.is_file(), "kept artifact must be copied into the new run dir"+ assert (store.run_dir("r-orig") / "artifacts" / "analyst" / "report.md").is_file(), (+ "the original run must stay untouched as a record"+ )+ assert by_id["t-b"].status == TaskStatus.completed+ assert by_id["t-b"].summary == "fresh-t-b", "failed task must re-execute"+ assert by_id["t-c"].status == TaskStatus.completed+ assert by_id["t-c"].summary == "fresh-t-c"++ # t-a was never dispatched; t-b and t-c ran, in DAG order.+ called = [tid for tid, _ in workers_succeed.calls]+ assert called == ["t-b", "t-c"], f"unexpected worker calls: {called}"+ t_c_upstream = dict(workers_succeed.calls[1][1])+ assert t_c_upstream.get("a") == "AAPL BUY 5 conviction", (+ "re-executed dependent must receive the kept task's summary"+ )+ assert t_c_upstream.get("b") == "fresh-t-b"+++def test_replay_emits_task_resumed_event(tmp_path, workers_succeed):+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ _seed_original_run(store, original)++ fresh = _make_dag_run("r-new", RunStatus.pending)+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ events_file = tmp_path / "r-new" / "events.jsonl"+ events = [json.loads(line) for line in events_file.read_text().splitlines() if line.strip()]+ resumed = [e for e in events if e.get("type") == "task_resumed"]+ assert len(resumed) == 1+ assert resumed[0]["task_id"] == "t-a"+ assert resumed[0]["data"]["source_run_id"] == "r-orig"+ assert resumed[0].get("agent_id") == "analyst"+++def test_replay_missing_prior_task_store_runs_fresh(tmp_path, workers_succeed):+ """A resume_from run whose task files are gone degrades to a full re-run."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ store.create_run(original) # run.json only — no task files++ fresh = _make_dag_run("r-new", RunStatus.pending)+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ reloaded = store.load_run("r-new")+ assert reloaded is not None+ assert reloaded.status == RunStatus.completed+ by_id = {t.id: t for t in reloaded.tasks}+ assert by_id["t-a"].summary == "fresh-t-a", (+ "no kept tasks => every task re-executes"+ )+ assert [tid for tid, _ in workers_succeed.calls] == ["t-a", "t-b", "t-c"]+++def test_replay_seeds_empty_kept_summary(tmp_path, workers_succeed):+ """A kept task with an empty summary still seeds its input_from key, so a+ re-executed dependent sees an empty-string upstream (matching live runs)."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ store.create_run(original)+ task_store = TaskStore(store.run_dir(original.id))+ for task in original.tasks:+ task_store.save_task(task)+ task_store.update_status(+ "t-a", TaskStatus.completed, summary="", completed_at="2026-08-20T09:01:00+00:00"+ )+ task_store.update_status("t-b", TaskStatus.failed, error="mock data rejected")+ task_store.update_status("t-c", TaskStatus.blocked, error="upstream failed")++ fresh = _make_dag_run("r-new", RunStatus.pending)+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ t_c_upstream = dict(workers_succeed.calls[-1][1])+ assert "a" in t_c_upstream, "empty kept summary must still seed the input_from key"+ assert t_c_upstream["a"] == ""+++def test_replay_does_not_keep_changed_task_definition(tmp_path, workers_succeed):+ """A completed task whose definition changed since the original run must be+ re-executed, not kept — a kept result must still mean the same thing."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ _seed_original_run(store, original)++ fresh = _make_dag_run("r-new", RunStatus.pending)+ # Redefine t-a's prompt (definition changed) — everything else identical.+ for task in fresh.tasks:+ if task.id == "t-a":+ task.prompt_template = "do a, but differently"+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ by_id = {t.id: t for t in store.load_run("r-new").tasks}+ assert by_id["t-a"].status == TaskStatus.completed+ assert by_id["t-a"].summary == "fresh-t-a", (+ "changed task definition must re-execute, not keep the old result"+ )+ assert [tid for tid, _ in workers_succeed.calls] == ["t-a", "t-b", "t-c"]+++def test_replay_rehomes_only_artifacts_subtree(tmp_path, workers_succeed):+ """Control-file paths stored as artifacts (run.json, tasks/*, absolute) must+ be dropped: they must never clobber the new run's own state files."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ store.create_run(original)+ task_store = TaskStore(store.run_dir(original.id))+ for task in original.tasks:+ task_store.save_task(task)+ task_store.update_status(+ "t-a",+ TaskStatus.completed,+ summary="KEPT",+ completed_at="2026-08-20T09:01:00+00:00",+ artifacts=["run.json", "tasks/task-t-b.json", "/etc/passwd", "artifacts/analyst/report.md"],+ )+ report = store.run_dir(original.id) / "artifacts" / "analyst" / "report.md"+ report.parent.mkdir(parents=True, exist_ok=True)+ report.write_text("# kept report", encoding="utf-8")+ task_store.update_status("t-b", TaskStatus.failed, error="mock data rejected")+ task_store.update_status("t-c", TaskStatus.blocked, error="upstream failed")++ fresh = _make_dag_run("r-new", RunStatus.pending)+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ by_id = {t.id: t for t in store.load_run("r-new").tasks}+ assert by_id["t-a"].artifacts == ["artifacts/analyst/report.md"], (+ f"control-file paths must be dropped, got {by_id['t-a'].artifacts}"+ )+ # The new run's own state file still parses as the new run's state.+ reloaded = store.load_run("r-new")+ assert reloaded is not None and reloaded.id == "r-new"+ assert reloaded.status == RunStatus.completed+++def test_replay_artifact_destination_symlink_escape_blocked(tmp_path, workers_succeed):+ """A symlink planted in the new run's artifact tree must not redirect a+ re-homed artifact outside the run directory."""+ outside = tmp_path / "outside"+ outside.mkdir()+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ _seed_original_run(store, original)++ fresh = _make_dag_run("r-new", RunStatus.pending)+ store.create_run(fresh)+ # Plant a symlink where the re-home would create the agent artifact dir.+ (store.run_dir("r-new") / "artifacts" / "analyst").symlink_to(+ outside, target_is_directory=True+ )++ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ by_id = {t.id: t for t in store.load_run("r-new").tasks}+ assert by_id["t-a"].artifacts == [], (+ "artifact re-home must refuse a destination that escapes the run dir"+ )+ assert list(outside.iterdir()) == [], "nothing may be written outside the run dir"+++def test_replay_cancel_before_first_layer_keeps_completed_tasks(tmp_path, workers_succeed):+ """A cancellation signalled before layer 0 must not relabel kept tasks as+ cancelled — the in-memory task list is synced from the store after the+ overlay, so _cancel_remaining_tasks sees kept tasks as completed."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ _seed_original_run(store, original)++ fresh = _make_dag_run("r-new", RunStatus.pending)+ store.create_run(fresh)+ cancel_event = threading.Event()+ cancel_event.set() # cancel BEFORE the first layer's cancel check+ runtime._execute_run(fresh, cancel_event, resume_from=original)++ by_id = {t.id: t for t in store.load_run("r-new").tasks}+ assert by_id["t-a"].status == TaskStatus.completed, (+ f"kept task must survive cancellation, got {by_id['t-a'].status}"+ )+ assert by_id["t-a"].summary == "AAPL BUY 5 conviction"+ assert by_id["t-b"].status == TaskStatus.cancelled+ assert by_id["t-c"].status == TaskStatus.cancelled+++def test_replay_keeps_completed_task_in_later_layer(tmp_path, workers_succeed):+ """Completed tasks in later layers are kept when their upstreams are kept;+ only the failed task re-executes."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ agents = [+ SwarmAgentSpec(id="analyst", role="Analyst", system_prompt="x", max_retries=0),+ SwarmAgentSpec(id="pm", role="PM", system_prompt="x", max_retries=0),+ ]+ original = SwarmRun(+ id="r-orig",+ preset_name="demo",+ created_at="2026-08-20T09:00:00+00:00",+ agents=agents,+ tasks=[+ SwarmTask(id="t-x", agent_id="analyst", prompt_template="do x"),+ SwarmTask(+ id="t-y",+ agent_id="pm",+ prompt_template="do y",+ depends_on=["t-x"],+ blocked_by=["t-x"],+ input_from={"x": "t-x"},+ ),+ SwarmTask(+ id="t-z",+ agent_id="pm",+ prompt_template="do z",+ depends_on=["t-y"],+ blocked_by=["t-y"],+ input_from={"y": "t-y"},+ ),+ ],+ )+ original.status = RunStatus.failed+ store.create_run(original)+ task_store = TaskStore(store.run_dir(original.id))+ for task in original.tasks:+ task_store.save_task(task)+ # DAG-gated shape: t-x and t-y completed, the run failed at the last task.+ task_store.update_status(+ "t-x",+ TaskStatus.completed,+ summary="KEPT-X",+ completed_at="2026-08-20T09:01:00+00:00",+ artifacts=[],+ )+ task_store.update_status(+ "t-y",+ TaskStatus.completed,+ summary="KEPT-LAYER2",+ completed_at="2026-08-20T09:02:00+00:00",+ artifacts=[],+ )+ task_store.update_status("t-z", TaskStatus.failed, error="mock data rejected")++ fresh = original.model_copy(deep=True)+ fresh.id = "r-new"+ fresh.status = RunStatus.pending+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ reloaded = store.load_run("r-new")+ by_id = {t.id: t for t in reloaded.tasks}+ assert reloaded.status == RunStatus.completed+ assert by_id["t-x"].summary == "KEPT-X"+ assert by_id["t-y"].status == TaskStatus.completed+ assert by_id["t-y"].summary == "KEPT-LAYER2", "completed later-layer task must be kept"+ assert by_id["t-z"].summary == "fresh-t-z"+ t_z_upstream = dict(workers_succeed.calls[-1][1])+ assert t_z_upstream.get("y") == "KEPT-LAYER2"+ # Only t-z ran; t-x and t-y were never dispatched.+ assert [tid for tid, _ in workers_succeed.calls] == ["t-z"]+++def test_replay_rejects_dotdot_artifact_clobber(tmp_path, workers_succeed):+ """artifacts/../run.json normalizes into the run root and must be dropped —+ it would otherwise overwrite the new run's control file."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ store.create_run(original)+ task_store = TaskStore(store.run_dir(original.id))+ for task in original.tasks:+ task_store.save_task(task)+ task_store.update_status(+ "t-a",+ TaskStatus.completed,+ summary="KEPT",+ completed_at="2026-08-20T09:01:00+00:00",+ artifacts=["artifacts/../run.json"],+ )+ task_store.update_status("t-b", TaskStatus.failed, error="mock data rejected")+ task_store.update_status("t-c", TaskStatus.blocked, error="upstream failed")++ fresh = _make_dag_run("r-new", RunStatus.pending)+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ by_id = {t.id: t for t in store.load_run("r-new").tasks}+ assert by_id["t-a"].artifacts == [], (+ "dotdot artifact path must be dropped, not clobber the run's control file"+ )+ reloaded = store.load_run("r-new")+ assert reloaded.id == "r-new", "new run's control file must not be overwritten"+ assert reloaded.status == RunStatus.completed+++def test_replay_source_symlink_artifact_dropped(tmp_path, workers_succeed):+ """A symlinked component in the SOURCE artifact tree (artifacts/analyst ->+ tasks) must not redirect a re-homed artifact onto task files."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ store.create_run(original)+ task_store = TaskStore(store.run_dir(original.id))+ for task in original.tasks:+ task_store.save_task(task)+ task_store.update_status(+ "t-a",+ TaskStatus.completed,+ summary="KEPT",+ completed_at="2026-08-20T09:01:00+00:00",+ artifacts=["artifacts/analyst/task-t-a.json"],+ )+ task_store.update_status("t-b", TaskStatus.failed, error="mock data rejected")+ task_store.update_status("t-c", TaskStatus.blocked, error="upstream failed")+ # Plant an internal symlink: artifacts/analyst -> tasks.+ run_dir = store.run_dir(original.id)+ analyst = run_dir / "artifacts" / "analyst"+ analyst.parent.mkdir(parents=True, exist_ok=True)+ analyst.symlink_to(run_dir / "tasks", target_is_directory=True)++ fresh = _make_dag_run("r-new", RunStatus.pending)+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ by_id = {t.id: t for t in store.load_run("r-new").tasks}+ assert by_id["t-a"].artifacts == [], (+ "artifact resolving outside the source artifacts subtree must be dropped"+ )+ assert {t.id for t in store.load_run("r-new").tasks} == {"t-a", "t-b", "t-c"}, (+ "new run's task files must remain intact"+ )+++def test_replay_does_not_keep_changed_agent_definition(tmp_path, workers_succeed):+ """A completed task must not be kept when the agent that executes it changed+ (here its system prompt) even though the task fields are identical."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ _seed_original_run(store, original)++ fresh = _make_dag_run("r-new", RunStatus.pending)+ for agent in fresh.agents:+ if agent.id == "analyst":+ agent.system_prompt = "a completely different analyst"+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ by_id = {t.id: t for t in store.load_run("r-new").tasks}+ assert by_id["t-a"].status == TaskStatus.completed+ assert by_id["t-a"].summary == "fresh-t-a", (+ "changed agent definition must re-execute, not keep the old result"+ )+ assert [tid for tid, _ in workers_succeed.calls] == ["t-a", "t-b", "t-c"]+++def test_replay_reexecutes_dependents_of_reexecuted_upstream(tmp_path, workers_succeed):+ """When a kept-candidate's upstream re-executes (definition changed), the+ dependent must also re-execute — its kept summary would be stale relative+ to the fresh upstream output flowing into it."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ agents = [+ SwarmAgentSpec(id="analyst", role="Analyst", system_prompt="x", max_retries=0),+ SwarmAgentSpec(id="pm", role="PM", system_prompt="x", max_retries=0),+ ]+ original = SwarmRun(+ id="r-orig",+ preset_name="demo",+ created_at="2026-08-20T09:00:00+00:00",+ agents=agents,+ tasks=[+ SwarmTask(id="t-x", agent_id="analyst", prompt_template="do x"),+ SwarmTask(+ id="t-y",+ agent_id="pm",+ prompt_template="do y",+ depends_on=["t-x"],+ blocked_by=["t-x"],+ input_from={"x": "t-x"},+ ),+ SwarmTask(+ id="t-z",+ agent_id="pm",+ prompt_template="do z",+ depends_on=["t-y"],+ blocked_by=["t-y"],+ input_from={"y": "t-y"},+ ),+ ],+ )+ original.status = RunStatus.failed+ store.create_run(original)+ task_store = TaskStore(store.run_dir(original.id))+ for task in original.tasks:+ task_store.save_task(task)+ task_store.update_status(+ "t-x", TaskStatus.completed, summary="OLD-X",+ completed_at="2026-08-20T09:01:00+00:00", artifacts=[],+ )+ task_store.update_status(+ "t-y", TaskStatus.completed, summary="OLD-Y",+ completed_at="2026-08-20T09:02:00+00:00", artifacts=[],+ )+ task_store.update_status("t-z", TaskStatus.failed, error="mock data rejected")++ fresh = original.model_copy(deep=True)+ fresh.id = "r-new"+ fresh.status = RunStatus.pending+ for task in fresh.tasks:+ if task.id == "t-x":+ task.prompt_template = "do x, but differently"+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ reloaded = store.load_run("r-new")+ by_id = {t.id: t for t in reloaded.tasks}+ assert reloaded.status == RunStatus.completed+ assert by_id["t-x"].summary == "fresh-t-x", "changed upstream must re-execute"+ assert by_id["t-y"].summary == "fresh-t-y", (+ "dependent of a re-executed upstream must re-execute, not keep a stale summary"+ )+ assert by_id["t-z"].summary == "fresh-t-z"+ assert [tid for tid, _ in workers_succeed.calls] == ["t-x", "t-y", "t-z"]+++def test_replay_destination_artifacts_dir_symlink_escape_blocked(tmp_path, workers_succeed):+ """A symlink replacing the NEW run's whole artifacts/ dir must not let a+ re-homed artifact write outside the run — the containment check is against+ both the run root and the artifacts subtree, so the symlink cannot redefine+ the safe boundary."""+ outside = tmp_path / "outside"+ outside.mkdir()+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ _seed_original_run(store, original)++ fresh = _make_dag_run("r-new", RunStatus.pending)+ store.create_run(fresh)+ # Replace the new run's real artifacts/ dir with a symlink to an external+ # dir (as a tampering process with run-dir access could before the overlay).+ artifacts_rd = store.run_dir("r-new") / "artifacts"+ artifacts_rd.rmdir()+ artifacts_rd.symlink_to(outside, target_is_directory=True)++ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ by_id = {t.id: t for t in store.load_run("r-new").tasks}+ assert by_id["t-a"].artifacts == [], (+ "artifact re-home must refuse a destination outside the run dir"+ )+ assert list(outside.iterdir()) == [], "nothing may be written outside the run dir"+++def test_replay_does_not_keep_run_level_model_change(tmp_path, workers_succeed):+ """A global model/provider swap between runs must invalidate kept results+ when agents use the run-level default (model_name=None) — the effective+ model that produced a kept result must still mean the same thing."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ original.model = "deepseek-v1"+ original.provider = "openai"+ _seed_original_run(store, original)++ fresh = _make_dag_run("r-new", RunStatus.pending)+ fresh.model = "deepseek-v2" # global default changed; agents unchanged+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ by_id = {t.id: t for t in store.load_run("r-new").tasks}+ assert by_id["t-a"].status == TaskStatus.completed+ assert by_id["t-a"].summary == "fresh-t-a", (+ "run-level model change must invalidate kept results produced under the old default"+ )+ assert [tid for tid, _ in workers_succeed.calls] == ["t-a", "t-b", "t-c"]+++def test_replay_seeds_none_summary_key_for_downstream(tmp_path, workers_succeed):+ """A kept task whose stored summary is None must still contribute its+ input_from key to re-executed dependents — the live run path seeds the key+ unconditionally, so a silent None must not behave differently."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ agents = [+ SwarmAgentSpec(id="analyst", role="Analyst", system_prompt="x", max_retries=0),+ SwarmAgentSpec(id="pm", role="PM", system_prompt="x", max_retries=0),+ ]+ original = SwarmRun(+ id="r-orig",+ preset_name="demo",+ created_at="2026-08-20T09:00:00+00:00",+ agents=agents,+ tasks=[+ SwarmTask(id="t-a", agent_id="analyst", prompt_template="do a"),+ SwarmTask(+ id="t-b",+ agent_id="pm",+ prompt_template="do b",+ depends_on=["t-a"],+ blocked_by=["t-a"],+ input_from={"a": "t-a"},+ ),+ ],+ )+ original.status = RunStatus.failed+ store.create_run(original)+ task_store = TaskStore(store.run_dir(original.id))+ for task in original.tasks:+ task_store.save_task(task)+ # t-a completed but its stored summary is None (legacy/corrupt old state).+ task_store.update_status(+ "t-a",+ TaskStatus.completed,+ summary=None,+ completed_at="2026-08-20T09:01:00+00:00",+ artifacts=[],+ )+ task_store.update_status("t-b", TaskStatus.failed, error="mock data rejected")++ fresh = original.model_copy(deep=True)+ fresh.id = "r-new"+ fresh.status = RunStatus.pending+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ reloaded = store.load_run("r-new")+ by_id = {t.id: t for t in reloaded.tasks}+ assert reloaded.status == RunStatus.completed+ assert by_id["t-a"].summary is None, "kept task keeps its (None) summary"+ assert by_id["t-b"].status == TaskStatus.completed+ t_b_upstream = dict(workers_succeed.calls[-1][1])+ assert "a" in t_b_upstream, (+ "None-summary kept task must still contribute its input_from key"+ )+++def test_replay_partially_missing_store_keeps_present_completed(tmp_path, workers_succeed):+ """A task file missing from an otherwise-readable store does not trigger the+ fresh-run fallback: present completed tasks are kept, missing ones re-execute+ (documented partial-resume contract, distinct from whole-store degradation)."""+ store = SwarmStore(base_dir=tmp_path)+ runtime = rt.SwarmRuntime(store=store)+ original = _make_dag_run("r-orig", RunStatus.failed)+ _seed_original_run(store, original)+ # Delete t-b's task file: it exists in the run model but not the store.+ (store.run_dir("r-orig") / "tasks" / "task-t-b.json").unlink()++ fresh = _make_dag_run("r-new", RunStatus.pending)+ store.create_run(fresh)+ runtime._execute_run(fresh, threading.Event(), resume_from=original)++ reloaded = store.load_run("r-new")+ by_id = {t.id: t for t in reloaded.tasks}+ assert reloaded.status == RunStatus.completed+ assert by_id["t-a"].summary == "AAPL BUY 5 conviction", (+ "present completed task with a readable file must still be kept"+ )+ assert by_id["t-b"].summary == "fresh-t-b", "missing task file must re-execute"diff --git a/agent/tests/test_swarm_retry.py b/agent/tests/test_swarm_retry.pyindex b982e4741..65f47533b 100644--- a/agent/tests/test_swarm_retry.py+++ b/agent/tests/test_swarm_retry.py@@ -93,3 +93,148 @@ def test_retry_run_relaunches_failed_run_with_same_preset(tmp_path, monkeypatch)# A fresh run id is returned, not the original.assert payload["run_id"] == "r-retry"assert payload["status"] == "running"+++def test_retry_run_default_resume_passes_no_resume_from(tmp_path, monkeypatch):+ """Default retry stays a full re-run: no resume_from handed to the runtime."""+ store = SwarmStore(base_dir=tmp_path)+ original = _make_run("r-failed", RunStatus.failed)+ store.create_run(original)+ monkeypatch.setattr(mcp_server, "_get_swarm_store", lambda: store)++ captured: dict[str, object] = {}++ def fake_start_run(self, preset_name, variables, **kwargs):+ captured["resume_from"] = kwargs.get("resume_from")+ new = _make_run("r-retry", RunStatus.running)+ self._store.create_run(new)+ return new++ monkeypatch.setattr(rt.SwarmRuntime, "start_run", fake_start_run)++ json.loads(mcp_server.retry_run("r-failed"))++ assert captured["resume_from"] is None+++def test_retry_run_resume_true_passes_reconciled_run(tmp_path, monkeypatch):+ """resume=True replays: the reconciled original run is handed to the runtime."""+ store = SwarmStore(base_dir=tmp_path)+ original = _make_run("r-failed", RunStatus.failed)+ store.create_run(original)+ monkeypatch.setattr(mcp_server, "_get_swarm_store", lambda: store)++ captured: dict[str, object] = {}++ def fake_start_run(self, preset_name, variables, **kwargs):+ captured["resume_from"] = kwargs.get("resume_from")+ new = _make_run("r-retry", RunStatus.running)+ self._store.create_run(new)+ return new++ monkeypatch.setattr(rt.SwarmRuntime, "start_run", fake_start_run)++ json.loads(mcp_server.retry_run("r-failed", resume=True))++ resume_from = captured["resume_from"]+ assert resume_from is not None+ assert resume_from.id == "r-failed"+ assert resume_from.preset_name == "demo"+++def test_retry_run_resume_rejects_completed_run(tmp_path, monkeypatch):+ """resume=True is refused for a completed run (failed/cancelled only)."""+ store = SwarmStore(base_dir=tmp_path)+ original = _make_run("r-completed", RunStatus.completed)+ store.create_run(original)+ monkeypatch.setattr(mcp_server, "_get_swarm_store", lambda: store)++ payload = json.loads(mcp_server.retry_run("r-completed", resume=True))++ assert payload["status"] == "error"+ assert "failed or cancelled" in payload["error"]++ # Plain retry of a completed run stays allowed (backward compatible).+ captured: dict[str, object] = {}++ def fake_start_run(self, preset_name, variables, **kwargs):+ captured["resume_from"] = kwargs.get("resume_from")+ new = _make_run("r-retry", RunStatus.running)+ self._store.create_run(new)+ return new++ monkeypatch.setattr(rt.SwarmRuntime, "start_run", fake_start_run)++ json.loads(mcp_server.retry_run("r-completed"))+ assert captured["resume_from"] is None+++def test_http_retry_route_resume_rejects_completed_run(tmp_path, monkeypatch):+ """POST /swarm/runs/{id}/retry?resume=true on a completed run -> 409."""+ import api_server+ from fastapi.testclient import TestClient++ from src.api import swarm_routes++ monkeypatch.delenv("API_AUTH_KEY", raising=False)+ monkeypatch.setattr(api_server, "_API_KEY", "")+ client = TestClient(api_server.app, client=("127.0.0.1", 50000))++ store = SwarmStore(base_dir=tmp_path)+ original = _make_run("r-http-done", RunStatus.completed)+ store.create_run(original)++ monkeypatch.setattr(swarm_routes, "_get_swarm_runtime", lambda: type("R", (), {"_store": store})())+ monkeypatch.setattr(+ api_server, "_shell_tools_enabled_for_request", lambda request: False+ )++ resp = client.post("/swarm/runs/r-http-done/retry?resume=true")+ assert resp.status_code == 409+ assert "failed or cancelled" in resp.json()["detail"]+++def test_http_retry_route_resume_query_plumbs_resume_from(tmp_path, monkeypatch):+ """POST /swarm/runs/{id}/retry?resume=true hands the reconciled run to+ start_run as resume_from; without the query it stays a full re-run."""+ import api_server+ from fastapi.testclient import TestClient++ from src.api import swarm_routes++ monkeypatch.delenv("API_AUTH_KEY", raising=False)+ monkeypatch.setattr(api_server, "_API_KEY", "")+ client = TestClient(api_server.app, client=("127.0.0.1", 50000))++ store = SwarmStore(base_dir=tmp_path)+ original = _make_run("r-http", RunStatus.failed)+ store.create_run(original)++ captured: dict[str, object] = {}+ call_count = 0++ class _FakeRuntime:+ _store = store++ def start_run(self, preset_name, variables, **kwargs):+ nonlocal call_count+ call_count += 1+ captured["resume_from"] = kwargs.get("resume_from")+ new = _make_run(f"r-http-retry{call_count}", RunStatus.running)+ self._store.create_run(new)+ return new++ monkeypatch.setattr(swarm_routes, "_get_swarm_runtime", lambda: _FakeRuntime())+ monkeypatch.setattr(+ api_server, "_shell_tools_enabled_for_request", lambda request: False+ )++ resp = client.post("/swarm/runs/r-http/retry?resume=true")+ assert resp.status_code == 200+ resume_from = captured["resume_from"]+ assert resume_from is not None+ assert resume_from.id == "r-http"++ resp = client.post("/swarm/runs/r-http/retry")+ assert resp.status_code == 200+ assert captured["resume_from"] is None