Open-source design agent: on-brand stills, decks, diagrams, charts and motion graphics from a chat thread, built on Deep Agents.
JavaScript agentagentsai-agentdeepagentsdesignhyperframeslanggraphmanaged-agentsmotion-graphicspaperslackvideo
"""Pipelines as code: a production's order, gates and retries in Python; its judgment in helpers.
Run from instructions, one model turn per step, a long production lets the lead skip a review or
start assembling before an act was checked. A pipeline makes the flow code (which step runs when,
what waits for what, which review gates what) and leaves every creative call to the briefs its
helpers get.
A pipeline is an async function `fn(ctx, **args)`, registered by name with a `WorkflowRunner`:
- `ctx.agent(brief, effort=...)` runs one helper through the host's `HelperRunner` and returns its
report. With a `schema` it returns the JSON the report ends with, validated; a report that doesn't
match goes back to a helper with what is wrong, up to `attempts` times.
- `ctx.parallel(thunks)` runs steps at once and waits for all of them; a failure becomes None.
`ctx.pipeline(items, *stages)` moves each item to its next stage as soon as its current stage
finishes, with no barrier between stages.
- `ctx.phase(name)` and `ctx.note(text)` say where the run is; `ctx.ask(question, options)` parks the
run until someone answers.
Every step is memoized in the store under its run, keyed by the sha256 of (brief, effort, schema,
label). A run is re-run from the top after an answer, after a restart, or when the lead resumes it,
and each step it already finished replays at once from the memo: only unfinished work runs again.
The key is the step's content, not its position (LangGraph's `@task` replays by position), so adding
or reordering steps keeps every unchanged step's result. A failed step is memoized too, so pipeline
code that handles a failure takes the same path on every replay; resuming a failed or cancelled run
clears those failures, and the steps run again.
What happens is published on the bus with `source=<run id>` and `label=<pipeline name>`:
- `TASK_STARTED` when the run starts (or resumes);
- `PIPELINE_STEP` when a step starts, retries, fails or finishes (with its report), and for each
note: held for the lead's next wake-up;
- `JOB_PHASE` when a phase starts (recorded);
- `JOB_ATTENTION` for a question: it wakes the lead and pings a person;
- one `TASK_DONE` with the result (`data["report"]`), or `TASK_FAILED`, or `TASK_CANCELLED`;
- `TASK_INTERRUPTED` from `WorkflowRunner.sweep` when a restart ended the run (a run keeps a
heartbeat on its record while a process drives it, as `harness.ledger` does for helpers).
So the lead is woken once, with the result, like any helper, and hears every step on the way. A
re-run publishes nothing for the steps it replays.
A pipeline must be deterministic: the same arguments and step results must produce the same briefs,
or their keys change and replay stops working. Don't read the clock, randomness or the environment
inside one; a brief carries durations ("stop within 45 minutes of starting"), not clock times.
"""
from __future__ import annotations
import asyncio
import hashlib
import inspect
import json
import logging
import re
import time
import uuid
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from dataclasses import asdict, dataclass, field
from typing import Any
from design_agent.harness import events as ev
from design_agent.harness.protocols import HelperResult, HelperRunner
from design_agent.harness.tasks import minutes
_LOGGER = logging.getLogger(__name__)
#: A run's memoized steps live under ("harness", "workflows", <run id>), one item per step key.
MEMO = ("harness", "workflows")
#: One record per run: its pipeline, arguments, status, questions and answers, and result.
RUNS = ("harness", "workflow_runs")
#: Helpers one run keeps in flight at once.
CONCURRENCY = 8
#: A running run's record is refreshed this often, so another process can tell it is alive.
HEARTBEAT_SECONDS = 60.0
#: A record still `running` whose refresh is older than this has no process driving it.
STALE_SECONDS = 180.0
#: Characters of a report carried in a step's event; the whole report stays in the memo.
EVENT_REPORT_CHARS = 1500
#: A pipeline: `async def fn(ctx: WorkflowContext, **args) -> result` (JSON-able).
Pipeline = Callable[..., Awaitable[Any]]
_ENDED = frozenset({"done", "failed", "cancelled"})
class WorkflowError(Exception):
"""A request the runner can't carry out: an unknown pipeline or run, bad arguments, nothing to
answer. The message says why, for the lead."""
class StepFailed(Exception):
"""A step's helper failed, or its report never matched the schema."""
def __init__(self, label: str, reason: str, *, task_id: str = "") -> None:
super().__init__(f"{label}: {reason}")
self.label = label
self.reason = reason
self.task_id = task_id
class Suspended(BaseException): # a signal, like CancelledError, not an error
"""Raised by `WorkflowContext.ask` to park the run until someone answers.
A BaseException, like `asyncio.CancelledError`, so a pipeline's `except Exception` can't
swallow it and carry on without the answer.
"""
def __init__(self, question: str) -> None:
super().__init__(question)
self.question = question
@dataclass(frozen=True)
class Step:
"""A finished step: the helper's report (or, with a schema, the JSON it ended with), the task
that produced it, and whether it was replayed from the memo instead of run."""
value: Any
task_id: str
cached: bool = False
@dataclass
class RunRecord:
"""One run of a pipeline, as the store keeps it (namespace `RUNS`, key: the run id)."""
id: str
name: str
thread_id: str
args: dict[str, Any] = field(default_factory=dict)
#: running, waiting (for an answer), done, failed, cancelled, or interrupted (by a restart).
status: str = "running"
#: The phase the run last entered.
phase: str = ""
started_at: float = field(default_factory=time.time)
finished_at: float | None = None
#: When a process last saved it; a running record that stops refreshing was interrupted.
alive_at: float = field(default_factory=time.time)
#: Steps a helper finished for this run (replays don't count).
steps: int = 0
#: Questions by ask key: question, options, asked_at, answer (None until answered), returned.
asks: dict[str, dict[str, Any]] = field(default_factory=dict)
result: Any = None
error: str = ""
#: Host data a later process needs (the wake target).
meta: dict[str, Any] = field(default_factory=dict)
@property
def waiting(self) -> list[dict[str, Any]]:
"""Questions asked and not yet answered, oldest first."""
pending = [ask for ask in self.asks.values() if ask.get("answer") is None]
return sorted(pending, key=lambda ask: float(ask.get("asked_at") or 0))
def to_value(self) -> dict[str, Any]:
value: dict[str, Any] = _jsonable(asdict(self))
return value
@classmethod
def from_value(cls, value: Mapping[str, Any]) -> RunRecord:
data = _jsonable(dict(value))
finished = data.get("finished_at")
return cls(
id=str(data.get("id") or ""),
name=str(data.get("name") or ""),
thread_id=str(data.get("thread_id") or ""),
args=dict(data.get("args") or {}),
status=str(data.get("status") or "running"),
phase=str(data.get("phase") or ""),
started_at=float(data.get("started_at") or 0.0),
finished_at=float(finished) if finished is not None else None,
alive_at=float(data.get("alive_at") or 0.0),
steps=int(data.get("steps") or 0),
asks=dict(data.get("asks") or {}),
result=data.get("result"),
error=str(data.get("error") or ""),
meta=dict(data.get("meta") or {}),
)
def describe(self) -> str:
"""One line for the lead (`check_tasks`), with the result when it is done."""
head = f"{self.id} ({self.name})"
if self.waiting:
ask = self.waiting[0]
options = " / ".join(ask.get("options") or [])
return f"{head}: waiting for an answer to: {ask['question']}" + (
f" Options: {options}." if options else ""
)
if self.status == "running":
return f"{head}: running, phase {self.phase or 'starting'}, {self.steps} steps done."
if self.status == "failed":
return f"{head}: failed. {self.error}"
if self.status == "done":
return f"{head}: done.\n{report_text(self.result)}"
return f"{head}: {self.status}."
def new_run_id() -> str:
return f"p-{uuid.uuid4().hex[:6]}"
def _digest(parts: Sequence[Any]) -> str:
payload = json.dumps(
list(parts), sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str
)
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def step_key(brief: str, effort: str, schema: Mapping[str, Any] | None, label: str) -> str:
"""A step's memo key: the sha256 of its content, so it holds when steps are added or moved."""
return _digest([brief, effort, schema, label])
def ask_key(question: str, options: Sequence[str]) -> str:
return _digest(["ask", question, list(options)])
def report_text(result: Any) -> str:
"""A pipeline's result as the text of its `TASK_DONE`, which is what the lead's notice shows."""
if result is None:
return "Finished."
if isinstance(result, str):
return result
return json.dumps(result, indent=1, ensure_ascii=False, default=str)
# -- reading a report's JSON ------------------------------------------------------------------------
_FENCE = re.compile(r"```[^\n`]*\n(.*?)```", re.DOTALL)
def last_json(text: str) -> Any:
"""The JSON a report ends with: the last fenced block that parses, or the last bare object.
Whichever ends later wins, so a fenced example early in a report never shadows the object the
report ends with. Raises ValueError when there is none.
"""
best: tuple[int, Any] | None = None
for match in _FENCE.finditer(text):
try:
best = (match.end(), json.loads(match.group(1)))
except ValueError:
continue
decoder = json.JSONDecoder()
start = text.find("{")
while start != -1:
try:
value, end = decoder.raw_decode(text, start)
except ValueError:
start = text.find("{", start + 1)
continue
if best is None or end > best[0]:
best = (end, value)
start = text.find("{", end)
if best is None:
raise ValueError("the report has no JSON object")
return best[1]
def _where(path: Iterable[Any]) -> str:
return "$" + "".join(f"[{part}]" if isinstance(part, int) else f".{part}" for part in path)
def _show(value: Any) -> str:
return repr(value)[:80] # as jsonschema's messages quote values
_TYPES: Mapping[str, Callable[[Any], bool]] = {
"object": lambda v: isinstance(v, dict),
"array": lambda v: isinstance(v, list),
"string": lambda v: isinstance(v, str),
"number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
"integer": lambda v: (
(isinstance(v, int) and not isinstance(v, bool))
or (isinstance(v, float) and v.is_integer())
),
"boolean": lambda v: isinstance(v, bool),
"null": lambda v: v is None,
}
def basic_schema_errors(
value: Any, schema: Mapping[str, Any], path: Sequence[Any] = ()
) -> list[str]:
"""What `value` breaks in `schema`, from JSON Schema's `type`, `enum`, `required`,
`properties`, `items` and `minItems`: what a step's report needs, without `jsonschema`."""
where = _where(path)
kinds = schema.get("type")
if kinds is not None:
names = [kinds] if isinstance(kinds, str) else list(kinds)
if not any(_TYPES.get(name, lambda _v: True)(value) for name in names):
return [f"{where}: {_show(value)} is not of type {' or '.join(map(repr, names))}"]
errors: list[str] = []
if "enum" in schema and value not in schema["enum"]:
errors.append(f"{where}: {_show(value)} is not one of {list(schema['enum'])!r}")
if isinstance(value, dict):
errors += [
f"{where}: {key!r} is a required property"
for key in schema.get("required", ())
if key not in value
]
for key, sub in (schema.get("properties") or {}).items():
if key in value and isinstance(sub, Mapping):
errors += basic_schema_errors(value[key], sub, (*path, key))
if isinstance(value, list):
least = int(schema.get("minItems", 0))
if len(value) < least:
errors.append(f"{where}: {_show(value)} should have at least {least} item(s)")
items = schema.get("items")
if isinstance(items, Mapping):
for index, item in enumerate(value):
errors += basic_schema_errors(item, items, (*path, index))
return errors
def schema_errors(value: Any, schema: Mapping[str, Any]) -> list[str]:
"""What `value` breaks in `schema`, one line each; empty when it matches. Uses `jsonschema`
when the environment has it, the basic check otherwise."""
try:
from jsonschema.validators import validator_for # optional dependency
except ImportError: # pragma: no cover - exercised directly in the tests
return basic_schema_errors(value, schema)
validator = validator_for(schema)(schema)
found = sorted(validator.iter_errors(value), key=lambda e: [str(p) for p in e.absolute_path])
return [f"{_where(error.absolute_path)}: {error.message}" for error in found]
def parse_report(report: str, schema: Mapping[str, Any]) -> Any:
"""The JSON a report ends with, checked against `schema`. Raises ValueError saying what's wrong."""
value = last_json(report)
errors = schema_errors(value, schema)
if errors:
raise ValueError("; ".join(errors[:6]))
return value
def _retry_brief(brief: str, problem: str, schema: Mapping[str, Any], report: str) -> str:
return (
f"{brief}\n\n--- From the pipeline, about your last reply ---\n"
f"Its JSON didn't match what this step needs: {problem}.\n"
"Your work is done and on disk, so don't redo it. Reply again with your report, ending "
f"with one JSON object that matches this schema:\n{json.dumps(schema)}\n\n"
f"Your last reply ended:\n{report[-2000:]}"
)
def _title(brief: str) -> str:
lines = [line.strip() for line in brief.strip().splitlines() if line.strip()]
return lines[0][:60] if lines else "step"
def _excerpt(text: str) -> str:
text = text.strip()
return text if len(text) <= EVENT_REPORT_CHARS else text[:EVENT_REPORT_CHARS] + " …"
def _jsonable(value: Any) -> Any:
return json.loads(json.dumps(value, default=str))
# -- what a pipeline sees ------------------------------------------------------------------------
@dataclass
class _Live:
"""A run this process holds: driving it, or holding it while it waits for an answer."""
record: RunRecord
helpers: HelperRunner | None
task: asyncio.Task[None] | None = None
#: Steps whose helpers are in flight: step key -> label.
running: dict[str, str] = field(default_factory=dict)
class WorkflowContext:
"""What a pipeline function gets: its run, and the step primitives (module doc).
One context per execution. A re-run (after an answer, a restart or a resume) starts in replay:
phases are recorded but not published and notes are dropped, until the first step that really
runs or the first new answer, so the lead never hears the same thing twice.
"""
def __init__(self, runner: WorkflowRunner, live: _Live, *, replaying: bool) -> None:
self._runner = runner
self._live = live
self._replaying = replaying
self._phase = ""
self._slots = asyncio.Semaphore(runner.concurrency)
@property
def run_id(self) -> str:
return self._live.record.id
@property
def thread_id(self) -> str:
return self._live.record.thread_id
@property
def name(self) -> str:
return self._live.record.name
@property
def args(self) -> Mapping[str, Any]:
return dict(self._live.record.args)
# -- steps ----------------------------------------------------------------------------------
async def agent(
self,
brief: str,
*,
effort: str,
label: str = "",
phase: str = "",
schema: Mapping[str, Any] | None = None,
attempts: int = 3,
) -> Step:
"""Run one helper with `brief` at `effort`; return its report as a `Step`.
With `schema`, the value is the JSON the report ends with, validated; a report that doesn't
match is sent back with the problem appended, up to `attempts` runs in all. A helper that
fails raises `StepFailed`. `label` names the step in events and status lines (default: the
brief's first line); `phase` tags it (default: the current phase).
The same brief, effort, schema and label is the same step: it replays from the memo. Give
work you mean to repeat a distinct label, and carry an earlier step's result in the brief of
a step that depends on it, so a change upstream reaches it.
"""
key = step_key(brief, effort, schema, label)
memo = await self._runner._memo_get(self.run_id, key)
name = label or _title(brief)
if memo is not None:
task_id = str(memo.get("task_id") or "")
if memo.get("status") == "failed":
raise StepFailed(name, str(memo.get("error") or "failed"), task_id=task_id)
return Step(memo.get("value"), task_id, cached=True)
return await self._run(
key,
brief,
effort=effort,
label=name,
phase=phase or self._phase,
schema=schema,
attempts=max(1, attempts),
)
async def _run(
self,
key: str,
brief: str,
*,
effort: str,
label: str,
phase: str,
schema: Mapping[str, Any] | None,
attempts: int,
) -> Step:
helpers = self._live.helpers
if helpers is None:
raise StepFailed(label, "this run has no helper runner")
self._go_live()
facts = {"step": label, "effort": effort, "phase": phase}
async with self._slots:
started = time.time()
self._live.running[key] = label
self._say(f"{label}: started ({effort}).", state="started", **facts)
try:
value, result, attempt = await self._attempts(
helpers, brief, effort=effort, label=label, schema=schema, attempts=attempts
)
except StepFailed as failure:
failed = {
**facts,
"status": "failed",
"error": failure.reason,
"task_id": failure.task_id,
"started_at": started,
"finished_at": time.time(),
}
await self._runner._memo_put(self.run_id, key, failed)
self._say(
f"{label}: failed after {minutes(time.time() - started)}. {failure.reason}",
state="failed",
failed=True,
task_id=failure.task_id,
**facts,
)
raise
finally:
self._live.running.pop(key, None)
finished = time.time()
done = {
**facts,
"status": "done",
"task_id": result.task_id,
"value": value,
"attempts": attempt,
"started_at": started,
"finished_at": finished,
}
if schema is not None:
done["report"] = result.report
await self._runner._memo_put(self.run_id, key, done)
record = self._live.record
record.steps += 1
await self._runner._save(record)
shown = json.dumps(value, ensure_ascii=False) if schema is not None else result.report
self._say(
f"{label}: done in {minutes(finished - started)}. {_excerpt(shown)}",
state="done",
task_id=result.task_id,
**facts,
)
return Step(value, result.task_id)
async def _attempts(
self,
helpers: HelperRunner,
brief: str,
*,
effort: str,
label: str,
schema: Mapping[str, Any] | None,
attempts: int,
) -> tuple[Any, HelperResult, int]:
prompt, problem, task_id = brief, "", ""
for attempt in range(1, attempts + 1):
try:
result = await helpers.run(prompt, effort=effort, label=label, wave=self.run_id)
except Exception as error: # the host's error fails the step, not the run
raise StepFailed(
label, f"its helper could not run: {type(error).__name__}: {error}"
) from error
task_id = result.task_id
if not result.ok:
reason = f"its helper {result.status}: {result.error or 'no report'}"
raise StepFailed(label, reason, task_id=task_id)
if schema is None:
return result.report, result, attempt
try:
return parse_report(result.report, schema), result, attempt
except ValueError as error:
problem = str(error)
if attempt < attempts:
self._say(
f"{label}: its report's JSON didn't match ({problem}); sent back, attempt "
f"{attempt + 1} of {attempts}.",
state="retry",
step=label,
task_id=task_id,
)
prompt = _retry_brief(brief, problem, schema, result.report)
raise StepFailed(
label, f"no valid JSON after {attempts} attempts: {problem}", task_id=task_id
)
async def parallel(self, thunks: Iterable[Any]) -> list[Any]:
"""Run every thunk (a function returning an awaitable, or an awaitable) at once, and wait
for all of them. A thunk that fails gives None and the failure is published. An `ask` inside
one parks the run once the others have settled, so their work is saved first."""
outcomes = await asyncio.gather(
*(self._branch(thunk) for thunk in thunks), return_exceptions=True
)
for kind in (asyncio.CancelledError, Suspended, BaseException):
for outcome in outcomes:
if isinstance(outcome, kind):
raise outcome
return list(outcomes)
async def _branch(self, thunk: Any) -> Any:
try:
value = thunk() if callable(thunk) else thunk
if inspect.isawaitable(value):
value = await value
return value
except StepFailed:
return None # published by the step when it failed
except Exception as error: # noqa: BLE001 - a failed branch is None, by contract
if not self._replaying:
self._say(
f"A branch failed with {type(error).__name__}: {error}. It counts as None.",
state="failed",
failed=True,
)
return None
async def pipeline(self, items: Iterable[Any], *stages: Callable[[Any], Any]) -> list[Any]:
"""Run every item through `stages` in order, all items at once, with no barrier between
stages: an item moves on as soon as its current stage finishes. Each stage gets the previous
stage's result (the first gets the item). An item whose stage fails gives None and skips its
later stages; the others go on. Returns the last stage's results, in item order."""
def flow(item: Any) -> Callable[[], Awaitable[Any]]:
async def run() -> Any:
value = item
for stage in stages:
value = stage(value)
if inspect.isawaitable(value):
value = await value
return value
return run
return await self.parallel([flow(item) for item in items])
# -- telling people ---------------------------------------------------------------------------
def phase(self, name: str) -> None:
"""Enter a phase (`JOB_PHASE`, recorded)."""
self._phase = name
self._live.record.phase = name
if not self._replaying:
self._runner._publish(self._live.record, ev.JOB_PHASE, name, phase=name)
def note(self, text: str) -> None:
"""Tell the lead something (`PIPELINE_STEP`, held for its next wake-up)."""
if not self._replaying:
self._say(text, note=True)
async def ask(self, question: str, options: Sequence[str] = ()) -> str:
"""Ask a person, through the lead, and park the run until the answer comes.
The first time, it publishes `JOB_ATTENTION` (the lead is woken, a person is pinged), saves
the question and raises `Suspended`. `WorkflowRunner.answer` stores the answer and re-runs
the pipeline from the top; finished steps replay, and this call returns the answer. The same
question with the same options is the same ask: put what changed (a round, a score) in the
question when you ask again.
"""
record = self._live.record
key = ask_key(question, options)
asked = record.asks.get(key)
if asked is not None and asked.get("answer") is not None:
if not asked.get("returned"):
asked["returned"] = True
self._go_live()
return str(asked["answer"])
if asked is None:
self._go_live()
record.asks[key] = {
"key": key,
"question": question,
"options": list(options),
"asked_at": time.time(),
"answer": None,
"returned": False,
}
await self._runner._save(record)
choices = f" Options: {' / '.join(options)}." if options else ""
self._runner._publish(
record,
ev.JOB_ATTENTION,
f"{question}{choices} Pipeline {record.id} waits for the answer.",
ping=True,
options=list(options),
question=question,
)
raise Suspended(question)
def _go_live(self) -> None:
"""Replay is over: from here on, what happens is new. Show the phase the run is in."""
if self._replaying:
self._replaying = False
if self._phase:
self._runner._publish(
self._live.record, ev.JOB_PHASE, self._phase, phase=self._phase
)
def _say(self, text: str, **data: Any) -> None:
self._runner._publish(self._live.record, ev.PIPELINE_STEP, text, **data)
# -- the runner ------------------------------------------------------------------------------------
class WorkflowRunner:
"""Runs registered pipelines as background tasks of this process (module doc).
`store` is a LangGraph `BaseStore` (the deployment's; `InMemoryStore` in tests). `runner` runs the
helpers of any run that isn't given its own; a host can give each run its own, bound to the
thread it belongs to. Events go to `bus`.
"""
def __init__(
self,
store: Any,
runner: HelperRunner | None = None,
bus: ev.EventBus | None = None,
*,
concurrency: int = CONCURRENCY,
) -> None:
self._store = store
self._helpers = runner
self._bus = bus or ev.BUS
self.concurrency = max(1, concurrency)
self._pipelines: dict[str, Pipeline] = {}
self._runs: dict[str, _Live] = {}
self._background: set[asyncio.Task[None]] = set()
def register(self, name: str, fn: Pipeline) -> None:
self._pipelines[name] = fn
@property
def pipelines(self) -> Mapping[str, Pipeline]:
return dict(self._pipelines)
# -- the lead's operations ------------------------------------------------------------------
async def start(
self,
name: str,
args: Mapping[str, Any] | None,
thread_id: str,
*,
runner: HelperRunner | None = None,
meta: Mapping[str, Any] | None = None,
) -> str:
"""Start pipeline `name` with `args` in the background; return its run id (`p-…`) at once.
`runner` runs this run's helpers (default: the runner's own); `meta` is host data saved
with the run. Raises `WorkflowError` for an unknown pipeline or arguments it can't take.
"""
fn = self._pipelines.get(name)
if fn is None:
known = ", ".join(sorted(self._pipelines)) or "none"
raise WorkflowError(f"No pipeline named {name!r}. Pipelines: {known}.")
values = dict(args or {})
try:
inspect.signature(fn).bind(None, **values)
json.dumps(values)
except TypeError as error:
raise WorkflowError(f"{name} can't run with these arguments: {error}.") from error
helpers = runner or self._helpers
if helpers is None:
raise WorkflowError("No helper runner: pass one to start() or to the WorkflowRunner.")
record = RunRecord(
id=new_run_id(), name=name, thread_id=thread_id, args=values, meta=dict(meta or {})
)
await self._save(record)
live = _Live(record, helpers)
self._runs[record.id] = live
self._publish(
record,
ev.TASK_STARTED,
f"Started pipeline {name}.",
args=values,
started_at=record.started_at,
)
self._launch(live, replay=False)
return record.id
async def answer(
self, run_id: str, text: str, *, runner: HelperRunner | None = None
) -> RunRecord:
"""Answer the oldest question the run is waiting on, and let it continue: it re-runs from
the top, its finished steps replay, and the `ask` returns `text`."""
live = self._runs.get(run_id)
record = live.record if live is not None else await self._load(run_id)
if record is None:
raise WorkflowError(f"No pipeline run {run_id}.")
pending = record.waiting
if record.status in _ENDED or not pending:
raise WorkflowError(f"{run_id} is {record.status}; nothing is waiting for an answer.")
if record.name not in self._pipelines:
raise WorkflowError(f"Pipeline {record.name!r} isn't registered in this process.")
helpers = runner or (live.helpers if live is not None else None) or self._helpers
if helpers is None:
raise WorkflowError("No helper runner: pass one to answer().")
ask = pending[0]
ask["answer"], ask["answered_at"] = text, time.time()
self._publish(
record, ev.PIPELINE_STEP, f'Answered "{ask["question"]}": {text}', answered=True
)
if live is not None and live.task is not None and not live.task.done():
# Other branches are still settling; the run goes on with the answer when they have.
live.helpers = helpers
await self._save(record)
return record
if record.waiting: # another question is still open
await self._save(record)
self._runs[run_id] = live or _Live(record, helpers)
return record
record.status = "running"
await self._save(record)
live = _Live(record, helpers)
self._runs[run_id] = live
self._launch(live, replay=True)
return record
async def resume(
self, run_id: str, *, runner: HelperRunner | None = None, reason: str = ""
) -> RunRecord:
"""Run `run_id` again from the top: after a restart ended it (interrupted), or after it
failed or was cancelled. Finished steps replay from the memo; after a failure or a cancel,
failed steps run again. A run that is done, running here, or waiting for an answer is left
as it is."""
live = self._runs.get(run_id)
if live is not None and live.task is not None and not live.task.done():
return live.record
record = live.record if live is not None else await self._load(run_id)
if record is None:
raise WorkflowError(f"No pipeline run {run_id}.")
if record.name not in self._pipelines:
raise WorkflowError(f"Pipeline {record.name!r} isn't registered in this process.")
if record.status == "done":
return record
if record.waiting:
if record.status != "waiting":
record.status = "waiting"
await self._save(record)
self._runs[run_id] = live or _Live(record, runner or self._helpers)
return record
helpers = runner or (live.helpers if live is not None else None) or self._helpers
if helpers is None:
raise WorkflowError("No helper runner: pass one to resume().")
if record.status in ("failed", "cancelled"):
await self._forget_failures(run_id)
record.status, record.error, record.finished_at = "running", "", None
await self._save(record)
live = _Live(record, helpers)
self._runs[run_id] = live
self._publish(
record,
ev.TASK_STARTED,
f"Resumed pipeline {record.name}{' ' + reason if reason else ''}: finished steps "
"replay from memo, the rest runs.",
resumed=True,
started_at=record.started_at,
)
self._launch(live, replay=True)
return record
async def cancel(self, run_id: str) -> RunRecord:
"""Stop a run and the helpers it has in flight. Its finished steps stay saved."""
live = self._runs.get(run_id)
record = live.record if live is not None else await self._load(run_id)
if record is None:
raise WorkflowError(f"No pipeline run {run_id}.")
if record.status not in _ENDED:
self._stop(record, live)
await self._save(record)
return record
async def status(self, run_id: str) -> RunRecord | None:
live = self._runs.get(run_id)
return live.record if live is not None else await self._load(run_id)
async def wait(self, run_id: str) -> RunRecord:
"""Wait until the run stops: done, failed, cancelled, or parked on a question."""
while True:
live = self._runs.get(run_id)
if live is None or live.task is None or live.task.done():
break
await asyncio.wait({live.task})
record = await self.status(run_id)
if record is None:
raise WorkflowError(f"No pipeline run {run_id}.")
return record
async def steps(self, run_id: str) -> list[dict[str, Any]]:
"""The run's memoized steps, oldest first: label, effort, status, task id, value or error."""
items = await self._store.asearch((*MEMO, run_id), limit=1000)
values = [{**_jsonable(item.value or {}), "key": item.key} for item in items]
return sorted(values, key=lambda value: float(value.get("finished_at") or 0))
async def sweep(
self,
*,
on_found: Callable[[RunRecord], Awaitable[None]] | None = None,
now: float | None = None,
) -> list[RunRecord]:
"""Find the runs a restart ended (still `running`, but no process has refreshed them for
`STALE_SECONDS`), mark each `interrupted` and publish `TASK_INTERRUPTED` for it, once.
`on_found` runs before each is published, so the host can say where to wake its lead.
The lead resumes each with `resume`. Called by the host's sweep, like `harness.ledger`'s."""
now = time.time() if now is None else now
items = await self._store.asearch(RUNS, filter={"status": "running"}, limit=200)
found = []
for item in items:
record = RunRecord.from_value(item.value or {})
last_seen = record.alive_at
if record.id in self._runs or now - last_seen < STALE_SECONDS:
continue
record.status = "interrupted"
await self._save(record)
if on_found is not None:
await on_found(record)
self._publish(
record,
ev.TASK_INTERRUPTED,
f"Pipeline {record.name} was in its {record.phase or 'first'} phase. Its finished "
"steps are saved: resuming it runs only the rest.",
started_at=record.started_at,
finished_at=last_seen,
)
found.append(record)
return found
# -- hooks for the host's status lines and cancel tool ---------------------------------------
def running_lines(self, thread_id: str) -> list[str]:
"""One line per run this process holds on the thread, for "still running" lists
(`harness.notices.RUNNING_PROVIDERS`)."""
now, lines = time.time(), []
for live in self._runs.values():
record = live.record
if record.thread_id != thread_id:
continue
if record.waiting:
question = record.waiting[0]["question"]
lines.append(
f"{record.id} (pipeline {record.name}, waiting for an answer: {question})"
)
elif record.status == "running":
steps = ", ".join(live.running.values())
doing = f"{record.phase or 'starting'}{': ' + steps if steps else ''}"
lines.append(
f"{record.id} (pipeline {record.name}, {doing}, {minutes(now - record.started_at)})"
)
return lines
def canceller(self, task_id: str, thread_id: str) -> str | None:
"""Cancel a run this process holds on the thread; None when `task_id` isn't one (for the
host's `cancel_task`, which asks each canceller in turn)."""
live = self._runs.get(task_id)
if live is None or live.record.thread_id != thread_id:
return None
record = live.record
self._stop(record, live)
self._spawn(self._save(record))
return (
f"Cancelled pipeline {task_id} ({record.name}) and the helpers it had running. Its "
"finished steps stay saved: resuming it continues from them."
)
# -- internals --------------------------------------------------------------------------------
def _launch(self, live: _Live, *, replay: bool) -> None:
live.task = asyncio.get_running_loop().create_task(
self._drive(live, replay=replay), name=f"pipeline {live.record.id}"
)
async def _drive(self, live: _Live, *, replay: bool) -> None:
record = live.record
fn = self._pipelines[record.name]
beat = asyncio.get_running_loop().create_task(
self._beat(record), name=f"pipeline heartbeat {record.id}"
)
try:
while True:
context = WorkflowContext(self, live, replaying=replay)
try:
result = await fn(context, **record.args)
except Suspended:
if record.waiting:
record.status = "waiting"
await self._save(record)
if record.waiting:
return
record.status = "running" # answered while it was being saved
replay = True
continue
except asyncio.CancelledError:
# A cancel has already recorded it; a shutdown leaves it running, for resume.
raise
except Exception as error: # noqa: BLE001 - reported to the lead, never raised
self._end(record, "failed", error=f"{type(error).__name__}: {error}"[:1500])
await self._save(record)
self._publish(
record,
ev.TASK_FAILED,
f"{record.error} Its finished steps are saved: resuming it runs only the "
"rest.",
error=record.error,
started_at=record.started_at,
finished_at=record.finished_at,
)
return
self._end(record, "done", result=_jsonable(result))
await self._save(record)
self._publish(
record,
ev.TASK_DONE,
report_text(record.result),
report=record.result,
started_at=record.started_at,
finished_at=record.finished_at,
)
return
finally:
beat.cancel()
def _end(self, record: RunRecord, status: str, *, result: Any = None, error: str = "") -> None:
record.status, record.result, record.error = status, result, error
record.finished_at = time.time()
live = self._runs.get(record.id)
if live is not None and live.record is record:
del self._runs[record.id]
def _stop(self, record: RunRecord, live: _Live | None) -> None:
self._end(record, "cancelled")
if live is not None and live.task is not None and not live.task.done():
live.task.cancel()
self._publish(
record,
ev.TASK_CANCELLED,
"Cancelled.",
started_at=record.started_at,
finished_at=record.finished_at,
)
async def _beat(self, record: RunRecord) -> None:
while True:
await asyncio.sleep(HEARTBEAT_SECONDS)
await self._save(record)
def _spawn(self, work: Awaitable[None]) -> None:
async def run() -> None:
await work
task = asyncio.get_running_loop().create_task(run())
self._background.add(task)
task.add_done_callback(self._background.discard)
def _publish(self, record: RunRecord, kind: str, text: str = "", **data: Any) -> None:
self._bus.publish(
ev.Event(
kind,
record.thread_id,
record.id,
label=record.name,
text=text,
data={"run_id": record.id, "pipeline": record.name, **data},
)
)
async def _save(self, record: RunRecord) -> None:
record.alive_at = time.time()
try:
await self._store.aput(RUNS, record.id, record.to_value())
except Exception as error: # noqa: BLE001 - the run goes on; only a restart would miss it
_LOGGER.warning("[harness] saving %s failed: %s", record.id, type(error).__name__)
async def _load(self, run_id: str) -> RunRecord | None:
item = await self._store.aget(RUNS, run_id)
return RunRecord.from_value(item.value) if item is not None and item.value else None
async def _memo_get(self, run_id: str, key: str) -> dict[str, Any] | None:
try:
item = await self._store.aget((*MEMO, run_id), key)
except Exception as error: # noqa: BLE001 - a miss: the step runs again
_LOGGER.warning(
"[harness] reading a step of %s failed: %s", run_id, type(error).__name__
)
return None
return _jsonable(item.value) if item is not None and item.value else None
async def _memo_put(self, run_id: str, key: str, value: Mapping[str, Any]) -> None:
try:
await self._store.aput((*MEMO, run_id), key, _jsonable(dict(value)))
except Exception as error: # noqa: BLE001 - only a replay would miss it (and re-run it)
_LOGGER.warning(
"[harness] saving a step of %s failed: %s", run_id, type(error).__name__
)
async def _forget_failures(self, run_id: str) -> None:
namespace = (*MEMO, run_id)
for item in await self._store.asearch(namespace, filter={"status": "failed"}, limit=1000):
await self._store.adelete(namespace, item.key)