Skip to content

Open-source design agent: on-brand stills, decks, diagrams, charts and motion graphics from a chat thread, built on Deep Agents.

56.4 KB
Raw Blame
"""The render watcher: a background loop that turns a job's log or status file into events.

A final render runs for minutes in the sandbox and prints nothing a person sees until it ends, so a
stuck render looks exactly like a slow one, and nothing tells the lead when a detached job ends.

A watch is a small loop in this process. Every `poll_seconds` it reads what is new from a source (a
growing log, a status file, a command's output), hands it to a parser, and publishes what changed:
- `WATCH_STARTED` once;
- `WATCH_PROGRESS` with `fraction` and `eta_seconds`, when the fraction moves a point, the phase
  changes, or at least every minute while output flows. It is recorded, and the running lines of
  `check_tasks` and every notice show the latest; the lead is not woken for it;
- `WATCH_DONE`, or `WATCH_ERROR` with the error lines, either of which ends the watch;
- `WATCH_STALLED` once when nothing has come for `stall_seconds` (default 5 minutes): no new output
  and no progress. Silence is not success: a render the OOM killer took prints nothing. The watch
  keeps reading, and if output resumes it says so with an urgent progress event, since the lead was
  told it stalled;
- `WATCH_STALLED` with `timeout` when an optional overall timeout passes, which ends the watch.

Sources and parsers are separate so each stays small. A source only fetches new text: byte offsets
over `SandboxExec` (`wc -c`, then `tail -c +N`), robust to a truncated log. A parser only knows a
format (`parser_for`): `hyperframes` (the `hyperframes render` CLI), `ffmpeg` (`-progress`), `json`
(a status file), `regex` (any N/M counter) or `heartbeat` (the default: any output is a sign of
life, a failure line ends it). The filter must match every terminal state (errors, kills, OOM), not
only success. `harness.offload` reuses the runner for detached jobs.

`WatchLedger` keeps each watch's record in the deployment store, so a restart neither loses an end
the lead had not heard nor a watch it cut off: that one is reported as interrupted and stays
resumable for `RESUMABLE_SECONDS` (a day).

Deep Agents' `execute` runs a command to completion inside the tool call, with no background
command and no watcher; this module would sit beside it upstream.
"""

from __future__ import annotations

import asyncio
import builtins
import inspect
import json
import logging
import math
import re
import shlex
import time
from collections import deque
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass, field, replace
from typing import Any, Literal, Protocol, runtime_checkable

from design_agent.harness import events as ev
from design_agent.harness.ledger import HEARTBEAT_SECONDS, STALE_SECONDS
from design_agent.harness.protocols import SandboxExec
from design_agent.harness.tasks import minutes, new_id

_LOGGER = logging.getLogger(__name__)

#: A watch the lead stopped is recorded only. events.py has no watch kind for that yet,
#: so it goes out as `task.cancelled`, which has that delivery and ends its source; `data["watch"]`
#: marks it. Swap in a `watch.cancelled` kind once events.py and policy.py have one.
WATCH_CANCELLED = ev.TASK_CANCELLED

#: Progress events: when the fraction moves this much, or at least this often while output flows.
PROGRESS_STEP = 0.01
PROGRESS_EVERY_SECONDS = 60.0
#: The ETA is a straight line through the progress of this recent window.
ETA_WINDOW_SECONDS = 300.0
#: Lines kept for context (the error text, a stall, a failed job's report).
TAIL_LINES = 20
#: A read that takes longer is abandoned and counted as failed, so a hung call can't hide a stall.
READ_TIMEOUT_SECONDS = 120.0
#: Ended watches stay listed this long.
KEEP_ENDED_SECONDS = 24 * 3600.0

#: Lines that mean a job failed, for any log: every terminal state, not only success. `Error\b`
#: catches `TypeError:` but not `ErrorBoundary`.
FAILURE = (
    r"Error\b|error:|✖|Traceback|ENOENT|\bKilled\b|\bOOM\b|out of memory|Segmentation fault"
    r"|exited with code [1-9]"
)
#: The ffmpeg messages that end it. "Error while decoding" is routine and ffmpeg carries on.
FFMPEG_FAILURE = (
    r"(?i)conversion failed|error opening (?:input|output)|error initializing"
    r"|no such file or directory|unrecognized option|unknown encoder|invalid argument"
    r"|\bkilled\b|out of memory|exited with code [1-9]"
)

Pattern = str | re.Pattern[str]


# -- what a parser reads ----------------------------------------------------------------------------


@dataclass(frozen=True)
class Update:
    """What a parser found in new output.

    `kind` is `progress` (a counter or status moved), `error` (a line that means it failed), `done`
    (a line that means it finished) or `line` (anything else, kept for context). `fraction` is 0-1
    when known; `eta_seconds` when the source states one; `phase` names the stage ("rendering",
    "encoding", a status file's `step`); `data` carries extra facts (`cost_usd`, `output`) into
    the event.
    """

    kind: Literal["progress", "error", "done", "line"]
    fraction: float | None = None
    eta_seconds: float | None = None
    text: str = ""
    phase: str = ""
    data: Mapping[str, Any] = field(default_factory=dict)


@runtime_checkable
class WatchSource(Protocol):
    """Where a watch reads from.

    `read` returns the text that is new since the last read, or "" when nothing is. `closed` turns
    true once nothing more can come. `note` says why the last read had nothing (the file does not
    exist yet, the JSON is torn), for the stall message.
    """

    closed: bool
    note: str

    async def read(self) -> str: ...


@runtime_checkable
class WatchParser(Protocol):
    def parse(self, text: str) -> list[Update]:
        """Updates for a source's new text. "" means the source was quiet this poll; a line parser
        then finishes a partial line it was holding."""
        ...


# -- parsers ----------------------------------------------------------------------------------------

_ANSI = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)")
_BREAK = re.compile(r"\r\n|\r|\n")


def _patterns(value: Pattern | Sequence[Pattern] | None) -> list[re.Pattern[str]]:
    if value is None or isinstance(value, str | re.Pattern):
        value = [value] if value else []
    return [p if isinstance(p, re.Pattern) else re.compile(p) for p in value if p]


class LineParser:
    """Line-oriented output: split on newlines and carriage returns (a TTY progress bar redraws with
    `\\r`), colors stripped, a partial last line held until it completes or the source goes quiet.
    Each line is checked for done, then error, then progress (`progress`, which subclasses define).
    """

    def __init__(
        self,
        *,
        error: Pattern | Sequence[Pattern] | None = FAILURE,
        done: Pattern | Sequence[Pattern] | None = None,
    ) -> None:
        self._error = _patterns(error)
        self._done = _patterns(done)
        self._held = ""

    def parse(self, text: str) -> list[Update]:
        if text:
            pieces = _BREAK.split(self._held + text)
            self._held = pieces.pop()
        else:
            pieces, self._held = [self._held], ""
        updates: list[Update] = []
        for piece in pieces:
            line = _ANSI.sub("", piece).strip()
            if line:
                updates.extend(self.parse_line(line))
        return updates

    def parse_line(self, line: str) -> list[Update]:
        if any(p.search(line) for p in self._done):
            return [Update("done", fraction=1.0, text=line)]
        if any(p.search(line) for p in self._error):
            return [Update("error", text=line)]
        return self.progress(line)

    def progress(self, line: str) -> list[Update]:
        return [Update("line", text=line)]


def _counter(match: re.Match[str] | None) -> tuple[int, int] | None:
    """A done/total pair from a match, or None when it can't be a counter (0 total, done > total:
    a partial line)."""
    if match is None:
        return None
    done, total = int(match[1]), int(match[2])
    return (done, total) if 0 < total and 0 <= done <= total else None


class HeartbeatParser(LineParser):
    """Any log, read for signs of life only: each new line resets the stall timer and is kept for
    context, and a line that means failure (`FAILURE`) ends the watch. It reports no percentage.
    The default format: a detached job's exit code still says how it ended."""


#: What ends a HyperFrames render as failed: its error box (`✗  Render failed`, `✗  Not a
#: directory: …`), a `Failed:` or `Render cancelled` progress line, or the process being killed.
#: Generic `Error` lines are left out on purpose: a best-effort render logs the page's own script
#: errors as warnings and still finishes.
HYPERFRAMES_FAILURE = (
    r"^✗\s+\S",
    r"^[█░]*\s*\d{1,3}%\s+(?:Failed:|Render cancelled)",
    r"\bKilled\b|\bOOM\b|out of memory|Segmentation fault|exited with code [1-9]",
)

#: Phase names for HyperFrames' stage codes and progress-line labels, matched by keyword in order.
_HF_PHASES = (
    ("compil", "compiling"),
    ("extract", "extracting video"),
    ("audio", "processing audio"),
    ("start", "starting capture"),
    ("browser", "starting capture"),
    ("captur", "capturing frames"),
    ("stream", "capturing frames"),
    ("encod", "encoding"),
    ("assembl", "assembling"),
)
_COUNTS = re.compile(r"\([^)]*\)|\d+\s*/\s*\d+")
_FRAME_PHASES = frozenset({"capturing frames", "encoding"})


def _percent(value: object) -> float | None:
    """A percent (0-100) as a fraction (0-1); None when it is not a number."""
    number = _number(value)
    return None if number is None or number < 0 else min(1.0, number / 100)


def _hf_phase(stage: str) -> str:
    """One phase name for a stage, whether it came as a code (`capture`) or a label
    (`Streaming frame 12/60`), so the two forms of one step never read as a phase change."""
    lowered = stage.lower()
    for key, phase in _HF_PHASES:
        if key in lowered:
            return phase
    return " ".join(_COUNTS.sub("", lowered).replace("_", " ").split()) or "rendering"


class HyperFramesParser(LineParser):
    """`hyperframes render` output, as a detached job's log (no TTY) or a terminal's.

    Without a TTY the CLI writes `@hf-progress {"code": …, "pct": …, "done": …, "total": …}` for
    every step, beside a plain progress line (`████░░  42%  Streaming frame 25/60`); under a TTY
    only the progress line is drawn, redrawn with `\\r`. Either gives the fraction (`pct` / 100), a
    phase (compiling, capturing frames, encoding, assembling) and frame counts. Done at
    `100%  Render complete`, with the output path from the `◇  <path>` line after it (the done line
    waits one read for it, and a quiet poll gives up). Failed on `HYPERFRAMES_FAILURE`.
    """

    MACHINE = re.compile(r"^@hf-progress\s+(\{.*\})\s*$")
    BAR = re.compile(r"^[█░]+\s+(\d{1,3})%\s+(\S.*)$")
    COMPLETE = r"^[█░]*\s*100%\s+Render complete\b"
    OUTPUT = re.compile(r"^◇\s+(/\S.*)$")
    LOG_TAG = re.compile(r"\[[\w:-]+\]")
    GLYPHS = re.compile(r"^[█░]+\s+")

    def __init__(
        self,
        *,
        error: Pattern | Sequence[Pattern] | None = HYPERFRAMES_FAILURE,
        done: Pattern | Sequence[Pattern] | None = None,
    ) -> None:
        super().__init__(error=error, done=[self.COMPLETE, *_patterns(done)])
        #: A done line waiting one read for the output path that follows it.
        self._held_done: Update | None = None

    def parse(self, text: str) -> list[Update]:
        updates = [replace(u, text=self.GLYPHS.sub("", u.text)) for u in super().parse(text)]
        if self._held_done is not None:
            updates.insert(0, self._held_done)
            self._held_done = None
        for index, update in enumerate(updates):
            if update.kind != "done":
                continue
            for later in updates[index + 1 :]:
                found = self.OUTPUT.match(later.text)
                if found is not None:
                    data = {**update.data, "output": found[1]}
                    shown = f"Render complete: {found[1]}"
                    updates[index] = replace(update, text=shown, data=data)
                    break
            else:
                if text:  # the path may be in the next read; a quiet poll gives up on it
                    self._held_done = updates.pop(index)
            break
        return updates

    def progress(self, line: str) -> list[Update]:
        machine = self.MACHINE.match(line)
        if machine is not None:
            return self._machine(line, machine[1])
        bar = self.BAR.match(line)
        if bar is None:
            return [Update("line", text=line)]
        # Under a TTY the bar has no newline, so the next log line can land on the same line.
        stage = self.LOG_TAG.split(bar[2], maxsplit=1)[0].strip() or bar[2]
        phase = _hf_phase(stage)
        counts = _counter(re.search(r"(\d+)\s*/\s*(\d+)", stage))
        data = (
            {"frames": counts[0], "total_frames": counts[1]}
            if counts is not None and phase in _FRAME_PHASES
            else {}
        )
        return [Update("progress", _percent(bar[1]), text=stage, phase=phase, data=data)]

    def _machine(self, line: str, payload: str) -> list[Update]:
        try:
            value = json.loads(payload)
        except ValueError:
            return [Update("line", text=line)]
        if not isinstance(value, Mapping):
            return [Update("line", text=line)]
        phase = _hf_phase(str(value.get("code") or ""))
        done, total = value.get("done"), value.get("total")
        counted = isinstance(done, int) and isinstance(total, int) and 0 < total
        data = {"frames": done, "total_frames": total} if counted and phase in _FRAME_PHASES else {}
        text = f"{phase} {done}/{total}" if counted else phase
        return [Update("progress", _percent(value.get("pct")), text=text, phase=phase, data=data)]


def _number(value: object) -> float | None:
    try:
        number = float(str(value).strip().rstrip("x"))
    except ValueError:
        return None
    return number if math.isfinite(number) else None


def _clock_seconds(value: str) -> float | None:
    parts = value.strip().split(":")
    if len(parts) != 3 or value.strip().startswith("-"):
        return None
    numbers = [_number(p) for p in parts]
    if any(n is None for n in numbers):
        return None
    hours, mins, secs = (n or 0.0 for n in numbers)
    return hours * 3600 + mins * 60 + secs


class FfmpegProgressParser(LineParser):
    """ffmpeg's `-progress` blocks (`frame=`, `out_time_us=`, `speed=`, ending `progress=continue`
    or `progress=end`, every half second), and its ordinary stats line as a fallback.

    The fraction needs `total_frames` or `duration_seconds`; without them progress carries the frame
    and time only. `progress=end` is reported done once a quiet poll confirms nothing follows it,
    because on a failure ffmpeg prints its last block before "Conversion failed!". Only messages
    that end ffmpeg count as errors (`FFMPEG_FAILURE`).
    """

    KEY = re.compile(r"^([a-z_0-9]+)=(\S*)$")
    STATS = re.compile(
        r"frame=\s*(\d+)(?:.*?fps=\s*([\d.]+))?.*?time=\s*(-?[\d:.]+)(?:.*?speed=\s*([\d.]+)x)?"
    )

    def __init__(
        self,
        total_frames: int | None = None,
        duration_seconds: float | None = None,
        *,
        error: Pattern | Sequence[Pattern] | None = FFMPEG_FAILURE,
        done: Pattern | Sequence[Pattern] | None = None,
    ) -> None:
        super().__init__(error=error, done=done)
        self._total = total_frames or None
        self._duration = duration_seconds or None
        self._block: dict[str, str] = {}
        self._ended = False

    def parse(self, text: str) -> list[Update]:
        updates = super().parse(text)
        if not text and self._ended:
            self._ended = False
            updates.append(Update("done", fraction=1.0, text="ffmpeg finished (progress=end)."))
        return updates

    def progress(self, line: str) -> list[Update]:
        key = self.KEY.match(line)
        if key is not None:
            self._block[key[1]] = key[2]
            if key[1] != "progress":
                return []
            block, self._block = self._block, {}
            if key[2] == "end":
                self._ended = True
            seconds = next(
                (
                    n / 1e6
                    for n in (_number(block.get("out_time_us")), _number(block.get("out_time_ms")))
                    if n is not None and n >= 0
                ),
                _clock_seconds(block.get("out_time", "")),
            )
            frame = _number(block.get("frame"))
            return [
                self._update(frame, seconds, _number(block.get("speed")), _number(block.get("fps")))
            ]
        stats = self.STATS.search(line)
        if stats is None:
            return [Update("line", text=line)]
        return [
            self._update(
                _number(stats[1]), _clock_seconds(stats[3]), _number(stats[4]), _number(stats[2])
            )
        ]

    def _update(
        self, frame: float | None, seconds: float | None, speed: float | None, fps: float | None
    ) -> Update:
        fraction = eta = None
        if self._total and frame is not None:
            fraction = min(1.0, frame / self._total)
            eta = max(0.0, (self._total - frame) / fps) if fps else None
        elif self._duration and seconds is not None:
            fraction = min(1.0, seconds / self._duration)
            eta = max(0.0, (self._duration - seconds) / speed) if speed else None
        parts = []
        if frame is not None:
            parts.append(f"frame {int(frame)}" + (f"/{self._total}" if self._total else ""))
        if seconds is not None:
            parts.append(f"{seconds:.1f}s" + (f" of {self._duration:g}s" if self._duration else ""))
        if speed:
            parts.append(f"{speed:g}x")
        data = {
            key: value
            for key, value in (("frame", frame), ("out_seconds", seconds), ("speed", speed))
            if value is not None
        }
        return Update("progress", fraction, eta, ", ".join(parts), phase="encoding", data=data)


def _fraction(value: object) -> float | None:
    number = _number(value)
    if number is None or number < 0:
        return None
    return min(1.0, number / 100 if number > 1 else number)


def _eta(value: object) -> float | None:
    if isinstance(value, str) and ":" in value:
        return _clock_seconds(value)
    if isinstance(value, str) and value.strip().endswith(("s", "m", "h")):
        number = _number(value.strip()[:-1])
        scale = {"s": 1, "m": 60, "h": 3600}[value.strip()[-1]]
        return number * scale if number is not None else None
    number = _number(value)
    return number if number is not None and number >= 0 else None


class JsonStatusParser:
    """A JSON status file, such as a project's run.json: `status`, `step`, `progress` or `fraction`
    (0-1, or a percent), `eta` (seconds, "90s", "3m" or "00:03:00"). A status of done, success,
    complete or finished ends the watch as done; failed or error, as an error with the file's
    `error` or `message`. A `cost` or `output` field rides along in the event."""

    DONE = frozenset({"done", "success", "succeeded", "complete", "completed", "finished"})
    FAILED = frozenset({"failed", "failure", "error", "errored"})

    def parse(self, text: str) -> list[Update]:
        if not text.strip():
            return []
        try:
            value = json.loads(text)
        except ValueError:
            return [Update("line", text=text.strip()[:300])]
        if not isinstance(value, Mapping):
            return [Update("line", text=text.strip()[:300])]
        status = str(value.get("status") or "").strip()
        step = str(value.get("step") or "").strip()
        data = {
            key: value[key]
            for key in ("cost", "cost_usd", "output")
            if value.get(key) not in (None, "")
        }
        summary = ", ".join(part for part in (status, step) if part) or text.strip()[:200]
        if status.lower() in self.DONE:
            return [Update("done", fraction=1.0, text=summary, data=data)]
        if status.lower() in self.FAILED:
            reason = str(value.get("error") or value.get("message") or "").strip()
            return [Update("error", text=f"{summary}: {reason}" if reason else summary, data=data)]
        fraction = _fraction(value.get("progress", value.get("fraction")))
        return [
            Update(
                "progress",
                fraction=fraction,
                eta_seconds=_eta(value.get("eta")),
                text=summary,
                phase=step or status,
                data=data,
            )
        ]


class RegexParser(LineParser):
    """Any log with a counter. `progress` has two groups (done, total) or one (a percent); the
    default reads "120/300", as in `render: 120/300 frames`. `error` and
    `done` are the lines that end the watch (by default errors are `FAILURE`, and nothing is done:
    such a job's end shows as a stall unless something prints a done line)."""

    def __init__(
        self,
        progress: Pattern | None = r"(\d+)\s*/\s*(\d+)",
        *,
        error: Pattern | Sequence[Pattern] | None = FAILURE,
        done: Pattern | Sequence[Pattern] | None = None,
    ) -> None:
        super().__init__(error=error, done=done)
        self._progress = _patterns(progress)

    def progress(self, line: str) -> list[Update]:
        match = next((m for p in self._progress if (m := p.search(line))), None)
        groups = [g for g in (match.groups() if match else ()) if g is not None]
        numbers = [_number(g) for g in groups[:2]]
        fraction = None
        if len(numbers) == 2 and numbers[0] is not None and numbers[1]:
            if 0 <= numbers[0] <= numbers[1]:
                fraction = numbers[0] / numbers[1]
        elif len(numbers) == 1 and numbers[0] is not None and 0 <= numbers[0] <= 100:
            fraction = numbers[0] / 100
        if fraction is None:
            return [Update("line", text=line)]
        return [Update("progress", fraction=fraction, text=line)]


# -- choosing a parser ------------------------------------------------------------------------------

ParserKind = Literal["hyperframes", "ffmpeg", "json", "regex", "heartbeat"]
PARSER_KINDS: tuple[ParserKind, ...] = ("hyperframes", "ffmpeg", "json", "regex", "heartbeat")
DEFAULT_PARSER: ParserKind = "heartbeat"


def parser_for(
    kind: str = DEFAULT_PARSER,
    *,
    total_frames: int = 0,
    duration_seconds: float = 0,
    progress_regex: str = "",
    error_regex: str = "",
    done_regex: str = "",
) -> WatchParser:
    """The parser for output of `kind` (`PARSER_KINDS`).

    `error_regex` replaces the lines that mean failure; `done_regex` adds a line that means done;
    `progress_regex` is the `regex` kind's counter; `total_frames` or `duration_seconds` give
    `ffmpeg` a percentage. `json` reads a status file, so its source is `SandboxJsonSource`.
    Raises `ValueError` for an unknown kind and `re.error` for a bad pattern.
    """
    done = done_regex or None
    if kind == "hyperframes":
        return HyperFramesParser(error=error_regex or HYPERFRAMES_FAILURE, done=done)
    if kind == "ffmpeg":
        return FfmpegProgressParser(
            total_frames or None,
            duration_seconds or None,
            error=error_regex or FFMPEG_FAILURE,
            done=done,
        )
    if kind == "json":
        return JsonStatusParser()
    if kind == "regex":
        return RegexParser(
            progress_regex or r"(\d+)\s*/\s*(\d+)", error=error_regex or FAILURE, done=done
        )
    if kind == "heartbeat":
        return HeartbeatParser(error=error_regex or FAILURE, done=done)
    raise ValueError(f"unknown kind {kind!r}; use {', '.join(PARSER_KINDS)}")


# -- sources ----------------------------------------------------------------------------------------

#: One read returns at most this many new bytes. A log that grew more between polls is read from its
#: last `READ_BYTES`: the newest output is what says how the job is doing.
READ_BYTES = 256 * 1024
#: Seconds one sandbox command may run.
EXEC_TIMEOUT = 60
_MARK = "@@watch"
_END = "@@watch-end"


class WatchReadError(RuntimeError):
    """A source could not read (its sandbox command failed). The watch keeps polling; the failure
    goes into the stall message if reads keep failing."""


def _after_mark(output: str) -> tuple[str, str]:
    """The header line after the marker, and everything after it."""
    at = output.find(_MARK)
    if at < 0:
        return "", output
    header, _, body = output[at:].partition("\n")
    return header.strip(), body


class SandboxFileSource:
    """New bytes of a growing file in the sandbox, by byte offset: `wc -c`, then `tail -c +N`.

    A file that shrank (truncated, or rewritten from the start) is read again from the top. A file
    that doesn't exist yet reads as nothing, so a watch can start before the job writes its log.
    The output ends with a marker so a trailing newline survives transport, and stderr is kept out
    of it (some sandbox backends append stderr to stdout).
    """

    def __init__(
        self,
        exec: SandboxExec,
        path: str,
        *,
        max_bytes: int = READ_BYTES,
        timeout: int = EXEC_TIMEOUT,
    ) -> None:
        self.path = path
        self.offset = 0
        self.closed = False
        self.note = ""
        self._exec = exec
        self._max = max_bytes
        self._timeout = timeout

    def command(self) -> str:
        return (
            f"f={shlex.quote(self.path)}; o={self.offset}; cap={self._max}\n"
            f'if [ ! -f "$f" ]; then echo "{_MARK} missing"; exit 0; fi\n'
            's=$( { wc -c < "$f"; } 2>/dev/null | tr -d " \\n"); s=${s:-0}\n'
            'if [ "$s" -lt "$o" ]; then o=0; fi\n'
            'if [ $((s - o)) -gt "$cap" ]; then o=$((s - cap)); fi\n'
            f'echo "{_MARK} $s $o"\n'
            'tail -c +$((o + 1)) "$f" 2>/dev/null | head -c $((s - o)) 2>/dev/null\n'
            "printf '\\n" + _END + "\\n'"
        )

    async def read(self) -> str:
        result = await self._exec.execute(self.command(), timeout=self._timeout)
        header, body = _after_mark(result.output)
        if header == f"{_MARK} missing":
            self.offset = 0  # a file that comes back is a new file
            self.note = f"{self.path} does not exist yet"
            return ""
        fields = header.split()
        end = body.rfind("\n" + _END)
        if len(fields) != 3 or end < 0 or not all(f.isdigit() for f in fields[1:]):
            raise WatchReadError(f"reading {self.path} failed: {result.output.strip()[-300:]}")
        body = body[:end]
        size, start = int(fields[1]), int(fields[2])
        truncated = size < self.offset
        if start > 0 and start != self.offset:
            body = body.partition("\n")[2]  # it starts mid-line after the skipped bytes
        self.offset = size
        self.note = f"{self.path} was truncated; reading it again from the top" if truncated else ""
        return body


class SandboxJsonSource:
    """A small JSON status file (a project's run.json), re-read each poll and returned, in
    canonical form, when its content changes. A torn or half-written file is skipped until it
    parses."""

    def __init__(
        self,
        exec: SandboxExec,
        path: str,
        *,
        max_bytes: int = 64 * 1024,
        timeout: int = EXEC_TIMEOUT,
    ) -> None:
        self.path = path
        self.closed = False
        self.note = ""
        self._exec = exec
        self._max = max_bytes
        self._timeout = timeout
        self._last: str | None = None

    def command(self) -> str:
        return (
            f"f={shlex.quote(self.path)}\n"
            f'if [ -f "$f" ]; then echo "{_MARK} file"; head -c {self._max} "$f" 2>/dev/null; '
            f'else echo "{_MARK} missing"; fi'
        )

    async def read(self) -> str:
        result = await self._exec.execute(self.command(), timeout=self._timeout)
        header, body = _after_mark(result.output)
        if header == f"{_MARK} missing":
            self.note = f"{self.path} does not exist yet"
            return ""
        if header != f"{_MARK} file":
            raise WatchReadError(f"reading {self.path} failed: {result.output.strip()[-300:]}")
        try:
            value = json.loads(body)
        except ValueError:
            self.note = f"{self.path} is not valid JSON (yet)"
            return ""
        self.note = ""
        text = json.dumps(value, sort_keys=True)
        if text == self._last:
            return ""
        self._last = text
        return text


class CommandSource:
    """A command's output, run each poll and returned when it differs from the last: a status
    endpoint, or a frame count (`echo "$(ls frames | wc -l)/300"`)."""

    def __init__(self, exec: SandboxExec, command: str, *, timeout: int = EXEC_TIMEOUT) -> None:
        self.command = command
        self.closed = False
        self.note = ""
        self._exec = exec
        self._timeout = timeout
        self._last: str | None = None

    async def read(self) -> str:
        result = await self._exec.execute(self.command, timeout=self._timeout)
        ok = result.exit_code in (0, None)
        self.note = "" if ok else f"the command exited with {result.exit_code}"
        if result.output == self._last:
            return ""
        self._last = result.output
        if not result.output.strip():
            return ""
        return result.output if result.output.endswith("\n") else result.output + "\n"


# -- the runner -------------------------------------------------------------------------------------

Status = Literal["watching", "done", "error", "timed_out", "cancelled", "interrupted"]
#: Called after every poll and once when the watch ends (not on a shutdown), so an adapter can keep
#: a heartbeat record.
Beat = Callable[["WatchRecord"], Awaitable[None] | None]


@dataclass
class WatchRecord:
    """One watch and what it has seen."""

    id: str
    thread_id: str
    label: str
    started_at: float
    poll_seconds: float
    stall_seconds: float
    timeout_seconds: float | None = None
    status: Status = "watching"
    fraction: float | None = None
    eta_seconds: float | None = None
    phase: str = ""
    #: The latest progress line.
    text: str = ""
    stalled: bool = False
    polls: int = 0
    last_output_at: float | None = None
    ended_at: float | None = None
    #: The event that ended it, for an adapter that keeps it until the lead has heard.
    final: ev.Event | None = None
    #: Adapter facts (the file, the parser kind, a job id), published with `WATCH_STARTED`.
    meta: dict[str, Any] = field(default_factory=dict)
    tail: deque[str] = field(default_factory=lambda: deque(maxlen=TAIL_LINES), repr=False)
    handle: asyncio.Task[None] | None = field(default=None, repr=False)
    #: Set by `cancel`, so a stop the lead asked for is told apart from a shutdown.
    cancel_requested: bool = False

    def describe(self) -> str:
        """One line: where it stands."""
        state = {
            "watching": "stalled" if self.stalled else "watching",
            "timed_out": "timed out",
        }.get(self.status, self.status)
        parts = [state]
        if self.fraction is not None:
            parts.append(f"{self.fraction * 100:.0f}%")
        if self.phase:
            parts.append(self.phase)
        if self.status == "watching" and self.eta_seconds is not None:
            parts.append(f"about {_duration(self.eta_seconds)} left")
        line = f"{self.id} ({self.label}): {', '.join(parts)}"
        return f"{line}; last line: {self.tail[-1]}" if self.tail else line


def _duration(seconds: float) -> str:
    return f"{max(1, round(seconds))} s" if seconds < 90 else minutes(seconds)


class _Eta:
    """A straight line through the recent (time, fraction) points."""

    def __init__(self, window: float) -> None:
        self._window = window
        self._points: deque[tuple[float, float]] = deque()

    def add(self, at: float, fraction: float) -> None:
        if self._points and fraction < self._points[-1][1]:
            self._points.clear()  # it went back: a new pass or a restarted job
        self._points.append((at, fraction))
        while len(self._points) > 2 and at - self._points[0][0] > self._window:
            self._points.popleft()

    def estimate(self) -> float | None:
        if len(self._points) < 2:
            return None
        (t0, f0), (t1, f1) = self._points[0], self._points[-1]
        if f1 <= f0 or t1 <= t0:
            return None
        return max(0.0, (1.0 - f1) * (t1 - t0) / (f1 - f0))


@dataclass
class _Loop:
    """A watch's working state, beside its public record."""

    eta: _Eta
    last_activity: float
    published_at: float = 0.0
    published_fraction: float | None = None
    published_phase: str = ""
    failure: str = ""


class WatchRunner:
    """Runs watches as asyncio tasks in this process (module doc). One runner per process is
    enough; an adapter keeps it at module level because a watch outlives the call that started it.
    """

    def __init__(
        self,
        *,
        clock: Callable[[], float] = time.time,
        progress_step: float = PROGRESS_STEP,
        progress_every_seconds: float = PROGRESS_EVERY_SECONDS,
        eta_window_seconds: float = ETA_WINDOW_SECONDS,
        read_timeout_seconds: float = READ_TIMEOUT_SECONDS,
    ) -> None:
        self._clock = clock
        self._step = progress_step
        self._every = progress_every_seconds
        self._window = eta_window_seconds
        self._read_timeout = read_timeout_seconds
        self._records: dict[str, WatchRecord] = {}

    def start(
        self,
        thread_id: str,
        *,
        label: str,
        source: WatchSource,
        parser: WatchParser,
        poll_seconds: float = 15.0,
        stall_seconds: float = 300.0,
        timeout_seconds: float | None = None,
        bus: ev.EventBus | None = None,
        on_beat: Beat | None = None,
        meta: Mapping[str, Any] | None = None,
        watch_id: str | None = None,
    ) -> str:
        """Start watching and return the watch id (`r-xxxxxx`) at once. Needs a running loop."""
        bus = bus or ev.BUS
        self._prune()
        record = WatchRecord(
            id=watch_id or new_id("r"),
            thread_id=thread_id,
            label=label,
            started_at=self._clock(),
            poll_seconds=poll_seconds,
            stall_seconds=stall_seconds,
            timeout_seconds=timeout_seconds or None,
            meta=dict(meta or {}),
        )
        self._records[record.id] = record
        bus.publish(
            ev.Event(
                ev.WATCH_STARTED,
                thread_id,
                record.id,
                label=label,
                text=f"Watching {label}.",
                data={
                    **record.meta,
                    "poll_seconds": poll_seconds,
                    "stall_seconds": stall_seconds,
                    "timeout_seconds": record.timeout_seconds,
                },
            )
        )
        record.handle = asyncio.get_running_loop().create_task(
            self._watch(record, source, parser, bus, on_beat), name=f"watch {record.id}"
        )
        return record.id

    def cancel(self, watch_id: str) -> bool:
        """Stop a running watch. Its end is recorded only; nothing wakes the lead."""
        record = self._records.get(watch_id)
        if record is None or record.status != "watching" or record.handle is None:
            return False
        if record.cancel_requested or record.handle.done():
            return False
        record.cancel_requested = True
        record.handle.cancel()
        return True

    def get(self, watch_id: str) -> WatchRecord | None:
        return self._records.get(watch_id)

    def running_lines(self, thread_id: str) -> builtins.list[str]:
        """One line per running watch, for notices and `check_tasks`:
        'r-1a2b3c (watch, final render, 42%, about 3 min left, 12 min)'."""
        now = self._clock()
        lines = []
        for record in self.list(thread_id):
            if record.status != "watching":
                continue
            parts = ["watch", record.label]
            if record.stalled:
                parts.append("stalled")
            if record.fraction is not None:
                parts.append(f"{record.fraction * 100:.0f}%")
            if record.eta_seconds is not None and not record.stalled:
                parts.append(f"about {_duration(record.eta_seconds)} left")
            parts.append(minutes(now - record.started_at))
            lines.append(f"{record.id} ({', '.join(parts)})")
        return lines

    def clear(self) -> None:
        """Stop every watch and forget them all (tests, shutdown)."""
        for record in self._records.values():
            if record.handle is not None and not record.handle.done():
                record.handle.cancel()
        self._records.clear()

    def _prune(self) -> None:
        cutoff = self._clock() - KEEP_ENDED_SECONDS
        for watch_id, record in builtins.list(self._records.items()):
            if record.ended_at is not None and record.ended_at < cutoff:
                del self._records[watch_id]

    # -- the loop -----------------------------------------------------------------------------

    async def _watch(
        self,
        record: WatchRecord,
        source: WatchSource,
        parser: WatchParser,
        bus: ev.EventBus,
        on_beat: Beat | None,
    ) -> None:
        loop = _Loop(eta=_Eta(self._window), last_activity=record.started_at)
        try:
            while not await self._poll(record, source, parser, bus, loop):
                await _beat(on_beat, record)
                await asyncio.sleep(record.poll_seconds)
        except asyncio.CancelledError:
            if record.status == "watching":
                record.ended_at = self._clock()
                if record.cancel_requested:
                    record.status = "cancelled"
                    record.final = self._publish(
                        bus, record, WATCH_CANCELLED, "Stopped watching.", {"watch": True}
                    )
                    await _beat(on_beat, record)
                else:
                    # A shutdown: the heartbeat record stays behind for the sweep to report.
                    record.status = "interrupted"
            raise
        except Exception as error:  # noqa: BLE001 - a watch that breaks says so, never silently
            _LOGGER.warning("[harness] watch %s broke: %s", record.id, type(error).__name__)
            self._end(
                record, bus, ev.WATCH_ERROR, "error", f"The watch itself failed: {error}"[:500]
            )
        await _beat(on_beat, record)

    async def _read(self, source: WatchSource, loop: _Loop) -> str:
        try:
            text = await asyncio.wait_for(source.read(), timeout=self._read_timeout)
        except asyncio.CancelledError:
            raise
        except Exception as error:  # noqa: BLE001 - counted, reported with the stall
            loop.failure = f"{type(error).__name__}: {str(error)[:300]}"
            _LOGGER.info("[harness] a watch read failed: %s", loop.failure)
            return ""
        loop.failure = ""
        return text

    async def _poll(
        self,
        record: WatchRecord,
        source: WatchSource,
        parser: WatchParser,
        bus: ev.EventBus,
        loop: _Loop,
    ) -> bool:
        """One read and everything it calls for. True when the watch has ended."""
        text = await self._read(source, loop)
        now = self._clock()
        record.polls += 1
        updates = parser.parse(text)
        if source.closed:
            updates += parser.parse("")
        moved = any(
            u.kind == "progress" and u.fraction is not None and u.fraction != record.fraction
            for u in updates
        )
        silent_for = now - loop.last_activity
        resumed = record.stalled and bool(text or moved)
        if text or moved:
            loop.last_activity = now
            record.last_output_at = now
            record.stalled = False

        lines = [u.text for u in updates if u.text]
        record.tail.extend(lines)
        errors = [i for i, u in enumerate(updates) if u.kind == "error"]
        done = next((u for u in updates if u.kind == "done"), None)
        if errors:
            texts = [u.text for u in updates[errors[0] :] if u.text][:TAIL_LINES]
            data: dict[str, Any] = {"lines": texts, "tail": builtins.list(record.tail)}
            if done is not None:
                data["done_line"] = done.text
            self._end(record, bus, ev.WATCH_ERROR, "error", "\n".join(texts)[:2000], data)
            return True
        if done is not None:
            record.fraction = done.fraction if done.fraction is not None else 1.0
            data = {"took_seconds": round(now - record.started_at, 1), **done.data}
            self._end(record, bus, ev.WATCH_DONE, "done", done.text or "Finished.", data)
            return True

        self._progress(record, updates, now, bus, loop, resumed, silent_for)

        if source.closed:
            note = f" ({source.note})" if source.note else ""
            tail = "\n".join(record.tail)
            text = f"It ended without a done or error line{note}." + (
                f" Last lines:\n{tail}" if tail else ""
            )
            self._end(record, bus, ev.WATCH_ERROR, "error", text[:2000])
            return True
        if not record.stalled and now - loop.last_activity >= record.stall_seconds:
            record.stalled = True
            self._stalled(record, bus, loop, source, now)
        if record.timeout_seconds and now - record.started_at >= record.timeout_seconds:
            text = f"Stopped watching after {_duration(record.timeout_seconds)}" + self._where(
                record
            )
            data = {"timeout": True, "ended": True, "fraction": record.fraction}
            self._end(record, bus, ev.WATCH_STALLED, "timed_out", text, data)
            return True
        return False

    def _progress(
        self,
        record: WatchRecord,
        updates: Sequence[Update],
        now: float,
        bus: ev.EventBus,
        loop: _Loop,
        resumed: bool,
        silent_for: float,
    ) -> None:
        fresh = next((u for u in reversed(updates) if u.kind == "progress"), None)
        if fresh is None and not resumed:
            return
        data: dict[str, Any] = {}
        if fresh is not None:
            if fresh.fraction is not None:
                record.fraction = fresh.fraction
                loop.eta.add(now, fresh.fraction)
            record.phase = fresh.phase or record.phase
            record.text = fresh.text or record.text
            record.eta_seconds = (
                fresh.eta_seconds if fresh.eta_seconds is not None else loop.eta.estimate()
            )
            data.update(fresh.data)
        moved = record.fraction is not None and (
            loop.published_fraction is None
            or abs(record.fraction - loop.published_fraction) >= self._step - 1e-9
        )
        due = now - loop.published_at >= self._every
        if not (moved or record.phase != loop.published_phase or due or resumed):
            return
        text = self._state(record)
        if resumed:
            # The lead was woken to hear it stalled, so it hears this too.
            data.update({"resumed": True, "urgent": True, "silent_seconds": round(silent_for)})
            text = f"Output resumed after {_duration(silent_for)} of silence: {text}"
        data.update(
            {"fraction": record.fraction, "eta_seconds": record.eta_seconds, "phase": record.phase}
        )
        loop.published_at = now
        loop.published_fraction = record.fraction
        loop.published_phase = record.phase
        self._publish(bus, record, ev.WATCH_PROGRESS, text, data)

    def _stalled(
        self, record: WatchRecord, bus: ev.EventBus, loop: _Loop, source: WatchSource, now: float
    ) -> None:
        silent = now - loop.last_activity
        text = f"No new output for {_duration(silent)}" + self._where(record)
        if loop.failure:
            text += f" Reading it is failing: {loop.failure}."
        elif source.note:
            text += f" Note: {source.note}."
        data = {
            "fraction": record.fraction,
            "silent_seconds": round(silent),
            "last_line": record.tail[-1] if record.tail else "",
            "read_error": loop.failure,
        }
        self._publish(bus, record, ev.WATCH_STALLED, text, data)

    @staticmethod
    def _state(record: WatchRecord) -> str:
        parts = []
        if record.fraction is not None:
            parts.append(f"{record.fraction * 100:.0f}%")
        if record.phase:
            parts.append(record.phase)
        if record.eta_seconds is not None:
            parts.append(f"about {_duration(record.eta_seconds)} left")
        head = ", ".join(parts)
        if record.text and head:
            return f"{head} ({record.text})"
        return head or record.text

    @staticmethod
    def _where(record: WatchRecord) -> str:
        at = f" at {record.fraction * 100:.0f}%" if record.fraction is not None else ""
        last = f" Last line: {record.tail[-1]}" if record.tail else ""
        return f"{at}.{last}"

    def _end(
        self,
        record: WatchRecord,
        bus: ev.EventBus,
        kind: str,
        status: Status,
        text: str,
        data: Mapping[str, Any] | None = None,
    ) -> None:
        record.status = status
        record.ended_at = self._clock()
        record.final = self._publish(bus, record, kind, text, data or {})

    def _publish(
        self,
        bus: ev.EventBus,
        record: WatchRecord,
        kind: str,
        text: str,
        data: Mapping[str, Any],
    ) -> ev.Event:
        event = ev.Event(
            kind, record.thread_id, record.id, label=record.label, text=text, data=data
        )
        bus.publish(event)
        return event

    def list(self, thread_id: str) -> builtins.list[WatchRecord]:
        """This thread's watches, running or ended, oldest first."""
        return [r for r in self._records.values() if r.thread_id == thread_id]


async def _beat(on_beat: Beat | None, record: WatchRecord) -> None:
    if on_beat is None:
        return
    try:
        result = on_beat(record)
        if inspect.isawaitable(result):
            await result
    except asyncio.CancelledError:
        raise
    except Exception as error:  # noqa: BLE001 - a missed beat is retried next poll
        _LOGGER.warning("[harness] beat for %s failed: %s", record.id, type(error).__name__)


# -- surviving a restart ----------------------------------------------------------------------------

#: The store namespace each watch keeps its heartbeat record in, beside the helpers' ledger.
WATCH_LEDGER = ("background_watches",)
#: A watch a restart cut off can be resumed for this long after it was reported.
RESUMABLE_SECONDS = 24 * 3600.0

#: Called by `WatchLedger.sweep` with each record it found, before it publishes the record's event,
#: so the host can say where to wake that thread's lead.
OnFoundWatch = Callable[[str, dict[str, Any]], Awaitable[None]]


class WatchLedger:
    """Each watch's record in the deployment store, so a restart loses neither a watch nor its end.

    `beat(store, extra)` is a watch's `on_beat`. It writes the record every `heartbeat_seconds`
    while the watch runs: what it reads and how (`extra`, enough for the host to start it again
    under the same id), where it stands, and the host's wake data. After the watch ends the record
    stays, refreshed, until the lead has heard the end (`delivered`, an `Inbox.on_delivered`
    callback); a watch the lead stopped drops it at once.

    `sweep` handles what a restart left behind, for records no live watch holds whose heartbeat is
    older than `stale_seconds`: an end whose notice was lost is published again, and a watch the
    restart cut off is published as `TASK_INTERRUPTED` (marked `data["watch"]`) and kept, marked
    reported, so `resumable` finds it for `resumable_seconds`.
    """

    def __init__(
        self,
        watches: WatchRunner,
        *,
        clock: Callable[[], float] = time.time,
        heartbeat_seconds: float = HEARTBEAT_SECONDS,
        stale_seconds: float = STALE_SECONDS,
        resumable_seconds: float = RESUMABLE_SECONDS,
    ) -> None:
        self.watches = watches
        self._clock = clock
        self._heartbeat = heartbeat_seconds
        self._stale = stale_seconds
        self._resumable = resumable_seconds
        #: Ended watches whose end the lead has not heard: their records stay, refreshed.
        self._unheard: dict[str, tuple[Any, WatchRecord, dict[str, Any]]] = {}
        #: Ends the lead heard before their record was written: watch id -> its final event id.
        self._heard: dict[str, str] = {}
        self._refresh: asyncio.Task[None] | None = None

    def value(self, record: WatchRecord, extra: Mapping[str, Any]) -> dict[str, Any]:
        """The record as stored."""
        return {
            **record.meta,
            **extra,
            "thread_id": record.thread_id,
            "label": record.label,
            "status": record.status,
            "fraction": record.fraction,
            "started_at": record.started_at,
            "alive_at": self._clock(),
            "final": record.final.to_value() if record.final is not None else None,
        }

    async def _put(self, store: Any, record: WatchRecord, extra: Mapping[str, Any]) -> None:
        try:
            await store.aput(WATCH_LEDGER, record.id, self.value(record, extra))
        except Exception as error:  # noqa: BLE001 - a missed beat is retried next time
            _LOGGER.warning("[harness] record for %s failed: %s", record.id, type(error).__name__)

    async def _drop(self, store: Any, watch_id: str) -> None:
        try:
            await store.adelete(WATCH_LEDGER, watch_id)
        except Exception as error:  # noqa: BLE001 - a leftover is swept and reported once more
            _LOGGER.warning("[harness] dropping %s failed: %s", watch_id, type(error).__name__)

    def beat(self, store: Any, extra: Mapping[str, Any]) -> Beat:
        """A watch's `on_beat` that keeps its record in `store` (class doc)."""
        last = -math.inf
        kept = dict(extra)

        async def beat(record: WatchRecord) -> None:
            nonlocal last
            if store is None:
                return
            if record.status == "watching":
                if self._clock() - last >= self._heartbeat:
                    last = self._clock()
                    await self._put(store, record, kept)
            elif record.status == "cancelled":
                self._unheard.pop(record.id, None)
                await self._drop(store, record.id)
            elif record.final is not None and self._heard.get(record.id) == record.final.id:
                self._heard.pop(record.id, None)
                await self._drop(store, record.id)
            else:
                self._unheard[record.id] = (store, record, kept)
                await self._put(store, record, kept)
                if record.id in self._unheard:
                    self._ensure_refresh()
                else:  # heard while its record was being written
                    await self._drop(store, record.id)

        return beat

    def _ensure_refresh(self) -> None:
        if self._refresh is None or self._refresh.done():
            self._refresh = asyncio.get_running_loop().create_task(
                self._refresh_unheard(), name="watch records"
            )

    async def _refresh_unheard(self) -> None:
        while self._unheard:
            await asyncio.sleep(self._heartbeat)
            for watch_id, (store, record, extra) in builtins.list(self._unheard.items()):
                if watch_id not in self._unheard:
                    continue
                await self._put(store, record, extra)
                if watch_id not in self._unheard:  # heard while it was being refreshed
                    await self._drop(store, watch_id)

    async def delivered(self, batch: Sequence[ev.Event]) -> None:
        """Once the lead has heard how a watch ended, its record goes."""
        for event in batch:
            held = self._unheard.pop(event.source, None)
            if held is not None and held[1].final is not None and held[1].final.id == event.id:
                await self._drop(held[0], event.source)
                continue
            if held is not None:  # not its end: keep waiting for that
                self._unheard[event.source] = held
            record = self.watches.get(event.source)
            if record is not None and record.final is not None and record.final.id == event.id:
                self._heard[event.source] = event.id
                for gone in [w for w in self._heard if self.watches.get(w) is None]:
                    del self._heard[gone]

    async def sweep(
        self,
        store: Any,
        *,
        on_found: OnFoundWatch | None = None,
        bus: ev.EventBus | None = None,
        now: float | None = None,
    ) -> int:
        """Publish what a restart left behind (class doc) and return how many records it found.
        A reported record is told once, and deleted once it is past `resumable_seconds`."""
        now = self._clock() if now is None else now
        found = 0
        for item in await store.asearch(WATCH_LEDGER, limit=200):
            value = dict(item.value or {})
            if value.get("reported_at") is not None:
                if now - float(value["reported_at"]) > self._resumable:
                    await store.adelete(WATCH_LEDGER, item.key)
                continue
            record = self.watches.get(item.key)
            held = record is not None and (record.status == "watching" or item.key in self._unheard)
            if held or now - float(value.get("alive_at") or 0) < self._stale:
                continue
            if on_found is not None:
                await on_found(item.key, value)
            final = value.get("final")
            if isinstance(final, Mapping):
                await store.adelete(WATCH_LEDGER, item.key)
                event = ev.Event.from_value(final)
            else:
                await store.aput(
                    WATCH_LEDGER, item.key, {**value, "status": "interrupted", "reported_at": now}
                )
                event = interrupted_event(item.key, value)
            (bus or ev.BUS).publish(event)
            found += 1
        return found

    async def resumable(
        self, store: Any, watch_id: str, thread_id: str, *, now: float | None = None
    ) -> dict[str, Any] | None:
        """The record of a watch a restart cut off on this thread, for the host to watch it again
        under the same id; None when there is nothing to resume."""
        if store is None:
            return None
        item = await store.aget(WATCH_LEDGER, watch_id)
        value = dict(item.value or {}) if item is not None else {}
        reported = value.get("reported_at")
        if value.get("thread_id") != thread_id or reported is None:
            return None
        now = self._clock() if now is None else now
        return value if now - float(reported) <= self._resumable else None

    def stop(self) -> None:
        """Stop refreshing records and forget what is unheard (tests, shutdown)."""
        if self._refresh is not None and not self._refresh.done():
            self._refresh.cancel()
        self._unheard.clear()
        self._heard.clear()


def interrupted_event(watch_id: str, value: Mapping[str, Any]) -> ev.Event:
    """What the lead hears about a watch a restart cut off: `task.interrupted`, as for a helper or
    a pipeline, marked `data["watch"]`."""
    fraction = value.get("fraction")
    at = f" at {float(fraction) * 100:.0f}%" if isinstance(fraction, int | float) else ""
    log = str(value.get("log_path") or value.get("path") or "")
    job = bool(value.get("command"))
    where = f" The job itself may still be running: its log is {log}." if log else ""
    text = (
        f"The watch stopped{at} when the server restarted.{where} resume_task('{watch_id}') "
        f"watches it again, re-reading the log{' and the exit code' if job else ''}, so a job that "
        "ended meanwhile says how. Nothing read the sandbox while the server was down; a sandbox "
        "that stops when idle may have stopped the job with it."
    )
    return ev.Event(
        ev.TASK_INTERRUPTED,
        str(value.get("thread_id") or ""),
        watch_id,
        label=str(value.get("label") or watch_id),
        text=text,
        data={
            "watch": True,
            "fraction": fraction,
            "started_at": value.get("started_at"),
            "finished_at": value.get("alive_at"),
        },
    )