Python
feat(observability): sandbox setup and restarts as steps (OB-13)
Devika Vermada21e3038e9f
diff --git a/coworker/agent.py b/coworker/agent.pyindex 831a23d7..3d2eccc8 100644--- a/coworker/agent.py+++ b/coworker/agent.py@@ -776,6 +776,8 @@ def build_engine(if executor is not None:engine.permissions.shell_cwd = engine.current_shell_cwdengine.sandbox_workspace = sandbox_workspace # type: ignore[attr-defined]+ if sandbox_workspace is not None and hasattr(sandbox_workspace, "observer"):+ sandbox_workspace.observer = engine.on_sandbox # setup step and restarts (OB-13)# Whether this machine has a sandbox switched on (stamp.sandbox.expected, format 1.2), for# the "ran without the sandbox" check: the setting, whatever this session ended up using.engine.sandbox_expected = _sandbox_provider_name(config.sandbox_provider) != "direct"diff --git a/coworker/engine.py b/coworker/engine.pyindex 28e99932..be5d508a 100644--- a/coworker/engine.py+++ b/coworker/engine.py@@ -16,6 +16,7 @@ import loggingfrom copy import deepcopyimport asyncio+import inspectimport functoolsimport jsonimport time@@ -1114,7 +1115,10 @@ class TurnEngine:provider = str(getattr(getattr(workspace, "provider", None), "name", "") or "sandbox")yield Event(EventType.SANDBOX_PREPARING, {"provider": provider})try:- await asyncio.to_thread(start)+ if "trigger" in inspect.signature(start).parameters:+ await asyncio.to_thread(start, "turn_start")+ else:+ await asyncio.to_thread(start)except Exception as exc:text = f"The {provider} sandbox could not start: {exc}"self._append_notice("error", text)@@ -1124,6 +1128,34 @@ class TurnEngine:self._note_stamp() # the runner's version is known only now (V2-28)yield Event(EventType.SANDBOX_READY, {k: info.get(k) for k in ("provider", "enforcement", "reason", "sandbox") if info.get(k) is not None})+ def on_sandbox(self, event: str, data: dict[str, Any]) -> None:+ """What the sandbox workspace tells the recording (observability OB-13): its start as a+ `sandbox` setup step with its stages, and a restart as a `sandbox_restarted` notice.+ Called from the thread that starts or restarts the sandbox; outside a turn nothing is+ written."""+ obs = self._obs+ if not obs.enabled or not obs.in_turn:+ return+ provider = str(getattr(getattr(getattr(self, "sandbox_workspace", None), "provider", None), "name", "") or "sandbox")+ if event == "setup_started":+ self._sandbox_setup_step = obs.start_step(+ kind="sandbox", name="setup",+ attrs={"provider": provider, "stages": [], "failed_stage": None, "trigger": data.get("trigger") or "turn_start"},+ )+ elif event == "setup_ended":+ step_id, self._sandbox_setup_step = getattr(self, "_sandbox_setup_step", None), None+ log = data.get("log")+ if step_id is None or log is None:+ return+ failed = data.get("error") is not None+ obs.end_step(step_id, status="error" if failed else "ok",+ attrs=log.attrs(provider, data.get("trigger") or "turn_start"))+ if not failed and data.get("trigger") == "first_tool":+ self._note_stamp() # the runner's version, as after a start at the turn's start+ elif event == "restarted":+ obs.instant_step(kind="notice", name="sandbox_restarted", status="ok",+ attrs={"provider": provider, "reason": data.get("reason"), "how": data.get("how")})+@_records_turn("resume")@_summarizes_turnasync def resume(self) -> AsyncIterator[Event]:@@ -2970,6 +3002,15 @@ class TurnEngine:logger.exception("observability: could not record a tool call's facts")return out+ @staticmethod+ def _sandbox_notice_id(text: Any) -> Optional[str]:+ """`restart` or `regrant` for the sandbox notice a command's result carries, else None."""+ if not isinstance(text, str) or not text:+ return None+ from .sandbox.workspace import REGRANT_NOTICE, RESTART_NOTICE++ return "restart" if text == RESTART_NOTICE else "regrant" if text == REGRANT_NOTICE else None+def _sandbox_facts(self, result: Any) -> Optional[dict[str, Any]]:"""Per-call sandbox facts (observability E13, §9.3): whether this sandbox can reportblocked sites now, and how many connections it refused during the call. None when@@ -2978,10 +3019,13 @@ class TurnEngine:if ws is None or not hasattr(ws, "reports_blocked"):return Noneblocked = result.get("_blocked") if isinstance(result, dict) else None+ told = result.get("sandbox_notice") if isinstance(result, dict) else Nonereturn {"blocked_known": bool(getattr(ws, "reports_blocked", False)),"blocked_count": sum(int(b.get("count") or 0) for b in blocked or []),- "notice": None,+ # The restart the agent was told of with this result (OB-13): the same short id+ # as the `sandbox_restarted` notice's `how`.+ "notice": self._sandbox_notice_id(told),}def _execute_call(self, tool_call: ToolCall) -> tuple[Any, str]:diff --git a/coworker/observability/calculator.py b/coworker/observability/calculator.pyindex f6334c69..af55ae5f 100644--- a/coworker/observability/calculator.py+++ b/coworker/observability/calculator.py@@ -10,7 +10,7 @@ from __future__ import annotationsfrom typing import Any, Iterable, Optional-BUCKET_ORDER = ["waiting", "tools", "model", "auto_approver", "summary"]+BUCKET_ORDER = ["waiting", "tools", "model", "auto_approver", "summary", "sandbox"]# -- tokens (§6.3) -------------------------------------------------------------------------------@@ -126,12 +126,15 @@ def _bucket_of(step: dict[str, Any]) -> Optional[str]:return "auto_approver"if kind == "summary":return "summary"+ if kind == "sandbox":+ return "sandbox" # 1.2: sandbox setup, its own total so a first download is not "Other"return Nonedef buckets(recording: dict[str, Any], turn: dict[str, Any], now: Optional[int] = None) -> dict[str, Any]:"""Where the turn's time went: each bucket is the union of its steps' time, overlaps go- to the first bucket in the order waiting, tools, model, auto-approver, summary; Other is+ to the first bucket in the order waiting, tools, model, auto-approver, summary, sandbox+ setup (1.2, present only when the turn has one); Other isthe rest, shown when it is 1% of the total or more (M-R5)."""start = turn["started_at"]# Steps found open when the app started have no time (status unknown): not counted (TD-12).@@ -155,6 +158,8 @@ def buckets(recording: dict[str, Any], turn: dict[str, Any], now: Optional[int]own = _subtract(raw[b], taken)out[b] = _length(own)taken = _merge(taken + own)+ if not any(s.get("kind") == "sandbox" for s in steps):+ del out["sandbox"] # 1.1 recordings read back with the same buckets as beforetotal = end_of_turn - startout["total"] = totalout["other"] = total - _length(taken)diff --git a/coworker/observability/export_spans.py b/coworker/observability/export_spans.pyindex 987ba4d6..aeb45223 100644--- a/coworker/observability/export_spans.py+++ b/coworker/observability/export_spans.py@@ -654,6 +654,18 @@ def build_turn_spans(attrs[f"openworker.context.{k}"] = a[k]elif s["kind"] == "sandbox":name = "openworker.sandbox.setup"+ for k in ("provider", "trigger", "failed_stage"):+ if a.get(k) is not None:+ attrs[f"openworker.sandbox.{k}"] = a[k]+ # One event per stage (tracing-spec §9); the error's text stays on this machine.+ for st in a.get("stages") or []:+ ev_attrs = {"openworker.sandbox.stage.status": st.get("status") or "ok"}+ if st.get("duration_ms") is not None:+ ev_attrs["openworker.sandbox.stage.duration_ms"] = st["duration_ms"]+ if st.get("uncertain"):+ ev_attrs["openworker.sandbox.stage.uncertain"] = True+ events.append({"name": f"openworker.sandbox.stage.{st.get('name')}",+ "at": st.get("started_at") or s.get("started_at") or turn["started_at"], "attrs": ev_attrs})else:name = f"openworker.{s['kind']}"diff --git a/coworker/observability/recorder.py b/coworker/observability/recorder.pyindex 6c765ad9..4ebcf775 100644--- a/coworker/observability/recorder.py+++ b/coworker/observability/recorder.py@@ -47,6 +47,7 @@ KIND_ORDER = {"wait": 4,"tool": 5,"notice": 6,+ "sandbox": 7, # 1.2; it never shares a start with the others in practice}TURN_FIELDS = (@@ -145,6 +146,9 @@ def fill_unended(kind: str, attrs: dict[str, Any]) -> dict[str, Any]:elif kind == "summary":for key in ("tokens_before", "model_step_id"):attrs.setdefault(key, None)+ elif kind == "sandbox":+ attrs.setdefault("stages", [])+ attrs.setdefault("failed_stage", None)return attrsdiff --git a/coworker/sandbox/providers/openshell.py b/coworker/sandbox/providers/openshell.pyindex 5a87bc61..d641e225 100644--- a/coworker/sandbox/providers/openshell.py+++ b/coworker/sandbox/providers/openshell.py@@ -40,6 +40,7 @@ from ..bundle import build_runner_zipappfrom ..registry import REGISTRY_LABEL, registry_idfrom ..transport import Transportfrom .. import credentials as creds+from .. import stagesfrom . import openshell_policy as policyfrom . import openshell_wire as wire@@ -92,15 +93,22 @@ def _cli(*args: str, timeout: float = _CLI_TIMEOUT, check: bool = True) -> subprreturn done+# What the last preflight learnt of the base image (True, False, None = cannot tell), for+# the recording's setup stages: None means `sandbox create` may pull it (OB-13).+_last_image_present: Optional[bool] = None++def preflight() -> dict[str, Any]:"""Refuse early, with a message a person can act on."""+ global _last_image_presentversion = _cli("--version", timeout=15).stdout.strip().split()[-1]if version != PINNED_VERSION:raise OpenShellUnavailable(f"OpenShell {version} is installed; OpenWorker is tested with {PINNED_VERSION} only. Install that version (`openworker machine sandbox setup`).")status = _cli("status", timeout=30, check=False)if status.returncode != 0 or "Connected" not in status.stdout:raise OpenShellUnavailable("The OpenShell gateway is not running or cannot be reached (`openshell status`). On a headless machine, check `systemctl --user status openshell-gateway` and `loginctl enable-linger`.")- if image_present() is False:+ present = _last_image_present = image_present()+ if present is False:raise OpenShellImageMissing(f"{IMAGE_MISSING_PREFIX} (about 5 GB, one time). Run `openworker machine sandbox setup`, "f"or `{image_tool() or 'docker'} pull {sandbox_image()}`, then start the session again."@@ -288,6 +296,7 @@ class DenialLog:class OpenShellProvider:name = "openshell"+ reports_stages = True # its start's inner stages go to the recording (stages.stage)def __init__(self,@@ -352,17 +361,26 @@ class OpenShellProvider:raisedef _create(self) -> None:+ global _last_image_present+ _last_image_present = Nonepreflight()for root in self.roots:if not os.path.isdir(root["path"]):raise RuntimeError(f"folder does not exist on this machine: {root['path']}")uid, gid = (os.getuid(), os.getgid()) if sys.platform.startswith("linux") else (None, None)+ if self.grants:+ stages.stage(self, "copy_logins")# The sandbox's private home with the copied credentials (section 11b), mounted at# the same path it has on the machine, read-write. With no grants it holds only# git's settings, and HOME stays the runtime folder.self.copied = creds.copy_in(self.grants, self._tmp) if self.grants else Nonehome = self.copied.home if self.copied is not None else Noneself._identity = (uid, gid, home)+ stages.stage(self, "create")+ if _last_image_present is None:+ # Nobody here can say whether the image is present: `sandbox create` may pull it,+ # and how long that took cannot be told apart from the create.+ stages.mark(self, "download_image", uncertain=True)policy_file = self._write_policy()driver_config = json.dumps(policy.mounts(self.roots, str(self._runner.parent), home))command = [policy.PYTHON, "-S", f"{policy.RUNNER_MOUNT}/{self._runner.name}", "serve", "--socket", _SOCKET, "--cwd", self.cwd]@@ -396,6 +414,7 @@ class OpenShellProvider:if docker_host_network() is False:raise OpenShellUnavailable(f"{DOCKER_HOST_NETWORK_FIX}. Then start the session again.") from excraise+ stages.stage(self, "ready")self._wait_ready()self._denials = DenialLog(self.sandbox_name) # what it refuses, for the agent and the card# Spike finding K: the first exec after a sandbox turns Ready hangs until it timesdiff --git a/coworker/sandbox/providers/seatbelt.py b/coworker/sandbox/providers/seatbelt.pyindex b9b63bfb..6b5e9dce 100644--- a/coworker/sandbox/providers/seatbelt.py+++ b/coworker/sandbox/providers/seatbelt.py@@ -29,7 +29,7 @@ from pathlib import Pathfrom typing import Any, Callable, Optional, Sequencefrom .. import credentials as creds-from .. import netproxy, network_profiles+from .. import netproxy, network_profiles, stagesfrom ..bundle import build_runner_zipappfrom ..launch import read_paths, runner_commandfrom ..transport import PipeTransport, Transport@@ -80,6 +80,7 @@ def clean_environment(base: Optional[dict[str, str]] = None) -> dict[str, str]:class SeatbeltProvider:name = "seatbelt"+ reports_stages = True # its start's inner stages go to the recording (stages.stage)def __init__(self,@@ -163,6 +164,8 @@ class SeatbeltProvider:def _create(self) -> None:preflight()os.makedirs(os.path.join(self._dir, "tmp", "cache"), exist_ok=True)+ if self.grants:+ stages.stage(self, "copy_logins")if self.network and not network_profiles.is_open(self.profile):hosts = sorted({*self.extra_hosts, *(h for g in self.grants for h in g.hosts)})# Each session has its own proxy: its allow list can grow while it runs (add_hosts).@@ -175,6 +178,7 @@ class SeatbeltProvider:proxy_port=self._proxy.port if self._proxy is not None else None,ssh_proxy_command=creds.mac_ssh_proxy_command(self._proxy.port) if self._proxy is not None else None,)+ stages.stage(self, "create")log = open(os.path.join(self._dir, "daemon.log"), "wb")self._daemon = subprocess.Popen([diff --git a/coworker/sandbox/providers/windows.py b/coworker/sandbox/providers/windows.pyindex 5a57c741..6a4d1eb7 100644--- a/coworker/sandbox/providers/windows.py+++ b/coworker/sandbox/providers/windows.py@@ -40,7 +40,7 @@ from pathlib import Pathfrom typing import Any, Optional, Sequencefrom .. import credentials as creds-from .. import netproxy, network_profiles+from .. import netproxy, network_profiles, stagesfrom ..bundle import build_runner_zipappfrom ..launch import runner_command, serve_arguments, spawn_kwargs, wait_for_runnerfrom ..runner import winpipe@@ -76,6 +76,7 @@ def preflight() -> None:class WindowsProvider:name = "windows"+ reports_stages = True # its start's inner stages go to the recording (stages.stage)def __init__(self,@@ -155,6 +156,8 @@ class WindowsProvider:preflight()from .. import winsec+ stages.stage(self, "create")+hosts = sorted({*self.extra_hosts, *(h for g in self.grants for h in g.hosts)})if self.network and not self.open_network:# Each session has its own proxy: its allow list can grow while it runs (add_hosts).diff --git a/coworker/sandbox/stages.py b/coworker/sandbox/stages.pynew file mode 100644index 00000000..9ea9e0b0--- /dev/null+++ b/coworker/sandbox/stages.py@@ -0,0 +1,85 @@+"""The stages of one sandbox start, for the recording's setup step (observability OB-13,+format 1.2 `sandbox_attrs`). Stage names come from a fixed list; each stage is closed when+the next one begins, and the last one when the start ends."""++from __future__ import annotations++import threading+import time+from typing import Any, Optional++STAGES = ("preparing", "download_image", "copy_logins", "create", "connect", "verify", "ready")+++class StageLog:+ def __init__(self) -> None:+ self._lock = threading.Lock()+ self.stages: list[dict[str, Any]] = []+ self._open: Optional[tuple[dict[str, Any], float]] = None+ self.failed_stage: Optional[str] = None+ self.error: Optional[dict[str, Any]] = None+ self.started_at = int(time.time() * 1000)++ def begin(self, name: str) -> None:+ """Close the stage in progress as `ok` and start `name`."""+ if name not in STAGES:+ return+ with self._lock:+ self._close("ok")+ stage = {"name": name, "started_at": int(time.time() * 1000), "duration_ms": None, "status": "ok"}+ self.stages.append(stage)+ self._open = (stage, time.monotonic())++ def mark(self, name: str, **extra: Any) -> None:+ """A stage that cannot be timed on its own (OpenShell's image pull happens inside+ `sandbox create`): written with no duration, the stage in progress left open."""+ if name not in STAGES:+ return+ with self._lock:+ self.stages.append({"name": name, "started_at": int(time.time() * 1000), "duration_ms": None, "status": "ok", **extra})++ def end(self, error: Optional[BaseException] = None) -> None:+ """The start is over. With `error`, the stage in progress failed."""+ with self._lock:+ if error is not None:+ if self._open is not None:+ self.failed_stage = self._open[0]["name"]+ elif self.stages:+ self.failed_stage = self.stages[-1]["name"]+ else:+ self.failed_stage = "preparing"+ text = str(error).strip()+ self.error = {"type": type(error).__name__, "text": text or None}+ self._close("error" if error is not None else "ok")++ def _close(self, status: str) -> None:+ if self._open is None:+ return+ stage, mono = self._open+ stage["duration_ms"] = max(0, int((time.monotonic() - mono) * 1000))+ stage["status"] = status+ self._open = None++ def attrs(self, provider: str, trigger: str) -> dict[str, Any]:+ out: dict[str, Any] = {+ "provider": provider,+ "stages": [dict(s) for s in self.stages],+ "failed_stage": self.failed_stage,+ "trigger": trigger,+ }+ if self.error is not None:+ out["error"] = dict(self.error)+ return out+++def stage(provider: Any, name: str) -> None:+ """A provider says it moved to stage `name` of its start. Nothing when nobody listens."""+ log = getattr(provider, "stage_log", None)+ if isinstance(log, StageLog):+ log.begin(name)+++def mark(provider: Any, name: str, **extra: Any) -> None:+ log = getattr(provider, "stage_log", None)+ if isinstance(log, StageLog):+ log.mark(name, **extra)diff --git a/coworker/sandbox/workspace.py b/coworker/sandbox/workspace.pyindex 61d01d45..f457d175 100644--- a/coworker/sandbox/workspace.py+++ b/coworker/sandbox/workspace.py@@ -20,6 +20,7 @@ from pathlib import Pathfrom typing import Any, Callable, Optional, Sequencefrom .runner.executor import Executor+from .stages import StageLogPROVIDER_ENV = "OPENWORKER_SANDBOX_PROVIDER"DIRECT = "direct"@@ -106,46 +107,89 @@ class RunnerWorkspace(Workspace):# Refusals up to here have been told to the agent. One that reaches us after its# command's result (OpenShell's log lags) is told with the next result, not lost.self._blocked_told = time.time()+ # What the recording hears of this sandbox (observability OB-13): `observer(event,+ # data)` with `setup_started`, `setup_ended` and `restarted`. Set by the engine; it+ # is called from whatever thread starts or restarts the sandbox.+ self.observer: Optional[Callable[[str, dict[str, Any]], None]] = None+ self.last_start: Optional[StageLog] = None+ self._restarts_seen = 0 # the client's restart count already toldif start:self.ensure_started()else:self.client.starter = self.ensure_started- def ensure_started(self) -> None:+ def ensure_started(self, trigger: str = "first_tool") -> None:"""Make the sandbox, connect to its runner and prove the wall. Once; a failure- leaves nothing behind and the next call tries again."""+ leaves nothing behind and the next call tries again. `trigger` is for the recording:+ `turn_start` when a turn starts it, `first_tool` when the first runner call does."""with self._start_lock:if self.started:return- registry = self.registry- if registry is not None:- registry.reap() # sandboxes left behind by a server that is gone- registry.check_room()- # Reserve the name before the sandbox exists: engine builds run concurrently- # (OPE-206), and another build's reap() would otherwise delete this one while- # it is still provisioning, because it is not yet in the registry.- self._record(state="creating")+ log = StageLog()+ self.last_start = log+ self._tell("setup_started", {"trigger": trigger, "log": log})try:- self.provider.create()- except Exception:- self.provider.destroy() # a half-made sandbox may already hold copied credentials- if registry is not None and self._registered:- registry.close(self._registered)- self._registered = None- raise- try:- self.hello = self.client.connect()- self._after_connect()- except Exception:- self.client.detach()- self.provider.destroy()- if registry is not None and self._registered:- registry.close(self._registered)- self._registered = None+ self._start(log)+ except BaseException as exc:+ log.end(exc)+ self._tell("setup_ended", {"trigger": trigger, "log": log, "error": exc})raise- self._record()- self.started = True- self.client.starter = None+ finally:+ if getattr(self.provider, "stage_log", None) is log:+ self.provider.stage_log = None # a later restart's create is not this start+ log.end()+ self._restarts_seen = self.client.restarts+ self._tell("setup_ended", {"trigger": trigger, "log": log, "error": None})++ def _start(self, log: StageLog) -> None:+ # Providers that know their own inner stages report them through `stage_log`; for+ # the others the whole `create` is one stage.+ try:+ self.provider.stage_log = log+ except AttributeError:+ pass+ log.begin("preparing")+ registry = self.registry+ if registry is not None:+ registry.reap() # sandboxes left behind by a server that is gone+ registry.check_room()+ # Reserve the name before the sandbox exists: engine builds run concurrently+ # (OPE-206), and another build's reap() would otherwise delete this one while+ # it is still provisioning, because it is not yet in the registry.+ self._record(state="creating")+ if not getattr(self.provider, "reports_stages", False):+ log.begin("create")+ try:+ self.provider.create()+ except Exception:+ self.provider.destroy() # a half-made sandbox may already hold copied credentials+ if registry is not None and self._registered:+ registry.close(self._registered)+ self._registered = None+ raise+ try:+ log.begin("connect")+ self.hello = self.client.connect()+ self._after_connect(log)+ except Exception:+ self.client.detach()+ self.provider.destroy()+ if registry is not None and self._registered:+ registry.close(self._registered)+ self._registered = None+ raise+ self._record()+ self.started = True+ self.client.starter = None++ def _tell(self, event: str, data: dict[str, Any]) -> None:+ observer = self.observer+ if observer is None:+ return+ try:+ observer(event, data)+ except Exception: # noqa: BLE001 - the recording never stops a sandbox+ passdef _record(self, state: str = "ready") -> None:"""Enter this sandbox in the registry; after a restart that replaced it, under its@@ -173,6 +217,10 @@ class RunnerWorkspace(Workspace):started with (design ruling 15). Returns what to tell the agent, or None. Runsbefore every command, so it is also where a sandbox not yet started starts."""self.ensure_started()+ if self.client.restarts > self._restarts_seen:+ # The runner came back as a new instance on its own (a crash and reconnect).+ self._restarts_seen = self.client.restarts+ self._tell("restarted", {"reason": "runner_restarted", "how": "restart"})regrant = getattr(self.provider, "regrant", None)if regrant is None or self._live_roots is None:return None@@ -187,23 +235,30 @@ class RunnerWorkspace(Workspace):regrant(wanted, before_create=lambda: self._record(state="creating"))self.client.connect() # a new runner: the client notes the restartself._after_connect()+ self._restarts_seen = self.client.restartselse:regrant(wanted) # the same sandbox, re-granted in place (Windows: entries on folders)verify = getattr(self.provider, "verify", None)if verify is not None:verify(self.client)self._record()+ how = "restart" if restarts else "regrant"+ self._tell("restarted", {"reason": "folders_changed", "how": how})return RESTART_NOTICE if restarts else REGRANT_NOTICE- def _after_connect(self) -> None:+ def _after_connect(self, log: Optional[StageLog] = None) -> None:"""What a provider does once it can talk to its runner: prove the wall is up, thenput in what it could not put in before the runner existed (copies that must beowned by the sandbox's own account)."""verify = getattr(self.provider, "verify", None)if verify is not None:+ if log is not None:+ log.begin("verify")verify(self.client) # e.g. every folder is really reachable insideprovision = getattr(self.provider, "provision", None)if provision is not None:+ if log is not None:+ log.begin("copy_logins")provision(self.client)@property@@ -408,7 +463,9 @@ def open_workspace(if name == RUNNER_LOCAL:from .providers.runner_local import RunnerLocalProvider- return RunnerWorkspace(RunnerLocalProvider(cwd=cwd), cwd=cwd)+ # Like the real providers, it starts when the first turn needs it (`start`), so its+ # start is a turn's setup step too (observability OB-13).+ return RunnerWorkspace(RunnerLocalProvider(cwd=cwd), cwd=cwd, start=start)listed = [{"path": str(r.path), "writable": bool(r.writable)} for r in (roots or [])]listed = listed or [{"path": str(cwd), "writable": True}]from .credentials import granteddiff --git a/surfaces/gui/src/components/StepsPanel.test.tsx b/surfaces/gui/src/components/StepsPanel.test.tsxindex e06ba488..20848b4e 100644--- a/surfaces/gui/src/components/StepsPanel.test.tsx+++ b/surfaces/gui/src/components/StepsPanel.test.tsx@@ -267,6 +267,56 @@ describe("StepsPanel", () => {});});+describe("Sandbox setup (OB-13)", () => {+ const setupTurn = { ...turn, buckets: { ...turn.buckets, sandbox: 4000, other: 0, other_shown: false } };+ const stages = [+ { name: "preparing", started_at: 1000, duration_ms: 100, status: "ok" },+ { name: "copy_logins", started_at: 1100, duration_ms: 100, status: "ok" },+ { name: "create", started_at: 1200, duration_ms: 3000, status: "ok" },+ { name: "download_image", started_at: 1200, duration_ms: null, status: "ok", uncertain: true },+ { name: "connect", started_at: 4200, duration_ms: 400, status: "ok" },+ { name: "verify", started_at: 4600, duration_ms: 400, status: "error" },+ ];+ const setup = step({ step_id: "sb", kind: "sandbox", name: "setup", duration_ms: 4000, status: "error",+ attrs: { provider: "seatbelt", stages, failed_stage: "verify", trigger: "turn_start",+ error: { type: "SeatbeltUnavailable", text: "the home folder was listable from inside" } } });+ const restart = step({ step_id: "rs", kind: "notice", name: "sandbox_restarted", started_at: 6000, duration_ms: 0,+ attrs: { provider: "seatbelt", reason: "folders_changed", how: "restart" } });++ it("draws the setup row with its stages, its own total and the restart row", async () => {+ vi.mocked(getRecordedTurns).mockResolvedValue([setupTurn] as any);+ vi.mocked(getTurnSteps).mockResolvedValue([setup, restart] as any);+ render(<StepsPanel sessionId="s1" onClose={() => {}} />);+ await waitFor(() => expect(screen.getAllByTestId("steps-row")).toHaveLength(2));+ expect(screen.getByTestId("steps-tiles").textContent).toContain("Sandbox setup");+ const [row, notice] = screen.getAllByTestId("steps-row").map((r) => r.textContent || "");+ expect(row).toContain("Sandbox setup");+ expect(row).toContain("failed at verify");+ expect(notice).toContain("Sandbox restarted");+ expect(notice).toContain("folders");+ const stageRows = screen.getAllByTestId("steps-stage").map((r) => r.textContent || "");+ // The pull OpenShell cannot time is said on the Create row, not as a row of its own.+ expect(stageRows).toHaveLength(5);+ expect(stageRows[2]).toContain("Create · image pulled?");+ expect(stageRows[4]).toContain("Verify");+ expect(stageRows[4]).toContain("error");+ });++ it("says in the details which stage failed and what the sandbox said", async () => {+ vi.mocked(getRecordedTurns).mockResolvedValue([setupTurn] as any);+ vi.mocked(getTurnSteps).mockResolvedValue([setup] as any);+ vi.mocked(getStepDetail).mockResolvedValue(setup as any);+ render(<StepsPanel sessionId="s1" onClose={() => {}} />);+ await waitFor(() => expect(screen.getAllByTestId("steps-row")).toHaveLength(1));+ fireEvent.click(screen.getAllByTestId("steps-row")[0]);+ const failed = await screen.findByTestId("steps-sandbox-failed");+ expect(failed.textContent).toContain("could not start at verify (SeatbeltUnavailable)");+ const details = screen.getByTestId("steps-details").textContent || "";+ expect(details).toContain("at the turn's start");+ expect(details).toContain("the home folder was listable from inside");+ });+});+describe("formatDuration", () => {it("reads like the mock", () => {expect(formatDuration(100)).toBe("0.1s");diff --git a/surfaces/gui/src/components/StepsPanel.tsx b/surfaces/gui/src/components/StepsPanel.tsxindex cc1181f4..fe007518 100644--- a/surfaces/gui/src/components/StepsPanel.tsx+++ b/surfaces/gui/src/components/StepsPanel.tsx@@ -5,7 +5,7 @@// §12); `refreshKey` moves when live step events or a turn's end arrive, so the panel follows a// running turn.-import { useEffect, useMemo, useState } from "react";+import { Fragment, useEffect, useMemo, useState } from "react";import { useTranslation } from "react-i18next";import {getRecordedTurns,@@ -23,7 +23,7 @@ import { formatTokens } from "../usage";type T = (k: string, o?: any) => string;-const BUCKETS = ["waiting", "tools", "model", "auto_approver", "summary"] as const;+const BUCKETS = ["waiting", "tools", "model", "auto_approver", "summary", "sandbox"] as const;/** 0.1s, 3.4s, 41s, 2m 14s: the panel's one duration format. */export function formatDuration(ms: number | null | undefined): string {@@ -125,7 +125,10 @@ export function stepLabel(step: RecordedStep, t: T): string {const arrow = before != null && after != null ? ` · ${formatTokens(before)}→${formatTokens(after)}` : "";return (step.name === "trim" ? t("steps.row.trim") : t("steps.row.summary")) + arrow;}+ case "sandbox":+ return t("steps.row.sandbox_setup");case "notice":+ if (step.name === "sandbox_restarted" && a.how === "regrant") return t("steps.notice.sandbox_regranted");return t(`steps.notice.${step.name}`, { defaultValue: step.name });default:return step.name;@@ -159,6 +162,11 @@ export function stepFlags(step: RecordedStep, t: T): Flag[] {if (a.decision?.origin === "reviewer") flags.push({ cls: "info", text: t("steps.flag.auto_approved") });} else if (step.kind === "auto_approver") {if (step.status === "interrupted") flags.push({ cls: "err", text: t("steps.status.interrupted") });+ } else if (step.kind === "sandbox" && step.status === "error") {+ const stage = a.failed_stage || "preparing";+ flags.push({ cls: "err", text: t("steps.flag.sandbox_failed", { stage: t(`steps.sbx_stage.${stage}`, { defaultValue: stage }).toLowerCase() }) });+ } else if (step.kind === "notice" && step.name === "sandbox_restarted") {+ if (a.reason) flags.push({ cls: "info", text: t(`steps.flag.restart_${a.reason}`, { defaultValue: a.reason }) });} else if (["error", "denied", "interrupted", "skipped", "unknown"].includes(step.status)) {flags.push({ cls: "err", text: t(`steps.status.${step.status}`) });}@@ -187,8 +195,47 @@ function nest(steps: RecordedStep[]): { step: RecordedStep; depth: number }[] {const KIND_CLASS: Record<string, string> = {context: "ctx", model: "model", tool: "tool", wait: "wait", auto_approver: "review", summary: "sum", notice: "ctx",+ sandbox: "sbx",};+/** The class of a row: a sandbox restart notice is drawn in the sandbox colour (mock ob13). */+function kindClass(step: RecordedStep): string {+ if (step.kind === "notice" && step.name === "sandbox_restarted") return "sbx";+ return KIND_CLASS[step.kind] || "ctx";+}++type SandboxStage = { name: string; started_at: number; duration_ms: number | null; status: string; uncertain?: boolean };++/** A sandbox setup step's stages as rows under it, in the order run (OB-13). An image pull+ * the provider cannot time is not a row of its own: it is said on the Create row. */+function StageRows({ step, depth, turn, total }: { step: RecordedStep; depth: number; turn: RecordedTurn; total: number }) {+ const { t } = useTranslation();+ const stages: SandboxStage[] = Array.isArray(step.attrs?.stages) ? step.attrs.stages : [];+ const pulled = stages.some((s) => s.name === "download_image" && s.uncertain);+ return (+ <>+ {stages.filter((s) => !(s.name === "download_image" && s.uncertain)).map((s, i) => {+ const left = Math.min(100, Math.max(0, ((s.started_at - turn.started_at) / total) * 100));+ const width = s.duration_ms ? Math.min(100 - left, (s.duration_ms / total) * 100) : 0;+ const name = t(`steps.sbx_stage.${s.name}`, { defaultValue: s.name }) + (s.name === "create" && pulled ? ` · ${t("steps.sbx_stage.image_maybe")}` : "");+ return (+ <div key={`${s.name}-${i}`} className="steps-row steps-stage k-sbx" data-testid="steps-stage">+ <span className="steps-label" style={{ paddingLeft: (depth + 1) * 16 }}>+ <span className="steps-kind" />+ <span className="steps-name">{name}</span>+ {s.status === "error" && <span className="steps-badge err">{t("steps.status.error")}</span>}+ </span>+ <span className="steps-track">+ <span className="steps-bar" style={{ left: `${left}%`, width: width ? `${width}%` : undefined }} />+ </span>+ <span className="steps-time">{formatDuration(s.duration_ms)}</span>+ </div>+ );+ })}+ </>+ );+}+export function StepsPanel({sessionId,refreshKey = 0,@@ -290,17 +337,19 @@ export function StepsPanel({<span className="steps-axis-time">{t("steps.axis.time")}</span></div>{rows.map(({ step, depth }) => (- <StepRow- key={step.step_id}- step={step}- depth={depth}- turn={turn}- total={total}- label={stepLabel(step, t)}- selected={selected === step.step_id}- issue={flagged.has(step.step_id)}- onSelect={() => setSelected(selected === step.step_id ? null : step.step_id)}- />+ <Fragment key={step.step_id}>+ <StepRow+ step={step}+ depth={depth}+ turn={turn}+ total={total}+ label={stepLabel(step, t)}+ selected={selected === step.step_id}+ issue={flagged.has(step.step_id)}+ onSelect={() => setSelected(selected === step.step_id ? null : step.step_id)}+ />+ {step.kind === "sandbox" && <StageRows step={step} depth={depth} turn={turn} total={total} />}+ </Fragment>))}</div>);@@ -412,7 +461,7 @@ function StepRow({ step, depth, turn, total, label, selected, issue, onSelect }:const { t } = useTranslation();const left = Math.min(100, Math.max(0, ((step.started_at - turn.started_at) / total) * 100));const width = step.duration_ms ? Math.min(100 - left, (step.duration_ms / total) * 100) : 0;- const kind = KIND_CLASS[step.kind] || "ctx";+ const kind = kindClass(step);return (<buttonclassName={`steps-row k-${kind}` + (selected ? " selected" : "") + (step.shadow ? " shadow" : "")}@@ -655,7 +704,7 @@ function statusPill(d: any, t: T): { cls: string; text: string } {function Inspector({ d, turn, steps, sessionId, t }: { d: any; turn: RecordedTurn; steps: RecordedStep[]; sessionId: string; t: T }) {const a = d.attrs || {};- const kind = KIND_CLASS[d.kind] || "ctx";+ const kind = kindClass(d);const started = clock(d.started_at - turn.started_at);const ctxStep = steps.find((s) => s.kind === "context");const exchanges = (d.exchange_rebuilt || []) as any[];@@ -813,12 +862,45 @@ function Inspector({ d, turn, steps, sessionId, t }: { d: any; turn: RecordedTur[t("steps.detail.tokens_after"), a.tokens_after == null ? "—" : formatTokens(a.tokens_after)]);}if (d.kind === "context") rows.push([t("steps.detail.session_prompt"), <span className="steps-mono">{a.session_prompt_hash || "—"}</span>]);+ if (d.kind === "sandbox") {+ rows.push([t("steps.sbx.provider"), a.provider || "—"],+ [t("steps.sbx.trigger"), t(`steps.sbx.trigger_${a.trigger}`, { defaultValue: a.trigger || "—" })]);+ }+ if (d.kind === "notice" && d.name === "sandbox_restarted") {+ rows.push([t("steps.sbx.provider"), a.provider || "—"],+ [t("steps.sbx.reason"), t(`steps.sbx.reason_${a.reason}`, { defaultValue: a.reason || "—" })],+ [t("steps.sbx.how"), t(`steps.sbx.how_${a.how}`, { defaultValue: a.how || "—" })]);+ }+ const sbxStages: SandboxStage[] = d.kind === "sandbox" && Array.isArray(a.stages) ? a.stages : [];const changed = d.kind === "notice" && d.name === "stamp_changed" ? stampChanges(a, t) : [];const showsContext = d.kind === "context" || (d.kind === "notice" && d.name === "context_changed");return (<><h3 className="steps-h3"><span className={`steps-kind k-${kind}`} />{stepLabel(d, t)}<span className={`steps-pill ${pill.cls}`}>{pill.text}</span></h3>+ {d.kind === "sandbox" && d.status === "error" && (+ <div className="steps-callout err" data-testid="steps-sandbox-failed">+ {t("steps.sbx.failed", {+ stage: t(`steps.sbx_stage.${a.failed_stage || "preparing"}`, { defaultValue: a.failed_stage || "preparing" }).toLowerCase(),+ type: a.error?.type || "error",+ })}+ </div>+ )}<Kv rows={rows} />+ {sbxStages.length > 0 && (+ <>+ <Sec label={t("steps.sbx.stages")} />+ <Kv rows={sbxStages.map((s) => [+ t(`steps.sbx_stage.${s.name}`, { defaultValue: s.name }),+ s.uncertain ? t("steps.sbx.image_note") : `${formatDuration(s.duration_ms)}${s.status === "error" ? ` · ${t("steps.status.error")}` : ""}`,+ ] as [string, React.ReactNode])} />+ </>+ )}+ {d.kind === "sandbox" && a.error?.text && (+ <>+ <Sec label={t("steps.sbx.error_text")} />+ <div className="steps-box">{a.error.text}</div>+ </>+ )}{changed.length > 0 && (<><Sec label={t("steps.insp.changed")} />diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.jsonindex 751d1c56..986cd766 100644--- a/surfaces/gui/src/locales/en.json+++ b/surfaces/gui/src/locales/en.json@@ -3037,6 +3037,32 @@}},"steps": {+ "sbx_stage": {+ "preparing": "Preparing",+ "download_image": "Download image",+ "copy_logins": "Copy logins",+ "create": "Create",+ "connect": "Connect",+ "verify": "Verify",+ "ready": "Wait until ready",+ "image_maybe": "image pulled?"+ },+ "sbx": {+ "provider": "Sandbox",+ "trigger": "Started",+ "trigger_turn_start": "at the turn's start",+ "trigger_first_tool": "on first use",+ "failed": "The sandbox could not start at {{stage}} ({{type}}). The turn ended; Retry tries again.",+ "error_text": "What the sandbox said",+ "stages": "Stages",+ "reason": "Why",+ "how": "What happened",+ "reason_folders_changed": "the session's folders changed",+ "reason_runner_restarted": "the runner came back as a new instance",+ "how_restart": "a new sandbox; shells and variables started again",+ "how_regrant": "folder grants changed in place; shells kept",+ "image_note": "The image may have been downloaded inside Create; the provider cannot say how long that took."+ },"issues": {"tab": "Issues","tab_n": "Issues · {{n}}",@@ -3184,7 +3210,8 @@"model": "Model","auto_approver": "Auto-approver","summary": "Summary",- "other": "Other"+ "other": "Other",+ "sandbox": "Sandbox setup"},"checks_line_one": "Auto-approve: {{count}} check · {{allowed}} allowed · {{asked}} asked you · {{blocked}} blocked · {{time}} · {{reviewer}}","checks_line_other": "Auto-approve: {{count}} checks · {{allowed}} allowed · {{asked}} asked you · {{blocked}} blocked · {{time}} · {{reviewer}}",@@ -3221,7 +3248,8 @@"summary": "Summary","trim": "Trim","wait_inbox": "Waiting · parked in Inbox",- "shadow": "Shadow check"+ "shadow": "Shadow check",+ "sandbox_setup": "Sandbox setup"},"would": {"allow": "would have allowed",@@ -3235,7 +3263,10 @@"no_usage": "no usage","exit": "exit {{code}}","blocked": "blocked by sandbox",- "auto_approved": "auto-approved"+ "auto_approved": "auto-approved",+ "sandbox_failed": "failed at {{stage}}",+ "restart_folders_changed": "folders",+ "restart_runner_restarted": "runner came back"},"purpose": {"main": "main",@@ -3280,7 +3311,9 @@"context_changed": "Context changed","stamp_changed": "Settings changed","summary_failed": "Summary failed",- "recording_gap": "Recording gap"+ "recording_gap": "Recording gap",+ "sandbox_restarted": "Sandbox restarted",+ "sandbox_regranted": "Sandbox folders updated"},"status": {"ok": "ok",diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.jsonindex 2fabf3f4..a078aa21 100644--- a/surfaces/gui/src/locales/zh.json+++ b/surfaces/gui/src/locales/zh.json@@ -2993,6 +2993,32 @@}},"steps": {+ "sbx_stage": {+ "preparing": "准备",+ "download_image": "下载镜像",+ "copy_logins": "复制登录信息",+ "create": "创建",+ "connect": "连接",+ "verify": "验证",+ "ready": "等待就绪",+ "image_maybe": "可能拉取了镜像"+ },+ "sbx": {+ "provider": "沙箱",+ "trigger": "启动于",+ "trigger_turn_start": "回合开始时",+ "trigger_first_tool": "首次使用时",+ "failed": "沙箱在{{stage}}阶段无法启动({{type}})。本回合已结束;重试会再次尝试。",+ "error_text": "沙箱返回的信息",+ "stages": "阶段",+ "reason": "原因",+ "how": "发生了什么",+ "reason_folders_changed": "会话的文件夹已变化",+ "reason_runner_restarted": "运行器作为新实例重新连接",+ "how_restart": "新的沙箱;终端和变量重新开始",+ "how_regrant": "就地更改了文件夹授权;终端保留",+ "image_note": "镜像可能在“创建”阶段下载;提供方无法说明用了多长时间。"+ },"issues": {"tab": "问题","tab_n": "问题 · {{n}}",@@ -3136,7 +3162,8 @@"model": "模型","auto_approver": "自动审批","summary": "摘要",- "other": "其他"+ "other": "其他",+ "sandbox": "沙箱启动"},"checks_line_other": "自动审批:{{count}} 次检查 · {{allowed}} 次允许 · {{asked}} 次询问你 · {{blocked}} 次阻止 · {{time}} · {{reviewer}}","unknown_line_one": "{{count}} 个步骤在 OpenWorker 停止时仍在运行,其用时未知。",@@ -3172,7 +3199,8 @@"summary": "摘要","trim": "裁剪","wait_inbox": "等待 · 已放入收件箱",- "shadow": "影子检查"+ "shadow": "影子检查",+ "sandbox_setup": "沙箱启动"},"would": {"allow": "本会允许",@@ -3186,7 +3214,10 @@"no_usage": "未报告用量","exit": "退出码 {{code}}","blocked": "被沙箱阻止",- "auto_approved": "已自动批准"+ "auto_approved": "已自动批准",+ "sandbox_failed": "在{{stage}}失败",+ "restart_folders_changed": "文件夹",+ "restart_runner_restarted": "运行器已重启"},"purpose": {"main": "主要",@@ -3231,7 +3262,9 @@"context_changed": "上下文已变化","stamp_changed": "设置已变化","summary_failed": "摘要失败",- "recording_gap": "记录缺口"+ "recording_gap": "记录缺口",+ "sandbox_restarted": "沙箱已重启",+ "sandbox_regranted": "沙箱文件夹已更新"},"status": {"ok": "正常",diff --git a/surfaces/gui/src/styles.css b/surfaces/gui/src/styles.cssindex 7eb407b5..077738c3 100644--- a/surfaces/gui/src/styles.css+++ b/surfaces/gui/src/styles.css@@ -2430,8 +2430,8 @@ html[data-platform="linux"] ::-webkit-scrollbar-thumb:hover { background-color:/* Observability Phase 1 (UX-059 OB-1, OB-2, OB-11, OB-4): the Steps panel in the right rail,styled as the UX-059 mock (docs/ux-improvements/mocks/UX-059-observability-desktop.html).The step colours are the mock's --bar-* tokens, light and dark. */-:root { --bar-model:#2563eb; --bar-tool:#0f766e; --bar-wait:#d8dce1; --bar-review:#7c3aed; --bar-sum:#92400e; --bar-ctx:#6f7680; --bar-write:#f59e0b; }-html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3a3e46; --bar-review:#a78bfa; --bar-sum:#e8b04b; --bar-ctx:#9aa1ab; --bar-write:#f59e0b; }+:root { --bar-model:#2563eb; --bar-tool:#0f766e; --bar-wait:#d8dce1; --bar-review:#7c3aed; --bar-sum:#92400e; --bar-ctx:#6f7680; --bar-write:#f59e0b; --bar-sbx:var(--ok); }+html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3a3e46; --bar-review:#a78bfa; --bar-sum:#e8b04b; --bar-ctx:#9aa1ab; --bar-write:#f59e0b; --bar-sbx:var(--ok); }.team-view.steps-panel { background:var(--panel); font-size:var(--fs-ui); line-height:1.45; container-type:inline-size; }/* A step's details open beside the list: the rail widens to hold both (mock: list + 520px). */.main:has(.steps-panel.inspecting) { --artifact-rail-w:min(78vw, 1180px); }@@ -2485,6 +2485,9 @@ html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3.k-review .steps-kind, .steps-kind.k-review, .k-review .steps-bar { background:var(--bar-review); }.k-sum .steps-kind, .steps-kind.k-sum, .k-sum .steps-bar { background:var(--bar-sum); }.k-ctx .steps-kind, .steps-kind.k-ctx, .k-ctx .steps-bar { background:var(--bar-ctx); }+.k-sbx .steps-kind, .steps-kind.k-sbx, .k-sbx .steps-bar { background:var(--bar-sbx); }+.steps-stage { cursor:default; }+.steps-stage .steps-bar { opacity:.75; }.k-wait .steps-kind, .steps-kind.k-wait { background:var(--bar-wait); outline:1px dashed var(--faint); }.steps-track { position:relative; height:12px; background:repeating-linear-gradient(90deg, transparent 0 24.6%, var(--line) 24.6% 25%); }.steps-bar { position:absolute; top:2px; height:8px; min-width:3px; border-radius:3px; background:var(--bar-ctx); }diff --git a/tests/test_observability_sandbox_setup.py b/tests/test_observability_sandbox_setup.pynew file mode 100644index 00000000..6241582b--- /dev/null+++ b/tests/test_observability_sandbox_setup.py@@ -0,0 +1,292 @@+"""OB-13: the sandbox's start as a `sandbox` setup step with its stages, and restarts as a+`sandbox_restarted` notice (phase-2 design §12, recording format 1.2).++A fake provider and a fake runner client stand in for a real sandbox: the stages, their+order and the recording are what is tested, not a sandbox."""++from __future__ import annotations++import asyncio+import importlib+import json+from pathlib import Path++import jsonschema++from coworker.observability import calculator, signals+from coworker.observability.export_spans import Masker, build_turn_spans+from coworker.providers import AssistantTurn+from coworker.sandbox import stages+from coworker.sandbox.workspace import RunnerWorkspace++base = importlib.import_module("test_observability_stamps_context")++SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))+++def _check(recording):+ validator = jsonschema.validators.validator_for(SCHEMA)(SCHEMA)+ errors = sorted(validator.iter_errors(recording), key=lambda e: list(e.absolute_path))+ assert not errors, "\n".join(f"{list(e.absolute_path)}: {e.message[:300]}" for e in errors[:8])+++class ImageMissing(RuntimeError):+ pass+++class _Provider:+ """A provider that reports its own inner stages, like OpenShell."""++ name = "openshell"+ reports_stages = True+ profile = "allowlist"+ extra_hosts: list = []++ def __init__(self, *, fail_at=None, grants=True, image_known=False, regrant=False):+ self.fail_at, self.grants, self.image_known = fail_at, grants, image_known+ self.roots = [{"path": "/a", "writable": True}]+ self.destroyed = 0+ if regrant:+ self.regrant = self._regrant++ def create(self):+ if self.grants:+ stages.stage(self, "copy_logins")+ stages.stage(self, "create")+ if not self.image_known:+ stages.mark(self, "download_image", uncertain=True)+ if self.fail_at == "create":+ raise ImageMissing("The sandbox base image is not downloaded yet")+ stages.stage(self, "ready")++ def verify(self, client):+ if self.fail_at == "verify":+ raise RuntimeError("the home folder was listable from inside")++ def destroy(self):+ self.destroyed += 1++ def describe(self):+ return {"provider": self.name, "enforcement": "full", "sandbox": "sbx-1"}++ def _regrant(self, roots, *, before_create=None):+ self.roots = list(roots)+ self.create() # a restart makes a new sandbox: not part of the first start's stages+++class _Client:+ def __init__(self):+ self.restarts = 0+ self.starter = None+ self.hello = {}++ def connect(self):+ return {"instance_id": "i1"}++ def detach(self):+ pass+++class _Root:+ def __init__(self, path, writable=True):+ self.path, self.writable = path, writable+++def _workspace(provider, roots=None):+ ws = RunnerWorkspace(provider, cwd=".", start=False, live_roots=roots)+ ws.client = _Client()+ return ws+++# -- the stage log ---------------------------------------------------------------------------+++def test_stages_close_in_order_and_a_failure_names_its_stage():+ log = stages.StageLog()+ log.begin("preparing")+ log.begin("create")+ log.mark("download_image", uncertain=True)+ log.end(ImageMissing("no image"))+ names = [(s["name"], s["status"]) for s in log.stages]+ assert names == [("preparing", "ok"), ("create", "error"), ("download_image", "ok")]+ assert log.stages[2]["duration_ms"] is None and log.stages[2]["uncertain"] is True+ assert all(isinstance(s["duration_ms"], int) for s in log.stages[:2])+ assert log.failed_stage == "create"+ assert log.error == {"type": "ImageMissing", "text": "no image"}+ log.begin("nonsense") # not a stage name: ignored+ assert len(log.stages) == 3+++def test_the_workspace_records_each_stage_of_its_start():+ told = []+ ws = _workspace(_Provider())+ ws.observer = lambda event, data: told.append((event, data.get("trigger")))+ ws.ensure_started()+ assert [s["name"] for s in ws.last_start.stages] == ["preparing", "copy_logins", "create", "download_image", "ready", "connect", "verify"]+ assert ws.last_start.failed_stage is None+ assert told == [("setup_started", "first_tool"), ("setup_ended", "first_tool")]+ assert getattr(ws.provider, "stage_log", None) is None # a later restart is not this start+++def test_a_provider_without_stages_gets_one_create_stage():+ provider = _Provider()+ provider.reports_stages = False+ ws = _workspace(provider)+ ws.ensure_started("turn_start")+ assert [s["name"] for s in ws.last_start.stages][:2] == ["preparing", "create"]+++# -- in a turn -------------------------------------------------------------------------------++_WS: dict = {}+++def run_cmd(command: str) -> dict:+ """Run a command in the sandbox.++ Args:+ command: the command+ """+ # What the executor does before every command: restart when the folders changed.+ return {"exit_code": 0, "output": "", "sandbox_notice": _WS["ws"].sync_roots()}+++def needs_runner(path: str) -> dict:+ """Read something in the sandbox.++ Args:+ path: what to read+ """+ _WS["ws"].ensure_started() # what the runner client's starter does on first use+ return {"path": path}+++def _engine_with(tmp_path, provider, turns, roots=None):+ engine, rec = base._engine(tmp_path, turns)+ for fn in (run_cmd, needs_runner): # before any turn: the engine offers what is registered+ engine.registry.register(fn, metadata=base._ReadMeta())+ ws = _workspace(provider, roots)+ ws.observer = engine.on_sandbox+ engine.sandbox_workspace = ws+ _WS["ws"] = ws+ return engine, rec, ws+++def _turns(engine, *steps):+ """Several turns on one event loop, as in the app; a callable between them changes+ something mid-session. (A loop per turn leaves the engine's Stop flag on the old loop.)"""++ async def go():+ for step in steps:+ if callable(step):+ step()+ else:+ async for _ in engine.run(step, trigger="user"):+ pass++ asyncio.run(go())+++async def _no_start():+ if False: # pragma: no cover - an empty async generator+ yield None+++def test_the_first_turn_records_the_setup_step(tmp_path):+ engine, rec, ws = _engine_with(tmp_path, _Provider(), [AssistantTurn(text="ok"), AssistantTurn(text="again")])+ _turns(engine, "go")+ d = rec.as_dict()+ [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]+ assert setup["name"] == "setup" and setup["status"] == "ok" and setup["parent_step_id"] is None+ a = setup["attrs"]+ assert a["provider"] == "openshell" and a["trigger"] == "turn_start" and a["failed_stage"] is None+ assert [s["name"] for s in a["stages"]] == ["preparing", "copy_logins", "create", "download_image", "ready", "connect", "verify"]+ assert "error" not in a+ _check(d)+ turn = d["turns"][0]+ assert "sandbox" in calculator.buckets(d, turn) # its own total (mock ob13)+ assert signals.check_turn(turn, d["steps"])["warn"] == 0+ _turns(engine, "again") # the sandbox is running: no setup step+ assert len([s for s in rec.as_dict()["steps"] if s["kind"] == "sandbox"]) == 1+++def test_a_failed_start_names_its_stage_and_is_an_issue(tmp_path):+ engine, rec, ws = _engine_with(tmp_path, _Provider(fail_at="create"), [AssistantTurn(text="never")])+ events = base._run(engine)+ assert any(e.type.name == "ERROR" for e in events)+ d = rec.as_dict()+ [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]+ assert setup["status"] == "error"+ assert setup["attrs"]["failed_stage"] == "create"+ assert setup["attrs"]["error"] == {"type": "ImageMissing", "text": "The sandbox base image is not downloaded yet"}+ assert d["turns"][0]["status"] == "error"+ _check(d)+ found = signals.check_turn(d["turns"][0], d["steps"])+ assert [i["check"] for i in found["issues"]] == ["sandbox_failed"]+ assert "create" in found["issues"][0]["text"] and "ImageMissing" in found["issues"][0]["text"]+ assert ws.provider.destroyed == 1+++def test_a_lazy_start_inside_a_tool_is_recorded_as_first_tool(tmp_path):+ """On resume nothing starts the sandbox at the turn's start: the first runner call does."""+ engine, rec, ws = _engine_with(tmp_path, _Provider(grants=False, image_known=True),+ [base._call("c1", "needs_runner", {"path": "x"}), AssistantTurn(text="ok")])+ engine._start_sandbox = _no_start+ base._run(engine)+ d = rec.as_dict()+ [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]+ assert setup["attrs"]["trigger"] == "first_tool"+ assert [s["name"] for s in setup["attrs"]["stages"]] == ["preparing", "create", "ready", "connect", "verify"]+ assert setup["parent_step_id"] is None and setup["turn_id"] == d["turns"][0]["turn_id"]+ [tool] = [s for s in d["steps"] if s["kind"] == "tool"]+ assert tool["started_at"] <= setup["started_at"] # it started inside the call that needed it+ _check(d)+++def test_a_restart_for_changed_folders_is_a_notice_and_the_tool_says_so(tmp_path):+ roots = [_Root("/a")]+ engine, rec, ws = _engine_with(tmp_path, _Provider(regrant=True),+ [AssistantTurn(text="ok"), base._call("c1", "run_cmd", {"command": "ls"}),+ AssistantTurn(text="done")], roots=roots)+ _turns(engine, "go", lambda: roots.append(_Root("/b")), "again") # a folder granted mid-session+ d = rec.as_dict()+ [notice] = [s for s in d["steps"] if s["kind"] == "notice" and s["name"] == "sandbox_restarted"]+ assert notice["attrs"] == {"provider": "openshell", "reason": "folders_changed", "how": "restart"}+ [tool] = [s for s in d["steps"] if s["kind"] == "tool"]+ assert tool["attrs"]["sandbox"]["notice"] == "restart"+ assert notice["started_at"] >= tool["started_at"]+ # The first start's stages did not grow with the restart's create.+ [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]+ assert [s["name"] for s in setup["attrs"]["stages"]].count("create") == 1+ _check(d)+++def test_a_runner_that_came_back_on_its_own_is_a_notice(tmp_path):+ engine, rec, ws = _engine_with(tmp_path, _Provider(),+ [AssistantTurn(text="ok"), base._call("c1", "run_cmd", {"command": "ls"}),+ AssistantTurn(text="done")])+ def came_back():+ ws.client.restarts += 1 # the client saw a new runner instance after a reconnect++ _turns(engine, "go", came_back, "again")+ d = rec.as_dict()+ [notice] = [s for s in d["steps"] if s["name"] == "sandbox_restarted"]+ assert notice["attrs"]["reason"] == "runner_restarted"+ [tool] = [s for s in d["steps"] if s["kind"] == "tool"]+ assert tool["attrs"]["sandbox"]["notice"] is None # the agent was told nothing new+ _check(d)+++def test_the_setup_span_has_one_event_per_stage(tmp_path):+ engine, rec, ws = _engine_with(tmp_path, _Provider(fail_at="verify"), [AssistantTurn(text="never")])+ base._run(engine)+ d = rec.as_dict()+ turn = d["turns"][0]+ steps = [s for s in d["steps"] if s["turn_id"] == turn["turn_id"]]+ spans = build_turn_spans(turn, steps, sw={}, dialect="otel", get_piece=lambda k: None, masker=Masker())+ [span] = [s for s in spans if s["name"] == "openworker.sandbox.setup"]+ assert span["attrs"]["openworker.sandbox.failed_stage"] == "verify"+ assert span["error"] == "RuntimeError"+ names = [e["name"] for e in span["events"]]+ assert names[0] == "openworker.sandbox.stage.preparing" and names[-1] == "openworker.sandbox.stage.verify"+ assert "listable" not in json.dumps(span) # the error's text stays on this machine