Skip to content
Python

feat(observability): sandbox setup and restarts as steps (OB-13)

Devika Vermada21e3038e9f
diff --git a/coworker/agent.py b/coworker/agent.py
index 831a23d7..3d2eccc8 100644
--- a/coworker/agent.py
+++ b/coworker/agent.py
@@ -776,6 +776,8 @@ def build_engine(
if executor is not None:
engine.permissions.shell_cwd = engine.current_shell_cwd
engine.sandbox_workspace = sandbox_workspace # type: ignore[attr-defined]
+ if sandbox_workspace is not None and hasattr(sandbox_workspace, "observer"):
+ sandbox_workspace.observer = engine.on_sandbox # setup step and restarts (OB-13)
# Whether this machine has a sandbox switched on (stamp.sandbox.expected, format 1.2), for
# the "ran without the sandbox" check: the setting, whatever this session ended up using.
engine.sandbox_expected = _sandbox_provider_name(config.sandbox_provider) != "direct"
diff --git a/coworker/engine.py b/coworker/engine.py
index 28e99932..be5d508a 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -16,6 +16,7 @@ import logging
from copy import deepcopy
import asyncio
+import inspect
import functools
import json
import time
@@ -1114,7 +1115,10 @@ class TurnEngine:
provider = str(getattr(getattr(workspace, "provider", None), "name", "") or "sandbox")
yield Event(EventType.SANDBOX_PREPARING, {"provider": provider})
try:
- await asyncio.to_thread(start)
+ if "trigger" in inspect.signature(start).parameters:
+ await asyncio.to_thread(start, "turn_start")
+ else:
+ await asyncio.to_thread(start)
except Exception as exc:
text = f"The {provider} sandbox could not start: {exc}"
self._append_notice("error", text)
@@ -1124,6 +1128,34 @@ class TurnEngine:
self._note_stamp() # the runner's version is known only now (V2-28)
yield Event(EventType.SANDBOX_READY, {k: info.get(k) for k in ("provider", "enforcement", "reason", "sandbox") if info.get(k) is not None})
+ def on_sandbox(self, event: str, data: dict[str, Any]) -> None:
+ """What the sandbox workspace tells the recording (observability OB-13): its start as a
+ `sandbox` setup step with its stages, and a restart as a `sandbox_restarted` notice.
+ Called from the thread that starts or restarts the sandbox; outside a turn nothing is
+ written."""
+ obs = self._obs
+ if not obs.enabled or not obs.in_turn:
+ return
+ provider = str(getattr(getattr(getattr(self, "sandbox_workspace", None), "provider", None), "name", "") or "sandbox")
+ if event == "setup_started":
+ self._sandbox_setup_step = obs.start_step(
+ kind="sandbox", name="setup",
+ attrs={"provider": provider, "stages": [], "failed_stage": None, "trigger": data.get("trigger") or "turn_start"},
+ )
+ elif event == "setup_ended":
+ step_id, self._sandbox_setup_step = getattr(self, "_sandbox_setup_step", None), None
+ log = data.get("log")
+ if step_id is None or log is None:
+ return
+ failed = data.get("error") is not None
+ obs.end_step(step_id, status="error" if failed else "ok",
+ attrs=log.attrs(provider, data.get("trigger") or "turn_start"))
+ if not failed and data.get("trigger") == "first_tool":
+ self._note_stamp() # the runner's version, as after a start at the turn's start
+ elif event == "restarted":
+ obs.instant_step(kind="notice", name="sandbox_restarted", status="ok",
+ attrs={"provider": provider, "reason": data.get("reason"), "how": data.get("how")})
+
@_records_turn("resume")
@_summarizes_turn
async def resume(self) -> AsyncIterator[Event]:
@@ -2970,6 +3002,15 @@ class TurnEngine:
logger.exception("observability: could not record a tool call's facts")
return out
+ @staticmethod
+ def _sandbox_notice_id(text: Any) -> Optional[str]:
+ """`restart` or `regrant` for the sandbox notice a command's result carries, else None."""
+ if not isinstance(text, str) or not text:
+ return None
+ from .sandbox.workspace import REGRANT_NOTICE, RESTART_NOTICE
+
+ return "restart" if text == RESTART_NOTICE else "regrant" if text == REGRANT_NOTICE else None
+
def _sandbox_facts(self, result: Any) -> Optional[dict[str, Any]]:
"""Per-call sandbox facts (observability E13, §9.3): whether this sandbox can report
blocked sites now, and how many connections it refused during the call. None when
@@ -2978,10 +3019,13 @@ class TurnEngine:
if ws is None or not hasattr(ws, "reports_blocked"):
return None
blocked = result.get("_blocked") if isinstance(result, dict) else None
+ told = result.get("sandbox_notice") if isinstance(result, dict) else None
return {
"blocked_known": bool(getattr(ws, "reports_blocked", False)),
"blocked_count": sum(int(b.get("count") or 0) for b in blocked or []),
- "notice": None,
+ # The restart the agent was told of with this result (OB-13): the same short id
+ # as the `sandbox_restarted` notice's `how`.
+ "notice": self._sandbox_notice_id(told),
}
def _execute_call(self, tool_call: ToolCall) -> tuple[Any, str]:
diff --git a/coworker/observability/calculator.py b/coworker/observability/calculator.py
index f6334c69..af55ae5f 100644
--- a/coworker/observability/calculator.py
+++ b/coworker/observability/calculator.py
@@ -10,7 +10,7 @@ from __future__ import annotations
from typing import Any, Iterable, Optional
-BUCKET_ORDER = ["waiting", "tools", "model", "auto_approver", "summary"]
+BUCKET_ORDER = ["waiting", "tools", "model", "auto_approver", "summary", "sandbox"]
# -- tokens (§6.3) -------------------------------------------------------------------------------
@@ -126,12 +126,15 @@ def _bucket_of(step: dict[str, Any]) -> Optional[str]:
return "auto_approver"
if kind == "summary":
return "summary"
+ if kind == "sandbox":
+ return "sandbox" # 1.2: sandbox setup, its own total so a first download is not "Other"
return None
def buckets(recording: dict[str, Any], turn: dict[str, Any], now: Optional[int] = None) -> dict[str, Any]:
"""Where the turn's time went: each bucket is the union of its steps' time, overlaps go
- to the first bucket in the order waiting, tools, model, auto-approver, summary; Other is
+ to the first bucket in the order waiting, tools, model, auto-approver, summary, sandbox
+ setup (1.2, present only when the turn has one); Other is
the rest, shown when it is 1% of the total or more (M-R5)."""
start = turn["started_at"]
# Steps found open when the app started have no time (status unknown): not counted (TD-12).
@@ -155,6 +158,8 @@ def buckets(recording: dict[str, Any], turn: dict[str, Any], now: Optional[int]
own = _subtract(raw[b], taken)
out[b] = _length(own)
taken = _merge(taken + own)
+ if not any(s.get("kind") == "sandbox" for s in steps):
+ del out["sandbox"] # 1.1 recordings read back with the same buckets as before
total = end_of_turn - start
out["total"] = total
out["other"] = total - _length(taken)
diff --git a/coworker/observability/export_spans.py b/coworker/observability/export_spans.py
index 987ba4d6..aeb45223 100644
--- a/coworker/observability/export_spans.py
+++ b/coworker/observability/export_spans.py
@@ -654,6 +654,18 @@ def build_turn_spans(
attrs[f"openworker.context.{k}"] = a[k]
elif s["kind"] == "sandbox":
name = "openworker.sandbox.setup"
+ for k in ("provider", "trigger", "failed_stage"):
+ if a.get(k) is not None:
+ attrs[f"openworker.sandbox.{k}"] = a[k]
+ # One event per stage (tracing-spec §9); the error's text stays on this machine.
+ for st in a.get("stages") or []:
+ ev_attrs = {"openworker.sandbox.stage.status": st.get("status") or "ok"}
+ if st.get("duration_ms") is not None:
+ ev_attrs["openworker.sandbox.stage.duration_ms"] = st["duration_ms"]
+ if st.get("uncertain"):
+ ev_attrs["openworker.sandbox.stage.uncertain"] = True
+ events.append({"name": f"openworker.sandbox.stage.{st.get('name')}",
+ "at": st.get("started_at") or s.get("started_at") or turn["started_at"], "attrs": ev_attrs})
else:
name = f"openworker.{s['kind']}"
diff --git a/coworker/observability/recorder.py b/coworker/observability/recorder.py
index 6c765ad9..4ebcf775 100644
--- a/coworker/observability/recorder.py
+++ b/coworker/observability/recorder.py
@@ -47,6 +47,7 @@ KIND_ORDER = {
"wait": 4,
"tool": 5,
"notice": 6,
+ "sandbox": 7, # 1.2; it never shares a start with the others in practice
}
TURN_FIELDS = (
@@ -145,6 +146,9 @@ def fill_unended(kind: str, attrs: dict[str, Any]) -> dict[str, Any]:
elif kind == "summary":
for key in ("tokens_before", "model_step_id"):
attrs.setdefault(key, None)
+ elif kind == "sandbox":
+ attrs.setdefault("stages", [])
+ attrs.setdefault("failed_stage", None)
return attrs
diff --git a/coworker/sandbox/providers/openshell.py b/coworker/sandbox/providers/openshell.py
index 5a87bc61..d641e225 100644
--- a/coworker/sandbox/providers/openshell.py
+++ b/coworker/sandbox/providers/openshell.py
@@ -40,6 +40,7 @@ from ..bundle import build_runner_zipapp
from ..registry import REGISTRY_LABEL, registry_id
from ..transport import Transport
from .. import credentials as creds
+from .. import stages
from . import openshell_policy as policy
from . import openshell_wire as wire
@@ -92,15 +93,22 @@ def _cli(*args: str, timeout: float = _CLI_TIMEOUT, check: bool = True) -> subpr
return done
+# What the last preflight learnt of the base image (True, False, None = cannot tell), for
+# the recording's setup stages: None means `sandbox create` may pull it (OB-13).
+_last_image_present: Optional[bool] = None
+
+
def preflight() -> dict[str, Any]:
"""Refuse early, with a message a person can act on."""
+ global _last_image_present
version = _cli("--version", timeout=15).stdout.strip().split()[-1]
if version != PINNED_VERSION:
raise OpenShellUnavailable(f"OpenShell {version} is installed; OpenWorker is tested with {PINNED_VERSION} only. Install that version (`openworker machine sandbox setup`).")
status = _cli("status", timeout=30, check=False)
if status.returncode != 0 or "Connected" not in status.stdout:
raise OpenShellUnavailable("The OpenShell gateway is not running or cannot be reached (`openshell status`). On a headless machine, check `systemctl --user status openshell-gateway` and `loginctl enable-linger`.")
- if image_present() is False:
+ present = _last_image_present = image_present()
+ if present is False:
raise OpenShellImageMissing(
f"{IMAGE_MISSING_PREFIX} (about 5 GB, one time). Run `openworker machine sandbox setup`, "
f"or `{image_tool() or 'docker'} pull {sandbox_image()}`, then start the session again."
@@ -288,6 +296,7 @@ class DenialLog:
class OpenShellProvider:
name = "openshell"
+ reports_stages = True # its start's inner stages go to the recording (stages.stage)
def __init__(
self,
@@ -352,17 +361,26 @@ class OpenShellProvider:
raise
def _create(self) -> None:
+ global _last_image_present
+ _last_image_present = None
preflight()
for root in self.roots:
if not os.path.isdir(root["path"]):
raise RuntimeError(f"folder does not exist on this machine: {root['path']}")
uid, gid = (os.getuid(), os.getgid()) if sys.platform.startswith("linux") else (None, None)
+ if self.grants:
+ stages.stage(self, "copy_logins")
# The sandbox's private home with the copied credentials (section 11b), mounted at
# the same path it has on the machine, read-write. With no grants it holds only
# git's settings, and HOME stays the runtime folder.
self.copied = creds.copy_in(self.grants, self._tmp) if self.grants else None
home = self.copied.home if self.copied is not None else None
self._identity = (uid, gid, home)
+ stages.stage(self, "create")
+ if _last_image_present is None:
+ # Nobody here can say whether the image is present: `sandbox create` may pull it,
+ # and how long that took cannot be told apart from the create.
+ stages.mark(self, "download_image", uncertain=True)
policy_file = self._write_policy()
driver_config = json.dumps(policy.mounts(self.roots, str(self._runner.parent), home))
command = [policy.PYTHON, "-S", f"{policy.RUNNER_MOUNT}/{self._runner.name}", "serve", "--socket", _SOCKET, "--cwd", self.cwd]
@@ -396,6 +414,7 @@ class OpenShellProvider:
if docker_host_network() is False:
raise OpenShellUnavailable(f"{DOCKER_HOST_NETWORK_FIX}. Then start the session again.") from exc
raise
+ stages.stage(self, "ready")
self._wait_ready()
self._denials = DenialLog(self.sandbox_name) # what it refuses, for the agent and the card
# Spike finding K: the first exec after a sandbox turns Ready hangs until it times
diff --git a/coworker/sandbox/providers/seatbelt.py b/coworker/sandbox/providers/seatbelt.py
index b9b63bfb..6b5e9dce 100644
--- a/coworker/sandbox/providers/seatbelt.py
+++ b/coworker/sandbox/providers/seatbelt.py
@@ -29,7 +29,7 @@ from pathlib import Path
from typing import Any, Callable, Optional, Sequence
from .. import credentials as creds
-from .. import netproxy, network_profiles
+from .. import netproxy, network_profiles, stages
from ..bundle import build_runner_zipapp
from ..launch import read_paths, runner_command
from ..transport import PipeTransport, Transport
@@ -80,6 +80,7 @@ def clean_environment(base: Optional[dict[str, str]] = None) -> dict[str, str]:
class SeatbeltProvider:
name = "seatbelt"
+ reports_stages = True # its start's inner stages go to the recording (stages.stage)
def __init__(
self,
@@ -163,6 +164,8 @@ class SeatbeltProvider:
def _create(self) -> None:
preflight()
os.makedirs(os.path.join(self._dir, "tmp", "cache"), exist_ok=True)
+ if self.grants:
+ stages.stage(self, "copy_logins")
if self.network and not network_profiles.is_open(self.profile):
hosts = sorted({*self.extra_hosts, *(h for g in self.grants for h in g.hosts)})
# Each session has its own proxy: its allow list can grow while it runs (add_hosts).
@@ -175,6 +178,7 @@ class SeatbeltProvider:
proxy_port=self._proxy.port if self._proxy is not None else None,
ssh_proxy_command=creds.mac_ssh_proxy_command(self._proxy.port) if self._proxy is not None else None,
)
+ stages.stage(self, "create")
log = open(os.path.join(self._dir, "daemon.log"), "wb")
self._daemon = subprocess.Popen(
[
diff --git a/coworker/sandbox/providers/windows.py b/coworker/sandbox/providers/windows.py
index 5a57c741..6a4d1eb7 100644
--- a/coworker/sandbox/providers/windows.py
+++ b/coworker/sandbox/providers/windows.py
@@ -40,7 +40,7 @@ from pathlib import Path
from typing import Any, Optional, Sequence
from .. import credentials as creds
-from .. import netproxy, network_profiles
+from .. import netproxy, network_profiles, stages
from ..bundle import build_runner_zipapp
from ..launch import runner_command, serve_arguments, spawn_kwargs, wait_for_runner
from ..runner import winpipe
@@ -76,6 +76,7 @@ def preflight() -> None:
class WindowsProvider:
name = "windows"
+ reports_stages = True # its start's inner stages go to the recording (stages.stage)
def __init__(
self,
@@ -155,6 +156,8 @@ class WindowsProvider:
preflight()
from .. import winsec
+ stages.stage(self, "create")
+
hosts = sorted({*self.extra_hosts, *(h for g in self.grants for h in g.hosts)})
if self.network and not self.open_network:
# Each session has its own proxy: its allow list can grow while it runs (add_hosts).
diff --git a/coworker/sandbox/stages.py b/coworker/sandbox/stages.py
new file mode 100644
index 00000000..9ea9e0b0
--- /dev/null
+++ b/coworker/sandbox/stages.py
@@ -0,0 +1,85 @@
+"""The stages of one sandbox start, for the recording's setup step (observability OB-13,
+format 1.2 `sandbox_attrs`). Stage names come from a fixed list; each stage is closed when
+the next one begins, and the last one when the start ends."""
+
+from __future__ import annotations
+
+import threading
+import time
+from typing import Any, Optional
+
+STAGES = ("preparing", "download_image", "copy_logins", "create", "connect", "verify", "ready")
+
+
+class StageLog:
+ def __init__(self) -> None:
+ self._lock = threading.Lock()
+ self.stages: list[dict[str, Any]] = []
+ self._open: Optional[tuple[dict[str, Any], float]] = None
+ self.failed_stage: Optional[str] = None
+ self.error: Optional[dict[str, Any]] = None
+ self.started_at = int(time.time() * 1000)
+
+ def begin(self, name: str) -> None:
+ """Close the stage in progress as `ok` and start `name`."""
+ if name not in STAGES:
+ return
+ with self._lock:
+ self._close("ok")
+ stage = {"name": name, "started_at": int(time.time() * 1000), "duration_ms": None, "status": "ok"}
+ self.stages.append(stage)
+ self._open = (stage, time.monotonic())
+
+ def mark(self, name: str, **extra: Any) -> None:
+ """A stage that cannot be timed on its own (OpenShell's image pull happens inside
+ `sandbox create`): written with no duration, the stage in progress left open."""
+ if name not in STAGES:
+ return
+ with self._lock:
+ self.stages.append({"name": name, "started_at": int(time.time() * 1000), "duration_ms": None, "status": "ok", **extra})
+
+ def end(self, error: Optional[BaseException] = None) -> None:
+ """The start is over. With `error`, the stage in progress failed."""
+ with self._lock:
+ if error is not None:
+ if self._open is not None:
+ self.failed_stage = self._open[0]["name"]
+ elif self.stages:
+ self.failed_stage = self.stages[-1]["name"]
+ else:
+ self.failed_stage = "preparing"
+ text = str(error).strip()
+ self.error = {"type": type(error).__name__, "text": text or None}
+ self._close("error" if error is not None else "ok")
+
+ def _close(self, status: str) -> None:
+ if self._open is None:
+ return
+ stage, mono = self._open
+ stage["duration_ms"] = max(0, int((time.monotonic() - mono) * 1000))
+ stage["status"] = status
+ self._open = None
+
+ def attrs(self, provider: str, trigger: str) -> dict[str, Any]:
+ out: dict[str, Any] = {
+ "provider": provider,
+ "stages": [dict(s) for s in self.stages],
+ "failed_stage": self.failed_stage,
+ "trigger": trigger,
+ }
+ if self.error is not None:
+ out["error"] = dict(self.error)
+ return out
+
+
+def stage(provider: Any, name: str) -> None:
+ """A provider says it moved to stage `name` of its start. Nothing when nobody listens."""
+ log = getattr(provider, "stage_log", None)
+ if isinstance(log, StageLog):
+ log.begin(name)
+
+
+def mark(provider: Any, name: str, **extra: Any) -> None:
+ log = getattr(provider, "stage_log", None)
+ if isinstance(log, StageLog):
+ log.mark(name, **extra)
diff --git a/coworker/sandbox/workspace.py b/coworker/sandbox/workspace.py
index 61d01d45..f457d175 100644
--- a/coworker/sandbox/workspace.py
+++ b/coworker/sandbox/workspace.py
@@ -20,6 +20,7 @@ from pathlib import Path
from typing import Any, Callable, Optional, Sequence
from .runner.executor import Executor
+from .stages import StageLog
PROVIDER_ENV = "OPENWORKER_SANDBOX_PROVIDER"
DIRECT = "direct"
@@ -106,46 +107,89 @@ class RunnerWorkspace(Workspace):
# Refusals up to here have been told to the agent. One that reaches us after its
# command's result (OpenShell's log lags) is told with the next result, not lost.
self._blocked_told = time.time()
+ # What the recording hears of this sandbox (observability OB-13): `observer(event,
+ # data)` with `setup_started`, `setup_ended` and `restarted`. Set by the engine; it
+ # is called from whatever thread starts or restarts the sandbox.
+ self.observer: Optional[Callable[[str, dict[str, Any]], None]] = None
+ self.last_start: Optional[StageLog] = None
+ self._restarts_seen = 0 # the client's restart count already told
if start:
self.ensure_started()
else:
self.client.starter = self.ensure_started
- def ensure_started(self) -> None:
+ def ensure_started(self, trigger: str = "first_tool") -> None:
"""Make the sandbox, connect to its runner and prove the wall. Once; a failure
- leaves nothing behind and the next call tries again."""
+ leaves nothing behind and the next call tries again. `trigger` is for the recording:
+ `turn_start` when a turn starts it, `first_tool` when the first runner call does."""
with self._start_lock:
if self.started:
return
- registry = self.registry
- if registry is not None:
- registry.reap() # sandboxes left behind by a server that is gone
- registry.check_room()
- # Reserve the name before the sandbox exists: engine builds run concurrently
- # (OPE-206), and another build's reap() would otherwise delete this one while
- # it is still provisioning, because it is not yet in the registry.
- self._record(state="creating")
+ log = StageLog()
+ self.last_start = log
+ self._tell("setup_started", {"trigger": trigger, "log": log})
try:
- self.provider.create()
- except Exception:
- self.provider.destroy() # a half-made sandbox may already hold copied credentials
- if registry is not None and self._registered:
- registry.close(self._registered)
- self._registered = None
- raise
- try:
- self.hello = self.client.connect()
- self._after_connect()
- except Exception:
- self.client.detach()
- self.provider.destroy()
- if registry is not None and self._registered:
- registry.close(self._registered)
- self._registered = None
+ self._start(log)
+ except BaseException as exc:
+ log.end(exc)
+ self._tell("setup_ended", {"trigger": trigger, "log": log, "error": exc})
raise
- self._record()
- self.started = True
- self.client.starter = None
+ finally:
+ if getattr(self.provider, "stage_log", None) is log:
+ self.provider.stage_log = None # a later restart's create is not this start
+ log.end()
+ self._restarts_seen = self.client.restarts
+ self._tell("setup_ended", {"trigger": trigger, "log": log, "error": None})
+
+ def _start(self, log: StageLog) -> None:
+ # Providers that know their own inner stages report them through `stage_log`; for
+ # the others the whole `create` is one stage.
+ try:
+ self.provider.stage_log = log
+ except AttributeError:
+ pass
+ log.begin("preparing")
+ registry = self.registry
+ if registry is not None:
+ registry.reap() # sandboxes left behind by a server that is gone
+ registry.check_room()
+ # Reserve the name before the sandbox exists: engine builds run concurrently
+ # (OPE-206), and another build's reap() would otherwise delete this one while
+ # it is still provisioning, because it is not yet in the registry.
+ self._record(state="creating")
+ if not getattr(self.provider, "reports_stages", False):
+ log.begin("create")
+ try:
+ self.provider.create()
+ except Exception:
+ self.provider.destroy() # a half-made sandbox may already hold copied credentials
+ if registry is not None and self._registered:
+ registry.close(self._registered)
+ self._registered = None
+ raise
+ try:
+ log.begin("connect")
+ self.hello = self.client.connect()
+ self._after_connect(log)
+ except Exception:
+ self.client.detach()
+ self.provider.destroy()
+ if registry is not None and self._registered:
+ registry.close(self._registered)
+ self._registered = None
+ raise
+ self._record()
+ self.started = True
+ self.client.starter = None
+
+ def _tell(self, event: str, data: dict[str, Any]) -> None:
+ observer = self.observer
+ if observer is None:
+ return
+ try:
+ observer(event, data)
+ except Exception: # noqa: BLE001 - the recording never stops a sandbox
+ pass
def _record(self, state: str = "ready") -> None:
"""Enter this sandbox in the registry; after a restart that replaced it, under its
@@ -173,6 +217,10 @@ class RunnerWorkspace(Workspace):
started with (design ruling 15). Returns what to tell the agent, or None. Runs
before every command, so it is also where a sandbox not yet started starts."""
self.ensure_started()
+ if self.client.restarts > self._restarts_seen:
+ # The runner came back as a new instance on its own (a crash and reconnect).
+ self._restarts_seen = self.client.restarts
+ self._tell("restarted", {"reason": "runner_restarted", "how": "restart"})
regrant = getattr(self.provider, "regrant", None)
if regrant is None or self._live_roots is None:
return None
@@ -187,23 +235,30 @@ class RunnerWorkspace(Workspace):
regrant(wanted, before_create=lambda: self._record(state="creating"))
self.client.connect() # a new runner: the client notes the restart
self._after_connect()
+ self._restarts_seen = self.client.restarts
else:
regrant(wanted) # the same sandbox, re-granted in place (Windows: entries on folders)
verify = getattr(self.provider, "verify", None)
if verify is not None:
verify(self.client)
self._record()
+ how = "restart" if restarts else "regrant"
+ self._tell("restarted", {"reason": "folders_changed", "how": how})
return RESTART_NOTICE if restarts else REGRANT_NOTICE
- def _after_connect(self) -> None:
+ def _after_connect(self, log: Optional[StageLog] = None) -> None:
"""What a provider does once it can talk to its runner: prove the wall is up, then
put in what it could not put in before the runner existed (copies that must be
owned by the sandbox's own account)."""
verify = getattr(self.provider, "verify", None)
if verify is not None:
+ if log is not None:
+ log.begin("verify")
verify(self.client) # e.g. every folder is really reachable inside
provision = getattr(self.provider, "provision", None)
if provision is not None:
+ if log is not None:
+ log.begin("copy_logins")
provision(self.client)
@property
@@ -408,7 +463,9 @@ def open_workspace(
if name == RUNNER_LOCAL:
from .providers.runner_local import RunnerLocalProvider
- return RunnerWorkspace(RunnerLocalProvider(cwd=cwd), cwd=cwd)
+ # Like the real providers, it starts when the first turn needs it (`start`), so its
+ # start is a turn's setup step too (observability OB-13).
+ return RunnerWorkspace(RunnerLocalProvider(cwd=cwd), cwd=cwd, start=start)
listed = [{"path": str(r.path), "writable": bool(r.writable)} for r in (roots or [])]
listed = listed or [{"path": str(cwd), "writable": True}]
from .credentials import granted
diff --git a/surfaces/gui/src/components/StepsPanel.test.tsx b/surfaces/gui/src/components/StepsPanel.test.tsx
index e06ba488..20848b4e 100644
--- a/surfaces/gui/src/components/StepsPanel.test.tsx
+++ b/surfaces/gui/src/components/StepsPanel.test.tsx
@@ -267,6 +267,56 @@ describe("StepsPanel", () => {
});
});
+describe("Sandbox setup (OB-13)", () => {
+ const setupTurn = { ...turn, buckets: { ...turn.buckets, sandbox: 4000, other: 0, other_shown: false } };
+ const stages = [
+ { name: "preparing", started_at: 1000, duration_ms: 100, status: "ok" },
+ { name: "copy_logins", started_at: 1100, duration_ms: 100, status: "ok" },
+ { name: "create", started_at: 1200, duration_ms: 3000, status: "ok" },
+ { name: "download_image", started_at: 1200, duration_ms: null, status: "ok", uncertain: true },
+ { name: "connect", started_at: 4200, duration_ms: 400, status: "ok" },
+ { name: "verify", started_at: 4600, duration_ms: 400, status: "error" },
+ ];
+ const setup = step({ step_id: "sb", kind: "sandbox", name: "setup", duration_ms: 4000, status: "error",
+ attrs: { provider: "seatbelt", stages, failed_stage: "verify", trigger: "turn_start",
+ error: { type: "SeatbeltUnavailable", text: "the home folder was listable from inside" } } });
+ const restart = step({ step_id: "rs", kind: "notice", name: "sandbox_restarted", started_at: 6000, duration_ms: 0,
+ attrs: { provider: "seatbelt", reason: "folders_changed", how: "restart" } });
+
+ it("draws the setup row with its stages, its own total and the restart row", async () => {
+ vi.mocked(getRecordedTurns).mockResolvedValue([setupTurn] as any);
+ vi.mocked(getTurnSteps).mockResolvedValue([setup, restart] as any);
+ render(<StepsPanel sessionId="s1" onClose={() => {}} />);
+ await waitFor(() => expect(screen.getAllByTestId("steps-row")).toHaveLength(2));
+ expect(screen.getByTestId("steps-tiles").textContent).toContain("Sandbox setup");
+ const [row, notice] = screen.getAllByTestId("steps-row").map((r) => r.textContent || "");
+ expect(row).toContain("Sandbox setup");
+ expect(row).toContain("failed at verify");
+ expect(notice).toContain("Sandbox restarted");
+ expect(notice).toContain("folders");
+ const stageRows = screen.getAllByTestId("steps-stage").map((r) => r.textContent || "");
+ // The pull OpenShell cannot time is said on the Create row, not as a row of its own.
+ expect(stageRows).toHaveLength(5);
+ expect(stageRows[2]).toContain("Create · image pulled?");
+ expect(stageRows[4]).toContain("Verify");
+ expect(stageRows[4]).toContain("error");
+ });
+
+ it("says in the details which stage failed and what the sandbox said", async () => {
+ vi.mocked(getRecordedTurns).mockResolvedValue([setupTurn] as any);
+ vi.mocked(getTurnSteps).mockResolvedValue([setup] as any);
+ vi.mocked(getStepDetail).mockResolvedValue(setup as any);
+ render(<StepsPanel sessionId="s1" onClose={() => {}} />);
+ await waitFor(() => expect(screen.getAllByTestId("steps-row")).toHaveLength(1));
+ fireEvent.click(screen.getAllByTestId("steps-row")[0]);
+ const failed = await screen.findByTestId("steps-sandbox-failed");
+ expect(failed.textContent).toContain("could not start at verify (SeatbeltUnavailable)");
+ const details = screen.getByTestId("steps-details").textContent || "";
+ expect(details).toContain("at the turn's start");
+ expect(details).toContain("the home folder was listable from inside");
+ });
+});
+
describe("formatDuration", () => {
it("reads like the mock", () => {
expect(formatDuration(100)).toBe("0.1s");
diff --git a/surfaces/gui/src/components/StepsPanel.tsx b/surfaces/gui/src/components/StepsPanel.tsx
index cc1181f4..fe007518 100644
--- a/surfaces/gui/src/components/StepsPanel.tsx
+++ b/surfaces/gui/src/components/StepsPanel.tsx
@@ -5,7 +5,7 @@
// §12); `refreshKey` moves when live step events or a turn's end arrive, so the panel follows a
// running turn.
-import { useEffect, useMemo, useState } from "react";
+import { Fragment, useEffect, useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import {
getRecordedTurns,
@@ -23,7 +23,7 @@ import { formatTokens } from "../usage";
type T = (k: string, o?: any) => string;
-const BUCKETS = ["waiting", "tools", "model", "auto_approver", "summary"] as const;
+const BUCKETS = ["waiting", "tools", "model", "auto_approver", "summary", "sandbox"] as const;
/** 0.1s, 3.4s, 41s, 2m 14s: the panel's one duration format. */
export function formatDuration(ms: number | null | undefined): string {
@@ -125,7 +125,10 @@ export function stepLabel(step: RecordedStep, t: T): string {
const arrow = before != null && after != null ? ` · ${formatTokens(before)}→${formatTokens(after)}` : "";
return (step.name === "trim" ? t("steps.row.trim") : t("steps.row.summary")) + arrow;
}
+ case "sandbox":
+ return t("steps.row.sandbox_setup");
case "notice":
+ if (step.name === "sandbox_restarted" && a.how === "regrant") return t("steps.notice.sandbox_regranted");
return t(`steps.notice.${step.name}`, { defaultValue: step.name });
default:
return step.name;
@@ -159,6 +162,11 @@ export function stepFlags(step: RecordedStep, t: T): Flag[] {
if (a.decision?.origin === "reviewer") flags.push({ cls: "info", text: t("steps.flag.auto_approved") });
} else if (step.kind === "auto_approver") {
if (step.status === "interrupted") flags.push({ cls: "err", text: t("steps.status.interrupted") });
+ } else if (step.kind === "sandbox" && step.status === "error") {
+ const stage = a.failed_stage || "preparing";
+ flags.push({ cls: "err", text: t("steps.flag.sandbox_failed", { stage: t(`steps.sbx_stage.${stage}`, { defaultValue: stage }).toLowerCase() }) });
+ } else if (step.kind === "notice" && step.name === "sandbox_restarted") {
+ if (a.reason) flags.push({ cls: "info", text: t(`steps.flag.restart_${a.reason}`, { defaultValue: a.reason }) });
} else if (["error", "denied", "interrupted", "skipped", "unknown"].includes(step.status)) {
flags.push({ cls: "err", text: t(`steps.status.${step.status}`) });
}
@@ -187,8 +195,47 @@ function nest(steps: RecordedStep[]): { step: RecordedStep; depth: number }[] {
const KIND_CLASS: Record<string, string> = {
context: "ctx", model: "model", tool: "tool", wait: "wait", auto_approver: "review", summary: "sum", notice: "ctx",
+ sandbox: "sbx",
};
+/** The class of a row: a sandbox restart notice is drawn in the sandbox colour (mock ob13). */
+function kindClass(step: RecordedStep): string {
+ if (step.kind === "notice" && step.name === "sandbox_restarted") return "sbx";
+ return KIND_CLASS[step.kind] || "ctx";
+}
+
+type SandboxStage = { name: string; started_at: number; duration_ms: number | null; status: string; uncertain?: boolean };
+
+/** A sandbox setup step's stages as rows under it, in the order run (OB-13). An image pull
+ * the provider cannot time is not a row of its own: it is said on the Create row. */
+function StageRows({ step, depth, turn, total }: { step: RecordedStep; depth: number; turn: RecordedTurn; total: number }) {
+ const { t } = useTranslation();
+ const stages: SandboxStage[] = Array.isArray(step.attrs?.stages) ? step.attrs.stages : [];
+ const pulled = stages.some((s) => s.name === "download_image" && s.uncertain);
+ return (
+ <>
+ {stages.filter((s) => !(s.name === "download_image" && s.uncertain)).map((s, i) => {
+ const left = Math.min(100, Math.max(0, ((s.started_at - turn.started_at) / total) * 100));
+ const width = s.duration_ms ? Math.min(100 - left, (s.duration_ms / total) * 100) : 0;
+ const name = t(`steps.sbx_stage.${s.name}`, { defaultValue: s.name }) + (s.name === "create" && pulled ? ` · ${t("steps.sbx_stage.image_maybe")}` : "");
+ return (
+ <div key={`${s.name}-${i}`} className="steps-row steps-stage k-sbx" data-testid="steps-stage">
+ <span className="steps-label" style={{ paddingLeft: (depth + 1) * 16 }}>
+ <span className="steps-kind" />
+ <span className="steps-name">{name}</span>
+ {s.status === "error" && <span className="steps-badge err">{t("steps.status.error")}</span>}
+ </span>
+ <span className="steps-track">
+ <span className="steps-bar" style={{ left: `${left}%`, width: width ? `${width}%` : undefined }} />
+ </span>
+ <span className="steps-time">{formatDuration(s.duration_ms)}</span>
+ </div>
+ );
+ })}
+ </>
+ );
+}
+
export function StepsPanel({
sessionId,
refreshKey = 0,
@@ -290,17 +337,19 @@ export function StepsPanel({
<span className="steps-axis-time">{t("steps.axis.time")}</span>
</div>
{rows.map(({ step, depth }) => (
- <StepRow
- key={step.step_id}
- step={step}
- depth={depth}
- turn={turn}
- total={total}
- label={stepLabel(step, t)}
- selected={selected === step.step_id}
- issue={flagged.has(step.step_id)}
- onSelect={() => setSelected(selected === step.step_id ? null : step.step_id)}
- />
+ <Fragment key={step.step_id}>
+ <StepRow
+ step={step}
+ depth={depth}
+ turn={turn}
+ total={total}
+ label={stepLabel(step, t)}
+ selected={selected === step.step_id}
+ issue={flagged.has(step.step_id)}
+ onSelect={() => setSelected(selected === step.step_id ? null : step.step_id)}
+ />
+ {step.kind === "sandbox" && <StageRows step={step} depth={depth} turn={turn} total={total} />}
+ </Fragment>
))}
</div>
);
@@ -412,7 +461,7 @@ function StepRow({ step, depth, turn, total, label, selected, issue, onSelect }:
const { t } = useTranslation();
const left = Math.min(100, Math.max(0, ((step.started_at - turn.started_at) / total) * 100));
const width = step.duration_ms ? Math.min(100 - left, (step.duration_ms / total) * 100) : 0;
- const kind = KIND_CLASS[step.kind] || "ctx";
+ const kind = kindClass(step);
return (
<button
className={`steps-row k-${kind}` + (selected ? " selected" : "") + (step.shadow ? " shadow" : "")}
@@ -655,7 +704,7 @@ function statusPill(d: any, t: T): { cls: string; text: string } {
function Inspector({ d, turn, steps, sessionId, t }: { d: any; turn: RecordedTurn; steps: RecordedStep[]; sessionId: string; t: T }) {
const a = d.attrs || {};
- const kind = KIND_CLASS[d.kind] || "ctx";
+ const kind = kindClass(d);
const started = clock(d.started_at - turn.started_at);
const ctxStep = steps.find((s) => s.kind === "context");
const exchanges = (d.exchange_rebuilt || []) as any[];
@@ -813,12 +862,45 @@ function Inspector({ d, turn, steps, sessionId, t }: { d: any; turn: RecordedTur
[t("steps.detail.tokens_after"), a.tokens_after == null ? "—" : formatTokens(a.tokens_after)]);
}
if (d.kind === "context") rows.push([t("steps.detail.session_prompt"), <span className="steps-mono">{a.session_prompt_hash || "—"}</span>]);
+ if (d.kind === "sandbox") {
+ rows.push([t("steps.sbx.provider"), a.provider || "—"],
+ [t("steps.sbx.trigger"), t(`steps.sbx.trigger_${a.trigger}`, { defaultValue: a.trigger || "—" })]);
+ }
+ if (d.kind === "notice" && d.name === "sandbox_restarted") {
+ rows.push([t("steps.sbx.provider"), a.provider || "—"],
+ [t("steps.sbx.reason"), t(`steps.sbx.reason_${a.reason}`, { defaultValue: a.reason || "—" })],
+ [t("steps.sbx.how"), t(`steps.sbx.how_${a.how}`, { defaultValue: a.how || "—" })]);
+ }
+ const sbxStages: SandboxStage[] = d.kind === "sandbox" && Array.isArray(a.stages) ? a.stages : [];
const changed = d.kind === "notice" && d.name === "stamp_changed" ? stampChanges(a, t) : [];
const showsContext = d.kind === "context" || (d.kind === "notice" && d.name === "context_changed");
return (
<>
<h3 className="steps-h3"><span className={`steps-kind k-${kind}`} />{stepLabel(d, t)}<span className={`steps-pill ${pill.cls}`}>{pill.text}</span></h3>
+ {d.kind === "sandbox" && d.status === "error" && (
+ <div className="steps-callout err" data-testid="steps-sandbox-failed">
+ {t("steps.sbx.failed", {
+ stage: t(`steps.sbx_stage.${a.failed_stage || "preparing"}`, { defaultValue: a.failed_stage || "preparing" }).toLowerCase(),
+ type: a.error?.type || "error",
+ })}
+ </div>
+ )}
<Kv rows={rows} />
+ {sbxStages.length > 0 && (
+ <>
+ <Sec label={t("steps.sbx.stages")} />
+ <Kv rows={sbxStages.map((s) => [
+ t(`steps.sbx_stage.${s.name}`, { defaultValue: s.name }),
+ s.uncertain ? t("steps.sbx.image_note") : `${formatDuration(s.duration_ms)}${s.status === "error" ? ` · ${t("steps.status.error")}` : ""}`,
+ ] as [string, React.ReactNode])} />
+ </>
+ )}
+ {d.kind === "sandbox" && a.error?.text && (
+ <>
+ <Sec label={t("steps.sbx.error_text")} />
+ <div className="steps-box">{a.error.text}</div>
+ </>
+ )}
{changed.length > 0 && (
<>
<Sec label={t("steps.insp.changed")} />
diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.json
index 751d1c56..986cd766 100644
--- a/surfaces/gui/src/locales/en.json
+++ b/surfaces/gui/src/locales/en.json
@@ -3037,6 +3037,32 @@
}
},
"steps": {
+ "sbx_stage": {
+ "preparing": "Preparing",
+ "download_image": "Download image",
+ "copy_logins": "Copy logins",
+ "create": "Create",
+ "connect": "Connect",
+ "verify": "Verify",
+ "ready": "Wait until ready",
+ "image_maybe": "image pulled?"
+ },
+ "sbx": {
+ "provider": "Sandbox",
+ "trigger": "Started",
+ "trigger_turn_start": "at the turn's start",
+ "trigger_first_tool": "on first use",
+ "failed": "The sandbox could not start at {{stage}} ({{type}}). The turn ended; Retry tries again.",
+ "error_text": "What the sandbox said",
+ "stages": "Stages",
+ "reason": "Why",
+ "how": "What happened",
+ "reason_folders_changed": "the session's folders changed",
+ "reason_runner_restarted": "the runner came back as a new instance",
+ "how_restart": "a new sandbox; shells and variables started again",
+ "how_regrant": "folder grants changed in place; shells kept",
+ "image_note": "The image may have been downloaded inside Create; the provider cannot say how long that took."
+ },
"issues": {
"tab": "Issues",
"tab_n": "Issues · {{n}}",
@@ -3184,7 +3210,8 @@
"model": "Model",
"auto_approver": "Auto-approver",
"summary": "Summary",
- "other": "Other"
+ "other": "Other",
+ "sandbox": "Sandbox setup"
},
"checks_line_one": "Auto-approve: {{count}} check · {{allowed}} allowed · {{asked}} asked you · {{blocked}} blocked · {{time}} · {{reviewer}}",
"checks_line_other": "Auto-approve: {{count}} checks · {{allowed}} allowed · {{asked}} asked you · {{blocked}} blocked · {{time}} · {{reviewer}}",
@@ -3221,7 +3248,8 @@
"summary": "Summary",
"trim": "Trim",
"wait_inbox": "Waiting · parked in Inbox",
- "shadow": "Shadow check"
+ "shadow": "Shadow check",
+ "sandbox_setup": "Sandbox setup"
},
"would": {
"allow": "would have allowed",
@@ -3235,7 +3263,10 @@
"no_usage": "no usage",
"exit": "exit {{code}}",
"blocked": "blocked by sandbox",
- "auto_approved": "auto-approved"
+ "auto_approved": "auto-approved",
+ "sandbox_failed": "failed at {{stage}}",
+ "restart_folders_changed": "folders",
+ "restart_runner_restarted": "runner came back"
},
"purpose": {
"main": "main",
@@ -3280,7 +3311,9 @@
"context_changed": "Context changed",
"stamp_changed": "Settings changed",
"summary_failed": "Summary failed",
- "recording_gap": "Recording gap"
+ "recording_gap": "Recording gap",
+ "sandbox_restarted": "Sandbox restarted",
+ "sandbox_regranted": "Sandbox folders updated"
},
"status": {
"ok": "ok",
diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.json
index 2fabf3f4..a078aa21 100644
--- a/surfaces/gui/src/locales/zh.json
+++ b/surfaces/gui/src/locales/zh.json
@@ -2993,6 +2993,32 @@
}
},
"steps": {
+ "sbx_stage": {
+ "preparing": "准备",
+ "download_image": "下载镜像",
+ "copy_logins": "复制登录信息",
+ "create": "创建",
+ "connect": "连接",
+ "verify": "验证",
+ "ready": "等待就绪",
+ "image_maybe": "可能拉取了镜像"
+ },
+ "sbx": {
+ "provider": "沙箱",
+ "trigger": "启动于",
+ "trigger_turn_start": "回合开始时",
+ "trigger_first_tool": "首次使用时",
+ "failed": "沙箱在{{stage}}阶段无法启动({{type}})。本回合已结束;重试会再次尝试。",
+ "error_text": "沙箱返回的信息",
+ "stages": "阶段",
+ "reason": "原因",
+ "how": "发生了什么",
+ "reason_folders_changed": "会话的文件夹已变化",
+ "reason_runner_restarted": "运行器作为新实例重新连接",
+ "how_restart": "新的沙箱;终端和变量重新开始",
+ "how_regrant": "就地更改了文件夹授权;终端保留",
+ "image_note": "镜像可能在“创建”阶段下载;提供方无法说明用了多长时间。"
+ },
"issues": {
"tab": "问题",
"tab_n": "问题 · {{n}}",
@@ -3136,7 +3162,8 @@
"model": "模型",
"auto_approver": "自动审批",
"summary": "摘要",
- "other": "其他"
+ "other": "其他",
+ "sandbox": "沙箱启动"
},
"checks_line_other": "自动审批:{{count}} 次检查 · {{allowed}} 次允许 · {{asked}} 次询问你 · {{blocked}} 次阻止 · {{time}} · {{reviewer}}",
"unknown_line_one": "{{count}} 个步骤在 OpenWorker 停止时仍在运行,其用时未知。",
@@ -3172,7 +3199,8 @@
"summary": "摘要",
"trim": "裁剪",
"wait_inbox": "等待 · 已放入收件箱",
- "shadow": "影子检查"
+ "shadow": "影子检查",
+ "sandbox_setup": "沙箱启动"
},
"would": {
"allow": "本会允许",
@@ -3186,7 +3214,10 @@
"no_usage": "未报告用量",
"exit": "退出码 {{code}}",
"blocked": "被沙箱阻止",
- "auto_approved": "已自动批准"
+ "auto_approved": "已自动批准",
+ "sandbox_failed": "在{{stage}}失败",
+ "restart_folders_changed": "文件夹",
+ "restart_runner_restarted": "运行器已重启"
},
"purpose": {
"main": "主要",
@@ -3231,7 +3262,9 @@
"context_changed": "上下文已变化",
"stamp_changed": "设置已变化",
"summary_failed": "摘要失败",
- "recording_gap": "记录缺口"
+ "recording_gap": "记录缺口",
+ "sandbox_restarted": "沙箱已重启",
+ "sandbox_regranted": "沙箱文件夹已更新"
},
"status": {
"ok": "正常",
diff --git a/surfaces/gui/src/styles.css b/surfaces/gui/src/styles.css
index 7eb407b5..077738c3 100644
--- a/surfaces/gui/src/styles.css
+++ b/surfaces/gui/src/styles.css
@@ -2430,8 +2430,8 @@ html[data-platform="linux"] ::-webkit-scrollbar-thumb:hover { background-color:
/* Observability Phase 1 (UX-059 OB-1, OB-2, OB-11, OB-4): the Steps panel in the right rail,
styled as the UX-059 mock (docs/ux-improvements/mocks/UX-059-observability-desktop.html).
The step colours are the mock's --bar-* tokens, light and dark. */
-:root { --bar-model:#2563eb; --bar-tool:#0f766e; --bar-wait:#d8dce1; --bar-review:#7c3aed; --bar-sum:#92400e; --bar-ctx:#6f7680; --bar-write:#f59e0b; }
-html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3a3e46; --bar-review:#a78bfa; --bar-sum:#e8b04b; --bar-ctx:#9aa1ab; --bar-write:#f59e0b; }
+:root { --bar-model:#2563eb; --bar-tool:#0f766e; --bar-wait:#d8dce1; --bar-review:#7c3aed; --bar-sum:#92400e; --bar-ctx:#6f7680; --bar-write:#f59e0b; --bar-sbx:var(--ok); }
+html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3a3e46; --bar-review:#a78bfa; --bar-sum:#e8b04b; --bar-ctx:#9aa1ab; --bar-write:#f59e0b; --bar-sbx:var(--ok); }
.team-view.steps-panel { background:var(--panel); font-size:var(--fs-ui); line-height:1.45; container-type:inline-size; }
/* A step's details open beside the list: the rail widens to hold both (mock: list + 520px). */
.main:has(.steps-panel.inspecting) { --artifact-rail-w:min(78vw, 1180px); }
@@ -2485,6 +2485,9 @@ html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3
.k-review .steps-kind, .steps-kind.k-review, .k-review .steps-bar { background:var(--bar-review); }
.k-sum .steps-kind, .steps-kind.k-sum, .k-sum .steps-bar { background:var(--bar-sum); }
.k-ctx .steps-kind, .steps-kind.k-ctx, .k-ctx .steps-bar { background:var(--bar-ctx); }
+.k-sbx .steps-kind, .steps-kind.k-sbx, .k-sbx .steps-bar { background:var(--bar-sbx); }
+.steps-stage { cursor:default; }
+.steps-stage .steps-bar { opacity:.75; }
.k-wait .steps-kind, .steps-kind.k-wait { background:var(--bar-wait); outline:1px dashed var(--faint); }
.steps-track { position:relative; height:12px; background:repeating-linear-gradient(90deg, transparent 0 24.6%, var(--line) 24.6% 25%); }
.steps-bar { position:absolute; top:2px; height:8px; min-width:3px; border-radius:3px; background:var(--bar-ctx); }
diff --git a/tests/test_observability_sandbox_setup.py b/tests/test_observability_sandbox_setup.py
new file mode 100644
index 00000000..6241582b
--- /dev/null
+++ b/tests/test_observability_sandbox_setup.py
@@ -0,0 +1,292 @@
+"""OB-13: the sandbox's start as a `sandbox` setup step with its stages, and restarts as a
+`sandbox_restarted` notice (phase-2 design §12, recording format 1.2).
+
+A fake provider and a fake runner client stand in for a real sandbox: the stages, their
+order and the recording are what is tested, not a sandbox."""
+
+from __future__ import annotations
+
+import asyncio
+import importlib
+import json
+from pathlib import Path
+
+import jsonschema
+
+from coworker.observability import calculator, signals
+from coworker.observability.export_spans import Masker, build_turn_spans
+from coworker.providers import AssistantTurn
+from coworker.sandbox import stages
+from coworker.sandbox.workspace import RunnerWorkspace
+
+base = importlib.import_module("test_observability_stamps_context")
+
+SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))
+
+
+def _check(recording):
+ validator = jsonschema.validators.validator_for(SCHEMA)(SCHEMA)
+ errors = sorted(validator.iter_errors(recording), key=lambda e: list(e.absolute_path))
+ assert not errors, "\n".join(f"{list(e.absolute_path)}: {e.message[:300]}" for e in errors[:8])
+
+
+class ImageMissing(RuntimeError):
+ pass
+
+
+class _Provider:
+ """A provider that reports its own inner stages, like OpenShell."""
+
+ name = "openshell"
+ reports_stages = True
+ profile = "allowlist"
+ extra_hosts: list = []
+
+ def __init__(self, *, fail_at=None, grants=True, image_known=False, regrant=False):
+ self.fail_at, self.grants, self.image_known = fail_at, grants, image_known
+ self.roots = [{"path": "/a", "writable": True}]
+ self.destroyed = 0
+ if regrant:
+ self.regrant = self._regrant
+
+ def create(self):
+ if self.grants:
+ stages.stage(self, "copy_logins")
+ stages.stage(self, "create")
+ if not self.image_known:
+ stages.mark(self, "download_image", uncertain=True)
+ if self.fail_at == "create":
+ raise ImageMissing("The sandbox base image is not downloaded yet")
+ stages.stage(self, "ready")
+
+ def verify(self, client):
+ if self.fail_at == "verify":
+ raise RuntimeError("the home folder was listable from inside")
+
+ def destroy(self):
+ self.destroyed += 1
+
+ def describe(self):
+ return {"provider": self.name, "enforcement": "full", "sandbox": "sbx-1"}
+
+ def _regrant(self, roots, *, before_create=None):
+ self.roots = list(roots)
+ self.create() # a restart makes a new sandbox: not part of the first start's stages
+
+
+class _Client:
+ def __init__(self):
+ self.restarts = 0
+ self.starter = None
+ self.hello = {}
+
+ def connect(self):
+ return {"instance_id": "i1"}
+
+ def detach(self):
+ pass
+
+
+class _Root:
+ def __init__(self, path, writable=True):
+ self.path, self.writable = path, writable
+
+
+def _workspace(provider, roots=None):
+ ws = RunnerWorkspace(provider, cwd=".", start=False, live_roots=roots)
+ ws.client = _Client()
+ return ws
+
+
+# -- the stage log ---------------------------------------------------------------------------
+
+
+def test_stages_close_in_order_and_a_failure_names_its_stage():
+ log = stages.StageLog()
+ log.begin("preparing")
+ log.begin("create")
+ log.mark("download_image", uncertain=True)
+ log.end(ImageMissing("no image"))
+ names = [(s["name"], s["status"]) for s in log.stages]
+ assert names == [("preparing", "ok"), ("create", "error"), ("download_image", "ok")]
+ assert log.stages[2]["duration_ms"] is None and log.stages[2]["uncertain"] is True
+ assert all(isinstance(s["duration_ms"], int) for s in log.stages[:2])
+ assert log.failed_stage == "create"
+ assert log.error == {"type": "ImageMissing", "text": "no image"}
+ log.begin("nonsense") # not a stage name: ignored
+ assert len(log.stages) == 3
+
+
+def test_the_workspace_records_each_stage_of_its_start():
+ told = []
+ ws = _workspace(_Provider())
+ ws.observer = lambda event, data: told.append((event, data.get("trigger")))
+ ws.ensure_started()
+ assert [s["name"] for s in ws.last_start.stages] == ["preparing", "copy_logins", "create", "download_image", "ready", "connect", "verify"]
+ assert ws.last_start.failed_stage is None
+ assert told == [("setup_started", "first_tool"), ("setup_ended", "first_tool")]
+ assert getattr(ws.provider, "stage_log", None) is None # a later restart is not this start
+
+
+def test_a_provider_without_stages_gets_one_create_stage():
+ provider = _Provider()
+ provider.reports_stages = False
+ ws = _workspace(provider)
+ ws.ensure_started("turn_start")
+ assert [s["name"] for s in ws.last_start.stages][:2] == ["preparing", "create"]
+
+
+# -- in a turn -------------------------------------------------------------------------------
+
+_WS: dict = {}
+
+
+def run_cmd(command: str) -> dict:
+ """Run a command in the sandbox.
+
+ Args:
+ command: the command
+ """
+ # What the executor does before every command: restart when the folders changed.
+ return {"exit_code": 0, "output": "", "sandbox_notice": _WS["ws"].sync_roots()}
+
+
+def needs_runner(path: str) -> dict:
+ """Read something in the sandbox.
+
+ Args:
+ path: what to read
+ """
+ _WS["ws"].ensure_started() # what the runner client's starter does on first use
+ return {"path": path}
+
+
+def _engine_with(tmp_path, provider, turns, roots=None):
+ engine, rec = base._engine(tmp_path, turns)
+ for fn in (run_cmd, needs_runner): # before any turn: the engine offers what is registered
+ engine.registry.register(fn, metadata=base._ReadMeta())
+ ws = _workspace(provider, roots)
+ ws.observer = engine.on_sandbox
+ engine.sandbox_workspace = ws
+ _WS["ws"] = ws
+ return engine, rec, ws
+
+
+def _turns(engine, *steps):
+ """Several turns on one event loop, as in the app; a callable between them changes
+ something mid-session. (A loop per turn leaves the engine's Stop flag on the old loop.)"""
+
+ async def go():
+ for step in steps:
+ if callable(step):
+ step()
+ else:
+ async for _ in engine.run(step, trigger="user"):
+ pass
+
+ asyncio.run(go())
+
+
+async def _no_start():
+ if False: # pragma: no cover - an empty async generator
+ yield None
+
+
+def test_the_first_turn_records_the_setup_step(tmp_path):
+ engine, rec, ws = _engine_with(tmp_path, _Provider(), [AssistantTurn(text="ok"), AssistantTurn(text="again")])
+ _turns(engine, "go")
+ d = rec.as_dict()
+ [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]
+ assert setup["name"] == "setup" and setup["status"] == "ok" and setup["parent_step_id"] is None
+ a = setup["attrs"]
+ assert a["provider"] == "openshell" and a["trigger"] == "turn_start" and a["failed_stage"] is None
+ assert [s["name"] for s in a["stages"]] == ["preparing", "copy_logins", "create", "download_image", "ready", "connect", "verify"]
+ assert "error" not in a
+ _check(d)
+ turn = d["turns"][0]
+ assert "sandbox" in calculator.buckets(d, turn) # its own total (mock ob13)
+ assert signals.check_turn(turn, d["steps"])["warn"] == 0
+ _turns(engine, "again") # the sandbox is running: no setup step
+ assert len([s for s in rec.as_dict()["steps"] if s["kind"] == "sandbox"]) == 1
+
+
+def test_a_failed_start_names_its_stage_and_is_an_issue(tmp_path):
+ engine, rec, ws = _engine_with(tmp_path, _Provider(fail_at="create"), [AssistantTurn(text="never")])
+ events = base._run(engine)
+ assert any(e.type.name == "ERROR" for e in events)
+ d = rec.as_dict()
+ [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]
+ assert setup["status"] == "error"
+ assert setup["attrs"]["failed_stage"] == "create"
+ assert setup["attrs"]["error"] == {"type": "ImageMissing", "text": "The sandbox base image is not downloaded yet"}
+ assert d["turns"][0]["status"] == "error"
+ _check(d)
+ found = signals.check_turn(d["turns"][0], d["steps"])
+ assert [i["check"] for i in found["issues"]] == ["sandbox_failed"]
+ assert "create" in found["issues"][0]["text"] and "ImageMissing" in found["issues"][0]["text"]
+ assert ws.provider.destroyed == 1
+
+
+def test_a_lazy_start_inside_a_tool_is_recorded_as_first_tool(tmp_path):
+ """On resume nothing starts the sandbox at the turn's start: the first runner call does."""
+ engine, rec, ws = _engine_with(tmp_path, _Provider(grants=False, image_known=True),
+ [base._call("c1", "needs_runner", {"path": "x"}), AssistantTurn(text="ok")])
+ engine._start_sandbox = _no_start
+ base._run(engine)
+ d = rec.as_dict()
+ [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]
+ assert setup["attrs"]["trigger"] == "first_tool"
+ assert [s["name"] for s in setup["attrs"]["stages"]] == ["preparing", "create", "ready", "connect", "verify"]
+ assert setup["parent_step_id"] is None and setup["turn_id"] == d["turns"][0]["turn_id"]
+ [tool] = [s for s in d["steps"] if s["kind"] == "tool"]
+ assert tool["started_at"] <= setup["started_at"] # it started inside the call that needed it
+ _check(d)
+
+
+def test_a_restart_for_changed_folders_is_a_notice_and_the_tool_says_so(tmp_path):
+ roots = [_Root("/a")]
+ engine, rec, ws = _engine_with(tmp_path, _Provider(regrant=True),
+ [AssistantTurn(text="ok"), base._call("c1", "run_cmd", {"command": "ls"}),
+ AssistantTurn(text="done")], roots=roots)
+ _turns(engine, "go", lambda: roots.append(_Root("/b")), "again") # a folder granted mid-session
+ d = rec.as_dict()
+ [notice] = [s for s in d["steps"] if s["kind"] == "notice" and s["name"] == "sandbox_restarted"]
+ assert notice["attrs"] == {"provider": "openshell", "reason": "folders_changed", "how": "restart"}
+ [tool] = [s for s in d["steps"] if s["kind"] == "tool"]
+ assert tool["attrs"]["sandbox"]["notice"] == "restart"
+ assert notice["started_at"] >= tool["started_at"]
+ # The first start's stages did not grow with the restart's create.
+ [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]
+ assert [s["name"] for s in setup["attrs"]["stages"]].count("create") == 1
+ _check(d)
+
+
+def test_a_runner_that_came_back_on_its_own_is_a_notice(tmp_path):
+ engine, rec, ws = _engine_with(tmp_path, _Provider(),
+ [AssistantTurn(text="ok"), base._call("c1", "run_cmd", {"command": "ls"}),
+ AssistantTurn(text="done")])
+ def came_back():
+ ws.client.restarts += 1 # the client saw a new runner instance after a reconnect
+
+ _turns(engine, "go", came_back, "again")
+ d = rec.as_dict()
+ [notice] = [s for s in d["steps"] if s["name"] == "sandbox_restarted"]
+ assert notice["attrs"]["reason"] == "runner_restarted"
+ [tool] = [s for s in d["steps"] if s["kind"] == "tool"]
+ assert tool["attrs"]["sandbox"]["notice"] is None # the agent was told nothing new
+ _check(d)
+
+
+def test_the_setup_span_has_one_event_per_stage(tmp_path):
+ engine, rec, ws = _engine_with(tmp_path, _Provider(fail_at="verify"), [AssistantTurn(text="never")])
+ base._run(engine)
+ d = rec.as_dict()
+ turn = d["turns"][0]
+ steps = [s for s in d["steps"] if s["turn_id"] == turn["turn_id"]]
+ spans = build_turn_spans(turn, steps, sw={}, dialect="otel", get_piece=lambda k: None, masker=Masker())
+ [span] = [s for s in spans if s["name"] == "openworker.sandbox.setup"]
+ assert span["attrs"]["openworker.sandbox.failed_stage"] == "verify"
+ assert span["error"] == "RuntimeError"
+ names = [e["name"] for e in span["events"]]
+ assert names[0] == "openworker.sandbox.stage.preparing" and names[-1] == "openworker.sandbox.stage.verify"
+ assert "listable" not in json.dumps(span) # the error's text stays on this machine