Skip to content
Python
11.7 KB
Raw Blame
"""OB-13: the sandbox's start as a `sandbox` setup step with its stages, and restarts as a
`sandbox_restarted` notice (phase-2 design §12, recording format 1.2).

A fake provider and a fake runner client stand in for a real sandbox: the stages, their
order and the recording are what is tested, not a sandbox."""

from __future__ import annotations

import asyncio
import importlib
import json
from pathlib import Path

import jsonschema

from coworker.observability import calculator, signals
from coworker.observability.export_spans import Masker, build_turn_spans
from coworker.providers import AssistantTurn
from coworker.sandbox import stages
from coworker.sandbox.workspace import RunnerWorkspace

base = importlib.import_module("test_observability_stamps_context")

SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))


def _check(recording):
    validator = jsonschema.validators.validator_for(SCHEMA)(SCHEMA)
    errors = sorted(validator.iter_errors(recording), key=lambda e: list(e.absolute_path))
    assert not errors, "\n".join(f"{list(e.absolute_path)}: {e.message[:300]}" for e in errors[:8])


class ImageMissing(RuntimeError):
    pass


class _Provider:
    """A provider that reports its own inner stages, like OpenShell."""

    name = "openshell"
    reports_stages = True
    profile = "allowlist"
    extra_hosts: list = []

    def __init__(self, *, fail_at=None, grants=True, image_known=False, regrant=False):
        self.fail_at, self.grants, self.image_known = fail_at, grants, image_known
        self.roots = [{"path": "/a", "writable": True}]
        self.destroyed = 0
        if regrant:
            self.regrant = self._regrant

    def create(self):
        if self.grants:
            stages.stage(self, "copy_logins")
        stages.stage(self, "create")
        if not self.image_known:
            stages.mark(self, "download_image", uncertain=True)
        if self.fail_at == "create":
            raise ImageMissing("The sandbox base image is not downloaded yet")
        stages.stage(self, "ready")

    def verify(self, client):
        if self.fail_at == "verify":
            raise RuntimeError("the home folder was listable from inside")

    def destroy(self):
        self.destroyed += 1

    def describe(self):
        return {"provider": self.name, "enforcement": "full", "sandbox": "sbx-1"}

    def _regrant(self, roots, *, before_create=None):
        self.roots = list(roots)
        self.create()  # a restart makes a new sandbox: not part of the first start's stages


class _Client:
    def __init__(self):
        self.restarts = 0
        self.starter = None
        self.hello = {}

    def connect(self):
        return {"instance_id": "i1"}

    def detach(self):
        pass


class _Root:
    def __init__(self, path, writable=True):
        self.path, self.writable = path, writable


def _workspace(provider, roots=None):
    ws = RunnerWorkspace(provider, cwd=".", start=False, live_roots=roots)
    ws.client = _Client()
    return ws


# -- the stage log ---------------------------------------------------------------------------


def test_stages_close_in_order_and_a_failure_names_its_stage():
    log = stages.StageLog()
    log.begin("preparing")
    log.begin("create")
    log.mark("download_image", uncertain=True)
    log.end(ImageMissing("no image"))
    names = [(s["name"], s["status"]) for s in log.stages]
    assert names == [("preparing", "ok"), ("create", "error"), ("download_image", "ok")]
    assert log.stages[2]["duration_ms"] is None and log.stages[2]["uncertain"] is True
    assert all(isinstance(s["duration_ms"], int) for s in log.stages[:2])
    assert log.failed_stage == "create"
    assert log.error == {"type": "ImageMissing", "text": "no image"}
    log.begin("nonsense")  # not a stage name: ignored
    assert len(log.stages) == 3


def test_the_workspace_records_each_stage_of_its_start():
    told = []
    ws = _workspace(_Provider())
    ws.observer = lambda event, data: told.append((event, data.get("trigger")))
    ws.ensure_started()
    assert [s["name"] for s in ws.last_start.stages] == ["preparing", "copy_logins", "create", "download_image", "ready", "connect", "verify"]
    assert ws.last_start.failed_stage is None
    assert told == [("setup_started", "first_tool"), ("setup_ended", "first_tool")]
    assert getattr(ws.provider, "stage_log", None) is None  # a later restart is not this start


def test_a_provider_without_stages_gets_one_create_stage():
    provider = _Provider()
    provider.reports_stages = False
    ws = _workspace(provider)
    ws.ensure_started("turn_start")
    assert [s["name"] for s in ws.last_start.stages][:2] == ["preparing", "create"]


# -- in a turn -------------------------------------------------------------------------------

_WS: dict = {}


def run_cmd(command: str) -> dict:
    """Run a command in the sandbox.

    Args:
        command: the command
    """
    # What the executor does before every command: restart when the folders changed.
    return {"exit_code": 0, "output": "", "sandbox_notice": _WS["ws"].sync_roots()}


def needs_runner(path: str) -> dict:
    """Read something in the sandbox.

    Args:
        path: what to read
    """
    _WS["ws"].ensure_started()  # what the runner client's starter does on first use
    return {"path": path}


def _engine_with(tmp_path, provider, turns, roots=None):
    engine, rec = base._engine(tmp_path, turns)
    for fn in (run_cmd, needs_runner):  # before any turn: the engine offers what is registered
        engine.registry.register(fn, metadata=base._ReadMeta())
    ws = _workspace(provider, roots)
    ws.observer = engine.on_sandbox
    engine.sandbox_workspace = ws
    _WS["ws"] = ws
    return engine, rec, ws


def _turns(engine, *steps):
    """Several turns on one event loop, as in the app; a callable between them changes
    something mid-session. (A loop per turn leaves the engine's Stop flag on the old loop.)"""

    async def go():
        for step in steps:
            if callable(step):
                step()
            else:
                async for _ in engine.run(step, trigger="user"):
                    pass

    asyncio.run(go())


async def _no_start():
    if False:  # pragma: no cover - an empty async generator
        yield None


def test_the_first_turn_records_the_setup_step(tmp_path):
    engine, rec, ws = _engine_with(tmp_path, _Provider(), [AssistantTurn(text="ok"), AssistantTurn(text="again")])
    _turns(engine, "go")
    d = rec.as_dict()
    [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]
    assert setup["name"] == "setup" and setup["status"] == "ok" and setup["parent_step_id"] is None
    a = setup["attrs"]
    assert a["provider"] == "openshell" and a["trigger"] == "turn_start" and a["failed_stage"] is None
    assert [s["name"] for s in a["stages"]] == ["preparing", "copy_logins", "create", "download_image", "ready", "connect", "verify"]
    assert "error" not in a
    _check(d)
    turn = d["turns"][0]
    assert "sandbox" in calculator.buckets(d, turn)  # its own total (mock ob13)
    assert signals.check_turn(turn, d["steps"])["warn"] == 0
    _turns(engine, "again")  # the sandbox is running: no setup step
    assert len([s for s in rec.as_dict()["steps"] if s["kind"] == "sandbox"]) == 1


def test_a_failed_start_names_its_stage_and_is_an_issue(tmp_path):
    engine, rec, ws = _engine_with(tmp_path, _Provider(fail_at="create"), [AssistantTurn(text="never")])
    events = base._run(engine)
    assert any(e.type.name == "ERROR" for e in events)
    d = rec.as_dict()
    [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]
    assert setup["status"] == "error"
    assert setup["attrs"]["failed_stage"] == "create"
    assert setup["attrs"]["error"] == {"type": "ImageMissing", "text": "The sandbox base image is not downloaded yet"}
    assert d["turns"][0]["status"] == "error"
    _check(d)
    found = signals.check_turn(d["turns"][0], d["steps"])
    assert [i["check"] for i in found["issues"]] == ["sandbox_failed"]
    assert "create" in found["issues"][0]["text"] and "ImageMissing" in found["issues"][0]["text"]
    assert ws.provider.destroyed == 1


def test_a_lazy_start_inside_a_tool_is_recorded_as_first_tool(tmp_path):
    """On resume nothing starts the sandbox at the turn's start: the first runner call does."""
    engine, rec, ws = _engine_with(tmp_path, _Provider(grants=False, image_known=True),
                                   [base._call("c1", "needs_runner", {"path": "x"}), AssistantTurn(text="ok")])
    engine._start_sandbox = _no_start
    base._run(engine)
    d = rec.as_dict()
    [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]
    assert setup["attrs"]["trigger"] == "first_tool"
    assert [s["name"] for s in setup["attrs"]["stages"]] == ["preparing", "create", "ready", "connect", "verify"]
    assert setup["parent_step_id"] is None and setup["turn_id"] == d["turns"][0]["turn_id"]
    [tool] = [s for s in d["steps"] if s["kind"] == "tool"]
    assert tool["started_at"] <= setup["started_at"]  # it started inside the call that needed it
    _check(d)


def test_a_restart_for_changed_folders_is_a_notice_and_the_tool_says_so(tmp_path):
    roots = [_Root("/a")]
    engine, rec, ws = _engine_with(tmp_path, _Provider(regrant=True),
                                   [AssistantTurn(text="ok"), base._call("c1", "run_cmd", {"command": "ls"}),
                                    AssistantTurn(text="done")], roots=roots)
    _turns(engine, "go", lambda: roots.append(_Root("/b")), "again")  # a folder granted mid-session
    d = rec.as_dict()
    [notice] = [s for s in d["steps"] if s["kind"] == "notice" and s["name"] == "sandbox_restarted"]
    assert notice["attrs"] == {"provider": "openshell", "reason": "folders_changed", "how": "restart"}
    [tool] = [s for s in d["steps"] if s["kind"] == "tool"]
    assert tool["attrs"]["sandbox"]["notice"] == "restart"
    assert notice["started_at"] >= tool["started_at"]
    # The first start's stages did not grow with the restart's create.
    [setup] = [s for s in d["steps"] if s["kind"] == "sandbox"]
    assert [s["name"] for s in setup["attrs"]["stages"]].count("create") == 1
    _check(d)


def test_a_runner_that_came_back_on_its_own_is_a_notice(tmp_path):
    engine, rec, ws = _engine_with(tmp_path, _Provider(),
                                   [AssistantTurn(text="ok"), base._call("c1", "run_cmd", {"command": "ls"}),
                                    AssistantTurn(text="done")])
    def came_back():
        ws.client.restarts += 1  # the client saw a new runner instance after a reconnect

    _turns(engine, "go", came_back, "again")
    d = rec.as_dict()
    [notice] = [s for s in d["steps"] if s["name"] == "sandbox_restarted"]
    assert notice["attrs"]["reason"] == "runner_restarted"
    [tool] = [s for s in d["steps"] if s["kind"] == "tool"]
    assert tool["attrs"]["sandbox"]["notice"] is None  # the agent was told nothing new
    _check(d)


def test_the_setup_span_has_one_event_per_stage(tmp_path):
    engine, rec, ws = _engine_with(tmp_path, _Provider(fail_at="verify"), [AssistantTurn(text="never")])
    base._run(engine)
    d = rec.as_dict()
    turn = d["turns"][0]
    steps = [s for s in d["steps"] if s["turn_id"] == turn["turn_id"]]
    spans = build_turn_spans(turn, steps, sw={}, dialect="otel", get_piece=lambda k: None, masker=Masker())
    [span] = [s for s in spans if s["name"] == "openworker.sandbox.setup"]
    assert span["attrs"]["openworker.sandbox.failed_stage"] == "verify"
    assert span["error"] == "RuntimeError"
    names = [e["name"] for e in span["events"]]
    assert names[0] == "openworker.sandbox.stage.preparing" and names[-1] == "openworker.sandbox.stage.verify"
    assert "listable" not in json.dumps(span)  # the error's text stays on this machine