Python
Tool runner: JSON-RPC daemon, attach relay, client and the Workspace interface
Devika Vermae70181f776f6
diff --git a/coworker/agent.py b/coworker/agent.pyindex 84012ec0..70b46963 100644--- a/coworker/agent.py+++ b/coworker/agent.py@@ -48,7 +48,7 @@ from .tools.toolreq import request_tool_toolfrom .tools.subagent import explorer_toolsfrom .web import make_web_fetch_tool, make_web_search_toolfrom .workspace_trust import WorkspaceTrustStore-from .tools.shell import LocalExecutor+from .sandbox.workspace import open_workspacefrom .tools.todo import TodoList# Appended each turn while discuss mode is active: enforcement-only read-only, with no@@ -336,7 +336,10 @@ def build_engine(# OPE-176: the reasoning-effort level takes the same route; providers translate it.if config.reasoning_effort and "reasoning_effort" not in (model_settings or {}):model_settings = {**(model_settings or {}), "reasoning_effort": config.reasoning_effort}- executor = LocalExecutor(cwd=ws) if ws is not None else None+ # The session's workspace decides where commands run: in this process (`direct`, the+ # default, today's behaviour) or in a tool runner behind a sandbox provider.+ sandbox_workspace = open_workspace(cwd=ws) if ws is not None else None+ executor = sandbox_workspace.executor if sandbox_workspace is not None else Nonetodo = TodoList()context = AgentContext(workspace=ws, executor=executor, todo=todo, roots=root_list or None@@ -699,6 +702,7 @@ def build_engine(if _compaction_overrides:engine.compaction_settings = lambda: dict(_compaction_overrides)engine.executor = executor # type: ignore[attr-defined]+ engine.sandbox_workspace = sandbox_workspace # type: ignore[attr-defined]engine.todo = todo # type: ignore[attr-defined]engine.agent_name = agent.name # type: ignore[attr-defined]engine.roots = root_list # type: ignore[attr-defined] # shared list; Slice C mutates in placediff --git a/coworker/sandbox/__init__.py b/coworker/sandbox/__init__.pyindex e69de29b..7f5db0b4 100644--- a/coworker/sandbox/__init__.py+++ b/coworker/sandbox/__init__.py@@ -0,0 +1,2 @@+"""Sandboxed tool execution: the Workspace interface, the tool runner and its client.+Design: ocw-context/docs/sandbox-design.md."""diff --git a/coworker/sandbox/bundle.py b/coworker/sandbox/bundle.pynew file mode 100644index 00000000..1151d962--- /dev/null+++ b/coworker/sandbox/bundle.py@@ -0,0 +1,52 @@+"""Pack the tool runner into one file.++The runner is delivered to a sandbox as a read-only mount of a single zipapp, so it always+matches this server's version and no image has to carry it. The file lives in the state+directory (`<state dir>/sandbox/`), which is a real folder on the machine, so a provider can+mount that folder into a sandbox whether the server runs on the host or in a sandbox itself.+"""++from __future__ import annotations++import hashlib+import os+import zipfile+from pathlib import Path+from typing import Optional++from ..secrets import state_dir++_PACKAGE = "owrunner" # the name the runner package has inside the zipapp+_MAIN = f"from {_PACKAGE}.__main__ import main\nraise SystemExit(main())\n"+++def _sources() -> list[Path]:+ root = Path(__file__).parent / "runner"+ return sorted(p for p in root.glob("*.py"))+++def runner_dir() -> Path:+ return state_dir() / "sandbox"+++def build_runner_zipapp(dest_dir: Optional[Path] = None) -> Path:+ """Build (or reuse) the runner zipapp and return its path. The name carries a hash of+ the sources, so an upgraded or edited runner never reuses a stale file."""+ sources = _sources()+ digest = hashlib.sha256()+ for src in sources:+ digest.update(src.name.encode())+ digest.update(src.read_bytes())+ dest_dir = Path(dest_dir) if dest_dir is not None else runner_dir()+ dest_dir.mkdir(parents=True, exist_ok=True)+ target = dest_dir / f"runner-{digest.hexdigest()[:12]}.pyz"+ if target.exists():+ return target+ tmp = target.with_suffix(f".tmp{os.getpid()}")+ with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED) as zf:+ zf.writestr("__main__.py", _MAIN)+ for src in sources:+ zf.write(src, f"{_PACKAGE}/{src.name}")+ os.chmod(tmp, 0o644)+ os.replace(tmp, target)+ return targetdiff --git a/coworker/sandbox/client.py b/coworker/sandbox/client.pynew file mode 100644index 00000000..019a5b43--- /dev/null+++ b/coworker/sandbox/client.py@@ -0,0 +1,220 @@+"""The workspace client: JSON-RPC to a tool runner over a pipe.++It does not know what is on the other end of the pipe (a local process, an OpenShell+sandbox, later another machine). It gets a function that opens a fresh pipe, and it uses+that function again whenever the pipe breaks.++After a break (design doc, section 4c):+- same runner instance: send `session.resume`; the runner replays the results we missed,+ and requests still waiting are sent again under the SAME id, which the runner runs once;+- a new runner instance (the sandbox restarted): every waiting request fails with+ RUNNER_RESTARTED. A command is never sent again by the client on its own, because+ commands are not safe to repeat.+"""++from __future__ import annotations++import logging+import threading+import time+import uuid+from typing import Any, Callable, Optional++from .runner import protocol as P+from .transport import Transport++log = logging.getLogger(__name__)++NotifyHandler = Callable[[str, dict[str, Any]], None]+_ACK_EVERY = 20+_RECONNECT_TRIES = 5+++class _Pending:+ def __init__(self, frame: bytes, on_notify: Optional[NotifyHandler]) -> None:+ self.frame = frame+ self.on_notify = on_notify+ self.done = threading.Event()+ self.result: Optional[dict[str, Any]] = None+ self.error: Optional[P.RunnerError] = None+++class RunnerClient:+ def __init__(self, open_transport: Callable[[], Transport], *, ping_seconds: float = P.PING_SECONDS) -> None:+ self._open = open_transport+ self._ping_seconds = ping_seconds+ self._transport: Optional[Transport] = None+ self._send_lock = threading.Lock()+ self._state_lock = threading.Lock()+ self._connect_lock = threading.Lock() # one (re)connect at a time+ self._pending: dict[str, _Pending] = {}+ self._last_seq = 0+ self._acked = 0+ self._generation = 0+ self._closed = False+ self.instance_id: Optional[str] = None+ self.hello: dict[str, Any] = {}+ self.restarts = 0 # how many times the runner came back as a new instance++ # -- connection -------------------------------------------------------------------+ def connect(self) -> dict[str, Any]:+ with self._connect_lock:+ return self._connect_locked()++ def _connect_locked(self) -> dict[str, Any]:+ transport = self._open()+ with self._state_lock:+ self._generation += 1+ generation = self._generation+ old, self._transport = self._transport, transport+ if old is not None:+ old.close()+ threading.Thread(target=self._read_loop, args=(transport, generation), daemon=True).start()+ hello = self._call_on(transport, "runner.hello", {"protocol_version": P.PROTOCOL_VERSION}, timeout=20)+ previous, self.instance_id, self.hello = self.instance_id, hello["instance_id"], hello+ if previous is None:+ threading.Thread(target=self._ping_loop, daemon=True).start()+ elif previous != self.instance_id:+ self.restarts += 1+ self._last_seq = self._acked = 0+ self._fail_all(P.RunnerError(P.RUNNER_RESTARTED, "the sandbox restarted; the outcome of this command is unknown"))+ else:+ self._call_on(transport, "session.resume", {"last_seq": self._last_seq}, timeout=20)+ with self._state_lock:+ waiting = [p.frame for p in self._pending.values() if not p.done.is_set()]+ for frame in waiting: # same ids: the runner answers, it does not run them twice+ self._send(transport, frame)+ return hello++ def _reconnect(self, generation: int) -> None:+ with self._connect_lock:+ if self._closed or generation != self._generation:+ return # someone already reconnected, or we are shutting down+ for attempt in range(_RECONNECT_TRIES):+ try:+ self._connect_locked()+ return+ except Exception as exc:+ log.warning("runner reconnect %d/%d failed: %s", attempt + 1, _RECONNECT_TRIES, exc)+ time.sleep(min(0.25 * 2**attempt, 3.0))+ self._fail_all(P.RunnerError(P.INTERNAL_ERROR, "lost the connection to the tool runner"))++ def close(self) -> None:+ self._closed = True+ with self._state_lock:+ transport, self._transport = self._transport, None+ if transport is not None:+ transport.close()+ self._fail_all(P.RunnerError(P.INTERNAL_ERROR, "the workspace client was closed"))++ # -- calls ------------------------------------------------------------------------+ def call(+ self,+ method: str,+ params: Optional[dict[str, Any]] = None,+ *,+ timeout: Optional[float] = None,+ on_notify: Optional[NotifyHandler] = None,+ ) -> dict[str, Any]:+ with self._state_lock:+ transport = self._transport+ if transport is None:+ raise P.RunnerError(P.INTERNAL_ERROR, "not connected to a tool runner")+ return self._call_on(transport, method, params or {}, timeout=timeout, on_notify=on_notify)++ def _call_on(+ self,+ transport: Transport,+ method: str,+ params: dict[str, Any],+ *,+ timeout: Optional[float],+ on_notify: Optional[NotifyHandler] = None,+ ) -> dict[str, Any]:+ req_id = f"r-{uuid.uuid4().hex[:12]}"+ pending = _Pending(P.encode(P.request(req_id, method, params)), on_notify)+ with self._state_lock:+ self._pending[req_id] = pending+ try:+ self._send(transport, pending.frame)+ if not pending.done.wait(timeout):+ raise P.RunnerError(P.INTERNAL_ERROR, f"no answer from the tool runner to {method}")+ finally:+ with self._state_lock:+ self._pending.pop(req_id, None)+ if pending.error is not None:+ raise pending.error+ assert pending.result is not None+ return pending.result++ def _send(self, transport: Transport, frame: bytes) -> None:+ try:+ with self._send_lock:+ transport.send(frame)+ except (OSError, ValueError):+ pass # the read loop sees the break and reconnects; the request is sent again then++ def _fail_all(self, error: P.RunnerError) -> None:+ with self._state_lock:+ waiting = list(self._pending.values())+ for pending in waiting:+ if not pending.done.is_set():+ pending.error = error+ pending.done.set()++ # -- incoming ---------------------------------------------------------------------+ def _read_loop(self, transport: Transport, generation: int) -> None:+ while True:+ try:+ line = transport.recv_line()+ except (OSError, ValueError):+ line = None+ if line is None:+ break+ frame = P.decode(line)+ if frame is None:+ log.debug("dropped a line that is not a frame: %r", line[:120])+ continue+ self._handle(frame)+ if not self._closed and generation == self._generation:+ threading.Thread(target=self._reconnect, args=(generation,), daemon=True).start()++ def _handle(self, frame: dict[str, Any]) -> None:+ if "method" in frame: # a notification: the live view+ params = frame.get("params") if isinstance(frame.get("params"), dict) else {}+ with self._state_lock:+ pending = self._pending.get(str(params.get("id")))+ if pending is not None and pending.on_notify is not None:+ try:+ pending.on_notify(frame["method"], params)+ except Exception:+ log.exception("live-output handler failed")+ return+ body = frame.get("result") if "result" in frame else (frame.get("error") or {}).get("data") or {}+ seq = body.get("seq") if isinstance(body, dict) else None+ if isinstance(seq, int):+ self._last_seq = max(self._last_seq, seq)+ if self._last_seq - self._acked >= _ACK_EVERY:+ self._acked = self._last_seq+ with self._state_lock:+ transport = self._transport+ if transport is not None:+ self._send(transport, P.encode(P.notification("session.ack", {"seq": self._acked})))+ with self._state_lock:+ pending = self._pending.get(str(frame.get("id")))+ if pending is None or pending.done.is_set():+ return # a replayed result nobody waits for any more+ if "error" in frame:+ err = frame["error"] or {}+ pending.error = P.RunnerError(int(err.get("code", P.INTERNAL_ERROR)), str(err.get("message", "")), err.get("data"))+ else:+ pending.result = frame.get("result") or {}+ pending.done.set()++ def _ping_loop(self) -> None:+ while not self._closed:+ time.sleep(self._ping_seconds)+ with self._state_lock:+ transport = self._transport+ if transport is not None and not self._closed:+ self._send(transport, P.encode(P.notification("$/ping")))diff --git a/coworker/sandbox/executor.py b/coworker/sandbox/executor.pynew file mode 100644index 00000000..20bdb0ca--- /dev/null+++ b/coworker/sandbox/executor.py@@ -0,0 +1,101 @@+"""`RunnerExecutor`: the existing `Executor` contract, served by a tool runner.++The shell tool does not change. It gets an executor; whether that executor drives a bash+in this process (`LocalExecutor`) or one inside a sandbox (this class) is decided by the+workspace. Results have exactly the keys `LocalExecutor` returns, so a session looks the+same to the model in both modes.+"""++from __future__ import annotations++import threading+from typing import Any, Callable, Optional++from .client import RunnerClient+from .runner import protocol as P+from .runner.executor import Executor++_RESULT_KEYS = ("command", "cwd", "exit_code", "output", "timed_out", "truncated", "error")+_DEFAULT_TIMEOUT = 120.0+_GRACE_SECONDS = 20.0 # the runner enforces the command timeout; this only covers the pipe+++class RunnerExecutor(Executor):+ def __init__(+ self,+ client: RunnerClient,+ *,+ cwd: str,+ shell: str = "main",+ default_timeout: float = _DEFAULT_TIMEOUT,+ on_output: Optional[Callable[[str], None]] = None,+ ) -> None:+ self._client = client+ self.shell = shell+ self.cwd = str(cwd) # the last folder the runner reported; a reopened shell starts here+ self.default_timeout = default_timeout+ self._on_output = on_output++ def _failed(self, command: str, message: str) -> dict[str, Any]:+ return {+ "command": command,+ "cwd": self.cwd,+ "exit_code": None,+ "output": "",+ "timed_out": False,+ "truncated": False,+ "error": message,+ }++ def run(self, command: str, timeout: Optional[float] = None) -> dict[str, Any]:+ timeout = timeout or self.default_timeout++ def live(method: str, params: dict[str, Any]) -> None:+ if method == "shell.output" and self._on_output is not None:+ self._on_output(str(params.get("text", "")))++ try:+ result = self._client.call(+ "shell.run",+ {"shell": self.shell, "command": command, "timeout_s": timeout, "cwd": self.cwd},+ timeout=timeout + _GRACE_SECONDS,+ on_notify=live,+ )+ except P.RunnerError as exc:+ return self._failed(command, exc.message)+ if isinstance(result.get("cwd"), str) and result["cwd"]:+ self.cwd = result["cwd"]+ return {key: result[key] for key in _RESULT_KEYS if key in result}++ def run_background(self, command: str) -> dict[str, Any]:+ return self._simple("task.start", {"command": command, "cwd": self.cwd})++ def background_output(self, task_id: str) -> dict[str, Any]:+ return self._simple("task.output", {"task_id": task_id})++ def background_kill(self, task_id: str) -> dict[str, Any]:+ return self._simple("task.kill", {"task_id": task_id})++ def _simple(self, method: str, params: dict[str, Any]) -> dict[str, Any]:+ try:+ result = self._client.call(method, {"shell": self.shell, **params}, timeout=30)+ except P.RunnerError as exc:+ return {"error": exc.message}+ result.pop("seq", None)+ return result++ def interrupt_now(self) -> None:+ """User Stop. Must not block the caller, so the request goes out on its own thread."""+ threading.Thread(target=self.interrupt, daemon=True).start()++ def interrupt(self) -> None:+ try:+ self._client.call("shell.interrupt", {"shell": self.shell}, timeout=10)+ except P.RunnerError:+ pass++ def close(self) -> None:+ try:+ self._client.call("shell.close", {"shell": self.shell}, timeout=10)+ except P.RunnerError:+ passdiff --git a/coworker/sandbox/providers/__init__.py b/coworker/sandbox/providers/__init__.pynew file mode 100644index 00000000..cecf2835--- /dev/null+++ b/coworker/sandbox/providers/__init__.py@@ -0,0 +1,2 @@+"""Sandbox providers. A provider creates a confined place, starts the tool runner in it and+hands back a pipe to it. One provider per sandbox technology."""diff --git a/coworker/sandbox/providers/runner_local.py b/coworker/sandbox/providers/runner_local.pynew file mode 100644index 00000000..fa828b33--- /dev/null+++ b/coworker/sandbox/providers/runner_local.py@@ -0,0 +1,82 @@+"""`runner-local`: the tool runner as a plain local process, with NO confinement.++For developers and tests only. It exists to exercise the protocol end to end and to+measure what the runner path costs on its own. It has the same shape as a real provider:+a daemon that outlives any one connection, and an attach relay per pipe.+"""++from __future__ import annotations++import os+import shutil+import subprocess+import sys+import tempfile+import time+from pathlib import Path+from typing import Any, Optional++from ..bundle import build_runner_zipapp+from ..transport import PipeTransport, Transport+++class RunnerLocalProvider:+ name = "runner-local"++ def __init__(self, *, cwd: str | Path, runner_path: Optional[Path] = None, relay_silence_seconds: Optional[float] = None) -> None:+ self.cwd = str(Path(cwd).expanduser().resolve())+ self._runner = Path(runner_path) if runner_path is not None else build_runner_zipapp()+ self._relay_silence = relay_silence_seconds+ # A Unix socket path is limited to about 100 bytes, and temp folders on macOS are+ # long, so the socket gets a short folder of its own.+ self._dir = tempfile.mkdtemp(prefix="owr-", dir="/tmp" if os.path.isdir("/tmp") else None)+ self.socket_path = os.path.join(self._dir, "r.sock")+ self._daemon: Optional[subprocess.Popen] = None++ def describe(self) -> dict[str, Any]:+ return {"provider": self.name, "enforcement": "none", "reason": "a local process without any sandbox (developer mode)"}++ def create(self) -> None:+ self._daemon = subprocess.Popen(+ [sys.executable, str(self._runner), "serve", "--socket", self.socket_path, "--cwd", self.cwd, "--exit-with-parent"],+ stdin=subprocess.DEVNULL,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ start_new_session=True,+ )+ deadline = time.monotonic() + 15+ while not os.path.exists(self.socket_path):+ if self._daemon.poll() is not None:+ raise RuntimeError(f"the tool runner exited at once (code {self._daemon.returncode})")+ if time.monotonic() > deadline:+ raise RuntimeError("the tool runner did not come up in 15 seconds")+ time.sleep(0.02)++ def open_runner(self) -> Transport:+ argv = [sys.executable, str(self._runner), "attach", "--socket", self.socket_path]+ if self._relay_silence is not None:+ argv += ["--silence-seconds", str(self._relay_silence)]+ return PipeTransport(+ subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, bufsize=0)+ )++ def restart_daemon(self) -> None:+ """Tests only: what a sandbox restart looks like from the client's side."""+ self._stop_daemon()+ self.create()++ def _stop_daemon(self) -> None:+ if self._daemon is not None and self._daemon.poll() is None:+ self._daemon.terminate()+ try:+ self._daemon.wait(timeout=5)+ except subprocess.TimeoutExpired:+ self._daemon.kill()+ try:+ os.unlink(self.socket_path)+ except OSError:+ pass++ def destroy(self) -> None:+ self._stop_daemon()+ shutil.rmtree(self._dir, ignore_errors=True)diff --git a/coworker/sandbox/runner/__init__.py b/coworker/sandbox/runner/__init__.pyindex e69de29b..f22d3e4e 100644--- a/coworker/sandbox/runner/__init__.py+++ b/coworker/sandbox/runner/__init__.py@@ -0,0 +1,3 @@+"""The OpenWorker tool runner. Everything in this package uses the standard library only+and imports nothing outside the package: it is packed into one file and run inside a+sandbox, in whatever image the sandbox uses."""diff --git a/coworker/sandbox/runner/__main__.py b/coworker/sandbox/runner/__main__.pynew file mode 100644index 00000000..e25ce7ad--- /dev/null+++ b/coworker/sandbox/runner/__main__.py@@ -0,0 +1,37 @@+"""Entry point of the tool runner: `serve` (the daemon) and `attach` (the relay).++Runs as `python -m coworker.sandbox.runner ...` from a checkout, or as+`python runner.pyz ...` from the packed single file that is mounted into a sandbox.+"""++from __future__ import annotations++import argparse+import sys++from .daemon import RUNNER_VERSION, Daemon+from .relay import run as run_relay+++def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser(prog="openworker tool-runner", description="OpenWorker tool runner")+ parser.add_argument("--version", action="version", version=RUNNER_VERSION)+ sub = parser.add_subparsers(dest="command", required=True)+ serve = sub.add_parser("serve", help="run the daemon (the sandbox's main process)")+ serve.add_argument("--socket", required=True, help="path of the Unix socket file to listen on")+ serve.add_argument("--cwd", default=None, help="folder new shells start in (default: current folder)")+ serve.add_argument("--exit-with-parent", action="store_true", help="stop when the starting process is gone (local use)")+ attach = sub.add_parser("attach", help="connect this process's stdin/stdout to the daemon")+ attach.add_argument("--socket", required=True)+ attach.add_argument("--silence-seconds", type=float, default=None, help="leave after this much client silence (0 = never)")+ args = parser.parse_args(argv)+ if args.command == "serve":+ Daemon(args.socket, args.cwd, args.exit_with_parent).serve_forever()+ return 0+ if args.silence_seconds is None:+ return run_relay(args.socket)+ return run_relay(args.socket, args.silence_seconds)+++if __name__ == "__main__":+ sys.exit(main())diff --git a/coworker/sandbox/runner/daemon.py b/coworker/sandbox/runner/daemon.pynew file mode 100644index 00000000..e124b1cd--- /dev/null+++ b/coworker/sandbox/runner/daemon.py@@ -0,0 +1,463 @@+"""The runner daemon: the long-lived process inside a sandbox. Standard library only.++It owns the persistent shells (one per agent that uses this sandbox), their background+tasks and the file operations, and serves them over a Unix socket FILE (not a network+port). A client never talks to it directly: each connection comes through a small attach+relay that the sandbox provider starts, and that relay dies with its stream while this+process and its shells live on.++Trust: this process is on the untrusted side. The agent's commands run as the same user+and can tamper with it. It holds no keys and decides nothing; the server treats what it+returns as data.++Delivery rules (design doc, section 4c):+- The RESULT of a request is the record. Results are kept until acknowledged and replayed+ after a reconnect (`session.resume`).+- `shell.output` / `shell.progress` notifications are a live view and may be dropped.+- The newest connection wins; an older one is closed.+- A request id seen twice runs once.+"""++from __future__ import annotations++import collections+import os+import platform+import socket+import subprocess+import sys+import threading+import time+import uuid+from pathlib import Path+from typing import Any, Callable, Optional++from . import fsops, protocol as P+from .executor import LocalExecutor++RUNNER_VERSION = "0.1.0"+_RESULT_LOG_MAX = 5000+_REQUEST_MEMORY_MAX = 5000+++class _Conn:+ """One attached client (through a relay). Writes are serialized."""++ def __init__(self, sock: socket.socket) -> None:+ self.sock = sock+ self._lock = threading.Lock()+ self.alive = True++ def send(self, frame: bytes) -> bool:+ if not self.alive:+ return False+ try:+ with self._lock:+ self.sock.sendall(frame)+ return True+ except OSError:+ self.alive = False+ return False++ def close(self) -> None:+ self.alive = False+ try:+ self.sock.shutdown(socket.SHUT_RDWR)+ except OSError:+ pass+ try:+ self.sock.close()+ except OSError:+ pass+++class _Shell:+ def __init__(self, name: str, executor: LocalExecutor) -> None:+ self.name = name+ self.executor = executor+ self.busy = threading.Lock()+++class Daemon:+ def __init__(self, socket_path: str, default_cwd: Optional[str] = None, exit_with_parent: bool = False) -> None:+ self.socket_path = socket_path+ # Outside a sandbox nothing else ends this process when its server goes away.+ self._parent = os.getppid() if exit_with_parent else None+ self.default_cwd = str(Path(default_cwd or os.getcwd()).expanduser().resolve())+ self.instance_id = uuid.uuid4().hex+ self._shells: dict[str, _Shell] = {}+ self._shells_lock = threading.Lock()+ self._seq = 0+ self._out_lock = threading.Lock() # guards seq, the result log and the active conn+ self._results: "collections.deque[tuple[int, bytes]]" = collections.deque(maxlen=_RESULT_LOG_MAX)+ self._gap = False # a result fell off the log before it was acknowledged+ self._requests: "collections.OrderedDict[str, Optional[bytes]]" = collections.OrderedDict()+ self._conn: Optional[_Conn] = None+ self._stop = threading.Event()++ # -- outgoing ---------------------------------------------------------------------+ def _emit(self, frame: dict[str, Any], *, record: bool) -> None:+ """Stamp a sequence number, remember results, send to whoever is attached."""+ with self._out_lock:+ self._seq += 1+ seq = self._seq+ if "result" in frame:+ frame["result"]["seq"] = seq+ elif "error" in frame:+ frame["error"].setdefault("data", {})["seq"] = seq+ else:+ frame.setdefault("params", {})["seq"] = seq+ data = P.encode(frame)+ if record:+ if len(self._results) == self._results.maxlen:+ self._gap = True+ self._results.append((seq, data))+ req_id = frame.get("id")+ if isinstance(req_id, str) and req_id in self._requests:+ self._requests[req_id] = data+ conn = self._conn+ if conn is not None:+ conn.send(data)++ def _reply(self, req_id: str, value: dict[str, Any]) -> None:+ self._emit(P.result(req_id, value), record=True)++ def _fail(self, req_id: Optional[str], code: int, message: str) -> None:+ self._emit(P.error(req_id, code, message), record=req_id is not None)++ def _notify(self, method: str, params: dict[str, Any]) -> None:+ self._emit(P.notification(method, params), record=False)++ # -- shells -----------------------------------------------------------------------+ def _shell(self, name: str, *, cwd: Optional[str] = None, create: bool = True) -> Optional[_Shell]:+ with self._shells_lock:+ shell = self._shells.get(name)+ if shell is None and create:+ start = cwd or self.default_cwd+ if not os.path.isdir(start):+ start = self.default_cwd+ shell = _Shell(name, LocalExecutor(cwd=start))+ self._shells[name] = shell+ return shell++ def _resolve(self, params: dict[str, Any], key: str = "path") -> str:+ raw = params.get(key)+ if not isinstance(raw, str) or not raw:+ raise ValueError(f"'{key}' is required")+ path = Path(raw).expanduser()+ if not path.is_absolute():+ shell = self._shell(str(params.get("shell") or ""), create=False) if params.get("shell") else None+ base = shell.executor.cwd if shell is not None else self.default_cwd+ path = Path(base) / path+ return str(path)++ # -- methods ----------------------------------------------------------------------+ def _hello(self, params: dict[str, Any]) -> dict[str, Any]:+ theirs = params.get("protocol_version")+ if theirs is not None and theirs != P.PROTOCOL_VERSION:+ raise P.RunnerError(P.VERSION_MISMATCH, f"runner speaks protocol {P.PROTOCOL_VERSION}, client sent {theirs}")+ return {+ "protocol_version": P.PROTOCOL_VERSION,+ "runner_version": RUNNER_VERSION,+ "instance_id": self.instance_id,+ "pid": os.getpid(),+ "os": platform.system(),+ "os_release": platform.release(),+ "machine": platform.machine(),+ "python": platform.python_version(),+ "shell": "powershell.exe" if sys.platform == "win32" else "/bin/bash",+ "cwd": self.default_cwd,+ "shells": sorted(self._shells),+ }++ def _resume(self, params: dict[str, Any]) -> dict[str, Any]:+ last = int(params.get("last_seq") or 0)+ with self._out_lock:+ pending = [data for seq, data in self._results if seq > last]+ gap, self._gap = self._gap, False+ conn = self._conn+ for data in pending:+ if conn is not None:+ conn.send(data)+ return {"instance_id": self.instance_id, "replayed": len(pending), "gap": gap}++ def _ack(self, params: dict[str, Any]) -> None:+ upto = int(params.get("seq") or 0)+ with self._out_lock:+ while self._results and self._results[0][0] <= upto:+ self._results.popleft()++ def _shell_run(self, req_id: str, params: dict[str, Any]) -> dict[str, Any]:+ command = params.get("command")+ if not isinstance(command, str):+ raise ValueError("'command' is required")+ name = str(params.get("shell") or "main")+ shell = self._shell(name, cwd=params.get("cwd"))+ assert shell is not None+ if not shell.busy.acquire(blocking=False):+ raise P.RunnerError(P.SHELL_BUSY, f"shell '{name}' is running another command")+ started = time.monotonic()+ live = _LiveView(lambda method, extra: self._notify(method, {"id": req_id, "shell": name, **extra}))+ try:+ timeout = params.get("timeout_s")+ result = shell.executor.run(+ command,+ timeout=float(timeout) if isinstance(timeout, (int, float)) and timeout > 0 else None,+ on_output=live.feed,+ )+ finally:+ live.stop()+ shell.busy.release()+ result["duration_ms"] = int((time.monotonic() - started) * 1000)+ result["output_bytes"] = live.total_bytes+ return result++ def _proc_run(self, params: dict[str, Any]) -> dict[str, Any]:+ argv = params.get("argv")+ if not isinstance(argv, list) or not argv or not all(isinstance(a, str) for a in argv):+ raise ValueError("'argv' must be a non-empty list of strings")+ cwd = self._resolve(params, "cwd") if params.get("cwd") else self.default_cwd+ env = {**os.environ, **{str(k): str(v) for k, v in (params.get("env") or {}).items()}}+ timeout = params.get("timeout_s")+ try:+ done = subprocess.run(+ argv,+ cwd=cwd,+ env=env,+ input=params.get("stdin"),+ capture_output=True,+ text=True,+ errors="replace",+ timeout=float(timeout) if isinstance(timeout, (int, float)) and timeout > 0 else None,+ )+ except subprocess.TimeoutExpired as exc:+ return {"exit_code": None, "stdout": exc.stdout or "", "stderr": exc.stderr or "", "timed_out": True}+ except OSError as exc:+ return {"exit_code": None, "stdout": "", "stderr": "", "timed_out": False, "error": f"{type(exc).__name__}: {exc}"}+ return {"exit_code": done.returncode, "stdout": done.stdout, "stderr": done.stderr, "timed_out": False}++ def _dispatch(self, req_id: str, method: str, params: dict[str, Any]) -> dict[str, Any]:+ if method == "runner.hello":+ return self._hello(params)+ if method == "session.resume":+ return self._resume(params)+ if method == "shell.open":+ shell = self._shell(str(params.get("shell") or "main"), cwd=params.get("cwd"))+ assert shell is not None+ return {"shell": shell.name, "cwd": shell.executor.cwd}+ if method == "shell.run":+ return self._shell_run(req_id, params)+ if method in ("shell.interrupt", "shell.close", "task.start", "task.output", "task.kill", "task.list"):+ name = str(params.get("shell") or "main")+ # A task call opens the shell if needed, so an unknown task id is reported as+ # that (as the in-process executor does), not as a missing shell.+ shell = self._shell(name, cwd=params.get("cwd"), create=method.startswith("task."))+ if shell is None:+ return {"shell": name, "closed": True} if method == "shell.close" else {"shell": name}+ ex = shell.executor+ if method == "shell.interrupt":+ ex.interrupt_now()+ return {"shell": name}+ if method == "shell.close":+ with self._shells_lock:+ self._shells.pop(name, None)+ ex.close()+ return {"shell": name, "closed": True}+ if method == "task.start":+ return ex.run_background(str(params.get("command") or ""))+ if method == "task.output":+ return ex.background_output(str(params.get("task_id") or ""))+ if method == "task.kill":+ return ex.background_kill(str(params.get("task_id") or ""))+ return {+ "tasks": [+ {"task_id": tid, "command": t.command, "status": "running" if t.proc.poll() is None else "exited"}+ for tid, t in ex._bg_tasks.items()+ ]+ }+ if method == "proc.run":+ return self._proc_run(params)+ if method.startswith("fs."):+ path = self._resolve(params)+ try:+ if method == "fs.read":+ return fsops.read(path, **_pick(params, "offset_line", "limit_lines", "offset_bytes", "limit_bytes"))+ if method == "fs.write":+ return fsops.write(path, **_pick(params, "text", "data_b64", "if_match_sha256", "append", "make_parents"))+ if method == "fs.stat":+ return fsops.stat(path)+ if method == "fs.list":+ return fsops.list_dir(path, **_pick(params, "limit"))+ if method == "fs.mkdir":+ return fsops.mkdir(path, **_pick(params, "parents"))+ if method == "fs.remove":+ return fsops.remove(path, **_pick(params, "recursive"))+ if method == "fs.search":+ return fsops.search(path, str(params.get("pattern") or ""), **_pick(params, "max_matches", "ignore_case"))+ except fsops.FsError as exc:+ raise P.RunnerError(P.FS_ERROR, str(exc)) from exc+ if method == "runner.shutdown":+ self._stop.set()+ return {"stopping": True}+ raise P.RunnerError(P.METHOD_NOT_FOUND, f"unknown method: {method}")++ def _run_request(self, req_id: str, method: str, params: dict[str, Any]) -> None:+ try:+ self._reply(req_id, self._dispatch(req_id, method, params))+ except P.RunnerError as exc:+ self._fail(req_id, exc.code, exc.message)+ except (ValueError, TypeError) as exc:+ self._fail(req_id, P.INVALID_PARAMS, str(exc))+ except Exception as exc: # never let one request take the daemon down+ self._fail(req_id, P.INTERNAL_ERROR, f"{type(exc).__name__}: {exc}")+ if method == "runner.shutdown":+ self._shutdown()++ # -- connections ------------------------------------------------------------------+ def _serve_conn(self, conn: _Conn) -> None:+ with self._out_lock:+ old, self._conn = self._conn, conn+ if old is not None:+ old.close() # the newest connection wins+ reader = conn.sock.makefile("rb")+ try:+ for line in reader:+ frame = P.decode(line)+ if frame is None:+ continue+ method = frame.get("method")+ if not isinstance(method, str):+ continue+ params = frame.get("params") if isinstance(frame.get("params"), dict) else {}+ req_id = frame.get("id")+ if req_id is None: # a notification from the client+ if method == "session.ack":+ self._ack(params)+ continue # "$/ping" and anything unknown: nothing to do+ req_id = str(req_id)+ with self._out_lock:+ seen = req_id in self._requests+ stored = self._requests.get(req_id)+ if not seen:+ self._requests[req_id] = None+ while len(self._requests) > _REQUEST_MEMORY_MAX:+ self._requests.popitem(last=False)+ if seen:+ if stored is not None:+ conn.send(stored) # already done: same answer, not a second run+ continue+ if method in ("runner.hello", "session.resume"):+ self._run_request(req_id, method, params) # inline: keeps their order+ else:+ threading.Thread(target=self._run_request, args=(req_id, method, params), daemon=True).start()+ except OSError:+ pass+ finally:+ conn.close()+ with self._out_lock:+ if self._conn is conn:+ self._conn = None++ def _shutdown(self) -> None:+ with self._shells_lock:+ shells, self._shells = list(self._shells.values()), {}+ for shell in shells:+ for task_id in list(shell.executor._bg_tasks):+ shell.executor.background_kill(task_id)+ shell.executor.close()+ try:+ os.unlink(self.socket_path)+ except OSError:+ pass++ def serve_forever(self) -> None:+ path = self.socket_path+ Path(path).parent.mkdir(parents=True, exist_ok=True)+ try:+ os.unlink(path)+ except FileNotFoundError:+ pass+ server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)+ old_mask = os.umask(0o077) # the socket file is for this user only+ try:+ server.bind(path)+ finally:+ os.umask(old_mask)+ server.listen(16)+ server.settimeout(0.5)+ print(f"openworker tool runner {RUNNER_VERSION} ready on {path} (instance {self.instance_id})", file=sys.stderr, flush=True)+ try:+ while not self._stop.is_set():+ try:+ sock, _ = server.accept()+ except socket.timeout:+ if self._parent is not None and os.getppid() != self._parent:+ break # the server that started us is gone+ continue+ except OSError:+ break+ threading.Thread(target=self._serve_conn, args=(_Conn(sock),), daemon=True).start()+ finally:+ server.close()+ self._shutdown()+++class _LiveView:+ """Turns a command's lines into `shell.output` notifications: a chunk when it reaches+ LIVE_CHUNK_BYTES or LIVE_CHUNK_SECONDS of age, nothing at all for a command that ends+ sooner than that, and only progress ticks once LIVE_BUDGET_BYTES has been sent."""++ def __init__(self, notify: Callable[[str, dict[str, Any]], None]) -> None:+ self._notify = notify+ self._lock = threading.Lock()+ self._pending: list[str] = []+ self._pending_bytes = 0+ self._sent_bytes = 0+ self.total_bytes = 0+ self._started = time.monotonic()+ self._last_progress = self._started+ self._done = threading.Event()+ self._ticker = threading.Thread(target=self._tick, daemon=True)+ self._ticker.start()++ def feed(self, line: str) -> None:+ size = len(line.encode("utf-8", errors="replace"))+ with self._lock:+ self.total_bytes += size+ if self._sent_bytes >= P.LIVE_BUDGET_BYTES:+ return+ self._pending.append(line)+ self._pending_bytes += size+ flush = self._pending_bytes >= P.LIVE_CHUNK_BYTES+ if flush:+ self._flush()++ def _flush(self) -> None:+ with self._lock:+ if not self._pending:+ return+ text = "".join(self._pending)+ self._sent_bytes += self._pending_bytes+ self._pending, self._pending_bytes = [], 0+ self._notify("shell.output", {"text": text})++ def _tick(self) -> None:+ while not self._done.wait(P.LIVE_CHUNK_SECONDS):+ self._flush()+ now = time.monotonic()+ with self._lock:+ paused = self._sent_bytes >= P.LIVE_BUDGET_BYTES+ total = self.total_bytes+ if paused and now - self._last_progress >= P.PROGRESS_SECONDS:+ self._last_progress = now+ self._notify(+ "shell.progress",+ {"live_output_paused": True, "output_bytes": total, "elapsed_s": int(now - self._started)},+ )++ def stop(self) -> None:+ self._done.set()+++def _pick(params: dict[str, Any], *names: str) -> dict[str, Any]:+ return {name: params[name] for name in names if params.get(name) is not None}diff --git a/coworker/sandbox/runner/executor.py b/coworker/sandbox/runner/executor.pyindex 24d26ebe..05dca357 100644--- a/coworker/sandbox/runner/executor.py+++ b/coworker/sandbox/runner/executor.py@@ -35,7 +35,7 @@ import timeimport uuidfrom abc import ABC, abstractmethodfrom pathlib import Path-from typing import Any, Optional, Sequence+from typing import Any, Callable, Optional, Sequence_IS_WINDOWS = sys.platform == "win32"@@ -227,7 +227,14 @@ class LocalExecutor(Executor):finally:self._queue.put(None) # EOF sentinel- def run(self, command: str, timeout: Optional[float] = None) -> dict[str, Any]:+ def run(+ self,+ command: str,+ timeout: Optional[float] = None,+ on_output: Optional[Callable[[str], None]] = None,+ ) -> dict[str, Any]:+ """`on_output`, when given, is called with each line as it arrives (the tool runner+ uses it for the live view). The returned result is the same either way."""if self._proc.poll() is not None:# Shell exited (e.g. hard-closed after a prior command's timeout). Respawn so# the session self-heals rather than wedging every future command.@@ -293,6 +300,11 @@ class LocalExecutor(Executor):self.cwd = cwdbreaklines.append(item)+ if on_output is not None:+ try:+ on_output(item)+ except Exception: # the live view must never break the command+ on_output = Noneoutput = "".join(lines)truncated = len(output) > self.max_output_charsdiff --git a/coworker/sandbox/runner/fsops.py b/coworker/sandbox/runner/fsops.pynew file mode 100644index 00000000..986ca032--- /dev/null+++ b/coworker/sandbox/runner/fsops.py@@ -0,0 +1,227 @@+"""File operations served by the tool runner. Standard library only.++Every function takes plain values and returns a plain dict, so the daemon can put the+result straight into a frame. Paths may be relative; the daemon resolves them against the+calling shell's current folder before they get here.+"""++from __future__ import annotations++import base64+import hashlib+import os+import re+import shutil+from pathlib import Path+from typing import Any, Optional++from .protocol import MAX_READ_BYTES+++class FsError(Exception):+ pass+++def _sha256(data: bytes) -> str:+ return hashlib.sha256(data).hexdigest()+++def read(+ path: str,+ *,+ offset_line: Optional[int] = None,+ limit_lines: Optional[int] = None,+ offset_bytes: Optional[int] = None,+ limit_bytes: Optional[int] = None,+) -> dict[str, Any]:+ """A page of a file. By lines (text) unless a byte range is asked for (then base64).+ Never more than MAX_READ_BYTES in one call; `eof` says whether there is more."""+ p = Path(path)+ try:+ size = p.stat().st_size+ if offset_bytes is not None or limit_bytes is not None:+ start = max(0, offset_bytes or 0)+ want = min(limit_bytes or MAX_READ_BYTES, MAX_READ_BYTES)+ with p.open("rb") as fh:+ fh.seek(start)+ data = fh.read(want)+ return {+ "data_b64": base64.b64encode(data).decode("ascii"),+ "offset_bytes": start,+ "length": len(data),+ "total_bytes": size,+ "eof": start + len(data) >= size,+ }+ start_line = max(1, offset_line or 1)+ out: list[str] = []+ used = 0+ total = 0+ last = start_line - 1+ full = False+ with p.open("r", encoding="utf-8", errors="replace", newline="") as fh:+ for number, line in enumerate(fh, start=1):+ total = number+ if number < start_line or full:+ continue+ if limit_lines is not None and len(out) >= limit_lines:+ full = True+ continue+ encoded = len(line.encode("utf-8"))+ if out and used + encoded > MAX_READ_BYTES:+ full = True+ continue+ out.append(line)+ used += encoded+ last = number+ return {+ "text": "".join(out),+ "start_line": start_line,+ "end_line": last,+ "total_lines": total,+ "total_bytes": size,+ "eof": last >= total,+ }+ except OSError as exc:+ raise FsError(f"{type(exc).__name__}: {exc}") from exc+++def write(+ path: str,+ *,+ text: Optional[str] = None,+ data_b64: Optional[str] = None,+ if_match_sha256: Optional[str] = None,+ append: bool = False,+ make_parents: bool = True,+) -> dict[str, Any]:+ """Write a file. `if_match_sha256` makes an edit fail when the file changed underneath+ (pass the hash of the content the edit was based on; "" means "must not exist")."""+ p = Path(path)+ data = base64.b64decode(data_b64) if data_b64 is not None else (text or "").encode("utf-8")+ try:+ if if_match_sha256 is not None:+ current = _sha256(p.read_bytes()) if p.exists() else ""+ if current != if_match_sha256:+ raise FsError("the file changed since it was read")+ if make_parents:+ p.parent.mkdir(parents=True, exist_ok=True)+ with p.open("ab" if append else "wb") as fh:+ fh.write(data)+ return {"path": str(p), "bytes_written": len(data), "sha256": _sha256(p.read_bytes())}+ except OSError as exc:+ raise FsError(f"{type(exc).__name__}: {exc}") from exc+++def stat(path: str) -> dict[str, Any]:+ p = Path(path)+ try:+ st = p.lstat()+ except FileNotFoundError:+ return {"path": str(p), "exists": False}+ except OSError as exc:+ raise FsError(f"{type(exc).__name__}: {exc}") from exc+ kind = "symlink" if p.is_symlink() else "dir" if p.is_dir() else "file"+ info: dict[str, Any] = {+ "path": str(p),+ "exists": True,+ "kind": kind,+ "size": st.st_size,+ "mtime": st.st_mtime,+ "mode": st.st_mode & 0o7777,+ }+ if kind == "file" and st.st_size <= MAX_READ_BYTES:+ try:+ info["sha256"] = _sha256(p.read_bytes())+ except OSError:+ pass+ return info+++def list_dir(path: str, *, limit: int = 2000) -> dict[str, Any]:+ p = Path(path)+ entries: list[dict[str, Any]] = []+ truncated = False+ try:+ with os.scandir(p) as it:+ for entry in sorted(it, key=lambda e: e.name):+ if len(entries) >= limit:+ truncated = True+ break+ kind = "symlink" if entry.is_symlink() else "dir" if entry.is_dir() else "file"+ size = None+ if kind == "file":+ try:+ size = entry.stat().st_size+ except OSError:+ pass+ entries.append({"name": entry.name, "kind": kind, "size": size})+ except OSError as exc:+ raise FsError(f"{type(exc).__name__}: {exc}") from exc+ return {"path": str(p), "entries": entries, "truncated": truncated}+++def mkdir(path: str, *, parents: bool = True) -> dict[str, Any]:+ try:+ Path(path).mkdir(parents=parents, exist_ok=True)+ except OSError as exc:+ raise FsError(f"{type(exc).__name__}: {exc}") from exc+ return {"path": str(Path(path))}+++def remove(path: str, *, recursive: bool = False) -> dict[str, Any]:+ p = Path(path)+ try:+ if p.is_dir() and not p.is_symlink():+ if recursive:+ shutil.rmtree(p)+ else:+ p.rmdir()+ else:+ p.unlink()+ except OSError as exc:+ raise FsError(f"{type(exc).__name__}: {exc}") from exc+ return {"path": str(p), "removed": True}+++def search(+ path: str,+ pattern: str,+ *,+ max_matches: int = 200,+ ignore_case: bool = False,+ max_file_bytes: int = 5 * 1024 * 1024,+) -> dict[str, Any]:+ """Regular-expression search in one file or under one folder. Pure Python, so it+ works in any image; hidden folders and very large or binary files are skipped."""+ try:+ rx = re.compile(pattern, re.IGNORECASE if ignore_case else 0)+ except re.error as exc:+ raise FsError(f"bad pattern: {exc}") from exc+ root = Path(path)+ matches: list[dict[str, Any]] = []+ truncated = False++ def scan(file: Path) -> bool:+ nonlocal truncated+ try:+ if file.stat().st_size > max_file_bytes and file != root:+ return True+ with file.open("r", encoding="utf-8", errors="strict") as fh:+ for number, line in enumerate(fh, start=1):+ if rx.search(line):+ if len(matches) >= max_matches:+ truncated = True+ return False+ matches.append({"path": str(file), "line": number, "text": line.rstrip("\n")[:500]})+ except (OSError, UnicodeDecodeError):+ pass+ return True++ if root.is_file():+ scan(root)+ else:+ for folder, dirs, files in os.walk(root):+ dirs[:] = sorted(d for d in dirs if not d.startswith(".") and d != "node_modules")+ if not all(scan(Path(folder) / name) for name in sorted(files)):+ break+ return {"matches": matches, "truncated": truncated}diff --git a/coworker/sandbox/runner/protocol.py b/coworker/sandbox/runner/protocol.pynew file mode 100644index 00000000..10de8a06--- /dev/null+++ b/coworker/sandbox/runner/protocol.py@@ -0,0 +1,93 @@+"""Wire format between the workspace client and the tool runner. Standard library only.++One JSON-RPC 2.0 object per line, UTF-8. Tool output never goes onto the pipe raw: it+travels inside a JSON string, where a line break is the two characters `\\n`, so a frame+can never contain a real line break however many lines the output has.+"""++from __future__ import annotations++import json+from typing import Any, Optional++PROTOCOL_VERSION = 1++# Live output (`shell.output` notifications) is a view, not the record: the result carries+# the output. A chunk goes out when it reaches this size or this age, whichever is first,+# and live text stops after the budget (progress notifications continue).+LIVE_CHUNK_BYTES = 64 * 1024+LIVE_CHUNK_SECONDS = 0.2+LIVE_BUDGET_BYTES = 1024 * 1024+PROGRESS_SECONDS = 5.0++# One fs.read call returns at most this much.+MAX_READ_BYTES = 1024 * 1024++# The client sends a ping this often; the relay exits after this much silence.+PING_SECONDS = 10.0+RELAY_SILENCE_SECONDS = 45.0++# JSON-RPC error codes. -32xxx are the standard ones; ours start at 1.+PARSE_ERROR = -32700+INVALID_REQUEST = -32600+METHOD_NOT_FOUND = -32601+INVALID_PARAMS = -32602+INTERNAL_ERROR = -32603+SHELL_BUSY = 1+UNKNOWN_SHELL = 2+UNKNOWN_TASK = 3+FS_ERROR = 4+VERSION_MISMATCH = 5+RUNNER_RESTARTED = 6 # raised by the client, never sent by the runner+++def encode(obj: dict[str, Any]) -> bytes:+ """One frame: compact JSON, one line. `ensure_ascii` keeps every byte printable."""+ return (json.dumps(obj, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii")+++def decode(line: bytes | str) -> Optional[dict[str, Any]]:+ """A frame, or None when the line is not a JSON object (a stray print, a banner).+ The reader drops such lines instead of failing."""+ if isinstance(line, bytes):+ try:+ line = line.decode("utf-8")+ except UnicodeDecodeError:+ return None+ line = line.strip()+ if not line.startswith("{"):+ return None+ try:+ obj = json.loads(line)+ except ValueError:+ return None+ return obj if isinstance(obj, dict) else None+++def request(req_id: str, method: str, params: Optional[dict[str, Any]] = None) -> dict[str, Any]:+ return {"jsonrpc": "2.0", "id": req_id, "method": method, "params": params or {}}+++def notification(method: str, params: Optional[dict[str, Any]] = None) -> dict[str, Any]:+ return {"jsonrpc": "2.0", "method": method, "params": params or {}}+++def result(req_id: str, value: dict[str, Any]) -> dict[str, Any]:+ return {"jsonrpc": "2.0", "id": req_id, "result": value}+++def error(req_id: Optional[str], code: int, message: str, data: Optional[dict] = None) -> dict[str, Any]:+ err: dict[str, Any] = {"code": code, "message": message}+ if data:+ err["data"] = data+ return {"jsonrpc": "2.0", "id": req_id, "error": err}+++class RunnerError(Exception):+ """An error response, as an exception on the client side."""++ def __init__(self, code: int, message: str, data: Optional[dict] = None) -> None:+ super().__init__(message)+ self.code = code+ self.message = message+ self.data = data or {}diff --git a/coworker/sandbox/runner/relay.py b/coworker/sandbox/runner/relay.pynew file mode 100644index 00000000..f650dd66--- /dev/null+++ b/coworker/sandbox/runner/relay.py@@ -0,0 +1,72 @@+"""The attach relay: copies bytes between its own stdin/stdout and the daemon's socket.+Standard library only.++One relay per client stream. It never reads frames. It ends when its stdin ends, when the+daemon closes the connection, or when the client has been silent for too long (the client+pings; a provider may leave a relay behind for a while after a stream is cut, and this is+how such a leftover goes away by itself).+"""++from __future__ import annotations++import os+import socket+import sys+import threading+import time++from .protocol import RELAY_SILENCE_SECONDS+++def run(socket_path: str, silence_seconds: float = RELAY_SILENCE_SECONDS) -> int:+ # Keep the pipe clean: move the real stdout to a private descriptor and point+ # descriptor 1 at stderr, so a stray print can never land inside a frame.+ out_fd = os.dup(1)+ os.dup2(2, 1)++ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)+ try:+ sock.connect(socket_path)+ except OSError as exc:+ print(f"attach: cannot reach the runner at {socket_path}: {exc}", file=sys.stderr)+ return 2++ last_heard = [time.monotonic()]+ done = threading.Event()++ def upstream() -> None:+ try:+ while True:+ data = os.read(0, 65536)+ if not data:+ break+ last_heard[0] = time.monotonic()+ sock.sendall(data)+ except OSError:+ pass+ done.set()++ def downstream() -> None:+ try:+ while True:+ data = sock.recv(65536)+ if not data:+ break+ view = memoryview(data)+ while view:+ view = view[os.write(out_fd, view):]+ except OSError:+ pass+ done.set()++ threading.Thread(target=upstream, daemon=True).start()+ threading.Thread(target=downstream, daemon=True).start()+ while not done.wait(1.0):+ if silence_seconds and time.monotonic() - last_heard[0] > silence_seconds:+ print("attach: no word from the client; leaving", file=sys.stderr)+ break+ try:+ sock.close()+ except OSError:+ pass+ return 0diff --git a/coworker/sandbox/transport.py b/coworker/sandbox/transport.pynew file mode 100644index 00000000..6bfb288b--- /dev/null+++ b/coworker/sandbox/transport.py@@ -0,0 +1,56 @@+"""The pipe between the workspace client and a tool runner.++The client needs three things from a pipe: send a line, receive a line, close. A local+process pipe (this file), an OpenShell exec stream and, later, a stream over the machine+channel all fit that shape, which is what keeps the client independent of the sandbox+technology and of which machine the runner is on.+"""++from __future__ import annotations++import subprocess+from typing import Optional, Protocol+++class Transport(Protocol):+ def send(self, frame: bytes) -> None: ...++ def recv_line(self) -> Optional[bytes]:+ """The next line, or None when the pipe is closed."""++ def close(self) -> None: ...+++class PipeTransport:+ """A child process whose stdin and stdout are the pipe (the attach relay, locally)."""++ def __init__(self, proc: subprocess.Popen) -> None:+ self.proc = proc++ def send(self, frame: bytes) -> None:+ if self.proc.stdin is None:+ raise OSError("pipe is closed")+ self.proc.stdin.write(frame)+ self.proc.stdin.flush()++ def recv_line(self) -> Optional[bytes]:+ if self.proc.stdout is None:+ return None+ line = self.proc.stdout.readline()+ return line or None++ def close(self) -> None:+ for stream in (self.proc.stdin, self.proc.stdout):+ try:+ if stream is not None:+ stream.close()+ except OSError:+ pass+ try:+ self.proc.terminate()+ self.proc.wait(timeout=3)+ except (OSError, subprocess.TimeoutExpired):+ try:+ self.proc.kill()+ except OSError:+ passdiff --git a/coworker/sandbox/workspace.py b/coworker/sandbox/workspace.pynew file mode 100644index 00000000..4a890ae7--- /dev/null+++ b/coworker/sandbox/workspace.py@@ -0,0 +1,97 @@+"""The Workspace: the one thing tools use to touch files and run commands.++Two implementations:+- `DirectWorkspace`: today's behaviour, in this process. No runner, no JSON-RPC, no extra+ process. This is `direct` mode and the default, kept so that benchmark readings stay+ comparable. It reports enforcement `none`.+- `RunnerWorkspace`: a tool runner behind a provider (a sandbox, or `runner-local`).++Step 1 routes the shell through the workspace. The file, git and search tools follow in+step 2 (design doc, section 11).+"""++from __future__ import annotations++import os+from abc import ABC, abstractmethod+from pathlib import Path+from typing import Any, Optional++from .runner.executor import Executor++PROVIDER_ENV = "OPENWORKER_SANDBOX_PROVIDER"+DIRECT = "direct"+RUNNER_LOCAL = "runner-local"+++class Workspace(ABC):+ @property+ @abstractmethod+ def executor(self) -> Executor: ...++ @abstractmethod+ def describe(self) -> dict[str, Any]:+ """Which provider this is and how much it enforces: `full`, `partial` or `none`."""++ def close(self) -> None:+ self.executor.close()+++class DirectWorkspace(Workspace):+ def __init__(self, *, cwd: str | Path) -> None:+ from ..tools.shell import LocalExecutor # here, not at the top: tools.shell imports us++ self._executor = LocalExecutor(cwd=cwd)++ @property+ def executor(self) -> Executor:+ return self._executor++ def describe(self) -> dict[str, Any]:+ return {"provider": DIRECT, "enforcement": "none", "reason": "commands run in the OpenWorker process, unconfined"}+++class RunnerWorkspace(Workspace):+ def __init__(self, provider: Any, *, cwd: str | Path, shell: str = "main") -> None:+ from .client import RunnerClient+ from .executor import RunnerExecutor++ self.provider = provider+ provider.create()+ try:+ self.client = RunnerClient(provider.open_runner)+ self.hello = self.client.connect()+ except Exception:+ provider.destroy()+ raise+ self._executor = RunnerExecutor(self.client, cwd=str(Path(cwd).expanduser().resolve()), shell=shell)++ @property+ def executor(self) -> Executor:+ return self._executor++ def describe(self) -> dict[str, Any]:+ return {**self.provider.describe(), "runner": {k: self.hello.get(k) for k in ("runner_version", "os", "machine", "instance_id")}}++ def close(self) -> None:+ try:+ self._executor.close()+ finally:+ self.client.close()+ self.provider.destroy()+++def provider_name(explicit: Optional[str] = None) -> str:+ return (explicit or os.environ.get(PROVIDER_ENV) or DIRECT).strip().lower()+++def open_workspace(*, cwd: str | Path, provider: Optional[str] = None) -> Workspace:+ """The session's workspace for the configured provider. `direct` unless told otherwise."""+ name = provider_name(provider)+ if name == DIRECT:+ return DirectWorkspace(cwd=cwd)+ if name == RUNNER_LOCAL:+ from .providers.runner_local import RunnerLocalProvider++ return RunnerWorkspace(RunnerLocalProvider(cwd=cwd), cwd=cwd)+ raise ValueError(f"unknown sandbox provider: {name!r} (known: {DIRECT}, {RUNNER_LOCAL})")diff --git a/tests/test_sandbox_runner.py b/tests/test_sandbox_runner.pynew file mode 100644index 00000000..05ffe2dd--- /dev/null+++ b/tests/test_sandbox_runner.py@@ -0,0 +1,322 @@+"""The tool runner: protocol, daemon, relay, client (design doc `sandbox-design.md`).++These run the real thing end to end: the runner packed into its single file, a daemon+process, an attach relay per pipe, and the client. No sandbox technology is involved+(`runner-local`), so what is tested here is the protocol and its delivery rules.+"""++from __future__ import annotations++import ast+import base64+import sys+import threading+import time+import zipfile+from pathlib import Path++import pytest++pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="the tool runner needs Unix sockets")++from coworker.sandbox.bundle import build_runner_zipapp # noqa: E402+from coworker.sandbox.client import RunnerClient # noqa: E402+from coworker.sandbox.executor import RunnerExecutor # noqa: E402+from coworker.sandbox.providers.runner_local import RunnerLocalProvider # noqa: E402+from coworker.sandbox.runner import protocol as P # noqa: E402+from coworker.sandbox.workspace import DirectWorkspace, RunnerWorkspace, open_workspace # noqa: E402++RUNNER_DIR = Path(__file__).resolve().parents[1] / "coworker" / "sandbox" / "runner"+++def runner(tmp_path):+ """(provider, client) on a fresh daemon. The relay never leaves on silence here."""+ zipapp = build_runner_zipapp(tmp_path / "dist")+ provider = RunnerLocalProvider(cwd=tmp_path, runner_path=zipapp, relay_silence_seconds=0)+ provider.create()+ client = RunnerClient(provider.open_runner, ping_seconds=0.5)+ client.connect()+ yield provider, client+ client.close()+ provider.destroy()+++# -- packaging --------------------------------------------------------------------------+++def test_runner_uses_only_the_standard_library():+ """The runner is mounted into any image that has Python, so it may import nothing else."""+ allowed = set(sys.stdlib_module_names)+ for source in RUNNER_DIR.glob("*.py"):+ for node in ast.walk(ast.parse(source.read_text(encoding="utf-8"))):+ if isinstance(node, ast.Import):+ names = [alias.name for alias in node.names]+ elif isinstance(node, ast.ImportFrom) and node.level == 0:+ names = [node.module or ""]+ else:+ continue # relative imports stay inside the package+ for name in names:+ assert name.split(".")[0] in allowed, f"{source.name} imports {name}"+++def test_zipapp_is_one_file_named_by_content(tmp_path):+ first = build_runner_zipapp(tmp_path)+ assert first == build_runner_zipapp(tmp_path) # reused, not rebuilt+ with zipfile.ZipFile(first) as zf:+ names = set(zf.namelist())+ assert "__main__.py" in names and "owrunner/daemon.py" in names and "owrunner/executor.py" in names+++# -- framing ----------------------------------------------------------------------------+++def test_a_frame_is_always_one_line():+ frame = P.encode(P.result("r-1", {"output": "line one\nline two\r\n\ttabbed ☃ \x00"}))+ assert frame.count(b"\n") == 1 and frame.endswith(b"\n")+ assert P.decode(frame)["result"]["output"] == "line one\nline two\r\n\ttabbed ☃ \x00"+++def test_lines_that_are_not_frames_are_dropped():+ assert P.decode(b"Welcome to Ubuntu 24.04\n") is None+ assert P.decode(b"[1, 2, 3]\n") is None+ assert P.decode(b"{not json}\n") is None+ assert P.decode(b"\xff\xfe\n") is None+++def test_multi_line_output_and_shell_syntax_survive_the_pipe(runner, tmp_path):+ _, client = runner+ ex = RunnerExecutor(client, cwd=str(tmp_path))+ assert ex.run("printf 'a\\nb\\nc\\n'")["output"].startswith("a\nb\nc\n")+ # Redirection, a pipeline and a heredoc go to a real bash unchanged.+ assert ex.run("printf 'x\\ny\\nx\\n' > out.log 2>&1")["output"].strip() == ""+ assert ex.run("sort out.log | uniq -c | sort -rn | head -1")["output"].split() == ["2", "x"]+ ex.run("cat > notes.md <<'EOF'\n# Notes\nsecond \"line\"\nEOF")+ assert (tmp_path / "notes.md").read_text() == '# Notes\nsecond "line"\n'+++# -- the contract -----------------------------------------------------------------------+++def test_hello_reports_the_runner_environment(runner):+ _, client = runner+ assert client.hello["protocol_version"] == P.PROTOCOL_VERSION+ assert client.hello["instance_id"] == client.instance_id+ assert client.hello["os"] and client.hello["shell"]+++def test_a_client_with_another_protocol_version_is_refused(runner):+ _, client = runner+ with pytest.raises(P.RunnerError) as err:+ client.call("runner.hello", {"protocol_version": 999}, timeout=10)+ assert err.value.code == P.VERSION_MISMATCH+++def test_results_look_the_same_as_the_in_process_executor(runner, tmp_path):+ """A session must look the same to the model in both modes: same keys, same values."""+ _, client = runner+ direct = DirectWorkspace(cwd=tmp_path)+ try:+ ours = RunnerExecutor(client, cwd=str(tmp_path)).run("echo hi; false")+ theirs = direct.executor.run("echo hi; false")+ finally:+ direct.close()+ assert ours == theirs+++def test_each_shell_keeps_its_own_folder_and_variables(runner, tmp_path):+ _, client = runner+ (tmp_path / "a").mkdir()+ (tmp_path / "b").mkdir()+ one = RunnerExecutor(client, cwd=str(tmp_path), shell="agent-1")+ two = RunnerExecutor(client, cwd=str(tmp_path), shell="agent-2")+ one.run("cd a && export WHO=one")+ two.run("cd b && export WHO=two")+ assert one.run("echo $WHO $(basename $PWD)")["output"].split() == ["one", "a"]+ assert two.run("echo $WHO $(basename $PWD)")["output"].split() == ["two", "b"]+++def test_a_slow_shell_does_not_block_another(runner, tmp_path):+ _, client = runner+ slow = RunnerExecutor(client, cwd=str(tmp_path), shell="slow")+ fast = RunnerExecutor(client, cwd=str(tmp_path), shell="fast")+ worker = threading.Thread(target=lambda: slow.run("sleep 2"))+ worker.start()+ time.sleep(0.3)+ started = time.monotonic()+ assert fast.run("echo quick")["exit_code"] == 0+ assert time.monotonic() - started < 1.0+ worker.join()+++def test_one_shell_runs_one_command_at_a_time(runner, tmp_path):+ _, client = runner+ ex = RunnerExecutor(client, cwd=str(tmp_path))+ worker = threading.Thread(target=lambda: ex.run("sleep 1.5"))+ worker.start()+ time.sleep(0.3)+ second = ex.run("echo too-soon")+ worker.join()+ assert "running another command" in second["error"] and second["exit_code"] is None+++def test_live_output_is_a_view_and_the_result_is_the_record(runner, tmp_path):+ _, client = runner+ seen: list[str] = []+ ex = RunnerExecutor(client, cwd=str(tmp_path), on_output=seen.append)+ fast = ex.run("echo instant")+ assert fast["output"].startswith("instant") and seen == [] # too quick for a live frame+ slow = ex.run("for i in 1 2 3; do echo tick $i; sleep 0.4; done")+ assert "tick 1" in "".join(seen) # the view arrived while it ran+ assert all(f"tick {i}" in slow["output"] for i in (1, 2, 3)) # the record is complete+++def test_user_stop_ends_the_command_and_the_session_carries_on(runner, tmp_path):+ """Same behaviour as the in-process executor: Stop ends the command within moments and+ the session carries on in the same folder. (Today a Stop also restarts the shell, in+ both modes, so variables are lost. That is a separate, older bug; this test does not+ pin it.)"""+ _, client = runner+ (tmp_path / "work").mkdir()+ ex = RunnerExecutor(client, cwd=str(tmp_path))+ ex.run("cd work")+ threading.Timer(0.5, ex.interrupt_now).start()+ started = time.monotonic()+ stopped = ex.run("sleep 20", timeout=30)+ assert time.monotonic() - started < 10+ assert stopped["error"] == "interrupted by user"+ assert ex.run("basename $PWD")["output"].strip() == "work"+++# -- delivery after a break -------------------------------------------------------------+++def test_a_cut_stream_resumes_and_no_result_is_lost(runner, tmp_path):+ provider, client = runner+ ex = RunnerExecutor(client, cwd=str(tmp_path))+ ex.run("cd /tmp && export KEEP=yes")+ instance = client.instance_id+ box: dict = {}+ worker = threading.Thread(target=lambda: box.update(ex.run("sleep 1.5; echo finished-while-away")))+ worker.start()+ time.sleep(0.4)+ client._transport.proc.kill() # the stream is cut while the command runs+ worker.join(timeout=20)+ assert "finished-while-away" in box.get("output", "") # delivered after the resume+ assert client.instance_id == instance and client.restarts == 0 # same daemon+ assert ex.run("echo $KEEP $PWD")["output"].split() == ["yes", "/tmp"] # same shell+++def test_the_same_request_id_runs_once(runner, tmp_path):+ _, client = runner+ target = tmp_path / "count.txt"+ frame = P.encode(P.request("r-fixed", "shell.run", {"shell": "main", "command": f"echo x >> {target}", "cwd": str(tmp_path)}))+ for _ in range(3):+ client._send(client._transport, frame)+ deadline = time.monotonic() + 10+ while not target.exists() and time.monotonic() < deadline:+ time.sleep(0.05)+ time.sleep(0.5)+ assert target.read_text() == "x\n"+++def test_a_restarted_sandbox_fails_the_running_command_and_reopens_in_the_last_folder(runner, tmp_path):+ provider, client = runner+ (tmp_path / "work").mkdir()+ ex = RunnerExecutor(client, cwd=str(tmp_path))+ ex.run("cd work && export GONE=soon")+ box: dict = {}+ worker = threading.Thread(target=lambda: box.update(ex.run("sleep 30", timeout=60)))+ worker.start()+ time.sleep(0.4)+ provider.restart_daemon() # a new runner instance, as after a sandbox restart+ worker.join(timeout=30)+ assert "outcome of this command is unknown" in box["error"] # never re-sent by the client+ assert client.restarts == 1+ after = ex.run("echo [$GONE] $(basename $PWD)")+ assert after["output"].split() == ["[]", "work"] # variables lost, folder kept+++def test_a_relay_left_behind_goes_away_by_itself(tmp_path):+ """Spike finding C: a provider may leave the relay running after a stream is cut."""+ import subprocess++ zipapp = build_runner_zipapp(tmp_path / "dist")+ provider = RunnerLocalProvider(cwd=tmp_path, runner_path=zipapp)+ provider.create()+ try:+ relay = subprocess.Popen(+ [sys.executable, str(zipapp), "attach", "--socket", provider.socket_path, "--silence-seconds", "1"],+ stdin=subprocess.PIPE,+ stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL,+ )+ assert relay.wait(timeout=10) == 0 # nobody spoke: it left+ finally:+ provider.destroy()+++# -- files ------------------------------------------------------------------------------+++def test_files_are_read_in_pages_and_written_with_a_precondition(runner, tmp_path):+ _, client = runner+ big = tmp_path / "out.log"+ big.write_text("".join(f"line {i}\n" for i in range(1, 1001)))+ page = client.call("fs.read", {"path": str(big), "offset_line": 10, "limit_lines": 3}, timeout=10)+ assert page["text"] == "line 10\nline 11\nline 12\n"+ assert (page["start_line"], page["end_line"], page["total_lines"], page["eof"]) == (10, 12, 1000, False)++ raw = client.call("fs.read", {"path": str(big), "offset_bytes": 0, "limit_bytes": 6}, timeout=10)+ assert base64.b64decode(raw["data_b64"]) == b"line 1" and raw["eof"] is False++ note = tmp_path / "note.txt"+ written = client.call("fs.write", {"path": str(note), "text": "one\n"}, timeout=10)+ client.call("fs.write", {"path": str(note), "text": "two\n", "if_match_sha256": written["sha256"]}, timeout=10)+ with pytest.raises(P.RunnerError) as err: # based on content that is no longer there+ client.call("fs.write", {"path": str(note), "text": "three\n", "if_match_sha256": written["sha256"]}, timeout=10)+ assert err.value.code == P.FS_ERROR and note.read_text() == "two\n"++ found = client.call("fs.search", {"path": str(tmp_path), "pattern": r"^line 99\d$"}, timeout=10)+ assert [m["line"] for m in found["matches"]] == list(range(990, 1000))+++def test_a_relative_path_resolves_against_the_shells_folder(runner, tmp_path):+ _, client = runner+ (tmp_path / "sub").mkdir()+ RunnerExecutor(client, cwd=str(tmp_path), shell="agent-1").run("cd sub && echo hello > here.txt")+ got = client.call("fs.read", {"path": "here.txt", "shell": "agent-1"}, timeout=10)+ assert got["text"] == "hello\n"+ listing = client.call("fs.list", {"path": ".", "shell": "agent-1"}, timeout=10)+ assert [e["name"] for e in listing["entries"]] == ["here.txt"]+++def test_proc_run_takes_an_argument_list_without_a_shell(runner, tmp_path):+ _, client = runner+ done = client.call("proc.run", {"argv": ["printf", "%s|", "a b", "$HOME"], "cwd": str(tmp_path)}, timeout=10)+ assert done["exit_code"] == 0 and done["stdout"] == "a b|$HOME|" # no shell: nothing expanded+++# -- choosing a workspace ---------------------------------------------------------------+++def test_direct_is_the_default_and_reports_no_enforcement(tmp_path, monkeypatch):+ monkeypatch.delenv("OPENWORKER_SANDBOX_PROVIDER", raising=False)+ ws = open_workspace(cwd=tmp_path)+ try:+ assert isinstance(ws, DirectWorkspace)+ assert ws.describe()["provider"] == "direct" and ws.describe()["enforcement"] == "none"+ finally:+ ws.close()+++def test_the_provider_setting_selects_the_runner(tmp_path, monkeypatch):+ monkeypatch.setenv("OPENWORKER_SANDBOX_PROVIDER", "runner-local")+ ws = open_workspace(cwd=tmp_path)+ try:+ assert isinstance(ws, RunnerWorkspace) and ws.describe()["provider"] == "runner-local"+ assert ws.executor.run("echo via-runner")["output"].strip() == "via-runner"+ finally:+ ws.close()+ with pytest.raises(ValueError):+ open_workspace(cwd=tmp_path, provider="no-such-thing")diff --git a/tests/test_shell.py b/tests/test_shell.pyindex f0a842e2..6b71f385 100644--- a/tests/test_shell.py+++ b/tests/test_shell.py@@ -31,11 +31,32 @@ PRINT_1000 = ()-def executor(tmp_path):- ex = LocalExecutor(cwd=tmp_path, default_timeout=10)- yield ex- ex.close()+# Every behaviour below is checked twice: on the in-process executor (direct mode) and on+# the same contract served by a tool runner in another process (the sandbox path). The two+# must not drift apart. The runner needs Unix sockets, so it is skipped on Windows.+_EXECUTORS = ["direct"] if _WIN else ["direct", "runner"]++[email protected](params=_EXECUTORS)+def executor(request, tmp_path):+ if request.param == "direct":+ ex = LocalExecutor(cwd=tmp_path, default_timeout=10)+ yield ex+ ex.close()+ return+ from coworker.sandbox.providers.runner_local import RunnerLocalProvider+ from coworker.sandbox.workspace import RunnerWorkspace++ ws = RunnerWorkspace(RunnerLocalProvider(cwd=tmp_path, runner_path=_runner_zipapp(tmp_path)), cwd=tmp_path)+ ws.executor.default_timeout = 10+ yield ws.executor+ ws.close()+++def _runner_zipapp(tmp_path):+ from coworker.sandbox.bundle import build_runner_zipapp++ return build_runner_zipapp(tmp_path.parent / "runner-dist")def test_cwd_persists_across_calls(executor, tmp_path):