Python
"""The Seatbelt provider: the tool runner inside the macOS sandbox.
The first half runs anywhere: the profile text, the allow-list proxy, the environment that
is passed on. The second half runs only on a Mac where `sandbox-exec` works, and checks the
wall itself with real commands.
"""
from __future__ import annotations
import os
import socket
import sys
import threading
from pathlib import Path
import pytest
from coworker.agents.base import AgentContext # noqa: E402 (before catalog: import order)
from coworker import catalog # noqa: E402, I001
from coworker.roots import RootDir
from coworker.sandbox import netproxy, network_profiles
from coworker.sandbox.providers import seatbelt, seatbelt_profile
from coworker.sandbox.selection import select
# -- the profile ----------------------------------------------------------------------
posix_paths = pytest.mark.skipif(sys.platform == "win32", reason="the profile is written for macOS paths")
@posix_paths
def test_profile_lists_the_folders_and_nothing_else_under_home(tmp_path):
home = tmp_path / "home"
(home / ".nvm").mkdir(parents=True)
(home / ".ssh").mkdir()
(home / ".gitconfig").write_text("[user]\n")
project, reference = tmp_path / "project", tmp_path / "reference"
project.mkdir()
reference.mkdir()
text = seatbelt_profile.render(
[{"path": str(project), "writable": True}, {"path": str(reference), "writable": False}],
runtime_dir=str(tmp_path / "run"),
home=str(home),
)
assert text.startswith("(version 1)\n(deny default)\n")
real = os.path.realpath
write_rule = next(line for line in text.splitlines() if line.startswith("(allow file-read* file-write* (subpath"))
assert real(project) in write_rule and real(tmp_path / "run") in write_rule
assert real(reference) not in write_rule # a read-only folder is never writable
read_rule = next(line for line in text.splitlines() if line.startswith("(allow file-read* (subpath") and real(reference) in line)
assert real(home / ".nvm") in read_rule and real(home / ".gitconfig") in read_rule
assert ".ssh" not in text
assert "remote tcp" not in text # no proxy port given: no network at all
@posix_paths
def test_profile_allows_only_the_proxy_port_and_quotes_paths(tmp_path):
odd = tmp_path / 'has "quotes" and spaces'
odd.mkdir()
text = seatbelt_profile.render([{"path": str(odd), "writable": True}], runtime_dir=str(tmp_path), proxy_port=45678, home=str(tmp_path))
assert '(allow network-outbound (remote tcp "localhost:45678"))' in text
assert text.count("remote tcp") == 1
assert 'has \\"quotes\\" and spaces' in text
def test_secrets_in_the_servers_environment_are_not_passed_on():
env = seatbelt.clean_environment(
{
"PATH": "/usr/bin",
"LANG": "en_US.UTF-8",
"NVM_DIR": "/Users/sam/.nvm",
"ANTHROPIC_API_KEY": "x",
"GITHUB_TOKEN": "x",
"AWS_SECRET_ACCESS_KEY": "x",
"AWS_SESSION_TOKEN": "x",
"COWORKER_API_TOKEN": "x",
"OPENWORKER_HEADLESS": "1",
"SSH_AUTH_SOCK": "/tmp/agent",
"DB_PASSWORD": "x",
}
)
assert env == {"PATH": "/usr/bin", "LANG": "en_US.UTF-8", "NVM_DIR": "/Users/sam/.nvm"}
# -- the allow-list proxy -------------------------------------------------------------
def _ask(proxy: netproxy.AllowListProxy, request: bytes) -> bytes:
with socket.create_connection(("127.0.0.1", proxy.port), timeout=5) as s:
s.sendall(request)
s.settimeout(5)
chunks = []
while True:
try:
data = s.recv(4096)
except socket.timeout:
break
if not data:
break
chunks.append(data)
if b"pong" in data:
break
return b"".join(chunks)
def test_proxy_refuses_hosts_that_are_not_listed_and_says_why():
proxy = netproxy.AllowListProxy("allowlist", extra_hosts=["github.com:443"])
try:
answer = _ask(proxy, b"CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n")
assert answer.startswith(b"HTTP/1.1 403 ")
assert b"example.com:443 is not on this sandbox's allow list" in answer
assert list(proxy.denied) == ["example.com:443"]
# a listed host on another port, and plain HTTP, are refused too
assert _ask(proxy, b"CONNECT github.com:22 HTTP/1.1\r\n\r\n").startswith(b"HTTP/1.1 403 ")
assert _ask(proxy, b"GET http://github.com/ HTTP/1.1\r\nHost: github.com\r\n\r\n").startswith(b"HTTP/1.1 403 ")
finally:
proxy.close()
def test_proxy_tunnels_a_listed_host(monkeypatch):
upstream = socket.socket()
upstream.bind(("127.0.0.1", 0))
upstream.listen(1)
def serve() -> None:
conn, _ = upstream.accept()
with conn:
if conn.recv(16) == b"ping":
conn.sendall(b"pong")
threading.Thread(target=serve, daemon=True).start()
wanted: list[tuple[str, int]] = []
real_connect = socket.create_connection
def fake_connect(address, *args, **kwargs):
if address[0] == "github.com":
wanted.append(address)
return real_connect(upstream.getsockname(), *args, **kwargs)
return real_connect(address, *args, **kwargs)
monkeypatch.setattr(netproxy.socket, "create_connection", fake_connect)
proxy = netproxy.AllowListProxy("allowlist", extra_hosts=["github.com:443"])
try:
with real_connect(("127.0.0.1", proxy.port), timeout=5) as s:
s.sendall(b"CONNECT github.com:443 HTTP/1.1\r\nHost: github.com:443\r\n\r\n")
s.settimeout(5)
assert s.recv(4096).startswith(b"HTTP/1.0 200 ")
s.sendall(b"ping")
assert s.recv(16) == b"pong"
assert wanted == [("github.com", 443)]
finally:
proxy.close()
upstream.close()
def test_the_catalogue_offers_sites_and_an_unknown_profile_is_refused():
assert network_profiles.hosts("allowlist") == [] # nothing until the machine ticks sites
assert {"github.com", "pypi.org", "api.tavily.com"} <= {h for hosts in network_profiles.SITES.values() for h in hosts}
for gone in ("strict", "standard", "wide-open"):
with pytest.raises(ValueError):
netproxy.AllowListProxy(gone)
def test_a_machines_own_hosts_are_cleaned_and_join_the_list():
clean = network_profiles.clean_host
assert clean("Registry.Acme.dev") == "registry.acme.dev:443"
assert clean("sentry.io:8443") == "sentry.io:8443" and clean("*.acme.dev") == "*.acme.dev:443"
assert clean("https://api.acme.dev/v1") == "api.acme.dev:443"
for bad in ("", "localhost", "acme dev", "api.acme.dev:0", "api.acme.dev:http", "a..b"):
with pytest.raises(ValueError):
clean(bad)
assert network_profiles.clean_hosts(["a.dev", "A.dev:443", "nope"]) == ["a.dev:443"]
proxy = netproxy.AllowListProxy("allowlist", extra_hosts=["registry.acme.dev:443"])
try:
assert proxy.allows("registry.acme.dev", 443) and not proxy.allows("other.acme.dev", 443)
finally:
proxy.close()
def test_a_running_proxy_takes_more_hosts():
"""The person allowed a site while the session runs (OPE-219): its proxy lets the host
out from then on, and only that proxy, since every session has its own."""
proxy, other = netproxy.AllowListProxy("allowlist"), netproxy.AllowListProxy("allowlist")
try:
assert not proxy.allows("weather.com", 443)
proxy.add_hosts(["weather.com:443", "not a host"])
assert proxy.allows("weather.com", 443) and not proxy.allows("weather.com", 80)
assert not other.allows("weather.com", 443)
finally:
proxy.close()
other.close()
def test_the_open_profile_has_no_list_and_is_the_default_on_windows_only(tmp_path):
assert network_profiles.hosts("open") == [] and network_profiles.is_open("open")
assert network_profiles.default_profile("win32") == "open"
assert network_profiles.default_profile("darwin") == "allowlist" and network_profiles.default_profile("linux") == "allowlist"
# Seatbelt: the files stay confined, the network clause opens
text = seatbelt_profile.render([{"path": str(tmp_path), "writable": True}], runtime_dir=str(tmp_path), open_network=True, home=str(tmp_path))
assert "(allow network*)" in text and "localhost:" not in text and "(deny default)" in text
@posix_paths
def test_profile_reads_the_machines_toolchain_list_when_given(tmp_path):
home = tmp_path / "home"
(home / "tools").mkdir(parents=True)
(home / ".nvm").mkdir()
project = tmp_path / "project"
project.mkdir()
text = seatbelt_profile.render([{"path": str(project), "writable": True}], runtime_dir=str(tmp_path / "rt"), home=str(home), tool_dirs=[str(home / "tools")])
assert seatbelt_profile.real(str(home / "tools")) in text
assert seatbelt_profile.real(str(home / ".nvm")) not in text # the list given replaces the shipped one
# -- selection ------------------------------------------------------------------------
def test_an_explicit_seatbelt_setting_refuses_when_it_cannot_be_used(monkeypatch):
def broken() -> None:
raise seatbelt.SeatbeltUnavailable("The Seatbelt sandbox exists only on macOS.")
monkeypatch.delenv("OPENWORKER_SANDBOX_PROVIDER", raising=False)
monkeypatch.setattr(seatbelt, "preflight", broken)
with pytest.raises(seatbelt.SeatbeltUnavailable, match="no session will start"):
select("seatbelt")
monkeypatch.setattr(seatbelt, "preflight", lambda: None)
assert select("seatbelt") == type(select("direct"))("seatbelt", explicit=True)
# -- the wall itself (macOS only) -----------------------------------------------------
def _usable() -> bool:
if sys.platform != "darwin":
return False
try:
seatbelt.preflight()
return True
except seatbelt.SeatbeltUnavailable:
return False
live = pytest.mark.skipif(not _usable(), reason="needs a Mac where sandbox-exec can apply a profile")
@pytest.fixture
def sandbox(tmp_path, monkeypatch):
"""(workspace, project folder, read-only folder, a folder that was NOT granted)."""
from coworker.sandbox.bundle import build_runner_zipapp
from coworker.sandbox.workspace import RunnerWorkspace
monkeypatch.setenv("ANTHROPIC_API_KEY", "not-a-real-key")
project, reference, private = tmp_path / "project", tmp_path / "reference", tmp_path / "private"
for folder in (project, reference, private):
folder.mkdir()
(reference / "ref.txt").write_text("reference\n")
(private / "secret.txt").write_text("classified-content\n")
roots = [RootDir(path=project, writable=True), RootDir(path=reference, writable=False)]
provider = seatbelt.SeatbeltProvider(
roots=[{"path": str(r.path), "writable": r.writable} for r in roots],
cwd=project,
runner_path=build_runner_zipapp(tmp_path / "dist"),
)
ws = RunnerWorkspace(provider, cwd=project, live_roots=roots)
ws.executor.default_timeout = 20
yield ws, project, reference, private, roots
ws.close()
@live
def test_files_outside_the_sessions_folders_cannot_be_read_or_written(sandbox):
ws, project, reference, private, _ = sandbox
run = ws.executor.run
assert ws.describe()["enforcement"] == "full"
assert run("echo made > made.txt && cat made.txt")["output"].strip() == "made"
assert (project / "made.txt").read_text() == "made\n"
assert "reference" in run(f"cat {reference}/ref.txt")["output"]
assert run(f"echo no > {reference}/new.txt")["exit_code"] != 0
assert not (reference / "new.txt").exists()
assert "classified-content" not in run(f"cat {private}/secret.txt 2>&1")["output"]
assert run(f"echo no > {private}/new.txt")["exit_code"] != 0
home = Path.home()
assert "not permitted" in run(f"ls {home} 2>&1")["output"]
assert run(f"echo no > {home}/openworker-seatbelt-test.txt")["exit_code"] != 0
assert not (home / "openworker-seatbelt-test.txt").exists()
assert run("touch /usr/local/bin/openworker-seatbelt-test")["exit_code"] != 0
@live
def test_the_servers_secrets_do_not_reach_a_command(sandbox):
ws, *_ = sandbox
out = ws.executor.run("env")["output"]
assert "not-a-real-key" not in out and "ANTHROPIC_API_KEY" not in out
assert "COWORKER_" not in out
@live
def test_the_network_is_only_the_proxy(sandbox):
"""No internet needed: a listener on this machine stands in for 'somewhere else'."""
ws, *_ = sandbox
elsewhere = socket.socket()
elsewhere.bind(("127.0.0.1", 0))
elsewhere.listen(1)
try:
port = elsewhere.getsockname()[1]
direct = ws.executor.run(f"curl -sS -m 5 --noproxy '*' http://127.0.0.1:{port}/ 2>&1")
assert direct["exit_code"] != 0 # the kernel refuses the connection
refused = ws.executor.run("curl -sS -m 5 https://example.com 2>&1")
assert "403" in refused["output"]
finally:
elsewhere.close()
@live
def test_our_own_tools_run_behind_the_wall(sandbox):
ws, project, reference, private, roots = sandbox
ctx = AgentContext(workspace=project, executor=ws.executor, roots=roots, sandbox=ws)
tools = {t.__name__: t for t in catalog.expand(["code_files", "search"], ctx)}
tools["write_file"]("notes.md", "one\ntwo\n")
assert (project / "notes.md").read_text() == "one\ntwo\n"
assert "two" in tools["read_file"]("notes.md")["content"]
assert "reference" in tools["read_file"](str(reference / "ref.txt"))["content"]
with pytest.raises(PermissionError):
tools["write_file"](str(reference / "new.txt"), "no")
@live
def test_granting_a_folder_restarts_the_sandbox_and_says_so(sandbox):
from coworker.sandbox.workspace import RESTART_NOTICE
ws, project, reference, private, roots = sandbox
run = ws.executor.run
run("export KEEP=yes")
assert "classified-content" not in run(f"cat {private}/secret.txt 2>&1")["output"]
before = ws.client.restarts
roots.append(RootDir(path=private, writable=True))
granted = run(f"cat {private}/secret.txt; echo KEEP=$KEEP")
assert "classified-content" in granted["output"]
assert granted["sandbox_notice"] == RESTART_NOTICE
assert "KEEP=yes" not in granted["output"] # a new shell, as the notice says
assert ws.client.restarts == before + 1
assert "sandbox_notice" not in run("true") # said once
roots.pop() # taken away again: the wall goes back up
assert "classified-content" not in run(f"cat {private}/secret.txt 2>&1")["output"]