Python
Reserve a sandbox's registry row before creating it, so a concurrent build's reap() cannot delete it mid-provisioning
Devika Vermae1957477f7a5
diff --git a/coworker/sandbox/providers/openshell.py b/coworker/sandbox/providers/openshell.pyindex a76f9da8..809fd542 100644--- a/coworker/sandbox/providers/openshell.py+++ b/coworker/sandbox/providers/openshell.py@@ -29,7 +29,7 @@ import threadingimport timeimport uuidfrom pathlib import Path-from typing import Any, Optional, Sequence+from typing import Any, Callable, Optional, Sequenceimport yaml@@ -284,13 +284,17 @@ class OpenShellProvider:if "entries" not in info:raise RuntimeError(f"the folder {root['path']} is not reachable inside the sandbox")- def regrant(self, roots: Sequence[dict[str, Any]]) -> None:+ def regrant(self, roots: Sequence[dict[str, Any]], *, before_create: Optional[Callable[[], None]] = None) -> None:"""The session's folders changed. Mounts and the file policy are fixed when a sandbox- is created, so this one is deleted and a new one is created with the new folders."""+ is created, so this one is deleted and a new one is created with the new folders.+ `before_create` runs once the new name is chosen and before the sandbox exists: the+ workspace reserves the name in the registry there (see RunnerWorkspace)."""self._delete()self.roots = [{"path": str(Path(r["path"]).expanduser().resolve()), "writable": bool(r.get("writable"))} for r in roots]self.sandbox_name = f"ow-{uuid.uuid4().hex[:12]}"self.sandbox_id = None+ if before_create is not None:+ before_create()self.create()def _delete(self) -> None:diff --git a/coworker/sandbox/providers/seatbelt.py b/coworker/sandbox/providers/seatbelt.pyindex ba84d2b8..e5ab2d15 100644--- a/coworker/sandbox/providers/seatbelt.py+++ b/coworker/sandbox/providers/seatbelt.py@@ -26,7 +26,7 @@ import tempfileimport timeimport uuidfrom pathlib import Path-from typing import Any, Optional, Sequence+from typing import Any, Callable, Optional, Sequencefrom .. import credentials as credsfrom .. import netproxy, network_profiles@@ -211,11 +211,15 @@ class SeatbeltProvider:returnraise SeatbeltUnavailable("the sandbox did not take effect: the home folder can be listed from inside")- def regrant(self, roots: Sequence[dict[str, Any]]) -> None:+ def regrant(self, roots: Sequence[dict[str, Any]], *, before_create: Optional[Callable[[], None]] = None) -> None:"""The session's folders changed. A profile is fixed when a process starts, so the- daemon is started again with a new one; the client sees a runner restart."""+ daemon is started again with a new one; the client sees a runner restart.+ `before_create` is the workspace's registry reservation (a no-op here: Seatbelt has+ no gateway-side list for a reap to consult, but the hook keeps providers alike)."""self.roots = [{"path": seatbelt_profile.real(r["path"]), "writable": bool(r.get("writable"))} for r in roots]self._stop_daemon()+ if before_create is not None:+ before_create()self.create()def restart_daemon(self) -> None:diff --git a/coworker/sandbox/registry.py b/coworker/sandbox/registry.pyindex 490fb77b..d5ba05f5 100644--- a/coworker/sandbox/registry.py+++ b/coworker/sandbox/registry.py@@ -90,12 +90,16 @@ class SandboxRegistry:profile: str = "",enforcement: str = "",machine_id: str = LOCAL_MACHINE,+ state: str = "ready",) -> None:+ """`state`: "creating" reserves the name BEFORE the sandbox exists, so that another+ engine build's `reap()` (they run concurrently since OPE-206) does not take the+ half-built sandbox for an orphan and delete it; "ready" once it is connected."""now = time.time()with self._lock, self._connect() as db:db.execute("INSERT OR REPLACE INTO sandboxes VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",- (name, provider, machine_id, session_id, agent, json.dumps(roots or []), profile, enforcement, "ready", os.getpid(), now, now),+ (name, provider, machine_id, session_id, agent, json.dumps(roots or []), profile, enforcement, state, os.getpid(), now, now),)def touch(self, name: str) -> None:diff --git a/coworker/sandbox/workspace.py b/coworker/sandbox/workspace.pyindex 7b2a80f1..81f3a5e6 100644--- a/coworker/sandbox/workspace.py+++ b/coworker/sandbox/workspace.py@@ -83,10 +83,16 @@ class RunnerWorkspace(Workspace):if registry is not None:registry.reap() # sandboxes left behind by a server that is goneregistry.check_room()+ # Reserve the name before the sandbox exists: engine builds run concurrently+ # (OPE-206), and another build's reap() would otherwise delete this one while+ # it is still provisioning, because it is not yet in the registry.+ self._record(state="creating")try:provider.create()except Exception:provider.destroy() # a half-made sandbox may already hold copied credentials+ if registry is not None and self._registered:+ registry.close(self._registered)raisetry:self.client = RunnerClient(provider.open_runner)@@ -102,9 +108,9 @@ class RunnerWorkspace(Workspace):)self._record()- def _record(self) -> None:+ def _record(self, state: str = "ready") -> None:"""Enter this sandbox in the registry; after a restart that replaced it, under its- new name."""+ new name. `state="creating"` reserves the name before the sandbox exists."""if self.registry is None:returninfo = self.provider.describe()@@ -120,6 +126,7 @@ class RunnerWorkspace(Workspace):roots=getattr(self.provider, "roots", None),profile=getattr(self.provider, "profile", ""),enforcement=info.get("enforcement", ""),+ state=state,)def sync_roots(self) -> Optional[str]:@@ -132,7 +139,9 @@ class RunnerWorkspace(Workspace):if not wanted or _same_roots(wanted, getattr(self.provider, "roots", [])):return Noneself.client.detach()- regrant(wanted)+ # Reserve the replacement's name before it exists, for the same reason as at start:+ # a concurrent build's reap() must not take it for an orphan.+ regrant(wanted, before_create=lambda: self._record(state="creating"))self.client.connect() # a new runner: the client notes the restartverify = getattr(self.provider, "verify", None)if verify is not None:diff --git a/tests/test_sandbox_openshell.py b/tests/test_sandbox_openshell.pyindex 44ab51ea..4ad77ac5 100644--- a/tests/test_sandbox_openshell.py+++ b/tests/test_sandbox_openshell.py@@ -176,6 +176,50 @@ def test_registry_counts_caps_and_forgets(tmp_path, monkeypatch):assert [r["name"] for r in reg.list()] == ["ow-b"]+def test_reap_leaves_a_sandbox_that_a_living_server_is_still_creating(tmp_path, monkeypatch):+ # Engine builds run concurrently (OPE-206). A sandbox that exists at the gateway but was+ # not yet connected used to look like an orphan to another build's reap() and got+ # deleted mid-provisioning ("The operation was cancelled: stream closed"). The name is+ # now reserved in the registry, state "creating", before the sandbox is created.+ from coworker.sandbox import registry as registry_mod++ reg = registry_mod.SandboxRegistry(tmp_path / "registry.db")+ reg.record("ow-building", provider="openshell", session_id="s1", state="creating") # this server, alive+ monkeypatch.setattr(registry_mod, "_openshell_present", lambda: True)+ monkeypatch.setattr(openshell, "list_our_sandboxes", lambda: [{"name": "ow-building"}, {"name": "ow-orphan"}])+ deleted: list[str] = []+ monkeypatch.setattr(openshell, "_cli", lambda *args, **kw: deleted.append(args[2]))+ assert reg.reap() == ["ow-orphan"]+ assert deleted == ["ow-orphan"] # the one being built by a live server is left alone+ assert reg.find("s1")["state"] == "creating"+++def test_a_workspace_reserves_its_name_before_creating_the_sandbox(tmp_path):+ from coworker.sandbox import registry as registry_mod+ from coworker.sandbox.workspace import RunnerWorkspace++ reg = registry_mod.SandboxRegistry(tmp_path / "registry.db")+ seen: dict = {}++ class Provider:+ roots, profile = [{"path": str(tmp_path), "writable": True}], "strict"++ def describe(self):+ return {"provider": "openshell", "enforcement": "full", "sandbox": "ow-reserved"}++ def create(self):+ seen["row_at_create"] = reg.find("s9") # what reap() would see while we provision+ raise RuntimeError("create failed on purpose")++ def destroy(self):+ seen["destroyed"] = True++ with pytest.raises(RuntimeError, match="on purpose"):+ RunnerWorkspace(Provider(), cwd=tmp_path, registry=reg, session_id="s9", agent="lead")+ assert seen["row_at_create"]["name"] == "ow-reserved" and seen["row_at_create"]["state"] == "creating"+ assert seen["destroyed"] and reg.find("s9") is None # a failed create leaves no reservation behind++def test_registry_reaps_rows_of_a_server_that_is_gone(tmp_path, monkeypatch):from coworker.sandbox import registry as registry_mod