Skip to content
Python

Reserve a sandbox's registry row before creating it, so a concurrent build's reap() cannot delete it mid-provisioning

Devika Vermae1957477f7a5
diff --git a/coworker/sandbox/providers/openshell.py b/coworker/sandbox/providers/openshell.py
index a76f9da8..809fd542 100644
--- a/coworker/sandbox/providers/openshell.py
+++ b/coworker/sandbox/providers/openshell.py
@@ -29,7 +29,7 @@ import threading
import time
import uuid
from pathlib import Path
-from typing import Any, Optional, Sequence
+from typing import Any, Callable, Optional, Sequence
import yaml
@@ -284,13 +284,17 @@ class OpenShellProvider:
if "entries" not in info:
raise RuntimeError(f"the folder {root['path']} is not reachable inside the sandbox")
- def regrant(self, roots: Sequence[dict[str, Any]]) -> None:
+ def regrant(self, roots: Sequence[dict[str, Any]], *, before_create: Optional[Callable[[], None]] = None) -> None:
"""The session's folders changed. Mounts and the file policy are fixed when a sandbox
- is created, so this one is deleted and a new one is created with the new folders."""
+ is created, so this one is deleted and a new one is created with the new folders.
+ `before_create` runs once the new name is chosen and before the sandbox exists: the
+ workspace reserves the name in the registry there (see RunnerWorkspace)."""
self._delete()
self.roots = [{"path": str(Path(r["path"]).expanduser().resolve()), "writable": bool(r.get("writable"))} for r in roots]
self.sandbox_name = f"ow-{uuid.uuid4().hex[:12]}"
self.sandbox_id = None
+ if before_create is not None:
+ before_create()
self.create()
def _delete(self) -> None:
diff --git a/coworker/sandbox/providers/seatbelt.py b/coworker/sandbox/providers/seatbelt.py
index ba84d2b8..e5ab2d15 100644
--- a/coworker/sandbox/providers/seatbelt.py
+++ b/coworker/sandbox/providers/seatbelt.py
@@ -26,7 +26,7 @@ import tempfile
import time
import uuid
from pathlib import Path
-from typing import Any, Optional, Sequence
+from typing import Any, Callable, Optional, Sequence
from .. import credentials as creds
from .. import netproxy, network_profiles
@@ -211,11 +211,15 @@ class SeatbeltProvider:
return
raise SeatbeltUnavailable("the sandbox did not take effect: the home folder can be listed from inside")
- def regrant(self, roots: Sequence[dict[str, Any]]) -> None:
+ def regrant(self, roots: Sequence[dict[str, Any]], *, before_create: Optional[Callable[[], None]] = None) -> None:
"""The session's folders changed. A profile is fixed when a process starts, so the
- daemon is started again with a new one; the client sees a runner restart."""
+ daemon is started again with a new one; the client sees a runner restart.
+ `before_create` is the workspace's registry reservation (a no-op here: Seatbelt has
+ no gateway-side list for a reap to consult, but the hook keeps providers alike)."""
self.roots = [{"path": seatbelt_profile.real(r["path"]), "writable": bool(r.get("writable"))} for r in roots]
self._stop_daemon()
+ if before_create is not None:
+ before_create()
self.create()
def restart_daemon(self) -> None:
diff --git a/coworker/sandbox/registry.py b/coworker/sandbox/registry.py
index 490fb77b..d5ba05f5 100644
--- a/coworker/sandbox/registry.py
+++ b/coworker/sandbox/registry.py
@@ -90,12 +90,16 @@ class SandboxRegistry:
profile: str = "",
enforcement: str = "",
machine_id: str = LOCAL_MACHINE,
+ state: str = "ready",
) -> None:
+ """`state`: "creating" reserves the name BEFORE the sandbox exists, so that another
+ engine build's `reap()` (they run concurrently since OPE-206) does not take the
+ half-built sandbox for an orphan and delete it; "ready" once it is connected."""
now = time.time()
with self._lock, self._connect() as db:
db.execute(
"INSERT OR REPLACE INTO sandboxes VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
- (name, provider, machine_id, session_id, agent, json.dumps(roots or []), profile, enforcement, "ready", os.getpid(), now, now),
+ (name, provider, machine_id, session_id, agent, json.dumps(roots or []), profile, enforcement, state, os.getpid(), now, now),
)
def touch(self, name: str) -> None:
diff --git a/coworker/sandbox/workspace.py b/coworker/sandbox/workspace.py
index 7b2a80f1..81f3a5e6 100644
--- a/coworker/sandbox/workspace.py
+++ b/coworker/sandbox/workspace.py
@@ -83,10 +83,16 @@ class RunnerWorkspace(Workspace):
if registry is not None:
registry.reap() # sandboxes left behind by a server that is gone
registry.check_room()
+ # Reserve the name before the sandbox exists: engine builds run concurrently
+ # (OPE-206), and another build's reap() would otherwise delete this one while
+ # it is still provisioning, because it is not yet in the registry.
+ self._record(state="creating")
try:
provider.create()
except Exception:
provider.destroy() # a half-made sandbox may already hold copied credentials
+ if registry is not None and self._registered:
+ registry.close(self._registered)
raise
try:
self.client = RunnerClient(provider.open_runner)
@@ -102,9 +108,9 @@ class RunnerWorkspace(Workspace):
)
self._record()
- def _record(self) -> None:
+ def _record(self, state: str = "ready") -> None:
"""Enter this sandbox in the registry; after a restart that replaced it, under its
- new name."""
+ new name. `state="creating"` reserves the name before the sandbox exists."""
if self.registry is None:
return
info = self.provider.describe()
@@ -120,6 +126,7 @@ class RunnerWorkspace(Workspace):
roots=getattr(self.provider, "roots", None),
profile=getattr(self.provider, "profile", ""),
enforcement=info.get("enforcement", ""),
+ state=state,
)
def sync_roots(self) -> Optional[str]:
@@ -132,7 +139,9 @@ class RunnerWorkspace(Workspace):
if not wanted or _same_roots(wanted, getattr(self.provider, "roots", [])):
return None
self.client.detach()
- regrant(wanted)
+ # Reserve the replacement's name before it exists, for the same reason as at start:
+ # a concurrent build's reap() must not take it for an orphan.
+ regrant(wanted, before_create=lambda: self._record(state="creating"))
self.client.connect() # a new runner: the client notes the restart
verify = getattr(self.provider, "verify", None)
if verify is not None:
diff --git a/tests/test_sandbox_openshell.py b/tests/test_sandbox_openshell.py
index 44ab51ea..4ad77ac5 100644
--- a/tests/test_sandbox_openshell.py
+++ b/tests/test_sandbox_openshell.py
@@ -176,6 +176,50 @@ def test_registry_counts_caps_and_forgets(tmp_path, monkeypatch):
assert [r["name"] for r in reg.list()] == ["ow-b"]
+def test_reap_leaves_a_sandbox_that_a_living_server_is_still_creating(tmp_path, monkeypatch):
+ # Engine builds run concurrently (OPE-206). A sandbox that exists at the gateway but was
+ # not yet connected used to look like an orphan to another build's reap() and got
+ # deleted mid-provisioning ("The operation was cancelled: stream closed"). The name is
+ # now reserved in the registry, state "creating", before the sandbox is created.
+ from coworker.sandbox import registry as registry_mod
+
+ reg = registry_mod.SandboxRegistry(tmp_path / "registry.db")
+ reg.record("ow-building", provider="openshell", session_id="s1", state="creating") # this server, alive
+ monkeypatch.setattr(registry_mod, "_openshell_present", lambda: True)
+ monkeypatch.setattr(openshell, "list_our_sandboxes", lambda: [{"name": "ow-building"}, {"name": "ow-orphan"}])
+ deleted: list[str] = []
+ monkeypatch.setattr(openshell, "_cli", lambda *args, **kw: deleted.append(args[2]))
+ assert reg.reap() == ["ow-orphan"]
+ assert deleted == ["ow-orphan"] # the one being built by a live server is left alone
+ assert reg.find("s1")["state"] == "creating"
+
+
+def test_a_workspace_reserves_its_name_before_creating_the_sandbox(tmp_path):
+ from coworker.sandbox import registry as registry_mod
+ from coworker.sandbox.workspace import RunnerWorkspace
+
+ reg = registry_mod.SandboxRegistry(tmp_path / "registry.db")
+ seen: dict = {}
+
+ class Provider:
+ roots, profile = [{"path": str(tmp_path), "writable": True}], "strict"
+
+ def describe(self):
+ return {"provider": "openshell", "enforcement": "full", "sandbox": "ow-reserved"}
+
+ def create(self):
+ seen["row_at_create"] = reg.find("s9") # what reap() would see while we provision
+ raise RuntimeError("create failed on purpose")
+
+ def destroy(self):
+ seen["destroyed"] = True
+
+ with pytest.raises(RuntimeError, match="on purpose"):
+ RunnerWorkspace(Provider(), cwd=tmp_path, registry=reg, session_id="s9", agent="lead")
+ assert seen["row_at_create"]["name"] == "ow-reserved" and seen["row_at_create"]["state"] == "creating"
+ assert seen["destroyed"] and reg.find("s9") is None # a failed create leaves no reservation behind
+
+
def test_registry_reaps_rows_of_a_server_that_is_gone(tmp_path, monkeypatch):
from coworker.sandbox import registry as registry_mod