Skip to content
Python

OpenShell: keep a session's private folder under the state dir, not /tmp (OPE-208)

Devika Verma0bb74aae24eb
diff --git a/coworker/sandbox/providers/openshell.py b/coworker/sandbox/providers/openshell.py
index 10749602..05debc3d 100644
--- a/coworker/sandbox/providers/openshell.py
+++ b/coworker/sandbox/providers/openshell.py
@@ -33,6 +33,7 @@ from typing import Any, Callable, Optional, Sequence
import yaml
+from ...secrets import state_dir
from ..bundle import build_runner_zipapp
from ..transport import Transport
from .. import credentials as creds
@@ -157,6 +158,41 @@ def image_present(image: Optional[str] = None) -> Optional[bool]:
return done.returncode == 0
+RUNTIME_DIR_NAME = "sandbox-runtime" # a sibling of `sandbox/`, which is mounted into sandboxes
+
+
+def runtime_dir() -> Path:
+ """Where a session's private folder lives: on the real filesystem (the gateway can see
+ it), owner-only, and mounted into no sandbox as a whole."""
+ path = state_dir() / RUNTIME_DIR_NAME
+ path.mkdir(parents=True, exist_ok=True)
+ try:
+ os.chmod(path, 0o700)
+ except OSError:
+ pass
+ return path
+
+
+def reap_runtime_dirs(alive: Callable[[int], bool]) -> list[str]:
+ """Remove the private folders of servers that are gone (`ow-openshell-<pid>-...`).
+ Their sandboxes are removed by the registry's reap; this is the matching sweep for the
+ copied credentials, which the OS would have cleared from /tmp and no longer does."""
+ removed: list[str] = []
+ try:
+ entries = list(runtime_dir().iterdir())
+ except OSError:
+ return removed
+ for entry in entries:
+ parts = entry.name.split("-")
+ if len(parts) < 4 or parts[0] != "ow" or parts[1] != "openshell" or not parts[2].isdigit():
+ continue
+ if alive(int(parts[2])):
+ continue
+ shutil.rmtree(entry, ignore_errors=True)
+ removed.append(entry.name)
+ return removed
+
+
def _gateway() -> tuple[str, Path]:
"""(host:port, folder with the client certificate) of the active gateway."""
home = Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") / "openshell"
@@ -192,7 +228,14 @@ class OpenShellProvider:
self.sandbox_name = f"ow-{uuid.uuid4().hex[:12]}"
self.sandbox_id: Optional[str] = None
self._runner = build_runner_zipapp()
- self._tmp = tempfile.mkdtemp(prefix="ow-openshell-")
+ # The session's private folder (copied credentials, the policy file). NOT under
+ # /tmp: the gateway installed by NVIDIA's installer is a systemd service with
+ # PrivateTmp=true, so a bind-mount source under /tmp does not exist for it and the
+ # sandbox fails to create (OPE-208). NOT under the state dir's `sandbox/` either:
+ # that whole folder is mounted read-only into every sandbox for the runner, and
+ # copied keys must never be readable from another session's sandbox. The server's
+ # pid is in the name so the registry's reap can remove folders of a dead server.
+ self._tmp = tempfile.mkdtemp(prefix=f"ow-openshell-{os.getpid()}-", dir=runtime_dir())
self.grants = list(credentials)
self.copied: Optional[creds.CopiedCredentials] = None
diff --git a/coworker/sandbox/registry.py b/coworker/sandbox/registry.py
index c5b3db2d..b301b271 100644
--- a/coworker/sandbox/registry.py
+++ b/coworker/sandbox/registry.py
@@ -164,6 +164,13 @@ class SandboxRegistry:
removed.append(name)
except Exception:
pass
+ if _openshell_present():
+ from .providers import openshell
+
+ try:
+ openshell.reap_runtime_dirs(_alive) # copied credentials of dead servers
+ except Exception:
+ pass
for row in dead:
self.close(row["name"])
if row["name"] not in removed and row["provider"] != "openshell":
diff --git a/docs/openshell.md b/docs/openshell.md
index 19fe72eb..c1132b90 100644
--- a/docs/openshell.md
+++ b/docs/openshell.md
@@ -135,7 +135,9 @@ You can add your own entries (a name, a path under your home folder, the hosts i
An enabled entry is **copied** into a private home that is mounted into the sandbox when
it starts, owner-only, and deleted with the sandbox; the real files are never opened for
-writing. For `ssh`, the copy is wired so that `ssh` and `git` inside use the copied keys
+writing. The copy lives under `~/.config/coworker/sandbox-runtime/`, never under `/tmp`:
+the gateway NVIDIA's installer sets up runs with a private `/tmp` and could not mount a
+folder there. For `ssh`, the copy is wired so that `ssh` and `git` inside use the copied keys
and known hosts, with no agent. Connectors always run in OpenWorker itself, outside every
sandbox, with their own tokens.
diff --git a/tests/test_sandbox_openshell.py b/tests/test_sandbox_openshell.py
index 4ad77ac5..cc3fb23b 100644
--- a/tests/test_sandbox_openshell.py
+++ b/tests/test_sandbox_openshell.py
@@ -8,6 +8,7 @@ messages. The second half needs a machine with OpenShell running and is skipped
from __future__ import annotations
import os
+import shutil
import subprocess
import threading
import time
@@ -220,6 +221,34 @@ def test_a_workspace_reserves_its_name_before_creating_the_sandbox(tmp_path):
assert seen["destroyed"] and reg.find("s9") is None # a failed create leaves no reservation behind
+def test_the_private_folder_is_visible_to_the_gateway_and_to_no_sandbox(tmp_path, monkeypatch):
+ # OPE-208: copied credentials used to go under /tmp, which the installer's gateway
+ # (PrivateTmp=true) cannot see, so a session with a shared credential failed to create.
+ # The folder is now under the state dir, in a sibling of `sandbox/`: `sandbox/` itself
+ # is mounted read-only into every sandbox, so a copy there would leak across sessions.
+ from coworker.sandbox import bundle
+
+ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path))
+ monkeypatch.setattr(bundle, "build_runner_zipapp", lambda: tmp_path / "sandbox" / "runner-x.pyz")
+ monkeypatch.setattr(openshell, "build_runner_zipapp", lambda: tmp_path / "sandbox" / "runner-x.pyz")
+ provider = openshell.OpenShellProvider(roots=ROOTS)
+ private = Path(provider._tmp)
+ assert private.parent == tmp_path / openshell.RUNTIME_DIR_NAME
+ assert not str(private).startswith(str(tmp_path / "sandbox") + os.sep) # not inside the mounted folder
+ assert not str(private).startswith("/tmp")
+ assert private.name.startswith(f"ow-openshell-{os.getpid()}-")
+ if os.name != "nt":
+ assert (private.parent.stat().st_mode & 0o777) == 0o700
+ # A dead server's folders are swept; a live one's are kept.
+ stale = private.parent / "ow-openshell-424242-abc"
+ stale.mkdir()
+ assert openshell.reap_runtime_dirs(lambda pid: pid == os.getpid()) == [stale.name]
+ assert private.exists() and not stale.exists()
+ provider.destroy = lambda: shutil.rmtree(private, ignore_errors=True) # no gateway here
+ provider.destroy()
+ assert not private.exists()
+
+
def test_registry_reaps_rows_of_a_server_that_is_gone(tmp_path, monkeypatch):
from coworker.sandbox import registry as registry_mod