Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: harden the tool sandbox around host launches, probes and workdir paths (#14429)

Daniel Han96e7534c8bc4
diff --git a/studio/backend/core/inference/os_sandbox.py b/studio/backend/core/inference/os_sandbox.py
index fb8263e5322..2f32222229b 100644
--- a/studio/backend/core/inference/os_sandbox.py
+++ b/studio/backend/core/inference/os_sandbox.py
@@ -462,7 +462,12 @@ def clear_stale_tool_ipc(workdir: str, deadline: "float | None" = None) -> tuple
return ()
if not _looks_like_our_scratch_dir(scratch):
return ()
- for base, dirs, names in os.walk(scratch, followlinks = False):
+ # By directory fd: the jail can swap a directory for a link mid-sweep. Windows has no fwalk.
+ if hasattr(os, "fwalk"):
+ walk = os.fwalk(scratch, follow_symlinks = False)
+ else:
+ walk = ((*step, None) for step in os.walk(scratch, followlinks = False))
+ for base, dirs, names, dirfd in walk:
dirs[:] = [name for name in dirs if not os.path.ismount(os.path.join(base, name))]
for name in names:
if deadline is not None and time.monotonic() > deadline:
@@ -470,7 +475,7 @@ def clear_stale_tool_ipc(workdir: str, deadline: "float | None" = None) -> tuple
return tuple(removed)
path = os.path.join(base, name)
try:
- entry = os.lstat(path)
+ entry = os.lstat(path) if dirfd is None else os.lstat(name, dir_fd = dirfd)
except OSError:
continue
mode = entry.st_mode
@@ -485,7 +490,14 @@ def clear_stale_tool_ipc(workdir: str, deadline: "float | None" = None) -> tuple
)
continue
try:
- os.unlink(path)
+ if dirfd is None:
+ os.unlink(path)
+ else:
+ # The probe went by path: unlink only the entry the walk saw.
+ again = os.lstat(name, dir_fd = dirfd)
+ if (again.st_dev, again.st_ino) != (entry.st_dev, entry.st_ino):
+ continue
+ os.unlink(name, dir_fd = dirfd)
except OSError:
continue
removed.append(path)
@@ -929,9 +941,20 @@ def capability_snapshot(
identity,
)
- from .sandbox_probe import probe
+ from .sandbox_probe import UndecidedReason, probe
available, reason = probe(backend, force = force)
+ if not available and isinstance(reason, UndecidedReason):
+ # Tool code can exhaust the host; that says nothing about the backend, so auto must not fall back on it.
+ return replace(
+ _unavailable(
+ reason,
+ "The sandbox check ran out of host resources (disk space, memory or open files). "
+ "Free them and try again.",
+ identity,
+ ),
+ protection_state = "undecided",
+ )
if not available:
remediation = (
linux_unavailable_remediation()
@@ -1232,6 +1255,12 @@ def prepare_tool_launch(plan: ToolLaunchPlan) -> PreparedSandboxLaunch:
raise SandboxBuildError("execution cancelled before sandbox launch")
if not capability.available:
+ if capability.protection_state == "undecided":
+ # Typed as a build failure: tools.py re-raises it rather than falling back to the host.
+ raise SandboxBuildError(
+ f"OS_ISOLATION_UNAVAILABLE: {capability.reason}",
+ remediation = capability.remediation,
+ )
if plan.requested_mode == "required":
raise SandboxUnavailableError(
f"OS_ISOLATION_UNAVAILABLE: {capability.reason}",
diff --git a/studio/backend/core/inference/sandbox_linux.py b/studio/backend/core/inference/sandbox_linux.py
index b18fc484091..6e801ff5777 100644
--- a/studio/backend/core/inference/sandbox_linux.py
+++ b/studio/backend/core/inference/sandbox_linux.py
@@ -843,11 +843,18 @@ def _model_cache_binds(workdir: str) -> dict[str, str]:
"""Inner cache subdirectory -> host directory, per component (HF_HUB_CACHE=/mnt/models is NOT /mnt/models/hub)."""
binds: dict[str, str] = {}
try:
- from utils.hf_cache_settings import get_hf_cache_paths
+ from utils.hf_cache_settings import effective_cache_home, get_hf_cache_paths
paths = get_hf_cache_paths()
- home = os.path.abspath(str(paths.cache_home))
- resolved = {"hub": str(paths.hub_cache), "xet": str(paths.xet_cache)}
+ # datasets/assets live under the real HF home, not the display home (which can be the hub itself).
+ home = os.path.abspath(str(effective_cache_home()))
+ resolved = {
+ "hub": str(paths.hub_cache),
+ "xet": str(paths.xet_cache),
+ # Portable installs point these at their own root while HF_HOME stays the default.
+ "datasets": os.path.expanduser(os.environ.get("HF_DATASETS_CACHE", "").strip()),
+ "assets": os.path.expanduser(os.environ.get("HF_ASSETS_CACHE", "").strip()),
+ }
except Exception: # noqa: BLE001 - a launch never fails over a cache lookup
logger.debug("could not resolve the configured Hugging Face cache", exc_info = True)
user_home = os.path.expanduser("~")
@@ -983,7 +990,12 @@ def prepare(plan: ToolLaunchPlan) -> PreparedSandboxLaunch:
"ALL",
"--seccomp",
str(seccomp.fileno()),
- *(("--tmpfs", "/proc") if empty_proc else ("--proc", "/proc")),
+ # A root-run Studio leaves the tool at uid 0: host sysctls stay read-only. The empty layout keeps /proc empty.
+ *(
+ ("--tmpfs", "/proc")
+ if empty_proc
+ else ("--proc", "/proc", "--ro-bind-try", "/proc/sys", "/proc/sys")
+ ),
"--dev",
"/dev",
"--dir",
diff --git a/studio/backend/core/inference/sandbox_probe.py b/studio/backend/core/inference/sandbox_probe.py
index 21bfe480520..8538156a4e2 100644
--- a/studio/backend/core/inference/sandbox_probe.py
+++ b/studio/backend/core/inference/sandbox_probe.py
@@ -5,7 +5,9 @@
from __future__ import annotations
import ctypes
+import errno
import os
+import re
import shutil
import socket
import subprocess
@@ -44,6 +46,56 @@ _MAX_PROBE_BASE_LEN = 59
# A stale PASS is served while one background re-probe runs; a FAIL never is.
_STALE_GRACE_SECONDS = 3600.0
+# Tool code can exhaust these from the jail: undecided, never cached, and auto refuses rather than falls back.
+_RESOURCE_ERRNOS = frozenset(
+ code
+ for code in (
+ getattr(errno, name, None)
+ for name in ("ENOSPC", "EDQUOT", "EMFILE", "ENFILE", "ENOMEM", "ENOBUFS", "EAGAIN")
+ )
+ if code is not None
+)
+_RESOURCE_TEXTS = tuple(os.strerror(code) for code in _RESOURCE_ERRNOS)
+# bwrap reports a user-namespace limit as ENOSPC: a host without a sandbox only when the quota is zero.
+_NAMESPACE_LIMIT = re.compile(
+ r"bwrap: .*(?:namespace|unshare).*" + re.escape(os.strerror(errno.ENOSPC)), re.I
+)
+
+
+def _user_namespaces_disabled() -> bool:
+ """A zero quota is a host without a sandbox; a used-up one is exhaustion."""
+ try:
+ with open("/proc/sys/user/max_user_namespaces", encoding = "ascii") as handle:
+ return int(handle.read().strip() or "0") == 0
+ except (OSError, ValueError):
+ return True
+
+
+class UndecidedReason(str):
+ """A probe failure caused by host resource exhaustion rather than by the backend."""
+
+
+def _exhausted(exc: BaseException | None) -> bool:
+ seen = 0
+ while exc is not None and seen < 8:
+ if isinstance(exc, MemoryError) or (
+ isinstance(exc, OSError) and exc.errno in _RESOURCE_ERRNOS
+ ):
+ return True
+ exc = exc.__cause__ or exc.__context__
+ seen += 1
+ return False
+
+
+def _classified(reason: str, exc: BaseException | None = None) -> str:
+ """By errno when there is an exception; by strerror text only for the host-side control's own output."""
+ if exc is not None:
+ exhausted = _exhausted(exc)
+ else:
+ exhausted = any(text in reason for text in _RESOURCE_TEXTS)
+ return UndecidedReason(reason) if exhausted else reason
+
+
_cache_lock = threading.Lock()
_cache: dict[tuple[str, str], tuple[float, bool, str]] = {}
_refreshing: set[tuple[str, str]] = set()
@@ -121,7 +173,8 @@ def _refresh_in_background(backend: Any, key: tuple[str, str]) -> None:
try:
from .os_sandbox import ToolLaunchPlan
available, reason = _run_probe(backend, key[0], ToolLaunchPlan)
- _cache_put(key, available, reason, generation = generation)
+ if not isinstance(reason, UndecidedReason):
+ _cache_put(key, available, reason, generation = generation)
except Exception as exc: # noqa: BLE001 - the stale PASS just ages out
logger.debug("background sandbox re-probe failed: %s", exc)
finally:
@@ -312,13 +365,15 @@ def _host_positive_controls(workdir: str, sentinel: str, outside: str, env: dict
if handle.read() != _SENTINEL_TOKEN:
return "the probe sentinel did not contain what was written to it"
except OSError as exc:
- return f"the host itself could not read the probe sentinel ({exc})"
+ return _classified(f"the host itself could not read the probe sentinel ({exc})", exc)
try:
with open(outside, "w", encoding = "utf-8") as handle:
handle.write("host-write-ok")
os.unlink(outside)
except OSError as exc:
- return f"the host itself could not write outside the probe workdir ({exc})"
+ return _classified(
+ f"the host itself could not write outside the probe workdir ({exc})", exc
+ )
try:
completed = subprocess.run(
(sys.executable, "-I", "-S", "-c", _host_payload(workdir)),
@@ -330,10 +385,12 @@ def _host_positive_controls(workdir: str, sentinel: str, outside: str, env: dict
timeout = PROBE_TIMEOUT_SECONDS,
)
except (OSError, subprocess.SubprocessError) as exc:
- return f"the host itself could not run the probe's positive controls ({exc})"
+ return _classified(
+ f"the host itself could not run the probe's positive controls ({exc})", exc
+ )
if completed.returncode != 0 or completed.stdout.strip() != PROBE_TOKEN.encode():
detail = (completed.stderr or b"").decode("utf-8", "replace").strip()[-300:]
- return (
+ return _classified(
"the host itself could not pass the probe's positive controls "
f"(exit {completed.returncode}): {detail or 'no output'}"
)
@@ -363,6 +420,9 @@ def probe(backend: Any, *, force: bool = False) -> tuple[bool, str]:
with _cache_lock:
generation = _generation
available, reason = _run_probe(backend, backend_name, ToolLaunchPlan)
+ if isinstance(reason, UndecidedReason):
+ # Re-probed on the next call: the resource may be freed by then.
+ return False, reason
_cache_put(key, available, reason, generation = generation)
return available, reason
@@ -433,10 +493,14 @@ def _run_probe(backend: Any, backend_name: str, plan_cls: Any) -> tuple[bool, st
detail = (completed.stderr or b"").decode("utf-8", "replace").strip()[-300:]
if not detail:
detail = (completed.stdout or b"").decode("utf-8", "replace").strip()[-300:]
- return False, (
+ reason = (
f"the {backend_name} live probe failed "
f"(exit {completed.returncode}): {detail or 'no output'}"
)
+ namespaces_off = _NAMESPACE_LIMIT.search(detail) and _user_namespaces_disabled()
+ if any(text in detail for text in _RESOURCE_TEXTS) and not namespaces_off:
+ return False, UndecidedReason(reason)
+ return False, reason
if _host_saw_the_write(outside):
return False, (
f"the {backend_name} live probe wrote through to the host: a file created "
@@ -452,7 +516,9 @@ def _run_probe(backend: Any, backend_name: str, plan_cls: Any) -> tuple[bool, st
return False, f"the {backend_name} live probe timed out after {PROBE_TIMEOUT_SECONDS:.0f}s"
except Exception as exc: # noqa: BLE001 - see the docstring on probe()
logger.debug("sandbox probe for %s could not run", backend_name, exc_info = True)
- return False, f"the {backend_name} live probe could not run: {type(exc).__name__}: {exc}"
+ return False, _classified(
+ f"the {backend_name} live probe could not run: {type(exc).__name__}: {exc}", exc
+ )
finally:
if abstract_listener is not None:
abstract_listener.close()
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index e99e6c885bb..564ae55efd6 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -10178,15 +10178,68 @@ def _requested_execution_mode(tool_execution_mode: str, disable_sandbox: bool) -
_with_session_packages = os_sandbox.with_session_packages
-def _software_safeguards_launch(plan, fault: str):
- """Prepare an unisolated launch, retaining session packages and recording *fault*."""
+def _host_access_launch(plan, script: "str | None") -> "tuple[tuple, dict, str]":
+ """argv and env for an approved host-access launch: nothing it loads comes from the sandbox-writable workdir.
+
+ HOME and (for the python tool) the script move to a private per-call directory outside every workdir, and the
+ session packages stay off PYTHONPATH/PATH; earlier sandboxed calls can write all of those in the workdir. The
+ cwd stays the workdir, so relative paths in the approved code resolve as before.
+ """
+ private = tempfile.mkdtemp(prefix = "unsloth-host-call-")
+ try:
+ env = dict(plan.env)
+ env["HOME"] = private
+ if sys.platform == "win32":
+ env["HOMEDRIVE"], env["HOMEPATH"] = os.path.splitdrive(private)
+ argv = tuple(plan.argv)
+ if script is not None and argv:
+ # Written here, not read back from the workdir copy: a sandboxed call could have replaced that file.
+ path = os.path.join(private, os.path.basename(argv[-1]))
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ with os.fdopen(fd, "w", encoding = "utf-8") as f:
+ f.write(script)
+ packages = os.path.join(plan.workdir, os_sandbox.SESSION_PACKAGES_RELPATH)
+ if os.path.isdir(packages):
+ # Chat-installed packages go last on sys.path, so the sandbox-writable folder cannot shadow anything.
+ launcher = os.path.join(private, "unsloth_host_launch.py")
+ fd = os.open(launcher, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ with os.fdopen(fd, "w", encoding = "utf-8") as f:
+ f.write(
+ "import runpy, sys\n"
+ f"sys.path.append({packages!r})\n"
+ f"sys.argv[0] = {path!r}\n"
+ f"runpy.run_path({path!r}, run_name = '__main__')\n"
+ )
+ path = launcher
+ argv = (*argv[:-1], path)
+ # The script's own folder stays off sys.path too (CPython >= 3.11).
+ env["PYTHONSAFEPATH"] = "1"
+ return argv, env, private
+ except BaseException:
+ shutil.rmtree(private, ignore_errors = True)
+ raise
+
+
+def _software_safeguards_launch(
+ plan,
+ fault: str,
+ *,
+ off_workdir: bool = False,
+ host_script: "str | None" = None,
+):
+ """Prepare an unisolated launch, retaining session packages (unless *off_workdir*) and recording *fault*."""
full = plan.requested_mode == "full"
+ argv, env, cleanup_paths = plan.argv, None, []
+ if off_workdir:
+ argv, env, private = _host_access_launch(plan, host_script)
+ cleanup_paths.append(private)
return os_sandbox.PreparedSandboxLaunch(
- argv = plan.argv,
+ argv = argv,
workdir = plan.workdir,
- env = _with_session_packages(plan.env, plan.workdir),
+ env = env if env is not None else _with_session_packages(plan.env, plan.workdir),
preexec_fn = plan.preexec_fn,
backend = "software-safeguards",
+ cleanup_paths = cleanup_paths,
timeout_seconds = plan.timeout_seconds,
close_fds = plan.close_fds,
terminate_descendants = plan.terminate_descendants,
@@ -10239,12 +10292,19 @@ def _prepare_tool_launch(
*,
host_access_approved: bool = False,
user_folder: bool = False,
+ host_script: "str | None" = None,
):
"""Fall back only when the backend is unavailable; unsafe workdirs, build failures and (outside `full`) planner errors refuse."""
if user_folder and plan.requested_mode == "auto":
os_sandbox.recheck_overran_workdir(plan.workdir)
if host_access_approved and plan.requested_mode == "auto":
- return _software_safeguards_launch(plan, "user_approved_host_access")
+ # A chosen project folder is never a sandbox workdir in auto, so its calls keep importing from it.
+ return _software_safeguards_launch(
+ plan,
+ "user_approved_host_access",
+ off_workdir = not user_folder,
+ host_script = host_script,
+ )
# A project folder the owner picked is host access they granted: the isolation's workdir scan would refuse common
# repositories (hard links from git clone --local, pnpm, uv) and walk large trees on every call. Decided up front,
# never after a scan finding, which tool code could plant to drop its own boundary.
@@ -11072,13 +11132,26 @@ def _get_project_workdir(session_id: str) -> str | None:
return None
root_real = os.path.realpath(root_path)
sandbox_real = os.path.realpath(sandbox_path)
- if sandbox_real != root_real and not sandbox_real.startswith(root_real + os.sep):
+ # Same rule as _get_workdir: a link left by tool code is dropped, and the root itself is never the workdir.
+ if (
+ os.path.islink(sandbox_path)
+ or _is_windows_reparse_point(sandbox_path)
+ or not sandbox_real.startswith(root_real + os.sep)
+ ):
return None
from storage.studio_db import available_project_working_folder
return available_project_working_folder(project) or sandbox_real
+def _is_windows_reparse_point(path: str) -> bool:
+ """A Windows junction, which ``os.path.islink`` does not report."""
+ try:
+ return bool(getattr(os.lstat(path), "st_file_attributes", 0) & 0x400)
+ except OSError:
+ return False
+
+
# Dropped in every session directory we create. The root can be an existing shared folder the user pointed us at, and
# a chat id can name something already in there; this is the only evidence the directory is ours to delete.
_SANDBOX_MARKER = ".unsloth_sandbox"
@@ -23794,6 +23867,7 @@ def _python_exec(
),
host_access_approved = host_access_approved and _reaches_host_paths("python", code),
user_folder = _is_user_project_folder(session_id, workdir),
+ host_script = code,
)
proc = os_sandbox.spawn_prepared_launch(
prepared, **_apply_prepared_launch(prepared, popen_kwargs)
diff --git a/studio/backend/tests/test_sandbox_linux.py b/studio/backend/tests/test_sandbox_linux.py
index 2dbc38ed9a9..176fe609470 100644
--- a/studio/backend/tests/test_sandbox_linux.py
+++ b/studio/backend/tests/test_sandbox_linux.py
@@ -1316,7 +1316,11 @@ def _share_cache_paths(monkeypatch, cache):
module = types.ModuleType("utils.hf_cache_settings")
module.get_hf_cache_paths = lambda: _Paths()
+ module.effective_cache_home = lambda: cache
monkeypatch.setitem(sys.modules, "utils.hf_cache_settings", module)
+ # The resolver prefers these over cache_home; a caller's own settings must not leak in.
+ monkeypatch.delenv("HF_DATASETS_CACHE", raising = False)
+ monkeypatch.delenv("HF_ASSETS_CACHE", raising = False)
def test_a_symlinked_runtime_entry_inside_the_workdir_is_protected_by_name(tmp_path, monkeypatch):
@@ -1558,11 +1562,15 @@ def test_the_cache_studio_actually_uses_is_the_one_shared(tmp_path, monkeypatch)
path.mkdir(parents = True)
import utils.hf_cache_settings as cache_settings
+ # datasets/assets follow the real HF home, which a Studio-selected folder leaves in place.
+ monkeypatch.setenv("HF_HOME", str(home))
+ monkeypatch.delenv("HF_DATASETS_CACHE", raising = False)
+ monkeypatch.delenv("HF_ASSETS_CACHE", raising = False)
monkeypatch.setattr(
cache_settings,
"get_hf_cache_paths",
lambda: cache_settings.HuggingFaceCachePaths(
- cache_home = home, hub_cache = hub, xet_cache = xet, source = "studio"
+ cache_home = hub.parent, hub_cache = hub, xet_cache = xet, source = "studio"
),
)
binds = sandbox_linux._model_cache_binds(str(tmp_path / "session"))
@@ -1579,7 +1587,10 @@ def _real_cache(monkeypatch, home):
paths = types.SimpleNamespace(cache_home = home, hub_cache = home / "hub", xet_cache = home / "xet")
module = types.ModuleType("utils.hf_cache_settings")
module.get_hf_cache_paths = lambda: paths
+ module.effective_cache_home = lambda: home
monkeypatch.setitem(sys.modules, "utils.hf_cache_settings", module)
+ monkeypatch.delenv("HF_DATASETS_CACHE", raising = False)
+ monkeypatch.delenv("HF_ASSETS_CACHE", raising = False)
def test_a_cache_component_holding_an_ipc_node_is_not_shared(tmp_path, monkeypatch):
@@ -2051,6 +2062,7 @@ def test_the_cache_bind_itself_drops_a_component_holding_studio_state(monkeypatc
hub_cache = str(state),
xet_cache = str(elsewhere / "xet"),
)
+ settings.effective_cache_home = lambda: elsewhere
monkeypatch.setitem(sys.modules, "utils.hf_cache_settings", settings)
monkeypatch.setattr(sandbox_linux, "_cache_hazard_within_deadline", lambda name, path: None)
@@ -2080,6 +2092,7 @@ def test_a_cache_configured_as_a_home_directory_is_not_shared(monkeypatch, tmp_p
hub_cache = str(home),
xet_cache = str(models / "xet"),
)
+ settings.effective_cache_home = lambda: models
monkeypatch.setitem(sys.modules, "utils.hf_cache_settings", settings)
monkeypatch.setattr(sandbox_linux, "_cache_hazard_within_deadline", lambda name, path: None)
diff --git a/studio/backend/tests/test_tool_sandbox_hardening.py b/studio/backend/tests/test_tool_sandbox_hardening.py
new file mode 100644
index 00000000000..a6da8626563
--- /dev/null
+++ b/studio/backend/tests/test_tool_sandbox_hardening.py
@@ -0,0 +1,443 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Nothing the sandboxed tool code can write decides what runs, or where, outside the sandbox."""
+
+from __future__ import annotations
+
+import errno
+import os
+import socket
+import subprocess
+import sys
+
+import pytest
+
+from core.inference import os_sandbox, sandbox_probe, tools
+from core.inference.os_sandbox import ToolLaunchPlan
+
+_POSIX = pytest.mark.skipif(sys.platform == "win32", reason = "POSIX launch layout")
+
+
+def _planted_workdir(tmp_path):
+ workdir = tmp_path / "chat"
+ packages = workdir / os_sandbox.SESSION_PACKAGES_RELPATH
+ packages.mkdir(parents = True)
+ for folder in (workdir, packages):
+ (folder / "csv.py").write_text("print('PLANTED', __file__)\n", encoding = "utf-8")
+ (packages / "chatpkg.py").write_text("VALUE = 'from-session-packages'\n", encoding = "utf-8")
+ (workdir / ".gitconfig").write_text("[core]\n\tfsmonitor = planted\n", encoding = "utf-8")
+ return workdir
+
+
+@_POSIX
+def test_an_approved_host_call_loads_nothing_from_the_workdir(tmp_path, monkeypatch):
+ workdir = _planted_workdir(tmp_path)
+ host_file = tmp_path / "host" / "notes.csv"
+ host_file.parent.mkdir()
+ host_file.write_text("a,b\n", encoding = "utf-8")
+ monkeypatch.setattr(tools, "_get_workdir", lambda session_id = None: str(workdir))
+ code = (
+ "import csv, os, sys\n"
+ f"rows = list(csv.reader(open({str(host_file)!r})))\n"
+ "print('ROWS', rows)\n"
+ "print('HOME', os.environ['HOME'])\n"
+ "print('FILE', __file__)\n"
+ "print('CWD', os.getcwd())\n"
+ "import chatpkg\n"
+ "print('PKG', chatpkg.VALUE)\n"
+ )
+ out = tools._python_exec(code, None, 60, "hardening-chat", host_access_approved = True)
+ assert "ROWS [['a', 'b']]" in out, out
+ assert "PLANTED" not in out, out
+ # A package pip-installed in the chat is still importable, behind the stdlib.
+ assert "PKG from-session-packages" in out, out
+ lines = dict(line.split(" ", 1) for line in out.splitlines() if " " in line)
+ assert os.path.realpath(lines["HOME"]) != os.path.realpath(workdir)
+ assert not lines["FILE"].startswith(str(workdir)), "the approved script ran from the workdir"
+ # Relative paths in the approved code still resolve in the chat folder.
+ assert os.path.realpath(lines["CWD"]) == os.path.realpath(workdir)
+ # The private per-call directory is gone afterwards.
+ assert not os.path.exists(os.path.dirname(lines["FILE"]))
+
+
+@_POSIX
+def test_an_approved_host_launch_env_drops_every_workdir_input(tmp_path):
+ workdir = _planted_workdir(tmp_path)
+ script = workdir / "studio_exec_x.py"
+ script.write_text("print('swapped')\n", encoding = "utf-8")
+ plan = ToolLaunchPlan(
+ argv = (sys.executable, "-u", str(script)),
+ workdir = str(workdir),
+ env = tools._build_safe_env(str(workdir)),
+ requested_mode = "auto",
+ execution_kind = "python",
+ )
+ prepared = tools._prepare_tool_launch(
+ plan, host_access_approved = True, host_script = "print('approved')\n"
+ )
+ try:
+ assert prepared.execution_record.os_isolation is False
+ assert prepared.env["PYTHONSAFEPATH"] == "1"
+ assert prepared.env["HOME"] != str(workdir)
+ packages = str(workdir / os_sandbox.SESSION_PACKAGES_RELPATH)
+ assert packages not in prepared.env.get("PYTHONPATH", "").split(os.pathsep)
+ assert not any(part.startswith(packages) for part in prepared.env["PATH"].split(os.pathsep))
+ assert not prepared.argv[-1].startswith(str(workdir))
+ # The approved code, not whatever the workdir copy was replaced with.
+ ran = subprocess.run(
+ prepared.argv,
+ cwd = prepared.workdir,
+ env = prepared.env,
+ capture_output = True,
+ text = True,
+ timeout = 60,
+ )
+ assert ran.stdout.strip() == "approved", ran
+ finally:
+ prepared.cleanup()
+
+
+@_POSIX
+def test_launches_that_are_not_approved_host_access_keep_the_workdir_env(tmp_path):
+ workdir = _planted_workdir(tmp_path)
+ script = str(workdir / "studio_exec_x.py")
+ plan = ToolLaunchPlan(
+ argv = (sys.executable, "-u", script),
+ workdir = str(workdir),
+ env = tools._build_safe_env(str(workdir)),
+ requested_mode = "auto",
+ execution_kind = "python",
+ )
+ packages = str(workdir / os_sandbox.SESSION_PACKAGES_RELPATH)
+ for prepared in (
+ tools._software_safeguards_launch(plan, "sandbox_became_unavailable"),
+ # A chosen project folder is never a sandbox workdir in auto: its calls keep importing from it.
+ tools._prepare_tool_launch(plan, host_access_approved = True, user_folder = True),
+ ):
+ assert prepared.argv[-1] == script
+ assert prepared.env["HOME"] == str(workdir)
+ assert "PYTHONSAFEPATH" not in prepared.env
+ assert packages in prepared.env["PYTHONPATH"].split(os.pathsep)
+ assert prepared.cleanup_paths == []
+
+
[email protected](sys.platform != "linux", reason = "the bubblewrap backend is Linux only")
+def test_the_jail_sees_host_sysctls_read_only(tmp_path, monkeypatch):
+ from core.inference import sandbox_linux
+
+ if sandbox_linux.shutil.which("bwrap") is None:
+ pytest.skip("bubblewrap is not installed on this host")
+ monkeypatch.setattr(sandbox_linux, "empty_proc_layout", lambda bwrap = None: False)
+ monkeypatch.setattr(sandbox_linux, "_model_cache_binds", lambda workdir: {})
+ launch = sandbox_linux.prepare(
+ ToolLaunchPlan(argv = ("/bin/true",), workdir = str(tmp_path), env = {"PATH": "/usr/bin"})
+ )
+ try:
+ argv = list(launch.argv)
+ at = next(i for i in range(len(argv) - 1) if argv[i : i + 2] == ["--proc", "/proc"])
+ assert argv[at + 2 : at + 5] == ["--ro-bind-try", "/proc/sys", "/proc/sys"]
+ finally:
+ launch.cleanup()
+
+
[email protected](sys.platform != "linux", reason = "the bubblewrap backend is Linux only")
+def test_the_empty_proc_layout_binds_nothing_from_host_proc(tmp_path, monkeypatch):
+ from core.inference import sandbox_linux
+
+ if sandbox_linux.shutil.which("bwrap") is None:
+ pytest.skip("bubblewrap is not installed on this host")
+ monkeypatch.setattr(sandbox_linux, "empty_proc_layout", lambda bwrap = None: True)
+ monkeypatch.setattr(sandbox_linux, "_model_cache_binds", lambda workdir: {})
+ launch = sandbox_linux.prepare(
+ ToolLaunchPlan(argv = ("/bin/true",), workdir = str(tmp_path), env = {"PATH": "/usr/bin"})
+ )
+ try:
+ assert "/proc/sys" not in list(launch.argv)
+ finally:
+ launch.cleanup()
+
+
[email protected](sys.platform != "linux", reason = "the bubblewrap backend is Linux only")
+def test_datasets_and_assets_are_bound_from_the_real_hf_home(tmp_path, monkeypatch):
+ """HF_HUB_CACHE=/mnt/models makes the display home the hub itself, which the jail can write."""
+ from core.inference import sandbox_linux
+ import utils.hf_cache_settings as cache_settings
+
+ hub = tmp_path / "mnt" / "models"
+ real_home = tmp_path / "hf_home"
+ for path in (hub / "datasets", hub / "assets", real_home / "datasets", real_home / "assets"):
+ path.mkdir(parents = True)
+ monkeypatch.setenv("HF_HOME", str(real_home))
+ monkeypatch.delenv("HF_DATASETS_CACHE", raising = False)
+ monkeypatch.delenv("HF_ASSETS_CACHE", raising = False)
+ monkeypatch.setattr(
+ cache_settings,
+ "get_hf_cache_paths",
+ lambda: cache_settings.HuggingFaceCachePaths(
+ cache_home = hub, hub_cache = hub, xet_cache = hub / "xet", source = "environment"
+ ),
+ )
+ monkeypatch.setattr(
+ sandbox_linux,
+ "_cache_hazard_within_deadline",
+ lambda name, path: None if os.path.isdir(path) else sandbox_linux.CACHE_MISSING,
+ )
+ binds = sandbox_linux._model_cache_binds(str(tmp_path / "session"))
+ assert binds["hub"] == str(hub)
+ assert binds["datasets"] == str(real_home / "datasets")
+ assert binds["assets"] == str(real_home / "assets")
+
+
+def test_portable_datasets_and_assets_caches_are_bound_where_they_point(tmp_path, monkeypatch):
+ """Portable installs set HF_DATASETS_CACHE / HF_ASSETS_CACHE under their root and leave HF_HOME alone."""
+ from core.inference import sandbox_linux
+ import utils.hf_cache_settings as cache_settings
+
+ default_home = tmp_path / "default_hf"
+ portable = tmp_path / "portable" / "huggingface"
+ for path in (
+ default_home / "hub",
+ default_home / "datasets",
+ default_home / "assets",
+ portable / "datasets",
+ portable / "assets",
+ ):
+ path.mkdir(parents = True)
+ monkeypatch.setenv("HF_HOME", str(default_home))
+ monkeypatch.setenv("HF_DATASETS_CACHE", str(portable / "datasets"))
+ monkeypatch.setenv("HF_ASSETS_CACHE", str(portable / "assets"))
+ monkeypatch.setattr(
+ cache_settings,
+ "get_hf_cache_paths",
+ lambda: cache_settings.HuggingFaceCachePaths(
+ cache_home = default_home,
+ hub_cache = default_home / "hub",
+ xet_cache = default_home / "xet",
+ source = "environment",
+ ),
+ )
+ monkeypatch.setattr(
+ sandbox_linux,
+ "_cache_hazard_within_deadline",
+ lambda name, path: None if os.path.isdir(path) else sandbox_linux.CACHE_MISSING,
+ )
+ binds = sandbox_linux._model_cache_binds(str(tmp_path / "session"))
+ assert binds["datasets"] == str(portable / "datasets")
+ assert binds["assets"] == str(portable / "assets")
+
+
+def _auto_plan(tmp_path):
+ return ToolLaunchPlan(
+ argv = (sys.executable, "-c", "print(1)"),
+ workdir = str(tmp_path),
+ env = {"PATH": "/usr/bin"},
+ requested_mode = "auto",
+ execution_kind = "python",
+ )
+
+
+def fresh_probe_cache():
+ sandbox_probe.reset_probe_cache()
+ yield
+ sandbox_probe.reset_probe_cache()
+
+
[email protected](sys.platform not in ("linux", "darwin"), reason = "the probe backends")
[email protected]("code", [errno.ENOSPC, errno.EMFILE, errno.EAGAIN])
+def test_a_probe_out_of_resources_refuses_rather_than_running_on_the_host(
+ tmp_path, monkeypatch, fresh_probe_cache, code
+):
+ calls = []
+
+ def exhausted():
+ calls.append(True)
+ raise OSError(code, os.strerror(code))
+
+ monkeypatch.setattr(sandbox_probe, "_probe_base", exhausted)
+ for _ in range(2):
+ with pytest.raises(os_sandbox.SandboxBuildError, match = "OS_ISOLATION_UNAVAILABLE"):
+ os_sandbox.prepare_tool_launch(_auto_plan(tmp_path))
+ # Undecided verdicts are not cached: the next call checks again.
+ assert len(calls) == 2
+ # tools.py does not turn it into a host launch either.
+ with pytest.raises(os_sandbox.SandboxBuildError):
+ tools._prepare_tool_launch(_auto_plan(tmp_path))
+
+
[email protected](sys.platform not in ("linux", "darwin"), reason = "the probe backends")
+def test_a_host_control_that_runs_out_of_disk_is_undecided_too(
+ tmp_path, monkeypatch, fresh_probe_cache
+):
+ monkeypatch.setattr(
+ sandbox_probe,
+ "_host_payload",
+ lambda workdir: "import errno, os\nraise OSError(errno.ENOSPC, os.strerror(errno.ENOSPC))\n",
+ )
+ with pytest.raises(os_sandbox.SandboxBuildError):
+ os_sandbox.prepare_tool_launch(_auto_plan(tmp_path))
+
+
[email protected](sys.platform not in ("linux", "darwin"), reason = "the probe backends")
+def test_a_host_without_a_sandbox_still_falls_back_in_auto(
+ tmp_path, monkeypatch, fresh_probe_cache
+):
+ monkeypatch.setattr(
+ sandbox_probe,
+ "_run_probe",
+ lambda backend, name, plan_cls: (False, "bubblewrap is not installed"),
+ )
+ prepared = os_sandbox.prepare_tool_launch(_auto_plan(tmp_path))
+ assert prepared.backend == "software-safeguards"
+ assert prepared.execution_record.os_isolation is False
+
+
+class _FailingLauncher:
+ """A backend whose launcher starts, then exits with *message* on stderr."""
+
+ def __init__(self, message):
+ self.message = message
+
+ def prepare(self, plan):
+ from types import SimpleNamespace
+ code = f"import sys; sys.stderr.write({self.message!r}); sys.exit(1)"
+ return SimpleNamespace(
+ argv = [sys.executable, "-c", code],
+ workdir = plan.workdir,
+ env = dict(plan.env),
+ preexec_fn = None,
+ pass_fds = (),
+ cleanup = lambda: None,
+ )
+
+
[email protected](sys.platform not in ("linux", "darwin"), reason = "the probe backends")
+ ("message", "undecided"),
+ [
+ (f"bwrap: Creating new namespace failed: {os.strerror(errno.EAGAIN)}", True),
+ (f"bwrap: Creating new namespace failed: {os.strerror(errno.ENOMEM)}", True),
+ # The payload itself filling the disk says nothing about the backend.
+ (f"OSError: [Errno 28] {os.strerror(errno.ENOSPC)}: 'work/x'", True),
+ # bwrap reports a user-namespace limit as ENOSPC: no sandbox only when the quota is zero.
+ (f"bwrap: Creating new namespace failed: {os.strerror(errno.ENOSPC)}", "quota"),
+ ],
+)
+def test_a_launcher_that_exits_out_of_resources_is_undecided(message, undecided, monkeypatch):
+ if undecided == "quota":
+ monkeypatch.setattr(sandbox_probe, "_user_namespaces_disabled", lambda: True)
+ launcher = _FailingLauncher(message)
+ _available, reason = sandbox_probe._run_probe(launcher, "bubblewrap", ToolLaunchPlan)
+ assert not isinstance(reason, sandbox_probe.UndecidedReason), reason
+ # A quota other launches used up is exhaustion, not a host without a sandbox.
+ monkeypatch.setattr(sandbox_probe, "_user_namespaces_disabled", lambda: False)
+ undecided = True
+ # bwrap forks inside the namespace it made; running out there is an exit, not an exception.
+ launcher = _FailingLauncher(message)
+ available, reason = sandbox_probe._run_probe(launcher, "bubblewrap", ToolLaunchPlan)
+ assert available is False
+ assert isinstance(reason, sandbox_probe.UndecidedReason) is undecided, reason
+
+
+def test_dedicated_cache_paths_are_expanded(tmp_path, monkeypatch):
+ from core.inference import sandbox_linux
+ import utils.hf_cache_settings as cache_settings
+
+ home = tmp_path / "home"
+ (home / "portable" / "datasets").mkdir(parents = True)
+ monkeypatch.setenv("HOME", str(home))
+ monkeypatch.setenv("HF_DATASETS_CACHE", " ~/portable/datasets ")
+ monkeypatch.delenv("HF_ASSETS_CACHE", raising = False)
+ monkeypatch.setattr(
+ cache_settings,
+ "get_hf_cache_paths",
+ lambda: cache_settings.HuggingFaceCachePaths(
+ cache_home = home / "hf",
+ hub_cache = home / "hf" / "hub",
+ xet_cache = home / "hf" / "xet",
+ source = "environment",
+ ),
+ )
+ monkeypatch.setattr(
+ sandbox_linux,
+ "_cache_hazard_within_deadline",
+ lambda name, path: None if os.path.isdir(path) else sandbox_linux.CACHE_MISSING,
+ )
+ binds = sandbox_linux._model_cache_binds(str(tmp_path / "session"))
+ assert binds["datasets"] == str(home / "portable" / "datasets")
+
+
+@_POSIX
+def test_the_stale_ipc_sweep_cannot_be_redirected_onto_the_host(tmp_path, monkeypatch):
+ workdir = tmp_path / "w"
+ scratch = workdir / os_sandbox.TOOL_TEMP_DIRNAME
+ inner = scratch / "d"
+ inner.mkdir(parents = True)
+ os.chmod(scratch, os_sandbox.TOOL_TEMP_MODE)
+ host = tmp_path / "host"
+ host.mkdir()
+ (host / "sock").write_text("host data", encoding = "utf-8")
+ monkeypatch.chdir(inner)
+ listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ listener.bind("sock")
+ listener.close()
+ monkeypatch.chdir(tmp_path)
+
+ def swap_then_report_dead(path):
+ os.rename(inner, scratch / "moved")
+ os.symlink(host, inner)
+ return True
+
+ monkeypatch.setattr(os_sandbox, "_ipc_endpoint_is_dead", swap_then_report_dead)
+ os_sandbox.clear_stale_tool_ipc(str(workdir))
+ assert (host / "sock").read_text(encoding = "utf-8") == "host data"
+ # The socket the walk actually saw is the one removed.
+ assert not os.path.lexists(scratch / "moved" / "sock")
+
+
+def project(tmp_path, monkeypatch):
+ from storage import studio_db
+
+ root = tmp_path / "Projects" / "p1"
+ (root / "sandbox").mkdir(parents = True)
+ record = {"id": "p1", "rootPath": str(root), "sandboxPath": str(root / "sandbox")}
+ monkeypatch.setattr(tools, "is_owner_context", lambda: True)
+ monkeypatch.setattr(tools, "_thread_exists", lambda *a, **k: False)
+ monkeypatch.setattr(studio_db, "ensure_chat_project_workspace", lambda project_id: record)
+ monkeypatch.setattr(studio_db, "available_project_working_folder", lambda project: None)
+ return root
+
+
+def test_a_project_sandbox_folder_is_used_as_it_is(project):
+ assert tools._get_project_workdir("project-p1") == os.path.realpath(project / "sandbox")
+
+
+@_POSIX
[email protected]("target", [".", "elsewhere"])
+def test_a_project_sandbox_swapped_for_a_link_is_not_followed(project, target):
+ (project / "elsewhere").mkdir()
+ os.rename(project / "sandbox", project / "old")
+ os.symlink(target, project / "sandbox")
+ assert tools._get_project_workdir("project-p1") is None
+
+
+def test_a_project_sandbox_that_is_a_windows_junction_is_not_followed(project, monkeypatch):
+ # islink() does not report a junction; its reparse-point attribute does.
+ monkeypatch.setattr(
+ tools,
+ "_is_windows_reparse_point",
+ lambda path: os.path.basename(path) == "sandbox",
+ )
+ assert tools._get_project_workdir("project-p1") is None
+
+
[email protected](sys.platform not in ("linux", "darwin"), reason = "the probe backends")
+def test_a_probe_out_of_python_memory_is_undecided(tmp_path, monkeypatch, fresh_probe_cache):
+ def exhausted():
+ raise MemoryError()
+
+ monkeypatch.setattr(sandbox_probe, "_probe_base", exhausted)
+ with pytest.raises(os_sandbox.SandboxBuildError, match = "OS_ISOLATION_UNAVAILABLE"):
+ os_sandbox.prepare_tool_launch(_auto_plan(tmp_path))