Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop the macOS sandbox probe from leaking processes that spin forever (#15345)
Michael Han6285c116e992
diff --git a/studio/backend/core/inference/mxc_probe.py b/studio/backend/core/inference/mxc_probe.pyindex da023f12b93..93913fea204 100644--- a/studio/backend/core/inference/mxc_probe.py+++ b/studio/backend/core/inference/mxc_probe.py@@ -18,12 +18,9 @@ from types import SimpleNamespacefrom . import mxc_adapter, mxc_drive_alias, mxc_policy, mxc_read_grants, mxc_runtime_lock = threading.Lock()-_cache: dict[tuple, tuple[float, bool, str]] = {}+# No TTL, as sandbox_probe: the key and invalidate_cache() decide; a timer only re-ran the container.+_cache: dict[tuple, tuple[bool, str]] = {}_inflight: dict[tuple, threading.Event] = {}-POSITIVE_TTL = 300.0-NEGATIVE_TTL = 30.0-# Re-probing a shell that cannot start costs ~15s of DACL-tier launches; the key changes with the runtime or the shell.-INCOMPATIBLE_TTL = 6 * 3600.0MSYS_NAMESPACE_REASON = ("Git Bash (MSYS2) cannot start in the MXC container, which denies the global named-object ""directory it creates (microsoft/mxc#1061)."@@ -33,13 +30,17 @@ NO_BUILTIN_CONTAINER_REASON = ()-_host_prep_cache: dict[str, tuple[float, str | None]] = {}+_host_prep_cache: dict[str, str | None] = {}+# Bumped by invalidate_cache(): a check started before it must not publish.+_generation = 0def invalidate_cache() -> None:+ global _generationwith _lock:_cache.clear()_host_prep_cache.clear()+ _generation += 1# Only when already loaded: the MXC child paths import this module without the tool stack.os_sandbox = sys.modules.get(f"{__package__}.os_sandbox")if os_sandbox is not None:@@ -64,9 +65,9 @@ def host_prep_remediation() -> str | None:except Exception: # noqa: BLE001 - no runtime, nothing to preparereturn Nonewith _lock:- cached = _host_prep_cache.get(identity)- if cached is not None and time.monotonic() < cached[0]:- return cached[1]+ if identity in _host_prep_cache:+ return _host_prep_cache[identity]+ generation = _generationsteps = mxc_runtime.probe_host_prep_steps(env = mxc_adapter._control_environment())advice = Noneif steps:@@ -82,7 +83,9 @@ def host_prep_remediation() -> str | None:"Prepare this PC.")with _lock:- _host_prep_cache[identity] = (time.monotonic() + NEGATIVE_TTL, advice)+ # None = wxc-exec could not tell: asked again next time rather than kept as "nothing missing".+ if generation == _generation and steps is not None:+ _host_prep_cache[identity] = advicereturn advice@@ -374,12 +377,13 @@ def probe(while True:with _lock:cached = _cache.get(key)- if cached is not None and not force and time.monotonic() < cached[0]:- return cached[1], cached[2]+ if cached is not None and not force:+ return cachedflight = _inflight.get(key)if flight is None:flight = threading.Event()_inflight[key] = flight+ generation = _generationbreakwhile not flight.wait(0.05):if cancel_event is not None and cancel_event.is_set():@@ -388,11 +392,8 @@ def probe(try:result = _probe(selected_executable, execution_kind, cancel_event)with _lock:- if cancel_event is None or not cancel_event.is_set():- ttl = POSITIVE_TTL if result[0] else NEGATIVE_TTL- if result[1] == MSYS_NAMESPACE_REASON:- ttl = INCOMPATIBLE_TTL- _cache[key] = (time.monotonic() + ttl, *result)+ if generation == _generation and (cancel_event is None or not cancel_event.is_set()):+ _cache[key] = resultreturn resultfinally:with _lock:diff --git a/studio/backend/core/inference/os_sandbox.py b/studio/backend/core/inference/os_sandbox.pyindex 27d89901ca7..77427120985 100644--- a/studio/backend/core/inference/os_sandbox.py+++ b/studio/backend/core/inference/os_sandbox.py@@ -1045,11 +1045,10 @@ def capability_snapshot()-# Last answer per tool; the "off" permission gate reads it on every call without probing.-_TOOL_ISOLATION_TTL_SECONDS = 60.0+# Last answer per tool, read by the "off" gate; a reset, a real launch or a new _host_key() replaces it.ISOLATED_TOOLS = ("python", "terminal")_tool_isolation_lock = threading.Lock()-_tool_isolation: dict[str, tuple[float, bool, str, str]] = {}+_tool_isolation: dict[str, tuple[bool, str, str, str]] = {}_tool_isolation_refreshing: set[str] = set()# A reset raises the generation floor; answers from older checks are dropped._tool_isolation_generation = 0@@ -1088,6 +1087,10 @@ def note_tool_isolation(global _tool_isolation_generationif tool not in ISOLATED_TOOLS:return+ from .sandbox_probe import UndecidedReason++ # Host resource exhaustion is not a verdict: an empty key makes the next read check again.+ key = "" if isinstance(reason, UndecidedReason) else _host_key()with _tool_isolation_lock:if generation is None:_tool_isolation_generation += 1@@ -1095,7 +1098,7 @@ def note_tool_isolation(elif generation < _tool_isolation_floor or generation < _tool_isolation_noted.get(tool, 0):return_tool_isolation_noted[tool] = generation- _tool_isolation[tool] = (time.monotonic(), bool(available), backend, reason)+ _tool_isolation[tool] = (bool(available), backend, reason, key)def tool_isolation_generation() -> int:@@ -1154,15 +1157,27 @@ def cached_tool_isolation(tool: str) -> bool | None:return None if cached is None else cached[0]+def _host_key() -> str:+ """Stat-only (no bwrap preflight): moves when the venv is swapped or bwrap installed."""+ key = os.path.realpath(sys.executable)+ if sys.platform == "linux":+ from . import sandbox_linux+ try:+ key += repr(sandbox_linux._bwrap_file_identity(sandbox_linux._trusted_bwrap_path()))+ except Exception as exc: # noqa: BLE001 - no trusted bwrap is a key too+ key += type(exc).__name__+ return key++def cached_tool_capability(tool: str) -> tuple[bool, str, str] | None:- """Never blocks: last (available, backend, reason) or None; stale starts a background refresh."""- now = time.monotonic()+ """Never blocks: last (available, backend, reason) or None; a missing or other-key answer starts one check."""+ key = _host_key()with _tool_isolation_lock:entry = _tool_isolation.get(tool)start = (tool in ISOLATED_TOOLSand not _background_probes_disabled()- and (entry is None or now - entry[0] >= _TOOL_ISOLATION_TTL_SECONDS)+ and (entry is None or entry[3] != key)and tool not in _tool_isolation_refreshing)if start:@@ -1178,7 +1193,7 @@ def cached_tool_capability(tool: str) -> tuple[bool, str, str] | None:except RuntimeError:with _tool_isolation_lock:_tool_isolation_refreshing.discard(tool)- return None if entry is None else (entry[1], entry[2], entry[3])+ return None if entry is None else entry[:3]def warm_tool_isolation() -> None:@@ -1366,7 +1381,7 @@ def prepare_tool_launch(plan: ToolLaunchPlan) -> PreparedSandboxLaunch:except (WorkdirUnsafeError, SandboxBuildError):raise # can be tool-induced; says nothing about the backend itselfexcept SandboxUnavailableError:- # Cached pass is stale: reset so "off" asks instead of failing until expiry.+ # Cached pass is stale: reset so "off" asks instead of failing on it until a Refresh.from .sandbox_probe import reset_probe_cachereset_probe_cache()raisediff --git a/studio/backend/core/inference/sandbox_macos.py b/studio/backend/core/inference/sandbox_macos.pyindex 903b02fe339..a1637aedf75 100644--- a/studio/backend/core/inference/sandbox_macos.py+++ b/studio/backend/core/inference/sandbox_macos.py@@ -597,6 +597,8 @@ def build_profile(# bash process substitution opens /dev/fd/63.'(allow file-read* (regex #"^/dev/fd/[0-9]+$"))','(allow file-write* (regex #"^/dev/fd/[0-9]+$"))',+ # Unlistable, CPython close()s every fd up to RLIMIT_NOFILE before exec: forever when unlimited.+ '(allow file-read* (literal "/dev/fd"))',_rule("allow file-ioctl", device_filters),"(allow ipc-posix-sem)",# write-data: libomp writes its registration into the segment.diff --git a/studio/backend/core/inference/sandbox_probe.py b/studio/backend/core/inference/sandbox_probe.pyindex b34af098adf..2f1c5014b78 100644--- a/studio/backend/core/inference/sandbox_probe.py+++ b/studio/backend/core/inference/sandbox_probe.py@@ -6,15 +6,16 @@from __future__ import annotationsimport ctypesimport errno+import itertoolsimport osimport reimport shutil+import signalimport socketimport subprocessimport sysimport tempfileimport threading-import timefrom typing import Anyfrom loggers import get_logger@@ -26,6 +27,15 @@ try:_libc = ctypes.CDLL(None, use_errno = True) if sys.platform == "linux" else Noneexcept OSError: # pragma: no cover - a libc that will not load_libc = None+try:+ import resource as _resource+except ImportError: # pragma: no cover - Windows never probes+ _resource = None++# The tool launch's own fd cap (tools._sandbox_preexec), so the probe qualifies what tools get.+_PROBE_NOFILE = 16384+# Kernel-enforced: ends a payload process nobody is left to kill.+_PROBE_CPU_SECONDS = 60# ALONE on stdout: a payload that printed it early and died would satisfy "in".PROBE_TOKEN = "UNSLOTH_SANDBOX_PROBE_OK"@@ -34,18 +44,12 @@ _SENTINEL_TOKEN = "unsloth-host-sentinel-must-not-be-readable"_OUTSIDE_WRITE_TOKEN = "unsloth-sandbox-escaped-to-the-host"PROBE_TIMEOUT_SECONDS = 30.0-# Correctness does not rest on the TTL: the key carries the runtime identity and a failed launch invalidates.-_CACHE_TTL_SECONDS = 900.0-# UNAVAILABLE keeps a short TTL: the user fixing their host does not move the cache key.-_CACHE_TTL_UNAVAILABLE_SECONDS = 60.0+# No TTL: re-probing on a timer turned one hung probe into one more each minute on macOS._CACHE_MAX_ENTRIES = 8# ``sun_path`` is 108 bytes and multiprocessing appends about 32._MAX_PROBE_BASE_LEN = 59-# A stale PASS is served while one background re-probe runs; a FAIL never is.-_STALE_GRACE_SECONDS = 3600.0-# Tool code can exhaust these from the jail: undecided, never cached, and auto refuses rather than falls back._RESOURCE_ERRNOS = frozenset(code@@ -97,9 +101,11 @@ def _classified(reason: str, exc: BaseException | None = None) -> str:_cache_lock = threading.Lock()-_cache: dict[tuple[str, str], tuple[float, bool, str]] = {}-_refreshing: set[tuple[str, str]] = set()+_cache: dict[tuple[str, str], tuple[bool, str]] = {}_generation = 0+# A slower, older probe must not replace a newer (forced) verdict.+_tickets = itertools.count(1)+_published: dict[tuple[str, str], int] = {}def reset_probe_cache() -> None:@@ -107,6 +113,7 @@ def reset_probe_cache() -> None:global _generationwith _cache_lock:_cache.clear()+ _published.clear()_generation += 1if sys.platform == "linux":try:@@ -121,73 +128,26 @@ def reset_probe_cache() -> None:forget_tool_isolation()-def _cache_get(key: tuple[str, str]) -> tuple[bool, str, bool] | None:- """(available, reason, stale); None when nothing may be served."""- now = time.monotonic()- with _cache_lock:- entry = _cache.get(key)- if entry is None:- return None- expires_at, available, reason = entry- if expires_at > now:- return available, reason, False- if available and now < expires_at + _STALE_GRACE_SECONDS:- return available, reason, True- _cache.pop(key, None)- return None--def _cache_put(key: tuple[str, str],available: bool,reason: str,*,generation: int | None = None,+ ticket: int = 0,) -> None:with _cache_lock:if generation is not None and generation != _generation:return- ttl = _CACHE_TTL_SECONDS if available else _CACHE_TTL_UNAVAILABLE_SECONDS+ if ticket < _published.get(key, 0):+ return+ _published[key] = ticket_cache.pop(key, None)- _cache[key] = (time.monotonic() + ttl, available, reason)+ _cache[key] = (available, reason)while len(_cache) > _CACHE_MAX_ENTRIES:_cache.pop(next(iter(_cache)))-def _background_probes_disabled() -> bool:- try:- from .os_sandbox import _background_probes_disabled as disabled- except Exception: # noqa: BLE001 - a partially imported package: stay in the foreground- return True- return disabled()---def _refresh_in_background(backend: Any, key: tuple[str, str]) -> None:- with _cache_lock:- if key in _refreshing:- return- _refreshing.add(key)- generation = _generation-- def run() -> None:- try:- from .os_sandbox import ToolLaunchPlan- available, reason = _run_probe(backend, key[0], ToolLaunchPlan)- if not isinstance(reason, UndecidedReason):- _cache_put(key, available, reason, generation = generation)- except Exception as exc: # noqa: BLE001 - the stale PASS just ages out- logger.debug("background sandbox re-probe failed: %s", exc)- finally:- with _cache_lock:- _refreshing.discard(key)-- try:- threading.Thread(target = run, name = "unsloth-sandbox-reprobe", daemon = True).start()- except RuntimeError:- with _cache_lock:- _refreshing.discard(key)--_PREAMBLE = '''import multiprocessing.reduction, os, socket, subprocess, sys@@ -296,6 +256,53 @@ def _no_new_privs() -> None:logger.warning("The sandbox probe could not set PR_SET_NO_NEW_PRIVS")+def _probe_preexec() -> None:+ """In the forked child, before the backend's wrapper: the payload's limits, then no_new_privs."""+ if _resource is not None:+ try:+ _soft, hard = _resource.getrlimit(_resource.RLIMIT_NOFILE)+ nofile = _PROBE_NOFILE if hard == _resource.RLIM_INFINITY else min(_PROBE_NOFILE, hard)+ _resource.setrlimit(_resource.RLIMIT_NOFILE, (nofile, nofile))+ except (ValueError, OSError):+ pass+ try:+ _resource.setrlimit(_resource.RLIMIT_CPU, (_PROBE_CPU_SECONDS, _PROBE_CPU_SECONDS))+ except (ValueError, OSError):+ pass+ _no_new_privs()+++def _end_probe_group(proc: subprocess.Popen) -> None:+ """killpg, not kill: the payload's own children outlive a killed payload, and on macOS one spun forever.+ Only while the leader is unreaped: once reaped and its group empty, the id can name someone else's group."""+ if proc.returncode is None:+ try:+ os.killpg(proc.pid, signal.SIGKILL)+ except OSError:+ proc.kill()+ proc.wait()+ for pipe in (proc.stdout, proc.stderr):+ if pipe is not None:+ pipe.close()+++def _run_grouped(argv, **popen_kwargs) -> subprocess.CompletedProcess:+ """subprocess.run in its own session; the whole group ends with it."""+ proc = subprocess.Popen(+ argv,+ stdin = subprocess.DEVNULL,+ stdout = subprocess.PIPE,+ stderr = subprocess.PIPE,+ start_new_session = True,+ **popen_kwargs,+ )+ try:+ stdout, stderr = proc.communicate(timeout = PROBE_TIMEOUT_SECONDS)+ finally:+ _end_probe_group(proc)+ return subprocess.CompletedProcess(argv, proc.returncode, stdout, stderr)++def _abstract_control() -> "tuple[bytes | None, Any]":"""Nothing without a scope to test, or when the host cannot reach its own socket."""if sys.platform != "linux":@@ -375,14 +382,11 @@ def _host_positive_controls(workdir: str, sentinel: str, outside: str, env: dictf"the host itself could not write outside the probe workdir ({exc})", exc)try:- completed = subprocess.run(+ completed = _run_grouped((sys.executable, "-I", "-S", "-c", _host_payload(workdir)),cwd = workdir,env = env,- stdin = subprocess.DEVNULL,- stdout = subprocess.PIPE,- stderr = subprocess.PIPE,- timeout = PROBE_TIMEOUT_SECONDS,+ preexec_fn = _probe_preexec,)except (OSError, subprocess.SubprocessError) as exc:return _classified(@@ -407,23 +411,19 @@ def probe(backend: Any, *, force: bool = False) -> tuple[bool, str]:return False, f"the sandbox probe could not identify this runtime: {exc}"if not force:- cached = _cache_get(key)+ with _cache_lock:+ cached = _cache.get(key)if cached is not None:- available, reason, stale = cached- if not stale:- return available, reason- if not _background_probes_disabled():- _refresh_in_background(backend, key)- return available, reason-- # Background probes are off: re-probe here.+ return cached+with _cache_lock:generation = _generation+ ticket = next(_tickets)available, reason = _run_probe(backend, backend_name, ToolLaunchPlan)if isinstance(reason, UndecidedReason):# Re-probed on the next call: the resource may be freed by then.return False, reason- _cache_put(key, available, reason, generation = generation)+ _cache_put(key, available, reason, generation = generation, ticket = ticket)return available, reason@@ -473,21 +473,17 @@ def _run_probe(backend: Any, backend_name: str, plan_cls: Any) -> tuple[bool, stworkdir = workdir,env = env,# A setuid bwrap dies under no_new_privs, so without this the probe qualifies a launch that fails after Popen.- preexec_fn = _no_new_privs,+ preexec_fn = _probe_preexec,requested_mode = "required",execution_kind = "python",)prepared = backend.prepare(plan)- completed = subprocess.run(+ completed = _run_grouped(prepared.argv,cwd = prepared.workdir,env = prepared.env,- stdin = subprocess.DEVNULL,- stdout = subprocess.PIPE,- stderr = subprocess.PIPE,preexec_fn = prepared.preexec_fn,pass_fds = tuple(prepared.pass_fds),- timeout = PROBE_TIMEOUT_SECONDS,)if completed.returncode != 0 or completed.stdout.strip() != PROBE_TOKEN.encode():detail = (completed.stderr or b"").decode("utf-8", "replace").strip()[-300:]diff --git a/studio/backend/tests/test_mxc_git_bash_namespace.py b/studio/backend/tests/test_mxc_git_bash_namespace.pyindex 6c4121e7bb1..94a4e3af83a 100644--- a/studio/backend/tests/test_mxc_git_bash_namespace.py+++ b/studio/backend/tests/test_mxc_git_bash_namespace.py@@ -111,7 +111,12 @@ def test_a_missing_builtin_container_is_named(monkeypatch, tmp_path, executable,def _count_live_probes(monkeypatch, reason):clock = [1000.0]calls = []- monkeypatch.setattr(mxc_probe.time, "monotonic", lambda: clock[0])++ def leaping_monotonic():+ clock[0] += 86400.0+ return clock[0]++ monkeypatch.setattr(mxc_probe.time, "monotonic", leaping_monotonic)monkeypatch.setattr(mxc_probe.mxc_runtime, "installation_identity", lambda: "runtime-a")def live_probe(@@ -126,28 +131,46 @@ def _count_live_probes(monkeypatch, reason):return clock, calls-def test_an_incompatible_shell_is_not_probed_again_each_negative_ttl(monkeypatch, tmp_path):- clock, calls = _count_live_probes(monkeypatch, mxc_probe.MSYS_NAMESPACE_REASON)+ "reason", [mxc_probe.MSYS_NAMESPACE_REASON, "the live MXC probe did not complete cleanly"]+)+def test_a_failure_is_held_until_invalidated_not_until_a_timer(monkeypatch, tmp_path, reason):+ _clock, calls = _count_live_probes(monkeypatch, reason)bash = str(tmp_path / "bash.exe")for _ in range(5):- assert mxc_probe.probe(bash, execution_kind = "terminal") == (- False,- mxc_probe.MSYS_NAMESPACE_REASON,- )- clock[0] += mxc_probe.NEGATIVE_TTL + 1+ assert mxc_probe.probe(bash, execution_kind = "terminal") == (False, reason)assert len(calls) == 1- clock[0] += mxc_probe.INCOMPATIBLE_TTL+ mxc_probe.invalidate_cache()mxc_probe.probe(bash, execution_kind = "terminal")assert len(calls) == 2+ mxc_probe.probe(bash, execution_kind = "terminal", force = True)+ assert len(calls) == 3++def test_an_invalidate_during_a_probe_keeps_its_result_out(monkeypatch, tmp_path):+ import threading-def test_a_generic_failure_still_expires_at_the_negative_ttl(monkeypatch, tmp_path):- clock, calls = _count_live_probes(monkeypatch, "the live MXC probe did not complete cleanly")+ _clock, calls = _count_live_probes(monkeypatch, "blocked before the fix")+ started, release = threading.Event(), threading.Event()+ live = mxc_probe._probe++ def slow(*args, **kwargs):+ started.set()+ release.wait(5)+ return live(*args, **kwargs)++ monkeypatch.setattr(mxc_probe, "_probe", slow)bash = str(tmp_path / "bash.exe")+ worker = threading.Thread(+ target = mxc_probe.probe, args = (bash,), kwargs = {"execution_kind": "terminal"}+ )+ worker.start()+ assert started.wait(5)+ mxc_probe.invalidate_cache()+ release.set()+ worker.join(5)mxc_probe.probe(bash, execution_kind = "terminal")- clock[0] += mxc_probe.NEGATIVE_TTL + 1- mxc_probe.probe(bash, execution_kind = "terminal")- assert len(calls) == 2+ assert len(calls) == 2, "a verdict from before the invalidate was cached"def test_a_new_runtime_or_opt_in_probes_the_shell_again(monkeypatch, tmp_path):diff --git a/studio/backend/tests/test_mxc_windows_backend.py b/studio/backend/tests/test_mxc_windows_backend.pyindex 2695cdee82a..edcd0210721 100644--- a/studio/backend/tests/test_mxc_windows_backend.py+++ b/studio/backend/tests/test_mxc_windows_backend.py@@ -113,6 +113,21 @@ def test_unavailable_capability_names_the_host_prep_command(monkeypatch, tmp_patassert "install_mxc_prebuilt.py" in capability.remediation or not advised+def test_an_inconclusive_host_prep_check_is_asked_again(monkeypatch):+ from core.inference import mxc_probe++ monkeypatch.setattr(mxc_probe, "_host_prep_cache", {})+ monkeypatch.setattr(mxc_probe.mxc_runtime, "installation_identity", lambda: "runner")+ answers = [None, ("prepare-null-device",)]+ monkeypatch.setattr(+ mxc_probe.mxc_runtime, "probe_host_prep_steps", lambda **_kwargs: answers.pop(0)+ )+ assert mxc_probe.host_prep_remediation() is None+ assert "prepare-null-device" in mxc_probe.host_prep_remediation()+ assert "prepare-null-device" in mxc_probe.host_prep_remediation()+ assert answers == []++def test_host_prep_is_not_advised_without_the_dacl_opt_in(monkeypatch, tmp_path):from core.inference import mxc_probe, sandbox_windows_mxcdiff --git a/studio/backend/tests/test_permission_mode_degrade.py b/studio/backend/tests/test_permission_mode_degrade.pyindex 397f9ac9415..1d597c8a6a1 100644--- a/studio/backend/tests/test_permission_mode_degrade.py+++ b/studio/backend/tests/test_permission_mode_degrade.py@@ -319,27 +319,30 @@ def test_no_startup_warmup_on_windows(monkeypatch):assert called == []-class _Clock:+class _LeapingClock:+ """Each read is a day later, so any TTL left in the code expires between two calls."""+def __init__(self):self.now = 1000.0def __call__(self):+ self.now += 86400.0return self.now+def _leap_time(monkeypatch, module):+ import types+ clock = types.SimpleNamespace(monotonic = _LeapingClock(), time = time.time, sleep = time.sleep)+ monkeypatch.setattr(module, "time", clock, raising = False)++@pytest.fixturedef probe_env(monkeypatch):monkeypatch.setenv(os_sandbox.WARMUP_DISABLE_ENV, "0")sandbox_probe.reset_probe_cache()- clock = _Clock()- import types-- monkeypatch.setattr(- sandbox_probe,- "time",- types.SimpleNamespace(monotonic = clock, time = time.time, sleep = time.sleep),- )- monkeypatch.setattr(os_sandbox, "_runtime_identity", lambda: "identity")+ _leap_time(monkeypatch, sandbox_probe)+ identity = ["identity"]+ monkeypatch.setattr(os_sandbox, "_runtime_identity", lambda: identity[0])verdicts = []gate = threading.Event()gate.set()@@ -351,7 +354,7 @@ def probe_env(monkeypatch):return verdicts.pop(0)monkeypatch.setattr(sandbox_probe, "_run_probe", run_probe)- yield clock, verdicts, gate, calls+ yield identity, verdicts, gate, callssandbox_probe.reset_probe_cache()@@ -366,73 +369,129 @@ def _wait_for(predicate, timeout = 5.0):return predicate()-def test_a_stale_pass_is_served_while_one_reprobe_runs(probe_env):- clock, verdicts, gate, calls = probe_env- verdicts.extend([(True, "first"), (True, "second")])- assert sandbox_probe.probe(_Backend()) == (True, "first")- clock.now += sandbox_probe._CACHE_TTL_SECONDS + 1[email protected]("first", [(True, "ok"), (False, "blocked")])+def test_a_verdict_is_held_until_an_event_not_a_timer(probe_env, first):+ identity, verdicts, gate, calls = probe_env+ verdicts.extend([first, (True, "after reset"), (False, "forced"), (True, "new runtime")])+ assert sandbox_probe.probe(_Backend()) == first+ for _ in range(20):+ assert sandbox_probe.probe(_Backend()) == first+ assert len(calls) == 1+ sandbox_probe.reset_probe_cache()+ assert sandbox_probe.probe(_Backend()) == (True, "after reset")+ assert sandbox_probe.probe(_Backend(), force = True) == (False, "forced")+ identity[0] = "swapped venv"+ assert sandbox_probe.probe(_Backend()) == (True, "new runtime")+ assert len(calls) == 4+ assert all(name == threading.current_thread().name for name in calls)+++def test_an_undecided_verdict_is_retried_by_the_next_caller(probe_env):+ identity, verdicts, gate, calls = probe_env+ verdicts.extend([(False, sandbox_probe.UndecidedReason("out of disk")), (True, "ok")])+ assert sandbox_probe.probe(_Backend())[0] is False+ assert sandbox_probe.probe(_Backend()) == (True, "ok")+ assert sandbox_probe.probe(_Backend()) == (True, "ok")+ assert len(calls) == 2+++def test_a_reset_during_a_probe_keeps_its_result_out(probe_env):+ identity, verdicts, gate, calls = probe_env+ verdicts.extend([(True, "from before the reset"), (False, "after")])gate.clear()- assert sandbox_probe.probe(_Backend()) == (True, "first")- assert sandbox_probe.probe(_Backend()) == (True, "first")- assert _wait_for(lambda: len(calls) == 2)- assert calls[1] == "unsloth-sandbox-reprobe"+ worker = threading.Thread(target = sandbox_probe.probe, args = (_Backend(),))+ worker.start()+ assert _wait_for(lambda: len(calls) == 1)+ sandbox_probe.reset_probe_cache()gate.set()- assert _wait_for(lambda: sandbox_probe.probe(_Backend()) == (True, "second"))- assert len(calls) == 2+ worker.join(5)+ assert sandbox_probe.probe(_Backend()) == (False, "after")-def test_a_failed_reprobe_replaces_the_stale_pass(probe_env):- clock, verdicts, gate, calls = probe_env- verdicts.extend([(True, "ok"), (False, "now blocked")])- sandbox_probe.probe(_Backend())- clock.now += sandbox_probe._CACHE_TTL_SECONDS + 1- assert sandbox_probe.probe(_Backend()) == (True, "ok")- assert _wait_for(lambda: sandbox_probe.probe(_Backend()) == (False, "now blocked"))+def test_an_older_probe_finishing_last_never_replaces_a_newer_verdict(probe_env, monkeypatch):+ """With no TTL, a slow ordinary probe landing after a forced Refresh would have stuck for good."""+ identity, verdicts, gate, calls = probe_env+ slow_started, release_slow = threading.Event(), threading.Event()+ def run_probe(backend, name, plan_cls):+ if not slow_started.is_set():+ slow_started.set()+ release_slow.wait(5)+ return False, "old failure"+ return True, "forced pass"-def test_a_fail_is_never_served_stale(probe_env):- clock, verdicts, gate, calls = probe_env- verdicts.extend([(False, "blocked"), (True, "fixed")])- assert sandbox_probe.probe(_Backend()) == (False, "blocked")- clock.now += sandbox_probe._CACHE_TTL_UNAVAILABLE_SECONDS + 1- assert sandbox_probe.probe(_Backend()) == (True, "fixed")- assert all(name != "unsloth-sandbox-reprobe" for name in calls)+ monkeypatch.setattr(sandbox_probe, "_run_probe", run_probe)+ slow = threading.Thread(target = sandbox_probe.probe, args = (_Backend(),))+ slow.start()+ assert slow_started.wait(5)+ assert sandbox_probe.probe(_Backend(), force = True) == (True, "forced pass")+ release_slow.set()+ slow.join(5)+ assert sandbox_probe.probe(_Backend()) == (True, "forced pass")-def test_a_pass_past_the_grace_is_probed_in_the_foreground(probe_env):- clock, verdicts, gate, calls = probe_env- verdicts.extend([(True, "old"), (True, "new")])- sandbox_probe.probe(_Backend())- clock.now += sandbox_probe._CACHE_TTL_SECONDS + sandbox_probe._STALE_GRACE_SECONDS + 1- assert sandbox_probe.probe(_Backend()) == (True, "new")- assert all(name != "unsloth-sandbox-reprobe" for name in calls)+def test_an_undecided_answer_is_checked_again_by_the_next_read(monkeypatch):+ monkeypatch.delenv(os_sandbox.WARMUP_DISABLE_ENV, raising = False)+ started = []+ def refresh(tool, *, force = False):+ started.append(tool)+ os_sandbox.note_tool_isolation(tool, True, backend = "bubblewrap")+ return True-def test_a_stale_pass_is_reprobed_in_the_foreground_when_background_probes_are_off(- probe_env, monkeypatch-):- clock, verdicts, gate, calls = probe_env- monkeypatch.setenv(os_sandbox.WARMUP_DISABLE_ENV, "1")- verdicts.extend([(True, "old"), (False, "broke")])- sandbox_probe.probe(_Backend())- clock.now += sandbox_probe._CACHE_TTL_SECONDS + 1- assert sandbox_probe.probe(_Backend()) == (False, "broke")- assert all(name != "unsloth-sandbox-reprobe" for name in calls)- assert not sandbox_probe._refreshing---def test_a_reset_during_a_reprobe_keeps_its_result_out(probe_env):- clock, verdicts, gate, calls = probe_env- verdicts.extend([(True, "ok"), (True, "from before the reset"), (False, "after")])- sandbox_probe.probe(_Backend())- clock.now += sandbox_probe._CACHE_TTL_SECONDS + 1- gate.clear()- sandbox_probe.probe(_Backend())- assert _wait_for(lambda: len(calls) == 2)- sandbox_probe.reset_probe_cache()- gate.set()- assert _wait_for(lambda: not sandbox_probe._refreshing)- assert sandbox_probe.probe(_Backend()) == (False, "after")+ monkeypatch.setattr(os_sandbox, "refresh_tool_isolation", refresh)+ os_sandbox.note_tool_isolation(+ "python", False, backend = "none", reason = sandbox_probe.UndecidedReason("out of disk")+ )+ assert os_sandbox.cached_tool_capability("python")[0] is False+ assert _wait_for(lambda: os_sandbox.cached_tool_isolation("python") is True)+ for _ in range(5):+ os_sandbox.cached_tool_capability("python")+ assert started == ["python"]+++def test_polling_an_answered_tool_never_probes_again(monkeypatch):+ monkeypatch.delenv(os_sandbox.WARMUP_DISABLE_ENV, raising = False)+ _leap_time(monkeypatch, os_sandbox)+ started = []+ import types++ monkeypatch.setattr(+ os_sandbox,+ "threading",+ types.SimpleNamespace(+ Thread = lambda *a, **k: started.append(k.get("name")) or threading.Thread()+ ),+ )+ os_sandbox.note_tool_isolation("python", False, backend = "none", reason = "blocked")+ for _ in range(50):+ assert os_sandbox.cached_tool_capability("python")[:3] == (False, "none", "blocked")+ assert tool_policy.off_mode_still_gates("python") is True+ assert started == [], "polling an answered tool started a re-probe"+++def test_an_answer_noted_under_another_host_key_is_checked_once(monkeypatch):+ monkeypatch.delenv(os_sandbox.WARMUP_DISABLE_ENV, raising = False)+ key = ["no bwrap"]+ monkeypatch.setattr(os_sandbox, "_host_key", lambda: key[0])+ started = []++ def refresh(tool, *, force = False):+ started.append(tool)+ os_sandbox.note_tool_isolation(tool, True, backend = "bubblewrap")+ return True++ monkeypatch.setattr(os_sandbox, "refresh_tool_isolation", refresh)+ os_sandbox.note_tool_isolation("python", False, backend = "none", reason = "no bwrap")+ for _ in range(5):+ assert os_sandbox.cached_tool_capability("python")[0] is False+ assert started == []+ key[0] = "bwrap installed"+ os_sandbox.cached_tool_capability("python")+ assert _wait_for(lambda: os_sandbox.cached_tool_isolation("python") is True)+ for _ in range(5):+ os_sandbox.cached_tool_capability("python")+ assert started == ["python"]@pytest.fixture(autouse = True)diff --git a/studio/backend/tests/test_sandbox_linux.py b/studio/backend/tests/test_sandbox_linux.pyindex 0baae705f2a..d1aa3edbf6f 100644--- a/studio/backend/tests/test_sandbox_linux.py+++ b/studio/backend/tests/test_sandbox_linux.py@@ -22,6 +22,7 @@ import subprocessimport sysimport sysconfigimport tempfile+import textwrapimport threadingimport time@@ -1077,12 +1078,15 @@ def test_the_probe_launch_sets_no_new_privs_like_a_real_one(tmp_path):assert sandbox_probe._PR_SET_NO_NEW_PRIVS == 38source = inspect.getsource(sandbox_probe._run_probe)- assert "preexec_fn = _no_new_privs" in source- assert not [- node- for node in ast.walk(ast.parse(inspect.getsource(sandbox_probe._no_new_privs)))- if isinstance(node, (ast.Import, ast.ImportFrom))- ]+ assert "preexec_fn = _probe_preexec" in source+ assert "_no_new_privs()" in inspect.getsource(sandbox_probe._probe_preexec)+ for hook in (sandbox_probe._no_new_privs, sandbox_probe._probe_preexec):+ # Runs between fork and exec: an import there can deadlock on the import lock.+ assert not [+ node+ for node in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(hook))))+ if isinstance(node, (ast.Import, ast.ImportFrom))+ ]def test_the_pip_targets_script_directory_is_last_on_path(prepared):diff --git a/studio/backend/tests/test_sandbox_macos.py b/studio/backend/tests/test_sandbox_macos.pyindex 52a261926de..1229cc491d9 100644--- a/studio/backend/tests/test_sandbox_macos.py+++ b/studio/backend/tests/test_sandbox_macos.py@@ -76,6 +76,10 @@ def test_profile_is_deny_default(profile):assert "(allow default)" not in profile+def test_the_fd_directory_is_listable_so_spawning_never_brute_forces_closes(profile):+ assert '(allow file-read* (literal "/dev/fd"))' in profile.splitlines()++def test_every_rule_is_a_balanced_s_expression(profile):depth = 0in_string = False@@ -411,6 +415,60 @@ def test_profile_compiles_under_sandbox_exec(tmp_path):prepared.cleanup()+@_darwin_only+def test_a_child_spawns_promptly_under_an_unlimited_fd_limit(tmp_path):+ """Before /dev/fd was listable, this spawn spun forever at 100% CPU, unkillable by its parent's timeout."""+ import resource+ import signal++ _soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)+ if hard != resource.RLIM_INFINITY:+ pytest.skip("needs an unlimited hard RLIMIT_NOFILE to raise the soft limit to")++ def unlimited():+ resource.setrlimit(resource.RLIMIT_NOFILE, (resource.RLIM_INFINITY, hard))++ workdir = tmp_path / "session"+ workdir.mkdir()+ spawn = (+ "import subprocess, time\n"+ "start = time.monotonic()\n"+ "subprocess.run(['/usr/bin/true'], check = True)\n"+ "print(time.monotonic() - start)\n"+ )+ prepared = backend.prepare(+ ToolLaunchPlan(+ argv = (sys.executable, "-I", "-S", "-c", spawn),+ workdir = str(workdir),+ env = {"PATH": "/usr/bin:/bin"},+ preexec_fn = unlimited,+ )+ )+ proc = subprocess.Popen(+ prepared.argv,+ env = prepared.env,+ stdout = subprocess.PIPE,+ stderr = subprocess.PIPE,+ text = True,+ preexec_fn = prepared.preexec_fn,+ start_new_session = True,+ )+ try:+ stdout, stderr = proc.communicate(timeout = 30)+ except subprocess.TimeoutExpired:+ pytest.fail("spawning a child inside the sandbox did not finish within 30s")+ finally:+ if proc.returncode is None:+ try:+ os.killpg(proc.pid, signal.SIGKILL)+ except OSError:+ pass+ proc.wait()+ prepared.cleanup()+ assert proc.returncode == 0, stderr+ assert float(stdout.strip()) < 5.0++@_darwin_onlydef test_home_is_unreadable_inside_the_sandbox(tmp_path):"""The file is created and proven readable on the host first: a read that fails for an unrelated reason proves nothing."""diff --git a/studio/backend/tests/test_sandbox_probe.py b/studio/backend/tests/test_sandbox_probe.pyindex 4b536c081ce..d0bbbf939be 100644--- a/studio/backend/tests/test_sandbox_probe.py+++ b/studio/backend/tests/test_sandbox_probe.py@@ -5,6 +5,7 @@from __future__ import annotations+import astimport inspectimport osimport pathlib@@ -13,6 +14,7 @@ import shutilimport subprocessimport sysimport tempfile+import timeimport pytest@@ -257,6 +259,104 @@ def test_a_wedged_backend_times_out_instead_of_hanging(monkeypatch):assert "timed out" in reason+def _zombie(pid: int) -> bool:+ """Killed but unreaped: a container whose PID 1 never reaps orphans keeps it in the table."""+ try:+ with open(f"/proc/{pid}/stat", encoding = "ascii") as handle:+ return handle.read().rpartition(")")[2].split()[0] == "Z"+ except (OSError, IndexError):+ return False+++def _gone(pid: int, within: float = 10.0) -> bool:+ deadline = time.monotonic() + within+ while time.monotonic() < deadline:+ try:+ os.kill(pid, 0)+ except ProcessLookupError:+ return True+ if _zombie(pid):+ return True+ time.sleep(0.05)+ return False+++def test_a_timed_out_probe_also_ends_what_the_payload_spawned(monkeypatch, tmp_path):+ """The macOS leak: a killed payload left its stuck child spinning, re-probed every minute."""+ monkeypatch.setattr(sandbox_probe, "PROBE_TIMEOUT_SECONDS", 1.0)+ pid_file = tmp_path / "grandchild.pid"+ payload = (+ "import subprocess, sys, time\n"+ "child = subprocess.Popen([sys.executable, '-I', '-S', '-c', 'import time; time.sleep(120)'])\n"+ f"open({str(pid_file)!r}, 'w').write(str(child.pid))\n"+ "time.sleep(120)\n"+ )++ def wedged_with_a_child(plan):+ return PreparedSandboxLaunch(+ argv = (sys.executable, "-I", "-S", "-c", payload),+ workdir = plan.workdir,+ env = plan.env,+ preexec_fn = plan.preexec_fn,+ backend = "wedged",+ )++ available, reason = sandbox_probe.probe(_Backend("wedged", wedged_with_a_child))+ assert available is False and "timed out" in reason+ grandchild = int(pid_file.read_text())+ assert _gone(grandchild), f"the payload's child {grandchild} outlived the probe"+++def test_a_finished_probe_never_signals_a_group_it_no_longer_pins(monkeypatch):+ """After communicate() reaps the leader, an emptied group's id can be reused by another process."""+ signalled = []+ monkeypatch.setattr(sandbox_probe.os, "killpg", lambda pgid, sig: signalled.append(pgid))+ assert sandbox_probe.probe(_Backend("confining", _confining))[0] is True+ assert signalled == []+++def test_a_timed_out_host_control_also_ends_what_it_spawned(monkeypatch, tmp_path):+ monkeypatch.setattr(sandbox_probe, "PROBE_TIMEOUT_SECONDS", 1.0)+ pid_file = tmp_path / "grandchild.pid"+ monkeypatch.setattr(+ sandbox_probe,+ "_host_payload",+ lambda workdir: (+ "import subprocess, sys, time\n"+ "child = subprocess.Popen([sys.executable, '-I', '-S', '-c', 'import time; time.sleep(120)'])\n"+ f"open({str(pid_file)!r}, 'w').write(str(child.pid))\n"+ "time.sleep(120)\n"+ ),+ )+ available, reason = sandbox_probe.probe(_Backend("confining", _confining))+ assert available is False and "positive controls" in reason+ grandchild = int(pid_file.read_text())+ assert _gone(grandchild), f"the host control's child {grandchild} outlived the probe"+++def test_the_payload_runs_under_the_tool_launch_fd_and_cpu_caps(tmp_path):+ limits_file = tmp_path / "limits.txt"+ payload = (+ "import resource\n"+ f"open({str(limits_file)!r}, 'w').write(repr(("+ "resource.getrlimit(resource.RLIMIT_NOFILE), resource.getrlimit(resource.RLIMIT_CPU))))\n"+ )++ def report_limits(plan):+ return PreparedSandboxLaunch(+ argv = (sys.executable, "-I", "-S", "-c", payload),+ workdir = plan.workdir,+ env = plan.env,+ preexec_fn = plan.preexec_fn,+ backend = "limits",+ )++ sandbox_probe.probe(_Backend("limits", report_limits))+ (nofile_soft, _), (cpu_soft, cpu_hard) = ast.literal_eval(limits_file.read_text())+ assert nofile_soft <= sandbox_probe._PROBE_NOFILE+ assert cpu_soft == cpu_hard == sandbox_probe._PROBE_CPU_SECONDS++def test_a_backend_that_only_prints_the_token_is_not_believed():def liar(plan):return PreparedSandboxLaunch(@@ -372,16 +472,25 @@ def test_linux_runtime_identity_includes_the_verified_bwrap(monkeypatch):assert os_sandbox._runtime_identity() != first-def test_an_expired_verdict_is_re_probed(monkeypatch):- """Set before the first probe: the expiry is stamped when the verdict is stored. With no stale- grace the re-probe is in the foreground (test_permission_mode_degrade covers the grace)."""- monkeypatch.setattr(sandbox_probe, "_CACHE_TTL_SECONDS", 0.0)- monkeypatch.setattr(sandbox_probe, "_STALE_GRACE_SECONDS", 0.0)+def test_a_verdict_is_never_re_probed_on_its_own(monkeypatch):+ import types++ leap = {"now": 1000.0}++ def monotonic():+ leap["now"] += 86400.0+ return leap["now"]++ monkeypatch.setattr(+ sandbox_probe, "time", types.SimpleNamespace(monotonic = monotonic), raising = False+ )backend = _Backend("confining", _confining)- sandbox_probe.probe(backend)+ for _ in range(5):+ assert sandbox_probe.probe(backend)[0] is True+ assert backend.calls == 1+ sandbox_probe.reset_probe_cache()sandbox_probe.probe(backend)assert backend.calls == 2- assert not sandbox_probe._refreshingdef test_the_cache_cannot_grow_without_bound():@@ -489,53 +598,7 @@ def test_the_landlock_helper_imports_where_it_will_never_be_used():sandbox_landlock.abstract_scope_supported.cache_clear()-def test_a_cached_verdict_outlives_the_old_sixty_second_window(monkeypatch, tmp_path):- """The TTL was 60s, so an idle chat paid the cold probe again on its next message: a call 70s after the last one cost 1.24s against 0.040s unsandboxed."""- from core.inference import sandbox_probe-- assert sandbox_probe._CACHE_TTL_SECONDS >= 600, (- "the probe TTL is back under 10 minutes; an idle chat pays the cold "- "probe again on its next message"- )-- calls = []-- class Backend:- BACKEND_NAME = "ttl-probe"-- @staticmethod- def prepare(plan):- calls.append(plan)- raise AssertionError("the probe should not have re-run inside the TTL")-- sandbox_probe.reset_probe_cache()- clock = {"now": 1000.0}- monkeypatch.setattr(sandbox_probe.time, "monotonic", lambda: clock["now"])-- from core.inference import os_sandbox as _os_sandbox-- key = (Backend.BACKEND_NAME, _os_sandbox._runtime_identity())- sandbox_probe._cache[key] = (- clock["now"] + sandbox_probe._CACHE_TTL_SECONDS,- True,- "seeded",- )-- clock["now"] += 70.0- available, reason = sandbox_probe.probe(Backend)- assert (available, reason) == (True, "seeded")- assert calls == [], "the probe re-ran 70s later, which is the cost this fixed"---def test_an_unavailable_verdict_is_not_cached_for_the_long_window(monkeypatch, tmp_path):- """A user who installs bubblewrap must not wait 15 minutes for it to count."""- from core.inference import os_sandbox as _os_sandbox- from core.inference import sandbox_probe-- assert (- sandbox_probe._CACHE_TTL_UNAVAILABLE_SECONDS <= 120- ), "an unavailable verdict is cached long enough to outlive the user's own remediation"-+def test_an_unavailable_verdict_waits_for_refresh_or_a_new_runtime(monkeypatch):probes = []class Backend:@@ -546,24 +609,17 @@ def test_an_unavailable_verdict_is_not_cached_for_the_long_window(monkeypatch, tprobes.append(plan)raise RuntimeError("bubblewrap is not installed")+ identity = ["before"]+ monkeypatch.setattr(os_sandbox, "_runtime_identity", lambda: identity[0])sandbox_probe.reset_probe_cache()- clock = {"now": 5000.0}- monkeypatch.setattr(sandbox_probe.time, "monotonic", lambda: clock["now"])-- available, _ = sandbox_probe.probe(Backend)- assert available is False+ for _ in range(5):+ assert sandbox_probe.probe(Backend)[0] is Falseassert len(probes) == 1-- clock["now"] += 30.0+ assert sandbox_probe.probe(Backend, force = True)[0] is False+ assert len(probes) == 2+ identity[0] = "bubblewrap installed"sandbox_probe.probe(Backend)- assert len(probes) == 1, "an unavailable verdict should still be cached briefly"-- clock["now"] += 100.0- sandbox_probe.probe(Backend)- assert len(probes) == 2, (- "the host was never re-examined, so installing bubblewrap does not "- "take effect until the long TTL expires"- )+ assert len(probes) == 3def test_the_capability_names_the_empty_proc_layout_it_was_probed_with(monkeypatch):