Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop the macOS sandbox probe from leaking processes that spin forever (#15345)

Michael Han6285c116e992
diff --git a/studio/backend/core/inference/mxc_probe.py b/studio/backend/core/inference/mxc_probe.py
index da023f12b93..93913fea204 100644
--- a/studio/backend/core/inference/mxc_probe.py
+++ b/studio/backend/core/inference/mxc_probe.py
@@ -18,12 +18,9 @@ from types import SimpleNamespace
from . import mxc_adapter, mxc_drive_alias, mxc_policy, mxc_read_grants, mxc_runtime
_lock = threading.Lock()
-_cache: dict[tuple, tuple[float, bool, str]] = {}
+# No TTL, as sandbox_probe: the key and invalidate_cache() decide; a timer only re-ran the container.
+_cache: dict[tuple, tuple[bool, str]] = {}
_inflight: dict[tuple, threading.Event] = {}
-POSITIVE_TTL = 300.0
-NEGATIVE_TTL = 30.0
-# Re-probing a shell that cannot start costs ~15s of DACL-tier launches; the key changes with the runtime or the shell.
-INCOMPATIBLE_TTL = 6 * 3600.0
MSYS_NAMESPACE_REASON = (
"Git Bash (MSYS2) cannot start in the MXC container, which denies the global named-object "
"directory it creates (microsoft/mxc#1061)."
@@ -33,13 +30,17 @@ NO_BUILTIN_CONTAINER_REASON = (
)
-_host_prep_cache: dict[str, tuple[float, str | None]] = {}
+_host_prep_cache: dict[str, str | None] = {}
+# Bumped by invalidate_cache(): a check started before it must not publish.
+_generation = 0
def invalidate_cache() -> None:
+ global _generation
with _lock:
_cache.clear()
_host_prep_cache.clear()
+ _generation += 1
# Only when already loaded: the MXC child paths import this module without the tool stack.
os_sandbox = sys.modules.get(f"{__package__}.os_sandbox")
if os_sandbox is not None:
@@ -64,9 +65,9 @@ def host_prep_remediation() -> str | None:
except Exception: # noqa: BLE001 - no runtime, nothing to prepare
return None
with _lock:
- cached = _host_prep_cache.get(identity)
- if cached is not None and time.monotonic() < cached[0]:
- return cached[1]
+ if identity in _host_prep_cache:
+ return _host_prep_cache[identity]
+ generation = _generation
steps = mxc_runtime.probe_host_prep_steps(env = mxc_adapter._control_environment())
advice = None
if steps:
@@ -82,7 +83,9 @@ def host_prep_remediation() -> str | None:
"Prepare this PC."
)
with _lock:
- _host_prep_cache[identity] = (time.monotonic() + NEGATIVE_TTL, advice)
+ # None = wxc-exec could not tell: asked again next time rather than kept as "nothing missing".
+ if generation == _generation and steps is not None:
+ _host_prep_cache[identity] = advice
return advice
@@ -374,12 +377,13 @@ def probe(
while True:
with _lock:
cached = _cache.get(key)
- if cached is not None and not force and time.monotonic() < cached[0]:
- return cached[1], cached[2]
+ if cached is not None and not force:
+ return cached
flight = _inflight.get(key)
if flight is None:
flight = threading.Event()
_inflight[key] = flight
+ generation = _generation
break
while not flight.wait(0.05):
if cancel_event is not None and cancel_event.is_set():
@@ -388,11 +392,8 @@ def probe(
try:
result = _probe(selected_executable, execution_kind, cancel_event)
with _lock:
- if cancel_event is None or not cancel_event.is_set():
- ttl = POSITIVE_TTL if result[0] else NEGATIVE_TTL
- if result[1] == MSYS_NAMESPACE_REASON:
- ttl = INCOMPATIBLE_TTL
- _cache[key] = (time.monotonic() + ttl, *result)
+ if generation == _generation and (cancel_event is None or not cancel_event.is_set()):
+ _cache[key] = result
return result
finally:
with _lock:
diff --git a/studio/backend/core/inference/os_sandbox.py b/studio/backend/core/inference/os_sandbox.py
index 27d89901ca7..77427120985 100644
--- a/studio/backend/core/inference/os_sandbox.py
+++ b/studio/backend/core/inference/os_sandbox.py
@@ -1045,11 +1045,10 @@ def capability_snapshot(
)
-# Last answer per tool; the "off" permission gate reads it on every call without probing.
-_TOOL_ISOLATION_TTL_SECONDS = 60.0
+# Last answer per tool, read by the "off" gate; a reset, a real launch or a new _host_key() replaces it.
ISOLATED_TOOLS = ("python", "terminal")
_tool_isolation_lock = threading.Lock()
-_tool_isolation: dict[str, tuple[float, bool, str, str]] = {}
+_tool_isolation: dict[str, tuple[bool, str, str, str]] = {}
_tool_isolation_refreshing: set[str] = set()
# A reset raises the generation floor; answers from older checks are dropped.
_tool_isolation_generation = 0
@@ -1088,6 +1087,10 @@ def note_tool_isolation(
global _tool_isolation_generation
if tool not in ISOLATED_TOOLS:
return
+ from .sandbox_probe import UndecidedReason
+
+ # Host resource exhaustion is not a verdict: an empty key makes the next read check again.
+ key = "" if isinstance(reason, UndecidedReason) else _host_key()
with _tool_isolation_lock:
if generation is None:
_tool_isolation_generation += 1
@@ -1095,7 +1098,7 @@ def note_tool_isolation(
elif generation < _tool_isolation_floor or generation < _tool_isolation_noted.get(tool, 0):
return
_tool_isolation_noted[tool] = generation
- _tool_isolation[tool] = (time.monotonic(), bool(available), backend, reason)
+ _tool_isolation[tool] = (bool(available), backend, reason, key)
def tool_isolation_generation() -> int:
@@ -1154,15 +1157,27 @@ def cached_tool_isolation(tool: str) -> bool | None:
return None if cached is None else cached[0]
+def _host_key() -> str:
+ """Stat-only (no bwrap preflight): moves when the venv is swapped or bwrap installed."""
+ key = os.path.realpath(sys.executable)
+ if sys.platform == "linux":
+ from . import sandbox_linux
+ try:
+ key += repr(sandbox_linux._bwrap_file_identity(sandbox_linux._trusted_bwrap_path()))
+ except Exception as exc: # noqa: BLE001 - no trusted bwrap is a key too
+ key += type(exc).__name__
+ return key
+
+
def cached_tool_capability(tool: str) -> tuple[bool, str, str] | None:
- """Never blocks: last (available, backend, reason) or None; stale starts a background refresh."""
- now = time.monotonic()
+ """Never blocks: last (available, backend, reason) or None; a missing or other-key answer starts one check."""
+ key = _host_key()
with _tool_isolation_lock:
entry = _tool_isolation.get(tool)
start = (
tool in ISOLATED_TOOLS
and not _background_probes_disabled()
- and (entry is None or now - entry[0] >= _TOOL_ISOLATION_TTL_SECONDS)
+ and (entry is None or entry[3] != key)
and tool not in _tool_isolation_refreshing
)
if start:
@@ -1178,7 +1193,7 @@ def cached_tool_capability(tool: str) -> tuple[bool, str, str] | None:
except RuntimeError:
with _tool_isolation_lock:
_tool_isolation_refreshing.discard(tool)
- return None if entry is None else (entry[1], entry[2], entry[3])
+ return None if entry is None else entry[:3]
def warm_tool_isolation() -> None:
@@ -1366,7 +1381,7 @@ def prepare_tool_launch(plan: ToolLaunchPlan) -> PreparedSandboxLaunch:
except (WorkdirUnsafeError, SandboxBuildError):
raise # can be tool-induced; says nothing about the backend itself
except SandboxUnavailableError:
- # Cached pass is stale: reset so "off" asks instead of failing until expiry.
+ # Cached pass is stale: reset so "off" asks instead of failing on it until a Refresh.
from .sandbox_probe import reset_probe_cache
reset_probe_cache()
raise
diff --git a/studio/backend/core/inference/sandbox_macos.py b/studio/backend/core/inference/sandbox_macos.py
index 903b02fe339..a1637aedf75 100644
--- a/studio/backend/core/inference/sandbox_macos.py
+++ b/studio/backend/core/inference/sandbox_macos.py
@@ -597,6 +597,8 @@ def build_profile(
# bash process substitution opens /dev/fd/63.
'(allow file-read* (regex #"^/dev/fd/[0-9]+$"))',
'(allow file-write* (regex #"^/dev/fd/[0-9]+$"))',
+ # Unlistable, CPython close()s every fd up to RLIMIT_NOFILE before exec: forever when unlimited.
+ '(allow file-read* (literal "/dev/fd"))',
_rule("allow file-ioctl", device_filters),
"(allow ipc-posix-sem)",
# write-data: libomp writes its registration into the segment.
diff --git a/studio/backend/core/inference/sandbox_probe.py b/studio/backend/core/inference/sandbox_probe.py
index b34af098adf..2f1c5014b78 100644
--- a/studio/backend/core/inference/sandbox_probe.py
+++ b/studio/backend/core/inference/sandbox_probe.py
@@ -6,15 +6,16 @@
from __future__ import annotations
import ctypes
import errno
+import itertools
import os
import re
import shutil
+import signal
import socket
import subprocess
import sys
import tempfile
import threading
-import time
from typing import Any
from loggers import get_logger
@@ -26,6 +27,15 @@ try:
_libc = ctypes.CDLL(None, use_errno = True) if sys.platform == "linux" else None
except OSError: # pragma: no cover - a libc that will not load
_libc = None
+try:
+ import resource as _resource
+except ImportError: # pragma: no cover - Windows never probes
+ _resource = None
+
+# The tool launch's own fd cap (tools._sandbox_preexec), so the probe qualifies what tools get.
+_PROBE_NOFILE = 16384
+# Kernel-enforced: ends a payload process nobody is left to kill.
+_PROBE_CPU_SECONDS = 60
# ALONE on stdout: a payload that printed it early and died would satisfy "in".
PROBE_TOKEN = "UNSLOTH_SANDBOX_PROBE_OK"
@@ -34,18 +44,12 @@ _SENTINEL_TOKEN = "unsloth-host-sentinel-must-not-be-readable"
_OUTSIDE_WRITE_TOKEN = "unsloth-sandbox-escaped-to-the-host"
PROBE_TIMEOUT_SECONDS = 30.0
-# Correctness does not rest on the TTL: the key carries the runtime identity and a failed launch invalidates.
-_CACHE_TTL_SECONDS = 900.0
-# UNAVAILABLE keeps a short TTL: the user fixing their host does not move the cache key.
-_CACHE_TTL_UNAVAILABLE_SECONDS = 60.0
+# No TTL: re-probing on a timer turned one hung probe into one more each minute on macOS.
_CACHE_MAX_ENTRIES = 8
# ``sun_path`` is 108 bytes and multiprocessing appends about 32.
_MAX_PROBE_BASE_LEN = 59
-# A stale PASS is served while one background re-probe runs; a FAIL never is.
-_STALE_GRACE_SECONDS = 3600.0
-
# Tool code can exhaust these from the jail: undecided, never cached, and auto refuses rather than falls back.
_RESOURCE_ERRNOS = frozenset(
code
@@ -97,9 +101,11 @@ def _classified(reason: str, exc: BaseException | None = None) -> str:
_cache_lock = threading.Lock()
-_cache: dict[tuple[str, str], tuple[float, bool, str]] = {}
-_refreshing: set[tuple[str, str]] = set()
+_cache: dict[tuple[str, str], tuple[bool, str]] = {}
_generation = 0
+# A slower, older probe must not replace a newer (forced) verdict.
+_tickets = itertools.count(1)
+_published: dict[tuple[str, str], int] = {}
def reset_probe_cache() -> None:
@@ -107,6 +113,7 @@ def reset_probe_cache() -> None:
global _generation
with _cache_lock:
_cache.clear()
+ _published.clear()
_generation += 1
if sys.platform == "linux":
try:
@@ -121,73 +128,26 @@ def reset_probe_cache() -> None:
forget_tool_isolation()
-def _cache_get(key: tuple[str, str]) -> tuple[bool, str, bool] | None:
- """(available, reason, stale); None when nothing may be served."""
- now = time.monotonic()
- with _cache_lock:
- entry = _cache.get(key)
- if entry is None:
- return None
- expires_at, available, reason = entry
- if expires_at > now:
- return available, reason, False
- if available and now < expires_at + _STALE_GRACE_SECONDS:
- return available, reason, True
- _cache.pop(key, None)
- return None
-
-
def _cache_put(
key: tuple[str, str],
available: bool,
reason: str,
*,
generation: int | None = None,
+ ticket: int = 0,
) -> None:
with _cache_lock:
if generation is not None and generation != _generation:
return
- ttl = _CACHE_TTL_SECONDS if available else _CACHE_TTL_UNAVAILABLE_SECONDS
+ if ticket < _published.get(key, 0):
+ return
+ _published[key] = ticket
_cache.pop(key, None)
- _cache[key] = (time.monotonic() + ttl, available, reason)
+ _cache[key] = (available, reason)
while len(_cache) > _CACHE_MAX_ENTRIES:
_cache.pop(next(iter(_cache)))
-def _background_probes_disabled() -> bool:
- try:
- from .os_sandbox import _background_probes_disabled as disabled
- except Exception: # noqa: BLE001 - a partially imported package: stay in the foreground
- return True
- return disabled()
-
-
-def _refresh_in_background(backend: Any, key: tuple[str, str]) -> None:
- with _cache_lock:
- if key in _refreshing:
- return
- _refreshing.add(key)
- generation = _generation
-
- def run() -> None:
- try:
- from .os_sandbox import ToolLaunchPlan
- available, reason = _run_probe(backend, key[0], ToolLaunchPlan)
- if not isinstance(reason, UndecidedReason):
- _cache_put(key, available, reason, generation = generation)
- except Exception as exc: # noqa: BLE001 - the stale PASS just ages out
- logger.debug("background sandbox re-probe failed: %s", exc)
- finally:
- with _cache_lock:
- _refreshing.discard(key)
-
- try:
- threading.Thread(target = run, name = "unsloth-sandbox-reprobe", daemon = True).start()
- except RuntimeError:
- with _cache_lock:
- _refreshing.discard(key)
-
-
_PREAMBLE = '''import multiprocessing.reduction, os, socket, subprocess, sys
@@ -296,6 +256,53 @@ def _no_new_privs() -> None:
logger.warning("The sandbox probe could not set PR_SET_NO_NEW_PRIVS")
+def _probe_preexec() -> None:
+ """In the forked child, before the backend's wrapper: the payload's limits, then no_new_privs."""
+ if _resource is not None:
+ try:
+ _soft, hard = _resource.getrlimit(_resource.RLIMIT_NOFILE)
+ nofile = _PROBE_NOFILE if hard == _resource.RLIM_INFINITY else min(_PROBE_NOFILE, hard)
+ _resource.setrlimit(_resource.RLIMIT_NOFILE, (nofile, nofile))
+ except (ValueError, OSError):
+ pass
+ try:
+ _resource.setrlimit(_resource.RLIMIT_CPU, (_PROBE_CPU_SECONDS, _PROBE_CPU_SECONDS))
+ except (ValueError, OSError):
+ pass
+ _no_new_privs()
+
+
+def _end_probe_group(proc: subprocess.Popen) -> None:
+ """killpg, not kill: the payload's own children outlive a killed payload, and on macOS one spun forever.
+ Only while the leader is unreaped: once reaped and its group empty, the id can name someone else's group."""
+ if proc.returncode is None:
+ try:
+ os.killpg(proc.pid, signal.SIGKILL)
+ except OSError:
+ proc.kill()
+ proc.wait()
+ for pipe in (proc.stdout, proc.stderr):
+ if pipe is not None:
+ pipe.close()
+
+
+def _run_grouped(argv, **popen_kwargs) -> subprocess.CompletedProcess:
+ """subprocess.run in its own session; the whole group ends with it."""
+ proc = subprocess.Popen(
+ argv,
+ stdin = subprocess.DEVNULL,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.PIPE,
+ start_new_session = True,
+ **popen_kwargs,
+ )
+ try:
+ stdout, stderr = proc.communicate(timeout = PROBE_TIMEOUT_SECONDS)
+ finally:
+ _end_probe_group(proc)
+ return subprocess.CompletedProcess(argv, proc.returncode, stdout, stderr)
+
+
def _abstract_control() -> "tuple[bytes | None, Any]":
"""Nothing without a scope to test, or when the host cannot reach its own socket."""
if sys.platform != "linux":
@@ -375,14 +382,11 @@ def _host_positive_controls(workdir: str, sentinel: str, outside: str, env: dict
f"the host itself could not write outside the probe workdir ({exc})", exc
)
try:
- completed = subprocess.run(
+ completed = _run_grouped(
(sys.executable, "-I", "-S", "-c", _host_payload(workdir)),
cwd = workdir,
env = env,
- stdin = subprocess.DEVNULL,
- stdout = subprocess.PIPE,
- stderr = subprocess.PIPE,
- timeout = PROBE_TIMEOUT_SECONDS,
+ preexec_fn = _probe_preexec,
)
except (OSError, subprocess.SubprocessError) as exc:
return _classified(
@@ -407,23 +411,19 @@ def probe(backend: Any, *, force: bool = False) -> tuple[bool, str]:
return False, f"the sandbox probe could not identify this runtime: {exc}"
if not force:
- cached = _cache_get(key)
+ with _cache_lock:
+ cached = _cache.get(key)
if cached is not None:
- available, reason, stale = cached
- if not stale:
- return available, reason
- if not _background_probes_disabled():
- _refresh_in_background(backend, key)
- return available, reason
-
- # Background probes are off: re-probe here.
+ return cached
+
with _cache_lock:
generation = _generation
+ ticket = next(_tickets)
available, reason = _run_probe(backend, backend_name, ToolLaunchPlan)
if isinstance(reason, UndecidedReason):
# Re-probed on the next call: the resource may be freed by then.
return False, reason
- _cache_put(key, available, reason, generation = generation)
+ _cache_put(key, available, reason, generation = generation, ticket = ticket)
return available, reason
@@ -473,21 +473,17 @@ def _run_probe(backend: Any, backend_name: str, plan_cls: Any) -> tuple[bool, st
workdir = workdir,
env = env,
# A setuid bwrap dies under no_new_privs, so without this the probe qualifies a launch that fails after Popen.
- preexec_fn = _no_new_privs,
+ preexec_fn = _probe_preexec,
requested_mode = "required",
execution_kind = "python",
)
prepared = backend.prepare(plan)
- completed = subprocess.run(
+ completed = _run_grouped(
prepared.argv,
cwd = prepared.workdir,
env = prepared.env,
- stdin = subprocess.DEVNULL,
- stdout = subprocess.PIPE,
- stderr = subprocess.PIPE,
preexec_fn = prepared.preexec_fn,
pass_fds = tuple(prepared.pass_fds),
- timeout = PROBE_TIMEOUT_SECONDS,
)
if completed.returncode != 0 or completed.stdout.strip() != PROBE_TOKEN.encode():
detail = (completed.stderr or b"").decode("utf-8", "replace").strip()[-300:]
diff --git a/studio/backend/tests/test_mxc_git_bash_namespace.py b/studio/backend/tests/test_mxc_git_bash_namespace.py
index 6c4121e7bb1..94a4e3af83a 100644
--- a/studio/backend/tests/test_mxc_git_bash_namespace.py
+++ b/studio/backend/tests/test_mxc_git_bash_namespace.py
@@ -111,7 +111,12 @@ def test_a_missing_builtin_container_is_named(monkeypatch, tmp_path, executable,
def _count_live_probes(monkeypatch, reason):
clock = [1000.0]
calls = []
- monkeypatch.setattr(mxc_probe.time, "monotonic", lambda: clock[0])
+
+ def leaping_monotonic():
+ clock[0] += 86400.0
+ return clock[0]
+
+ monkeypatch.setattr(mxc_probe.time, "monotonic", leaping_monotonic)
monkeypatch.setattr(mxc_probe.mxc_runtime, "installation_identity", lambda: "runtime-a")
def live_probe(
@@ -126,28 +131,46 @@ def _count_live_probes(monkeypatch, reason):
return clock, calls
-def test_an_incompatible_shell_is_not_probed_again_each_negative_ttl(monkeypatch, tmp_path):
- clock, calls = _count_live_probes(monkeypatch, mxc_probe.MSYS_NAMESPACE_REASON)
+ "reason", [mxc_probe.MSYS_NAMESPACE_REASON, "the live MXC probe did not complete cleanly"]
+)
+def test_a_failure_is_held_until_invalidated_not_until_a_timer(monkeypatch, tmp_path, reason):
+ _clock, calls = _count_live_probes(monkeypatch, reason)
bash = str(tmp_path / "bash.exe")
for _ in range(5):
- assert mxc_probe.probe(bash, execution_kind = "terminal") == (
- False,
- mxc_probe.MSYS_NAMESPACE_REASON,
- )
- clock[0] += mxc_probe.NEGATIVE_TTL + 1
+ assert mxc_probe.probe(bash, execution_kind = "terminal") == (False, reason)
assert len(calls) == 1
- clock[0] += mxc_probe.INCOMPATIBLE_TTL
+ mxc_probe.invalidate_cache()
mxc_probe.probe(bash, execution_kind = "terminal")
assert len(calls) == 2
+ mxc_probe.probe(bash, execution_kind = "terminal", force = True)
+ assert len(calls) == 3
+
+def test_an_invalidate_during_a_probe_keeps_its_result_out(monkeypatch, tmp_path):
+ import threading
-def test_a_generic_failure_still_expires_at_the_negative_ttl(monkeypatch, tmp_path):
- clock, calls = _count_live_probes(monkeypatch, "the live MXC probe did not complete cleanly")
+ _clock, calls = _count_live_probes(monkeypatch, "blocked before the fix")
+ started, release = threading.Event(), threading.Event()
+ live = mxc_probe._probe
+
+ def slow(*args, **kwargs):
+ started.set()
+ release.wait(5)
+ return live(*args, **kwargs)
+
+ monkeypatch.setattr(mxc_probe, "_probe", slow)
bash = str(tmp_path / "bash.exe")
+ worker = threading.Thread(
+ target = mxc_probe.probe, args = (bash,), kwargs = {"execution_kind": "terminal"}
+ )
+ worker.start()
+ assert started.wait(5)
+ mxc_probe.invalidate_cache()
+ release.set()
+ worker.join(5)
mxc_probe.probe(bash, execution_kind = "terminal")
- clock[0] += mxc_probe.NEGATIVE_TTL + 1
- mxc_probe.probe(bash, execution_kind = "terminal")
- assert len(calls) == 2
+ assert len(calls) == 2, "a verdict from before the invalidate was cached"
def test_a_new_runtime_or_opt_in_probes_the_shell_again(monkeypatch, tmp_path):
diff --git a/studio/backend/tests/test_mxc_windows_backend.py b/studio/backend/tests/test_mxc_windows_backend.py
index 2695cdee82a..edcd0210721 100644
--- a/studio/backend/tests/test_mxc_windows_backend.py
+++ b/studio/backend/tests/test_mxc_windows_backend.py
@@ -113,6 +113,21 @@ def test_unavailable_capability_names_the_host_prep_command(monkeypatch, tmp_pat
assert "install_mxc_prebuilt.py" in capability.remediation or not advised
+def test_an_inconclusive_host_prep_check_is_asked_again(monkeypatch):
+ from core.inference import mxc_probe
+
+ monkeypatch.setattr(mxc_probe, "_host_prep_cache", {})
+ monkeypatch.setattr(mxc_probe.mxc_runtime, "installation_identity", lambda: "runner")
+ answers = [None, ("prepare-null-device",)]
+ monkeypatch.setattr(
+ mxc_probe.mxc_runtime, "probe_host_prep_steps", lambda **_kwargs: answers.pop(0)
+ )
+ assert mxc_probe.host_prep_remediation() is None
+ assert "prepare-null-device" in mxc_probe.host_prep_remediation()
+ assert "prepare-null-device" in mxc_probe.host_prep_remediation()
+ assert answers == []
+
+
def test_host_prep_is_not_advised_without_the_dacl_opt_in(monkeypatch, tmp_path):
from core.inference import mxc_probe, sandbox_windows_mxc
diff --git a/studio/backend/tests/test_permission_mode_degrade.py b/studio/backend/tests/test_permission_mode_degrade.py
index 397f9ac9415..1d597c8a6a1 100644
--- a/studio/backend/tests/test_permission_mode_degrade.py
+++ b/studio/backend/tests/test_permission_mode_degrade.py
@@ -319,27 +319,30 @@ def test_no_startup_warmup_on_windows(monkeypatch):
assert called == []
-class _Clock:
+class _LeapingClock:
+ """Each read is a day later, so any TTL left in the code expires between two calls."""
+
def __init__(self):
self.now = 1000.0
def __call__(self):
+ self.now += 86400.0
return self.now
+def _leap_time(monkeypatch, module):
+ import types
+ clock = types.SimpleNamespace(monotonic = _LeapingClock(), time = time.time, sleep = time.sleep)
+ monkeypatch.setattr(module, "time", clock, raising = False)
+
+
@pytest.fixture
def probe_env(monkeypatch):
monkeypatch.setenv(os_sandbox.WARMUP_DISABLE_ENV, "0")
sandbox_probe.reset_probe_cache()
- clock = _Clock()
- import types
-
- monkeypatch.setattr(
- sandbox_probe,
- "time",
- types.SimpleNamespace(monotonic = clock, time = time.time, sleep = time.sleep),
- )
- monkeypatch.setattr(os_sandbox, "_runtime_identity", lambda: "identity")
+ _leap_time(monkeypatch, sandbox_probe)
+ identity = ["identity"]
+ monkeypatch.setattr(os_sandbox, "_runtime_identity", lambda: identity[0])
verdicts = []
gate = threading.Event()
gate.set()
@@ -351,7 +354,7 @@ def probe_env(monkeypatch):
return verdicts.pop(0)
monkeypatch.setattr(sandbox_probe, "_run_probe", run_probe)
- yield clock, verdicts, gate, calls
+ yield identity, verdicts, gate, calls
sandbox_probe.reset_probe_cache()
@@ -366,73 +369,129 @@ def _wait_for(predicate, timeout = 5.0):
return predicate()
-def test_a_stale_pass_is_served_while_one_reprobe_runs(probe_env):
- clock, verdicts, gate, calls = probe_env
- verdicts.extend([(True, "first"), (True, "second")])
- assert sandbox_probe.probe(_Backend()) == (True, "first")
- clock.now += sandbox_probe._CACHE_TTL_SECONDS + 1
[email protected]("first", [(True, "ok"), (False, "blocked")])
+def test_a_verdict_is_held_until_an_event_not_a_timer(probe_env, first):
+ identity, verdicts, gate, calls = probe_env
+ verdicts.extend([first, (True, "after reset"), (False, "forced"), (True, "new runtime")])
+ assert sandbox_probe.probe(_Backend()) == first
+ for _ in range(20):
+ assert sandbox_probe.probe(_Backend()) == first
+ assert len(calls) == 1
+ sandbox_probe.reset_probe_cache()
+ assert sandbox_probe.probe(_Backend()) == (True, "after reset")
+ assert sandbox_probe.probe(_Backend(), force = True) == (False, "forced")
+ identity[0] = "swapped venv"
+ assert sandbox_probe.probe(_Backend()) == (True, "new runtime")
+ assert len(calls) == 4
+ assert all(name == threading.current_thread().name for name in calls)
+
+
+def test_an_undecided_verdict_is_retried_by_the_next_caller(probe_env):
+ identity, verdicts, gate, calls = probe_env
+ verdicts.extend([(False, sandbox_probe.UndecidedReason("out of disk")), (True, "ok")])
+ assert sandbox_probe.probe(_Backend())[0] is False
+ assert sandbox_probe.probe(_Backend()) == (True, "ok")
+ assert sandbox_probe.probe(_Backend()) == (True, "ok")
+ assert len(calls) == 2
+
+
+def test_a_reset_during_a_probe_keeps_its_result_out(probe_env):
+ identity, verdicts, gate, calls = probe_env
+ verdicts.extend([(True, "from before the reset"), (False, "after")])
gate.clear()
- assert sandbox_probe.probe(_Backend()) == (True, "first")
- assert sandbox_probe.probe(_Backend()) == (True, "first")
- assert _wait_for(lambda: len(calls) == 2)
- assert calls[1] == "unsloth-sandbox-reprobe"
+ worker = threading.Thread(target = sandbox_probe.probe, args = (_Backend(),))
+ worker.start()
+ assert _wait_for(lambda: len(calls) == 1)
+ sandbox_probe.reset_probe_cache()
gate.set()
- assert _wait_for(lambda: sandbox_probe.probe(_Backend()) == (True, "second"))
- assert len(calls) == 2
+ worker.join(5)
+ assert sandbox_probe.probe(_Backend()) == (False, "after")
-def test_a_failed_reprobe_replaces_the_stale_pass(probe_env):
- clock, verdicts, gate, calls = probe_env
- verdicts.extend([(True, "ok"), (False, "now blocked")])
- sandbox_probe.probe(_Backend())
- clock.now += sandbox_probe._CACHE_TTL_SECONDS + 1
- assert sandbox_probe.probe(_Backend()) == (True, "ok")
- assert _wait_for(lambda: sandbox_probe.probe(_Backend()) == (False, "now blocked"))
+def test_an_older_probe_finishing_last_never_replaces_a_newer_verdict(probe_env, monkeypatch):
+ """With no TTL, a slow ordinary probe landing after a forced Refresh would have stuck for good."""
+ identity, verdicts, gate, calls = probe_env
+ slow_started, release_slow = threading.Event(), threading.Event()
+ def run_probe(backend, name, plan_cls):
+ if not slow_started.is_set():
+ slow_started.set()
+ release_slow.wait(5)
+ return False, "old failure"
+ return True, "forced pass"
-def test_a_fail_is_never_served_stale(probe_env):
- clock, verdicts, gate, calls = probe_env
- verdicts.extend([(False, "blocked"), (True, "fixed")])
- assert sandbox_probe.probe(_Backend()) == (False, "blocked")
- clock.now += sandbox_probe._CACHE_TTL_UNAVAILABLE_SECONDS + 1
- assert sandbox_probe.probe(_Backend()) == (True, "fixed")
- assert all(name != "unsloth-sandbox-reprobe" for name in calls)
+ monkeypatch.setattr(sandbox_probe, "_run_probe", run_probe)
+ slow = threading.Thread(target = sandbox_probe.probe, args = (_Backend(),))
+ slow.start()
+ assert slow_started.wait(5)
+ assert sandbox_probe.probe(_Backend(), force = True) == (True, "forced pass")
+ release_slow.set()
+ slow.join(5)
+ assert sandbox_probe.probe(_Backend()) == (True, "forced pass")
-def test_a_pass_past_the_grace_is_probed_in_the_foreground(probe_env):
- clock, verdicts, gate, calls = probe_env
- verdicts.extend([(True, "old"), (True, "new")])
- sandbox_probe.probe(_Backend())
- clock.now += sandbox_probe._CACHE_TTL_SECONDS + sandbox_probe._STALE_GRACE_SECONDS + 1
- assert sandbox_probe.probe(_Backend()) == (True, "new")
- assert all(name != "unsloth-sandbox-reprobe" for name in calls)
+def test_an_undecided_answer_is_checked_again_by_the_next_read(monkeypatch):
+ monkeypatch.delenv(os_sandbox.WARMUP_DISABLE_ENV, raising = False)
+ started = []
+ def refresh(tool, *, force = False):
+ started.append(tool)
+ os_sandbox.note_tool_isolation(tool, True, backend = "bubblewrap")
+ return True
-def test_a_stale_pass_is_reprobed_in_the_foreground_when_background_probes_are_off(
- probe_env, monkeypatch
-):
- clock, verdicts, gate, calls = probe_env
- monkeypatch.setenv(os_sandbox.WARMUP_DISABLE_ENV, "1")
- verdicts.extend([(True, "old"), (False, "broke")])
- sandbox_probe.probe(_Backend())
- clock.now += sandbox_probe._CACHE_TTL_SECONDS + 1
- assert sandbox_probe.probe(_Backend()) == (False, "broke")
- assert all(name != "unsloth-sandbox-reprobe" for name in calls)
- assert not sandbox_probe._refreshing
-
-
-def test_a_reset_during_a_reprobe_keeps_its_result_out(probe_env):
- clock, verdicts, gate, calls = probe_env
- verdicts.extend([(True, "ok"), (True, "from before the reset"), (False, "after")])
- sandbox_probe.probe(_Backend())
- clock.now += sandbox_probe._CACHE_TTL_SECONDS + 1
- gate.clear()
- sandbox_probe.probe(_Backend())
- assert _wait_for(lambda: len(calls) == 2)
- sandbox_probe.reset_probe_cache()
- gate.set()
- assert _wait_for(lambda: not sandbox_probe._refreshing)
- assert sandbox_probe.probe(_Backend()) == (False, "after")
+ monkeypatch.setattr(os_sandbox, "refresh_tool_isolation", refresh)
+ os_sandbox.note_tool_isolation(
+ "python", False, backend = "none", reason = sandbox_probe.UndecidedReason("out of disk")
+ )
+ assert os_sandbox.cached_tool_capability("python")[0] is False
+ assert _wait_for(lambda: os_sandbox.cached_tool_isolation("python") is True)
+ for _ in range(5):
+ os_sandbox.cached_tool_capability("python")
+ assert started == ["python"]
+
+
+def test_polling_an_answered_tool_never_probes_again(monkeypatch):
+ monkeypatch.delenv(os_sandbox.WARMUP_DISABLE_ENV, raising = False)
+ _leap_time(monkeypatch, os_sandbox)
+ started = []
+ import types
+
+ monkeypatch.setattr(
+ os_sandbox,
+ "threading",
+ types.SimpleNamespace(
+ Thread = lambda *a, **k: started.append(k.get("name")) or threading.Thread()
+ ),
+ )
+ os_sandbox.note_tool_isolation("python", False, backend = "none", reason = "blocked")
+ for _ in range(50):
+ assert os_sandbox.cached_tool_capability("python")[:3] == (False, "none", "blocked")
+ assert tool_policy.off_mode_still_gates("python") is True
+ assert started == [], "polling an answered tool started a re-probe"
+
+
+def test_an_answer_noted_under_another_host_key_is_checked_once(monkeypatch):
+ monkeypatch.delenv(os_sandbox.WARMUP_DISABLE_ENV, raising = False)
+ key = ["no bwrap"]
+ monkeypatch.setattr(os_sandbox, "_host_key", lambda: key[0])
+ started = []
+
+ def refresh(tool, *, force = False):
+ started.append(tool)
+ os_sandbox.note_tool_isolation(tool, True, backend = "bubblewrap")
+ return True
+
+ monkeypatch.setattr(os_sandbox, "refresh_tool_isolation", refresh)
+ os_sandbox.note_tool_isolation("python", False, backend = "none", reason = "no bwrap")
+ for _ in range(5):
+ assert os_sandbox.cached_tool_capability("python")[0] is False
+ assert started == []
+ key[0] = "bwrap installed"
+ os_sandbox.cached_tool_capability("python")
+ assert _wait_for(lambda: os_sandbox.cached_tool_isolation("python") is True)
+ for _ in range(5):
+ os_sandbox.cached_tool_capability("python")
+ assert started == ["python"]
@pytest.fixture(autouse = True)
diff --git a/studio/backend/tests/test_sandbox_linux.py b/studio/backend/tests/test_sandbox_linux.py
index 0baae705f2a..d1aa3edbf6f 100644
--- a/studio/backend/tests/test_sandbox_linux.py
+++ b/studio/backend/tests/test_sandbox_linux.py
@@ -22,6 +22,7 @@ import subprocess
import sys
import sysconfig
import tempfile
+import textwrap
import threading
import time
@@ -1077,12 +1078,15 @@ def test_the_probe_launch_sets_no_new_privs_like_a_real_one(tmp_path):
assert sandbox_probe._PR_SET_NO_NEW_PRIVS == 38
source = inspect.getsource(sandbox_probe._run_probe)
- assert "preexec_fn = _no_new_privs" in source
- assert not [
- node
- for node in ast.walk(ast.parse(inspect.getsource(sandbox_probe._no_new_privs)))
- if isinstance(node, (ast.Import, ast.ImportFrom))
- ]
+ assert "preexec_fn = _probe_preexec" in source
+ assert "_no_new_privs()" in inspect.getsource(sandbox_probe._probe_preexec)
+ for hook in (sandbox_probe._no_new_privs, sandbox_probe._probe_preexec):
+ # Runs between fork and exec: an import there can deadlock on the import lock.
+ assert not [
+ node
+ for node in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(hook))))
+ if isinstance(node, (ast.Import, ast.ImportFrom))
+ ]
def test_the_pip_targets_script_directory_is_last_on_path(prepared):
diff --git a/studio/backend/tests/test_sandbox_macos.py b/studio/backend/tests/test_sandbox_macos.py
index 52a261926de..1229cc491d9 100644
--- a/studio/backend/tests/test_sandbox_macos.py
+++ b/studio/backend/tests/test_sandbox_macos.py
@@ -76,6 +76,10 @@ def test_profile_is_deny_default(profile):
assert "(allow default)" not in profile
+def test_the_fd_directory_is_listable_so_spawning_never_brute_forces_closes(profile):
+ assert '(allow file-read* (literal "/dev/fd"))' in profile.splitlines()
+
+
def test_every_rule_is_a_balanced_s_expression(profile):
depth = 0
in_string = False
@@ -411,6 +415,60 @@ def test_profile_compiles_under_sandbox_exec(tmp_path):
prepared.cleanup()
+@_darwin_only
+def test_a_child_spawns_promptly_under_an_unlimited_fd_limit(tmp_path):
+ """Before /dev/fd was listable, this spawn spun forever at 100% CPU, unkillable by its parent's timeout."""
+ import resource
+ import signal
+
+ _soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+ if hard != resource.RLIM_INFINITY:
+ pytest.skip("needs an unlimited hard RLIMIT_NOFILE to raise the soft limit to")
+
+ def unlimited():
+ resource.setrlimit(resource.RLIMIT_NOFILE, (resource.RLIM_INFINITY, hard))
+
+ workdir = tmp_path / "session"
+ workdir.mkdir()
+ spawn = (
+ "import subprocess, time\n"
+ "start = time.monotonic()\n"
+ "subprocess.run(['/usr/bin/true'], check = True)\n"
+ "print(time.monotonic() - start)\n"
+ )
+ prepared = backend.prepare(
+ ToolLaunchPlan(
+ argv = (sys.executable, "-I", "-S", "-c", spawn),
+ workdir = str(workdir),
+ env = {"PATH": "/usr/bin:/bin"},
+ preexec_fn = unlimited,
+ )
+ )
+ proc = subprocess.Popen(
+ prepared.argv,
+ env = prepared.env,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.PIPE,
+ text = True,
+ preexec_fn = prepared.preexec_fn,
+ start_new_session = True,
+ )
+ try:
+ stdout, stderr = proc.communicate(timeout = 30)
+ except subprocess.TimeoutExpired:
+ pytest.fail("spawning a child inside the sandbox did not finish within 30s")
+ finally:
+ if proc.returncode is None:
+ try:
+ os.killpg(proc.pid, signal.SIGKILL)
+ except OSError:
+ pass
+ proc.wait()
+ prepared.cleanup()
+ assert proc.returncode == 0, stderr
+ assert float(stdout.strip()) < 5.0
+
+
@_darwin_only
def test_home_is_unreadable_inside_the_sandbox(tmp_path):
"""The file is created and proven readable on the host first: a read that fails for an unrelated reason proves nothing."""
diff --git a/studio/backend/tests/test_sandbox_probe.py b/studio/backend/tests/test_sandbox_probe.py
index 4b536c081ce..d0bbbf939be 100644
--- a/studio/backend/tests/test_sandbox_probe.py
+++ b/studio/backend/tests/test_sandbox_probe.py
@@ -5,6 +5,7 @@
from __future__ import annotations
+import ast
import inspect
import os
import pathlib
@@ -13,6 +14,7 @@ import shutil
import subprocess
import sys
import tempfile
+import time
import pytest
@@ -257,6 +259,104 @@ def test_a_wedged_backend_times_out_instead_of_hanging(monkeypatch):
assert "timed out" in reason
+def _zombie(pid: int) -> bool:
+ """Killed but unreaped: a container whose PID 1 never reaps orphans keeps it in the table."""
+ try:
+ with open(f"/proc/{pid}/stat", encoding = "ascii") as handle:
+ return handle.read().rpartition(")")[2].split()[0] == "Z"
+ except (OSError, IndexError):
+ return False
+
+
+def _gone(pid: int, within: float = 10.0) -> bool:
+ deadline = time.monotonic() + within
+ while time.monotonic() < deadline:
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return True
+ if _zombie(pid):
+ return True
+ time.sleep(0.05)
+ return False
+
+
+def test_a_timed_out_probe_also_ends_what_the_payload_spawned(monkeypatch, tmp_path):
+ """The macOS leak: a killed payload left its stuck child spinning, re-probed every minute."""
+ monkeypatch.setattr(sandbox_probe, "PROBE_TIMEOUT_SECONDS", 1.0)
+ pid_file = tmp_path / "grandchild.pid"
+ payload = (
+ "import subprocess, sys, time\n"
+ "child = subprocess.Popen([sys.executable, '-I', '-S', '-c', 'import time; time.sleep(120)'])\n"
+ f"open({str(pid_file)!r}, 'w').write(str(child.pid))\n"
+ "time.sleep(120)\n"
+ )
+
+ def wedged_with_a_child(plan):
+ return PreparedSandboxLaunch(
+ argv = (sys.executable, "-I", "-S", "-c", payload),
+ workdir = plan.workdir,
+ env = plan.env,
+ preexec_fn = plan.preexec_fn,
+ backend = "wedged",
+ )
+
+ available, reason = sandbox_probe.probe(_Backend("wedged", wedged_with_a_child))
+ assert available is False and "timed out" in reason
+ grandchild = int(pid_file.read_text())
+ assert _gone(grandchild), f"the payload's child {grandchild} outlived the probe"
+
+
+def test_a_finished_probe_never_signals_a_group_it_no_longer_pins(monkeypatch):
+ """After communicate() reaps the leader, an emptied group's id can be reused by another process."""
+ signalled = []
+ monkeypatch.setattr(sandbox_probe.os, "killpg", lambda pgid, sig: signalled.append(pgid))
+ assert sandbox_probe.probe(_Backend("confining", _confining))[0] is True
+ assert signalled == []
+
+
+def test_a_timed_out_host_control_also_ends_what_it_spawned(monkeypatch, tmp_path):
+ monkeypatch.setattr(sandbox_probe, "PROBE_TIMEOUT_SECONDS", 1.0)
+ pid_file = tmp_path / "grandchild.pid"
+ monkeypatch.setattr(
+ sandbox_probe,
+ "_host_payload",
+ lambda workdir: (
+ "import subprocess, sys, time\n"
+ "child = subprocess.Popen([sys.executable, '-I', '-S', '-c', 'import time; time.sleep(120)'])\n"
+ f"open({str(pid_file)!r}, 'w').write(str(child.pid))\n"
+ "time.sleep(120)\n"
+ ),
+ )
+ available, reason = sandbox_probe.probe(_Backend("confining", _confining))
+ assert available is False and "positive controls" in reason
+ grandchild = int(pid_file.read_text())
+ assert _gone(grandchild), f"the host control's child {grandchild} outlived the probe"
+
+
+def test_the_payload_runs_under_the_tool_launch_fd_and_cpu_caps(tmp_path):
+ limits_file = tmp_path / "limits.txt"
+ payload = (
+ "import resource\n"
+ f"open({str(limits_file)!r}, 'w').write(repr(("
+ "resource.getrlimit(resource.RLIMIT_NOFILE), resource.getrlimit(resource.RLIMIT_CPU))))\n"
+ )
+
+ def report_limits(plan):
+ return PreparedSandboxLaunch(
+ argv = (sys.executable, "-I", "-S", "-c", payload),
+ workdir = plan.workdir,
+ env = plan.env,
+ preexec_fn = plan.preexec_fn,
+ backend = "limits",
+ )
+
+ sandbox_probe.probe(_Backend("limits", report_limits))
+ (nofile_soft, _), (cpu_soft, cpu_hard) = ast.literal_eval(limits_file.read_text())
+ assert nofile_soft <= sandbox_probe._PROBE_NOFILE
+ assert cpu_soft == cpu_hard == sandbox_probe._PROBE_CPU_SECONDS
+
+
def test_a_backend_that_only_prints_the_token_is_not_believed():
def liar(plan):
return PreparedSandboxLaunch(
@@ -372,16 +472,25 @@ def test_linux_runtime_identity_includes_the_verified_bwrap(monkeypatch):
assert os_sandbox._runtime_identity() != first
-def test_an_expired_verdict_is_re_probed(monkeypatch):
- """Set before the first probe: the expiry is stamped when the verdict is stored. With no stale
- grace the re-probe is in the foreground (test_permission_mode_degrade covers the grace)."""
- monkeypatch.setattr(sandbox_probe, "_CACHE_TTL_SECONDS", 0.0)
- monkeypatch.setattr(sandbox_probe, "_STALE_GRACE_SECONDS", 0.0)
+def test_a_verdict_is_never_re_probed_on_its_own(monkeypatch):
+ import types
+
+ leap = {"now": 1000.0}
+
+ def monotonic():
+ leap["now"] += 86400.0
+ return leap["now"]
+
+ monkeypatch.setattr(
+ sandbox_probe, "time", types.SimpleNamespace(monotonic = monotonic), raising = False
+ )
backend = _Backend("confining", _confining)
- sandbox_probe.probe(backend)
+ for _ in range(5):
+ assert sandbox_probe.probe(backend)[0] is True
+ assert backend.calls == 1
+ sandbox_probe.reset_probe_cache()
sandbox_probe.probe(backend)
assert backend.calls == 2
- assert not sandbox_probe._refreshing
def test_the_cache_cannot_grow_without_bound():
@@ -489,53 +598,7 @@ def test_the_landlock_helper_imports_where_it_will_never_be_used():
sandbox_landlock.abstract_scope_supported.cache_clear()
-def test_a_cached_verdict_outlives_the_old_sixty_second_window(monkeypatch, tmp_path):
- """The TTL was 60s, so an idle chat paid the cold probe again on its next message: a call 70s after the last one cost 1.24s against 0.040s unsandboxed."""
- from core.inference import sandbox_probe
-
- assert sandbox_probe._CACHE_TTL_SECONDS >= 600, (
- "the probe TTL is back under 10 minutes; an idle chat pays the cold "
- "probe again on its next message"
- )
-
- calls = []
-
- class Backend:
- BACKEND_NAME = "ttl-probe"
-
- @staticmethod
- def prepare(plan):
- calls.append(plan)
- raise AssertionError("the probe should not have re-run inside the TTL")
-
- sandbox_probe.reset_probe_cache()
- clock = {"now": 1000.0}
- monkeypatch.setattr(sandbox_probe.time, "monotonic", lambda: clock["now"])
-
- from core.inference import os_sandbox as _os_sandbox
-
- key = (Backend.BACKEND_NAME, _os_sandbox._runtime_identity())
- sandbox_probe._cache[key] = (
- clock["now"] + sandbox_probe._CACHE_TTL_SECONDS,
- True,
- "seeded",
- )
-
- clock["now"] += 70.0
- available, reason = sandbox_probe.probe(Backend)
- assert (available, reason) == (True, "seeded")
- assert calls == [], "the probe re-ran 70s later, which is the cost this fixed"
-
-
-def test_an_unavailable_verdict_is_not_cached_for_the_long_window(monkeypatch, tmp_path):
- """A user who installs bubblewrap must not wait 15 minutes for it to count."""
- from core.inference import os_sandbox as _os_sandbox
- from core.inference import sandbox_probe
-
- assert (
- sandbox_probe._CACHE_TTL_UNAVAILABLE_SECONDS <= 120
- ), "an unavailable verdict is cached long enough to outlive the user's own remediation"
-
+def test_an_unavailable_verdict_waits_for_refresh_or_a_new_runtime(monkeypatch):
probes = []
class Backend:
@@ -546,24 +609,17 @@ def test_an_unavailable_verdict_is_not_cached_for_the_long_window(monkeypatch, t
probes.append(plan)
raise RuntimeError("bubblewrap is not installed")
+ identity = ["before"]
+ monkeypatch.setattr(os_sandbox, "_runtime_identity", lambda: identity[0])
sandbox_probe.reset_probe_cache()
- clock = {"now": 5000.0}
- monkeypatch.setattr(sandbox_probe.time, "monotonic", lambda: clock["now"])
-
- available, _ = sandbox_probe.probe(Backend)
- assert available is False
+ for _ in range(5):
+ assert sandbox_probe.probe(Backend)[0] is False
assert len(probes) == 1
-
- clock["now"] += 30.0
+ assert sandbox_probe.probe(Backend, force = True)[0] is False
+ assert len(probes) == 2
+ identity[0] = "bubblewrap installed"
sandbox_probe.probe(Backend)
- assert len(probes) == 1, "an unavailable verdict should still be cached briefly"
-
- clock["now"] += 100.0
- sandbox_probe.probe(Backend)
- assert len(probes) == 2, (
- "the host was never re-examined, so installing bubblewrap does not "
- "take effect until the long TTL expires"
- )
+ assert len(probes) == 3
def test_the_capability_names_the_empty_proc_layout_it_was_probed_with(monkeypatch):