Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stop ten test assertions from measuring the runner instead of the code (#10897)

Daniel Hanf82efb6ed9b7
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index c9346c86eaf..e34ed020d8f 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -42,6 +42,15 @@ _backend_root = Path(__file__).resolve().parent.parent
if str(_backend_root) not in sys.path:
sys.path.insert(0, str(_backend_root))
+# tests/_shared, as tests/conftest.py does for its own trees. Module scope, not a fixture:
+# a test module imports from it at collection, before any fixture runs.
+for _up in Path(__file__).resolve().parents:
+ _repo_shared = _up / "tests" / "_shared"
+ if (_repo_shared / "growth.py").is_file():
+ if str(_repo_shared) not in sys.path:
+ sys.path.insert(0, str(_repo_shared))
+ break
+
# Let the diffusion patch backend lazily import unsloth_zoo on a CPU-only test host: unsloth_zoo runs accelerator
# detection at import and raises without a GPU unless this is set. setdefault so an explicit override wins.
os.environ.setdefault("UNSLOTH_ALLOW_CPU", "1")
diff --git a/studio/backend/tests/test_account_model_access.py b/studio/backend/tests/test_account_model_access.py
index c16c9573ca6..297e685a9c9 100644
--- a/studio/backend/tests/test_account_model_access.py
+++ b/studio/backend/tests/test_account_model_access.py
@@ -211,11 +211,28 @@ def test_concurrent_misses_ask_the_hub_once_per_repo(monkeypatch):
def test_a_listing_probes_its_unknown_repos_together(monkeypatch):
- monkeypatch.setattr(access, "_hub_public_answer", lambda *a: time.sleep(0.1) or True)
+ """All eight probes must be in flight at once, not walked one at a time.
+
+ Asked with a barrier rather than a stopwatch, the way
+ `test_concurrent_misses_ask_the_hub_once_per_repo` above already does. Eight
+ sleeping probes finish in ~0.1s concurrently and ~0.8s serially, so `< 0.4`
+ looks like it separates the two -- but it is still eight threads on a shared
+ 2-vCPU runner under `pytest -n 4`, and descheduling reads as serialization.
+ A Barrier(8) only clears if all eight really are inside the probe together,
+ which is the claim, and it cannot be answered by how busy the box is.
+ """
rows = [{"repo_id": f"org/slow-{index}"} for index in range(8)]
- start = time.perf_counter()
+ # Sized off the fan-out, so _PROBE_FANOUT = 1 fails rather than satisfying a Barrier(1).
+ together = min(access._PROBE_FANOUT, len(rows))
+ assert together > 1, f"_PROBE_FANOUT is {access._PROBE_FANOUT}; the listing is serial"
+ barrier = threading.Barrier(together)
+
+ def answer(*_args):
+ barrier.wait(timeout = 30)
+ return True
+
+ monkeypatch.setattr(access, "_hub_public_answer", answer)
assert len(run_as(ALICE, access.filter_model_rows, rows)) == 8
- assert time.perf_counter() - start < 0.4
def test_a_warm_listing_asks_the_hub_nothing(monkeypatch):
diff --git a/studio/backend/tests/test_liveness_reports_inference_active.py b/studio/backend/tests/test_liveness_reports_inference_active.py
index 1b80402710b..409181aed05 100644
--- a/studio/backend/tests/test_liveness_reports_inference_active.py
+++ b/studio/backend/tests/test_liveness_reports_inference_active.py
@@ -27,6 +27,7 @@ CPU-only, no network, no GPU, no weights.
from __future__ import annotations
import json
+import re
import subprocess
import sys
from pathlib import Path
@@ -64,18 +65,40 @@ hw.ensure_hardware_detected = must_not_run
app = FastAPI()
app.add_api_route("/api/liveness", main.liveness_check, methods = ["GET"])
+
+# A route that does nothing, served by the same app through the same TestClient. The
+# claim here is "liveness costs about what answering costs", and a bare answer is the
+# only honest zero: everything a runner charges the real probe, it charges this too.
+async def _nothing():
+ return {"ok": True}
+
+app.add_api_route("/api/nothing", _nothing, methods = ["GET"])
client = TestClient(app)
-def probe():
+def _timed(path):
started = time.perf_counter()
- response = client.get("/api/liveness")
- elapsed = time.perf_counter() - started
+ response = client.get(path)
+ return time.perf_counter() - started, response
+
+def probe():
+ # Interleaved, and best of three on each side. Taken in one batch after the fact, a
+ # control shares no scheduler delay with what it is compared against, so a pause that
+ # lands on the measured request alone is not divided out. Alternating them gives each
+ # side the same chance of being unlucky, and a minimum over three is not moved by a
+ # pause that has to hit all three to count.
+ mine, controls, response = [], [], None
+ for _ in range(3):
+ control_elapsed, _ = _timed("/api/nothing")
+ elapsed, response = _timed("/api/liveness")
+ controls.append(control_elapsed)
+ mine.append(elapsed)
body = response.json()
return {
"status_code": response.status_code,
"status": body.get("status"),
"service": body.get("service"),
- "elapsed": elapsed,
+ "elapsed": min(mine),
+ "control": min(controls),
"inference_active": body.get("inference_active"),
"has_busy_key": "inference_active" in body,
}
@@ -168,15 +191,47 @@ def test_the_marker_disappears_once_nothing_is_generating():
)
+def _watchdog_probe_budget_s() -> float:
+ """The launcher's per-probe HTTP budget, read out of the Rust that owns it.
+
+ A ceiling on this route has to sit under the number the watchdog actually allows, or a
+ regression that makes /api/liveness block for most of a probe passes here while every
+ real probe times out. Derived rather than written down so the two cannot drift apart,
+ the way test_health_answers_within_probe_budget.py derives its own budget.
+ """
+ assert _COMMANDS_RS.is_file(), f"{_COMMANDS_RS} moved; update this guard"
+ match = re.search(
+ r"const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs\((\d+)\)",
+ _COMMANDS_RS.read_text(encoding = "utf-8"),
+ )
+ assert match, "commands.rs no longer sets a whole-seconds probe timeout"
+ return float(match.group(1))
+
+
def test_the_marker_costs_nothing_to_read():
"""A probe every 15s cannot pay for anything that waits, which is why the route reads
a registry len() rather than asking the backend what it is doing."""
result = _probe()["probes"]
+ # Two bounds, because they answer different questions and neither covers the other.
+ #
+ # The first is the one that matters: liveness against a route in the same app that
+ # only returns a dict. Both pay the same interpreter, the same TestClient and the same
+ # scheduler, so what is left is the route's own work, and a second of new I/O shows up
+ # as a ratio however slow the runner is. Generous at 40x, since the floor here is tens
+ # of microseconds and small absolute jitter is a large ratio.
+ #
+ # The second is the absolute one the watchdog imposes: at or over its per-probe budget
+ # every real probe times out. A relative bound cannot see that, because a control that
+ # somehow took seconds would scale with it.
for state, sample in result.items():
- assert sample["elapsed"] < 0.5, (
- f"/api/liveness took {sample['elapsed']:.2f}s while {state}; it must read the "
- f"registry rather than wait on the generations in it"
+ control = sample["control"]
+ relative = max(control * 40, 0.05)
+ ceiling = min(relative, _watchdog_probe_budget_s() / 2)
+ assert sample["elapsed"] < ceiling, (
+ f"/api/liveness took {sample['elapsed'] * 1000:.1f}ms while {state}, against "
+ f"{control * 1000:.1f}ms to answer a route that does nothing; it must read "
+ f"the registry rather than wait on the generations in it"
)
diff --git a/studio/backend/tests/test_liveness_reports_warmup_state.py b/studio/backend/tests/test_liveness_reports_warmup_state.py
index da28b641094..dc70b606453 100644
--- a/studio/backend/tests/test_liveness_reports_warmup_state.py
+++ b/studio/backend/tests/test_liveness_reports_warmup_state.py
@@ -94,16 +94,37 @@ hw.ensure_hardware_detected = must_not_run
app = FastAPI()
app.add_api_route("/api/liveness", main.liveness_check, methods = ["GET"])
+
+# A route that does nothing, served by the same app through the same TestClient. The
+# claim here is "liveness costs about what answering costs", and a bare answer is the
+# only honest zero: everything a runner charges the real probe, it charges this too.
+async def _nothing():
+ return {"ok": True}
+
+app.add_api_route("/api/nothing", _nothing, methods = ["GET"])
client = TestClient(app)
-started = time.perf_counter()
-response = client.get("/api/liveness")
-elapsed = time.perf_counter() - started
+# Interleaved, and best of three on each side. Taken in one batch before the measured
+# request, a control shares no scheduler delay with it, so a pause that lands on the
+# measurement alone is not divided out. Alternating them gives each side the same chance
+# of being unlucky, and a minimum over three is not moved by a pause that has to hit all
+# three to count.
+_controls, _mine, response = [], [], None
+for _ in range(3):
+ _c0 = time.perf_counter()
+ client.get("/api/nothing")
+ _controls.append(time.perf_counter() - _c0)
+ _m0 = time.perf_counter()
+ response = client.get("/api/liveness")
+ _mine.append(time.perf_counter() - _m0)
+control = min(_controls)
+elapsed = min(_mine)
body = response.json()
print("RESULT" + json.dumps({
"status_code": response.status_code,
"elapsed": elapsed,
+ "control": control,
"status": body.get("status"),
"service": body.get("service"),
"hardware_detecting": body.get("hardware_detecting"),
@@ -219,14 +240,41 @@ def test_a_warm_retired_mid_stage_is_not_reported_as_warming_forever():
)
+def _watchdog_probe_budget_s() -> float:
+ """The launcher's per-probe HTTP budget, read out of the Rust that owns it.
+
+ A ceiling on this route has to sit under the number the watchdog actually allows, or a
+ regression that makes /api/liveness block for most of a probe passes here while every
+ real probe times out. Derived rather than written down so the two cannot drift apart,
+ the way test_health_answers_within_probe_budget.py derives its own budget.
+ """
+ assert _COMMANDS_RS.is_file(), f"{_COMMANDS_RS} moved; update this guard"
+ match = re.search(
+ r"const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs\((\d+)\)",
+ _COMMANDS_RS.read_text(encoding = "utf-8"),
+ )
+ assert match, "commands.rs no longer sets a whole-seconds probe timeout"
+ return float(match.group(1))
+
+
def test_liveness_answers_immediately_and_never_starts_detection():
"""The route exists because health's detection wait is too expensive to probe every
15s. The stub raises if detection is started, so returning at all proves it was not."""
result = _probe(settled = False)
- assert result["elapsed"] < 0.5, (
- f"/api/liveness took {result['elapsed']:.2f}s; it must read the settled snapshot "
- f"rather than wait for one"
+ # Returning at all is most of the assertion: `must_not_run` raises if detection starts,
+ # and DETECTION_COMPLETE is cleared, so a route awaiting one never returns.
+ #
+ # Two bounds, as in test_liveness_reports_inference_active.py. The relative one is
+ # against a route in the same app that only returns a dict, so what is left after both
+ # pay the same interpreter and scheduler is the route's own work; the absolute one is
+ # the watchdog's, since a control that somehow took seconds would scale with it.
+ relative = max(result["control"] * 40, 0.05)
+ ceiling = min(relative, _watchdog_probe_budget_s() / 2)
+ assert result["elapsed"] < ceiling, (
+ f"/api/liveness took {result['elapsed'] * 1000:.1f}ms against "
+ f"{result['control'] * 1000:.1f}ms to answer a route that does nothing, so it was "
+ "waiting on something rather than reading the snapshot"
)
# Still the full port-validation payload the launcher matches on. The key must be
# present because the launcher reads it; its value is environment-derived and is
diff --git a/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py b/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py
index 2823f64e4a7..a1adb7b5286 100644
--- a/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py
+++ b/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py
@@ -148,16 +148,78 @@ def _kw(**extra):
return base
+# Captured before the patch, so a deliberately slow probe still sleeps for real.
+_real_sleep = time.sleep
+
+
+class _Sleeps:
+ """The naps ``_wait_for_vram_settle`` asks for, in order.
+
+ Every claim in this file is about the helper's sleeping: did it skip the wait, did
+ it nap once per poll, did it clip the last nap to the deadline. Measuring that as
+ wall clock made each one a budget -- ``elapsed < 0.05`` around a short-circuit, which
+ a GC pause on a shared runner exceeds while the helper did exactly the right thing.
+ The durations it requests are the same facts without the runner in them, so ask for
+ those and keep one generous ceiling for the case where it never returns at all.
+ """
+
+ def __init__(self, clock = None):
+ self.durations: list[float] = []
+ # With a clock, the nap is charged to it instead of taken: the helper's deadline
+ # then advances exactly as far as it asked to wait, with no runner in between.
+ self._clock = clock
+
+ def __enter__(self):
+ self._patch = patch.object(time, "sleep", self._record)
+ self._patch.start()
+ return self
+
+ def __exit__(self, *exc):
+ self._patch.stop()
+ return False
+
+ def _record(self, seconds):
+ self.durations.append(seconds)
+ if self._clock is not None:
+ return self._clock.advance(seconds)
+ return _real_sleep(seconds)
+
+ @property
+ def total(self) -> float:
+ return sum(self.durations)
+
+
+class _Clock:
+ """``time.monotonic`` as a number the test moves, for the one claim real time cannot
+ answer: that a nap was cut to what was left of the budget. Only what is charged to it
+ advances it, so there is no scheduler to eat the window being measured."""
+
+ def __init__(self):
+ # Not 0.0: the helper reads `since_kill <= 0.0` as cold start, and `_kw()` takes
+ # that stamp off this clock, so an origin of zero short-circuits the whole wait.
+ self.now = 1000.0
+
+ def __enter__(self):
+ self._patch = patch.object(time, "monotonic", lambda: self.now)
+ self._patch.start()
+ return self
+
+ def __exit__(self, *exc):
+ self._patch.stop()
+ return False
+
+ def advance(self, seconds):
+ self.now += seconds
+
+
def test_cold_start_returns_immediately_without_probing():
"""Default ``since_kill=0.0`` is cold-start: no kill recorded, so the
helper short-circuits without invoking the probe."""
ctx, state = _patch_probe([[(0, 10000)], [(0, 10000)]])
- with ctx:
- start = time.monotonic()
+ with ctx, _Sleeps() as sleeps:
LlamaCppBackend._wait_for_vram_settle(max_wait = 2.0, interval = 0.25)
- elapsed = time.monotonic() - start
assert state["calls"] == 0, "cold start must skip the probe entirely"
- assert elapsed < 0.05
+ assert sleeps.durations == [], f"cold start slept anyway: {sleeps.durations}"
def test_stale_kill_skips_wait():
@@ -174,12 +236,12 @@ def test_stale_kill_skips_wait():
def test_empty_first_sample_returns_immediately():
"""CPU-only host: probe returns [] → no wait, no further polls."""
ctx, state = _patch_probe([[]])
- with ctx:
- start = time.monotonic()
+ with ctx, _Sleeps() as sleeps:
LlamaCppBackend._wait_for_vram_settle(**_kw(max_wait = 2.0, interval = 0.25))
- elapsed = time.monotonic() - start
assert state["calls"] == 1
- assert elapsed < 0.5, "CPU-only short-circuit must not sleep through the interval"
+ assert (
+ sleeps.durations == []
+ ), f"CPU-only short-circuit slept through the interval: {sleeps.durations}"
def test_first_probe_raises_returns_without_polling():
@@ -200,13 +262,10 @@ def test_two_consecutive_samples_within_tolerance_settles():
[(0, 11550)],
]
)
- with ctx:
- start = time.monotonic()
+ with ctx, _Sleeps() as sleeps:
LlamaCppBackend._wait_for_vram_settle(**_kw(max_wait = 2.0, interval = 0.05))
- elapsed = time.monotonic() - start
assert state["calls"] == 3
- # interval * 2 sleeps = 0.10; allow slack for scheduler jitter.
- assert elapsed < 1.0
+ assert sleeps.durations == [0.05, 0.05], sleeps.durations
def test_probe_raises_mid_loop_returns():
@@ -231,32 +290,48 @@ def test_max_wait_respected_when_never_settles():
drift["v"] += 500
return [(0, drift["v"])]
+ # On the fake clock, for the same reason the slow-probe test is: the lower bound below
+ # says "did not stop early", and real time spent outside sleep() -- a GC pause, a
+ # deschedule -- retires the deadline without appearing in the naps, so a correct helper
+ # returns having napped less than the window and reads as an early exit.
ctx, _state = _patch_probe([_drifty])
- with ctx:
- start = time.monotonic()
+ with ctx, _Clock() as clock, _Sleeps(clock) as sleeps:
LlamaCppBackend._wait_for_vram_settle(**_kw(max_wait = 0.5, interval = 0.1))
- elapsed = time.monotonic() - start
- # Must stop near max_wait, not run forever. Generous upper bound for CI.
- assert 0.3 <= elapsed < 2.0, f"helper ignored max_wait: elapsed={elapsed:.3f}s"
+ # Both ends, because only the pair says "polled for the whole window and no longer".
+ # A loop accidentally capped at two iterations records [0.1, 0.1] and returns after
+ # 0.2s while the VRAM is still moving, which is how the next model gets launched
+ # early; a ceiling alone calls that a pass.
+ assert sleeps.total <= 0.5 + 1e-9, f"helper napped past max_wait: {sleeps.durations}"
+ assert sleeps.total >= 0.5 - 1e-9, f"helper gave up inside max_wait: {sleeps.durations}"
def test_max_wait_respected_when_probe_is_slow():
- """Slow probe: clipped sleep keeps the wall-clock bound honest."""
+ """Slow probe: clipped sleep keeps the wall-clock bound honest.
+
+ On a fake clock, because this one cannot be asked in real time. The claim is that
+ the nap AFTER the slow probe is cut to what is left of the budget, so the test needs
+ a nap to have been requested at all -- and if a real scheduler pause eats the 0.1s
+ that remained, a correct helper returns at the deadline check without napping and
+ the assertion fails on a run that did nothing wrong. Nothing here is about how fast
+ the box is, so the box is taken out: the probe charges its cost to the clock instead
+ of sleeping, and every sample is exact.
+ """
def _slow_probe():
- time.sleep(0.30)
+ clock.advance(0.30) # the probe's own cost, not one of the helper's naps
return [(0, 10000)]
ctx, _state = _patch_probe([_slow_probe])
- with ctx:
- start = time.monotonic()
+ with ctx, _Clock() as clock, _Sleeps(clock) as sleeps:
LlamaCppBackend._wait_for_vram_settle(
**_kw(max_wait = 0.4, interval = 0.25),
)
- elapsed = time.monotonic() - start
- # First probe (0.30 s) + at most one clipped sleep + bail.
- # Hard cap well below the old 0.30 + 0.25 + 0.30 = 0.85.
- assert elapsed < 0.85, f"helper exceeded the deadline due to slow probes: {elapsed:.3f}s"
+ # The probe burned 0.30 of a 0.4s budget, so the nap after it must be clipped.
+ assert sleeps.durations, "helper never napped, so nothing was clipped"
+ assert (
+ sleeps.durations[-1] < 0.25
+ ), f"helper slept the full interval past the deadline: {sleeps.durations}"
+ assert sleeps.total <= 0.4 + 1e-9, f"helper napped past max_wait: {sleeps.durations}"
def test_gpu_index_set_change_returns():
diff --git a/studio/backend/tests/test_mcp_http_sessions.py b/studio/backend/tests/test_mcp_http_sessions.py
index 4436e9c3550..2add86ac35d 100644
--- a/studio/backend/tests/test_mcp_http_sessions.py
+++ b/studio/backend/tests/test_mcp_http_sessions.py
@@ -81,6 +81,12 @@ class RecordingClient:
instances: list["RecordingClient"] = []
+ # Class-level: two chats mean two clients, and no single instance sees their overlap.
+ # Reset by the `clients` fixture.
+ live_lock = threading.Lock()
+ live_now = 0
+ live_peak = 0
+
def __init__(self, url: str, headers, use_oauth: bool):
self.url = url
self.headers = headers
@@ -131,6 +137,9 @@ class RecordingClient:
with self._lock:
self.live += 1
self.max_live = max(self.max_live, self.live)
+ with RecordingClient.live_lock:
+ RecordingClient.live_now += 1
+ RecordingClient.live_peak = max(RecordingClient.live_peak, RecordingClient.live_now)
try:
if self.call_delay:
await asyncio.sleep(self.call_delay)
@@ -140,11 +149,15 @@ class RecordingClient:
finally:
with self._lock:
self.live -= 1
+ with RecordingClient.live_lock:
+ RecordingClient.live_now -= 1
@pytest.fixture
def clients(monkeypatch):
RecordingClient.instances = []
+ RecordingClient.live_now = 0
+ RecordingClient.live_peak = 0
monkeypatch.setattr(
mcp_client,
"_client",
@@ -377,7 +390,7 @@ def test_two_http_calls_in_one_chat_run_concurrently(monkeypatch, clients):
assert len(out) == 2
assert len(clients) == 1, "the two calls should share one client"
assert clients[0].max_live == 2, "the server never saw them overlap"
- assert elapsed < 0.75, f"calls were serialized: {elapsed:.2f}s"
+ assert elapsed < 30.0, f"the parallel batch never came back: {elapsed:.2f}s"
def test_two_stdio_calls_in_one_chat_stay_serialized(monkeypatch, clients):
@@ -414,7 +427,9 @@ def test_calls_in_different_chats_run_concurrently(monkeypatch, clients):
out, elapsed = _parallel(HTTP_URL, [SCOPE, SCOPE_B])
assert len(out) == 2
assert len(clients) == 2
- assert elapsed < 0.75, f"different chats were serialized: {elapsed:.2f}s"
+ # Two clients, so only the class-level peak sees the overlap.
+ assert RecordingClient.live_peak == 2, "different chats were serialized"
+ assert elapsed < 30.0, f"the parallel batch never came back: {elapsed:.2f}s"
def test_concurrent_first_calls_publish_one_session(monkeypatch, clients):
@@ -624,11 +639,20 @@ def test_closing_many_sessions_does_not_run_serially(monkeypatch, clients):
"""A popular HTTP server holds a session per chat, and close runs on the
request thread during an edit or delete."""
closes = []
+ # Sized off the configured fan-out, the way the probe test sizes off _PROBE_FANOUT.
+ # `_close_all` batches at _MAX_CLOSE_THREADS, so a fixed Barrier(6) would deadlock a
+ # correctly batched close configured any narrower and report it as serial. One batch
+ # is what has to overlap, so open exactly that many sessions and wait for that many:
+ # a partial last batch would leave stragglers waiting for a party that never arrives.
+ together = min(mcp_client._MAX_CLOSE_THREADS, 6)
+ assert together > 1, f"_MAX_CLOSE_THREADS is {mcp_client._MAX_CLOSE_THREADS}; close is serial"
+ overlapping = threading.Barrier(together, timeout = 30)
class SlowExit(RecordingClient):
async def __aexit__(self, *exc):
closes.append(time.monotonic())
- await asyncio.sleep(0.4)
+ # A serial close deadlocks here and fails, rather than merely running slowly.
+ await asyncio.to_thread(overlapping.wait)
return await super().__aexit__(*exc)
monkeypatch.setattr(
@@ -636,13 +660,14 @@ def test_closing_many_sessions_does_not_run_serially(monkeypatch, clients):
"_client",
lambda url, headers, use_oauth = False: SlowExit(url, headers, use_oauth),
)
- for i in range(6):
+ for i in range(together):
_call(HTTP_URL, scope = f"chat-{i}")
started = time.monotonic()
close_mcp_sessions()
elapsed = time.monotonic() - started
- assert len(closes) == 6
- assert elapsed < 6 * 0.4 * 0.75, f"closes ran serially: {elapsed:.2f}s"
+ assert len(closes) == together
+ assert not overlapping.broken, f"closes ran serially: the {together} never overlapped"
+ assert elapsed < 60.0, f"close_mcp_sessions never came back: {elapsed:.2f}s"
def test_a_slow_but_live_idle_session_survives_the_recheck(monkeypatch, clients):
@@ -674,12 +699,19 @@ def test_evicting_another_scope_does_not_run_on_the_callers_deadline(monkeypatch
"""Whoever happens to trip the cap is mid tool call. The victim belongs to a
different chat and nobody is waiting on it, so its teardown must not be
charged to that caller's budget."""
- monkeypatch.setattr(mcp_client, "_MAX_SESSIONS", 1)
+ tearing_down = threading.Event()
+ release = threading.Event()
closed = threading.Event()
+ closed_by = []
- class SlowExit(RecordingClient):
+ class HeldExit(RecordingClient):
async def __aexit__(self, *exc):
- await asyncio.sleep(1.5)
+ # Held rather than slow: a sleep only has to outlast the assertion, so a
+ # caller that joins the teardown for PART of it still reads as prompt. This
+ # never finishes until the test says so, leaving a caller that waits on it
+ # at all no way to return, so there is no partial wait to get away with.
+ tearing_down.set()
+ await asyncio.get_running_loop().run_in_executor(None, release.wait)
out = await super().__aexit__(*exc)
closed.set()
return out
@@ -687,13 +719,52 @@ def test_evicting_another_scope_does_not_run_on_the_callers_deadline(monkeypatch
monkeypatch.setattr(
mcp_client,
"_client",
- lambda url, headers, use_oauth = False: SlowExit(url, headers, use_oauth),
+ lambda url, headers, use_oauth = False: HeldExit(url, headers, use_oauth),
+ )
+ # Who CALLS close() is the whole question, and it is exact. `__aexit__` cannot answer
+ # it: that always runs on the session's own loop thread, whoever is waiting on it.
+ # Every timing form needs the measured call and a reference taken at different moments,
+ # so a runner pause on either decides the verdict; this is the claim in the name of
+ # this test and reads the same however loaded the box is.
+ closing = mcp_client._McpSession.close
+ monkeypatch.setattr(
+ mcp_client._McpSession,
+ "close",
+ lambda self: (closed_by.append(threading.get_ident()), closing(self))[1],
)
+
+ # Handing the teardown off and then waiting for the worker anyway spends the same
+ # deadline, and none of the checks below can see it: a bounded `join(timeout = 5)`
+ # leaves `closed` unset and `closed_by` free of the caller. So watch the join too.
+ joined_worker = []
+ joining = threading.Thread.join
+
+ def record_join(self, timeout = None):
+ if self.name == "mcp-cleanup":
+ joined_worker.append((threading.get_ident(), timeout))
+ return joining(self, timeout)
+
+ monkeypatch.setattr(threading.Thread, "join", record_join)
+
_call(HTTP_URL, scope = SCOPE) # fills the cache
+ monkeypatch.setattr(mcp_client, "_MAX_SESSIONS", 1)
+ caller = threading.get_ident()
started = time.monotonic()
assert _call(HTTP_URL, scope = SCOPE_B) == "call-1"
elapsed = time.monotonic() - started
- assert elapsed < 0.5, f"the caller paid for an unrelated eviction: {elapsed:.2f}s"
+
+ assert tearing_down.wait(10), "the eviction never started, so nothing was under test"
+ assert closed_by and caller not in closed_by, (
+ "the calling thread closed the evicted session itself, so an unrelated chat's "
+ f"teardown is spending the deadline meant for this tool call: {closed_by}"
+ )
+ assert not [entry for entry in joined_worker if entry[0] == caller], (
+ "the calling thread handed the teardown off and then waited on the cleanup worker, "
+ f"which spends the same deadline the handoff was meant to protect: {joined_worker}"
+ )
+ assert not closed.is_set(), "the caller waited out an unrelated eviction"
+ assert elapsed < 30.0, f"the call never came back: {elapsed:.2f}s"
+ release.set()
assert closed.wait(10), "the evicted session was never closed"
diff --git a/studio/backend/tests/test_pr5624_regressions.py b/studio/backend/tests/test_pr5624_regressions.py
index 2e7c51fc989..f3c7ca1f3b1 100644
--- a/studio/backend/tests/test_pr5624_regressions.py
+++ b/studio/backend/tests/test_pr5624_regressions.py
@@ -30,6 +30,8 @@ from core.inference.tool_call_parser import (
strip_tool_markup,
)
+from growth import assert_linear # tests/_shared, on sys.path via tests/conftest.py
+
# GLM string-vs-JSON-encoded value coercion (finding B in plan)
@@ -222,96 +224,12 @@ def test_kimi_bare_counter_id_is_dropped():
# DeepSeek truncated mid-stream
-def _growth(
- build,
- units: int,
- factor: int = 4,
- repeats: int = 3,
-) -> tuple[float, float, list]:
- """How much more `factor` times the input costs. Returns (ratio, big_seconds, big_result).
-
- The MEDIAN of `repeats` PAIRED ratios: each pair times the small input and then the big
- one back to back, and the ratio is formed inside the pair before anything is aggregated.
-
- This replaces an absolute ``elapsed < 1.0`` budget at one size. That budget read 0.20s on
- a quiet runner and 1.41s on a busy one, so it failed for the wrong reason on unrelated
- PRs, and it also sat close enough to the line that a REAL regression (#10507, which made
- the R1 path quadratic again) only tipped it over some of the time. A ratio answers the
- question these tests are named for: linear is ~`factor`, quadratic is ~`factor ** 2`.
-
- It also replaces ``min(big over repeats) / min(small over repeats)``, which looked like
- it cancelled the machine out and did not. Those two minima come from batches run at
- different times, so a quiet window during the small batch and a busy one during the big
- batch multiply instead of cancelling, and the quotient of two separately-taken minima is
- not a minimum of anything. Measured on a 2-vCPU box under load, 15 trials per shape:
-
- strategy R1 R1-distant GLM V3 >= 6.0
- min/min max 8.69 max 8.23 max 8.27 max 8.59 7 of 60
- paired median max 4.26 max 4.83 max 4.83 max 5.42 0 of 60
-
- That 12% false-fail rate is not hypothetical: it is why this test failed on #10825 and
- again on #10864 at 6.56, both times on branches that touch none of this code.
-
- Pairing is what fixes it. Contention hits both halves of a pair roughly equally and
- divides out, which is what the old comment claimed for the unpaired form. The median
- then discards a pair that got unlucky, in either direction, rather than trusting one
- reading.
-
- Detection power is kept, which is the half worth checking before loosening anything. On
- a synthetic path with a true 16x profile this reports 8 of 8 over the bar, same as the
- old form. On the marginal 6.7x shape (#10832's partially fixed sweep) it reports 9 of 12
- against the old form's 10 of 12, a difference well inside the noise at that sample size,
- and that shape's sibling test measures 12.2x when broken, so the suite still catches it.
- """
- import statistics as _statistics
- import time as _time
-
- def once(text: str) -> tuple[float, list]:
- start = _time.perf_counter()
- result = parse_tool_calls_from_text(text)
- return _time.perf_counter() - start, result
-
- small_text, big_text = build(units), build(units * factor)
- ratios, big, result = [], None, None
- for _ in range(repeats):
- small_elapsed, _ = once(small_text)
- big_elapsed, result = once(big_text)
- # The backstop below reads the BEST big time, not the worst. Contention only adds, so
- # the minimum is the closest this size got to its own cost, and the backstop should
- # fire on a path that is genuinely too slow rather than on a runner that stalled once.
- big = big_elapsed if big is None else min(big, big_elapsed)
- # A timer's own resolution must not read as superlinear growth on a very fast machine.
- ratios.append(big_elapsed / max(small_elapsed, 1e-4))
-
- return _statistics.median(ratios), big, result
-
-
-def _assert_linear(
- build,
- label: str,
- units: int,
- *,
- factor: int = 4,
- tolerance: float = 6.0,
-):
- """`build(n)` must cost ~`factor`x, not ~`factor ** 2`x, for `factor`x the input."""
- ratio, big, result = _growth(build, units, factor)
- # Backstop: a regression bad enough to make the ratio unmeasurable still has to fail, and
- # fail quickly, rather than run until the job's own timeout kills it with no explanation.
- assert big < 60.0, f"{label} path took {big:.1f}s on {units * factor} units"
- assert ratio < tolerance, (
- f"{label} path is not linear: {factor}x the input cost {ratio:.1f}x the time "
- f"(linear is ~{factor}, quadratic is ~{factor ** 2})"
- )
- return result
-
-
def test_deepseek_v3_1_huge_truncated_body_is_linear():
"""Adversarial input: DeepSeek envelope with no JSON brace and a long
body. A regex-based ``[^\\n<]+?`` name capture is O(N^2) here; the
parser uses ``str.find`` on the sep marker so it stays linear."""
build = lambda n: "<|tool▁calls▁begin|><|tool▁call▁begin|>fn<|tool▁sep|>" + "x" * n
- assert _assert_linear(build, "V3", 50_000) == []
+ assert assert_linear(parse_tool_calls_from_text, build, "V3", 50_000) == []
def test_deepseek_r1_huge_fenceless_body_is_linear():
@@ -319,7 +237,7 @@ def test_deepseek_r1_huge_fenceless_body_is_linear():
fence-less body of repeated ``function<sep>`` tokens. The parser now scans with
``str.find``."""
build = lambda n: "<|tool▁calls▁begin|>" + "function<|tool▁sep|>a" * n
- assert _assert_linear(build, "R1", 10_000) == []
+ assert assert_linear(parse_tool_calls_from_text, build, "R1", 10_000) == []
def test_deepseek_r1_fenceless_body_with_one_distant_object_is_linear():
@@ -328,7 +246,7 @@ def test_deepseek_r1_fenceless_body_with_one_distant_object_is_linear():
rescanned the tail to reach it, so the sweep stayed quadratic while the fence-less
case above had gone linear."""
build = lambda n: "<|tool▁calls▁begin|>" + "function<|tool▁sep|>a" * n + '{"a": 1}'
- _assert_linear(build, "R1 distant-object", 10_000)
+ assert_linear(parse_tool_calls_from_text, build, "R1 distant-object", 10_000)
def test_glm_unclosed_body_many_arg_keys_is_linear():
@@ -336,7 +254,7 @@ def test_glm_unclosed_body_many_arg_keys_is_linear():
over many bare ``<arg_key>`` tokens was O(N^2). The parser now walks pairs with
``str.find``."""
build = lambda n: "<tool_call>foo\n" + "<arg_key>k" * n
- _assert_linear(build, "GLM", 10_000)
+ assert_linear(parse_tool_calls_from_text, build, "GLM", 10_000)
def test_deepseek_r1_fenced_json_parses():
diff --git a/studio/backend/tests/test_profile_stats.py b/studio/backend/tests/test_profile_stats.py
index 3b862f3a130..97bc99a5cf5 100644
--- a/studio/backend/tests/test_profile_stats.py
+++ b/studio/backend/tests/test_profile_stats.py
@@ -278,7 +278,11 @@ def test_terminal_callback_returns_promptly_while_locked_db_eventually_persists(
started = time.perf_counter()
monitor.finish(entry_id)
elapsed = time.perf_counter() - started
- assert elapsed < 0.2
+ # The row count below is the sharp claim; this only rules out queueing on the lock.
+ # A second is already three orders of magnitude past a put_nowait, and this file is
+ # in the serial leg of studio-backend-ci.yml rather than under `-n 4`, so there is no
+ # contention to buy slack for: a looser bound just lets a stalling callback through.
+ assert elapsed < 1.0, f"finish() blocked on the locked database: {elapsed:.3f}s"
assert (
lock_conn.execute(
"SELECT COUNT(*) FROM api_usage_events WHERE id = ?", (entry_id,)
@@ -393,11 +397,24 @@ def test_writer_busy_shutdown_is_bounded_then_drains_after_unlock(monkeypatch, c
assert writer.submit(receipt)
assert entered.wait(timeout = 1)
+ # What stop() hands its join, not how long the box took to get back: a wall-clock
+ # budget loose enough to survive `-n 4` is also loose enough to pass a stop() that
+ # waited whole seconds, which is the regression named here.
+ joins: list[float | None] = []
+ joining = writer._thread.join
+
+ def record_join(timeout = None):
+ joins.append(timeout)
+ return joining(timeout = timeout)
+
+ monkeypatch.setattr(writer._thread, "join", record_join)
+
try:
started = time.perf_counter()
assert writer.stop(timeout = 0.03) is False
elapsed = time.perf_counter() - started
- assert elapsed < 0.2
+ assert joins == [0.03], f"stop() did not wait for exactly its own timeout: {joins}"
+ assert elapsed < 1.0, f"stop() ignored its 0.03s timeout: {elapsed:.3f}s"
assert not writer.submit(_api_receipt("rejected-after-stop", datetime.now(timezone.utc)))
assert writer._thread.is_alive()
assert "may be lost if the process exits" in caplog.text
diff --git a/studio/backend/tests/test_rag_store.py b/studio/backend/tests/test_rag_store.py
index 98b1429fc82..c4862712c0b 100644
--- a/studio/backend/tests/test_rag_store.py
+++ b/studio/backend/tests/test_rag_store.py
@@ -11,6 +11,7 @@ import pytest
from core.rag import store
from core.rag.chunking import Chunk
+from growth import assert_linear # tests/_shared, on sys.path via tests/conftest.py
VOCAB = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel"]
@@ -408,14 +409,13 @@ def test_query_shaping_stays_cheap_on_a_pasted_log():
takes about 4.6s of CPU; linear it takes about 6ms. A 1.0s ceiling is unreachable by
a linear implementation on any machine that can run this suite at all.
"""
- import time
-
- question = f"what is the current value of ZQXVARA123\n{_pasted_prose(6000)}"
- started = time.perf_counter()
- expressions = store.conversation_match_queries(question)
- elapsed = time.perf_counter() - started
+ expressions = assert_linear(
+ store.conversation_match_queries,
+ lambda n: f"what is the current value of ZQXVARA123\n{_pasted_prose(n)}",
+ "paste shaping",
+ 1_500,
+ )
assert expressions and expressions[0] == '"zqxvara123"'
- assert elapsed < 1.0, f"shaping a 6000-word paste took {elapsed:.2f}s"
def test_a_quoted_function_word_survives_the_stopword_filter():
diff --git a/studio/backend/tests/test_tool_xml_strip.py b/studio/backend/tests/test_tool_xml_strip.py
index 16bf1af2f52..ad8e8576463 100644
--- a/studio/backend/tests/test_tool_xml_strip.py
+++ b/studio/backend/tests/test_tool_xml_strip.py
@@ -39,6 +39,8 @@ from core.inference.tool_call_parser import (
_strip_mistral_closed_calls,
)
+from growth import assert_linear # tests/_shared, on sys.path via tests/conftest.py
+
from typing import Optional as _Optional
_ns = {
@@ -446,25 +448,29 @@ def test_gdpval_parameter_orphans_get_stripped(leak):
def test_no_catastrophic_backtracking_on_open_bracket_spam():
- # 256KB of '<' must fail fast (literal mismatch char 2), not backtrack.
- import time
+ """'<' spam must fail fast on the literal mismatch at char 2, not backtrack.
- adv = "<" * (1024 * 256) + "X"
- t0 = time.perf_counter()
- _TOOL_XML_RE.sub("", adv)
- elapsed = time.perf_counter() - t0
- assert elapsed < 0.5, f"regex took {elapsed*1000:.0f}ms on 256KB '<' spam"
+ Asked as growth rather than as a deadline. Backtracking here is superlinear by
+ definition, so 4x the spam costing ~4x the time IS the property; `elapsed < 0.5`
+ was a budget that said as much about the runner as about the regex.
+ """
+ assert_linear(
+ lambda text: _TOOL_XML_RE.sub("", text),
+ lambda n: "<" * n + "X",
+ "'<' spam",
+ 64 * 1024,
+ )
def test_no_catastrophic_backtracking_on_orphan_opening_spam():
- # 1000 unclosed openings: first alt must consume them all greedily.
- import time
-
- adv = "<tool_call>X" * 1000
- t0 = time.perf_counter()
- cleaned = _TOOL_XML_RE.sub("", adv)
- elapsed = time.perf_counter() - t0
- assert elapsed < 0.1, f"regex took {elapsed*1000:.0f}ms on 1000x orphan opens"
+ """1000 unclosed openings: the first alternative must consume them all greedily."""
+ cleaned = assert_linear(
+ lambda text: _TOOL_XML_RE.sub("", text),
+ lambda n: "<tool_call>X" * n,
+ "orphan opens",
+ # 250, so the big leg is the 1000 openings this was previously measured at.
+ 250,
+ )
assert "<tool_call>" not in cleaned
diff --git a/tests/_shared/growth.py b/tests/_shared/growth.py
new file mode 100644
index 00000000000..394f9e0e186
--- /dev/null
+++ b/tests/_shared/growth.py
@@ -0,0 +1,125 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Is this path linear, asked without the runner answering for it.
+
+A guard against a quadratic blow-up -- a regex that backtracks, a sweep that rescans its
+own tail -- reads naturally as "it must finish in under X seconds". On a shared runner
+that is a budget, not a property: it says as much about who else is on the box as about
+the code, and a value low enough to catch a real regression is low enough to fail on a
+quiet branch. `test_pr5624_regressions.py` carried one and cost two unrelated PRs a red
+check before #10868 replaced it with the paired-ratio form below.
+
+The property those tests are actually named for is growth: `factor` times the input must
+cost about `factor` times the time, not `factor ** 2` times. That is a ratio, and a ratio
+of two measurements taken back to back divides the machine out. Lives in tests/_shared so both test trees reach it (see
+tests/_shared/real_accelerator.py for the same arrangement) and the next guard of this
+shape reuses the statistics instead of inventing another budget.
+"""
+
+from __future__ import annotations
+
+
+def growth(
+ run,
+ build,
+ units: int,
+ factor: int = 4,
+ repeats: int = 3,
+ abort_over_s: float = None,
+):
+ """How much more `factor` times the input costs. Returns (ratio, big_seconds, big_result).
+
+ `build(n)` makes an input of size n and `run(text)` is the thing being measured.
+
+ The MEDIAN of `repeats` PAIRED ratios: each pair times the small input and then the big
+ one back to back, and the ratio is formed inside the pair before anything is aggregated.
+
+ This replaces an absolute ``elapsed < 1.0`` budget at one size. That budget read 0.20s on
+ a quiet runner and 1.41s on a busy one, so it failed for the wrong reason on unrelated
+ PRs, and it also sat close enough to the line that a REAL regression (#10507, which made
+ the R1 path quadratic again) only tipped it over some of the time. A ratio answers the
+ question these tests are named for: linear is ~`factor`, quadratic is ~`factor ** 2`.
+
+ It also replaces ``min(big over repeats) / min(small over repeats)``, which looked like
+ it cancelled the machine out and did not. Those two minima come from batches run at
+ different times, so a quiet window during the small batch and a busy one during the big
+ batch multiply instead of cancelling, and the quotient of two separately-taken minima is
+ not a minimum of anything. Measured on a 2-vCPU box under load, 15 trials per shape:
+
+ strategy R1 R1-distant GLM V3 >= 6.0
+ min/min max 8.69 max 8.23 max 8.27 max 8.59 7 of 60
+ paired median max 4.26 max 4.83 max 4.83 max 5.42 0 of 60
+
+ That 12% false-fail rate is not hypothetical: it is why this test failed on #10825 and
+ again on #10864 at 6.56, both times on branches that touch none of this code.
+
+ Pairing is what fixes it. Contention hits both halves of a pair roughly equally and
+ divides out, which is what the old comment claimed for the unpaired form. The median
+ then discards a pair that got unlucky, in either direction, rather than trusting one
+ reading.
+
+ Detection power is kept, which is the half worth checking before loosening anything. On
+ a synthetic path with a true 16x profile this reports 8 of 8 over the bar, same as the
+ old form. On the marginal 6.7x shape (#10832's partially fixed sweep) it reports 9 of 12
+ against the old form's 10 of 12, a difference well inside the noise at that sample size,
+ and that shape's sibling test measures 12.2x when broken, so the suite still catches it.
+ """
+ import statistics as _statistics
+ import time as _time
+
+ def once(text):
+ start = _time.perf_counter()
+ result = run(text)
+ return _time.perf_counter() - start, result
+
+ small_text, big_text = build(units), build(units * factor)
+ ratios, big, result = [], None, None
+ for _ in range(repeats):
+ small_elapsed, _ = once(small_text)
+ big_elapsed, result = once(big_text)
+ # The backstop below reads the BEST big time, not the worst. Contention only adds, so
+ # the minimum is the closest this size got to its own cost, and the backstop should
+ # fire on a path that is genuinely too slow rather than on a runner that stalled once.
+ big = big_elapsed if big is None else min(big, big_elapsed)
+ # A timer's own resolution must not read as superlinear growth on a very fast machine.
+ ratios.append(big_elapsed / max(small_elapsed, 1e-4))
+ # Checked here, not after the loop: on the regression these guards exist for, the
+ # big leg is the minutes-long one, so finishing all `repeats` of it to report a
+ # number the caller will reject anyway is the slow way to reach the same verdict.
+ if abort_over_s is not None and big_elapsed > abort_over_s:
+ break
+
+ return _statistics.median(ratios), big, result
+
+
+def assert_linear(
+ run,
+ build,
+ label: str,
+ units: int,
+ *,
+ factor: int = 4,
+ tolerance: float = 6.0,
+):
+ """`run(build(n))` must cost ~`factor`x, not ~`factor ** 2`x, for `factor`x the input.
+
+ `units` is the SMALL size. The largest input actually run is `units * factor`, so when
+ this replaces an absolute budget, pass the old size DIVIDED by `factor` -- otherwise the
+ big leg is `factor`x bigger than anything that was ever measured, and on the regression
+ being guarded against that leg is `factor ** 2`x slower again. A guard whose broken case
+ takes a minute at the old size would then take a quarter of an hour, and the job's own
+ timeout kills it before the ratio below can say why.
+ """
+ budget = 60.0
+ # Passed down rather than checked here: on a path slow enough to trip it, every repeat
+ # is another minute spent measuring something already known to be too slow.
+ ratio, big, result = growth(run, build, units, factor, abort_over_s = budget)
+ # Backstop: a regression bad enough to make the ratio unmeasurable still has to fail, and
+ # fail quickly, rather than run until the job's own timeout kills it with no explanation.
+ assert big < budget, f"{label} path took {big:.1f}s on {units * factor} units"
+ assert ratio < tolerance, (
+ f"{label} path is not linear: {factor}x the input cost {ratio:.1f}x the time "
+ f"(linear is ~{factor}, quadratic is ~{factor ** 2})"
+ )
+ return result
diff --git a/tests/studio/test_backend_ci_parallel_isolation.py b/tests/studio/test_backend_ci_parallel_isolation.py
index 577f7dea415..a0af9b3568f 100644
--- a/tests/studio/test_backend_ci_parallel_isolation.py
+++ b/tests/studio/test_backend_ci_parallel_isolation.py
@@ -515,7 +515,7 @@ def test_every_tight_elapsed_bound_is_isolated():
)
-def test_the_scan_finds_all_three_shapes():
+def test_the_scan_finds_all_three_shapes(tmp_path):
"""A scan that matched nothing would pass the test above on an empty set.
One of each form the suite actually uses, because each needed its own handling and
@@ -523,30 +523,66 @@ def test_the_scan_finds_all_three_shapes():
elapsed < 0.05 a name assigned from a difference
time.monotonic() - started < 0.2 the difference written inline
_elapsed(big) < 8 * _elapsed(small) a helper that returns a difference
+
+ Written out here rather than named as three real files. Naming them made this test a
+ second, invisible reason those files had to keep their fragile bounds: rewriting
+ `test_llama_cpp_wait_for_vram_settle.py` to assert on the naps the helper asks for
+ instead of on how long they took -- which is the outcome the scan exists to push
+ people toward -- failed HERE, in a file about CI topology, with a message about a
+ sample. The scan's own coverage should not depend on the suite still containing the
+ thing it is trying to remove.
"""
- found = {
- path.name: _fragile_timing_asserts(path)
- for path in sorted(BACKEND_TESTS.glob("*.py"))
- if _fragile_timing_asserts(path)
+ shapes = {
+ "assigned name": (
+ "import time\n"
+ "def test_x():\n"
+ " started = time.monotonic()\n"
+ " work()\n"
+ " elapsed = time.monotonic() - started\n"
+ " assert elapsed < 0.05\n"
+ ),
+ "inline difference": (
+ "import time\n"
+ "def test_x():\n"
+ " started = time.monotonic()\n"
+ " work()\n"
+ " assert time.monotonic() - started < 0.05\n"
+ ),
+ "helper, relative": (
+ "import time\n"
+ "def _elapsed(fn):\n"
+ " started = time.perf_counter()\n"
+ " fn()\n"
+ " return time.perf_counter() - started\n"
+ "def test_x():\n"
+ " assert _elapsed(big) < 8 * _elapsed(small)\n"
+ ),
}
- assert "test_llama_cpp_wait_for_vram_settle.py" in found, found # named
- assert "test_tool_xml_strip.py" in found, found # named
- assert "test_diffusion_checkpoint_resume.py" in found, found # helper, relative
+ for label, source in shapes.items():
+ sample = tmp_path / f"test_{label.replace(' ', '_').replace(',', '')}.py"
+ sample.write_text(source, encoding = "utf-8")
+ assert _fragile_timing_asserts(sample), (
+ f"the scan does not recognise the {label} shape, so a test written that way "
+ "could carry a 50ms bound into the -n 4 run unnoticed"
+ )
- # The inline form, which the suite currently uses only at 0.2s, above the threshold.
- inline = ast.parse(
+ # And quiet on a bound with headroom, or every anti-hang ceiling goes serial for nothing.
+ roomy = tmp_path / "test_roomy.py"
+ roomy.write_text(
"import time\n"
- "def t():\n"
- " started = 0\n"
- " assert time.monotonic() - started < 0.05\n"
- )
- names, helpers = _timed_names(inline), _timing_helpers(inline)
- node = [n for n in ast.walk(inline) if isinstance(n, ast.Assert)][0]
- compare = node.test
- assert _is_timed(compare.left, names, helpers), (
- "an inline clock difference is not recognised as a duration, so a test written "
- "that way could assert a 20ms bound and run under -n 4 unnoticed"
+ "def test_x():\n"
+ " started = time.monotonic()\n"
+ " work()\n"
+ " elapsed = time.monotonic() - started\n"
+ " assert elapsed < 30.0\n",
+ encoding = "utf-8",
)
+ assert not _fragile_timing_asserts(roomy), _fragile_timing_asserts(roomy)
+
+ # Deliberately nothing about the live suite here. The synthetic files above already
+ # exercise every shape the scanner knows, and a "the live suite still has some"
+ # assertion would turn cleaning the last one into a failure, which is the same
+ # dependency on real files this rewrite exists to remove.
def test_an_isolated_file_never_shadows_an_installed_library_with_a_stub():
diff --git a/tests/studio/test_stream_cancel_registration_timing.py b/tests/studio/test_stream_cancel_registration_timing.py
index 7117b257bd8..6862d963281 100644
--- a/tests/studio/test_stream_cancel_registration_timing.py
+++ b/tests/studio/test_stream_cancel_registration_timing.py
@@ -973,26 +973,27 @@ def test_audio_stream_stays_responsive_under_blocking_next():
async def _run(loop_coro):
return await asyncio.gather(loop_coro, _fire_early())
+ # Counted in chunks: the blocking loop never lets _fire_early run, so it drains all
+ # eight before seeing a cancel that arrived at 50ms; the awaiting loop stops at the first.
cancel_event.clear()
t0 = time.monotonic()
prefix_seen, _ = asyncio.run(_run(_prefix_loop()))
prefix_elapsed = time.monotonic() - t0
- assert prefix_elapsed >= 0.13, (
- f"pre-fix pattern should block event loop for >=1 chunk time "
- f"(~150ms); got {prefix_elapsed:.3f}s, {len(prefix_seen)} chunks"
+ assert len(prefix_seen) == 8, (
+ "pre-fix pattern let the cancel through, so it is no longer modelling a blocked "
+ f"event loop; got {len(prefix_seen)} chunks"
)
cancel_event.clear()
- t0 = time.monotonic()
postfix_seen, _ = asyncio.run(_run(_postfix_loop()))
- postfix_elapsed = time.monotonic() - t0
- assert postfix_elapsed < prefix_elapsed, (
- f"post-fix pattern must exit faster than pre-fix (blocking) "
- f"pattern; post={postfix_elapsed:.3f}s vs pre={prefix_elapsed:.3f}s"
- )
assert (
len(postfix_seen) < 8
), f"post-fix loop must not drain all chunks; got {len(postfix_seen)}"
+ assert len(postfix_seen) < len(prefix_seen), (
+ f"post-fix pattern saw as much as the blocking one: post={len(postfix_seen)} "
+ f"vs pre={len(prefix_seen)}"
+ )
+ assert prefix_elapsed < 30.0, f"the blocking loop never returned: {prefix_elapsed:.3f}s"
def test_unsloth_stream_loop_emits_zero_tokens_on_preset_cancel():
diff --git a/tests/studio/test_update_release_notes.py b/tests/studio/test_update_release_notes.py
index 26e29999e81..c49ae4dfb8a 100644
--- a/tests/studio/test_update_release_notes.py
+++ b/tests/studio/test_update_release_notes.py
@@ -21,6 +21,7 @@ from dataclasses import dataclass
from pathlib import Path
import pytest
+from growth import assert_linear # tests/_shared, on sys.path via tests/conftest.py
REPO = Path(__file__).resolve().parents[2]
BACKEND = REPO / "studio/backend"
@@ -1545,13 +1546,19 @@ def test_setext_headings_are_release_boundaries(notes_module):
def test_a_long_backtick_run_does_not_stall_the_parser(notes_module):
- """The code-span guard used to backtrack: 20k backticks took over a minute."""
- import time
+ """The code-span guard used to backtrack: 20k backticks took over a minute.
- text = "## 1.0\n\n- " + "`" * 20_000 + " <!--\n"
- started = time.perf_counter()
- parse_sections(notes_module, text)
- assert time.perf_counter() - started < 1.0
+ Asked as growth. `< 1.0` is a budget, and this runs in the `-n 4` CPU leg where a
+ second of wall clock says as much about the other three workers as about the parser.
+ Backtracking is superlinear, so 4x the backticks costing ~4x the time is the property.
+ """
+ assert_linear(
+ lambda text: parse_sections(notes_module, text),
+ lambda n: "## 1.0\n\n- " + "`" * n + " <!--\n",
+ "backtick run",
+ # 5_000, so the big leg is the 20k previously measured. See assert_linear.
+ 5_000,
+ )
def test_the_remote_fetch_has_a_total_deadline(notes_module):
@@ -2206,16 +2213,19 @@ def test_stripping_comments_stays_linear_in_the_code_spans(notes_module):
"""The comment scanner restarted its code-span search per opener, so N spans
cost N squared. A 203 KiB line is well inside the 2 MiB accepted, and notes
are reparsed on every request, so one held a worker for over ten seconds."""
- line = "`a` <!--x--> " * 16_000
- assert len(line) < notes_module.RELEASES_MAX_BYTES
- started = time.monotonic()
- visible, in_comment = notes_module._strip_comments(line, False, False)
- elapsed = time.monotonic() - started
- # Roughly 40ms scanning forward against roughly 11s restarting each time.
- assert elapsed < 2.0, f"comment stripping took {elapsed:.1f}s"
+ # A quarter of the budget form's size: the big leg is the 16_000 it was checked at.
+ spans = 4_000
+ assert len("`a` <!--x--> " * (spans * 4)) < notes_module.RELEASES_MAX_BYTES
+ # N spans cost N squared when the scanner restarts per opener: 11s against 40ms.
+ visible, in_comment = assert_linear(
+ lambda text: notes_module._strip_comments(text, False, False),
+ lambda n: "`a` <!--x--> " * n,
+ "comment stripping",
+ spans,
+ )
# Same result as before: the spans survive and the comments are gone.
assert in_comment is False
- assert "<!--" not in visible and visible.count("`a`") == 16_000
+ assert "<!--" not in visible and visible.count("`a`") == spans * 4
def test_the_three_scanners_share_one_list_column_rule():