Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Take the wall clock out of two backend guards that fail on a loaded runner (#10952)

Daniel Han4938d2bef592
diff --git a/studio/backend/tests/test_api_key_expiry.py b/studio/backend/tests/test_api_key_expiry.py
index 0dacb4c61e8..229f69072da 100644
--- a/studio/backend/tests/test_api_key_expiry.py
+++ b/studio/backend/tests/test_api_key_expiry.py
@@ -165,15 +165,31 @@ def test_cache_does_not_bypass_revocation():
def test_cache_does_not_bypass_expiry():
+ # Same shape as test_cache_does_not_bypass_revocation: warm the cache on a key that is
+ # comfortably valid, then expire it in the row and re-validate. Expiring it by wall
+ # clock instead would race the first call against the KDF, which on a loaded runner can
+ # outlast any margin short enough to keep the sleep cheap.
seed_user()
- # Expires between the two calls: the first warms the cache, the second is still rejected.
- near = (datetime.now(timezone.utc) + timedelta(milliseconds = 600)).isoformat()
- raw = make_key(near)
- assert storage.validate_api_key(raw) == storage.DEFAULT_ADMIN_USERNAME
- import time
-
- time.sleep(0.8)
- assert storage.validate_api_key(raw) is None
+ raw, row = storage.create_api_key(
+ username = storage.DEFAULT_ADMIN_USERNAME,
+ name = "expire-after-cache",
+ expires_at = iso_from_now(days = 1),
+ )
+ assert storage.validate_api_key(raw) == storage.DEFAULT_ADMIN_USERNAME # cached
+ cache_id = storage._api_key_cache_id(raw)
+ assert cache_id in storage._api_key_hash_cache # the second call takes the cache path
+
+ conn = storage.get_connection()
+ try:
+ conn.execute(
+ "UPDATE api_keys SET expires_at = ? WHERE id = ?",
+ (iso_from_now(days = -1), int(row["id"])),
+ )
+ conn.commit()
+ finally:
+ conn.close()
+
+ assert storage.validate_api_key(raw) is None # cache hit still re-checks expires_at
def test_unknown_key_not_cached():
diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py
index a27ad227d4b..77555d7f378 100644
--- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py
+++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py
@@ -646,6 +646,28 @@ def test_no_reprompt_on_crlf_complete_python_game():
# ── ReDoS guards ───────────────────────────────────────────────────
+# What these guards are looking for is catastrophic backtracking, which costs seconds or
+# minutes, not a few extra milliseconds. A wall clock cannot tell a regressed quantifier from a
+# shared runner descheduling the process mid-match: the tilde case below measures about 11ms of
+# work and has been seen at 60.9ms on CI for that reason alone. The quantity the budget is about
+# is the regex's own CPU time, so measure that directly with process_time, which does not run
+# while this process is off the CPU. Best of several runs on top, for the contention
+# process_time cannot see: cache and memory pressure from a neighbour are real work here. A
+# genuine blow-up survives both, since every repeat pays it in full. Same pairing as
+# test_tool_loop_controller.py.
+_REDOS_BUDGET_MS = 50
+
+
+def _guard_ms(payload, repeats = 5):
+ best = None
+ for _ in range(repeats):
+ t0 = time.process_time()
+ _has_answer_artifact(payload)
+ elapsed_ms = (time.process_time() - t0) * 1000
+ if best is None or elapsed_ms < best:
+ best = elapsed_ms
+ return best
+
def test_no_backtrack_on_crlf_spam():
"""10K of `\\r\\n` repeats must complete fast.
@@ -655,10 +677,8 @@ def test_no_backtrack_on_crlf_spam():
10 KB). The current `[ \\t]*` indent restriction plus length-bounded
`[\\s\\S]{...}?` runs keep every alternative linear."""
payload = "\r\n" * 5000
- t0 = time.time()
- _has_answer_artifact(payload)
- elapsed_ms = (time.time() - t0) * 1000
- assert elapsed_ms < 50, f"guard took {elapsed_ms:.1f}ms on 10KB CRLF spam"
+ elapsed_ms = _guard_ms(payload)
+ assert elapsed_ms < _REDOS_BUDGET_MS, f"guard took {elapsed_ms:.1f}ms on 10KB CRLF spam"
def test_no_backtrack_on_open_html_spam():
@@ -666,10 +686,8 @@ def test_no_backtrack_on_open_html_spam():
quickly. Bounded `[\\s\\S]{0,4000}?` between the open and close caps
the scan per occurrence."""
payload = "<html " * 200 # ~1200 chars, under _REPROMPT_MAX_CHARS
- t0 = time.time()
- _has_answer_artifact(payload)
- elapsed_ms = (time.time() - t0) * 1000
- assert elapsed_ms < 50, f"guard took {elapsed_ms:.1f}ms on <html spam"
+ elapsed_ms = _guard_ms(payload)
+ assert elapsed_ms < _REDOS_BUDGET_MS, f"guard took {elapsed_ms:.1f}ms on <html spam"
def test_no_backtrack_on_doctype_html_alternation_worst_case():
@@ -680,19 +698,15 @@ def test_no_backtrack_on_doctype_html_alternation_worst_case():
generous budget so future quantifier changes that drop the inner
``{0,4000}`` bound fail loudly."""
payload = ("<!doctype html><html foo " * 60)[:1999]
- t0 = time.time()
- _has_answer_artifact(payload)
- elapsed_ms = (time.time() - t0) * 1000
- assert elapsed_ms < 50, f"guard took {elapsed_ms:.1f}ms on doctype/html alt"
+ elapsed_ms = _guard_ms(payload)
+ assert elapsed_ms < _REDOS_BUDGET_MS, f"guard took {elapsed_ms:.1f}ms on doctype/html alt"
def test_no_backtrack_on_tilde_fence_spam():
"""Open ``~~~`` fences without close must terminate quickly."""
payload = "~~~a\n" * 400 # ~2000 chars, near _REPROMPT_MAX_CHARS
- t0 = time.time()
- _has_answer_artifact(payload)
- elapsed_ms = (time.time() - t0) * 1000
- assert elapsed_ms < 50, f"guard took {elapsed_ms:.1f}ms on ~~~ spam"
+ elapsed_ms = _guard_ms(payload)
+ assert elapsed_ms < _REDOS_BUDGET_MS, f"guard took {elapsed_ms:.1f}ms on ~~~ spam"
# ── Closing-fence-must-end-line edge cases ────────────────────────
diff --git a/tests/studio/test_mac_tab_capability_warm_window.py b/tests/studio/test_mac_tab_capability_warm_window.py
index 6d1940644dd..913f235cfc4 100644
--- a/tests/studio/test_mac_tab_capability_warm_window.py
+++ b/tests/studio/test_mac_tab_capability_warm_window.py
@@ -953,18 +953,25 @@ def _serve_trickling_headers():
srv.listen(4)
srv.settimeout(0.25)
port = srv.getsockname()[1]
- state = {"accepted": 0, "lines": 0}
+ # Bytes, never whole header lines: whole lines hit http.client's _MAXHEADERS cap of 100
+ # and end the call themselves in ~2s, passing against a build with no deadline at all.
+ chunk = b"x"
+ # urllib's per-socket-operation timeout is reset by any traffic. While max_gap stays under
+ # it no per-operation timeout can have fired, so a probe that returned anyway returned on
+ # the whole-request deadline. That is the only thing here that tells the two bounds apart.
+ state = {"accepted": 0, "lines": 0, "chunk": chunk, "max_gap": 0.0, "last_write": None}
stop = threading.Event()
def drip(conn):
try:
conn.recv(4096)
- # One header line that is never terminated, a byte at a time.
- # Whole header LINES would hit http.client's own _MAXHEADERS cap of 100 and end the call on their own at
- # about two seconds, which would make this fixture pass against a build that has no deadline at all.
conn.sendall(b"HTTP/1.1 200 OK\r\nX-Pad: ")
while not stop.is_set():
- conn.sendall(b"x")
+ conn.sendall(chunk)
+ now = time.monotonic()
+ if state["last_write"] is not None:
+ state["max_gap"] = max(state["max_gap"], now - state["last_write"])
+ state["last_write"] = now
state["lines"] += 1
time.sleep(0.02)
except Exception:
@@ -1022,21 +1029,41 @@ def test_trickling_response_headers_cannot_outlive_the_probe_budget(tmp_path, mo
urlopen has not returned, so nothing inside it is running yet and only urllib's
per-socket-operation timeout applies, which a peer resets by dribbling. The budget is
whole-request for that reason: connect, headers and body under one deadline."""
+ socket_timeout = 0.5
mod = _load(tmp_path, monkeypatch)
base, state, shutdown = _serve_trickling_headers()
monkeypatch.setattr(mod, "BASE", base)
try:
- returned, value, elapsed = _probe_bounded(mod, "/api/liveness", timeout = 0.5, wait = 6.0)
+ returned, value, elapsed = _probe_bounded(
+ mod, "/api/liveness", timeout = socket_timeout, wait = 6.0
+ )
+ probe_ended = time.monotonic()
+ lines, max_gap, last_write = state["lines"], state["max_gap"], state["last_write"]
finally:
shutdown()
- # Fixture preconditions first, so a server that stopped trickling fails loudly instead of letting the probe return
- # fast and passing for free.
+
assert state["accepted"] >= 1, "fixture never accepted a connection"
- assert state["lines"] >= 3, f"fixture stopped trickling headers after {state['lines']} bytes"
- assert state["lines"] < 100, (
+ assert b"\n" not in state["chunk"] and b"\r" not in state["chunk"], (
"fixture is sending whole header lines again; http.client's _MAXHEADERS would "
- "end the call by itself and this would pass without any deadline"
+ f"end the call by itself and this would pass without any deadline: {state['chunk']!r}"
+ )
+
+ # Contention that breaks the gap invariant yields no evidence, not a failure: report that
+ # rather than pass on a build with no deadline or go red on a busy machine.
+ trickled_throughout = (
+ lines >= 3
+ and last_write is not None
+ and max_gap < socket_timeout
+ and probe_ended - last_write < socket_timeout
)
+ if not trickled_throughout:
+ pytest.skip(
+ "fixture could not keep inside the per-operation timeout on this runner, so a "
+ f"timeout here would not distinguish the two bounds: {lines} bytes, "
+ f"max gap {max_gap:.3f}s, last write {probe_ended - (last_write or probe_ended):.3f}s "
+ f"before the probe returned, socket timeout {socket_timeout}s"
+ )
+
assert returned, "probe never returned: urlopen is outside the deadline again"
assert value[2] == "timeout", value
assert 0.4 <= elapsed < 4.0, f"probe took {elapsed:.2f}s against a 0.5s whole-request budget"