Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

tuner: waiting for the box lock does not count against the time budget; session tests

Niko1221e6b3af588907
diff --git a/strata_control/autotune.py b/strata_control/autotune.py
index f250d63bc..20a712fb2 100644
--- a/strata_control/autotune.py
+++ b/strata_control/autotune.py
@@ -97,6 +97,7 @@ class TuneResult:
message: str = ""
mode: str = "quick"
seconds: float = 0.0
+ waited_s: float = 0.0 # of those seconds, spent waiting (box lock, busy PC, memory)
runs: int = 0
base: dict[str, Any] = field(default_factory=dict)
chosen: dict[str, Any] = field(default_factory=dict)
@@ -193,10 +194,14 @@ class Tuner:
self.total_vram = total_vram_mib
self.quiet = quiet
self.storm = False
+ self.waited_s = 0.0 # time spent waiting for the box lock, the PC to be quiet or memory: not the user's budget
# ---- bookkeeping
+ def _elapsed(self) -> float:
+ return self.clock() - self.started - self.waited_s
+
def _say(self, stage: str, message: str, **extra: Any) -> None:
- elapsed = self.clock() - self.started
+ elapsed = self._elapsed()
avg = statistics.mean(self.run_times) if self.run_times else 150.0
remaining = max(0, self.planned_runs - self.done_runs)
eta = min(max(0.0, self.o.budget_s - elapsed), remaining * avg) if self.planned_runs else None
@@ -208,7 +213,7 @@ class Tuner:
raise Cancelled()
def _over_budget(self, reserve_s: float = 0.0) -> bool:
- return (self.clock() - self.started) + reserve_s > self.o.budget_s
+ return self._elapsed() + reserve_s > self.o.budget_s
def _avg_run(self) -> float:
return statistics.mean(self.run_times) if self.run_times else 150.0
@@ -270,12 +275,18 @@ class Tuner:
self._check_cancel()
t0 = self.clock()
+ entered: list[float] = []
+
def pre_start() -> None:
# checked again once the box lock is ours: another stream's engine may have come and gone while we waited
+ lock_got = self.clock()
+ self.waited_s += lock_got - t0
self._wait_for_quiet()
why = self._ram_guard(delta_mib, label, wait_s=60.0)
+ self.waited_s += self.clock() - lock_got
if why:
raise StartFailed("SKIPPED", why)
+ entered.append(self.clock())
try:
with self.session.engine(args, env, pre_start=pre_start) as eng:
@@ -284,7 +295,7 @@ class Tuner:
points(eng)
except StartFailed as e:
res = RunResult(status=e.status, reason=e.reason)
- dt = self.clock() - t0
+ dt = self.clock() - (entered[0] if entered else t0) # the run itself, not the waiting before it
if res.status != "SKIPPED":
self.run_times.append(dt)
self.done_runs += 1
@@ -304,6 +315,7 @@ class Tuner:
except Cancelled:
r.outcome, r.message = "cancelled", "Stopped. Your settings were not changed."
r.seconds = round(self.clock() - self.started, 1)
+ r.waited_s = round(self.waited_s, 1)
return r
def _spec(self) -> RequestSet:
@@ -353,7 +365,7 @@ class Tuner:
if o.mode == "thorough":
n_arms = len(planned)
else:
- left = int((o.budget_s - (self.clock() - self.started)) / max(1.0, self._avg_run()))
+ left = int((o.budget_s - self._elapsed()) / max(1.0, self._avg_run()))
fixed = (per_arm - 1) + 1 + 2 # extra base rounds, the final base, a combined-confirm margin
n_arms = min(len(planned), max(0, (left - fixed) // per_arm))
for p in planned[n_arms:]:
diff --git a/tests/tuning/test_session.py b/tests/tuning/test_session.py
new file mode 100644
index 000000000..9d6a526ec
--- /dev/null
+++ b/tests/tuning/test_session.py
@@ -0,0 +1,115 @@
+"""The real session's request side, against a tiny HTTP server that answers like the worker does (a `timings` block on the
+engine's own clock)."""
+
+from __future__ import annotations
+
+import json
+import tempfile
+import threading
+import unittest
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+
+from strata_control import tune_session
+from strata_control.tune_session import RequestSet, WorkerEngine, WorkerSession, parse_facts, failure_kind, make_prompt
+
+
+class Handler(BaseHTTPRequestHandler):
+ seen: list[dict] = []
+
+ def log_message(self, *a): # noqa: ANN002
+ pass
+
+ def do_GET(self): # noqa: N802
+ body = json.dumps({"data": [{"id": "stub-model"}]}).encode()
+ self.send_response(200)
+ self.send_header("content-length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def do_POST(self): # noqa: N802
+ req = json.loads(self.rfile.read(int(self.headers["content-length"])))
+ Handler.seen.append({"auth": self.headers.get("authorization"), **req})
+ n = req["max_tokens"]
+ prompt_tokens = len(req["messages"][0]["content"]) // 4
+ tune = req.get("strata_tune") or {}
+ rate = 80.0 * (1.1 if tune.get("spec_min_p") == 0.7 else 1.0)
+ body = json.dumps({"choices": [{"message": {"content": "x"}}], "usage": {"prompt_tokens": prompt_tokens, "completion_tokens": n},
+ "timings": {"cache_n": 0, "prompt_n": prompt_tokens, "prompt_ms": prompt_tokens / 2.0,
+ "prompt_per_second": 2000.0, "predicted_n": n, "predicted_ms": n / rate * 1000,
+ "predicted_per_second": rate}}).encode()
+ self.send_response(200)
+ self.send_header("content-length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+
+class FakeProc:
+ pid = 0
+
+ def poll(self):
+ return None
+
+
+class SessionTest(unittest.TestCase):
+ def setUp(self):
+ Handler.seen = []
+ self.srv = HTTPServer(("127.0.0.1", 0), Handler)
+ threading.Thread(target=self.srv.serve_forever, daemon=True).start()
+ self.addCleanup(self.srv.shutdown)
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+ s = WorkerSession(base_config={"args": []}, serve_dir=".", workdir=self.tmp.name)
+ log = Path(self.tmp.name) / "e.log"
+ log.write_text("strata generate: expert cache 3160 slots, 4.26 GiB of VRAM\nstrata generate: 5 pool workers on logical processors 2,4\n")
+ self.eng = WorkerEngine(s, FakeProc(), self.srv.server_address[1], "k", [log], 12.0, None, lambda x: None)
+
+ def test_measure_reads_the_engines_own_clock(self):
+ r = self.eng.measure(RequestSet(warmups=1, decodes=3, decode_tokens=200, pp_tokens=4096))
+ self.assertEqual(r.status, "PASS")
+ self.assertAlmostEqual(r.decode_tok_s, 80.0, places=1)
+ self.assertEqual(r.pp_tok_s, 2000.0)
+ self.assertEqual(r.facts["slots"], 3160)
+ self.assertEqual(r.facts["workers"], 5)
+ self.assertTrue(all(h["auth"] == "Bearer k" for h in Handler.seen))
+ self.assertTrue(all(h["model"] == "stub-model" for h in Handler.seen))
+
+ def test_prompts_are_unique_and_unchecked_by_cache(self):
+ self.eng.prompt_once(4096)
+ self.eng.prompt_once(4096)
+ a, b = [h["messages"][0]["content"] for h in Handler.seen[-2:]]
+ self.assertNotEqual(a[:40], b[:40])
+ self.assertTrue(all(h.get("strata_checkpoint") is False for h in Handler.seen))
+
+ def test_per_request_settings_are_sent_and_change_the_rate(self):
+ base = self.eng.decode_once(None, 200)
+ fast = self.eng.decode_once({"spec_min_p": 0.7}, 200)
+ self.assertAlmostEqual(fast / base, 1.1, places=3)
+ self.assertEqual(Handler.seen[-1]["strata_tune"], {"spec_min_p": 0.7})
+
+ def test_prompt_length_calibrates_once(self):
+ s = self.eng.s
+ before = s.chars_per_token
+ self.eng.prompt_once(4096)
+ self.assertTrue(s.calibrated)
+ self.assertNotEqual(s.chars_per_token, 0)
+ text = make_prompt(1000, "abc", s.chars_per_token)
+ self.assertIn("Reference abc.", text)
+
+
+class ParsingTest(unittest.TestCase):
+ def test_failure_kinds(self):
+ self.assertEqual(failure_kind("error: unknown argument: --adapt-min-gain\nusage...", 2)[0], "UNSUPPORTED")
+ self.assertEqual(failure_kind("native dense upload ... out of memory", 1)[0], "OOM")
+ self.assertEqual(failure_kind("", -9)[0], "OOM")
+ self.assertEqual(failure_kind("something else broke", 1)[0], "FAIL_BOOT")
+
+ def test_facts(self):
+ f = parse_facts("strata generate: 6 CPU threads\nhybrid experts: 37% on the GPU\n")
+ self.assertEqual(f["workers"], 6)
+ self.assertTrue(f["hybrid_experts"])
+ self.assertTrue(parse_facts("resident RAM mode on")["resident_ram"])
+
+
+if __name__ == "__main__":
+ unittest.main()