Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
calibrate: an engine that dies mid-measurement drops only its candidate (#3317)
Dzmitry Senchankau41188ec73495
diff --git a/src/core/verify.cpp b/src/core/verify.cppindex b82181c25..c8b9aa143 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -2809,12 +2809,30 @@ void Verifier::accumulate_profile_inner(const unsigned long long* stamps) {bool Verifier::adapt_gate_wait(int32_t layer, std::string& err) {const Clock::time_point t0 = Clock::now();uint32_t spins = 0;+ bool announced = false; // the watchdog's label is set only once the wait really starts: no cost when the copies landedwhile (adapt_gate_->reached() < layer) {+ if (!announced) {+ progress_at("verify window: waiting for the adaptive tier's copies of layer", layer);+ announced = true;+ }_mm_pause();if ((++spins & 1023u) != 0) continue;if (Clock::now() - t0 > std::chrono::seconds(20)) {+ // the release drains the window's own streams, never the tier's copy stream: say so, and hand out the+ // gate's words and this window's GPU breadcrumbs (STRATA_VERIFY_TRACE) - the last landed layer is the+ // exact place the stream stopped, which the released window's error line alone never told (#2188's class)+ const int32_t reach = adapt_gate_->reached(), land = adapt_gate_->landed.load();+ const int32_t hl = adapt_gate_->hland != nullptr ? *adapt_gate_->hland : -2;+ if (g_trace) trace_ev("GATE-TIMEOUT", -1, layer, reach);+ std::fprintf(stderr, "strata verify: the adaptive gate waited %lld ms for layer %d: reached %d "+ "(atomic %d, copy signal %d) - the tier's copy stream was not drained by the release\n",+ (long long) ms_since(t0), (int) layer, (int) reach, (int) land, (int) hl);+ if (g_trace) trace_dump(stderr);err = "verify: the adaptive tier's copies for layer " + std::to_string(layer) + " never landed" +- released_note(release_gpu_waits(5000));+ released_note(release_gpu_waits(5000)) ++ " (the copy stream itself is not drained: reached " + std::to_string(reach) + " of " ++ std::to_string(layer) + ", STRATA_VERIFY_TRACE=1 prints where the stream stopped)";+ std::fflush(stderr);return false;}}diff --git a/tools/calibrate.py b/tools/calibrate.pyindex 182c5a4d2..5fbe48f4a 100644--- a/tools/calibrate.py+++ b/tools/calibrate.py@@ -361,7 +361,8 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_workesettings["--pcie-frac"] = f"{chosen[0]:.2f}"settings["--spec-min-p"] = f"{chosen[1]:.2f}"base_rate = statistics.median(confirm[chosen]) if confirm.get(chosen) else None- # 4. fewer CPU workers (a restart each), with the chosen settings+ # 4. fewer CPU workers (a restart each), with the chosen settings. #1337's rule covers a death during a+ # measurement too: an engine that exits mid-rate is that candidate losing; the measurements so far stay.if d_workers and len(worker_candidates(d_workers, extra_workers)) > 1:tuned = with_arg(with_arg(base_args, "--pcie-frac", f"{chosen[0]:.2f}"), "--spec-min-p", f"{chosen[1]:.2f}")by_workers = {}@@ -370,18 +371,8 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_workefor index, w in enumerate(workers, 1):_check_cancel(cancel_event)say(f" Measuring with {w} CPU workers (restarts the engine) ...")- e = restart(start_engine, with_arg(tuned, "--pool-workers", None if w == d_workers else str(w)), f"{w} workers",- say, report)- if e is None:- _progress(progress, "cpu_workers", index, len(workers))- continue- try:- sw = Session(e, ids_list, cancel_event)- sw.warm_up(1)- by_workers[w] = [sw.rate(), sw.rate()]- say(f" {w} workers: {statistics.median(by_workers[w]):.1f} tok/s")- finally:- close(e)+ measure_candidate(start_engine, with_arg(tuned, "--pool-workers", None if w == d_workers else str(w)),+ f"{w} workers", say, report, by_workers, w, ids_list, cancel_event=cancel_event)_progress(progress, "cpu_workers", index, len(workers))report["workers"] = {str(k): v for k, v in by_workers.items()}w_best = pick(by_workers, d_workers) if by_workers else d_workers@@ -392,7 +383,9 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_workebase_rate = statistics.median(by_workers[d_workers])else:_progress(progress, "cpu_workers", 1, 1)- # 5. the adaptive expert tier (a restart each), with everything chosen so far+ # 5. the adaptive expert tier (a restart each), with everything chosen so far. A candidate whose engine dies+ # during its own measurement is dropped like a failed start (#1337); the other candidates and every earlier+ # result survive.tuned = apply(base_args, settings)by_adapt = {}_progress(progress, "expert_tier", 0, len(ADAPT_CANDIDATES))@@ -403,17 +396,8 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_workefor flag, v in zip(ADAPT_FLAGS, cand or (None,) * len(ADAPT_FLAGS)):args = with_arg(args, flag, v)say(f" Measuring the expert tier {'(the engine default)' if cand is None else 'every ' + cand[0] + ', ' + cand[1] + ' swaps, decay ' + cand[2]} (restarts the engine) ...")- e = restart(start_engine, args, "the expert tier " + key, say, report)- if e is None:- _progress(progress, "expert_tier", index, len(ADAPT_CANDIDATES))- continue- try:- sa = Session(e, ids_list, cancel_event)- sa.warm_up(1)- by_adapt[key] = [sa.rate(max_new=TIER_MAX_NEW), sa.rate(max_new=TIER_MAX_NEW)]- say(f" {statistics.median(by_adapt[key]):.1f} tok/s")- finally:- close(e)+ measure_candidate(start_engine, args, "the expert tier " + key, say, report, by_adapt, key, ids_list,+ TIER_MAX_NEW, cancel_event=cancel_event)_progress(progress, "expert_tier", index, len(ADAPT_CANDIDATES))a_best = pick(by_adapt, "default") if by_adapt else "default"report["adapt"] = by_adapt@@ -433,16 +417,8 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_worke_check_cancel(cancel_event)say(f" Measuring the shared expert {'on its own stream' if v == '1' else 'on the main stream (the AMD default)'}"" (restarts the engine) ...")- e = restart(lambda a: start_engine(a, {SH_STREAM: v}), tuned, f"{SH_STREAM}={v}", say, report)- if e is None:- continue- try:- sf = Session(e, ids_list, cancel_event)- sf.warm_up(1)- by_fork[v] = [sf.rate(), sf.rate()]- say(f" {statistics.median(by_fork[v]):.1f} tok/s")- finally:- close(e)+ measure_candidate(lambda a: start_engine(a, {SH_STREAM: v}), tuned, f"{SH_STREAM}={v}", say, report,+ by_fork, v, ids_list, cancel_event=cancel_event)report["sh_stream"] = by_forkif pick(by_fork, "0") == "1":env[SH_STREAM] = "1"@@ -455,6 +431,10 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_workereturn {"settings": settings, "env": env, "report": report}+def first_line(exc: Exception) -> str:+ return str(exc).strip().splitlines()[0][:300] if str(exc).strip() else type(exc).__name__++def restart(start_engine, args, what, say, report):"""#1337: a later step's engine start that fails is a candidate that loses, not a failed run: what the earliersteps measured (and the settings they chose) must survive it. Returns the engine, or None after saying why and@@ -463,12 +443,36 @@ def restart(start_engine, args, what, say, report):try:return start_engine(args)except Exception as e: # noqa: BLE001 - whatever the start raised (RuntimeError, EngineDied, OSError ...)- why = str(e).strip().splitlines()[0][:300] if str(e).strip() else type(e).__name__+ why = first_line(e)say(f" the engine did not start for {what}: {why} - this candidate is dropped, the earlier results are kept")report.setdefault("failed_starts", {})[what] = whyreturn None+def measure_candidate(start_engine, args, what, say, report, by: dict, key, ids_list, max_new: int | None = None,+ cancel_event=None):+ """One restart-and-measure candidate (#1337): a start that fails and an engine that exits mid-measurement are+ both that candidate losing - `by[key]` gets its two rates, or report["failed_starts"] its reason. Everything+ measured earlier and the other candidates survive. An operator cancel is not a candidate death: it raises."""+ e = restart(start_engine, args, what, say, report)+ if e is None:+ return+ try:+ s = Session(e, ids_list, cancel_event)+ s.warm_up(1)+ by[key] = [s.rate(max_new=max_new), s.rate(max_new=max_new)]+ say(f" {what}: {statistics.median(by[key]):.1f} tok/s")+ except CalibrationCancelled:+ raise+ except Exception as exc: # noqa: BLE001 - the engine died mid-measurement: this candidate loses (#1337)+ why = first_line(exc)+ say(f" the engine died while measuring {what}: {why} - this candidate is dropped, "+ "the earlier results are kept")+ report.setdefault("failed_starts", {})[what] = why+ finally:+ close(e)++def close(eng):proc = getattr(eng, "proc", None)if proc is None:diff --git a/tools/test_calibrate.py b/tools/test_calibrate.pyindex 97e335977..7ffe1fb63 100644--- a/tools/test_calibrate.py+++ b/tools/test_calibrate.py@@ -21,9 +21,10 @@ import calibrate as CAL # noqa: E402class FakeEngine:- """Speed = f(pcie_frac, spec_min_p, workers): the GEN line's tune keys arrive as `strata_tune`."""+ """Speed = f(pcie_frac, spec_min_p, workers): the GEN line's tune keys arrive as `strata_tune`.+ `dies`: setting values whose engine exits during its first generation (the #267 class, mid-measurement)."""- def __init__(self, args, speed, info_workers=6, starts=None, adapt=None, read=None, reads=None):+ def __init__(self, args, speed, info_workers=6, starts=None, adapt=None, read=None, reads=None, dies=()):self.args = list(args)self.read, self.reads = read, reads # prompt read speed = read(prefill_pipe); the values it read withself.adapt_bonus = (adapt or {}).get(CAL.arg_value(args, "--adapt-swaps"), 1.0)@@ -33,10 +34,16 @@ class FakeEngine:self.speed = speedself.last = {}self.proc = None+ self.dies = CAL.arg_value(args, "--adapt-swaps") in dies or CAL.arg_value(args, "--pool-workers") in dies+ self.dead = Falseif starts is not None:starts.append(list(args))def generate(self, ids, max_new, sampling, cancel):+ if self.dies and not self.dead:+ self.dead = True+ raise RuntimeError("the engine exited: ERR verify: the adaptive tier's copies for layer 47 never landed"+ " (the copy stream itself is not drained); restart the engine")tune = sampling.get("strata_tune") or {}rate = self.speed(tune.get("pcie_frac", 0.55), tune.get("spec_min_p", self.info["spec_min_p"]), self.workers)rate *= self.adapt_bonus@@ -54,9 +61,10 @@ BASE = ["--pack", "p", "--spec", "4", "--spec-min-p", "0.5", "--max-context", "8class Calibrate(unittest.TestCase):- def run_with(self, speed, workers=6, base=BASE, adapt=None, read=None, reads=None):+ def run_with(self, speed, workers=6, base=BASE, adapt=None, read=None, reads=None, dies=()):starts = []- res = CAL.measure(base, [[1, 2, 3]] * 3, lambda a: FakeEngine(a, speed, workers, starts, adapt, read, reads),+ res = CAL.measure(base, [[1, 2, 3]] * 3,+ lambda a: FakeEngine(a, speed, workers, starts, adapt, read, reads, dies),say=lambda *_: None, long_prompt=lambda i, n: [1] * n)return res, starts@@ -233,6 +241,22 @@ class Calibrate(unittest.TestCase):self.assertEqual(CAL.arg_value(a, "--adapt-swaps"), "160")self.assertIsNone(CAL.arg_value(CAL.apply(a, {}), "--adapt-swaps")) # an old calibration's tier goes+ def test_engine_death_during_tier_measurement_drops_only_that_candidate(self):+ # the 160-swap candidate's engine dies during its own measurement (the adaptive gate's #267 livelock):+ # that candidate loses, 80 (10% faster) is still chosen, and the run returns with the earlier results+ res, _ = self.run_with(lambda f, p, w: 50.0, adapt={"80": 1.10}, dies=("160",))+ self.assertEqual(res["settings"].get("--adapt-swaps"), "80")+ self.assertEqual(list(res["report"]["adapt"]), ["default", "1/80/0.97"])+ self.assertIn("the expert tier 1/160/0.97", res["report"]["failed_starts"])++ def test_engine_death_during_a_worker_measurement_keeps_the_run(self):+ # 4 workers' engine dies mid-measurement: 3 and 2 are still measured, the default count stays, the tier runs+ res, _ = self.run_with(lambda f, p, w: 50.0, workers=6, dies=("4",))+ self.assertNotIn("--pool-workers", res["settings"])+ self.assertEqual(sorted(int(k.split()[0]) for k in res["report"]["workers"]), [2, 3, 6])+ self.assertIn("4 workers", res["report"]["failed_starts"])+ self.assertEqual(list(res["report"]["adapt"]), ["default", "1/80/0.97", "1/160/0.97"])+def test_old_calibration_is_the_baseline_reset(self):# a config tuned earlier: the measurement starts from the product defaults, not from those valuesbase = CAL.apply(BASE, {"--pcie-frac": "0.20", "--pool-workers": "3", "--spec-min-p": "0.70"})@@ -277,6 +301,18 @@ class Calibrate(unittest.TestCase):self.assertEqual(res["env"], {})self.assertIn(f"{CAL.SH_STREAM}=1", res["report"]["failed_starts"])+ def test_engine_death_during_a_fork_measurement_keeps_the_run(self):+ # the forked engine exits mid-measurement (#267's class): that candidate loses, the run returns+ def start(args, env=None):+ e = FakeEngine(args, lambda f, p, w: 50.0, 6)+ if (env or {}).get(CAL.SH_STREAM) == "1":+ e.dies = True+ return e+ res = CAL.measure(BASE, [[1, 2, 3]] * 3, start, say=lambda *_: None, hip=True)+ self.assertEqual(res["env"], {})+ self.assertEqual(list(res["report"]["sh_stream"]), ["0"])+ self.assertIn(f"{CAL.SH_STREAM}=1", res["report"]["failed_starts"])+def test_apply_env(self):old = {"STRATA_ROCBLAS_TUNING": "tools/hip", CAL.SH_STREAM: "1"}self.assertEqual(CAL.apply_env(old, {}), {"STRATA_ROCBLAS_TUNING": "tools/hip"}) # an older choice goes