Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

calibrate: an engine that dies mid-measurement drops only its candidate (#3317)

Dzmitry Senchankau41188ec73495
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index b82181c25..c8b9aa143 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -2809,12 +2809,30 @@ void Verifier::accumulate_profile_inner(const unsigned long long* stamps) {
bool Verifier::adapt_gate_wait(int32_t layer, std::string& err) {
const Clock::time_point t0 = Clock::now();
uint32_t spins = 0;
+ bool announced = false; // the watchdog's label is set only once the wait really starts: no cost when the copies landed
while (adapt_gate_->reached() < layer) {
+ if (!announced) {
+ progress_at("verify window: waiting for the adaptive tier's copies of layer", layer);
+ announced = true;
+ }
_mm_pause();
if ((++spins & 1023u) != 0) continue;
if (Clock::now() - t0 > std::chrono::seconds(20)) {
+ // the release drains the window's own streams, never the tier's copy stream: say so, and hand out the
+ // gate's words and this window's GPU breadcrumbs (STRATA_VERIFY_TRACE) - the last landed layer is the
+ // exact place the stream stopped, which the released window's error line alone never told (#2188's class)
+ const int32_t reach = adapt_gate_->reached(), land = adapt_gate_->landed.load();
+ const int32_t hl = adapt_gate_->hland != nullptr ? *adapt_gate_->hland : -2;
+ if (g_trace) trace_ev("GATE-TIMEOUT", -1, layer, reach);
+ std::fprintf(stderr, "strata verify: the adaptive gate waited %lld ms for layer %d: reached %d "
+ "(atomic %d, copy signal %d) - the tier's copy stream was not drained by the release\n",
+ (long long) ms_since(t0), (int) layer, (int) reach, (int) land, (int) hl);
+ if (g_trace) trace_dump(stderr);
err = "verify: the adaptive tier's copies for layer " + std::to_string(layer) + " never landed" +
- released_note(release_gpu_waits(5000));
+ released_note(release_gpu_waits(5000)) +
+ " (the copy stream itself is not drained: reached " + std::to_string(reach) + " of " +
+ std::to_string(layer) + ", STRATA_VERIFY_TRACE=1 prints where the stream stopped)";
+ std::fflush(stderr);
return false;
}
}
diff --git a/tools/calibrate.py b/tools/calibrate.py
index 182c5a4d2..5fbe48f4a 100644
--- a/tools/calibrate.py
+++ b/tools/calibrate.py
@@ -361,7 +361,8 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_worke
settings["--pcie-frac"] = f"{chosen[0]:.2f}"
settings["--spec-min-p"] = f"{chosen[1]:.2f}"
base_rate = statistics.median(confirm[chosen]) if confirm.get(chosen) else None
- # 4. fewer CPU workers (a restart each), with the chosen settings
+ # 4. fewer CPU workers (a restart each), with the chosen settings. #1337's rule covers a death during a
+ # measurement too: an engine that exits mid-rate is that candidate losing; the measurements so far stay.
if d_workers and len(worker_candidates(d_workers, extra_workers)) > 1:
tuned = with_arg(with_arg(base_args, "--pcie-frac", f"{chosen[0]:.2f}"), "--spec-min-p", f"{chosen[1]:.2f}")
by_workers = {}
@@ -370,18 +371,8 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_worke
for index, w in enumerate(workers, 1):
_check_cancel(cancel_event)
say(f" Measuring with {w} CPU workers (restarts the engine) ...")
- e = restart(start_engine, with_arg(tuned, "--pool-workers", None if w == d_workers else str(w)), f"{w} workers",
- say, report)
- if e is None:
- _progress(progress, "cpu_workers", index, len(workers))
- continue
- try:
- sw = Session(e, ids_list, cancel_event)
- sw.warm_up(1)
- by_workers[w] = [sw.rate(), sw.rate()]
- say(f" {w} workers: {statistics.median(by_workers[w]):.1f} tok/s")
- finally:
- close(e)
+ measure_candidate(start_engine, with_arg(tuned, "--pool-workers", None if w == d_workers else str(w)),
+ f"{w} workers", say, report, by_workers, w, ids_list, cancel_event=cancel_event)
_progress(progress, "cpu_workers", index, len(workers))
report["workers"] = {str(k): v for k, v in by_workers.items()}
w_best = pick(by_workers, d_workers) if by_workers else d_workers
@@ -392,7 +383,9 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_worke
base_rate = statistics.median(by_workers[d_workers])
else:
_progress(progress, "cpu_workers", 1, 1)
- # 5. the adaptive expert tier (a restart each), with everything chosen so far
+ # 5. the adaptive expert tier (a restart each), with everything chosen so far. A candidate whose engine dies
+ # during its own measurement is dropped like a failed start (#1337); the other candidates and every earlier
+ # result survive.
tuned = apply(base_args, settings)
by_adapt = {}
_progress(progress, "expert_tier", 0, len(ADAPT_CANDIDATES))
@@ -403,17 +396,8 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_worke
for flag, v in zip(ADAPT_FLAGS, cand or (None,) * len(ADAPT_FLAGS)):
args = with_arg(args, flag, v)
say(f" Measuring the expert tier {'(the engine default)' if cand is None else 'every ' + cand[0] + ', ' + cand[1] + ' swaps, decay ' + cand[2]} (restarts the engine) ...")
- e = restart(start_engine, args, "the expert tier " + key, say, report)
- if e is None:
- _progress(progress, "expert_tier", index, len(ADAPT_CANDIDATES))
- continue
- try:
- sa = Session(e, ids_list, cancel_event)
- sa.warm_up(1)
- by_adapt[key] = [sa.rate(max_new=TIER_MAX_NEW), sa.rate(max_new=TIER_MAX_NEW)]
- say(f" {statistics.median(by_adapt[key]):.1f} tok/s")
- finally:
- close(e)
+ measure_candidate(start_engine, args, "the expert tier " + key, say, report, by_adapt, key, ids_list,
+ TIER_MAX_NEW, cancel_event=cancel_event)
_progress(progress, "expert_tier", index, len(ADAPT_CANDIDATES))
a_best = pick(by_adapt, "default") if by_adapt else "default"
report["adapt"] = by_adapt
@@ -433,16 +417,8 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_worke
_check_cancel(cancel_event)
say(f" Measuring the shared expert {'on its own stream' if v == '1' else 'on the main stream (the AMD default)'}"
" (restarts the engine) ...")
- e = restart(lambda a: start_engine(a, {SH_STREAM: v}), tuned, f"{SH_STREAM}={v}", say, report)
- if e is None:
- continue
- try:
- sf = Session(e, ids_list, cancel_event)
- sf.warm_up(1)
- by_fork[v] = [sf.rate(), sf.rate()]
- say(f" {statistics.median(by_fork[v]):.1f} tok/s")
- finally:
- close(e)
+ measure_candidate(lambda a: start_engine(a, {SH_STREAM: v}), tuned, f"{SH_STREAM}={v}", say, report,
+ by_fork, v, ids_list, cancel_event=cancel_event)
report["sh_stream"] = by_fork
if pick(by_fork, "0") == "1":
env[SH_STREAM] = "1"
@@ -455,6 +431,10 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_worke
return {"settings": settings, "env": env, "report": report}
+def first_line(exc: Exception) -> str:
+ return str(exc).strip().splitlines()[0][:300] if str(exc).strip() else type(exc).__name__
+
+
def restart(start_engine, args, what, say, report):
"""#1337: a later step's engine start that fails is a candidate that loses, not a failed run: what the earlier
steps measured (and the settings they chose) must survive it. Returns the engine, or None after saying why and
@@ -463,12 +443,36 @@ def restart(start_engine, args, what, say, report):
try:
return start_engine(args)
except Exception as e: # noqa: BLE001 - whatever the start raised (RuntimeError, EngineDied, OSError ...)
- why = str(e).strip().splitlines()[0][:300] if str(e).strip() else type(e).__name__
+ why = first_line(e)
say(f" the engine did not start for {what}: {why} - this candidate is dropped, the earlier results are kept")
report.setdefault("failed_starts", {})[what] = why
return None
+def measure_candidate(start_engine, args, what, say, report, by: dict, key, ids_list, max_new: int | None = None,
+ cancel_event=None):
+ """One restart-and-measure candidate (#1337): a start that fails and an engine that exits mid-measurement are
+ both that candidate losing - `by[key]` gets its two rates, or report["failed_starts"] its reason. Everything
+ measured earlier and the other candidates survive. An operator cancel is not a candidate death: it raises."""
+ e = restart(start_engine, args, what, say, report)
+ if e is None:
+ return
+ try:
+ s = Session(e, ids_list, cancel_event)
+ s.warm_up(1)
+ by[key] = [s.rate(max_new=max_new), s.rate(max_new=max_new)]
+ say(f" {what}: {statistics.median(by[key]):.1f} tok/s")
+ except CalibrationCancelled:
+ raise
+ except Exception as exc: # noqa: BLE001 - the engine died mid-measurement: this candidate loses (#1337)
+ why = first_line(exc)
+ say(f" the engine died while measuring {what}: {why} - this candidate is dropped, "
+ "the earlier results are kept")
+ report.setdefault("failed_starts", {})[what] = why
+ finally:
+ close(e)
+
+
def close(eng):
proc = getattr(eng, "proc", None)
if proc is None:
diff --git a/tools/test_calibrate.py b/tools/test_calibrate.py
index 97e335977..7ffe1fb63 100644
--- a/tools/test_calibrate.py
+++ b/tools/test_calibrate.py
@@ -21,9 +21,10 @@ import calibrate as CAL # noqa: E402
class FakeEngine:
- """Speed = f(pcie_frac, spec_min_p, workers): the GEN line's tune keys arrive as `strata_tune`."""
+ """Speed = f(pcie_frac, spec_min_p, workers): the GEN line's tune keys arrive as `strata_tune`.
+ `dies`: setting values whose engine exits during its first generation (the #267 class, mid-measurement)."""
- def __init__(self, args, speed, info_workers=6, starts=None, adapt=None, read=None, reads=None):
+ def __init__(self, args, speed, info_workers=6, starts=None, adapt=None, read=None, reads=None, dies=()):
self.args = list(args)
self.read, self.reads = read, reads # prompt read speed = read(prefill_pipe); the values it read with
self.adapt_bonus = (adapt or {}).get(CAL.arg_value(args, "--adapt-swaps"), 1.0)
@@ -33,10 +34,16 @@ class FakeEngine:
self.speed = speed
self.last = {}
self.proc = None
+ self.dies = CAL.arg_value(args, "--adapt-swaps") in dies or CAL.arg_value(args, "--pool-workers") in dies
+ self.dead = False
if starts is not None:
starts.append(list(args))
def generate(self, ids, max_new, sampling, cancel):
+ if self.dies and not self.dead:
+ self.dead = True
+ raise RuntimeError("the engine exited: ERR verify: the adaptive tier's copies for layer 47 never landed"
+ " (the copy stream itself is not drained); restart the engine")
tune = sampling.get("strata_tune") or {}
rate = self.speed(tune.get("pcie_frac", 0.55), tune.get("spec_min_p", self.info["spec_min_p"]), self.workers)
rate *= self.adapt_bonus
@@ -54,9 +61,10 @@ BASE = ["--pack", "p", "--spec", "4", "--spec-min-p", "0.5", "--max-context", "8
class Calibrate(unittest.TestCase):
- def run_with(self, speed, workers=6, base=BASE, adapt=None, read=None, reads=None):
+ def run_with(self, speed, workers=6, base=BASE, adapt=None, read=None, reads=None, dies=()):
starts = []
- res = CAL.measure(base, [[1, 2, 3]] * 3, lambda a: FakeEngine(a, speed, workers, starts, adapt, read, reads),
+ res = CAL.measure(base, [[1, 2, 3]] * 3,
+ lambda a: FakeEngine(a, speed, workers, starts, adapt, read, reads, dies),
say=lambda *_: None, long_prompt=lambda i, n: [1] * n)
return res, starts
@@ -233,6 +241,22 @@ class Calibrate(unittest.TestCase):
self.assertEqual(CAL.arg_value(a, "--adapt-swaps"), "160")
self.assertIsNone(CAL.arg_value(CAL.apply(a, {}), "--adapt-swaps")) # an old calibration's tier goes
+ def test_engine_death_during_tier_measurement_drops_only_that_candidate(self):
+ # the 160-swap candidate's engine dies during its own measurement (the adaptive gate's #267 livelock):
+ # that candidate loses, 80 (10% faster) is still chosen, and the run returns with the earlier results
+ res, _ = self.run_with(lambda f, p, w: 50.0, adapt={"80": 1.10}, dies=("160",))
+ self.assertEqual(res["settings"].get("--adapt-swaps"), "80")
+ self.assertEqual(list(res["report"]["adapt"]), ["default", "1/80/0.97"])
+ self.assertIn("the expert tier 1/160/0.97", res["report"]["failed_starts"])
+
+ def test_engine_death_during_a_worker_measurement_keeps_the_run(self):
+ # 4 workers' engine dies mid-measurement: 3 and 2 are still measured, the default count stays, the tier runs
+ res, _ = self.run_with(lambda f, p, w: 50.0, workers=6, dies=("4",))
+ self.assertNotIn("--pool-workers", res["settings"])
+ self.assertEqual(sorted(int(k.split()[0]) for k in res["report"]["workers"]), [2, 3, 6])
+ self.assertIn("4 workers", res["report"]["failed_starts"])
+ self.assertEqual(list(res["report"]["adapt"]), ["default", "1/80/0.97", "1/160/0.97"])
+
def test_old_calibration_is_the_baseline_reset(self):
# a config tuned earlier: the measurement starts from the product defaults, not from those values
base = CAL.apply(BASE, {"--pcie-frac": "0.20", "--pool-workers": "3", "--spec-min-p": "0.70"})
@@ -277,6 +301,18 @@ class Calibrate(unittest.TestCase):
self.assertEqual(res["env"], {})
self.assertIn(f"{CAL.SH_STREAM}=1", res["report"]["failed_starts"])
+ def test_engine_death_during_a_fork_measurement_keeps_the_run(self):
+ # the forked engine exits mid-measurement (#267's class): that candidate loses, the run returns
+ def start(args, env=None):
+ e = FakeEngine(args, lambda f, p, w: 50.0, 6)
+ if (env or {}).get(CAL.SH_STREAM) == "1":
+ e.dies = True
+ return e
+ res = CAL.measure(BASE, [[1, 2, 3]] * 3, start, say=lambda *_: None, hip=True)
+ self.assertEqual(res["env"], {})
+ self.assertEqual(list(res["report"]["sh_stream"]), ["0"])
+ self.assertIn(f"{CAL.SH_STREAM}=1", res["report"]["failed_starts"])
+
def test_apply_env(self):
old = {"STRATA_ROCBLAS_TUNING": "tools/hip", CAL.SH_STREAM: "1"}
self.assertEqual(CAL.apply_env(old, {}), {"STRATA_ROCBLAS_TUNING": "tools/hip"}) # an older choice goes