Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
"""Tests for tools/calibrate.py and setup's use of it, without a GPU: a stand-in engine whose decode speed is a
function of the settings it runs with.
python -m unittest tools.test_calibrate
"""
from __future__ import annotations
import json
import os
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tools"))
sys.path.insert(0, str(ROOT))
import calibrate as CAL # noqa: E402
class FakeEngine:
"""Speed = f(pcie_frac, spec_min_p, workers): the GEN line's tune keys arrive as `strata_tune`.
`dies`: setting values whose engine exits during its first generation (the #267 class, mid-measurement)."""
def __init__(self, args, speed, info_workers=6, starts=None, adapt=None, read=None, reads=None, dies=()):
self.args = list(args)
self.read, self.reads = read, reads # prompt read speed = read(prefill_pipe); the values it read with
self.adapt_bonus = (adapt or {}).get(CAL.arg_value(args, "--adapt-swaps"), 1.0)
w = CAL.arg_value(args, "--pool-workers")
self.workers = int(w) if w else info_workers
self.info = {"pool_workers": info_workers, "pcie_frac": 0.55, "spec_min_p": float(CAL.arg_value(args, "--spec-min-p") or 0)}
self.speed = speed
self.last = {}
self.proc = None
self.dies = CAL.arg_value(args, "--adapt-swaps") in dies or CAL.arg_value(args, "--pool-workers") in dies
self.dead = False
if starts is not None:
starts.append(list(args))
def generate(self, ids, max_new, sampling, cancel):
if self.dies and not self.dead:
self.dead = True
raise RuntimeError("the engine exited: ERR verify: the adaptive tier's copies for layer 47 never landed"
" (the copy stream itself is not drained); restart the engine")
tune = sampling.get("strata_tune") or {}
rate = self.speed(tune.get("pcie_frac", 0.55), tune.get("spec_min_p", self.info["spec_min_p"]), self.workers)
rate *= self.adapt_bonus
for _ in range(max_new):
yield 1
self.last = {"generated": max_new, "decode_ms": max_new / rate * 1000.0}
if "prefill_pipe" in tune and self.read is not None:
n = len(ids)
self.last.update(prompt_tokens=n, prompt_read=n, prompt_ms=n / self.read(tune["prefill_pipe"]) * 1000.0)
if self.reads is not None:
self.reads.append(tune["prefill_pipe"])
BASE = ["--pack", "p", "--spec", "4", "--spec-min-p", "0.5", "--max-context", "8192"]
class Calibrate(unittest.TestCase):
def run_with(self, speed, workers=6, base=BASE, adapt=None, read=None, reads=None, dies=()):
starts = []
res = CAL.measure(base, [[1, 2, 3]] * 3,
lambda a: FakeEngine(a, speed, workers, starts, adapt, read, reads, dies),
say=lambda *_: None, long_prompt=lambda i, n: [1] * n)
return res, starts
def test_prefill_pipe_on_a_layer_split(self):
# two stages that read fastest with b/a 1536: kept, and only one engine start measures it
split = BASE + ["--layer-split", "auto"]
reads = []
res, _ = self.run_with(lambda f, p, w: 50.0, base=split, reads=reads,
read=lambda v: {1536: 600.0, 768: 560.0, 1: 520.0}.get(v, 500.0))
self.assertEqual(res["settings"].get("--prefill-pipe"), "1536")
self.assertEqual(sorted(set(reads)), sorted(set(CAL.PIPE_VALUES)))
self.assertIn("pipe_sweep", res["report"])
self.assertEqual(CAL.arg_value(CAL.apply(split, res["settings"]), "--prefill-pipe"), "1536")
def test_prefill_pipe_noise_stays_off(self):
# 2% faster evened out: below MIN_GAIN, so the engine default (off) stays
res, _ = self.run_with(lambda f, p, w: 50.0, base=BASE + ["--layer-split", "auto"],
read=lambda v: 510.0 if v == 1 else 500.0)
self.assertNotIn("--prefill-pipe", res["settings"])
def test_prefill_pipe_must_win_every_round(self):
# 1536 looks 12% faster in the sweep and the median, but one paired round reads slower: noise, it stays off
calls = {}
def read(v):
calls[v] = calls.get(v, 0) + 1
if v != 1536:
return 500.0
return [560.0, 560.0, 560.0, 480.0, 560.0, 560.0][min(calls[v], 6) - 1]
res, _ = self.run_with(lambda f, p, w: 50.0, base=BASE + ["--layer-split", "auto"], read=read)
self.assertNotIn("--prefill-pipe", res["settings"])
self.assertEqual(len(res["report"]["pipe_confirm"]["1536"]), 3)
def test_prefill_pipe_reads_the_same_text(self):
# every read: the same stretch of text after a header of its own (only the chunks differ between values)
tok = type("T", (), {"encode": lambda s, t, **k: [ord(c) for c in t]})()
make = CAL.doc_prompt(tok)
a, b = make(1, 300), make(2, 300)
self.assertNotEqual(a[:40], b[:40])
self.assertEqual(a[-320:], b[-320:])
def test_prefill_pipe_only_on_a_layer_split(self):
reads = []
res, _ = self.run_with(lambda f, p, w: 50.0, read=lambda v: 600.0 if v == 1536 else 500.0, reads=reads)
self.assertEqual(reads, [])
self.assertNotIn("--prefill-pipe", res["settings"])
# an old calibration's value goes when a new one does not keep it
self.assertIsNone(CAL.arg_value(CAL.apply(BASE + ["--prefill-pipe", "800"], {}), "--prefill-pipe"))
def test_prefill_pipe_follows_the_engines_stage_count(self):
# --layer-split that left one prompt stage (the engine says prefill_stages=1): nothing to measure
reads = []
class OneStage(FakeEngine):
def __init__(self, *a, **k):
super().__init__(*a, **k)
self.info["prefill_stages"] = 1
res = CAL.measure(BASE + ["--layer-split", "auto"], [[1, 2, 3]] * 3,
lambda a: OneStage(a, lambda f, p, w: 50.0, 6, None, None,
lambda v: 600.0 if v == 1536 else 500.0, reads),
say=lambda *_: None, long_prompt=lambda i, n: [1] * n)
self.assertEqual(reads, [])
self.assertNotIn("--prefill-pipe", res["settings"])
def test_defaults_kept_when_flat(self):
res, _ = self.run_with(lambda f, p, w: 50.0)
self.assertEqual(res["settings"], {})
def test_measure_reports_bounded_stages_and_stops_between_samples(self):
events = []
stopped = threading.Event()
def progress(stage, completed, total):
events.append((stage, completed, total))
if stage == "pcie_share" and completed == 1:
stopped.set()
with self.assertRaises(CAL.CalibrationCancelled):
CAL.measure(BASE, [[1, 2, 3]] * 3,
lambda args: FakeEngine(args, lambda *_: 50.0),
say=lambda *_: None, progress=progress, cancel_event=stopped)
self.assertIn(("loading", 1, 1), events)
self.assertIn(("pcie_share", 1, len(CAL.PCIE_FRACS) * CAL.SWEEP_ROUNDS), events)
self.assertNotIn("draft_floor", [stage for stage, _, _ in events])
def test_active_generation_receives_shared_cancel_event(self):
stopped = threading.Event()
seen = []
class Engine:
def generate(self, ids, max_new, sampling, cancel):
seen.append(cancel)
cancel.set()
yield None
session = CAL.Session(Engine(), [[1, 2, 3]], stopped)
with self.assertRaises(CAL.CalibrationCancelled):
session.rate()
self.assertEqual(seen, [stopped])
def test_successful_sweep_covers_every_progress_stage(self):
events = []
CAL.measure(BASE, [[1, 2, 3]] * 3,
lambda args: FakeEngine(args, lambda *_: 50.0),
say=lambda *_: None, progress=lambda *item: events.append(item))
stages = list(dict.fromkeys(stage for stage, _, _ in events))
self.assertEqual(stages, ["loading", "pcie_share", "draft_floor", "confirmation",
"cpu_workers", "expert_tier"])
for stage in stages:
readings = [(done, total) for current, done, total in events if current == stage]
self.assertEqual(readings[0][0], 0)
self.assertEqual(readings[-1][0], readings[-1][1])
self.assertTrue(all(before[0] <= after[0] and before[1] == after[1]
for before, after in zip(readings, readings[1:])))
def test_small_gain_is_noise(self):
# 2% better at pcie 0.35: below MIN_GAIN, so the default stays
res, _ = self.run_with(lambda f, p, w: 51.0 if abs(f - 0.35) < 1e-6 else 50.0)
self.assertEqual(res["settings"], {})
def test_finds_pcie_and_min_p(self):
def speed(f, p, w):
return 50.0 + (10.0 if abs(f - 0.2) < 1e-6 else 0.0) + (5.0 if abs(p - 0.7) < 1e-6 else 0.0)
res, _ = self.run_with(speed)
self.assertEqual(res["settings"].get("--pcie-frac"), "0.20")
self.assertEqual(res["settings"].get("--spec-min-p"), "0.70")
self.assertNotIn("--pool-workers", res["settings"])
def test_fewer_workers(self):
# a hybrid CPU: half the workers is 20% faster
res, starts = self.run_with(lambda f, p, w: 60.0 if w == 3 else 50.0, workers=6)
self.assertEqual(res["settings"].get("--pool-workers"), "3")
# one start per worker count and per expert-tier candidate, plus the sweep
self.assertEqual(len(starts), 1 + len(CAL.worker_candidates(6)) + len(CAL.ADAPT_CANDIDATES))
self.assertEqual(CAL.arg_value(starts[0], "--spec-min-p"), "0.5") # measured against the product default
def test_a_failed_restart_keeps_the_measurements_1337(self):
# the top PCIe share wins by far; every later restart (the worker counts, the expert tier) fails to start
n = {"starts": 0}
def start(args):
n["starts"] += 1
if n["starts"] > 1:
raise RuntimeError("the engine exited before it was ready: cudaMalloc failed (213 MiB free)")
return FakeEngine(args, lambda f, p, w: 80.0 if f == 0.75 else 50.0, 6)
said = []
res = CAL.measure(BASE, [[1, 2, 3]] * 3, start, say=said.append)
self.assertEqual(res["settings"].get("--pcie-frac"), "0.75") # steps 1-3 survived
self.assertGreater(len(res["report"]["failed_starts"]), 1)
self.assertTrue(any("did not start" in x and "cudaMalloc" in x for x in said))
self.assertEqual(res["report"]["workers"], {})
self.assertEqual(res["report"]["adapt"], {})
def test_one_failing_worker_count_only_drops_that_candidate_1337(self):
def start(args):
if CAL.arg_value(args, "--pool-workers") == "3":
raise RuntimeError("out of memory")
return FakeEngine(args, lambda f, p, w: 60.0 if w == 2 else 50.0, 6)
res = CAL.measure(BASE, [[1, 2, 3]] * 3, start, say=lambda *_: None)
self.assertEqual(res["settings"].get("--pool-workers"), "2")
self.assertEqual(list(res["report"]["failed_starts"]), ["3 workers"])
def test_adaptive_tier(self):
# a slow-RAM PC: swapping 160 experts per round is 10% faster, 80 is 2% (noise)
res, starts = self.run_with(lambda f, p, w: 50.0, adapt={"160": 1.10, "80": 1.02})
self.assertEqual([res["settings"].get(f) for f in CAL.ADAPT_FLAGS], ["1", "160", "0.97"])
flat, _ = self.run_with(lambda f, p, w: 50.0, adapt={"160": 1.02})
self.assertFalse(any(f in flat["settings"] for f in CAL.ADAPT_FLAGS))
# the candidates are measured with the worker count chosen before
res, starts = self.run_with(lambda f, p, w: 60.0 if w == 3 else 50.0, workers=6, adapt={"160": 1.10})
self.assertEqual(res["settings"].get("--pool-workers"), "3")
self.assertTrue(all(CAL.arg_value(a, "--pool-workers") == "3" for a in starts[-len(CAL.ADAPT_CANDIDATES):]))
a = CAL.apply(BASE, res["settings"])
self.assertEqual(CAL.arg_value(a, "--adapt-swaps"), "160")
self.assertIsNone(CAL.arg_value(CAL.apply(a, {}), "--adapt-swaps")) # an old calibration's tier goes
def test_engine_death_during_tier_measurement_drops_only_that_candidate(self):
# the 160-swap candidate's engine dies during its own measurement (the adaptive gate's #267 livelock):
# that candidate loses, 80 (10% faster) is still chosen, and the run returns with the earlier results
res, _ = self.run_with(lambda f, p, w: 50.0, adapt={"80": 1.10}, dies=("160",))
self.assertEqual(res["settings"].get("--adapt-swaps"), "80")
self.assertEqual(list(res["report"]["adapt"]), ["default", "1/80/0.97"])
self.assertIn("the expert tier 1/160/0.97", res["report"]["failed_starts"])
def test_engine_death_during_a_worker_measurement_keeps_the_run(self):
# 4 workers' engine dies mid-measurement: 3 and 2 are still measured, the default count stays, the tier runs
res, _ = self.run_with(lambda f, p, w: 50.0, workers=6, dies=("4",))
self.assertNotIn("--pool-workers", res["settings"])
self.assertEqual(sorted(int(k.split()[0]) for k in res["report"]["workers"]), [2, 3, 6])
self.assertIn("4 workers", res["report"]["failed_starts"])
self.assertEqual(list(res["report"]["adapt"]), ["default", "1/80/0.97", "1/160/0.97"])
def test_old_calibration_is_the_baseline_reset(self):
# a config tuned earlier: the measurement starts from the product defaults, not from those values
base = CAL.apply(BASE, {"--pcie-frac": "0.20", "--pool-workers": "3", "--spec-min-p": "0.70"})
_, starts = self.run_with(lambda f, p, w: 50.0, base=base)
self.assertIsNone(CAL.arg_value(starts[0], "--pcie-frac"))
self.assertIsNone(CAL.arg_value(starts[0], "--pool-workers"))
self.assertEqual(CAL.arg_value(starts[0], "--spec-min-p"), "0.5")
def test_sh_stream_is_measured_on_hip_only(self):
# #1272 / #816: the shared-expert fork is measured off (the HIP default) against on, each with a restart and
# the engine variable set explicitly; it is kept only when it wins by more than MIN_GAIN
def run(bonus, hip=True):
starts, envs = [], []
def start(args, env=None):
envs.append(dict(env or {}))
fork = (env or {}).get(CAL.SH_STREAM) == "1"
return FakeEngine(args, lambda f, p, w: 50.0 * (bonus if fork else 1.0), 6, starts)
res = CAL.measure(BASE, [[1, 2, 3]] * 3, start, say=lambda *_: None, hip=hip)
return res, starts, envs
res, starts, envs = run(1.08)
self.assertEqual(res["env"], {CAL.SH_STREAM: "1"})
self.assertEqual(envs[-2:], [{CAL.SH_STREAM: "0"}, {CAL.SH_STREAM: "1"}])
self.assertEqual(starts[-1], starts[-2]) # the same args for both: only the variable differs
self.assertGreater(res["report"]["tok_s"], 52.0) # the reported speed is the fork's
flat, _, _ = run(1.02) # within noise: the default stays
self.assertEqual(flat["env"], {})
self.assertEqual(set(flat["report"]["sh_stream"]), {"0", "1"})
slower, _, _ = run(0.80)
self.assertEqual(slower["env"], {})
cuda, _, envs = run(1.50, hip=False) # not AMD: no step, no engine variable
self.assertEqual(cuda["env"], {})
self.assertNotIn("sh_stream", cuda["report"])
self.assertTrue(all(e == {} for e in envs))
def test_a_failed_fork_start_keeps_the_default(self):
def start(args, env=None):
if (env or {}).get(CAL.SH_STREAM) == "1":
raise RuntimeError("out of memory")
return FakeEngine(args, lambda f, p, w: 50.0, 6)
res = CAL.measure(BASE, [[1, 2, 3]] * 3, start, say=lambda *_: None, hip=True)
self.assertEqual(res["env"], {})
self.assertIn(f"{CAL.SH_STREAM}=1", res["report"]["failed_starts"])
def test_engine_death_during_a_fork_measurement_keeps_the_run(self):
# the forked engine exits mid-measurement (#267's class): that candidate loses, the run returns
def start(args, env=None):
e = FakeEngine(args, lambda f, p, w: 50.0, 6)
if (env or {}).get(CAL.SH_STREAM) == "1":
e.dies = True
return e
res = CAL.measure(BASE, [[1, 2, 3]] * 3, start, say=lambda *_: None, hip=True)
self.assertEqual(res["env"], {})
self.assertEqual(list(res["report"]["sh_stream"]), ["0"])
self.assertIn(f"{CAL.SH_STREAM}=1", res["report"]["failed_starts"])
def test_apply_env(self):
old = {"STRATA_ROCBLAS_TUNING": "tools/hip", CAL.SH_STREAM: "1"}
self.assertEqual(CAL.apply_env(old, {}), {"STRATA_ROCBLAS_TUNING": "tools/hip"}) # an older choice goes
self.assertEqual(CAL.apply_env({"X": "1"}, {CAL.SH_STREAM: "1"}), {"X": "1", CAL.SH_STREAM: "1"})
self.assertEqual(CAL.apply_env(None, None), {})
def test_apply(self):
a = CAL.apply(BASE, {"--pcie-frac": "0.35", "--pool-workers": "4"})
self.assertEqual(CAL.arg_value(a, "--pcie-frac"), "0.35")
self.assertEqual(CAL.arg_value(a, "--pool-workers"), "4")
self.assertEqual(CAL.arg_value(a, "--spec-min-p"), "0.5")
b = CAL.apply(a, {}) # back to the defaults
self.assertIsNone(CAL.arg_value(b, "--pcie-frac"))
self.assertIsNone(CAL.arg_value(b, "--pool-workers"))
self.assertEqual(b.count("--spec-min-p"), 1)
def test_engine_args_are_the_servers(self):
# #447: a "gpu" list is a layer split only with two or more cards; "0,2" is one too; split_skip_if_fits counts
helper = {"args": BASE + ["--expert-cache-device1", "1800"], "gpu": [0], "env": {"CUDA_VISIBLE_DEVICES": "0,1"}}
self.assertEqual(CAL.engine_args(helper), helper["args"]) # one stage + a helper card: no split
self.assertEqual(CAL.engine_args({"args": BASE, "gpu": 1}), BASE)
self.assertEqual(CAL.engine_args({"args": BASE, "gpu": [0, 2]}), BASE + ["--layer-split", "auto"])
self.assertEqual(CAL.engine_args({"args": BASE, "gpu": "0,2", "layer_split": "18"}),
BASE + ["--layer-split", "18"])
self.assertEqual(CAL.engine_args({"args": BASE, "gpu": [0, 1], "split_skip_if_fits": True}),
BASE + ["--layer-split", "auto", "--split-skip-if-fits"])
own = BASE + ["--layer-split", ""] # the config's own value wins
self.assertEqual(CAL.engine_args({"args": own, "gpu": [0, 1]}), own)
def test_run_measures_with_the_servers_args(self):
with tempfile.TemporaryDirectory() as d:
tok = Path(d)
(tok / "vocab.json").write_text(json.dumps({"a": 0, "b": 1}))
(tok / "merges.txt").write_text("")
(tok / "token_type.json").write_text(json.dumps([1, 1]))
seen = []
saved = CAL.measure
CAL.measure = lambda args, ids_list, start_engine, say=print, extra=(), long_prompt=None, hip=False: (
seen.append((args, hip)) or {})
fake = type("ST", (), {"Tokenizer": lambda *a: type("T", (), {"encode": lambda s, t, **k: [0]})()})
try:
with mock.patch.dict(sys.modules, {"strata_tokenizer": fake}):
CAL.run({"tokenizer": d, "args": list(BASE), "gpu": [0]}, start_engine=lambda a: None)
CAL.run({"tokenizer": d, "args": list(BASE), "gpu": [0], "backend": "hip"}, start_engine=lambda a: None)
finally:
CAL.measure = saved
self.assertEqual(seen, [(BASE, False), (BASE, True)]) # the shared-expert step runs on AMD only
def test_engine_error(self):
with tempfile.TemporaryDirectory() as d:
log = Path(d) / "strata-x.log"
log.write_text("strata generate: an old error\n", encoding="utf-8")
since = log.stat().st_size
with open(log, "a", encoding="utf-8") as f:
f.write("loading ...\nstrata generate: --expert-cache-remote with a layer split needs a GPU that runs "
"no stage (2 visible, 2 used by the split)\n\n")
self.assertEqual(CAL.engine_error(str(log), since), "strata generate: --expert-cache-remote with a layer "
"split needs a GPU that runs no stage (2 visible, 2 used by the split)")
with open(log, "a", encoding="utf-8") as f:
f.write("Segmentation fault\n")
self.assertIn("--expert-cache-remote", CAL.engine_error(str(log), since)) # the engine's own line first
self.assertEqual(CAL.engine_error(str(log), log.stat().st_size), None) # nothing new since
self.assertIsNone(CAL.engine_error(None))
self.assertIsNone(CAL.engine_error(str(Path(d) / "missing.log")))
def test_worker_candidates(self):
self.assertEqual(CAL.worker_candidates(6), [6, 4, 3, 2])
self.assertEqual(CAL.worker_candidates(23), [23, 15, 12, 6]) # a quarter: 4 beat 15 on 8P + 16E
self.assertEqual(CAL.worker_candidates(23, (7,)), [23, 15, 12, 6, 7]) # P-cores - 1
self.assertEqual(CAL.worker_candidates(35, (17,)), [35, 23, 18, 9, 17]) # 2 sockets: one socket's cores - 1
self.assertEqual(CAL.worker_candidates(6, (6, 1, 9)), [6, 4, 3, 2]) # none above the default, none below 2
self.assertEqual(CAL.worker_candidates(3), [3, 2])
self.assertEqual(CAL.worker_candidates(1), [1])
def test_pick(self):
self.assertEqual(CAL.pick({"a": [50, 51, 49], "b": [53, 52, 60]}, "a"), "b")
self.assertEqual(CAL.pick({"a": [50, 51, 49], "b": [51, 51.5, 51]}, "a"), "a")
self.assertEqual(CAL.pick({}, "a"), "a")
class SetupIntegration(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.old = {k: os.environ.get(k) for k in ("APPDATA", "XDG_CONFIG_HOME")}
os.environ["APPDATA"] = self.tmp.name
os.environ["XDG_CONFIG_HOME"] = self.tmp.name
import setup as S
self.S = S
self.saved = (S.gpu_info, S.cpu_info, S.ram_gb, CAL.run)
S.gpu_info = lambda pick=None: {"name": "RTX Test", "vram_gb": 12.0, "arch": "120", "count": 1, "index": 0}
S.cpu_info = lambda: ("Test CPU", True, True)
S.ram_gb = lambda: 64.0
def tearDown(self):
self.S.gpu_info, self.S.cpu_info, self.S.ram_gb, CAL.run = self.saved
for k, v in self.old.items():
if v is None:
os.environ.pop(k, None)
else:
os.environ[k] = v
self.tmp.cleanup()
def test_calibrate_config_writes_and_remembers(self):
cfg_path = Path(self.tmp.name) / "strata-q2_0.json"
cfg = {"exe": "x", "args": list(BASE), "model_name": "qwen3.8-flash-next-q2_0"}
cfg_path.write_text(json.dumps(cfg))
CAL.run = lambda c, say=print, start_engine=None: {"settings": {"--pcie-frac": "0.20"}, "report": {"tok_s": 61.2}}
self.assertTrue(self.S.calibrate_config(cfg_path))
written = json.loads(cfg_path.read_text())
self.assertEqual(CAL.arg_value(written["args"], "--pcie-frac"), "0.20")
saved = self.S.saved_calibration(written)
self.assertEqual(saved["settings"], {"--pcie-frac": "0.20"})
# another model on the same PC has no calibration yet
self.assertIsNone(self.S.saved_calibration({**written, "model_name": "swift-iq2_xs"}))
# another context size is another key (its KV cache changes the expert cache)
other = dict(written, args=CAL.with_arg(written["args"], "--max-context", "131072"))
self.assertIsNone(self.S.saved_calibration(other))
def test_calibrate_config_writes_the_env_choice(self):
# HIP: the measured engine variable goes into the config's "env" and the settings file; a re-run that finds it
# no longer faster removes it again, and leaves the config's other variables alone
cfg_path = Path(self.tmp.name) / "strata-iq3_s.json"
cfg = {"exe": "x", "args": list(BASE), "model_name": "m", "backend": "hip", "env": {"STRATA_ROCBLAS_TUNING": "t"}}
cfg_path.write_text(json.dumps(cfg))
CAL.run = lambda c, say=print, start_engine=None: {"settings": {}, "env": {CAL.SH_STREAM: "1"}, "report": {"tok_s": 70.1}}
self.assertTrue(self.S.calibrate_config(cfg_path))
written = json.loads(cfg_path.read_text())
self.assertEqual(written["env"], {"STRATA_ROCBLAS_TUNING": "t", CAL.SH_STREAM: "1"})
self.assertEqual(self.S.saved_calibration(written)["env"], {CAL.SH_STREAM: "1"})
CAL.run = lambda c, say=print, start_engine=None: {"settings": {}, "env": {}, "report": {"tok_s": 65.0}}
self.assertTrue(self.S.calibrate_config(cfg_path))
self.assertEqual(json.loads(cfg_path.read_text())["env"], {"STRATA_ROCBLAS_TUNING": "t"})
def test_failed_calibration_keeps_defaults(self):
cfg_path = Path(self.tmp.name) / "strata-q2_0.json"
cfg_path.write_text(json.dumps({"exe": "x", "args": list(BASE), "model_name": "m"}))
def boom(*a, **k):
raise RuntimeError("the engine exited before it was ready")
CAL.run = boom
self.assertFalse(self.S.calibrate_config(cfg_path))
self.assertEqual(json.loads(cfg_path.read_text())["args"], BASE)
self.assertIsNone(self.S.saved_calibration({"args": BASE, "model_name": "m"}))
def test_failed_calibration_says_the_engines_reason(self):
# #447: the engine's last error line is printed, and the --calibrate run repeats the failure before it starts
import contextlib
import io
log = Path(self.tmp.name) / "strata-q2_0.log"
log.write_text("strata generate: an error of an earlier start\n", encoding="utf-8")
cfg_path = Path(self.tmp.name) / "strata-q2_0.json"
cfg_path.write_text(json.dumps({"exe": "x", "args": list(BASE), "model_name": "m", "log": str(log)}))
def boom(*a, **k):
with open(log, "a", encoding="utf-8") as f:
f.write("strata generate: --expert-cache-remote with a layer split needs a GPU that runs no stage\n")
raise RuntimeError(f"the engine exited before it was ready (see {log})")
CAL.run = boom
out = io.StringIO()
with contextlib.redirect_stdout(out):
self.assertFalse(self.S.calibrate_config(cfg_path))
self.assertIn("the engine said: strata generate: --expert-cache-remote with a layer split", out.getvalue())
self.assertNotIn("an earlier start", out.getvalue())
out = io.StringIO()
with mock.patch.object(self.S, "ROOT", Path(self.tmp.name)), \
mock.patch.object(self.S, "data_folder", lambda d, *a, **k: (Path(self.tmp.name) / "data", [])), \
mock.patch.object(self.S, "update_installed_engine", lambda *a: None), \
mock.patch.object(sys, "argv", ["setup.py", "--calibrate", "--no-start"]), \
contextlib.redirect_stdout(out):
self.assertEqual(self.S.main(), 0)
self.assertIn("this PC is NOT tuned: the tuning failed (the reason is above); the model keeps the default "
"settings", out.getvalue())
if __name__ == "__main__":
unittest.main()