Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
tuner knobs from TUNING-SPEC draft 2: kv-resident 65536 at long contexts and the prefill chunk as exact candidates (prompt metric), pin-by-need/host-core/kv-grow last, no workers two below, single best knob
Niko122195e4f27f1781
diff --git a/strata_control/autotune.py b/strata_control/autotune.pyindex c95280743..fe51ada2c 100644--- a/strata_control/autotune.py+++ b/strata_control/autotune.py@@ -129,6 +129,9 @@ def metric(run: dict[str, Any] | RunResult, use: str) -> float | None:"""The number a configuration is judged on (6.1): decode speed for chat; for coding the geometric mean of decodespeed and long-prompt speed (the 4K prompt when no long one was measured)."""get = (lambda k: run.get(k)) if isinstance(run, dict) else (lambda k: getattr(run, k, None))+ if use == "prompt": # a prompt-only knob (the chunk size): long-prompt speed, the 4K prompt otherwise+ p = get("pp_long_tok_s") or get("pp_tok_s")+ return float(p) if p else Noned = get("decode_tok_s")if not d:return None@@ -322,7 +325,10 @@ class Tuner:def _spec(self) -> RequestSet:long_tokens = 0- if self.o.use == "coding" and self.o.mode == "thorough":+ if self.ctx.context_tokens >= 65536:+ # long contexts mean long prompts: the chunk size (spec 4.3) only shows on a long one+ long_tokens = min(98304, int(0.75 * self.ctx.context_tokens))+ elif self.o.use == "coding" and self.o.mode == "thorough":long_tokens = min(28000, int(0.8 * self.ctx.context_tokens))return RequestSet(warmups=2 if self.o.mode == "thorough" else 1, decodes=5, decode_tokens=200,pp_tokens=4096 if self.ctx.context_tokens >= 6000 else 1024, long_tokens=long_tokens)@@ -375,6 +381,7 @@ class Tuner:planned = planned[:n_arms]arms = [ArmReport(p.arm.id, p.arm.label, dict(p.args), dict(p.env), p.arm.answer) for p in planned]by_id = {a.id: a for a in arms}+ by_use = {p.arm.id: ("prompt" if p.arm.metric == "prompt" else o.use) for p in planned}self.planned_runs = self.done_runs + per_arm * (len(planned) + 1) - 1 + 1 + (o.confirm_pairs or 0) * 2if not planned:self._say("plan", "No other setting is worth trying on this PC")@@ -444,7 +451,8 @@ class Tuner:# a candidate that lost by more than the margin against its own round's base is dropped from later roundsfor p in list(alive):b, c = round_runs[rnd].get("base"), round_runs[rnd].get(p.arm.id)- m_b, m_c = (metric(b, o.use) if b else None), (metric(c, o.use) if c else None)+ um = by_use[p.arm.id]+ m_b, m_c = (metric(b, um) if b else None), (metric(c, um) if c else None)if m_b and m_c and m_c / m_b < 1.0 - max(o.margin, EARLY_DROP):by_id[p.arm.id].status = "dropped"by_id[p.arm.id].reason = f"slower by {100 * (1 - m_c / m_b):.1f}% in round {rnd + 1}"@@ -476,11 +484,16 @@ class Tuner:if rep.status in ("failed", "skipped") or not rows:continueratios, pps = [], []+ um = by_use[p.arm.id]for b, c in rows:- mc, mbb = metric(c, o.use), metric(b, o.use)+ mc, mbb = metric(c, um), metric(b, um)if mc and mbb:ratios.append(mc / mbb)- if c.pp_tok_s and b.pp_tok_s:+ # the other side of the trade: prompt speed for a decode knob, decode speed for a prompt knob+ if um == "prompt":+ if c.decode_tok_s and b.decode_tok_s:+ pps.append(c.decode_tok_s / b.decode_tok_s)+ elif c.pp_tok_s and b.pp_tok_s:pps.append(c.pp_tok_s / b.pp_tok_s)rep.pairs = len(ratios)rep.faster = sum(1 for x in ratios if x > 1.0)@@ -499,26 +512,27 @@ class Tuner:r.notes.append(f"the PC was noisy ({spread * 100:.1f}% between identical runs)")# ---- stage 4: stack the winners, then (thorough) confirm against the base with more pairs+ fu = o.usechosen_args: dict[str, Any] = {}chosen_env: dict[str, str | None] = {}final_pairs: list[tuple[RunResult, RunResult]] = []if winners:winners.sort(key=lambda a: a.ratio_decode or 0.0, reverse=True)- incumbent = [winners[0]]- for extra in winners[1:]:- conflict = set(extra.settings) & {k for w in incumbent for k in w.settings}- if not conflict:- incumbent.append(extra)+ incumbent = [winners[0]] # gains did not stack in three stacking runs (spec 1.10): the best single knob+ if len(winners) > 1:+ r.notes.append("more than one setting helped; only the best one is kept because gains did not add up in the study")for w in incumbent:chosen_args.update(w.settings)chosen_env.update(w.env)- need_confirm = len(incumbent) > 1 or (o.confirm_pairs or 0) > 0+ fu = by_use[incumbent[0].id]+ need_confirm = (o.confirm_pairs or 0) > 0if need_confirm and not self._over_budget(reserve_s=2 * self._avg_run()):n = o.confirm_pairs or 2self._say("confirm", "Confirming the combined result against your current settings")- final_pairs = self._confirm(chosen_args, chosen_env, spec, n)- ratios = [metric(c, o.use) / metric(b, o.use) for b, c in final_pairs if metric(c, o.use) and metric(b, o.use)]- pps = [c.pp_tok_s / b.pp_tok_s for b, c in final_pairs if c.pp_tok_s and b.pp_tok_s]+ final_pairs = self._confirm(chosen_args, chosen_env, spec, n, fu)+ ratios = [metric(c, fu) / metric(b, fu) for b, c in final_pairs if metric(c, fu) and metric(b, fu)]+ pps = [c.decode_tok_s / b.decode_tok_s for b, c in final_pairs if c.decode_tok_s and b.decode_tok_s] if fu == "prompt" \+ else [c.pp_tok_s / b.pp_tok_s for b, c in final_pairs if c.pp_tok_s and b.pp_tok_s]need = max(2, int(math.ceil(0.8 * len(ratios)))) if len(ratios) >= 5 else len(ratios)if not (ratios and is_win(ratios, pps, threshold=threshold, need=need)):# the stack did not hold; fall back to the best single winner, which already won its own pairs@@ -539,6 +553,7 @@ class Tuner:if not final_pairs and winners:final_pairs = list(data[winners[0].id])+ fu_final = fu# ---- per-request winners (found in the base engine): keep only the ones that won every roundper_request_chosen = {row["key"]: row["value"] for row in r.per_request if row.get("won")}if per_request_chosen and not self.storm and not self.cancel.is_set() and not self._over_budget(reserve_s=self._avg_run()):@@ -554,7 +569,7 @@ class Tuner:# ---- resultif end_base is not None and end_base.status == "PASS" and drift > 0.03 and winners:- gain_guess = statistics.median([metric(c, o.use) / metric(b, o.use) for b, c in final_pairs]) - 1.0 if final_pairs else 0.0+ gain_guess = statistics.median([metric(c, fu) / metric(b, fu) for b, c in final_pairs]) - 1.0 if final_pairs else 0.0if gain_guess < 2 * drift:chosen_args, chosen_env, winners, final_pairs = {}, {}, [], []per_request_chosen = {}@@ -588,7 +603,8 @@ class Tuner:fields.append(key)r.answers = {"class": cls, "fields": sorted(set(fields))}if final_pairs:- ratios = [metric(c, o.use) / metric(b, o.use) for b, c in final_pairs if metric(c, o.use) and metric(b, o.use)]+ fu = fu_final+ ratios = [metric(c, fu) / metric(b, fu) for b, c in final_pairs if metric(c, fu) and metric(b, fu)]pps = [c.pp_tok_s / b.pp_tok_s for b, c in final_pairs if c.pp_tok_s and b.pp_tok_s]r.pairs = len(ratios)r.faster_pairs = sum(1 for x in ratios if x > 1.0)@@ -599,7 +615,7 @@ class Tuner:c_runs = [c for _, c in final_pairs]r.chosen = {"decode_tok_s": round(statistics.median([c.decode_tok_s for c in c_runs if c.decode_tok_s]), 2)if any(c.decode_tok_s for c in c_runs) else None,- "pp_tok_s": round(statistics.median([c.pp_tok_s for c in c_runs if c.pp_tok_s]), 1)+ "pp_tok_s": round(statistics.median([c.pp_long_tok_s or c.pp_tok_s for c in c_runs if c.pp_long_tok_s or c.pp_tok_s]), 1)if any(c.pp_tok_s for c in c_runs) else None,"rss_peak_mib": max([c.rss_peak_mib for c in c_runs if c.rss_peak_mib] or [None]) if any(c.rss_peak_mib for c in c_runs) else None,"vram_used_mib": max([c.vram_used_mib for c in c_runs if c.vram_used_mib] or [None]) if any(c.vram_used_mib for c in c_runs) else None}@@ -636,7 +652,7 @@ class Tuner:return "it would not leave the graphics memory you wanted free"return None- def _confirm(self, args: dict[str, Any], env: dict[str, str | None], spec: RequestSet, pairs: int) -> list[tuple[RunResult, RunResult]]:+ def _confirm(self, args: dict[str, Any], env: dict[str, str | None], spec: RequestSet, pairs: int, use: str | None = None) -> list[tuple[RunResult, RunResult]]:"""Interleaved pairs of the combined settings against the base (the order alternates)."""out: list[tuple[RunResult, RunResult]] = []cand_args = tune_knobs.apply_to_argv(self.base_args, args)@@ -652,7 +668,8 @@ class Tuner:runs[who] = self._run(cand_args if who == "cand" else self.base_args, env if who == "cand" else {}, spec,f"confirm {who} {i + 1}")if runs["cand"].status == "PASS" and runs["base"].status == "PASS":- if self.o.use and metric(runs["cand"], self.o.use) and metric(runs["base"], self.o.use):+ use = use or self.o.use+ if metric(runs["cand"], use) and metric(runs["base"], use):out.append((runs["base"], runs["cand"]))self.base_runs.append(runs["base"])else:diff --git a/strata_control/tune_knobs.py b/strata_control/tune_knobs.pyindex b6a296ddf..2e2dbb0bf 100644--- a/strata_control/tune_knobs.py+++ b/strata_control/tune_knobs.py@@ -53,6 +53,8 @@ class Arm:goals: tuple[str, ...] = ("speed", "coding", "chat", "room")ram_delta_mib: int = 0 # extra memory the engine is predicted to take over the base rundeep_only: bool = False+ #: what a win is judged on: "decode" (the goal's metric) or "prompt" (long-prompt speed; the knob costs no decode speed)+ metric: str = "decode"#: a dynamic value: name of a computed number ("workers-2" ...), filled from the base run's engine logdynamic: dict[str, str] = field(default_factory=dict)@@ -61,43 +63,44 @@ class Arm:ARMS: tuple[Arm, ...] = (- # --- class A: RAM-bound resident mode (6.2) ---------------------------------------------------------------+ # --- long contexts (spec 4.3, 3.3): exact, measured +5.4% decode (5/5) at 128K on the R9700 --------------------------------+ Arm("kv_resident_64k", "Keep more of the conversation in graphics memory", "exact", {"--kv-resident": 65536},+ classes=("A", "B", "C"), backends=("cuda", "hip"), needs=("ctx_ge_128k",)),+ # --- prompt speed: the chunk size is exact (10/10) and costs no decode speed on a card with VRAM to spare ------------------+ Arm("prefill_32k", "Larger prompt chunks (faster very long prompts)", "exact", {"--prefill": "auto:32768"},+ classes=("B",), goals=("coding", "chat"), ram_delta_mib=4096, needs=("vram_ge_24g", "long_prompts"), metric="prompt"),+ # --- class A: RAM-bound resident mode (RTX 3060 / 46 GB): the one measured gain is adapt-async, behind its own switch ------Arm("adapt_async", "Learn which experts to keep while you chat, in the background", "nonrepeatable",{"--adapt-async": 1}, backends=("cuda",), classes=("A",), needs=("resident_ram", "one_gpu")),- Arm("pin_by_need", "Pin memory only when needed", "exact", {}, {"STRATA_PIN_BY_NEED": "1"},- backends=("cuda",), classes=("A",), needs=("one_gpu",)),- Arm("host_core_last", "Run the control thread on the last CPU core", "exact", {"--host-core": "last"},- classes=("A", "F")),- # --- class B: large graphics memory (experts mostly resident) ---------------------------------------------+ # --- placement-dependent (behind the owner's switch for answers that may differ slightly) ---------------------------------Arm("adapt_tier_fast", "Re-balance the expert cache faster", "placement",- {"--adapt-every": 1, "--adapt-swaps": 80, "--adapt-decay": 0.97}, classes=("B",)),- Arm("spec3", "Shorter speculative window (3 tokens)", "placement", {"--spec": 3}, classes=("A", "B")),- Arm("prefill_32k", "Larger prompt chunks (faster very long prompts)", "exact", {"--prefill": "auto:32768"},- classes=("B",), goals=("coding",), ram_delta_mib=4096, needs=("vram_ge_24g",)),- # --- class C: mid graphics memory, PCIe-bound -------------------------------------------------------------+ {"--adapt-every": 1, "--adapt-swaps": 80, "--adapt-decay": 0.97}, classes=("B", "C")),Arm("adapt_decay_09", "Forget old expert use a little faster", "placement", {"--adapt-decay": 0.9},- classes=("C", "G"), backends=("cuda", "hip", "sycl")),+ classes=("B", "C", "G"), backends=("cuda", "hip", "sycl")),+ Arm("spec3", "Shorter speculative window (3 tokens)", "placement", {"--spec": 3}, classes=("A", "B")),+ # --- class C: mid graphics memory, PCIe-bound -------------------------------------------------------------------------------Arm("trim_stage", "Trim stage weights on two cards", "exact", {"--trim-stage-weights": True},backends=("cuda",), classes=("C",), needs=("two_gpus",)),- Arm("workers_minus2", "Two fewer CPU helper threads", "exact", {}, classes=("C",),- dynamic={"--pool-workers": "workers-2"}),- # --- class F: hybrid CPU (performance + efficiency cores) -------------------------------------------------- Arm("workers_minus1", "One fewer CPU helper thread", "exact", {}, classes=("F",),- dynamic={"--pool-workers": "workers-1"}, needs=("hybrid_cpu",)),- # --- class G: Intel Arc (SYCL) ----------------------------------------------------------------------------+ # --- class G: Intel Arc (SYCL): +5.5% (5/5, twice) ------------------------------------------------------------------------Arm("pcie_start_02", "Start with a smaller PCIe share", "placement", {"--pcie-frac": 0.2},backends=("sycl",), classes=("G",)),- # --- Qwen3.6 hybrid (class D) -----------------------------------------------------------------------------+ # --- Qwen3.6 hybrid (class D): exact while the experts fit the card, placement-dependent when CPU experts take part ------Arm("q36_workers_explicit", "Fix the CPU helper thread count", "exact", {}, families=("qwen36",),backends=("cuda", "hip"), classes=("D",), dynamic={"--pool-workers": "workers"}, needs=("hybrid_experts",)),Arm("q36_mtp1", "Shorter draft window (1)", "placement", {"--mtp": 1}, families=("qwen36",),backends=("cuda", "hip"), classes=("D",), needs=("hybrid_experts",)),- Arm("q36_prefill_4096", "Larger prompt chunks (twice the prompt speed, a little slower answers)", "exact",+ Arm("q36_prefill_4096", "Larger prompt chunks (twice the prompt speed, a little slower answers)", "placement",{"--prefill": 4096}, families=("qwen36",), backends=("cuda", "hip"), classes=("D",), goals=("coding",),needs=("hybrid_experts",)),- Arm("q36_hybrid_rows_256", "Larger hybrid row blocks", "exact", {"--hybrid-rows": 256}, families=("qwen36",),+ Arm("q36_hybrid_rows_256", "Larger hybrid row blocks", "placement", {"--hybrid-rows": 256}, families=("qwen36",),backends=("cuda", "hip"), classes=("D",), needs=("hybrid_experts",), deep_only=True),- # --- thorough only: effects measured as flat, tried last --------------------------------------------------+ # --- thorough only: measured as no effect (spec 4.2-4.4), tried last ------------------------------------------------------+ Arm("pin_by_need", "Pin memory only when needed", "exact", {}, {"STRATA_PIN_BY_NEED": "1"},+ backends=("cuda",), classes=("C",), needs=("one_gpu",), deep_only=True),+ Arm("host_core_last", "Run the control thread on the last CPU core", "exact", {"--host-core": "last"},+ classes=("B", "C"), deep_only=True),+ Arm("workers_minus1", "One fewer CPU helper thread", "exact", {}, classes=("F",),+ dynamic={"--pool-workers": "workers-1"}, needs=("hybrid_cpu",), deep_only=True),Arm("kv_grow", "Allocate the conversation memory as it grows", "exact", {"--kv-grow": True},classes=("B", "C"), backends=("cuda", "hip", "sycl"), deep_only=True),)@@ -126,6 +129,7 @@ BOUNDS: dict[str, dict[str, Any]] = {"--adapt-decay": {"min": 0.7, "max": 0.97},"--hybrid-rows": {"min": 64, "max": 256},"--host-core": {"choices": ("first", "last")},+ "--kv-resident": {"choices": (32768, 49152, 65536)},"--prefill": {"choices": ("auto", "auto:16384", "auto:32768", 2048, 4096)},}@@ -142,7 +146,7 @@ NEVER: tuple[dict[str, Any], ...] = ({"flag": "--ple-io", "values": ("ram", "mmap"), "when": ("ram_bound",),"reason": "it needs the table in memory, which this PC does not have to spare (-61% speed)"},{"flag": "--adapt-every", "test": "at_least", "limit": 100000, "reason": "turning the expert re-balancing off cost 3% to 44%"},- {"flag": "--kv-resident", "test": "below_context", "reason": "it made answers 27% to 34% slower and changes them"},+ {"flag": "--kv-resident", "test": "kv_resident", "reason": "below the context it made answers 22% to 34% slower (and changes them) except at 128K, where only 49152 and 65536 were measured as flat or better"},{"flag": "--pcie-frac", "test": "at_least", "limit": 0.75, "when": ("pcie_bound",),"reason": "a high PCIe share made answers 14% to 37% slower"},{"flag": "--prefill", "test": "fixed_below_auto", "reason": "smaller prompt chunks made long prompts 11% to 75% slower"},@@ -161,7 +165,7 @@ ANSWER_OF_FLAG: dict[str, str] = {"--expert-cache": "placement", "--vram-reserve-mib": "placement", "--adapt-every": "placement","--adapt-swaps": "placement", "--adapt-decay": "placement", "--adapt-min-gain": "placement","--pcie-frac": "placement", "--spec": "placement", "--mtp": "placement", "--lookup-chain": "placement",- "--kv-resident": "placement", "--kv": "quality", "--adapt-async": "nonrepeatable",+ "--kv": "quality", "--adapt-async": "nonrepeatable",}@@ -203,6 +207,8 @@ def condition_holds(name: str, ctx: Context) -> bool:"gfx12": bool(ctx.arch and ctx.arch.startswith("gfx12")),"pcie_bound": 0 < ctx.vram_mib < 24000,"ram_bound": bool(ctx.ram_gib and ctx.model_gib and ctx.ram_gib - ctx.model_gib < 30),+ "ctx_ge_128k": ctx.context_tokens >= 98304,+ "long_prompts": ctx.context_tokens >= 65536 or ctx.use == "coding",}.get(name, False)@@ -273,9 +279,11 @@ def check_setting(args: dict[str, Any], env: dict[str, str | None], ctx: Contextelif test == "at_most":n = _as_number(value)hit = n is not None and n <= rule["limit"] and flag == "--spec"- elif test == "below_context":+ elif test == "kv_resident":n = _as_number(value)- hit = n is not None and n < ctx_tokens+ own = _as_number(_current(ctx.base_args, "--kv-resident"))+ # the config's own value is always fine; at a long context 49152 and 65536 were measured; anything else below the context is not+ hit = n is not None and n < ctx_tokens and n != own and not (ctx_tokens >= 98304 and n in (49152, 65536))elif test == "fixed_below_auto":hit = _prefill_below_auto(value) and ctx.family == "flash-next"if ctx.family == "qwen36":@@ -342,7 +350,7 @@ def dynamic_value(spec: str, ctx: Context) -> Any:return int(workers) if workers else Noneif spec.startswith("workers-") and workers:value = int(workers) - int(spec[len("workers-"):])- floor = max(2, int(workers) // 4 + 1) # never below N/4 (6.6)+ floor = max(2, int(workers) - 1) # one step down at most (6.2 F); never two below (6.2 C)return value if value >= floor else Nonereturn Nonediff --git a/tests/tuning/test_autotune.py b/tests/tuning/test_autotune.pyindex 8addab956..50845041b 100644--- a/tests/tuning/test_autotune.py+++ b/tests/tuning/test_autotune.py@@ -39,6 +39,13 @@ def speed_with(gains: dict[str, float], base=60.0):return model+def make_ctx_small():+ c = make_ctx()+ c.context_tokens = 16384+ c.base_args = ["--max-context", "16384"]+ return c++def run_tuner(model, *, opts=None, ctx=None, clock=None, session_kw=None, cancel=None, progress=None, **kw):clock = clock or FakeClock()session = StubSession(model=model, clock=clock, **(session_kw or {}))@@ -57,11 +64,68 @@ class RulesTest(unittest.TestCase):({}, {"STRATA_RDNA2_SGEMM": "all"}), ({}, {"STRATA_HIP_WMMA": "1"}), ({"--pool-workers": 8}, {})]for args, env in bad:self.assertTrue(tune_knobs.check_setting(args, env, ctx), (args, env))- ok = [({"--spec": 3}, {}), ({"--host-core": "last"}, {}), ({"--pool-workers": 4}, {}), ({"--prefill": "auto:32768"}, {}),+ ok = [({"--spec": 3}, {}), ({"--kv-resident": 65536}, {}), ({"--pool-workers": 4}, {}), ({"--prefill": "auto:32768"}, {}),({"--adapt-every": 1, "--adapt-swaps": 80, "--adapt-decay": 0.97}, {}), ({"--pcie-frac": 0.2}, {})]for args, env in ok:self.assertEqual(tune_knobs.check_setting(args, env, ctx), [], (args, env))+ def test_kv_resident_rules_follow_the_context(self):+ long = make_ctx() # 128K, the config's own --kv-resident is 32768+ self.assertEqual(tune_knobs.check_setting({"--kv-resident": 65536}, {}, long), [])+ self.assertEqual(tune_knobs.check_setting({"--kv-resident": 49152}, {}, long), [])+ self.assertTrue(tune_knobs.check_setting({"--kv-resident": 20480}, {}, long)) # below the context and not measured+ short = make_ctx()+ short.context_tokens = 32768+ short.base_args = ["--max-context", "32768"]+ self.assertTrue(tune_knobs.check_setting({"--kv-resident": 20480}, {}, short)) # -29.5% at 32K+ self.assertEqual(tune_knobs.check_setting({"--kv-resident": 65536}, {}, make_ctx_small()), []) # above the context: no streaming at all++ def test_long_context_candidates_exist_and_are_exact(self):+ ctx = make_ctx()+ ctx.vram_mib = 32624 # the R9700: class B+ ctx.facts = {"workers": 31}+ arms, _ = tune_knobs.plan_arms(ctx, mode="quick")+ ids = [p.arm.id for p in arms]+ self.assertEqual(ids[:2], ["kv_resident_64k", "prefill_32k"])+ self.assertTrue(all(p.arm.answer == "exact" for p in arms))+ short = make_ctx()+ short.vram_mib = 32624+ short.context_tokens = 32768+ short.base_args = ["--max-context", "32768"]+ self.assertNotIn("kv_resident_64k", [p.arm.id for p in tune_knobs.plan_arms(short, mode="quick")[0]])++ def test_a_prompt_knob_is_judged_on_prompt_speed_and_may_not_cost_decode(self):+ ctx = make_ctx()+ ctx.vram_mib = 32624+ def pp(args, env):+ return 1500.0 * (1.06 if flag(args, "--prefill") == "auto:32768" else 1.0)+ # decode unchanged, long prompt +6%: wins on the prompt metric+ clock = FakeClock()+ sess = StubSession(model=lambda a, e, tune=None: 60.0, pp=pp, clock=clock, noise=0.001)+ t = autotune.Tuner(sess, BASE, {}, ctx, autotune.Options(budget_s=3600), clock=clock, sleep=clock.sleep, system=sess.system,+ total_ram_mib=64000, total_vram_mib=32000, quiet=lambda: None)+ r = t.run()+ self.assertEqual(r.settings["args"], {"--prefill": "auto:32768"}, r.notes)+ # the same prompt gain with a 6% decode loss does not+ sess2 = StubSession(model=lambda a, e, tune=None: 60.0 * (0.94 if flag(a, "--prefill") == "auto:32768" else 1.0), pp=pp, clock=FakeClock(), noise=0.001)+ c2 = sess2.clock+ t2 = autotune.Tuner(sess2, BASE, {}, ctx, autotune.Options(budget_s=3600), clock=c2, sleep=c2.sleep, system=sess2.system,+ total_ram_mib=64000, total_vram_mib=32000, quiet=lambda: None)+ self.assertEqual(t2.run().settings["args"], {})++ def test_only_the_best_single_knob_is_kept(self):+ ctx = make_ctx()+ ctx.vram_mib = 32624+ model = speed_with({("--kv-resident", "65536"): 1.06})+ pp = lambda a, e: 1500.0 * (1.07 if flag(a, "--prefill") == "auto:32768" else 1.0)+ clock = FakeClock()+ sess = StubSession(model=model, pp=pp, clock=clock, noise=0.001)+ t = autotune.Tuner(sess, BASE, {}, ctx, autotune.Options(budget_s=3600), clock=clock, sleep=clock.sleep, system=sess.system,+ total_ram_mib=64000, total_vram_mib=32000, quiet=lambda: None)+ r = t.run()+ self.assertEqual(len(r.settings["args"]), 1)+ self.assertTrue(any("only the best one" in n for n in r.notes))+def test_hybrid_cpu_affinity_never(self):ctx = make_ctx()ctx.hybrid_cpu = True@@ -101,7 +165,7 @@ class RulesTest(unittest.TestCase):ctx = make_ctx()ctx.engine_flags = ["--spec"]arms, skipped = tune_knobs.plan_arms(ctx, mode="thorough")- self.assertFalse(any(p.arm.id == "host_core_last" for p in arms))+ self.assertFalse(any(p.arm.id == "kv_resident_64k" for p in arms))self.assertTrue(any("does not accept" in s["reason"] for s in skipped))def test_qwen36_and_sycl_lists(self):@@ -110,7 +174,7 @@ class RulesTest(unittest.TestCase):q.facts = {"workers": 6, "hybrid_experts": True}ids = [p.arm.id for p in tune_knobs.plan_arms(q, mode="thorough")[0]]self.assertIn("q36_workers_explicit", ids)- self.assertNotIn("host_core_last", ids)+ self.assertNotIn("kv_resident_64k", ids)s = make_ctx(allow_placement=True)s.backend = "sycl"ids = [p.arm.id for p in tune_knobs.plan_arms(s, mode="thorough")[0]]@@ -119,17 +183,18 @@ class RulesTest(unittest.TestCase):def test_workers_arm_never_goes_below_a_quarter(self):c = make_ctx()c.facts = {"workers": 4}- self.assertEqual(tune_knobs.dynamic_value("workers-2", c), 2)+ self.assertEqual(tune_knobs.dynamic_value("workers-1", c), 3) # one step down at most+ self.assertIsNone(tune_knobs.dynamic_value("workers-2", c)) # never two below (spec 6.2 C)c.facts = {"workers": 2}- self.assertIsNone(tune_knobs.dynamic_value("workers-2", c))+ self.assertIsNone(tune_knobs.dynamic_value("workers-1", c))class TunerTest(unittest.TestCase):def test_finds_a_real_gain_and_changes_one_knob(self):- model = speed_with({("--host-core", "last"): 1.08})+ model = speed_with({("--kv-resident", "65536"): 1.08})t, s, r = run_tuner(model, ctx=make_ctx())self.assertEqual(r.outcome, "improved", r.notes)- self.assertEqual(r.settings["args"], {"--host-core": "last"})+ self.assertEqual(r.settings["args"], {"--kv-resident": 65536})self.assertAlmostEqual(r.gain_pct, 8.0, delta=2.0)self.assertEqual(r.answers["class"], "exact")for args, env in s.runs: # every run kept the user's own flags@@ -144,23 +209,23 @@ class TunerTest(unittest.TestCase):self.assertIn("already the fastest", r.message)def test_a_gain_below_the_margin_is_not_taken(self):- model = speed_with({("--host-core", "last"): 1.02})+ model = speed_with({("--kv-resident", "65536"): 1.02})t, s, r = run_tuner(model, session_kw={"noise": 0.001})self.assertEqual(r.settings["args"], {})def test_losing_candidates_are_dropped_after_one_loss(self):- model = speed_with({("--host-core", "last"): 0.85})+ model = speed_with({("--kv-resident", "65536"): 0.85})t, s, r = run_tuner(model, session_kw={"noise": 0.001})- arm = [a for a in r.arms if a["id"] == "host_core_last"][0]+ arm = [a for a in r.arms if a["id"] == "kv_resident_64k"][0]self.assertEqual(arm["status"], "dropped")- self.assertEqual(sum(1 for args, _ in s.runs if flag(args, "--host-core") == "last"), 1)+ self.assertEqual(sum(1 for args, _ in s.runs if flag(args, "--kv-resident") == "65536"), 1)def test_machine_drift_is_not_mistaken_for_a_gain(self):clock = FakeClock()counter = {"n": 0}def model(args, env, tune=None): # the machine slows 1.2% per engine start- return 60.0 * (1.0 - 0.012 * len(session.runs)) * (1.03 if flag(args, "--host-core") == "last" else 1.0)+ return 60.0 * (1.0 - 0.012 * len(session.runs)) * (1.03 if flag(args, "--kv-resident") == "65536" else 1.0)session = StubSession(model=model, clock=clock, noise=0.0005)t = autotune.Tuner(session, BASE, {}, make_ctx(), autotune.Options(), clock=clock, sleep=clock.sleep, system=session.system,total_ram_mib=64000, quiet=lambda: None)@@ -168,7 +233,7 @@ class TunerTest(unittest.TestCase):self.assertEqual(r.settings["args"], {}, r.notes)def test_candidate_that_does_not_fit_is_never_started(self):- model = speed_with({("--host-core", "last"): 1.10})+ model = speed_with({("--kv-resident", "65536"): 1.10})clock = FakeClock()session = StubSession(model=model, clock=clock, avail=9000, rss=lambda a, e: 12000.0)calls = {"n": 0}@@ -187,34 +252,34 @@ class TunerTest(unittest.TestCase):def test_first_oom_stops_the_search(self):def fail(args, env):- return StartFailed("OOM", "out of memory") if flag(args, "--host-core") == "last" else None+ return StartFailed("OOM", "out of memory") if flag(args, "--kv-resident") == "65536" else Nonet, s, r = run_tuner(speed_with({}), session_kw={"fail": fail})self.assertTrue(any("ran out of memory" in n for n in r.notes))- self.assertEqual(sum(1 for a, _ in s.runs if flag(a, "--host-core") == "last"), 1)+ self.assertEqual(sum(1 for a, _ in s.runs if flag(a, "--kv-resident") == "65536"), 1)self.assertEqual(r.settings["args"], {})def test_unsupported_candidate_fails_alone(self):def fail(args, env):- return StartFailed("UNSUPPORTED", "unknown argument: --host-core") if flag(args, "--host-core") else None+ return StartFailed("UNSUPPORTED", "unknown argument: --kv-resident") if flag(args, "--kv-resident") == "65536" else Nonectx = make_ctx()ctx.facts = {"workers": 6}- t, s, r = run_tuner(speed_with({("--pool-workers", "4"): 1.1}), ctx=ctx, session_kw={"fail": fail})+ t, s, r = run_tuner(speed_with({}), ctx=ctx, session_kw={"fail": fail})failed = [a for a in r.arms if a["status"] == "failed"]self.assertTrue(failed and "UNSUPPORTED" in failed[0]["reason"])- self.assertEqual(r.settings["args"], {"--pool-workers": 4})+ self.assertEqual(r.settings["args"], {})def test_room_goal_refuses_a_faster_candidate_that_takes_more_memory(self):- model = speed_with({("--host-core", "last"): 1.1})- rss = lambda a, e: 20000.0 if flag(a, "--host-core") == "last" else 12000.0+ model = speed_with({("--kv-resident", "65536"): 1.1})+ rss = lambda a, e: 20000.0 if flag(a, "--kv-resident") == "65536" else 12000.0t, s, r = run_tuner(model, opts=autotune.Options(goal="room", ram_wish_pct=20, reserve_mib=1536), session_kw={"rss": rss})self.assertEqual(r.settings["args"], {})self.assertTrue(any("more memory" in a["reason"] for a in r.arms))t, s, r = run_tuner(model, opts=autotune.Options(goal="speed"), session_kw={"rss": rss})- self.assertEqual(r.settings["args"], {"--host-core": "last"})+ self.assertEqual(r.settings["args"], {"--kv-resident": 65536})def test_cancel_changes_nothing(self):cancel = threading.Event()- model = speed_with({("--host-core", "last"): 1.1})+ model = speed_with({("--kv-resident", "65536"): 1.1})n = {"i": 0}def progress(p):@@ -280,15 +345,15 @@ class TunerTest(unittest.TestCase):self.assertAlmostEqual(r.gain_pct, 6.0, delta=1.5)def test_thorough_confirms_with_five_pairs(self):- model = speed_with({("--host-core", "last"): 1.06})+ model = speed_with({("--kv-resident", "65536"): 1.06})t, s, r = run_tuner(model, opts=autotune.Options(mode="thorough", budget_s=7200), session_kw={"noise": 0.003})self.assertEqual(r.outcome, "improved")self.assertEqual(r.pairs, 5)self.assertGreaterEqual(r.faster_pairs, 4)def test_prompt_regression_blocks_a_decode_win(self):- model = speed_with({("--host-core", "last"): 1.08})- pp = lambda a, e: 1500.0 * (0.8 if flag(a, "--host-core") == "last" else 1.0)+ model = speed_with({("--kv-resident", "65536"): 1.08})+ pp = lambda a, e: 1500.0 * (0.8 if flag(a, "--kv-resident") == "65536" else 1.0)t, s, r = run_tuner(model, session_kw={"pp": pp, "noise": 0.001})self.assertEqual(r.settings["args"], {})