Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

tuner knobs from TUNING-SPEC draft 2: kv-resident 65536 at long contexts and the prefill chunk as exact candidates (prompt metric), pin-by-need/host-core/kv-grow last, no workers two below, single best knob

Niko122195e4f27f1781
diff --git a/strata_control/autotune.py b/strata_control/autotune.py
index c95280743..fe51ada2c 100644
--- a/strata_control/autotune.py
+++ b/strata_control/autotune.py
@@ -129,6 +129,9 @@ def metric(run: dict[str, Any] | RunResult, use: str) -> float | None:
"""The number a configuration is judged on (6.1): decode speed for chat; for coding the geometric mean of decode
speed and long-prompt speed (the 4K prompt when no long one was measured)."""
get = (lambda k: run.get(k)) if isinstance(run, dict) else (lambda k: getattr(run, k, None))
+ if use == "prompt": # a prompt-only knob (the chunk size): long-prompt speed, the 4K prompt otherwise
+ p = get("pp_long_tok_s") or get("pp_tok_s")
+ return float(p) if p else None
d = get("decode_tok_s")
if not d:
return None
@@ -322,7 +325,10 @@ class Tuner:
def _spec(self) -> RequestSet:
long_tokens = 0
- if self.o.use == "coding" and self.o.mode == "thorough":
+ if self.ctx.context_tokens >= 65536:
+ # long contexts mean long prompts: the chunk size (spec 4.3) only shows on a long one
+ long_tokens = min(98304, int(0.75 * self.ctx.context_tokens))
+ elif self.o.use == "coding" and self.o.mode == "thorough":
long_tokens = min(28000, int(0.8 * self.ctx.context_tokens))
return RequestSet(warmups=2 if self.o.mode == "thorough" else 1, decodes=5, decode_tokens=200,
pp_tokens=4096 if self.ctx.context_tokens >= 6000 else 1024, long_tokens=long_tokens)
@@ -375,6 +381,7 @@ class Tuner:
planned = planned[:n_arms]
arms = [ArmReport(p.arm.id, p.arm.label, dict(p.args), dict(p.env), p.arm.answer) for p in planned]
by_id = {a.id: a for a in arms}
+ by_use = {p.arm.id: ("prompt" if p.arm.metric == "prompt" else o.use) for p in planned}
self.planned_runs = self.done_runs + per_arm * (len(planned) + 1) - 1 + 1 + (o.confirm_pairs or 0) * 2
if not planned:
self._say("plan", "No other setting is worth trying on this PC")
@@ -444,7 +451,8 @@ class Tuner:
# a candidate that lost by more than the margin against its own round's base is dropped from later rounds
for p in list(alive):
b, c = round_runs[rnd].get("base"), round_runs[rnd].get(p.arm.id)
- m_b, m_c = (metric(b, o.use) if b else None), (metric(c, o.use) if c else None)
+ um = by_use[p.arm.id]
+ m_b, m_c = (metric(b, um) if b else None), (metric(c, um) if c else None)
if m_b and m_c and m_c / m_b < 1.0 - max(o.margin, EARLY_DROP):
by_id[p.arm.id].status = "dropped"
by_id[p.arm.id].reason = f"slower by {100 * (1 - m_c / m_b):.1f}% in round {rnd + 1}"
@@ -476,11 +484,16 @@ class Tuner:
if rep.status in ("failed", "skipped") or not rows:
continue
ratios, pps = [], []
+ um = by_use[p.arm.id]
for b, c in rows:
- mc, mbb = metric(c, o.use), metric(b, o.use)
+ mc, mbb = metric(c, um), metric(b, um)
if mc and mbb:
ratios.append(mc / mbb)
- if c.pp_tok_s and b.pp_tok_s:
+ # the other side of the trade: prompt speed for a decode knob, decode speed for a prompt knob
+ if um == "prompt":
+ if c.decode_tok_s and b.decode_tok_s:
+ pps.append(c.decode_tok_s / b.decode_tok_s)
+ elif c.pp_tok_s and b.pp_tok_s:
pps.append(c.pp_tok_s / b.pp_tok_s)
rep.pairs = len(ratios)
rep.faster = sum(1 for x in ratios if x > 1.0)
@@ -499,26 +512,27 @@ class Tuner:
r.notes.append(f"the PC was noisy ({spread * 100:.1f}% between identical runs)")
# ---- stage 4: stack the winners, then (thorough) confirm against the base with more pairs
+ fu = o.use
chosen_args: dict[str, Any] = {}
chosen_env: dict[str, str | None] = {}
final_pairs: list[tuple[RunResult, RunResult]] = []
if winners:
winners.sort(key=lambda a: a.ratio_decode or 0.0, reverse=True)
- incumbent = [winners[0]]
- for extra in winners[1:]:
- conflict = set(extra.settings) & {k for w in incumbent for k in w.settings}
- if not conflict:
- incumbent.append(extra)
+ incumbent = [winners[0]] # gains did not stack in three stacking runs (spec 1.10): the best single knob
+ if len(winners) > 1:
+ r.notes.append("more than one setting helped; only the best one is kept because gains did not add up in the study")
for w in incumbent:
chosen_args.update(w.settings)
chosen_env.update(w.env)
- need_confirm = len(incumbent) > 1 or (o.confirm_pairs or 0) > 0
+ fu = by_use[incumbent[0].id]
+ need_confirm = (o.confirm_pairs or 0) > 0
if need_confirm and not self._over_budget(reserve_s=2 * self._avg_run()):
n = o.confirm_pairs or 2
self._say("confirm", "Confirming the combined result against your current settings")
- final_pairs = self._confirm(chosen_args, chosen_env, spec, n)
- ratios = [metric(c, o.use) / metric(b, o.use) for b, c in final_pairs if metric(c, o.use) and metric(b, o.use)]
- pps = [c.pp_tok_s / b.pp_tok_s for b, c in final_pairs if c.pp_tok_s and b.pp_tok_s]
+ final_pairs = self._confirm(chosen_args, chosen_env, spec, n, fu)
+ ratios = [metric(c, fu) / metric(b, fu) for b, c in final_pairs if metric(c, fu) and metric(b, fu)]
+ pps = [c.decode_tok_s / b.decode_tok_s for b, c in final_pairs if c.decode_tok_s and b.decode_tok_s] if fu == "prompt" \
+ else [c.pp_tok_s / b.pp_tok_s for b, c in final_pairs if c.pp_tok_s and b.pp_tok_s]
need = max(2, int(math.ceil(0.8 * len(ratios)))) if len(ratios) >= 5 else len(ratios)
if not (ratios and is_win(ratios, pps, threshold=threshold, need=need)):
# the stack did not hold; fall back to the best single winner, which already won its own pairs
@@ -539,6 +553,7 @@ class Tuner:
if not final_pairs and winners:
final_pairs = list(data[winners[0].id])
+ fu_final = fu
# ---- per-request winners (found in the base engine): keep only the ones that won every round
per_request_chosen = {row["key"]: row["value"] for row in r.per_request if row.get("won")}
if per_request_chosen and not self.storm and not self.cancel.is_set() and not self._over_budget(reserve_s=self._avg_run()):
@@ -554,7 +569,7 @@ class Tuner:
# ---- result
if end_base is not None and end_base.status == "PASS" and drift > 0.03 and winners:
- gain_guess = statistics.median([metric(c, o.use) / metric(b, o.use) for b, c in final_pairs]) - 1.0 if final_pairs else 0.0
+ gain_guess = statistics.median([metric(c, fu) / metric(b, fu) for b, c in final_pairs]) - 1.0 if final_pairs else 0.0
if gain_guess < 2 * drift:
chosen_args, chosen_env, winners, final_pairs = {}, {}, [], []
per_request_chosen = {}
@@ -588,7 +603,8 @@ class Tuner:
fields.append(key)
r.answers = {"class": cls, "fields": sorted(set(fields))}
if final_pairs:
- ratios = [metric(c, o.use) / metric(b, o.use) for b, c in final_pairs if metric(c, o.use) and metric(b, o.use)]
+ fu = fu_final
+ ratios = [metric(c, fu) / metric(b, fu) for b, c in final_pairs if metric(c, fu) and metric(b, fu)]
pps = [c.pp_tok_s / b.pp_tok_s for b, c in final_pairs if c.pp_tok_s and b.pp_tok_s]
r.pairs = len(ratios)
r.faster_pairs = sum(1 for x in ratios if x > 1.0)
@@ -599,7 +615,7 @@ class Tuner:
c_runs = [c for _, c in final_pairs]
r.chosen = {"decode_tok_s": round(statistics.median([c.decode_tok_s for c in c_runs if c.decode_tok_s]), 2)
if any(c.decode_tok_s for c in c_runs) else None,
- "pp_tok_s": round(statistics.median([c.pp_tok_s for c in c_runs if c.pp_tok_s]), 1)
+ "pp_tok_s": round(statistics.median([c.pp_long_tok_s or c.pp_tok_s for c in c_runs if c.pp_long_tok_s or c.pp_tok_s]), 1)
if any(c.pp_tok_s for c in c_runs) else None,
"rss_peak_mib": max([c.rss_peak_mib for c in c_runs if c.rss_peak_mib] or [None]) if any(c.rss_peak_mib for c in c_runs) else None,
"vram_used_mib": max([c.vram_used_mib for c in c_runs if c.vram_used_mib] or [None]) if any(c.vram_used_mib for c in c_runs) else None}
@@ -636,7 +652,7 @@ class Tuner:
return "it would not leave the graphics memory you wanted free"
return None
- def _confirm(self, args: dict[str, Any], env: dict[str, str | None], spec: RequestSet, pairs: int) -> list[tuple[RunResult, RunResult]]:
+ def _confirm(self, args: dict[str, Any], env: dict[str, str | None], spec: RequestSet, pairs: int, use: str | None = None) -> list[tuple[RunResult, RunResult]]:
"""Interleaved pairs of the combined settings against the base (the order alternates)."""
out: list[tuple[RunResult, RunResult]] = []
cand_args = tune_knobs.apply_to_argv(self.base_args, args)
@@ -652,7 +668,8 @@ class Tuner:
runs[who] = self._run(cand_args if who == "cand" else self.base_args, env if who == "cand" else {}, spec,
f"confirm {who} {i + 1}")
if runs["cand"].status == "PASS" and runs["base"].status == "PASS":
- if self.o.use and metric(runs["cand"], self.o.use) and metric(runs["base"], self.o.use):
+ use = use or self.o.use
+ if metric(runs["cand"], use) and metric(runs["base"], use):
out.append((runs["base"], runs["cand"]))
self.base_runs.append(runs["base"])
else:
diff --git a/strata_control/tune_knobs.py b/strata_control/tune_knobs.py
index b6a296ddf..2e2dbb0bf 100644
--- a/strata_control/tune_knobs.py
+++ b/strata_control/tune_knobs.py
@@ -53,6 +53,8 @@ class Arm:
goals: tuple[str, ...] = ("speed", "coding", "chat", "room")
ram_delta_mib: int = 0 # extra memory the engine is predicted to take over the base run
deep_only: bool = False
+ #: what a win is judged on: "decode" (the goal's metric) or "prompt" (long-prompt speed; the knob costs no decode speed)
+ metric: str = "decode"
#: a dynamic value: name of a computed number ("workers-2" ...), filled from the base run's engine log
dynamic: dict[str, str] = field(default_factory=dict)
@@ -61,43 +63,44 @@ class Arm:
ARMS: tuple[Arm, ...] = (
- # --- class A: RAM-bound resident mode (6.2) ---------------------------------------------------------------
+ # --- long contexts (spec 4.3, 3.3): exact, measured +5.4% decode (5/5) at 128K on the R9700 --------------------------------
+ Arm("kv_resident_64k", "Keep more of the conversation in graphics memory", "exact", {"--kv-resident": 65536},
+ classes=("A", "B", "C"), backends=("cuda", "hip"), needs=("ctx_ge_128k",)),
+ # --- prompt speed: the chunk size is exact (10/10) and costs no decode speed on a card with VRAM to spare ------------------
+ Arm("prefill_32k", "Larger prompt chunks (faster very long prompts)", "exact", {"--prefill": "auto:32768"},
+ classes=("B",), goals=("coding", "chat"), ram_delta_mib=4096, needs=("vram_ge_24g", "long_prompts"), metric="prompt"),
+ # --- class A: RAM-bound resident mode (RTX 3060 / 46 GB): the one measured gain is adapt-async, behind its own switch ------
Arm("adapt_async", "Learn which experts to keep while you chat, in the background", "nonrepeatable",
{"--adapt-async": 1}, backends=("cuda",), classes=("A",), needs=("resident_ram", "one_gpu")),
- Arm("pin_by_need", "Pin memory only when needed", "exact", {}, {"STRATA_PIN_BY_NEED": "1"},
- backends=("cuda",), classes=("A",), needs=("one_gpu",)),
- Arm("host_core_last", "Run the control thread on the last CPU core", "exact", {"--host-core": "last"},
- classes=("A", "F")),
- # --- class B: large graphics memory (experts mostly resident) ---------------------------------------------
+ # --- placement-dependent (behind the owner's switch for answers that may differ slightly) ---------------------------------
Arm("adapt_tier_fast", "Re-balance the expert cache faster", "placement",
- {"--adapt-every": 1, "--adapt-swaps": 80, "--adapt-decay": 0.97}, classes=("B",)),
- Arm("spec3", "Shorter speculative window (3 tokens)", "placement", {"--spec": 3}, classes=("A", "B")),
- Arm("prefill_32k", "Larger prompt chunks (faster very long prompts)", "exact", {"--prefill": "auto:32768"},
- classes=("B",), goals=("coding",), ram_delta_mib=4096, needs=("vram_ge_24g",)),
- # --- class C: mid graphics memory, PCIe-bound -------------------------------------------------------------
+ {"--adapt-every": 1, "--adapt-swaps": 80, "--adapt-decay": 0.97}, classes=("B", "C")),
Arm("adapt_decay_09", "Forget old expert use a little faster", "placement", {"--adapt-decay": 0.9},
- classes=("C", "G"), backends=("cuda", "hip", "sycl")),
+ classes=("B", "C", "G"), backends=("cuda", "hip", "sycl")),
+ Arm("spec3", "Shorter speculative window (3 tokens)", "placement", {"--spec": 3}, classes=("A", "B")),
+ # --- class C: mid graphics memory, PCIe-bound -------------------------------------------------------------------------------
Arm("trim_stage", "Trim stage weights on two cards", "exact", {"--trim-stage-weights": True},
backends=("cuda",), classes=("C",), needs=("two_gpus",)),
- Arm("workers_minus2", "Two fewer CPU helper threads", "exact", {}, classes=("C",),
- dynamic={"--pool-workers": "workers-2"}),
- # --- class F: hybrid CPU (performance + efficiency cores) -------------------------------------------------
- Arm("workers_minus1", "One fewer CPU helper thread", "exact", {}, classes=("F",),
- dynamic={"--pool-workers": "workers-1"}, needs=("hybrid_cpu",)),
- # --- class G: Intel Arc (SYCL) ----------------------------------------------------------------------------
+ # --- class G: Intel Arc (SYCL): +5.5% (5/5, twice) ------------------------------------------------------------------------
Arm("pcie_start_02", "Start with a smaller PCIe share", "placement", {"--pcie-frac": 0.2},
backends=("sycl",), classes=("G",)),
- # --- Qwen3.6 hybrid (class D) -----------------------------------------------------------------------------
+ # --- Qwen3.6 hybrid (class D): exact while the experts fit the card, placement-dependent when CPU experts take part ------
Arm("q36_workers_explicit", "Fix the CPU helper thread count", "exact", {}, families=("qwen36",),
backends=("cuda", "hip"), classes=("D",), dynamic={"--pool-workers": "workers"}, needs=("hybrid_experts",)),
Arm("q36_mtp1", "Shorter draft window (1)", "placement", {"--mtp": 1}, families=("qwen36",),
backends=("cuda", "hip"), classes=("D",), needs=("hybrid_experts",)),
- Arm("q36_prefill_4096", "Larger prompt chunks (twice the prompt speed, a little slower answers)", "exact",
+ Arm("q36_prefill_4096", "Larger prompt chunks (twice the prompt speed, a little slower answers)", "placement",
{"--prefill": 4096}, families=("qwen36",), backends=("cuda", "hip"), classes=("D",), goals=("coding",),
needs=("hybrid_experts",)),
- Arm("q36_hybrid_rows_256", "Larger hybrid row blocks", "exact", {"--hybrid-rows": 256}, families=("qwen36",),
+ Arm("q36_hybrid_rows_256", "Larger hybrid row blocks", "placement", {"--hybrid-rows": 256}, families=("qwen36",),
backends=("cuda", "hip"), classes=("D",), needs=("hybrid_experts",), deep_only=True),
- # --- thorough only: effects measured as flat, tried last --------------------------------------------------
+ # --- thorough only: measured as no effect (spec 4.2-4.4), tried last ------------------------------------------------------
+ Arm("pin_by_need", "Pin memory only when needed", "exact", {}, {"STRATA_PIN_BY_NEED": "1"},
+ backends=("cuda",), classes=("C",), needs=("one_gpu",), deep_only=True),
+ Arm("host_core_last", "Run the control thread on the last CPU core", "exact", {"--host-core": "last"},
+ classes=("B", "C"), deep_only=True),
+ Arm("workers_minus1", "One fewer CPU helper thread", "exact", {}, classes=("F",),
+ dynamic={"--pool-workers": "workers-1"}, needs=("hybrid_cpu",), deep_only=True),
Arm("kv_grow", "Allocate the conversation memory as it grows", "exact", {"--kv-grow": True},
classes=("B", "C"), backends=("cuda", "hip", "sycl"), deep_only=True),
)
@@ -126,6 +129,7 @@ BOUNDS: dict[str, dict[str, Any]] = {
"--adapt-decay": {"min": 0.7, "max": 0.97},
"--hybrid-rows": {"min": 64, "max": 256},
"--host-core": {"choices": ("first", "last")},
+ "--kv-resident": {"choices": (32768, 49152, 65536)},
"--prefill": {"choices": ("auto", "auto:16384", "auto:32768", 2048, 4096)},
}
@@ -142,7 +146,7 @@ NEVER: tuple[dict[str, Any], ...] = (
{"flag": "--ple-io", "values": ("ram", "mmap"), "when": ("ram_bound",),
"reason": "it needs the table in memory, which this PC does not have to spare (-61% speed)"},
{"flag": "--adapt-every", "test": "at_least", "limit": 100000, "reason": "turning the expert re-balancing off cost 3% to 44%"},
- {"flag": "--kv-resident", "test": "below_context", "reason": "it made answers 27% to 34% slower and changes them"},
+ {"flag": "--kv-resident", "test": "kv_resident", "reason": "below the context it made answers 22% to 34% slower (and changes them) except at 128K, where only 49152 and 65536 were measured as flat or better"},
{"flag": "--pcie-frac", "test": "at_least", "limit": 0.75, "when": ("pcie_bound",),
"reason": "a high PCIe share made answers 14% to 37% slower"},
{"flag": "--prefill", "test": "fixed_below_auto", "reason": "smaller prompt chunks made long prompts 11% to 75% slower"},
@@ -161,7 +165,7 @@ ANSWER_OF_FLAG: dict[str, str] = {
"--expert-cache": "placement", "--vram-reserve-mib": "placement", "--adapt-every": "placement",
"--adapt-swaps": "placement", "--adapt-decay": "placement", "--adapt-min-gain": "placement",
"--pcie-frac": "placement", "--spec": "placement", "--mtp": "placement", "--lookup-chain": "placement",
- "--kv-resident": "placement", "--kv": "quality", "--adapt-async": "nonrepeatable",
+ "--kv": "quality", "--adapt-async": "nonrepeatable",
}
@@ -203,6 +207,8 @@ def condition_holds(name: str, ctx: Context) -> bool:
"gfx12": bool(ctx.arch and ctx.arch.startswith("gfx12")),
"pcie_bound": 0 < ctx.vram_mib < 24000,
"ram_bound": bool(ctx.ram_gib and ctx.model_gib and ctx.ram_gib - ctx.model_gib < 30),
+ "ctx_ge_128k": ctx.context_tokens >= 98304,
+ "long_prompts": ctx.context_tokens >= 65536 or ctx.use == "coding",
}.get(name, False)
@@ -273,9 +279,11 @@ def check_setting(args: dict[str, Any], env: dict[str, str | None], ctx: Context
elif test == "at_most":
n = _as_number(value)
hit = n is not None and n <= rule["limit"] and flag == "--spec"
- elif test == "below_context":
+ elif test == "kv_resident":
n = _as_number(value)
- hit = n is not None and n < ctx_tokens
+ own = _as_number(_current(ctx.base_args, "--kv-resident"))
+ # the config's own value is always fine; at a long context 49152 and 65536 were measured; anything else below the context is not
+ hit = n is not None and n < ctx_tokens and n != own and not (ctx_tokens >= 98304 and n in (49152, 65536))
elif test == "fixed_below_auto":
hit = _prefill_below_auto(value) and ctx.family == "flash-next"
if ctx.family == "qwen36":
@@ -342,7 +350,7 @@ def dynamic_value(spec: str, ctx: Context) -> Any:
return int(workers) if workers else None
if spec.startswith("workers-") and workers:
value = int(workers) - int(spec[len("workers-"):])
- floor = max(2, int(workers) // 4 + 1) # never below N/4 (6.6)
+ floor = max(2, int(workers) - 1) # one step down at most (6.2 F); never two below (6.2 C)
return value if value >= floor else None
return None
diff --git a/tests/tuning/test_autotune.py b/tests/tuning/test_autotune.py
index 8addab956..50845041b 100644
--- a/tests/tuning/test_autotune.py
+++ b/tests/tuning/test_autotune.py
@@ -39,6 +39,13 @@ def speed_with(gains: dict[str, float], base=60.0):
return model
+def make_ctx_small():
+ c = make_ctx()
+ c.context_tokens = 16384
+ c.base_args = ["--max-context", "16384"]
+ return c
+
+
def run_tuner(model, *, opts=None, ctx=None, clock=None, session_kw=None, cancel=None, progress=None, **kw):
clock = clock or FakeClock()
session = StubSession(model=model, clock=clock, **(session_kw or {}))
@@ -57,11 +64,68 @@ class RulesTest(unittest.TestCase):
({}, {"STRATA_RDNA2_SGEMM": "all"}), ({}, {"STRATA_HIP_WMMA": "1"}), ({"--pool-workers": 8}, {})]
for args, env in bad:
self.assertTrue(tune_knobs.check_setting(args, env, ctx), (args, env))
- ok = [({"--spec": 3}, {}), ({"--host-core": "last"}, {}), ({"--pool-workers": 4}, {}), ({"--prefill": "auto:32768"}, {}),
+ ok = [({"--spec": 3}, {}), ({"--kv-resident": 65536}, {}), ({"--pool-workers": 4}, {}), ({"--prefill": "auto:32768"}, {}),
({"--adapt-every": 1, "--adapt-swaps": 80, "--adapt-decay": 0.97}, {}), ({"--pcie-frac": 0.2}, {})]
for args, env in ok:
self.assertEqual(tune_knobs.check_setting(args, env, ctx), [], (args, env))
+ def test_kv_resident_rules_follow_the_context(self):
+ long = make_ctx() # 128K, the config's own --kv-resident is 32768
+ self.assertEqual(tune_knobs.check_setting({"--kv-resident": 65536}, {}, long), [])
+ self.assertEqual(tune_knobs.check_setting({"--kv-resident": 49152}, {}, long), [])
+ self.assertTrue(tune_knobs.check_setting({"--kv-resident": 20480}, {}, long)) # below the context and not measured
+ short = make_ctx()
+ short.context_tokens = 32768
+ short.base_args = ["--max-context", "32768"]
+ self.assertTrue(tune_knobs.check_setting({"--kv-resident": 20480}, {}, short)) # -29.5% at 32K
+ self.assertEqual(tune_knobs.check_setting({"--kv-resident": 65536}, {}, make_ctx_small()), []) # above the context: no streaming at all
+
+ def test_long_context_candidates_exist_and_are_exact(self):
+ ctx = make_ctx()
+ ctx.vram_mib = 32624 # the R9700: class B
+ ctx.facts = {"workers": 31}
+ arms, _ = tune_knobs.plan_arms(ctx, mode="quick")
+ ids = [p.arm.id for p in arms]
+ self.assertEqual(ids[:2], ["kv_resident_64k", "prefill_32k"])
+ self.assertTrue(all(p.arm.answer == "exact" for p in arms))
+ short = make_ctx()
+ short.vram_mib = 32624
+ short.context_tokens = 32768
+ short.base_args = ["--max-context", "32768"]
+ self.assertNotIn("kv_resident_64k", [p.arm.id for p in tune_knobs.plan_arms(short, mode="quick")[0]])
+
+ def test_a_prompt_knob_is_judged_on_prompt_speed_and_may_not_cost_decode(self):
+ ctx = make_ctx()
+ ctx.vram_mib = 32624
+ def pp(args, env):
+ return 1500.0 * (1.06 if flag(args, "--prefill") == "auto:32768" else 1.0)
+ # decode unchanged, long prompt +6%: wins on the prompt metric
+ clock = FakeClock()
+ sess = StubSession(model=lambda a, e, tune=None: 60.0, pp=pp, clock=clock, noise=0.001)
+ t = autotune.Tuner(sess, BASE, {}, ctx, autotune.Options(budget_s=3600), clock=clock, sleep=clock.sleep, system=sess.system,
+ total_ram_mib=64000, total_vram_mib=32000, quiet=lambda: None)
+ r = t.run()
+ self.assertEqual(r.settings["args"], {"--prefill": "auto:32768"}, r.notes)
+ # the same prompt gain with a 6% decode loss does not
+ sess2 = StubSession(model=lambda a, e, tune=None: 60.0 * (0.94 if flag(a, "--prefill") == "auto:32768" else 1.0), pp=pp, clock=FakeClock(), noise=0.001)
+ c2 = sess2.clock
+ t2 = autotune.Tuner(sess2, BASE, {}, ctx, autotune.Options(budget_s=3600), clock=c2, sleep=c2.sleep, system=sess2.system,
+ total_ram_mib=64000, total_vram_mib=32000, quiet=lambda: None)
+ self.assertEqual(t2.run().settings["args"], {})
+
+ def test_only_the_best_single_knob_is_kept(self):
+ ctx = make_ctx()
+ ctx.vram_mib = 32624
+ model = speed_with({("--kv-resident", "65536"): 1.06})
+ pp = lambda a, e: 1500.0 * (1.07 if flag(a, "--prefill") == "auto:32768" else 1.0)
+ clock = FakeClock()
+ sess = StubSession(model=model, pp=pp, clock=clock, noise=0.001)
+ t = autotune.Tuner(sess, BASE, {}, ctx, autotune.Options(budget_s=3600), clock=clock, sleep=clock.sleep, system=sess.system,
+ total_ram_mib=64000, total_vram_mib=32000, quiet=lambda: None)
+ r = t.run()
+ self.assertEqual(len(r.settings["args"]), 1)
+ self.assertTrue(any("only the best one" in n for n in r.notes))
+
def test_hybrid_cpu_affinity_never(self):
ctx = make_ctx()
ctx.hybrid_cpu = True
@@ -101,7 +165,7 @@ class RulesTest(unittest.TestCase):
ctx = make_ctx()
ctx.engine_flags = ["--spec"]
arms, skipped = tune_knobs.plan_arms(ctx, mode="thorough")
- self.assertFalse(any(p.arm.id == "host_core_last" for p in arms))
+ self.assertFalse(any(p.arm.id == "kv_resident_64k" for p in arms))
self.assertTrue(any("does not accept" in s["reason"] for s in skipped))
def test_qwen36_and_sycl_lists(self):
@@ -110,7 +174,7 @@ class RulesTest(unittest.TestCase):
q.facts = {"workers": 6, "hybrid_experts": True}
ids = [p.arm.id for p in tune_knobs.plan_arms(q, mode="thorough")[0]]
self.assertIn("q36_workers_explicit", ids)
- self.assertNotIn("host_core_last", ids)
+ self.assertNotIn("kv_resident_64k", ids)
s = make_ctx(allow_placement=True)
s.backend = "sycl"
ids = [p.arm.id for p in tune_knobs.plan_arms(s, mode="thorough")[0]]
@@ -119,17 +183,18 @@ class RulesTest(unittest.TestCase):
def test_workers_arm_never_goes_below_a_quarter(self):
c = make_ctx()
c.facts = {"workers": 4}
- self.assertEqual(tune_knobs.dynamic_value("workers-2", c), 2)
+ self.assertEqual(tune_knobs.dynamic_value("workers-1", c), 3) # one step down at most
+ self.assertIsNone(tune_knobs.dynamic_value("workers-2", c)) # never two below (spec 6.2 C)
c.facts = {"workers": 2}
- self.assertIsNone(tune_knobs.dynamic_value("workers-2", c))
+ self.assertIsNone(tune_knobs.dynamic_value("workers-1", c))
class TunerTest(unittest.TestCase):
def test_finds_a_real_gain_and_changes_one_knob(self):
- model = speed_with({("--host-core", "last"): 1.08})
+ model = speed_with({("--kv-resident", "65536"): 1.08})
t, s, r = run_tuner(model, ctx=make_ctx())
self.assertEqual(r.outcome, "improved", r.notes)
- self.assertEqual(r.settings["args"], {"--host-core": "last"})
+ self.assertEqual(r.settings["args"], {"--kv-resident": 65536})
self.assertAlmostEqual(r.gain_pct, 8.0, delta=2.0)
self.assertEqual(r.answers["class"], "exact")
for args, env in s.runs: # every run kept the user's own flags
@@ -144,23 +209,23 @@ class TunerTest(unittest.TestCase):
self.assertIn("already the fastest", r.message)
def test_a_gain_below_the_margin_is_not_taken(self):
- model = speed_with({("--host-core", "last"): 1.02})
+ model = speed_with({("--kv-resident", "65536"): 1.02})
t, s, r = run_tuner(model, session_kw={"noise": 0.001})
self.assertEqual(r.settings["args"], {})
def test_losing_candidates_are_dropped_after_one_loss(self):
- model = speed_with({("--host-core", "last"): 0.85})
+ model = speed_with({("--kv-resident", "65536"): 0.85})
t, s, r = run_tuner(model, session_kw={"noise": 0.001})
- arm = [a for a in r.arms if a["id"] == "host_core_last"][0]
+ arm = [a for a in r.arms if a["id"] == "kv_resident_64k"][0]
self.assertEqual(arm["status"], "dropped")
- self.assertEqual(sum(1 for args, _ in s.runs if flag(args, "--host-core") == "last"), 1)
+ self.assertEqual(sum(1 for args, _ in s.runs if flag(args, "--kv-resident") == "65536"), 1)
def test_machine_drift_is_not_mistaken_for_a_gain(self):
clock = FakeClock()
counter = {"n": 0}
def model(args, env, tune=None): # the machine slows 1.2% per engine start
- return 60.0 * (1.0 - 0.012 * len(session.runs)) * (1.03 if flag(args, "--host-core") == "last" else 1.0)
+ return 60.0 * (1.0 - 0.012 * len(session.runs)) * (1.03 if flag(args, "--kv-resident") == "65536" else 1.0)
session = StubSession(model=model, clock=clock, noise=0.0005)
t = autotune.Tuner(session, BASE, {}, make_ctx(), autotune.Options(), clock=clock, sleep=clock.sleep, system=session.system,
total_ram_mib=64000, quiet=lambda: None)
@@ -168,7 +233,7 @@ class TunerTest(unittest.TestCase):
self.assertEqual(r.settings["args"], {}, r.notes)
def test_candidate_that_does_not_fit_is_never_started(self):
- model = speed_with({("--host-core", "last"): 1.10})
+ model = speed_with({("--kv-resident", "65536"): 1.10})
clock = FakeClock()
session = StubSession(model=model, clock=clock, avail=9000, rss=lambda a, e: 12000.0)
calls = {"n": 0}
@@ -187,34 +252,34 @@ class TunerTest(unittest.TestCase):
def test_first_oom_stops_the_search(self):
def fail(args, env):
- return StartFailed("OOM", "out of memory") if flag(args, "--host-core") == "last" else None
+ return StartFailed("OOM", "out of memory") if flag(args, "--kv-resident") == "65536" else None
t, s, r = run_tuner(speed_with({}), session_kw={"fail": fail})
self.assertTrue(any("ran out of memory" in n for n in r.notes))
- self.assertEqual(sum(1 for a, _ in s.runs if flag(a, "--host-core") == "last"), 1)
+ self.assertEqual(sum(1 for a, _ in s.runs if flag(a, "--kv-resident") == "65536"), 1)
self.assertEqual(r.settings["args"], {})
def test_unsupported_candidate_fails_alone(self):
def fail(args, env):
- return StartFailed("UNSUPPORTED", "unknown argument: --host-core") if flag(args, "--host-core") else None
+ return StartFailed("UNSUPPORTED", "unknown argument: --kv-resident") if flag(args, "--kv-resident") == "65536" else None
ctx = make_ctx()
ctx.facts = {"workers": 6}
- t, s, r = run_tuner(speed_with({("--pool-workers", "4"): 1.1}), ctx=ctx, session_kw={"fail": fail})
+ t, s, r = run_tuner(speed_with({}), ctx=ctx, session_kw={"fail": fail})
failed = [a for a in r.arms if a["status"] == "failed"]
self.assertTrue(failed and "UNSUPPORTED" in failed[0]["reason"])
- self.assertEqual(r.settings["args"], {"--pool-workers": 4})
+ self.assertEqual(r.settings["args"], {})
def test_room_goal_refuses_a_faster_candidate_that_takes_more_memory(self):
- model = speed_with({("--host-core", "last"): 1.1})
- rss = lambda a, e: 20000.0 if flag(a, "--host-core") == "last" else 12000.0
+ model = speed_with({("--kv-resident", "65536"): 1.1})
+ rss = lambda a, e: 20000.0 if flag(a, "--kv-resident") == "65536" else 12000.0
t, s, r = run_tuner(model, opts=autotune.Options(goal="room", ram_wish_pct=20, reserve_mib=1536), session_kw={"rss": rss})
self.assertEqual(r.settings["args"], {})
self.assertTrue(any("more memory" in a["reason"] for a in r.arms))
t, s, r = run_tuner(model, opts=autotune.Options(goal="speed"), session_kw={"rss": rss})
- self.assertEqual(r.settings["args"], {"--host-core": "last"})
+ self.assertEqual(r.settings["args"], {"--kv-resident": 65536})
def test_cancel_changes_nothing(self):
cancel = threading.Event()
- model = speed_with({("--host-core", "last"): 1.1})
+ model = speed_with({("--kv-resident", "65536"): 1.1})
n = {"i": 0}
def progress(p):
@@ -280,15 +345,15 @@ class TunerTest(unittest.TestCase):
self.assertAlmostEqual(r.gain_pct, 6.0, delta=1.5)
def test_thorough_confirms_with_five_pairs(self):
- model = speed_with({("--host-core", "last"): 1.06})
+ model = speed_with({("--kv-resident", "65536"): 1.06})
t, s, r = run_tuner(model, opts=autotune.Options(mode="thorough", budget_s=7200), session_kw={"noise": 0.003})
self.assertEqual(r.outcome, "improved")
self.assertEqual(r.pairs, 5)
self.assertGreaterEqual(r.faster_pairs, 4)
def test_prompt_regression_blocks_a_decode_win(self):
- model = speed_with({("--host-core", "last"): 1.08})
- pp = lambda a, e: 1500.0 * (0.8 if flag(a, "--host-core") == "last" else 1.0)
+ model = speed_with({("--kv-resident", "65536"): 1.08})
+ pp = lambda a, e: 1500.0 * (0.8 if flag(a, "--kv-resident") == "65536" else 1.0)
t, s, r = run_tuner(model, session_kw={"pp": pp, "noise": 0.001})
self.assertEqual(r.settings["args"], {})