Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
"""The auto-tuner: find the fastest settings for this PC, for this model, for what the user asked for.
What it does (TUNING-SPEC.md sections 5-6, measured on five machines):
1. **Starts from the user's own configuration** (the "base") and changes one knob at a time (`tune_knobs`). It never starts
a candidate at "the first value of every dimension".
2. Measures with the real worker over HTTP, on the engine's own clock (`tune_session`), the same fixed request set every
time. Restart knobs are tried as **interleaved rounds**: every round runs the base and each candidate once, in a
rotating order, so machine drift cannot favour one of them. Per-request knobs need no restart and are tried first, in
one engine.
3. Accepts a candidate only if the **median of its per-round ratios** to the base beats the noise: at least +3%, and twice
the drift and twice the A/A gap measured on the base runs, in every round (quick) or at least 4 of 5 (thorough
confirm), without losing more than 2% elsewhere (prompt speed). A gain the noise can explain is not a gain: the result
is then "your current settings are already the fastest" and nothing changes.
4. Guards before every start: free RAM, free graphics memory, other engines running, the CPU busy with something else.
A candidate that does not fit is skipped, never started. A candidate must not use more memory than the base the user
asked for (so "leave 20% for my work" is respected). The first crash stops the stage.
5. Only knobs whose answers are byte-identical (class exact) are chosen on their own; the others need the user's tick.
Progress and time left are reported through a callback; a cancel stops at the next run and keeps nothing half-measured.
"""
from __future__ import annotations
import math
import statistics
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Callable
from strata_control import engine_component, tune_knobs, tune_machine
from strata_control.tune_session import RequestSet, RunResult, Session, StartFailed
MIN_FREE_RAM_MIB = 3584 # 3.5 GiB left for the system at every start (6.5)
EARLY_DROP = 0.05 # a candidate this much slower than its own round's base is dropped after one round (run-to-run noise is 2-8%)
STAGE_GAP_NOISY = 0.04 # an A/A gap above this: the machine is noisy (6.5)
STAGNANT_WAIT_S = 60.0 # free memory that has not grown by STAGNANT_MIB after this long is not going to
STAGNANT_MIB = 512
class Cancelled(Exception):
"""The user stopped the tune; what was measured so far is kept in the result."""
# --------------------------------------------------------------------------- options and result
@dataclass
class Options:
mode: str = "quick" # quick | thorough
goal: str = "speed" # speed | room
reserve_mib: int = 0 # graphics memory to leave free (goal room)
ram_wish_pct: int = 0 # system memory to leave free, in percent (goal room, e.g. 20)
use: str = "chat" # chat | coding
allow_placement: bool = False
allow_nonrepeatable: bool = False
budget_s: float | None = None # quick 20 min, thorough 2 h
margin: float = 0.03
rounds: int | None = None # quick 2, thorough 3 (screen); confirm 5 in thorough
confirm_pairs: int | None = None # thorough: 5
per_request: bool = True
min_free_ram_mib: int = MIN_FREE_RAM_MIB
wait_ram_s: float = 600.0
busy_pct: float = 25.0
wait_busy_s: float = 300.0
check_busy: bool = True
def resolved(self) -> "Options":
o = Options(**self.__dict__)
thorough = o.mode == "thorough"
o.budget_s = o.budget_s or (7200.0 if thorough else 1200.0)
o.rounds = o.rounds or (3 if thorough else 2)
o.confirm_pairs = o.confirm_pairs if o.confirm_pairs is not None else (5 if thorough else 0)
return o
@dataclass
class ArmReport:
id: str
label: str
settings: dict[str, Any]
env: dict[str, Any]
answer: str
status: str = "pending" # pending | tested | dropped | failed | skipped
reason: str = ""
runs: list[dict[str, Any]] = field(default_factory=list) # per round: ratios vs that round's base
ratio_decode: float | None = None
ratio_pp: float | None = None
pairs: int = 0
faster: int = 0
won: bool = False
def to_dict(self) -> dict[str, Any]:
return dict(self.__dict__)
@dataclass
class TuneResult:
outcome: str = "default_confirmed" # improved | default_confirmed | inconclusive | base_failed | cancelled
#: what a screen shows, one of: faster | already_fastest | busy_pc | failed | stopped (see `finalize`)
code: str = "already_fastest"
#: why, for busy_pc: noisy | drift | not_tried (no memory or time to compare anything); for failed: start_failed | no_engine
reason_code: str = ""
#: the numbers a screen puts in its sentence, all plain numbers: gain_pct, base_tok_s, tuned_tok_s, noise_pct, drift_pct,
#: minutes, pairs, faster_pairs, preset_name
values: dict[str, Any] = field(default_factory=dict)
message: str = ""
mode: str = "quick"
seconds: float = 0.0
waited_s: float = 0.0 # of those seconds, spent waiting (box lock, busy PC, memory)
runs: int = 0
base: dict[str, Any] = field(default_factory=dict)
chosen: dict[str, Any] = field(default_factory=dict)
gain_pct: float = 0.0
gain_decode_pct: float = 0.0
gain_pp_pct: float = 0.0
settings: dict[str, Any] = field(default_factory=lambda: {"args": {}, "env": {}, "per_request": {}})
answers: dict[str, Any] = field(default_factory=lambda: {"class": "exact", "fields": []})
arms: list[dict[str, Any]] = field(default_factory=list)
skipped: list[dict[str, str]] = field(default_factory=list)
noise: dict[str, Any] = field(default_factory=dict)
pairs: int = 0
faster_pairs: int = 0
per_request: list[dict[str, Any]] = field(default_factory=list)
per_request_confirm: dict[str, Any] | None = None
notes: list[str] = field(default_factory=list)
classes: list[str] = field(default_factory=list)
facts: dict[str, Any] = field(default_factory=dict)
trials: list[dict[str, Any]] = field(default_factory=list) # every engine run, for the history store
def to_dict(self) -> dict[str, Any]:
return dict(self.__dict__)
# --------------------------------------------------------------------------- pure decision rules
def metric(run: dict[str, Any] | RunResult, use: str) -> float | None:
"""The number a configuration is judged on (6.1): decode speed for chat; for coding the geometric mean of decode
speed and long-prompt speed (the 4K prompt when no long one was measured)."""
get = (lambda k: run.get(k)) if isinstance(run, dict) else (lambda k: getattr(run, k, None))
if use == "prompt": # a prompt-only knob (the chunk size): long-prompt speed, the 4K prompt otherwise
p = get("pp_long_tok_s") or get("pp_tok_s")
return float(p) if p else None
d = get("decode_tok_s")
if not d:
return None
if use == "coding":
p = get("pp_long_tok_s") or get("pp_tok_s")
if p:
return math.sqrt(d * p)
return float(d)
def median_ratio(ratios: list[float]) -> float | None:
return statistics.median(ratios) if ratios else None
def accept_threshold(margin: float, aa_gap: float, drift: float) -> float:
"""How far above 1.0 a median ratio has to be: the margin, or twice the noise the base runs showed (6.7 rule 3)."""
return 1.0 + max(margin, 2.0 * aa_gap, 2.0 * drift)
def is_win(ratios: list[float], pp_ratios: list[float], *, threshold: float, need: int | None = None) -> bool:
"""A win: the median ratio beats the threshold, enough rounds were faster, and the prompt speed lost at most 2%."""
if not ratios:
return False
need = len(ratios) if need is None else need
faster = sum(1 for r in ratios if r > 1.0)
if faster < need or statistics.median(ratios) < threshold:
return False
if pp_ratios and statistics.median(pp_ratios) < 0.98:
return False
return True
def rotate(items: list[Any], r: int) -> list[Any]:
k = r % len(items) if items else 0
return items[k:] + items[:k]
# --------------------------------------------------------------------------- the tuner
class Tuner:
def __init__(self, session: Session, base_args: list[str], base_env: dict[str, str], ctx: tune_knobs.Context,
options: Options, *, progress: Callable[[dict[str, Any]], None] | None = None,
cancel: threading.Event | None = None, history_bad: set[str] | None = None,
clock: Callable[[], float] = time.monotonic, sleep: Callable[[float], None] = time.sleep,
system: Callable[[], dict[str, Any]] | None = None, total_ram_mib: int | None = None,
total_vram_mib: int | None = None, quiet: Callable[[], str | None] | None = None) -> None:
self.session = session
self.base_args = list(base_args)
self.base_env = dict(base_env)
self.ctx = ctx
self.o = options.resolved()
self.progress = progress or (lambda p: None)
self.cancel = cancel or threading.Event()
self.history_bad = history_bad or set()
self._ram_gave_up: tuple[float, float] | None = None
self.clock = clock
self.sleep = sleep
self.system = system or session.system
self.result = TuneResult(mode=self.o.mode)
self.started = 0.0
self.run_times: list[float] = []
self.planned_runs = 0
self.done_runs = 0
self.base_runs: list[RunResult] = []
self.total_ram = total_ram_mib or (tune_machine.memory().get("total_mib") or 0)
self.total_vram = total_vram_mib
self.quiet = quiet
self.storm = False
self.speeds: list[dict[str, Any]] = []
self.waited_s = 0.0 # time spent waiting for the box lock, the PC to be quiet or memory: not the user's budget
# ---- bookkeeping
def _elapsed(self) -> float:
return self.clock() - self.started - self.waited_s
def _say(self, stage: str, message: str, **extra: Any) -> None:
elapsed = self._elapsed()
avg = statistics.mean(self.run_times) if self.run_times else 150.0
remaining = max(0, self.planned_runs - self.done_runs)
eta = min(max(0.0, self.o.budget_s - elapsed), remaining * avg) if self.planned_runs else None
self.progress({"stage": stage, "message": message, "elapsed_s": round(elapsed), "runs_done": self.done_runs,
"runs_planned": self.planned_runs, "eta_s": None if eta is None else round(eta),
# what each measurement so far came to, for the live readout (label, decode tok/s as the goal judges it)
"speeds": list(self.speeds), **extra})
def _check_cancel(self) -> None:
if self.cancel.is_set():
raise Cancelled()
def _over_budget(self, reserve_s: float = 0.0) -> bool:
return self._elapsed() + reserve_s > self.o.budget_s
def _avg_run(self) -> float:
return statistics.mean(self.run_times) if self.run_times else 150.0
# ---- guards
def _wait_for_quiet(self) -> None:
"""Pause while another engine runs or the CPU is busy with something else (6.5, 6.7 rule 5)."""
if not self.o.check_busy:
return
waited = 0.0
while True:
why = self.quiet() if self.quiet else self._machine_busy()
if not why:
return
if waited >= self.o.wait_busy_s:
self.result.notes.append(f"measured while the machine was busy ({why})")
return
self._say("waiting", f"Waiting for the PC to be quiet ({why})")
self._check_cancel()
self.sleep(10.0)
waited += 10.0
def _machine_busy(self) -> str | None:
eng = tune_machine.other_engines()
if eng:
return "another Strata engine is running"
busy = tune_machine.cpu_busy_percent(1.0)
if busy is not None and busy > self.o.busy_pct:
return f"the CPU is {busy:.0f}% busy"
return None
def _ram_guard(self, delta_mib: int, label: str, wait_s: float | None = None) -> str | None:
"""None when a start fits; otherwise the plain reason it does not. Waiting ends early when the free memory does not
grow (a loaded model keeps it): ten minutes per candidate made a tuning run last for hours (#3490)."""
waited = 0.0
first_avail: float | None = None
while True:
mem = self.system().get("memory", {})
avail = mem.get("available_mib")
if avail is None:
return None # unreadable: the engine's own guard decides
need = self.o.min_free_ram_mib + delta_mib
base_rss = self.base_runs[0].rss_peak_mib if self.base_runs else None
if base_rss:
need += base_rss
elif self.total_ram:
need = max(need, self.o.min_free_ram_mib)
if avail >= need:
return None
reason = (f"not enough free memory to start it safely ({avail / 1024:.1f} GiB free, "
f"{need / 1024:.1f} GiB needed)")
prior = self._ram_gave_up
if prior is not None and need >= prior[0] and avail <= prior[1] + STAGNANT_MIB:
return reason # an earlier wait found nothing freed and this needs no less
if first_avail is None:
first_avail = avail
stagnant = waited >= STAGNANT_WAIT_S and avail <= first_avail + STAGNANT_MIB
if stagnant or waited >= (self.o.wait_ram_s if wait_s is None else wait_s):
self._ram_gave_up = (need, avail)
return reason
self._say("waiting", f"Waiting for memory to be freed before trying {label} "
f"({avail / 1024:.1f} GiB free, {need / 1024:.1f} GiB needed)")
self._check_cancel()
self.sleep(10.0)
waited += 10.0
# ---- one engine run
def _run(self, args: list[str], env: dict[str, str | None], spec: RequestSet, label: str,
points: Callable[[Any], None] | None = None, delta_mib: int = 0) -> RunResult:
self._check_cancel()
t0 = self.clock()
entered: list[float] = []
def pre_start() -> None:
# checked again once the box lock is ours: another stream's engine may have come and gone while we waited
lock_got = self.clock()
self.waited_s += lock_got - t0
self._wait_for_quiet()
why = self._ram_guard(delta_mib, label, wait_s=60.0)
self.waited_s += self.clock() - lock_got
if why:
raise StartFailed("SKIPPED", why)
entered.append(self.clock())
try:
with self.session.engine(args, env, pre_start=pre_start) as eng:
res = eng.measure(spec)
if points is not None and res.status == "PASS":
points(eng)
except StartFailed as e:
res = RunResult(status=e.status, reason=e.reason)
dt = self.clock() - (entered[0] if entered else t0) # the run itself, not the waiting before it
if res.status != "SKIPPED":
self.run_times.append(dt)
self.done_runs += 1
self.result.runs += 1
res.wall_s = round(dt, 1)
self.result.trials.append({"label": label, "args": args, "env": env, "status": res.status, "reason": res.reason,
"decode_tok_s": res.decode_tok_s, "pp_tok_s": res.pp_tok_s,
"rss_peak_mib": res.rss_peak_mib, "load_s": res.load_s, "at": time.time()})
return res
# ---- the whole job
def run(self) -> TuneResult:
self.started = self.clock()
r = self.result
try:
self._run_stages()
except Cancelled:
r.outcome, r.message = "cancelled", "Stopped. Your settings were not changed."
r.seconds = round(self.clock() - self.started, 1)
r.waited_s = round(self.waited_s, 1)
self.finalize(r)
return r
@staticmethod
def finalize(r: "TuneResult") -> None:
"""The structured outcome a screen shows inside the tune dialog (owner feedback, item 13): a code and plain numbers, so no
screen has to read a sentence. faster / already_fastest / busy_pc (noisy, drifting, or nothing could be compared) /
failed / stopped. `busy_pc` is a result, not an error: nothing was changed and trying again later is the answer."""
r.code = {"improved": "faster", "default_confirmed": "already_fastest", "inconclusive": "busy_pc",
"base_failed": "failed", "cancelled": "stopped"}.get(r.outcome, "failed")
if r.outcome == "default_confirmed" and r.noise.get("noisy"):
r.reason_code = r.reason_code or "noisy_but_nothing_better"
base = (r.base or {}).get("decode_tok_s")
tuned = (r.chosen or {}).get("decode_tok_s")
r.values = {"gain_pct": r.gain_pct, "gain_decode_pct": r.gain_decode_pct, "base_tok_s": base, "tuned_tok_s": tuned,
"noise_pct": round(100 * (r.noise or {}).get("spread", 0.0), 1) if r.noise else None,
"drift_pct": round(100 * (r.noise or {}).get("drift", 0.0), 1) if r.noise else None,
"minutes": round(r.seconds / 60.0 - r.waited_s / 60.0, 1), "pairs": r.pairs, "faster_pairs": r.faster_pairs}
def _spec(self) -> RequestSet:
long_tokens = 0
if self.ctx.context_tokens >= 65536:
# long contexts mean long prompts: the chunk size (spec 4.3) only shows on a long one
long_tokens = min(98304, int(0.75 * self.ctx.context_tokens))
elif self.o.use == "coding" and self.o.mode == "thorough":
long_tokens = min(28000, int(0.8 * self.ctx.context_tokens))
return RequestSet(warmups=2 if self.o.mode == "thorough" else 1, decodes=5, decode_tokens=200,
pp_tokens=4096 if self.ctx.context_tokens >= 6000 else 1024, long_tokens=long_tokens)
def _run_stages(self) -> None:
r, o = self.result, self.o
spec = self._spec()
self.planned_runs = 6 # corrected once the candidates are known
self._say("start", "Checking your PC before the first measurement")
# ---- stage 1: the base run (the user's configuration, untouched), with the per-request guard in the same engine
why = self._ram_guard(0, "your current settings")
if why:
r.outcome, r.message = "base_failed", "Not tuned: " + why
r.reason_code = "no_memory"
return
pr_rows: list[dict[str, Any]] = []
def per_request_points(eng: Any) -> None:
if o.per_request:
pr_rows.extend(self._per_request_stage(eng))
self._say("base", "Measuring your current settings")
base1 = self._run(self.base_args, {}, spec, "base", points=per_request_points)
self.base_runs.append(base1)
if base1.status == "PASS" and metric(base1, o.use):
self.speeds.append({"label": "base", "tok_s": round(float(metric(base1, o.use)), 1)})
self._say("base", "Measured your current settings")
if base1.status != "PASS" or not metric(base1, o.use):
r.outcome = "base_failed"
r.reason_code = "start_failed"
r.message = "Your current settings could not be measured: " + (base1.reason or base1.status)
return
r.base = base1.to_dict()
r.facts = dict(base1.facts)
self.ctx.facts = dict(base1.facts)
if base1.vram_used_mib is not None and self.total_vram is None:
cards = [g for g in self.system().get("gpus", []) if g.get("vram_total_mib")]
self.total_vram = sum(g["vram_total_mib"] for g in cards) if cards else None
r.per_request = pr_rows
r.classes = tune_knobs.machine_classes(self.ctx)
# ---- stage 2: plan the candidates for this machine; how many depends on the time chosen
planned, skipped = tune_knobs.plan_arms(self.ctx, mode=o.mode, history_bad=self.history_bad)
r.skipped = skipped
per_arm = o.rounds or 2
if o.mode == "thorough":
n_arms = len(planned)
else:
left = int((o.budget_s - self._elapsed()) / max(1.0, self._avg_run()))
fixed = (per_arm - 1) + 1 # extra base rounds, the final base
n_arms = min(len(planned), max(0, (left - fixed) // per_arm))
for p in planned[n_arms:]:
skipped.append({"id": p.arm.id, "label": p.arm.label, "reason": "left out to stay inside the time you chose"})
planned = planned[:n_arms]
arms = [ArmReport(p.arm.id, p.arm.label, dict(p.args), dict(p.env), p.arm.answer) for p in planned]
by_id = {a.id: a for a in arms}
by_use = {p.arm.id: ("prompt" if p.arm.metric == "prompt" else o.use) for p in planned}
self.planned_runs = self.done_runs + per_arm * (len(planned) + 1) - 1 + 1 + (o.confirm_pairs or 0) * 2
if not planned:
self._say("plan", "No other setting is worth trying on this PC")
# ---- stage 3: interleaved rounds. Every round runs the base and each live candidate once, in a rotating order.
round_runs: list[dict[str, RunResult]] = [{} for _ in range(per_arm)]
round_runs[0]["base"] = base1
alive = list(planned)
stop = False
for rnd in range(per_arm):
if stop or self.storm or (rnd > 0 and not alive):
break
for item in rotate(["base"] + alive, rnd):
self._check_cancel()
if self._over_budget(reserve_s=2 * self._avg_run()):
r.notes.append("the time you chose ran out before every round finished")
stop = True
break
if item == "base":
if rnd == 0:
continue # round 1's base is the run above
if self._ram_guard(0, "your current settings"):
r.notes.append("not enough free memory for another measurement of your current settings; the search stopped")
stop = True
break
self._say("rounds", f"Measuring your current settings again (round {rnd + 1} of {per_arm})")
b = self._run(self.base_args, {}, spec, f"base r{rnd + 1}")
if b.status == "SKIPPED":
r.notes.append("not enough free memory for another measurement of your current settings; the search stopped")
self.storm = stop = True
break
if b.status != "PASS":
r.notes.append("your current settings failed to start a second time; the search stopped")
self.storm = stop = True
break
self.base_runs.append(b)
round_runs[rnd]["base"] = b
continue
p = item
rep = by_id[p.arm.id]
why = self._ram_guard(p.arm.ram_delta_mib, p.arm.label)
if why:
rep.status, rep.reason = "skipped", why
alive = [x for x in alive if x.arm.id != p.arm.id]
continue
self._say("rounds", f"Trying: {p.arm.label} (round {rnd + 1} of {per_arm})", arm=p.arm.id)
res = self._run(tune_knobs.apply_to_argv(self.base_args, p.args), dict(p.env), spec, p.arm.id,
delta_mib=p.arm.ram_delta_mib)
if res.status == "SKIPPED":
rep.status, rep.reason = "skipped", res.reason
alive = [x for x in alive if x.arm.id != p.arm.id]
continue
if res.status != "PASS":
rep.status, rep.reason = "failed", f"{res.status}: {res.reason}"
alive = [x for x in alive if x.arm.id != p.arm.id]
if res.status == "OOM":
self.storm = stop = True
r.notes.append("an engine ran out of memory; the search stopped (nothing was retried)")
break
continue
if metric(res, o.use):
self.speeds.append({"label": p.arm.label, "tok_s": round(float(metric(res, o.use)), 1)})
over = self._memory_overrun(res, base1)
if over:
rep.status, rep.reason = "failed", over
alive = [x for x in alive if x.arm.id != p.arm.id]
continue
round_runs[rnd][p.arm.id] = res
# a candidate that lost by more than the margin against its own round's base is dropped from later rounds
for p in list(alive):
b, c = round_runs[rnd].get("base"), round_runs[rnd].get(p.arm.id)
um = by_use[p.arm.id]
m_b, m_c = (metric(b, um) if b else None), (metric(c, um) if c else None)
if m_b and m_c and m_c / m_b < 1.0 - max(o.margin, EARLY_DROP):
by_id[p.arm.id].status = "dropped"
by_id[p.arm.id].reason = f"slower by {100 * (1 - m_c / m_b):.1f}% in round {rnd + 1}"
alive = [x for x in alive if x.arm.id != p.arm.id]
data: dict[str, list[tuple[RunResult, RunResult]]] = {
a.id: [(rr["base"], rr[a.id]) for rr in round_runs if a.id in rr and "base" in rr] for a in arms}
# ---- noise: the A/A gap between base runs, the drift against a final base run
end_base: RunResult | None = None
ran_any = any(len(v) for v in data.values())
if not self.storm and not self.cancel.is_set() and ran_any and not self._over_budget() and not self._ram_guard(0, "your current settings"):
self._say("drift", "Measuring your current settings once more to check the PC did not drift")
end_base = self._run(self.base_args, {}, spec, "base end")
if end_base.status == "PASS":
self.base_runs.append(end_base)
mb = [metric(b, o.use) for b in self.base_runs if metric(b, o.use)]
aa = abs(mb[1] / mb[0] - 1.0) if len(mb) >= 2 else 0.0
drift = abs(mb[-1] / mb[0] - 1.0) if len(mb) >= 3 else (aa if len(mb) == 2 else 0.0)
spread = (max(mb) / min(mb) - 1.0) if len(mb) >= 2 else 0.0
r.noise = {"aa_gap": round(aa, 4), "drift": round(drift, 4), "spread": round(spread, 4),
"base_runs": [round(x, 2) for x in mb], "noisy": spread > STAGE_GAP_NOISY}
threshold = accept_threshold(o.margin, aa, drift)
# ---- evaluate the candidates
winners: list[ArmReport] = []
for p in planned:
rep = by_id[p.arm.id]
rows = data[p.arm.id]
if rep.status in ("failed", "skipped") or not rows:
continue
ratios, pps = [], []
um = by_use[p.arm.id]
for b, c in rows:
mc, mbb = metric(c, um), metric(b, um)
if mc and mbb:
ratios.append(mc / mbb)
# the other side of the trade: prompt speed for a decode knob, decode speed for a prompt knob
if um == "prompt":
if c.decode_tok_s and b.decode_tok_s:
pps.append(c.decode_tok_s / b.decode_tok_s)
elif c.pp_tok_s and b.pp_tok_s:
pps.append(c.pp_tok_s / b.pp_tok_s)
rep.pairs = len(ratios)
rep.faster = sum(1 for x in ratios if x > 1.0)
rep.ratio_decode = round(statistics.median(ratios), 4) if ratios else None
rep.ratio_pp = round(statistics.median(pps), 4) if pps else None
rep.runs = [{"ratio": round(x, 4)} for x in ratios]
if rep.status == "dropped":
continue
rep.status = "tested"
if len(ratios) >= per_arm and is_win(ratios, pps, threshold=threshold):
rep.won = True
winners.append(rep)
r.arms = [a.to_dict() for a in arms]
if r.noise["noisy"] and not winners:
r.notes.append(f"the PC was noisy ({spread * 100:.1f}% between identical runs)")
# ---- stage 4: stack the winners, then (thorough) confirm against the base with more pairs
fu = o.use
chosen_args: dict[str, Any] = {}
chosen_env: dict[str, str | None] = {}
final_pairs: list[tuple[RunResult, RunResult]] = []
if winners:
winners.sort(key=lambda a: a.ratio_decode or 0.0, reverse=True)
incumbent = [winners[0]] # gains did not stack in three stacking runs (spec 1.10): the best single knob
if len(winners) > 1:
r.notes.append("more than one setting helped; only the best one is kept because gains did not add up in the study")
for w in incumbent:
chosen_args.update(w.settings)
chosen_env.update(w.env)
fu = by_use[incumbent[0].id]
need_confirm = (o.confirm_pairs or 0) > 0
if need_confirm and not self._over_budget(reserve_s=2 * self._avg_run()):
n = o.confirm_pairs or 2
self._say("confirm", "Confirming the combined result against your current settings")
final_pairs = self._confirm(chosen_args, chosen_env, spec, n, fu)
ratios = [metric(c, fu) / metric(b, fu) for b, c in final_pairs if metric(c, fu) and metric(b, fu)]
pps = [c.decode_tok_s / b.decode_tok_s for b, c in final_pairs if c.decode_tok_s and b.decode_tok_s] if fu == "prompt" \
else [c.pp_tok_s / b.pp_tok_s for b, c in final_pairs if c.pp_tok_s and b.pp_tok_s]
need = max(2, int(math.ceil(0.8 * len(ratios)))) if len(ratios) >= 5 else len(ratios)
if not (ratios and is_win(ratios, pps, threshold=threshold, need=need)):
# the stack did not hold; fall back to the best single winner, which already won its own pairs
if len(incumbent) > 1:
r.notes.append("the combination did not beat the best single change; keeping the best single change")
best = incumbent[0]
chosen_args, chosen_env = dict(best.settings), dict(best.env)
final_pairs = [(b, c) for b, c in data[best.id]]
else:
winners, chosen_args, chosen_env, final_pairs = [], {}, {}, []
r.notes.append("the gain did not hold up in the confirmation, so nothing was changed")
else:
pass
else:
best = incumbent[0]
chosen_args, chosen_env = dict(best.settings), dict(best.env)
final_pairs = list(data[best.id])
if not final_pairs and winners:
final_pairs = list(data[winners[0].id])
fu_final = fu
# ---- per-request winners (found in the base engine): keep only the ones that won every round
per_request_chosen = {row["key"]: row["value"] for row in r.per_request if row.get("won")}
if per_request_chosen and not self.storm and not self.cancel.is_set() and not self._over_budget(reserve_s=self._avg_run()):
# a per-request win from one engine's quick points is only a candidate: confirm it in a fresh engine, 5 interleaved rounds
self._say("confirm", "Confirming the settings that need no restart")
r.per_request_confirm = self._confirm_per_request(per_request_chosen, spec, threshold)
if not r.per_request_confirm["ok"]:
r.notes.append("a per-request setting looked faster in the first test but did not hold in the confirmation, so it was not kept")
per_request_chosen = {}
elif per_request_chosen:
r.notes.append("a per-request setting looked faster but could not be confirmed in the time left, so it was not kept")
per_request_chosen = {}
# ---- result
if end_base is not None and end_base.status == "PASS" and drift > 0.03 and winners:
gain_guess = statistics.median([metric(c, fu) / metric(b, fu) for b, c in final_pairs]) - 1.0 if final_pairs else 0.0
if gain_guess < 2 * drift:
chosen_args, chosen_env, winners, final_pairs = {}, {}, [], []
per_request_chosen = {}
r.outcome = "inconclusive"
r.reason_code = "drift"
r.message = ("The PC changed speed while measuring (about %.0f%%), more than the gain found, so nothing "
"was changed. Try again when nothing else is running." % (drift * 100))
if r.outcome != "inconclusive":
if chosen_args or chosen_env or per_request_chosen:
r.outcome = "improved"
elif r.noise["noisy"] and (spread > 2 * STAGE_GAP_NOISY):
r.outcome = "inconclusive"
r.reason_code = "noisy"
r.message = "The PC was too noisy to tell settings apart, so nothing was changed. Try again when it is idle."
elif arms and not any(a.status in ("tested", "dropped", "failed") for a in arms) and not r.per_request:
# candidates were planned but none could be measured (no free memory, no time): that is not "already the fastest"
r.outcome = "inconclusive"
r.reason_code = "not_tried"
r.message = ("Other settings could not be tried (not enough free memory or time), so nothing was compared and nothing was "
"changed. Try again when the PC is quiet.")
else:
r.outcome = "default_confirmed"
r.message = "Your current settings are already the fastest we found. Nothing was changed."
if r.noise.get("noisy"):
r.message += " (The PC was busy while measuring, so a small gain could be hidden.)"
r.settings = {"args": chosen_args, "env": chosen_env, "per_request": per_request_chosen}
cls, fields = "exact", []
for rep in arms:
if rep.won and any(k in chosen_args for k in rep.settings) or (rep.won and any(k in chosen_env for k in rep.env)):
if tune_knobs.ANSWER_ORDER.index(rep.answer) > tune_knobs.ANSWER_ORDER.index(cls):
cls = rep.answer
if rep.answer != "exact":
fields += list(rep.settings) + list(rep.env)
for key in per_request_chosen:
for row in tune_knobs.PER_REQUEST:
if row["key"] == key and tune_knobs.ANSWER_ORDER.index(row["answer"]) > tune_knobs.ANSWER_ORDER.index(cls):
cls = row["answer"]
if row["key"] == key and row["answer"] != "exact":
fields.append(key)
r.answers = {"class": cls, "fields": sorted(set(fields)), "default_repeatable": not tune_knobs.default_unsteady(self.ctx)}
if final_pairs:
fu = fu_final
ratios = [metric(c, fu) / metric(b, fu) for b, c in final_pairs if metric(c, fu) and metric(b, fu)]
pps = [c.pp_tok_s / b.pp_tok_s for b, c in final_pairs if c.pp_tok_s and b.pp_tok_s]
r.pairs = len(ratios)
r.faster_pairs = sum(1 for x in ratios if x > 1.0)
r.gain_pct = round(100 * (statistics.median(ratios) - 1.0), 1) if ratios else 0.0
r.gain_pp_pct = round(100 * (statistics.median(pps) - 1.0), 1) if pps else 0.0
dec = [c.decode_tok_s / b.decode_tok_s for b, c in final_pairs if c.decode_tok_s and b.decode_tok_s]
r.gain_decode_pct = round(100 * (statistics.median(dec) - 1.0), 1) if dec else 0.0
c_runs = [c for _, c in final_pairs]
r.chosen = {"decode_tok_s": round(statistics.median([c.decode_tok_s for c in c_runs if c.decode_tok_s]), 2)
if any(c.decode_tok_s for c in c_runs) else None,
"pp_tok_s": round(statistics.median([c.pp_long_tok_s or c.pp_tok_s for c in c_runs if c.pp_long_tok_s or c.pp_tok_s]), 1)
if any(c.pp_tok_s for c in c_runs) else None,
"rss_peak_mib": max([c.rss_peak_mib for c in c_runs if c.rss_peak_mib] or [None]) if any(c.rss_peak_mib for c in c_runs) else None,
"vram_used_mib": max([c.vram_used_mib for c in c_runs if c.vram_used_mib] or [None]) if any(c.vram_used_mib for c in c_runs) else None}
elif r.outcome == "improved" and per_request_chosen:
r.gain_pct = r.gain_decode_pct = round(100 * ((r.per_request_confirm or {}).get("ratio", 1.0) - 1.0), 1)
r.pairs = len((r.per_request_confirm or {}).get("ratios", []))
r.faster_pairs = sum(1 for x in (r.per_request_confirm or {}).get("ratios", []) if x > 1.0)
r.chosen = dict(r.base)
else:
r.chosen = {k: r.base.get(k) for k in ("decode_tok_s", "pp_tok_s", "rss_peak_mib", "vram_used_mib")}
if r.outcome == "improved":
r.message = "+%.0f%% faster answers" % r.gain_pct if r.gain_pct >= 0.5 else "slightly faster"
self._say("done", r.message or "Done")
# ---- helpers
def _memory_overrun(self, res: RunResult, base: RunResult) -> str | None:
"""A candidate that takes more memory than the base the user asked for is not a candidate (their wish, 20% left
for work, is a promise about memory)."""
if res.rss_peak_mib and base.rss_peak_mib and res.rss_peak_mib > base.rss_peak_mib * 1.05 + 512 and self.o.goal == "room":
return "it needs more memory than your current settings, which would eat into the memory you wanted left free"
if res.min_avail_mib is not None and res.min_avail_mib < self.o.min_free_ram_mib and (
base.min_avail_mib is None or res.min_avail_mib < base.min_avail_mib - 512): # a RAM-bound base can sit below the floor itself
return "it left the PC with too little free memory"
if self.o.goal == "room" and self.o.ram_wish_pct and self.total_ram and res.min_avail_mib is not None and \
base.min_avail_mib is not None:
if res.min_avail_mib < base.min_avail_mib - max(512, 0.03 * self.total_ram):
return "it left less memory free than your current settings"
if self.o.goal == "room" and res.vram_used_mib is not None and base.vram_used_mib is not None and \
res.vram_used_mib > base.vram_used_mib + 256:
return "it uses more graphics memory than your current settings"
if self.o.goal == "room" and self.o.reserve_mib and self.total_vram and res.vram_used_mib is not None and \
base.vram_used_mib is not None and self.total_vram - base.vram_used_mib >= self.o.reserve_mib - 128 and \
self.total_vram - res.vram_used_mib < self.o.reserve_mib - 128:
return "it would not leave the graphics memory you wanted free"
return None
def _confirm(self, args: dict[str, Any], env: dict[str, str | None], spec: RequestSet, pairs: int, use: str | None = None) -> list[tuple[RunResult, RunResult]]:
"""Interleaved pairs of the combined settings against the base (the order alternates)."""
out: list[tuple[RunResult, RunResult]] = []
cand_args = tune_knobs.apply_to_argv(self.base_args, args)
for i in range(pairs):
self._check_cancel()
if self._over_budget(reserve_s=2 * self._avg_run()):
self.result.notes.append("confirmation stopped early: the time you chose ran out")
break
first = "cand" if i % 2 == 0 else "base"
runs: dict[str, RunResult] = {}
for who in (first, "base" if first == "cand" else "cand"):
self._say("confirm", f"Confirming (pair {i + 1} of {pairs})")
runs[who] = self._run(cand_args if who == "cand" else self.base_args, env if who == "cand" else {}, spec,
f"confirm {who} {i + 1}")
if runs["cand"].status == "PASS" and runs["base"].status == "PASS":
use = use or self.o.use
if metric(runs["cand"], use) and metric(runs["base"], use):
out.append((runs["base"], runs["cand"]))
self.base_runs.append(runs["base"])
else:
break
return out
def _per_request_stage(self, eng: Any) -> list[dict[str, Any]]:
"""Per-request settings against the engine's own choice, in this one engine, 3 interleaved rounds (6.3 step 2).
A **control** point sends a per-request value equal to what the engine already uses (the configuration's own
`--spec-min-p`), so it changes nothing: whatever it "gains" is the bias of sending a setting at all (the first run
after an idle gap, the order). A value must beat the default by the margin AND beat the control by the margin in every
round. A win here is only a candidate: the tuner confirms it afterwards in a fresh engine. One value per key can win."""
points: list[tuple[str, Any]] = []
for spec in tune_knobs.PER_REQUEST:
if self.ctx.backend not in spec["backends"]:
continue
if spec["answer"] != "exact" and not tune_knobs.allowed(spec["answer"], self.ctx):
continue
for v in spec["values"]:
points.append((spec["key"], v))
if not points:
return []
own = tune_knobs._current(self.ctx.base_args, "--spec-min-p")
control = ("control", float(own)) if own not in (None, "") and self.ctx.backend != "metal" else None
rounds = 3
default: dict[int, list[float]] = {}
vals: dict[tuple[str, Any], dict[int, float]] = {pt: {} for pt in points + ([control] if control else [])}
self._say("per_request", "Trying settings that need no restart")
for rnd in range(rounds):
order = rotate([None] + points + ([control] if control else []) + [None], rnd)
for item in order:
self._check_cancel()
if item is None:
d = eng.decode_once(None, 200)
if d:
default.setdefault(rnd, []).append(d)
else:
key = "spec_min_p" if item[0] == "control" else item[0]
d = eng.decode_once({key: item[1]}, 200)
if d:
vals[item][rnd] = d
ref = {i: statistics.mean(v) for i, v in default.items() if v}
ctl = [vals[control][i] / ref[i] for i in range(rounds) if control and i in vals[control] and i in ref] if control else []
rows: list[dict[str, Any]] = []
for pt in points:
ratios = [vals[pt][i] / ref[i] for i in range(rounds) if i in vals[pt] and i in ref]
adj = [ratios[j] / ctl[j] for j in range(min(len(ratios), len(ctl)))] if ctl else ratios
won = len(ratios) == rounds and len(adj) == rounds and all(x >= 1.0 + self.o.margin for x in adj) \
and all(x >= 1.0 + self.o.margin for x in ratios)
rows.append({"key": pt[0], "value": pt[1], "ratios": [round(x, 4) for x in ratios], "control_ratios": [round(x, 4) for x in ctl],
"ratio": round(statistics.median(adj), 4) if adj else None, "won": won})
best: dict[str, dict[str, Any]] = {}
for row in rows:
if row["won"] and (row["key"] not in best or row["ratio"] > best[row["key"]]["ratio"]):
best[row["key"]] = row
for row in rows:
row["won"] = best.get(row["key"]) is row
return rows
def _confirm_per_request(self, chosen: dict[str, Any], spec: RequestSet, threshold: float) -> dict[str, Any]:
"""5 interleaved rounds in a fresh engine of the chosen per-request settings against the engine's own choice (two decodes
each, the order alternating). Kept only if the median ratio beats the threshold in at least 4 of 5 rounds."""
ratios: list[float] = []
def points(eng: Any) -> None:
for i in range(5):
self._check_cancel()
got: dict[str, list[float]] = {"d": [], "t": []}
for who in (("d", "t") if i % 2 == 0 else ("t", "d")):
for _ in range(2):
v = eng.decode_once(chosen if who == "t" else None, 200)
if v:
got[who].append(v)
if got["d"] and got["t"]:
ratios.append(statistics.mean(got["t"]) / statistics.mean(got["d"]))
res = self._run(self.base_args, {}, spec, "confirm per-request", points=points)
ok = res.status == "PASS" and len(ratios) == 5 and sum(1 for x in ratios if x > 1.0) >= 4 \
and statistics.median(ratios) >= threshold
return {"ok": ok, "ratios": [round(x, 4) for x in ratios], "ratio": round(statistics.median(ratios), 4) if ratios else None,
"status": res.status}
# --------------------------------------------------------------------------- building a tuner from a run configuration
def split_args(argv: list[str]) -> list[str]:
return [str(a) for a in argv]
def detect_family(args: list[str]) -> str:
for i, a in enumerate(args):
if a == "--mtp" and i + 1 < len(args):
return "qwen36" if str(args[i + 1]).isdigit() else "flash-next"
if "--pack" in args or "--native" in args:
return "flash-next"
return "flash-next"
def model_gib_of(args: list[str]) -> float:
"""Total size of the model files a configuration names, shards included."""
from pathlib import Path
import re
total = 0
seen: set[str] = set()
for i, a in enumerate(args):
if a in ("--native", "--model", "--ple-gguf") and i + 1 < len(args):
p = Path(str(args[i + 1]))
if not p.is_file() or str(p) in seen:
continue
m = re.match(r"(.*)-(\d{5})-of-(\d{5})\.gguf$", p.name)
group = sorted(p.parent.glob(f"{m.group(1)}-*-of-{m.group(3)}.gguf")) if m else [p]
for f in group:
if str(f) not in seen:
seen.add(str(f))
total += f.stat().st_size
return round(total / 1024 ** 3, 1)
def context_from_config(cfg: dict[str, Any], *, engine_backend: str | None, opts: Options,
engine_flags: list[str] | None = None, system: dict[str, Any] | None = None,
family: str | None = None) -> tune_knobs.Context:
args = split_args(cfg.get("args") or [])
system = system or {"memory": tune_machine.memory(), "gpus": tune_machine.gpus(), "cpu": tune_machine.cpu()}
gpu_cfg = cfg.get("gpu")
wanted = gpu_cfg if isinstance(gpu_cfg, list) else ([gpu_cfg] if isinstance(gpu_cfg, int) else None)
cards = system.get("gpus") or []
used = [g for g in cards if not wanted or g.get("index") in wanted] or cards
ctx_tokens = 32768
for i, a in enumerate(args):
if a == "--max-context" and i + 1 < len(args) and str(args[i + 1]).isdigit():
ctx_tokens = int(args[i + 1])
mem = system.get("memory") or {}
cpu = system.get("cpu") or {}
backend = engine_backend or ("cuda" if any(g.get("vendor") == "nvidia" for g in cards) else
"hip" if any(g.get("vendor") == "amd" for g in cards) else "cpu")
arch = None
if backend == "hip":
try:
archs = engine_component.hip_archs_present()
arch = archs[0] if archs else None
except Exception: # noqa: BLE001
arch = None
return tune_knobs.Context(
backend=backend, family=family or detect_family(args), gpu_count=max(1, len(used)),
vram_mib=min((g["vram_total_mib"] for g in used if g.get("vram_total_mib")), default=0),
ram_gib=int((mem.get("total_mib") or 0) // 1024), model_gib=model_gib_of(args),
hybrid_cpu=bool(cpu.get("hybrid")), logical_threads=int(cpu.get("logical") or 0), arch=arch,
context_tokens=ctx_tokens, goal=opts.goal, use=opts.use, allow_placement=opts.allow_placement,
allow_nonrepeatable=opts.allow_nonrepeatable, engine_flags=engine_flags, base_args=args)
# --------------------------------------------------------------------------- the short "still good?" re-check (8.3)
def recheck(session: Session, base_args: list[str], preset: dict[str, Any], *, ctx: tune_knobs.Context,
engine_changed: bool = False, progress: Callable[[dict[str, Any]], None] | None = None,
cancel: threading.Event | None = None, clock: Callable[[], float] = time.monotonic) -> dict[str, Any]:
"""Is a saved preset still as fast as when it was made? One restart with the preset's settings and 30-45 s of requests.
Pass rule (against the stored numbers, not a fresh baseline): decode and the 4K prompt at least 95% of what was stored,
memory within the stored +5%. A failing prompt rate is measured once more with a second start before it counts (the
prompt rate of some cards is bimodal between engine starts). When the engine changed in a way that can move speed
(``tuning_epoch``) a baseline start is added and the preset must be at least as fast as doing nothing.
The result is ``keep``, ``retune`` (suggested, never forced) or ``inconclusive`` (noise or a competing process)."""
progress = progress or (lambda p: None)
cancel = cancel or threading.Event()
t0 = clock()
s = preset.get("settings") or {}
stored = preset.get("measured") or {}
args = tune_knobs.apply_to_argv(base_args, s.get("args") or {})
env = dict(s.get("env") or {})
spec = RequestSet(warmups=1, decodes=3, decode_tokens=200, pp_tokens=4096 if ctx.context_tokens >= 6000 else 1024, long_tokens=0)
if ctx.backend == "sycl":
spec.pp_tokens = 0 # the A750's 4K prompt takes a minute: decode only (8.3)
out: dict[str, Any] = {"verdict": "inconclusive", "request_set": RECHECK_SET_NAME, "notes": []}
def one(a: list[str], e: dict[str, str | None], label: str) -> RunResult:
if cancel.is_set():
raise Cancelled()
progress({"stage": "recheck", "message": label, "elapsed_s": round(clock() - t0)})
try:
with session.engine(a, e) as eng:
r = eng.measure(spec)
except StartFailed as ex:
return RunResult(status=ex.status, reason=ex.reason)
return r
try:
run = one(args, env, "Checking your saved settings")
if run.status != "PASS":
out.update(verdict="retune", reason=f"the saved settings did not start now: {run.status} {run.reason}")
out["seconds"] = round(clock() - t0, 1)
return out
out["measured"] = {"decode_tok_s": run.decode_tok_s, "pp4k_tok_s": run.pp_tok_s, "rss_peak_mib": run.rss_peak_mib,
"vram_peak_mib": run.vram_used_mib}
sd, sp = stored.get("decode_tok_s"), stored.get("pp4k_tok_s")
problems: list[str] = []
if sd and run.decode_tok_s and run.decode_tok_s < 0.95 * sd:
problems.append(f"answers are {100 * (1 - run.decode_tok_s / sd):.0f}% slower than when it was saved")
if sp and run.pp_tok_s and run.pp_tok_s < 0.95 * sp:
again = one(args, env, "Checking prompt speed once more")
if again.status == "PASS" and again.pp_tok_s and again.pp_tok_s >= 0.95 * sp:
out["notes"].append("prompt speed varied between starts; the second start was fine")
else:
problems.append("long prompts are slower than when it was saved")
for key, got in (("rss_peak_mib", run.rss_peak_mib), ("vram_peak_mib", run.vram_used_mib)):
ref = stored.get(key)
if ref and got and got > 1.05 * ref:
problems.append("it now uses more memory than when it was saved")
break
if engine_changed:
base = one(base_args, {}, "Checking against the standard settings after the engine update")
if base.status == "PASS" and base.decode_tok_s and run.decode_tok_s and run.decode_tok_s < base.decode_tok_s:
problems.append("the standard settings are now faster than the saved ones")
out["baseline"] = {"decode_tok_s": base.decode_tok_s, "pp4k_tok_s": base.pp_tok_s}
out["problems"] = problems
out["verdict"] = "retune" if problems else "keep"
out["reason"] = ("Still as fast as when it was saved." if not problems else "; ".join(problems))
except Cancelled:
out.update(verdict="inconclusive", reason="Stopped.")
out["seconds"] = round(clock() - t0, 1)
return out
RECHECK_SET_NAME = "recheck/1"