Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
trellis: swap comparison and greedy sanity scripts
Niko1221adf7ca97296a
diff --git a/tools/trellis/sanity_greedy.py b/tools/trellis/sanity_greedy.pynew file mode 100644index 000000000..c6e2bc2ec--- /dev/null+++ b/tools/trellis/sanity_greedy.py@@ -0,0 +1,76 @@+"""sanity_greedy.py - a few greedy answers from an engine config (the same config files tools/quant_quality.py uses), through the+normal batched prompt path (not the yardstick's decode windows).++ python3 tools/trellis/sanity_greedy.py --config CFG.json --out answers.txt [--wrap "flock ... env HIP_VISIBLE_DEVICES=3"]+"""+from __future__ import annotations++import argparse+import json+import os+import pathlib+import subprocess+import sys++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE.parent))+sys.path.insert(0, str(HERE))+import make_corpus # noqa: E402++PROMPTS = [+ "What is the capital of Australia, and what is it known for? Answer in two sentences.",+ "Write a Python function that returns the n-th Fibonacci number iteratively, with a docstring.",+ "Translate into German: 'The library opens at nine and closes at five, except on Sundays.'",+ "A train leaves at 14:20 and arrives at 17:05. How long is the trip? Think step by step, then give the answer.",+]+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--config", required=True)+ ap.add_argument("--out", required=True)+ ap.add_argument("--wrap", default="")+ ap.add_argument("--new", type=int, default=80)+ a = ap.parse_args()+ cfg = json.load(open(a.config))+ tok = make_corpus.load_tokenizer(cfg["tokenizer"])+ env = dict(os.environ)+ env.update(cfg.get("env", {}))+ env["LD_LIBRARY_PATH"] = ":".join(cfg.get("lib_dirs", []) + [env.get("LD_LIBRARY_PATH", "")])+ args = [cfg["exe"], "--serve"] + [x for x in cfg["args"] if x != "--serve"]+ if a.wrap:+ args = a.wrap.split() + args+ p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=open(a.out + ".stderr", "w"), text=True,+ cwd=cfg.get("cwd"), env=env, bufsize=1)+ for line in p.stdout:+ if line.startswith("READY"):+ break+ else:+ raise SystemExit("the engine ended before READY")+ out = []+ for q in PROMPTS:+ text = "<|im_start|>user\n%s<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n" % q+ ids = tok.encode(text, parse_special=True)+ p.stdin.write("GEN %d %s\n" % (a.new, ",".join(map(str, ids))))+ p.stdin.flush()+ gen = []+ for line in p.stdout:+ if line.startswith("T "):+ gen.append(int(line.split()[1]))+ elif line.startswith(("DONE", "ERR")):+ done = line.strip()+ break+ out.append("### %s\n%s\n[%s]\n" % (q, tok.decode(gen), done))+ print(out[-1], flush=True)+ p.stdin.write("QUIT\n")+ p.stdin.flush()+ try:+ p.wait(timeout=300)+ except subprocess.TimeoutExpired:+ p.kill()+ pathlib.Path(a.out).write_text("\n".join(out), encoding="utf-8")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/trellis/swap_compare.py b/tools/trellis/swap_compare.pynew file mode 100644index 000000000..81ccee24b--- /dev/null+++ b/tools/trellis/swap_compare.py@@ -0,0 +1,96 @@+"""swap_compare.py - KL / NLL of Gyro-S swap variants against the Q8_0-swap reference, with paired differences.++ python3 tools/trellis/swap_compare.py REF.tsv shipped=RUN.tsv plain=RUN.tsv diag=RUN.tsv [...] [--base shipped]++Per run: mean KL(REF || run) over REF's top-64 (tools/quant_quality.py's definition), top-1 agreement with REF, mean negative+log-likelihood of the text (and its difference to REF's), each with a standard error over segments. Then the paired+difference of every run against --base (same positions): mean +- standard error over segments, and the share of segments+where the run is better. Per text kind (english / multilingual / code / docs / agent) too.+"""+from __future__ import annotations++import argparse+import math+import pathlib+import sys++import numpy as np++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE.parent))+import quant_quality as QQ # noqa: E402+++def per_position(ref, run):+ """{segment: (kl[], nll[], agree[])} for positions both runs have."""+ out = {}+ for name, rr in ref.items():+ ra = run.get(name, {})+ kls, nll, ag = [], [], []+ for q in sorted(set(rr) & set(ra)):+ tr, lr, topr, kr = rr[q]+ ta, la, topa, ka = ra[q]+ if tr != ta:+ raise SystemExit("%s pos %d: different text" % (name, q))+ da = dict(ka)+ floor = min(b for _, b in ka)+ pr = np.array([math.exp(b) for _, b in kr])+ pa = np.array([math.exp(da.get(i, floor)) for i, _ in kr])+ pr /= pr.sum()+ pa /= pa.sum()+ kls.append(float(np.sum(pr * np.log(pr / pa))))+ nll.append(-la)+ ag.append(float(topr == topa))+ if kls:+ out[name] = (np.array(kls), np.array(nll), np.array(ag))+ return out+++def kind(name):+ return name.rsplit("_", 1)[0]+++def se(x):+ return float(np.std(x, ddof=1) / math.sqrt(len(x))) if len(x) > 1 else float("nan")+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("ref")+ ap.add_argument("runs", nargs="+", help="label=file.tsv")+ ap.add_argument("--base", default=None)+ a = ap.parse_args()+ ref = QQ.read_run(a.ref)+ runs = {}+ for r in a.runs:+ lab, _, path = r.partition("=")+ runs[lab] = per_position(ref, QQ.read_run(path))+ runs.setdefault('ref', per_position(ref, ref))+ base = a.base or next(iter(runs))+ segs = sorted(set.intersection(*[set(v) for v in runs.values()]))+ kinds = sorted({kind(s) for s in segs})+ print("segments %d, positions %d (reference %s)\n" % (len(segs), sum(len(runs[base][s][0]) for s in segs), a.ref))++ def seg_mean(lab, idx, ss):+ return np.array([runs[lab][s][idx].mean() for s in ss])++ for title, ss in [("ALL", segs)] + [(k, [s for s in segs if kind(s) == k]) for k in kinds]:+ if not ss:+ continue+ print("== %s (%d segments)" % (title, len(ss)))+ print(" %-10s %-22s %-14s %-20s %s" % ("run", "KL(ref||run)", "top-1 vs ref", "NLL of the text", "paired KL difference vs " + base))+ for lab in runs:+ kl, ag, nl = seg_mean(lab, 0, ss), seg_mean(lab, 2, ss), seg_mean(lab, 1, ss)+ kb = seg_mean(base, 0, ss)+ d = kl - kb+ nb = seg_mean(base, 1, ss)+ print(" %-10s %.5f +- %.5f %.4f %.4f +- %.4f %s" % (+ lab, kl.mean(), se(kl), ag.mean(), nl.mean(), se(nl),+ "(base)" if lab == base else "%+.5f +- %.5f better in %d/%d segments; NLL %+.4f +- %.4f" % (+ d.mean(), se(d), int((d < 0).sum()), len(d), (nl - nb).mean(), se(nl - nb))))+ print()+ return 0+++if __name__ == "__main__":+ sys.exit(main())