Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

trellis: swap comparison and greedy sanity scripts

Niko1221adf7ca97296a
diff --git a/tools/trellis/sanity_greedy.py b/tools/trellis/sanity_greedy.py
new file mode 100644
index 000000000..c6e2bc2ec
--- /dev/null
+++ b/tools/trellis/sanity_greedy.py
@@ -0,0 +1,76 @@
+"""sanity_greedy.py - a few greedy answers from an engine config (the same config files tools/quant_quality.py uses), through the
+normal batched prompt path (not the yardstick's decode windows).
+
+ python3 tools/trellis/sanity_greedy.py --config CFG.json --out answers.txt [--wrap "flock ... env HIP_VISIBLE_DEVICES=3"]
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import pathlib
+import subprocess
+import sys
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+sys.path.insert(0, str(HERE))
+import make_corpus # noqa: E402
+
+PROMPTS = [
+ "What is the capital of Australia, and what is it known for? Answer in two sentences.",
+ "Write a Python function that returns the n-th Fibonacci number iteratively, with a docstring.",
+ "Translate into German: 'The library opens at nine and closes at five, except on Sundays.'",
+ "A train leaves at 14:20 and arrives at 17:05. How long is the trip? Think step by step, then give the answer.",
+]
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--config", required=True)
+ ap.add_argument("--out", required=True)
+ ap.add_argument("--wrap", default="")
+ ap.add_argument("--new", type=int, default=80)
+ a = ap.parse_args()
+ cfg = json.load(open(a.config))
+ tok = make_corpus.load_tokenizer(cfg["tokenizer"])
+ env = dict(os.environ)
+ env.update(cfg.get("env", {}))
+ env["LD_LIBRARY_PATH"] = ":".join(cfg.get("lib_dirs", []) + [env.get("LD_LIBRARY_PATH", "")])
+ args = [cfg["exe"], "--serve"] + [x for x in cfg["args"] if x != "--serve"]
+ if a.wrap:
+ args = a.wrap.split() + args
+ p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=open(a.out + ".stderr", "w"), text=True,
+ cwd=cfg.get("cwd"), env=env, bufsize=1)
+ for line in p.stdout:
+ if line.startswith("READY"):
+ break
+ else:
+ raise SystemExit("the engine ended before READY")
+ out = []
+ for q in PROMPTS:
+ text = "<|im_start|>user\n%s<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n" % q
+ ids = tok.encode(text, parse_special=True)
+ p.stdin.write("GEN %d %s\n" % (a.new, ",".join(map(str, ids))))
+ p.stdin.flush()
+ gen = []
+ for line in p.stdout:
+ if line.startswith("T "):
+ gen.append(int(line.split()[1]))
+ elif line.startswith(("DONE", "ERR")):
+ done = line.strip()
+ break
+ out.append("### %s\n%s\n[%s]\n" % (q, tok.decode(gen), done))
+ print(out[-1], flush=True)
+ p.stdin.write("QUIT\n")
+ p.stdin.flush()
+ try:
+ p.wait(timeout=300)
+ except subprocess.TimeoutExpired:
+ p.kill()
+ pathlib.Path(a.out).write_text("\n".join(out), encoding="utf-8")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/trellis/swap_compare.py b/tools/trellis/swap_compare.py
new file mode 100644
index 000000000..81ccee24b
--- /dev/null
+++ b/tools/trellis/swap_compare.py
@@ -0,0 +1,96 @@
+"""swap_compare.py - KL / NLL of Gyro-S swap variants against the Q8_0-swap reference, with paired differences.
+
+ python3 tools/trellis/swap_compare.py REF.tsv shipped=RUN.tsv plain=RUN.tsv diag=RUN.tsv [...] [--base shipped]
+
+Per run: mean KL(REF || run) over REF's top-64 (tools/quant_quality.py's definition), top-1 agreement with REF, mean negative
+log-likelihood of the text (and its difference to REF's), each with a standard error over segments. Then the paired
+difference of every run against --base (same positions): mean +- standard error over segments, and the share of segments
+where the run is better. Per text kind (english / multilingual / code / docs / agent) too.
+"""
+from __future__ import annotations
+
+import argparse
+import math
+import pathlib
+import sys
+
+import numpy as np
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+import quant_quality as QQ # noqa: E402
+
+
+def per_position(ref, run):
+ """{segment: (kl[], nll[], agree[])} for positions both runs have."""
+ out = {}
+ for name, rr in ref.items():
+ ra = run.get(name, {})
+ kls, nll, ag = [], [], []
+ for q in sorted(set(rr) & set(ra)):
+ tr, lr, topr, kr = rr[q]
+ ta, la, topa, ka = ra[q]
+ if tr != ta:
+ raise SystemExit("%s pos %d: different text" % (name, q))
+ da = dict(ka)
+ floor = min(b for _, b in ka)
+ pr = np.array([math.exp(b) for _, b in kr])
+ pa = np.array([math.exp(da.get(i, floor)) for i, _ in kr])
+ pr /= pr.sum()
+ pa /= pa.sum()
+ kls.append(float(np.sum(pr * np.log(pr / pa))))
+ nll.append(-la)
+ ag.append(float(topr == topa))
+ if kls:
+ out[name] = (np.array(kls), np.array(nll), np.array(ag))
+ return out
+
+
+def kind(name):
+ return name.rsplit("_", 1)[0]
+
+
+def se(x):
+ return float(np.std(x, ddof=1) / math.sqrt(len(x))) if len(x) > 1 else float("nan")
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("ref")
+ ap.add_argument("runs", nargs="+", help="label=file.tsv")
+ ap.add_argument("--base", default=None)
+ a = ap.parse_args()
+ ref = QQ.read_run(a.ref)
+ runs = {}
+ for r in a.runs:
+ lab, _, path = r.partition("=")
+ runs[lab] = per_position(ref, QQ.read_run(path))
+ runs.setdefault('ref', per_position(ref, ref))
+ base = a.base or next(iter(runs))
+ segs = sorted(set.intersection(*[set(v) for v in runs.values()]))
+ kinds = sorted({kind(s) for s in segs})
+ print("segments %d, positions %d (reference %s)\n" % (len(segs), sum(len(runs[base][s][0]) for s in segs), a.ref))
+
+ def seg_mean(lab, idx, ss):
+ return np.array([runs[lab][s][idx].mean() for s in ss])
+
+ for title, ss in [("ALL", segs)] + [(k, [s for s in segs if kind(s) == k]) for k in kinds]:
+ if not ss:
+ continue
+ print("== %s (%d segments)" % (title, len(ss)))
+ print(" %-10s %-22s %-14s %-20s %s" % ("run", "KL(ref||run)", "top-1 vs ref", "NLL of the text", "paired KL difference vs " + base))
+ for lab in runs:
+ kl, ag, nl = seg_mean(lab, 0, ss), seg_mean(lab, 2, ss), seg_mean(lab, 1, ss)
+ kb = seg_mean(base, 0, ss)
+ d = kl - kb
+ nb = seg_mean(base, 1, ss)
+ print(" %-10s %.5f +- %.5f %.4f %.4f +- %.4f %s" % (
+ lab, kl.mean(), se(kl), ag.mean(), nl.mean(), se(nl),
+ "(base)" if lab == base else "%+.5f +- %.5f better in %d/%d segments; NLL %+.4f +- %.4f" % (
+ d.mean(), se(d), int((d < 0).sum()), len(d), (nl - nb).mean(), se(nl - nb))))
+ print()
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())