Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
5.2 KB
Raw Blame
"""tools/qwen35/kl.py - teacher-forced KL for Qwen3.6-35B-A3B (NEW-MODELS-PLAN.md T1/T2): on one token sequence per
prompt (the prompt + llama.cpp GPU's greedy answer, from a parity.py run), the next-token distributions of
llama.cpp HIP (the reference P), llama.cpp CPU (the noise floor) and strata-q36ref.  Reported: mean KL(P || Q) and
the top-1 agreement over all steps, per engine; the plan's gate is Strata's KL within 1.5x of the floor.

    python tools/qwen35/kl.py --ref BIN --dump BIN --model GGUF --parity DIR --out DIR [--lock "..."] [--cpu-lock "..."]
"""
from __future__ import annotations

import argparse
import shlex
import subprocess
import sys
from pathlib import Path

import numpy as np

V = 248320
EOS = 248046                                    # <|im_end|>


def ids(path: Path, key: str) -> list[int]:
    for line in path.read_text(encoding="utf-8").splitlines():
        if line.startswith(key + ":"):
            return [int(x) for x in line.split(":", 1)[1].split()]
    raise SystemExit(f"no {key}: in {path}")


def run(cmd: list[str], lock: str, log: Path) -> None:
    r = subprocess.run((shlex.split(lock) if lock else []) + cmd, capture_output=True, text=True, timeout=3600)
    log.write_text(r.stdout + "\n--- stderr ---\n" + r.stderr, encoding="utf-8")
    if r.returncode != 0:
        raise SystemExit(f"{cmd[0]} failed (rc {r.returncode}), see {log}")


def log_softmax(x: np.ndarray) -> np.ndarray:
    x = x.astype(np.float64)
    m = x.max(axis=1, keepdims=True)
    return x - m - np.log(np.exp(x - m).sum(axis=1, keepdims=True))


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--ref", required=True)
    ap.add_argument("--dump", required=True)
    ap.add_argument("--model", required=True)
    ap.add_argument("--parity", required=True, help="a parity.py output dir (its llama.cpp logs give the sequences)")
    ap.add_argument("--out", required=True)
    ap.add_argument("--lock", default="")
    ap.add_argument("--cpu-lock", default="")
    ap.add_argument("--cache", default="", help="where llama.cpp's logits are kept between runs (default --out)")
    ap.add_argument("--ref-args", default="", help="extra strata-q36ref arguments (e.g. '--act f32')")
    a = ap.parse_args()
    out, par = Path(a.out), Path(a.parity)
    out.mkdir(parents=True, exist_ok=True)
    tags = sorted(p.name.split(".")[0] for p in par.glob("p*.llama.log"))
    kl = {"cpu": [], "strata": []}
    top1 = {"cpu": [], "strata": []}
    kl_ans = {"cpu": [], "strata": []}
    top1_ans = {"cpu": [], "strata": []}
    for tag in tags:
        prompt, gen = ids(par / f"{tag}.llama.log", "prompt"), ids(par / f"{tag}.llama.log", "gen")
        force = ",".join(map(str, gen))
        toks = ",".join(map(str, prompt))
        cache = Path(a.cache) if a.cache else out
        f = {k: (out if k == "strata" else cache) / f"{tag}.{k}.full" for k in ("gpu", "cpu", "strata")}
        # llama.cpp's two runs depend only on the model and the sequence: kept in --out and reused (64 MB each)
        if not f["gpu"].exists():
            run([a.dump, "--model", a.model, "--tokens", toks, "--force", force, "--ngl", "99", "--dump-full-logits",
                 str(f["gpu"])], a.lock, cache / f"{tag}.gpu.log")
        if not f["cpu"].exists():
            run([a.dump, "--model", a.model, "--tokens", toks, "--force", force, "--ngl", "0", "--dump-full-logits",
                 str(f["cpu"])], a.cpu_lock, cache / f"{tag}.cpu.log")
        run([a.ref, "--model", a.model, "--tokens", toks, "--force", force, "--dump-full-logits", str(f["strata"]),
             *shlex.split(a.ref_args)], a.lock, out / f"{tag}.strata.log")
        P = log_softmax(np.fromfile(f["gpu"], dtype=np.float32).reshape(-1, V))
        # the answer: the steps up to and including the first <|im_end|> of the forced sequence (past it the context is
        # one no chat reaches, and its distributions are the noisiest); "all" keeps every step
        ans = gen.index(EOS) + 1 if EOS in gen else len(P)
        line = f"{tag}: {len(P)} steps ({ans} answer)"
        per = {}
        for k in ("cpu", "strata"):
            Q = log_softmax(np.fromfile(f[k], dtype=np.float32).reshape(-1, V))
            n = min(len(P), len(Q))
            d = (np.exp(P[:n]) * (P[:n] - Q[:n])).sum(axis=1)
            t1 = P[:n].argmax(axis=1) == Q[:n].argmax(axis=1)
            kl[k].extend(d.tolist())
            top1[k].extend(t1.tolist())
            kl_ans[k].extend(d[:ans].tolist())
            top1_ans[k].extend(t1[:ans].tolist())
            per[k] = d
            line += f"; {k}: KL {d.mean():.5f} (answer {d[:ans].mean():.5f}) top-1 {t1.mean():.3f}"
        np.savez(out / f"{tag}.kl.npz", answer=ans, **per)
        f["strata"].unlink(missing_ok=True)
        print(line, flush=True)
    for name, K, T in (("all steps", kl, top1), ("answers only", kl_ans, top1_ans)):
        print(f"-- {name}")
        for k in ("cpu", "strata"):
            print(f"{k:6s} vs llama.cpp HIP: mean KL {np.mean(K[k]):.5f} (median {np.median(K[k]):.6f}, p99 "
                  f"{np.percentile(K[k], 99):.4f}), top-1 {np.mean(T[k]):.4f} over {len(K[k])} steps")
        print(f"ratio strata / floor: {np.mean(K['strata']) / max(np.mean(K['cpu']), 1e-12):.2f}")
    return 0


if __name__ == "__main__":
    sys.exit(main())