Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: tools to turn the llama.cpp reference's 10 greedy prompts into ids and compare the engine's greedy output (serve protocol)
Niko1221c3e9a998ad91
diff --git a/tools/dsv41/mkprompts.py b/tools/dsv41/mkprompts.pynew file mode 100644index 000000000..bf467e86f--- /dev/null+++ b/tools/dsv41/mkprompts.py@@ -0,0 +1,48 @@+#!/usr/bin/env python3+"""mkprompts.py: the 10 greedy prompts of the llama.cpp reference run (runs/base1/answers_{chat,think}) as token ids.++Renders each request with the GGUF's chat template (jinja2, thinking_mode chat | thinking), tokenizes with llama-tokenize+(--no-bos (special tokens parse by default): the template writes the BOS text itself), and keeps the reference's greedy output ids.+Writes OUT/prompts.json: [{"id", "mode", "ids": [...], "expect_ids": [...], "expect_text", "finish"}].+"""+import json, os, subprocess, sys, tempfile+import jinja2+from jinja2.sandbox import ImmutableSandboxedEnvironment++ref_dir, template, shard, tokenize, out_dir = sys.argv[1:6]+env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True)+env.filters["tojson"] = lambda x, **k: json.dumps(x, ensure_ascii=False)+def raise_exception(m):+ raise jinja2.exceptions.TemplateError(m)+env.globals["raise_exception"] = raise_exception+tpl = env.from_string(open(template, encoding="utf-8").read())+libs = os.environ.get("LD_LIBRARY_PATH", "")++def tok(text):+ with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False, encoding="utf-8") as f:+ f.write(text)+ p = f.name+ r = subprocess.run([tokenize, "-m", shard, "-f", p, "--ids", "--no-bos", "--no-escape"],+ capture_output=True, text=True, env=dict(os.environ, LD_LIBRARY_PATH=libs))+ os.unlink(p)+ line = [l for l in r.stdout.splitlines() if l.startswith("[")][-1]+ return json.loads(line)++out = []+for mode_dir, mode in (("answers_chat", "chat"), ("answers_think", "thinking")):+ d = os.path.join(ref_dir, mode_dir)+ for fn in sorted(os.listdir(d)):+ if fn == "summary.json" or not fn.endswith(".json"):+ continue+ r = json.load(open(os.path.join(d, fn), encoding="utf-8"))+ b = r["request_body"]+ text = tpl.render(messages=b["messages"], tools=b.get("tools"), thinking_mode=mode, reasoning_effort=None,+ add_generation_prompt=True)+ ids = tok(text)+ ch = r["response"]["choices"][0]+ out.append({"id": fn[:-5], "mode": mode, "ids": ids, "max_tokens": b.get("max_tokens", 256),+ "expect_ids": [t["id"] for t in ch["logprobs"]["content"]],+ "expect_text": ch["message"].get("content"), "finish": ch["finish_reason"]})+ print(mode, fn[:-5], len(ids), "prompt tokens;", len(out[-1]["expect_ids"]), "expected")+os.makedirs(out_dir, exist_ok=True)+json.dump(out, open(os.path.join(out_dir, "prompts.json"), "w"))diff --git a/tools/dsv41/serve_prompts.py b/tools/dsv41/serve_prompts.pynew file mode 100644index 000000000..8f0b6e93d--- /dev/null+++ b/tools/dsv41/serve_prompts.py@@ -0,0 +1,63 @@+#!/usr/bin/env python3+"""serve_prompts.py PROMPTS.json OUT.json -- ENGINE CMD...+Starts the engine in --serve mode (appended), waits for READY, sends GEN for every prompt (greedy), compares the+generated ids with the llama.cpp reference's greedy ids (expect_ids), and prints one line each:+ id mode prompt_tokens n_gen match_prefix/n_expect first_divergence prefill ms decode tok/s+"""+import json, subprocess, sys, time++prompts_path, out_path = sys.argv[1], sys.argv[2]+cmd = sys.argv[sys.argv.index("--") + 1:] + ["--serve"]+prompts = json.load(open(prompts_path))+only = [a for a in sys.argv[3:sys.argv.index("--")]]+if only:+ prompts = [p for p in prompts if p["id"] in only or (p["mode"] + ":" + p["id"]) in only]+p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=sys.stderr, text=True, bufsize=1)+t0 = time.time()+while True:+ line = p.stdout.readline()+ if not line:+ print("engine exited before READY", file=sys.stderr)+ sys.exit(2)+ if line.startswith("READY"):+ print("READY after %.0f s: %s" % (time.time() - t0, line.strip()), flush=True)+ break+ print("engine>", line.rstrip()[:200], flush=True)+res = []+for pr in prompts:+ ids = pr["ids"]+ mx = min(pr["max_tokens"], max(len(pr["expect_ids"]) + 4, 8))+ p.stdin.write("GEN %d %s\n" % (mx, ",".join(map(str, ids))))+ p.stdin.flush()+ gen, done = [], None+ while True:+ line = p.stdout.readline()+ if not line:+ break+ if line.startswith("T "):+ gen.append(int(line.split()[1]))+ elif line.startswith("DONE"):+ done = line.split()+ break+ exp = pr["expect_ids"]+ n = 0+ while n < min(len(gen), len(exp)) and gen[n] == exp[n]:+ n += 1+ full = n == len(exp) or (n == len(gen) and n >= len(exp) - 1)+ # DONE <generated> <prompt> <prompt ms> <decode ms> <stop|length|cancel> ...+ pms = float(done[3]) if done and len(done) > 4 else 0.0+ dms = float(done[4]) if done and len(done) > 4 else 0.0+ row = {"id": pr["id"], "mode": pr["mode"], "prompt_tokens": len(ids), "n_gen": len(gen), "n_expect": len(exp),+ "match_prefix": n, "first_div": None if n >= min(len(gen), len(exp)) else n, "prompt_ms": pms,+ "decode_ms": dms, "decode_tps": (len(gen) * 1000.0 / dms) if dms > 0 else 0.0,+ "prompt_tps": (len(ids) * 1000.0 / pms) if pms > 0 else 0.0, "gen": gen}+ res.append(row)+ print("%-26s %-8s prompt %4d (%.1f tok/s) gen %3d match %3d/%3d div %s decode %.2f tok/s" % (+ pr["id"], pr["mode"], len(ids), row["prompt_tps"], len(gen), n, len(exp), row["first_div"], row["decode_tps"]), flush=True)+ json.dump(res, open(out_path, "w"))+p.stdin.write("QUIT\n")+p.stdin.flush()+try:+ p.wait(timeout=60)+except Exception:+ p.terminate()