Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: tools to turn the llama.cpp reference's 10 greedy prompts into ids and compare the engine's greedy output (serve protocol)

Niko1221c3e9a998ad91
diff --git a/tools/dsv41/mkprompts.py b/tools/dsv41/mkprompts.py
new file mode 100644
index 000000000..bf467e86f
--- /dev/null
+++ b/tools/dsv41/mkprompts.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""mkprompts.py: the 10 greedy prompts of the llama.cpp reference run (runs/base1/answers_{chat,think}) as token ids.
+
+Renders each request with the GGUF's chat template (jinja2, thinking_mode chat | thinking), tokenizes with llama-tokenize
+(--no-bos (special tokens parse by default): the template writes the BOS text itself), and keeps the reference's greedy output ids.
+Writes OUT/prompts.json: [{"id", "mode", "ids": [...], "expect_ids": [...], "expect_text", "finish"}].
+"""
+import json, os, subprocess, sys, tempfile
+import jinja2
+from jinja2.sandbox import ImmutableSandboxedEnvironment
+
+ref_dir, template, shard, tokenize, out_dir = sys.argv[1:6]
+env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True)
+env.filters["tojson"] = lambda x, **k: json.dumps(x, ensure_ascii=False)
+def raise_exception(m):
+ raise jinja2.exceptions.TemplateError(m)
+env.globals["raise_exception"] = raise_exception
+tpl = env.from_string(open(template, encoding="utf-8").read())
+libs = os.environ.get("LD_LIBRARY_PATH", "")
+
+def tok(text):
+ with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False, encoding="utf-8") as f:
+ f.write(text)
+ p = f.name
+ r = subprocess.run([tokenize, "-m", shard, "-f", p, "--ids", "--no-bos", "--no-escape"],
+ capture_output=True, text=True, env=dict(os.environ, LD_LIBRARY_PATH=libs))
+ os.unlink(p)
+ line = [l for l in r.stdout.splitlines() if l.startswith("[")][-1]
+ return json.loads(line)
+
+out = []
+for mode_dir, mode in (("answers_chat", "chat"), ("answers_think", "thinking")):
+ d = os.path.join(ref_dir, mode_dir)
+ for fn in sorted(os.listdir(d)):
+ if fn == "summary.json" or not fn.endswith(".json"):
+ continue
+ r = json.load(open(os.path.join(d, fn), encoding="utf-8"))
+ b = r["request_body"]
+ text = tpl.render(messages=b["messages"], tools=b.get("tools"), thinking_mode=mode, reasoning_effort=None,
+ add_generation_prompt=True)
+ ids = tok(text)
+ ch = r["response"]["choices"][0]
+ out.append({"id": fn[:-5], "mode": mode, "ids": ids, "max_tokens": b.get("max_tokens", 256),
+ "expect_ids": [t["id"] for t in ch["logprobs"]["content"]],
+ "expect_text": ch["message"].get("content"), "finish": ch["finish_reason"]})
+ print(mode, fn[:-5], len(ids), "prompt tokens;", len(out[-1]["expect_ids"]), "expected")
+os.makedirs(out_dir, exist_ok=True)
+json.dump(out, open(os.path.join(out_dir, "prompts.json"), "w"))
diff --git a/tools/dsv41/serve_prompts.py b/tools/dsv41/serve_prompts.py
new file mode 100644
index 000000000..8f0b6e93d
--- /dev/null
+++ b/tools/dsv41/serve_prompts.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""serve_prompts.py PROMPTS.json OUT.json -- ENGINE CMD...
+Starts the engine in --serve mode (appended), waits for READY, sends GEN for every prompt (greedy), compares the
+generated ids with the llama.cpp reference's greedy ids (expect_ids), and prints one line each:
+ id mode prompt_tokens n_gen match_prefix/n_expect first_divergence prefill ms decode tok/s
+"""
+import json, subprocess, sys, time
+
+prompts_path, out_path = sys.argv[1], sys.argv[2]
+cmd = sys.argv[sys.argv.index("--") + 1:] + ["--serve"]
+prompts = json.load(open(prompts_path))
+only = [a for a in sys.argv[3:sys.argv.index("--")]]
+if only:
+ prompts = [p for p in prompts if p["id"] in only or (p["mode"] + ":" + p["id"]) in only]
+p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=sys.stderr, text=True, bufsize=1)
+t0 = time.time()
+while True:
+ line = p.stdout.readline()
+ if not line:
+ print("engine exited before READY", file=sys.stderr)
+ sys.exit(2)
+ if line.startswith("READY"):
+ print("READY after %.0f s: %s" % (time.time() - t0, line.strip()), flush=True)
+ break
+ print("engine>", line.rstrip()[:200], flush=True)
+res = []
+for pr in prompts:
+ ids = pr["ids"]
+ mx = min(pr["max_tokens"], max(len(pr["expect_ids"]) + 4, 8))
+ p.stdin.write("GEN %d %s\n" % (mx, ",".join(map(str, ids))))
+ p.stdin.flush()
+ gen, done = [], None
+ while True:
+ line = p.stdout.readline()
+ if not line:
+ break
+ if line.startswith("T "):
+ gen.append(int(line.split()[1]))
+ elif line.startswith("DONE"):
+ done = line.split()
+ break
+ exp = pr["expect_ids"]
+ n = 0
+ while n < min(len(gen), len(exp)) and gen[n] == exp[n]:
+ n += 1
+ full = n == len(exp) or (n == len(gen) and n >= len(exp) - 1)
+ # DONE <generated> <prompt> <prompt ms> <decode ms> <stop|length|cancel> ...
+ pms = float(done[3]) if done and len(done) > 4 else 0.0
+ dms = float(done[4]) if done and len(done) > 4 else 0.0
+ row = {"id": pr["id"], "mode": pr["mode"], "prompt_tokens": len(ids), "n_gen": len(gen), "n_expect": len(exp),
+ "match_prefix": n, "first_div": None if n >= min(len(gen), len(exp)) else n, "prompt_ms": pms,
+ "decode_ms": dms, "decode_tps": (len(gen) * 1000.0 / dms) if dms > 0 else 0.0,
+ "prompt_tps": (len(ids) * 1000.0 / pms) if pms > 0 else 0.0, "gen": gen}
+ res.append(row)
+ print("%-26s %-8s prompt %4d (%.1f tok/s) gen %3d match %3d/%3d div %s decode %.2f tok/s" % (
+ pr["id"], pr["mode"], len(ids), row["prompt_tps"], len(gen), n, len(exp), row["first_div"], row["decode_tps"]), flush=True)
+ json.dump(res, open(out_path, "w"))
+p.stdin.write("QUIT\n")
+p.stdin.flush()
+try:
+ p.wait(timeout=60)
+except Exception:
+ p.terminate()