Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

tools/dflash2: q8test, the drafter with Q8_0 matrices keeps its acceptance

Niko12215f426c38cb4b
diff --git a/tools/dflash2/q8test.py b/tools/dflash2/q8test.py
new file mode 100644
index 000000000..30f2a1545
--- /dev/null
+++ b/tools/dflash2/q8test.py
@@ -0,0 +1,60 @@
+"""Does a Q8_0 drafter (the matrices block-quantized like ggml Q8_0) keep the acceptance? Walk over two prompts, block 4 and 8."""
+import json, sys, time
+from pathlib import Path
+import numpy as np
+import torch
+sys.path.insert(0, r"C:\dflash2-work")
+import dflash2_ref as R
+
+W = Path(r"C:\dflash2-work")
+embed = R.load_dequant(str(W / "embd_iq4xs.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.IQ4_XS)
+head = R.load_dequant(str(W / "head_q6k.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.Q6_K)
+d = R.Drafter(str(W / "drafter.safetensors"), str(W / "drafter_config.json"), embed, head, threads=6)
+d.enable_fast_head()
+
+
+def q8_0(w):
+ """ggml Q8_0 along the last dimension: blocks of 32, scale = amax/127 stored as fp16, int8 codes"""
+ shp = w.shape
+ x = w.reshape(-1, 32)
+ amax = x.abs().amax(dim=1, keepdim=True)
+ sc = (amax / 127.0).to(torch.float16).float()
+ sc_safe = torch.where(sc == 0, torch.ones_like(sc), sc)
+ q = torch.round(x / sc_safe).clamp(-127, 127)
+ return (q * sc).reshape(shp)
+
+
+def run(name, b, tag):
+ seqj = json.load(open(W / "p" / ("%s.seq.json" % name)))
+ seq = seqj["prompt"] + seqj["out"]
+ layers, pos, tok, taps = R.read_taps(str(W / "taps" / ("%s.bin" % name)))
+ keep = pos < len(seq)
+ pos, taps = pos[keep], taps[keep]
+ feats = np.zeros((int(pos.max()) + 1, taps.shape[1]), np.float32)
+ for i in range(len(pos)):
+ feats[pos[i]] = taps[i]
+ feats[0] = feats[1]
+ d.set_context(torch.from_numpy(feats))
+ s = len(seqj["prompt"])
+ cyc = []
+ with torch.no_grad():
+ while s + b <= len(seq):
+ path, _ = d.draft(s, seq[s], b)
+ truth = seq[s + 1:s + b]
+ a = 0
+ while a < b - 1 and path[a] == truth[a]:
+ a += 1
+ cyc.append(a + 1)
+ s += a + 1
+ print("%s %s block %d: %.3f tokens per round (%d rounds)" % (tag, name, b, np.mean(cyc), len(cyc)), flush=True)
+
+
+names = sys.argv[1].split(",")
+orig = {k: v.clone() for k, v in d.w.items() if k.endswith("proj.weight") or k == "fc.weight" or k.endswith("kernel_projection.weight")}
+for tag in ("fp32", "q8_0"):
+ if tag == "q8_0":
+ for k in orig:
+ d.w[k] = q8_0(orig[k])
+ for n in names:
+ run(n, 4, tag)
+ run(n, 8, tag)