Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
tools/dflash2: q8test, the drafter with Q8_0 matrices keeps its acceptance
Niko12215f426c38cb4b
diff --git a/tools/dflash2/q8test.py b/tools/dflash2/q8test.pynew file mode 100644index 000000000..30f2a1545--- /dev/null+++ b/tools/dflash2/q8test.py@@ -0,0 +1,60 @@+"""Does a Q8_0 drafter (the matrices block-quantized like ggml Q8_0) keep the acceptance? Walk over two prompts, block 4 and 8."""+import json, sys, time+from pathlib import Path+import numpy as np+import torch+sys.path.insert(0, r"C:\dflash2-work")+import dflash2_ref as R++W = Path(r"C:\dflash2-work")+embed = R.load_dequant(str(W / "embd_iq4xs.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.IQ4_XS)+head = R.load_dequant(str(W / "head_q6k.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.Q6_K)+d = R.Drafter(str(W / "drafter.safetensors"), str(W / "drafter_config.json"), embed, head, threads=6)+d.enable_fast_head()+++def q8_0(w):+ """ggml Q8_0 along the last dimension: blocks of 32, scale = amax/127 stored as fp16, int8 codes"""+ shp = w.shape+ x = w.reshape(-1, 32)+ amax = x.abs().amax(dim=1, keepdim=True)+ sc = (amax / 127.0).to(torch.float16).float()+ sc_safe = torch.where(sc == 0, torch.ones_like(sc), sc)+ q = torch.round(x / sc_safe).clamp(-127, 127)+ return (q * sc).reshape(shp)+++def run(name, b, tag):+ seqj = json.load(open(W / "p" / ("%s.seq.json" % name)))+ seq = seqj["prompt"] + seqj["out"]+ layers, pos, tok, taps = R.read_taps(str(W / "taps" / ("%s.bin" % name)))+ keep = pos < len(seq)+ pos, taps = pos[keep], taps[keep]+ feats = np.zeros((int(pos.max()) + 1, taps.shape[1]), np.float32)+ for i in range(len(pos)):+ feats[pos[i]] = taps[i]+ feats[0] = feats[1]+ d.set_context(torch.from_numpy(feats))+ s = len(seqj["prompt"])+ cyc = []+ with torch.no_grad():+ while s + b <= len(seq):+ path, _ = d.draft(s, seq[s], b)+ truth = seq[s + 1:s + b]+ a = 0+ while a < b - 1 and path[a] == truth[a]:+ a += 1+ cyc.append(a + 1)+ s += a + 1+ print("%s %s block %d: %.3f tokens per round (%d rounds)" % (tag, name, b, np.mean(cyc), len(cyc)), flush=True)+++names = sys.argv[1].split(",")+orig = {k: v.clone() for k, v in d.w.items() if k.endswith("proj.weight") or k == "fc.weight" or k.endswith("kernel_projection.weight")}+for tag in ("fp32", "q8_0"):+ if tag == "q8_0":+ for k in orig:+ d.w[k] = q8_0(orig[k])+ for n in names:+ run(n, 4, tag)+ run(n, 8, tag)