Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

tools/dflash2: the torch port of the DFlash2 drafter and the scripts of the acceptance experiment (research tooling)

Niko1221128223891e3a
diff --git a/tools/dflash2/README.md b/tools/dflash2/README.md
new file mode 100644
index 000000000..ff5c71558
--- /dev/null
+++ b/tools/dflash2/README.md
@@ -0,0 +1,20 @@
+# DFlash2 acceptance experiment (research tooling, not shipped)
+
+Measures how many tokens per round the incoai DFlash2 drafter for Qwen3.8-Flash-Next would get on OUR quantized target, offline, from the hidden states
+the engine dumps. Results and method: phaseA-tests/notes-dflash2.md. All scripts keep the paths of the run (C:\dflash2-work on the owner PC, C:\strata-test\dflash2
+on the test box): edit them before reuse.
+
+Engine side (both off unless the variable is set):
+
+- `STRATA_TAP_FILE=<file>` (+ `STRATA_TAP_LAYERS`, default `5,15,25,35,45`): `Verifier::record_window` copies each verified row's completed layer output (the mean of
+ the 4 hyper-connection streams, the quantity the drafter was trained on) and `Verifier::run` appends `pos, token, taps` records.
+- `STRATA_SPEC_T_CYCLE=1,2,...,8` (CLI bench only): the verify window size cycles; the report line `window cost (ms/round)` gives the cost of each width.
+
+Scripts:
+
+- `mk_prompts.py` builds chat-template prompt ids with the server's own tokenizer and template.
+- `orch.py <prompt> <max_new> [gen,genf,genfm,tap]` drives `run_one.ps1` on the test box: free generation (ids, MTP statistics), the same text with MTP only, and the
+ teacher-forced tap dump with oracle drafts (`--spec 8 --spec-oracle --spec-follow`, a 2-token prompt: a native pack never starts the speculative loop from a 1-token prompt).
+- `dflash2_ref.py` is a plain-torch port of z-lab's `DFlash2DraftModel` (checked against it by `test_port.py`), with the target's embedding and head dequantized from the GGUF.
+- `run_eval.py` (walk, a few block sizes), `run_all2.py` (every position, resumable: per-anchor correctness and the selector's confidences), `analyze2.py` and
+ `final_tables.py` (policies, speed estimates from the measured window costs).
diff --git a/tools/dflash2/analyze2.py b/tools/dflash2/analyze2.py
new file mode 100644
index 000000000..7db05068d
--- /dev/null
+++ b/tools/dflash2/analyze2.py
@@ -0,0 +1,73 @@
+import json
+import numpy as np
+
+TV = {1: 39.96, 2: 43.21, 3: 48.39, 4: 53.68, 5: 59.40, 6: 66.00, 7: 74.66, 8: 80.34} # ms per verify window, gfx1151 IQ3_S (STRATA_SPEC_T_CYCLE run)
+HOST = 0.07
+PROD = {"chat_explain": (2.39, 43.33, 2.38, 43.97), "code_py": (2.27, 43.32, 2.22, 42.72), "code_cpp": (2.11, 40.60, 2.05, 39.93),
+ "math": (2.99, 52.38, 2.95, 52.29), "summarize": (2.81, 48.25, 2.75, 48.06), "json_extract": (3.25, 53.91, 3.22, 54.22),
+ "creative": (2.30, 42.77, 2.29, 42.65), "chat_advice": (1.94, 38.59, 1.92, 38.47)} # default tau, default tok/s, MTP-only tau, MTP-only tok/s
+names = list(PROD)
+A8 = json.load(open(r"C:\dflash2-work\anchors1_b8.json"))
+A4 = json.load(open(r"C:\dflash2-work\anchors1_b4.json"))
+SEQ = {n: json.load(open(r"C:\dflash2-work\p\%s.seq.json" % n)) for n in names}
+
+
+def walk(rows, T_of, D, n):
+ by = {r["s"]: r for r in rows}
+ s = len(SEQ[n]["prompt"])
+ tok = 0
+ ms = 0.0
+ rounds = 0
+ hist = np.zeros(10)
+ while s in by:
+ r = by[s]
+ T = T_of(r)
+ a = 0
+ while a < T - 1 and a < len(r["ok"]) and r["ok"][a]:
+ a += 1
+ tok += a + 1
+ ms += TV[T] + D + HOST
+ rounds += 1
+ hist[T] += 1
+ s += a + 1
+ return tok / rounds, ms / rounds, 1000.0 * tok / ms, hist
+
+
+def conf_T(theta, cap=8):
+ def f(r):
+ T = 1
+ for p in r["p"][:cap - 1]:
+ if p >= theta:
+ T += 1
+ else:
+ break
+ return T
+ return f
+
+
+print("check, fixed b=8 walk tau vs the earlier walk: ", {n: round(walk(A8[n], lambda r: 8, 8, n)[0], 3) for n in names})
+print("check, fixed b=4 drafter walk tau: ", {n: round(walk(A4[n], lambda r: 4, 8, n)[0], 3) for n in names})
+out = {}
+for D in (5.0, 8.0, 12.0):
+ print("\n=== drafter cost D = %.0f ms per round (verify cost + D + 0.07) ===" % D)
+ cfgs = [("fixed4 block-4 drafter", A4, lambda r: 4), ("fixed3 block-8 drafter", A8, lambda r: 3),
+ ("fixed4 block-8 drafter", A8, lambda r: 4), ("fixed5", A8, lambda r: 5), ("fixed6", A8, lambda r: 6),
+ ("fixed7", A8, lambda r: 7), ("fixed8", A8, lambda r: 8)]
+ for th in (0.3, 0.4, 0.5, 0.6, 0.7, 0.8):
+ for cap in (4, 5, 6, 8):
+ cfgs.append(("conf>=%.1f cap %d" % (th, cap), A8, conf_T(th, cap)))
+ print("%-26s" % "policy" + "".join("%15s" % n[:13] for n in names) + " mean-of-ratios")
+ best = None
+ for label, A, f in cfgs:
+ ratios, row = [], "%-26s" % label
+ for n in names:
+ tau, ms, tps, hist = walk(A[n], f, D, n)
+ ratios.append(tps / PROD[n][1])
+ row += "%15s" % ("%.2f/%.1f(%+.0f%%)" % (tau, tps, 100 * (tps / PROD[n][1] - 1)))
+ m = 100 * (np.mean(ratios) - 1)
+ if D == 8.0 or label.startswith("fixed") or True:
+ print(row + " %+6.1f%%" % m)
+ if best is None or m > best[0]:
+ best = (m, label)
+ out[(D, label)] = m
+ print("best:", best)
diff --git a/tools/dflash2/dflash2_ref.py b/tools/dflash2/dflash2_ref.py
new file mode 100644
index 000000000..6702a8c61
--- /dev/null
+++ b/tools/dflash2/dflash2_ref.py
@@ -0,0 +1,201 @@
+"""A plain-torch port of the DFlash2 drafter (z-lab/dflash model.py, DFlash2DraftModel) for the offline acceptance experiment.
+
+Everything is fp32 on the CPU. The target's embedding table and LM head are the dequantized tensors of OUR GGUF (IQ4_XS embedding,
+Q6_K head), the context features are the hidden states dumped by the engine's STRATA_TAP_FILE hook.
+"""
+import json, math, struct, sys
+import numpy as np
+import torch
+import torch.nn.functional as F
+from safetensors import safe_open
+
+sys.path.insert(0, r"C:\Users\AI-Server\Desktop\Strata\Public\Engine\third_party\llama.cpp\gguf-py")
+import gguf
+from gguf.quants import dequantize
+
+
+def rms(x, w, eps=1e-6):
+ v = x.pow(2).mean(-1, keepdim=True)
+ return (x * torch.rsqrt(v + eps)) * w
+
+
+def rotate_half(x):
+ h = x.shape[-1] // 2
+ return torch.cat((-x[..., h:], x[..., :h]), dim=-1)
+
+
+def load_dequant(path, rows, cols, qtype):
+ raw = np.fromfile(path, dtype=np.uint8)
+ bs, ts = gguf.GGML_QUANT_SIZES[qtype]
+ raw = raw.reshape(rows, cols // bs * ts)
+ return torch.from_numpy(dequantize(raw, qtype).astype(np.float32).reshape(rows, cols))
+
+
+class Drafter:
+ def __init__(self, st_path, cfg_path, embed, head, threads=6):
+ torch.set_num_threads(threads)
+ cfg = json.load(open(cfg_path))
+ dc = cfg["dflash_config"]
+ self.H = cfg["hidden_size"]
+ self.L = cfg["num_hidden_layers"]
+ self.NH = cfg["num_attention_heads"]
+ self.NKV = cfg["num_key_value_heads"]
+ self.HD = cfg["head_dim"]
+ self.win = cfg["sliding_window"]
+ self.theta = cfg["rope_parameters"]["rope_theta"]
+ self.eps = cfg["rms_norm_eps"]
+ self.gs = dc["conv_group_size"]
+ self.ks = dc["conv_kernel_size"]
+ self.mask_id = dc["mask_token_id"]
+ self.top_k = dc["selector_top_k"]
+ self.taps = dc["target_layer_ids"]
+ self.embed = embed
+ self.head = head
+ self.w = {}
+ with safe_open(st_path, "pt") as f:
+ for k in f.keys():
+ t = f.get_tensor(k)
+ # the two codebooks stay bf16 (only gathered rows are used); everything else fp32
+ self.w[k] = t if "codebook" in k else t.float()
+ self.inv_freq = 1.0 / (self.theta ** (torch.arange(0, self.HD, 2, dtype=torch.float32) / self.HD))
+ self.Kc = None
+ self.fast_head = False
+ self.head_bf16 = None
+
+ def enable_fast_head(self):
+ self.head_bf16 = self.head.to(torch.bfloat16)
+ self.fast_head = True
+
+ def cos_sin(self, pos):
+ fr = pos.float()[:, None] * self.inv_freq[None, :]
+ emb = torch.cat([fr, fr], dim=-1)
+ return emb.cos(), emb.sin()
+
+ # ---- context: fc + hidden_norm over the tapped features, then every layer's K (norm + rope) and V
+ def set_context(self, feats):
+ """feats: [n, taps*H] float32, row j = the features of position j."""
+ w = self.w
+ h = rms(feats @ w["fc.weight"].T, w["hidden_norm.weight"], self.eps)
+ n = h.shape[0]
+ cos, sin = self.cos_sin(torch.arange(n))
+ self.Kc, self.Vc = [], []
+ for l in range(self.L):
+ p = "layers.%d.self_attn." % l
+ k = (h @ w[p + "k_proj.weight"].T).view(n, self.NKV, self.HD)
+ v = (h @ w[p + "v_proj.weight"].T).view(n, self.NKV, self.HD)
+ k = rms(k, w[p + "k_norm.weight"], self.eps)
+ k = k * cos[:, None, :] + rotate_half(k) * sin[:, None, :]
+ self.Kc.append(k.permute(1, 0, 2).contiguous()) # [NKV, n, HD]
+ self.Vc.append(v.permute(1, 0, 2).contiguous())
+
+ def conv(self, x, dyn, base):
+ """_grouped_dynamic_convolve over a block x [b, H]; dyn [b, ks, groups]; base [ks, H]."""
+ b, H = x.shape
+ groups = H // self.gs
+ blocks = x.view(b, groups, self.gs)
+ out = torch.zeros_like(blocks)
+ for o in range(base.shape[0]):
+ vals = blocks if o == 0 else F.pad(blocks[:-o], (0, 0, 0, 0, o, 0))
+ kern = base[o].float().view(1, groups, self.gs)
+ out = out + kern * vals
+ out = out + dyn[:, o, :, None] * vals
+ return out.view(b, H)
+
+ def prepare(self, x, p):
+ w = self.w
+ groups = self.H // self.gs
+ dyn = (x @ w[p + "kernel_projection.weight"].T).view(x.shape[0], 2, self.ks, groups)
+ return self.conv(x, dyn[:, 0], w[p + "base_kernel"][0]), dyn[:, 1]
+
+ def finish(self, x, dyn, p):
+ return self.conv(x, dyn, self.w[p + "base_kernel"][1])
+
+ # ---- one block: anchor token at position s (s >= 1), block size b; returns the drafted path (b-1 tokens), the candidate lists
+ def draft(self, s, anchor, b, greedy=True):
+ w = self.w
+ H, NH, NKV, HD = self.H, self.NH, self.NKV, self.HD
+ ids = torch.tensor([anchor] + [self.mask_id] * (b - 1))
+ h = self.embed[ids] # [b, H]
+ pos = torch.arange(s, s + b)
+ cos, sin = self.cos_sin(pos)
+ # keys: context positions max(0, s-win+1) .. s-1 (a block row i sees ctx j when s+i-j < win), then the block
+ lo = max(0, s - self.win + 1)
+ for l in range(self.L):
+ p = "layers.%d." % l
+ r = h
+ x = rms(h, w[p + "input_layernorm.weight"], self.eps)
+ x, kdyn = self.prepare(x, p + "attention_conv.")
+ q = (x @ w[p + "self_attn.q_proj.weight"].T).view(b, NH, HD)
+ q = rms(q, w[p + "self_attn.q_norm.weight"], self.eps)
+ kn = rms((x @ w[p + "self_attn.k_proj.weight"].T).view(b, NKV, HD), w[p + "self_attn.k_norm.weight"], self.eps)
+ vn = (x @ w[p + "self_attn.v_proj.weight"].T).view(b, NKV, HD)
+ q = q * cos[:, None, :] + rotate_half(q) * sin[:, None, :]
+ kn = kn * cos[:, None, :] + rotate_half(kn) * sin[:, None, :]
+ Kc = self.Kc[l][:, lo:s, :] # [NKV, c, HD]
+ Vc = self.Vc[l][:, lo:s, :]
+ K = torch.cat([Kc, kn.permute(1, 0, 2)], dim=1) # [NKV, c+b, HD]
+ V = torch.cat([Vc, vn.permute(1, 0, 2)], dim=1)
+ c = Kc.shape[1]
+ kpos = torch.cat([torch.arange(lo, s), pos])
+ vis = (pos[:, None] - kpos[None, :]).abs() < self.win # [b, c+b]
+ rep = NH // NKV
+ Kx = K.repeat_interleave(rep, dim=0) # [NH, c+b, HD]
+ Vx = V.repeat_interleave(rep, dim=0)
+ sc = torch.einsum("bhd,hkd->hbk", q, Kx) * (HD ** -0.5)
+ sc = sc.masked_fill(~vis[None], float("-inf"))
+ att = torch.softmax(sc, dim=-1)
+ o = torch.einsum("hbk,hkd->bhd", att, Vx).reshape(b, NH * HD)
+ o = o @ w[p + "self_attn.o_proj.weight"].T
+ o = self.finish(o, kdyn, p + "attention_conv.")
+ h = r + o
+ r = h
+ x = rms(h, w[p + "post_attention_layernorm.weight"], self.eps)
+ x, mdyn = self.prepare(x, p + "mlp_conv.")
+ m = (F.silu(x @ w[p + "mlp.gate_proj.weight"].T) * (x @ w[p + "mlp.up_proj.weight"].T)) @ w[p + "mlp.down_proj.weight"].T
+ m = self.finish(m, mdyn, p + "mlp_conv.")
+ h = r + m
+ self.last_all = rms(h, w["norm.weight"], self.eps)
+ out = self.last_all[1:] # rows 1..b-1
+ if self.fast_head:
+ # candidates from a bf16 pass over the table (7x faster on this CPU), then the exact fp32 logits of the 32 best decide the top 16
+ lb = (out.to(torch.bfloat16) @ self.head_bf16.T)
+ c32 = torch.topk(lb, 32, dim=-1, sorted=False).indices # [b-1, 32]
+ ex = torch.einsum("rkh,rh->rk", self.head[c32], out) # [b-1, 32] exact
+ unary, j16 = torch.topk(ex, self.top_k, dim=-1, sorted=False)
+ cand = c32.gather(-1, j16)
+ logits = None
+ else:
+ logits = out @ self.head.T # [b-1, V]
+ unary, cand = torch.topk(logits, self.top_k, dim=-1, sorted=False)
+ hp = out @ w["candidate_selector.hidden_projection.weight"].T # [b-1, rank]
+ A = w["candidate_selector.predecessor_codebook"]
+ B = w["candidate_selector.successor_codebook"]
+ pred = anchor
+ path, probs = [], []
+ for i in range(b - 1):
+ sc = unary[i] + (B[cand[i]].float() @ (A[pred].float() * hp[i]))
+ j = int(torch.argmax(sc))
+ probs.append(float(torch.softmax(sc, dim=-1)[j])) # the selector's own probability of the pick (a confidence)
+ pred = int(cand[i][j])
+ path.append(pred)
+ top1 = (cand.gather(-1, unary.argmax(-1, keepdim=True))[:, 0] if logits is None else logits.argmax(-1)).tolist()
+ self.last_probs = probs
+ return path, top1
+
+
+def read_taps(path):
+ f = open(path, "rb")
+ magic, ver, N, nt = struct.unpack("<4i", f.read(16))
+ assert magic == 0x50415453 and ver == 1, (hex(magic), ver)
+ layers = struct.unpack("<%di" % nt, f.read(4 * nt))
+ rec = 8 + 4 * nt * N
+ data = f.read()
+ n = len(data) // rec
+ pos = np.empty(n, np.int32)
+ tok = np.empty(n, np.int32)
+ taps = np.empty((n, nt * N), np.float32)
+ for i in range(n):
+ o = i * rec
+ pos[i], tok[i] = struct.unpack_from("<2i", data, o)
+ taps[i] = np.frombuffer(data, np.float32, nt * N, o + 8)
+ return layers, pos, tok, taps
diff --git a/tools/dflash2/final_tables.py b/tools/dflash2/final_tables.py
new file mode 100644
index 000000000..9caa872bd
--- /dev/null
+++ b/tools/dflash2/final_tables.py
@@ -0,0 +1,61 @@
+import json
+import numpy as np
+import sys
+sys.path.insert(0, r"C:\dflash2-work")
+exec(open(r"C:\dflash2-work\analyze2.py").read().split("print(\"check")[0]) # TV, PROD, A8, A4, walk, conf_T, names
+
+fixed8 = {n: walk(A8[n], lambda r: 8, 8, n)[0] for n in names}
+fixed4 = {n: walk(A4[n], lambda r: 4, 8, n)[0] for n in names}
+fixed5 = {n: walk(A8[n], lambda r: 5, 8, n)[0] for n in names}
+print("| prompt | MTP+lookup (default) tokens/round, tok/s | MTP only tokens/round, tok/s | DFlash2 block 4 | block 8 |")
+print("| --- | --- | --- | --- | --- |")
+for n in names:
+ p = PROD[n]
+ print("| %s | %.2f, %.1f | %.2f, %.1f | %.2f | %.2f |" % (n, p[0], p[1], p[2], p[3], fixed4[n], fixed8[n]))
+print("| mean | %.2f, %.1f | %.2f, %.1f | %.2f | %.2f |" % (np.mean([PROD[n][0] for n in names]), np.mean([PROD[n][1] for n in names]),
+ np.mean([PROD[n][2] for n in names]), np.mean([PROD[n][3] for n in names]), np.mean(list(fixed4.values())), np.mean(list(fixed8.values()))))
+print("gain of block-4 tokens/round over MTP default: mean of ratios %+.1f%%, per prompt %s" % (
+ 100 * (np.mean([fixed4[n] / PROD[n][0] for n in names]) - 1), {n: round(100 * (fixed4[n] / PROD[n][0] - 1)) for n in names}))
+print("gain of block-8 tokens/round over MTP default: mean of ratios %+.1f%%" % (100 * (np.mean([fixed8[n] / PROD[n][0] for n in names]) - 1)))
+
+# conditional acceptance by draft position (block 8, all anchors, conditional on all earlier correct)
+cond = np.zeros(7); cn = np.zeros(7)
+for n in names:
+ for r in A8[n]:
+ for i in range(7):
+ cn[i] += 1
+ if r["ok"][i]:
+ cond[i] += 1
+ else:
+ break
+print("block-8 conditional acceptance by position (all prompts):", (cond / cn).round(3).tolist())
+cond4 = np.zeros(3); cn4 = np.zeros(3)
+for n in names:
+ for r in A4[n]:
+ for i in range(3):
+ cn4[i] += 1
+ if r["ok"][i]:
+ cond4[i] += 1
+ else:
+ break
+print("block-4 conditional acceptance by position (all prompts):", (cond4 / cn4).round(3).tolist())
+
+print("\nspeed estimate table (tok/s), D = drafter cost per round")
+pol = [("fixed 4 (block-4 drafter)", A4, lambda r: 4), ("fixed 5 (block-8 drafter)", A8, lambda r: 5), ("confidence>=0.5, up to 8", A8, conf_T(0.5, 8)),
+ ("confidence>=0.4, up to 6", A8, conf_T(0.4, 6))]
+for label, A, f in pol:
+ for D in (5.0, 8.0, 12.0):
+ r_ = []
+ tps_ = []
+ for n in names:
+ tau, ms, tps, hist = walk(A[n], f, D, n)
+ r_.append(tps / PROD[n][1])
+ tps_.append(tps)
+ print("%-28s D=%4.0f ms: mean %.1f tok/s (default %.1f): mean-of-ratios %+.1f%%, best %+.0f%% (%s), worst %+.0f%% (%s)" % (
+ label, D, np.mean(tps_), np.mean([PROD[n][1] for n in names]), 100 * (np.mean(r_) - 1),
+ 100 * (max(r_) - 1), names[int(np.argmax(r_))], 100 * (min(r_) - 1), names[int(np.argmin(r_))]))
+# window histogram of the confidence policy
+print()
+for n in names:
+ tau, ms, tps, hist = walk(A8[n], conf_T(0.5, 8), 8.0, n)
+ print(n, "conf>=0.5 cap 8: windows", {int(i): int(h) for i, h in enumerate(hist) if h}, "tau %.2f" % tau)
diff --git a/tools/dflash2/mk_prompts.py b/tools/dflash2/mk_prompts.py
new file mode 100644
index 000000000..78910f51e
--- /dev/null
+++ b/tools/dflash2/mk_prompts.py
@@ -0,0 +1,40 @@
+"""Builds the chat-templated prompt ids for the DFlash2 acceptance experiment (same tokenizer and template the server uses)."""
+import json, sys
+from pathlib import Path
+
+REPO = Path(r"C:\Users\AI-Server\Desktop\Strata\GitHub\wt-022-dflash2")
+sys.path.insert(0, str(REPO / "tools"))
+sys.path.insert(0, str(REPO))
+import strata_tokenizer as ST
+from serve.frontend import ChatTemplate
+
+tp = Path(r"C:\dflash2-work\tok")
+vocab = json.loads((tp / "vocab.json").read_text(encoding="utf-8"))
+tokens = [None] * len(vocab)
+for t, i in vocab.items():
+ tokens[i] = t
+merges = (tp / "merges.txt").read_text(encoding="utf-8").split("\n")
+types = json.loads((tp / "token_type.json").read_text())
+tok = ST.Tokenizer(tokens, merges, types)
+tpl = ChatTemplate(tp / "chat_template.jinja")
+
+readme = (REPO / "README.md").read_text(encoding="utf-8")[:2600]
+
+PROMPTS = {
+ "chat_explain": "Explain how a hash map works and why lookups are O(1) on average. Then say when that guarantee breaks down. Keep it clear for a junior developer.",
+ "code_py": "Write a Python function that reads a web server log file, parses the timestamps (ISO-8601) and returns the five busiest one-minute windows with their request counts. Include a small test with pytest.",
+ "code_cpp": "Implement an LRU cache in C++17 with O(1) get and put using std::list and std::unordered_map. Show the class and a short main() that exercises it.",
+ "math": "A train leaves city A at 9:00 traveling at 80 km/h. A second train leaves city B, 450 km away, at 10:30 traveling toward A at 95 km/h. At what time do they meet, and how far from city A? Work it out step by step and check the result.",
+ "summarize": "Summarize the following text in exactly five bullet points, then give a one-sentence takeaway.\n\n" + readme,
+ "json_extract": "Extract the data below as JSON with the keys name, role, city, start_date (ISO format) and skills (a list). Output only the JSON.\n\nMaria Gonzalez joined the company on the 3rd of March 2021 as a senior data engineer. She works from the Lisbon office and her main tools are Spark, dbt, Airflow and a bit of Rust. Her colleague Tom Becker, a product designer from Vienna, started a year later, on 14 June 2022, and is good at Figma, user research and prototyping.",
+ "creative": "Write a short story (about 400 words) about a lighthouse keeper who discovers that the light has been answering messages from the sea. Give it a quiet, slightly eerie tone.",
+ "chat_advice": "I have a 12 GB graphics card and 64 GB of RAM. What are the trade-offs between running a large mixture-of-experts model with most of its experts in RAM versus a smaller dense model fully on the GPU? Give me a recommendation for daily coding help.",
+}
+
+out = {}
+for name, text in PROMPTS.items():
+ s = tpl.render([{"role": "user", "content": text}])
+ ids = tok.encode(s, parse_special=True)
+ out[name] = {"ids": ids, "text": s}
+ print(name, len(ids), repr(s[-60:]))
+json.dump(out, open(r"C:\dflash2-work\prompts.json", "w"))
diff --git a/tools/dflash2/orch.py b/tools/dflash2/orch.py
new file mode 100644
index 000000000..fda18f596
--- /dev/null
+++ b/tools/dflash2/orch.py
@@ -0,0 +1,67 @@
+"""Runs the engine side of the DFlash2 experiment for one prompt on strix (foreground; one engine at a time)."""
+import json, re, subprocess, sys, time
+from pathlib import Path
+
+W = Path(r"C:\dflash2-work")
+RW = "C:\\strata-test\\dflash2"
+
+
+def ssh(cmd, timeout=1500):
+ r = subprocess.run(["ssh", "-o", "ServerAliveInterval=30", "strix", cmd], capture_output=True, text=True, timeout=timeout)
+ out = "\n".join(l for l in (r.stdout + r.stderr).splitlines() if not l.startswith("**"))
+ return out
+
+
+def scp_up(local, remote):
+ subprocess.run(["scp", "-q", str(local), "strix:" + remote.replace("\\", "/")], check=True, capture_output=True)
+
+
+def scp_down(remote, local):
+ subprocess.run(["scp", "-q", "strix:" + remote.replace("\\", "/"), str(local)], check=True, capture_output=True)
+
+
+def run_mode(mode, name, extra=""):
+ t = time.time()
+ out = ssh('powershell -NoProfile -ExecutionPolicy Bypass -File %s\\run_one.ps1 -Mode %s -Name %s %s' % (RW, mode, name, extra))
+ print("[%s %s] %.0f s" % (mode, name, time.time() - t))
+ print(out)
+ (W / "logs").mkdir(exist_ok=True)
+ (W / "logs" / ("%s_%s.log" % (mode, name))).write_text(out)
+ scp_down("%s\\out\\%s_%s.txt" % (RW, mode, name), W / "logs" / ("%s_%s.txt" % (mode, name)))
+ scp_down("%s\\out\\%s_%s.err" % (RW, mode, name), W / "logs" / ("%s_%s.err" % (mode, name)))
+ return out
+
+
+def produced_ids(path):
+ txt = Path(path).read_text()
+ m = re.search(r"^output :(.*)$", txt, re.M)
+ return [int(x) for x in m.group(1).split()] if m else []
+
+
+def main(name, max_new, steps):
+ prompts = json.load(open(W / "prompts.json"))
+ pids = prompts[name]["ids"]
+ if "gen" in steps:
+ run_mode("gen", name, "-MaxNew %d" % max_new)
+ out = produced_ids(W / "logs" / ("gen_%s.txt" % name))
+ print(name, "prompt", len(pids), "generated", len(out), "tail", out[-3:])
+ seq = pids + out
+ (W / "p" / ("%s.out.ids" % name)).write_text(",".join(map(str, out)))
+ (W / "p" / ("%s.first.ids" % name)).write_text(",".join(map(str, seq[:2])))
+ (W / "p" / ("%s.rest.ids" % name)).write_text(",".join(map(str, seq[2:])))
+ json.dump({"prompt": pids, "out": out}, open(W / "p" / ("%s.seq.json" % name), "w"))
+ for k in ("out", "first", "rest"):
+ scp_up(W / "p" / ("%s.%s.ids" % (name, k)), "%s\\p\\%s.%s.ids" % (RW, name, k))
+ for mode in ("genf", "genfm", "tap"):
+ if mode in steps:
+ run_mode(mode, name)
+ if "tap" in steps:
+ (W / "taps").mkdir(exist_ok=True)
+ scp_down("%s\\out\\tap_%s.bin" % (RW, name), W / "taps" / ("%s.bin" % name))
+
+
+if __name__ == "__main__":
+ name = sys.argv[1]
+ max_new = int(sys.argv[2])
+ steps = sys.argv[3].split(",") if len(sys.argv) > 3 else ["gen", "genf", "genfm", "tap"]
+ main(name, max_new, steps)
diff --git a/tools/dflash2/run_all2.py b/tools/dflash2/run_all2.py
new file mode 100644
index 000000000..bc30dd72d
--- /dev/null
+++ b/tools/dflash2/run_all2.py
@@ -0,0 +1,53 @@
+"""Per-anchor drafter results at EVERY generated position (stride 1), resumable: the exact path and the selector's confidences."""
+import json, sys, time
+from pathlib import Path
+import numpy as np
+import torch
+sys.path.insert(0, r"C:\dflash2-work")
+import dflash2_ref as R
+
+W = Path(r"C:\dflash2-work")
+names = sys.argv[1].split(",")
+b = int(sys.argv[2])
+threads = int(sys.argv[3])
+budget = float(sys.argv[4]) # seconds this call may spend before it saves and stops
+t_start = time.time()
+embed = R.load_dequant(str(W / "embd_iq4xs.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.IQ4_XS)
+head = R.load_dequant(str(W / "head_q6k.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.Q6_K)
+d = R.Drafter(str(W / "drafter.safetensors"), str(W / "drafter_config.json"), embed, head, threads=threads)
+d.enable_fast_head()
+out_path = W / ("anchors1_b%d.json" % b)
+res = json.load(open(out_path)) if out_path.exists() else {}
+for name in names:
+ if res.get(name + "#done"):
+ continue
+ seqj = json.load(open(W / "p" / ("%s.seq.json" % name)))
+ seq = seqj["prompt"] + seqj["out"]
+ layers, pos, tok, taps = R.read_taps(str(W / "taps" / ("%s.bin" % name)))
+ keep = pos < len(seq)
+ pos, tok, taps = pos[keep], tok[keep], taps[keep]
+ n = int(pos.max()) + 1
+ feats = np.zeros((n, taps.shape[1]), np.float32)
+ for i in range(len(pos)):
+ feats[pos[i]] = taps[i]
+ feats[0] = feats[1]
+ d.set_context(torch.from_numpy(feats))
+ rows = res.get(name, [])
+ s0 = rows[-1]["s"] + 1 if rows else len(seqj["prompt"])
+ t1 = time.time()
+ finished = True
+ with torch.no_grad():
+ for s in range(s0, len(seq) - b):
+ path, top1 = d.draft(s, seq[s], b)
+ truth = seq[s + 1:s + b]
+ rows.append({"s": s, "ok": [int(path[i] == truth[i]) for i in range(b - 1)], "p": [round(x, 4) for x in d.last_probs]})
+ if time.time() - t_start > budget:
+ finished = False
+ break
+ res[name] = rows
+ if finished:
+ res[name + "#done"] = True
+ json.dump(res, open(out_path, "w"))
+ print(name, b, len(rows), "anchors so far, %.0f s" % (time.time() - t1), "done" if finished else "PAUSED", flush=True)
+ if not finished:
+ break
diff --git a/tools/dflash2/run_eval.py b/tools/dflash2/run_eval.py
new file mode 100644
index 000000000..77056a023
--- /dev/null
+++ b/tools/dflash2/run_eval.py
@@ -0,0 +1,70 @@
+"""Offline DFlash2 acceptance on OUR quantized target: walks each prompt's dumped sequence the way the speculative loop would
+(anchor, draft block, accept the leading matches of the drafted path against the engine's own greedy tokens, move on)."""
+import json, sys, time
+from pathlib import Path
+import numpy as np
+import torch
+sys.path.insert(0, r"C:\dflash2-work")
+import dflash2_ref as R
+
+W = Path(r"C:\dflash2-work")
+names = sys.argv[1].split(",")
+blocks = [int(x) for x in sys.argv[2].split(",")] if len(sys.argv) > 2 else [4, 8]
+threads = int(sys.argv[3]) if len(sys.argv) > 3 else 6
+
+t0 = time.time()
+embed = R.load_dequant(str(W / "embd_iq4xs.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.IQ4_XS)
+head = R.load_dequant(str(W / "head_q6k.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.Q6_K)
+print("dequantized embed/head in %.0f s" % (time.time() - t0), flush=True)
+d = R.Drafter(str(W / "drafter.safetensors"), str(W / "drafter_config.json"), embed, head, threads=threads)
+res_path = W / "eval_results.json"
+results = json.load(open(res_path)) if res_path.exists() else {}
+
+for name in names:
+ seqj = json.load(open(W / "p" / ("%s.seq.json" % name)))
+ prompt, out = seqj["prompt"], seqj["out"]
+ seq = prompt + out
+ layers, pos, tok, taps = R.read_taps(str(W / "taps" / ("%s.bin" % name)))
+ keep = pos < len(seq) # the tail of the last window is padding (zeros) past the sequence
+ pos, tok, taps = pos[keep], tok[keep], taps[keep]
+ n = int(pos.max()) + 1
+ feats = np.zeros((n, taps.shape[1]), np.float32)
+ seen = np.zeros(n, bool)
+ for i in range(len(pos)): # the last record of a position wins
+ feats[pos[i]] = taps[i]
+ seen[pos[i]] = True
+ assert tok[i] == seq[pos[i]], ("token mismatch at", pos[i], tok[i], seq[pos[i]])
+ if not seen[0] and seen[1]: # position 0 goes through the batched prefill (no tap): the next row stands in
+ feats[0] = feats[1]
+ seen[0] = True
+ assert seen.all(), "positions missing: %s" % np.where(~seen)[0][:5]
+ d.set_context(torch.from_numpy(feats))
+ print("%s: prompt %d, generated %d, tap records %d, layers %s" % (name, len(prompt), len(out), len(pos), layers), flush=True)
+ results.setdefault(name, {})
+ for b in blocks:
+ s = len(prompt) # the first generated token is the first anchor, as after a prefill
+ cyc, cond_ok, cond_n = [], np.zeros(b - 1), np.zeros(b - 1)
+ top1_ok = np.zeros(b - 1)
+ t1 = time.time()
+ with torch.no_grad():
+ while s + b <= len(seq) - 0 and s < n:
+ path, top1 = d.draft(s, seq[s], b)
+ truth = seq[s + 1:s + b]
+ a = 0
+ while a < b - 1 and path[a] == truth[a]:
+ a += 1
+ for i in range(b - 1): # conditional on all earlier correct
+ if i <= a:
+ cond_n[i] += 1
+ cond_ok[i] += (i < a)
+ if i <= a and top1[i] == truth[i]:
+ top1_ok[i] += 1
+ cyc.append(a + 1)
+ s += a + 1
+ tau = float(np.mean(cyc))
+ results[name]["b%d" % b] = {"anchors": len(cyc), "tokens_per_round": tau,
+ "cond_accept": (cond_ok / np.maximum(cond_n, 1)).round(3).tolist(),
+ "hist": np.bincount(cyc, minlength=b + 1).tolist()}
+ print(" block %d: %d rounds, %.3f tokens per round, cond %s (%.0f s)" %
+ (b, len(cyc), tau, (cond_ok / np.maximum(cond_n, 1)).round(2).tolist(), time.time() - t1), flush=True)
+ json.dump(results, open(res_path, "w"), indent=1)
diff --git a/tools/dflash2/run_one.ps1 b/tools/dflash2/run_one.ps1
new file mode 100644
index 000000000..dc0c38bcf
--- /dev/null
+++ b/tools/dflash2/run_one.ps1
@@ -0,0 +1,50 @@
+# DFlash2 experiment runner on strix. One engine process at a time; reads ids from C:\strata-test\dflash2\p\<name>.*.ids
+# usage: run_one.ps1 -Mode gen|genf|genfm|tap|tcycle -Name <prompt> [-MaxNew N] [-Cycle 1,2,...]
+param([string]$Mode, [string]$Name, [int]$MaxNew = 900, [string]$Cycle = "", [string]$Exe = "strata-hook.exe", [string]$Tag = "", [string]$Extra = "")
+$ErrorActionPreference = "Continue"
+$W = "C:\strata-test\dflash2"
+$D = "C:\Users\Administrator\Documents\Strata"
+$OWN = "C:\Users\Administrator\AppData\Local\Strata\terminal\0.1.46.1"
+$env:LOCALAPPDATA = "C:\strata-test\la"
+$env:APPDATA = "C:\strata-test\la"
+New-Item -ItemType Directory -Force -Path $env:LOCALAPPDATA | Out-Null
+New-Item -ItemType Directory -Force -Path "$W\out" | Out-Null
+$env:STRATA_ARENA_PIN_GIB = "none"
+$env:PATH = "$W\eng\rocm\bin;" + $env:PATH
+$mtp = $D + "\mtp\rt"
+$base = @("--pack", "$D\packs\iq3_s",
+ "--native", "$D\models\IQ3_S\Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf",
+ "--ple-gguf", "$D\models\IQ3_S\Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00002-of-00002.gguf",
+ "--expert-profile", "$OWN\data\expert-profile.bin",
+ "--expert-cache", "auto", "--prefill", "auto", "--max-context", "4096", "--kv", "int8", "--vram-reserve-mib", "700", "--greedy")
+$exe = "$W\eng\$Exe"
+$out = "$W\out\${Mode}_${Name}${Tag}.txt"
+$err = "$W\out\${Mode}_${Name}${Tag}.err"
+Remove-Item $out, $err -ErrorAction SilentlyContinue
+if ($Mode -eq "gen") {
+ $ids = (Get-Content "$W\p\$Name.prompt.ids" -Raw).Trim()
+ $a = $base + @("--spec", "4", "--spec-min-p", "0.5", "--mtp", $mtp, "--max-new", "$MaxNew", "--stop-eos", "--tokens", $ids)
+} elseif ($Mode -eq "genf" -or $Mode -eq "genfm") {
+ # the production flags on a fixed text: the drafts are judged against the continuation in <name>.out.ids
+ $ids = (Get-Content "$W\p\$Name.prompt.ids" -Raw).Trim()
+ $n = (Get-Content "$W\p\$Name.out.ids" -Raw).Split(",").Count
+ $a = $base + @("--spec", "4", "--spec-min-p", "0.5", "--mtp", $mtp, "--spec-follow", "$W\p\$Name.out.ids", "--max-new", "$n", "--tokens", $ids)
+ if ($Mode -eq "genfm") { $a += @("--suffix-draft", "0") }
+} elseif ($Mode -eq "tap") {
+ $env:STRATA_TAP_FILE = "$W\out\tap_$Name.bin"
+ Remove-Item $env:STRATA_TAP_FILE -ErrorAction SilentlyContinue
+ $first = (Get-Content "$W\p\$Name.first.ids" -Raw).Trim()
+ $n = (Get-Content "$W\p\$Name.rest.ids" -Raw).Split(",").Count
+ $a = $base + @("--spec", "8", "--spec-oracle", "$W\p\$Name.rest.ids", "--spec-follow", "$W\p\$Name.rest.ids", "--max-new", "$n", "--tokens", $first)
+} elseif ($Mode -eq "tcycle") {
+ $env:STRATA_SPEC_T_CYCLE = $Cycle
+ $first = (Get-Content "$W\p\$Name.first.ids" -Raw).Trim()
+ $a = $base + @("--spec", "8", "--spec-oracle", "$W\p\$Name.rest.ids", "--spec-follow", "$W\p\$Name.rest.ids", "--max-new", "$MaxNew", "--tokens", $first)
+} else { "bad mode"; exit 2 }
+if ($Extra -ne "") { $a += $Extra.Split("_") }
+Set-Location $OWN
+$t0 = Get-Date
+$p = Start-Process -FilePath $exe -ArgumentList $a -RedirectStandardOutput $out -RedirectStandardError $err -NoNewWindow -PassThru
+$p.WaitForExit()
+"EXIT " + $p.ExitCode + " after " + [int]((Get-Date) - $t0).TotalSeconds + " s"
+Get-Content $out | Select-String -Pattern "^decode|^speculation|^window|^accepted|^mtp|^suffix|^verify window" | ForEach-Object { $_.Line.Substring(0, [Math]::Min(300, $_.Line.Length)) }
diff --git a/tools/dflash2/test_port.py b/tools/dflash2/test_port.py
new file mode 100644
index 000000000..335363bda
--- /dev/null
+++ b/tools/dflash2/test_port.py
@@ -0,0 +1,45 @@
+import sys, importlib.util, json, torch
+sys.path.insert(0, r"C:\dflash2-work")
+import dflash2_ref as R
+torch.manual_seed(0)
+# the reference (z-lab) model on transformers 4.57
+spec = importlib.util.spec_from_file_location("ghm", r"C:\Users\AI-Server\Desktop\Strata\phaseA-tests\ornith_tmp\gh_model.py")
+ghm = importlib.util.module_from_spec(spec); spec.loader.exec_module(ghm)
+from transformers.models.qwen3.modeling_qwen3 import Qwen3Config
+cfgd = json.load(open(r"C:\dflash2-work\drafter_config.json"))
+c = Qwen3Config(**{k: v for k, v in cfgd.items() if k in ("hidden_size","intermediate_size","num_hidden_layers","num_attention_heads","num_key_value_heads","head_dim","vocab_size","rms_norm_eps","max_position_embeddings","attention_bias","sliding_window","layer_types","use_sliding_window","max_window_layers")})
+c.rope_theta = cfgd["rope_parameters"]["rope_theta"]
+c.dflash_config = cfgd["dflash_config"]
+c.is_causal = False
+c.num_target_layers = 48
+m = ghm.DFlash2DraftModel(c)
+from safetensors.torch import load_file
+sd = load_file(r"C:\dflash2-work\drafter.safetensors")
+sd = {(k + ".weight" if k.startswith("candidate_selector.") and k.endswith("codebook") else k): v for k, v in sd.items()}
+miss = m.load_state_dict(sd, strict=False); print("missing", miss.missing_keys[:5], "unexpected", miss.unexpected_keys[:5])
+m = m.float().eval()
+# my port with a tiny random embed/head
+V = 248320; H = 2560
+embed = torch.randn(V, H) * 0.02
+head = torch.randn(V, H) * 0.02
+d = R.Drafter(r"C:\dflash2-work\drafter.safetensors", r"C:\dflash2-work\drafter_config.json", embed, head)
+n = 300; b = 8; s = 250
+feats = torch.randn(n, 5 * H) * 2.0
+d.set_context(feats)
+anchor = 1234
+mine_path, _ = d.draft(s, anchor, b)
+ids = torch.tensor([[anchor] + [d.mask_id] * (b - 1)])
+noise = embed[ids]
+pos = torch.arange(0, s + b)[None] # context 0..s-1 then block
+with torch.no_grad():
+ out = m(position_ids=pos, noise_embedding=noise, target_hidden=feats[None, :s], past_key_values=None, use_cache=False)
+print("shape", out.shape, "mine", d.last_all.shape)
+err = (out[0] - d.last_all).abs().max().item()
+print("max abs err all rows", err, "ref scale", out.abs().mean().item())
+# the selector path against the reference
+logits = (out[:, 1:] @ head.T)
+class Tgt: pass
+with torch.no_grad():
+ path, idx, q = m.candidate_selector.select(out[:, 1:], logits, torch.tensor([anchor]), 0.0)
+print("ref path ", path[0].tolist())
+print("mine path", mine_path)