Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
tools/dflash2: the torch port of the DFlash2 drafter and the scripts of the acceptance experiment (research tooling)
Niko1221128223891e3a
diff --git a/tools/dflash2/README.md b/tools/dflash2/README.mdnew file mode 100644index 000000000..ff5c71558--- /dev/null+++ b/tools/dflash2/README.md@@ -0,0 +1,20 @@+# DFlash2 acceptance experiment (research tooling, not shipped)++Measures how many tokens per round the incoai DFlash2 drafter for Qwen3.8-Flash-Next would get on OUR quantized target, offline, from the hidden states+the engine dumps. Results and method: phaseA-tests/notes-dflash2.md. All scripts keep the paths of the run (C:\dflash2-work on the owner PC, C:\strata-test\dflash2+on the test box): edit them before reuse.++Engine side (both off unless the variable is set):++- `STRATA_TAP_FILE=<file>` (+ `STRATA_TAP_LAYERS`, default `5,15,25,35,45`): `Verifier::record_window` copies each verified row's completed layer output (the mean of+ the 4 hyper-connection streams, the quantity the drafter was trained on) and `Verifier::run` appends `pos, token, taps` records.+- `STRATA_SPEC_T_CYCLE=1,2,...,8` (CLI bench only): the verify window size cycles; the report line `window cost (ms/round)` gives the cost of each width.++Scripts:++- `mk_prompts.py` builds chat-template prompt ids with the server's own tokenizer and template.+- `orch.py <prompt> <max_new> [gen,genf,genfm,tap]` drives `run_one.ps1` on the test box: free generation (ids, MTP statistics), the same text with MTP only, and the+ teacher-forced tap dump with oracle drafts (`--spec 8 --spec-oracle --spec-follow`, a 2-token prompt: a native pack never starts the speculative loop from a 1-token prompt).+- `dflash2_ref.py` is a plain-torch port of z-lab's `DFlash2DraftModel` (checked against it by `test_port.py`), with the target's embedding and head dequantized from the GGUF.+- `run_eval.py` (walk, a few block sizes), `run_all2.py` (every position, resumable: per-anchor correctness and the selector's confidences), `analyze2.py` and+ `final_tables.py` (policies, speed estimates from the measured window costs).diff --git a/tools/dflash2/analyze2.py b/tools/dflash2/analyze2.pynew file mode 100644index 000000000..7db05068d--- /dev/null+++ b/tools/dflash2/analyze2.py@@ -0,0 +1,73 @@+import json+import numpy as np++TV = {1: 39.96, 2: 43.21, 3: 48.39, 4: 53.68, 5: 59.40, 6: 66.00, 7: 74.66, 8: 80.34} # ms per verify window, gfx1151 IQ3_S (STRATA_SPEC_T_CYCLE run)+HOST = 0.07+PROD = {"chat_explain": (2.39, 43.33, 2.38, 43.97), "code_py": (2.27, 43.32, 2.22, 42.72), "code_cpp": (2.11, 40.60, 2.05, 39.93),+ "math": (2.99, 52.38, 2.95, 52.29), "summarize": (2.81, 48.25, 2.75, 48.06), "json_extract": (3.25, 53.91, 3.22, 54.22),+ "creative": (2.30, 42.77, 2.29, 42.65), "chat_advice": (1.94, 38.59, 1.92, 38.47)} # default tau, default tok/s, MTP-only tau, MTP-only tok/s+names = list(PROD)+A8 = json.load(open(r"C:\dflash2-work\anchors1_b8.json"))+A4 = json.load(open(r"C:\dflash2-work\anchors1_b4.json"))+SEQ = {n: json.load(open(r"C:\dflash2-work\p\%s.seq.json" % n)) for n in names}+++def walk(rows, T_of, D, n):+ by = {r["s"]: r for r in rows}+ s = len(SEQ[n]["prompt"])+ tok = 0+ ms = 0.0+ rounds = 0+ hist = np.zeros(10)+ while s in by:+ r = by[s]+ T = T_of(r)+ a = 0+ while a < T - 1 and a < len(r["ok"]) and r["ok"][a]:+ a += 1+ tok += a + 1+ ms += TV[T] + D + HOST+ rounds += 1+ hist[T] += 1+ s += a + 1+ return tok / rounds, ms / rounds, 1000.0 * tok / ms, hist+++def conf_T(theta, cap=8):+ def f(r):+ T = 1+ for p in r["p"][:cap - 1]:+ if p >= theta:+ T += 1+ else:+ break+ return T+ return f+++print("check, fixed b=8 walk tau vs the earlier walk: ", {n: round(walk(A8[n], lambda r: 8, 8, n)[0], 3) for n in names})+print("check, fixed b=4 drafter walk tau: ", {n: round(walk(A4[n], lambda r: 4, 8, n)[0], 3) for n in names})+out = {}+for D in (5.0, 8.0, 12.0):+ print("\n=== drafter cost D = %.0f ms per round (verify cost + D + 0.07) ===" % D)+ cfgs = [("fixed4 block-4 drafter", A4, lambda r: 4), ("fixed3 block-8 drafter", A8, lambda r: 3),+ ("fixed4 block-8 drafter", A8, lambda r: 4), ("fixed5", A8, lambda r: 5), ("fixed6", A8, lambda r: 6),+ ("fixed7", A8, lambda r: 7), ("fixed8", A8, lambda r: 8)]+ for th in (0.3, 0.4, 0.5, 0.6, 0.7, 0.8):+ for cap in (4, 5, 6, 8):+ cfgs.append(("conf>=%.1f cap %d" % (th, cap), A8, conf_T(th, cap)))+ print("%-26s" % "policy" + "".join("%15s" % n[:13] for n in names) + " mean-of-ratios")+ best = None+ for label, A, f in cfgs:+ ratios, row = [], "%-26s" % label+ for n in names:+ tau, ms, tps, hist = walk(A[n], f, D, n)+ ratios.append(tps / PROD[n][1])+ row += "%15s" % ("%.2f/%.1f(%+.0f%%)" % (tau, tps, 100 * (tps / PROD[n][1] - 1)))+ m = 100 * (np.mean(ratios) - 1)+ if D == 8.0 or label.startswith("fixed") or True:+ print(row + " %+6.1f%%" % m)+ if best is None or m > best[0]:+ best = (m, label)+ out[(D, label)] = m+ print("best:", best)diff --git a/tools/dflash2/dflash2_ref.py b/tools/dflash2/dflash2_ref.pynew file mode 100644index 000000000..6702a8c61--- /dev/null+++ b/tools/dflash2/dflash2_ref.py@@ -0,0 +1,201 @@+"""A plain-torch port of the DFlash2 drafter (z-lab/dflash model.py, DFlash2DraftModel) for the offline acceptance experiment.++Everything is fp32 on the CPU. The target's embedding table and LM head are the dequantized tensors of OUR GGUF (IQ4_XS embedding,+Q6_K head), the context features are the hidden states dumped by the engine's STRATA_TAP_FILE hook.+"""+import json, math, struct, sys+import numpy as np+import torch+import torch.nn.functional as F+from safetensors import safe_open++sys.path.insert(0, r"C:\Users\AI-Server\Desktop\Strata\Public\Engine\third_party\llama.cpp\gguf-py")+import gguf+from gguf.quants import dequantize+++def rms(x, w, eps=1e-6):+ v = x.pow(2).mean(-1, keepdim=True)+ return (x * torch.rsqrt(v + eps)) * w+++def rotate_half(x):+ h = x.shape[-1] // 2+ return torch.cat((-x[..., h:], x[..., :h]), dim=-1)+++def load_dequant(path, rows, cols, qtype):+ raw = np.fromfile(path, dtype=np.uint8)+ bs, ts = gguf.GGML_QUANT_SIZES[qtype]+ raw = raw.reshape(rows, cols // bs * ts)+ return torch.from_numpy(dequantize(raw, qtype).astype(np.float32).reshape(rows, cols))+++class Drafter:+ def __init__(self, st_path, cfg_path, embed, head, threads=6):+ torch.set_num_threads(threads)+ cfg = json.load(open(cfg_path))+ dc = cfg["dflash_config"]+ self.H = cfg["hidden_size"]+ self.L = cfg["num_hidden_layers"]+ self.NH = cfg["num_attention_heads"]+ self.NKV = cfg["num_key_value_heads"]+ self.HD = cfg["head_dim"]+ self.win = cfg["sliding_window"]+ self.theta = cfg["rope_parameters"]["rope_theta"]+ self.eps = cfg["rms_norm_eps"]+ self.gs = dc["conv_group_size"]+ self.ks = dc["conv_kernel_size"]+ self.mask_id = dc["mask_token_id"]+ self.top_k = dc["selector_top_k"]+ self.taps = dc["target_layer_ids"]+ self.embed = embed+ self.head = head+ self.w = {}+ with safe_open(st_path, "pt") as f:+ for k in f.keys():+ t = f.get_tensor(k)+ # the two codebooks stay bf16 (only gathered rows are used); everything else fp32+ self.w[k] = t if "codebook" in k else t.float()+ self.inv_freq = 1.0 / (self.theta ** (torch.arange(0, self.HD, 2, dtype=torch.float32) / self.HD))+ self.Kc = None+ self.fast_head = False+ self.head_bf16 = None++ def enable_fast_head(self):+ self.head_bf16 = self.head.to(torch.bfloat16)+ self.fast_head = True++ def cos_sin(self, pos):+ fr = pos.float()[:, None] * self.inv_freq[None, :]+ emb = torch.cat([fr, fr], dim=-1)+ return emb.cos(), emb.sin()++ # ---- context: fc + hidden_norm over the tapped features, then every layer's K (norm + rope) and V+ def set_context(self, feats):+ """feats: [n, taps*H] float32, row j = the features of position j."""+ w = self.w+ h = rms(feats @ w["fc.weight"].T, w["hidden_norm.weight"], self.eps)+ n = h.shape[0]+ cos, sin = self.cos_sin(torch.arange(n))+ self.Kc, self.Vc = [], []+ for l in range(self.L):+ p = "layers.%d.self_attn." % l+ k = (h @ w[p + "k_proj.weight"].T).view(n, self.NKV, self.HD)+ v = (h @ w[p + "v_proj.weight"].T).view(n, self.NKV, self.HD)+ k = rms(k, w[p + "k_norm.weight"], self.eps)+ k = k * cos[:, None, :] + rotate_half(k) * sin[:, None, :]+ self.Kc.append(k.permute(1, 0, 2).contiguous()) # [NKV, n, HD]+ self.Vc.append(v.permute(1, 0, 2).contiguous())++ def conv(self, x, dyn, base):+ """_grouped_dynamic_convolve over a block x [b, H]; dyn [b, ks, groups]; base [ks, H]."""+ b, H = x.shape+ groups = H // self.gs+ blocks = x.view(b, groups, self.gs)+ out = torch.zeros_like(blocks)+ for o in range(base.shape[0]):+ vals = blocks if o == 0 else F.pad(blocks[:-o], (0, 0, 0, 0, o, 0))+ kern = base[o].float().view(1, groups, self.gs)+ out = out + kern * vals+ out = out + dyn[:, o, :, None] * vals+ return out.view(b, H)++ def prepare(self, x, p):+ w = self.w+ groups = self.H // self.gs+ dyn = (x @ w[p + "kernel_projection.weight"].T).view(x.shape[0], 2, self.ks, groups)+ return self.conv(x, dyn[:, 0], w[p + "base_kernel"][0]), dyn[:, 1]++ def finish(self, x, dyn, p):+ return self.conv(x, dyn, self.w[p + "base_kernel"][1])++ # ---- one block: anchor token at position s (s >= 1), block size b; returns the drafted path (b-1 tokens), the candidate lists+ def draft(self, s, anchor, b, greedy=True):+ w = self.w+ H, NH, NKV, HD = self.H, self.NH, self.NKV, self.HD+ ids = torch.tensor([anchor] + [self.mask_id] * (b - 1))+ h = self.embed[ids] # [b, H]+ pos = torch.arange(s, s + b)+ cos, sin = self.cos_sin(pos)+ # keys: context positions max(0, s-win+1) .. s-1 (a block row i sees ctx j when s+i-j < win), then the block+ lo = max(0, s - self.win + 1)+ for l in range(self.L):+ p = "layers.%d." % l+ r = h+ x = rms(h, w[p + "input_layernorm.weight"], self.eps)+ x, kdyn = self.prepare(x, p + "attention_conv.")+ q = (x @ w[p + "self_attn.q_proj.weight"].T).view(b, NH, HD)+ q = rms(q, w[p + "self_attn.q_norm.weight"], self.eps)+ kn = rms((x @ w[p + "self_attn.k_proj.weight"].T).view(b, NKV, HD), w[p + "self_attn.k_norm.weight"], self.eps)+ vn = (x @ w[p + "self_attn.v_proj.weight"].T).view(b, NKV, HD)+ q = q * cos[:, None, :] + rotate_half(q) * sin[:, None, :]+ kn = kn * cos[:, None, :] + rotate_half(kn) * sin[:, None, :]+ Kc = self.Kc[l][:, lo:s, :] # [NKV, c, HD]+ Vc = self.Vc[l][:, lo:s, :]+ K = torch.cat([Kc, kn.permute(1, 0, 2)], dim=1) # [NKV, c+b, HD]+ V = torch.cat([Vc, vn.permute(1, 0, 2)], dim=1)+ c = Kc.shape[1]+ kpos = torch.cat([torch.arange(lo, s), pos])+ vis = (pos[:, None] - kpos[None, :]).abs() < self.win # [b, c+b]+ rep = NH // NKV+ Kx = K.repeat_interleave(rep, dim=0) # [NH, c+b, HD]+ Vx = V.repeat_interleave(rep, dim=0)+ sc = torch.einsum("bhd,hkd->hbk", q, Kx) * (HD ** -0.5)+ sc = sc.masked_fill(~vis[None], float("-inf"))+ att = torch.softmax(sc, dim=-1)+ o = torch.einsum("hbk,hkd->bhd", att, Vx).reshape(b, NH * HD)+ o = o @ w[p + "self_attn.o_proj.weight"].T+ o = self.finish(o, kdyn, p + "attention_conv.")+ h = r + o+ r = h+ x = rms(h, w[p + "post_attention_layernorm.weight"], self.eps)+ x, mdyn = self.prepare(x, p + "mlp_conv.")+ m = (F.silu(x @ w[p + "mlp.gate_proj.weight"].T) * (x @ w[p + "mlp.up_proj.weight"].T)) @ w[p + "mlp.down_proj.weight"].T+ m = self.finish(m, mdyn, p + "mlp_conv.")+ h = r + m+ self.last_all = rms(h, w["norm.weight"], self.eps)+ out = self.last_all[1:] # rows 1..b-1+ if self.fast_head:+ # candidates from a bf16 pass over the table (7x faster on this CPU), then the exact fp32 logits of the 32 best decide the top 16+ lb = (out.to(torch.bfloat16) @ self.head_bf16.T)+ c32 = torch.topk(lb, 32, dim=-1, sorted=False).indices # [b-1, 32]+ ex = torch.einsum("rkh,rh->rk", self.head[c32], out) # [b-1, 32] exact+ unary, j16 = torch.topk(ex, self.top_k, dim=-1, sorted=False)+ cand = c32.gather(-1, j16)+ logits = None+ else:+ logits = out @ self.head.T # [b-1, V]+ unary, cand = torch.topk(logits, self.top_k, dim=-1, sorted=False)+ hp = out @ w["candidate_selector.hidden_projection.weight"].T # [b-1, rank]+ A = w["candidate_selector.predecessor_codebook"]+ B = w["candidate_selector.successor_codebook"]+ pred = anchor+ path, probs = [], []+ for i in range(b - 1):+ sc = unary[i] + (B[cand[i]].float() @ (A[pred].float() * hp[i]))+ j = int(torch.argmax(sc))+ probs.append(float(torch.softmax(sc, dim=-1)[j])) # the selector's own probability of the pick (a confidence)+ pred = int(cand[i][j])+ path.append(pred)+ top1 = (cand.gather(-1, unary.argmax(-1, keepdim=True))[:, 0] if logits is None else logits.argmax(-1)).tolist()+ self.last_probs = probs+ return path, top1+++def read_taps(path):+ f = open(path, "rb")+ magic, ver, N, nt = struct.unpack("<4i", f.read(16))+ assert magic == 0x50415453 and ver == 1, (hex(magic), ver)+ layers = struct.unpack("<%di" % nt, f.read(4 * nt))+ rec = 8 + 4 * nt * N+ data = f.read()+ n = len(data) // rec+ pos = np.empty(n, np.int32)+ tok = np.empty(n, np.int32)+ taps = np.empty((n, nt * N), np.float32)+ for i in range(n):+ o = i * rec+ pos[i], tok[i] = struct.unpack_from("<2i", data, o)+ taps[i] = np.frombuffer(data, np.float32, nt * N, o + 8)+ return layers, pos, tok, tapsdiff --git a/tools/dflash2/final_tables.py b/tools/dflash2/final_tables.pynew file mode 100644index 000000000..9caa872bd--- /dev/null+++ b/tools/dflash2/final_tables.py@@ -0,0 +1,61 @@+import json+import numpy as np+import sys+sys.path.insert(0, r"C:\dflash2-work")+exec(open(r"C:\dflash2-work\analyze2.py").read().split("print(\"check")[0]) # TV, PROD, A8, A4, walk, conf_T, names++fixed8 = {n: walk(A8[n], lambda r: 8, 8, n)[0] for n in names}+fixed4 = {n: walk(A4[n], lambda r: 4, 8, n)[0] for n in names}+fixed5 = {n: walk(A8[n], lambda r: 5, 8, n)[0] for n in names}+print("| prompt | MTP+lookup (default) tokens/round, tok/s | MTP only tokens/round, tok/s | DFlash2 block 4 | block 8 |")+print("| --- | --- | --- | --- | --- |")+for n in names:+ p = PROD[n]+ print("| %s | %.2f, %.1f | %.2f, %.1f | %.2f | %.2f |" % (n, p[0], p[1], p[2], p[3], fixed4[n], fixed8[n]))+print("| mean | %.2f, %.1f | %.2f, %.1f | %.2f | %.2f |" % (np.mean([PROD[n][0] for n in names]), np.mean([PROD[n][1] for n in names]),+ np.mean([PROD[n][2] for n in names]), np.mean([PROD[n][3] for n in names]), np.mean(list(fixed4.values())), np.mean(list(fixed8.values()))))+print("gain of block-4 tokens/round over MTP default: mean of ratios %+.1f%%, per prompt %s" % (+ 100 * (np.mean([fixed4[n] / PROD[n][0] for n in names]) - 1), {n: round(100 * (fixed4[n] / PROD[n][0] - 1)) for n in names}))+print("gain of block-8 tokens/round over MTP default: mean of ratios %+.1f%%" % (100 * (np.mean([fixed8[n] / PROD[n][0] for n in names]) - 1)))++# conditional acceptance by draft position (block 8, all anchors, conditional on all earlier correct)+cond = np.zeros(7); cn = np.zeros(7)+for n in names:+ for r in A8[n]:+ for i in range(7):+ cn[i] += 1+ if r["ok"][i]:+ cond[i] += 1+ else:+ break+print("block-8 conditional acceptance by position (all prompts):", (cond / cn).round(3).tolist())+cond4 = np.zeros(3); cn4 = np.zeros(3)+for n in names:+ for r in A4[n]:+ for i in range(3):+ cn4[i] += 1+ if r["ok"][i]:+ cond4[i] += 1+ else:+ break+print("block-4 conditional acceptance by position (all prompts):", (cond4 / cn4).round(3).tolist())++print("\nspeed estimate table (tok/s), D = drafter cost per round")+pol = [("fixed 4 (block-4 drafter)", A4, lambda r: 4), ("fixed 5 (block-8 drafter)", A8, lambda r: 5), ("confidence>=0.5, up to 8", A8, conf_T(0.5, 8)),+ ("confidence>=0.4, up to 6", A8, conf_T(0.4, 6))]+for label, A, f in pol:+ for D in (5.0, 8.0, 12.0):+ r_ = []+ tps_ = []+ for n in names:+ tau, ms, tps, hist = walk(A[n], f, D, n)+ r_.append(tps / PROD[n][1])+ tps_.append(tps)+ print("%-28s D=%4.0f ms: mean %.1f tok/s (default %.1f): mean-of-ratios %+.1f%%, best %+.0f%% (%s), worst %+.0f%% (%s)" % (+ label, D, np.mean(tps_), np.mean([PROD[n][1] for n in names]), 100 * (np.mean(r_) - 1),+ 100 * (max(r_) - 1), names[int(np.argmax(r_))], 100 * (min(r_) - 1), names[int(np.argmin(r_))]))+# window histogram of the confidence policy+print()+for n in names:+ tau, ms, tps, hist = walk(A8[n], conf_T(0.5, 8), 8.0, n)+ print(n, "conf>=0.5 cap 8: windows", {int(i): int(h) for i, h in enumerate(hist) if h}, "tau %.2f" % tau)diff --git a/tools/dflash2/mk_prompts.py b/tools/dflash2/mk_prompts.pynew file mode 100644index 000000000..78910f51e--- /dev/null+++ b/tools/dflash2/mk_prompts.py@@ -0,0 +1,40 @@+"""Builds the chat-templated prompt ids for the DFlash2 acceptance experiment (same tokenizer and template the server uses)."""+import json, sys+from pathlib import Path++REPO = Path(r"C:\Users\AI-Server\Desktop\Strata\GitHub\wt-022-dflash2")+sys.path.insert(0, str(REPO / "tools"))+sys.path.insert(0, str(REPO))+import strata_tokenizer as ST+from serve.frontend import ChatTemplate++tp = Path(r"C:\dflash2-work\tok")+vocab = json.loads((tp / "vocab.json").read_text(encoding="utf-8"))+tokens = [None] * len(vocab)+for t, i in vocab.items():+ tokens[i] = t+merges = (tp / "merges.txt").read_text(encoding="utf-8").split("\n")+types = json.loads((tp / "token_type.json").read_text())+tok = ST.Tokenizer(tokens, merges, types)+tpl = ChatTemplate(tp / "chat_template.jinja")++readme = (REPO / "README.md").read_text(encoding="utf-8")[:2600]++PROMPTS = {+ "chat_explain": "Explain how a hash map works and why lookups are O(1) on average. Then say when that guarantee breaks down. Keep it clear for a junior developer.",+ "code_py": "Write a Python function that reads a web server log file, parses the timestamps (ISO-8601) and returns the five busiest one-minute windows with their request counts. Include a small test with pytest.",+ "code_cpp": "Implement an LRU cache in C++17 with O(1) get and put using std::list and std::unordered_map. Show the class and a short main() that exercises it.",+ "math": "A train leaves city A at 9:00 traveling at 80 km/h. A second train leaves city B, 450 km away, at 10:30 traveling toward A at 95 km/h. At what time do they meet, and how far from city A? Work it out step by step and check the result.",+ "summarize": "Summarize the following text in exactly five bullet points, then give a one-sentence takeaway.\n\n" + readme,+ "json_extract": "Extract the data below as JSON with the keys name, role, city, start_date (ISO format) and skills (a list). Output only the JSON.\n\nMaria Gonzalez joined the company on the 3rd of March 2021 as a senior data engineer. She works from the Lisbon office and her main tools are Spark, dbt, Airflow and a bit of Rust. Her colleague Tom Becker, a product designer from Vienna, started a year later, on 14 June 2022, and is good at Figma, user research and prototyping.",+ "creative": "Write a short story (about 400 words) about a lighthouse keeper who discovers that the light has been answering messages from the sea. Give it a quiet, slightly eerie tone.",+ "chat_advice": "I have a 12 GB graphics card and 64 GB of RAM. What are the trade-offs between running a large mixture-of-experts model with most of its experts in RAM versus a smaller dense model fully on the GPU? Give me a recommendation for daily coding help.",+}++out = {}+for name, text in PROMPTS.items():+ s = tpl.render([{"role": "user", "content": text}])+ ids = tok.encode(s, parse_special=True)+ out[name] = {"ids": ids, "text": s}+ print(name, len(ids), repr(s[-60:]))+json.dump(out, open(r"C:\dflash2-work\prompts.json", "w"))diff --git a/tools/dflash2/orch.py b/tools/dflash2/orch.pynew file mode 100644index 000000000..fda18f596--- /dev/null+++ b/tools/dflash2/orch.py@@ -0,0 +1,67 @@+"""Runs the engine side of the DFlash2 experiment for one prompt on strix (foreground; one engine at a time)."""+import json, re, subprocess, sys, time+from pathlib import Path++W = Path(r"C:\dflash2-work")+RW = "C:\\strata-test\\dflash2"+++def ssh(cmd, timeout=1500):+ r = subprocess.run(["ssh", "-o", "ServerAliveInterval=30", "strix", cmd], capture_output=True, text=True, timeout=timeout)+ out = "\n".join(l for l in (r.stdout + r.stderr).splitlines() if not l.startswith("**"))+ return out+++def scp_up(local, remote):+ subprocess.run(["scp", "-q", str(local), "strix:" + remote.replace("\\", "/")], check=True, capture_output=True)+++def scp_down(remote, local):+ subprocess.run(["scp", "-q", "strix:" + remote.replace("\\", "/"), str(local)], check=True, capture_output=True)+++def run_mode(mode, name, extra=""):+ t = time.time()+ out = ssh('powershell -NoProfile -ExecutionPolicy Bypass -File %s\\run_one.ps1 -Mode %s -Name %s %s' % (RW, mode, name, extra))+ print("[%s %s] %.0f s" % (mode, name, time.time() - t))+ print(out)+ (W / "logs").mkdir(exist_ok=True)+ (W / "logs" / ("%s_%s.log" % (mode, name))).write_text(out)+ scp_down("%s\\out\\%s_%s.txt" % (RW, mode, name), W / "logs" / ("%s_%s.txt" % (mode, name)))+ scp_down("%s\\out\\%s_%s.err" % (RW, mode, name), W / "logs" / ("%s_%s.err" % (mode, name)))+ return out+++def produced_ids(path):+ txt = Path(path).read_text()+ m = re.search(r"^output :(.*)$", txt, re.M)+ return [int(x) for x in m.group(1).split()] if m else []+++def main(name, max_new, steps):+ prompts = json.load(open(W / "prompts.json"))+ pids = prompts[name]["ids"]+ if "gen" in steps:+ run_mode("gen", name, "-MaxNew %d" % max_new)+ out = produced_ids(W / "logs" / ("gen_%s.txt" % name))+ print(name, "prompt", len(pids), "generated", len(out), "tail", out[-3:])+ seq = pids + out+ (W / "p" / ("%s.out.ids" % name)).write_text(",".join(map(str, out)))+ (W / "p" / ("%s.first.ids" % name)).write_text(",".join(map(str, seq[:2])))+ (W / "p" / ("%s.rest.ids" % name)).write_text(",".join(map(str, seq[2:])))+ json.dump({"prompt": pids, "out": out}, open(W / "p" / ("%s.seq.json" % name), "w"))+ for k in ("out", "first", "rest"):+ scp_up(W / "p" / ("%s.%s.ids" % (name, k)), "%s\\p\\%s.%s.ids" % (RW, name, k))+ for mode in ("genf", "genfm", "tap"):+ if mode in steps:+ run_mode(mode, name)+ if "tap" in steps:+ (W / "taps").mkdir(exist_ok=True)+ scp_down("%s\\out\\tap_%s.bin" % (RW, name), W / "taps" / ("%s.bin" % name))+++if __name__ == "__main__":+ name = sys.argv[1]+ max_new = int(sys.argv[2])+ steps = sys.argv[3].split(",") if len(sys.argv) > 3 else ["gen", "genf", "genfm", "tap"]+ main(name, max_new, steps)diff --git a/tools/dflash2/run_all2.py b/tools/dflash2/run_all2.pynew file mode 100644index 000000000..bc30dd72d--- /dev/null+++ b/tools/dflash2/run_all2.py@@ -0,0 +1,53 @@+"""Per-anchor drafter results at EVERY generated position (stride 1), resumable: the exact path and the selector's confidences."""+import json, sys, time+from pathlib import Path+import numpy as np+import torch+sys.path.insert(0, r"C:\dflash2-work")+import dflash2_ref as R++W = Path(r"C:\dflash2-work")+names = sys.argv[1].split(",")+b = int(sys.argv[2])+threads = int(sys.argv[3])+budget = float(sys.argv[4]) # seconds this call may spend before it saves and stops+t_start = time.time()+embed = R.load_dequant(str(W / "embd_iq4xs.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.IQ4_XS)+head = R.load_dequant(str(W / "head_q6k.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.Q6_K)+d = R.Drafter(str(W / "drafter.safetensors"), str(W / "drafter_config.json"), embed, head, threads=threads)+d.enable_fast_head()+out_path = W / ("anchors1_b%d.json" % b)+res = json.load(open(out_path)) if out_path.exists() else {}+for name in names:+ if res.get(name + "#done"):+ continue+ seqj = json.load(open(W / "p" / ("%s.seq.json" % name)))+ seq = seqj["prompt"] + seqj["out"]+ layers, pos, tok, taps = R.read_taps(str(W / "taps" / ("%s.bin" % name)))+ keep = pos < len(seq)+ pos, tok, taps = pos[keep], tok[keep], taps[keep]+ n = int(pos.max()) + 1+ feats = np.zeros((n, taps.shape[1]), np.float32)+ for i in range(len(pos)):+ feats[pos[i]] = taps[i]+ feats[0] = feats[1]+ d.set_context(torch.from_numpy(feats))+ rows = res.get(name, [])+ s0 = rows[-1]["s"] + 1 if rows else len(seqj["prompt"])+ t1 = time.time()+ finished = True+ with torch.no_grad():+ for s in range(s0, len(seq) - b):+ path, top1 = d.draft(s, seq[s], b)+ truth = seq[s + 1:s + b]+ rows.append({"s": s, "ok": [int(path[i] == truth[i]) for i in range(b - 1)], "p": [round(x, 4) for x in d.last_probs]})+ if time.time() - t_start > budget:+ finished = False+ break+ res[name] = rows+ if finished:+ res[name + "#done"] = True+ json.dump(res, open(out_path, "w"))+ print(name, b, len(rows), "anchors so far, %.0f s" % (time.time() - t1), "done" if finished else "PAUSED", flush=True)+ if not finished:+ breakdiff --git a/tools/dflash2/run_eval.py b/tools/dflash2/run_eval.pynew file mode 100644index 000000000..77056a023--- /dev/null+++ b/tools/dflash2/run_eval.py@@ -0,0 +1,70 @@+"""Offline DFlash2 acceptance on OUR quantized target: walks each prompt's dumped sequence the way the speculative loop would+(anchor, draft block, accept the leading matches of the drafted path against the engine's own greedy tokens, move on)."""+import json, sys, time+from pathlib import Path+import numpy as np+import torch+sys.path.insert(0, r"C:\dflash2-work")+import dflash2_ref as R++W = Path(r"C:\dflash2-work")+names = sys.argv[1].split(",")+blocks = [int(x) for x in sys.argv[2].split(",")] if len(sys.argv) > 2 else [4, 8]+threads = int(sys.argv[3]) if len(sys.argv) > 3 else 6++t0 = time.time()+embed = R.load_dequant(str(W / "embd_iq4xs.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.IQ4_XS)+head = R.load_dequant(str(W / "head_q6k.bin"), 248320, 2560, R.gguf.GGMLQuantizationType.Q6_K)+print("dequantized embed/head in %.0f s" % (time.time() - t0), flush=True)+d = R.Drafter(str(W / "drafter.safetensors"), str(W / "drafter_config.json"), embed, head, threads=threads)+res_path = W / "eval_results.json"+results = json.load(open(res_path)) if res_path.exists() else {}++for name in names:+ seqj = json.load(open(W / "p" / ("%s.seq.json" % name)))+ prompt, out = seqj["prompt"], seqj["out"]+ seq = prompt + out+ layers, pos, tok, taps = R.read_taps(str(W / "taps" / ("%s.bin" % name)))+ keep = pos < len(seq) # the tail of the last window is padding (zeros) past the sequence+ pos, tok, taps = pos[keep], tok[keep], taps[keep]+ n = int(pos.max()) + 1+ feats = np.zeros((n, taps.shape[1]), np.float32)+ seen = np.zeros(n, bool)+ for i in range(len(pos)): # the last record of a position wins+ feats[pos[i]] = taps[i]+ seen[pos[i]] = True+ assert tok[i] == seq[pos[i]], ("token mismatch at", pos[i], tok[i], seq[pos[i]])+ if not seen[0] and seen[1]: # position 0 goes through the batched prefill (no tap): the next row stands in+ feats[0] = feats[1]+ seen[0] = True+ assert seen.all(), "positions missing: %s" % np.where(~seen)[0][:5]+ d.set_context(torch.from_numpy(feats))+ print("%s: prompt %d, generated %d, tap records %d, layers %s" % (name, len(prompt), len(out), len(pos), layers), flush=True)+ results.setdefault(name, {})+ for b in blocks:+ s = len(prompt) # the first generated token is the first anchor, as after a prefill+ cyc, cond_ok, cond_n = [], np.zeros(b - 1), np.zeros(b - 1)+ top1_ok = np.zeros(b - 1)+ t1 = time.time()+ with torch.no_grad():+ while s + b <= len(seq) - 0 and s < n:+ path, top1 = d.draft(s, seq[s], b)+ truth = seq[s + 1:s + b]+ a = 0+ while a < b - 1 and path[a] == truth[a]:+ a += 1+ for i in range(b - 1): # conditional on all earlier correct+ if i <= a:+ cond_n[i] += 1+ cond_ok[i] += (i < a)+ if i <= a and top1[i] == truth[i]:+ top1_ok[i] += 1+ cyc.append(a + 1)+ s += a + 1+ tau = float(np.mean(cyc))+ results[name]["b%d" % b] = {"anchors": len(cyc), "tokens_per_round": tau,+ "cond_accept": (cond_ok / np.maximum(cond_n, 1)).round(3).tolist(),+ "hist": np.bincount(cyc, minlength=b + 1).tolist()}+ print(" block %d: %d rounds, %.3f tokens per round, cond %s (%.0f s)" %+ (b, len(cyc), tau, (cond_ok / np.maximum(cond_n, 1)).round(2).tolist(), time.time() - t1), flush=True)+ json.dump(results, open(res_path, "w"), indent=1)diff --git a/tools/dflash2/run_one.ps1 b/tools/dflash2/run_one.ps1new file mode 100644index 000000000..dc0c38bcf--- /dev/null+++ b/tools/dflash2/run_one.ps1@@ -0,0 +1,50 @@+# DFlash2 experiment runner on strix. One engine process at a time; reads ids from C:\strata-test\dflash2\p\<name>.*.ids+# usage: run_one.ps1 -Mode gen|genf|genfm|tap|tcycle -Name <prompt> [-MaxNew N] [-Cycle 1,2,...]+param([string]$Mode, [string]$Name, [int]$MaxNew = 900, [string]$Cycle = "", [string]$Exe = "strata-hook.exe", [string]$Tag = "", [string]$Extra = "")+$ErrorActionPreference = "Continue"+$W = "C:\strata-test\dflash2"+$D = "C:\Users\Administrator\Documents\Strata"+$OWN = "C:\Users\Administrator\AppData\Local\Strata\terminal\0.1.46.1"+$env:LOCALAPPDATA = "C:\strata-test\la"+$env:APPDATA = "C:\strata-test\la"+New-Item -ItemType Directory -Force -Path $env:LOCALAPPDATA | Out-Null+New-Item -ItemType Directory -Force -Path "$W\out" | Out-Null+$env:STRATA_ARENA_PIN_GIB = "none"+$env:PATH = "$W\eng\rocm\bin;" + $env:PATH+$mtp = $D + "\mtp\rt"+$base = @("--pack", "$D\packs\iq3_s",+ "--native", "$D\models\IQ3_S\Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf",+ "--ple-gguf", "$D\models\IQ3_S\Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00002-of-00002.gguf",+ "--expert-profile", "$OWN\data\expert-profile.bin",+ "--expert-cache", "auto", "--prefill", "auto", "--max-context", "4096", "--kv", "int8", "--vram-reserve-mib", "700", "--greedy")+$exe = "$W\eng\$Exe"+$out = "$W\out\${Mode}_${Name}${Tag}.txt"+$err = "$W\out\${Mode}_${Name}${Tag}.err"+Remove-Item $out, $err -ErrorAction SilentlyContinue+if ($Mode -eq "gen") {+ $ids = (Get-Content "$W\p\$Name.prompt.ids" -Raw).Trim()+ $a = $base + @("--spec", "4", "--spec-min-p", "0.5", "--mtp", $mtp, "--max-new", "$MaxNew", "--stop-eos", "--tokens", $ids)+} elseif ($Mode -eq "genf" -or $Mode -eq "genfm") {+ # the production flags on a fixed text: the drafts are judged against the continuation in <name>.out.ids+ $ids = (Get-Content "$W\p\$Name.prompt.ids" -Raw).Trim()+ $n = (Get-Content "$W\p\$Name.out.ids" -Raw).Split(",").Count+ $a = $base + @("--spec", "4", "--spec-min-p", "0.5", "--mtp", $mtp, "--spec-follow", "$W\p\$Name.out.ids", "--max-new", "$n", "--tokens", $ids)+ if ($Mode -eq "genfm") { $a += @("--suffix-draft", "0") }+} elseif ($Mode -eq "tap") {+ $env:STRATA_TAP_FILE = "$W\out\tap_$Name.bin"+ Remove-Item $env:STRATA_TAP_FILE -ErrorAction SilentlyContinue+ $first = (Get-Content "$W\p\$Name.first.ids" -Raw).Trim()+ $n = (Get-Content "$W\p\$Name.rest.ids" -Raw).Split(",").Count+ $a = $base + @("--spec", "8", "--spec-oracle", "$W\p\$Name.rest.ids", "--spec-follow", "$W\p\$Name.rest.ids", "--max-new", "$n", "--tokens", $first)+} elseif ($Mode -eq "tcycle") {+ $env:STRATA_SPEC_T_CYCLE = $Cycle+ $first = (Get-Content "$W\p\$Name.first.ids" -Raw).Trim()+ $a = $base + @("--spec", "8", "--spec-oracle", "$W\p\$Name.rest.ids", "--spec-follow", "$W\p\$Name.rest.ids", "--max-new", "$MaxNew", "--tokens", $first)+} else { "bad mode"; exit 2 }+if ($Extra -ne "") { $a += $Extra.Split("_") }+Set-Location $OWN+$t0 = Get-Date+$p = Start-Process -FilePath $exe -ArgumentList $a -RedirectStandardOutput $out -RedirectStandardError $err -NoNewWindow -PassThru+$p.WaitForExit()+"EXIT " + $p.ExitCode + " after " + [int]((Get-Date) - $t0).TotalSeconds + " s"+Get-Content $out | Select-String -Pattern "^decode|^speculation|^window|^accepted|^mtp|^suffix|^verify window" | ForEach-Object { $_.Line.Substring(0, [Math]::Min(300, $_.Line.Length)) }diff --git a/tools/dflash2/test_port.py b/tools/dflash2/test_port.pynew file mode 100644index 000000000..335363bda--- /dev/null+++ b/tools/dflash2/test_port.py@@ -0,0 +1,45 @@+import sys, importlib.util, json, torch+sys.path.insert(0, r"C:\dflash2-work")+import dflash2_ref as R+torch.manual_seed(0)+# the reference (z-lab) model on transformers 4.57+spec = importlib.util.spec_from_file_location("ghm", r"C:\Users\AI-Server\Desktop\Strata\phaseA-tests\ornith_tmp\gh_model.py")+ghm = importlib.util.module_from_spec(spec); spec.loader.exec_module(ghm)+from transformers.models.qwen3.modeling_qwen3 import Qwen3Config+cfgd = json.load(open(r"C:\dflash2-work\drafter_config.json"))+c = Qwen3Config(**{k: v for k, v in cfgd.items() if k in ("hidden_size","intermediate_size","num_hidden_layers","num_attention_heads","num_key_value_heads","head_dim","vocab_size","rms_norm_eps","max_position_embeddings","attention_bias","sliding_window","layer_types","use_sliding_window","max_window_layers")})+c.rope_theta = cfgd["rope_parameters"]["rope_theta"]+c.dflash_config = cfgd["dflash_config"]+c.is_causal = False+c.num_target_layers = 48+m = ghm.DFlash2DraftModel(c)+from safetensors.torch import load_file+sd = load_file(r"C:\dflash2-work\drafter.safetensors")+sd = {(k + ".weight" if k.startswith("candidate_selector.") and k.endswith("codebook") else k): v for k, v in sd.items()}+miss = m.load_state_dict(sd, strict=False); print("missing", miss.missing_keys[:5], "unexpected", miss.unexpected_keys[:5])+m = m.float().eval()+# my port with a tiny random embed/head+V = 248320; H = 2560+embed = torch.randn(V, H) * 0.02+head = torch.randn(V, H) * 0.02+d = R.Drafter(r"C:\dflash2-work\drafter.safetensors", r"C:\dflash2-work\drafter_config.json", embed, head)+n = 300; b = 8; s = 250+feats = torch.randn(n, 5 * H) * 2.0+d.set_context(feats)+anchor = 1234+mine_path, _ = d.draft(s, anchor, b)+ids = torch.tensor([[anchor] + [d.mask_id] * (b - 1)])+noise = embed[ids]+pos = torch.arange(0, s + b)[None] # context 0..s-1 then block+with torch.no_grad():+ out = m(position_ids=pos, noise_embedding=noise, target_hidden=feats[None, :s], past_key_values=None, use_cache=False)+print("shape", out.shape, "mine", d.last_all.shape)+err = (out[0] - d.last_all).abs().max().item()+print("max abs err all rows", err, "ref scale", out.abs().mean().item())+# the selector path against the reference+logits = (out[:, 1:] @ head.T)+class Tgt: pass+with torch.no_grad():+ path, idx, q = m.candidate_selector.select(out[:, 1:], logits, torch.tensor([anchor]), 0.0)+print("ref path ", path[0].tolist())+print("mine path", mine_path)