Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
APR port, stages 1-2: map, trellis type tables, prism.hadamard packing
Laurent Zuijdwijk89febede8b5b
diff --git a/include/strata/artifact/gguf_reader.hpp b/include/strata/artifact/gguf_reader.hppindex 7e646a4b1..5b67f54ba 100644--- a/include/strata/artifact/gguf_reader.hpp+++ b/include/strata/artifact/gguf_reader.hpp@@ -115,6 +115,15 @@ inline const char* ggml_type_name(uint32_t t) {return "Q1_0";case 42:return "Q2_0";+ // Agention's llama.cpp fork: the APR trellis expert types (docs/GYRO.md)+ case 143:+ return "PTQ1_0";+ case 144:+ return "TQ2_T";+ case 145:+ return "TQK6";+ case 146:+ return "TQK7";default:return "?";}@@ -225,6 +234,24 @@ inline bool block_geometry(uint32_t t, int& elems, int& bytes) {elems = 64;bytes = 18;return true;+ // APR trellis types: 128 values = fp16 scale + 32 steps of K bits (K = 8 TQ2_T, 6 TQK6, 7 TQK7);+ // PTQ1_0 = 26 B of base-3 digits + fp16 scale+ case 143:+ elems = 128;+ bytes = 28;+ return true;+ case 144:+ elems = 128;+ bytes = 34;+ return true;+ case 145:+ elems = 128;+ bytes = 26;+ return true;+ case 146:+ elems = 128;+ bytes = 30;+ return true;default:return false;}diff --git a/tools/embd_bf16_pack.py b/tools/embd_bf16_pack.pyindex 1ed2d6423..a633b3323 100644--- a/tools/embd_bf16_pack.py+++ b/tools/embd_bf16_pack.py@@ -7,6 +7,12 @@ and reads one row per token, so BF16 costs 0.6 GB of RAM more and no VRAM; `--emThe output is a one-tensor GGUF: `token_embd.weight`, ne = [n_embd, n_vocab], type BF16, the bytes copied straightout of the safetensors file - nothing decoded or rounded.++ python tools/embd_bf16_pack.py --gguf <model.gguf> --out <token-embd-bf16.gguf>++instead dequantizes a GGUF's own token_embd.weight to BF16 (round-to-nearest-even, through gguf-py), for a file whose+embedding type has no GPU dequantizer in the engine (Agention's Gyro-S stores it as Q6_K). That is the GGUF's values+rounded to BF16, not the checkpoint's."""import argparseimport json@@ -35,11 +41,63 @@ def kv_string(key, val):return gguf_string(key) + struct.pack("<I", GGUF_TYPE_STRING) + gguf_string(val)+def write_header(dim, vocab, source, what):+ kvs = [kv_string("general.architecture", "strata-embd"),+ kv_string("general.name", what),+ kv_string("strata.embd.source", source)]+ head = b"GGUF" + struct.pack("<IQQ", 3, 1, len(kvs)) + b"".join(kvs)+ head += gguf_string("token_embd.weight") + struct.pack("<I", 2) + struct.pack("<QQ", dim, vocab)+ head += struct.pack("<I", GGML_TYPE_BF16) + struct.pack("<Q", 0)+ head += b"\0" * ((-len(head)) % ALIGN)+ return head+++def from_gguf(src, out):+ """token_embd.weight of a GGUF, dequantized and rounded to BF16 (nearest-even), in row chunks."""+ import numpy as np+ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))+ import gguf_reader as G+ from _paths import add_gguf_py+ add_gguf_py()+ from gguf import GGMLQuantizationType as Q, quants+ g = G.GGUFFile(src)+ t = next((t for t in g.tensors if t.name == "token_embd.weight"), None)+ if t is None or len(t.shape) != 2 or t.expected_bytes() is None:+ sys.exit("%s: no 2-D token_embd.weight of a known type" % src)+ dim, vocab = int(t.shape[0]), int(t.shape[1])+ row = t.expected_bytes() // vocab+ mm = np.memmap(src, dtype=np.uint8, mode="r")+ base = g.data_start + t.offset+ print("token_embd.weight: %d x %d %s -> BF16, %.2f GB" % (vocab, dim, t.type_name, vocab * dim * 2 / 1e9), flush=True)+ head = write_header(dim, vocab, src.name, "token embedding, %s dequantized to BF16" % t.type_name)+ out.parent.mkdir(parents=True, exist_ok=True)+ tmp = out.with_suffix(out.suffix + ".part")+ rows = 8192+ with open(tmp, "wb") as w:+ w.write(head)+ for r0 in range(0, vocab, rows):+ r1 = min(vocab, r0 + rows)+ raw = np.asarray(mm[base + r0 * row: base + r1 * row]).reshape(r1 - r0, row)+ vals = quants.dequantize(raw, Q[t.type_name]).astype(np.float32)+ if not np.isfinite(vals).all():+ sys.exit("token_embd.weight has non-finite values")+ w.write(quants.quantize(vals, Q.BF16).tobytes())+ if tmp.stat().st_size != len(head) + vocab * dim * 2:+ sys.exit("size check failed")+ tmp.replace(out)+ print("wrote %s (header %d B)" % (out, len(head)))++def main():ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])- ap.add_argument("--model", required=True, help="the checkpoint directory (model.safetensors.index.json)")+ src = ap.add_mutually_exclusive_group(required=True)+ src.add_argument("--model", help="the checkpoint directory (model.safetensors.index.json)")+ src.add_argument("--gguf", help="a model GGUF whose token_embd.weight is dequantized to BF16")ap.add_argument("--out", required=True)a = ap.parse_args()+ if a.gguf:+ from_gguf(pathlib.Path(a.gguf).absolute(), pathlib.Path(a.out))+ returnmodel, out = pathlib.Path(a.model), pathlib.Path(a.out)wmap = json.loads((model / "model.safetensors.index.json").read_text(encoding="utf-8"))["weight_map"]@@ -57,13 +115,7 @@ def main():sys.exit("%s: data size does not match its shape" % name)print("%s: %d x %d BF16, %.2f GB" % (name, vocab, dim, n / 1e9), flush=True)- kvs = [kv_string("general.architecture", "strata-embd"),- kv_string("general.name", "token embedding, BF16 as shipped"),- kv_string("strata.embd.source", model.name)]- head = b"GGUF" + struct.pack("<IQQ", 3, 1, len(kvs)) + b"".join(kvs)- head += gguf_string("token_embd.weight") + struct.pack("<I", 2) + struct.pack("<QQ", dim, vocab)- head += struct.pack("<I", GGML_TYPE_BF16) + struct.pack("<Q", 0)- head += b"\0" * ((-len(head)) % ALIGN)+ head = write_header(dim, vocab, model.name, "token embedding, BF16 as shipped")out.parent.mkdir(parents=True, exist_ok=True)tmp = out.with_suffix(out.suffix + ".part")diff --git a/tools/gguf_reader.py b/tools/gguf_reader.pyindex 237d5fdb4..4da9d2d24 100644--- a/tools/gguf_reader.py+++ b/tools/gguf_reader.py@@ -25,6 +25,8 @@ GGML_TYPES: dict[int, str] = {28: "F64", 29: "IQ1_M", 30: "BF16", 31: "Q4_0_4_4", 32: "Q4_0_4_8", 33: "Q4_0_8_8",34: "TQ1_0", 35: "TQ2_0", 36: "IQ4_NL_4_4", 37: "IQ4_NL_4_8", 38: "IQ4_NL_8_8",39: "MXFP4", 40: "NVFP4", 41: "Q4_0_8_8", 42: "Q2_0",+ # Agention's llama.cpp fork (agentionai/llama.cpp): the APR trellis expert types (128-value blocks)+ 143: "PTQ1_0", 144: "TQ2_T", 145: "TQK6", 146: "TQK7",}# GGUF metadata value type ids@@ -45,6 +47,8 @@ BLOCK_GEOMETRY: dict[str, tuple[int, int]] = {"IQ2_XXS": (256, 66), "IQ2_XS": (256, 74), "IQ3_XXS": (256, 98), "IQ1_S": (256, 50),"IQ4_NL": (32, 18), "IQ3_S": (256, 110), "IQ2_S": (256, 82), "IQ4_XS": (256, 136),"IQ1_M": (256, 56), "Q2_0": (64, 18), "MXFP4": (32, 17), "NVFP4": (64, 36),+ # fp16 scale + trellis stream (32 steps x 8 / 6 / 7 bits); PTQ1_0: 26 B of trits + fp16 scale+ "TQ2_T": (128, 34), "TQK6": (128, 26), "TQK7": (128, 30), "PTQ1_0": (128, 28),}diff --git a/tools/iq_pack.py b/tools/iq_pack.pyindex cb3618247..997be21d0 100644--- a/tools/iq_pack.py+++ b/tools/iq_pack.py@@ -488,6 +488,77 @@ def expert_layout(model: Model, src: pathlib.Path):return layout, head + "".join(line + "\n" for line in lines), n_expert, offset+HADAMARD_ROLE = re.compile(r"^blk\.(\d+)\.ffn_(gate|up|down)_exps\.weight$")+++def hadamard_spec(meta: dict, layout, n_embd: int, n_ff: int):+ """prism.hadamard.* (Agention's APR files: routed experts stored in a block-Hadamard-rotated basis; the runtime+ must rotate their input activations, see include/strata/kernels/cpu/hadamard.hpp) -> the text of hadamard.txt,+ None for a model without the metadata, or an error string. Anything this engine cannot apply is refused: a+ folded weight it would run unrotated computes plausible garbage."""+ if "prism.hadamard.version" not in meta:+ return None+ if int(meta["prism.hadamard.version"]) != 1:+ return "unsupported prism.hadamard.version %s" % meta["prism.hadamard.version"]+ block = int(meta.get("prism.hadamard.block_size", 0))+ if block <= 1 or block & (block - 1):+ return "invalid prism.hadamard.block_size %r" % block+ if meta.get("prism.hadamard.transform") != "normalized-sylvester-walsh-hadamard":+ return "unsupported prism.hadamard.transform %r" % meta.get("prism.hadamard.transform")+ if meta.get("prism.hadamard.axis") != "input-last-dimension":+ return "unsupported prism.hadamard.axis %r" % meta.get("prism.hadamard.axis")+ mode = meta.get("prism.hadamard.sign_mode")+ if mode not in ("identity", "explicit"):+ return "unsupported prism.hadamard.sign_mode %r" % mode+ if meta.get("prism.hadamard.inverse_weight_names"):+ return ("prism.hadamard.inverse_weight_names (a rotated token embedding) is not supported by this engine: "+ "%s" % ", ".join(meta["prism.hadamard.inverse_weight_names"][:4]))+ names = meta.get("prism.hadamard.weight_names") or []+ if not names:+ return "prism.hadamard.weight_names is empty"+ n_layers = len(layout)+ roles = [0] * n_layers+ for name in names:+ m = HADAMARD_ROLE.match(name)+ if not m or int(m.group(1)) >= n_layers:+ return ("prism.hadamard folds %s; this engine applies the activation rotation to the routed experts "+ "(blk.N.ffn_{gate,up,down}_exps) only" % name)+ bit = {"gate": 1, "up": 2, "down": 4}[m.group(2)]+ if roles[int(m.group(1))] & bit:+ return "duplicate prism.hadamard weight %s" % name+ roles[int(m.group(1))] |= bit+ for l, r in enumerate(roles):+ if bool(r & 1) != bool(r & 2):+ return "layer %d: prism.hadamard folds only one of gate/up (they share one input)" % l+ if r and 42 in (layout[l][1] if r & 3 else -1, layout[l][2] if r & 4 else -1):+ return "layer %d: a Hadamard-folded Q2_0 expert (the engine's Q2_0 rows cannot rotate)" % l+ lines = ["# strata hadamard v1: prism.hadamard (normalized-sylvester-walsh-hadamard, input-last-dimension, "+ "sign_mode %s), written by tools/iq_pack.py" % mode, "block %d" % block]+ widths = {n_embd for r in roles if r & 3} | {n_ff for r in roles if r & 4}+ if any(w % block for w in widths):+ return "prism.hadamard block %d does not divide the expert widths %s" % (block, sorted(widths))+ if mode == "explicit":+ sw = [int(w) for w in meta.get("prism.hadamard.sign_widths") or []]+ sv = [int(v) for v in meta.get("prism.hadamard.sign_values") or []]+ if not sw or sum(sw) != len(sv) or any(w <= 0 or w % block for w in sw) or len(set(sw)) != len(sw):+ return "prism.hadamard sign_widths / sign_values are inconsistent"+ if any(v not in (1, -1) for v in sv):+ return "prism.hadamard sign values must be +1/-1"+ at, table = 0, {}+ for w in sw:+ table[w] = sv[at:at + w]+ at += w+ missing = sorted(widths - set(table))+ if missing:+ return "prism.hadamard has no sign vector for width(s) %s" % missing+ for w in sorted(table):+ lines.append("signs %d %s" % (w, " ".join(str(v) for v in table[w])))+ for l, r in enumerate(roles):+ if r:+ lines.append("layer %d %d %d %d" % (l, r & 1, (r >> 1) & 1, (r >> 2) & 1))+ return "\n".join(lines) + "\n"++def experts_source(model: Model, text: str, total: int) -> dict:"""What experts.bin is cut from: the shards (names and sizes) and the hash of native_experts.txt (everyper-role file and offset, the formats and the blob sizes). A same-size experts.bin of another model or@@ -539,6 +610,21 @@ def main() -> int:layout, text, n_expert, offset = gotif any(t.type_name == "NVFP4" for *_, ts in layout for t in ts):a.experts_bin = True # the GGUF holds the weights without the scale tails: the engine must read experts.bin+ # ---- prism.hadamard: rotated experts need hadamard.txt beside native_experts.txt, which becomes v5 so an engine+ # that cannot rotate refuses the pack (instead of running the folded experts on unrotated activations)+ t0 = layout[0][5]+ had = hadamard_spec(g.metadata, layout, int(t0[0].shape[0]), int(t0[2].shape[0]))+ if isinstance(had, str) and not had.startswith("# strata hadamard"):+ print("prism.hadamard: " + had)+ return 1+ if had is not None:+ head, rest = text.split("\n", 1)+ head = re.sub(r"^# strata native experts v[34]:", "# strata native experts v5:", head)+ text = head + " - v5: Hadamard-folded experts, see hadamard.txt\n" + rest+ print("prism.hadamard: %d layer(s) of Hadamard-folded experts (block %s, %s signs): hadamard.txt, "+ "native_experts.txt v5" % (sum(1 for ln in had.splitlines() if ln.startswith("layer ")),+ g.metadata["prism.hadamard.block_size"],+ g.metadata["prism.hadamard.sign_mode"]))path = out / "experts.bin"sidecar = out / "experts.bin.src.json"want = experts_source(model, text, offset)@@ -608,6 +694,12 @@ def main() -> int:# ---- the experts. native_experts.txt is written to a temporary name and renamed only when every layer is# in: a stop part-way (a layer split across shards, #171) left a partial native_experts.txt that the next setup# run took as a finished pack (#172). It is the pack's completion marker, so it is published last.+ htmp = out / "hadamard.txt.tmp"+ if had is not None:+ htmp.write_text(had, encoding="utf-8", newline="\n")+ htmp.replace(out / "hadamard.txt")+ else:+ (out / "hadamard.txt").unlink(missing_ok=True) # a previous Hadamard model's, packed into this foldertmp = out / "native_experts.txt.tmp"with open(tmp, "w", encoding="utf-8", newline="\n") as fo:fo.write(text)diff --git a/tools/test_iq_pack.py b/tools/test_iq_pack.pyindex 9758b41e1..538c08276 100644--- a/tools/test_iq_pack.py+++ b/tools/test_iq_pack.py@@ -548,5 +548,55 @@ class SplitArtifactTests(unittest.TestCase):self.assertEqual((pack / "experts.bin").read_bytes(), want)+class HadamardTests(unittest.TestCase):+ """prism.hadamard (Agention's APR files) -> hadamard.txt; anything the engine cannot rotate is refused."""+ N_EMBD, N_FF = 256, 128++ def meta(self, **kw):+ names = ["blk.%d.ffn_%s_exps.weight" % (l, r) for l in range(2) for r in ("gate", "up", "down")]+ m = {"prism.hadamard.version": 1, "prism.hadamard.block_size": 128,+ "prism.hadamard.transform": "normalized-sylvester-walsh-hadamard",+ "prism.hadamard.axis": "input-last-dimension", "prism.hadamard.sign_mode": "explicit",+ "prism.hadamard.weight_names": names,+ "prism.hadamard.sign_widths": [self.N_FF, self.N_EMBD],+ "prism.hadamard.sign_values": [1, -1] * (self.N_FF // 2) + [-1, 1] * (self.N_EMBD // 2)}+ m.update(kw)+ return m++ def layout(self, gu=145, d=146):+ return [(l, gu, d, 0, 0, None) for l in range(2)]++ def test_no_metadata(self):+ self.assertIsNone(iq_pack.hadamard_spec({}, self.layout(), self.N_EMBD, self.N_FF))++ def test_explicit_signs_written(self):+ text = iq_pack.hadamard_spec(self.meta(), self.layout(), self.N_EMBD, self.N_FF)+ lines = text.splitlines()+ self.assertTrue(lines[0].startswith("# strata hadamard v1"))+ self.assertEqual(lines[1], "block 128")+ self.assertEqual(lines[2].split()[:4], ["signs", str(self.N_FF), "1", "-1"])+ self.assertEqual(len(lines[3].split()), 2 + self.N_EMBD)+ self.assertEqual(lines[4:], ["layer 0 1 1 1", "layer 1 1 1 1"])++ def test_refusals(self):+ bad = {+ "transform": self.meta(**{"prism.hadamard.transform": "dct"}),+ "dense weight": self.meta(**{"prism.hadamard.weight_names": ["blk.0.attn_q.weight"]}),+ "inverse": self.meta(**{"prism.hadamard.inverse_weight_names": ["token_embd.weight"]}),+ "gate only": self.meta(**{"prism.hadamard.weight_names": ["blk.0.ffn_gate_exps.weight"]}),+ "missing width": self.meta(**{"prism.hadamard.sign_widths": [self.N_EMBD],+ "prism.hadamard.sign_values": [1] * self.N_EMBD}),+ "bad sign": self.meta(**{"prism.hadamard.sign_values": [2] * (self.N_FF + self.N_EMBD)}),+ "block": self.meta(**{"prism.hadamard.block_size": 96}),+ }+ for what, m in bad.items():+ with self.subTest(what):+ got = iq_pack.hadamard_spec(m, self.layout(), self.N_EMBD, self.N_FF)+ self.assertIsInstance(got, str)+ self.assertFalse(got.startswith("# strata hadamard"))+ got = iq_pack.hadamard_spec(self.meta(), self.layout(gu=42), self.N_EMBD, self.N_FF)+ self.assertIn("Q2_0", got)++if __name__ == "__main__":unittest.main()