Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

APR port, stages 1-2: map, trellis type tables, prism.hadamard packing

Laurent Zuijdwijk89febede8b5b
diff --git a/include/strata/artifact/gguf_reader.hpp b/include/strata/artifact/gguf_reader.hpp
index 7e646a4b1..5b67f54ba 100644
--- a/include/strata/artifact/gguf_reader.hpp
+++ b/include/strata/artifact/gguf_reader.hpp
@@ -115,6 +115,15 @@ inline const char* ggml_type_name(uint32_t t) {
return "Q1_0";
case 42:
return "Q2_0";
+ // Agention's llama.cpp fork: the APR trellis expert types (docs/GYRO.md)
+ case 143:
+ return "PTQ1_0";
+ case 144:
+ return "TQ2_T";
+ case 145:
+ return "TQK6";
+ case 146:
+ return "TQK7";
default:
return "?";
}
@@ -225,6 +234,24 @@ inline bool block_geometry(uint32_t t, int& elems, int& bytes) {
elems = 64;
bytes = 18;
return true;
+ // APR trellis types: 128 values = fp16 scale + 32 steps of K bits (K = 8 TQ2_T, 6 TQK6, 7 TQK7);
+ // PTQ1_0 = 26 B of base-3 digits + fp16 scale
+ case 143:
+ elems = 128;
+ bytes = 28;
+ return true;
+ case 144:
+ elems = 128;
+ bytes = 34;
+ return true;
+ case 145:
+ elems = 128;
+ bytes = 26;
+ return true;
+ case 146:
+ elems = 128;
+ bytes = 30;
+ return true;
default:
return false;
}
diff --git a/tools/embd_bf16_pack.py b/tools/embd_bf16_pack.py
index 1ed2d6423..a633b3323 100644
--- a/tools/embd_bf16_pack.py
+++ b/tools/embd_bf16_pack.py
@@ -7,6 +7,12 @@ and reads one row per token, so BF16 costs 0.6 GB of RAM more and no VRAM; `--em
The output is a one-tensor GGUF: `token_embd.weight`, ne = [n_embd, n_vocab], type BF16, the bytes copied straight
out of the safetensors file - nothing decoded or rounded.
+
+ python tools/embd_bf16_pack.py --gguf <model.gguf> --out <token-embd-bf16.gguf>
+
+instead dequantizes a GGUF's own token_embd.weight to BF16 (round-to-nearest-even, through gguf-py), for a file whose
+embedding type has no GPU dequantizer in the engine (Agention's Gyro-S stores it as Q6_K). That is the GGUF's values
+rounded to BF16, not the checkpoint's.
"""
import argparse
import json
@@ -35,11 +41,63 @@ def kv_string(key, val):
return gguf_string(key) + struct.pack("<I", GGUF_TYPE_STRING) + gguf_string(val)
+def write_header(dim, vocab, source, what):
+ kvs = [kv_string("general.architecture", "strata-embd"),
+ kv_string("general.name", what),
+ kv_string("strata.embd.source", source)]
+ head = b"GGUF" + struct.pack("<IQQ", 3, 1, len(kvs)) + b"".join(kvs)
+ head += gguf_string("token_embd.weight") + struct.pack("<I", 2) + struct.pack("<QQ", dim, vocab)
+ head += struct.pack("<I", GGML_TYPE_BF16) + struct.pack("<Q", 0)
+ head += b"\0" * ((-len(head)) % ALIGN)
+ return head
+
+
+def from_gguf(src, out):
+ """token_embd.weight of a GGUF, dequantized and rounded to BF16 (nearest-even), in row chunks."""
+ import numpy as np
+ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
+ import gguf_reader as G
+ from _paths import add_gguf_py
+ add_gguf_py()
+ from gguf import GGMLQuantizationType as Q, quants
+ g = G.GGUFFile(src)
+ t = next((t for t in g.tensors if t.name == "token_embd.weight"), None)
+ if t is None or len(t.shape) != 2 or t.expected_bytes() is None:
+ sys.exit("%s: no 2-D token_embd.weight of a known type" % src)
+ dim, vocab = int(t.shape[0]), int(t.shape[1])
+ row = t.expected_bytes() // vocab
+ mm = np.memmap(src, dtype=np.uint8, mode="r")
+ base = g.data_start + t.offset
+ print("token_embd.weight: %d x %d %s -> BF16, %.2f GB" % (vocab, dim, t.type_name, vocab * dim * 2 / 1e9), flush=True)
+ head = write_header(dim, vocab, src.name, "token embedding, %s dequantized to BF16" % t.type_name)
+ out.parent.mkdir(parents=True, exist_ok=True)
+ tmp = out.with_suffix(out.suffix + ".part")
+ rows = 8192
+ with open(tmp, "wb") as w:
+ w.write(head)
+ for r0 in range(0, vocab, rows):
+ r1 = min(vocab, r0 + rows)
+ raw = np.asarray(mm[base + r0 * row: base + r1 * row]).reshape(r1 - r0, row)
+ vals = quants.dequantize(raw, Q[t.type_name]).astype(np.float32)
+ if not np.isfinite(vals).all():
+ sys.exit("token_embd.weight has non-finite values")
+ w.write(quants.quantize(vals, Q.BF16).tobytes())
+ if tmp.stat().st_size != len(head) + vocab * dim * 2:
+ sys.exit("size check failed")
+ tmp.replace(out)
+ print("wrote %s (header %d B)" % (out, len(head)))
+
+
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
- ap.add_argument("--model", required=True, help="the checkpoint directory (model.safetensors.index.json)")
+ src = ap.add_mutually_exclusive_group(required=True)
+ src.add_argument("--model", help="the checkpoint directory (model.safetensors.index.json)")
+ src.add_argument("--gguf", help="a model GGUF whose token_embd.weight is dequantized to BF16")
ap.add_argument("--out", required=True)
a = ap.parse_args()
+ if a.gguf:
+ from_gguf(pathlib.Path(a.gguf).absolute(), pathlib.Path(a.out))
+ return
model, out = pathlib.Path(a.model), pathlib.Path(a.out)
wmap = json.loads((model / "model.safetensors.index.json").read_text(encoding="utf-8"))["weight_map"]
@@ -57,13 +115,7 @@ def main():
sys.exit("%s: data size does not match its shape" % name)
print("%s: %d x %d BF16, %.2f GB" % (name, vocab, dim, n / 1e9), flush=True)
- kvs = [kv_string("general.architecture", "strata-embd"),
- kv_string("general.name", "token embedding, BF16 as shipped"),
- kv_string("strata.embd.source", model.name)]
- head = b"GGUF" + struct.pack("<IQQ", 3, 1, len(kvs)) + b"".join(kvs)
- head += gguf_string("token_embd.weight") + struct.pack("<I", 2) + struct.pack("<QQ", dim, vocab)
- head += struct.pack("<I", GGML_TYPE_BF16) + struct.pack("<Q", 0)
- head += b"\0" * ((-len(head)) % ALIGN)
+ head = write_header(dim, vocab, model.name, "token embedding, BF16 as shipped")
out.parent.mkdir(parents=True, exist_ok=True)
tmp = out.with_suffix(out.suffix + ".part")
diff --git a/tools/gguf_reader.py b/tools/gguf_reader.py
index 237d5fdb4..4da9d2d24 100644
--- a/tools/gguf_reader.py
+++ b/tools/gguf_reader.py
@@ -25,6 +25,8 @@ GGML_TYPES: dict[int, str] = {
28: "F64", 29: "IQ1_M", 30: "BF16", 31: "Q4_0_4_4", 32: "Q4_0_4_8", 33: "Q4_0_8_8",
34: "TQ1_0", 35: "TQ2_0", 36: "IQ4_NL_4_4", 37: "IQ4_NL_4_8", 38: "IQ4_NL_8_8",
39: "MXFP4", 40: "NVFP4", 41: "Q4_0_8_8", 42: "Q2_0",
+ # Agention's llama.cpp fork (agentionai/llama.cpp): the APR trellis expert types (128-value blocks)
+ 143: "PTQ1_0", 144: "TQ2_T", 145: "TQK6", 146: "TQK7",
}
# GGUF metadata value type ids
@@ -45,6 +47,8 @@ BLOCK_GEOMETRY: dict[str, tuple[int, int]] = {
"IQ2_XXS": (256, 66), "IQ2_XS": (256, 74), "IQ3_XXS": (256, 98), "IQ1_S": (256, 50),
"IQ4_NL": (32, 18), "IQ3_S": (256, 110), "IQ2_S": (256, 82), "IQ4_XS": (256, 136),
"IQ1_M": (256, 56), "Q2_0": (64, 18), "MXFP4": (32, 17), "NVFP4": (64, 36),
+ # fp16 scale + trellis stream (32 steps x 8 / 6 / 7 bits); PTQ1_0: 26 B of trits + fp16 scale
+ "TQ2_T": (128, 34), "TQK6": (128, 26), "TQK7": (128, 30), "PTQ1_0": (128, 28),
}
diff --git a/tools/iq_pack.py b/tools/iq_pack.py
index cb3618247..997be21d0 100644
--- a/tools/iq_pack.py
+++ b/tools/iq_pack.py
@@ -488,6 +488,77 @@ def expert_layout(model: Model, src: pathlib.Path):
return layout, head + "".join(line + "\n" for line in lines), n_expert, offset
+HADAMARD_ROLE = re.compile(r"^blk\.(\d+)\.ffn_(gate|up|down)_exps\.weight$")
+
+
+def hadamard_spec(meta: dict, layout, n_embd: int, n_ff: int):
+ """prism.hadamard.* (Agention's APR files: routed experts stored in a block-Hadamard-rotated basis; the runtime
+ must rotate their input activations, see include/strata/kernels/cpu/hadamard.hpp) -> the text of hadamard.txt,
+ None for a model without the metadata, or an error string. Anything this engine cannot apply is refused: a
+ folded weight it would run unrotated computes plausible garbage."""
+ if "prism.hadamard.version" not in meta:
+ return None
+ if int(meta["prism.hadamard.version"]) != 1:
+ return "unsupported prism.hadamard.version %s" % meta["prism.hadamard.version"]
+ block = int(meta.get("prism.hadamard.block_size", 0))
+ if block <= 1 or block & (block - 1):
+ return "invalid prism.hadamard.block_size %r" % block
+ if meta.get("prism.hadamard.transform") != "normalized-sylvester-walsh-hadamard":
+ return "unsupported prism.hadamard.transform %r" % meta.get("prism.hadamard.transform")
+ if meta.get("prism.hadamard.axis") != "input-last-dimension":
+ return "unsupported prism.hadamard.axis %r" % meta.get("prism.hadamard.axis")
+ mode = meta.get("prism.hadamard.sign_mode")
+ if mode not in ("identity", "explicit"):
+ return "unsupported prism.hadamard.sign_mode %r" % mode
+ if meta.get("prism.hadamard.inverse_weight_names"):
+ return ("prism.hadamard.inverse_weight_names (a rotated token embedding) is not supported by this engine: "
+ "%s" % ", ".join(meta["prism.hadamard.inverse_weight_names"][:4]))
+ names = meta.get("prism.hadamard.weight_names") or []
+ if not names:
+ return "prism.hadamard.weight_names is empty"
+ n_layers = len(layout)
+ roles = [0] * n_layers
+ for name in names:
+ m = HADAMARD_ROLE.match(name)
+ if not m or int(m.group(1)) >= n_layers:
+ return ("prism.hadamard folds %s; this engine applies the activation rotation to the routed experts "
+ "(blk.N.ffn_{gate,up,down}_exps) only" % name)
+ bit = {"gate": 1, "up": 2, "down": 4}[m.group(2)]
+ if roles[int(m.group(1))] & bit:
+ return "duplicate prism.hadamard weight %s" % name
+ roles[int(m.group(1))] |= bit
+ for l, r in enumerate(roles):
+ if bool(r & 1) != bool(r & 2):
+ return "layer %d: prism.hadamard folds only one of gate/up (they share one input)" % l
+ if r and 42 in (layout[l][1] if r & 3 else -1, layout[l][2] if r & 4 else -1):
+ return "layer %d: a Hadamard-folded Q2_0 expert (the engine's Q2_0 rows cannot rotate)" % l
+ lines = ["# strata hadamard v1: prism.hadamard (normalized-sylvester-walsh-hadamard, input-last-dimension, "
+ "sign_mode %s), written by tools/iq_pack.py" % mode, "block %d" % block]
+ widths = {n_embd for r in roles if r & 3} | {n_ff for r in roles if r & 4}
+ if any(w % block for w in widths):
+ return "prism.hadamard block %d does not divide the expert widths %s" % (block, sorted(widths))
+ if mode == "explicit":
+ sw = [int(w) for w in meta.get("prism.hadamard.sign_widths") or []]
+ sv = [int(v) for v in meta.get("prism.hadamard.sign_values") or []]
+ if not sw or sum(sw) != len(sv) or any(w <= 0 or w % block for w in sw) or len(set(sw)) != len(sw):
+ return "prism.hadamard sign_widths / sign_values are inconsistent"
+ if any(v not in (1, -1) for v in sv):
+ return "prism.hadamard sign values must be +1/-1"
+ at, table = 0, {}
+ for w in sw:
+ table[w] = sv[at:at + w]
+ at += w
+ missing = sorted(widths - set(table))
+ if missing:
+ return "prism.hadamard has no sign vector for width(s) %s" % missing
+ for w in sorted(table):
+ lines.append("signs %d %s" % (w, " ".join(str(v) for v in table[w])))
+ for l, r in enumerate(roles):
+ if r:
+ lines.append("layer %d %d %d %d" % (l, r & 1, (r >> 1) & 1, (r >> 2) & 1))
+ return "\n".join(lines) + "\n"
+
+
def experts_source(model: Model, text: str, total: int) -> dict:
"""What experts.bin is cut from: the shards (names and sizes) and the hash of native_experts.txt (every
per-role file and offset, the formats and the blob sizes). A same-size experts.bin of another model or
@@ -539,6 +610,21 @@ def main() -> int:
layout, text, n_expert, offset = got
if any(t.type_name == "NVFP4" for *_, ts in layout for t in ts):
a.experts_bin = True # the GGUF holds the weights without the scale tails: the engine must read experts.bin
+ # ---- prism.hadamard: rotated experts need hadamard.txt beside native_experts.txt, which becomes v5 so an engine
+ # that cannot rotate refuses the pack (instead of running the folded experts on unrotated activations)
+ t0 = layout[0][5]
+ had = hadamard_spec(g.metadata, layout, int(t0[0].shape[0]), int(t0[2].shape[0]))
+ if isinstance(had, str) and not had.startswith("# strata hadamard"):
+ print("prism.hadamard: " + had)
+ return 1
+ if had is not None:
+ head, rest = text.split("\n", 1)
+ head = re.sub(r"^# strata native experts v[34]:", "# strata native experts v5:", head)
+ text = head + " - v5: Hadamard-folded experts, see hadamard.txt\n" + rest
+ print("prism.hadamard: %d layer(s) of Hadamard-folded experts (block %s, %s signs): hadamard.txt, "
+ "native_experts.txt v5" % (sum(1 for ln in had.splitlines() if ln.startswith("layer ")),
+ g.metadata["prism.hadamard.block_size"],
+ g.metadata["prism.hadamard.sign_mode"]))
path = out / "experts.bin"
sidecar = out / "experts.bin.src.json"
want = experts_source(model, text, offset)
@@ -608,6 +694,12 @@ def main() -> int:
# ---- the experts. native_experts.txt is written to a temporary name and renamed only when every layer is
# in: a stop part-way (a layer split across shards, #171) left a partial native_experts.txt that the next setup
# run took as a finished pack (#172). It is the pack's completion marker, so it is published last.
+ htmp = out / "hadamard.txt.tmp"
+ if had is not None:
+ htmp.write_text(had, encoding="utf-8", newline="\n")
+ htmp.replace(out / "hadamard.txt")
+ else:
+ (out / "hadamard.txt").unlink(missing_ok=True) # a previous Hadamard model's, packed into this folder
tmp = out / "native_experts.txt.tmp"
with open(tmp, "w", encoding="utf-8", newline="\n") as fo:
fo.write(text)
diff --git a/tools/test_iq_pack.py b/tools/test_iq_pack.py
index 9758b41e1..538c08276 100644
--- a/tools/test_iq_pack.py
+++ b/tools/test_iq_pack.py
@@ -548,5 +548,55 @@ class SplitArtifactTests(unittest.TestCase):
self.assertEqual((pack / "experts.bin").read_bytes(), want)
+class HadamardTests(unittest.TestCase):
+ """prism.hadamard (Agention's APR files) -> hadamard.txt; anything the engine cannot rotate is refused."""
+ N_EMBD, N_FF = 256, 128
+
+ def meta(self, **kw):
+ names = ["blk.%d.ffn_%s_exps.weight" % (l, r) for l in range(2) for r in ("gate", "up", "down")]
+ m = {"prism.hadamard.version": 1, "prism.hadamard.block_size": 128,
+ "prism.hadamard.transform": "normalized-sylvester-walsh-hadamard",
+ "prism.hadamard.axis": "input-last-dimension", "prism.hadamard.sign_mode": "explicit",
+ "prism.hadamard.weight_names": names,
+ "prism.hadamard.sign_widths": [self.N_FF, self.N_EMBD],
+ "prism.hadamard.sign_values": [1, -1] * (self.N_FF // 2) + [-1, 1] * (self.N_EMBD // 2)}
+ m.update(kw)
+ return m
+
+ def layout(self, gu=145, d=146):
+ return [(l, gu, d, 0, 0, None) for l in range(2)]
+
+ def test_no_metadata(self):
+ self.assertIsNone(iq_pack.hadamard_spec({}, self.layout(), self.N_EMBD, self.N_FF))
+
+ def test_explicit_signs_written(self):
+ text = iq_pack.hadamard_spec(self.meta(), self.layout(), self.N_EMBD, self.N_FF)
+ lines = text.splitlines()
+ self.assertTrue(lines[0].startswith("# strata hadamard v1"))
+ self.assertEqual(lines[1], "block 128")
+ self.assertEqual(lines[2].split()[:4], ["signs", str(self.N_FF), "1", "-1"])
+ self.assertEqual(len(lines[3].split()), 2 + self.N_EMBD)
+ self.assertEqual(lines[4:], ["layer 0 1 1 1", "layer 1 1 1 1"])
+
+ def test_refusals(self):
+ bad = {
+ "transform": self.meta(**{"prism.hadamard.transform": "dct"}),
+ "dense weight": self.meta(**{"prism.hadamard.weight_names": ["blk.0.attn_q.weight"]}),
+ "inverse": self.meta(**{"prism.hadamard.inverse_weight_names": ["token_embd.weight"]}),
+ "gate only": self.meta(**{"prism.hadamard.weight_names": ["blk.0.ffn_gate_exps.weight"]}),
+ "missing width": self.meta(**{"prism.hadamard.sign_widths": [self.N_EMBD],
+ "prism.hadamard.sign_values": [1] * self.N_EMBD}),
+ "bad sign": self.meta(**{"prism.hadamard.sign_values": [2] * (self.N_FF + self.N_EMBD)}),
+ "block": self.meta(**{"prism.hadamard.block_size": 96}),
+ }
+ for what, m in bad.items():
+ with self.subTest(what):
+ got = iq_pack.hadamard_spec(m, self.layout(), self.N_EMBD, self.N_FF)
+ self.assertIsInstance(got, str)
+ self.assertFalse(got.startswith("# strata hadamard"))
+ got = iq_pack.hadamard_spec(self.meta(), self.layout(gu=42), self.N_EMBD, self.N_FF)
+ self.assertIn("Q2_0", got)
+
+
if __name__ == "__main__":
unittest.main()