Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Metal: Gyro in the Mac setup and app model lists, the n-gram table stays out of the dense copy, decode-check test

Niko1221c3472caf8e55
diff --git a/metal/setup_mac.py b/metal/setup_mac.py
index 834de130a..b939e2460 100644
--- a/metal/setup_mac.py
+++ b/metal/setup_mac.py
@@ -66,6 +66,8 @@ UD_SHARDS = {
def model_shard_names(model: str) -> list[str]:
"""The model's files in order (shard 1 first)."""
n = MAC_MODELS[model].get("shards", 2)
+ if MAC_MODELS[model].get("family") == "gyro": # agentionai's one file (the n-gram table is inside it)
+ return [S.model_file(S.FAMILIES["gyro"], model, 1)]
if MAC_MODELS[model].get("family") == "unsloth":
return [f"Qwen3.8-Flash-Next-{model}-{i:05d}-of-{n:05d}.gguf" for i in range(1, n + 1)]
return [S.model_file(S.FAMILIES["qwen"], model, i) for i in range(1, n + 1)]
@@ -324,8 +326,9 @@ def start(cfg_path: Path, port: int | None, open_browser: bool = True) -> int:
# ------------------------------------------------------------------------------------------------ main
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
- ap.add_argument("--family", choices=["qwen", "unsloth"], default=None,
- help="qwen = Qwen3.8-Flash-Next (GSQ-RCO), unsloth = Unsloth's UD-*_K_XL (follows --model)")
+ ap.add_argument("--family", choices=["qwen", "unsloth", "gyro"], default=None,
+ help="qwen = Qwen3.8-Flash-Next (GSQ-RCO), unsloth = Unsloth's UD-*_K_XL, gyro = agentionai's Gyro-S / Gyro-M "
+ "(follows --model)")
ap.add_argument("--model", choices=list(MAC_MODELS))
ap.add_argument("--context", type=int)
ap.add_argument("--port", type=int)
@@ -491,6 +494,8 @@ def main(argv=None) -> int:
S.mark(native)
ple = ple_shard(files)
shards = [native, ple]
+ elif family == "gyro":
+ shards = [files[0], files[0]] # one file: the model and its n-gram table (--native, --ple-gguf)
else:
shards = files
if not (pack / "native_experts.txt").exists() or not (pack / "tokenizer" / "vocab.json").exists():
diff --git a/metal/src/engine/stream.cpp b/metal/src/engine/stream.cpp
index b06aac61c..4d90b9853 100644
--- a/metal/src/engine/stream.cpp
+++ b/metal/src/engine/stream.cpp
@@ -183,6 +183,9 @@ bool ExpertStream::load_dense(gguf::File& f, mtl::Context& cx, const std::vector
for (auto& kv : f.tensors) {
gguf::Tensor& t = kv.second;
if (t.name.find("_exps") != std::string::npos) continue;
+ // the n-gram table is read by the host from the SSD, a few rows at a time (src/ngram/ple_reader.cpp). It is its own shard in
+ // the ISTA / Unsloth files; in agentionai's Gyro files it is the first 28.8 GB of the one file, and must not be read in here
+ if (t.name == "per_layer_token_embd.weight") continue;
if (std::find(skip.begin(), skip.end(), t.name) != skip.end()) { t.offset = 0; continue; }
ts.push_back(&t);
tot += (t.bytes + 255) / 256 * 256;
diff --git a/metal/tests/expert_parity.cpp b/metal/tests/expert_parity.cpp
index 492855447..70c985f39 100644
--- a/metal/tests/expert_parity.cpp
+++ b/metal/tests/expert_parity.cpp
@@ -32,6 +32,9 @@
#include <string>
#include <vector>
+#include "ggml-common-tq.h"
+const uint16_t* tq_lut_for_test() { return tq_lut_f16; }
+
namespace {
constexpr int N_EMBD = 2560, N_FF = 640, TOPK = 10;
@@ -216,7 +219,7 @@ double median(std::vector<double> v) {
int main(int argc, char** argv) {
int gu = ref::Q2_0, dn = ref::Q2_0, ntok = 1, layers = 8, reps = 96;
Cfg cgu{32, 2, 4}, cdn{8, 2, 4};
- bool bench = false, sweep = false, had_test = false;
+ bool bench = false, sweep = false, had_test = false, decode_check = false;
double share = 0.0;
std::string gguf_path, had_pack;
int real_layer = 0;
@@ -240,6 +243,7 @@ int main(int argc, char** argv) {
else if (a == "--layer") real_layer = std::atoi(nx());
else if (a == "--had-pack") had_pack = nx();
else if (a == "--had-test") had_test = true;
+ else if (a == "--decode-check") decode_check = true;
else if (a == "--seed") seed = std::strtoull(nx(), nullptr, 10);
else { std::fprintf(stderr, "unknown argument %s\n", a.c_str()); return 2; }
}
@@ -284,6 +288,54 @@ int main(int argc, char** argv) {
g_had_h = !had_test;
std::printf("Hadamard: block %d, signs for %s%s\n", strata::kernels::cpu::hadamard_spec().block, sgx ? "2560 " : "", sgh ? "640" : "(none)");
}
+ if (decode_check) {
+ // the decoder the Metal kernels and ref_quant share the meaning of, written a third time straight from the format's
+ // definition (ggml-common-tq.h): step t's state = 16 stream bits from bit (31 - t) * K, LSB first, circular modulo 32 K
+ // bits, masked to 15 bits (TQ2_T: qs[(t + 31) % 32] << 8 | qs[t]); x = state * 0x9e3779b1; points lut[x >> 21] and
+ // lut[(x >> 10) & 2047]; weights = fp16 values * d. Compared bit for bit with ref::dequant on the real tensors.
+ extern const uint16_t* tq_lut_for_test();
+ long blocks = 0, bad = 0;
+ const gguf::Tensor* ts[3] = {tg_, tu_, td_};
+ if (!tg_) { std::fprintf(stderr, "--decode-check needs --gguf\n"); return 2; }
+ for (const gguf::Tensor* t : ts) {
+ const int ty = t->type, K = ty == ref::TQ2_T ? 8 : ty == ref::TQK6 ? 6 : 7, bb = ref::block_bytes(ty);
+ const uint8_t* base = (const uint8_t*)gf.map + t->offset;
+ const size_t nb = t->bytes / bb;
+ const size_t step = std::max<size_t>(1, nb / 200000); // up to 200k blocks a tensor, spread over it
+ for (size_t i = 0; i < nb; i += step) {
+ const uint8_t* b = base + i * bb;
+ float want[128], got[128];
+ const float d = ref::f16_to_f32((uint16_t)(b[0] | (b[1] << 8)));
+ const uint8_t* qs = b + 2;
+ for (int st = 0; st < 32; ++st) {
+ uint32_t state;
+ if (K == 8) state = (((uint32_t)qs[(st + 31) % 32] << 8) | qs[st]) & 0x7fff;
+ else {
+ const int off = ((31 - st) * K) % (32 * K);
+ uint32_t v = 0;
+ for (int bit = 0; bit < 16; ++bit) {
+ const int pos = (off + bit) % (32 * K);
+ v |= (uint32_t)((qs[pos / 8] >> (pos % 8)) & 1) << bit;
+ }
+ state = v & 0x7fff;
+ }
+ const uint32_t x = state * 0x9e3779b1u;
+ const uint16_t* lut = tq_lut_for_test();
+ const uint32_t pa = x >> 21, pb = (x >> 10) & 2047u;
+ want[4 * st] = ref::f16_to_f32(lut[2 * pa]) * d;
+ want[4 * st + 1] = ref::f16_to_f32(lut[2 * pa + 1]) * d;
+ want[4 * st + 2] = ref::f16_to_f32(lut[2 * pb]) * d;
+ want[4 * st + 3] = ref::f16_to_f32(lut[2 * pb + 1]) * d;
+ }
+ ref::dequant(ty, b, got, 128);
+ bad += std::memcmp(want, got, sizeof want) != 0;
+ ++blocks;
+ }
+ }
+ std::printf("decode-check layer %d: %ld blocks of the real tensors, %ld differ from the format's definition %s\n", real_layer,
+ blocks, bad, bad == 0 ? "PASS" : "FAIL");
+ return bad == 0 ? 0 : 1;
+ }
if (had_test) {
// the Metal rotation kernels against the CPU's (hadamard_rotate), bit for bit
int bad = 0;
diff --git a/setup.py b/setup.py
index af4d1e340..33f59563e 100644
--- a/setup.py
+++ b/setup.py
@@ -7281,8 +7281,8 @@ def mac_control_setup(a) -> int:
fail(problems[0])
step(2, "your choices")
- app_sizes = [m for m, d in MF.MAC_MODELS.items() if d.get("family", "qwen") == "qwen"]
- if a.family not in (None, "qwen"):
+ app_sizes = [m for m, d in MF.MAC_MODELS.items() if d.get("family", "qwen") in ("qwen", "gyro")] # Gyro: one file, no repack
+ if a.family not in (None, "qwen", "gyro"):
fail(f"the app on a Mac runs the Qwen3.8-Flash-Next sizes ({', '.join(app_sizes)}); {a.family} is not one of them",
"choose Qwen3.8-Flash-Next")
rec, notes = MF.recommend_model(ram)
@@ -7298,7 +7298,8 @@ def mac_control_setup(a) -> int:
if ctx > MF.TESTED_CONTEXT:
say(f" context {ctx}: measured on the Mac up to {MF.TESTED_CONTEXT} (needles found at 8K-32K); longer is kept")
ok(f"{model}, context {ctx}")
- fam = FAMILIES["qwen"]
+ family = MF.MAC_MODELS[model].get("family", "qwen")
+ fam = FAMILIES[family]
tag = fam["tag"] + model
stream = MF.streams(model, ram)
@@ -7319,6 +7320,15 @@ def mac_control_setup(a) -> int:
if vision != "none":
warn("pictures are off for now: the picture reader comes with the engine (run setup again once it is installed)")
vision = "none"
+ else:
+ if family == "gyro": # an engine from before the trellis kernels refuses the file when it loads
+ try:
+ has_tq = b"inputs are rotated on the GPU" in Path(found.exe).read_bytes()
+ except (OSError, AttributeError):
+ has_tq = True # cannot tell: not a reason to hold the install back
+ if not has_tq:
+ warn(f"the Metal engine component {found.version} has no kernels for Gyro's trellis experts yet: the model is "
+ "downloaded and prepared, and it runs once the app or its update feed brings a newer engine")
ok("engine: the engine component (looked up when the model loads)")
ok("images: " + {"none": "off", "gpu": "on (read on the Mac's GPU)", "cpu": "on (read on the CPU)"}[vision])
@@ -7367,7 +7377,8 @@ def mac_control_setup(a) -> int:
pack = find_in(roots, f"packs/{tag.lower()}") or data / "packs" / tag.lower()
env = dict(os.environ, STRATA_GGUF_PY=str(llama / "gguf-py"))
if not (pack / "native_experts.txt").exists() or not (pack / "tokenizer" / "vocab.json").exists():
- run([sys.executable, str(ROOT / "tools" / "iq_pack.py"), "--gguf", str(files[0]), "--out", str(pack)], env=env)
+ run([sys.executable, str(ROOT / "tools" / "iq_pack.py"), "--gguf", str(files[0]), "--out", str(pack),
+ *fam.get("pack_args", [])], env=env) # Gyro: --compat-bf16
ok(f"model prepared: {pack}")
use_mtp = MF.stream_mtp(stream, ram)
mtp = (find_in(roots, "mtp/mtp-q2_0.gguf") or data / "mtp" / "mtp-q2_0.gguf").parent
@@ -7388,7 +7399,8 @@ def mac_control_setup(a) -> int:
step(7, "writing the run settings")
port = a.port or 8080
- args = MF.engine_args(pack, files, rt, ctx, stream, ram, ROOT / "data" / "expert-profile.bin")
+ args = MF.engine_args(pack, [files[0], files[0]] if family == "gyro" else files, rt, ctx, stream, ram,
+ ROOT / "data" / "expert-profile.bin") # Gyro: one file, the model and its n-gram table
cfg = {"exe": engine_exe(Path("engine")), "args": args, "cwd": str(ROOT), "tokenizer": str(pack / "tokenizer"),
"model_name": f"{fam['name']}-{model.lower()}", "log": str(home() / f"strata-{tag.lower()}.log"),
"lib_dirs": [], "port": port, "backend": "metal", "env": {"STRATA_METAL_STREAM": "1" if stream else "0"}}
diff --git a/strata_control/mac_fit.py b/strata_control/mac_fit.py
index 94183e156..68a7b1063 100644
--- a/strata_control/mac_fit.py
+++ b/strata_control/mac_fit.py
@@ -26,6 +26,11 @@ MAC_MODELS = {
"UD-Q4_K_XL": {"shard1_gb": 82.6, "tested": True, "family": "unsloth", "shards": 4},
"UD-Q5_K_XL": {"shard1_gb": 103.9, "tested": True, "family": "unsloth", "shards": 6},
"UD-Q6_K_XL": {"shard1_gb": 114.8, "tested": True, "family": "unsloth", "shards": 6},
+ # agentionai's Gyro files (family "gyro", docs/GYRO.md): ONE file with the n-gram table inside (58.5 / 92.0 GB to download);
+ # shard1_gb is what the GPU needs of it, without that table (the engine reads the table from the SSD a few rows at a time).
+ # The trellis experts run on the Metal engine from the release that added metal/kernels' TQ types (docs/MAC.md).
+ "Gyro-S": {"shard1_gb": 29.7, "tested": True, "family": "gyro", "shards": 1, "file_gb": 58.5},
+ "Gyro-M": {"shard1_gb": 63.2, "tested": False, "family": "gyro", "shards": 1, "file_gb": 92.0},
}
# The recommended sizes (owner's decision): IQ3_S is the default where it stays resident, UD-Q4_K_XL the higher-quality one
@@ -57,7 +62,9 @@ CONTEXTS = [8192, 32768, 65536, 131072]
MAC_ABOUT = {"Q2_0": "2-bit, the fastest", "IQ3_XXS": "3-bit i-quant, better quality",
"IQ3_S": "3.5-bit i-quant, the best quality (matches the full model), the slowest",
"UD-Q4_K_XL": "Unsloth ~4.5-bit (111 GB download)", "UD-Q5_K_XL": "Unsloth ~5.5-bit (158 GB download)",
- "UD-Q6_K_XL": "Unsloth ~6.5-bit (169 GB download)"}
+ "UD-Q6_K_XL": "Unsloth ~6.5-bit (169 GB download)",
+ "Gyro-S": "agentionai's trellis experts, the smallest (58 GB download, the n-gram table is inside it)",
+ "Gyro-M": "agentionai's trellis experts, about IQ3_XXS quality in less memory (92 GB download)"}
TESTED_CONTEXT = 32768 # needles and the KL harness ran to 32K on the Mac
# Below this much unified memory the app says a model does not fit (a Mac this small was never run: 32 GB is the smallest that was).
# The Terminal keeps its rule of recommending and never forcing: the person can still pick one there.
diff --git a/tools/strata_mcp.py b/tools/strata_mcp.py
index c863e3b41..64c2f6ef6 100644
--- a/tools/strata_mcp.py
+++ b/tools/strata_mcp.py
@@ -1014,11 +1014,12 @@ class Strata:
hw["nvidia_smi"] = False
return hw
- def _mac_sizes(self, models: dict) -> list:
- """The sizes the app offers on a Mac: the GSQ-RCO files of the Qwen family (two shards, no repack). Unsloth's many-shard files need
- metal/tools/metal_repack.py, which the Terminal's setup uses; the app does not offer them yet."""
+ def _mac_sizes(self, models: dict, family: str = "qwen") -> list:
+ """The sizes the app offers on a Mac: the GSQ-RCO files of the Qwen family (two shards, no repack) and agentionai's Gyro files (one
+ file, no repack). Unsloth's many-shard files need metal/tools/metal_repack.py, which the Terminal's setup uses; the app does not
+ offer them yet."""
mf = self.mac()
- return [m for m, d in mf.MAC_MODELS.items() if d.get("family", "qwen") == "qwen" and m in models] if mf else []
+ return [m for m, d in mf.MAC_MODELS.items() if d.get("family", "qwen") == family and m in models] if mf else []
def _mac_verdict(self, model: str, ram: float) -> str:
mf = self.mac()
@@ -1061,18 +1062,24 @@ class Strata:
models, families, _ctx, _src = self.tables()
ram = float(hw.get("ram_gb") or 0)
have = {c.stem[len("strata-"):] for c in self.configs()}
- fd = families["qwen"]
- sizes = []
- for m in self._mac_sizes(models):
- d = models[m]
- tag = (fd.get("tag", "") + m).lower()
- sizes.append({"model": m, "about": d["about"], "download_gb": d["download_gb"], "experimental": False,
- "images": d.get("vision", fd.get("vision", True)) is not False, # the Mac engine carries the picture reader (Metal)
- "ram_needed_gb": d["ram_gb"], "experts_gb": d.get("arena_gb"), "mem_total": False,
- "memory_gb": round(float(d.get("arena_gb") or d["ram_gb"]) + 1.0, 1),
- "on_this_pc": self._mac_verdict(m, ram) if ram else None, "installed": tag in have, "id": tag})
- return [{"family": "qwen", "title": fd["title"], "about": fd.get("about"), "experimental": False,
- "images": any(x["images"] for x in sizes), "sizes": sizes}]
+ out = []
+ for family in ("qwen", "gyro"): # Gyro after the regular sizes, as on a PC: never the default
+ fd = families.get(family)
+ if fd is None:
+ continue
+ sizes = []
+ for m in self._mac_sizes(models, family):
+ d = models[m]
+ tag = (fd.get("tag", "") + m).lower()
+ sizes.append({"model": m, "about": d["about"], "download_gb": d["download_gb"], "experimental": False,
+ "images": d.get("vision", fd.get("vision", True)) is not False, # the Mac engine carries the picture reader (Metal)
+ "ram_needed_gb": d["ram_gb"], "experts_gb": d.get("arena_gb"), "mem_total": False,
+ "memory_gb": round(float(d.get("arena_gb") or d["ram_gb"]) + 1.0, 1),
+ "on_this_pc": self._mac_verdict(m, ram) if ram else None, "installed": tag in have, "id": tag})
+ if sizes:
+ out.append({"family": family, "title": fd["title"], "about": fd.get("about"), "experimental": False,
+ "images": any(x["images"] for x in sizes), "sizes": sizes})
+ return out
def recommend(self, hw: dict) -> dict:
"""setup.py's own rules: the size it preselects (IQ3_XXS from 60 GB of RAM, else the first, Q2_0), the