Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Metal: Gyro in the Mac setup and app model lists, the n-gram table stays out of the dense copy, decode-check test
Niko1221c3472caf8e55
diff --git a/metal/setup_mac.py b/metal/setup_mac.pyindex 834de130a..b939e2460 100644--- a/metal/setup_mac.py+++ b/metal/setup_mac.py@@ -66,6 +66,8 @@ UD_SHARDS = {def model_shard_names(model: str) -> list[str]:"""The model's files in order (shard 1 first)."""n = MAC_MODELS[model].get("shards", 2)+ if MAC_MODELS[model].get("family") == "gyro": # agentionai's one file (the n-gram table is inside it)+ return [S.model_file(S.FAMILIES["gyro"], model, 1)]if MAC_MODELS[model].get("family") == "unsloth":return [f"Qwen3.8-Flash-Next-{model}-{i:05d}-of-{n:05d}.gguf" for i in range(1, n + 1)]return [S.model_file(S.FAMILIES["qwen"], model, i) for i in range(1, n + 1)]@@ -324,8 +326,9 @@ def start(cfg_path: Path, port: int | None, open_browser: bool = True) -> int:# ------------------------------------------------------------------------------------------------ maindef main(argv=None) -> int:ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])- ap.add_argument("--family", choices=["qwen", "unsloth"], default=None,- help="qwen = Qwen3.8-Flash-Next (GSQ-RCO), unsloth = Unsloth's UD-*_K_XL (follows --model)")+ ap.add_argument("--family", choices=["qwen", "unsloth", "gyro"], default=None,+ help="qwen = Qwen3.8-Flash-Next (GSQ-RCO), unsloth = Unsloth's UD-*_K_XL, gyro = agentionai's Gyro-S / Gyro-M "+ "(follows --model)")ap.add_argument("--model", choices=list(MAC_MODELS))ap.add_argument("--context", type=int)ap.add_argument("--port", type=int)@@ -491,6 +494,8 @@ def main(argv=None) -> int:S.mark(native)ple = ple_shard(files)shards = [native, ple]+ elif family == "gyro":+ shards = [files[0], files[0]] # one file: the model and its n-gram table (--native, --ple-gguf)else:shards = filesif not (pack / "native_experts.txt").exists() or not (pack / "tokenizer" / "vocab.json").exists():diff --git a/metal/src/engine/stream.cpp b/metal/src/engine/stream.cppindex b06aac61c..4d90b9853 100644--- a/metal/src/engine/stream.cpp+++ b/metal/src/engine/stream.cpp@@ -183,6 +183,9 @@ bool ExpertStream::load_dense(gguf::File& f, mtl::Context& cx, const std::vectorfor (auto& kv : f.tensors) {gguf::Tensor& t = kv.second;if (t.name.find("_exps") != std::string::npos) continue;+ // the n-gram table is read by the host from the SSD, a few rows at a time (src/ngram/ple_reader.cpp). It is its own shard in+ // the ISTA / Unsloth files; in agentionai's Gyro files it is the first 28.8 GB of the one file, and must not be read in here+ if (t.name == "per_layer_token_embd.weight") continue;if (std::find(skip.begin(), skip.end(), t.name) != skip.end()) { t.offset = 0; continue; }ts.push_back(&t);tot += (t.bytes + 255) / 256 * 256;diff --git a/metal/tests/expert_parity.cpp b/metal/tests/expert_parity.cppindex 492855447..70c985f39 100644--- a/metal/tests/expert_parity.cpp+++ b/metal/tests/expert_parity.cpp@@ -32,6 +32,9 @@#include <string>#include <vector>+#include "ggml-common-tq.h"+const uint16_t* tq_lut_for_test() { return tq_lut_f16; }+namespace {constexpr int N_EMBD = 2560, N_FF = 640, TOPK = 10;@@ -216,7 +219,7 @@ double median(std::vector<double> v) {int main(int argc, char** argv) {int gu = ref::Q2_0, dn = ref::Q2_0, ntok = 1, layers = 8, reps = 96;Cfg cgu{32, 2, 4}, cdn{8, 2, 4};- bool bench = false, sweep = false, had_test = false;+ bool bench = false, sweep = false, had_test = false, decode_check = false;double share = 0.0;std::string gguf_path, had_pack;int real_layer = 0;@@ -240,6 +243,7 @@ int main(int argc, char** argv) {else if (a == "--layer") real_layer = std::atoi(nx());else if (a == "--had-pack") had_pack = nx();else if (a == "--had-test") had_test = true;+ else if (a == "--decode-check") decode_check = true;else if (a == "--seed") seed = std::strtoull(nx(), nullptr, 10);else { std::fprintf(stderr, "unknown argument %s\n", a.c_str()); return 2; }}@@ -284,6 +288,54 @@ int main(int argc, char** argv) {g_had_h = !had_test;std::printf("Hadamard: block %d, signs for %s%s\n", strata::kernels::cpu::hadamard_spec().block, sgx ? "2560 " : "", sgh ? "640" : "(none)");}+ if (decode_check) {+ // the decoder the Metal kernels and ref_quant share the meaning of, written a third time straight from the format's+ // definition (ggml-common-tq.h): step t's state = 16 stream bits from bit (31 - t) * K, LSB first, circular modulo 32 K+ // bits, masked to 15 bits (TQ2_T: qs[(t + 31) % 32] << 8 | qs[t]); x = state * 0x9e3779b1; points lut[x >> 21] and+ // lut[(x >> 10) & 2047]; weights = fp16 values * d. Compared bit for bit with ref::dequant on the real tensors.+ extern const uint16_t* tq_lut_for_test();+ long blocks = 0, bad = 0;+ const gguf::Tensor* ts[3] = {tg_, tu_, td_};+ if (!tg_) { std::fprintf(stderr, "--decode-check needs --gguf\n"); return 2; }+ for (const gguf::Tensor* t : ts) {+ const int ty = t->type, K = ty == ref::TQ2_T ? 8 : ty == ref::TQK6 ? 6 : 7, bb = ref::block_bytes(ty);+ const uint8_t* base = (const uint8_t*)gf.map + t->offset;+ const size_t nb = t->bytes / bb;+ const size_t step = std::max<size_t>(1, nb / 200000); // up to 200k blocks a tensor, spread over it+ for (size_t i = 0; i < nb; i += step) {+ const uint8_t* b = base + i * bb;+ float want[128], got[128];+ const float d = ref::f16_to_f32((uint16_t)(b[0] | (b[1] << 8)));+ const uint8_t* qs = b + 2;+ for (int st = 0; st < 32; ++st) {+ uint32_t state;+ if (K == 8) state = (((uint32_t)qs[(st + 31) % 32] << 8) | qs[st]) & 0x7fff;+ else {+ const int off = ((31 - st) * K) % (32 * K);+ uint32_t v = 0;+ for (int bit = 0; bit < 16; ++bit) {+ const int pos = (off + bit) % (32 * K);+ v |= (uint32_t)((qs[pos / 8] >> (pos % 8)) & 1) << bit;+ }+ state = v & 0x7fff;+ }+ const uint32_t x = state * 0x9e3779b1u;+ const uint16_t* lut = tq_lut_for_test();+ const uint32_t pa = x >> 21, pb = (x >> 10) & 2047u;+ want[4 * st] = ref::f16_to_f32(lut[2 * pa]) * d;+ want[4 * st + 1] = ref::f16_to_f32(lut[2 * pa + 1]) * d;+ want[4 * st + 2] = ref::f16_to_f32(lut[2 * pb]) * d;+ want[4 * st + 3] = ref::f16_to_f32(lut[2 * pb + 1]) * d;+ }+ ref::dequant(ty, b, got, 128);+ bad += std::memcmp(want, got, sizeof want) != 0;+ ++blocks;+ }+ }+ std::printf("decode-check layer %d: %ld blocks of the real tensors, %ld differ from the format's definition %s\n", real_layer,+ blocks, bad, bad == 0 ? "PASS" : "FAIL");+ return bad == 0 ? 0 : 1;+ }if (had_test) {// the Metal rotation kernels against the CPU's (hadamard_rotate), bit for bitint bad = 0;diff --git a/setup.py b/setup.pyindex af4d1e340..33f59563e 100644--- a/setup.py+++ b/setup.py@@ -7281,8 +7281,8 @@ def mac_control_setup(a) -> int:fail(problems[0])step(2, "your choices")- app_sizes = [m for m, d in MF.MAC_MODELS.items() if d.get("family", "qwen") == "qwen"]- if a.family not in (None, "qwen"):+ app_sizes = [m for m, d in MF.MAC_MODELS.items() if d.get("family", "qwen") in ("qwen", "gyro")] # Gyro: one file, no repack+ if a.family not in (None, "qwen", "gyro"):fail(f"the app on a Mac runs the Qwen3.8-Flash-Next sizes ({', '.join(app_sizes)}); {a.family} is not one of them","choose Qwen3.8-Flash-Next")rec, notes = MF.recommend_model(ram)@@ -7298,7 +7298,8 @@ def mac_control_setup(a) -> int:if ctx > MF.TESTED_CONTEXT:say(f" context {ctx}: measured on the Mac up to {MF.TESTED_CONTEXT} (needles found at 8K-32K); longer is kept")ok(f"{model}, context {ctx}")- fam = FAMILIES["qwen"]+ family = MF.MAC_MODELS[model].get("family", "qwen")+ fam = FAMILIES[family]tag = fam["tag"] + modelstream = MF.streams(model, ram)@@ -7319,6 +7320,15 @@ def mac_control_setup(a) -> int:if vision != "none":warn("pictures are off for now: the picture reader comes with the engine (run setup again once it is installed)")vision = "none"+ else:+ if family == "gyro": # an engine from before the trellis kernels refuses the file when it loads+ try:+ has_tq = b"inputs are rotated on the GPU" in Path(found.exe).read_bytes()+ except (OSError, AttributeError):+ has_tq = True # cannot tell: not a reason to hold the install back+ if not has_tq:+ warn(f"the Metal engine component {found.version} has no kernels for Gyro's trellis experts yet: the model is "+ "downloaded and prepared, and it runs once the app or its update feed brings a newer engine")ok("engine: the engine component (looked up when the model loads)")ok("images: " + {"none": "off", "gpu": "on (read on the Mac's GPU)", "cpu": "on (read on the CPU)"}[vision])@@ -7367,7 +7377,8 @@ def mac_control_setup(a) -> int:pack = find_in(roots, f"packs/{tag.lower()}") or data / "packs" / tag.lower()env = dict(os.environ, STRATA_GGUF_PY=str(llama / "gguf-py"))if not (pack / "native_experts.txt").exists() or not (pack / "tokenizer" / "vocab.json").exists():- run([sys.executable, str(ROOT / "tools" / "iq_pack.py"), "--gguf", str(files[0]), "--out", str(pack)], env=env)+ run([sys.executable, str(ROOT / "tools" / "iq_pack.py"), "--gguf", str(files[0]), "--out", str(pack),+ *fam.get("pack_args", [])], env=env) # Gyro: --compat-bf16ok(f"model prepared: {pack}")use_mtp = MF.stream_mtp(stream, ram)mtp = (find_in(roots, "mtp/mtp-q2_0.gguf") or data / "mtp" / "mtp-q2_0.gguf").parent@@ -7388,7 +7399,8 @@ def mac_control_setup(a) -> int:step(7, "writing the run settings")port = a.port or 8080- args = MF.engine_args(pack, files, rt, ctx, stream, ram, ROOT / "data" / "expert-profile.bin")+ args = MF.engine_args(pack, [files[0], files[0]] if family == "gyro" else files, rt, ctx, stream, ram,+ ROOT / "data" / "expert-profile.bin") # Gyro: one file, the model and its n-gram tablecfg = {"exe": engine_exe(Path("engine")), "args": args, "cwd": str(ROOT), "tokenizer": str(pack / "tokenizer"),"model_name": f"{fam['name']}-{model.lower()}", "log": str(home() / f"strata-{tag.lower()}.log"),"lib_dirs": [], "port": port, "backend": "metal", "env": {"STRATA_METAL_STREAM": "1" if stream else "0"}}diff --git a/strata_control/mac_fit.py b/strata_control/mac_fit.pyindex 94183e156..68a7b1063 100644--- a/strata_control/mac_fit.py+++ b/strata_control/mac_fit.py@@ -26,6 +26,11 @@ MAC_MODELS = {"UD-Q4_K_XL": {"shard1_gb": 82.6, "tested": True, "family": "unsloth", "shards": 4},"UD-Q5_K_XL": {"shard1_gb": 103.9, "tested": True, "family": "unsloth", "shards": 6},"UD-Q6_K_XL": {"shard1_gb": 114.8, "tested": True, "family": "unsloth", "shards": 6},+ # agentionai's Gyro files (family "gyro", docs/GYRO.md): ONE file with the n-gram table inside (58.5 / 92.0 GB to download);+ # shard1_gb is what the GPU needs of it, without that table (the engine reads the table from the SSD a few rows at a time).+ # The trellis experts run on the Metal engine from the release that added metal/kernels' TQ types (docs/MAC.md).+ "Gyro-S": {"shard1_gb": 29.7, "tested": True, "family": "gyro", "shards": 1, "file_gb": 58.5},+ "Gyro-M": {"shard1_gb": 63.2, "tested": False, "family": "gyro", "shards": 1, "file_gb": 92.0},}# The recommended sizes (owner's decision): IQ3_S is the default where it stays resident, UD-Q4_K_XL the higher-quality one@@ -57,7 +62,9 @@ CONTEXTS = [8192, 32768, 65536, 131072]MAC_ABOUT = {"Q2_0": "2-bit, the fastest", "IQ3_XXS": "3-bit i-quant, better quality","IQ3_S": "3.5-bit i-quant, the best quality (matches the full model), the slowest","UD-Q4_K_XL": "Unsloth ~4.5-bit (111 GB download)", "UD-Q5_K_XL": "Unsloth ~5.5-bit (158 GB download)",- "UD-Q6_K_XL": "Unsloth ~6.5-bit (169 GB download)"}+ "UD-Q6_K_XL": "Unsloth ~6.5-bit (169 GB download)",+ "Gyro-S": "agentionai's trellis experts, the smallest (58 GB download, the n-gram table is inside it)",+ "Gyro-M": "agentionai's trellis experts, about IQ3_XXS quality in less memory (92 GB download)"}TESTED_CONTEXT = 32768 # needles and the KL harness ran to 32K on the Mac# Below this much unified memory the app says a model does not fit (a Mac this small was never run: 32 GB is the smallest that was).# The Terminal keeps its rule of recommending and never forcing: the person can still pick one there.diff --git a/tools/strata_mcp.py b/tools/strata_mcp.pyindex c863e3b41..64c2f6ef6 100644--- a/tools/strata_mcp.py+++ b/tools/strata_mcp.py@@ -1014,11 +1014,12 @@ class Strata:hw["nvidia_smi"] = Falsereturn hw- def _mac_sizes(self, models: dict) -> list:- """The sizes the app offers on a Mac: the GSQ-RCO files of the Qwen family (two shards, no repack). Unsloth's many-shard files need- metal/tools/metal_repack.py, which the Terminal's setup uses; the app does not offer them yet."""+ def _mac_sizes(self, models: dict, family: str = "qwen") -> list:+ """The sizes the app offers on a Mac: the GSQ-RCO files of the Qwen family (two shards, no repack) and agentionai's Gyro files (one+ file, no repack). Unsloth's many-shard files need metal/tools/metal_repack.py, which the Terminal's setup uses; the app does not+ offer them yet."""mf = self.mac()- return [m for m, d in mf.MAC_MODELS.items() if d.get("family", "qwen") == "qwen" and m in models] if mf else []+ return [m for m, d in mf.MAC_MODELS.items() if d.get("family", "qwen") == family and m in models] if mf else []def _mac_verdict(self, model: str, ram: float) -> str:mf = self.mac()@@ -1061,18 +1062,24 @@ class Strata:models, families, _ctx, _src = self.tables()ram = float(hw.get("ram_gb") or 0)have = {c.stem[len("strata-"):] for c in self.configs()}- fd = families["qwen"]- sizes = []- for m in self._mac_sizes(models):- d = models[m]- tag = (fd.get("tag", "") + m).lower()- sizes.append({"model": m, "about": d["about"], "download_gb": d["download_gb"], "experimental": False,- "images": d.get("vision", fd.get("vision", True)) is not False, # the Mac engine carries the picture reader (Metal)- "ram_needed_gb": d["ram_gb"], "experts_gb": d.get("arena_gb"), "mem_total": False,- "memory_gb": round(float(d.get("arena_gb") or d["ram_gb"]) + 1.0, 1),- "on_this_pc": self._mac_verdict(m, ram) if ram else None, "installed": tag in have, "id": tag})- return [{"family": "qwen", "title": fd["title"], "about": fd.get("about"), "experimental": False,- "images": any(x["images"] for x in sizes), "sizes": sizes}]+ out = []+ for family in ("qwen", "gyro"): # Gyro after the regular sizes, as on a PC: never the default+ fd = families.get(family)+ if fd is None:+ continue+ sizes = []+ for m in self._mac_sizes(models, family):+ d = models[m]+ tag = (fd.get("tag", "") + m).lower()+ sizes.append({"model": m, "about": d["about"], "download_gb": d["download_gb"], "experimental": False,+ "images": d.get("vision", fd.get("vision", True)) is not False, # the Mac engine carries the picture reader (Metal)+ "ram_needed_gb": d["ram_gb"], "experts_gb": d.get("arena_gb"), "mem_total": False,+ "memory_gb": round(float(d.get("arena_gb") or d["ram_gb"]) + 1.0, 1),+ "on_this_pc": self._mac_verdict(m, ram) if ram else None, "installed": tag in have, "id": tag})+ if sizes:+ out.append({"family": family, "title": fd["title"], "about": fd.get("about"), "experimental": False,+ "images": any(x["images"] for x in sizes), "sizes": sizes})+ return outdef recommend(self, hw: dict) -> dict:"""setup.py's own rules: the size it preselects (IQ3_XXS from 60 GB of RAM, else the first, Q2_0), the