Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
strata-vision reads only the vocabulary and header of the text model; Gyro reads pictures
Niko12213fe84ec28d13
diff --git a/setup.py b/setup.pyindex 26457ae5d..4e0224aee 100644--- a/setup.py+++ b/setup.py@@ -366,15 +366,17 @@ MODELS = {# checked end to end on an RTX 5070 and an RTX 3060 against IQ3_XXS's answers on the same picture.# Gyro (agentionai): "ram_gb" is the memory the weights need in total, VRAM and RAM together ("mem_total"): Gyro-S# 27.6 GB, Gyro-M 35 GB (the rest of the download is the n-gram table, read from the SSD). Neither is anyone's- # default: setup and the MCP server mention them where the other sizes do not fit. No images yet (untested).+ # default: setup and the MCP server mention them where the other sizes do not fit. Images: the same mmproj as every+ # Flash-Next model; strata-vision reads only the vocabulary and header of the Gyro GGUF (tools/vision/gguf_meta.h),+ # whose TQK6 / TQK7 tensors llama.cpp does not know."Gyro-S": {"about": "the smallest option, for PCs with little memory (27.6 GB of VRAM + RAM together); lower quality ""than the 2-3-bit sizes, and the 2-bit Q2_0 decodes faster","download_gb": 58.5, "ram_gb": 27.6, "arena_gb": 25.8, "families": ("gyro",), "mem_total": True,- "file": "Qwen3.8-Flash-Next-Gyro-S-TQ1_0.gguf", "shards": 1, "engine": GYRO_ENGINE, "vision": False},+ "file": "Qwen3.8-Flash-Next-Gyro-S-TQ1_0.gguf", "shards": 1, "engine": GYRO_ENGINE, "vision": True},"Gyro-M": {"about": "about IQ3_XXS quality with about 20% less GPU memory (35 GB of VRAM + RAM together); decodes ""about 20% slower than Q2_0 on 12 GB cards","download_gb": 92.0, "ram_gb": 35.0, "arena_gb": 32.1, "families": ("gyro",), "mem_total": True,- "file": "Qwen3.8-Flash-Next-Gyro-M-TQ2_0.gguf", "shards": 1, "engine": GYRO_ENGINE, "vision": False},+ "file": "Qwen3.8-Flash-Next-Gyro-M-TQ2_0.gguf", "shards": 1, "engine": GYRO_ENGINE, "vision": True},"NVFP4": {"about": "best quality of the NVFP4 versions (GPTQ, sergqwer's strata-nvfp4 pack); RTX 50 has FP4 tensor cores, ""other NVIDIA cards run it at 8-bit precision", "download_gb": 128.0, "ram_gb": 80, "arena_gb": 68.0,"families": ("nvfp4",), "nvidia_only": True, "vision": True, "repo": NVFP4_GPTQ_REPO,@@ -511,7 +513,7 @@ FAMILIES = {"about": "Gyro-S (27.6 GB of memory) and Gyro-M (35 GB): the smallest sizes, for PCs with little memory; ""58 and 92 GB downloads (the files include the n-gram table)","hf": hf("agentionai/Qwen3.8-Flash-Next-Gyro-GGUF"), "file": "Qwen3.8-Flash-Next-{q}.gguf", "shards": 1,- "tag": "gyro-", "name": "qwen3.8-flash-next-gyro", "vision": False, "pack_args": ["--compat-bf16"],+ "tag": "gyro-", "name": "qwen3.8-flash-next-gyro", "vision": True, "pack_args": ["--compat-bf16"],"mmproj_hf": hf("ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF"), "mmproj": "mmproj-Qwen3.8-Flash-Next-BF16.gguf","sha256": GYRO_FILES,"license": "Qwen Community License 1.0 (Qwen3.8-Flash-Next's; the Gyro files are agentionai's quantization): "diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.pyindex 76b7d2382..95cd8f586 100644--- a/tests/terminal/test_launch.py+++ b/tests/terminal/test_launch.py@@ -681,7 +681,7 @@ class FlagsCommand(unittest.TestCase):self.assertIn("Did you mean 'Office'", err)def test_vision_is_turned_off_for_a_model_without_pictures(self):- p = self.plan("--model", "gyro-s", "--vision", "on")+ p = self.plan("--model", "nvfp4-qad", "--vision", "on")self.assertEqual(p["settings"]["vision"], "off")self.assertTrue(any("cannot read pictures" in w for w in p["warnings"]))diff --git a/tools/strata_mcp.py b/tools/strata_mcp.pyindex 61af7795a..5fb540c79 100644--- a/tools/strata_mcp.py+++ b/tools/strata_mcp.py@@ -80,11 +80,11 @@ FALLBACK_MODELS = {"Gyro-S": {"about": "the smallest option, for PCs with little memory (27.6 GB of VRAM + RAM together); lower quality ""than the 2-3-bit sizes, and the 2-bit Q2_0 decodes faster","download_gb": 58.5, "ram_gb": 27.6, "arena_gb": 25.8, "families": ("gyro",), "mem_total": True,- "vision": False},+ "vision": True},"Gyro-M": {"about": "about IQ3_XXS quality with about 20% less GPU memory (35 GB of VRAM + RAM together); decodes ""about 20% slower than Q2_0 on 12 GB cards","download_gb": 92.0, "ram_gb": 35.0, "arena_gb": 32.1, "families": ("gyro",), "mem_total": True,- "vision": False},+ "vision": True},"NVFP4": {"about": "4-bit NVFP4 experts, GPTQ (sergqwer's strata-nvfp4 pack); RTX 50 has FP4 tensor cores, other ""NVIDIA cards run it at 8-bit precision", "download_gb": 128.0, "ram_gb": 80, "arena_gb": 68.0,"families": ("nvfp4",), "vision": True},diff --git a/tools/vision/CMakeLists.txt b/tools/vision/CMakeLists.txtindex d6518c71d..ed42e6254 100644--- a/tools/vision/CMakeLists.txt+++ b/tools/vision/CMakeLists.txt@@ -59,3 +59,9 @@ add_executable(strata-vision strata_vision.cpp)target_link_libraries(strata-vision PRIVATE mtmd llama)target_compile_features(strata-vision PRIVATE cxx_std_17)set_target_properties(strata-vision PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)++# opens a GGUF that has tensor types llama.cpp does not know (a Gyro quantization's TQK6 / TQK7) for its vocabulary only+add_executable(strata-vision-gguf-test EXCLUDE_FROM_ALL gguf_meta_test.cpp)+target_link_libraries(strata-vision-gguf-test PRIVATE llama)+target_compile_features(strata-vision-gguf-test PRIVATE cxx_std_17)+set_target_properties(strata-vision-gguf-test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)diff --git a/tools/vision/gguf_meta.h b/tools/vision/gguf_meta.hnew file mode 100644index 000000000..2741a71df--- /dev/null+++ b/tools/vision/gguf_meta.h@@ -0,0 +1,156 @@+// tools/vision/gguf_meta.h - open a model GGUF for its vocabulary only, whatever its weight tensors are.+//+// The picture reader needs the text model for one thing: its tokenizer (and the header values that go with it). llama.cpp's+// loader reads the whole tensor directory first and refuses a file with a tensor type it does not know - and a Gyro+// quantization (TQK6 / TQK7, ids 145 and 146) and other Strata-private types are exactly that - so even a vocab-only+// load failed with "cannot open the text model". strata_open_text_model() therefore copies the key/value header of the+// GGUF (everything before the tensor directory) into a small temporary GGUF without tensors, and loads THAT vocab-only.+// The weights are never read, mapped or type-checked, and the split keys are dropped so the loader does not go looking+// for the other shards. If the header cannot be read (a type this reader does not know), it falls back to loading the+// file itself, as before.+#pragma once++#include "gguf.h"+#include "llama.h"++#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <string>+#include <vector>++#if defined(_WIN32)+#include <process.h>+#define STRATA_GETPID _getpid+#else+#include <unistd.h>+#define STRATA_GETPID getpid+#endif++namespace strata_gguf {++struct Reader {+ FILE* f = nullptr;+ std::string err;+ bool take(void* dst, size_t n) {+ if (std::fread(dst, 1, n, f) != n) { err = "the GGUF header is cut short"; return false; }+ return true;+ }+ template <class T> bool num(T& v) { return take(&v, sizeof v); }+ bool str(std::string& s, uint64_t cap = (1ull << 26)) {+ uint64_t n = 0;+ if (!num(n)) return false;+ if (n > cap) { err = "a string in the GGUF header is too long"; return false; }+ s.resize((size_t) n);+ return n == 0 || take(&s[0], (size_t) n);+ }+};++inline size_t scalar_size(uint32_t t) {+ switch (t) {+ case GGUF_TYPE_UINT8: case GGUF_TYPE_INT8: case GGUF_TYPE_BOOL: return 1;+ case GGUF_TYPE_UINT16: case GGUF_TYPE_INT16: return 2;+ case GGUF_TYPE_UINT32: case GGUF_TYPE_INT32: case GGUF_TYPE_FLOAT32: return 4;+ case GGUF_TYPE_UINT64: case GGUF_TYPE_INT64: case GGUF_TYPE_FLOAT64: return 8;+ default: return 0;+ }+}++inline bool dropped_key(const std::string& k) {+ return k == "split.no" || k == "split.count" || k == "split.tensors.count";+}++// Copy the key/value header of `src` into `dst` as a GGUF with no tensors. False (and `err`) when the header holds+// something this reader does not handle.+inline bool copy_meta(const std::string& src, const std::string& dst, std::string& err) {+ Reader r;+ r.f = std::fopen(src.c_str(), "rb");+ if (!r.f) { err = "cannot open " + src; return false; }+ struct Closer { FILE* f; ~Closer() { if (f) std::fclose(f); } } closer{r.f};+ char magic[4];+ uint32_t version = 0;+ uint64_t n_tensors = 0, n_kv = 0;+ if (!r.take(magic, 4) || std::memcmp(magic, "GGUF", 4) != 0) { err = r.err.empty() ? "not a GGUF file" : r.err; return false; }+ if (!r.num(version) || !r.num(n_tensors) || !r.num(n_kv)) { err = r.err; return false; }+ if (version < 2 || version > 3) { err = "GGUF version " + std::to_string(version) + " is not handled"; return false; }+ if (n_kv > 100000) { err = "too many keys in the GGUF header"; return false; }+ gguf_context* out = gguf_init_empty();+ if (!out) { err = "out of memory"; return false; }+ struct Freer { gguf_context* g; ~Freer() { gguf_free(g); } } freer{out};+ for (uint64_t i = 0; i < n_kv; ++i) {+ std::string key;+ uint32_t type = 0;+ if (!r.str(key, 1 << 16) || !r.num(type)) { err = r.err; return false; }+ const bool keep = !dropped_key(key);+ if (type == GGUF_TYPE_STRING) {+ std::string v;+ if (!r.str(v)) { err = r.err; return false; }+ if (keep) gguf_set_val_str(out, key.c_str(), v.c_str());+ } else if (type == GGUF_TYPE_ARRAY) {+ uint32_t et = 0;+ uint64_t n = 0;+ if (!r.num(et) || !r.num(n)) { err = r.err; return false; }+ if (n > 50'000'000ull) { err = "an array in the GGUF header is too long"; return false; }+ if (et == GGUF_TYPE_STRING) {+ std::vector<std::string> items((size_t) n);+ for (auto& s : items) if (!r.str(s)) { err = r.err; return false; }+ if (keep) {+ std::vector<const char*> ptrs;+ ptrs.reserve(items.size());+ for (const auto& s : items) ptrs.push_back(s.c_str());+ gguf_set_arr_str(out, key.c_str(), ptrs.data(), ptrs.size());+ }+ } else if (scalar_size(et)) {+ std::vector<uint8_t> raw((size_t) (n * scalar_size(et)));+ if (!raw.empty() && !r.take(raw.data(), raw.size())) { err = r.err; return false; }+ if (keep) gguf_set_arr_data(out, key.c_str(), (gguf_type) et, raw.data(), (size_t) n);+ } else { err = "an array of arrays in the GGUF header is not handled"; return false; }+ } else if (scalar_size(type)) {+ uint64_t raw = 0;+ if (!r.take(&raw, scalar_size(type))) { err = r.err; return false; } // little endian, as the file+ if (!keep) continue;+ switch (type) {+ case GGUF_TYPE_UINT8: gguf_set_val_u8(out, key.c_str(), (uint8_t) raw); break;+ case GGUF_TYPE_INT8: gguf_set_val_i8(out, key.c_str(), (int8_t) raw); break;+ case GGUF_TYPE_UINT16: gguf_set_val_u16(out, key.c_str(), (uint16_t) raw); break;+ case GGUF_TYPE_INT16: gguf_set_val_i16(out, key.c_str(), (int16_t) raw); break;+ case GGUF_TYPE_UINT32: gguf_set_val_u32(out, key.c_str(), (uint32_t) raw); break;+ case GGUF_TYPE_INT32: gguf_set_val_i32(out, key.c_str(), (int32_t) raw); break;+ case GGUF_TYPE_FLOAT32: { float v; std::memcpy(&v, &raw, 4); gguf_set_val_f32(out, key.c_str(), v); break; }+ case GGUF_TYPE_BOOL: gguf_set_val_bool(out, key.c_str(), raw != 0); break;+ case GGUF_TYPE_UINT64: gguf_set_val_u64(out, key.c_str(), raw); break;+ case GGUF_TYPE_INT64: gguf_set_val_i64(out, key.c_str(), (int64_t) raw); break;+ case GGUF_TYPE_FLOAT64: { double v; std::memcpy(&v, &raw, 8); gguf_set_val_f64(out, key.c_str(), v); break; }+ default: break;+ }+ } else { err = "a value type in the GGUF header is not handled"; return false; }+ }+ if (!gguf_write_to_file(out, dst.c_str(), true)) { err = "cannot write " + dst; return false; }+ return true;+}++} // namespace strata_gguf++// The text model for the picture reader: vocabulary and header only. `how` says which way it was opened.+inline llama_model* strata_open_text_model(const std::string& model, std::string* how = nullptr) {+ llama_model_params mp = llama_model_default_params();+ mp.vocab_only = true;+ const char* tmpdir = std::getenv("TMPDIR");+ if (!tmpdir || !*tmpdir) tmpdir = std::getenv("TEMP");+ if (!tmpdir || !*tmpdir) tmpdir = std::getenv("TMP");+ if (!tmpdir || !*tmpdir) tmpdir = "/tmp";+ const std::string meta = std::string(tmpdir) + "/strata-vision-meta-" + std::to_string((long) STRATA_GETPID()) + ".gguf";+ std::string err;+ llama_model* m = nullptr;+ if (strata_gguf::copy_meta(model, meta, err)) {+ m = llama_model_load_from_file(meta.c_str(), mp);+ if (how) *how = m ? "header only" : "header copy was not accepted";+ } else if (how) *how = "header copy failed: " + err;+ std::remove(meta.c_str());+ if (!m) { // the file as it is (a type of header this reader does not handle, or an llama.cpp that wants more)+ m = llama_model_load_from_file(model.c_str(), mp);+ if (how) *how += m ? ", loaded the file itself" : ", the file itself did not load either";+ }+ return m;+}diff --git a/tools/vision/gguf_meta_test.cpp b/tools/vision/gguf_meta_test.cppnew file mode 100644index 000000000..c0262eafd--- /dev/null+++ b/tools/vision/gguf_meta_test.cpp@@ -0,0 +1,110 @@+// tools/vision/gguf_meta_test.cpp - strata-vision opens a GGUF whose weight tensors have types llama.cpp does not know.+//+// Writes a small GGUF by hand (a tokenizer, two tensors of the Strata-private types TQK6 = 145 and TQK7 = 146, and split+// keys that point at shards which are not there), then checks that+// - llama.cpp itself cannot load it (the reason the picture reader used to say "cannot open the text model"),+// - strata_open_text_model() can, and sees the same vocabulary.+// Exit code 0 = all checks passed.+#include "gguf_meta.h"++#include <cstdint>+#include <cstdio>+#include <string>+#include <vector>++namespace {++struct Out {+ std::vector<uint8_t> b;+ void raw(const void* p, size_t n) { const auto* c = (const uint8_t*) p; b.insert(b.end(), c, c + n); }+ template <class T> void num(T v) { raw(&v, sizeof v); }+ void str(const std::string& s) { num<uint64_t>(s.size()); raw(s.data(), s.size()); }+ void kv_str(const std::string& k, const std::string& v) { str(k); num<uint32_t>(GGUF_TYPE_STRING); str(v); }+ void kv_u32(const std::string& k, uint32_t v) { str(k); num<uint32_t>(GGUF_TYPE_UINT32); num<uint32_t>(v); }+ void kv_strs(const std::string& k, const std::vector<std::string>& v) {+ str(k); num<uint32_t>(GGUF_TYPE_ARRAY); num<uint32_t>(GGUF_TYPE_STRING); num<uint64_t>(v.size());+ for (const auto& s : v) str(s);+ }+ void kv_i32s(const std::string& k, const std::vector<int32_t>& v) {+ str(k); num<uint32_t>(GGUF_TYPE_ARRAY); num<uint32_t>(GGUF_TYPE_INT32); num<uint64_t>(v.size());+ raw(v.data(), v.size() * 4);+ }+};++int failures = 0;+void check(bool ok, const char* what) {+ std::printf("%s %s\n", ok ? "ok " : "FAIL", what);+ if (!ok) ++failures;+}++bool write_fixture(const std::string& path, size_t& n_vocab, bool split) {+ std::vector<std::string> tokens;+ for (int i = 0; i < 256; ++i) tokens.push_back(std::string("t") + std::to_string(i));+ tokens.push_back("<|endoftext|>");+ n_vocab = tokens.size();+ std::vector<int32_t> types(tokens.size(), 1);+ types.back() = 3;+ Out o;+ o.raw("GGUF", 4);+ o.num<uint32_t>(3);+ o.num<uint64_t>(2); // two tensors+ const uint64_t n_kv = split ? 9 : 6;+ o.num<uint64_t>(n_kv);+ o.kv_str("general.architecture", "llama");+ o.kv_str("tokenizer.ggml.model", "gpt2");+ o.kv_str("tokenizer.ggml.pre", "default");+ o.kv_strs("tokenizer.ggml.tokens", tokens);+ o.kv_i32s("tokenizer.ggml.token_type", types);+ o.kv_strs("tokenizer.ggml.merges", {"t1 t2"});+ if (split) {+ o.kv_u32("split.no", 0);+ o.kv_u32("split.count", 2);+ o.kv_u32("split.tensors.count", 4);+ }+ // two tensors of types llama.cpp's ggml does not have (145 = TQK6, 146 = TQK7), 128 elements each+ for (int i = 0; i < 2; ++i) {+ o.str(i == 0 ? "blk.0.ffn_gate_exps.weight" : "blk.0.ffn_up_exps.weight");+ o.num<uint32_t>(1);+ o.num<uint64_t>(128);+ o.num<uint32_t>(145 + i);+ o.num<uint64_t>(i * 64);+ }+ while (o.b.size() % 32) o.b.push_back(0);+ o.b.resize(o.b.size() + 128, 0);+ FILE* f = std::fopen(path.c_str(), "wb");+ if (!f) return false;+ const bool ok = std::fwrite(o.b.data(), 1, o.b.size(), f) == o.b.size();+ std::fclose(f);+ return ok;+}++} // namespace++int main(int argc, char** argv) {+ const std::string dir = argc > 1 ? argv[1] : ".";+ llama_backend_init();+ for (int split = 0; split < 2; ++split) {+ const std::string label = split ? " (a first shard whose other shards are missing)" : "";+ const std::string path = dir + "/strata-vision-gguf-test-" + std::to_string(split) + ".gguf";+ size_t n_vocab = 0;+ check(write_fixture(path, n_vocab, split == 1), ("wrote a GGUF with tensor types 145 and 146" + label).c_str());+ llama_model_params mp = llama_model_default_params();+ mp.vocab_only = true;+ llama_model* plain = llama_model_load_from_file(path.c_str(), mp);+ std::printf("note llama.cpp itself %s this file\n", plain ? "accepts" : "refuses");+ if (plain) llama_model_free(plain);+ std::string how;+ llama_model* m = strata_open_text_model(path, &how);+ check(m != nullptr, ("strata_open_text_model opens it (" + how + ")" + label).c_str());+ if (m) {+ check((size_t) llama_vocab_n_tokens(llama_model_get_vocab(m)) == n_vocab, "and sees the whole vocabulary");+ llama_model_free(m);+ }+ check(how.find("header only") == 0, "by reading the header, not the weights");+ std::remove(path.c_str());+ }+ check(strata_open_text_model(dir + "/does-not-exist.gguf") == nullptr, "a file that is not there gives no model");+ llama_backend_free();+ std::printf("%s\n", failures ? "GGUF-META FAIL" : "GGUF-META PASS");+ return failures ? 1 : 0;+}diff --git a/tools/vision/strata_vision.cpp b/tools/vision/strata_vision.cppindex 9a06ce1fa..bb58a2a15 100644--- a/tools/vision/strata_vision.cpp+++ b/tools/vision/strata_vision.cpp@@ -15,6 +15,7 @@// row i at grid position (x = i % nx, y = i / nx). The text model is opened vocab-only (no weights).#include "ggml-cpu.h"#include "gguf.h"+#include "gguf_meta.h"#include "llama.h"#include "mtmd.h"#include "mtmd-helper.h"@@ -112,10 +113,11 @@ int main(int argc, char** argv) {}}#endif- llama_model_params mp = llama_model_default_params();- mp.vocab_only = true;- llama_model* text = llama_model_load_from_file(model.c_str(), mp);- if (!text) { std::printf("ERR cannot open the text model %s\n", model.c_str()); std::fflush(stdout); return 1; }+ // the vocabulary and header only: the weights (and their tensor types, which can be Strata-private ones such as+ // Gyro's TQK6 / TQK7) are never touched - see gguf_meta.h+ std::string how;+ llama_model* text = strata_open_text_model(model, &how);+ if (!text) { std::printf("ERR cannot open the text model %s (%s)\n", model.c_str(), how.c_str()); std::fflush(stdout); return 1; }mtmd_context_params cp = mtmd_context_params_default();cp.use_gpu = gpu;