Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

strata-vision reads only the vocabulary and header of the text model; Gyro reads pictures

Niko12213fe84ec28d13
diff --git a/setup.py b/setup.py
index 26457ae5d..4e0224aee 100644
--- a/setup.py
+++ b/setup.py
@@ -366,15 +366,17 @@ MODELS = {
# checked end to end on an RTX 5070 and an RTX 3060 against IQ3_XXS's answers on the same picture.
# Gyro (agentionai): "ram_gb" is the memory the weights need in total, VRAM and RAM together ("mem_total"): Gyro-S
# 27.6 GB, Gyro-M 35 GB (the rest of the download is the n-gram table, read from the SSD). Neither is anyone's
- # default: setup and the MCP server mention them where the other sizes do not fit. No images yet (untested).
+ # default: setup and the MCP server mention them where the other sizes do not fit. Images: the same mmproj as every
+ # Flash-Next model; strata-vision reads only the vocabulary and header of the Gyro GGUF (tools/vision/gguf_meta.h),
+ # whose TQK6 / TQK7 tensors llama.cpp does not know.
"Gyro-S": {"about": "the smallest option, for PCs with little memory (27.6 GB of VRAM + RAM together); lower quality "
"than the 2-3-bit sizes, and the 2-bit Q2_0 decodes faster",
"download_gb": 58.5, "ram_gb": 27.6, "arena_gb": 25.8, "families": ("gyro",), "mem_total": True,
- "file": "Qwen3.8-Flash-Next-Gyro-S-TQ1_0.gguf", "shards": 1, "engine": GYRO_ENGINE, "vision": False},
+ "file": "Qwen3.8-Flash-Next-Gyro-S-TQ1_0.gguf", "shards": 1, "engine": GYRO_ENGINE, "vision": True},
"Gyro-M": {"about": "about IQ3_XXS quality with about 20% less GPU memory (35 GB of VRAM + RAM together); decodes "
"about 20% slower than Q2_0 on 12 GB cards",
"download_gb": 92.0, "ram_gb": 35.0, "arena_gb": 32.1, "families": ("gyro",), "mem_total": True,
- "file": "Qwen3.8-Flash-Next-Gyro-M-TQ2_0.gguf", "shards": 1, "engine": GYRO_ENGINE, "vision": False},
+ "file": "Qwen3.8-Flash-Next-Gyro-M-TQ2_0.gguf", "shards": 1, "engine": GYRO_ENGINE, "vision": True},
"NVFP4": {"about": "best quality of the NVFP4 versions (GPTQ, sergqwer's strata-nvfp4 pack); RTX 50 has FP4 tensor cores, "
"other NVIDIA cards run it at 8-bit precision", "download_gb": 128.0, "ram_gb": 80, "arena_gb": 68.0,
"families": ("nvfp4",), "nvidia_only": True, "vision": True, "repo": NVFP4_GPTQ_REPO,
@@ -511,7 +513,7 @@ FAMILIES = {
"about": "Gyro-S (27.6 GB of memory) and Gyro-M (35 GB): the smallest sizes, for PCs with little memory; "
"58 and 92 GB downloads (the files include the n-gram table)",
"hf": hf("agentionai/Qwen3.8-Flash-Next-Gyro-GGUF"), "file": "Qwen3.8-Flash-Next-{q}.gguf", "shards": 1,
- "tag": "gyro-", "name": "qwen3.8-flash-next-gyro", "vision": False, "pack_args": ["--compat-bf16"],
+ "tag": "gyro-", "name": "qwen3.8-flash-next-gyro", "vision": True, "pack_args": ["--compat-bf16"],
"mmproj_hf": hf("ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF"), "mmproj": "mmproj-Qwen3.8-Flash-Next-BF16.gguf",
"sha256": GYRO_FILES,
"license": "Qwen Community License 1.0 (Qwen3.8-Flash-Next's; the Gyro files are agentionai's quantization): "
diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.py
index 76b7d2382..95cd8f586 100644
--- a/tests/terminal/test_launch.py
+++ b/tests/terminal/test_launch.py
@@ -681,7 +681,7 @@ class FlagsCommand(unittest.TestCase):
self.assertIn("Did you mean 'Office'", err)
def test_vision_is_turned_off_for_a_model_without_pictures(self):
- p = self.plan("--model", "gyro-s", "--vision", "on")
+ p = self.plan("--model", "nvfp4-qad", "--vision", "on")
self.assertEqual(p["settings"]["vision"], "off")
self.assertTrue(any("cannot read pictures" in w for w in p["warnings"]))
diff --git a/tools/strata_mcp.py b/tools/strata_mcp.py
index 61af7795a..5fb540c79 100644
--- a/tools/strata_mcp.py
+++ b/tools/strata_mcp.py
@@ -80,11 +80,11 @@ FALLBACK_MODELS = {
"Gyro-S": {"about": "the smallest option, for PCs with little memory (27.6 GB of VRAM + RAM together); lower quality "
"than the 2-3-bit sizes, and the 2-bit Q2_0 decodes faster",
"download_gb": 58.5, "ram_gb": 27.6, "arena_gb": 25.8, "families": ("gyro",), "mem_total": True,
- "vision": False},
+ "vision": True},
"Gyro-M": {"about": "about IQ3_XXS quality with about 20% less GPU memory (35 GB of VRAM + RAM together); decodes "
"about 20% slower than Q2_0 on 12 GB cards",
"download_gb": 92.0, "ram_gb": 35.0, "arena_gb": 32.1, "families": ("gyro",), "mem_total": True,
- "vision": False},
+ "vision": True},
"NVFP4": {"about": "4-bit NVFP4 experts, GPTQ (sergqwer's strata-nvfp4 pack); RTX 50 has FP4 tensor cores, other "
"NVIDIA cards run it at 8-bit precision", "download_gb": 128.0, "ram_gb": 80, "arena_gb": 68.0,
"families": ("nvfp4",), "vision": True},
diff --git a/tools/vision/CMakeLists.txt b/tools/vision/CMakeLists.txt
index d6518c71d..ed42e6254 100644
--- a/tools/vision/CMakeLists.txt
+++ b/tools/vision/CMakeLists.txt
@@ -59,3 +59,9 @@ add_executable(strata-vision strata_vision.cpp)
target_link_libraries(strata-vision PRIVATE mtmd llama)
target_compile_features(strata-vision PRIVATE cxx_std_17)
set_target_properties(strata-vision PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+
+# opens a GGUF that has tensor types llama.cpp does not know (a Gyro quantization's TQK6 / TQK7) for its vocabulary only
+add_executable(strata-vision-gguf-test EXCLUDE_FROM_ALL gguf_meta_test.cpp)
+target_link_libraries(strata-vision-gguf-test PRIVATE llama)
+target_compile_features(strata-vision-gguf-test PRIVATE cxx_std_17)
+set_target_properties(strata-vision-gguf-test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
diff --git a/tools/vision/gguf_meta.h b/tools/vision/gguf_meta.h
new file mode 100644
index 000000000..2741a71df
--- /dev/null
+++ b/tools/vision/gguf_meta.h
@@ -0,0 +1,156 @@
+// tools/vision/gguf_meta.h - open a model GGUF for its vocabulary only, whatever its weight tensors are.
+//
+// The picture reader needs the text model for one thing: its tokenizer (and the header values that go with it). llama.cpp's
+// loader reads the whole tensor directory first and refuses a file with a tensor type it does not know - and a Gyro
+// quantization (TQK6 / TQK7, ids 145 and 146) and other Strata-private types are exactly that - so even a vocab-only
+// load failed with "cannot open the text model". strata_open_text_model() therefore copies the key/value header of the
+// GGUF (everything before the tensor directory) into a small temporary GGUF without tensors, and loads THAT vocab-only.
+// The weights are never read, mapped or type-checked, and the split keys are dropped so the loader does not go looking
+// for the other shards. If the header cannot be read (a type this reader does not know), it falls back to loading the
+// file itself, as before.
+#pragma once
+
+#include "gguf.h"
+#include "llama.h"
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#if defined(_WIN32)
+#include <process.h>
+#define STRATA_GETPID _getpid
+#else
+#include <unistd.h>
+#define STRATA_GETPID getpid
+#endif
+
+namespace strata_gguf {
+
+struct Reader {
+ FILE* f = nullptr;
+ std::string err;
+ bool take(void* dst, size_t n) {
+ if (std::fread(dst, 1, n, f) != n) { err = "the GGUF header is cut short"; return false; }
+ return true;
+ }
+ template <class T> bool num(T& v) { return take(&v, sizeof v); }
+ bool str(std::string& s, uint64_t cap = (1ull << 26)) {
+ uint64_t n = 0;
+ if (!num(n)) return false;
+ if (n > cap) { err = "a string in the GGUF header is too long"; return false; }
+ s.resize((size_t) n);
+ return n == 0 || take(&s[0], (size_t) n);
+ }
+};
+
+inline size_t scalar_size(uint32_t t) {
+ switch (t) {
+ case GGUF_TYPE_UINT8: case GGUF_TYPE_INT8: case GGUF_TYPE_BOOL: return 1;
+ case GGUF_TYPE_UINT16: case GGUF_TYPE_INT16: return 2;
+ case GGUF_TYPE_UINT32: case GGUF_TYPE_INT32: case GGUF_TYPE_FLOAT32: return 4;
+ case GGUF_TYPE_UINT64: case GGUF_TYPE_INT64: case GGUF_TYPE_FLOAT64: return 8;
+ default: return 0;
+ }
+}
+
+inline bool dropped_key(const std::string& k) {
+ return k == "split.no" || k == "split.count" || k == "split.tensors.count";
+}
+
+// Copy the key/value header of `src` into `dst` as a GGUF with no tensors. False (and `err`) when the header holds
+// something this reader does not handle.
+inline bool copy_meta(const std::string& src, const std::string& dst, std::string& err) {
+ Reader r;
+ r.f = std::fopen(src.c_str(), "rb");
+ if (!r.f) { err = "cannot open " + src; return false; }
+ struct Closer { FILE* f; ~Closer() { if (f) std::fclose(f); } } closer{r.f};
+ char magic[4];
+ uint32_t version = 0;
+ uint64_t n_tensors = 0, n_kv = 0;
+ if (!r.take(magic, 4) || std::memcmp(magic, "GGUF", 4) != 0) { err = r.err.empty() ? "not a GGUF file" : r.err; return false; }
+ if (!r.num(version) || !r.num(n_tensors) || !r.num(n_kv)) { err = r.err; return false; }
+ if (version < 2 || version > 3) { err = "GGUF version " + std::to_string(version) + " is not handled"; return false; }
+ if (n_kv > 100000) { err = "too many keys in the GGUF header"; return false; }
+ gguf_context* out = gguf_init_empty();
+ if (!out) { err = "out of memory"; return false; }
+ struct Freer { gguf_context* g; ~Freer() { gguf_free(g); } } freer{out};
+ for (uint64_t i = 0; i < n_kv; ++i) {
+ std::string key;
+ uint32_t type = 0;
+ if (!r.str(key, 1 << 16) || !r.num(type)) { err = r.err; return false; }
+ const bool keep = !dropped_key(key);
+ if (type == GGUF_TYPE_STRING) {
+ std::string v;
+ if (!r.str(v)) { err = r.err; return false; }
+ if (keep) gguf_set_val_str(out, key.c_str(), v.c_str());
+ } else if (type == GGUF_TYPE_ARRAY) {
+ uint32_t et = 0;
+ uint64_t n = 0;
+ if (!r.num(et) || !r.num(n)) { err = r.err; return false; }
+ if (n > 50'000'000ull) { err = "an array in the GGUF header is too long"; return false; }
+ if (et == GGUF_TYPE_STRING) {
+ std::vector<std::string> items((size_t) n);
+ for (auto& s : items) if (!r.str(s)) { err = r.err; return false; }
+ if (keep) {
+ std::vector<const char*> ptrs;
+ ptrs.reserve(items.size());
+ for (const auto& s : items) ptrs.push_back(s.c_str());
+ gguf_set_arr_str(out, key.c_str(), ptrs.data(), ptrs.size());
+ }
+ } else if (scalar_size(et)) {
+ std::vector<uint8_t> raw((size_t) (n * scalar_size(et)));
+ if (!raw.empty() && !r.take(raw.data(), raw.size())) { err = r.err; return false; }
+ if (keep) gguf_set_arr_data(out, key.c_str(), (gguf_type) et, raw.data(), (size_t) n);
+ } else { err = "an array of arrays in the GGUF header is not handled"; return false; }
+ } else if (scalar_size(type)) {
+ uint64_t raw = 0;
+ if (!r.take(&raw, scalar_size(type))) { err = r.err; return false; } // little endian, as the file
+ if (!keep) continue;
+ switch (type) {
+ case GGUF_TYPE_UINT8: gguf_set_val_u8(out, key.c_str(), (uint8_t) raw); break;
+ case GGUF_TYPE_INT8: gguf_set_val_i8(out, key.c_str(), (int8_t) raw); break;
+ case GGUF_TYPE_UINT16: gguf_set_val_u16(out, key.c_str(), (uint16_t) raw); break;
+ case GGUF_TYPE_INT16: gguf_set_val_i16(out, key.c_str(), (int16_t) raw); break;
+ case GGUF_TYPE_UINT32: gguf_set_val_u32(out, key.c_str(), (uint32_t) raw); break;
+ case GGUF_TYPE_INT32: gguf_set_val_i32(out, key.c_str(), (int32_t) raw); break;
+ case GGUF_TYPE_FLOAT32: { float v; std::memcpy(&v, &raw, 4); gguf_set_val_f32(out, key.c_str(), v); break; }
+ case GGUF_TYPE_BOOL: gguf_set_val_bool(out, key.c_str(), raw != 0); break;
+ case GGUF_TYPE_UINT64: gguf_set_val_u64(out, key.c_str(), raw); break;
+ case GGUF_TYPE_INT64: gguf_set_val_i64(out, key.c_str(), (int64_t) raw); break;
+ case GGUF_TYPE_FLOAT64: { double v; std::memcpy(&v, &raw, 8); gguf_set_val_f64(out, key.c_str(), v); break; }
+ default: break;
+ }
+ } else { err = "a value type in the GGUF header is not handled"; return false; }
+ }
+ if (!gguf_write_to_file(out, dst.c_str(), true)) { err = "cannot write " + dst; return false; }
+ return true;
+}
+
+} // namespace strata_gguf
+
+// The text model for the picture reader: vocabulary and header only. `how` says which way it was opened.
+inline llama_model* strata_open_text_model(const std::string& model, std::string* how = nullptr) {
+ llama_model_params mp = llama_model_default_params();
+ mp.vocab_only = true;
+ const char* tmpdir = std::getenv("TMPDIR");
+ if (!tmpdir || !*tmpdir) tmpdir = std::getenv("TEMP");
+ if (!tmpdir || !*tmpdir) tmpdir = std::getenv("TMP");
+ if (!tmpdir || !*tmpdir) tmpdir = "/tmp";
+ const std::string meta = std::string(tmpdir) + "/strata-vision-meta-" + std::to_string((long) STRATA_GETPID()) + ".gguf";
+ std::string err;
+ llama_model* m = nullptr;
+ if (strata_gguf::copy_meta(model, meta, err)) {
+ m = llama_model_load_from_file(meta.c_str(), mp);
+ if (how) *how = m ? "header only" : "header copy was not accepted";
+ } else if (how) *how = "header copy failed: " + err;
+ std::remove(meta.c_str());
+ if (!m) { // the file as it is (a type of header this reader does not handle, or an llama.cpp that wants more)
+ m = llama_model_load_from_file(model.c_str(), mp);
+ if (how) *how += m ? ", loaded the file itself" : ", the file itself did not load either";
+ }
+ return m;
+}
diff --git a/tools/vision/gguf_meta_test.cpp b/tools/vision/gguf_meta_test.cpp
new file mode 100644
index 000000000..c0262eafd
--- /dev/null
+++ b/tools/vision/gguf_meta_test.cpp
@@ -0,0 +1,110 @@
+// tools/vision/gguf_meta_test.cpp - strata-vision opens a GGUF whose weight tensors have types llama.cpp does not know.
+//
+// Writes a small GGUF by hand (a tokenizer, two tensors of the Strata-private types TQK6 = 145 and TQK7 = 146, and split
+// keys that point at shards which are not there), then checks that
+// - llama.cpp itself cannot load it (the reason the picture reader used to say "cannot open the text model"),
+// - strata_open_text_model() can, and sees the same vocabulary.
+// Exit code 0 = all checks passed.
+#include "gguf_meta.h"
+
+#include <cstdint>
+#include <cstdio>
+#include <string>
+#include <vector>
+
+namespace {
+
+struct Out {
+ std::vector<uint8_t> b;
+ void raw(const void* p, size_t n) { const auto* c = (const uint8_t*) p; b.insert(b.end(), c, c + n); }
+ template <class T> void num(T v) { raw(&v, sizeof v); }
+ void str(const std::string& s) { num<uint64_t>(s.size()); raw(s.data(), s.size()); }
+ void kv_str(const std::string& k, const std::string& v) { str(k); num<uint32_t>(GGUF_TYPE_STRING); str(v); }
+ void kv_u32(const std::string& k, uint32_t v) { str(k); num<uint32_t>(GGUF_TYPE_UINT32); num<uint32_t>(v); }
+ void kv_strs(const std::string& k, const std::vector<std::string>& v) {
+ str(k); num<uint32_t>(GGUF_TYPE_ARRAY); num<uint32_t>(GGUF_TYPE_STRING); num<uint64_t>(v.size());
+ for (const auto& s : v) str(s);
+ }
+ void kv_i32s(const std::string& k, const std::vector<int32_t>& v) {
+ str(k); num<uint32_t>(GGUF_TYPE_ARRAY); num<uint32_t>(GGUF_TYPE_INT32); num<uint64_t>(v.size());
+ raw(v.data(), v.size() * 4);
+ }
+};
+
+int failures = 0;
+void check(bool ok, const char* what) {
+ std::printf("%s %s\n", ok ? "ok " : "FAIL", what);
+ if (!ok) ++failures;
+}
+
+bool write_fixture(const std::string& path, size_t& n_vocab, bool split) {
+ std::vector<std::string> tokens;
+ for (int i = 0; i < 256; ++i) tokens.push_back(std::string("t") + std::to_string(i));
+ tokens.push_back("<|endoftext|>");
+ n_vocab = tokens.size();
+ std::vector<int32_t> types(tokens.size(), 1);
+ types.back() = 3;
+ Out o;
+ o.raw("GGUF", 4);
+ o.num<uint32_t>(3);
+ o.num<uint64_t>(2); // two tensors
+ const uint64_t n_kv = split ? 9 : 6;
+ o.num<uint64_t>(n_kv);
+ o.kv_str("general.architecture", "llama");
+ o.kv_str("tokenizer.ggml.model", "gpt2");
+ o.kv_str("tokenizer.ggml.pre", "default");
+ o.kv_strs("tokenizer.ggml.tokens", tokens);
+ o.kv_i32s("tokenizer.ggml.token_type", types);
+ o.kv_strs("tokenizer.ggml.merges", {"t1 t2"});
+ if (split) {
+ o.kv_u32("split.no", 0);
+ o.kv_u32("split.count", 2);
+ o.kv_u32("split.tensors.count", 4);
+ }
+ // two tensors of types llama.cpp's ggml does not have (145 = TQK6, 146 = TQK7), 128 elements each
+ for (int i = 0; i < 2; ++i) {
+ o.str(i == 0 ? "blk.0.ffn_gate_exps.weight" : "blk.0.ffn_up_exps.weight");
+ o.num<uint32_t>(1);
+ o.num<uint64_t>(128);
+ o.num<uint32_t>(145 + i);
+ o.num<uint64_t>(i * 64);
+ }
+ while (o.b.size() % 32) o.b.push_back(0);
+ o.b.resize(o.b.size() + 128, 0);
+ FILE* f = std::fopen(path.c_str(), "wb");
+ if (!f) return false;
+ const bool ok = std::fwrite(o.b.data(), 1, o.b.size(), f) == o.b.size();
+ std::fclose(f);
+ return ok;
+}
+
+} // namespace
+
+int main(int argc, char** argv) {
+ const std::string dir = argc > 1 ? argv[1] : ".";
+ llama_backend_init();
+ for (int split = 0; split < 2; ++split) {
+ const std::string label = split ? " (a first shard whose other shards are missing)" : "";
+ const std::string path = dir + "/strata-vision-gguf-test-" + std::to_string(split) + ".gguf";
+ size_t n_vocab = 0;
+ check(write_fixture(path, n_vocab, split == 1), ("wrote a GGUF with tensor types 145 and 146" + label).c_str());
+ llama_model_params mp = llama_model_default_params();
+ mp.vocab_only = true;
+ llama_model* plain = llama_model_load_from_file(path.c_str(), mp);
+ std::printf("note llama.cpp itself %s this file\n", plain ? "accepts" : "refuses");
+ if (plain) llama_model_free(plain);
+ std::string how;
+ llama_model* m = strata_open_text_model(path, &how);
+ check(m != nullptr, ("strata_open_text_model opens it (" + how + ")" + label).c_str());
+ if (m) {
+ check((size_t) llama_vocab_n_tokens(llama_model_get_vocab(m)) == n_vocab, "and sees the whole vocabulary");
+ llama_model_free(m);
+ }
+ check(how.find("header only") == 0, "by reading the header, not the weights");
+ std::remove(path.c_str());
+ }
+ check(strata_open_text_model(dir + "/does-not-exist.gguf") == nullptr, "a file that is not there gives no model");
+ llama_backend_free();
+ std::printf("%s\n", failures ? "GGUF-META FAIL" : "GGUF-META PASS");
+ return failures ? 1 : 0;
+}
diff --git a/tools/vision/strata_vision.cpp b/tools/vision/strata_vision.cpp
index 9a06ce1fa..bb58a2a15 100644
--- a/tools/vision/strata_vision.cpp
+++ b/tools/vision/strata_vision.cpp
@@ -15,6 +15,7 @@
// row i at grid position (x = i % nx, y = i / nx). The text model is opened vocab-only (no weights).
#include "ggml-cpu.h"
#include "gguf.h"
+#include "gguf_meta.h"
#include "llama.h"
#include "mtmd.h"
#include "mtmd-helper.h"
@@ -112,10 +113,11 @@ int main(int argc, char** argv) {
}
}
#endif
- llama_model_params mp = llama_model_default_params();
- mp.vocab_only = true;
- llama_model* text = llama_model_load_from_file(model.c_str(), mp);
- if (!text) { std::printf("ERR cannot open the text model %s\n", model.c_str()); std::fflush(stdout); return 1; }
+ // the vocabulary and header only: the weights (and their tensor types, which can be Strata-private ones such as
+ // Gyro's TQK6 / TQK7) are never touched - see gguf_meta.h
+ std::string how;
+ llama_model* text = strata_open_text_model(model, &how);
+ if (!text) { std::printf("ERR cannot open the text model %s (%s)\n", model.c_str(), how.c_str()); std::fflush(stdout); return 1; }
mtmd_context_params cp = mtmd_context_params_default();
cp.use_gpu = gpu;