Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: pack_gguf takes unsloth's DSpark drafter GGUF (dflash) and the mmproj vision GGUF into the pack; the vision loader reads F32 rows and gate/up halves
Niko12212d718defc072
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 4999ddc24..4ed6ef3aa 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -4212,6 +4212,20 @@ struct Model::Impl {std::string vision_err; // why the tower did not load (pictures off), or emptystd::vector<uint16_t> img_mark[3]; // image_start, image_newline, image_end: BF16 [dim]std::vector<Image> imgs; // the pictures of the prompt being fed, by start+ // a BF16 (or, rounded to nearest even, F32) tensor of n values into dst+ bool read_bf16(const Tensor& t, size_t n, void* dst) {+ if (t.dtype == "BF16") return t.bytes == n * 2 && pk.read(t, dst);+ if (t.dtype != "F32" || t.bytes != n * 4) return false;+ std::vector<float> f(n);+ if (!pk.read(t, f.data())) return false;+ uint16_t* o = (uint16_t*) dst;+ for (size_t i = 0; i < n; ++i) {+ uint32_t u;+ std::memcpy(&u, &f[i], 4);+ o[i] = (uint16_t) ((u + 0x7FFFu + ((u >> 16) & 1u)) >> 16);+ }+ return true;+ }bool load_vision(std::string& err) {vg.layers = (int) pk.num("vision.num_hidden_layers", 0);const int tok = (int) pk.num("image_token_id", -1);@@ -4224,18 +4238,31 @@ struct Model::Impl {for (int i = 0; i < 3; ++i) {const Tensor* t = pk.get(mark[i]);img_mark[i].resize((size_t) g.dim);- if (t == nullptr || t->dtype != "BF16" || t->bytes != (uint64_t) g.dim * 2 || !pk.read(*t, img_mark[i].data())) {- err = std::string(mark[i]) + ": not a BF16 row of the model's width";+ if (t == nullptr || !read_bf16(*t, (size_t) g.dim, img_mark[i].data())) {+ err = std::string(mark[i]) + ": not a BF16 (or F32) row of the model's width";return false;}}const double t0 = now_ms();Dev& d = devs[0];+ // the tower's tensors as BF16: the checkpoint's own, or a GGUF's mmproj (F32 norms / biases / marker rows, which+ // came from BF16, converted back exactly; mlp.w1 as the two files' gate and up, the halves of its rows)auto read = [this](const std::string& n, std::vector<int64_t>& shape, void* dst) {const Tensor* t = pk.get(n);- if (t == nullptr || t->dtype != "BF16") return false;+ if (t == nullptr && n.size() > 13 && n.compare(n.size() - 13, 13, "mlp.w1.weight") == 0) {+ const std::string pre = n.substr(0, n.size() - 13);+ const Tensor* tg = pk.get(pre + "mlp.gate.weight");+ const Tensor* tu = pk.get(pre + "mlp.up.weight");+ if (tg == nullptr || tu == nullptr || tg->shape.size() != 2 || tg->shape != tu->shape) return false;+ shape = {tg->shape[0] * 2, tg->shape[1]};+ const size_t half = (size_t) tg->shape[0] * (size_t) tg->shape[1];+ return dst == nullptr || (read_bf16(*tg, half, dst) && read_bf16(*tu, half, (uint16_t*) dst + half));+ }+ if (t == nullptr) return false;shape = t->shape;- return dst == nullptr || pk.read(*t, dst);+ size_t n_el = 1;+ for (int64_t x : t->shape) n_el *= (size_t) x;+ return dst == nullptr || read_bf16(*t, n_el, dst);};if (!vt.load(vg, d.id, read, err)) return false;img_tok = tok;diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.pyindex 055b33765..d1fd7a5e4 100644--- a/tools/dsv41/pack_gguf.py+++ b/tools/dsv41/pack_gguf.py@@ -323,6 +323,11 @@ def main() -> int:ap.add_argument("--model", required=True, help="the directory of the GGUF's shards")ap.add_argument("--out", help="default: <model>/maya-ds4")ap.add_argument("--vision", help="vision_pack.py's folder (default: <model>/../vision; taken when it is there)")+ ap.add_argument("--dspark-gguf", help="unsloth's DSpark drafter GGUF (dspark-*-Q8_0.gguf, architecture dflash): its three "+ "blocks become mtp.0-2 beside the trunk's tensors (default: the first dspark-*.gguf "+ "in <model>/.. when there is one)")+ ap.add_argument("--mmproj", help="unsloth's vision tower GGUF (mmproj-*.gguf, clip / deepseek41v): its tensors become the "+ "checkpoint's vision.* / aligner.* (default: the first mmproj*.gguf in <model>/..)")ap.add_argument("--drafter", help="the DSpark drafter's .json (drafter_pack.py's; default <model>/../dspark/dspark.json)")ap.add_argument("--engram", help="the folder of the Engram tables' GGUF set, for a quant that leaves them out ""(default: <model>/../Engram, when it is whole)")@@ -431,10 +436,118 @@ def main() -> int:files = namesif eg_from:print("Engram tables from %s (%d shards)" % (eg_dir, len(eg_from)))+ # the DSpark drafter from unsloth's GGUF (architecture dflash): blk.N.* -> mtp.N.*, the head tensors under the+ # checkpoint's names (fc -> main_proj, enc.output_norm -> main_norm, output_norm -> norm of the last block,+ # conf_proj -> confidence_head.proj, markov_w1 / w2 -> markov_head.embed / head)+ ds_path = pathlib.Path(a.dspark_gguf).expanduser().resolve() if a.dspark_gguf else next(+ iter(sorted(model.parent.glob("dspark-*.gguf"))), None)+ dspark_gguf = False+ if ds_path is not None:+ if not ds_path.is_file():+ raise SystemExit("%s: no DSpark GGUF there" % ds_path)+ dg = Gguf(ds_path)+ if dg.kv.get("general.architecture") != "dflash":+ raise SystemExit("%s: not a DSpark (dflash) GGUF" % ds_path)+ nb = int(dg.kv["dflash.block_count"])+ files.append(rel(ds_path))+ fi = len(files) - 1+ HEAD = {"fc.weight": "mtp.0.main_proj.weight", "enc.output_norm.weight": "mtp.0.main_norm.weight",+ "output_norm.weight": "mtp.%d.norm.weight" % (nb - 1),+ "conf_proj.weight": "mtp.%d.confidence_head.proj.weight" % (nb - 1),+ "markov_w1.weight": "mtp.%d.markov_head.embed.weight" % (nb - 1),+ "markov_w2.weight": "mtp.%d.markov_head.head.weight" % (nb - 1)}+ groups = int(kv["%s.attention.output_group_count" % kv["general.architecture"]])+ bad = []+ for name, typ, dims, off in dg.tensors:+ if typ not in TYPES:+ raise SystemExit("%s: %s has ggml type %d, which the engine does not read" % (ds_path.name, name, typ))+ tname, per, bpb = TYPES[typ]+ nel = 1+ for d in dims:+ nel *= d+ nbytes = nel // per * bpb+ m = re.match(r"blk\.(\d+)\.(.+)$", name)+ if m and m.group(2) in LAYER:+ ck = "mtp.%s.%s" % (m.group(1), LAYER[m.group(2)])+ elif name in HEAD:+ ck = HEAD[name]+ else:+ bad.append(name)+ continue+ shape = list(reversed(dims))+ base = dg.data_start + off+ if ck.endswith(".attn.wo_a.slice"):+ part = nbytes // groups+ for i in range(groups):+ tensors.append(("%s.%d.weight" % (ck, i), tname, fi, base + i * part, part, [shape[0] // groups, shape[1]]))+ elif ".ffn.experts." in ck:+ n, part = shape[0], nbytes // shape[0]+ pre, role = ck.rsplit(".", 1)+ for i in range(n):+ tensors.append(("%s.%d.%s.weight" % (pre, i, role), tname, fi, base + i * part, part, shape[1:]))+ else:+ tensors.append((ck, tname, fi, base, nbytes, shape))+ if bad:+ raise SystemExit("DSpark GGUF tensors the pack does not map (%d): %s" % (len(bad), ", ".join(bad[:12])))+ dspark_gguf = True+ print("DSpark drafter from %s: %d blocks" % (ds_path.name, nb))+ # the vision tower from unsloth's mmproj GGUF (clip, projector deepseek41v): llama.cpp's names -> the checkpoint's;+ # the F32 norms / biases / marker rows are read as BF16 by the engine; mlp.w1 is the file's gate and up+ mm_path = pathlib.Path(a.mmproj).expanduser().resolve() if a.mmproj else next(+ iter(sorted(model.parent.glob("mmproj*.gguf"))), None)+ mm_geo = {}+ if mm_path is not None:+ if not mm_path.is_file():+ raise SystemExit("%s: no mmproj GGUF there" % mm_path)+ mg = Gguf(mm_path)+ if mg.kv.get("clip.projector_type") != "deepseek41v":+ raise SystemExit("%s: not a DeepSeek-V4.1 vision GGUF (projector %r)" % (mm_path, mg.kv.get("clip.projector_type")))+ files.append(rel(mm_path))+ fi = len(files) - 1+ FIX = {"v.patch_embd.weight": "vision.patch_embed.proj.weight", "v.patch_embd.bias": "vision.patch_embed.proj.bias",+ "v.post_ln.weight": "vision.norm.weight", "mm.1.weight": "aligner.w1.weight", "mm.1.bias": "aligner.w1.bias",+ "mm.2.weight": "aligner.w2.weight", "mm.2.bias": "aligner.w2.bias", "v.token_embd.img_start": "image_start",+ "v.image_newline": "image_newline", "v.token_embd.img_end": "image_end"}+ BLK = {"ln1.weight": "norm1.weight", "ln2.weight": "norm2.weight", "attn_qkv.weight": "attn.wqkv.weight",+ "attn_qkv.bias": "attn.wqkv.bias", "attn_out.weight": "attn.wo.weight", "attn_out.bias": "attn.wo.bias",+ "ffn_gate.weight": "mlp.gate.weight", "ffn_up.weight": "mlp.up.weight", "ffn_down.weight": "mlp.w2.weight"}+ bad = []+ for name, typ, dims, off in mg.tensors:+ if typ not in TYPES:+ raise SystemExit("%s: %s has ggml type %d, which the engine does not read" % (mm_path.name, name, typ))+ tname, per, bpb = TYPES[typ]+ nel = 1+ for d in dims:+ nel *= d+ m = re.match(r"v\.blk\.(\d+)\.(.+)$", name)+ if m and m.group(2) in BLK:+ ck = "vision.blocks.%s.%s" % (m.group(1), BLK[m.group(2)])+ elif name in FIX:+ ck = FIX[name]+ else:+ bad.append(name)+ continue+ shape = list(reversed(dims))+ if name == "v.patch_embd.weight": # conv [out][3][14][14] -> [out][3 * 14 * 14], the same bytes+ shape = [shape[0], nel // shape[0]]+ tensors.append((ck, tname, fi, mg.data_start + off, nel // per * bpb, shape))+ if bad:+ raise SystemExit("mmproj tensors the pack does not map (%d): %s" % (len(bad), ", ".join(bad[:12])))+ k = lambda n, d=None: mg.kv.get("clip.vision." + n, d) # noqa: E731+ toks = kv.get("tokenizer.ggml.tokens", [])+ img_id = next((i for i, t in enumerate(toks) if t == "<|deepseek_image|>"), None)+ if img_id is None:+ raise SystemExit("the vocabulary has no <|deepseek_image|> token")+ mm_geo = {"vision.num_hidden_layers": k("block_count"), "vision.hidden_size": k("embedding_length"),+ "vision.num_attention_heads": k("attention.head_count"), "vision.intermediate_size": k("feed_forward_length"),+ "vision.patch_size": k("patch_size"), "vision.downsample_ratio": k("projector.scale_factor"),+ "vision.rope_theta": 10000, "vision.max_image_tokens": 1024, "vision.min_pixels": k("image_min_pixels"),+ "image_token_id": img_id}+ print("vision tower from %s: %d blocks, image token %d" % (mm_path.name, mm_geo["vision.num_hidden_layers"], img_id))drafter = pathlib.Path(a.drafter).expanduser().resolve() if a.drafter else model.parent / "dspark" / "dspark.json"if a.drafter and not drafter.is_file():raise SystemExit("%s: no drafter there" % drafter)- drafter_in = drafter.is_file() and drafter.with_suffix(".bin").is_file()+ drafter_in = (drafter.is_file() and drafter.with_suffix(".bin").is_file()) and not dspark_ggufif drafter_in:files.append(rel(drafter.with_suffix(".bin")))groups = int(kv["%s.attention.output_group_count" % kv["general.architecture"]])@@ -461,6 +574,8 @@ def main() -> int:for name, dt, off, nb, shape in info["tensors"]:tensors.append((name, dt, len(files) - 1, off, nb, shape))vision = info["config"]+ if mm_geo and not vision:+ vision = mm_geovocab = next(t[5][0] for t in tensors if t[0] == "embed.weight")geo = geometry(kv, vocab)@@ -492,7 +607,7 @@ def main() -> int:by = {}for t in tensors:by[t[1]] = by.get(t[1], 0) + t[4]- extras = [w for w, there in (("the DSpark drafter", drafter_in), ("the vision tower", bool(vision))) if there]+ extras = [w for w, there in (("the DSpark drafter", drafter_in or dspark_gguf), ("the vision tower", bool(vision))) if there]print("%s: %d tensors in %d files (%s), Engram %d layers x %d hash columns%s -> %s" %(model.name, len(tensors), len(files), ", ".join("%s %.1f GB" % (k, v / 1e9) for k, v in sorted(by.items())),nl, cols, "".join(", " + w for w in extras), out))