Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: pack_gguf uses the GGUF's own chat template (fixes Maya #299), drop Maya's template and the EXL3 packer
Niko1221a0dbad807883
diff --git a/tools/dsv41/chat_template.jinja b/tools/dsv41/chat_template.jinjadeleted file mode 100644index 7b983a45f..000000000--- a/tools/dsv41/chat_template.jinja+++ /dev/null@@ -1,93 +0,0 @@-{#- DeepSeek-V4.1's prompt format (the model repo's encoding/encoding.py), as a chat template for Maya's server.- Turns: <|User|> ... <|Assistant|> (reasoning</think>)answer<|end▁of▁sentence|>. Thinking mode ends the prompt- with <think> and starts with a reasoning-effort line (low 50, high 75, max 100); chat mode ends it with </think>.- Tool results (role "tool") are <tool_result> blocks of a user turn, merged with the user messages around them;- tool calls are DSML blocks; the tools are listed in the system turn. Earlier turns keep their reasoning only when- tools are given (encoding.py's drop_thinking). A picture in a message's content list is its placeholder- <|deepseek_image|>, a block of its own between the text blocks (the server expands it into the picture's span). -#}-{%- set thinking = enable_thinking is not defined or enable_thinking -%}-{%- set efforts = {'low': 50, 'medium': 75, 'high': 75, 'xhigh': 100, 'max': 100} -%}-{%- if reasoning_effort is defined and reasoning_effort is number -%}-{%- set budget = reasoning_effort -%}-{%- elif reasoning_effort is defined and reasoning_effort in efforts -%}-{%- set budget = efforts[reasoning_effort] -%}-{%- else -%}-{%- set budget = 75 -%}-{%- endif -%}-{%- set has_tools = tools is defined and tools is not none and tools | length > 0 -%}-{%- set keep = has_tools -%}-{%- set ns = namespace(last_user=-1, schemas='') -%}-{%- if has_tools -%}{%- for t in tools -%}{%- set ns.schemas = ns.schemas ~ (t | tojson) ~ ('' if loop.last else '\n') -%}{%- endfor -%}{%- endif -%}-{%- for m in messages -%}{%- if m.role == 'user' -%}{%- set ns.last_user = loop.index0 -%}{%- endif -%}{%- endfor -%}-{%- macro text(c) -%}-{%- if c is string -%}{{ c }}{%- elif c is iterable -%}-{%- for p in c if p.type in ['text', 'image'] -%}{%- if not loop.first -%}{{ '\n\n' }}{%- endif -%}{{ p.text if p.type == 'text' else '<|deepseek_image|>' }}{%- endfor -%}-{%- endif -%}-{%- endmacro -%}-{%- macro tools_block() -%}-## Tools--You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:--<|DSML| calls>-<|DSML| invoke name="$TOOL_NAME">-<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>-...-</|DSML| invoke>-<|DSML| invoke name="$TOOL_NAME2">-...-</|DSML| invoke>-</|DSML| calls>--String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.--If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.--Otherwise, output directly after </think> with tool calls or final response.--### Available Tool Schemas--{{ ns.schemas }}--You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.-{% endmacro -%}-{{- '<|begin▁of▁sentence|>' -}}-{%- set effort_line = ('Reasoning Effort: ' ~ budget ~ ' (range 1-100, the higher the value, the more thorough the reasoning)\n\n') if thinking else '' -%}-{%- if messages and messages[0].role == 'system' -%}-{{- '<|System|>' ~ effort_line ~ text(messages[0].content) -}}-{%- if has_tools -%}{{- '\n\n' ~ tools_block() -}}{%- endif -%}-{%- elif has_tools -%}-{{- '<|System|>' ~ effort_line ~ '\n\n' ~ tools_block() -}}-{%- elif thinking -%}-{{- '<|System|>' ~ effort_line -}}-{%- endif -%}-{%- for m in messages -%}-{%- set i = loop.index0 -%}-{%- set prev = messages[i - 1].role if i > 0 else '' -%}-{%- set next = messages[i + 1].role if i + 1 < messages | length else '' -%}-{%- if m.role == 'system' and i == 0 -%}-{%- elif m.role in ['user', 'tool', 'system'] -%}-{%- if prev in ['user', 'tool'] or (prev == 'system' and i - 1 > 0) -%}{{- '\n\n' -}}{%- else -%}{{- '<|User|>' -}}{%- endif -%}-{%- if m.role == 'tool' -%}{{- '<tool_result>' ~ text(m.content) ~ '</tool_result>' -}}{%- else -%}{{- text(m.content) -}}{%- endif -%}-{%- if next not in ['user', 'tool', 'system'] -%}-{%- if thinking and (keep or i >= ns.last_user) -%}{{- '<|Assistant|><think>' -}}{%- else -%}{{- '<|Assistant|></think>' -}}{%- endif -%}-{%- endif -%}-{%- elif m.role == 'assistant' -%}-{%- if thinking and (keep or i > ns.last_user) -%}{{- (m.reasoning_content or '') ~ '</think>' -}}{%- endif -%}-{{- text(m.content) -}}-{%- if m.tool_calls -%}-{{- '\n\n<|DSML| calls>\n' -}}-{%- for c in m.tool_calls -%}-{%- set f = c.function if c.function is defined else c -%}-{{- '<|DSML| invoke name="' ~ f.name ~ '">\n' -}}-{%- for k, v in (f.arguments or {}).items() -%}-{{- '<|DSML| parameter name="' ~ k ~ '" string="' ~ ('true' if v is string else 'false') ~ '">' ~ (v if v is string else v | tojson) ~ '</|DSML| parameter>' -}}-{%- if not loop.last -%}{{- '\n' -}}{%- endif -%}-{%- endfor -%}-{{- '\n</|DSML| invoke>' -}}{%- if not loop.last -%}{{- '\n' -}}{%- endif -%}-{%- endfor -%}-{{- '\n</|DSML| calls>' -}}-{%- endif -%}-{{- '<|end▁of▁sentence|>' -}}-{%- endif -%}-{%- endfor -%}diff --git a/tools/dsv41/exl3_torch.py b/tools/dsv41/exl3_torch.pydeleted file mode 100644index 48505de07..000000000--- a/tools/dsv41/exl3_torch.py+++ /dev/null@@ -1,148 +0,0 @@-# Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).-# Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.-"""tools/ds4/exl3_torch.py - EXL3 (exllamav3) matrices decoded on the GPU with torch, for the DeepSeek-V4.1 reference.--The format is include/strata/kernels/exl3.hpp's; this is tools/exl3_pack.py's numpy decoder (exl3_states, exl3_values,-exl3_rows) on torch tensors. decode(st, prefix) returns the full weight as [n, k] (out x in, F.linear's layout):-W = diag(suh) H W' H diag(svh), so x W is H(H(x * suh) W') * svh.-"""-from __future__ import annotations--import math--import numpy as np-import torch--_P = torch.arange(256)-_ROW = (_P >> 3 & 3) * 2 + (_P & 1) + 8 * (_P >> 1 & 1)-_COL = (_P >> 5) + 8 * (_P >> 2 & 1)-_PERM = _ROW * 16 + _COL # tile[..., _PERM[p]] = value of step p---def _sylvester(n: int) -> torch.Tensor:- h = torch.ones(1, 1, dtype=torch.float64)- while h.shape[0] < n:- h = torch.cat([torch.cat([h, h], 1), torch.cat([h, -h], 1)], 0)- return h / math.sqrt(n)---_HAD = {}---def had(dev) -> torch.Tensor:- if dev not in _HAD:- _HAD[dev] = _sylvester(128).to(dev, torch.float32)- return _HAD[dev]---def _f16(b: int) -> float:- return float(np.array([b], dtype=np.uint16).view(np.float16)[0])---K_INV, K_BIAS = _f16(0x1EEE), _f16(0xC931)---def states(words: torch.Tensor, K: int) -> torch.Tensor:- """(..., 8 K) int32 tile words -> (..., 256) 16-bit states (MSB-first stream, tail-biting), int64."""- W = 8 * K- dev = words.device- p = _P.to(dev)- b1 = (p + 1 + 256) * K- i1, i0 = (b1 - 1) >> 5, (b1 - 16) >> 5- sh = ((i1 + 1) << 5) - b1- w = words.to(torch.int64) & 0xFFFFFFFF- m = (w[..., i0 % W] << 32) | w[..., i1 % W]- return (m >> sh) & 0xFFFF---def values(st: torch.Tensor, cb: int) -> torch.Tensor:- """States -> FP16-rounded codebook values (float32)."""- if cb == 2:- x = (st * 0x83DCD12D) & 0xFFFFFFFF- bs = (x & 0xFF) + (x >> 8 & 0xFF) + (x >> 16 & 0xFF) + (x >> 24 & 0xFF)- v = (1024.0 + bs.to(torch.float64)) * K_INV + K_BIAS- return v.to(torch.float16).to(torch.float32)- x = st * (0xCBAC1FED if cb == 1 else 89226354)- if cb == 0:- x = x + 64248484- x = ((x & 0xFFFFFFFF) & 0x8FFF8FFF) ^ 0x3B603B60- lo = (x & 0xFFFF).to(torch.int16).view(torch.float16).to(torch.float64) # bit patterns as FP16- hi = (x >> 16).to(torch.int16).view(torch.float16).to(torch.float64)- return (lo + hi).to(torch.float16).to(torch.float32)---def decode_raw(trellis: torch.Tensor, suh: torch.Tensor, svh: torch.Tensor, cb: int) -> torch.Tensor:- """trellis int16 [k/16, n/16, 16 K] (device), suh F16 [k], svh F16 [n] -> W [n, k] float32."""- kt, nt, w16 = trellis.shape- K = w16 // 16- k, n = kt * 16, nt * 16- dev = trellis.device- words = trellis.contiguous().view(torch.int32) # [kt, nt, 8K]- out = torch.empty(n, k, dtype=torch.float32, device=dev)- H = had(dev)- step = max(1, (1 << 27) // (nt * 256 * 8)) # 16-row slices per chunk (bounded scratch)- step = max(8, step - step % 8) # whole 128-row Hadamard blocks- perm = _PERM.to(dev)- for a in range(0, kt, step):- b = min(kt, a + step)- v = values(states(words[a:b], K), cb) # [b-a, nt, 256] in step order- tile = torch.empty_like(v)- tile[..., perm] = v- inner = tile.view(b - a, nt, 16, 16).permute(0, 2, 1, 3).reshape((b - a) * 16, n)- inner = (inner.view(-1, n // 128, 128) @ H).view(-1, n) # over n, in blocks of 128- inner = (H @ inner.view(-1, 128, n)).view(-1, n) # over k, in blocks of 128- out[:, a * 16:b * 16] = inner.T- out *= svh.to(torch.float32)[:, None]- out *= suh.to(torch.float32)[None, :]- return out---def decode_many(st, prefixes: list[str], dev) -> list[torch.Tensor]:- """Several EXL3 matrices of one shape (a layer's experts' w1, say) decoded together: their trellises side by side- along n (both Hadamards act inside blocks of 128, so the stacked matrix decodes as each would), then each one's- channel scales. Returns [n, k] float32 views, one a prefix."""- trs = [tensor(st, p + ".trellis", dev) for p in prefixes]- kt, nt, _ = trs[0].shape- cb = codebook(st, prefixes[0])- ones_k = torch.ones(kt * 16, dtype=torch.float16, device=dev)- ones_n = torch.ones(nt * 16 * len(prefixes), dtype=torch.float16, device=dev)- inner = decode_raw(torch.cat(trs, dim=1), ones_k, ones_n, cb) # [N n, k]- n = nt * 16- out = []- for i, p in enumerate(prefixes):- w = inner[i * n:(i + 1) * n]- w *= tensor(st, p + ".svh", dev).to(torch.float32)[:, None]- w *= tensor(st, p + ".suh", dev).to(torch.float32)[None, :]- out.append(w)- return out---def tensor(st, name: str, dev) -> torch.Tensor:- _, dt, shape, _, _ = st.t[name]- r = st.raw(name)- np_dt = {"F32": np.float32, "F16": np.float16, "I16": np.int16, "I32": np.int32, "BF16": np.uint16}[dt]- a = torch.from_numpy(np.array(r.view(np_dt)).reshape(shape))- if dt == "BF16":- a = a.view(torch.int16).view(torch.bfloat16)- return a.to(dev)---def is_exl3(st, prefix: str) -> bool:- return st.has(prefix + ".trellis")---def codebook(st, prefix: str) -> int:- return 2 if st.has(prefix + ".mul1") else 1 if st.has(prefix + ".mcg") else 0---def decode(st, prefix: str, dev) -> torch.Tensor:- """The EXL3 matrix at `prefix` (.trellis/.suh/.svh) as W [n, k] float32 on dev."""- return decode_raw(tensor(st, prefix + ".trellis", dev), tensor(st, prefix + ".suh", dev),- tensor(st, prefix + ".svh", dev), codebook(st, prefix))---def weight(st, prefix: str, dev) -> torch.Tensor:- """A linear weight [n, k] float32: EXL3 decoded, or the stored float tensor (prefix.weight)."""- if is_exl3(st, prefix):- return decode(st, prefix, dev)- return torch.from_numpy(np.array(st.f32(prefix + ".weight"))).to(dev)diff --git a/tools/dsv41/pack.py b/tools/dsv41/pack.pydeleted file mode 100644index e0e895085..000000000--- a/tools/dsv41/pack.py+++ /dev/null@@ -1,154 +0,0 @@-# Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).-# Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.-"""tools/ds4/pack.py - the engine's pack for a DeepSeek-V4.1 EXL3 model, beside the model's files (no weights copied).-- python tools/ds4/pack.py --model ~/models/DSV4--writes <model>/maya-ds4/:- ds4.txt the geometry, `key value` lines (lists comma-separated), from config.json's text_config (and its- vision_config as vision.<key>, image_token_id)- tensors.txt every tensor of the directory's safetensors (the index's 43 shards and the base repo's Engram shards- 47 / 48 beside them): `file <i> <name>` lines, then `t <name> <dtype> <file> <offset> <bytes> <dims...>`- engram.bin the Engram hash tables (tools/ds4/engram_hash.py): int32 magic 'DSE1', vocab, layers, max_ngram,- heads, head_dim, pad (compressed); int32 token_map[vocab]; int64 mult[layers][max_ngram];- int64 primes[layers][cols]; int64 offsets[layers][cols] (cols = (max_ngram - 1) * heads)- tokenizer/ the server's tokenizer (tools/strata_tokenizer.py's layout, pre "deepseek3") and chat template--The compressed token map needs the `tokenizers` package (it normalizes every token as DeepSeek's training did).-"""-from __future__ import annotations--import argparse-import json-import pathlib-import struct-import sys--import numpy as np--HERE = pathlib.Path(__file__).resolve().parent-sys.path.insert(0, str(HERE))-import engram_hash as EH # noqa: E402-from st import SafeTensors # noqa: E402--TEMPLATE = HERE / "chat_template.jinja"---def need(st: SafeTensors, t: dict) -> list[str]:- """The tensors the engine reads; a missing one stops the pack here instead of the engine's start."""- miss = []- for li in range(t["num_hidden_layers"]):- L = "layers.%d." % li- for n in ("attn.wq_a.trellis", "attn.wq_b.trellis", "attn.wkv.trellis", "attn.wo_b.trellis",- "attn.attn_sink", "attn.q_norm.weight", "attn.kv_norm.weight", "attn_norm.weight", "ffn_norm.weight",- "ffn.gate.weight", "ffn.gate.bias", "hc_attn_fn", "hc_ffn_fn", "ffn.shared_experts.w1.trellis"):- if not st.has(L + n):- miss.append(L + n)- if li in t["engram_layer_ids"] and not st.has(L + "engram.embed.weight"):- miss.append(L + "engram.embed.weight (shards 47 / 48 of deepseek-ai/DeepSeek-V4.1-Flash-1017)")- for n in ("embed.weight", "norm.weight", "head.trellis"):- if not st.has(n):- miss.append(n)- return miss---def write_engram(out: pathlib.Path, model: pathlib.Path, t: dict):- lay = EH.Layout.from_config(t)- tm, n = EH.compressed_token_map(model / "tokenizer.json")- if n != t["engram_compressed_vocab_size"]:- raise SystemExit("engram: the compressed vocab has %d ids, the config says %d" %- (n, t["engram_compressed_vocab_size"]))- vocab = t["vocab_size"]- full = np.zeros(vocab, dtype=np.int32) # ids past the tokenizer's (none in use) map to 0- full[:len(tm)] = tm- mult = EH.multipliers(lay, n)- cols = (lay.max_ngram - 1) * lay.n_heads- with out.open("wb") as f:- f.write(struct.pack("<4s6i", b"DSE1", vocab, len(lay.layer_ids), lay.max_ngram, lay.n_heads, lay.head_dim,- int(tm[t["engram_pad_token_id"]])))- f.write(full.astype("<i4").tobytes())- f.write(mult.astype("<i8").tobytes())- f.write(lay.flat_primes().astype("<i8").tobytes())- f.write(lay.offsets().astype("<i8").tobytes())- return len(lay.layer_ids), cols---def write_tokenizer(out: pathlib.Path, model: pathlib.Path):- """vocab.json (id order), merges.txt, token_type.json, tokenizer.json {pre, special_ids} and the chat template."""- tj = json.loads((model / "tokenizer.json").read_text(encoding="utf-8"))- vocab = tj["model"]["vocab"]- added = {a["id"]: a for a in tj["added_tokens"]}- n = max(max(vocab.values()), max(added)) + 1- tokens = ["<|unused▁%d|>" % i for i in range(n)] # (ids no token names: none in this vocabulary)- types = [1] * n # llama.cpp: 1 normal, 3 control, 4 user-defined- for s, i in vocab.items():- tokens[i] = s- for i, a in added.items():- tokens[i] = a["content"]- types[i] = 3 if a["special"] else 4- merges = tj["model"]["merges"]- merges = [m if isinstance(m, str) else " ".join(m) for m in merges]- out.mkdir(parents=True, exist_ok=True)- if len(set(tokens)) != n:- raise SystemExit("tokenizer.json: duplicate token strings")- (out / "vocab.json").write_text(json.dumps({t: i for i, t in enumerate(tokens)}, ensure_ascii=False),- encoding="utf-8")- (out / "merges.txt").write_text("\n".join(merges), encoding="utf-8")- (out / "token_type.json").write_text(json.dumps(types))- (out / "tokenizer.json").write_text(json.dumps({"pre": "deepseek3", "special_ids": {- "bos": 0, "eos": 1, "pad": 2}}, indent=1))- if TEMPLATE.exists():- (out / "chat_template.jinja").write_text(TEMPLATE.read_text(encoding="utf-8"), encoding="utf-8")---def main() -> int:- ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)- ap.add_argument("--model", required=True)- ap.add_argument("--out", help="default: <model>/maya-ds4")- a = ap.parse_args()- model = pathlib.Path(a.model).expanduser().resolve()- out = pathlib.Path(a.out).expanduser() if a.out else model / "maya-ds4"- cfg = json.loads((model / "config.json").read_text())- t = cfg["text_config"]- q = cfg.get("quantization_config", {})- if q.get("quant_method") != "exl3":- raise SystemExit("%s: not an EXL3 model (quant_method %r)" % (model, q.get("quant_method")))- st = SafeTensors(model)- miss = need(st, t)- if miss:- raise SystemExit("missing tensors:\n " + "\n ".join(miss[:20]))- out.mkdir(parents=True, exist_ok=True)-- with (out / "ds4.txt").open("w") as f:- for k, v in t.items():- if isinstance(v, bool):- v = int(v)- if isinstance(v, (int, float)):- f.write("%s %s\n" % (k, v))- elif isinstance(v, list) and all(isinstance(x, (int, float)) for x in v):- f.write("%s %s\n" % (k, ",".join(str(x) for x in v)))- elif isinstance(v, dict):- for k2, v2 in v.items():- if isinstance(v2, (int, float)) and not isinstance(v2, bool):- f.write("%s.%s %s\n" % (k, k2, v2))- # the vision tower's geometry (vision.<key>) and the token an image's span carries- for k, v in (cfg.get("vision_config") or {}).items():- if isinstance(v, (int, float)) and not isinstance(v, bool):- f.write("vision.%s %s\n" % (k, v))- if isinstance(cfg.get("image_token_id"), int):- f.write("image_token_id %d\n" % cfg["image_token_id"])- f.write("model_dir %s\n" % model)- with (out / "tensors.txt").open("w") as f:- for i, fn in enumerate(st.files):- f.write("file %d %s\n" % (i, fn))- for name, (fn, dt, shape, off, nb) in st.t.items():- f.write("t %s %s %d %d %d %s\n" % (name, dt, st.files.index(fn), off, nb, " ".join(map(str, shape))))- nl, cols = write_engram(out / "engram.bin", model, t)- write_tokenizer(out / "tokenizer", model)- print("%s: %d tensors in %d files, Engram %d layers x %d hash columns -> %s" %- (model.name, len(st.t), len(st.files), nl, cols, out))- return 0---if __name__ == "__main__":- sys.exit(main())diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.pyindex 5638fe837..055b33765 100644--- a/tools/dsv41/pack_gguf.py+++ b/tools/dsv41/pack_gguf.py@@ -482,10 +482,13 @@ def main() -> int:for name, dt, fi, off, nb, shape in tensors:f.write("t %s %s %d %d %d %s\n" % (name, dt, fi, off, nb, " ".join(map(str, shape))))nl, cols = write_engram(out / "engram.bin", kv, vocab)- # the server's chat template (pack.py's, so both formats chat alike); the GGUF's own when it is not there- template = pathlib.Path(__file__).resolve().parent / "chat_template.jinja"- write_tokenizer(out / "tokenizer", kv,- template.read_text(encoding="utf-8") if template.exists() else kv.get("tokenizer.chat_template"))+ # the server's chat template: the GGUF's own (unsloth's, byte-identical to DeepSeek's encoding.py on the 9 cases we+ # checked: chat, thinking, effort, system, multi-turn, tools, tool results). Project Maya's tools/ds4/chat_template.jinja+ # lost the tool associations and later system instructions (their #299), so it is NOT used here.+ template = kv.get("tokenizer.chat_template")+ if not template:+ sys.exit("%s: the GGUF has no tokenizer.chat_template; refusing to fall back to the old Maya template (#299)" % model)+ write_tokenizer(out / "tokenizer", kv, template)by = {}for t in tensors:by[t[1]] = by.get(t[1], 0) + t[4]