Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
nvfp4_local_pack: slim (the dense-only GGUF), convert --dense-only
Niko12217f9e719c8a4e
diff --git a/tools/nvfp4_local_pack.py b/tools/nvfp4_local_pack.pyindex 2d903fb03..7b527964c 100644--- a/tools/nvfp4_local_pack.py+++ b/tools/nvfp4_local_pack.py@@ -170,11 +170,13 @@ def _orig_name(ck: Checkpoint, name: str):return None-def overlay(tensors: dict, ck: Checkpoint, stats: dict) -> dict:+def overlay(tensors: dict, ck: Checkpoint, stats: dict, dense_only: bool = False) -> dict:"""Replace the CSF stream pairs by plain `weight_scale` tensors, the MXFP8 weights by BF16, and drop whatthe PLE writer and the MTP packer own."""import torchout = dict(tensors)+ if dense_only: # the GGUF the engine reads the dense weights from: no experts in it+ out = {k: v for k, v in out.items() if ".mlp.experts." not in k}for name in list(out):if name not in out:continue@@ -211,7 +213,7 @@ def overlay(tensors: dict, ck: Checkpoint, stats: dict) -> dict:return out-def install(ck: Checkpoint, stats: dict):+def install(ck: Checkpoint, stats: dict, dense_only: bool = False):"""Import the converter, give it nvfp4_convert's type policy, and hook the tensor index."""import nvfp4_convert as basebase.install_policy()@@ -220,7 +222,7 @@ def install(ck: Checkpoint, stats: dict):orig_index = ModelBase.index_tensorsdef index_tensors(self, remote_hf_model_id=None):- return overlay(orig_index(self, remote_hf_model_id), ck, stats)+ return overlay(orig_index(self, remote_hf_model_id), ck, stats, dense_only)ModelBase.index_tensors = index_tensors# NVIDIA's repo keeps the shards in a `tensors/` folder that its index names; the converter only lists the top level@@ -251,7 +253,7 @@ def cmd_convert(a) -> int:model = pathlib.Path(a.model)ck = Checkpoint(model)stats: dict = {}- base = install(ck, stats)+ base = install(ck, stats, a.dense_only)import convert_hf_to_gguf as Cout = pathlib.Path(a.outfile)out.parent.mkdir(parents=True, exist_ok=True)@@ -410,6 +412,43 @@ def cmd_mtp(a) -> int:return 0+# ------------------------------------------------------------------------------------------------------ the slim GGUF+def cmd_slim(a) -> int:+ """Copy a converted GGUF without its routed experts (and their .scale / .input_scale tensors): the file the engine+ reads the dense weights from (--native / --native-dense-gguf), ~9 GB instead of ~74 GB, once experts.bin is cut."""+ from _paths import add_gguf_py+ add_gguf_py()+ import gguf+ from gguf import GGUFReader, GGUFWriter, GGUFValueType as VT+ rd = GGUFReader(a.gguf)+ arch = rd.fields["general.architecture"].contents()+ w = GGUFWriter(a.out, arch)+ for key, f in rd.fields.items():+ if key.startswith("GGUF.") or key == "general.architecture":+ continue+ vt = f.types[0]+ w.add_key_value(key, f.contents(), vt, sub_type=f.types[-1] if vt == VT.ARRAY else None)+ kept = skipped = 0+ for t in rd.tensors:+ if "_exps." in t.name:+ skipped += 1+ continue+ data = np.asarray(t.data)+ qt = t.tensor_type+ if qt in (gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16):+ w.add_tensor(t.name, data)+ else: # raw bytes with the type: the quantized blocks (BF16 rides as uint16)+ raw = np.ascontiguousarray(data).view(np.uint8)+ w.add_tensor(t.name, raw, raw_dtype=qt, raw_shape=[int(x) for x in raw.shape])+ kept += 1+ w.write_header_to_file()+ w.write_kv_data_to_file()+ w.write_tensors_to_file()+ w.close()+ print("slim: %d tensors kept, %d expert tensors left out -> %s" % (kept, skipped, a.out))+ return 0++# ------------------------------------------------------------------------------------------------------ verifyingdef cmd_verify(a) -> int:"""Experts: decode the checkpoint (CSF + ModelOpt definition) and compare with gguf-py's dequantization of the@@ -451,18 +490,23 @@ def main() -> int:c.add_argument("--model", required=True)c.add_argument("--outfile", required=True)c.add_argument("--dry-run", action="store_true")+ c.add_argument("--dense-only", action="store_true",+ help="leave the routed experts out: the GGUF that --native-dense-gguf reads (~9 GB) beside an experts.bin")p = sub.add_parser("ple")p.add_argument("--model", required=True)p.add_argument("--out", required=True)m = sub.add_parser("mtp")m.add_argument("--model", required=True)m.add_argument("--out", required=True)+ sl = sub.add_parser("slim")+ sl.add_argument("--gguf", required=True)+ sl.add_argument("--out", required=True)v = sub.add_parser("verify")v.add_argument("--model", required=True)v.add_argument("--gguf", required=True)v.add_argument("--experts", type=int, default=4)a = ap.parse_args()- return {"convert": cmd_convert, "ple": cmd_ple, "mtp": cmd_mtp, "verify": cmd_verify}[a.cmd](a)+ return {"convert": cmd_convert, "ple": cmd_ple, "mtp": cmd_mtp, "slim": cmd_slim, "verify": cmd_verify}[a.cmd](a)if __name__ == "__main__":