Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
6.6 KB
Raw Blame
#!/usr/bin/env python3
"""tools/dflash2_pack.py reads a .safetensors file with numpy alone (the person's PC has neither torch nor the safetensors package) and writes
the same runtime file the torch-based version wrote.

    python tools/test_dflash2_pack.py
"""
import importlib.util
import json
import struct
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

import numpy as np

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import dflash2_pack as new  # noqa: E402

L, H, INTER, HEADS, KV, HD, RANK, VOCAB = 2, 64, 96, 2, 1, 32, 32, 128
CFG = {"num_hidden_layers": L, "hidden_size": H, "num_attention_heads": HEADS, "num_key_value_heads": KV, "head_dim": HD,
       "intermediate_size": INTER, "sliding_window": 128, "rope_parameters": {"rope_theta": 1000000.0}, "rms_norm_eps": 1e-6, "vocab_size": VOCAB,
       "dflash_config": {"block_size": 8, "conv_group_size": 16, "conv_kernel_size": 2, "mask_token_id": 7, "selector_rank": RANK, "selector_top_k": 4,
                         "target_layer_ids": [1, 2, 3]}}


def bf16(a):
    """float32 -> BF16 bytes by truncation (what torch's .to(bfloat16) does for these values up to rounding; the test only needs a valid BF16 file)."""
    return (np.ascontiguousarray(a, np.float32).view(np.uint32) >> 16).astype(np.uint16)


def tensors():
    rng = np.random.default_rng(7)
    r = lambda *shape: rng.standard_normal(shape).astype(np.float32) * 0.1
    t = {"fc.weight": r(H, H * 3), "hidden_norm.weight": r(H), "norm.weight": r(H)}
    for l in range(L):
        p = "layers.%d." % l
        t[p + "self_attn.q_proj.weight"] = r(HEADS * HD, H)
        t[p + "self_attn.k_proj.weight"] = r(KV * HD, H)
        t[p + "self_attn.v_proj.weight"] = r(KV * HD, H)
        t[p + "self_attn.o_proj.weight"] = r(H, HEADS * HD)
        t[p + "mlp.gate_proj.weight"] = r(INTER, H)
        t[p + "mlp.up_proj.weight"] = r(INTER, H)
        t[p + "mlp.down_proj.weight"] = r(H, INTER)
        t[p + "attention_conv.kernel_projection.weight"] = r(32, H)
        t[p + "mlp_conv.kernel_projection.weight"] = r(32, H)
        for n in ("input_layernorm", "post_attention_layernorm", "self_attn.q_norm", "self_attn.k_norm"):
            t[p + n + ".weight"] = r(HD if "_norm" in n else H)
        t[p + "attention_conv.base_kernel"] = r(2, 16)
        t[p + "mlp_conv.base_kernel"] = r(2, 16)
    t["candidate_selector.hidden_projection.weight"] = r(RANK, H)
    t["candidate_selector.predecessor_codebook"] = r(VOCAB, RANK)
    t["candidate_selector.successor_codebook"] = r(VOCAB, RANK)
    return t


def write_safetensors(path, arrays):
    """A BF16 safetensors file written by hand (the format is documented: u64 header length, JSON header, raw data)."""
    header, blobs, at = {}, [], 0
    for name, a in arrays.items():
        data = bf16(a).tobytes()
        header[name] = {"dtype": "BF16", "shape": list(a.shape), "data_offsets": [at, at + len(data)]}
        blobs.append(data)
        at += len(data)
    js = json.dumps(header).encode()
    js += b" " * ((8 - len(js) % 8) % 8)
    with open(path, "wb") as fh:
        fh.write(struct.pack("<Q", len(js)) + js + b"".join(blobs))


class PackWithoutTorch(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.tmp = tempfile.TemporaryDirectory()
        cls.dir = Path(cls.tmp.name)
        cls.arrays = tensors()
        write_safetensors(cls.dir / "model.safetensors", cls.arrays)
        (cls.dir / "config.json").write_text(json.dumps(CFG))

    @classmethod
    def tearDownClass(cls):
        cls.tmp.cleanup()

    def pack(self, out):
        return new.pack(str(self.dir / "model.safetensors"), str(self.dir / "config.json"), str(out), say=lambda *_: None)

    def test_reader_widens_bf16_exactly(self):
        f = new.SafeTensors(str(self.dir / "model.safetensors"))
        got = f.get_float32("fc.weight")
        want = (bf16(self.arrays["fc.weight"]).astype(np.uint32) << 16).view(np.float32)
        self.assertTrue(np.array_equal(got, want))
        self.assertEqual(f.get_bf16_bits("candidate_selector.predecessor_codebook").dtype, np.uint16)
        self.assertEqual(f.raw("norm.weight")[0], "BF16")

    def test_the_file_has_the_header_the_engine_reads(self):
        out = self.dir / "out.pack"
        size = self.pack(out)
        data = out.read_bytes()
        self.assertEqual(data[:4], b"DF2P")
        version, index_len = struct.unpack("<II", data[4:12])
        self.assertEqual(version, 1)
        self.assertEqual((12 + index_len) % 4096, 0, "the data starts on a 4096 multiple")
        index = data[12:12 + index_len].decode().splitlines()
        self.assertTrue(any(l.startswith("cfg hidden %d" % H) for l in index))
        self.assertIn("end", [l.strip() for l in index])
        names = [l.split()[1] for l in index if l.startswith("t ")]
        self.assertIn("fc.weight", names)
        self.assertIn("candidate_selector.successor_codebook", names)
        self.assertEqual(len(data), size)

    def test_a_matrix_is_ggml_q8_0_of_the_widened_values(self):
        out = self.dir / "q8.pack"
        self.pack(out)
        data = out.read_bytes()
        index_len = struct.unpack("<I", data[8:12])[0]
        base = 12 + index_len
        row = next(l.split() for l in data[12:base].decode().splitlines() if l.startswith("t fc.weight "))
        _t, _name, dtype, rows, cols, off, nbytes = row
        self.assertEqual(dtype, "q8_0")
        got = data[base + int(off): base + int(off) + int(nbytes)]
        widened = (bf16(self.arrays["fc.weight"]).astype(np.uint32) << 16).view(np.float32)
        self.assertEqual(got, new.quant_q8_0(widened))

    def test_it_matches_the_torch_version_when_torch_is_here(self):
        try:
            import torch  # noqa: F401
            import safetensors  # noqa: F401
        except ImportError:
            self.skipTest("torch or safetensors is not installed here (the point of the change)")
        old_src = subprocess.run(["git", "show", "f1b6a2144:tools/dflash2_pack.py"], cwd=HERE, capture_output=True, text=True)
        if old_src.returncode != 0:
            self.skipTest("the earlier version is not in this clone")
        old_path = self.dir / "old_pack.py"
        old_path.write_text(old_src.stdout)
        spec = importlib.util.spec_from_file_location("old_pack", old_path)
        old = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(old)
        a, b = self.dir / "a.pack", self.dir / "b.pack"
        sys.argv = ["x", str(self.dir / "model.safetensors"), str(self.dir / "config.json"), str(a)]
        old.main()
        self.pack(b)
        self.assertEqual(a.read_bytes(), b.read_bytes(), "byte for byte the same file")


if __name__ == "__main__":
    unittest.main()