Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
4.0 KB
Raw Blame
#!/usr/bin/env python3
"""Convert trained pregate tensors to a memory-mappable safetensors file.

The converter stacks the per-layer heads, stores linear weights transposed for batched
matrix multiplication, preserves their fp16 values, and records the owning layer IDs.
It accepts an NPZ or safetensors source.

Usage:
    convert_pregate.py <input.npz|input.safetensors> <output.safetensors>
"""

import json
import struct
import sys
from pathlib import Path

import numpy as np


def load_safetensors(path):
    """Small NumPy-only reader so deployment does not require another package."""
    with path.open("rb") as handle:
        header_size = struct.unpack("<Q", handle.read(8))[0]
        header = json.loads(handle.read(header_size))
    base = 8 + header_size
    dtype = {"F16": np.dtype("<f2"), "F32": np.dtype("<f4"), "I32": np.dtype("<i4")}
    result = {}
    for name, entry in header.items():
        if name == "__metadata__":
            continue
        start, end = entry["data_offsets"]
        result[name] = np.memmap(
            path, mode="r", dtype=dtype[entry["dtype"]], offset=base + start,
            shape=tuple(entry["shape"]), order="C")
    return result


def save_safetensors(tensors, path):
    """Write the subset of safetensors used by these converted fp16/int32 heads."""
    dtype = {np.dtype("<f2"): "F16", np.dtype("<f4"): "F32", np.dtype("<i4"): "I32"}
    header = {}
    offset = 0
    for name, value in tensors.items():
        value = np.ascontiguousarray(value)
        tensors[name] = value
        size = value.nbytes
        header[name] = {
            "dtype": dtype[value.dtype], "shape": list(value.shape),
            "data_offsets": [offset, offset + size],
        }
        offset += size
    encoded = json.dumps(header, separators=(",", ":")).encode("utf-8")
    encoded += b" " * ((8 - len(encoded) % 8) % 8)
    temporary = path.with_suffix(path.suffix + ".partial")
    with temporary.open("wb", buffering=4 << 20) as handle:
        handle.write(struct.pack("<Q", len(encoded)))
        handle.write(encoded)
        for value in tensors.values():
            handle.write(memoryview(value).cast("B"))
    temporary.replace(path)


def main():
    if len(sys.argv) < 3:
        raise SystemExit(__doc__)
    source, destination = Path(sys.argv[1]), Path(sys.argv[2])

    # Accept both supported checkpoint containers.
    if source.suffix == ".safetensors":
        npz = load_safetensors(source)
    else:
        npz = dict(np.load(source))
    owners = sorted({int(name.split(".")[1]) for name in npz})
    if not owners:
        raise SystemExit(f"no `layers.N.*` tensors in {source}")

    # Normalize the checkpoint's `prerouter` names to the runtime's `pregate` names.
    prefix = "pregate"
    parts = {}
    for part in ("fc1", "fc2", "linear_init"):
        stack = []
        for owner in owners:
            key = f"layers.{owner}.{part}.weight"
            if key not in npz:
                raise SystemExit(f"{source.name} is missing {key}")
            # `[out, in]` -> `[in, out]`, contiguous. `ascontiguousarray` is not
            # decoration: safetensors stores a flat row-major buffer and a transposed
            # view is not one, so without it the file would either fail to write or, far
            # worse, write the untransposed bytes under the transposed shape.
            stack.append(np.ascontiguousarray(npz[key].T))
        merged = np.stack(stack, axis=0)
        if merged.dtype != np.float16:
            raise SystemExit(f"{part} is {merged.dtype}, expected float16")
        parts[f"{prefix}.{part}"] = merged

    parts[f"{prefix}.owners"] = np.array(owners, dtype=np.int32)

    save_safetensors(parts, destination)
    total = sum(v.nbytes for v in parts.values())
    print(f"  {destination}")
    for name, value in parts.items():
        print(f"    {name:24s} {str(list(value.shape)):20s} {value.dtype}")
    print(f"  owners {owners[0]}..{owners[-1]}  n={len(owners)}  "
          f"{total / 2**20:.1f} MiB")


if __name__ == "__main__":
    main()