Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
3.7 KB
Raw Blame
"""hf_slice.py - read single tensors (or slices of them) out of a remote safetensors repo with HTTP range requests, so a
quality test needs megabytes of a model, not the model.  No token is read or printed (public repos only; set HF_TOKEN in
the environment for gated ones - it is sent as a header and never printed).

  from hf_slice import Repo
  r = Repo("Qwen/Qwen3.8-Flash-Next")
  w = r.tensor("model.language_model.layers.0.mlp.experts.gate_up_proj", rows=slice(0, 1))   # expert 0 only
"""
from __future__ import annotations

import concurrent.futures as cf
import json
import os
import struct
import urllib.request

import numpy as np

_DT = {"BF16": (2, None), "F16": (2, np.float16), "F32": (4, np.float32), "F8_E4M3": (1, None), "I8": (1, np.int8),
       "U8": (1, np.uint8), "I32": (4, np.int32), "I64": (8, np.int64)}


def _get(url: str, start: int | None = None, end: int | None = None) -> bytes:
    req = urllib.request.Request(url)
    tok = os.environ.get("HF_TOKEN")
    if tok:
        req.add_header("Authorization", "Bearer " + tok)
    if start is not None:
        req.add_header("Range", "bytes=%d-%d" % (start, end))
    with urllib.request.urlopen(req, timeout=120) as r:
        return r.read()


def get_parallel(url: str, start: int, end: int, workers: int = 16, piece: int = 8 << 20) -> bytes:
    """bytes start..end (inclusive) of url, in `piece`-byte range requests on `workers` threads (one stream of a CDN
    object is slow, a dozen are not)."""
    spans = [(a, min(end, a + piece - 1)) for a in range(start, end + 1, piece)]
    if len(spans) == 1:
        return _get(url, start, end)

    def one(sp):
        for attempt in range(6):
            try:
                return _get(url, sp[0], sp[1])
            except Exception:
                if attempt == 5:
                    raise
        return b""
    with cf.ThreadPoolExecutor(workers) as ex:
        return b"".join(ex.map(one, spans))


def bf16_to_f32(raw: bytes | np.ndarray) -> np.ndarray:
    u = np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16
    return u.view(np.float32)


class Repo:
    def __init__(self, repo: str, rev: str = "main"):
        self.repo, self.rev = repo, rev
        self.base = "https://huggingface.co/%s/resolve/%s/" % (repo, rev)
        self.map = json.loads(_get(self.base + "model.safetensors.index.json"))["weight_map"]
        self._hdr: dict[str, tuple[dict, int]] = {}

    def header(self, shard: str):
        if shard not in self._hdr:
            n = struct.unpack("<Q", _get(self.base + shard, 0, 7))[0]
            h = json.loads(_get(self.base + shard, 8, 8 + n - 1))
            self._hdr[shard] = (h, 8 + n)
        return self._hdr[shard]

    def info(self, name: str):
        shard = self.map[name]
        h, base = self.header(shard)
        return shard, h[name], base

    def tensor(self, name: str, lead: slice | int | None = None, workers: int = 16) -> np.ndarray:
        """The tensor `name` as float32.  lead: an index or slice on the first axis (e.g. an expert), fetched alone."""
        shard, t, base = self.info(name)
        shape = t["shape"]
        item = _DT[t["dtype"]][0]
        s, e = t["data_offsets"]
        inner = int(np.prod(shape[1:])) * item if len(shape) > 1 else item
        if lead is None:
            i0, i1 = 0, shape[0]
        elif isinstance(lead, int):
            i0, i1 = lead, lead + 1
        else:
            i0, i1 = lead.start or 0, lead.stop
        raw = get_parallel(self.base + shard, base + s + i0 * inner, base + s + i1 * inner - 1, workers)
        out_shape = [i1 - i0] + shape[1:]
        if t["dtype"] == "BF16":
            return bf16_to_f32(raw).reshape(out_shape)
        return np.frombuffer(raw, dtype=_DT[t["dtype"]][1]).astype(np.float32).reshape(out_shape)