Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

7.7 KB
Raw Blame
"""Write header-only stubs of the GGUF files in HF repos: the KV metadata and tensor table, in a sparse file of the real size.

    python scripts/gguf_check/fetch_headers.py <out_dir> [<repo> ...] [--search <query>]

The weights stay holes that read as zeros, so a stub takes a few MB of disk and ``check.py`` reads it like the real file.
Stubs land at ``<out_dir>/<repo>/<path in repo>``; a stub already there with the right size is kept, so a rerun only fetches what is missing.
Prints one JSON line per file. Gated repos read ``HF_TOKEN``.
"""

from __future__ import annotations

import argparse
import concurrent.futures as cf
import json
import os
import struct
import sys
import threading

import requests
from huggingface_hub import HfApi, hf_hub_url
from huggingface_hub.utils import build_hf_headers

_FIRST_READ, _NEXT_READ = 24 << 20, 16 << 20
_SCALAR = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
_STRING, _ARRAY = 8, 9
_F32 = 0
# tensors whose values a parse_gguf_config reads, so their bytes go into the stub
VALUE_TENSORS = ("rope_freqs.weight",)

_local = threading.local()


def _get_range(url: str, start: int, n: int) -> requests.Response:
    """Bytes [start, start + n) of ``url``, or fewer at the end of the file; refuses a reply that is not exactly that range."""
    r = _session().get(url, headers={"Range": f"bytes={start}-{start + n - 1}"}, timeout=300)
    r.raise_for_status()
    got = r.headers.get("Content-Range", "")
    if r.status_code != 206 or not got.startswith(f"bytes {start}-"):
        raise ValueError(f"{url}: asked for bytes {start}-{start + n - 1}, got status {r.status_code} {got!r}")
    end = int(got.split("-", 1)[1].split("/", 1)[0])
    if len(r.content) != end - start + 1:
        raise ValueError(f"{url}: short read at byte {start}")
    return r


def _session() -> requests.Session:
    if not hasattr(_local, "session"):
        _local.session = requests.Session()
        _local.session.headers.update(build_hf_headers())
    return _local.session


class _Remote:
    """Reads a remote file front to back through range requests, keeping every byte read so far."""

    def __init__(self, url: str):
        self.url, self.buf, self.pos, self.size = url, bytearray(), 0, None

    def _need(self, n: int) -> None:
        while len(self.buf) < self.pos + n:
            start = len(self.buf)
            if self.size is not None and start >= self.size:
                raise EOFError(f"{self.url}: header runs past the end of the file")
            r = _get_range(self.url, start, max(_FIRST_READ if start == 0 else _NEXT_READ, self.pos + n - start))
            if self.size is None:
                self.size = int(r.headers["Content-Range"].rsplit("/", 1)[1])
            self.buf += r.content

    def read(self, n: int) -> bytes:
        self._need(n)
        self.pos += n
        return bytes(self.buf[self.pos - n : self.pos])

    def unpack(self, fmt: str) -> tuple:
        return struct.unpack("<" + fmt, self.read(struct.calcsize("<" + fmt)))

    def string(self) -> str:
        (n,) = self.unpack("Q")
        return self.read(n).decode("utf-8", "replace")

    def skip_value(self, kind: int) -> None:
        if kind in _SCALAR:
            self.read(struct.calcsize(_SCALAR[kind]))
        elif kind == _STRING:
            self.string()
        elif kind == _ARRAY:
            elem, n = self.unpack("I")[0], self.unpack("Q")[0]
            if elem in _SCALAR:
                self.read(struct.calcsize(_SCALAR[elem]) * n)
            else:
                for _ in range(n):
                    self.skip_value(elem)
        else:
            raise ValueError(f"{self.url}: unknown GGUF value type {kind}")


def fetch(repo: str, path: str, size: int, out_dir: str) -> dict:
    dst = os.path.join(out_dir, repo, path)
    rec = {"repo": repo, "path": path, "stub": dst, "size": size, "value_tensors": list(VALUE_TENSORS)}
    # the sidecar marks a finished stub and which tensors it holds the values of
    sidecar = dst + ".json"
    if os.path.isfile(dst) and os.path.getsize(dst) == size and os.path.isfile(sidecar):
        with open(sidecar) as f:
            if json.load(f).get("value_tensors") == rec["value_tensors"]:
                return {**rec, "ok": True, "cached": True}
    url = hf_hub_url(repo, path)
    try:
        f = _Remote(url)
        if f.read(4) != b"GGUF":
            raise ValueError("not a GGUF file")
        _, n_tensors, n_kv = f.unpack("IQQ")
        align = 32
        for _ in range(n_kv):
            key, kind = f.string(), f.unpack("I")[0]
            if key == "general.alignment" and kind in (4, 5):
                align = f.unpack("I")[0]
            else:
                f.skip_value(kind)
        tensors = {}
        for _ in range(n_tensors):
            name = f.string()
            (nd,) = f.unpack("I")
            dims = f.unpack(f"{nd}Q")
            kind, offset = f.unpack("IQ")
            tensors[name] = (dims, kind, offset)
        header_end = f.pos
        data_start = (header_end + align - 1) // align * align
        tmp = dst + ".part"
        os.makedirs(os.path.dirname(dst), exist_ok=True)
        with open(tmp, "wb") as out:
            out.write(f.buf[:header_end])
            out.truncate(f.size)
            for name in VALUE_TENSORS:
                if name not in tensors:
                    continue
                dims, kind, offset = tensors[name]
                if kind != _F32 or len(dims) != 1:
                    raise ValueError(f"{name} is type {kind} with dims {dims}, expected 1-D F32")
                start, n = data_start + offset, dims[0] * 4
                r = _get_range(url, start, n)
                if len(r.content) != n:
                    raise ValueError(f"{name} runs past the end of the file")
                out.seek(start)
                out.write(r.content)
        os.replace(tmp, dst)
        rec.update(header_bytes=header_end, tensors=n_tensors)
        with open(sidecar, "w") as f:
            json.dump(rec, f)
        return {**rec, "ok": True}
    except Exception as e:
        return {**rec, "ok": False, "error": f"{type(e).__name__}: {e}"[:300]}


def list_gguf(api: HfApi, repo: str) -> list[tuple[str, int]]:
    """(path, size) of every model GGUF in the repo; vision projectors and imatrix files are left out."""
    files = []
    for entry in api.list_repo_tree(repo, recursive=True):
        path = entry.path
        if path.endswith(".gguf") and not any(s in path.lower() for s in ("mmproj", "imatrix")):
            files.append((path, entry.size))
    return files


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("out_dir")
    ap.add_argument("repos", nargs="*")
    ap.add_argument("--search", action="append", default=[], help="also take every HF GGUF repo whose name matches this query (repeatable)")
    ap.add_argument("--workers", type=int, default=12)
    args = ap.parse_args()

    api = HfApi()
    repos = list(args.repos)
    for query in args.search:
        repos += [m.id for m in api.list_models(search=query, filter="gguf") if m.id not in repos]
    jobs = []
    for repo in repos:
        try:
            jobs += [(repo, path, size) for path, size in list_gguf(api, repo)]
        except Exception as e:
            print(json.dumps({"repo": repo, "ok": False, "error": f"{type(e).__name__}: {e}"[:300]}), flush=True)
    print(f"{len(jobs)} GGUF files in {len(repos)} repos", file=sys.stderr, flush=True)
    failed = 0
    with cf.ThreadPoolExecutor(args.workers) as ex:
        for rec in ex.map(lambda job: fetch(*job, args.out_dir), jobs):
            failed += not rec["ok"]
            print(json.dumps(rec), flush=True)
    return 1 if failed else 0


if __name__ == "__main__":
    sys.exit(main())