Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

12.4 KB
Raw Blame
import copy
import functools
import json
import os
from typing import Any, FrozenSet, Mapping

from huggingface_hub import hf_hub_download, snapshot_download
from huggingface_hub.utils import EntryNotFoundError
from tqdm.asyncio import tqdm
from transformers import (
    AutoConfig,
    AutoTokenizer,
    GenerationConfig,
    PretrainedConfig,
    PreTrainedTokenizerBase,
)
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME

from freetoken.utils.logger import init_logger

logger = init_logger(__name__)


class DisabledTqdm(tqdm):
    def __init__(self, *args, **kwargs):
        kwargs.pop("name", None)
        kwargs["disable"] = True
        super().__init__(*args, **kwargs)


def load_tokenizer(model_path: str) -> PreTrainedTokenizerBase:
    from freetoken.models.gguf.reader import gguf_config_source

    if (gguf_src := gguf_config_source(model_path)) is not None:
        from freetoken.models.gguf.tokenizer import load_gguf_tokenizer

        return load_gguf_tokenizer(gguf_src)
    tokenizer = AutoTokenizer.from_pretrained(model_path)
    # Some Mistral models store chat_template in a separate JSON file
    if not getattr(tokenizer, "chat_template", None):
        try:
            path = hf_hub_download(repo_id=model_path, filename="chat_template.json")
            with open(path, "r", encoding="utf-8") as f:
                tokenizer.chat_template = json.load(f)["chat_template"]
        except Exception:
            pass
    return tokenizer


def load_image_processor(model_path: str) -> Any:
    """The checkpoint's image processor; a GGUF builds it from the preprocessor_config its family derives."""
    from freetoken.models.gguf.reader import gguf_config_source

    if gguf_config_source(model_path) is not None:
        from transformers.models.auto.image_processing_auto import get_image_processor_class_from_name

        config = cached_load_hf_config(model_path).preprocessor_config
        if config is None:
            raise ValueError(f"{model_path}: GGUF has no vision mmproj to build an image processor from")
        return get_image_processor_class_from_name(config["image_processor_type"]).from_dict(config)
    from transformers import AutoImageProcessor

    return AutoImageProcessor.from_pretrained(model_path)


def load_eos_token_ids(
    model_path: str, tokenizer: PreTrainedTokenizerBase
) -> FrozenSet[int]:
    """Return the full set of stop-token ids for generation.

    Chat models often terminate a turn with a token other than ``tokenizer.eos_token``
    (e.g. Gemma uses ``<end_of_turn>``), and list every stop id in
    ``generation_config.json``'s ``eos_token_id``. Honor that list (unioned with the
    tokenizer's eos) so generation halts correctly instead of running to ``max_tokens``.
    """
    from freetoken.models.gguf.reader import gguf_config_source

    if (gguf_src := gguf_config_source(model_path)) is not None:
        from freetoken.models.gguf.tokenizer import gguf_eos_token_ids

        return frozenset(gguf_eos_token_ids(gguf_src))

    ids: set[int] = set()
    if tokenizer.eos_token_id is not None:
        ids.add(int(tokenizer.eos_token_id))
    try:
        gen_eos = GenerationConfig.from_pretrained(model_path).eos_token_id
    except Exception:
        gen_eos = None
    if isinstance(gen_eos, int):
        ids.add(gen_eos)
    elif isinstance(gen_eos, (list, tuple)):
        ids.update(int(x) for x in gen_eos)
    return frozenset(ids)


def load_toolcall_anchor_id(
    tokenizer: PreTrainedTokenizerBase, opener: str | None
) -> int | None:
    """The single token id of ``opener`` -- the wire format's unique tool-call opening
    marker, declared by the model's detector (``BaseFormatDetector.toolcall_opener``).
    None when there is no opener or the tokenizer spells it with more than one token:
    the scheduler's special-token checkpoint matches sampled ids one at a time, so only
    a one-token opener can anchor."""
    if not opener:
        return None
    try:
        ids = tokenizer.encode(opener, add_special_tokens=False)
    except Exception:
        return None
    return int(ids[0]) if len(ids) == 1 else None


def load_generation_sampling(model_path: str) -> dict[str, Any]:
    """Recommended sampling defaults from ``generation_config.json`` (sglang's
    ``sampling_defaults='model'``). Returns ``{temperature, top_k, top_p}`` for the keys
    present; if the model recommends greedy (``do_sample=false``) returns greedy
    (``temperature=0``). Empty dict if there is no generation config / no sampling info.

    Many reasoning models (e.g. Qwen3.5: temp 1.0, top_k 20, top_p 0.95) ship these here;
    applying them avoids the greedy/unfiltered repetition loops these models fall into.
    """
    # GGUF carries the recommended sampling in metadata (general.sampling.*), not a
    # generation_config.json -- read it directly so --sampling-defaults=model works.
    from freetoken.models.gguf.reader import gguf_config_source

    if (gguf_src := gguf_config_source(model_path)) is not None:
        from freetoken.models.gguf.reader import load_gguf_metadata

        meta = load_gguf_metadata(gguf_src)
        out: dict[str, Any] = {}
        if (v := meta.get("general.sampling.temp")) is not None:
            out["temperature"] = float(v)
        if (v := meta.get("general.sampling.top_k")) is not None:
            out["top_k"] = int(v)
        if (v := meta.get("general.sampling.top_p")) is not None:
            out["top_p"] = float(v)
        return out

    try:
        gc = GenerationConfig.from_pretrained(model_path)
    except Exception:
        return {}
    if getattr(gc, "do_sample", None) is False:
        return {"temperature": 0.0}
    out: dict[str, Any] = {}
    for key in ("temperature", "top_k", "top_p"):
        val = getattr(gc, key, None)
        if val is not None:
            out[key] = val
    return out


class RawConfigShim:
    """Attribute view over a checkpoint's raw ``config.json``.

    Fallback for model types NEWER than the installed transformers (AutoConfig raises
    on an unknown ``model_type`` when the checkpoint ships no ``auto_map``, e.g.
    Muse-Glimmer on transformers < 5.15). FreeToken only reads config fields
    (parse_config) and never instantiates modeling code, so the raw JSON is enough.
    ``*_config`` sub-dicts are wrapped for attribute access, matching
    PretrainedConfig's nested-config behavior; every other value is served raw
    (``rope_parameters`` and friends stay plain dicts, as they do on the real class).
    """

    def __init__(self, data: dict | None = None, **kwargs: Any) -> None:
        self._data = {**(data or {}), **kwargs}

    def __getattr__(self, name: str) -> Any:
        if name == "_name_or_path":
            # PretrainedConfig carries the checkpoint path under this name and
            # DSV4's parse_config reads it (to find inference/config.json), so it
            # must survive the underscore guard below.
            return self.__dict__.get("_data", {}).get("_name_or_path", "")
        if name.startswith("_"):
            raise AttributeError(name)
        try:
            value = self._data[name]
        except KeyError:
            raise AttributeError(name) from None
        if name.endswith("_config") and isinstance(value, dict):
            return RawConfigShim(value)
        return value

    def to_dict(self) -> dict:
        return json.loads(json.dumps(self._data))  # deep copy, callers may mutate


def optional_hf_file(model_path: str, filename: str) -> str | None:
    """Local path of ``filename`` in a checkpoint dir or Hub repo; None when the checkpoint has no such file."""
    if os.path.isdir(model_path):
        path = os.path.join(model_path, filename)
        return path if os.path.isfile(path) else None
    try:
        return hf_hub_download(repo_id=model_path, filename=filename)
    except EntryNotFoundError:
        return None


def _raw_config_json(model_path: str) -> dict:
    if os.path.isdir(model_path):
        path = os.path.join(model_path, "config.json")
    else:
        path = hf_hub_download(repo_id=model_path, filename="config.json")
    with open(path, encoding="utf-8") as f:
        return json.load(f)


def sidecar_quantization_config(model_path: str) -> dict | None:
    """The ``quantization_config`` an old ModelOpt export keeps only in ``hf_quant_config.json``, or None."""
    sidecar = optional_hf_file(model_path, "hf_quant_config.json")
    if sidecar is None:
        return None
    with open(sidecar, encoding="utf-8") as f:
        quant = json.load(f).get("quantization")
    if not isinstance(quant, dict):
        return None
    return {"quant_method": "modelopt", **quant}


def _merge_sidecar_quantization_config(config: Any, model_path: str) -> None:
    # config.json wins when it has one; the sidecar is only read for exports that never wrote it there
    if getattr(config, "quantization_config", None) is not None:
        return
    if getattr(getattr(config, "text_config", None), "quantization_config", None) is not None:
        return
    quant = sidecar_quantization_config(model_path)
    if quant is None:
        return
    if isinstance(config, RawConfigShim):
        config._data["quantization_config"] = quant
    else:
        config.quantization_config = quant


@functools.cache
def _load_hf_config(model_path: str) -> Any:
    # trust_remote_code: checkpoints that ship a custom config class via ``auto_map``
    # (e.g. MiniMax-M2) refuse to load without it. FreeToken only reads config fields
    # (parse_config) and never instantiates the checkpoint's modeling code.
    try:
        config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
    except ValueError as exc:
        # Unknown model_type on this transformers version: serve off the raw JSON.
        # Anything else (bad path, malformed JSON) stays fatal.
        if "model type" not in str(exc):
            raise
        config = RawConfigShim(_raw_config_json(model_path), _name_or_path=model_path)
    _merge_sidecar_quantization_config(config, model_path)
    return config


def _with_overrides(config: Any, data: dict, overrides: Mapping[str, Any]) -> dict:
    merged = dict(data)
    for key, value in overrides.items():
        section = getattr(config, key, None)
        if isinstance(value, Mapping) and isinstance(section, (PretrainedConfig, RawConfigShim)):
            merged[key] = _with_overrides(section, merged.get(key) or {}, value)
        else:
            merged[key] = copy.deepcopy(value)
    return merged


def cached_load_hf_config(
    model_path: str, overrides: Mapping[str, Any] | None = None
) -> PretrainedConfig:
    """A fresh copy of the checkpoint's config with ``overrides`` applied as vLLM's --hf-overrides
    does: nested config sections update key by key, every other value is replaced whole."""
    # A .gguf file (or an FTW dir converted from one) carries its own metadata (no HF
    # config.json); return a shim the model registry dispatches on instead of a
    # PretrainedConfig.
    from freetoken.models.gguf.reader import gguf_config_source

    if (gguf_src := gguf_config_source(model_path)) is not None:
        if overrides:
            raise ValueError("--hf-overrides applies to a HF config.json; a GGUF carries its own metadata")
        from freetoken.models.gguf.config import build_gguf_shim

        return build_gguf_shim(gguf_src)
    config = _load_hf_config(model_path)
    data = config.to_dict()
    if overrides:
        data = _with_overrides(config, data, overrides)
    if isinstance(config, RawConfigShim):
        return RawConfigShim(data)
    return type(config)(**data)


def _weight_allow_patterns(repo_id: str) -> list[str]:
    try:
        index = hf_hub_download(repo_id, SAFE_WEIGHTS_INDEX_NAME, tqdm_class=DisabledTqdm)
        with open(index, encoding="utf-8") as f:
            shards = sorted(set(json.load(f)["weight_map"].values()))
    except Exception as e:
        logger.warning(
            "no usable %s for %s (%s); falling back to *.safetensors",
            SAFE_WEIGHTS_INDEX_NAME, repo_id, e,
        )
        return ["*.safetensors"]
    return shards or ["*.safetensors"]


def download_hf_weight(model_path: str) -> str:
    if os.path.isdir(model_path):
        return model_path
    try:
        return snapshot_download(
            model_path,
            allow_patterns=_weight_allow_patterns(model_path),
            tqdm_class=DisabledTqdm,
        )
    except Exception as e:
        raise ValueError(
            f"Model path '{model_path}' is neither a local directory nor a valid model ID: {e}"
        )