Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

6.4 KB
Raw Blame
"""llama.cpp's ggml-cuda kernels (vendored under ``csrc/ggml``) as a tvm-ffi module: prebuilt in the kernel cache, else compiled on first use.

Weights stay in their packed ggml blocks. ``mul_mat`` / ``mul_mat_id`` pick MMVQ or MMQ the way llama.cpp's mul_mat / mul_mat_id do.
Activations are fp32, bf16 or fp16 and are quantized to q8_1 first; results come out in the activation dtype and are accumulated in fp32.
"""

from __future__ import annotations

import functools
import pathlib
from typing import TYPE_CHECKING

import torch

from .utils import _is_rocm, load_aot

if TYPE_CHECKING:
    from tvm_ffi import Module

_ROOT = pathlib.Path(__file__).parent / "csrc" / "ggml"


def load_module(build_directory: str | None = None) -> Module:
    cuda_dir = _ROOT / "ggml-cuda"
    sources = [
        _ROOT / "freetoken" / "ggml_shim.cu",
        _ROOT / "freetoken" / "ggml_ops.cu",
        *(cuda_dir / f for f in ("mmvq.cu", "mmq.cu", "mmid.cu", "quantize.cu", "convert.cu", "getrows.cu")),
        *sorted((cuda_dir / "template-instances").glob("mmq-instance-*.cu")),
    ]
    if _is_rocm():
        flags = ["-ffast-math", "-DGGML_USE_HIP"]
    else:
        # a multi-arch kernel-cache build is ~6x smaller with compressed SASS
        flags = ["-use_fast_math", "--extended-lambda", "--compress-mode=size"]
    return load_aot(
        "ggml",
        cuda_files=[str(s) for s in sources],
        extra_cuda_cflags=flags,
        extra_include_paths=[str(_ROOT / "include"), str(_ROOT / "src"), str(cuda_dir)],
        build_directory=build_directory,
    )


@functools.cache
def _module() -> Module:
    return load_module()


@functools.cache
def _mmq_tail_bytes(ggml_type: int, k: int) -> int:
    return _module().mmq_tail_bytes(ggml_type, k)


def _check_mmq_tail(w: torch.Tensor, ggml_type: int, k: int) -> None:
    """MMQ tiles read up to MATRIX_ROW_PADDING values past the last row of ``w``, so its storage must hold them."""
    tail = _mmq_tail_bytes(ggml_type, k)
    if tail == 0 or w.numel() == 0:
        return
    end = w.storage_offset() + sum((n - 1) * s for n, s in zip(w.shape, w.stride())) + 1
    if end + tail > w.untyped_storage().nbytes():
        raise ValueError(f"ggml type {ggml_type} K={k} weight needs {tail} bytes of zero padding after its last row for MMQ")


def mul_mat(w: torch.Tensor, ggml_type: int, x: torch.Tensor, *, out: torch.Tensor | None = None, xq: torch.Tensor | None = None) -> torch.Tensor:
    """``x[M, K] @ w[N, K]^T`` in ``x``'s dtype with ``w`` packed as ``[N, row_bytes]`` uint8, or written into the ``[M, N]`` view ``out``.

    ``xq`` is ``quantize_q8_1(x)``: the vector kernel reads it instead of quantizing ``x`` again.
    """
    m, k = x.shape
    if out is None:
        out = torch.empty(m, w.shape[0], dtype=x.dtype, device=x.device)
    if not uses_mmvq(ggml_type, m):
        _check_mmq_tail(w, ggml_type, k)
    _module().mul_mat(w, ggml_type, x.contiguous(), out, xq)
    return out


def quantize_q8_1(x: torch.Tensor) -> torch.Tensor:
    """``x[M, K]`` as the vector kernel's q8_1 blocks, to share among GEMVs that read the same ``x``."""
    m, k = x.shape
    out = torch.empty(m, _q8_1_row_bytes(k), dtype=torch.uint8, device=x.device)
    _module().quantize_q8_1(x.contiguous(), out)
    return out


@functools.cache
def _q8_1_row_bytes(k: int) -> int:
    return _module().q8_1_row_bytes(k)


@functools.cache
def uses_mmvq(ggml_type: int, m: int) -> bool:
    """Whether ``mul_mat`` of ``m`` rows on this GPU runs the vector kernel, the one that reads ``xq``."""
    return _module().uses_mmvq(ggml_type, m)


def mul_mat_id(w: torch.Tensor, ggml_type: int, x: torch.Tensor, ids: torch.Tensor, scale: torch.Tensor | None = None) -> torch.Tensor:
    """Per-route expert GEMM over ``w[E, N, row_bytes]``: returns ``[T, k, N]`` in ``x``'s dtype; ``x`` is ``[T, K]`` (shared by the k routes) or ``[T, k, K]``.

    ``scale`` is an fp32 ``[E]`` view (strided is fine) that multiplies each NVFP4 expert's rows; it needs ``T <= mmvq_max_tokens``.
    """
    t, top_k = ids.shape
    out = torch.empty(t, top_k, w.shape[1], dtype=x.dtype, device=x.device)
    if t > mmvq_max_tokens(ggml_type):
        _check_mmq_tail(w, ggml_type, x.shape[-1])
    _module().mul_mat_id(w, ggml_type, (x.unsqueeze(1) if x.dim() == 2 else x).contiguous(), ids, out, scale)
    return out


def mul_mat_id_glu(
    w: torch.Tensor,
    ggml_type: int,
    x: torch.Tensor,
    ids: torch.Tensor,
    glu: str,
    limit: float = 0.0,
    gate_scale: torch.Tensor | None = None,
    up_scale: torch.Tensor | None = None,
) -> torch.Tensor:
    """``glu(gate, up)`` per route in one pass for small ``T``: ``w[E, 2I, row_bytes]`` holds gate rows then up rows; returns ``[T, k, I]`` in ``x``'s dtype.

    ``glu`` is one of ``swiglu``, ``geglu`` (tanh GELU), ``swiglu_oai`` (alpha 1.702, limit 7) and ``swiglu_clamp`` (alpha 1, clamped at ``limit``).
    ``gate_scale`` / ``up_scale`` are fp32 ``[E]`` views that multiply each NVFP4 expert's gate / up results before the glu.
    """
    t, top_k = ids.shape
    out = torch.empty(t, top_k, w.shape[1] // 2, dtype=x.dtype, device=x.device)
    _module().mul_mat_id_glu(w, ggml_type, x.contiguous(), ids, out, glu, limit, gate_scale, up_scale)
    return out


def moe_weighted_sum(y: torch.Tensor, weights: torch.Tensor) -> torch.Tensor:
    """``sum_j weights[t, j] * y[t, j, :]`` over ``y[T, k, N]``, accumulated in fp32 and returned in ``y``'s dtype."""
    out = torch.empty(y.shape[0], y.shape[2], dtype=y.dtype, device=y.device)
    _module().moe_weighted_sum(y.contiguous(), weights.float().contiguous(), out)
    return out


@functools.cache
def mmvq_max_tokens(ggml_type: int) -> int:
    """Largest token count the small-batch MoE kernels take on this GPU."""
    return _module().mmvq_max_tokens(ggml_type)


def dequantize(w: torch.Tensor, ggml_type: int, k: int, dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
    out = torch.empty(w.numel() // w.shape[-1], k, dtype=dtype, device=w.device)
    _module().dequantize(w, ggml_type, k, out)
    return out


def get_rows(w: torch.Tensor, ggml_type: int, k: int, ids: torch.Tensor, dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
    out = torch.empty(ids.shape[0], k, dtype=dtype, device=ids.device)
    _module().get_rows(w, ggml_type, k, ids, out)
    return out


__all__ = [
    "dequantize",
    "get_rows",
    "load_module",
    "mmvq_max_tokens",
    "moe_weighted_sum",
    "mul_mat",
    "mul_mat_id",
    "mul_mat_id_glu",
    "quantize_q8_1",
    "uses_mmvq",
]