Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

15.0 KB
Raw Blame
from __future__ import annotations

import contextlib
import importlib
import os
import pathlib
import re
from functools import cache
from typing import TYPE_CHECKING, Iterator, List, NamedTuple, Tuple, TypeAlias, Union

if TYPE_CHECKING:
    from tvm_ffi import Module

KERNEL_PATH = pathlib.Path(__file__).parent / "csrc"
KERNEL_CACHE_PACKAGE = "freetoken_kernel_cache"
KERNEL_CACHE_DIR_ENV = "FREETOKEN_KERNEL_CACHE_DIR"
DISABLE_KERNEL_CACHE_ENV = "FREETOKEN_DISABLE_KERNEL_CACHE"
DISABLE_KERNEL_CACHE_VERSION_CHECK_ENV = "FREETOKEN_DISABLE_KERNEL_CACHE_VERSION_CHECK"
DISABLE_JIT_ENV = "FREETOKEN_DISABLE_JIT"
_TRUE_VALUES = {"1", "true", "yes", "on"}
DEFAULT_INCLUDE = [str(KERNEL_PATH / "include")]
DEFAULT_CFLAGS = ["-std=c++20", "-O3"]
DEFAULT_CUDA_CFLAGS = ["-std=c++20", "-O3", "--expt-relaxed-constexpr"]
DEFAULT_HIP_CFLAGS = ["-std=c++20", "-O3"]
DEFAULT_LDFLAGS = []


CUDA_ARCH_LIST_ENV = "TVM_FFI_CUDA_ARCH_LIST"
ROCM_ARCH_LIST_ENV = "TVM_FFI_ROCM_ARCH_LIST"


def _is_rocm() -> bool:
    import torch

    return getattr(torch.version, "hip", None) is not None


def _cuda_arch_list() -> List[str]:
    """Archs a CUDA build targets: the AOT build's TVM_FFI_CUDA_ARCH_LIST, else the GPU this process is bound to."""
    arch_list = os.getenv(CUDA_ARCH_LIST_ENV, "").split()
    if arch_list:
        return arch_list
    import torch

    if not torch.cuda.is_available():
        return []
    major, minor = torch.cuda.get_device_capability()
    return [f"{major}.{minor}"]


def _rocm_arch_list() -> List[str]:
    """The single ROCm GPU this process is bound to, or an explicit cross-compile target."""
    from freetoken.utils.arch import get_rocm_gfx_arch

    arch = get_rocm_gfx_arch()
    if arch is None:
        raise RuntimeError(
            "Could not determine the bound ROCm GPU architecture; select a visible GPU "
            "or set FREETOKEN_ROCM_ARCH for cross-compilation"
        )
    return [arch]


@contextlib.contextmanager
def _pin_tvm_ffi_arch_ctx(arch_list: List[str], env_var: str) -> Iterator[None]:
    """Pin tvm-ffi to the resolved targets for one build, then restore the caller's environment."""
    if not arch_list:
        yield
        return
    old_arch_list = os.environ.get(env_var)
    os.environ[env_var] = " ".join(arch_list)
    try:
        yield
    finally:
        if old_arch_list is None:
            os.environ.pop(env_var, None)
        else:
            os.environ[env_var] = old_arch_list


def _cuda_cflags(extra: List[str], arch_list: List[str]) -> List[str]:
    """CUDA nvcc flags for a kernel build. tvm-ffi emits one SASS cubin per arch in ``arch_list`` and no PTX, so add the PTX of the highest arch: a GPU newer than every listed arch still runs through the driver's PTX JIT. This flag also carries the arch into tvm-ffi's build hash, which skips tvm-ffi's own -gencode, so GPUs of different archs never share a cached .so."""
    flags = DEFAULT_CUDA_CFLAGS + extra
    if arch_list:
        def _rank(a: str) -> int:
            major, minor = a.rstrip("a").split(".")
            return int(major) * 100 + int(minor)

        cc = max(arch_list, key=_rank).rstrip("a").replace(".", "")
        flags = flags + [f"-gencode=arch=compute_{cc},code=compute_{cc}"]
    return flags


def _hip_cflags(extra: List[str], arch_list: List[str]) -> List[str]:
    """HIP flags for a kernel build on ROCm."""
    # TODO(ROCm): Triton autotune configs need RDNA-specific tuning (wave count, LDS size).
    return DEFAULT_HIP_CFLAGS + extra + [f"--offload-arch={arch}" for arch in arch_list]


@cache
def _rocm_link_flags() -> List[str]:
    """Make ROCm's runtime library discoverable to JIT link commands.

    Traditional ROCm installs provide ``libamdhip64.so`` under ``$ROCM_HOME/lib``.
    ROCm 7.14 Python SDK images only provide the versioned soname, while TVM-FFI
    still links with ``-lamdhip64``. Supply a cache-local unversioned symlink via
    an explicit linker search path without modifying the Python environment.
    """
    candidates: list[pathlib.Path] = []
    if os.getenv("ROCM_HOME"):
        candidates.append(pathlib.Path(os.environ["ROCM_HOME"]))
    try:
        from torch.utils.cpp_extension import ROCM_HOME

        if ROCM_HOME:
            candidates.append(pathlib.Path(ROCM_HOME))
    except ImportError:
        pass
    spec = importlib.util.find_spec("_rocm_sdk_core")
    if spec and spec.submodule_search_locations:
        candidates.append(pathlib.Path(next(iter(spec.submodule_search_locations))))
    candidates.append(pathlib.Path("/opt/rocm"))

    for rocm_home in dict.fromkeys(candidates):
        library_dir = rocm_home / "lib"
        unversioned = library_dir / "libamdhip64.so"
        link_dir = library_dir
        if not unversioned.exists():
            versioned = sorted(library_dir.glob("libamdhip64.so.*"))
            if not versioned:
                continue
            link_dir = pathlib.Path.home() / ".cache" / "freetoken" / "rocm-lib"
            link_dir.mkdir(parents=True, exist_ok=True)
            compat_link = link_dir / "libamdhip64.so"
            if not compat_link.exists() and not compat_link.is_symlink():
                try:
                    compat_link.symlink_to(versioned[-1])
                except FileExistsError:
                    # Multiple tensor-parallel ranks may prepare the same cache.
                    pass

        return [f"-L{link_dir}", f"-Wl,-rpath,{library_dir}"]

    raise RuntimeError("Unable to locate libamdhip64 for ROCm JIT linking")


CPP_TEMPLATE_TYPE: TypeAlias = Union[int, float, bool]


class CppArgList(list[str]):
    def __str__(self) -> str:
        return ", ".join(self)


class KernelConfig(NamedTuple):
    num_threads: int
    max_occupancy: int
    use_pdl: bool

    @property
    def template_args(self) -> str:
        pdl = "true" if self.use_pdl else "false"
        return f"{self.num_threads},{self.max_occupancy},{pdl}"


def _make_name(*args: str) -> str:
    return "freetoken__" + "_".join(str(arg) for arg in args)


def _env_enabled(name: str) -> bool:
    return os.getenv(name, "").strip().lower() in _TRUE_VALUES


def _freetoken_version() -> str:
    from freetoken.version import __version__

    return __version__


def _version_parts(version: str) -> Tuple[str, List[str]]:
    """Split a PEP 440 version string into (release, local segments):
    "0.1.1+cu130.g3f01615" -> ("0.1.1", ["cu130", "g3f01615"])."""
    base, _, local = version.partition("+")
    return base, local.split(".") if local else []


def _build_stamps(segments: List[str]) -> set[str]:
    """The `g<sha>` commit-stamp tokens of a local version segment list
    (stamped by scripts/build-release-wheels.sh)."""
    return {s for s in segments if re.fullmatch(r"g[0-9a-f]{7,40}", s)}


def _kernel_cache_version_ok(cache_version: str, runtime_version: str) -> bool:
    """Same release -- and, when both sides carry a `g<sha>` stamp, the same build.

    The cache wheel extends the runtime's version with local segments (`+cu130`,
    `.g<sha>`), so the old string-prefix test cannot pair a stamped runtime with its
    cache; and comparing the stamps rejects a runtime/cache pair from two different
    builds, which bare release numbers (both `0.1.1`) could never detect. Either side
    may lack a stamp (dev builds) -- then only the release part is compared."""
    cache_base, cache_local = _version_parts(cache_version)
    runtime_base, runtime_local = _version_parts(runtime_version)
    if cache_base != runtime_base:
        return False
    cache_stamps = _build_stamps(cache_local)
    runtime_stamps = _build_stamps(runtime_local)
    return not (cache_stamps and runtime_stamps and cache_stamps != runtime_stamps)


def _kernel_cache_dir() -> pathlib.Path | None:
    if _env_enabled(DISABLE_KERNEL_CACHE_ENV):
        return None

    override = os.getenv(KERNEL_CACHE_DIR_ENV)
    if override:
        return pathlib.Path(override).expanduser()

    try:
        package = importlib.import_module(KERNEL_CACHE_PACKAGE)
    except ModuleNotFoundError as exc:
        if exc.name == KERNEL_CACHE_PACKAGE:
            return None
        raise

    package_version = str(getattr(package, "__version__", "0.0.0+unknown"))
    runtime_version = _freetoken_version()
    if not _env_enabled(DISABLE_KERNEL_CACHE_VERSION_CHECK_ENV):
        if not _kernel_cache_version_ok(package_version, runtime_version):
            raise RuntimeError(
                "freetoken-kernel-cache version "
                f"{package_version!r} does not match freetoken version {runtime_version!r}"
            )
        cache_cuda = re.search(r"\+cu(\d{2,})", package_version)
        if cache_cuda is not None:
            from freetoken.kernel._toolchain import torch_cuda_major

            cache_major = int(cache_cuda.group(1)[:-1])
            torch_major = torch_cuda_major()
            if torch_major is not None and cache_major != torch_major:
                raise RuntimeError(
                    f"freetoken-kernel-cache {package_version!r} was built for CUDA "
                    f"{cache_major}.x but torch runs CUDA {torch_major}.x -- install "
                    "the kernel-cache wheel matching this torch build"
                )

    get_jit_cache_dir = getattr(package, "get_jit_cache_dir", None)
    if get_jit_cache_dir is None:
        raise RuntimeError(f"{KERNEL_CACHE_PACKAGE} does not expose get_jit_cache_dir()")
    return pathlib.Path(get_jit_cache_dir()).expanduser()


def _load_prebuilt(name: str) -> Module | None:
    cache_dir = _kernel_cache_dir()
    if cache_dir is None:
        if _env_enabled(DISABLE_JIT_ENV):
            raise RuntimeError(
                "JIT compilation is disabled by FREETOKEN_DISABLE_JIT, "
                f"but no prebuilt kernel cache is configured for {name!r}"
            )
        return None

    so_path = cache_dir / name / f"{name}.so"
    if so_path.exists():
        import tvm_ffi

        return tvm_ffi.load_module(str(so_path))

    if _env_enabled(DISABLE_JIT_ENV):
        raise RuntimeError(
            "JIT compilation is disabled by FREETOKEN_DISABLE_JIT, "
            f"but prebuilt kernel {name!r} was not found at {so_path}"
        )
    return None


def _make_wrapper(tup: Tuple[str, str]) -> str:
    export_name, kernel_name = tup
    return f"TVM_FFI_DLL_EXPORT_TYPED_FUNC({export_name}, ({kernel_name}));"


def make_cpp_args(*args: CPP_TEMPLATE_TYPE) -> CppArgList:
    def _convert(arg: CPP_TEMPLATE_TYPE) -> str:
        if isinstance(arg, bool):
            return "true" if arg else "false"
        if isinstance(arg, (int, float)):
            return str(arg)
        raise TypeError(f"Unsupported argument type for cpp template: {type(arg)}")

    return CppArgList(_convert(arg) for arg in args)


def load_aot(
    *args: str,
    cpp_files: List[str] | None = None,
    cuda_files: List[str] | None = None,
    extra_cflags: List[str] | None = None,
    extra_cuda_cflags: List[str] | None = None,
    extra_ldflags: List[str] | None = None,
    extra_include_paths: List[str] | None = None,
    build_directory: str | None = None,
) -> Module:
    name = _make_name(*args)
    prebuilt = _load_prebuilt(name)
    if prebuilt is not None:
        return prebuilt

    is_rocm = _is_rocm()
    arch_list: List[str] = []
    if cuda_files:
        if is_rocm:
            arch_list = _rocm_arch_list()
        else:
            from freetoken.kernel._toolchain import check_nvcc_matches_torch

            check_nvcc_matches_torch()
            arch_list = _cuda_arch_list()

    from tvm_ffi.cpp import load

    cpp_files = cpp_files or []
    cuda_files = cuda_files or []
    extra_cflags = extra_cflags or []
    extra_cuda_cflags = extra_cuda_cflags or []
    extra_ldflags = extra_ldflags or []
    extra_include_paths = extra_include_paths or []

    cpp_files = [str((KERNEL_PATH / "src" / f).resolve()) for f in cpp_files]
    cuda_files = [str((KERNEL_PATH / "src" / f).resolve()) for f in cuda_files]

    if is_rocm:
        cuda_cflags = _hip_cflags(extra_cuda_cflags, arch_list)
        runtime_ldflags = _rocm_link_flags()
        arch_list_env = ROCM_ARCH_LIST_ENV
    else:
        cuda_cflags = _cuda_cflags(extra_cuda_cflags, arch_list)
        runtime_ldflags = []
        arch_list_env = CUDA_ARCH_LIST_ENV

    with _pin_tvm_ffi_arch_ctx(arch_list, arch_list_env):
        return load(
            name,
            cpp_files=cpp_files,
            cuda_files=cuda_files,
            extra_cflags=DEFAULT_CFLAGS + extra_cflags,
            extra_cuda_cflags=cuda_cflags,
            extra_ldflags=DEFAULT_LDFLAGS + runtime_ldflags + extra_ldflags,
            extra_include_paths=DEFAULT_INCLUDE + extra_include_paths,
            build_directory=build_directory,
        )


def load_jit(
    *args: str,
    cpp_files: List[str] | None = None,
    cuda_files: List[str] | None = None,
    cpp_wrappers: List[Tuple[str, str]] | None = None,
    cuda_wrappers: List[Tuple[str, str]] | None = None,
    extra_cflags: List[str] | None = None,
    extra_cuda_cflags: List[str] | None = None,
    extra_ldflags: List[str] | None = None,
    extra_include_paths: List[str] | None = None,
    build_directory: str | None = None,
) -> Module:
    name = _make_name(*args)
    prebuilt = _load_prebuilt(name)
    if prebuilt is not None:
        return prebuilt

    is_rocm = _is_rocm()
    arch_list: List[str] = []
    if cuda_files or cuda_wrappers:
        if is_rocm:
            arch_list = _rocm_arch_list()
        else:
            from freetoken.kernel._toolchain import check_nvcc_matches_torch

            check_nvcc_matches_torch()
            arch_list = _cuda_arch_list()

    from tvm_ffi.cpp import load_inline

    cpp_files = cpp_files or []
    cuda_files = cuda_files or []
    cpp_wrappers = cpp_wrappers or []
    cuda_wrappers = cuda_wrappers or []
    extra_cflags = extra_cflags or []
    extra_cuda_cflags = extra_cuda_cflags or []
    extra_ldflags = extra_ldflags or []
    extra_include_paths = extra_include_paths or []

    # include cpp files
    cpp_paths = [(KERNEL_PATH / "jit" / f).resolve() for f in cpp_files]
    cpp_sources = [f'#include "{path}"' for path in cpp_paths]
    cpp_sources += [_make_wrapper(tup) for tup in cpp_wrappers]

    # include cuda files
    cuda_paths = [(KERNEL_PATH / "jit" / f).resolve() for f in cuda_files]
    cuda_sources = [f'#include "{path}"' for path in cuda_paths]
    cuda_sources += [_make_wrapper(tup) for tup in cuda_wrappers]

    if is_rocm:
        cuda_cflags = _hip_cflags(extra_cuda_cflags, arch_list)
        runtime_ldflags = _rocm_link_flags()
        arch_list_env = ROCM_ARCH_LIST_ENV
    else:
        cuda_cflags = _cuda_cflags(extra_cuda_cflags, arch_list)
        runtime_ldflags = []
        arch_list_env = CUDA_ARCH_LIST_ENV

    with _pin_tvm_ffi_arch_ctx(arch_list, arch_list_env):
        return load_inline(
            name,
            cpp_sources=cpp_sources,
            cuda_sources=cuda_sources,
            extra_cflags=DEFAULT_CFLAGS + extra_cflags,
            extra_cuda_cflags=cuda_cflags,
            extra_ldflags=DEFAULT_LDFLAGS + runtime_ldflags + extra_ldflags,
            extra_include_paths=DEFAULT_INCLUDE + extra_include_paths,
            build_directory=build_directory,
        )