FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
import contextlib
import importlib
import os
import pathlib
import re
from functools import cache
from typing import TYPE_CHECKING, Iterator, List, NamedTuple, Tuple, TypeAlias, Union
if TYPE_CHECKING:
from tvm_ffi import Module
KERNEL_PATH = pathlib.Path(__file__).parent / "csrc"
KERNEL_CACHE_PACKAGE = "freetoken_kernel_cache"
KERNEL_CACHE_DIR_ENV = "FREETOKEN_KERNEL_CACHE_DIR"
DISABLE_KERNEL_CACHE_ENV = "FREETOKEN_DISABLE_KERNEL_CACHE"
DISABLE_KERNEL_CACHE_VERSION_CHECK_ENV = "FREETOKEN_DISABLE_KERNEL_CACHE_VERSION_CHECK"
DISABLE_JIT_ENV = "FREETOKEN_DISABLE_JIT"
_TRUE_VALUES = {"1", "true", "yes", "on"}
DEFAULT_INCLUDE = [str(KERNEL_PATH / "include")]
DEFAULT_CFLAGS = ["-std=c++20", "-O3"]
DEFAULT_CUDA_CFLAGS = ["-std=c++20", "-O3", "--expt-relaxed-constexpr"]
DEFAULT_HIP_CFLAGS = ["-std=c++20", "-O3"]
DEFAULT_LDFLAGS = []
CUDA_ARCH_LIST_ENV = "TVM_FFI_CUDA_ARCH_LIST"
ROCM_ARCH_LIST_ENV = "TVM_FFI_ROCM_ARCH_LIST"
def _is_rocm() -> bool:
import torch
return getattr(torch.version, "hip", None) is not None
def _cuda_arch_list() -> List[str]:
"""Archs a CUDA build targets: the AOT build's TVM_FFI_CUDA_ARCH_LIST, else the GPU this process is bound to."""
arch_list = os.getenv(CUDA_ARCH_LIST_ENV, "").split()
if arch_list:
return arch_list
import torch
if not torch.cuda.is_available():
return []
major, minor = torch.cuda.get_device_capability()
return [f"{major}.{minor}"]
def _rocm_arch_list() -> List[str]:
"""The single ROCm GPU this process is bound to, or an explicit cross-compile target."""
from freetoken.utils.arch import get_rocm_gfx_arch
arch = get_rocm_gfx_arch()
if arch is None:
raise RuntimeError(
"Could not determine the bound ROCm GPU architecture; select a visible GPU "
"or set FREETOKEN_ROCM_ARCH for cross-compilation"
)
return [arch]
@contextlib.contextmanager
def _pin_tvm_ffi_arch_ctx(arch_list: List[str], env_var: str) -> Iterator[None]:
"""Pin tvm-ffi to the resolved targets for one build, then restore the caller's environment."""
if not arch_list:
yield
return
old_arch_list = os.environ.get(env_var)
os.environ[env_var] = " ".join(arch_list)
try:
yield
finally:
if old_arch_list is None:
os.environ.pop(env_var, None)
else:
os.environ[env_var] = old_arch_list
def _cuda_cflags(extra: List[str], arch_list: List[str]) -> List[str]:
"""CUDA nvcc flags for a kernel build. tvm-ffi emits one SASS cubin per arch in ``arch_list`` and no PTX, so add the PTX of the highest arch: a GPU newer than every listed arch still runs through the driver's PTX JIT. This flag also carries the arch into tvm-ffi's build hash, which skips tvm-ffi's own -gencode, so GPUs of different archs never share a cached .so."""
flags = DEFAULT_CUDA_CFLAGS + extra
if arch_list:
def _rank(a: str) -> int:
major, minor = a.rstrip("a").split(".")
return int(major) * 100 + int(minor)
cc = max(arch_list, key=_rank).rstrip("a").replace(".", "")
flags = flags + [f"-gencode=arch=compute_{cc},code=compute_{cc}"]
return flags
def _hip_cflags(extra: List[str], arch_list: List[str]) -> List[str]:
"""HIP flags for a kernel build on ROCm."""
# TODO(ROCm): Triton autotune configs need RDNA-specific tuning (wave count, LDS size).
return DEFAULT_HIP_CFLAGS + extra + [f"--offload-arch={arch}" for arch in arch_list]
@cache
def _rocm_link_flags() -> List[str]:
"""Make ROCm's runtime library discoverable to JIT link commands.
Traditional ROCm installs provide ``libamdhip64.so`` under ``$ROCM_HOME/lib``.
ROCm 7.14 Python SDK images only provide the versioned soname, while TVM-FFI
still links with ``-lamdhip64``. Supply a cache-local unversioned symlink via
an explicit linker search path without modifying the Python environment.
"""
candidates: list[pathlib.Path] = []
if os.getenv("ROCM_HOME"):
candidates.append(pathlib.Path(os.environ["ROCM_HOME"]))
try:
from torch.utils.cpp_extension import ROCM_HOME
if ROCM_HOME:
candidates.append(pathlib.Path(ROCM_HOME))
except ImportError:
pass
spec = importlib.util.find_spec("_rocm_sdk_core")
if spec and spec.submodule_search_locations:
candidates.append(pathlib.Path(next(iter(spec.submodule_search_locations))))
candidates.append(pathlib.Path("/opt/rocm"))
for rocm_home in dict.fromkeys(candidates):
library_dir = rocm_home / "lib"
unversioned = library_dir / "libamdhip64.so"
link_dir = library_dir
if not unversioned.exists():
versioned = sorted(library_dir.glob("libamdhip64.so.*"))
if not versioned:
continue
link_dir = pathlib.Path.home() / ".cache" / "freetoken" / "rocm-lib"
link_dir.mkdir(parents=True, exist_ok=True)
compat_link = link_dir / "libamdhip64.so"
if not compat_link.exists() and not compat_link.is_symlink():
try:
compat_link.symlink_to(versioned[-1])
except FileExistsError:
# Multiple tensor-parallel ranks may prepare the same cache.
pass
return [f"-L{link_dir}", f"-Wl,-rpath,{library_dir}"]
raise RuntimeError("Unable to locate libamdhip64 for ROCm JIT linking")
CPP_TEMPLATE_TYPE: TypeAlias = Union[int, float, bool]
class CppArgList(list[str]):
def __str__(self) -> str:
return ", ".join(self)
class KernelConfig(NamedTuple):
num_threads: int
max_occupancy: int
use_pdl: bool
@property
def template_args(self) -> str:
pdl = "true" if self.use_pdl else "false"
return f"{self.num_threads},{self.max_occupancy},{pdl}"
def _make_name(*args: str) -> str:
return "freetoken__" + "_".join(str(arg) for arg in args)
def _env_enabled(name: str) -> bool:
return os.getenv(name, "").strip().lower() in _TRUE_VALUES
def _freetoken_version() -> str:
from freetoken.version import __version__
return __version__
def _version_parts(version: str) -> Tuple[str, List[str]]:
"""Split a PEP 440 version string into (release, local segments):
"0.1.1+cu130.g3f01615" -> ("0.1.1", ["cu130", "g3f01615"])."""
base, _, local = version.partition("+")
return base, local.split(".") if local else []
def _build_stamps(segments: List[str]) -> set[str]:
"""The `g<sha>` commit-stamp tokens of a local version segment list
(stamped by scripts/build-release-wheels.sh)."""
return {s for s in segments if re.fullmatch(r"g[0-9a-f]{7,40}", s)}
def _kernel_cache_version_ok(cache_version: str, runtime_version: str) -> bool:
"""Same release -- and, when both sides carry a `g<sha>` stamp, the same build.
The cache wheel extends the runtime's version with local segments (`+cu130`,
`.g<sha>`), so the old string-prefix test cannot pair a stamped runtime with its
cache; and comparing the stamps rejects a runtime/cache pair from two different
builds, which bare release numbers (both `0.1.1`) could never detect. Either side
may lack a stamp (dev builds) -- then only the release part is compared."""
cache_base, cache_local = _version_parts(cache_version)
runtime_base, runtime_local = _version_parts(runtime_version)
if cache_base != runtime_base:
return False
cache_stamps = _build_stamps(cache_local)
runtime_stamps = _build_stamps(runtime_local)
return not (cache_stamps and runtime_stamps and cache_stamps != runtime_stamps)
def _kernel_cache_dir() -> pathlib.Path | None:
if _env_enabled(DISABLE_KERNEL_CACHE_ENV):
return None
override = os.getenv(KERNEL_CACHE_DIR_ENV)
if override:
return pathlib.Path(override).expanduser()
try:
package = importlib.import_module(KERNEL_CACHE_PACKAGE)
except ModuleNotFoundError as exc:
if exc.name == KERNEL_CACHE_PACKAGE:
return None
raise
package_version = str(getattr(package, "__version__", "0.0.0+unknown"))
runtime_version = _freetoken_version()
if not _env_enabled(DISABLE_KERNEL_CACHE_VERSION_CHECK_ENV):
if not _kernel_cache_version_ok(package_version, runtime_version):
raise RuntimeError(
"freetoken-kernel-cache version "
f"{package_version!r} does not match freetoken version {runtime_version!r}"
)
cache_cuda = re.search(r"\+cu(\d{2,})", package_version)
if cache_cuda is not None:
from freetoken.kernel._toolchain import torch_cuda_major
cache_major = int(cache_cuda.group(1)[:-1])
torch_major = torch_cuda_major()
if torch_major is not None and cache_major != torch_major:
raise RuntimeError(
f"freetoken-kernel-cache {package_version!r} was built for CUDA "
f"{cache_major}.x but torch runs CUDA {torch_major}.x -- install "
"the kernel-cache wheel matching this torch build"
)
get_jit_cache_dir = getattr(package, "get_jit_cache_dir", None)
if get_jit_cache_dir is None:
raise RuntimeError(f"{KERNEL_CACHE_PACKAGE} does not expose get_jit_cache_dir()")
return pathlib.Path(get_jit_cache_dir()).expanduser()
def _load_prebuilt(name: str) -> Module | None:
cache_dir = _kernel_cache_dir()
if cache_dir is None:
if _env_enabled(DISABLE_JIT_ENV):
raise RuntimeError(
"JIT compilation is disabled by FREETOKEN_DISABLE_JIT, "
f"but no prebuilt kernel cache is configured for {name!r}"
)
return None
so_path = cache_dir / name / f"{name}.so"
if so_path.exists():
import tvm_ffi
return tvm_ffi.load_module(str(so_path))
if _env_enabled(DISABLE_JIT_ENV):
raise RuntimeError(
"JIT compilation is disabled by FREETOKEN_DISABLE_JIT, "
f"but prebuilt kernel {name!r} was not found at {so_path}"
)
return None
def _make_wrapper(tup: Tuple[str, str]) -> str:
export_name, kernel_name = tup
return f"TVM_FFI_DLL_EXPORT_TYPED_FUNC({export_name}, ({kernel_name}));"
def make_cpp_args(*args: CPP_TEMPLATE_TYPE) -> CppArgList:
def _convert(arg: CPP_TEMPLATE_TYPE) -> str:
if isinstance(arg, bool):
return "true" if arg else "false"
if isinstance(arg, (int, float)):
return str(arg)
raise TypeError(f"Unsupported argument type for cpp template: {type(arg)}")
return CppArgList(_convert(arg) for arg in args)
def load_aot(
*args: str,
cpp_files: List[str] | None = None,
cuda_files: List[str] | None = None,
extra_cflags: List[str] | None = None,
extra_cuda_cflags: List[str] | None = None,
extra_ldflags: List[str] | None = None,
extra_include_paths: List[str] | None = None,
build_directory: str | None = None,
) -> Module:
name = _make_name(*args)
prebuilt = _load_prebuilt(name)
if prebuilt is not None:
return prebuilt
is_rocm = _is_rocm()
arch_list: List[str] = []
if cuda_files:
if is_rocm:
arch_list = _rocm_arch_list()
else:
from freetoken.kernel._toolchain import check_nvcc_matches_torch
check_nvcc_matches_torch()
arch_list = _cuda_arch_list()
from tvm_ffi.cpp import load
cpp_files = cpp_files or []
cuda_files = cuda_files or []
extra_cflags = extra_cflags or []
extra_cuda_cflags = extra_cuda_cflags or []
extra_ldflags = extra_ldflags or []
extra_include_paths = extra_include_paths or []
cpp_files = [str((KERNEL_PATH / "src" / f).resolve()) for f in cpp_files]
cuda_files = [str((KERNEL_PATH / "src" / f).resolve()) for f in cuda_files]
if is_rocm:
cuda_cflags = _hip_cflags(extra_cuda_cflags, arch_list)
runtime_ldflags = _rocm_link_flags()
arch_list_env = ROCM_ARCH_LIST_ENV
else:
cuda_cflags = _cuda_cflags(extra_cuda_cflags, arch_list)
runtime_ldflags = []
arch_list_env = CUDA_ARCH_LIST_ENV
with _pin_tvm_ffi_arch_ctx(arch_list, arch_list_env):
return load(
name,
cpp_files=cpp_files,
cuda_files=cuda_files,
extra_cflags=DEFAULT_CFLAGS + extra_cflags,
extra_cuda_cflags=cuda_cflags,
extra_ldflags=DEFAULT_LDFLAGS + runtime_ldflags + extra_ldflags,
extra_include_paths=DEFAULT_INCLUDE + extra_include_paths,
build_directory=build_directory,
)
def load_jit(
*args: str,
cpp_files: List[str] | None = None,
cuda_files: List[str] | None = None,
cpp_wrappers: List[Tuple[str, str]] | None = None,
cuda_wrappers: List[Tuple[str, str]] | None = None,
extra_cflags: List[str] | None = None,
extra_cuda_cflags: List[str] | None = None,
extra_ldflags: List[str] | None = None,
extra_include_paths: List[str] | None = None,
build_directory: str | None = None,
) -> Module:
name = _make_name(*args)
prebuilt = _load_prebuilt(name)
if prebuilt is not None:
return prebuilt
is_rocm = _is_rocm()
arch_list: List[str] = []
if cuda_files or cuda_wrappers:
if is_rocm:
arch_list = _rocm_arch_list()
else:
from freetoken.kernel._toolchain import check_nvcc_matches_torch
check_nvcc_matches_torch()
arch_list = _cuda_arch_list()
from tvm_ffi.cpp import load_inline
cpp_files = cpp_files or []
cuda_files = cuda_files or []
cpp_wrappers = cpp_wrappers or []
cuda_wrappers = cuda_wrappers or []
extra_cflags = extra_cflags or []
extra_cuda_cflags = extra_cuda_cflags or []
extra_ldflags = extra_ldflags or []
extra_include_paths = extra_include_paths or []
# include cpp files
cpp_paths = [(KERNEL_PATH / "jit" / f).resolve() for f in cpp_files]
cpp_sources = [f'#include "{path}"' for path in cpp_paths]
cpp_sources += [_make_wrapper(tup) for tup in cpp_wrappers]
# include cuda files
cuda_paths = [(KERNEL_PATH / "jit" / f).resolve() for f in cuda_files]
cuda_sources = [f'#include "{path}"' for path in cuda_paths]
cuda_sources += [_make_wrapper(tup) for tup in cuda_wrappers]
if is_rocm:
cuda_cflags = _hip_cflags(extra_cuda_cflags, arch_list)
runtime_ldflags = _rocm_link_flags()
arch_list_env = ROCM_ARCH_LIST_ENV
else:
cuda_cflags = _cuda_cflags(extra_cuda_cflags, arch_list)
runtime_ldflags = []
arch_list_env = CUDA_ARCH_LIST_ENV
with _pin_tvm_ffi_arch_ctx(arch_list, arch_list_env):
return load_inline(
name,
cpp_sources=cpp_sources,
cuda_sources=cuda_sources,
extra_cflags=DEFAULT_CFLAGS + extra_cflags,
extra_cuda_cflags=cuda_cflags,
extra_ldflags=DEFAULT_LDFLAGS + runtime_ldflags + extra_ldflags,
extra_include_paths=DEFAULT_INCLUDE + extra_include_paths,
build_directory=build_directory,
)