FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
import functools
import os
import re
from typing import Tuple
_GFX_ARCH_RE = re.compile(r"gfx\d+[a-z]?")
def _gfx_arch_from(value: object) -> str | None:
match = _GFX_ARCH_RE.search(str(value).lower())
return match.group(0) if match else None
@functools.cache
def is_rocm() -> bool:
"""True when torch is built for ROCm (AMD GPU) instead of CUDA."""
import torch
return getattr(torch.version, "hip", None) is not None
@functools.cache
def get_rocm_gfx_arch() -> str | None:
"""Return the current AMD GPU target (for example ``gfx1201``).
Prefer the runtime device because build variables may contain multiple
semicolon-separated targets. Environment variables remain useful for
cross-compilation and systems where no GPU is currently visible; in that
fallback mode the first target is returned. The result is process-cached
for FreeToken's one-process-per-GPU execution model, so callers must select
the intended device before the first call.
"""
if not is_rocm():
return None
import torch
if torch.cuda.is_available():
try:
props = torch.cuda.get_device_properties(torch.cuda.current_device())
for attr in ("gcnArchName", "arch"):
arch = _gfx_arch_from(getattr(props, attr, ""))
if arch:
return arch
except (AttributeError, RuntimeError):
pass
for env_var in ("FREETOKEN_ROCM_ARCH", "PYTORCH_ROCM_ARCH", "HCC_AMDGPU_TARGET"):
arch = _gfx_arch_from(os.getenv(env_var, ""))
if arch:
return arch
return None
@functools.cache
def _get_torch_cuda_version() -> Tuple[int, int] | None:
import torch
import torch.version
if is_rocm():
return None
if not torch.cuda.is_available() or not torch.version.cuda:
return None
return torch.cuda.get_device_capability()
def is_arch_supported(major: int, minor: int = 0) -> bool:
"""capability >= (major, minor). Open-ended: newer archs also pass. Only use this
for family-portable features (e.g. PDL); arch-specific kernels (sm_90a/sm_100a
cubins) need the closed is_smXX_family checks below."""
arch = _get_torch_cuda_version()
if arch is None:
return False
return arch >= (major, minor)
def _is_arch_family(major: int) -> bool:
arch = _get_torch_cuda_version()
return arch is not None and arch[0] == major
def is_sm90_family() -> bool:
"""Exactly major 9 (Hopper). For sm_90a-only kernels (e.g. FA3)."""
return _is_arch_family(9)
def is_sm100_family() -> bool:
"""Exactly major 10 (datacenter Blackwell). For sm_100a/103a-only kernels
(e.g. trtllm-gen) that consumer Blackwell (sm_120/121) cannot run."""
return _is_arch_family(10)
def is_sm90_supported() -> bool:
return is_arch_supported(9, 0)
def is_sm100_supported() -> bool:
return is_arch_supported(10, 0)