FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
from functools import lru_cache
from typing import TYPE_CHECKING
import torch
from .utils import load_jit
if TYPE_CHECKING:
from tvm_ffi import Module
@lru_cache(maxsize=None)
def _jit_batch_memcpy_module() -> Module:
return load_jit(
"batch_memcpy",
cuda_files=["batch_memcpy.cuh"],
cuda_wrappers=[("batch_memcpy", "&BatchMemcpy::run")],
)
def _probe(fn) -> None:
"""One real 16-byte H2D through the binding. A version-gated build (panic
branch) or a driver without batch-memcpy support loads cleanly and only fails
at call time; probing here turns every such mode into a load_batch_memcpy
exception the caller's fallback path can catch."""
src = torch.arange(16, dtype=torch.uint8).pin_memory()
dst = torch.zeros(16, dtype=torch.uint8, device="cuda")
stream = torch.cuda.Stream()
# Order the copy after dst's initialization on the current stream;
# otherwise the fill can overwrite the copied bytes.
stream.wait_stream(torch.cuda.current_stream())
fn(
torch.tensor([dst.data_ptr()]),
torch.tensor([src.data_ptr()]),
torch.tensor([16]),
stream.cuda_stream,
)
stream.synchronize()
if not torch.equal(dst.cpu(), src):
raise RuntimeError("cudaMemcpyBatchAsync probe copied wrong bytes")
def load_batch_memcpy():
"""Build (once), probe, and return the batch-memcpy entry point, or raise.
The 8-argument cudaMemcpyBatchAsync signature this binding uses is CUDA 13.0's
(12.8/12.9 had an extra failIdx parameter); gate on the torch runtime version
before paying for the JIT build, then verify with a real copy.
HIP uses its own signature, checked by building and probing a real copy.
"""
cuda = torch.version.cuda
if torch.version.hip is None and (
cuda is None or tuple(int(x) for x in cuda.split(".")[:2]) < (13, 0)
):
raise RuntimeError(f"cudaMemcpyBatchAsync binding requires CUDA >= 13.0 (torch built with {cuda})")
fn = _jit_batch_memcpy_module().batch_memcpy
_probe(fn)
return fn
def batch_memcpy_jit(
dst_ptrs: torch.Tensor,
src_ptrs: torch.Tensor,
sizes: torch.Tensor,
stream: int,
) -> None:
"""Enqueue one cudaMemcpyBatchAsync/hipMemcpyBatchAsync of ``len(sizes)`` independent copies.
``dst_ptrs``/``src_ptrs``/``sizes`` are same-length CPU int64 tensors of raw
addresses and byte counts; ``stream`` is a raw cudaStream_t/hipStream_t handle
(``torch.cuda.Stream.cuda_stream``), which must not be the legacy NULL stream.
"""
load_batch_memcpy()(dst_ptrs, src_ptrs, sizes, stream)