FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
build(deps): move to torch 2.14.1, triton 3.8 and sglang-kernel 0.4.9 (#630)
Xiaoze Fan74a387777708
diff --git a/freetoken-kernel-cache/pyproject.toml b/freetoken-kernel-cache/pyproject.tomlindex ec37efbf..bc75e1b2 100644--- a/freetoken-kernel-cache/pyproject.toml+++ b/freetoken-kernel-cache/pyproject.toml@@ -8,7 +8,7 @@ requires = [# 77 is the first release that understands the PEP 639 `license` SPDX string below."setuptools>=77","wheel",- "torch>=2.11,<2.12",+ "torch>=2.14,<2.15","apache-tvm-ffi==0.1.13.post3",]build-backend = "build_backend"@@ -22,7 +22,7 @@ requires-python = ">=3.10"license = "Apache-2.0"dependencies = ["apache-tvm-ffi==0.1.13.post3"]-# Provenance pin: PyPI's torch 2.11.0 is itself the cu130 build, but this index serves+# Provenance pin: PyPI's torch 2.14.1 is itself the cu130 build, but this index serves# ONLY cu130 wheels, so an isolated build can never resolve a different-CUDA torch and# stamp the wheel +cu126/+cu128. uv applies these to build-system.requires when this# directory is the build root. Mirrors ../pyproject.toml.diff --git a/install.sh b/install.shindex f623ffec..68738cc6 100755--- a/install.sh+++ b/install.sh@@ -206,7 +206,7 @@ mkdir -p "$FT_HOME"# re-install can't inherit a stale/mismatched torch (e.g. an old cu128 venv after a cu130 bump)."$UV" venv "$VENV" --python "$PY_VERSION" --clear-# PyPI's torch 2.11.0 is the same cu130 build the pytorch index+# PyPI's torch 2.14.1 is the same cu130 build the pytorch index# serves; the explicit index pins provenance to the cu130 channel. `unsafe-best-match`# is needed because the pytorch index also mirrors stale copies of common deps (e.g.# packaging<=24.1) that would shadow PyPI under uv's first-index strategy; all indexesdiff --git a/pyproject.toml b/pyproject.tomlindex 8d00d2bb..95b872f5 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -3,7 +3,7 @@# own, and a mismatch links the C++ extensions against the wrong libtorch.# setuptools floor: 77 is the first release that understands the PEP 639 `license`# SPDX string and `license-files` below.-requires = ["setuptools>=77", "torch>=2.11,<2.12", "wheel"]+requires = ["setuptools>=77", "torch>=2.14,<2.15", "wheel"]build-backend = "setuptools.build_meta"[project]@@ -53,17 +53,17 @@ dependencies = ["pydantic>=2.9,<3","pyzmq>=27,<28","safetensors>=0.6,<1",- # floor+ceiling: sglang-kernel 0.4.5 links libtorch symbols only 2.11 has.- # PyPI's torch 2.11.0 wheel is itself the cu130 build, so plain pip resolves+ # floor+ceiling: sglang-kernel 0.4.9 requires torch==2.14.1.+ # PyPI's torch 2.14.1 wheel is itself the cu130 build, so plain pip resolves# correctly from PyPI alone; uv additionally pins the index below.- "torch>=2.11,<2.12",- "torchvision>=0.26,<0.27",+ "torch>=2.14,<2.15",+ "torchvision>=0.29,<0.30","tqdm>=4.66,<5","transformers>=5.16,<5.17",- # CUDA torch 2.11 resolves Triton 3.6; AMD's ROCm 7.14 image supplies its+ # CUDA torch 2.14 resolves Triton 3.8; AMD's ROCm 7.14 image supplies its# gfx1201-enabled Triton 3.7 build. Keep both supported without replacing the# runtime-specific wheel selected by the PyTorch distribution.- "triton>=3.6,<3.8; platform_system == 'Linux'",+ "triton>=3.6,<3.9; platform_system == 'Linux'","uvicorn>=0.30,<1",]@@ -82,7 +82,7 @@ dev = ["pytest>=6.0"]# `freetoken[accel]` for the full native fast path.fi = ["flashinfer-python[cu13]==0.6.18.post1"]# renamed from sgl-kernel at 0.4; still imports as `sgl_kernel`, so never co-install both-sgl = ["sglang-kernel==0.4.5"]+sgl = ["sglang-kernel==0.4.9"]accel = ["freetoken[fi,sgl]"]# NOTE: the Marlin W4A16 NVFP4 expert-GEMM path (sm_80-99) borrows vLLM's AOT wheel# (vllm>=0.14,<0.15), which pins transformers>=4.56,<5 and so is INCOMPATIBLE with the@@ -91,7 +91,7 @@ accel = ["freetoken[fi,sgl]"]# in a dedicated environment when you specifically need the Marlin path.# uv-only provenance pins — pip ignores this section and resolves from PyPI, which-# works: PyPI's torch 2.11.0 is the same cu130 build this index serves.+# works: PyPI's torch 2.14.1 is the same cu130 build this index serves.# `explicit = true` scopes the index to the one package that needs it# (the torch index also mirrors stale copies of common deps, e.g. packaging<=24.1,# which would otherwise shadow PyPI under uv's first-index strategy).diff --git a/scripts/ci/manylinux-build.sh b/scripts/ci/manylinux-build.shindex d2dd3ca2..5fe0487f 100755--- a/scripts/ci/manylinux-build.sh+++ b/scripts/ci/manylinux-build.sh@@ -97,11 +97,11 @@ for cptag in $MATRIX; doVENV="/tmp/build-venv-$cptag"say "creating build venv ($cptag)"uv venv --quiet --python "$PYBIN" "$VENV"- # Provenance pin: PyPI's torch 2.11.0 is itself the cu130 build, but this index+ # Provenance pin: PyPI's torch 2.14.1 is itself the cu130 build, but this index# serves ONLY cu130 wheels, so the resolve can never pick a different-CUDA torch# and tag the kernel-cache wheel wrong; everything else is plain PyPI.uv pip install --quiet --python "$VENV/bin/python" \- --index-url https://download.pytorch.org/whl/cu130 "torch>=2.11,<2.12"+ --index-url https://download.pytorch.org/whl/cu130 "torch>=2.14,<2.15"uv pip install --quiet --python "$VENV/bin/python" \"setuptools>=77" wheel ninja "apache-tvm-ffi==0.1.13.post3"diff --git a/setup.py b/setup.pyindex 997745f1..b8e4693c 100644--- a/setup.py+++ b/setup.py@@ -80,12 +80,12 @@ if IS_ROCM:# These extensions contain host code only. BuildExtension supplies the ROCm# platform defines to the C++ compiler; offload architecture flags belong on# HIP device sources and would be rejected by the host compiler here.- extra_compile = ["-O3", "-std=c++17"]+ extra_compile = ["-O3", "-std=c++20"]else:runtime_include_dirs, runtime_library_dirs = _cuda_runtime_paths()runtime_lib = "cudart"runtime_link_args = []- extra_compile = ["-O3", "-std=c++17"]+ extra_compile = ["-O3", "-std=c++20"]_check_toolchain()@@ -126,7 +126,7 @@ setup(sources=["python/freetoken/kernel/csrc/row_store/row_store_ext.cpp",],- extra_compile_args=["-O3", "-std=c++17"],+ extra_compile_args=["-O3", "-std=c++20"],)] if sys.platform == "linux" else []),],