Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

4.9 KB
Raw Blame
[build-system]
# torch must match the runtime range below: build isolation resolves this list on its
# own, and a mismatch links the C++ extensions against the wrong libtorch.
# setuptools floor: 77 is the first release that understands the PEP 639 `license`
# SPDX string and `license-files` below.
requires = ["setuptools>=77", "torch>=2.14,<2.15", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "freetoken"
# Single source of truth: python/freetoken/version.py (read via the attr directive below).
dynamic = ["version"]
description = "Local MoE-offload LLM inference runtime with OpenAI- and Anthropic-compatible APIs"
readme = "README.md"
requires-python = ">=3.10"
license = "Apache-2.0"
license-files = ["LICENSE"]
authors = [{ name = "FlashML Team" }]
# No License:: classifier: setuptools>=77 rejects mixing one with the SPDX string above.
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Operating System :: POSIX :: Linux",
    "Environment :: GPU :: NVIDIA CUDA",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
]
# Version ranges are floor=last-verified, ceiling=next-major (kept loose enough to
# resolve, tight enough to bar a silent breaking bump). These ranges are the contract;
# there is no lockfile.
dependencies = [
    "apache-tvm-ffi==0.1.13.post3", # pin it
    "einops>=0.8,<1",
    "fastapi>=0.115,<1",
    # slot_cache: the device-side LRU admission kernel behind the MoE expert cache.
    "flashlib==0.4.0", # pin it
    "huggingface_hub>=1.5,<2",
    "msgpack>=1.1,<2",
    "modelscope>=1.37,<2",
    "numpy>=2.0,<3",
    "openai>=2.0,<3",
    "partial-json-parser>=0.2,<1",
    "pillow>=10,<13",
    "prompt_toolkit>=3.0,<4",
    "pydantic>=2.9,<3",
    "pyzmq>=27,<28",
    "safetensors>=0.6,<1",
    # floor+ceiling: sglang-kernel 0.4.9 requires torch==2.14.1.
    # PyPI's torch 2.14.1 wheel is itself the cu130 build, so plain pip resolves
    # correctly from PyPI alone; uv additionally pins the index below.
    "torch>=2.14,<2.15",
    "torchvision>=0.29,<0.30",
    "tqdm>=4.66,<5",
    "transformers>=5.18,<5.19",
    # CUDA torch 2.14 resolves Triton 3.8; AMD's ROCm 7.14 image supplies its
    # gfx1201-enabled Triton 3.7 build. Keep both supported without replacing the
    # runtime-specific wheel selected by the PyTorch distribution.
    "triton>=3.6,<3.9; platform_system == 'Linux'",
    "uvicorn>=0.30,<1",
]

[project.urls]
Homepage = "https://github.com/FlashML-org/FreeToken"
Repository = "https://github.com/FlashML-org/FreeToken"
Issues = "https://github.com/FlashML-org/FreeToken/issues"

[project.scripts]
ft = "freetoken.cli:main"

[project.optional-dependencies]
dev = ["pytest>=6.0"]
# Native fused-kernel packages. Without them the runtime falls back to the pure-Triton
# kernels in freetoken.kernel.triton (and the 'triton' attention backend). Install
# `freetoken[accel]` for the full native fast path.
fi = ["flashinfer-python[cu13]==0.6.18.post1"]
# renamed from sgl-kernel at 0.4; still imports as `sgl_kernel`, so never co-install both
sgl = ["sglang-kernel==0.4.9"]
accel = ["freetoken[fi,sgl]"]
# NOTE: the Marlin W4A16 NVFP4 expert-GEMM path (sm_80-99) borrows vLLM's AOT wheel
# (vllm>=0.14,<0.15), which pins transformers>=4.56,<5 and so is INCOMPATIBLE with the
# core transformers>=5.5 requirement. It is therefore not a lockable extra and is left
# out of the default resolution — install it separately (`pip install 'vllm>=0.14,<0.15'`)
# in a dedicated environment when you specifically need the Marlin path.

# uv-only provenance pins — pip ignores this section and resolves from PyPI, which
# works: PyPI's torch 2.14.1 is the same cu130 build this index serves.
# `explicit = true` scopes the index to the one package that needs it
# (the torch index also mirrors stale copies of common deps, e.g. packaging<=24.1,
# which would otherwise shadow PyPI under uv's first-index strategy).
[tool.uv.sources]
torch = { index = "pytorch-cu130" }

[[tool.uv.index]]
name = "pytorch-cu130"
url = "https://download.pytorch.org/whl/cu130"
explicit = true

[tool.setuptools]
include-package-data = true
package-dir = {"" = "python"}

[tool.setuptools.dynamic]
version = {attr = "freetoken.version.__version__"}

[tool.setuptools.packages.find]
where = ["python"]

[tool.setuptools.package-data]
"*" = ["csrc/**/*", "moe/configs/**/*.json", "third_party/*/LICENSE", "third_party/*/VERSION", "third_party/*/freetoken.patch"]

[tool.pytest.ini_options]
testpaths = ["tests"]
python_files = ["test_*.py"]
markers = [
    "slow: takes tens of seconds (big kernel sweeps, real-checkpoint reads); deselect with -m 'not slow'",
    "needs_weights: needs a real local checkpoint, gated behind an env var (see tests/README.md)",
]