Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Prebuilt CUDA wheels: compress the fatbins (flash-attn 1.49 GB -> ~285 MB) (#13854)
Daniel Han7c235de6f90c
diff --git a/.github/scripts/prebuilt_wheels.py b/.github/scripts/prebuilt_wheels.pyindex 89f61432a8a..9b4445665ea 100644--- a/.github/scripts/prebuilt_wheels.py+++ b/.github/scripts/prebuilt_wheels.py@@ -24,11 +24,13 @@ Subcommands:wheel-name print the single upstream-style filename for one cell.notes read `sha256 filename` lines on stdin and print the release body.sums read `sha256 filename` lines on stdin and print SHA256SUMS for the wheels.+ size-gate fail if a built wheel's CUDA extension is larger than its package allows.Usage:prebuilt_wheels.py matrixprebuilt_wheels.py wheel-name --package flash-attn --torch 2.13.0 --python 3.13 [--platform win_amd64]prebuilt_wheels.py notes --tag prebuilt-wheels-cu13 --repo unslothai/unsloth < SHA256SUMS+ prebuilt_wheels.py size-gate --package flash-attn dist/flash_attn-*.whl"""from __future__ import annotations@@ -37,6 +39,7 @@ import argparseimport jsonimport osimport sys+import zipfile# The CUDA major that every wheel here is built against, and the only one. It is the local# version segment upstream writes (cu13), not the toolkit patch level: upstream normalises@@ -54,6 +57,24 @@ CUDA_TOOLKIT = "13.0"# that pattern-matches upstream names has to find it here too.CXX11_ABI = "TRUE"+# Device code is most of every wheel. Upstream flash-attn 2.8.1 compressed all of it; the commit+# we build from dropped those flags, and in default mode its cubins went in uncompressed (the zip+# still gets 3.5x out of the 1.49 GB extension; upstream's is 139 MB). Measured on one flash-attn+# TU (nvcc 13.1, sm_80/90/100/120 + compute_120 PTX): 18.8 MB as is, 3.5 MB with these flags, and+# the module loads no slower. causal-conv1d's and mamba-ssm's large cubins are already compressed+# in default mode; repacking the released ones in size mode takes causal_conv1d_cuda from 248 to+# 103 MB and selective_scan_cuda to about two thirds.+# The workflow appends this to NVCC_APPEND_FLAGS (Windows already puts _USE_MATH_DEFINES there),+# which nvcc reads itself, so it reaches every nvcc call each setup.py makes without patching+# them; it is not in SPECS' env because that is word-split. --compress-mode=size needs a+# CUDA >= 12.4 driver to load, below the cu13 floor.+NVCC_APPEND_FLAGS = "-Xfatbin -compress-all --compress-mode=size"++# ccache hashes the nvcc command line, not NVCC_APPEND_FLAGS, and the Linux runners' ccache (4.5)+# has no namespaces, so the flags' identity goes into the actions/cache key instead: a cache+# saved before them would otherwise hand back uncompressed objects.+CCACHE_TAG = "fatbin-size"+# The Linux tag. The wheel is tagged linux_x86_64 rather than manylinux_*, exactly as upstream# tags its own, so pip installs it on any glibc without a floor check; the practical floor is the# runner's glibc, which is why the build runs on ubuntu-22.04 (glibc 2.35) and not on@@ -177,6 +198,9 @@ SPECS = {# Leave time to upload partial caches after a build timeout."build_timeout": "300m","import_names": ["flash_attn", "flash_attn_2_cuda"],+ # Largest CUDA extension the size gate lets through, in MB. Compressed it is ~280; the+ # uncompressed build that prompted the gate was 1,488.+ "max_extension_mb": 450,},"causal-conv1d": {"dist": "causal_conv1d",@@ -191,6 +215,8 @@ SPECS = {},"build_timeout": "120m","import_names": ["causal_conv1d", "causal_conv1d_cuda"],+ # ~105 with these flags, 263 in default mode.+ "max_extension_mb": 160,},"mamba-ssm": {"dist": "mamba_ssm",@@ -212,6 +238,9 @@ SPECS = {"patch": "cxx20","build_timeout": "180m","import_names": ["mamba_ssm", "selective_scan_cuda"],+ # ~230 with these flags, 358 in default mode: the gate only catches flags that never+ # reached nvcc.+ "max_extension_mb": 300,},}@@ -381,6 +410,9 @@ def build_matrix("build_timeout": spec["build_timeout"],"shards": spec.get("shards", 0),"build_env": _build_env(spec, platform),+ "nvcc_append_flags": NVCC_APPEND_FLAGS,+ "ccache_tag": CCACHE_TAG,+ "max_extension_mb": spec["max_extension_mb"],"import_names": " ".join(spec["import_names"]),"wheel_name": wheel_name(package, torch_version, python_version, platform@@ -621,6 +653,37 @@ def _read_digest_lines() -> list[tuple[str, str]]:return entries+def extension_sizes(wheel: str) -> dict[str, int]:+ """Uncompressed size of each compiled extension (.so / .pyd) in a wheel, by member name."""+ with zipfile.ZipFile(wheel) as archive:+ return {+ info.filename: info.file_size+ for info in archive.infolist()+ if info.filename.endswith((".so", ".pyd"))+ }+++def _cmd_size_gate(args: argparse.Namespace) -> int:+ if args.package not in SPECS:+ raise SystemExit(f"unknown package {args.package!r}")+ limit = SPECS[args.package]["max_extension_mb"]+ sizes = extension_sizes(args.wheel)+ if not sizes:+ raise SystemExit(f"{args.wheel}: no compiled extension in the wheel")+ over = False+ for name, size in sorted(sizes.items()):+ megabytes = size / 1e6+ print(f"{name}: {megabytes:.0f} MB (limit {limit} MB)")+ over = over or megabytes > limit+ if over:+ print(+ f"::error::{args.wheel}: a CUDA extension is over {limit} MB; is NVCC_APPEND_FLAGS "+ "reaching nvcc?"+ )+ return 1+ return 0++def _cmd_notes(args: argparse.Namespace) -> int:sys.stdout.write(render_notes(_read_digest_lines(), tag = args.tag, repo = args.repo))return 0@@ -659,6 +722,11 @@ def main(argv: list[str] | None = None) -> int:sums_parser = sub.add_parser("sums", help = "print SHA256SUMS for every wheel, digests on stdin")sums_parser.set_defaults(func = _cmd_sums)+ gate_parser = sub.add_parser("size-gate", help = "fail if the wheel's extension is too large")+ gate_parser.add_argument("--package", required = True)+ gate_parser.add_argument("wheel")+ gate_parser.set_defaults(func = _cmd_size_gate)+args = parser.parse_args(argv)return args.func(args)diff --git a/.github/workflows/prebuilt-cuda-wheels.yml b/.github/workflows/prebuilt-cuda-wheels.ymlindex 4b8eda42235..259e2358bc1 100644--- a/.github/workflows/prebuilt-cuda-wheels.yml+++ b/.github/workflows/prebuilt-cuda-wheels.yml@@ -143,6 +143,10 @@ jobs:CCACHE_MAXSIZE: 20G# torch writes this into build.ninja unquoted, so the Windows value has no spaces.PYTORCH_NVCC: ${{ matrix.pytorch_nvcc }}+ # Compress every fatbin image; see NVCC_APPEND_FLAGS in prebuilt_wheels.py. The compile+ # steps append it to NVCC_APPEND_FLAGS, which Windows already sets for _USE_MATH_DEFINES.+ # Warm and build must agree or the merged ccache would mix both.+ FATBIN_FLAGS: ${{ matrix.nvcc_append_flags }}steps:- name: Harden runner (audit)# Audit egress because package downloads use changing endpoints. Its agent is Linux-only.@@ -455,9 +459,10 @@ jobs:uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0with:path: ccache- key: prebuilt-ccache-${{ matrix.platform }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}+ # ccache_tag names the nvcc flags ccache cannot see, so an older cache is never restored.+ key: prebuilt-ccache-${{ matrix.platform }}-${{ matrix.ccache_tag }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}restore-keys: |- prebuilt-ccache-${{ matrix.platform }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-+ prebuilt-ccache-${{ matrix.platform }}-${{ matrix.ccache_tag }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-- name: Compile slice ${{ matrix.shard }} of ${{ matrix.shards }}working-directory: src@@ -474,6 +479,9 @@ jobs:export "${pair?}"echo "export $pair"done+ export NVCC_APPEND_FLAGS="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }$FATBIN_FLAGS"+ echo "NVCC_APPEND_FLAGS=$NVCC_APPEND_FLAGS"+ if [ -z "$FATBIN_FLAGS" ]; then echo "::error::FATBIN_FLAGS is empty"; exit 1; ficcache -z# Windows ends the whole process tree at the deadline; see prebuilt_wheels_timeout.py.if [ "$RUNNER_OS" = "Windows" ]; then@@ -508,7 +516,7 @@ jobs:uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0with:path: ccache- key: prebuilt-ccache-${{ matrix.platform }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}-${{ github.run_attempt }}+ key: prebuilt-ccache-${{ matrix.platform }}-${{ matrix.ccache_tag }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}-${{ github.run_attempt }}build:name: ${{ matrix.label }}@@ -535,6 +543,10 @@ jobs:CCACHE_MAXSIZE: 20G# torch writes this into build.ninja unquoted, so the Windows value has no spaces.PYTORCH_NVCC: ${{ matrix.pytorch_nvcc }}+ # Compress every fatbin image; see NVCC_APPEND_FLAGS in prebuilt_wheels.py. The compile+ # steps append it to NVCC_APPEND_FLAGS, which Windows already sets for _USE_MATH_DEFINES.+ # Warm and build must agree or the merged ccache would mix both.+ FATBIN_FLAGS: ${{ matrix.nvcc_append_flags }}steps:# Keep setup through the mamba patch in sync with warm.- name: Harden runner (audit)@@ -867,6 +879,9 @@ jobs:echo "export $pair"doneecho "MAX_JOBS=$MAX_JOBS NVCC_THREADS=$NVCC_THREADS timeout=$BUILD_TIMEOUT"+ export NVCC_APPEND_FLAGS="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }$FATBIN_FLAGS"+ echo "NVCC_APPEND_FLAGS=$NVCC_APPEND_FLAGS"+ if [ -z "$FATBIN_FLAGS" ]; then echo "::error::FATBIN_FLAGS is empty"; exit 1; fiif [ "$RUNNER_OS" = "Windows" ]; then# Git Bash puts /usr/bin first, and its coreutils `link.exe` (hard links) shadows MSVC's# linker: setuptools found it on PATH and every object compiled, then the final link@@ -914,6 +929,15 @@ jobs:mv "$built" "dist/${WHEEL_NAME}"ls -l dist/+ # An uncompressed fatbin still imports, so only its size gives it away (1.49 GB for+ # flash-attn instead of ~280 MB). Gate it before anything is signed or published.+ - name: Check the extension size+ working-directory: src+ env:+ PACKAGE: ${{ matrix.package }}+ WHEEL_NAME: ${{ matrix.wheel_name }}+ run: python ../unsloth/.github/scripts/prebuilt_wheels.py size-gate --package "$PACKAGE" "dist/${WHEEL_NAME}"+# The whole reason this release exists, so it is a gate and not a report. A fresh venv with# the matching torch, the wheel installed from the file, and the compiled extension# imported out of process. An upstream wheel on torch 2.13 dies exactly here, withdiff --git a/tests/python/test_prebuilt_cuda_wheels.py b/tests/python/test_prebuilt_cuda_wheels.pyindex e9712a2f9c8..15cbb7f974a 100644--- a/tests/python/test_prebuilt_cuda_wheels.py+++ b/tests/python/test_prebuilt_cuda_wheels.py@@ -762,6 +762,96 @@ class TestWarmWiring:assert uploaded[len(pattern) - 1 :] == "${{ matrix.shard }}"+class TestFatbinCompression:+ """nvcc compresses only PTX by default; uncompressed cubins made flash-attn 1.49 GB."""++ def test_every_cell_compresses_every_fatbin_image(self):+ flags = prebuilt_wheels.NVCC_APPEND_FLAGS.split()+ assert "-compress-all" in flags+ assert flags[flags.index("-compress-all") - 1] == "-Xfatbin"+ assert "--compress-mode=size" in flags+ everywhere = ",".join(prebuilt_wheels.PLATFORMS)+ cells = prebuilt_wheels.build_matrix(torches = "2.13.0,2.14.0", platforms = everywhere)+ cells += prebuilt_wheels.build_matrix(+ torches = "2.10.0,2.11.0,2.12.0", pythons = "3.12", platforms = "win_amd64"+ )+ assert {cell["platform"] for cell in cells} == set(prebuilt_wheels.PLATFORMS)+ for cell in cells:+ assert cell["nvcc_append_flags"] == prebuilt_wheels.NVCC_APPEND_FLAGS+ assert cell["ccache_tag"] == prebuilt_wheels.CCACHE_TAG+ assert (+ cell["max_extension_mb"]+ == prebuilt_wheels.SPECS[cell["package"]]["max_extension_mb"]+ )+ # Never through the word-split build_env.+ assert "compress" not in cell["build_env"]++ def test_warm_and_build_append_the_flags_before_compiling(self, workflow):+ for job, step_name in (+ ("warm", "Compile slice ${{ matrix.shard }} of ${{ matrix.shards }}"),+ ("build", "Build the wheel"),+ ):+ assert workflow["jobs"][job]["env"]["FATBIN_FLAGS"] == "${{ matrix.nvcc_append_flags }}"+ script = next(s for s in workflow["jobs"][job]["steps"] if s.get("name") == step_name)[+ "run"+ ]+ # Appended, so the Windows _USE_MATH_DEFINES already in NVCC_APPEND_FLAGS survives.+ assert (+ 'export NVCC_APPEND_FLAGS="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }$FATBIN_FLAGS"'+ in script+ )+ # An empty value would compile, and in warm save, uncompressed objects under the tag.+ assert 'if [ -z "$FATBIN_FLAGS" ]' in script+ assert script.index("NVCC_APPEND_FLAGS=") < script.index("prebuilt_wheels") or (+ script.index("NVCC_APPEND_FLAGS=") < script.index("bdist_wheel")+ )++ def test_no_ccache_from_before_the_flags_is_restored(self, workflow):+ """ccache does not hash NVCC_APPEND_FLAGS, so the cache key has to carry them."""+ keyed = [+ step+ for job in workflow["jobs"].values()+ for step in job.get("steps", [])+ if "actions/cache" in step.get("uses", "")+ ]+ assert len(keyed) == 2+ for step in keyed:+ keys = [step["with"]["key"], *step["with"].get("restore-keys", "").splitlines()]+ for key in filter(None, (k.strip() for k in keys)):+ assert key.startswith(+ "prebuilt-ccache-${{ matrix.platform }}-${{ matrix.ccache_tag }}-"+ ), key++ def test_the_size_gate_runs_before_the_wheel_leaves_the_build(self, workflow):+ names = [s.get("name") for s in workflow["jobs"]["build"]["steps"]]+ gate = names.index("Check the extension size")+ assert names.index("Name the wheel the way upstream names it") < gate+ assert gate < names.index("Smoke test the wheel in a fresh venv")+ run = workflow["jobs"]["build"]["steps"][gate]["run"]+ assert run.startswith("python ../unsloth/.github/scripts/prebuilt_wheels.py size-gate ")+ assert '--package "$PACKAGE"' in run++ @pytest.mark.parametrize("megabytes, status", [(0.001, 0), (0.0001, 1)])+ def test_size_gate(self, tmp_path, monkeypatch, megabytes, status):+ import zipfile++ wheel = tmp_path / "flash_attn-2.8.4-cp313-cp313-linux_x86_64.whl"+ with zipfile.ZipFile(wheel, "w") as archive:+ archive.writestr("flash_attn_2_cuda.cpython-313-x86_64-linux-gnu.so", b"x" * 500)+ archive.writestr("flash_attn/__init__.py", b"")+ monkeypatch.setitem(prebuilt_wheels.SPECS["flash-attn"], "max_extension_mb", megabytes)+ assert prebuilt_wheels.main(["size-gate", "--package", "flash-attn", str(wheel)]) == status++ def test_size_gate_refuses_a_wheel_without_an_extension(self, tmp_path):+ import zipfile++ wheel = tmp_path / "x.whl"+ with zipfile.ZipFile(wheel, "w") as archive:+ archive.writestr("x/__init__.py", b"")+ with pytest.raises(SystemExit):+ prebuilt_wheels.main(["size-gate", "--package", "flash-attn", str(wheel)])++def test_release_lookups_do_not_pass_arg_to_gh_jq():"""gh's --jq has no --arg flag: `--jq --arg tag X` fails the publish job before any release call."""workflow = (REPO / ".github" / "workflows" / "prebuilt-cuda-wheels.yml").read_text(