Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Prebuilt CUDA wheels: compress the fatbins (flash-attn 1.49 GB -> ~285 MB) (#13854)

Daniel Han7c235de6f90c
diff --git a/.github/scripts/prebuilt_wheels.py b/.github/scripts/prebuilt_wheels.py
index 89f61432a8a..9b4445665ea 100644
--- a/.github/scripts/prebuilt_wheels.py
+++ b/.github/scripts/prebuilt_wheels.py
@@ -24,11 +24,13 @@ Subcommands:
wheel-name print the single upstream-style filename for one cell.
notes read `sha256 filename` lines on stdin and print the release body.
sums read `sha256 filename` lines on stdin and print SHA256SUMS for the wheels.
+ size-gate fail if a built wheel's CUDA extension is larger than its package allows.
Usage:
prebuilt_wheels.py matrix
prebuilt_wheels.py wheel-name --package flash-attn --torch 2.13.0 --python 3.13 [--platform win_amd64]
prebuilt_wheels.py notes --tag prebuilt-wheels-cu13 --repo unslothai/unsloth < SHA256SUMS
+ prebuilt_wheels.py size-gate --package flash-attn dist/flash_attn-*.whl
"""
from __future__ import annotations
@@ -37,6 +39,7 @@ import argparse
import json
import os
import sys
+import zipfile
# The CUDA major that every wheel here is built against, and the only one. It is the local
# version segment upstream writes (cu13), not the toolkit patch level: upstream normalises
@@ -54,6 +57,24 @@ CUDA_TOOLKIT = "13.0"
# that pattern-matches upstream names has to find it here too.
CXX11_ABI = "TRUE"
+# Device code is most of every wheel. Upstream flash-attn 2.8.1 compressed all of it; the commit
+# we build from dropped those flags, and in default mode its cubins went in uncompressed (the zip
+# still gets 3.5x out of the 1.49 GB extension; upstream's is 139 MB). Measured on one flash-attn
+# TU (nvcc 13.1, sm_80/90/100/120 + compute_120 PTX): 18.8 MB as is, 3.5 MB with these flags, and
+# the module loads no slower. causal-conv1d's and mamba-ssm's large cubins are already compressed
+# in default mode; repacking the released ones in size mode takes causal_conv1d_cuda from 248 to
+# 103 MB and selective_scan_cuda to about two thirds.
+# The workflow appends this to NVCC_APPEND_FLAGS (Windows already puts _USE_MATH_DEFINES there),
+# which nvcc reads itself, so it reaches every nvcc call each setup.py makes without patching
+# them; it is not in SPECS' env because that is word-split. --compress-mode=size needs a
+# CUDA >= 12.4 driver to load, below the cu13 floor.
+NVCC_APPEND_FLAGS = "-Xfatbin -compress-all --compress-mode=size"
+
+# ccache hashes the nvcc command line, not NVCC_APPEND_FLAGS, and the Linux runners' ccache (4.5)
+# has no namespaces, so the flags' identity goes into the actions/cache key instead: a cache
+# saved before them would otherwise hand back uncompressed objects.
+CCACHE_TAG = "fatbin-size"
+
# The Linux tag. The wheel is tagged linux_x86_64 rather than manylinux_*, exactly as upstream
# tags its own, so pip installs it on any glibc without a floor check; the practical floor is the
# runner's glibc, which is why the build runs on ubuntu-22.04 (glibc 2.35) and not on
@@ -177,6 +198,9 @@ SPECS = {
# Leave time to upload partial caches after a build timeout.
"build_timeout": "300m",
"import_names": ["flash_attn", "flash_attn_2_cuda"],
+ # Largest CUDA extension the size gate lets through, in MB. Compressed it is ~280; the
+ # uncompressed build that prompted the gate was 1,488.
+ "max_extension_mb": 450,
},
"causal-conv1d": {
"dist": "causal_conv1d",
@@ -191,6 +215,8 @@ SPECS = {
},
"build_timeout": "120m",
"import_names": ["causal_conv1d", "causal_conv1d_cuda"],
+ # ~105 with these flags, 263 in default mode.
+ "max_extension_mb": 160,
},
"mamba-ssm": {
"dist": "mamba_ssm",
@@ -212,6 +238,9 @@ SPECS = {
"patch": "cxx20",
"build_timeout": "180m",
"import_names": ["mamba_ssm", "selective_scan_cuda"],
+ # ~230 with these flags, 358 in default mode: the gate only catches flags that never
+ # reached nvcc.
+ "max_extension_mb": 300,
},
}
@@ -381,6 +410,9 @@ def build_matrix(
"build_timeout": spec["build_timeout"],
"shards": spec.get("shards", 0),
"build_env": _build_env(spec, platform),
+ "nvcc_append_flags": NVCC_APPEND_FLAGS,
+ "ccache_tag": CCACHE_TAG,
+ "max_extension_mb": spec["max_extension_mb"],
"import_names": " ".join(spec["import_names"]),
"wheel_name": wheel_name(
package, torch_version, python_version, platform
@@ -621,6 +653,37 @@ def _read_digest_lines() -> list[tuple[str, str]]:
return entries
+def extension_sizes(wheel: str) -> dict[str, int]:
+ """Uncompressed size of each compiled extension (.so / .pyd) in a wheel, by member name."""
+ with zipfile.ZipFile(wheel) as archive:
+ return {
+ info.filename: info.file_size
+ for info in archive.infolist()
+ if info.filename.endswith((".so", ".pyd"))
+ }
+
+
+def _cmd_size_gate(args: argparse.Namespace) -> int:
+ if args.package not in SPECS:
+ raise SystemExit(f"unknown package {args.package!r}")
+ limit = SPECS[args.package]["max_extension_mb"]
+ sizes = extension_sizes(args.wheel)
+ if not sizes:
+ raise SystemExit(f"{args.wheel}: no compiled extension in the wheel")
+ over = False
+ for name, size in sorted(sizes.items()):
+ megabytes = size / 1e6
+ print(f"{name}: {megabytes:.0f} MB (limit {limit} MB)")
+ over = over or megabytes > limit
+ if over:
+ print(
+ f"::error::{args.wheel}: a CUDA extension is over {limit} MB; is NVCC_APPEND_FLAGS "
+ "reaching nvcc?"
+ )
+ return 1
+ return 0
+
+
def _cmd_notes(args: argparse.Namespace) -> int:
sys.stdout.write(render_notes(_read_digest_lines(), tag = args.tag, repo = args.repo))
return 0
@@ -659,6 +722,11 @@ def main(argv: list[str] | None = None) -> int:
sums_parser = sub.add_parser("sums", help = "print SHA256SUMS for every wheel, digests on stdin")
sums_parser.set_defaults(func = _cmd_sums)
+ gate_parser = sub.add_parser("size-gate", help = "fail if the wheel's extension is too large")
+ gate_parser.add_argument("--package", required = True)
+ gate_parser.add_argument("wheel")
+ gate_parser.set_defaults(func = _cmd_size_gate)
+
args = parser.parse_args(argv)
return args.func(args)
diff --git a/.github/workflows/prebuilt-cuda-wheels.yml b/.github/workflows/prebuilt-cuda-wheels.yml
index 4b8eda42235..259e2358bc1 100644
--- a/.github/workflows/prebuilt-cuda-wheels.yml
+++ b/.github/workflows/prebuilt-cuda-wheels.yml
@@ -143,6 +143,10 @@ jobs:
CCACHE_MAXSIZE: 20G
# torch writes this into build.ninja unquoted, so the Windows value has no spaces.
PYTORCH_NVCC: ${{ matrix.pytorch_nvcc }}
+ # Compress every fatbin image; see NVCC_APPEND_FLAGS in prebuilt_wheels.py. The compile
+ # steps append it to NVCC_APPEND_FLAGS, which Windows already sets for _USE_MATH_DEFINES.
+ # Warm and build must agree or the merged ccache would mix both.
+ FATBIN_FLAGS: ${{ matrix.nvcc_append_flags }}
steps:
- name: Harden runner (audit)
# Audit egress because package downloads use changing endpoints. Its agent is Linux-only.
@@ -455,9 +459,10 @@ jobs:
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ccache
- key: prebuilt-ccache-${{ matrix.platform }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}
+ # ccache_tag names the nvcc flags ccache cannot see, so an older cache is never restored.
+ key: prebuilt-ccache-${{ matrix.platform }}-${{ matrix.ccache_tag }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}
restore-keys: |
- prebuilt-ccache-${{ matrix.platform }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-
+ prebuilt-ccache-${{ matrix.platform }}-${{ matrix.ccache_tag }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-
- name: Compile slice ${{ matrix.shard }} of ${{ matrix.shards }}
working-directory: src
@@ -474,6 +479,9 @@ jobs:
export "${pair?}"
echo "export $pair"
done
+ export NVCC_APPEND_FLAGS="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }$FATBIN_FLAGS"
+ echo "NVCC_APPEND_FLAGS=$NVCC_APPEND_FLAGS"
+ if [ -z "$FATBIN_FLAGS" ]; then echo "::error::FATBIN_FLAGS is empty"; exit 1; fi
ccache -z
# Windows ends the whole process tree at the deadline; see prebuilt_wheels_timeout.py.
if [ "$RUNNER_OS" = "Windows" ]; then
@@ -508,7 +516,7 @@ jobs:
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ccache
- key: prebuilt-ccache-${{ matrix.platform }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}-${{ github.run_attempt }}
+ key: prebuilt-ccache-${{ matrix.platform }}-${{ matrix.ccache_tag }}-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}-${{ github.run_attempt }}
build:
name: ${{ matrix.label }}
@@ -535,6 +543,10 @@ jobs:
CCACHE_MAXSIZE: 20G
# torch writes this into build.ninja unquoted, so the Windows value has no spaces.
PYTORCH_NVCC: ${{ matrix.pytorch_nvcc }}
+ # Compress every fatbin image; see NVCC_APPEND_FLAGS in prebuilt_wheels.py. The compile
+ # steps append it to NVCC_APPEND_FLAGS, which Windows already sets for _USE_MATH_DEFINES.
+ # Warm and build must agree or the merged ccache would mix both.
+ FATBIN_FLAGS: ${{ matrix.nvcc_append_flags }}
steps:
# Keep setup through the mamba patch in sync with warm.
- name: Harden runner (audit)
@@ -867,6 +879,9 @@ jobs:
echo "export $pair"
done
echo "MAX_JOBS=$MAX_JOBS NVCC_THREADS=$NVCC_THREADS timeout=$BUILD_TIMEOUT"
+ export NVCC_APPEND_FLAGS="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }$FATBIN_FLAGS"
+ echo "NVCC_APPEND_FLAGS=$NVCC_APPEND_FLAGS"
+ if [ -z "$FATBIN_FLAGS" ]; then echo "::error::FATBIN_FLAGS is empty"; exit 1; fi
if [ "$RUNNER_OS" = "Windows" ]; then
# Git Bash puts /usr/bin first, and its coreutils `link.exe` (hard links) shadows MSVC's
# linker: setuptools found it on PATH and every object compiled, then the final link
@@ -914,6 +929,15 @@ jobs:
mv "$built" "dist/${WHEEL_NAME}"
ls -l dist/
+ # An uncompressed fatbin still imports, so only its size gives it away (1.49 GB for
+ # flash-attn instead of ~280 MB). Gate it before anything is signed or published.
+ - name: Check the extension size
+ working-directory: src
+ env:
+ PACKAGE: ${{ matrix.package }}
+ WHEEL_NAME: ${{ matrix.wheel_name }}
+ run: python ../unsloth/.github/scripts/prebuilt_wheels.py size-gate --package "$PACKAGE" "dist/${WHEEL_NAME}"
+
# The whole reason this release exists, so it is a gate and not a report. A fresh venv with
# the matching torch, the wheel installed from the file, and the compiled extension
# imported out of process. An upstream wheel on torch 2.13 dies exactly here, with
diff --git a/tests/python/test_prebuilt_cuda_wheels.py b/tests/python/test_prebuilt_cuda_wheels.py
index e9712a2f9c8..15cbb7f974a 100644
--- a/tests/python/test_prebuilt_cuda_wheels.py
+++ b/tests/python/test_prebuilt_cuda_wheels.py
@@ -762,6 +762,96 @@ class TestWarmWiring:
assert uploaded[len(pattern) - 1 :] == "${{ matrix.shard }}"
+class TestFatbinCompression:
+ """nvcc compresses only PTX by default; uncompressed cubins made flash-attn 1.49 GB."""
+
+ def test_every_cell_compresses_every_fatbin_image(self):
+ flags = prebuilt_wheels.NVCC_APPEND_FLAGS.split()
+ assert "-compress-all" in flags
+ assert flags[flags.index("-compress-all") - 1] == "-Xfatbin"
+ assert "--compress-mode=size" in flags
+ everywhere = ",".join(prebuilt_wheels.PLATFORMS)
+ cells = prebuilt_wheels.build_matrix(torches = "2.13.0,2.14.0", platforms = everywhere)
+ cells += prebuilt_wheels.build_matrix(
+ torches = "2.10.0,2.11.0,2.12.0", pythons = "3.12", platforms = "win_amd64"
+ )
+ assert {cell["platform"] for cell in cells} == set(prebuilt_wheels.PLATFORMS)
+ for cell in cells:
+ assert cell["nvcc_append_flags"] == prebuilt_wheels.NVCC_APPEND_FLAGS
+ assert cell["ccache_tag"] == prebuilt_wheels.CCACHE_TAG
+ assert (
+ cell["max_extension_mb"]
+ == prebuilt_wheels.SPECS[cell["package"]]["max_extension_mb"]
+ )
+ # Never through the word-split build_env.
+ assert "compress" not in cell["build_env"]
+
+ def test_warm_and_build_append_the_flags_before_compiling(self, workflow):
+ for job, step_name in (
+ ("warm", "Compile slice ${{ matrix.shard }} of ${{ matrix.shards }}"),
+ ("build", "Build the wheel"),
+ ):
+ assert workflow["jobs"][job]["env"]["FATBIN_FLAGS"] == "${{ matrix.nvcc_append_flags }}"
+ script = next(s for s in workflow["jobs"][job]["steps"] if s.get("name") == step_name)[
+ "run"
+ ]
+ # Appended, so the Windows _USE_MATH_DEFINES already in NVCC_APPEND_FLAGS survives.
+ assert (
+ 'export NVCC_APPEND_FLAGS="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }$FATBIN_FLAGS"'
+ in script
+ )
+ # An empty value would compile, and in warm save, uncompressed objects under the tag.
+ assert 'if [ -z "$FATBIN_FLAGS" ]' in script
+ assert script.index("NVCC_APPEND_FLAGS=") < script.index("prebuilt_wheels") or (
+ script.index("NVCC_APPEND_FLAGS=") < script.index("bdist_wheel")
+ )
+
+ def test_no_ccache_from_before_the_flags_is_restored(self, workflow):
+ """ccache does not hash NVCC_APPEND_FLAGS, so the cache key has to carry them."""
+ keyed = [
+ step
+ for job in workflow["jobs"].values()
+ for step in job.get("steps", [])
+ if "actions/cache" in step.get("uses", "")
+ ]
+ assert len(keyed) == 2
+ for step in keyed:
+ keys = [step["with"]["key"], *step["with"].get("restore-keys", "").splitlines()]
+ for key in filter(None, (k.strip() for k in keys)):
+ assert key.startswith(
+ "prebuilt-ccache-${{ matrix.platform }}-${{ matrix.ccache_tag }}-"
+ ), key
+
+ def test_the_size_gate_runs_before_the_wheel_leaves_the_build(self, workflow):
+ names = [s.get("name") for s in workflow["jobs"]["build"]["steps"]]
+ gate = names.index("Check the extension size")
+ assert names.index("Name the wheel the way upstream names it") < gate
+ assert gate < names.index("Smoke test the wheel in a fresh venv")
+ run = workflow["jobs"]["build"]["steps"][gate]["run"]
+ assert run.startswith("python ../unsloth/.github/scripts/prebuilt_wheels.py size-gate ")
+ assert '--package "$PACKAGE"' in run
+
+ @pytest.mark.parametrize("megabytes, status", [(0.001, 0), (0.0001, 1)])
+ def test_size_gate(self, tmp_path, monkeypatch, megabytes, status):
+ import zipfile
+
+ wheel = tmp_path / "flash_attn-2.8.4-cp313-cp313-linux_x86_64.whl"
+ with zipfile.ZipFile(wheel, "w") as archive:
+ archive.writestr("flash_attn_2_cuda.cpython-313-x86_64-linux-gnu.so", b"x" * 500)
+ archive.writestr("flash_attn/__init__.py", b"")
+ monkeypatch.setitem(prebuilt_wheels.SPECS["flash-attn"], "max_extension_mb", megabytes)
+ assert prebuilt_wheels.main(["size-gate", "--package", "flash-attn", str(wheel)]) == status
+
+ def test_size_gate_refuses_a_wheel_without_an_extension(self, tmp_path):
+ import zipfile
+
+ wheel = tmp_path / "x.whl"
+ with zipfile.ZipFile(wheel, "w") as archive:
+ archive.writestr("x/__init__.py", b"")
+ with pytest.raises(SystemExit):
+ prebuilt_wheels.main(["size-gate", "--package", "flash-attn", str(wheel)])
+
+
def test_release_lookups_do_not_pass_arg_to_gh_jq():
"""gh's --jq has no --arg flag: `--jq --arg tag X` fails the publish job before any release call."""
workflow = (REPO / ".github" / "workflows" / "prebuilt-cuda-wheels.yml").read_text(