Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Split the flash-attn prebuilt wheel build across parallel ccache jobs (#11812)

oobaboogabaa1da072bd2
diff --git a/.github/scripts/prebuilt_wheels.py b/.github/scripts/prebuilt_wheels.py
index c80cc1c3077..e403ac03b31 100644
--- a/.github/scripts/prebuilt_wheels.py
+++ b/.github/scripts/prebuilt_wheels.py
@@ -90,9 +90,9 @@ SPECS = {
# unlisted future card JITs rather than failing.
"FLASH_ATTN_CUDA_ARCHS": "80;90;100;120",
},
- # ~4.5 h for the four architectures at MAX_JOBS=1. GitHub kills a job at 6 h, so the
- # build is wrapped in `timeout` below that, to fail as a build timeout with logs
- # rather than as a job cancellation with none.
+ # Split the 6-8 hour compile across jobs to stay below GitHub's 6-hour limit.
+ "shards": 8,
+ # Leave time to upload partial caches after a build timeout.
"build_timeout": "300m",
"import_names": ["flash_attn", "flash_attn_2_cuda"],
},
@@ -232,6 +232,7 @@ def build_matrix(
"max_jobs": spec["max_jobs"],
"nvcc_threads": spec["nvcc_threads"],
"build_timeout": spec["build_timeout"],
+ "shards": spec.get("shards", 0),
"build_env": " ".join(
f"{key}={value}" for key, value in spec["env"].items()
),
@@ -247,6 +248,19 @@ def build_matrix(
return include
+def warm_matrix(include: list[dict]) -> list[dict]:
+ """One warm job per (cell, shard), for the cells whose package is sharded."""
+ return [
+ {
+ **cell,
+ "shard": shard,
+ "label": f"{cell['label']} / shard {shard + 1} of {cell['shards']}",
+ }
+ for cell in include
+ for shard in range(cell["shards"])
+ ]
+
+
def parse_wheel_name(name: str) -> dict | None:
"""Filename back to the facts the notes table needs, or None if it is not one of ours.
@@ -410,6 +424,7 @@ def _cmd_matrix(args: argparse.Namespace) -> int:
pythons = os.environ.get("UW_PYTHON_VERSIONS", ""),
)
matrix = json.dumps({"include": include}, separators = (",", ":"))
+ warm = warm_matrix(include)
print(matrix)
# Writing the step output here rather than echoing it in YAML keeps the JSON -- which is
@@ -420,6 +435,9 @@ def _cmd_matrix(args: argparse.Namespace) -> int:
with open(output, "a", encoding = "utf-8") as handle:
handle.write(f"matrix={matrix}\n")
handle.write(f"count={len(include)}\n")
+ warm_json = json.dumps({"include": warm}, separators = (",", ":"))
+ handle.write(f"warm_matrix={warm_json}\n")
+ handle.write(f"warm_count={len(warm)}\n")
summary = os.environ.get("GITHUB_STEP_SUMMARY")
if summary:
listing = "\n".join(f"- `{cell['wheel_name']}`" for cell in include)
@@ -461,7 +479,8 @@ def main(argv: list[str] | None = None) -> int:
matrix_parser.add_argument(
"--github",
action = "store_true",
- help = "also append matrix/count to $GITHUB_OUTPUT and a listing to $GITHUB_STEP_SUMMARY",
+ help = "also append matrix/count and warm_matrix/warm_count to $GITHUB_OUTPUT and a "
+ "listing to $GITHUB_STEP_SUMMARY",
)
matrix_parser.set_defaults(func = _cmd_matrix)
diff --git a/.github/scripts/prebuilt_wheels_shard.py b/.github/scripts/prebuilt_wheels_shard.py
new file mode 100644
index 00000000000..7153c31d730
--- /dev/null
+++ b/.github/scripts/prebuilt_wheels_shard.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Compile one shard of a torch extension's objects into ccache, then stop before linking.
+
+Only ninja's targets change; compiler commands must stay identical to the full build.
+"""
+
+import os
+import runpy
+import subprocess
+import sys
+
+
+def slice_objects(ninja_targets: str, shard: int, shards: int) -> list[str]:
+ """Select every SHARDS-th object from sorted, unique ninja targets, starting at SHARD."""
+ if not 0 <= shard < shards:
+ raise SystemExit(f"shard {shard} is outside 0..{shards - 1}")
+ objects = sorted(
+ {
+ target
+ for target in (line.split(":")[0] for line in ninja_targets.splitlines())
+ if target.endswith(".o")
+ }
+ )
+ return objects[shard::shards]
+
+
+def main() -> None:
+ import torch.utils.cpp_extension as cpp_extension
+
+ shard, shards = int(sys.argv[1]), int(sys.argv[2])
+
+ def compile_shard(build_directory, verbose, error_prefix):
+ listing = subprocess.run(
+ ["ninja", "-t", "targets", "all"],
+ cwd = build_directory,
+ capture_output = True,
+ text = True,
+ check = True,
+ ).stdout
+ mine = slice_objects(listing, shard, shards)
+ total = len(slice_objects(listing, 0, 1))
+ print(f"shard {shard + 1} of {shards}: {len(mine)} of {total} objects", flush = True)
+ jobs = os.environ.get("MAX_JOBS", "1")
+ subprocess.run(["ninja", "-v", "-j", jobs, *mine], cwd = build_directory, check = True)
+ raise SystemExit(0)
+
+ cpp_extension._run_ninja_build = compile_shard
+ sys.argv = ["setup.py", "build_ext"]
+ runpy.run_path("setup.py", run_name = "__main__")
+ raise SystemExit("setup.py returned without reaching the ninja step")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.github/workflows/prebuilt-cuda-wheels.yml b/.github/workflows/prebuilt-cuda-wheels.yml
index a0ac7f77cb4..563a366cde6 100644
--- a/.github/workflows/prebuilt-cuda-wheels.yml
+++ b/.github/workflows/prebuilt-cuda-wheels.yml
@@ -72,6 +72,8 @@ jobs:
outputs:
matrix: ${{ steps.plan.outputs.matrix }}
count: ${{ steps.plan.outputs.count }}
+ warm_matrix: ${{ steps.plan.outputs.warm_matrix }}
+ warm_count: ${{ steps.plan.outputs.warm_count }}
steps:
- name: Harden runner (egress block)
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
@@ -106,31 +108,31 @@ jobs:
# quotes and braces and there is no reason to send it through a shell round trip.
python3 .github/scripts/prebuilt_wheels.py matrix --github
- build:
- name: ${{ matrix.label }}
+ warm:
+ name: warm ${{ matrix.label }}
needs: plan
- if: needs.plan.outputs.count != '0'
- # ubuntu-22.04 deliberately, not ubuntu-latest. These wheels are tagged linux_x86_64, which
- # pip installs without checking glibc, so the runner's glibc IS the compatibility floor: 2.35
- # here against 2.39 on 24.04. Upstream builds its own wheels on 22.04 for the same reason.
+ if: needs.plan.outputs.warm_count != '0'
runs-on: ubuntu-22.04
- # flash-attn is ~4.5 h of nvcc at MAX_JOBS=1. GitHub's hard job limit is 6 h and a job that
- # hits it is reported as "cancelled" with no logs kept, so the build step carries its own
- # shorter `timeout` and this sits just above it.
+ # Best effort: the build job compiles whatever a warm job did not.
+ continue-on-error: true
timeout-minutes: 340
permissions:
contents: read
strategy:
fail-fast: false
- # One cell at a time would be safe and would also take a week. Five is what the free
- # runner pool serves without pushing every other workflow in the org behind it.
- max-parallel: 5
- matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
+ max-parallel: 16
+ matrix: ${{ fromJSON(needs.plan.outputs.warm_matrix) }}
+ env:
+ # Not a hidden directory: upload-artifact skips those by default.
+ CCACHE_DIR: ${{ github.workspace }}/ccache
+ CCACHE_BASEDIR: ${{ github.workspace }}
+ CCACHE_NOHASHDIR: 'true'
+ CCACHE_MAXSIZE: 20G
+ # torch reads this for the nvcc in build.ninja.
+ PYTORCH_NVCC: ccache /usr/local/cuda-13.0/bin/nvcc
steps:
- name: Harden runner (audit)
- # Audit, not block. This job talks to the apt CUDA repository, PyPI, the PyTorch wheel
- # index and two upstream GitHub repositories, and a blocklist tuned on one of those
- # moving would fail a four-hour build at the last step.
+ # Audit egress because package downloads use changing endpoints.
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
with:
egress-policy: audit
@@ -189,6 +191,19 @@ jobs:
echo "/usr/local/cuda-13.0/bin" >> "$GITHUB_PATH"
echo "CUDA_HOME=/usr/local/cuda-13.0" >> "$GITHUB_ENV"
+ # Cache hits require matching compiler commands and checkout paths in both jobs.
+ - name: Install ccache
+ timeout-minutes: 13
+ env:
+ RETRY_ATTEMPTS: '2'
+ RETRY_ATTEMPT_TIMEOUT: '300'
+ run: |
+ set -euo pipefail
+ bash unsloth/.github/scripts/retry-with-apt-lock.sh sudo sh -c \
+ 'apt-get update && apt-get install -y --no-install-recommends ccache'
+ ccache --version | head -1
+ mkdir -p "$CCACHE_DIR"
+
# 10 GB of swap. nvcc's peak RSS on the sm_100 flash-attn kernels is the reason MAX_JOBS is
# 1; swap is what keeps a single job that briefly exceeds the runner's 16 GB from being
# OOM-killed four hours in.
@@ -217,8 +232,7 @@ jobs:
sys.exit(f"installed torch {torch.__version__}, wanted {want_torch}")
if not torch.version.cuda.startswith("13."):
sys.exit(f"torch reports CUDA {torch.version.cuda}, wanted 13.x")
- # The wheel filename claims cxx11abiTRUE. If torch ever ships abiFALSE wheels again the
- # name would be a lie, and a lie in a filename is worse than a failure here.
+ # Verify the ABI matches the wheel's cxx11abiTRUE tag.
if not torch._C._GLIBCXX_USE_CXX11_ABI:
sys.exit("torch was built with _GLIBCXX_USE_CXX11_ABI=0; the cxx11abiTRUE tag would be wrong")
PY
@@ -235,6 +249,201 @@ jobs:
if: matrix.patch == 'cxx20'
run: python3 unsloth/.github/scripts/patch_mamba_cxx20.py src/setup.py
+ - name: Restore this slice's ccache from earlier runs
+ uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: ccache
+ key: prebuilt-ccache-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}
+ restore-keys: |
+ prebuilt-ccache-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-
+
+ - name: Compile slice ${{ matrix.shard }} of ${{ matrix.shards }}
+ working-directory: src
+ env:
+ MAX_JOBS: ${{ matrix.max_jobs }}
+ NVCC_THREADS: ${{ matrix.nvcc_threads }}
+ BUILD_ENV: ${{ matrix.build_env }}
+ BUILD_TIMEOUT: ${{ matrix.build_timeout }}
+ SHARD: ${{ matrix.shard }}
+ SHARDS: ${{ matrix.shards }}
+ run: |
+ set -euo pipefail
+ for pair in $BUILD_ENV; do
+ export "${pair?}"
+ echo "export $pair"
+ done
+ ccache -z
+ timeout --signal=INT --kill-after=120 "$BUILD_TIMEOUT" \
+ python ../unsloth/.github/scripts/prebuilt_wheels_shard.py "$SHARD" "$SHARDS"
+
+ - name: ccache stats
+ if: always()
+ run: ccache -s
+
+ - name: Hand the slice to the build job
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: ccache-${{ matrix.dist }}-torch${{ matrix.torch_mm }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}
+ path: ccache
+ if-no-files-found: warn
+ # ccache compresses its own entries.
+ compression-level: 0
+ retention-days: 1
+ # A re-run attempt uploads under the same name, which fails without this.
+ overwrite: true
+
+ # Save partial progress after failures. Immutable caches need a key per run attempt.
+ - name: Save this slice's ccache for later runs
+ if: always()
+ uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: ccache
+ key: prebuilt-ccache-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}-${{ github.run_attempt }}
+
+ build:
+ name: ${{ matrix.label }}
+ needs: [plan, warm]
+ # Run even if warm was skipped or failed; compile any cache misses here.
+ if: ${{ !cancelled() && needs.plan.outputs.count != '0' }}
+ # Keep the glibc floor at 2.35; pip does not check it for linux_x86_64 wheels.
+ runs-on: ubuntu-22.04
+ # Allow cleanup after the build step's timeout, below GitHub's 6-hour limit.
+ timeout-minutes: 340
+ permissions:
+ contents: read
+ strategy:
+ fail-fast: false
+ max-parallel: 5
+ matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
+ env:
+ CCACHE_DIR: ${{ github.workspace }}/ccache
+ CCACHE_BASEDIR: ${{ github.workspace }}
+ CCACHE_NOHASHDIR: 'true'
+ CCACHE_MAXSIZE: 20G
+ PYTORCH_NVCC: ccache /usr/local/cuda-13.0/bin/nvcc
+ steps:
+ # Keep setup through the mamba patch in sync with warm.
+ - name: Harden runner (audit)
+ # Audit egress because package downloads use changing endpoints.
+ uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
+ with:
+ egress-policy: audit
+
+ - name: Checkout Unsloth (build scripts only)
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ path: unsloth
+ persist-credentials: false
+ sparse-checkout: |
+ .github/scripts
+ sparse-checkout-cone-mode: false
+
+ - name: Checkout ${{ matrix.repo }}
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ repository: ${{ matrix.repo }}
+ ref: ${{ matrix.ref }}
+ path: src
+ submodules: ${{ matrix.submodules }}
+ persist-credentials: false
+
+ - name: Free up disk space
+ run: |
+ set -euo pipefail
+ df -h /
+ sudo rm -rf /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL /usr/local/lib/android || true
+ df -h /
+
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: ${{ matrix.python }}
+
+ - name: Install the CUDA ${{ matrix.cuda_tag }} toolkit
+ timeout-minutes: 25
+ env:
+ RETRY_ATTEMPTS: '2'
+ RETRY_ATTEMPT_TIMEOUT: '600'
+ run: |
+ set -euo pipefail
+ keyring=cuda-keyring_1.1-1_all.deb
+ curl --fail --silent --show-error --location --retry 5 --retry-delay 5 \
+ -o "$keyring" \
+ "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/$keyring"
+ sudo dpkg -i "$keyring"
+ bash unsloth/.github/scripts/retry-with-apt-lock.sh sudo sh -c \
+ 'apt-get update && apt-get install -y --no-install-recommends cuda-toolkit-13-0'
+ echo "/usr/local/cuda-13.0/bin" >> "$GITHUB_PATH"
+ echo "CUDA_HOME=/usr/local/cuda-13.0" >> "$GITHUB_ENV"
+
+ - name: Install ccache
+ timeout-minutes: 13
+ env:
+ RETRY_ATTEMPTS: '2'
+ RETRY_ATTEMPT_TIMEOUT: '300'
+ run: |
+ set -euo pipefail
+ bash unsloth/.github/scripts/retry-with-apt-lock.sh sudo sh -c \
+ 'apt-get update && apt-get install -y --no-install-recommends ccache'
+ ccache --version | head -1
+ mkdir -p "$CCACHE_DIR"
+
+ - name: Set up swap space
+ run: |
+ set -euo pipefail
+ sudo fallocate -l 10G /mnt/swapfile
+ sudo chmod 600 /mnt/swapfile
+ sudo mkswap /mnt/swapfile
+ sudo swapon /mnt/swapfile
+ free -h
+
+ - name: Install torch ${{ matrix.torch }}+cu130
+ run: |
+ set -euo pipefail
+ python -m pip install --upgrade pip
+ python -m pip install 'setuptools>=77' wheel ninja packaging
+ python -m pip install --no-cache-dir "torch==${{ matrix.torch }}" \
+ --index-url https://download.pytorch.org/whl/cu130
+ nvcc --version
+ python -c "import torch; print('torch', torch.__version__, 'cuda', torch.version.cuda, 'cxx11abi', torch._C._GLIBCXX_USE_CXX11_ABI)"
+ python - <<'PY'
+ import os, sys, torch
+ want_torch = os.environ["MATRIX_TORCH"]
+ if torch.__version__.split("+")[0] != want_torch:
+ sys.exit(f"installed torch {torch.__version__}, wanted {want_torch}")
+ if not torch.version.cuda.startswith("13."):
+ sys.exit(f"torch reports CUDA {torch.version.cuda}, wanted 13.x")
+ # Verify the ABI matches the wheel's cxx11abiTRUE tag.
+ if not torch._C._GLIBCXX_USE_CXX11_ABI:
+ sys.exit("torch was built with _GLIBCXX_USE_CXX11_ABI=0; the cxx11abiTRUE tag would be wrong")
+ PY
+ env:
+ MATRIX_TORCH: ${{ matrix.torch }}
+
+ - name: Patch mamba-ssm for the torch 2.13 C++20 requirement
+ if: matrix.patch == 'cxx20'
+ run: python3 unsloth/.github/scripts/patch_mamba_cxx20.py src/setup.py
+
+ - name: Download the warm slices
+ if: matrix.shards > 0
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ pattern: ccache-${{ matrix.dist }}-torch${{ matrix.torch_mm }}-${{ matrix.python_tag }}-shard*
+ path: ccache-slices
+
+ # The slices are disjoint, so copying them over one another loses nothing.
+ - name: Merge the warm slices into the ccache
+ if: matrix.shards > 0
+ run: |
+ set -euo pipefail
+ for slice in ccache-slices/*/; do
+ [ -d "$slice" ] || continue
+ echo "merging $slice ($(find "$slice" -type f | wc -l) files, $(du -sh "$slice" | cut -f1))"
+ cp -a "$slice." "$CCACHE_DIR/"
+ done
+ rm -rf ccache-slices
+ ccache -z
+
- name: Build the wheel
id: build
working-directory: src
@@ -255,6 +464,10 @@ jobs:
timeout --signal=INT --kill-after=120 "$BUILD_TIMEOUT" \
python setup.py bdist_wheel --dist-dir=dist
+ - name: ccache stats
+ if: always()
+ run: ccache -s
+
# Upstream's own naming, reproduced rather than approximated: the local version segment is
# what stops pip installing a torch 2.13 wheel into a torch 2.14 environment. Asserting the
# pre-rename name first is what makes it honest -- it proves the version that was BUILT is
@@ -308,6 +521,8 @@ jobs:
# wants a different one fails the step loudly instead of swapping torch under us.
"$RUNNER_TEMP/smoke/bin/python" -m pip freeze | grep -E '^(torch|triton)==' \
> "$RUNNER_TEMP/smoke-constraints.txt"
+ # tvm-ffi 0.1.12 with tilelang 0.1.8 breaks mamba_ssm imports (import test only).
+ echo 'apache-tvm-ffi==0.1.11' >> "$RUNNER_TEMP/smoke-constraints.txt"
cat "$RUNNER_TEMP/smoke-constraints.txt"
"$RUNNER_TEMP/smoke/bin/python" - "src/dist/${WHEEL_NAME}" "$RUNNER_TEMP/smoke-constraints.txt" <<'PY'
import email.parser, pathlib, re, subprocess, sys, zipfile
@@ -356,7 +571,8 @@ jobs:
gpu-smoke:
name: GPU forward pass
needs: [plan, build]
- if: inputs.gpu_smoke
+ # A skipped warm (no flash-attn) would otherwise skip every job downstream of build.
+ if: ${{ !cancelled() && inputs.gpu_smoke && needs.build.result == 'success' }}
runs-on: [self-hosted, gpu]
timeout-minutes: 90
permissions:
@@ -397,6 +613,7 @@ jobs:
sign:
name: Sign and attest
needs: [plan, build]
+ if: ${{ !cancelled() && needs.build.result == 'success' }}
runs-on: ubuntu-latest
timeout-minutes: 45
# Named for the same reason release-desktop.yml and woa-wheelhouse.yml name it: it is what
@@ -509,7 +726,7 @@ jobs:
publish:
name: Publish the release
needs: sign
- if: inputs.publish
+ if: ${{ !cancelled() && inputs.publish && needs.sign.result == 'success' }}
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
diff --git a/tests/python/test_prebuilt_cuda_wheels.py b/tests/python/test_prebuilt_cuda_wheels.py
index ad9d94c81d8..4fd94725ab9 100644
--- a/tests/python/test_prebuilt_cuda_wheels.py
+++ b/tests/python/test_prebuilt_cuda_wheels.py
@@ -53,6 +53,7 @@ def _load(name: str, path: Path):
prebuilt_wheels = _load("prebuilt_wheels", SCRIPTS / "prebuilt_wheels.py")
+prebuilt_wheels_shard = _load("prebuilt_wheels_shard", SCRIPTS / "prebuilt_wheels_shard.py")
def env(
@@ -537,8 +538,8 @@ class TestWorkflow:
inputs = triggers(workflow)["workflow_dispatch"]["inputs"]
assert inputs["publish"]["default"] is False
assert inputs["gpu_smoke"]["default"] is False
- assert workflow["jobs"]["publish"]["if"] == "inputs.publish"
- assert workflow["jobs"]["gpu-smoke"]["if"] == "inputs.gpu_smoke"
+ assert "inputs.publish &&" in workflow["jobs"]["publish"]["if"]
+ assert "inputs.gpu_smoke &&" in workflow["jobs"]["gpu-smoke"]["if"]
def test_only_publish_can_write_contents(self, workflow):
assert workflow["permissions"] == {"contents": "read"}
@@ -666,3 +667,101 @@ class TestWorkflow:
def test_concurrency_queues_rather_than_cancels(self, workflow):
assert workflow["concurrency"]["cancel-in-progress"] is False
+
+
+class TestWarmSlices:
+ def test_the_slices_compile_every_object_exactly_once(self):
+ objects = [f"/src/build/temp/csrc/kernel_{i:03d}.o" for i in range(97)]
+ # ninja lists rules and phony targets too, and nothing guarantees order or uniqueness.
+ listing = "\n".join(
+ [f"{o}: cuda_compile" for o in reversed(objects)]
+ + [f"{objects[0]}: cuda_compile", "/src/build/temp/flash_api.o: compile", "all: phony"]
+ )
+ expected = sorted(objects + ["/src/build/temp/flash_api.o"])
+ slices = [prebuilt_wheels_shard.slice_objects(listing, k, 8) for k in range(8)]
+ assert sorted(o for s in slices for o in s) == expected
+ assert max(map(len, slices)) - min(map(len, slices)) <= 1
+
+ @pytest.mark.parametrize("shard", [-1, 8])
+ def test_a_slice_outside_the_range_is_refused(self, shard):
+ with pytest.raises(SystemExit):
+ prebuilt_wheels_shard.slice_objects("a.o: compile", shard, 8)
+
+ def test_every_sharded_cell_gets_one_warm_job_per_slice(self):
+ include = prebuilt_wheels.build_matrix()
+ warm = prebuilt_wheels.warm_matrix(include)
+ for cell in include:
+ jobs = [w for w in warm if w["wheel_name"] == cell["wheel_name"]]
+ assert [w["shard"] for w in jobs] == list(range(cell["shards"]))
+ for w in jobs:
+ assert w["label"] == f"{cell['label']} / shard {w['shard'] + 1} of {cell['shards']}"
+ assert {w["package"] for w in warm} == {"flash-attn"}
+
+ def test_nothing_to_warm_without_flash_attn(self):
+ include = prebuilt_wheels.build_matrix(packages = "causal-conv1d,mamba-ssm")
+ assert prebuilt_wheels.warm_matrix(include) == []
+
+ def test_the_plan_step_writes_the_warm_outputs(self, tmp_path):
+ output = tmp_path / "output"
+ subprocess.run(
+ [sys.executable, str(SCRIPTS / "prebuilt_wheels.py"), "matrix", "--github"],
+ env = {
+ "GITHUB_OUTPUT": str(output),
+ "UW_PACKAGES": "flash-attn,mamba-ssm",
+ "UW_TORCH_VERSIONS": "2.13.0",
+ "UW_PYTHON_VERSIONS": "",
+ },
+ check = True,
+ capture_output = True,
+ )
+ values = dict(line.split("=", 1) for line in output.read_text().splitlines())
+ warm = json.loads(values["warm_matrix"])["include"]
+ assert (
+ values["warm_count"]
+ == str(len(warm))
+ == str(prebuilt_wheels.SPECS["flash-attn"]["shards"])
+ )
+ assert values["count"] == "2"
+
+
+class TestWarmWiring:
+ def test_warm_and_build_hash_the_same_compiles(self, workflow):
+ warm, build = workflow["jobs"]["warm"], workflow["jobs"]["build"]
+ assert warm["env"] == build["env"]
+ assert warm["runs-on"] == build["runs-on"] == "ubuntu-22.04"
+
+ def test_the_build_waits_for_warm_but_not_on_its_success(self, workflow):
+ warm, build = workflow["jobs"]["warm"], workflow["jobs"]["build"]
+ assert warm["continue-on-error"] is True
+ assert "warm" in build["needs"]
+ assert "!cancelled()" in build["if"]
+ assert warm["strategy"]["matrix"] == "${{ fromJSON(needs.plan.outputs.warm_matrix) }}"
+
+ def test_a_skipped_warm_does_not_skip_what_follows_the_build(self, workflow):
+ """Without a status function a skipped warm would silently skip signing and publishing."""
+ jobs = workflow["jobs"]
+
+ def needs(name):
+ value = jobs[name].get("needs", [])
+ return [value] if isinstance(value, str) else value
+
+ def ancestors(name):
+ return {a for n in needs(name) for a in {n} | ancestors(n)}
+
+ downstream = [name for name in jobs if "warm" in ancestors(name)]
+ assert set(downstream) == {"build", "gpu-smoke", "sign", "publish"}
+ for name in downstream:
+ assert "!cancelled()" in jobs[name]["if"], name
+ for parent in needs(name):
+ if parent not in ("plan", "warm"):
+ assert f"needs.{parent}.result == 'success'" in jobs[name]["if"], name
+
+ def test_the_build_downloads_the_names_warm_uploads(self, workflow):
+ def step(job, name):
+ return next(s for s in workflow["jobs"][job]["steps"] if s.get("name") == name)
+
+ uploaded = step("warm", "Hand the slice to the build job")["with"]["name"]
+ pattern = step("build", "Download the warm slices")["with"]["pattern"]
+ assert pattern.endswith("*")
+ assert uploaded.startswith(pattern[:-1])
+ assert uploaded[len(pattern) - 1 :] == "${{ matrix.shard }}"
diff --git a/tests/studio/test_cache_budget_discipline.py b/tests/studio/test_cache_budget_discipline.py
index 081e823eac6..56f845e3f2f 100644
--- a/tests/studio/test_cache_budget_discipline.py
+++ b/tests/studio/test_cache_budget_discipline.py
@@ -85,6 +85,9 @@ PIP_CACHE_JOBS = {
("version-compat-ci.yml", "grpo-fake-run"),
}
+# Partial saves on purpose: ccache checksums entries, so a truncated cache only costs misses.
+PARTIAL_SAVE_JOBS = {("prebuilt-cuda-wheels.yml", "warm")}
+
HEAVY = re.compile(
r"torch|transformers|trl|peft|vllm|bitsandbytes|sentence-transformers|diffusers"
r"|accelerate|datasets|requirements/"
@@ -676,6 +679,8 @@ def test_a_cache_save_of_downloaded_artifacts_waits_for_the_download_to_succeed(
"""
offenders = []
for name, jid, job in _jobs():
+ if (name, jid) in PARTIAL_SAVE_JOBS:
+ continue
steps = job.get("steps") or []
producers = {
s.get("id")