Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Split the flash-attn prebuilt wheel build across parallel ccache jobs (#11812)
oobaboogabaa1da072bd2
diff --git a/.github/scripts/prebuilt_wheels.py b/.github/scripts/prebuilt_wheels.pyindex c80cc1c3077..e403ac03b31 100644--- a/.github/scripts/prebuilt_wheels.py+++ b/.github/scripts/prebuilt_wheels.py@@ -90,9 +90,9 @@ SPECS = {# unlisted future card JITs rather than failing."FLASH_ATTN_CUDA_ARCHS": "80;90;100;120",},- # ~4.5 h for the four architectures at MAX_JOBS=1. GitHub kills a job at 6 h, so the- # build is wrapped in `timeout` below that, to fail as a build timeout with logs- # rather than as a job cancellation with none.+ # Split the 6-8 hour compile across jobs to stay below GitHub's 6-hour limit.+ "shards": 8,+ # Leave time to upload partial caches after a build timeout."build_timeout": "300m","import_names": ["flash_attn", "flash_attn_2_cuda"],},@@ -232,6 +232,7 @@ def build_matrix("max_jobs": spec["max_jobs"],"nvcc_threads": spec["nvcc_threads"],"build_timeout": spec["build_timeout"],+ "shards": spec.get("shards", 0),"build_env": " ".join(f"{key}={value}" for key, value in spec["env"].items()),@@ -247,6 +248,19 @@ def build_matrix(return include+def warm_matrix(include: list[dict]) -> list[dict]:+ """One warm job per (cell, shard), for the cells whose package is sharded."""+ return [+ {+ **cell,+ "shard": shard,+ "label": f"{cell['label']} / shard {shard + 1} of {cell['shards']}",+ }+ for cell in include+ for shard in range(cell["shards"])+ ]++def parse_wheel_name(name: str) -> dict | None:"""Filename back to the facts the notes table needs, or None if it is not one of ours.@@ -410,6 +424,7 @@ def _cmd_matrix(args: argparse.Namespace) -> int:pythons = os.environ.get("UW_PYTHON_VERSIONS", ""),)matrix = json.dumps({"include": include}, separators = (",", ":"))+ warm = warm_matrix(include)print(matrix)# Writing the step output here rather than echoing it in YAML keeps the JSON -- which is@@ -420,6 +435,9 @@ def _cmd_matrix(args: argparse.Namespace) -> int:with open(output, "a", encoding = "utf-8") as handle:handle.write(f"matrix={matrix}\n")handle.write(f"count={len(include)}\n")+ warm_json = json.dumps({"include": warm}, separators = (",", ":"))+ handle.write(f"warm_matrix={warm_json}\n")+ handle.write(f"warm_count={len(warm)}\n")summary = os.environ.get("GITHUB_STEP_SUMMARY")if summary:listing = "\n".join(f"- `{cell['wheel_name']}`" for cell in include)@@ -461,7 +479,8 @@ def main(argv: list[str] | None = None) -> int:matrix_parser.add_argument("--github",action = "store_true",- help = "also append matrix/count to $GITHUB_OUTPUT and a listing to $GITHUB_STEP_SUMMARY",+ help = "also append matrix/count and warm_matrix/warm_count to $GITHUB_OUTPUT and a "+ "listing to $GITHUB_STEP_SUMMARY",)matrix_parser.set_defaults(func = _cmd_matrix)diff --git a/.github/scripts/prebuilt_wheels_shard.py b/.github/scripts/prebuilt_wheels_shard.pynew file mode 100644index 00000000000..7153c31d730--- /dev/null+++ b/.github/scripts/prebuilt_wheels_shard.py@@ -0,0 +1,57 @@+#!/usr/bin/env python3+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Compile one shard of a torch extension's objects into ccache, then stop before linking.++Only ninja's targets change; compiler commands must stay identical to the full build.+"""++import os+import runpy+import subprocess+import sys+++def slice_objects(ninja_targets: str, shard: int, shards: int) -> list[str]:+ """Select every SHARDS-th object from sorted, unique ninja targets, starting at SHARD."""+ if not 0 <= shard < shards:+ raise SystemExit(f"shard {shard} is outside 0..{shards - 1}")+ objects = sorted(+ {+ target+ for target in (line.split(":")[0] for line in ninja_targets.splitlines())+ if target.endswith(".o")+ }+ )+ return objects[shard::shards]+++def main() -> None:+ import torch.utils.cpp_extension as cpp_extension++ shard, shards = int(sys.argv[1]), int(sys.argv[2])++ def compile_shard(build_directory, verbose, error_prefix):+ listing = subprocess.run(+ ["ninja", "-t", "targets", "all"],+ cwd = build_directory,+ capture_output = True,+ text = True,+ check = True,+ ).stdout+ mine = slice_objects(listing, shard, shards)+ total = len(slice_objects(listing, 0, 1))+ print(f"shard {shard + 1} of {shards}: {len(mine)} of {total} objects", flush = True)+ jobs = os.environ.get("MAX_JOBS", "1")+ subprocess.run(["ninja", "-v", "-j", jobs, *mine], cwd = build_directory, check = True)+ raise SystemExit(0)++ cpp_extension._run_ninja_build = compile_shard+ sys.argv = ["setup.py", "build_ext"]+ runpy.run_path("setup.py", run_name = "__main__")+ raise SystemExit("setup.py returned without reaching the ninja step")+++if __name__ == "__main__":+ main()diff --git a/.github/workflows/prebuilt-cuda-wheels.yml b/.github/workflows/prebuilt-cuda-wheels.ymlindex a0ac7f77cb4..563a366cde6 100644--- a/.github/workflows/prebuilt-cuda-wheels.yml+++ b/.github/workflows/prebuilt-cuda-wheels.yml@@ -72,6 +72,8 @@ jobs:outputs:matrix: ${{ steps.plan.outputs.matrix }}count: ${{ steps.plan.outputs.count }}+ warm_matrix: ${{ steps.plan.outputs.warm_matrix }}+ warm_count: ${{ steps.plan.outputs.warm_count }}steps:- name: Harden runner (egress block)uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1@@ -106,31 +108,31 @@ jobs:# quotes and braces and there is no reason to send it through a shell round trip.python3 .github/scripts/prebuilt_wheels.py matrix --github- build:- name: ${{ matrix.label }}+ warm:+ name: warm ${{ matrix.label }}needs: plan- if: needs.plan.outputs.count != '0'- # ubuntu-22.04 deliberately, not ubuntu-latest. These wheels are tagged linux_x86_64, which- # pip installs without checking glibc, so the runner's glibc IS the compatibility floor: 2.35- # here against 2.39 on 24.04. Upstream builds its own wheels on 22.04 for the same reason.+ if: needs.plan.outputs.warm_count != '0'runs-on: ubuntu-22.04- # flash-attn is ~4.5 h of nvcc at MAX_JOBS=1. GitHub's hard job limit is 6 h and a job that- # hits it is reported as "cancelled" with no logs kept, so the build step carries its own- # shorter `timeout` and this sits just above it.+ # Best effort: the build job compiles whatever a warm job did not.+ continue-on-error: truetimeout-minutes: 340permissions:contents: readstrategy:fail-fast: false- # One cell at a time would be safe and would also take a week. Five is what the free- # runner pool serves without pushing every other workflow in the org behind it.- max-parallel: 5- matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}+ max-parallel: 16+ matrix: ${{ fromJSON(needs.plan.outputs.warm_matrix) }}+ env:+ # Not a hidden directory: upload-artifact skips those by default.+ CCACHE_DIR: ${{ github.workspace }}/ccache+ CCACHE_BASEDIR: ${{ github.workspace }}+ CCACHE_NOHASHDIR: 'true'+ CCACHE_MAXSIZE: 20G+ # torch reads this for the nvcc in build.ninja.+ PYTORCH_NVCC: ccache /usr/local/cuda-13.0/bin/nvccsteps:- name: Harden runner (audit)- # Audit, not block. This job talks to the apt CUDA repository, PyPI, the PyTorch wheel- # index and two upstream GitHub repositories, and a blocklist tuned on one of those- # moving would fail a four-hour build at the last step.+ # Audit egress because package downloads use changing endpoints.uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1with:egress-policy: audit@@ -189,6 +191,19 @@ jobs:echo "/usr/local/cuda-13.0/bin" >> "$GITHUB_PATH"echo "CUDA_HOME=/usr/local/cuda-13.0" >> "$GITHUB_ENV"+ # Cache hits require matching compiler commands and checkout paths in both jobs.+ - name: Install ccache+ timeout-minutes: 13+ env:+ RETRY_ATTEMPTS: '2'+ RETRY_ATTEMPT_TIMEOUT: '300'+ run: |+ set -euo pipefail+ bash unsloth/.github/scripts/retry-with-apt-lock.sh sudo sh -c \+ 'apt-get update && apt-get install -y --no-install-recommends ccache'+ ccache --version | head -1+ mkdir -p "$CCACHE_DIR"+# 10 GB of swap. nvcc's peak RSS on the sm_100 flash-attn kernels is the reason MAX_JOBS is# 1; swap is what keeps a single job that briefly exceeds the runner's 16 GB from being# OOM-killed four hours in.@@ -217,8 +232,7 @@ jobs:sys.exit(f"installed torch {torch.__version__}, wanted {want_torch}")if not torch.version.cuda.startswith("13."):sys.exit(f"torch reports CUDA {torch.version.cuda}, wanted 13.x")- # The wheel filename claims cxx11abiTRUE. If torch ever ships abiFALSE wheels again the- # name would be a lie, and a lie in a filename is worse than a failure here.+ # Verify the ABI matches the wheel's cxx11abiTRUE tag.if not torch._C._GLIBCXX_USE_CXX11_ABI:sys.exit("torch was built with _GLIBCXX_USE_CXX11_ABI=0; the cxx11abiTRUE tag would be wrong")PY@@ -235,6 +249,201 @@ jobs:if: matrix.patch == 'cxx20'run: python3 unsloth/.github/scripts/patch_mamba_cxx20.py src/setup.py+ - name: Restore this slice's ccache from earlier runs+ uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0+ with:+ path: ccache+ key: prebuilt-ccache-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}+ restore-keys: |+ prebuilt-ccache-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-++ - name: Compile slice ${{ matrix.shard }} of ${{ matrix.shards }}+ working-directory: src+ env:+ MAX_JOBS: ${{ matrix.max_jobs }}+ NVCC_THREADS: ${{ matrix.nvcc_threads }}+ BUILD_ENV: ${{ matrix.build_env }}+ BUILD_TIMEOUT: ${{ matrix.build_timeout }}+ SHARD: ${{ matrix.shard }}+ SHARDS: ${{ matrix.shards }}+ run: |+ set -euo pipefail+ for pair in $BUILD_ENV; do+ export "${pair?}"+ echo "export $pair"+ done+ ccache -z+ timeout --signal=INT --kill-after=120 "$BUILD_TIMEOUT" \+ python ../unsloth/.github/scripts/prebuilt_wheels_shard.py "$SHARD" "$SHARDS"++ - name: ccache stats+ if: always()+ run: ccache -s++ - name: Hand the slice to the build job+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: ccache-${{ matrix.dist }}-torch${{ matrix.torch_mm }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}+ path: ccache+ if-no-files-found: warn+ # ccache compresses its own entries.+ compression-level: 0+ retention-days: 1+ # A re-run attempt uploads under the same name, which fails without this.+ overwrite: true++ # Save partial progress after failures. Immutable caches need a key per run attempt.+ - name: Save this slice's ccache for later runs+ if: always()+ uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0+ with:+ path: ccache+ key: prebuilt-ccache-${{ matrix.dist }}-${{ matrix.ref }}-torch${{ matrix.torch }}-${{ matrix.python_tag }}-shard${{ matrix.shard }}of${{ matrix.shards }}-${{ github.run_id }}-${{ github.run_attempt }}++ build:+ name: ${{ matrix.label }}+ needs: [plan, warm]+ # Run even if warm was skipped or failed; compile any cache misses here.+ if: ${{ !cancelled() && needs.plan.outputs.count != '0' }}+ # Keep the glibc floor at 2.35; pip does not check it for linux_x86_64 wheels.+ runs-on: ubuntu-22.04+ # Allow cleanup after the build step's timeout, below GitHub's 6-hour limit.+ timeout-minutes: 340+ permissions:+ contents: read+ strategy:+ fail-fast: false+ max-parallel: 5+ matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}+ env:+ CCACHE_DIR: ${{ github.workspace }}/ccache+ CCACHE_BASEDIR: ${{ github.workspace }}+ CCACHE_NOHASHDIR: 'true'+ CCACHE_MAXSIZE: 20G+ PYTORCH_NVCC: ccache /usr/local/cuda-13.0/bin/nvcc+ steps:+ # Keep setup through the mamba patch in sync with warm.+ - name: Harden runner (audit)+ # Audit egress because package downloads use changing endpoints.+ uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1+ with:+ egress-policy: audit++ - name: Checkout Unsloth (build scripts only)+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ path: unsloth+ persist-credentials: false+ sparse-checkout: |+ .github/scripts+ sparse-checkout-cone-mode: false++ - name: Checkout ${{ matrix.repo }}+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ repository: ${{ matrix.repo }}+ ref: ${{ matrix.ref }}+ path: src+ submodules: ${{ matrix.submodules }}+ persist-credentials: false++ - name: Free up disk space+ run: |+ set -euo pipefail+ df -h /+ sudo rm -rf /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL /usr/local/lib/android || true+ df -h /++ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: ${{ matrix.python }}++ - name: Install the CUDA ${{ matrix.cuda_tag }} toolkit+ timeout-minutes: 25+ env:+ RETRY_ATTEMPTS: '2'+ RETRY_ATTEMPT_TIMEOUT: '600'+ run: |+ set -euo pipefail+ keyring=cuda-keyring_1.1-1_all.deb+ curl --fail --silent --show-error --location --retry 5 --retry-delay 5 \+ -o "$keyring" \+ "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/$keyring"+ sudo dpkg -i "$keyring"+ bash unsloth/.github/scripts/retry-with-apt-lock.sh sudo sh -c \+ 'apt-get update && apt-get install -y --no-install-recommends cuda-toolkit-13-0'+ echo "/usr/local/cuda-13.0/bin" >> "$GITHUB_PATH"+ echo "CUDA_HOME=/usr/local/cuda-13.0" >> "$GITHUB_ENV"++ - name: Install ccache+ timeout-minutes: 13+ env:+ RETRY_ATTEMPTS: '2'+ RETRY_ATTEMPT_TIMEOUT: '300'+ run: |+ set -euo pipefail+ bash unsloth/.github/scripts/retry-with-apt-lock.sh sudo sh -c \+ 'apt-get update && apt-get install -y --no-install-recommends ccache'+ ccache --version | head -1+ mkdir -p "$CCACHE_DIR"++ - name: Set up swap space+ run: |+ set -euo pipefail+ sudo fallocate -l 10G /mnt/swapfile+ sudo chmod 600 /mnt/swapfile+ sudo mkswap /mnt/swapfile+ sudo swapon /mnt/swapfile+ free -h++ - name: Install torch ${{ matrix.torch }}+cu130+ run: |+ set -euo pipefail+ python -m pip install --upgrade pip+ python -m pip install 'setuptools>=77' wheel ninja packaging+ python -m pip install --no-cache-dir "torch==${{ matrix.torch }}" \+ --index-url https://download.pytorch.org/whl/cu130+ nvcc --version+ python -c "import torch; print('torch', torch.__version__, 'cuda', torch.version.cuda, 'cxx11abi', torch._C._GLIBCXX_USE_CXX11_ABI)"+ python - <<'PY'+ import os, sys, torch+ want_torch = os.environ["MATRIX_TORCH"]+ if torch.__version__.split("+")[0] != want_torch:+ sys.exit(f"installed torch {torch.__version__}, wanted {want_torch}")+ if not torch.version.cuda.startswith("13."):+ sys.exit(f"torch reports CUDA {torch.version.cuda}, wanted 13.x")+ # Verify the ABI matches the wheel's cxx11abiTRUE tag.+ if not torch._C._GLIBCXX_USE_CXX11_ABI:+ sys.exit("torch was built with _GLIBCXX_USE_CXX11_ABI=0; the cxx11abiTRUE tag would be wrong")+ PY+ env:+ MATRIX_TORCH: ${{ matrix.torch }}++ - name: Patch mamba-ssm for the torch 2.13 C++20 requirement+ if: matrix.patch == 'cxx20'+ run: python3 unsloth/.github/scripts/patch_mamba_cxx20.py src/setup.py++ - name: Download the warm slices+ if: matrix.shards > 0+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ pattern: ccache-${{ matrix.dist }}-torch${{ matrix.torch_mm }}-${{ matrix.python_tag }}-shard*+ path: ccache-slices++ # The slices are disjoint, so copying them over one another loses nothing.+ - name: Merge the warm slices into the ccache+ if: matrix.shards > 0+ run: |+ set -euo pipefail+ for slice in ccache-slices/*/; do+ [ -d "$slice" ] || continue+ echo "merging $slice ($(find "$slice" -type f | wc -l) files, $(du -sh "$slice" | cut -f1))"+ cp -a "$slice." "$CCACHE_DIR/"+ done+ rm -rf ccache-slices+ ccache -z+- name: Build the wheelid: buildworking-directory: src@@ -255,6 +464,10 @@ jobs:timeout --signal=INT --kill-after=120 "$BUILD_TIMEOUT" \python setup.py bdist_wheel --dist-dir=dist+ - name: ccache stats+ if: always()+ run: ccache -s+# Upstream's own naming, reproduced rather than approximated: the local version segment is# what stops pip installing a torch 2.13 wheel into a torch 2.14 environment. Asserting the# pre-rename name first is what makes it honest -- it proves the version that was BUILT is@@ -308,6 +521,8 @@ jobs:# wants a different one fails the step loudly instead of swapping torch under us."$RUNNER_TEMP/smoke/bin/python" -m pip freeze | grep -E '^(torch|triton)==' \> "$RUNNER_TEMP/smoke-constraints.txt"+ # tvm-ffi 0.1.12 with tilelang 0.1.8 breaks mamba_ssm imports (import test only).+ echo 'apache-tvm-ffi==0.1.11' >> "$RUNNER_TEMP/smoke-constraints.txt"cat "$RUNNER_TEMP/smoke-constraints.txt""$RUNNER_TEMP/smoke/bin/python" - "src/dist/${WHEEL_NAME}" "$RUNNER_TEMP/smoke-constraints.txt" <<'PY'import email.parser, pathlib, re, subprocess, sys, zipfile@@ -356,7 +571,8 @@ jobs:gpu-smoke:name: GPU forward passneeds: [plan, build]- if: inputs.gpu_smoke+ # A skipped warm (no flash-attn) would otherwise skip every job downstream of build.+ if: ${{ !cancelled() && inputs.gpu_smoke && needs.build.result == 'success' }}runs-on: [self-hosted, gpu]timeout-minutes: 90permissions:@@ -397,6 +613,7 @@ jobs:sign:name: Sign and attestneeds: [plan, build]+ if: ${{ !cancelled() && needs.build.result == 'success' }}runs-on: ubuntu-latesttimeout-minutes: 45# Named for the same reason release-desktop.yml and woa-wheelhouse.yml name it: it is what@@ -509,7 +726,7 @@ jobs:publish:name: Publish the releaseneeds: sign- if: inputs.publish+ if: ${{ !cancelled() && inputs.publish && needs.sign.result == 'success' }}runs-on: ubuntu-latesttimeout-minutes: 30permissions:diff --git a/tests/python/test_prebuilt_cuda_wheels.py b/tests/python/test_prebuilt_cuda_wheels.pyindex ad9d94c81d8..4fd94725ab9 100644--- a/tests/python/test_prebuilt_cuda_wheels.py+++ b/tests/python/test_prebuilt_cuda_wheels.py@@ -53,6 +53,7 @@ def _load(name: str, path: Path):prebuilt_wheels = _load("prebuilt_wheels", SCRIPTS / "prebuilt_wheels.py")+prebuilt_wheels_shard = _load("prebuilt_wheels_shard", SCRIPTS / "prebuilt_wheels_shard.py")def env(@@ -537,8 +538,8 @@ class TestWorkflow:inputs = triggers(workflow)["workflow_dispatch"]["inputs"]assert inputs["publish"]["default"] is Falseassert inputs["gpu_smoke"]["default"] is False- assert workflow["jobs"]["publish"]["if"] == "inputs.publish"- assert workflow["jobs"]["gpu-smoke"]["if"] == "inputs.gpu_smoke"+ assert "inputs.publish &&" in workflow["jobs"]["publish"]["if"]+ assert "inputs.gpu_smoke &&" in workflow["jobs"]["gpu-smoke"]["if"]def test_only_publish_can_write_contents(self, workflow):assert workflow["permissions"] == {"contents": "read"}@@ -666,3 +667,101 @@ class TestWorkflow:def test_concurrency_queues_rather_than_cancels(self, workflow):assert workflow["concurrency"]["cancel-in-progress"] is False+++class TestWarmSlices:+ def test_the_slices_compile_every_object_exactly_once(self):+ objects = [f"/src/build/temp/csrc/kernel_{i:03d}.o" for i in range(97)]+ # ninja lists rules and phony targets too, and nothing guarantees order or uniqueness.+ listing = "\n".join(+ [f"{o}: cuda_compile" for o in reversed(objects)]+ + [f"{objects[0]}: cuda_compile", "/src/build/temp/flash_api.o: compile", "all: phony"]+ )+ expected = sorted(objects + ["/src/build/temp/flash_api.o"])+ slices = [prebuilt_wheels_shard.slice_objects(listing, k, 8) for k in range(8)]+ assert sorted(o for s in slices for o in s) == expected+ assert max(map(len, slices)) - min(map(len, slices)) <= 1++ @pytest.mark.parametrize("shard", [-1, 8])+ def test_a_slice_outside_the_range_is_refused(self, shard):+ with pytest.raises(SystemExit):+ prebuilt_wheels_shard.slice_objects("a.o: compile", shard, 8)++ def test_every_sharded_cell_gets_one_warm_job_per_slice(self):+ include = prebuilt_wheels.build_matrix()+ warm = prebuilt_wheels.warm_matrix(include)+ for cell in include:+ jobs = [w for w in warm if w["wheel_name"] == cell["wheel_name"]]+ assert [w["shard"] for w in jobs] == list(range(cell["shards"]))+ for w in jobs:+ assert w["label"] == f"{cell['label']} / shard {w['shard'] + 1} of {cell['shards']}"+ assert {w["package"] for w in warm} == {"flash-attn"}++ def test_nothing_to_warm_without_flash_attn(self):+ include = prebuilt_wheels.build_matrix(packages = "causal-conv1d,mamba-ssm")+ assert prebuilt_wheels.warm_matrix(include) == []++ def test_the_plan_step_writes_the_warm_outputs(self, tmp_path):+ output = tmp_path / "output"+ subprocess.run(+ [sys.executable, str(SCRIPTS / "prebuilt_wheels.py"), "matrix", "--github"],+ env = {+ "GITHUB_OUTPUT": str(output),+ "UW_PACKAGES": "flash-attn,mamba-ssm",+ "UW_TORCH_VERSIONS": "2.13.0",+ "UW_PYTHON_VERSIONS": "",+ },+ check = True,+ capture_output = True,+ )+ values = dict(line.split("=", 1) for line in output.read_text().splitlines())+ warm = json.loads(values["warm_matrix"])["include"]+ assert (+ values["warm_count"]+ == str(len(warm))+ == str(prebuilt_wheels.SPECS["flash-attn"]["shards"])+ )+ assert values["count"] == "2"+++class TestWarmWiring:+ def test_warm_and_build_hash_the_same_compiles(self, workflow):+ warm, build = workflow["jobs"]["warm"], workflow["jobs"]["build"]+ assert warm["env"] == build["env"]+ assert warm["runs-on"] == build["runs-on"] == "ubuntu-22.04"++ def test_the_build_waits_for_warm_but_not_on_its_success(self, workflow):+ warm, build = workflow["jobs"]["warm"], workflow["jobs"]["build"]+ assert warm["continue-on-error"] is True+ assert "warm" in build["needs"]+ assert "!cancelled()" in build["if"]+ assert warm["strategy"]["matrix"] == "${{ fromJSON(needs.plan.outputs.warm_matrix) }}"++ def test_a_skipped_warm_does_not_skip_what_follows_the_build(self, workflow):+ """Without a status function a skipped warm would silently skip signing and publishing."""+ jobs = workflow["jobs"]++ def needs(name):+ value = jobs[name].get("needs", [])+ return [value] if isinstance(value, str) else value++ def ancestors(name):+ return {a for n in needs(name) for a in {n} | ancestors(n)}++ downstream = [name for name in jobs if "warm" in ancestors(name)]+ assert set(downstream) == {"build", "gpu-smoke", "sign", "publish"}+ for name in downstream:+ assert "!cancelled()" in jobs[name]["if"], name+ for parent in needs(name):+ if parent not in ("plan", "warm"):+ assert f"needs.{parent}.result == 'success'" in jobs[name]["if"], name++ def test_the_build_downloads_the_names_warm_uploads(self, workflow):+ def step(job, name):+ return next(s for s in workflow["jobs"][job]["steps"] if s.get("name") == name)++ uploaded = step("warm", "Hand the slice to the build job")["with"]["name"]+ pattern = step("build", "Download the warm slices")["with"]["pattern"]+ assert pattern.endswith("*")+ assert uploaded.startswith(pattern[:-1])+ assert uploaded[len(pattern) - 1 :] == "${{ matrix.shard }}"diff --git a/tests/studio/test_cache_budget_discipline.py b/tests/studio/test_cache_budget_discipline.pyindex 081e823eac6..56f845e3f2f 100644--- a/tests/studio/test_cache_budget_discipline.py+++ b/tests/studio/test_cache_budget_discipline.py@@ -85,6 +85,9 @@ PIP_CACHE_JOBS = {("version-compat-ci.yml", "grpo-fake-run"),}+# Partial saves on purpose: ccache checksums entries, so a truncated cache only costs misses.+PARTIAL_SAVE_JOBS = {("prebuilt-cuda-wheels.yml", "warm")}+HEAVY = re.compile(r"torch|transformers|trl|peft|vllm|bitsandbytes|sentence-transformers|diffusers"r"|accelerate|datasets|requirements/"@@ -676,6 +679,8 @@ def test_a_cache_save_of_downloaded_artifacts_waits_for_the_download_to_succeed("""offenders = []for name, jid, job in _jobs():+ if (name, jid) in PARTIAL_SAVE_JOBS:+ continuesteps = job.get("steps") or []producers = {s.get("id")