Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: prebuilt CUTLASS int8 GEMM with the dequant epilogue fused, raced per arch (FLUX.3 klein INT8 -19.8% on L4) (#17486)
Daniel Han52cea6699e1e
diff --git a/.github/workflows/prebuilt-int8-gemm.yml b/.github/workflows/prebuilt-int8-gemm.ymlnew file mode 100644index 00000000000..a885059eb51--- /dev/null+++ b/.github/workflows/prebuilt-int8-gemm.yml@@ -0,0 +1,234 @@+# Prebuilt int8 GEMM for Unsloth Studio's INT8 ConvRot diffusion path (studio/native_kernels/int8_gemm).+# C ABI + static cudart: one py3-none wheel per platform for every torch and CUDA 12/13 driver (built with 12.8).+# Dispatch only; `publish: false` builds, tests and signs; publishing shares prebuilt-cuda-wheels.yml's tag.+name: Prebuilt int8 GEMM++on:+ workflow_dispatch:+ inputs:+ release_tag:+ description: 'release tag to publish under'+ type: string+ default: 'prebuilt-wheels-cu13'+ publish:+ description: 'upload to the release (otherwise artifacts only)'+ type: boolean+ default: false+ draft:+ description: 'keep the release a draft'+ type: boolean+ default: true++permissions:+ contents: read++concurrency:+ group: prebuilt-int8-gemm-${{ github.repository }}+ cancel-in-progress: false++env:+ # CUTLASS is header-only here; pinned by commit so a rebuild is the same source.+ CUTLASS_SHA: b2f249acd20392beeb16f5e5a31f4f1a4e38b3bd++jobs:+ build:+ name: build ${{ matrix.platform }}+ runs-on: ${{ matrix.runner }}+ timeout-minutes: 90+ strategy:+ fail-fast: false+ matrix:+ include:+ - platform: manylinux_2_28_x86_64+ runner: ubuntu-latest+ - platform: win_amd64+ runner: windows-latest+ defaults:+ run:+ shell: bash+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false+ sparse-checkout: |+ studio/native_kernels/int8_gemm+ studio/backend/core/inference/diffusion_int8_cutlass.py+ sparse-checkout-cone-mode: false++ - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0+ with:+ python-version: '3.12'++ - name: CUTLASS at the pinned commit+ run: |+ set -euo pipefail+ git init -q cutlass+ git -C cutlass remote add origin https://github.com/NVIDIA/cutlass+ git -C cutlass fetch -q --depth 1 origin "$CUTLASS_SHA"+ git -C cutlass checkout -q FETCH_HEAD+ test "$(git -C cutlass rev-parse HEAD)" = "$CUTLASS_SHA"++ - name: Build (Linux, manylinux_2_28 container)+ if: runner.os == 'Linux'+ timeout-minutes: 60+ run: |+ set -euo pipefail+ docker run --rm -e CUTLASS_SHA -v "$PWD:/w" -w /w quay.io/pypa/manylinux_2_28_x86_64 bash -euo pipefail -c '+ dnf -y -q install dnf-plugins-core+ dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo+ dnf -y -q install cuda-nvcc-12-8 cuda-cudart-devel-12-8 cuda-crt-12-8+ /opt/python/cp312-cp312/bin/python studio/native_kernels/int8_gemm/build.py \+ --cutlass cutlass --out build --nvcc /usr/local/cuda-12.8/bin/nvcc --jobs "$(nproc)"+ /opt/python/cp312-cp312/bin/python studio/native_kernels/int8_gemm/make_wheel.py \+ --lib build/libunsloth_i8gemm.so --platform manylinux_2_28_x86_64 --out dist --cutlass-sha "$CUTLASS_SHA"+ '++ # Authenticode-checked NVIDIA installer: its output is what we sign.+ - name: Install the CUDA 12.8 toolkit (Windows)+ if: runner.os == 'Windows'+ timeout-minutes: 25+ shell: pwsh+ run: |+ $ErrorActionPreference = 'Stop'+ $version = '12.8.1'+ $url = "https://developer.download.nvidia.com/compute/cuda/$version/network_installers/cuda_${version}_windows_network.exe"+ $installer = Join-Path $env:RUNNER_TEMP 'cuda_network.exe'+ for ($attempt = 1; $attempt -le 3; $attempt++) {+ try { Invoke-WebRequest -Uri $url -OutFile $installer -TimeoutSec 300; break }+ catch { if ($attempt -eq 3) { throw }; Start-Sleep -Seconds 10 }+ }+ $signature = Get-AuthenticodeSignature -FilePath $installer+ if ($signature.Status -ne 'Valid' -or $signature.SignerCertificate.Subject -notmatch 'O=NVIDIA Corporation') {+ throw "CUDA installer signature: $($signature.Status) $($signature.SignerCertificate.Subject)"+ }+ $packages = @('nvcc', 'nvvm', 'crt', 'cudart', 'thrust') | ForEach-Object { "${_}_12.8" }+ $process = Start-Process -FilePath $installer -ArgumentList (@('-s') + $packages + @('-n')) -Wait -PassThru -NoNewWindow+ if ($process.ExitCode -ne 0) { throw "CUDA installer exited $($process.ExitCode)" }+ $root = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8'+ if (-not (Test-Path (Join-Path $root 'bin\nvcc.exe'))) { throw "no nvcc.exe under $root" }+ New-Item -ItemType Junction -Path 'C:\cuda' -Target $root | Out-Null+ 'C:\cuda\bin' | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8++ - name: Build (Windows, MSVC x64)+ if: runner.os == 'Windows'+ timeout-minutes: 60+ shell: pwsh+ run: |+ $ErrorActionPreference = 'Stop'+ $vs = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * `+ -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath+ cmd /c "`"$vs\VC\Auxiliary\Build\vcvars64.bat`" && set" | ForEach-Object {+ if ($_ -match '^([^=]+)=(.*)$') { [Environment]::SetEnvironmentVariable($matches[1], $matches[2]) }+ }+ python studio/native_kernels/int8_gemm/build.py --cutlass cutlass --out build --nvcc C:\cuda\bin\nvcc.exe --jobs $env:NUMBER_OF_PROCESSORS+ if ($LASTEXITCODE -ne 0) { throw "build exited $LASTEXITCODE" }+ python studio/native_kernels/int8_gemm/make_wheel.py --lib build/unsloth_i8gemm.dll --platform win_amd64 --out dist --cutlass-sha $env:CUTLASS_SHA+ if ($LASTEXITCODE -ne 0) { throw "make_wheel exited $LASTEXITCODE" }++ # No GPU: the library must still load and report its ABI and tile tables.+ - name: Smoke (install the wheel, load it the way Studio does)+ run: |+ set -euo pipefail+ python -m venv smoke+ py="$PWD/smoke/bin/python"; [ -x "$py" ] || py="$PWD/smoke/Scripts/python.exe"+ "$py" -m pip install --quiet --no-deps dist/*.whl+ cd "$RUNNER_TEMP"+ "$py" - <<'EOF'+ import ctypes, importlib.util, os+ import unsloth_studio_i8gemm as m+ lib = ctypes.CDLL(m.LIB_PATH)+ assert lib.ui8_abi_version() == m.ABI_VERSION == 1, lib.ui8_abi_version()+ assert lib.ui8_num_tiles(80) > 0 and lib.ui8_num_tiles(75) > 0+ print(m.LIB_PATH, os.path.getsize(m.LIB_PATH), "tiles", lib.ui8_num_tiles(80), lib.ui8_num_tiles(75))+ EOF++ - name: Linked only against the system runtime (Linux)+ if: runner.os == 'Linux'+ run: |+ set -euo pipefail+ ldd build/libunsloth_i8gemm.so+ ! ldd build/libunsloth_i8gemm.so | grep -E 'libcudart|libtorch|libpython'+ objdump -T build/libunsloth_i8gemm.so | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1++ - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2+ with:+ name: wheel-i8gemm-${{ matrix.platform }}+ path: dist/*.whl+ if-no-files-found: error++ sign:+ name: Sign+ needs: build+ runs-on: ubuntu-latest+ timeout-minutes: 30+ environment: release-signing+ permissions:+ id-token: write+ attestations: write+ contents: read+ steps:+ - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2+ with:+ pattern: wheel-i8gemm-*+ merge-multiple: true+ path: wheels++ - name: Both platforms arrived+ run: |+ set -euo pipefail+ ls -l wheels/+ test "$(find wheels -name '*.whl' | wc -l)" -eq 2+ cd wheels && sha256sum -- *.whl | sort -k2 > SHA256SUMS && cat SHA256SUMS++ - name: Sign every wheel with Sigstore+ uses: sigstore/gh-action-sigstore-python@790bc6befb9d733738f18d8f895854b453640ec9 # v3.5.0+ with:+ inputs: ./wheels/*.whl+ verify: true+ verify-cert-identity: https://github.com/${{ github.repository }}/.github/workflows/prebuilt-int8-gemm.yml@${{ github.ref }}+ verify-oidc-issuer: https://token.actions.githubusercontent.com+ upload-signing-artifacts: false+ release-signing-artifacts: false++ - name: Attest build provenance+ uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2+ with:+ subject-checksums: wheels/SHA256SUMS++ - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2+ with:+ name: prebuilt-int8-gemm-signed+ path: |+ wheels/*.whl+ wheels/*.sigstore.json+ wheels/SHA256SUMS+ if-no-files-found: error+ compression-level: 0++ publish:+ name: Publish to the release+ needs: sign+ if: ${{ inputs.publish }}+ runs-on: ubuntu-latest+ timeout-minutes: 20+ permissions:+ contents: write+ steps:+ - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2+ with:+ name: prebuilt-int8-gemm-signed+ path: wheels++ - name: Upload+ env:+ GH_TOKEN: ${{ github.token }}+ TAG: ${{ inputs.release_tag }}+ DRAFT: ${{ inputs.draft }}+ run: |+ set -euo pipefail+ if ! gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then+ gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --title "$TAG" --notes "Prebuilt wheels" \+ --latest=false --draft="$DRAFT"+ fi+ # SHA256SUMS is not uploaded here: prebuilt-cuda-wheels.yml owns that asset on the shared tag.+ gh release upload "$TAG" --repo "$GITHUB_REPOSITORY" --clobber wheels/*.whl wheels/*.sigstore.jsondiff --git a/studio/backend/core/inference/diffusion_int8_cutlass.py b/studio/backend/core/inference/diffusion_int8_cutlass.pynew file mode 100644index 00000000000..ec57c79ed1c--- /dev/null+++ b/studio/backend/core/inference/diffusion_int8_cutlass.py@@ -0,0 +1,536 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Prebuilt CUTLASS int8 GEMM (C ABI, ctypes) with torchao's epilogue fused: sm75 fp16 out, sm8x bf16 out.++Any load failure or tile failing the bit-exact probe keeps today's path; a tile replaces it only when it wins+the race by ``_MARGIN``. ``UNSLOTH_DIFFUSION_INT8_CUTLASS=0`` disables, ``=1`` skips the race."""++from __future__ import annotations++import ctypes+import os+import threading+from functools import lru_cache+from typing import Any, Callable, Optional++CUTLASS_ENV = "UNSLOTH_DIFFUSION_INT8_CUTLASS"+CUTLASS_LIB_ENV = "UNSLOTH_DIFFUSION_INT8_CUTLASS_LIB"+WHEEL_MODULE = "unsloth_studio_i8gemm"+ABI_VERSION = 1+LIB_NAMES = ("libunsloth_i8gemm.so", "unsloth_i8gemm.dll")++_STATUS = {+ 0: "ok",+ 1: "bad arguments",+ 2: "no kernel for this tile / dtype",+ 3: "cannot implement",+ 4: "initialize failed",+ 5: "launch failed",+}+# ties keep today's path+_MARGIN = 0.02+_RACE_M = (512, 4096)+_LOCK = threading.Lock()+_LOAD_ERROR: list = [None]+_CALLS = [0]+++def mode() -> str:+ raw = (os.environ.get(CUTLASS_ENV) or "").strip().lower()+ if raw in ("0", "off", "false", "no"):+ return "off"+ if raw in ("1", "on", "true", "yes", "force"):+ return "force"+ return "auto"+++def family(capability: Optional[tuple]) -> Optional[int]:+ if capability is None:+ return None+ cap = (int(capability[0]), int(capability[1]))+ if cap == (7, 5):+ return 75+ if cap[0] == 8:+ return 80+ return None+++def out_dtype_for(fam: int) -> Any:+ import torch+ return torch.float16 if fam == 75 else torch.bfloat16+++def _candidate_paths() -> list:+ paths = []+ explicit = (os.environ.get(CUTLASS_LIB_ENV) or "").strip()+ if explicit:+ paths.append(explicit)+ return paths+ try:+ import importlib.util+ spec = importlib.util.find_spec(WHEEL_MODULE)+ roots = list(spec.submodule_search_locations or ()) if spec is not None else []+ except Exception: # noqa: BLE001 - no wheel+ roots = []+ for root in roots:+ for name in LIB_NAMES:+ for sub in ("", "lib"):+ p = os.path.join(root, sub, name)+ if os.path.isfile(p):+ paths.append(p)+ return paths+++def _bind(lib: Any) -> Any:+ c = ctypes+ lib.ui8_abi_version.restype = c.c_int+ lib.ui8_abi_version.argtypes = []+ lib.ui8_num_tiles.restype = c.c_int+ lib.ui8_num_tiles.argtypes = [c.c_int]+ lib.ui8_tile_info.restype = c.c_int+ lib.ui8_tile_info.argtypes = [c.c_int, c.c_int, c.POINTER(c.c_int)]+ lib.ui8_has_kernel.restype = c.c_int+ lib.ui8_has_kernel.argtypes = [c.c_int] * 7+ lib.ui8_gemm.restype = c.c_int+ lib.ui8_gemm.argtypes = [c.c_int] * 6 + [c.c_void_p] * 6 + [c.c_int64] * 6 + [c.c_void_p]+ return lib+++@lru_cache(maxsize = 1)+def library() -> Optional[Any]:+ if mode() == "off":+ return None+ try:+ import torch++ from .rocm_bf16 import is_rocm_torch+ if not torch.cuda.is_available() or is_rocm_torch(torch):+ return None+ except Exception: # noqa: BLE001+ return None+ for path in _candidate_paths():+ try:+ lib = _bind(ctypes.CDLL(path))+ if lib.ui8_abi_version() != ABI_VERSION:+ _LOAD_ERROR[0] = f"{path}: ABI {lib.ui8_abi_version()} != {ABI_VERSION}"+ continue+ return lib+ except Exception as exc: # noqa: BLE001 - a broken binary keeps today's path+ _LOAD_ERROR[0] = f"{path}: {exc}"+ return None+++def load_error() -> Optional[str]:+ return _LOAD_ERROR[0]+++def library_path() -> Optional[str]:+ lib = library()+ return getattr(lib, "_name", None) if lib is not None else None+++@lru_cache(maxsize = 4)+def tiles(fam: int) -> tuple:+ lib = library()+ if lib is None:+ return ()+ out = []+ buf = (ctypes.c_int * 8)()+ for i in range(lib.ui8_num_tiles(fam)):+ if lib.ui8_tile_info(fam, i, buf) == 0:+ out.append(tuple(buf))+ return tuple(out)+++def _kind(dtype: Any) -> Optional[int]:+ import torch+ return {torch.float32: 0, torch.bfloat16: 1, torch.float16: 2}.get(dtype)+++EPI_TORCHAO = 0+EPI_NATIVE = 1+++def has_kernel(+ fam: int,+ tile: int,+ out_dtype: Any,+ xs_dtype: Any,+ ws_dtype: Any,+ bias: bool,+ epilogue: int = EPI_TORCHAO,+) -> bool:+ lib = library()+ kinds = (_kind(out_dtype), _kind(xs_dtype), _kind(ws_dtype))+ if lib is None or None in kinds:+ return False+ return bool(+ lib.ui8_has_kernel(fam, tile, kinds[0], kinds[1], kinds[2], int(bool(bias)), int(epilogue))+ )+++def eligible(a: Any, w: Any, xs: Any, ws: Any, bias: Any, out_dtype: Any) -> bool:+ import torch+ try:+ if a.dtype != torch.int8 or w.dtype != torch.int8 or a.dim() != 2 or w.dim() != 2:+ return False+ if a.device != w.device or not a.is_cuda:+ return False+ m, k = a.shape+ n = w.shape[0]+ if w.shape[1] != k or k % 16 or n % 8 or m < 1:+ return False+ if a.stride(1) != 1 or w.stride(1) != 1 or a.stride(0) % 16 or w.stride(0) % 16:+ return False+ if a.data_ptr() % 16 or w.data_ptr() % 16:+ return False+ for vec, size in ((xs, m), (ws, n)):+ if vec.device != a.device or vec.numel() != size or (vec.dim() and vec.stride(-1) != 1):+ return False+ if vec.data_ptr() % 16 or _kind(vec.dtype) is None:+ return False+ if bias is not None:+ if (+ bias.dtype != out_dtype+ or bias.numel() != n+ or bias.data_ptr() % 16+ or bias.device != a.device+ ):+ return False+ if bias.dim() and bias.stride(-1) != 1:+ return False+ return True+ except Exception: # noqa: BLE001+ return False+++def launch(+ a: Any,+ w: Any,+ xs: Any,+ ws: Any,+ bias: Any,+ fam: int,+ tile: int,+ out_dtype: Any,+ epilogue: int = EPI_TORCHAO,+) -> Any:+ import torch++ lib = library()+ if lib is None:+ raise RuntimeError("int8 CUTLASS library not loaded")+ m, k = a.shape+ n = w.shape[0]+ out = torch.empty((m, n), device = a.device, dtype = out_dtype)+ with torch.cuda.device(a.device):+ stream = torch.cuda.current_stream(a.device).cuda_stream+ status = lib.ui8_gemm(+ fam,+ tile,+ _kind(out_dtype),+ _kind(xs.dtype),+ _kind(ws.dtype),+ int(epilogue),+ a.data_ptr(),+ w.data_ptr(),+ xs.data_ptr(),+ ws.data_ptr(),+ bias.data_ptr() if bias is not None else None,+ out.data_ptr(),+ m,+ n,+ k,+ a.stride(0),+ w.stride(0),+ out.stride(0),+ stream,+ )+ if status != 0:+ raise RuntimeError(f"ui8_gemm tile {tile}: {_STATUS.get(status, status)}")+ _CALLS[0] += 1+ return out+++def reference(a: Any, w: Any, xs: Any, ws: Any, bias: Any, out_dtype: Any) -> Any:+ """torchao's int8 dynamic Linear epilogue op for op (fp16 x_scale computes in fp32)."""+ import torch++ m = a.shape[0]+ if m < 17: # _int_mm row floor+ a = torch.cat([a, a.new_zeros((17 - m, a.shape[1]))])+ c = torch._int_mm(a, w.t())[:m]+ inter = torch.float32 if xs.dtype == torch.float16 else xs.dtype+ y = (c * xs.reshape(-1, 1).to(inter)).to(out_dtype)+ y = y * ws.reshape(-1)+ if bias is not None:+ y = y + bias+ return y.to(out_dtype)+++def reference_native(a: Any, w: Any, xs: Any, ws: Any, out_dtype: Any) -> Any:+ """``NativeWeightOnlyLinear._int_mm_rows``: fp32 epilogue, one cast."""+ import torch++ m = a.shape[0]+ if m < 17:+ a = torch.cat([a, a.new_zeros((17 - m, a.shape[1]))])+ c = torch._int_mm(a, w.t())[:m]+ out = torch.mul(c, xs.reshape(-1, 1).float())+ return out.mul_(ws.reshape(-1).float()).to(out_dtype)+++# (M, N, K, bias): ragged tails, K below / above one tile+_PROBE_SHAPES = (+ (257, 384, 512, False),+ (33, 208, 144, True),+ (300, 528, 1040, True),+ (1, 64, 64, False),+)+++def _scale_dtypes(fam: int) -> tuple:+ import torch+ if fam == 75:+ return tuple(+ (x, w)+ for x in (torch.float16, torch.float32)+ for w in (torch.float16, torch.bfloat16, torch.float32)+ )+ return tuple(+ (x, w) for x in (torch.bfloat16, torch.float32) for w in (torch.bfloat16, torch.float32)+ )+++def _probe_tile(index: int, fam: int, tile: int, pairs: tuple) -> bool:+ import torch++ from .diffusion_int8_gemm import tie_operands++ dev = torch.device("cuda", index)+ out_dtype = out_dtype_for(fam)+ g = torch.Generator(device = "cpu").manual_seed(0)+ try:+ for xs_dt, ws_dt in pairs:+ for m, n, k, bias in _PROBE_SHAPES:+ if not has_kernel(fam, tile, out_dtype, xs_dt, ws_dt, bias):+ return False+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)+ xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(xs_dt).to(dev)+ ws = (torch.rand(n, generator = g) * 0.002 + 1e-5).to(ws_dt).to(dev)+ b = (torch.randn(n, generator = g) * 0.1).to(out_dtype).to(dev) if bias else None+ if not torch.equal(+ launch(a, w, xs, ws, b, fam, tile, out_dtype),+ reference(a, w, xs, ws, b, out_dtype),+ ):+ return False+ # near-midpoint products: the double rounding must match+ a, w = tie_operands(dev)+ xs = torch.ones(a.shape[0], device = dev, dtype = xs_dt)+ ws = torch.ones(w.shape[0], device = dev, dtype = ws_dt)+ if fam == 75:+ # keep fp16 out of inf+ xs = torch.full((a.shape[0],), 2.0**-12, device = dev, dtype = xs_dt)+ if not torch.equal(+ launch(a, w, xs, ws, None, fam, tile, out_dtype),+ reference(a, w, xs, ws, None, out_dtype),+ ):+ return False+ if has_kernel(fam, tile, out_dtype, torch.float32, torch.float32, False, EPI_NATIVE):+ for m, n, k, _bias in _PROBE_SHAPES:+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)+ xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(dev)+ ws = (torch.rand(n, generator = g) * 0.002 + 1e-5).to(dev)+ got = launch(a, w, xs, ws, None, fam, tile, out_dtype, EPI_NATIVE)+ if not torch.equal(got, reference_native(a, w, xs, ws, out_dtype)):+ return False+ a, w = tie_operands(dev)+ xs = torch.full((a.shape[0],), 2.0**-12, device = dev, dtype = torch.float32)+ ws = torch.ones(w.shape[0], device = dev, dtype = torch.float32)+ got = launch(a, w, xs, ws, None, fam, tile, out_dtype, EPI_NATIVE)+ if not torch.equal(got, reference_native(a, w, xs, ws, out_dtype)):+ return False+ torch.cuda.synchronize(dev)+ return True+ except Exception: # noqa: BLE001+ return False+++_PROBED: dict = {}+++def probed_tiles(index: int) -> tuple:+ if index in _PROBED:+ return _PROBED[index]+ ok: tuple = ()+ try:+ import torch+ fam = family(torch.cuda.get_device_capability(index))+ if fam is not None and library() is not None:+ pairs = _scale_dtypes(fam)+ ok = tuple(t for t in range(len(tiles(fam))) if _probe_tile(index, fam, t, pairs))+ except Exception: # noqa: BLE001+ ok = ()+ _PROBED[index] = ok+ return ok+++def available(index: int) -> bool:+ return bool(probed_tiles(index))+++def time_us(+ fns: dict,+ rounds: int = 3,+ reps: int = 3,+ warm: int = 1,+) -> dict:+ """{name: us}, min over interleaved rounds."""+ import torch++ start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)+ for fn in fns.values():+ for _ in range(warm):+ fn()+ torch.cuda.synchronize()+ best = {name: float("inf") for name in fns}+ for _ in range(rounds):+ for name, fn in fns.items():+ start.record()+ for _ in range(reps):+ fn()+ end.record()+ end.synchronize()+ best[name] = min(best[name], start.elapsed_time(end) * 1000.0 / reps)+ return best+++def race(+ index: int,+ w: Any,+ ws: Any,+ bias: Any,+ xs_dtype: Any,+ today: Optional[Callable],+ m: int,+ epilogue: int = EPI_TORCHAO,+) -> dict:+ """Best probed tile vs ``today`` (None = cuBLAS ``_int_mm``); tile None keeps today's path."""+ import torch++ fam = family(torch.cuda.get_device_capability(index))+ tiles_ok = probed_tiles(index)+ if fam is None or not tiles_ok:+ return {"tile": None, "us": {}}+ out_dtype = out_dtype_for(fam)+ dev = w.device+ k = w.shape[1]+ g = torch.Generator(device = "cpu").manual_seed(7)+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)+ xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(xs_dtype).to(dev)+ if not eligible(a, w, xs, ws, bias, out_dtype):+ return {"tile": None, "us": {}}+ cands = [+ t+ for t in tiles_ok+ if has_kernel(fam, t, out_dtype, xs_dtype, ws.dtype, bias is not None, epilogue)+ ]+ if not cands:+ return {"tile": None, "us": {}}++ def cut(t):+ return lambda: launch(a, w, xs, ws, bias, fam, t, out_dtype, epilogue)++ if today is None:+ base = lambda: torch._int_mm(a, w.t()).amax() # noqa: E731+ base_name = "cublas"+ else:+ base = lambda: today(a, w, xs, ws, bias) # noqa: E731+ base_name = "today"+ with torch.cuda.device(dev):+ screen = time_us({t: cut(t) for t in cands}, rounds = 1, reps = 2, warm = 1)+ top = sorted(cands, key = lambda t: screen[t])[:3]+ fns = {base_name: base}+ fns.update({t: cut(t) for t in top})+ timing = time_us(fns, rounds = 3, reps = 3, warm = 1)+ best = min(top, key = lambda t: timing[t])+ keep = timing[best] < timing[base_name] * (1.0 - _MARGIN)+ us = {str(k_): round(v, 1) for k_, v in timing.items()}+ return {"tile": best if keep else None, "us": us, "m": m}+++# (card, N, K) -> {race M: tile or None}, sm75 NativeWeightOnlyLinear+_NATIVE_PLAN: dict = {}+_NATIVE_RACES: dict = {}+_NATIVE_LARGE_M = 1024+++def native_prepare(weight_q: Any, x_dtype: Any, has_bias: bool) -> None:+ """Once per (card, N, K), outside any capture: race native-epilogue tiles (sm75, fp16, no bias)."""+ import torch+ try:+ if mode() == "off" or has_bias or x_dtype != torch.float16 or not weight_q.is_cuda:+ return+ index = weight_q.device.index+ key = (index, int(weight_q.shape[0]), int(weight_q.shape[1]))+ if key in _NATIVE_PLAN:+ return+ if torch.cuda.is_current_stream_capturing() or torch.compiler.is_compiling():+ return+ _NATIVE_PLAN[key] = {}+ if family(torch.cuda.get_device_capability(index)) != 75 or not probed_tiles(index):+ return+ g = torch.Generator(device = "cpu").manual_seed(13)+ ws = (torch.rand(weight_q.shape[0], generator = g) * 1e-3 + 1e-5).to(weight_q.device)+ today = lambda a, w_, x_, s_, b_: reference_native(a, w_, x_, s_, torch.float16) # noqa: E731+ entry = {}+ for m in _RACE_M:+ r = race(index, weight_q, ws, None, torch.float32, today, m, epilogue = EPI_NATIVE)+ if mode() == "force" and r.get("us"):+ digits = [t for t in r["us"] if t.isdigit()]+ r["tile"] = int(min(digits, key = lambda t: r["us"][t])) if digits else None+ entry[m] = r["tile"]+ _NATIVE_RACES.setdefault(key, {})[m] = r+ _NATIVE_PLAN[key] = entry+ except Exception: # noqa: BLE001+ return+++def native_tile(xq: Any, weight_q: Any, x_scale: Any, w_scale: Any) -> Optional[int]:+ import torch++ if torch.compiler.is_compiling():+ return None+ entry = _NATIVE_PLAN.get(+ (weight_q.device.index, int(weight_q.shape[0]), int(weight_q.shape[1]))+ )+ if not entry:+ return None+ tile = entry.get(max(_RACE_M) if xq.shape[0] >= _NATIVE_LARGE_M else min(_RACE_M))+ if tile is None:+ return None+ if x_scale.dtype != torch.float32 or w_scale.dtype != torch.float32:+ return None+ return (+ tile+ if eligible(xq, weight_q, x_scale.reshape(-1), w_scale.reshape(-1), None, torch.float16)+ else None+ )+++def native_gemm(xq: Any, weight_q: Any, x_scale: Any, w_scale: Any, tile: int) -> Any:+ import torch+ return launch(+ xq,+ weight_q,+ x_scale.reshape(-1),+ w_scale.reshape(-1),+ None,+ 75,+ tile,+ torch.float16,+ EPI_NATIVE,+ )diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex 5c28c386d46..42fbc9e431e 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -38,6 +38,9 @@ _MARK = "_unsloth_i8_gemm_prev"_FWD = "_unsloth_i8_gemm_fwd"# on a SwiGLU MLP whose down projection takes its codes from the gate / value halves_FF_MARK = "_unsloth_i8_gemm_swiglu_prev"+# fp16 input runs the sm75 prebuilt kernel+_F16 = "_unsloth_i8_gemm_f16"+_OP16_NAME = "int8_mm_dequant_f16"_SWIGLU_FF_CLASSES = ("QwenImage21SwiGLUFeedForward",)_NO_PREV = object()_LOCK = threading.Lock()@@ -483,11 +486,18 @@ def tma_tile_for(index: Any, m: int) -> Optional[tuple]:def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:- """Op body: the probed tile for this device and shape, else (launch failure, ``_aligned`` false) the stock- epilogue."""+ """Op body: race-won CUTLASS tile, else probed Triton tile, else the stock epilogue."""_CALLS[0] += 1a = a if a.stride(-1) == 1 else a.contiguous()index = a.device.index+ tile = _cutlass_tile(index, a, w, xs, ws, bias)+ if tile is not None:+ import torch+ from . import diffusion_int8_cutlass as cut+ try:+ return cut.launch(a, w, xs, ws, bias, _CUT_FAMILY[index], tile, torch.bfloat16)+ except Exception: # noqa: BLE001 - drops the prebuilt kernel on this card+ _CUT_PLAN.pop(index, None)cfg = tile_for(index, a.shape[0], w.shape[0], a.shape[1])if cfg is not None and w.stride(-1) == 1 and w.device == a.device and _aligned(a, w):tma = tma_tile_for(index, a.shape[0])@@ -510,6 +520,345 @@ def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:return reference(a, w, xs, ws, bias)+# index -> {(N, K, xs dtype, ws dtype, bias): {race M: tile or None}}; a large-M tile runs ``_cutlass_fused``+_CUT_PLAN: dict = {}+_CUT_FAMILY: dict = {}+_CUT_RACES: dict = {}+# index -> every shape key already planned (winners and losers), so a later load races only new shapes+_CUT_DONE: dict = {}+_CUT_LARGE_M = 1024+_CUT_CACHE_NAME = "diffusion_int8_cutlass_race.json"+_CUT_CACHE_MAX = 512+++def _cut_key(n: int, k: int, xs_dtype: Any, ws_dtype: Any, bias: bool) -> tuple:+ return (int(n), int(k), str(xs_dtype), str(ws_dtype), bool(bias))+++def _cutlass_tile(+ index: Any,+ a: Any,+ w: Any,+ xs: Any,+ ws: Any,+ bias: Any,+ out_dtype: Any = None,+) -> Optional[int]:+ plan = _CUT_PLAN.get(index)+ if not plan:+ return None+ entry = plan.get(_cut_key(w.shape[0], w.shape[1], xs.dtype, ws.dtype, bias is not None))+ if not entry:+ return None+ from . import diffusion_int8_cutlass as cut++ tile = entry.get(max(cut._RACE_M) if a.shape[0] >= _CUT_LARGE_M else min(cut._RACE_M))+ if tile is None:+ return None+ import torch++ return tile if cut.eligible(a, w, xs, ws, bias, out_dtype or torch.bfloat16) else None+++def _cutlass_fused(index: Any, n: int, k: int, xs_dtype: Any, ws_dtype: Any, bias: bool) -> bool:+ from . import diffusion_int8_cutlass as cut+ entry = (_CUT_PLAN.get(index) or {}).get(_cut_key(n, k, xs_dtype, ws_dtype, bias))+ return bool(entry) and entry.get(max(cut._RACE_M)) is not None+++def cutlass_wanted(capability: Optional[tuple]) -> bool:+ """The prebuilt kernel could serve this arch (switch, family, library loads); before any probe."""+ try:+ from . import diffusion_int8_cutlass as cut+ return (+ int8_gemm_mode() != "off"+ and cut.mode() != "off"+ and cut.family(capability) in (75, 80)+ and cut.library() is not None+ )+ except Exception: # noqa: BLE001+ return False+++def _cut_cache_key(index: int) -> Optional[str]:+ """Card, toolchain, library build and both sources: any change re-races."""+ try:+ import hashlib++ import torch++ from . import diffusion_int8_cutlass as cut++ h = hashlib.sha256(open(__file__, "rb").read())+ h.update(open(cut.__file__, "rb").read())+ lib = cut.library_path()+ if lib:+ h.update(open(lib, "rb").read())+ try:+ import triton as _triton+ tv = _triton.__version__+ except Exception: # noqa: BLE001+ tv = "none"+ cap = torch.cuda.get_device_capability(index)+ return "|".join(+ (+ torch.cuda.get_device_name(index),+ f"{cap[0]}.{cap[1]}",+ torch.__version__,+ str(torch.version.cuda),+ tv,+ os.environ.get(INT8_GEMM_TILES_ENV, ""),+ os.environ.get(INT8_GEMM_UNLISTED_ENV, ""),+ h.hexdigest()[:20],+ )+ )+ except Exception: # noqa: BLE001+ return None+++def _cut_cache_path() -> Optional[Any]:+ try:+ from pathlib import Path+ from utils.paths.storage_roots import cache_root+ return Path(cache_root()) / _CUT_CACHE_NAME+ except Exception: # noqa: BLE001+ return None+++def _cut_cache_read() -> dict:+ path = _cut_cache_path()+ try:+ import json+ table = (+ json.loads(path.read_text(encoding = "utf-8"))+ if path is not None and path.is_file()+ else {}+ )+ return table if isinstance(table, dict) else {}+ except Exception: # noqa: BLE001+ return {}+++def _cut_cache_write(table: dict) -> None:+ path = _cut_cache_path()+ if path is None:+ return+ try:+ import json++ table = dict(list(table.items())[-_CUT_CACHE_MAX:])+ path.parent.mkdir(parents = True, exist_ok = True)+ tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")+ tmp.write_text(json.dumps(table), encoding = "utf-8")+ os.replace(tmp, path)+ except Exception: # noqa: BLE001+ pass+++def _xs_dtype(+ kind: str,+ weight: Any,+ k: int,+ device: Any,+ dtype: Any = None,+) -> Any:+ """Activation-scale dtype of this Linear's act quant, read off a one-row quant."""+ import torch++ dtype = dtype or torch.bfloat16+ x = torch.ones((_MIN_ROWS, k), device = device, dtype = dtype)+ if dtype == torch.float16:+ from .diffusion_int8_fused import _act_quant+ return _act_quant(x, weight)[1].dtype+ _, xs = _act_quant_v1(x) if kind == "v1" else _act_quant_v2(x, weight)+ return xs.dtype+++def _cutlass_plan(+ index: int,+ recs: list,+ triton_on: bool,+ logger: Any = None,+) -> dict:+ """Race each int8 weight shape vs today's path, publish winners to ``_CUT_PLAN``; outside any capture."""+ import torch++ from . import diffusion_int8_cutlass as cut++ try:+ cap = torch.cuda.get_device_capability(index)+ if not cutlass_wanted(cap) or torch.cuda.is_current_stream_capturing():+ return {}+ if torch.compiler.is_compiling():+ return {}+ except Exception: # noqa: BLE001+ return {}+ done = _CUT_DONE.setdefault(index, set())+ plan: dict = dict(_CUT_PLAN.get(index) or {})+ races: dict = dict(_CUT_RACES.get(index) or {})+ try:+ if not cut.probed_tiles(index):+ _CUT_PLAN[index] = {}+ return {}+ fam = cut.family(cap)+ act = cut.out_dtype_for(fam)+ ckey = _cut_cache_key(index)+ table = _cut_cache_read() if ckey else {}+ cached = table.get(ckey) if ckey else None+ cached = cached if isinstance(cached, dict) else {}+ dev = torch.device("cuda", index)+ g = torch.Generator(device = "cpu").manual_seed(11)+ seen: dict = {}+ for module, rec in recs:+ kind, weight = rec[0], rec[3]+ wq, ws = _payload(kind, weight)+ n, k = int(wq.shape[0]), int(wq.shape[1])+ bias = getattr(module, "bias", None)+ if fam == 75 and kind != "v2":+ continue # fp16 family: torchao Int8Tensor only+ xs_dt = seen.get(("xs", kind, k))+ if xs_dt is None:+ xs_dt = _xs_dtype(kind, weight, k, dev, act)+ seen[("xs", kind, k)] = xs_dt+ key = _cut_key(n, k, xs_dt, ws.dtype, bias is not None)+ if key in plan or key in done:+ continue+ done.add(key)+ skey = repr(key)+ hit = cached.get(skey)+ if isinstance(hit, dict) and all(+ (v is None or v in cut.probed_tiles(index)) for v in hit.values()+ ):+ plan[key] = {int(m): v for m, v in hit.items()}+ continue+ # random operands: a streamed weight need not be resident+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)+ wsr = (torch.rand(n, generator = g) * 1e-3 + 1e-5).to(ws.dtype).to(dev)+ br = (torch.randn(n, generator = g) * 0.1).to(act).to(dev) if bias is not None else None+ entry = {}+ for m in cut._RACE_M:+ if triton_on:+ today = lambda a, w_, x_, s_, b_: _launch_or_ref(index, a, w_, x_, s_, b_) # noqa: E731+ elif m >= _CUT_LARGE_M:+ today = None # cuBLAS lower bound+ else:+ today = lambda a, w_, x_, s_, b_: cut.reference(a, w_, x_, s_, b_, act) # noqa: E731+ r = cut.race(index, w, wsr, br, xs_dt, today, m)+ if cut.mode() == "force" and r.get("us"):+ r["tile"] = min(+ (t for t in r["us"] if t.isdigit()), key = lambda t: r["us"][t], default = None+ )+ r["tile"] = int(r["tile"]) if r["tile"] is not None else None+ entry[m] = r["tile"]+ races.setdefault(skey, {})[m] = r+ plan[key] = entry+ cached[skey] = {str(m): v for m, v in entry.items()}+ del w, wsr, br+ if ckey:+ table = _cut_cache_read()+ table.pop(ckey, None)+ table[ckey] = cached+ _cut_cache_write(table)+ except Exception as exc: # noqa: BLE001+ if logger is not None:+ logger.info(+ "diffusion.int8_gemm: prebuilt CUTLASS race failed on cuda:%d (%s)", index, exc+ )+ plan = dict(_CUT_PLAN.get(index) or {})+ plan = {k_: v for k_, v in plan.items() if any(t is not None for t in v.values())}+ _CUT_PLAN[index] = plan+ _CUT_FAMILY[index] = cut.family(torch.cuda.get_device_capability(index))+ _CUT_RACES[index] = races+ return plan+++def _launch_or_ref(index: int, a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:+ cfg = tile_for(index, a.shape[0], w.shape[0], a.shape[1])+ if cfg is not None and _aligned(a, w):+ return _launch(a, w, xs, ws, bias, cfg)+ return reference(a, w, xs, ws, bias)+++def cutlass_call_count() -> int:+ from . import diffusion_int8_cutlass as cut+ return cut._CALLS[0]+++def _run16(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:+ """fp16 op body (sm75 family): the race winner, else torchao's fp16 epilogue after cuBLAS."""+ import torch++ from . import diffusion_int8_cutlass as cut++ _CALLS[0] += 1+ a = a if a.stride(-1) == 1 else a.contiguous()+ index = a.device.index+ tile = _cutlass_tile(index, a, w, xs, ws, bias, torch.float16)+ if tile is not None:+ try:+ return cut.launch(a, w, xs, ws, bias, _CUT_FAMILY[index], tile, torch.float16)+ except Exception: # noqa: BLE001+ _CUT_PLAN.pop(index, None)+ return cut.reference(a, w, xs, ws, bias, torch.float16)+++@lru_cache(maxsize = 1)+def _op16() -> Any:+ """``unsloth_studio::int8_mm_dequant_f16`` (opaque to Inductor), or None."""+ try:+ import torch+ except Exception: # noqa: BLE001+ return None+ ns = getattr(torch.ops, _OP_NAMESPACE, None)+ if ns is not None and hasattr(ns, _OP16_NAME):+ return getattr(ns, _OP16_NAME)+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)+ if custom_op is None:+ return None+ try:++ @custom_op(+ f"{_OP_NAMESPACE}::{_OP16_NAME}",+ mutates_args = (),+ schema = "(Tensor a, Tensor w, Tensor xs, Tensor ws, Tensor? bias) -> Tensor",+ )+ def _int8_mm_dequant_f16(a, w, xs, ws, bias):+ return _run16(a, w, xs, ws, bias)++ @_int8_mm_dequant_f16.register_fake+ def _(a, w, xs, ws, bias):+ return a.new_empty((a.shape[0], w.shape[0]), dtype = torch.float16)+ except Exception: # noqa: BLE001+ return None+ return getattr(torch.ops, _OP_NAMESPACE).int8_mm_dequant_f16+++def _linear_forward_f16(self: Any, x: Any, rec: tuple) -> Any:+ """sm75 fp16 forward via the prebuilt kernel, bit for bit the stock forward."""+ import torch++ kind, group, _rotq, weight, _fused = rec+ if self.weight is not weight or _OP16_HANDLE is None:+ return type(self).forward(self, x)+ wq, ws = _payload(kind, weight)+ if wq.device != x.device or x.numel() < _MIN_ROWS * x.shape[-1]:+ return type(self).forward(self, x)+ lead = x.shape[:-1]+ if group is not None:+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation+ x = rotate_convrot_activation(+ x, build_convrot_hadamard(group, device = x.device, dtype = x.dtype), group+ )+ from .diffusion_int8_fused import _act_quant++ xq, xs = _act_quant(x.reshape(-1, x.shape[-1]), weight)+ y = _OP16_HANDLE(xq, wq, xs.reshape(-1), ws, self.bias)+ return y.reshape(*lead, y.shape[-1])+++_OP16_HANDLE: Any = None++@lru_cache(maxsize = 1)def _op() -> Any:"""``unsloth_studio::int8_mm_dequant`` (opaque to Inductor; no host sync, allocation only), or None."""@@ -1055,7 +1404,13 @@ def _linear_forward(self: Any, x: Any) -> Any:import torchrec = self.__dict__.get(_REC)- if rec is None or x.dtype != torch.bfloat16 or not x.is_cuda:+ if rec is None or not x.is_cuda:+ return type(self).forward(self, x)+ if self.__dict__.get(_F16):+ if x.dtype == torch.float16:+ return _linear_forward_f16(self, x, rec)+ return type(self).forward(self, x)+ if x.dtype != torch.bfloat16:return type(self).forward(self, x)kind, group, rotq, weight, fused = recif self.weight is not weight: # weight replaced since install (reload / LoRA bake): stock@@ -1307,10 +1662,20 @@ def linear_from_q(q: Any, xs: Any, weight: Any, bias: Any) -> Optional[Any]:if _OP_HANDLE is None or int8_gemm_mode() == "off" or q.device.index is None:return None- if _DEVICE_CFG.get(q.device.index) is None:- return None# The fused MLP already admitted this weight (symmetric, and its stock epilogue ignores the zero point too).parts = _v2_parts(weight, check_zero_point = False)+ if _DEVICE_CFG.get(q.device.index) is None and not (+ parts is not None+ and _cutlass_fused(+ q.device.index,+ parts[0].shape[0],+ parts[0].shape[1],+ xs.dtype,+ parts[1].dtype,+ bias is not None,+ )+ ):+ return Noneif (parts is Noneor q.shape[0] < _MIN_ROWS@@ -1353,9 +1718,10 @@ def _eligible(module: Any) -> Optional[tuple]:bias = module.biasimport torch- if bias is not None and bias.dtype != torch.bfloat16:+ # fp16 bias only reaches the sm75 kernel (``_swap_on`` filters per card)+ if bias is not None and bias.dtype not in (torch.bfloat16, torch.float16):return None- # v1: bf16 scales only; v2 also fp32 (prequant checkpoints). Off-grid K runs masked loads, far slower than cuBLAS.+ # off-grid K runs masked loads, far slower than cuBLASout_f, in_f = getattr(module, "out_features", 0), getattr(module, "in_features", 0)if in_f % _K_ALIGN or out_f % _N_ALIGN:return None@@ -1363,11 +1729,25 @@ def _eligible(module: Any) -> Optional[tuple]:if parts is not None and parts[1].dtype == torch.bfloat16:return ("v1", group, parts[1], w)parts = _v2_parts(w)- if parts is not None and parts[1].dtype in (torch.bfloat16, torch.float32):+ if parts is not None and parts[1].dtype in (torch.bfloat16, torch.float32, torch.float16):return ("v2", group, parts[1], w)return None+def _card_dtypes_ok(module: Any, rec: tuple, f16_card: bool) -> bool:+ import torch++ bias = getattr(module, "bias", None)+ if f16_card:+ return rec[0] == "v2" and (bias is None or bias.dtype == torch.float16)+ scale_dtype = getattr(rec[2], "dtype", None)+ return (bias is None or bias.dtype == torch.bfloat16) and scale_dtype in (+ None,+ torch.bfloat16,+ torch.float32,+ )++def candidates(transformer: Any, rotated_only: bool = False) -> int:try:recs = (_eligible(m) for m in transformer.modules())@@ -1439,11 +1819,12 @@ def install(if not torch.cuda.is_available() or is_rocm_torch(torch):return 0- if not _triton_version_ok() or _kernels() is None:- return 0cap = torch.cuda.get_device_capability(torch.cuda.current_device())+ prebuilt = cutlass_wanted(cap)+ if not prebuilt and (not _triton_version_ok() or _kernels() is None):+ return 0gemm, rotq = _gates(cap)- gemm = gemm or unlisted_wanted(cap)+ gemm = gemm or unlisted_wanted(cap) or prebuiltif not (gemm or rotq):return 0except Exception: # noqa: BLE001@@ -1538,7 +1919,13 @@ def _swap_on(return 0from .diffusion_convrot_quant import rotquant_device_config+ f16_card = tuple(torch.cuda.get_device_capability(index)) == (7, 5)+ if f16_card:+ recs = [(m, r) for m, r in recs if _card_dtypes_ok(m, r, True)]+ if not recs:+ return 0gemm = device_config(index) is not None+ triton_gemm = gemmrotated = any(r[1] is not None for _, r in recs)rotq = rotated and rotquant_device_config(index) is not Noneif gemm and rotated and not rotq and index in _UNLISTED_CARDS:@@ -1559,20 +1946,51 @@ def _swap_on(t["fused"],t["stock"],)- if not gemm:- # no fused GEMM here: only rotated Linears gain (one kernel instead of rotation GEMM + act quant)- recs = [(m, r) for m, r in recs if r[1] is not None] if rotq else []if any(r[0] == "v1" for _, r in recs) and not _v1_act_quant_matches(index):recs = [(m, r) for m, r in recs if r[0] != "v1"]+ # bf16 cards: the Triton path takes any bias dtype; only the prebuilt kernel needs bf16 bias+ cut_recs = [(m, r) for m, r in recs if f16_card or _card_dtypes_ok(m, r, False)]+ plan = _cutlass_plan(index, cut_recs, triton_gemm, logger) if cut_recs else {}+ cut_fused: dict = {}+ if plan:+ import torch as _torch++ dev = _torch.device("cuda", index)+ xs_seen: dict = {}+ act = _torch.float16 if f16_card else _torch.bfloat16+ for m, r in cut_recs:+ wq, ws_ = _payload(r[0], r[3])+ k = int(wq.shape[1])+ if (r[0], k) not in xs_seen:+ xs_seen[(r[0], k)] = _xs_dtype(r[0], r[3], k, dev, act)+ cut_fused[id(m)] = _cutlass_fused(+ index,+ wq.shape[0],+ k,+ xs_seen[(r[0], k)],+ ws_.dtype,+ getattr(m, "bias", None) is not None,+ )+ gemm = gemm or any(cut_fused.values())+ if not triton_gemm:+ recs = [(m, r) for m, r in recs if (rotq and r[1] is not None) or cut_fused.get(id(m))]if not recs:return 0count = 0+ global _OP16_HANDLEwith _LOCK:- if gemm:+ if f16_card:+ _OP16_HANDLE = _op16()+ if _OP16_HANDLE is None:+ return 0+ elif gemm:_OP_HANDLE = _op()if _OP_HANDLE is None:return 0for module, rec in recs:+ fused = (triton_gemm or bool(cut_fused.get(id(module)))) and not f16_card+ if f16_card:+ module.__dict__[_F16] = Trueif _MARK in module.__dict__:if multi:module.__dict__[_REC] = (@@ -1580,7 +1998,7 @@ def _swap_on(rec[1],bool(rotq and rec[1] is not None),rec[3],- gemm,+ fused,)count += 1continue@@ -1590,7 +2008,7 @@ def _swap_on(rec[1],bool(rotq and rec[1] is not None),rec[3],- gemm,+ fused,)module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)module.forward = types.MethodType(_linear_forward, module)@@ -1605,6 +2023,16 @@ def _swap_on(if rotqelse 0)+ if logger is not None and plan:+ n_cut = sum(1 for v in cut_fused.values() if v)+ logger.info(+ "diffusion.int8_gemm: %d int8 Linear(s) on cuda:%d run the prebuilt CUTLASS GEMM (%d weight shape(s) "+ "won the race against %s)",+ n_cut,+ index,+ len(plan),+ "the Triton kernel" if triton_gemm else "cuBLAS",+ )if logger is not None and count:from .diffusion_convrot_quant import _H16_DEVICEtiming = _UNLISTED_TIMING.get(index) if gemm else None@@ -1704,6 +2132,7 @@ def uninstall(transformer: Any = None) -> None:fwd = module.__dict__.get("forward")if getattr(fwd, "__func__", None) is _swiglu_ff_forward:module.__dict__.pop("forward", None)+ module.__dict__.pop(_F16, None)prev = module.__dict__.pop(_MARK, None)if prev is None:continuediff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.pyindex 5ea074e1efa..312f8f3fd71 100644--- a/studio/backend/core/inference/diffusion_native_quant.py+++ b/studio/backend/core/inference/diffusion_native_quant.py@@ -691,8 +691,21 @@ def native_linear_class():if not self.rot_group and (torch.compiler.is_compiling() or not xq.is_contiguous()):op = getattr(self, "_row_major", None)xq = op(xq) if op is not None else xq.contiguous()- acc = torch._int_mm(xq, self.weight_q.t())w_scale = self.weight_scale.view(torch.float32)+ if (+ self.bias is None+ and x2.dtype == torch.float16+ and x2.is_cuda+ and not torch.compiler.is_compiling()+ ):+ from . import diffusion_int8_cutlass as _cut # noqa: PLC0415+ tile = _cut.native_tile(xq, self.weight_q, x_scale, w_scale)+ if tile is not None:+ try:+ return _cut.native_gemm(xq, self.weight_q, x_scale, w_scale, tile)+ except Exception: # noqa: BLE001 - the stock maths below+ _cut._NATIVE_PLAN.clear()+ acc = torch._int_mm(xq, self.weight_q.t())out = torch.mul(acc, x_scale)if self.bias is not None:out = torch.addcmul(self.bias.float(), out, w_scale)@@ -857,6 +870,16 @@ def native_linear_class():return out.reshape(*x.shape[:-1], self.out_features)def forward(self, x: Any) -> Any:+ if (+ self.act_int8+ and self.bias is None+ and x.is_cuda+ and x.dtype == torch.float16+ and not torch.compiler.is_compiling()+ ):+ # race once per card and shape, before any graph capture+ from . import diffusion_int8_cutlass as _cut # noqa: PLC0415+ _cut.native_prepare(self.weight_q, x.dtype, False)if self._int8pack_ok(x):out = self._forward_int8pack(x)if out is not None:diff --git a/studio/backend/tests/test_diffusion_int8_cutlass.py b/studio/backend/tests/test_diffusion_int8_cutlass.pynew file mode 100644index 00000000000..b17c060507e--- /dev/null+++ b/studio/backend/tests/test_diffusion_int8_cutlass.py@@ -0,0 +1,433 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Tests for the prebuilt CUTLASS int8 GEMM (``diffusion_int8_cutlass``) and its dispatch in ``diffusion_int8_gemm``.+The loader, gates, reference, plan dispatch and build generator run anywhere; the kernel tests need NVIDIA CUDA and+the built library (``UNSLOTH_DIFFUSION_INT8_CUTLASS_LIB`` or the ``unsloth-studio-i8gemm`` wheel)."""++from __future__ import annotations++import importlib.util+import os+import sys+from pathlib import Path++import pytest++from core.inference import diffusion_int8_cutlass as cut+from core.inference import diffusion_int8_gemm as g8++torch = pytest.importorskip("torch")++_BUILD = Path(__file__).resolve().parents[2] / "native_kernels" / "int8_gemm" / "build.py"+# read before the autouse fixture clears it, for the GPU tests at the bottom+_REAL_LIB = os.environ.get(cut.CUTLASS_LIB_ENV)++[email protected](autouse = True)+def _clean(monkeypatch):+ for env in (cut.CUTLASS_ENV, cut.CUTLASS_LIB_ENV, g8.INT8_GEMM_ENV):+ monkeypatch.delenv(env, raising = False)+ cut.library.cache_clear()+ cut.tiles.cache_clear()+ cut._PROBED.clear()+ cut._LOAD_ERROR[0] = None+ g8._CUT_PLAN.clear()+ g8._CUT_FAMILY.clear()+ g8._CUT_RACES.clear()+ g8._CUT_DONE.clear()+ yield+ monkeypatch.undo()+ cut.library.cache_clear()+ cut.tiles.cache_clear()+ cut._PROBED.clear()+ g8._CUT_PLAN.clear()+ g8._CUT_FAMILY.clear()+++ "cap, fam",+ [+ ((7, 5), 75),+ ((8, 0), 80),+ ((8, 6), 80),+ ((8, 7), 80),+ ((8, 9), 80),+ ((9, 0), None),+ ((10, 0), None),+ ((12, 0), None),+ ((7, 0), None),+ (None, None),+ ],+)+def test_family_covers_turing_ampere_ada_only(cap, fam):+ assert cut.family(cap) == fam+++ "raw, want", [("0", "off"), ("off", "off"), ("1", "force"), ("", "auto"), ("auto", "auto")]+)+def test_mode(monkeypatch, raw, want):+ monkeypatch.setenv(cut.CUTLASS_ENV, raw)+ assert cut.mode() == want+++def test_missing_or_broken_library_keeps_todays_path(monkeypatch, tmp_path):+ bad = tmp_path / "libunsloth_i8gemm.so"+ bad.write_bytes(b"not an ELF")+ monkeypatch.setenv(cut.CUTLASS_LIB_ENV, str(bad))+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ assert cut.library() is None+ assert cut.load_error() and str(bad) in cut.load_error()+ assert cut.tiles(80) == ()+ assert not g8.cutlass_wanted((8, 9))+++def test_kill_switch_never_loads(monkeypatch):+ monkeypatch.setenv(cut.CUTLASS_ENV, "0")+ calls = []+ monkeypatch.setattr(cut, "_candidate_paths", lambda: calls.append(1) or [])+ assert cut.library() is None+ assert calls == []+ assert not g8.cutlass_wanted((8, 0))+++def test_rocm_never_loads(monkeypatch):+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.version, "hip", "6.4", raising = False)+ monkeypatch.setattr(cut, "_candidate_paths", lambda: pytest.fail("probed on ROCm"))+ assert cut.library() is None+++def test_wheel_package_is_found_without_importing_it(monkeypatch, tmp_path):+ pkg = tmp_path / cut.WHEEL_MODULE+ (pkg / "lib").mkdir(parents = True)+ (pkg / "__init__.py").write_text("raise RuntimeError('must not be imported')\n")+ lib = pkg / "lib" / cut.LIB_NAMES[0]+ lib.write_bytes(b"x")+ monkeypatch.syspath_prepend(str(tmp_path))+ importlib.invalidate_caches()+ assert cut._candidate_paths() == [str(lib)]+ assert cut.WHEEL_MODULE not in sys.modules+++def test_explicit_path_wins(monkeypatch, tmp_path):+ monkeypatch.setenv(cut.CUTLASS_LIB_ENV, str(tmp_path / "x.so"))+ assert cut._candidate_paths() == [str(tmp_path / "x.so")]+++def test_cutlass_wanted_for_turing_ampere_ada_only(monkeypatch):+ monkeypatch.setattr(cut, "library", lambda: object())+ assert g8.cutlass_wanted((8, 9)) and g8.cutlass_wanted((8, 6)) and g8.cutlass_wanted((8, 0))+ assert g8.cutlass_wanted((7, 5))+ assert (+ not g8.cutlass_wanted((9, 0))+ and not g8.cutlass_wanted((12, 0))+ and not g8.cutlass_wanted((10, 0))+ )+ monkeypatch.setenv(g8.INT8_GEMM_ENV, "0")+ assert not g8.cutlass_wanted((8, 9))+++def _operands(+ m,+ n,+ k,+ xs_dt,+ ws_dt,+ bias,+ seed = 0,+ out = torch.bfloat16,+):+ g = torch.Generator().manual_seed(seed)+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8)+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8)+ xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(xs_dt)+ ws = (torch.rand(n, generator = g) * 0.002 + 1e-5).to(ws_dt)+ b = (torch.randn(n, generator = g) * 0.1).to(out) if bias else None+ return a, w, xs, ws, b+++def _cpu_int_mm_ok():+ try:+ torch._int_mm(torch.ones(17, 16, dtype = torch.int8), torch.ones(16, 8, dtype = torch.int8))+ return True+ except Exception: # noqa: BLE001+ return False++[email protected](not _cpu_int_mm_ok(), reason = "torch._int_mm needs CPU support")[email protected]("xs_dt", [torch.bfloat16, torch.float32])[email protected]("ws_dt", [torch.bfloat16, torch.float32])[email protected]("bias", [False, True])[email protected]("m", [3, 33])+def test_generic_reference_is_studios_bf16_reference(xs_dt, ws_dt, bias, m):+ a, w, xs, ws, b = _operands(m, 48, 64, xs_dt, ws_dt, bias)+ got = cut.reference(a, w, xs, ws, b, torch.bfloat16)+ want = g8.reference(a, w, xs, ws, b)+ assert got.dtype == torch.bfloat16+ assert torch.equal(got.view(torch.int16), want.view(torch.int16))++[email protected](not _cpu_int_mm_ok(), reason = "torch._int_mm needs CPU support")[email protected]("ws_dt", [torch.float16, torch.bfloat16, torch.float32])+def test_fp16_reference_is_the_fused_paths_torchao_epilogue(ws_dt):+ from core.inference.diffusion_int8_fused import _torchao_epilogue++ a, w, xs, ws, b = _operands(40, 32, 64, torch.float16, ws_dt, True, out = torch.float16)+ xs = xs * 0.01+ c = torch._int_mm(a, w.t())+ want = _torchao_epilogue(c, xs, ws, b, torch.float16)+ got = cut.reference(a, w, xs, ws, b, torch.float16)+ assert torch.equal(got.view(torch.int16), want.view(torch.int16))++[email protected](not _cpu_int_mm_ok(), reason = "torch._int_mm needs CPU support")+def test_op_body_runs_the_race_winner_then_falls_back(monkeypatch):+ a, w, xs, ws, b = _operands(1100, 64, 64, torch.bfloat16, torch.bfloat16, False)+ key = g8._cut_key(64, 64, xs.dtype, ws.dtype, False)+ g8._CUT_PLAN[None] = {key: {512: None, 4096: 3}}+ g8._CUT_FAMILY[None] = 80+ monkeypatch.setattr(cut, "eligible", lambda *a_: True)+ calls = []++ def fake_launch(a_, w_, xs_, ws_, b_, fam, tile, out):+ calls.append((fam, tile, a_.shape[0]))+ return torch.zeros(a_.shape[0], w_.shape[0], dtype = out)++ monkeypatch.setattr(cut, "launch", fake_launch)+ y = g8._run(a, w, xs, ws, b)+ assert calls == [(80, 3, 1100)] and not y.any()+ # small M lost its race: today's path+ small = a[:200]+ y = g8._run(small, w, xs[:200], ws, b)+ assert len(calls) == 1+ assert torch.equal(+ y.view(torch.int16), g8.reference(small, w, xs[:200], ws, b).view(torch.int16)+ )++ def broken(*a_):+ raise RuntimeError("launch failed")++ monkeypatch.setattr(cut, "launch", broken)+ y = g8._run(a, w, xs, ws, b)+ assert None not in g8._CUT_PLAN # dropped on this card+ assert torch.equal(y.view(torch.int16), g8.reference(a, w, xs, ws, b).view(torch.int16))+++def test_fused_needs_the_large_m_win():+ key = g8._cut_key(64, 128, torch.bfloat16, torch.float32, True)+ g8._CUT_PLAN[0] = {key: {512: 2, 4096: None}}+ assert not g8._cutlass_fused(0, 64, 128, torch.bfloat16, torch.float32, True)+ g8._CUT_PLAN[0] = {key: {512: None, 4096: 1}}+ assert g8._cutlass_fused(0, 64, 128, torch.bfloat16, torch.float32, True)+ assert not g8._cutlass_fused(+ 0, 64, 128, torch.bfloat16, torch.float32, False+ ) # bias is part of the key+ assert not g8._cutlass_fused(1, 64, 128, torch.bfloat16, torch.float32, True) # per card+++def test_race_margin_keeps_todays_path_on_a_tie(monkeypatch):+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda i = None: (8, 9))+ monkeypatch.setattr(cut, "probed_tiles", lambda i: (0, 1))+ monkeypatch.setattr(cut, "has_kernel", lambda *a: True)+ monkeypatch.setattr(cut, "eligible", lambda *a: True)+ monkeypatch.setattr(torch.cuda, "device", lambda d: __import__("contextlib").nullcontext())+ results = {}++ def fake_time(+ fns,+ rounds = 3,+ reps = 3,+ warm = 1,+ ):+ return {k: results[k] for k in fns}++ monkeypatch.setattr(cut, "time_us", fake_time)+ w = torch.zeros(64, 64, dtype = torch.int8)+ ws = torch.ones(64, dtype = torch.bfloat16)+ results.update({0: 100.0, 1: 99.0, "today": 100.0})+ r = cut.race(0, w, ws, None, torch.bfloat16, lambda *a: None, 64)+ assert r["tile"] is None # 1% faster is inside the margin+ results.update({0: 90.0, 1: 97.0, "today": 100.0})+ r = cut.race(0, w, ws, None, torch.bfloat16, lambda *a: None, 64)+ assert r["tile"] == 0+++def _load_build():+ spec = importlib.util.spec_from_file_location("i8gemm_build", _BUILD)+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ return mod+++def test_build_generator_instantiates_every_variant_per_tile():+ b = _load_build()+ for name, fam in b.FAMILIES.items():+ n_var = len(b.variants(fam))+ native = len(fam["out"]) if fam.get("native") else 0+ assert n_var == len(fam["out"]) * len(fam["xs"]) * len(fam["ws"]) * 2 + native+ for i in range(len(fam["tiles"])):+ src = b.tile_tu(name, fam, i)+ assert src.count("return &ui8::FusedGemm<") == n_var+ assert f"ui8_{name}_t{i}(" in src+ api = b.api_tu()+ for sym in ("ui8_abi_version", "ui8_num_tiles", "ui8_tile_info", "ui8_has_kernel", "ui8_gemm"):+ assert f"UI8_API int {sym}(" in api+ assert f"return {cut.ABI_VERSION};" in api # loader and library agree on the ABI+ assert "int epilogue" in api+ assert b.FAMILIES["sm75"].get("native") and not b.FAMILIES["sm80"].get("native")+ assert any(e == cut.EPI_NATIVE for *_, e in b.variants(b.FAMILIES["sm75"]))+++def test_python_kinds_match_the_library_kinds():+ b = _load_build()+ assert cut._kind(torch.float32) == b.KIND["float"]+ assert cut._kind(torch.bfloat16) == b.KIND["cutlass::bfloat16_t"]+ assert cut._kind(torch.float16) == b.KIND["cutlass::half_t"]+ assert set(b.FAMILY_ID) == set(b.FAMILIES) and set(b.FAMILY_ID.values()) == {75, 80}+ assert cut.family((7, 5)) == b.FAMILY_ID["sm75"] and cut.family((8, 9)) == b.FAMILY_ID["sm80"]++[email protected](not _cpu_int_mm_ok(), reason = "torch._int_mm needs CPU support")+def test_native_reference_is_the_native_linears_epilogue():+ a, w, _xs, _ws, _ = _operands(40, 32, 64, torch.float32, torch.float32, False)+ xs = torch.rand(40, 1) * 1e-3 + 1e-6+ ws = torch.rand(32) * 1e-3 + 1e-6+ acc = torch._int_mm(a, w.t())+ want = (+ torch.mul(acc, xs).mul_(ws).to(torch.float16)+ ) # NativeWeightOnlyLinear._int_mm_rows, no bias+ got = cut.reference_native(a, w, xs, ws, torch.float16)+ assert torch.equal(got.view(torch.int16), want.view(torch.int16))+++def test_native_plan_needs_fp16_no_bias_and_a_won_race(monkeypatch):+ w = torch.zeros(64, 64, dtype = torch.int8)+ cut._NATIVE_PLAN.clear()+ cut.native_prepare(w, torch.float16, False) # CPU weight: nothing raced+ cut.native_prepare(w, torch.bfloat16, False)+ assert cut._NATIVE_PLAN == {}+ cut._NATIVE_PLAN[(None, 64, 64)] = {512: None, 4096: 2}+ monkeypatch.setattr(cut, "eligible", lambda *a: True)+ xs = torch.ones(2000, 1)+ assert cut.native_tile(torch.zeros(2000, 64, dtype = torch.int8), w, xs, torch.ones(64)) == 2+ assert (+ cut.native_tile(torch.zeros(100, 64, dtype = torch.int8), w, xs[:100], torch.ones(64)) is None+ )+ assert (+ cut.native_tile(torch.zeros(2000, 64, dtype = torch.int8), w, xs.half(), torch.ones(64))+ is None+ )+ cut._NATIVE_PLAN.clear()+++# ------------------------------------------------------------------------------------------------- GPU (needs the lib)++_GPU = pytest.mark.skipif(+ not (torch.cuda.is_available() and _REAL_LIB),+ reason = "needs CUDA and UNSLOTH_DIFFUSION_INT8_CUTLASS_LIB",+)+++def real_lib(monkeypatch):+ monkeypatch.setenv(cut.CUTLASS_LIB_ENV, _REAL_LIB)+ cut.library.cache_clear()+ cut.tiles.cache_clear()+ cut._PROBED.clear()+++@_GPU+def test_library_probe_and_bit_exact_launch(real_lib):+ cap = torch.cuda.get_device_capability(0)+ fam = cut.family(cap)+ if fam is None:+ pytest.skip("arch outside the library")+ assert cut.library() is not None, cut.load_error()+ ok = cut.probed_tiles(0)+ assert ok, "no tile passed the bit-exact probe"+ out = cut.out_dtype_for(fam)+ for xs_dt, ws_dt in cut._scale_dtypes(fam):+ for bias in (False, True):+ a, w, xs, ws, b = _operands(1031, 528, 1040, xs_dt, ws_dt, bias, out = out)+ if fam == 75:+ xs = xs * 0.01+ a, w, xs, ws = a.cuda(), w.cuda(), xs.cuda(), ws.cuda()+ b = b.cuda() if b is not None else None+ ref = cut.reference(a, w, xs, ws, b, out)+ for tile in ok:+ y = cut.launch(a, w, xs, ws, b, fam, tile, out)+ assert torch.equal(y.view(torch.int16), ref.view(torch.int16)), (+ tile,+ xs_dt,+ ws_dt,+ bias,+ )+ if fam == 75: # the native W8A8 Linear's epilogue+ a, w, _x, _w, _ = _operands(1031, 528, 1040, torch.float32, torch.float32, False, out = out)+ a, w = a.cuda(), w.cuda()+ xs = (torch.rand(1031) * 1e-4 + 1e-6).cuda()+ ws = (torch.rand(528) * 1e-3 + 1e-6).cuda()+ ref = cut.reference_native(a, w, xs, ws, out)+ for tile in ok:+ y = cut.launch(a, w, xs, ws, None, fam, tile, out, cut.EPI_NATIVE)+ assert torch.equal(y.view(torch.int16), ref.view(torch.int16)), tile+++@_GPU+def test_launch_records_into_a_cuda_graph(real_lib):+ fam = cut.family(torch.cuda.get_device_capability(0))+ if fam is None or not cut.probed_tiles(0):+ pytest.skip("no probed tile")+ out = cut.out_dtype_for(fam)+ xs_dt, ws_dt = cut._scale_dtypes(fam)[0]+ a, w, xs, ws, _ = _operands(256, 256, 256, xs_dt, ws_dt, False, out = out)+ a, w, xs, ws = a.cuda(), w.cuda(), (xs * 0.01).cuda(), ws.cuda()+ tile = cut.probed_tiles(0)[0]+ cut.launch(a, w, xs, ws, None, fam, tile, out)+ torch.cuda.synchronize()+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ y = cut.launch(a, w, xs, ws, None, fam, tile, out)+ a.copy_(torch.randint(-127, 128, a.shape, dtype = torch.int8, device = "cuda"))+ graph.replay()+ torch.cuda.synchronize()+ assert torch.equal(+ y.view(torch.int16), cut.reference(a, w, xs, ws, None, out).view(torch.int16)+ )+++def test_a_later_load_plans_only_its_new_shapes(monkeypatch):+ class _W:+ def __init__(self, n, k):+ self.qdata = torch.zeros(n, k, dtype = torch.int8)+ self.scale = torch.ones(n, 1, dtype = torch.bfloat16)++ def _rec(n, k):+ return (torch.nn.Linear(k, n, bias = False), ("v2", None, None, _W(n, k)))++ a, b = _rec(64, 32), _rec(128, 64)+ key_a = g8._cut_key(64, 32, torch.float32, torch.bfloat16, False)+ key_b = g8._cut_key(128, 64, torch.float32, torch.bfloat16, False)+ table = {"ck": {repr(key_a): {"512": 1, "4096": 1}, repr(key_b): {"512": 2, "4096": 2}}}+ monkeypatch.setattr(g8, "cutlass_wanted", lambda cap: True)+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: (8, 0))+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)+ monkeypatch.setattr(cut, "probed_tiles", lambda index: (1, 2))+ monkeypatch.setattr(g8, "_cut_cache_key", lambda index: "ck")+ monkeypatch.setattr(g8, "_cut_cache_read", lambda: table)+ monkeypatch.setattr(g8, "_cut_cache_write", lambda t: None)+ monkeypatch.setattr(g8, "_xs_dtype", lambda *a_: torch.float32)+ races = []+ monkeypatch.setattr(cut, "race", lambda *a_, **k_: races.append(a_) or {"tile": None})++ assert set(g8._cutlass_plan(0, [a], True)) == {key_a}+ assert set(g8._cutlass_plan(0, [a, b], True)) == {key_a, key_b}+ assert g8._CUT_PLAN[0][key_b] == {512: 2, 4096: 2} and not races+++def test_the_kernel_sources_do_not_shadow_the_kernels_package():+ # CI and Studio put studio/ on sys.path, where a studio/kernels folder imports as the HF ``kernels`` package.+ studio = Path(__file__).resolve().parents[2]+ assert _BUILD.is_file() and not (studio / "kernels").exists()diff --git a/studio/backend/utils/wheel_utils.py b/studio/backend/utils/wheel_utils.pyindex 6d644e204fe..829b5af514d 100644--- a/studio/backend/utils/wheel_utils.py+++ b/studio/backend/utils/wheel_utils.py@@ -169,6 +169,25 @@ MAMBA_SSM_PURE_VERSION = "2.3.2.post1"MAMBA_SSM_PURE_WHEEL = f"mamba_ssm-{MAMBA_SSM_PURE_VERSION}-py3-none-any.whl"+# studio/native_kernels/int8_gemm: C ABI + static cudart, so one py3-none wheel per platform fits every torch and CUDA 12/13.+I8GEMM_DISTRIBUTION = "unsloth-studio-i8gemm"+I8GEMM_VERSION = "1.0.0"+_I8GEMM_PLATFORMS = {"linux_x86_64": "manylinux_2_28_x86_64", "win_amd64": "win_amd64"}+++def int8_gemm_wheel_url(env: dict[str, str] | None) -> str | None:+ """The prebuilt int8 GEMM wheel for an NVIDIA CUDA torch on x86_64 Linux / Windows, else None."""+ if env is None or env.get("hip_version") or not env.get("cuda_major"):+ return None+ platform = _I8GEMM_PLATFORMS.get(str(env.get("platform_tag") or ""))+ if platform is None:+ return None+ return (+ f"{UNSLOTH_PREBUILT_RELEASE_BASE_URL}/{UNSLOTH_PREBUILT_RELEASE_TAG}/"+ f"unsloth_studio_i8gemm-{I8GEMM_VERSION}-py3-none-{platform}.whl"+ )++def mamba_ssm_pure_wheel_url(env: dict[str, str] | None = None) -> str:"""The extension-free mamba_ssm wheel (the same file for every environment)."""return (diff --git a/studio/install_python_stack.py b/studio/install_python_stack.pyindex b54a1c59b76..6e55201023e 100644--- a/studio/install_python_stack.py+++ b/studio/install_python_stack.py@@ -61,8 +61,11 @@ from backend.utils.kernel_install import (uninstall_command,)from backend.utils.wheel_utils import (+ I8GEMM_DISTRIBUTION,+ I8GEMM_VERSION,flash_attn_package_version,flash_attn_wheel_url,+ int8_gemm_wheel_url,install_wheel,probe_torch_wheel_env,url_exists,@@ -7887,6 +7890,49 @@ def _remove_stale_flash_attn() -> None:)+def _int8_gemm_install_disabled() -> bool:+ if os.getenv("UNSLOTH_STUDIO_SKIP_I8GEMM_INSTALL") == "1":+ return True+ return (os.getenv("UNSLOTH_DIFFUSION_INT8_CUTLASS") or "").strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ )+++def _ensure_int8_gemm() -> None:+ """Optional prebuilt int8 GEMM wheel (diffusion INT8, sm75 / sm8x); every failure only warns."""+ if _int8_gemm_install_disabled() or NO_TORCH or IS_MACOS:+ return+ if _installed_distribution_version(I8GEMM_DISTRIBUTION) == I8GEMM_VERSION:+ return+ env = probe_torch_wheel_env(include_windows = True)+ wheel_url = int8_gemm_wheel_url(env)+ if not wheel_url or not url_exists(wheel_url):+ return+ _count_install_action()+ with _install_heartbeat("int8 GEMM"):+ outcome = install_prebuilt(+ wheel_url,+ install = install_wheel,+ verify = lambda: _installed_distribution_version(I8GEMM_DISTRIBUTION) == I8GEMM_VERSION,+ on_failed = lambda installer, wheel_result: _print_optional_install_failure(+ f"Installing the prebuilt int8 GEMM wheel with {installer}",+ wheel_result,+ ),+ use_uv = USE_UV,+ uv_needs_system = UV_NEEDS_SYSTEM,+ reinstall = _installed_distribution_version(I8GEMM_DISTRIBUTION) is not None,+ )+ if outcome != "installed":+ _step(+ "warning",+ "Continuing without the prebuilt int8 GEMM (diffusion INT8 keeps Triton / cuBLAS)",+ _warn,+ )++def _flash_attn_gpu_below_floor() -> bool:"""Visible CUDA GPU below sm80 (a T4); no visible GPU is False, since image builds install first."""return gpu_below_flash_attn_floor()@@ -12181,6 +12227,7 @@ def install_python_stack() -> int:_progress("flash-attn")_ensure_flash_attn()_remove_stale_ssm_kernels()+ _ensure_int8_gemm()# 8. Unsloth dependenciesif not _skip_step(REQ_ROOT / "studio.txt", "Unsloth Studio deps", no_deps = False):diff --git a/studio/native_kernels/int8_gemm/build.py b/studio/native_kernels/int8_gemm/build.pynew file mode 100644index 00000000000..465dde49a7d--- /dev/null+++ b/studio/native_kernels/int8_gemm/build.py@@ -0,0 +1,302 @@+#!/usr/bin/env python3+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Build the prebuilt int8 GEMM library, one nvcc unit per (family, tile), static CUDA runtime.++ python build.py --cutlass /path/to/cutlass --out build/ [--jobs 16] [--families sm80,sm75]++SASS only (no PTX): other archs keep Studio's other paths instead of a JIT build.+"""++from __future__ import annotations++import argparse+import concurrent.futures+import os+import shutil+import subprocess+import sys+from pathlib import Path++HERE = Path(__file__).resolve().parent+ABI_VERSION = 1++# (TBM, TBN, TBK, WM, WN, WK, stages, swizzle). The index is the config id the loader races; append only.+FAMILIES = {+ "sm80": {+ "arch_tag": "cutlass::arch::Sm80",+ "gencode": ("80", "86", "89"),+ "out": ("cutlass::bfloat16_t",),+ "xs": ("cutlass::bfloat16_t", "float"),+ "ws": ("cutlass::bfloat16_t", "float"),+ "tiles": (+ (128, 256, 64, 64, 64, 64, 3, 1),+ (256, 128, 64, 64, 64, 64, 3, 1),+ (128, 128, 64, 64, 64, 64, 4, 1),+ (128, 128, 128, 64, 64, 128, 3, 1),+ (64, 128, 64, 32, 64, 64, 4, 1),+ (128, 256, 64, 64, 64, 64, 3, 4),+ (128, 128, 64, 64, 64, 64, 4, 4),+ (64, 64, 64, 32, 32, 64, 5, 1),+ (256, 128, 64, 64, 64, 64, 3, 4),+ # 96 KB of stages fits sm86 / sm89's 99 KB+ (128, 256, 64, 64, 64, 64, 4, 1),+ (256, 128, 64, 64, 64, 64, 4, 1),+ (128, 128, 64, 64, 64, 64, 6, 1),+ (128, 256, 64, 64, 64, 64, 4, 4),+ (256, 128, 128, 64, 64, 128, 2, 1),+ (128, 256, 128, 64, 64, 128, 2, 1),+ ),+ },+ "sm75": {+ "arch_tag": "cutlass::arch::Sm75",+ "gencode": ("75",),+ "out": ("cutlass::half_t",),+ "xs": ("cutlass::half_t", "float"),+ "ws": ("cutlass::half_t", "cutlass::bfloat16_t", "float"),+ "native": True,+ "tiles": (+ (128, 256, 64, 64, 64, 64, 2, 1),+ (256, 128, 64, 64, 64, 64, 2, 1),+ (128, 128, 64, 64, 64, 64, 2, 1),+ (64, 128, 64, 32, 64, 64, 2, 1),+ (128, 256, 64, 64, 64, 64, 2, 4),+ (128, 128, 64, 64, 64, 64, 2, 4),+ (64, 64, 64, 32, 32, 64, 2, 1),+ ),+ },+}++KIND = {"float": 0, "cutlass::bfloat16_t": 1, "cutlass::half_t": 2}+FAMILY_ID = {"sm80": 80, "sm75": 75}+++def variants(fam: dict) -> list:+ """(out, xs, ws, bias, epilogue); epilogue 0 = torchao's, 1 = the native W8A8 Linear's."""+ out = [+ (o, x, w, b, 0)+ for o in fam["out"]+ for x in fam["xs"]+ for w in fam["ws"]+ for b in (False, True)+ ]+ if fam.get("native"):+ out += [(o, "float", "float", False, 1) for o in fam["out"]]+ return out+++def tile_tu(name: str, fam: dict, index: int) -> str:+ t = fam["tiles"][index]+ lines = [+ '#include "i8gemm_impl.cuh"',+ "namespace {",+ f"using T = ui8::Tile<{', '.join(str(v) for v in t)}>;",+ "}",+ f'extern "C" ui8::RunFn ui8_{name}_t{index}(int out, int xs, int ws, int bias, int epi) {{',+ ]+ for o, x, w, b, e in variants(fam):+ lines.append(+ f" if (out == {KIND[o]} && xs == {KIND[x]} && ws == {KIND[w]} && bias == {int(b)} && epi == {e}) "+ f"return &ui8::FusedGemm<{fam['arch_tag']}, T, {o}, {x}, {w}, {str(b).lower()}, {e}>::run;"+ )+ lines += [" return nullptr;", "}", ""]+ return "\n".join(lines)+++def api_tu() -> str:+ decls, tables = [], []+ for name, fam in FAMILIES.items():+ n = len(fam["tiles"])+ for i in range(n):+ decls.append(f'extern "C" ui8::RunFn ui8_{name}_t{i}(int, int, int, int, int);')+ tables.append(+ f"static ui8::RunFn (*const k_{name}[])(int, int, int, int, int) = {{"+ + ", ".join(f"&ui8_{name}_t{i}" for i in range(n))+ + "};"+ )+ tables.append(+ f"static const int k_{name}_tiles[][8] = {{"+ + ", ".join("{" + ", ".join(str(v) for v in t) + "}" for t in fam["tiles"])+ + "};"+ )+ fam_cases = "\n".join(+ f" case {FAMILY_ID[name]}: *table = k_{name}; *count = {len(fam['tiles'])}; *tiles = k_{name}_tiles; break;"+ for name, fam in FAMILIES.items()+ )+ return f"""#include "i8gemm_impl.cuh"+#if defined(_WIN32)+#define UI8_API __declspec(dllexport)+#else+#define UI8_API __attribute__((visibility("default")))+#endif+{chr(10).join(decls)}+{chr(10).join(tables)}++static bool family_table(int family, ui8::RunFn (*const** table)(int, int, int, int, int), int* count,+ const int (**tiles)[8]) {{+ switch (family) {{+{fam_cases}+ default: return false;+ }}+ return true;+}}++extern "C" {{+UI8_API int ui8_abi_version() {{ return {ABI_VERSION}; }}++// Number of tiles for a family (80 or 75), 0 when absent.+UI8_API int ui8_num_tiles(int family) {{+ ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;+ int count = 0;+ const int (*tiles)[8] = nullptr;+ return family_table(family, &table, &count, &tiles) ? count : 0;+}}++// The 8 tile parameters (TBM, TBN, TBK, WM, WN, WK, stages, swizzle) into out8; 0 on success.+UI8_API int ui8_tile_info(int family, int tile, int* out8) {{+ ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;+ int count = 0;+ const int (*tiles)[8] = nullptr;+ if (!family_table(family, &table, &count, &tiles) || tile < 0 || tile >= count || out8 == nullptr) return 1;+ for (int i = 0; i < 8; ++i) out8[i] = tiles[tile][i];+ return 0;+}}++// 1 when (family, tile, out, xs, ws, bias, epilogue) is compiled in; epilogue 0 torchao, 1 native W8A8.+UI8_API int ui8_has_kernel(int family, int tile, int out_kind, int xs_kind, int ws_kind, int has_bias, int epilogue) {{+ ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;+ int count = 0;+ const int (*tiles)[8] = nullptr;+ if (!family_table(family, &table, &count, &tiles) || tile < 0 || tile >= count) return 0;+ return table[tile](out_kind, xs_kind, ws_kind, has_bias ? 1 : 0, epilogue) != nullptr;+}}++// out[M, N] = dequant(a[M, K] @ w[N, K]^T); kinds: 0 fp32, 1 bf16, 2 fp16. bias may be null. Status code returned.+UI8_API int ui8_gemm(int family, int tile, int out_kind, int xs_kind, int ws_kind, int epilogue, const void* a, const void* w,+ const void* xs, const void* ws, const void* bias, void* out, int64_t M, int64_t N, int64_t K,+ int64_t lda, int64_t ldw, int64_t ldc, void* stream) {{+ ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;+ int count = 0;+ const int (*tiles)[8] = nullptr;+ if (!family_table(family, &table, &count, &tiles) || tile < 0 || tile >= count) return ui8::kNoKernel;+ if (M <= 0 || N <= 0 || K <= 0 || M > 0x7fffffff || N > 0x7fffffff || K > 0x7fffffff) return ui8::kBadArgs;+ if (a == nullptr || w == nullptr || xs == nullptr || ws == nullptr || out == nullptr) return ui8::kBadArgs;+ if (lda < K || ldw < K || ldc < N) return ui8::kBadArgs;+ ui8::RunFn fn = table[tile](out_kind, xs_kind, ws_kind, bias != nullptr ? 1 : 0, epilogue);+ if (fn == nullptr) return ui8::kNoKernel;+ return fn(a, w, xs, ws, bias, out, int(M), int(N), int(K), lda, ldw, ldc, static_cast<cudaStream_t>(stream));+}}+}}+"""+++def lib_name() -> str:+ return "unsloth_i8gemm.dll" if os.name == "nt" else "libunsloth_i8gemm.so"+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--cutlass", required = True, help = "CUTLASS checkout (its include/ is used)")+ ap.add_argument("--out", required = True)+ ap.add_argument("--nvcc", default = shutil.which("nvcc") or "nvcc")+ ap.add_argument("--jobs", type = int, default = os.cpu_count() or 4)+ ap.add_argument("--families", default = ",".join(FAMILIES))+ ap.add_argument(+ "--tiles", default = "", help = "comma separated tile ids to keep (default all; testing only)"+ )+ ap.add_argument(+ "--sm",+ default = "",+ help = "comma separated SASS targets to keep, e.g. 89 (default all; testing only)",+ )+ args = ap.parse_args()++ out = Path(args.out).resolve()+ src = out / "src"+ obj = out / "obj"+ src.mkdir(parents = True, exist_ok = True)+ obj.mkdir(parents = True, exist_ok = True)+ shutil.copy2(HERE / "i8gemm_impl.cuh", src / "i8gemm_impl.cuh")+ keep_fams = [f for f in args.families.split(",") if f]+ for name in list(FAMILIES):+ if name not in keep_fams:+ del FAMILIES[name]+ if args.sm:+ want = set(args.sm.split(","))+ for name in list(FAMILIES):+ FAMILIES[name]["gencode"] = tuple(sm for sm in FAMILIES[name]["gencode"] if sm in want)+ if not FAMILIES[name]["gencode"]:+ del FAMILIES[name]+ if args.tiles:+ keep = [int(t) for t in args.tiles.split(",")]+ for fam in FAMILIES.values():+ fam["tiles"] = tuple(t for i, t in enumerate(fam["tiles"]) if i in keep)++ include = Path(args.cutlass).resolve() / "include"+ common = [+ args.nvcc,+ "-std=c++17",+ "-O3",+ "-DNDEBUG",+ "--expt-relaxed-constexpr",+ f"-I{src}",+ f"-I{include}",+ "-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1",+ ]+ if os.name == "nt":+ # static MSVC runtime (no DLL dependency), the conforming preprocessor CCCL needs+ common += ["-Xcompiler", "/MT", "-Xcompiler", "/Zc:preprocessor", "--diag-suppress=177"]+ else:+ common += ["-Xcompiler", "-fPIC", "-Xcompiler", "-fvisibility=hidden"]+ units = []+ for name, fam in FAMILIES.items():+ gencode = []+ for sm in fam["gencode"]:+ gencode += ["-gencode", f"arch=compute_{sm},code=sm_{sm}"]+ for i in range(len(fam["tiles"])):+ path = src / f"{name}_t{i}.cu"+ path.write_text(tile_tu(name, fam, i), encoding = "utf-8")+ units.append((path, gencode))+ api = src / "i8gemm_api.cu"+ api.write_text(api_tu(), encoding = "utf-8")+ all_gencode = []+ for fam in FAMILIES.values():+ for sm in fam["gencode"]:+ all_gencode += ["-gencode", f"arch=compute_{sm},code=sm_{sm}"]+ units.append((api, all_gencode))++ def compile_one(unit):+ path, gencode = unit+ o = obj / (path.stem + (".obj" if os.name == "nt" else ".o"))+ cmd = common + gencode + ["-c", str(path), "-o", str(o)]+ res = subprocess.run(cmd, capture_output = True, text = True)+ return path.name, res.returncode, res.stdout + res.stderr, o++ objs = []+ failed = False+ with concurrent.futures.ThreadPoolExecutor(max_workers = max(1, args.jobs)) as pool:+ for name, rc, log, o in pool.map(compile_one, units):+ print(f"[{'ok' if rc == 0 else 'FAIL'}] {name}", flush = True)+ if rc != 0:+ failed = True+ print(log[-6000:], flush = True)+ objs.append(str(o))+ if failed:+ return 1+ lib = out / lib_name()+ link = [args.nvcc, "-shared", "-cudart", "static", "-o", str(lib)] + objs + all_gencode+ if os.name != "nt":+ # libstdc++ / libgcc static: the library then needs only glibc from the host+ link += ["-Xcompiler", "-static-libstdc++", "-Xcompiler", "-static-libgcc"]+ link += ["-Xlinker", "--exclude-libs,ALL", "-Xlinker", "-z,defs"]+ res = subprocess.run(link, capture_output = True, text = True)+ if res.returncode != 0:+ print(res.stdout + res.stderr)+ return 1+ print(f"built {lib} ({lib.stat().st_size / 1e6:.1f} MB)")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/studio/native_kernels/int8_gemm/i8gemm_impl.cuh b/studio/native_kernels/int8_gemm/i8gemm_impl.cuhnew file mode 100644index 00000000000..d8a253cf298--- /dev/null+++ b/studio/native_kernels/int8_gemm/i8gemm_impl.cuh@@ -0,0 +1,215 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+//+// s8 GEMM with torchao's int8 dequant epilogue fused bit for bit: __fmul_rn / __fadd_rn (no FMA contraction),+// every PyTorch cast reproduced. A [M, K], W [N, K], out [M, N], all row-major.++#pragma once++#include <cuda_bf16.h>+#include <cuda_fp16.h>+#include <cuda_runtime.h>++#include <cstdint>+#include <type_traits>++// clang-format off: the visitor headers need the GEMM headers first+#include "cutlass/cutlass.h"+#include "cutlass/gemm/device/gemm_universal_adapter.h"+#include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"+#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"+// clang-format on++namespace ui8 {++using namespace cute;++// shared with the Python loader+enum Status : int {+ kOk = 0,+ kBadArgs = 1,+ kNoKernel = 2,+ kCannotImplement = 3,+ kInitFailed = 4,+ kLaunchFailed = 5,+};++enum Kind : int { kFp32 = 0, kBf16 = 1, kFp16 = 2 };++template <class T>+CUTLASS_DEVICE float round_to(float x);+template <>+CUTLASS_DEVICE float round_to<cutlass::bfloat16_t>(float x) {+ return __bfloat162float(__float2bfloat16_rn(x));+}+template <>+CUTLASS_DEVICE float round_to<cutlass::half_t>(float x) {+ return __half2float(__float2half_rn(x));+}+template <>+CUTLASS_DEVICE float round_to<float>(float x) {+ return x;+}++template <class Out, class XsT, class WsT, bool Bias>+struct Dequant {+ // torch promotes int32 * bf16 to bf16; torchao widens an fp16 x_scale to fp32+ static constexpr bool kBf16Inter = std::is_same<XsT, cutlass::bfloat16_t>::value;+ static constexpr bool kRoundWs = std::is_same<WsT, Out>::value;++ CUTLASS_DEVICE static float scaled(float c, float xs, float ws) {+ float t = kBf16Inter ? round_to<Out>(__fmul_rn(round_to<Out>(c), xs)) : round_to<Out>(__fmul_rn(c, xs));+ float y = __fmul_rn(t, ws);+ return kRoundWs ? round_to<Out>(y) : y;+ }++ template <class T>+ struct Fn;++ template <int N>+ struct Fn<cutlass::Array<float, N>> {+ using Frag = cutlass::Array<float, N>;++ CUTLASS_DEVICE Frag operator()(Frag const& c, Frag const& xs, Frag const& ws) const {+ Frag y;+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < N; ++i) {+ y[i] = round_to<Out>(scaled(c[i], xs[i], ws[i]));+ }+ return y;+ }++ CUTLASS_DEVICE Frag operator()(Frag const& c, Frag const& xs, Frag const& ws, Frag const& b) const {+ Frag y;+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < N; ++i) {+ y[i] = round_to<Out>(__fadd_rn(scaled(c[i], xs[i], ws[i]), b[i]));+ }+ return y;+ }+ };+};++// diffusion_native_quant W8A8 Linear: fp32 epilogue, one cast, no bias+template <class Out>+struct NativeDequant {+ template <class T>+ struct Fn;++ template <int N>+ struct Fn<cutlass::Array<float, N>> {+ using Frag = cutlass::Array<float, N>;++ CUTLASS_DEVICE Frag operator()(Frag const& c, Frag const& xs, Frag const& ws) const {+ Frag y;+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < N; ++i) {+ y[i] = round_to<Out>(__fmul_rn(__fmul_rn(c[i], xs[i]), ws[i]));+ }+ return y;+ }+ };+};++enum Epilogue : int { kTorchao = 0, kNative = 1 };++template <class Out, class XsT, class WsT, bool Bias, int Epi>+struct EpiFn;+template <class Out, class XsT, class WsT, bool Bias>+struct EpiFn<Out, XsT, WsT, Bias, kTorchao> {+ template <class T>+ using Fn = typename Dequant<Out, XsT, WsT, Bias>::template Fn<T>;+};+template <class Out, class XsT, class WsT, bool Bias>+struct EpiFn<Out, XsT, WsT, Bias, kNative> {+ static_assert(!Bias && std::is_same<XsT, float>::value && std::is_same<WsT, float>::value,+ "native epilogue: fp32 scales, no bias");+ template <class T>+ using Fn = typename NativeDequant<Out>::template Fn<T>;+};++template <int TBM_, int TBN_, int TBK_, int WM_, int WN_, int WK_, int Stages_, int Swizzle_ = 1>+struct Tile {+ static constexpr int TBM = TBM_, TBN = TBN_, TBK = TBK_;+ static constexpr int WM = WM_, WN = WN_, WK = WK_;+ static constexpr int Stages = Stages_;+ static constexpr int Swizzle = Swizzle_;+};++template <class ArchTag>+struct ArchTraits;+template <>+struct ArchTraits<cutlass::arch::Sm80> {+ using Inst = cutlass::gemm::GemmShape<16, 8, 32>;+};+template <>+struct ArchTraits<cutlass::arch::Sm75> {+ using Inst = cutlass::gemm::GemmShape<8, 8, 16>;+};++template <class ArchTag, class T, class Out, class XsT, class WsT, bool Bias, int Epi = kTorchao>+struct FusedGemm {+ using ElementAcc = int32_t;+ using ElementCompute = float;+ static constexpr int kAlignAB = 16;+ static constexpr int kAlignC = 128 / cutlass::sizeof_bits<Out>::value;+ using TB = cutlass::gemm::GemmShape<T::TBM, T::TBN, T::TBK>;+ using Warp = cutlass::gemm::GemmShape<T::WM, T::WN, T::WK>;+ using Inst = typename ArchTraits<ArchTag>::Inst;+ static constexpr int kEVTStages = 1;++ using ThreadMap =+ cutlass::epilogue::threadblock::OutputTileThreadLayout<TB, Warp, Out, kAlignC, kEVTStages>;+ using Acc = cutlass::epilogue::threadblock::VisitorAccFetch;+ using XScale = cutlass::epilogue::threadblock::VisitorColBroadcast<ThreadMap, XsT, Stride<_1, _0, int32_t>>;+ using WScale = cutlass::epilogue::threadblock::VisitorRowBroadcast<ThreadMap, WsT, Stride<_0, _1, int32_t>>;+ using BiasRow = cutlass::epilogue::threadblock::VisitorRowBroadcast<ThreadMap, Out, Stride<_0, _1, int32_t>>;+ using Compute = cutlass::epilogue::threadblock::VisitorCompute<EpiFn<Out, XsT, WsT, Bias, Epi>::template Fn, float,+ float, cutlass::FloatRoundStyle::round_to_nearest>;+ using Body = typename std::conditional<Bias, cutlass::epilogue::threadblock::Sm80EVT<Compute, Acc, XScale, WScale, BiasRow>,+ cutlass::epilogue::threadblock::Sm80EVT<Compute, Acc, XScale, WScale>>::type;+ using Store = cutlass::epilogue::threadblock::VisitorAuxStore<+ ThreadMap, Out, cutlass::FloatRoundStyle::round_to_nearest, Stride<int64_t, _1, int64_t>>;+ using EVT = cutlass::epilogue::threadblock::Sm80EVT<Store, Body>;++ using Swizzle = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<T::Swizzle>;++ using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmWithVisitor<+ int8_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, kAlignAB, int8_t,+ cutlass::layout::ColumnMajor, cutlass::ComplexTransform::kNone, kAlignAB, Out, cutlass::layout::RowMajor,+ kAlignC, ElementAcc, ElementCompute, cutlass::arch::OpClassTensorOp, ArchTag, TB, Warp, Inst, EVT, Swizzle,+ T::Stages, cutlass::arch::OpMultiplyAddSaturate, kEVTStages>::GemmKernel;+ using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;++ static typename Body::Arguments body_args(const void* xs, const void* ws, const void* bias, int M, int N) {+ typename XScale::Arguments xa{static_cast<XsT const*>(xs), XsT(0), {_1{}, _0{}, M}};+ typename WScale::Arguments wa{static_cast<WsT const*>(ws), WsT(0), {_0{}, _1{}, N}};+ if constexpr (Bias) {+ typename BiasRow::Arguments ba{static_cast<Out const*>(bias), Out(0), {_0{}, _1{}, N}};+ return typename Body::Arguments{{}, xa, wa, ba, {}};+ } else {+ return typename Body::Arguments{{}, xa, wa, {}};+ }+ }++ static int run(const void* a, const void* w, const void* xs, const void* ws, const void* bias, void* out, int M,+ int N, int K, int64_t lda, int64_t ldw, int64_t ldc, cudaStream_t stream) {+ typename EVT::Arguments cb{body_args(xs, ws, bias, M, N),+ {static_cast<Out*>(out), {ldc, _1{}, int64_t(M) * ldc}}};+ cutlass::gemm::GemmCoord problem(M, N, K);+ typename Gemm::Arguments args(cutlass::gemm::GemmUniversalMode::kGemm, problem, 1, cb,+ const_cast<void*>(a), const_cast<void*>(w), nullptr, nullptr, int64_t(M) * lda,+ int64_t(N) * ldw, 0, 0, lda, ldw, 0, 0);+ Gemm gemm;+ if (gemm.can_implement(args) != cutlass::Status::kSuccess) return kCannotImplement;+ if (Gemm::get_workspace_size(args) != 0) return kCannotImplement;+ if (gemm.initialize(args, nullptr, stream) != cutlass::Status::kSuccess) return kInitFailed;+ if (gemm.run(stream) != cutlass::Status::kSuccess) return kLaunchFailed;+ return cudaGetLastError() == cudaSuccess ? kOk : kLaunchFailed;+ }+};++using RunFn = int (*)(const void*, const void*, const void*, const void*, const void*, void*, int, int, int, int64_t,+ int64_t, int64_t, cudaStream_t);++} // namespace ui8diff --git a/studio/native_kernels/int8_gemm/make_wheel.py b/studio/native_kernels/int8_gemm/make_wheel.pynew file mode 100644index 00000000000..9530f4ec071--- /dev/null+++ b/studio/native_kernels/int8_gemm/make_wheel.py@@ -0,0 +1,96 @@+#!/usr/bin/env python3+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Pack a built library into ``unsloth_studio_i8gemm-<version>-py3-none-<platform>.whl``.++python make_wheel.py --lib build/libunsloth_i8gemm.so --platform manylinux_2_28_x86_64 --out dist/+"""++from __future__ import annotations++import argparse+import base64+import hashlib+import sys+import zipfile+from pathlib import Path++NAME = "unsloth_studio_i8gemm"+DIST = "unsloth-studio-i8gemm"+VERSION = "1.0.0"+ABI_VERSION = 1+++def _record_line(path: str, data: bytes) -> str:+ digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode()+ return f"{path},sha256={digest},{len(data)}"+++def build(+ lib: Path,+ platform: str,+ out: Path,+ cutlass_sha: str = "",+) -> Path:+ tag = f"py3-none-{platform}"+ dist_info = f"{NAME}-{VERSION}.dist-info"+ init = (+ '"""Prebuilt int8 GEMM for Unsloth Studio (loaded with ctypes by diffusion_int8_cutlass)."""\n'+ "import os\n\n"+ f"ABI_VERSION = {ABI_VERSION}\n"+ f"CUTLASS_SHA = {cutlass_sha!r}\n"+ f"LIB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib', {lib.name!r})\n"+ ).encode()+ metadata = (+ "Metadata-Version: 2.1\n"+ f"Name: {DIST}\n"+ f"Version: {VERSION}\n"+ "Summary: Prebuilt CUTLASS int8 GEMM with a fused torchao dequant epilogue for Unsloth Studio\n"+ "License: AGPL-3.0-only\n"+ "Requires-Python: >=3.9\n"+ ).encode()+ wheel = (+ "Wheel-Version: 1.0\n"+ "Generator: unsloth-studio make_wheel.py\n"+ "Root-Is-Purelib: false\n"+ f"Tag: {tag}\n"+ ).encode()+ files = {+ f"{NAME}/__init__.py": init,+ f"{NAME}/lib/{lib.name}": lib.read_bytes(),+ f"{dist_info}/METADATA": metadata,+ f"{dist_info}/WHEEL": wheel,+ f"{dist_info}/top_level.txt": f"{NAME}\n".encode(),+ }+ record = [_record_line(p, d) for p, d in files.items()] + [f"{dist_info}/RECORD,,"]+ files[f"{dist_info}/RECORD"] = ("\n".join(record) + "\n").encode()+ out.mkdir(parents = True, exist_ok = True)+ path = out / f"{NAME}-{VERSION}-{tag}.whl"+ with zipfile.ZipFile(path, "w", compression = zipfile.ZIP_DEFLATED, compresslevel = 9) as zf:+ for name, data in files.items():+ info = zipfile.ZipInfo(name, date_time = (2026, 1, 1, 0, 0, 0))+ info.external_attr = (0o755 if name.endswith((".so", ".dll")) else 0o644) << 16+ info.compress_type = zipfile.ZIP_DEFLATED+ zf.writestr(info, data)+ return path+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--lib", required = True)+ ap.add_argument(+ "--platform",+ required = True,+ help = "wheel platform tag, e.g. manylinux_2_28_x86_64, win_amd64",+ )+ ap.add_argument("--out", required = True)+ ap.add_argument("--cutlass-sha", default = "")+ args = ap.parse_args()+ path = build(Path(args.lib), args.platform, Path(args.out), args.cutlass_sha)+ print(f"{path} ({path.stat().st_size / 1e6:.1f} MB)")+ return 0+++if __name__ == "__main__":+ sys.exit(main())