Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: prebuilt CUTLASS int8 GEMM with the dequant epilogue fused, raced per arch (FLUX.3 klein INT8 -19.8% on L4) (#17486)

Daniel Han52cea6699e1e
diff --git a/.github/workflows/prebuilt-int8-gemm.yml b/.github/workflows/prebuilt-int8-gemm.yml
new file mode 100644
index 00000000000..a885059eb51
--- /dev/null
+++ b/.github/workflows/prebuilt-int8-gemm.yml
@@ -0,0 +1,234 @@
+# Prebuilt int8 GEMM for Unsloth Studio's INT8 ConvRot diffusion path (studio/native_kernels/int8_gemm).
+# C ABI + static cudart: one py3-none wheel per platform for every torch and CUDA 12/13 driver (built with 12.8).
+# Dispatch only; `publish: false` builds, tests and signs; publishing shares prebuilt-cuda-wheels.yml's tag.
+name: Prebuilt int8 GEMM
+
+on:
+ workflow_dispatch:
+ inputs:
+ release_tag:
+ description: 'release tag to publish under'
+ type: string
+ default: 'prebuilt-wheels-cu13'
+ publish:
+ description: 'upload to the release (otherwise artifacts only)'
+ type: boolean
+ default: false
+ draft:
+ description: 'keep the release a draft'
+ type: boolean
+ default: true
+
+permissions:
+ contents: read
+
+concurrency:
+ group: prebuilt-int8-gemm-${{ github.repository }}
+ cancel-in-progress: false
+
+env:
+ # CUTLASS is header-only here; pinned by commit so a rebuild is the same source.
+ CUTLASS_SHA: b2f249acd20392beeb16f5e5a31f4f1a4e38b3bd
+
+jobs:
+ build:
+ name: build ${{ matrix.platform }}
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 90
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - platform: manylinux_2_28_x86_64
+ runner: ubuntu-latest
+ - platform: win_amd64
+ runner: windows-latest
+ defaults:
+ run:
+ shell: bash
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ sparse-checkout: |
+ studio/native_kernels/int8_gemm
+ studio/backend/core/inference/diffusion_int8_cutlass.py
+ sparse-checkout-cone-mode: false
+
+ - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+ with:
+ python-version: '3.12'
+
+ - name: CUTLASS at the pinned commit
+ run: |
+ set -euo pipefail
+ git init -q cutlass
+ git -C cutlass remote add origin https://github.com/NVIDIA/cutlass
+ git -C cutlass fetch -q --depth 1 origin "$CUTLASS_SHA"
+ git -C cutlass checkout -q FETCH_HEAD
+ test "$(git -C cutlass rev-parse HEAD)" = "$CUTLASS_SHA"
+
+ - name: Build (Linux, manylinux_2_28 container)
+ if: runner.os == 'Linux'
+ timeout-minutes: 60
+ run: |
+ set -euo pipefail
+ docker run --rm -e CUTLASS_SHA -v "$PWD:/w" -w /w quay.io/pypa/manylinux_2_28_x86_64 bash -euo pipefail -c '
+ dnf -y -q install dnf-plugins-core
+ dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo
+ dnf -y -q install cuda-nvcc-12-8 cuda-cudart-devel-12-8 cuda-crt-12-8
+ /opt/python/cp312-cp312/bin/python studio/native_kernels/int8_gemm/build.py \
+ --cutlass cutlass --out build --nvcc /usr/local/cuda-12.8/bin/nvcc --jobs "$(nproc)"
+ /opt/python/cp312-cp312/bin/python studio/native_kernels/int8_gemm/make_wheel.py \
+ --lib build/libunsloth_i8gemm.so --platform manylinux_2_28_x86_64 --out dist --cutlass-sha "$CUTLASS_SHA"
+ '
+
+ # Authenticode-checked NVIDIA installer: its output is what we sign.
+ - name: Install the CUDA 12.8 toolkit (Windows)
+ if: runner.os == 'Windows'
+ timeout-minutes: 25
+ shell: pwsh
+ run: |
+ $ErrorActionPreference = 'Stop'
+ $version = '12.8.1'
+ $url = "https://developer.download.nvidia.com/compute/cuda/$version/network_installers/cuda_${version}_windows_network.exe"
+ $installer = Join-Path $env:RUNNER_TEMP 'cuda_network.exe'
+ for ($attempt = 1; $attempt -le 3; $attempt++) {
+ try { Invoke-WebRequest -Uri $url -OutFile $installer -TimeoutSec 300; break }
+ catch { if ($attempt -eq 3) { throw }; Start-Sleep -Seconds 10 }
+ }
+ $signature = Get-AuthenticodeSignature -FilePath $installer
+ if ($signature.Status -ne 'Valid' -or $signature.SignerCertificate.Subject -notmatch 'O=NVIDIA Corporation') {
+ throw "CUDA installer signature: $($signature.Status) $($signature.SignerCertificate.Subject)"
+ }
+ $packages = @('nvcc', 'nvvm', 'crt', 'cudart', 'thrust') | ForEach-Object { "${_}_12.8" }
+ $process = Start-Process -FilePath $installer -ArgumentList (@('-s') + $packages + @('-n')) -Wait -PassThru -NoNewWindow
+ if ($process.ExitCode -ne 0) { throw "CUDA installer exited $($process.ExitCode)" }
+ $root = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8'
+ if (-not (Test-Path (Join-Path $root 'bin\nvcc.exe'))) { throw "no nvcc.exe under $root" }
+ New-Item -ItemType Junction -Path 'C:\cuda' -Target $root | Out-Null
+ 'C:\cuda\bin' | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8
+
+ - name: Build (Windows, MSVC x64)
+ if: runner.os == 'Windows'
+ timeout-minutes: 60
+ shell: pwsh
+ run: |
+ $ErrorActionPreference = 'Stop'
+ $vs = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * `
+ -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath
+ cmd /c "`"$vs\VC\Auxiliary\Build\vcvars64.bat`" && set" | ForEach-Object {
+ if ($_ -match '^([^=]+)=(.*)$') { [Environment]::SetEnvironmentVariable($matches[1], $matches[2]) }
+ }
+ python studio/native_kernels/int8_gemm/build.py --cutlass cutlass --out build --nvcc C:\cuda\bin\nvcc.exe --jobs $env:NUMBER_OF_PROCESSORS
+ if ($LASTEXITCODE -ne 0) { throw "build exited $LASTEXITCODE" }
+ python studio/native_kernels/int8_gemm/make_wheel.py --lib build/unsloth_i8gemm.dll --platform win_amd64 --out dist --cutlass-sha $env:CUTLASS_SHA
+ if ($LASTEXITCODE -ne 0) { throw "make_wheel exited $LASTEXITCODE" }
+
+ # No GPU: the library must still load and report its ABI and tile tables.
+ - name: Smoke (install the wheel, load it the way Studio does)
+ run: |
+ set -euo pipefail
+ python -m venv smoke
+ py="$PWD/smoke/bin/python"; [ -x "$py" ] || py="$PWD/smoke/Scripts/python.exe"
+ "$py" -m pip install --quiet --no-deps dist/*.whl
+ cd "$RUNNER_TEMP"
+ "$py" - <<'EOF'
+ import ctypes, importlib.util, os
+ import unsloth_studio_i8gemm as m
+ lib = ctypes.CDLL(m.LIB_PATH)
+ assert lib.ui8_abi_version() == m.ABI_VERSION == 1, lib.ui8_abi_version()
+ assert lib.ui8_num_tiles(80) > 0 and lib.ui8_num_tiles(75) > 0
+ print(m.LIB_PATH, os.path.getsize(m.LIB_PATH), "tiles", lib.ui8_num_tiles(80), lib.ui8_num_tiles(75))
+ EOF
+
+ - name: Linked only against the system runtime (Linux)
+ if: runner.os == 'Linux'
+ run: |
+ set -euo pipefail
+ ldd build/libunsloth_i8gemm.so
+ ! ldd build/libunsloth_i8gemm.so | grep -E 'libcudart|libtorch|libpython'
+ objdump -T build/libunsloth_i8gemm.so | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1
+
+ - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+ with:
+ name: wheel-i8gemm-${{ matrix.platform }}
+ path: dist/*.whl
+ if-no-files-found: error
+
+ sign:
+ name: Sign
+ needs: build
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ environment: release-signing
+ permissions:
+ id-token: write
+ attestations: write
+ contents: read
+ steps:
+ - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+ with:
+ pattern: wheel-i8gemm-*
+ merge-multiple: true
+ path: wheels
+
+ - name: Both platforms arrived
+ run: |
+ set -euo pipefail
+ ls -l wheels/
+ test "$(find wheels -name '*.whl' | wc -l)" -eq 2
+ cd wheels && sha256sum -- *.whl | sort -k2 > SHA256SUMS && cat SHA256SUMS
+
+ - name: Sign every wheel with Sigstore
+ uses: sigstore/gh-action-sigstore-python@790bc6befb9d733738f18d8f895854b453640ec9 # v3.5.0
+ with:
+ inputs: ./wheels/*.whl
+ verify: true
+ verify-cert-identity: https://github.com/${{ github.repository }}/.github/workflows/prebuilt-int8-gemm.yml@${{ github.ref }}
+ verify-oidc-issuer: https://token.actions.githubusercontent.com
+ upload-signing-artifacts: false
+ release-signing-artifacts: false
+
+ - name: Attest build provenance
+ uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
+ with:
+ subject-checksums: wheels/SHA256SUMS
+
+ - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+ with:
+ name: prebuilt-int8-gemm-signed
+ path: |
+ wheels/*.whl
+ wheels/*.sigstore.json
+ wheels/SHA256SUMS
+ if-no-files-found: error
+ compression-level: 0
+
+ publish:
+ name: Publish to the release
+ needs: sign
+ if: ${{ inputs.publish }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ permissions:
+ contents: write
+ steps:
+ - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+ with:
+ name: prebuilt-int8-gemm-signed
+ path: wheels
+
+ - name: Upload
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TAG: ${{ inputs.release_tag }}
+ DRAFT: ${{ inputs.draft }}
+ run: |
+ set -euo pipefail
+ if ! gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+ gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --title "$TAG" --notes "Prebuilt wheels" \
+ --latest=false --draft="$DRAFT"
+ fi
+ # SHA256SUMS is not uploaded here: prebuilt-cuda-wheels.yml owns that asset on the shared tag.
+ gh release upload "$TAG" --repo "$GITHUB_REPOSITORY" --clobber wheels/*.whl wheels/*.sigstore.json
diff --git a/studio/backend/core/inference/diffusion_int8_cutlass.py b/studio/backend/core/inference/diffusion_int8_cutlass.py
new file mode 100644
index 00000000000..ec57c79ed1c
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_int8_cutlass.py
@@ -0,0 +1,536 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Prebuilt CUTLASS int8 GEMM (C ABI, ctypes) with torchao's epilogue fused: sm75 fp16 out, sm8x bf16 out.
+
+Any load failure or tile failing the bit-exact probe keeps today's path; a tile replaces it only when it wins
+the race by ``_MARGIN``. ``UNSLOTH_DIFFUSION_INT8_CUTLASS=0`` disables, ``=1`` skips the race."""
+
+from __future__ import annotations
+
+import ctypes
+import os
+import threading
+from functools import lru_cache
+from typing import Any, Callable, Optional
+
+CUTLASS_ENV = "UNSLOTH_DIFFUSION_INT8_CUTLASS"
+CUTLASS_LIB_ENV = "UNSLOTH_DIFFUSION_INT8_CUTLASS_LIB"
+WHEEL_MODULE = "unsloth_studio_i8gemm"
+ABI_VERSION = 1
+LIB_NAMES = ("libunsloth_i8gemm.so", "unsloth_i8gemm.dll")
+
+_STATUS = {
+ 0: "ok",
+ 1: "bad arguments",
+ 2: "no kernel for this tile / dtype",
+ 3: "cannot implement",
+ 4: "initialize failed",
+ 5: "launch failed",
+}
+# ties keep today's path
+_MARGIN = 0.02
+_RACE_M = (512, 4096)
+_LOCK = threading.Lock()
+_LOAD_ERROR: list = [None]
+_CALLS = [0]
+
+
+def mode() -> str:
+ raw = (os.environ.get(CUTLASS_ENV) or "").strip().lower()
+ if raw in ("0", "off", "false", "no"):
+ return "off"
+ if raw in ("1", "on", "true", "yes", "force"):
+ return "force"
+ return "auto"
+
+
+def family(capability: Optional[tuple]) -> Optional[int]:
+ if capability is None:
+ return None
+ cap = (int(capability[0]), int(capability[1]))
+ if cap == (7, 5):
+ return 75
+ if cap[0] == 8:
+ return 80
+ return None
+
+
+def out_dtype_for(fam: int) -> Any:
+ import torch
+ return torch.float16 if fam == 75 else torch.bfloat16
+
+
+def _candidate_paths() -> list:
+ paths = []
+ explicit = (os.environ.get(CUTLASS_LIB_ENV) or "").strip()
+ if explicit:
+ paths.append(explicit)
+ return paths
+ try:
+ import importlib.util
+ spec = importlib.util.find_spec(WHEEL_MODULE)
+ roots = list(spec.submodule_search_locations or ()) if spec is not None else []
+ except Exception: # noqa: BLE001 - no wheel
+ roots = []
+ for root in roots:
+ for name in LIB_NAMES:
+ for sub in ("", "lib"):
+ p = os.path.join(root, sub, name)
+ if os.path.isfile(p):
+ paths.append(p)
+ return paths
+
+
+def _bind(lib: Any) -> Any:
+ c = ctypes
+ lib.ui8_abi_version.restype = c.c_int
+ lib.ui8_abi_version.argtypes = []
+ lib.ui8_num_tiles.restype = c.c_int
+ lib.ui8_num_tiles.argtypes = [c.c_int]
+ lib.ui8_tile_info.restype = c.c_int
+ lib.ui8_tile_info.argtypes = [c.c_int, c.c_int, c.POINTER(c.c_int)]
+ lib.ui8_has_kernel.restype = c.c_int
+ lib.ui8_has_kernel.argtypes = [c.c_int] * 7
+ lib.ui8_gemm.restype = c.c_int
+ lib.ui8_gemm.argtypes = [c.c_int] * 6 + [c.c_void_p] * 6 + [c.c_int64] * 6 + [c.c_void_p]
+ return lib
+
+
+@lru_cache(maxsize = 1)
+def library() -> Optional[Any]:
+ if mode() == "off":
+ return None
+ try:
+ import torch
+
+ from .rocm_bf16 import is_rocm_torch
+ if not torch.cuda.is_available() or is_rocm_torch(torch):
+ return None
+ except Exception: # noqa: BLE001
+ return None
+ for path in _candidate_paths():
+ try:
+ lib = _bind(ctypes.CDLL(path))
+ if lib.ui8_abi_version() != ABI_VERSION:
+ _LOAD_ERROR[0] = f"{path}: ABI {lib.ui8_abi_version()} != {ABI_VERSION}"
+ continue
+ return lib
+ except Exception as exc: # noqa: BLE001 - a broken binary keeps today's path
+ _LOAD_ERROR[0] = f"{path}: {exc}"
+ return None
+
+
+def load_error() -> Optional[str]:
+ return _LOAD_ERROR[0]
+
+
+def library_path() -> Optional[str]:
+ lib = library()
+ return getattr(lib, "_name", None) if lib is not None else None
+
+
+@lru_cache(maxsize = 4)
+def tiles(fam: int) -> tuple:
+ lib = library()
+ if lib is None:
+ return ()
+ out = []
+ buf = (ctypes.c_int * 8)()
+ for i in range(lib.ui8_num_tiles(fam)):
+ if lib.ui8_tile_info(fam, i, buf) == 0:
+ out.append(tuple(buf))
+ return tuple(out)
+
+
+def _kind(dtype: Any) -> Optional[int]:
+ import torch
+ return {torch.float32: 0, torch.bfloat16: 1, torch.float16: 2}.get(dtype)
+
+
+EPI_TORCHAO = 0
+EPI_NATIVE = 1
+
+
+def has_kernel(
+ fam: int,
+ tile: int,
+ out_dtype: Any,
+ xs_dtype: Any,
+ ws_dtype: Any,
+ bias: bool,
+ epilogue: int = EPI_TORCHAO,
+) -> bool:
+ lib = library()
+ kinds = (_kind(out_dtype), _kind(xs_dtype), _kind(ws_dtype))
+ if lib is None or None in kinds:
+ return False
+ return bool(
+ lib.ui8_has_kernel(fam, tile, kinds[0], kinds[1], kinds[2], int(bool(bias)), int(epilogue))
+ )
+
+
+def eligible(a: Any, w: Any, xs: Any, ws: Any, bias: Any, out_dtype: Any) -> bool:
+ import torch
+ try:
+ if a.dtype != torch.int8 or w.dtype != torch.int8 or a.dim() != 2 or w.dim() != 2:
+ return False
+ if a.device != w.device or not a.is_cuda:
+ return False
+ m, k = a.shape
+ n = w.shape[0]
+ if w.shape[1] != k or k % 16 or n % 8 or m < 1:
+ return False
+ if a.stride(1) != 1 or w.stride(1) != 1 or a.stride(0) % 16 or w.stride(0) % 16:
+ return False
+ if a.data_ptr() % 16 or w.data_ptr() % 16:
+ return False
+ for vec, size in ((xs, m), (ws, n)):
+ if vec.device != a.device or vec.numel() != size or (vec.dim() and vec.stride(-1) != 1):
+ return False
+ if vec.data_ptr() % 16 or _kind(vec.dtype) is None:
+ return False
+ if bias is not None:
+ if (
+ bias.dtype != out_dtype
+ or bias.numel() != n
+ or bias.data_ptr() % 16
+ or bias.device != a.device
+ ):
+ return False
+ if bias.dim() and bias.stride(-1) != 1:
+ return False
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
+def launch(
+ a: Any,
+ w: Any,
+ xs: Any,
+ ws: Any,
+ bias: Any,
+ fam: int,
+ tile: int,
+ out_dtype: Any,
+ epilogue: int = EPI_TORCHAO,
+) -> Any:
+ import torch
+
+ lib = library()
+ if lib is None:
+ raise RuntimeError("int8 CUTLASS library not loaded")
+ m, k = a.shape
+ n = w.shape[0]
+ out = torch.empty((m, n), device = a.device, dtype = out_dtype)
+ with torch.cuda.device(a.device):
+ stream = torch.cuda.current_stream(a.device).cuda_stream
+ status = lib.ui8_gemm(
+ fam,
+ tile,
+ _kind(out_dtype),
+ _kind(xs.dtype),
+ _kind(ws.dtype),
+ int(epilogue),
+ a.data_ptr(),
+ w.data_ptr(),
+ xs.data_ptr(),
+ ws.data_ptr(),
+ bias.data_ptr() if bias is not None else None,
+ out.data_ptr(),
+ m,
+ n,
+ k,
+ a.stride(0),
+ w.stride(0),
+ out.stride(0),
+ stream,
+ )
+ if status != 0:
+ raise RuntimeError(f"ui8_gemm tile {tile}: {_STATUS.get(status, status)}")
+ _CALLS[0] += 1
+ return out
+
+
+def reference(a: Any, w: Any, xs: Any, ws: Any, bias: Any, out_dtype: Any) -> Any:
+ """torchao's int8 dynamic Linear epilogue op for op (fp16 x_scale computes in fp32)."""
+ import torch
+
+ m = a.shape[0]
+ if m < 17: # _int_mm row floor
+ a = torch.cat([a, a.new_zeros((17 - m, a.shape[1]))])
+ c = torch._int_mm(a, w.t())[:m]
+ inter = torch.float32 if xs.dtype == torch.float16 else xs.dtype
+ y = (c * xs.reshape(-1, 1).to(inter)).to(out_dtype)
+ y = y * ws.reshape(-1)
+ if bias is not None:
+ y = y + bias
+ return y.to(out_dtype)
+
+
+def reference_native(a: Any, w: Any, xs: Any, ws: Any, out_dtype: Any) -> Any:
+ """``NativeWeightOnlyLinear._int_mm_rows``: fp32 epilogue, one cast."""
+ import torch
+
+ m = a.shape[0]
+ if m < 17:
+ a = torch.cat([a, a.new_zeros((17 - m, a.shape[1]))])
+ c = torch._int_mm(a, w.t())[:m]
+ out = torch.mul(c, xs.reshape(-1, 1).float())
+ return out.mul_(ws.reshape(-1).float()).to(out_dtype)
+
+
+# (M, N, K, bias): ragged tails, K below / above one tile
+_PROBE_SHAPES = (
+ (257, 384, 512, False),
+ (33, 208, 144, True),
+ (300, 528, 1040, True),
+ (1, 64, 64, False),
+)
+
+
+def _scale_dtypes(fam: int) -> tuple:
+ import torch
+ if fam == 75:
+ return tuple(
+ (x, w)
+ for x in (torch.float16, torch.float32)
+ for w in (torch.float16, torch.bfloat16, torch.float32)
+ )
+ return tuple(
+ (x, w) for x in (torch.bfloat16, torch.float32) for w in (torch.bfloat16, torch.float32)
+ )
+
+
+def _probe_tile(index: int, fam: int, tile: int, pairs: tuple) -> bool:
+ import torch
+
+ from .diffusion_int8_gemm import tie_operands
+
+ dev = torch.device("cuda", index)
+ out_dtype = out_dtype_for(fam)
+ g = torch.Generator(device = "cpu").manual_seed(0)
+ try:
+ for xs_dt, ws_dt in pairs:
+ for m, n, k, bias in _PROBE_SHAPES:
+ if not has_kernel(fam, tile, out_dtype, xs_dt, ws_dt, bias):
+ return False
+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)
+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)
+ xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(xs_dt).to(dev)
+ ws = (torch.rand(n, generator = g) * 0.002 + 1e-5).to(ws_dt).to(dev)
+ b = (torch.randn(n, generator = g) * 0.1).to(out_dtype).to(dev) if bias else None
+ if not torch.equal(
+ launch(a, w, xs, ws, b, fam, tile, out_dtype),
+ reference(a, w, xs, ws, b, out_dtype),
+ ):
+ return False
+ # near-midpoint products: the double rounding must match
+ a, w = tie_operands(dev)
+ xs = torch.ones(a.shape[0], device = dev, dtype = xs_dt)
+ ws = torch.ones(w.shape[0], device = dev, dtype = ws_dt)
+ if fam == 75:
+ # keep fp16 out of inf
+ xs = torch.full((a.shape[0],), 2.0**-12, device = dev, dtype = xs_dt)
+ if not torch.equal(
+ launch(a, w, xs, ws, None, fam, tile, out_dtype),
+ reference(a, w, xs, ws, None, out_dtype),
+ ):
+ return False
+ if has_kernel(fam, tile, out_dtype, torch.float32, torch.float32, False, EPI_NATIVE):
+ for m, n, k, _bias in _PROBE_SHAPES:
+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)
+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)
+ xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(dev)
+ ws = (torch.rand(n, generator = g) * 0.002 + 1e-5).to(dev)
+ got = launch(a, w, xs, ws, None, fam, tile, out_dtype, EPI_NATIVE)
+ if not torch.equal(got, reference_native(a, w, xs, ws, out_dtype)):
+ return False
+ a, w = tie_operands(dev)
+ xs = torch.full((a.shape[0],), 2.0**-12, device = dev, dtype = torch.float32)
+ ws = torch.ones(w.shape[0], device = dev, dtype = torch.float32)
+ got = launch(a, w, xs, ws, None, fam, tile, out_dtype, EPI_NATIVE)
+ if not torch.equal(got, reference_native(a, w, xs, ws, out_dtype)):
+ return False
+ torch.cuda.synchronize(dev)
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
+_PROBED: dict = {}
+
+
+def probed_tiles(index: int) -> tuple:
+ if index in _PROBED:
+ return _PROBED[index]
+ ok: tuple = ()
+ try:
+ import torch
+ fam = family(torch.cuda.get_device_capability(index))
+ if fam is not None and library() is not None:
+ pairs = _scale_dtypes(fam)
+ ok = tuple(t for t in range(len(tiles(fam))) if _probe_tile(index, fam, t, pairs))
+ except Exception: # noqa: BLE001
+ ok = ()
+ _PROBED[index] = ok
+ return ok
+
+
+def available(index: int) -> bool:
+ return bool(probed_tiles(index))
+
+
+def time_us(
+ fns: dict,
+ rounds: int = 3,
+ reps: int = 3,
+ warm: int = 1,
+) -> dict:
+ """{name: us}, min over interleaved rounds."""
+ import torch
+
+ start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)
+ for fn in fns.values():
+ for _ in range(warm):
+ fn()
+ torch.cuda.synchronize()
+ best = {name: float("inf") for name in fns}
+ for _ in range(rounds):
+ for name, fn in fns.items():
+ start.record()
+ for _ in range(reps):
+ fn()
+ end.record()
+ end.synchronize()
+ best[name] = min(best[name], start.elapsed_time(end) * 1000.0 / reps)
+ return best
+
+
+def race(
+ index: int,
+ w: Any,
+ ws: Any,
+ bias: Any,
+ xs_dtype: Any,
+ today: Optional[Callable],
+ m: int,
+ epilogue: int = EPI_TORCHAO,
+) -> dict:
+ """Best probed tile vs ``today`` (None = cuBLAS ``_int_mm``); tile None keeps today's path."""
+ import torch
+
+ fam = family(torch.cuda.get_device_capability(index))
+ tiles_ok = probed_tiles(index)
+ if fam is None or not tiles_ok:
+ return {"tile": None, "us": {}}
+ out_dtype = out_dtype_for(fam)
+ dev = w.device
+ k = w.shape[1]
+ g = torch.Generator(device = "cpu").manual_seed(7)
+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8).to(dev)
+ xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(xs_dtype).to(dev)
+ if not eligible(a, w, xs, ws, bias, out_dtype):
+ return {"tile": None, "us": {}}
+ cands = [
+ t
+ for t in tiles_ok
+ if has_kernel(fam, t, out_dtype, xs_dtype, ws.dtype, bias is not None, epilogue)
+ ]
+ if not cands:
+ return {"tile": None, "us": {}}
+
+ def cut(t):
+ return lambda: launch(a, w, xs, ws, bias, fam, t, out_dtype, epilogue)
+
+ if today is None:
+ base = lambda: torch._int_mm(a, w.t()).amax() # noqa: E731
+ base_name = "cublas"
+ else:
+ base = lambda: today(a, w, xs, ws, bias) # noqa: E731
+ base_name = "today"
+ with torch.cuda.device(dev):
+ screen = time_us({t: cut(t) for t in cands}, rounds = 1, reps = 2, warm = 1)
+ top = sorted(cands, key = lambda t: screen[t])[:3]
+ fns = {base_name: base}
+ fns.update({t: cut(t) for t in top})
+ timing = time_us(fns, rounds = 3, reps = 3, warm = 1)
+ best = min(top, key = lambda t: timing[t])
+ keep = timing[best] < timing[base_name] * (1.0 - _MARGIN)
+ us = {str(k_): round(v, 1) for k_, v in timing.items()}
+ return {"tile": best if keep else None, "us": us, "m": m}
+
+
+# (card, N, K) -> {race M: tile or None}, sm75 NativeWeightOnlyLinear
+_NATIVE_PLAN: dict = {}
+_NATIVE_RACES: dict = {}
+_NATIVE_LARGE_M = 1024
+
+
+def native_prepare(weight_q: Any, x_dtype: Any, has_bias: bool) -> None:
+ """Once per (card, N, K), outside any capture: race native-epilogue tiles (sm75, fp16, no bias)."""
+ import torch
+ try:
+ if mode() == "off" or has_bias or x_dtype != torch.float16 or not weight_q.is_cuda:
+ return
+ index = weight_q.device.index
+ key = (index, int(weight_q.shape[0]), int(weight_q.shape[1]))
+ if key in _NATIVE_PLAN:
+ return
+ if torch.cuda.is_current_stream_capturing() or torch.compiler.is_compiling():
+ return
+ _NATIVE_PLAN[key] = {}
+ if family(torch.cuda.get_device_capability(index)) != 75 or not probed_tiles(index):
+ return
+ g = torch.Generator(device = "cpu").manual_seed(13)
+ ws = (torch.rand(weight_q.shape[0], generator = g) * 1e-3 + 1e-5).to(weight_q.device)
+ today = lambda a, w_, x_, s_, b_: reference_native(a, w_, x_, s_, torch.float16) # noqa: E731
+ entry = {}
+ for m in _RACE_M:
+ r = race(index, weight_q, ws, None, torch.float32, today, m, epilogue = EPI_NATIVE)
+ if mode() == "force" and r.get("us"):
+ digits = [t for t in r["us"] if t.isdigit()]
+ r["tile"] = int(min(digits, key = lambda t: r["us"][t])) if digits else None
+ entry[m] = r["tile"]
+ _NATIVE_RACES.setdefault(key, {})[m] = r
+ _NATIVE_PLAN[key] = entry
+ except Exception: # noqa: BLE001
+ return
+
+
+def native_tile(xq: Any, weight_q: Any, x_scale: Any, w_scale: Any) -> Optional[int]:
+ import torch
+
+ if torch.compiler.is_compiling():
+ return None
+ entry = _NATIVE_PLAN.get(
+ (weight_q.device.index, int(weight_q.shape[0]), int(weight_q.shape[1]))
+ )
+ if not entry:
+ return None
+ tile = entry.get(max(_RACE_M) if xq.shape[0] >= _NATIVE_LARGE_M else min(_RACE_M))
+ if tile is None:
+ return None
+ if x_scale.dtype != torch.float32 or w_scale.dtype != torch.float32:
+ return None
+ return (
+ tile
+ if eligible(xq, weight_q, x_scale.reshape(-1), w_scale.reshape(-1), None, torch.float16)
+ else None
+ )
+
+
+def native_gemm(xq: Any, weight_q: Any, x_scale: Any, w_scale: Any, tile: int) -> Any:
+ import torch
+ return launch(
+ xq,
+ weight_q,
+ x_scale.reshape(-1),
+ w_scale.reshape(-1),
+ None,
+ 75,
+ tile,
+ torch.float16,
+ EPI_NATIVE,
+ )
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index 5c28c386d46..42fbc9e431e 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -38,6 +38,9 @@ _MARK = "_unsloth_i8_gemm_prev"
_FWD = "_unsloth_i8_gemm_fwd"
# on a SwiGLU MLP whose down projection takes its codes from the gate / value halves
_FF_MARK = "_unsloth_i8_gemm_swiglu_prev"
+# fp16 input runs the sm75 prebuilt kernel
+_F16 = "_unsloth_i8_gemm_f16"
+_OP16_NAME = "int8_mm_dequant_f16"
_SWIGLU_FF_CLASSES = ("QwenImage21SwiGLUFeedForward",)
_NO_PREV = object()
_LOCK = threading.Lock()
@@ -483,11 +486,18 @@ def tma_tile_for(index: Any, m: int) -> Optional[tuple]:
def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
- """Op body: the probed tile for this device and shape, else (launch failure, ``_aligned`` false) the stock
- epilogue."""
+ """Op body: race-won CUTLASS tile, else probed Triton tile, else the stock epilogue."""
_CALLS[0] += 1
a = a if a.stride(-1) == 1 else a.contiguous()
index = a.device.index
+ tile = _cutlass_tile(index, a, w, xs, ws, bias)
+ if tile is not None:
+ import torch
+ from . import diffusion_int8_cutlass as cut
+ try:
+ return cut.launch(a, w, xs, ws, bias, _CUT_FAMILY[index], tile, torch.bfloat16)
+ except Exception: # noqa: BLE001 - drops the prebuilt kernel on this card
+ _CUT_PLAN.pop(index, None)
cfg = tile_for(index, a.shape[0], w.shape[0], a.shape[1])
if cfg is not None and w.stride(-1) == 1 and w.device == a.device and _aligned(a, w):
tma = tma_tile_for(index, a.shape[0])
@@ -510,6 +520,345 @@ def _run(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
return reference(a, w, xs, ws, bias)
+# index -> {(N, K, xs dtype, ws dtype, bias): {race M: tile or None}}; a large-M tile runs ``_cutlass_fused``
+_CUT_PLAN: dict = {}
+_CUT_FAMILY: dict = {}
+_CUT_RACES: dict = {}
+# index -> every shape key already planned (winners and losers), so a later load races only new shapes
+_CUT_DONE: dict = {}
+_CUT_LARGE_M = 1024
+_CUT_CACHE_NAME = "diffusion_int8_cutlass_race.json"
+_CUT_CACHE_MAX = 512
+
+
+def _cut_key(n: int, k: int, xs_dtype: Any, ws_dtype: Any, bias: bool) -> tuple:
+ return (int(n), int(k), str(xs_dtype), str(ws_dtype), bool(bias))
+
+
+def _cutlass_tile(
+ index: Any,
+ a: Any,
+ w: Any,
+ xs: Any,
+ ws: Any,
+ bias: Any,
+ out_dtype: Any = None,
+) -> Optional[int]:
+ plan = _CUT_PLAN.get(index)
+ if not plan:
+ return None
+ entry = plan.get(_cut_key(w.shape[0], w.shape[1], xs.dtype, ws.dtype, bias is not None))
+ if not entry:
+ return None
+ from . import diffusion_int8_cutlass as cut
+
+ tile = entry.get(max(cut._RACE_M) if a.shape[0] >= _CUT_LARGE_M else min(cut._RACE_M))
+ if tile is None:
+ return None
+ import torch
+
+ return tile if cut.eligible(a, w, xs, ws, bias, out_dtype or torch.bfloat16) else None
+
+
+def _cutlass_fused(index: Any, n: int, k: int, xs_dtype: Any, ws_dtype: Any, bias: bool) -> bool:
+ from . import diffusion_int8_cutlass as cut
+ entry = (_CUT_PLAN.get(index) or {}).get(_cut_key(n, k, xs_dtype, ws_dtype, bias))
+ return bool(entry) and entry.get(max(cut._RACE_M)) is not None
+
+
+def cutlass_wanted(capability: Optional[tuple]) -> bool:
+ """The prebuilt kernel could serve this arch (switch, family, library loads); before any probe."""
+ try:
+ from . import diffusion_int8_cutlass as cut
+ return (
+ int8_gemm_mode() != "off"
+ and cut.mode() != "off"
+ and cut.family(capability) in (75, 80)
+ and cut.library() is not None
+ )
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _cut_cache_key(index: int) -> Optional[str]:
+ """Card, toolchain, library build and both sources: any change re-races."""
+ try:
+ import hashlib
+
+ import torch
+
+ from . import diffusion_int8_cutlass as cut
+
+ h = hashlib.sha256(open(__file__, "rb").read())
+ h.update(open(cut.__file__, "rb").read())
+ lib = cut.library_path()
+ if lib:
+ h.update(open(lib, "rb").read())
+ try:
+ import triton as _triton
+ tv = _triton.__version__
+ except Exception: # noqa: BLE001
+ tv = "none"
+ cap = torch.cuda.get_device_capability(index)
+ return "|".join(
+ (
+ torch.cuda.get_device_name(index),
+ f"{cap[0]}.{cap[1]}",
+ torch.__version__,
+ str(torch.version.cuda),
+ tv,
+ os.environ.get(INT8_GEMM_TILES_ENV, ""),
+ os.environ.get(INT8_GEMM_UNLISTED_ENV, ""),
+ h.hexdigest()[:20],
+ )
+ )
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _cut_cache_path() -> Optional[Any]:
+ try:
+ from pathlib import Path
+ from utils.paths.storage_roots import cache_root
+ return Path(cache_root()) / _CUT_CACHE_NAME
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _cut_cache_read() -> dict:
+ path = _cut_cache_path()
+ try:
+ import json
+ table = (
+ json.loads(path.read_text(encoding = "utf-8"))
+ if path is not None and path.is_file()
+ else {}
+ )
+ return table if isinstance(table, dict) else {}
+ except Exception: # noqa: BLE001
+ return {}
+
+
+def _cut_cache_write(table: dict) -> None:
+ path = _cut_cache_path()
+ if path is None:
+ return
+ try:
+ import json
+
+ table = dict(list(table.items())[-_CUT_CACHE_MAX:])
+ path.parent.mkdir(parents = True, exist_ok = True)
+ tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
+ tmp.write_text(json.dumps(table), encoding = "utf-8")
+ os.replace(tmp, path)
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _xs_dtype(
+ kind: str,
+ weight: Any,
+ k: int,
+ device: Any,
+ dtype: Any = None,
+) -> Any:
+ """Activation-scale dtype of this Linear's act quant, read off a one-row quant."""
+ import torch
+
+ dtype = dtype or torch.bfloat16
+ x = torch.ones((_MIN_ROWS, k), device = device, dtype = dtype)
+ if dtype == torch.float16:
+ from .diffusion_int8_fused import _act_quant
+ return _act_quant(x, weight)[1].dtype
+ _, xs = _act_quant_v1(x) if kind == "v1" else _act_quant_v2(x, weight)
+ return xs.dtype
+
+
+def _cutlass_plan(
+ index: int,
+ recs: list,
+ triton_on: bool,
+ logger: Any = None,
+) -> dict:
+ """Race each int8 weight shape vs today's path, publish winners to ``_CUT_PLAN``; outside any capture."""
+ import torch
+
+ from . import diffusion_int8_cutlass as cut
+
+ try:
+ cap = torch.cuda.get_device_capability(index)
+ if not cutlass_wanted(cap) or torch.cuda.is_current_stream_capturing():
+ return {}
+ if torch.compiler.is_compiling():
+ return {}
+ except Exception: # noqa: BLE001
+ return {}
+ done = _CUT_DONE.setdefault(index, set())
+ plan: dict = dict(_CUT_PLAN.get(index) or {})
+ races: dict = dict(_CUT_RACES.get(index) or {})
+ try:
+ if not cut.probed_tiles(index):
+ _CUT_PLAN[index] = {}
+ return {}
+ fam = cut.family(cap)
+ act = cut.out_dtype_for(fam)
+ ckey = _cut_cache_key(index)
+ table = _cut_cache_read() if ckey else {}
+ cached = table.get(ckey) if ckey else None
+ cached = cached if isinstance(cached, dict) else {}
+ dev = torch.device("cuda", index)
+ g = torch.Generator(device = "cpu").manual_seed(11)
+ seen: dict = {}
+ for module, rec in recs:
+ kind, weight = rec[0], rec[3]
+ wq, ws = _payload(kind, weight)
+ n, k = int(wq.shape[0]), int(wq.shape[1])
+ bias = getattr(module, "bias", None)
+ if fam == 75 and kind != "v2":
+ continue # fp16 family: torchao Int8Tensor only
+ xs_dt = seen.get(("xs", kind, k))
+ if xs_dt is None:
+ xs_dt = _xs_dtype(kind, weight, k, dev, act)
+ seen[("xs", kind, k)] = xs_dt
+ key = _cut_key(n, k, xs_dt, ws.dtype, bias is not None)
+ if key in plan or key in done:
+ continue
+ done.add(key)
+ skey = repr(key)
+ hit = cached.get(skey)
+ if isinstance(hit, dict) and all(
+ (v is None or v in cut.probed_tiles(index)) for v in hit.values()
+ ):
+ plan[key] = {int(m): v for m, v in hit.items()}
+ continue
+ # random operands: a streamed weight need not be resident
+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8).to(dev)
+ wsr = (torch.rand(n, generator = g) * 1e-3 + 1e-5).to(ws.dtype).to(dev)
+ br = (torch.randn(n, generator = g) * 0.1).to(act).to(dev) if bias is not None else None
+ entry = {}
+ for m in cut._RACE_M:
+ if triton_on:
+ today = lambda a, w_, x_, s_, b_: _launch_or_ref(index, a, w_, x_, s_, b_) # noqa: E731
+ elif m >= _CUT_LARGE_M:
+ today = None # cuBLAS lower bound
+ else:
+ today = lambda a, w_, x_, s_, b_: cut.reference(a, w_, x_, s_, b_, act) # noqa: E731
+ r = cut.race(index, w, wsr, br, xs_dt, today, m)
+ if cut.mode() == "force" and r.get("us"):
+ r["tile"] = min(
+ (t for t in r["us"] if t.isdigit()), key = lambda t: r["us"][t], default = None
+ )
+ r["tile"] = int(r["tile"]) if r["tile"] is not None else None
+ entry[m] = r["tile"]
+ races.setdefault(skey, {})[m] = r
+ plan[key] = entry
+ cached[skey] = {str(m): v for m, v in entry.items()}
+ del w, wsr, br
+ if ckey:
+ table = _cut_cache_read()
+ table.pop(ckey, None)
+ table[ckey] = cached
+ _cut_cache_write(table)
+ except Exception as exc: # noqa: BLE001
+ if logger is not None:
+ logger.info(
+ "diffusion.int8_gemm: prebuilt CUTLASS race failed on cuda:%d (%s)", index, exc
+ )
+ plan = dict(_CUT_PLAN.get(index) or {})
+ plan = {k_: v for k_, v in plan.items() if any(t is not None for t in v.values())}
+ _CUT_PLAN[index] = plan
+ _CUT_FAMILY[index] = cut.family(torch.cuda.get_device_capability(index))
+ _CUT_RACES[index] = races
+ return plan
+
+
+def _launch_or_ref(index: int, a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
+ cfg = tile_for(index, a.shape[0], w.shape[0], a.shape[1])
+ if cfg is not None and _aligned(a, w):
+ return _launch(a, w, xs, ws, bias, cfg)
+ return reference(a, w, xs, ws, bias)
+
+
+def cutlass_call_count() -> int:
+ from . import diffusion_int8_cutlass as cut
+ return cut._CALLS[0]
+
+
+def _run16(a: Any, w: Any, xs: Any, ws: Any, bias: Any) -> Any:
+ """fp16 op body (sm75 family): the race winner, else torchao's fp16 epilogue after cuBLAS."""
+ import torch
+
+ from . import diffusion_int8_cutlass as cut
+
+ _CALLS[0] += 1
+ a = a if a.stride(-1) == 1 else a.contiguous()
+ index = a.device.index
+ tile = _cutlass_tile(index, a, w, xs, ws, bias, torch.float16)
+ if tile is not None:
+ try:
+ return cut.launch(a, w, xs, ws, bias, _CUT_FAMILY[index], tile, torch.float16)
+ except Exception: # noqa: BLE001
+ _CUT_PLAN.pop(index, None)
+ return cut.reference(a, w, xs, ws, bias, torch.float16)
+
+
+@lru_cache(maxsize = 1)
+def _op16() -> Any:
+ """``unsloth_studio::int8_mm_dequant_f16`` (opaque to Inductor), or None."""
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return None
+ ns = getattr(torch.ops, _OP_NAMESPACE, None)
+ if ns is not None and hasattr(ns, _OP16_NAME):
+ return getattr(ns, _OP16_NAME)
+ custom_op = getattr(getattr(torch, "library", None), "custom_op", None)
+ if custom_op is None:
+ return None
+ try:
+
+ @custom_op(
+ f"{_OP_NAMESPACE}::{_OP16_NAME}",
+ mutates_args = (),
+ schema = "(Tensor a, Tensor w, Tensor xs, Tensor ws, Tensor? bias) -> Tensor",
+ )
+ def _int8_mm_dequant_f16(a, w, xs, ws, bias):
+ return _run16(a, w, xs, ws, bias)
+
+ @_int8_mm_dequant_f16.register_fake
+ def _(a, w, xs, ws, bias):
+ return a.new_empty((a.shape[0], w.shape[0]), dtype = torch.float16)
+ except Exception: # noqa: BLE001
+ return None
+ return getattr(torch.ops, _OP_NAMESPACE).int8_mm_dequant_f16
+
+
+def _linear_forward_f16(self: Any, x: Any, rec: tuple) -> Any:
+ """sm75 fp16 forward via the prebuilt kernel, bit for bit the stock forward."""
+ import torch
+
+ kind, group, _rotq, weight, _fused = rec
+ if self.weight is not weight or _OP16_HANDLE is None:
+ return type(self).forward(self, x)
+ wq, ws = _payload(kind, weight)
+ if wq.device != x.device or x.numel() < _MIN_ROWS * x.shape[-1]:
+ return type(self).forward(self, x)
+ lead = x.shape[:-1]
+ if group is not None:
+ from .diffusion_convrot import build_convrot_hadamard, rotate_convrot_activation
+ x = rotate_convrot_activation(
+ x, build_convrot_hadamard(group, device = x.device, dtype = x.dtype), group
+ )
+ from .diffusion_int8_fused import _act_quant
+
+ xq, xs = _act_quant(x.reshape(-1, x.shape[-1]), weight)
+ y = _OP16_HANDLE(xq, wq, xs.reshape(-1), ws, self.bias)
+ return y.reshape(*lead, y.shape[-1])
+
+
+_OP16_HANDLE: Any = None
+
+
@lru_cache(maxsize = 1)
def _op() -> Any:
"""``unsloth_studio::int8_mm_dequant`` (opaque to Inductor; no host sync, allocation only), or None."""
@@ -1055,7 +1404,13 @@ def _linear_forward(self: Any, x: Any) -> Any:
import torch
rec = self.__dict__.get(_REC)
- if rec is None or x.dtype != torch.bfloat16 or not x.is_cuda:
+ if rec is None or not x.is_cuda:
+ return type(self).forward(self, x)
+ if self.__dict__.get(_F16):
+ if x.dtype == torch.float16:
+ return _linear_forward_f16(self, x, rec)
+ return type(self).forward(self, x)
+ if x.dtype != torch.bfloat16:
return type(self).forward(self, x)
kind, group, rotq, weight, fused = rec
if self.weight is not weight: # weight replaced since install (reload / LoRA bake): stock
@@ -1307,10 +1662,20 @@ def linear_from_q(q: Any, xs: Any, weight: Any, bias: Any) -> Optional[Any]:
if _OP_HANDLE is None or int8_gemm_mode() == "off" or q.device.index is None:
return None
- if _DEVICE_CFG.get(q.device.index) is None:
- return None
# The fused MLP already admitted this weight (symmetric, and its stock epilogue ignores the zero point too).
parts = _v2_parts(weight, check_zero_point = False)
+ if _DEVICE_CFG.get(q.device.index) is None and not (
+ parts is not None
+ and _cutlass_fused(
+ q.device.index,
+ parts[0].shape[0],
+ parts[0].shape[1],
+ xs.dtype,
+ parts[1].dtype,
+ bias is not None,
+ )
+ ):
+ return None
if (
parts is None
or q.shape[0] < _MIN_ROWS
@@ -1353,9 +1718,10 @@ def _eligible(module: Any) -> Optional[tuple]:
bias = module.bias
import torch
- if bias is not None and bias.dtype != torch.bfloat16:
+ # fp16 bias only reaches the sm75 kernel (``_swap_on`` filters per card)
+ if bias is not None and bias.dtype not in (torch.bfloat16, torch.float16):
return None
- # v1: bf16 scales only; v2 also fp32 (prequant checkpoints). Off-grid K runs masked loads, far slower than cuBLAS.
+ # off-grid K runs masked loads, far slower than cuBLAS
out_f, in_f = getattr(module, "out_features", 0), getattr(module, "in_features", 0)
if in_f % _K_ALIGN or out_f % _N_ALIGN:
return None
@@ -1363,11 +1729,25 @@ def _eligible(module: Any) -> Optional[tuple]:
if parts is not None and parts[1].dtype == torch.bfloat16:
return ("v1", group, parts[1], w)
parts = _v2_parts(w)
- if parts is not None and parts[1].dtype in (torch.bfloat16, torch.float32):
+ if parts is not None and parts[1].dtype in (torch.bfloat16, torch.float32, torch.float16):
return ("v2", group, parts[1], w)
return None
+def _card_dtypes_ok(module: Any, rec: tuple, f16_card: bool) -> bool:
+ import torch
+
+ bias = getattr(module, "bias", None)
+ if f16_card:
+ return rec[0] == "v2" and (bias is None or bias.dtype == torch.float16)
+ scale_dtype = getattr(rec[2], "dtype", None)
+ return (bias is None or bias.dtype == torch.bfloat16) and scale_dtype in (
+ None,
+ torch.bfloat16,
+ torch.float32,
+ )
+
+
def candidates(transformer: Any, rotated_only: bool = False) -> int:
try:
recs = (_eligible(m) for m in transformer.modules())
@@ -1439,11 +1819,12 @@ def install(
if not torch.cuda.is_available() or is_rocm_torch(torch):
return 0
- if not _triton_version_ok() or _kernels() is None:
- return 0
cap = torch.cuda.get_device_capability(torch.cuda.current_device())
+ prebuilt = cutlass_wanted(cap)
+ if not prebuilt and (not _triton_version_ok() or _kernels() is None):
+ return 0
gemm, rotq = _gates(cap)
- gemm = gemm or unlisted_wanted(cap)
+ gemm = gemm or unlisted_wanted(cap) or prebuilt
if not (gemm or rotq):
return 0
except Exception: # noqa: BLE001
@@ -1538,7 +1919,13 @@ def _swap_on(
return 0
from .diffusion_convrot_quant import rotquant_device_config
+ f16_card = tuple(torch.cuda.get_device_capability(index)) == (7, 5)
+ if f16_card:
+ recs = [(m, r) for m, r in recs if _card_dtypes_ok(m, r, True)]
+ if not recs:
+ return 0
gemm = device_config(index) is not None
+ triton_gemm = gemm
rotated = any(r[1] is not None for _, r in recs)
rotq = rotated and rotquant_device_config(index) is not None
if gemm and rotated and not rotq and index in _UNLISTED_CARDS:
@@ -1559,20 +1946,51 @@ def _swap_on(
t["fused"],
t["stock"],
)
- if not gemm:
- # no fused GEMM here: only rotated Linears gain (one kernel instead of rotation GEMM + act quant)
- recs = [(m, r) for m, r in recs if r[1] is not None] if rotq else []
if any(r[0] == "v1" for _, r in recs) and not _v1_act_quant_matches(index):
recs = [(m, r) for m, r in recs if r[0] != "v1"]
+ # bf16 cards: the Triton path takes any bias dtype; only the prebuilt kernel needs bf16 bias
+ cut_recs = [(m, r) for m, r in recs if f16_card or _card_dtypes_ok(m, r, False)]
+ plan = _cutlass_plan(index, cut_recs, triton_gemm, logger) if cut_recs else {}
+ cut_fused: dict = {}
+ if plan:
+ import torch as _torch
+
+ dev = _torch.device("cuda", index)
+ xs_seen: dict = {}
+ act = _torch.float16 if f16_card else _torch.bfloat16
+ for m, r in cut_recs:
+ wq, ws_ = _payload(r[0], r[3])
+ k = int(wq.shape[1])
+ if (r[0], k) not in xs_seen:
+ xs_seen[(r[0], k)] = _xs_dtype(r[0], r[3], k, dev, act)
+ cut_fused[id(m)] = _cutlass_fused(
+ index,
+ wq.shape[0],
+ k,
+ xs_seen[(r[0], k)],
+ ws_.dtype,
+ getattr(m, "bias", None) is not None,
+ )
+ gemm = gemm or any(cut_fused.values())
+ if not triton_gemm:
+ recs = [(m, r) for m, r in recs if (rotq and r[1] is not None) or cut_fused.get(id(m))]
if not recs:
return 0
count = 0
+ global _OP16_HANDLE
with _LOCK:
- if gemm:
+ if f16_card:
+ _OP16_HANDLE = _op16()
+ if _OP16_HANDLE is None:
+ return 0
+ elif gemm:
_OP_HANDLE = _op()
if _OP_HANDLE is None:
return 0
for module, rec in recs:
+ fused = (triton_gemm or bool(cut_fused.get(id(module)))) and not f16_card
+ if f16_card:
+ module.__dict__[_F16] = True
if _MARK in module.__dict__:
if multi:
module.__dict__[_REC] = (
@@ -1580,7 +1998,7 @@ def _swap_on(
rec[1],
bool(rotq and rec[1] is not None),
rec[3],
- gemm,
+ fused,
)
count += 1
continue
@@ -1590,7 +2008,7 @@ def _swap_on(
rec[1],
bool(rotq and rec[1] is not None),
rec[3],
- gemm,
+ fused,
)
module.__dict__[_MARK] = module.__dict__.get("forward", _NO_PREV)
module.forward = types.MethodType(_linear_forward, module)
@@ -1605,6 +2023,16 @@ def _swap_on(
if rotq
else 0
)
+ if logger is not None and plan:
+ n_cut = sum(1 for v in cut_fused.values() if v)
+ logger.info(
+ "diffusion.int8_gemm: %d int8 Linear(s) on cuda:%d run the prebuilt CUTLASS GEMM (%d weight shape(s) "
+ "won the race against %s)",
+ n_cut,
+ index,
+ len(plan),
+ "the Triton kernel" if triton_gemm else "cuBLAS",
+ )
if logger is not None and count:
from .diffusion_convrot_quant import _H16_DEVICE
timing = _UNLISTED_TIMING.get(index) if gemm else None
@@ -1704,6 +2132,7 @@ def uninstall(transformer: Any = None) -> None:
fwd = module.__dict__.get("forward")
if getattr(fwd, "__func__", None) is _swiglu_ff_forward:
module.__dict__.pop("forward", None)
+ module.__dict__.pop(_F16, None)
prev = module.__dict__.pop(_MARK, None)
if prev is None:
continue
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.py
index 5ea074e1efa..312f8f3fd71 100644
--- a/studio/backend/core/inference/diffusion_native_quant.py
+++ b/studio/backend/core/inference/diffusion_native_quant.py
@@ -691,8 +691,21 @@ def native_linear_class():
if not self.rot_group and (torch.compiler.is_compiling() or not xq.is_contiguous()):
op = getattr(self, "_row_major", None)
xq = op(xq) if op is not None else xq.contiguous()
- acc = torch._int_mm(xq, self.weight_q.t())
w_scale = self.weight_scale.view(torch.float32)
+ if (
+ self.bias is None
+ and x2.dtype == torch.float16
+ and x2.is_cuda
+ and not torch.compiler.is_compiling()
+ ):
+ from . import diffusion_int8_cutlass as _cut # noqa: PLC0415
+ tile = _cut.native_tile(xq, self.weight_q, x_scale, w_scale)
+ if tile is not None:
+ try:
+ return _cut.native_gemm(xq, self.weight_q, x_scale, w_scale, tile)
+ except Exception: # noqa: BLE001 - the stock maths below
+ _cut._NATIVE_PLAN.clear()
+ acc = torch._int_mm(xq, self.weight_q.t())
out = torch.mul(acc, x_scale)
if self.bias is not None:
out = torch.addcmul(self.bias.float(), out, w_scale)
@@ -857,6 +870,16 @@ def native_linear_class():
return out.reshape(*x.shape[:-1], self.out_features)
def forward(self, x: Any) -> Any:
+ if (
+ self.act_int8
+ and self.bias is None
+ and x.is_cuda
+ and x.dtype == torch.float16
+ and not torch.compiler.is_compiling()
+ ):
+ # race once per card and shape, before any graph capture
+ from . import diffusion_int8_cutlass as _cut # noqa: PLC0415
+ _cut.native_prepare(self.weight_q, x.dtype, False)
if self._int8pack_ok(x):
out = self._forward_int8pack(x)
if out is not None:
diff --git a/studio/backend/tests/test_diffusion_int8_cutlass.py b/studio/backend/tests/test_diffusion_int8_cutlass.py
new file mode 100644
index 00000000000..b17c060507e
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_int8_cutlass.py
@@ -0,0 +1,433 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for the prebuilt CUTLASS int8 GEMM (``diffusion_int8_cutlass``) and its dispatch in ``diffusion_int8_gemm``.
+The loader, gates, reference, plan dispatch and build generator run anywhere; the kernel tests need NVIDIA CUDA and
+the built library (``UNSLOTH_DIFFUSION_INT8_CUTLASS_LIB`` or the ``unsloth-studio-i8gemm`` wheel)."""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+from core.inference import diffusion_int8_cutlass as cut
+from core.inference import diffusion_int8_gemm as g8
+
+torch = pytest.importorskip("torch")
+
+_BUILD = Path(__file__).resolve().parents[2] / "native_kernels" / "int8_gemm" / "build.py"
+# read before the autouse fixture clears it, for the GPU tests at the bottom
+_REAL_LIB = os.environ.get(cut.CUTLASS_LIB_ENV)
+
+
[email protected](autouse = True)
+def _clean(monkeypatch):
+ for env in (cut.CUTLASS_ENV, cut.CUTLASS_LIB_ENV, g8.INT8_GEMM_ENV):
+ monkeypatch.delenv(env, raising = False)
+ cut.library.cache_clear()
+ cut.tiles.cache_clear()
+ cut._PROBED.clear()
+ cut._LOAD_ERROR[0] = None
+ g8._CUT_PLAN.clear()
+ g8._CUT_FAMILY.clear()
+ g8._CUT_RACES.clear()
+ g8._CUT_DONE.clear()
+ yield
+ monkeypatch.undo()
+ cut.library.cache_clear()
+ cut.tiles.cache_clear()
+ cut._PROBED.clear()
+ g8._CUT_PLAN.clear()
+ g8._CUT_FAMILY.clear()
+
+
+ "cap, fam",
+ [
+ ((7, 5), 75),
+ ((8, 0), 80),
+ ((8, 6), 80),
+ ((8, 7), 80),
+ ((8, 9), 80),
+ ((9, 0), None),
+ ((10, 0), None),
+ ((12, 0), None),
+ ((7, 0), None),
+ (None, None),
+ ],
+)
+def test_family_covers_turing_ampere_ada_only(cap, fam):
+ assert cut.family(cap) == fam
+
+
+ "raw, want", [("0", "off"), ("off", "off"), ("1", "force"), ("", "auto"), ("auto", "auto")]
+)
+def test_mode(monkeypatch, raw, want):
+ monkeypatch.setenv(cut.CUTLASS_ENV, raw)
+ assert cut.mode() == want
+
+
+def test_missing_or_broken_library_keeps_todays_path(monkeypatch, tmp_path):
+ bad = tmp_path / "libunsloth_i8gemm.so"
+ bad.write_bytes(b"not an ELF")
+ monkeypatch.setenv(cut.CUTLASS_LIB_ENV, str(bad))
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ assert cut.library() is None
+ assert cut.load_error() and str(bad) in cut.load_error()
+ assert cut.tiles(80) == ()
+ assert not g8.cutlass_wanted((8, 9))
+
+
+def test_kill_switch_never_loads(monkeypatch):
+ monkeypatch.setenv(cut.CUTLASS_ENV, "0")
+ calls = []
+ monkeypatch.setattr(cut, "_candidate_paths", lambda: calls.append(1) or [])
+ assert cut.library() is None
+ assert calls == []
+ assert not g8.cutlass_wanted((8, 0))
+
+
+def test_rocm_never_loads(monkeypatch):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.version, "hip", "6.4", raising = False)
+ monkeypatch.setattr(cut, "_candidate_paths", lambda: pytest.fail("probed on ROCm"))
+ assert cut.library() is None
+
+
+def test_wheel_package_is_found_without_importing_it(monkeypatch, tmp_path):
+ pkg = tmp_path / cut.WHEEL_MODULE
+ (pkg / "lib").mkdir(parents = True)
+ (pkg / "__init__.py").write_text("raise RuntimeError('must not be imported')\n")
+ lib = pkg / "lib" / cut.LIB_NAMES[0]
+ lib.write_bytes(b"x")
+ monkeypatch.syspath_prepend(str(tmp_path))
+ importlib.invalidate_caches()
+ assert cut._candidate_paths() == [str(lib)]
+ assert cut.WHEEL_MODULE not in sys.modules
+
+
+def test_explicit_path_wins(monkeypatch, tmp_path):
+ monkeypatch.setenv(cut.CUTLASS_LIB_ENV, str(tmp_path / "x.so"))
+ assert cut._candidate_paths() == [str(tmp_path / "x.so")]
+
+
+def test_cutlass_wanted_for_turing_ampere_ada_only(monkeypatch):
+ monkeypatch.setattr(cut, "library", lambda: object())
+ assert g8.cutlass_wanted((8, 9)) and g8.cutlass_wanted((8, 6)) and g8.cutlass_wanted((8, 0))
+ assert g8.cutlass_wanted((7, 5))
+ assert (
+ not g8.cutlass_wanted((9, 0))
+ and not g8.cutlass_wanted((12, 0))
+ and not g8.cutlass_wanted((10, 0))
+ )
+ monkeypatch.setenv(g8.INT8_GEMM_ENV, "0")
+ assert not g8.cutlass_wanted((8, 9))
+
+
+def _operands(
+ m,
+ n,
+ k,
+ xs_dt,
+ ws_dt,
+ bias,
+ seed = 0,
+ out = torch.bfloat16,
+):
+ g = torch.Generator().manual_seed(seed)
+ a = torch.randint(-127, 128, (m, k), generator = g, dtype = torch.int8)
+ w = torch.randint(-127, 128, (n, k), generator = g, dtype = torch.int8)
+ xs = (torch.rand(m, generator = g) * 0.02 + 1e-4).to(xs_dt)
+ ws = (torch.rand(n, generator = g) * 0.002 + 1e-5).to(ws_dt)
+ b = (torch.randn(n, generator = g) * 0.1).to(out) if bias else None
+ return a, w, xs, ws, b
+
+
+def _cpu_int_mm_ok():
+ try:
+ torch._int_mm(torch.ones(17, 16, dtype = torch.int8), torch.ones(16, 8, dtype = torch.int8))
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
[email protected](not _cpu_int_mm_ok(), reason = "torch._int_mm needs CPU support")
[email protected]("xs_dt", [torch.bfloat16, torch.float32])
[email protected]("ws_dt", [torch.bfloat16, torch.float32])
[email protected]("bias", [False, True])
[email protected]("m", [3, 33])
+def test_generic_reference_is_studios_bf16_reference(xs_dt, ws_dt, bias, m):
+ a, w, xs, ws, b = _operands(m, 48, 64, xs_dt, ws_dt, bias)
+ got = cut.reference(a, w, xs, ws, b, torch.bfloat16)
+ want = g8.reference(a, w, xs, ws, b)
+ assert got.dtype == torch.bfloat16
+ assert torch.equal(got.view(torch.int16), want.view(torch.int16))
+
+
[email protected](not _cpu_int_mm_ok(), reason = "torch._int_mm needs CPU support")
[email protected]("ws_dt", [torch.float16, torch.bfloat16, torch.float32])
+def test_fp16_reference_is_the_fused_paths_torchao_epilogue(ws_dt):
+ from core.inference.diffusion_int8_fused import _torchao_epilogue
+
+ a, w, xs, ws, b = _operands(40, 32, 64, torch.float16, ws_dt, True, out = torch.float16)
+ xs = xs * 0.01
+ c = torch._int_mm(a, w.t())
+ want = _torchao_epilogue(c, xs, ws, b, torch.float16)
+ got = cut.reference(a, w, xs, ws, b, torch.float16)
+ assert torch.equal(got.view(torch.int16), want.view(torch.int16))
+
+
[email protected](not _cpu_int_mm_ok(), reason = "torch._int_mm needs CPU support")
+def test_op_body_runs_the_race_winner_then_falls_back(monkeypatch):
+ a, w, xs, ws, b = _operands(1100, 64, 64, torch.bfloat16, torch.bfloat16, False)
+ key = g8._cut_key(64, 64, xs.dtype, ws.dtype, False)
+ g8._CUT_PLAN[None] = {key: {512: None, 4096: 3}}
+ g8._CUT_FAMILY[None] = 80
+ monkeypatch.setattr(cut, "eligible", lambda *a_: True)
+ calls = []
+
+ def fake_launch(a_, w_, xs_, ws_, b_, fam, tile, out):
+ calls.append((fam, tile, a_.shape[0]))
+ return torch.zeros(a_.shape[0], w_.shape[0], dtype = out)
+
+ monkeypatch.setattr(cut, "launch", fake_launch)
+ y = g8._run(a, w, xs, ws, b)
+ assert calls == [(80, 3, 1100)] and not y.any()
+ # small M lost its race: today's path
+ small = a[:200]
+ y = g8._run(small, w, xs[:200], ws, b)
+ assert len(calls) == 1
+ assert torch.equal(
+ y.view(torch.int16), g8.reference(small, w, xs[:200], ws, b).view(torch.int16)
+ )
+
+ def broken(*a_):
+ raise RuntimeError("launch failed")
+
+ monkeypatch.setattr(cut, "launch", broken)
+ y = g8._run(a, w, xs, ws, b)
+ assert None not in g8._CUT_PLAN # dropped on this card
+ assert torch.equal(y.view(torch.int16), g8.reference(a, w, xs, ws, b).view(torch.int16))
+
+
+def test_fused_needs_the_large_m_win():
+ key = g8._cut_key(64, 128, torch.bfloat16, torch.float32, True)
+ g8._CUT_PLAN[0] = {key: {512: 2, 4096: None}}
+ assert not g8._cutlass_fused(0, 64, 128, torch.bfloat16, torch.float32, True)
+ g8._CUT_PLAN[0] = {key: {512: None, 4096: 1}}
+ assert g8._cutlass_fused(0, 64, 128, torch.bfloat16, torch.float32, True)
+ assert not g8._cutlass_fused(
+ 0, 64, 128, torch.bfloat16, torch.float32, False
+ ) # bias is part of the key
+ assert not g8._cutlass_fused(1, 64, 128, torch.bfloat16, torch.float32, True) # per card
+
+
+def test_race_margin_keeps_todays_path_on_a_tie(monkeypatch):
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda i = None: (8, 9))
+ monkeypatch.setattr(cut, "probed_tiles", lambda i: (0, 1))
+ monkeypatch.setattr(cut, "has_kernel", lambda *a: True)
+ monkeypatch.setattr(cut, "eligible", lambda *a: True)
+ monkeypatch.setattr(torch.cuda, "device", lambda d: __import__("contextlib").nullcontext())
+ results = {}
+
+ def fake_time(
+ fns,
+ rounds = 3,
+ reps = 3,
+ warm = 1,
+ ):
+ return {k: results[k] for k in fns}
+
+ monkeypatch.setattr(cut, "time_us", fake_time)
+ w = torch.zeros(64, 64, dtype = torch.int8)
+ ws = torch.ones(64, dtype = torch.bfloat16)
+ results.update({0: 100.0, 1: 99.0, "today": 100.0})
+ r = cut.race(0, w, ws, None, torch.bfloat16, lambda *a: None, 64)
+ assert r["tile"] is None # 1% faster is inside the margin
+ results.update({0: 90.0, 1: 97.0, "today": 100.0})
+ r = cut.race(0, w, ws, None, torch.bfloat16, lambda *a: None, 64)
+ assert r["tile"] == 0
+
+
+def _load_build():
+ spec = importlib.util.spec_from_file_location("i8gemm_build", _BUILD)
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ return mod
+
+
+def test_build_generator_instantiates_every_variant_per_tile():
+ b = _load_build()
+ for name, fam in b.FAMILIES.items():
+ n_var = len(b.variants(fam))
+ native = len(fam["out"]) if fam.get("native") else 0
+ assert n_var == len(fam["out"]) * len(fam["xs"]) * len(fam["ws"]) * 2 + native
+ for i in range(len(fam["tiles"])):
+ src = b.tile_tu(name, fam, i)
+ assert src.count("return &ui8::FusedGemm<") == n_var
+ assert f"ui8_{name}_t{i}(" in src
+ api = b.api_tu()
+ for sym in ("ui8_abi_version", "ui8_num_tiles", "ui8_tile_info", "ui8_has_kernel", "ui8_gemm"):
+ assert f"UI8_API int {sym}(" in api
+ assert f"return {cut.ABI_VERSION};" in api # loader and library agree on the ABI
+ assert "int epilogue" in api
+ assert b.FAMILIES["sm75"].get("native") and not b.FAMILIES["sm80"].get("native")
+ assert any(e == cut.EPI_NATIVE for *_, e in b.variants(b.FAMILIES["sm75"]))
+
+
+def test_python_kinds_match_the_library_kinds():
+ b = _load_build()
+ assert cut._kind(torch.float32) == b.KIND["float"]
+ assert cut._kind(torch.bfloat16) == b.KIND["cutlass::bfloat16_t"]
+ assert cut._kind(torch.float16) == b.KIND["cutlass::half_t"]
+ assert set(b.FAMILY_ID) == set(b.FAMILIES) and set(b.FAMILY_ID.values()) == {75, 80}
+ assert cut.family((7, 5)) == b.FAMILY_ID["sm75"] and cut.family((8, 9)) == b.FAMILY_ID["sm80"]
+
+
[email protected](not _cpu_int_mm_ok(), reason = "torch._int_mm needs CPU support")
+def test_native_reference_is_the_native_linears_epilogue():
+ a, w, _xs, _ws, _ = _operands(40, 32, 64, torch.float32, torch.float32, False)
+ xs = torch.rand(40, 1) * 1e-3 + 1e-6
+ ws = torch.rand(32) * 1e-3 + 1e-6
+ acc = torch._int_mm(a, w.t())
+ want = (
+ torch.mul(acc, xs).mul_(ws).to(torch.float16)
+ ) # NativeWeightOnlyLinear._int_mm_rows, no bias
+ got = cut.reference_native(a, w, xs, ws, torch.float16)
+ assert torch.equal(got.view(torch.int16), want.view(torch.int16))
+
+
+def test_native_plan_needs_fp16_no_bias_and_a_won_race(monkeypatch):
+ w = torch.zeros(64, 64, dtype = torch.int8)
+ cut._NATIVE_PLAN.clear()
+ cut.native_prepare(w, torch.float16, False) # CPU weight: nothing raced
+ cut.native_prepare(w, torch.bfloat16, False)
+ assert cut._NATIVE_PLAN == {}
+ cut._NATIVE_PLAN[(None, 64, 64)] = {512: None, 4096: 2}
+ monkeypatch.setattr(cut, "eligible", lambda *a: True)
+ xs = torch.ones(2000, 1)
+ assert cut.native_tile(torch.zeros(2000, 64, dtype = torch.int8), w, xs, torch.ones(64)) == 2
+ assert (
+ cut.native_tile(torch.zeros(100, 64, dtype = torch.int8), w, xs[:100], torch.ones(64)) is None
+ )
+ assert (
+ cut.native_tile(torch.zeros(2000, 64, dtype = torch.int8), w, xs.half(), torch.ones(64))
+ is None
+ )
+ cut._NATIVE_PLAN.clear()
+
+
+# ------------------------------------------------------------------------------------------------- GPU (needs the lib)
+
+_GPU = pytest.mark.skipif(
+ not (torch.cuda.is_available() and _REAL_LIB),
+ reason = "needs CUDA and UNSLOTH_DIFFUSION_INT8_CUTLASS_LIB",
+)
+
+
+def real_lib(monkeypatch):
+ monkeypatch.setenv(cut.CUTLASS_LIB_ENV, _REAL_LIB)
+ cut.library.cache_clear()
+ cut.tiles.cache_clear()
+ cut._PROBED.clear()
+
+
+@_GPU
+def test_library_probe_and_bit_exact_launch(real_lib):
+ cap = torch.cuda.get_device_capability(0)
+ fam = cut.family(cap)
+ if fam is None:
+ pytest.skip("arch outside the library")
+ assert cut.library() is not None, cut.load_error()
+ ok = cut.probed_tiles(0)
+ assert ok, "no tile passed the bit-exact probe"
+ out = cut.out_dtype_for(fam)
+ for xs_dt, ws_dt in cut._scale_dtypes(fam):
+ for bias in (False, True):
+ a, w, xs, ws, b = _operands(1031, 528, 1040, xs_dt, ws_dt, bias, out = out)
+ if fam == 75:
+ xs = xs * 0.01
+ a, w, xs, ws = a.cuda(), w.cuda(), xs.cuda(), ws.cuda()
+ b = b.cuda() if b is not None else None
+ ref = cut.reference(a, w, xs, ws, b, out)
+ for tile in ok:
+ y = cut.launch(a, w, xs, ws, b, fam, tile, out)
+ assert torch.equal(y.view(torch.int16), ref.view(torch.int16)), (
+ tile,
+ xs_dt,
+ ws_dt,
+ bias,
+ )
+ if fam == 75: # the native W8A8 Linear's epilogue
+ a, w, _x, _w, _ = _operands(1031, 528, 1040, torch.float32, torch.float32, False, out = out)
+ a, w = a.cuda(), w.cuda()
+ xs = (torch.rand(1031) * 1e-4 + 1e-6).cuda()
+ ws = (torch.rand(528) * 1e-3 + 1e-6).cuda()
+ ref = cut.reference_native(a, w, xs, ws, out)
+ for tile in ok:
+ y = cut.launch(a, w, xs, ws, None, fam, tile, out, cut.EPI_NATIVE)
+ assert torch.equal(y.view(torch.int16), ref.view(torch.int16)), tile
+
+
+@_GPU
+def test_launch_records_into_a_cuda_graph(real_lib):
+ fam = cut.family(torch.cuda.get_device_capability(0))
+ if fam is None or not cut.probed_tiles(0):
+ pytest.skip("no probed tile")
+ out = cut.out_dtype_for(fam)
+ xs_dt, ws_dt = cut._scale_dtypes(fam)[0]
+ a, w, xs, ws, _ = _operands(256, 256, 256, xs_dt, ws_dt, False, out = out)
+ a, w, xs, ws = a.cuda(), w.cuda(), (xs * 0.01).cuda(), ws.cuda()
+ tile = cut.probed_tiles(0)[0]
+ cut.launch(a, w, xs, ws, None, fam, tile, out)
+ torch.cuda.synchronize()
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ y = cut.launch(a, w, xs, ws, None, fam, tile, out)
+ a.copy_(torch.randint(-127, 128, a.shape, dtype = torch.int8, device = "cuda"))
+ graph.replay()
+ torch.cuda.synchronize()
+ assert torch.equal(
+ y.view(torch.int16), cut.reference(a, w, xs, ws, None, out).view(torch.int16)
+ )
+
+
+def test_a_later_load_plans_only_its_new_shapes(monkeypatch):
+ class _W:
+ def __init__(self, n, k):
+ self.qdata = torch.zeros(n, k, dtype = torch.int8)
+ self.scale = torch.ones(n, 1, dtype = torch.bfloat16)
+
+ def _rec(n, k):
+ return (torch.nn.Linear(k, n, bias = False), ("v2", None, None, _W(n, k)))
+
+ a, b = _rec(64, 32), _rec(128, 64)
+ key_a = g8._cut_key(64, 32, torch.float32, torch.bfloat16, False)
+ key_b = g8._cut_key(128, 64, torch.float32, torch.bfloat16, False)
+ table = {"ck": {repr(key_a): {"512": 1, "4096": 1}, repr(key_b): {"512": 2, "4096": 2}}}
+ monkeypatch.setattr(g8, "cutlass_wanted", lambda cap: True)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: (8, 0))
+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
+ monkeypatch.setattr(cut, "probed_tiles", lambda index: (1, 2))
+ monkeypatch.setattr(g8, "_cut_cache_key", lambda index: "ck")
+ monkeypatch.setattr(g8, "_cut_cache_read", lambda: table)
+ monkeypatch.setattr(g8, "_cut_cache_write", lambda t: None)
+ monkeypatch.setattr(g8, "_xs_dtype", lambda *a_: torch.float32)
+ races = []
+ monkeypatch.setattr(cut, "race", lambda *a_, **k_: races.append(a_) or {"tile": None})
+
+ assert set(g8._cutlass_plan(0, [a], True)) == {key_a}
+ assert set(g8._cutlass_plan(0, [a, b], True)) == {key_a, key_b}
+ assert g8._CUT_PLAN[0][key_b] == {512: 2, 4096: 2} and not races
+
+
+def test_the_kernel_sources_do_not_shadow_the_kernels_package():
+ # CI and Studio put studio/ on sys.path, where a studio/kernels folder imports as the HF ``kernels`` package.
+ studio = Path(__file__).resolve().parents[2]
+ assert _BUILD.is_file() and not (studio / "kernels").exists()
diff --git a/studio/backend/utils/wheel_utils.py b/studio/backend/utils/wheel_utils.py
index 6d644e204fe..829b5af514d 100644
--- a/studio/backend/utils/wheel_utils.py
+++ b/studio/backend/utils/wheel_utils.py
@@ -169,6 +169,25 @@ MAMBA_SSM_PURE_VERSION = "2.3.2.post1"
MAMBA_SSM_PURE_WHEEL = f"mamba_ssm-{MAMBA_SSM_PURE_VERSION}-py3-none-any.whl"
+# studio/native_kernels/int8_gemm: C ABI + static cudart, so one py3-none wheel per platform fits every torch and CUDA 12/13.
+I8GEMM_DISTRIBUTION = "unsloth-studio-i8gemm"
+I8GEMM_VERSION = "1.0.0"
+_I8GEMM_PLATFORMS = {"linux_x86_64": "manylinux_2_28_x86_64", "win_amd64": "win_amd64"}
+
+
+def int8_gemm_wheel_url(env: dict[str, str] | None) -> str | None:
+ """The prebuilt int8 GEMM wheel for an NVIDIA CUDA torch on x86_64 Linux / Windows, else None."""
+ if env is None or env.get("hip_version") or not env.get("cuda_major"):
+ return None
+ platform = _I8GEMM_PLATFORMS.get(str(env.get("platform_tag") or ""))
+ if platform is None:
+ return None
+ return (
+ f"{UNSLOTH_PREBUILT_RELEASE_BASE_URL}/{UNSLOTH_PREBUILT_RELEASE_TAG}/"
+ f"unsloth_studio_i8gemm-{I8GEMM_VERSION}-py3-none-{platform}.whl"
+ )
+
+
def mamba_ssm_pure_wheel_url(env: dict[str, str] | None = None) -> str:
"""The extension-free mamba_ssm wheel (the same file for every environment)."""
return (
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py
index b54a1c59b76..6e55201023e 100644
--- a/studio/install_python_stack.py
+++ b/studio/install_python_stack.py
@@ -61,8 +61,11 @@ from backend.utils.kernel_install import (
uninstall_command,
)
from backend.utils.wheel_utils import (
+ I8GEMM_DISTRIBUTION,
+ I8GEMM_VERSION,
flash_attn_package_version,
flash_attn_wheel_url,
+ int8_gemm_wheel_url,
install_wheel,
probe_torch_wheel_env,
url_exists,
@@ -7887,6 +7890,49 @@ def _remove_stale_flash_attn() -> None:
)
+def _int8_gemm_install_disabled() -> bool:
+ if os.getenv("UNSLOTH_STUDIO_SKIP_I8GEMM_INSTALL") == "1":
+ return True
+ return (os.getenv("UNSLOTH_DIFFUSION_INT8_CUTLASS") or "").strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _ensure_int8_gemm() -> None:
+ """Optional prebuilt int8 GEMM wheel (diffusion INT8, sm75 / sm8x); every failure only warns."""
+ if _int8_gemm_install_disabled() or NO_TORCH or IS_MACOS:
+ return
+ if _installed_distribution_version(I8GEMM_DISTRIBUTION) == I8GEMM_VERSION:
+ return
+ env = probe_torch_wheel_env(include_windows = True)
+ wheel_url = int8_gemm_wheel_url(env)
+ if not wheel_url or not url_exists(wheel_url):
+ return
+ _count_install_action()
+ with _install_heartbeat("int8 GEMM"):
+ outcome = install_prebuilt(
+ wheel_url,
+ install = install_wheel,
+ verify = lambda: _installed_distribution_version(I8GEMM_DISTRIBUTION) == I8GEMM_VERSION,
+ on_failed = lambda installer, wheel_result: _print_optional_install_failure(
+ f"Installing the prebuilt int8 GEMM wheel with {installer}",
+ wheel_result,
+ ),
+ use_uv = USE_UV,
+ uv_needs_system = UV_NEEDS_SYSTEM,
+ reinstall = _installed_distribution_version(I8GEMM_DISTRIBUTION) is not None,
+ )
+ if outcome != "installed":
+ _step(
+ "warning",
+ "Continuing without the prebuilt int8 GEMM (diffusion INT8 keeps Triton / cuBLAS)",
+ _warn,
+ )
+
+
def _flash_attn_gpu_below_floor() -> bool:
"""Visible CUDA GPU below sm80 (a T4); no visible GPU is False, since image builds install first."""
return gpu_below_flash_attn_floor()
@@ -12181,6 +12227,7 @@ def install_python_stack() -> int:
_progress("flash-attn")
_ensure_flash_attn()
_remove_stale_ssm_kernels()
+ _ensure_int8_gemm()
# 8. Unsloth dependencies
if not _skip_step(REQ_ROOT / "studio.txt", "Unsloth Studio deps", no_deps = False):
diff --git a/studio/native_kernels/int8_gemm/build.py b/studio/native_kernels/int8_gemm/build.py
new file mode 100644
index 00000000000..465dde49a7d
--- /dev/null
+++ b/studio/native_kernels/int8_gemm/build.py
@@ -0,0 +1,302 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Build the prebuilt int8 GEMM library, one nvcc unit per (family, tile), static CUDA runtime.
+
+ python build.py --cutlass /path/to/cutlass --out build/ [--jobs 16] [--families sm80,sm75]
+
+SASS only (no PTX): other archs keep Studio's other paths instead of a JIT build.
+"""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+ABI_VERSION = 1
+
+# (TBM, TBN, TBK, WM, WN, WK, stages, swizzle). The index is the config id the loader races; append only.
+FAMILIES = {
+ "sm80": {
+ "arch_tag": "cutlass::arch::Sm80",
+ "gencode": ("80", "86", "89"),
+ "out": ("cutlass::bfloat16_t",),
+ "xs": ("cutlass::bfloat16_t", "float"),
+ "ws": ("cutlass::bfloat16_t", "float"),
+ "tiles": (
+ (128, 256, 64, 64, 64, 64, 3, 1),
+ (256, 128, 64, 64, 64, 64, 3, 1),
+ (128, 128, 64, 64, 64, 64, 4, 1),
+ (128, 128, 128, 64, 64, 128, 3, 1),
+ (64, 128, 64, 32, 64, 64, 4, 1),
+ (128, 256, 64, 64, 64, 64, 3, 4),
+ (128, 128, 64, 64, 64, 64, 4, 4),
+ (64, 64, 64, 32, 32, 64, 5, 1),
+ (256, 128, 64, 64, 64, 64, 3, 4),
+ # 96 KB of stages fits sm86 / sm89's 99 KB
+ (128, 256, 64, 64, 64, 64, 4, 1),
+ (256, 128, 64, 64, 64, 64, 4, 1),
+ (128, 128, 64, 64, 64, 64, 6, 1),
+ (128, 256, 64, 64, 64, 64, 4, 4),
+ (256, 128, 128, 64, 64, 128, 2, 1),
+ (128, 256, 128, 64, 64, 128, 2, 1),
+ ),
+ },
+ "sm75": {
+ "arch_tag": "cutlass::arch::Sm75",
+ "gencode": ("75",),
+ "out": ("cutlass::half_t",),
+ "xs": ("cutlass::half_t", "float"),
+ "ws": ("cutlass::half_t", "cutlass::bfloat16_t", "float"),
+ "native": True,
+ "tiles": (
+ (128, 256, 64, 64, 64, 64, 2, 1),
+ (256, 128, 64, 64, 64, 64, 2, 1),
+ (128, 128, 64, 64, 64, 64, 2, 1),
+ (64, 128, 64, 32, 64, 64, 2, 1),
+ (128, 256, 64, 64, 64, 64, 2, 4),
+ (128, 128, 64, 64, 64, 64, 2, 4),
+ (64, 64, 64, 32, 32, 64, 2, 1),
+ ),
+ },
+}
+
+KIND = {"float": 0, "cutlass::bfloat16_t": 1, "cutlass::half_t": 2}
+FAMILY_ID = {"sm80": 80, "sm75": 75}
+
+
+def variants(fam: dict) -> list:
+ """(out, xs, ws, bias, epilogue); epilogue 0 = torchao's, 1 = the native W8A8 Linear's."""
+ out = [
+ (o, x, w, b, 0)
+ for o in fam["out"]
+ for x in fam["xs"]
+ for w in fam["ws"]
+ for b in (False, True)
+ ]
+ if fam.get("native"):
+ out += [(o, "float", "float", False, 1) for o in fam["out"]]
+ return out
+
+
+def tile_tu(name: str, fam: dict, index: int) -> str:
+ t = fam["tiles"][index]
+ lines = [
+ '#include "i8gemm_impl.cuh"',
+ "namespace {",
+ f"using T = ui8::Tile<{', '.join(str(v) for v in t)}>;",
+ "}",
+ f'extern "C" ui8::RunFn ui8_{name}_t{index}(int out, int xs, int ws, int bias, int epi) {{',
+ ]
+ for o, x, w, b, e in variants(fam):
+ lines.append(
+ f" if (out == {KIND[o]} && xs == {KIND[x]} && ws == {KIND[w]} && bias == {int(b)} && epi == {e}) "
+ f"return &ui8::FusedGemm<{fam['arch_tag']}, T, {o}, {x}, {w}, {str(b).lower()}, {e}>::run;"
+ )
+ lines += [" return nullptr;", "}", ""]
+ return "\n".join(lines)
+
+
+def api_tu() -> str:
+ decls, tables = [], []
+ for name, fam in FAMILIES.items():
+ n = len(fam["tiles"])
+ for i in range(n):
+ decls.append(f'extern "C" ui8::RunFn ui8_{name}_t{i}(int, int, int, int, int);')
+ tables.append(
+ f"static ui8::RunFn (*const k_{name}[])(int, int, int, int, int) = {{"
+ + ", ".join(f"&ui8_{name}_t{i}" for i in range(n))
+ + "};"
+ )
+ tables.append(
+ f"static const int k_{name}_tiles[][8] = {{"
+ + ", ".join("{" + ", ".join(str(v) for v in t) + "}" for t in fam["tiles"])
+ + "};"
+ )
+ fam_cases = "\n".join(
+ f" case {FAMILY_ID[name]}: *table = k_{name}; *count = {len(fam['tiles'])}; *tiles = k_{name}_tiles; break;"
+ for name, fam in FAMILIES.items()
+ )
+ return f"""#include "i8gemm_impl.cuh"
+#if defined(_WIN32)
+#define UI8_API __declspec(dllexport)
+#else
+#define UI8_API __attribute__((visibility("default")))
+#endif
+{chr(10).join(decls)}
+{chr(10).join(tables)}
+
+static bool family_table(int family, ui8::RunFn (*const** table)(int, int, int, int, int), int* count,
+ const int (**tiles)[8]) {{
+ switch (family) {{
+{fam_cases}
+ default: return false;
+ }}
+ return true;
+}}
+
+extern "C" {{
+UI8_API int ui8_abi_version() {{ return {ABI_VERSION}; }}
+
+// Number of tiles for a family (80 or 75), 0 when absent.
+UI8_API int ui8_num_tiles(int family) {{
+ ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;
+ int count = 0;
+ const int (*tiles)[8] = nullptr;
+ return family_table(family, &table, &count, &tiles) ? count : 0;
+}}
+
+// The 8 tile parameters (TBM, TBN, TBK, WM, WN, WK, stages, swizzle) into out8; 0 on success.
+UI8_API int ui8_tile_info(int family, int tile, int* out8) {{
+ ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;
+ int count = 0;
+ const int (*tiles)[8] = nullptr;
+ if (!family_table(family, &table, &count, &tiles) || tile < 0 || tile >= count || out8 == nullptr) return 1;
+ for (int i = 0; i < 8; ++i) out8[i] = tiles[tile][i];
+ return 0;
+}}
+
+// 1 when (family, tile, out, xs, ws, bias, epilogue) is compiled in; epilogue 0 torchao, 1 native W8A8.
+UI8_API int ui8_has_kernel(int family, int tile, int out_kind, int xs_kind, int ws_kind, int has_bias, int epilogue) {{
+ ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;
+ int count = 0;
+ const int (*tiles)[8] = nullptr;
+ if (!family_table(family, &table, &count, &tiles) || tile < 0 || tile >= count) return 0;
+ return table[tile](out_kind, xs_kind, ws_kind, has_bias ? 1 : 0, epilogue) != nullptr;
+}}
+
+// out[M, N] = dequant(a[M, K] @ w[N, K]^T); kinds: 0 fp32, 1 bf16, 2 fp16. bias may be null. Status code returned.
+UI8_API int ui8_gemm(int family, int tile, int out_kind, int xs_kind, int ws_kind, int epilogue, const void* a, const void* w,
+ const void* xs, const void* ws, const void* bias, void* out, int64_t M, int64_t N, int64_t K,
+ int64_t lda, int64_t ldw, int64_t ldc, void* stream) {{
+ ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;
+ int count = 0;
+ const int (*tiles)[8] = nullptr;
+ if (!family_table(family, &table, &count, &tiles) || tile < 0 || tile >= count) return ui8::kNoKernel;
+ if (M <= 0 || N <= 0 || K <= 0 || M > 0x7fffffff || N > 0x7fffffff || K > 0x7fffffff) return ui8::kBadArgs;
+ if (a == nullptr || w == nullptr || xs == nullptr || ws == nullptr || out == nullptr) return ui8::kBadArgs;
+ if (lda < K || ldw < K || ldc < N) return ui8::kBadArgs;
+ ui8::RunFn fn = table[tile](out_kind, xs_kind, ws_kind, bias != nullptr ? 1 : 0, epilogue);
+ if (fn == nullptr) return ui8::kNoKernel;
+ return fn(a, w, xs, ws, bias, out, int(M), int(N), int(K), lda, ldw, ldc, static_cast<cudaStream_t>(stream));
+}}
+}}
+"""
+
+
+def lib_name() -> str:
+ return "unsloth_i8gemm.dll" if os.name == "nt" else "libunsloth_i8gemm.so"
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--cutlass", required = True, help = "CUTLASS checkout (its include/ is used)")
+ ap.add_argument("--out", required = True)
+ ap.add_argument("--nvcc", default = shutil.which("nvcc") or "nvcc")
+ ap.add_argument("--jobs", type = int, default = os.cpu_count() or 4)
+ ap.add_argument("--families", default = ",".join(FAMILIES))
+ ap.add_argument(
+ "--tiles", default = "", help = "comma separated tile ids to keep (default all; testing only)"
+ )
+ ap.add_argument(
+ "--sm",
+ default = "",
+ help = "comma separated SASS targets to keep, e.g. 89 (default all; testing only)",
+ )
+ args = ap.parse_args()
+
+ out = Path(args.out).resolve()
+ src = out / "src"
+ obj = out / "obj"
+ src.mkdir(parents = True, exist_ok = True)
+ obj.mkdir(parents = True, exist_ok = True)
+ shutil.copy2(HERE / "i8gemm_impl.cuh", src / "i8gemm_impl.cuh")
+ keep_fams = [f for f in args.families.split(",") if f]
+ for name in list(FAMILIES):
+ if name not in keep_fams:
+ del FAMILIES[name]
+ if args.sm:
+ want = set(args.sm.split(","))
+ for name in list(FAMILIES):
+ FAMILIES[name]["gencode"] = tuple(sm for sm in FAMILIES[name]["gencode"] if sm in want)
+ if not FAMILIES[name]["gencode"]:
+ del FAMILIES[name]
+ if args.tiles:
+ keep = [int(t) for t in args.tiles.split(",")]
+ for fam in FAMILIES.values():
+ fam["tiles"] = tuple(t for i, t in enumerate(fam["tiles"]) if i in keep)
+
+ include = Path(args.cutlass).resolve() / "include"
+ common = [
+ args.nvcc,
+ "-std=c++17",
+ "-O3",
+ "-DNDEBUG",
+ "--expt-relaxed-constexpr",
+ f"-I{src}",
+ f"-I{include}",
+ "-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1",
+ ]
+ if os.name == "nt":
+ # static MSVC runtime (no DLL dependency), the conforming preprocessor CCCL needs
+ common += ["-Xcompiler", "/MT", "-Xcompiler", "/Zc:preprocessor", "--diag-suppress=177"]
+ else:
+ common += ["-Xcompiler", "-fPIC", "-Xcompiler", "-fvisibility=hidden"]
+ units = []
+ for name, fam in FAMILIES.items():
+ gencode = []
+ for sm in fam["gencode"]:
+ gencode += ["-gencode", f"arch=compute_{sm},code=sm_{sm}"]
+ for i in range(len(fam["tiles"])):
+ path = src / f"{name}_t{i}.cu"
+ path.write_text(tile_tu(name, fam, i), encoding = "utf-8")
+ units.append((path, gencode))
+ api = src / "i8gemm_api.cu"
+ api.write_text(api_tu(), encoding = "utf-8")
+ all_gencode = []
+ for fam in FAMILIES.values():
+ for sm in fam["gencode"]:
+ all_gencode += ["-gencode", f"arch=compute_{sm},code=sm_{sm}"]
+ units.append((api, all_gencode))
+
+ def compile_one(unit):
+ path, gencode = unit
+ o = obj / (path.stem + (".obj" if os.name == "nt" else ".o"))
+ cmd = common + gencode + ["-c", str(path), "-o", str(o)]
+ res = subprocess.run(cmd, capture_output = True, text = True)
+ return path.name, res.returncode, res.stdout + res.stderr, o
+
+ objs = []
+ failed = False
+ with concurrent.futures.ThreadPoolExecutor(max_workers = max(1, args.jobs)) as pool:
+ for name, rc, log, o in pool.map(compile_one, units):
+ print(f"[{'ok' if rc == 0 else 'FAIL'}] {name}", flush = True)
+ if rc != 0:
+ failed = True
+ print(log[-6000:], flush = True)
+ objs.append(str(o))
+ if failed:
+ return 1
+ lib = out / lib_name()
+ link = [args.nvcc, "-shared", "-cudart", "static", "-o", str(lib)] + objs + all_gencode
+ if os.name != "nt":
+ # libstdc++ / libgcc static: the library then needs only glibc from the host
+ link += ["-Xcompiler", "-static-libstdc++", "-Xcompiler", "-static-libgcc"]
+ link += ["-Xlinker", "--exclude-libs,ALL", "-Xlinker", "-z,defs"]
+ res = subprocess.run(link, capture_output = True, text = True)
+ if res.returncode != 0:
+ print(res.stdout + res.stderr)
+ return 1
+ print(f"built {lib} ({lib.stat().st_size / 1e6:.1f} MB)")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/studio/native_kernels/int8_gemm/i8gemm_impl.cuh b/studio/native_kernels/int8_gemm/i8gemm_impl.cuh
new file mode 100644
index 00000000000..d8a253cf298
--- /dev/null
+++ b/studio/native_kernels/int8_gemm/i8gemm_impl.cuh
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+//
+// s8 GEMM with torchao's int8 dequant epilogue fused bit for bit: __fmul_rn / __fadd_rn (no FMA contraction),
+// every PyTorch cast reproduced. A [M, K], W [N, K], out [M, N], all row-major.
+
+#pragma once
+
+#include <cuda_bf16.h>
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <type_traits>
+
+// clang-format off: the visitor headers need the GEMM headers first
+#include "cutlass/cutlass.h"
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
+#include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
+#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
+// clang-format on
+
+namespace ui8 {
+
+using namespace cute;
+
+// shared with the Python loader
+enum Status : int {
+ kOk = 0,
+ kBadArgs = 1,
+ kNoKernel = 2,
+ kCannotImplement = 3,
+ kInitFailed = 4,
+ kLaunchFailed = 5,
+};
+
+enum Kind : int { kFp32 = 0, kBf16 = 1, kFp16 = 2 };
+
+template <class T>
+CUTLASS_DEVICE float round_to(float x);
+template <>
+CUTLASS_DEVICE float round_to<cutlass::bfloat16_t>(float x) {
+ return __bfloat162float(__float2bfloat16_rn(x));
+}
+template <>
+CUTLASS_DEVICE float round_to<cutlass::half_t>(float x) {
+ return __half2float(__float2half_rn(x));
+}
+template <>
+CUTLASS_DEVICE float round_to<float>(float x) {
+ return x;
+}
+
+template <class Out, class XsT, class WsT, bool Bias>
+struct Dequant {
+ // torch promotes int32 * bf16 to bf16; torchao widens an fp16 x_scale to fp32
+ static constexpr bool kBf16Inter = std::is_same<XsT, cutlass::bfloat16_t>::value;
+ static constexpr bool kRoundWs = std::is_same<WsT, Out>::value;
+
+ CUTLASS_DEVICE static float scaled(float c, float xs, float ws) {
+ float t = kBf16Inter ? round_to<Out>(__fmul_rn(round_to<Out>(c), xs)) : round_to<Out>(__fmul_rn(c, xs));
+ float y = __fmul_rn(t, ws);
+ return kRoundWs ? round_to<Out>(y) : y;
+ }
+
+ template <class T>
+ struct Fn;
+
+ template <int N>
+ struct Fn<cutlass::Array<float, N>> {
+ using Frag = cutlass::Array<float, N>;
+
+ CUTLASS_DEVICE Frag operator()(Frag const& c, Frag const& xs, Frag const& ws) const {
+ Frag y;
+ CUTLASS_PRAGMA_UNROLL
+ for (int i = 0; i < N; ++i) {
+ y[i] = round_to<Out>(scaled(c[i], xs[i], ws[i]));
+ }
+ return y;
+ }
+
+ CUTLASS_DEVICE Frag operator()(Frag const& c, Frag const& xs, Frag const& ws, Frag const& b) const {
+ Frag y;
+ CUTLASS_PRAGMA_UNROLL
+ for (int i = 0; i < N; ++i) {
+ y[i] = round_to<Out>(__fadd_rn(scaled(c[i], xs[i], ws[i]), b[i]));
+ }
+ return y;
+ }
+ };
+};
+
+// diffusion_native_quant W8A8 Linear: fp32 epilogue, one cast, no bias
+template <class Out>
+struct NativeDequant {
+ template <class T>
+ struct Fn;
+
+ template <int N>
+ struct Fn<cutlass::Array<float, N>> {
+ using Frag = cutlass::Array<float, N>;
+
+ CUTLASS_DEVICE Frag operator()(Frag const& c, Frag const& xs, Frag const& ws) const {
+ Frag y;
+ CUTLASS_PRAGMA_UNROLL
+ for (int i = 0; i < N; ++i) {
+ y[i] = round_to<Out>(__fmul_rn(__fmul_rn(c[i], xs[i]), ws[i]));
+ }
+ return y;
+ }
+ };
+};
+
+enum Epilogue : int { kTorchao = 0, kNative = 1 };
+
+template <class Out, class XsT, class WsT, bool Bias, int Epi>
+struct EpiFn;
+template <class Out, class XsT, class WsT, bool Bias>
+struct EpiFn<Out, XsT, WsT, Bias, kTorchao> {
+ template <class T>
+ using Fn = typename Dequant<Out, XsT, WsT, Bias>::template Fn<T>;
+};
+template <class Out, class XsT, class WsT, bool Bias>
+struct EpiFn<Out, XsT, WsT, Bias, kNative> {
+ static_assert(!Bias && std::is_same<XsT, float>::value && std::is_same<WsT, float>::value,
+ "native epilogue: fp32 scales, no bias");
+ template <class T>
+ using Fn = typename NativeDequant<Out>::template Fn<T>;
+};
+
+template <int TBM_, int TBN_, int TBK_, int WM_, int WN_, int WK_, int Stages_, int Swizzle_ = 1>
+struct Tile {
+ static constexpr int TBM = TBM_, TBN = TBN_, TBK = TBK_;
+ static constexpr int WM = WM_, WN = WN_, WK = WK_;
+ static constexpr int Stages = Stages_;
+ static constexpr int Swizzle = Swizzle_;
+};
+
+template <class ArchTag>
+struct ArchTraits;
+template <>
+struct ArchTraits<cutlass::arch::Sm80> {
+ using Inst = cutlass::gemm::GemmShape<16, 8, 32>;
+};
+template <>
+struct ArchTraits<cutlass::arch::Sm75> {
+ using Inst = cutlass::gemm::GemmShape<8, 8, 16>;
+};
+
+template <class ArchTag, class T, class Out, class XsT, class WsT, bool Bias, int Epi = kTorchao>
+struct FusedGemm {
+ using ElementAcc = int32_t;
+ using ElementCompute = float;
+ static constexpr int kAlignAB = 16;
+ static constexpr int kAlignC = 128 / cutlass::sizeof_bits<Out>::value;
+ using TB = cutlass::gemm::GemmShape<T::TBM, T::TBN, T::TBK>;
+ using Warp = cutlass::gemm::GemmShape<T::WM, T::WN, T::WK>;
+ using Inst = typename ArchTraits<ArchTag>::Inst;
+ static constexpr int kEVTStages = 1;
+
+ using ThreadMap =
+ cutlass::epilogue::threadblock::OutputTileThreadLayout<TB, Warp, Out, kAlignC, kEVTStages>;
+ using Acc = cutlass::epilogue::threadblock::VisitorAccFetch;
+ using XScale = cutlass::epilogue::threadblock::VisitorColBroadcast<ThreadMap, XsT, Stride<_1, _0, int32_t>>;
+ using WScale = cutlass::epilogue::threadblock::VisitorRowBroadcast<ThreadMap, WsT, Stride<_0, _1, int32_t>>;
+ using BiasRow = cutlass::epilogue::threadblock::VisitorRowBroadcast<ThreadMap, Out, Stride<_0, _1, int32_t>>;
+ using Compute = cutlass::epilogue::threadblock::VisitorCompute<EpiFn<Out, XsT, WsT, Bias, Epi>::template Fn, float,
+ float, cutlass::FloatRoundStyle::round_to_nearest>;
+ using Body = typename std::conditional<Bias, cutlass::epilogue::threadblock::Sm80EVT<Compute, Acc, XScale, WScale, BiasRow>,
+ cutlass::epilogue::threadblock::Sm80EVT<Compute, Acc, XScale, WScale>>::type;
+ using Store = cutlass::epilogue::threadblock::VisitorAuxStore<
+ ThreadMap, Out, cutlass::FloatRoundStyle::round_to_nearest, Stride<int64_t, _1, int64_t>>;
+ using EVT = cutlass::epilogue::threadblock::Sm80EVT<Store, Body>;
+
+ using Swizzle = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<T::Swizzle>;
+
+ using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
+ int8_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, kAlignAB, int8_t,
+ cutlass::layout::ColumnMajor, cutlass::ComplexTransform::kNone, kAlignAB, Out, cutlass::layout::RowMajor,
+ kAlignC, ElementAcc, ElementCompute, cutlass::arch::OpClassTensorOp, ArchTag, TB, Warp, Inst, EVT, Swizzle,
+ T::Stages, cutlass::arch::OpMultiplyAddSaturate, kEVTStages>::GemmKernel;
+ using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+
+ static typename Body::Arguments body_args(const void* xs, const void* ws, const void* bias, int M, int N) {
+ typename XScale::Arguments xa{static_cast<XsT const*>(xs), XsT(0), {_1{}, _0{}, M}};
+ typename WScale::Arguments wa{static_cast<WsT const*>(ws), WsT(0), {_0{}, _1{}, N}};
+ if constexpr (Bias) {
+ typename BiasRow::Arguments ba{static_cast<Out const*>(bias), Out(0), {_0{}, _1{}, N}};
+ return typename Body::Arguments{{}, xa, wa, ba, {}};
+ } else {
+ return typename Body::Arguments{{}, xa, wa, {}};
+ }
+ }
+
+ static int run(const void* a, const void* w, const void* xs, const void* ws, const void* bias, void* out, int M,
+ int N, int K, int64_t lda, int64_t ldw, int64_t ldc, cudaStream_t stream) {
+ typename EVT::Arguments cb{body_args(xs, ws, bias, M, N),
+ {static_cast<Out*>(out), {ldc, _1{}, int64_t(M) * ldc}}};
+ cutlass::gemm::GemmCoord problem(M, N, K);
+ typename Gemm::Arguments args(cutlass::gemm::GemmUniversalMode::kGemm, problem, 1, cb,
+ const_cast<void*>(a), const_cast<void*>(w), nullptr, nullptr, int64_t(M) * lda,
+ int64_t(N) * ldw, 0, 0, lda, ldw, 0, 0);
+ Gemm gemm;
+ if (gemm.can_implement(args) != cutlass::Status::kSuccess) return kCannotImplement;
+ if (Gemm::get_workspace_size(args) != 0) return kCannotImplement;
+ if (gemm.initialize(args, nullptr, stream) != cutlass::Status::kSuccess) return kInitFailed;
+ if (gemm.run(stream) != cutlass::Status::kSuccess) return kLaunchFailed;
+ return cudaGetLastError() == cudaSuccess ? kOk : kLaunchFailed;
+ }
+};
+
+using RunFn = int (*)(const void*, const void*, const void*, const void*, const void*, void*, int, int, int, int64_t,
+ int64_t, int64_t, cudaStream_t);
+
+} // namespace ui8
diff --git a/studio/native_kernels/int8_gemm/make_wheel.py b/studio/native_kernels/int8_gemm/make_wheel.py
new file mode 100644
index 00000000000..9530f4ec071
--- /dev/null
+++ b/studio/native_kernels/int8_gemm/make_wheel.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Pack a built library into ``unsloth_studio_i8gemm-<version>-py3-none-<platform>.whl``.
+
+python make_wheel.py --lib build/libunsloth_i8gemm.so --platform manylinux_2_28_x86_64 --out dist/
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import sys
+import zipfile
+from pathlib import Path
+
+NAME = "unsloth_studio_i8gemm"
+DIST = "unsloth-studio-i8gemm"
+VERSION = "1.0.0"
+ABI_VERSION = 1
+
+
+def _record_line(path: str, data: bytes) -> str:
+ digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode()
+ return f"{path},sha256={digest},{len(data)}"
+
+
+def build(
+ lib: Path,
+ platform: str,
+ out: Path,
+ cutlass_sha: str = "",
+) -> Path:
+ tag = f"py3-none-{platform}"
+ dist_info = f"{NAME}-{VERSION}.dist-info"
+ init = (
+ '"""Prebuilt int8 GEMM for Unsloth Studio (loaded with ctypes by diffusion_int8_cutlass)."""\n'
+ "import os\n\n"
+ f"ABI_VERSION = {ABI_VERSION}\n"
+ f"CUTLASS_SHA = {cutlass_sha!r}\n"
+ f"LIB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib', {lib.name!r})\n"
+ ).encode()
+ metadata = (
+ "Metadata-Version: 2.1\n"
+ f"Name: {DIST}\n"
+ f"Version: {VERSION}\n"
+ "Summary: Prebuilt CUTLASS int8 GEMM with a fused torchao dequant epilogue for Unsloth Studio\n"
+ "License: AGPL-3.0-only\n"
+ "Requires-Python: >=3.9\n"
+ ).encode()
+ wheel = (
+ "Wheel-Version: 1.0\n"
+ "Generator: unsloth-studio make_wheel.py\n"
+ "Root-Is-Purelib: false\n"
+ f"Tag: {tag}\n"
+ ).encode()
+ files = {
+ f"{NAME}/__init__.py": init,
+ f"{NAME}/lib/{lib.name}": lib.read_bytes(),
+ f"{dist_info}/METADATA": metadata,
+ f"{dist_info}/WHEEL": wheel,
+ f"{dist_info}/top_level.txt": f"{NAME}\n".encode(),
+ }
+ record = [_record_line(p, d) for p, d in files.items()] + [f"{dist_info}/RECORD,,"]
+ files[f"{dist_info}/RECORD"] = ("\n".join(record) + "\n").encode()
+ out.mkdir(parents = True, exist_ok = True)
+ path = out / f"{NAME}-{VERSION}-{tag}.whl"
+ with zipfile.ZipFile(path, "w", compression = zipfile.ZIP_DEFLATED, compresslevel = 9) as zf:
+ for name, data in files.items():
+ info = zipfile.ZipInfo(name, date_time = (2026, 1, 1, 0, 0, 0))
+ info.external_attr = (0o755 if name.endswith((".so", ".dll")) else 0o644) << 16
+ info.compress_type = zipfile.ZIP_DEFLATED
+ zf.writestr(info, data)
+ return path
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--lib", required = True)
+ ap.add_argument(
+ "--platform",
+ required = True,
+ help = "wheel platform tag, e.g. manylinux_2_28_x86_64, win_amd64",
+ )
+ ap.add_argument("--out", required = True)
+ ap.add_argument("--cutlass-sha", default = "")
+ args = ap.parse_args()
+ path = build(Path(args.lib), args.platform, Path(args.out), args.cutlass_sha)
+ print(f"{path} ({path.stat().st_size / 1e6:.1f} MB)")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())