Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
#!/usr/bin/env python3
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Build the prebuilt int8 GEMM library, one nvcc unit per (family, tile), static CUDA runtime.
python build.py --cutlass /path/to/cutlass --out build/ [--jobs 16] [--families sm80,sm75]
SASS only (no PTX): other archs keep Studio's other paths instead of a JIT build.
"""
from __future__ import annotations
import argparse
import concurrent.futures
import os
import shutil
import subprocess
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
ABI_VERSION = 1
# (TBM, TBN, TBK, WM, WN, WK, stages, swizzle). The index is the config id the loader races; append only.
FAMILIES = {
"sm80": {
"arch_tag": "cutlass::arch::Sm80",
"gencode": ("80", "86", "89"),
"out": ("cutlass::bfloat16_t",),
"xs": ("cutlass::bfloat16_t", "float"),
"ws": ("cutlass::bfloat16_t", "float"),
"tiles": (
(128, 256, 64, 64, 64, 64, 3, 1),
(256, 128, 64, 64, 64, 64, 3, 1),
(128, 128, 64, 64, 64, 64, 4, 1),
(128, 128, 128, 64, 64, 128, 3, 1),
(64, 128, 64, 32, 64, 64, 4, 1),
(128, 256, 64, 64, 64, 64, 3, 4),
(128, 128, 64, 64, 64, 64, 4, 4),
(64, 64, 64, 32, 32, 64, 5, 1),
(256, 128, 64, 64, 64, 64, 3, 4),
# 96 KB of stages fits sm86 / sm89's 99 KB
(128, 256, 64, 64, 64, 64, 4, 1),
(256, 128, 64, 64, 64, 64, 4, 1),
(128, 128, 64, 64, 64, 64, 6, 1),
(128, 256, 64, 64, 64, 64, 4, 4),
(256, 128, 128, 64, 64, 128, 2, 1),
(128, 256, 128, 64, 64, 128, 2, 1),
),
},
"sm75": {
"arch_tag": "cutlass::arch::Sm75",
"gencode": ("75",),
"out": ("cutlass::half_t",),
"xs": ("cutlass::half_t", "float"),
"ws": ("cutlass::half_t", "cutlass::bfloat16_t", "float"),
"native": True,
"tiles": (
(128, 256, 64, 64, 64, 64, 2, 1),
(256, 128, 64, 64, 64, 64, 2, 1),
(128, 128, 64, 64, 64, 64, 2, 1),
(64, 128, 64, 32, 64, 64, 2, 1),
(128, 256, 64, 64, 64, 64, 2, 4),
(128, 128, 64, 64, 64, 64, 2, 4),
(64, 64, 64, 32, 32, 64, 2, 1),
),
},
}
KIND = {"float": 0, "cutlass::bfloat16_t": 1, "cutlass::half_t": 2}
FAMILY_ID = {"sm80": 80, "sm75": 75}
def variants(fam: dict) -> list:
"""(out, xs, ws, bias, epilogue); epilogue 0 = torchao's, 1 = the native W8A8 Linear's."""
out = [
(o, x, w, b, 0)
for o in fam["out"]
for x in fam["xs"]
for w in fam["ws"]
for b in (False, True)
]
if fam.get("native"):
out += [(o, "float", "float", False, 1) for o in fam["out"]]
return out
def tile_tu(name: str, fam: dict, index: int) -> str:
t = fam["tiles"][index]
lines = [
'#include "i8gemm_impl.cuh"',
"namespace {",
f"using T = ui8::Tile<{', '.join(str(v) for v in t)}>;",
"}",
f'extern "C" ui8::RunFn ui8_{name}_t{index}(int out, int xs, int ws, int bias, int epi) {{',
]
for o, x, w, b, e in variants(fam):
lines.append(
f" if (out == {KIND[o]} && xs == {KIND[x]} && ws == {KIND[w]} && bias == {int(b)} && epi == {e}) "
f"return &ui8::FusedGemm<{fam['arch_tag']}, T, {o}, {x}, {w}, {str(b).lower()}, {e}>::run;"
)
lines += [" return nullptr;", "}", ""]
return "\n".join(lines)
def api_tu() -> str:
decls, tables = [], []
for name, fam in FAMILIES.items():
n = len(fam["tiles"])
for i in range(n):
decls.append(f'extern "C" ui8::RunFn ui8_{name}_t{i}(int, int, int, int, int);')
tables.append(
f"static ui8::RunFn (*const k_{name}[])(int, int, int, int, int) = {{"
+ ", ".join(f"&ui8_{name}_t{i}" for i in range(n))
+ "};"
)
tables.append(
f"static const int k_{name}_tiles[][8] = {{"
+ ", ".join("{" + ", ".join(str(v) for v in t) + "}" for t in fam["tiles"])
+ "};"
)
fam_cases = "\n".join(
f" case {FAMILY_ID[name]}: *table = k_{name}; *count = {len(fam['tiles'])}; *tiles = k_{name}_tiles; break;"
for name, fam in FAMILIES.items()
)
return f"""#include "i8gemm_impl.cuh"
#if defined(_WIN32)
#define UI8_API __declspec(dllexport)
#else
#define UI8_API __attribute__((visibility("default")))
#endif
{chr(10).join(decls)}
{chr(10).join(tables)}
static bool family_table(int family, ui8::RunFn (*const** table)(int, int, int, int, int), int* count,
const int (**tiles)[8]) {{
switch (family) {{
{fam_cases}
default: return false;
}}
return true;
}}
extern "C" {{
UI8_API int ui8_abi_version() {{ return {ABI_VERSION}; }}
// Number of tiles for a family (80 or 75), 0 when absent.
UI8_API int ui8_num_tiles(int family) {{
ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;
int count = 0;
const int (*tiles)[8] = nullptr;
return family_table(family, &table, &count, &tiles) ? count : 0;
}}
// The 8 tile parameters (TBM, TBN, TBK, WM, WN, WK, stages, swizzle) into out8; 0 on success.
UI8_API int ui8_tile_info(int family, int tile, int* out8) {{
ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;
int count = 0;
const int (*tiles)[8] = nullptr;
if (!family_table(family, &table, &count, &tiles) || tile < 0 || tile >= count || out8 == nullptr) return 1;
for (int i = 0; i < 8; ++i) out8[i] = tiles[tile][i];
return 0;
}}
// 1 when (family, tile, out, xs, ws, bias, epilogue) is compiled in; epilogue 0 torchao, 1 native W8A8.
UI8_API int ui8_has_kernel(int family, int tile, int out_kind, int xs_kind, int ws_kind, int has_bias, int epilogue) {{
ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;
int count = 0;
const int (*tiles)[8] = nullptr;
if (!family_table(family, &table, &count, &tiles) || tile < 0 || tile >= count) return 0;
return table[tile](out_kind, xs_kind, ws_kind, has_bias ? 1 : 0, epilogue) != nullptr;
}}
// out[M, N] = dequant(a[M, K] @ w[N, K]^T); kinds: 0 fp32, 1 bf16, 2 fp16. bias may be null. Status code returned.
UI8_API int ui8_gemm(int family, int tile, int out_kind, int xs_kind, int ws_kind, int epilogue, const void* a, const void* w,
const void* xs, const void* ws, const void* bias, void* out, int64_t M, int64_t N, int64_t K,
int64_t lda, int64_t ldw, int64_t ldc, void* stream) {{
ui8::RunFn (*const* table)(int, int, int, int, int) = nullptr;
int count = 0;
const int (*tiles)[8] = nullptr;
if (!family_table(family, &table, &count, &tiles) || tile < 0 || tile >= count) return ui8::kNoKernel;
if (M <= 0 || N <= 0 || K <= 0 || M > 0x7fffffff || N > 0x7fffffff || K > 0x7fffffff) return ui8::kBadArgs;
if (a == nullptr || w == nullptr || xs == nullptr || ws == nullptr || out == nullptr) return ui8::kBadArgs;
if (lda < K || ldw < K || ldc < N) return ui8::kBadArgs;
ui8::RunFn fn = table[tile](out_kind, xs_kind, ws_kind, bias != nullptr ? 1 : 0, epilogue);
if (fn == nullptr) return ui8::kNoKernel;
return fn(a, w, xs, ws, bias, out, int(M), int(N), int(K), lda, ldw, ldc, static_cast<cudaStream_t>(stream));
}}
}}
"""
def lib_name() -> str:
return "unsloth_i8gemm.dll" if os.name == "nt" else "libunsloth_i8gemm.so"
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--cutlass", required = True, help = "CUTLASS checkout (its include/ is used)")
ap.add_argument("--out", required = True)
ap.add_argument("--nvcc", default = shutil.which("nvcc") or "nvcc")
ap.add_argument("--jobs", type = int, default = os.cpu_count() or 4)
ap.add_argument("--families", default = ",".join(FAMILIES))
ap.add_argument(
"--tiles", default = "", help = "comma separated tile ids to keep (default all; testing only)"
)
ap.add_argument(
"--sm",
default = "",
help = "comma separated SASS targets to keep, e.g. 89 (default all; testing only)",
)
args = ap.parse_args()
out = Path(args.out).resolve()
src = out / "src"
obj = out / "obj"
src.mkdir(parents = True, exist_ok = True)
obj.mkdir(parents = True, exist_ok = True)
shutil.copy2(HERE / "i8gemm_impl.cuh", src / "i8gemm_impl.cuh")
keep_fams = [f for f in args.families.split(",") if f]
for name in list(FAMILIES):
if name not in keep_fams:
del FAMILIES[name]
if args.sm:
want = set(args.sm.split(","))
for name in list(FAMILIES):
FAMILIES[name]["gencode"] = tuple(sm for sm in FAMILIES[name]["gencode"] if sm in want)
if not FAMILIES[name]["gencode"]:
del FAMILIES[name]
if args.tiles:
keep = [int(t) for t in args.tiles.split(",")]
for fam in FAMILIES.values():
fam["tiles"] = tuple(t for i, t in enumerate(fam["tiles"]) if i in keep)
include = Path(args.cutlass).resolve() / "include"
common = [
args.nvcc,
"-std=c++17",
"-O3",
"-DNDEBUG",
"--expt-relaxed-constexpr",
f"-I{src}",
f"-I{include}",
"-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1",
]
if os.name == "nt":
# static MSVC runtime (no DLL dependency), the conforming preprocessor CCCL needs
common += ["-Xcompiler", "/MT", "-Xcompiler", "/Zc:preprocessor", "--diag-suppress=177"]
else:
common += ["-Xcompiler", "-fPIC", "-Xcompiler", "-fvisibility=hidden"]
units = []
for name, fam in FAMILIES.items():
gencode = []
for sm in fam["gencode"]:
gencode += ["-gencode", f"arch=compute_{sm},code=sm_{sm}"]
for i in range(len(fam["tiles"])):
path = src / f"{name}_t{i}.cu"
path.write_text(tile_tu(name, fam, i), encoding = "utf-8")
units.append((path, gencode))
api = src / "i8gemm_api.cu"
api.write_text(api_tu(), encoding = "utf-8")
all_gencode = []
for fam in FAMILIES.values():
for sm in fam["gencode"]:
all_gencode += ["-gencode", f"arch=compute_{sm},code=sm_{sm}"]
units.append((api, all_gencode))
def compile_one(unit):
path, gencode = unit
o = obj / (path.stem + (".obj" if os.name == "nt" else ".o"))
cmd = common + gencode + ["-c", str(path), "-o", str(o)]
res = subprocess.run(cmd, capture_output = True, text = True)
return path.name, res.returncode, res.stdout + res.stderr, o
objs = []
failed = False
with concurrent.futures.ThreadPoolExecutor(max_workers = max(1, args.jobs)) as pool:
for name, rc, log, o in pool.map(compile_one, units):
print(f"[{'ok' if rc == 0 else 'FAIL'}] {name}", flush = True)
if rc != 0:
failed = True
print(log[-6000:], flush = True)
objs.append(str(o))
if failed:
return 1
lib = out / lib_name()
link = [args.nvcc, "-shared", "-cudart", "static", "-o", str(lib)] + objs + all_gencode
if os.name != "nt":
# libstdc++ / libgcc static: the library then needs only glibc from the host
link += ["-Xcompiler", "-static-libstdc++", "-Xcompiler", "-static-libgcc"]
link += ["-Xlinker", "--exclude-libs,ALL", "-Xlinker", "-z,defs"]
res = subprocess.run(link, capture_output = True, text = True)
if res.returncode != 0:
print(res.stdout + res.stderr)
return 1
print(f"built {lib} ({lib.stat().st_size / 1e6:.1f} MB)")
return 0
if __name__ == "__main__":
sys.exit(main())