Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Reduce cuda dylibs size (#963)
Davide Eynard3dd200539c6c
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patchnew file mode 100644index 00000000..6ef4957c--- /dev/null+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch@@ -0,0 +1,35 @@+diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu+--- a/llama.cpp/ggml/src/ggml-cuda/convert.cu++++ b/llama.cpp/ggml/src/ggml-cuda/convert.cu+@@ -736,6 +736,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {+ return dequantize_row_q5_K_cuda;+ case GGML_TYPE_Q6_K:+ return dequantize_row_q6_K_cuda;++#ifndef GGML_CUDA_NO_IQ_QUANTS+ case GGML_TYPE_IQ2_XXS:+ return dequantize_row_iq2_xxs_cuda;+ case GGML_TYPE_IQ2_XS:+@@ -754,6 +755,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {+ return dequantize_row_iq4_xs_cuda;+ case GGML_TYPE_IQ3_S:+ return dequantize_row_iq3_s_cuda;++#endif // GGML_CUDA_NO_IQ_QUANTS+ case GGML_TYPE_MXFP4:+ return dequantize_row_mxfp4_cuda;+ case GGML_TYPE_NVFP4:+@@ -791,6 +793,7 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {+ return dequantize_row_q5_K_cuda;+ case GGML_TYPE_Q6_K:+ return dequantize_row_q6_K_cuda;++#ifndef GGML_CUDA_NO_IQ_QUANTS+ case GGML_TYPE_IQ2_XXS:+ return dequantize_row_iq2_xxs_cuda;+ case GGML_TYPE_IQ2_XS:+@@ -809,6 +812,7 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {+ return dequantize_row_iq4_xs_cuda;+ case GGML_TYPE_IQ3_S:+ return dequantize_row_iq3_s_cuda;++#endif // GGML_CUDA_NO_IQ_QUANTS+ case GGML_TYPE_MXFP4:+ return dequantize_row_mxfp4_cuda;+ case GGML_TYPE_NVFP4:diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patchnew file mode 100644index 00000000..36166384--- /dev/null+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch@@ -0,0 +1,31 @@+diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu+--- a/llama.cpp/ggml/src/ggml-cuda/cpy.cu++++ b/llama.cpp/ggml/src/ggml-cuda/cpy.cu+@@ -360,6 +360,7 @@ static void ggml_cpy_q5_1_f32_cuda(+ ne10, ne11, ne12, nb10, nb11, nb12, nb13);+ }+++#ifndef GGML_CUDA_NO_IQ_QUANTS+ static void ggml_cpy_f32_iq4_nl_cuda(+ const char * cx, char * cdst, const int64_t ne,+ const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,+@@ -371,6 +372,7 @@ static void ggml_cpy_f32_iq4_nl_cuda(+ cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+ }++#endif // GGML_CUDA_NO_IQ_QUANTS++ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1) {+ const int64_t ne = ggml_nelements(src0);+@@ -465,9 +467,11 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg+ } else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_F32) {+ ggml_cpy_q5_0_f32_cuda+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);++#ifndef GGML_CUDA_NO_IQ_QUANTS+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) {+ ggml_cpy_f32_iq4_nl_cuda+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);++#endif // GGML_CUDA_NO_IQ_QUANTS+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) {+ ggml_cpy_f32_q5_1_cuda+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patchnew file mode 100644index 00000000..c54c0e0f--- /dev/null+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch@@ -0,0 +1,47 @@+diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu+--- a/llama.cpp/ggml/src/ggml-cuda/mmq.cu++++ b/llama.cpp/ggml/src/ggml-cuda/mmq.cu+@@ -44,6 +44,7 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con+ case GGML_TYPE_Q6_K:+ mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, args, stream);+ break;++#ifndef GGML_CUDA_NO_IQ_QUANTS+ case GGML_TYPE_IQ2_XXS:+ mul_mat_q_case<GGML_TYPE_IQ2_XXS>(ctx, args, stream);+ break;+@@ -68,6 +69,7 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con+ case GGML_TYPE_IQ4_NL:+ mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, args, stream);+ break;++#endif // GGML_CUDA_NO_IQ_QUANTS+ default:+ GGML_ABORT("fatal error");+ break;+@@ -286,6 +288,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:++#ifndef GGML_CUDA_NO_IQ_QUANTS+ case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ2_S:+@@ -294,6 +297,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_XS:+ case GGML_TYPE_IQ4_NL:++#endif // GGML_CUDA_NO_IQ_QUANTS+ mmq_supported = true;+ break;+ default:+@@ -356,9 +360,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t+ return ne11 <= 128;+ case GGML_TYPE_Q6_K:+ return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);++#ifndef GGML_CUDA_NO_IQ_QUANTS+ case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ2_S:+ return GGML_CUDA_CC_IS_RDNA3_5(cc) || ne11 <= 128;++#endif // GGML_CUDA_NO_IQ_QUANTS+ default:+ return true;+ }diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patchnew file mode 100644index 00000000..c52d0de0--- /dev/null+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch@@ -0,0 +1,19 @@+diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh+--- a/llama.cpp/ggml/src/ggml-cuda/mmq.cuh++++ b/llama.cpp/ggml/src/ggml-cuda/mmq.cuh+@@ -4088,6 +4088,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);+ extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);+ extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);+ extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);++#ifndef GGML_CUDA_NO_IQ_QUANTS+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);+@@ -4096,6 +4097,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);+ extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);+ extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);+ extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);++#endif // GGML_CUDA_NO_IQ_QUANTS++ // -------------------------------------------------------------------------------------------------------------------------+diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmvq.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmvq.cu.patchnew file mode 100644index 00000000..4c3fe11e--- /dev/null+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmvq.cu.patch@@ -0,0 +1,35 @@+diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu+--- a/llama.cpp/ggml/src/ggml-cuda/mmvq.cu++++ b/llama.cpp/ggml/src/ggml-cuda/mmvq.cu+@@ -22,6 +22,7 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type)+ case GGML_TYPE_Q4_K: return vec_dot_q4_K_q8_1;+ case GGML_TYPE_Q5_K: return vec_dot_q5_K_q8_1;+ case GGML_TYPE_Q6_K: return vec_dot_q6_K_q8_1;++#ifndef GGML_CUDA_NO_IQ_QUANTS+ case GGML_TYPE_IQ2_XXS: return vec_dot_iq2_xxs_q8_1;+ case GGML_TYPE_IQ2_XS: return vec_dot_iq2_xs_q8_1;+ case GGML_TYPE_IQ2_S: return vec_dot_iq2_s_q8_1;+@@ -31,6 +32,7 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type)+ case GGML_TYPE_IQ4_NL: return vec_dot_iq4_nl_q8_1;+ case GGML_TYPE_IQ4_XS: return vec_dot_iq4_xs_q8_1;+ case GGML_TYPE_IQ3_S: return vec_dot_iq3_s_q8_1;++#endif // GGML_CUDA_NO_IQ_QUANTS+ default: return nullptr;+ }+ }+@@ -50,6 +52,7 @@ static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) {+ case GGML_TYPE_Q4_K: return VDR_Q4_K_Q8_1_MMVQ;+ case GGML_TYPE_Q5_K: return VDR_Q5_K_Q8_1_MMVQ;+ case GGML_TYPE_Q6_K: return VDR_Q6_K_Q8_1_MMVQ;++#ifndef GGML_CUDA_NO_IQ_QUANTS+ case GGML_TYPE_IQ2_XXS: return VDR_IQ2_XXS_Q8_1_MMVQ;+ case GGML_TYPE_IQ2_XS: return VDR_IQ2_XS_Q8_1_MMVQ;+ case GGML_TYPE_IQ2_S: return VDR_IQ2_S_Q8_1_MMVQ;+@@ -57,6 +60,7 @@ static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) {+ case GGML_TYPE_IQ3_S: return VDR_IQ3_S_Q8_1_MMVQ;+ case GGML_TYPE_IQ4_NL: return VDR_IQ4_NL_Q8_1_MMVQ;+ case GGML_TYPE_IQ4_XS: return VDR_IQ4_XS_Q8_1_MMVQ;++#endif // GGML_CUDA_NO_IQ_QUANTS+ default: return 1;+ }+ }diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.shindex 3f81934d..590d3038 100644--- a/llamafile/build-functions.sh+++ b/llamafile/build-functions.sh@@ -98,17 +98,64 @@ setup_build_dir() {mkdir -p "$build_dir"}-# Collect CUDA/HIP source files+# Collect CUDA/HIP source files with selective template inclusion# Sets: CUDA_SOURCES, NUM_SOURCES-# Args: $1 = GGML_CUDA_DIR, $2 = extra sources (optional, e.g., tinyblas.cu path)+# Args: $1 = GGML_CUDA_DIR+# $2 = caller-supplied sources prepended to the list (e.g., tinyblas.cu+# for the default TinyBLAS build; empty for the --cublas build)+# $3 = NO_IQ_QUANTS (optional, "1" to exclude IQ quant MMQ templates)+# $4 = FA_ALL_QUANTS (optional, "1" to include all fattn-vec quant combos+# instead of the 3 default ones; mirrors upstream's GGML_CUDA_FA_ALL_QUANTS)collect_gpu_sources() {local ggml_cuda_dir="$1"- local extra_sources="$2"+ local caller_sources="$2"+ local no_iq_quants="${3:-0}"+ local fa_all_quants="${4:-0}"- CUDA_SOURCES="$extra_sources"+ CUDA_SOURCES="$caller_sources"- for f in "$ggml_cuda_dir"/*.cu "$ggml_cuda_dir/template-instances"/*.cu; do+ # 1. Main CUDA sources (always included)+ for f in "$ggml_cuda_dir"/*.cu; do+ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f"+ done++ local ti_dir="$ggml_cuda_dir/template-instances"++ # 2. fattn-mma and fattn-tile instances (always included)+ for f in "$ti_dir"/fattn-mma-*.cu "$ti_dir"/fattn-tile-*.cu; do+ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f"+ done++ # 3. fattn-vec: default to the 4 common quant combos (f16-f16, q4_0-q4_0,+ # q8_0-q8_0, bf16-bf16), matching upstream CMake. With FA_ALL_QUANTS=1+ # include all fattn-vec instances (mirrors upstream's+ # GGML_CUDA_FA_ALL_QUANTS opt-in).+ if [ "$fa_all_quants" = "1" ]; then+ for f in "$ti_dir"/fattn-vec-instance-*.cu; do+ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f"+ done+ else+ for f in "$ti_dir"/fattn-vec-instance-f16-f16.cu \+ "$ti_dir"/fattn-vec-instance-q4_0-q4_0.cu \+ "$ti_dir"/fattn-vec-instance-q8_0-q8_0.cu \+ "$ti_dir"/fattn-vec-instance-bf16-bf16.cu; do+ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f"+ done+ fi++ # 4. mmf instances (always included)+ for f in "$ti_dir"/mmf-*.cu; do+ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f"+ done++ # 5. mmq instances: include all, but optionally exclude IQ quant templates+ for f in "$ti_dir"/mmq-*.cu; doif [ -f "$f" ]; then+ if [ "$no_iq_quants" = "1" ]; then+ case "$(basename "$f")" in+ mmq-instance-iq*) continue ;;+ esac+ fiCUDA_SOURCES="$CUDA_SOURCES $f"fidonediff --git a/llamafile/cuda.bat b/llamafile/cuda.batindex ad0b428d..ba940c77 100644--- a/llamafile/cuda.bat+++ b/llamafile/cuda.bat@@ -22,14 +22,37 @@ for %%I in ("%~dp0..") do set "REPO_DIR=%%~fI"set "USE_CUBLAS=0"set "CLEAN=0"set "OUTPUT="+set "MINIMAL_ARCHS=0"+set "NO_IQ_QUANTS=0"+set "STRIP=0"+set "COMPRESS=0"+set "FA_ALL_QUANTS=0":parse_argsif "%~1"=="" goto done_args-if /i "%~1"=="--cublas" (set "USE_CUBLAS=1" & shift & goto parse_args)-if /i "%~1"=="--clean" (set "CLEAN=1" & shift & goto parse_args)-if /i "%~1"=="--output" (set "OUTPUT=%~2" & shift & shift & goto parse_args)+if /i "%~1"=="--cublas" (set "USE_CUBLAS=1" & shift & goto parse_args)+if /i "%~1"=="--clean" (set "CLEAN=1" & shift & goto parse_args)+if /i "%~1"=="--output" (set "OUTPUT=%~2" & shift & shift & goto parse_args)+if /i "%~1"=="--fa-all-quants" (set "FA_ALL_QUANTS=1" & shift & goto parse_args)+if /i "%~1"=="--minimize-size" (set "MINIMAL_ARCHS=1" & set "NO_IQ_QUANTS=1" & set "STRIP=1" & set "COMPRESS=1" & shift & goto parse_args)+if /i "%~1"=="--minimal-archs" (set "MINIMAL_ARCHS=1" & shift & goto parse_args)+if /i "%~1"=="--no-iq-quants" (set "NO_IQ_QUANTS=1" & shift & goto parse_args)+if /i "%~1"=="--strip" (set "STRIP=1" & shift & goto parse_args)+if /i "%~1"=="--compress" (set "COMPRESS=1" & shift & goto parse_args)if /i "%~1"=="--help" (echo Usage: cuda.bat [--clean] [--cublas] [--output PATH]+ echo --clean Clean build directory before building+ echo --cublas Use NVIDIA cuBLAS instead of TinyBLAS+ echo --output Output path for shared library+ echo --fa-all-quants Compile all flash-attention vec quant combos+ echo ^(default: f16-f16, q4_0-q4_0, q8_0-q8_0, bf16-bf16 only^)+ echo.+ echo Size reduction options ^(all off by default^):+ echo --minimize-size Enable all size reduction options below+ echo --minimal-archs Use virtual PTX for sm_75/sm_90, real SASS for sm_80/86/89+ echo --no-iq-quants Exclude IQ quant MMQ template instances+ echo --strip No-op on Windows ^(debug info is in a separate .pdb^)+ echo --compress Pass --compress-mode=size to nvcc ^(requires CUDA ^>= 12.8^)exit /b 0)echo Unknown option: %~1@@ -97,28 +120,50 @@ if errorlevel 1 (exit /b 1)-:: -------- detect CUDA version for Blackwell support --------+:: -------- architecture flags --------+:: Virtual PTX for less-used archs (JIT on first run), real SASS for popular archs.set "ARCH_FLAGS="-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_75,code=sm_75"-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_80,code=sm_80"-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_86,code=sm_86"-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_89,code=sm_89"-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_90,code=sm_90"--:: Check for CUDA 13.x to add Blackwell support-for /f "tokens=*" %%v in ('nvcc --version 2^>nul ^| findstr /r "release [0-9]"') do (- set "NVCC_VER_LINE=%%v"+if "%MINIMAL_ARCHS%"=="1" (+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_75,code=compute_75"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_80,code=sm_80"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_86,code=sm_86"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_89,code=sm_89"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_90,code=compute_90"+) else (+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_75,code=sm_75"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_80,code=sm_80"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_86,code=sm_86"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_89,code=sm_89"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_90,code=sm_90")++:: Detect CUDA version for Blackwell and compress support+:: nvcc prints e.g. "Cuda compilation tools, release 13.2, V13.2.51".+:: Splitting on ',', ' ', and '.' produces tokens 4..6 = "release", "13", "2".set "CUDA_MAJOR="-if defined NVCC_VER_LINE (- for /f "tokens=2 delims=," %%a in ("!NVCC_VER_LINE!") do (- for /f "tokens=2" %%b in ("%%a") do (- for /f "tokens=1 delims=." %%c in ("%%b") do set "CUDA_MAJOR=%%c"- )- )+set "CUDA_MINOR="+for /f "tokens=5,6 delims=, ." %%a in ('nvcc --version 2^>nul ^| findstr /r "release [0-9]"') do (+ set "CUDA_MAJOR=%%a"+ set "CUDA_MINOR=%%b")if "%CUDA_MAJOR%"=="13" (- set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_120f,code=sm_120f --compress-mode=size"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_120f,code=sm_120f"+)++:: Fall back to 0 if version parsing failed, to keep the numeric comparisons below valid.+if not defined CUDA_MAJOR set "CUDA_MAJOR=0"+if not defined CUDA_MINOR set "CUDA_MINOR=0"++:: --compress-mode=size: opt-in via --compress (or --minimize-size). Requires CUDA >= 12.8.+if "%COMPRESS%"=="1" (+ set "COMPRESS_OK=0"+ if !CUDA_MAJOR! gtr 12 set "COMPRESS_OK=1"+ if !CUDA_MAJOR! equ 12 if !CUDA_MINOR! geq 8 set "COMPRESS_OK=1"+ if "!COMPRESS_OK!"=="1" (+ set "ARCH_FLAGS=!ARCH_FLAGS! --compress-mode=size"+ ) else (+ echo Warning: --compress requested but CUDA !CUDA_MAJOR!.!CUDA_MINOR! ^< 12.8; ignoring.+ )):: -------- copy TinyBLAS files if needed --------@@ -138,6 +183,8 @@ set "COMMON_FLAGS=%COMMON_FLAGS% -Xcompiler="/nologo /EHsc /O2 /GR /MT /std:c++1set "COMMON_FLAGS=%COMMON_FLAGS% -diag-suppress 177 -diag-suppress 221 -diag-suppress 550"set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"set "COMMON_FLAGS=%COMMON_FLAGS% %BLAS_DEFINE%"+if "%NO_IQ_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_NO_IQ_QUANTS"+if "%FA_ALL_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_FA_ALL_QUANTS":: -------- extract GGML version --------set "GGML_VERSION=unknown"@@ -165,6 +212,14 @@ echo Version: !GGML_VERSION! (commit: !GGML_COMMIT!)echo Source: %GGML_CUDA_DIR%echo Output: %OUTPUT%echo Build: %BUILD_DIR%+if "%MINIMAL_ARCHS%%NO_IQ_QUANTS%%STRIP%%COMPRESS%" neq "0000" (+ echo Size reduction:+ if "%MINIMAL_ARCHS%"=="1" echo - Minimal archs ^(PTX for sm_75/sm_90^)+ if "%NO_IQ_QUANTS%"=="1" echo - No IQ quant templates+ if "%STRIP%"=="1" echo - Strip ^(no-op on Windows; debug info is in a separate .pdb^)+ if "%COMPRESS%"=="1" echo - Compress mode enabled+)+if "%FA_ALL_QUANTS%"=="1" echo FA all quants: all fattn-vec template instances includedecho.:: -------- collect and compile .cu sources --------@@ -205,8 +260,52 @@ for %%f in ("%GGML_CUDA_DIR%\*.cu") do (set "OBJ_FILES=!OBJ_FILES! "!OBJ!"")-:: Template-instances CUDA sources-for %%f in ("%GGML_CUDA_DIR%\template-instances\*.cu") do (+:: Template-instances: selectively included to mirror upstream CMake defaults.+set "TI_DIR=%GGML_CUDA_DIR%\template-instances"++:: 2. fattn-mma and fattn-tile instances (always included)+for %%f in ("%TI_DIR%\fattn-mma-*.cu" "%TI_DIR%\fattn-tile-*.cu") do (+ set /a COUNT+=1+ set "BASE=%%~nf"+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"+ set "SRC=%%f"+ if not exist "!OBJ!" (+ echo [!COUNT!] Compiling: ti-!BASE!.cu+ nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "!SRC!"+ if errorlevel 1 (echo Error compiling ti-!BASE!.cu & exit /b 1)+ set /a COMPILED+=1+ ) else (+ echo [!COUNT!] Skipping: ti-!BASE!.cu (up to date^)+ )+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""+)++:: 3. fattn-vec: only the 4 default quant combos unless --fa-all-quants.+if "%FA_ALL_QUANTS%"=="1" (+ set "FATTN_VEC_GLOB=%TI_DIR%\fattn-vec-instance-*.cu"+) else (+ set "FATTN_VEC_GLOB=%TI_DIR%\fattn-vec-instance-f16-f16.cu %TI_DIR%\fattn-vec-instance-q4_0-q4_0.cu %TI_DIR%\fattn-vec-instance-q8_0-q8_0.cu %TI_DIR%\fattn-vec-instance-bf16-bf16.cu"+)+for %%f in (!FATTN_VEC_GLOB!) do (+ if exist "%%f" (+ set /a COUNT+=1+ set "BASE=%%~nf"+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"+ set "SRC=%%f"+ if not exist "!OBJ!" (+ echo [!COUNT!] Compiling: ti-!BASE!.cu+ nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "!SRC!"+ if errorlevel 1 (echo Error compiling ti-!BASE!.cu & exit /b 1)+ set /a COMPILED+=1+ ) else (+ echo [!COUNT!] Skipping: ti-!BASE!.cu (up to date^)+ )+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""+ )+)++:: 4. mmf instances (always included)+for %%f in ("%TI_DIR%\mmf-*.cu") do (set /a COUNT+=1set "BASE=%%~nf"set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"@@ -222,6 +321,30 @@ for %%f in ("%GGML_CUDA_DIR%\template-instances\*.cu") do (set "OBJ_FILES=!OBJ_FILES! "!OBJ!"")+:: 5. mmq instances: all unless --no-iq-quants excludes mmq-instance-iq*+for %%f in ("%TI_DIR%\mmq-*.cu") do (+ set "BASE=%%~nf"+ set "SKIP=0"+ if "%NO_IQ_QUANTS%"=="1" (+ echo !BASE! | findstr /b /i "mmq-instance-iq" >nul+ if not errorlevel 1 set "SKIP=1"+ )+ if "!SKIP!"=="0" (+ set /a COUNT+=1+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"+ set "SRC=%%f"+ if not exist "!OBJ!" (+ echo [!COUNT!] Compiling: ti-!BASE!.cu+ nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "!SRC!"+ if errorlevel 1 (echo Error compiling ti-!BASE!.cu & exit /b 1)+ set /a COMPILED+=1+ ) else (+ echo [!COUNT!] Skipping: ti-!BASE!.cu (up to date^)+ )+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""+ )+)+echo.echo Compiled %COMPILED% of %COUNT% .cu files (rest were up to date)echo.@@ -276,6 +399,10 @@ if errorlevel 1 (exit /b 1)+if "%STRIP%"=="1" (+ echo Note: --strip is a no-op on Windows ^(debug info is in a separate .pdb^)+)+echo.echo Successfully built: %OUTPUT%for %%f in ("%OUTPUT%") do echo Size: %%~zf bytesdiff --git a/llamafile/cuda.sh b/llamafile/cuda.shindex d5e36d8d..3349b738 100755--- a/llamafile/cuda.sh+++ b/llamafile/cuda.sh@@ -43,6 +43,11 @@ source "$SCRIPT_DIR/build-functions.sh"#USE_CUBLAS=0+MINIMAL_ARCHS=0+NO_IQ_QUANTS=0+STRIP=0+COMPRESS=0+FA_ALL_QUANTS=0ARGS=()for arg in "$@"; do@@ -50,12 +55,42 @@ for arg in "$@"; do--cublas)USE_CUBLAS=1;;+ --fa-all-quants)+ FA_ALL_QUANTS=1+ ;;+ --minimize-size)+ MINIMAL_ARCHS=1+ NO_IQ_QUANTS=1+ STRIP=1+ COMPRESS=1+ ;;+ --minimal-archs)+ MINIMAL_ARCHS=1+ ;;+ --no-iq-quants)+ NO_IQ_QUANTS=1+ ;;+ --strip)+ STRIP=1+ ;;+ --compress)+ COMPRESS=1+ ;;--help)echo "Usage: $0 [-jN] [--clean] [--cublas] [--output PATH]"- echo " -jN Use N parallel jobs (default: auto-detect)"- echo " --clean Clean build directory before building"- echo " --cublas Use NVIDIA cuBLAS instead of TinyBLAS"- echo " --output Output path for shared library"+ echo " -jN Use N parallel jobs (default: auto-detect)"+ echo " --clean Clean build directory before building"+ echo " --cublas Use NVIDIA cuBLAS instead of TinyBLAS"+ echo " --output Output path for shared library"+ echo " --fa-all-quants Compile all flash-attention vec quant combos"+ echo " (default: f16-f16, q4_0-q4_0, q8_0-q8_0, bf16-bf16 only)"+ echo ""+ echo "Size reduction options (all off by default):"+ echo " --minimize-size Enable all size reduction options below"+ echo " --minimal-archs Use virtual PTX for sm_75/sm_90, real SASS for sm_80/86/89"+ echo " --no-iq-quants Exclude IQ quant MMQ template instances"+ echo " --strip Strip the final shared library"+ echo " --compress Pass --compress-mode=size to nvcc (requires CUDA >= 12.8)"exit 0;;*)@@ -128,6 +163,14 @@ echo " Source: $GGML_CUDA_DIR"echo " Output: $OUTPUT"echo " Build: $BUILD_DIR"echo " Jobs: $JOBS"+if [ "$MINIMAL_ARCHS" = "1" ] || [ "$NO_IQ_QUANTS" = "1" ] || [ "$STRIP" = "1" ] || [ "$COMPRESS" = "1" ]; then+ echo " Size reduction:"+ [ "$MINIMAL_ARCHS" = "1" ] && echo " - Minimal archs (PTX for sm_75/sm_90)"+ [ "$NO_IQ_QUANTS" = "1" ] && echo " - No IQ quant templates"+ [ "$STRIP" = "1" ] && echo " - Strip enabled"+ [ "$COMPRESS" = "1" ] && echo " - Compress mode enabled"+fi+[ "$FA_ALL_QUANTS" = "1" ] && echo " FA all quants: all fattn-vec template instances included"# Copy TinyBLAS files if neededif [ "$USE_CUBLAS" = "0" ]; then@@ -142,29 +185,50 @@ fi# sm_86: Ampere (RTX 3000 series mobile)# sm_89: Ada Lovelace (RTX 4000 series, L40S)# sm_90: Hopper (H100)-ARCH_FLAGS="\+if [ "$MINIMAL_ARCHS" = "1" ]; then+ # Virtual PTX for less-used archs (JIT-compiled and cached on first run),+ # real SASS for the most popular archs (no JIT penalty).+ ARCH_FLAGS="\+ -gencode arch=compute_75,code=compute_75 \+ -gencode arch=compute_80,code=sm_80 \+ -gencode arch=compute_86,code=sm_86 \+ -gencode arch=compute_89,code=sm_89 \+ -gencode arch=compute_90,code=compute_90"+else+ ARCH_FLAGS="\-gencode arch=compute_75,code=sm_75 \-gencode arch=compute_80,code=sm_80 \-gencode arch=compute_86,code=sm_86 \-gencode arch=compute_89,code=sm_89 \-gencode arch=compute_90,code=sm_90"+fi-# Enabling Blackwell native codegen requires CUDA 13.x+# Detect CUDA version for Blackwell and compress supportCUDA_VERSION=$("$NVCC" --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p')+CUDA_MAJOR="${CUDA_VERSION%%.*}"+CUDA_MINOR="${CUDA_VERSION#*.}"HOST_ARCH=$(uname -m)# Blackwell aarch64 non-server platforms (sm_110: Jetson Thor & family, sm_121: DGX Spark GB10)-if [ "$HOST_ARCH" = "aarch64" ] && [ "${CUDA_VERSION%%.*}" = "13" ]; then+if [ "$HOST_ARCH" = "aarch64" ] && [ "$CUDA_MAJOR" = "13" ]; thenARCH_FLAGS="\-gencode arch=compute_110f,code=sm_110f \- -gencode arch=compute_121a,code=sm_121a \- --compress-mode=size"+ -gencode arch=compute_121a,code=sm_121a"# Blackwell GPUs: CUDA 13.x append sm_120 family GPU support (RTX 5000 series, RTX PRO Blackwell)-elif [ "${CUDA_VERSION%%.*}" = "13" ]; then+elif [ "$CUDA_MAJOR" = "13" ]; thenARCH_FLAGS="$ARCH_FLAGS \- -gencode arch=compute_120f,code=sm_120f \- --compress-mode=size"+ -gencode arch=compute_120f,code=sm_120f"+fi++# --compress-mode=size: opt-in via --compress (or --minimize-size). Requires CUDA >= 12.8.+if [ "$COMPRESS" = "1" ]; then+ if [ "$CUDA_MAJOR" -gt 12 ] 2>/dev/null || \+ { [ "$CUDA_MAJOR" = "12" ] && [ "$CUDA_MINOR" -ge 8 ] 2>/dev/null; }; then+ ARCH_FLAGS="$ARCH_FLAGS --compress-mode=size"+ else+ echo "Warning: --compress requested but CUDA $CUDA_VERSION < 12.8; ignoring."+ fifi# NVCC compiler flags@@ -183,8 +247,15 @@ COMMON_FLAGS="\-DGGML_MULTIPLATFORM \$BLAS_DEFINE"+if [ "$NO_IQ_QUANTS" = "1" ]; then+ COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_NO_IQ_QUANTS"+fi+if [ "$FA_ALL_QUANTS" = "1" ]; then+ COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_FA_ALL_QUANTS"+fi+# Collect sources-collect_gpu_sources "$GGML_CUDA_DIR" "$EXTRA_SOURCES"+collect_gpu_sources "$GGML_CUDA_DIR" "$EXTRA_SOURCES" "$NO_IQ_QUANTS" "$FA_ALL_QUANTS"echo " Sources: $NUM_SOURCES .cu files"echo ""@@ -203,6 +274,12 @@ compile_ggml_core "$LLAMA_CPP_DIR" "$BUILD_DIR"# Linklink_shared_library "$NVCC" "--shared" "$ARCH_FLAGS" "$BUILD_DIR" "$OUTPUT" "$LINK_LIBS"+# Strip+if [ "$STRIP" = "1" ]; then+ echo "Stripping $OUTPUT..."+ strip --strip-unneeded "$OUTPUT"+fi+# Doneif [ "$USE_CUBLAS" = "1" ]; thenprint_build_summary "$OUTPUT" "$START_TIME" "Note: This library requires libcublas.so at runtime"diff --git a/llamafile/cuda_parallel.bat b/llamafile/cuda_parallel.batindex 65c63aec..a8891f0b 100644--- a/llamafile/cuda_parallel.bat+++ b/llamafile/cuda_parallel.bat@@ -26,14 +26,38 @@ set "CLEAN=0"set "OUTPUT="set "JOBS=%NUMBER_OF_PROCESSORS%"if not defined JOBS set "JOBS=4"+set "MINIMAL_ARCHS=0"+set "NO_IQ_QUANTS=0"+set "STRIP=0"+set "COMPRESS=0"+set "FA_ALL_QUANTS=0":parse_argsif "%~1"=="" goto done_args-if /i "%~1"=="--cublas" (set "USE_CUBLAS=1" & shift & goto parse_args)-if /i "%~1"=="--clean" (set "CLEAN=1" & shift & goto parse_args)-if /i "%~1"=="--output" (set "OUTPUT=%~2" & shift & shift & goto parse_args)+if /i "%~1"=="--cublas" (set "USE_CUBLAS=1" & shift & goto parse_args)+if /i "%~1"=="--clean" (set "CLEAN=1" & shift & goto parse_args)+if /i "%~1"=="--output" (set "OUTPUT=%~2" & shift & shift & goto parse_args)+if /i "%~1"=="--fa-all-quants" (set "FA_ALL_QUANTS=1" & shift & goto parse_args)+if /i "%~1"=="--minimize-size" (set "MINIMAL_ARCHS=1" & set "NO_IQ_QUANTS=1" & set "STRIP=1" & set "COMPRESS=1" & shift & goto parse_args)+if /i "%~1"=="--minimal-archs" (set "MINIMAL_ARCHS=1" & shift & goto parse_args)+if /i "%~1"=="--no-iq-quants" (set "NO_IQ_QUANTS=1" & shift & goto parse_args)+if /i "%~1"=="--strip" (set "STRIP=1" & shift & goto parse_args)+if /i "%~1"=="--compress" (set "COMPRESS=1" & shift & goto parse_args)if /i "%~1"=="--help" (echo Usage: cuda_parallel.bat [-jN] [--clean] [--cublas] [--output PATH]+ echo -jN Use N parallel jobs ^(default: NUMBER_OF_PROCESSORS^)+ echo --clean Clean build directory before building+ echo --cublas Use NVIDIA cuBLAS instead of TinyBLAS+ echo --output Output path for shared library+ echo --fa-all-quants Compile all flash-attention vec quant combos+ echo ^(default: f16-f16, q4_0-q4_0, q8_0-q8_0, bf16-bf16 only^)+ echo.+ echo Size reduction options ^(all off by default^):+ echo --minimize-size Enable all size reduction options below+ echo --minimal-archs Use virtual PTX for sm_75/sm_90, real SASS for sm_80/86/89+ echo --no-iq-quants Exclude IQ quant MMQ template instances+ echo --strip No-op on Windows ^(debug info is in a separate .pdb^)+ echo --compress Pass --compress-mode=size to nvcc ^(requires CUDA ^>= 12.8^)exit /b 0):: Handle -jN@@ -119,28 +143,50 @@ if not exist "%PARALLEL%" (echo.)-:: -------- detect CUDA version for Blackwell support --------+:: -------- architecture flags --------+:: Virtual PTX for less-used archs (JIT on first run), real SASS for popular archs.set "ARCH_FLAGS="-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_75,code=sm_75"-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_80,code=sm_80"-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_86,code=sm_86"-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_89,code=sm_89"-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_90,code=sm_90"--:: Check for CUDA 13.x to add Blackwell support-for /f "tokens=*" %%v in ('nvcc --version 2^>nul ^| findstr /r "release [0-9]"') do (- set "NVCC_VER_LINE=%%v"+if "%MINIMAL_ARCHS%"=="1" (+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_75,code=compute_75"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_80,code=sm_80"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_86,code=sm_86"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_89,code=sm_89"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_90,code=compute_90"+) else (+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_75,code=sm_75"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_80,code=sm_80"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_86,code=sm_86"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_89,code=sm_89"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_90,code=sm_90")++:: Detect CUDA version for Blackwell and compress support+:: nvcc prints e.g. "Cuda compilation tools, release 13.2, V13.2.51".+:: Splitting on ',', ' ', and '.' produces tokens 4..6 = "release", "13", "2".set "CUDA_MAJOR="-if defined NVCC_VER_LINE (- for /f "tokens=2 delims=," %%a in ("!NVCC_VER_LINE!") do (- for /f "tokens=2" %%b in ("%%a") do (- for /f "tokens=1 delims=." %%c in ("%%b") do set "CUDA_MAJOR=%%c"- )- )+set "CUDA_MINOR="+for /f "tokens=5,6 delims=, ." %%a in ('nvcc --version 2^>nul ^| findstr /r "release [0-9]"') do (+ set "CUDA_MAJOR=%%a"+ set "CUDA_MINOR=%%b")if "%CUDA_MAJOR%"=="13" (- set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_120f,code=sm_120f --compress-mode=size"+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_120f,code=sm_120f"+)++:: Fall back to 0 if version parsing failed, to keep the numeric comparisons below valid.+if not defined CUDA_MAJOR set "CUDA_MAJOR=0"+if not defined CUDA_MINOR set "CUDA_MINOR=0"++:: --compress-mode=size: opt-in via --compress (or --minimize-size). Requires CUDA >= 12.8.+if "%COMPRESS%"=="1" (+ set "COMPRESS_OK=0"+ if !CUDA_MAJOR! gtr 12 set "COMPRESS_OK=1"+ if !CUDA_MAJOR! equ 12 if !CUDA_MINOR! geq 8 set "COMPRESS_OK=1"+ if "!COMPRESS_OK!"=="1" (+ set "ARCH_FLAGS=!ARCH_FLAGS! --compress-mode=size"+ ) else (+ echo Warning: --compress requested but CUDA !CUDA_MAJOR!.!CUDA_MINOR! ^< 12.8; ignoring.+ )):: -------- copy TinyBLAS files if needed --------@@ -160,6 +206,8 @@ set "COMMON_FLAGS=%COMMON_FLAGS% -Xcompiler="/nologo /EHsc /O2 /GR /MT /std:c++1set "COMMON_FLAGS=%COMMON_FLAGS% -diag-suppress 177 -diag-suppress 221 -diag-suppress 550"set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"set "COMMON_FLAGS=%COMMON_FLAGS% %BLAS_DEFINE%"+if "%NO_IQ_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_NO_IQ_QUANTS"+if "%FA_ALL_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_FA_ALL_QUANTS":: -------- extract GGML version --------set "GGML_VERSION=unknown"@@ -188,6 +236,14 @@ echo Source: %GGML_CUDA_DIR%echo Output: %OUTPUT%echo Build: %BUILD_DIR%echo Jobs: %JOBS%+if "%MINIMAL_ARCHS%%NO_IQ_QUANTS%%STRIP%%COMPRESS%" neq "0000" (+ echo Size reduction:+ if "%MINIMAL_ARCHS%"=="1" echo - Minimal archs ^(PTX for sm_75/sm_90^)+ if "%NO_IQ_QUANTS%"=="1" echo - No IQ quant templates+ if "%STRIP%"=="1" echo - Strip ^(no-op on Windows; debug info is in a separate .pdb^)+ if "%COMPRESS%"=="1" echo - Compress mode enabled+)+if "%FA_ALL_QUANTS%"=="1" echo FA all quants: all fattn-vec template instances includedecho.:: -------- generate compilation command list --------@@ -219,8 +275,11 @@ for %%f in ("%GGML_CUDA_DIR%\*.cu") do (set "OBJ_FILES=!OBJ_FILES! "!OBJ!"")-:: Template-instances CUDA sources-for %%f in ("%GGML_CUDA_DIR%\template-instances\*.cu") do (+:: Template-instances: selectively included to mirror upstream CMake defaults.+set "TI_DIR=%GGML_CUDA_DIR%\template-instances"++:: 2. fattn-mma and fattn-tile instances (always included)+for %%f in ("%TI_DIR%\fattn-mma-*.cu" "%TI_DIR%\fattn-tile-*.cu") do (set "BASE=%%~nf"set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"if not exist "!OBJ!" (@@ -230,6 +289,53 @@ for %%f in ("%GGML_CUDA_DIR%\template-instances\*.cu") do (set "OBJ_FILES=!OBJ_FILES! "!OBJ!"")+:: 3. fattn-vec: only the 4 default quant combos unless --fa-all-quants.+if "%FA_ALL_QUANTS%"=="1" (+ set "FATTN_VEC_GLOB=%TI_DIR%\fattn-vec-instance-*.cu"+) else (+ set "FATTN_VEC_GLOB=%TI_DIR%\fattn-vec-instance-f16-f16.cu %TI_DIR%\fattn-vec-instance-q4_0-q4_0.cu %TI_DIR%\fattn-vec-instance-q8_0-q8_0.cu %TI_DIR%\fattn-vec-instance-bf16-bf16.cu"+)+for %%f in (!FATTN_VEC_GLOB!) do (+ if exist "%%f" (+ set "BASE=%%~nf"+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"+ if not exist "!OBJ!" (+ echo ti-!BASE!.cu:::nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "%%f">> "%CMD_FILE%"+ set /a CMD_COUNT+=1+ )+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""+ )+)++:: 4. mmf instances (always included)+for %%f in ("%TI_DIR%\mmf-*.cu") do (+ set "BASE=%%~nf"+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"+ if not exist "!OBJ!" (+ echo ti-!BASE!.cu:::nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "%%f">> "%CMD_FILE%"+ set /a CMD_COUNT+=1+ )+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""+)++:: 5. mmq instances: all unless --no-iq-quants excludes mmq-instance-iq*+for %%f in ("%TI_DIR%\mmq-*.cu") do (+ set "BASE=%%~nf"+ set "SKIP=0"+ if "%NO_IQ_QUANTS%"=="1" (+ echo !BASE! | findstr /b /i "mmq-instance-iq" >nul+ if not errorlevel 1 set "SKIP=1"+ )+ if "!SKIP!"=="0" (+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"+ if not exist "!OBJ!" (+ echo ti-!BASE!.cu:::nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "%%f">> "%CMD_FILE%"+ set /a CMD_COUNT+=1+ )+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""+ )+)+:: -------- compile .cu sources in parallel --------if !CMD_COUNT! gtr 0 (echo Compiling !CMD_COUNT! .cu files with %JOBS% parallel jobs...@@ -295,6 +401,10 @@ if errorlevel 1 (exit /b 1)+if "%STRIP%"=="1" (+ echo Note: --strip is a no-op on Windows ^(debug info is in a separate .pdb^)+)+echo.echo Successfully built: %OUTPUT%for %%f in ("%OUTPUT%") do echo Size: %%~zf bytes