Skip to content

Distribute and run LLMs with a single file.

Reduce cuda dylibs size (#963)

Davide Eynard3dd200539c6c
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch
new file mode 100644
index 00000000..6ef4957c
--- /dev/null
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch
@@ -0,0 +1,35 @@
+diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/convert.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/convert.cu
+@@ -736,6 +736,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
+ return dequantize_row_q5_K_cuda;
+ case GGML_TYPE_Q6_K:
+ return dequantize_row_q6_K_cuda;
++#ifndef GGML_CUDA_NO_IQ_QUANTS
+ case GGML_TYPE_IQ2_XXS:
+ return dequantize_row_iq2_xxs_cuda;
+ case GGML_TYPE_IQ2_XS:
+@@ -754,6 +755,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
+ return dequantize_row_iq4_xs_cuda;
+ case GGML_TYPE_IQ3_S:
+ return dequantize_row_iq3_s_cuda;
++#endif // GGML_CUDA_NO_IQ_QUANTS
+ case GGML_TYPE_MXFP4:
+ return dequantize_row_mxfp4_cuda;
+ case GGML_TYPE_NVFP4:
+@@ -791,6 +793,7 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {
+ return dequantize_row_q5_K_cuda;
+ case GGML_TYPE_Q6_K:
+ return dequantize_row_q6_K_cuda;
++#ifndef GGML_CUDA_NO_IQ_QUANTS
+ case GGML_TYPE_IQ2_XXS:
+ return dequantize_row_iq2_xxs_cuda;
+ case GGML_TYPE_IQ2_XS:
+@@ -809,6 +812,7 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {
+ return dequantize_row_iq4_xs_cuda;
+ case GGML_TYPE_IQ3_S:
+ return dequantize_row_iq3_s_cuda;
++#endif // GGML_CUDA_NO_IQ_QUANTS
+ case GGML_TYPE_MXFP4:
+ return dequantize_row_mxfp4_cuda;
+ case GGML_TYPE_NVFP4:
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch
new file mode 100644
index 00000000..36166384
--- /dev/null
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch
@@ -0,0 +1,31 @@
+diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/cpy.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/cpy.cu
+@@ -360,6 +360,7 @@ static void ggml_cpy_q5_1_f32_cuda(
+ ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+ }
+
++#ifndef GGML_CUDA_NO_IQ_QUANTS
+ static void ggml_cpy_f32_iq4_nl_cuda(
+ const char * cx, char * cdst, const int64_t ne,
+ const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,
+@@ -371,6 +372,7 @@ static void ggml_cpy_f32_iq4_nl_cuda(
+ cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, 1, 0, stream>>>
+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
+ }
++#endif // GGML_CUDA_NO_IQ_QUANTS
+
+ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1) {
+ const int64_t ne = ggml_nelements(src0);
+@@ -465,9 +467,11 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
+ } else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_F32) {
+ ggml_cpy_q5_0_f32_cuda
+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
++#ifndef GGML_CUDA_NO_IQ_QUANTS
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) {
+ ggml_cpy_f32_iq4_nl_cuda
+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
++#endif // GGML_CUDA_NO_IQ_QUANTS
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) {
+ ggml_cpy_f32_q5_1_cuda
+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch
new file mode 100644
index 00000000..c54c0e0f
--- /dev/null
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch
@@ -0,0 +1,47 @@
+diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/mmq.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/mmq.cu
+@@ -44,6 +44,7 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con
+ case GGML_TYPE_Q6_K:
+ mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, args, stream);
+ break;
++#ifndef GGML_CUDA_NO_IQ_QUANTS
+ case GGML_TYPE_IQ2_XXS:
+ mul_mat_q_case<GGML_TYPE_IQ2_XXS>(ctx, args, stream);
+ break;
+@@ -68,6 +69,7 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con
+ case GGML_TYPE_IQ4_NL:
+ mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, args, stream);
+ break;
++#endif // GGML_CUDA_NO_IQ_QUANTS
+ default:
+ GGML_ABORT("fatal error");
+ break;
+@@ -286,6 +288,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
++#ifndef GGML_CUDA_NO_IQ_QUANTS
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_S:
+@@ -294,6 +297,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ4_NL:
++#endif // GGML_CUDA_NO_IQ_QUANTS
+ mmq_supported = true;
+ break;
+ default:
+@@ -356,9 +360,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+ return ne11 <= 128;
+ case GGML_TYPE_Q6_K:
+ return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);
++#ifndef GGML_CUDA_NO_IQ_QUANTS
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_S:
+ return GGML_CUDA_CC_IS_RDNA3_5(cc) || ne11 <= 128;
++#endif // GGML_CUDA_NO_IQ_QUANTS
+ default:
+ return true;
+ }
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch
new file mode 100644
index 00000000..c52d0de0
--- /dev/null
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch
@@ -0,0 +1,19 @@
+diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
+--- a/llama.cpp/ggml/src/ggml-cuda/mmq.cuh
++++ b/llama.cpp/ggml/src/ggml-cuda/mmq.cuh
+@@ -4088,6 +4088,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);
+ extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);
+ extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);
+ extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);
++#ifndef GGML_CUDA_NO_IQ_QUANTS
+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
+ extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);
+@@ -4096,6 +4097,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);
+ extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
+ extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);
+ extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);
++#endif // GGML_CUDA_NO_IQ_QUANTS
+
+ // -------------------------------------------------------------------------------------------------------------------------
+
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmvq.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmvq.cu.patch
new file mode 100644
index 00000000..4c3fe11e
--- /dev/null
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmvq.cu.patch
@@ -0,0 +1,35 @@
+diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/mmvq.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/mmvq.cu
+@@ -22,6 +22,7 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type)
+ case GGML_TYPE_Q4_K: return vec_dot_q4_K_q8_1;
+ case GGML_TYPE_Q5_K: return vec_dot_q5_K_q8_1;
+ case GGML_TYPE_Q6_K: return vec_dot_q6_K_q8_1;
++#ifndef GGML_CUDA_NO_IQ_QUANTS
+ case GGML_TYPE_IQ2_XXS: return vec_dot_iq2_xxs_q8_1;
+ case GGML_TYPE_IQ2_XS: return vec_dot_iq2_xs_q8_1;
+ case GGML_TYPE_IQ2_S: return vec_dot_iq2_s_q8_1;
+@@ -31,6 +32,7 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type)
+ case GGML_TYPE_IQ4_NL: return vec_dot_iq4_nl_q8_1;
+ case GGML_TYPE_IQ4_XS: return vec_dot_iq4_xs_q8_1;
+ case GGML_TYPE_IQ3_S: return vec_dot_iq3_s_q8_1;
++#endif // GGML_CUDA_NO_IQ_QUANTS
+ default: return nullptr;
+ }
+ }
+@@ -50,6 +52,7 @@ static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) {
+ case GGML_TYPE_Q4_K: return VDR_Q4_K_Q8_1_MMVQ;
+ case GGML_TYPE_Q5_K: return VDR_Q5_K_Q8_1_MMVQ;
+ case GGML_TYPE_Q6_K: return VDR_Q6_K_Q8_1_MMVQ;
++#ifndef GGML_CUDA_NO_IQ_QUANTS
+ case GGML_TYPE_IQ2_XXS: return VDR_IQ2_XXS_Q8_1_MMVQ;
+ case GGML_TYPE_IQ2_XS: return VDR_IQ2_XS_Q8_1_MMVQ;
+ case GGML_TYPE_IQ2_S: return VDR_IQ2_S_Q8_1_MMVQ;
+@@ -57,6 +60,7 @@ static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) {
+ case GGML_TYPE_IQ3_S: return VDR_IQ3_S_Q8_1_MMVQ;
+ case GGML_TYPE_IQ4_NL: return VDR_IQ4_NL_Q8_1_MMVQ;
+ case GGML_TYPE_IQ4_XS: return VDR_IQ4_XS_Q8_1_MMVQ;
++#endif // GGML_CUDA_NO_IQ_QUANTS
+ default: return 1;
+ }
+ }
diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.sh
index 3f81934d..590d3038 100644
--- a/llamafile/build-functions.sh
+++ b/llamafile/build-functions.sh
@@ -98,17 +98,64 @@ setup_build_dir() {
mkdir -p "$build_dir"
}
-# Collect CUDA/HIP source files
+# Collect CUDA/HIP source files with selective template inclusion
# Sets: CUDA_SOURCES, NUM_SOURCES
-# Args: $1 = GGML_CUDA_DIR, $2 = extra sources (optional, e.g., tinyblas.cu path)
+# Args: $1 = GGML_CUDA_DIR
+# $2 = caller-supplied sources prepended to the list (e.g., tinyblas.cu
+# for the default TinyBLAS build; empty for the --cublas build)
+# $3 = NO_IQ_QUANTS (optional, "1" to exclude IQ quant MMQ templates)
+# $4 = FA_ALL_QUANTS (optional, "1" to include all fattn-vec quant combos
+# instead of the 3 default ones; mirrors upstream's GGML_CUDA_FA_ALL_QUANTS)
collect_gpu_sources() {
local ggml_cuda_dir="$1"
- local extra_sources="$2"
+ local caller_sources="$2"
+ local no_iq_quants="${3:-0}"
+ local fa_all_quants="${4:-0}"
- CUDA_SOURCES="$extra_sources"
+ CUDA_SOURCES="$caller_sources"
- for f in "$ggml_cuda_dir"/*.cu "$ggml_cuda_dir/template-instances"/*.cu; do
+ # 1. Main CUDA sources (always included)
+ for f in "$ggml_cuda_dir"/*.cu; do
+ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f"
+ done
+
+ local ti_dir="$ggml_cuda_dir/template-instances"
+
+ # 2. fattn-mma and fattn-tile instances (always included)
+ for f in "$ti_dir"/fattn-mma-*.cu "$ti_dir"/fattn-tile-*.cu; do
+ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f"
+ done
+
+ # 3. fattn-vec: default to the 4 common quant combos (f16-f16, q4_0-q4_0,
+ # q8_0-q8_0, bf16-bf16), matching upstream CMake. With FA_ALL_QUANTS=1
+ # include all fattn-vec instances (mirrors upstream's
+ # GGML_CUDA_FA_ALL_QUANTS opt-in).
+ if [ "$fa_all_quants" = "1" ]; then
+ for f in "$ti_dir"/fattn-vec-instance-*.cu; do
+ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f"
+ done
+ else
+ for f in "$ti_dir"/fattn-vec-instance-f16-f16.cu \
+ "$ti_dir"/fattn-vec-instance-q4_0-q4_0.cu \
+ "$ti_dir"/fattn-vec-instance-q8_0-q8_0.cu \
+ "$ti_dir"/fattn-vec-instance-bf16-bf16.cu; do
+ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f"
+ done
+ fi
+
+ # 4. mmf instances (always included)
+ for f in "$ti_dir"/mmf-*.cu; do
+ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f"
+ done
+
+ # 5. mmq instances: include all, but optionally exclude IQ quant templates
+ for f in "$ti_dir"/mmq-*.cu; do
if [ -f "$f" ]; then
+ if [ "$no_iq_quants" = "1" ]; then
+ case "$(basename "$f")" in
+ mmq-instance-iq*) continue ;;
+ esac
+ fi
CUDA_SOURCES="$CUDA_SOURCES $f"
fi
done
diff --git a/llamafile/cuda.bat b/llamafile/cuda.bat
index ad0b428d..ba940c77 100644
--- a/llamafile/cuda.bat
+++ b/llamafile/cuda.bat
@@ -22,14 +22,37 @@ for %%I in ("%~dp0..") do set "REPO_DIR=%%~fI"
set "USE_CUBLAS=0"
set "CLEAN=0"
set "OUTPUT="
+set "MINIMAL_ARCHS=0"
+set "NO_IQ_QUANTS=0"
+set "STRIP=0"
+set "COMPRESS=0"
+set "FA_ALL_QUANTS=0"
:parse_args
if "%~1"=="" goto done_args
-if /i "%~1"=="--cublas" (set "USE_CUBLAS=1" & shift & goto parse_args)
-if /i "%~1"=="--clean" (set "CLEAN=1" & shift & goto parse_args)
-if /i "%~1"=="--output" (set "OUTPUT=%~2" & shift & shift & goto parse_args)
+if /i "%~1"=="--cublas" (set "USE_CUBLAS=1" & shift & goto parse_args)
+if /i "%~1"=="--clean" (set "CLEAN=1" & shift & goto parse_args)
+if /i "%~1"=="--output" (set "OUTPUT=%~2" & shift & shift & goto parse_args)
+if /i "%~1"=="--fa-all-quants" (set "FA_ALL_QUANTS=1" & shift & goto parse_args)
+if /i "%~1"=="--minimize-size" (set "MINIMAL_ARCHS=1" & set "NO_IQ_QUANTS=1" & set "STRIP=1" & set "COMPRESS=1" & shift & goto parse_args)
+if /i "%~1"=="--minimal-archs" (set "MINIMAL_ARCHS=1" & shift & goto parse_args)
+if /i "%~1"=="--no-iq-quants" (set "NO_IQ_QUANTS=1" & shift & goto parse_args)
+if /i "%~1"=="--strip" (set "STRIP=1" & shift & goto parse_args)
+if /i "%~1"=="--compress" (set "COMPRESS=1" & shift & goto parse_args)
if /i "%~1"=="--help" (
echo Usage: cuda.bat [--clean] [--cublas] [--output PATH]
+ echo --clean Clean build directory before building
+ echo --cublas Use NVIDIA cuBLAS instead of TinyBLAS
+ echo --output Output path for shared library
+ echo --fa-all-quants Compile all flash-attention vec quant combos
+ echo ^(default: f16-f16, q4_0-q4_0, q8_0-q8_0, bf16-bf16 only^)
+ echo.
+ echo Size reduction options ^(all off by default^):
+ echo --minimize-size Enable all size reduction options below
+ echo --minimal-archs Use virtual PTX for sm_75/sm_90, real SASS for sm_80/86/89
+ echo --no-iq-quants Exclude IQ quant MMQ template instances
+ echo --strip No-op on Windows ^(debug info is in a separate .pdb^)
+ echo --compress Pass --compress-mode=size to nvcc ^(requires CUDA ^>= 12.8^)
exit /b 0
)
echo Unknown option: %~1
@@ -97,28 +120,50 @@ if errorlevel 1 (
exit /b 1
)
-:: -------- detect CUDA version for Blackwell support --------
+:: -------- architecture flags --------
+:: Virtual PTX for less-used archs (JIT on first run), real SASS for popular archs.
set "ARCH_FLAGS="
-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_75,code=sm_75"
-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_80,code=sm_80"
-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_86,code=sm_86"
-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_89,code=sm_89"
-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_90,code=sm_90"
-
-:: Check for CUDA 13.x to add Blackwell support
-for /f "tokens=*" %%v in ('nvcc --version 2^>nul ^| findstr /r "release [0-9]"') do (
- set "NVCC_VER_LINE=%%v"
+if "%MINIMAL_ARCHS%"=="1" (
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_75,code=compute_75"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_80,code=sm_80"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_86,code=sm_86"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_89,code=sm_89"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_90,code=compute_90"
+) else (
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_75,code=sm_75"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_80,code=sm_80"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_86,code=sm_86"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_89,code=sm_89"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_90,code=sm_90"
)
+
+:: Detect CUDA version for Blackwell and compress support
+:: nvcc prints e.g. "Cuda compilation tools, release 13.2, V13.2.51".
+:: Splitting on ',', ' ', and '.' produces tokens 4..6 = "release", "13", "2".
set "CUDA_MAJOR="
-if defined NVCC_VER_LINE (
- for /f "tokens=2 delims=," %%a in ("!NVCC_VER_LINE!") do (
- for /f "tokens=2" %%b in ("%%a") do (
- for /f "tokens=1 delims=." %%c in ("%%b") do set "CUDA_MAJOR=%%c"
- )
- )
+set "CUDA_MINOR="
+for /f "tokens=5,6 delims=, ." %%a in ('nvcc --version 2^>nul ^| findstr /r "release [0-9]"') do (
+ set "CUDA_MAJOR=%%a"
+ set "CUDA_MINOR=%%b"
)
if "%CUDA_MAJOR%"=="13" (
- set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_120f,code=sm_120f --compress-mode=size"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_120f,code=sm_120f"
+)
+
+:: Fall back to 0 if version parsing failed, to keep the numeric comparisons below valid.
+if not defined CUDA_MAJOR set "CUDA_MAJOR=0"
+if not defined CUDA_MINOR set "CUDA_MINOR=0"
+
+:: --compress-mode=size: opt-in via --compress (or --minimize-size). Requires CUDA >= 12.8.
+if "%COMPRESS%"=="1" (
+ set "COMPRESS_OK=0"
+ if !CUDA_MAJOR! gtr 12 set "COMPRESS_OK=1"
+ if !CUDA_MAJOR! equ 12 if !CUDA_MINOR! geq 8 set "COMPRESS_OK=1"
+ if "!COMPRESS_OK!"=="1" (
+ set "ARCH_FLAGS=!ARCH_FLAGS! --compress-mode=size"
+ ) else (
+ echo Warning: --compress requested but CUDA !CUDA_MAJOR!.!CUDA_MINOR! ^< 12.8; ignoring.
+ )
)
:: -------- copy TinyBLAS files if needed --------
@@ -138,6 +183,8 @@ set "COMMON_FLAGS=%COMMON_FLAGS% -Xcompiler="/nologo /EHsc /O2 /GR /MT /std:c++1
set "COMMON_FLAGS=%COMMON_FLAGS% -diag-suppress 177 -diag-suppress 221 -diag-suppress 550"
set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"
set "COMMON_FLAGS=%COMMON_FLAGS% %BLAS_DEFINE%"
+if "%NO_IQ_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_NO_IQ_QUANTS"
+if "%FA_ALL_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_FA_ALL_QUANTS"
:: -------- extract GGML version --------
set "GGML_VERSION=unknown"
@@ -165,6 +212,14 @@ echo Version: !GGML_VERSION! (commit: !GGML_COMMIT!)
echo Source: %GGML_CUDA_DIR%
echo Output: %OUTPUT%
echo Build: %BUILD_DIR%
+if "%MINIMAL_ARCHS%%NO_IQ_QUANTS%%STRIP%%COMPRESS%" neq "0000" (
+ echo Size reduction:
+ if "%MINIMAL_ARCHS%"=="1" echo - Minimal archs ^(PTX for sm_75/sm_90^)
+ if "%NO_IQ_QUANTS%"=="1" echo - No IQ quant templates
+ if "%STRIP%"=="1" echo - Strip ^(no-op on Windows; debug info is in a separate .pdb^)
+ if "%COMPRESS%"=="1" echo - Compress mode enabled
+)
+if "%FA_ALL_QUANTS%"=="1" echo FA all quants: all fattn-vec template instances included
echo.
:: -------- collect and compile .cu sources --------
@@ -205,8 +260,52 @@ for %%f in ("%GGML_CUDA_DIR%\*.cu") do (
set "OBJ_FILES=!OBJ_FILES! "!OBJ!""
)
-:: Template-instances CUDA sources
-for %%f in ("%GGML_CUDA_DIR%\template-instances\*.cu") do (
+:: Template-instances: selectively included to mirror upstream CMake defaults.
+set "TI_DIR=%GGML_CUDA_DIR%\template-instances"
+
+:: 2. fattn-mma and fattn-tile instances (always included)
+for %%f in ("%TI_DIR%\fattn-mma-*.cu" "%TI_DIR%\fattn-tile-*.cu") do (
+ set /a COUNT+=1
+ set "BASE=%%~nf"
+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"
+ set "SRC=%%f"
+ if not exist "!OBJ!" (
+ echo [!COUNT!] Compiling: ti-!BASE!.cu
+ nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "!SRC!"
+ if errorlevel 1 (echo Error compiling ti-!BASE!.cu & exit /b 1)
+ set /a COMPILED+=1
+ ) else (
+ echo [!COUNT!] Skipping: ti-!BASE!.cu (up to date^)
+ )
+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""
+)
+
+:: 3. fattn-vec: only the 4 default quant combos unless --fa-all-quants.
+if "%FA_ALL_QUANTS%"=="1" (
+ set "FATTN_VEC_GLOB=%TI_DIR%\fattn-vec-instance-*.cu"
+) else (
+ set "FATTN_VEC_GLOB=%TI_DIR%\fattn-vec-instance-f16-f16.cu %TI_DIR%\fattn-vec-instance-q4_0-q4_0.cu %TI_DIR%\fattn-vec-instance-q8_0-q8_0.cu %TI_DIR%\fattn-vec-instance-bf16-bf16.cu"
+)
+for %%f in (!FATTN_VEC_GLOB!) do (
+ if exist "%%f" (
+ set /a COUNT+=1
+ set "BASE=%%~nf"
+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"
+ set "SRC=%%f"
+ if not exist "!OBJ!" (
+ echo [!COUNT!] Compiling: ti-!BASE!.cu
+ nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "!SRC!"
+ if errorlevel 1 (echo Error compiling ti-!BASE!.cu & exit /b 1)
+ set /a COMPILED+=1
+ ) else (
+ echo [!COUNT!] Skipping: ti-!BASE!.cu (up to date^)
+ )
+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""
+ )
+)
+
+:: 4. mmf instances (always included)
+for %%f in ("%TI_DIR%\mmf-*.cu") do (
set /a COUNT+=1
set "BASE=%%~nf"
set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"
@@ -222,6 +321,30 @@ for %%f in ("%GGML_CUDA_DIR%\template-instances\*.cu") do (
set "OBJ_FILES=!OBJ_FILES! "!OBJ!""
)
+:: 5. mmq instances: all unless --no-iq-quants excludes mmq-instance-iq*
+for %%f in ("%TI_DIR%\mmq-*.cu") do (
+ set "BASE=%%~nf"
+ set "SKIP=0"
+ if "%NO_IQ_QUANTS%"=="1" (
+ echo !BASE! | findstr /b /i "mmq-instance-iq" >nul
+ if not errorlevel 1 set "SKIP=1"
+ )
+ if "!SKIP!"=="0" (
+ set /a COUNT+=1
+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"
+ set "SRC=%%f"
+ if not exist "!OBJ!" (
+ echo [!COUNT!] Compiling: ti-!BASE!.cu
+ nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "!SRC!"
+ if errorlevel 1 (echo Error compiling ti-!BASE!.cu & exit /b 1)
+ set /a COMPILED+=1
+ ) else (
+ echo [!COUNT!] Skipping: ti-!BASE!.cu (up to date^)
+ )
+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""
+ )
+)
+
echo.
echo Compiled %COMPILED% of %COUNT% .cu files (rest were up to date)
echo.
@@ -276,6 +399,10 @@ if errorlevel 1 (
exit /b 1
)
+if "%STRIP%"=="1" (
+ echo Note: --strip is a no-op on Windows ^(debug info is in a separate .pdb^)
+)
+
echo.
echo Successfully built: %OUTPUT%
for %%f in ("%OUTPUT%") do echo Size: %%~zf bytes
diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh
index d5e36d8d..3349b738 100755
--- a/llamafile/cuda.sh
+++ b/llamafile/cuda.sh
@@ -43,6 +43,11 @@ source "$SCRIPT_DIR/build-functions.sh"
#
USE_CUBLAS=0
+MINIMAL_ARCHS=0
+NO_IQ_QUANTS=0
+STRIP=0
+COMPRESS=0
+FA_ALL_QUANTS=0
ARGS=()
for arg in "$@"; do
@@ -50,12 +55,42 @@ for arg in "$@"; do
--cublas)
USE_CUBLAS=1
;;
+ --fa-all-quants)
+ FA_ALL_QUANTS=1
+ ;;
+ --minimize-size)
+ MINIMAL_ARCHS=1
+ NO_IQ_QUANTS=1
+ STRIP=1
+ COMPRESS=1
+ ;;
+ --minimal-archs)
+ MINIMAL_ARCHS=1
+ ;;
+ --no-iq-quants)
+ NO_IQ_QUANTS=1
+ ;;
+ --strip)
+ STRIP=1
+ ;;
+ --compress)
+ COMPRESS=1
+ ;;
--help)
echo "Usage: $0 [-jN] [--clean] [--cublas] [--output PATH]"
- echo " -jN Use N parallel jobs (default: auto-detect)"
- echo " --clean Clean build directory before building"
- echo " --cublas Use NVIDIA cuBLAS instead of TinyBLAS"
- echo " --output Output path for shared library"
+ echo " -jN Use N parallel jobs (default: auto-detect)"
+ echo " --clean Clean build directory before building"
+ echo " --cublas Use NVIDIA cuBLAS instead of TinyBLAS"
+ echo " --output Output path for shared library"
+ echo " --fa-all-quants Compile all flash-attention vec quant combos"
+ echo " (default: f16-f16, q4_0-q4_0, q8_0-q8_0, bf16-bf16 only)"
+ echo ""
+ echo "Size reduction options (all off by default):"
+ echo " --minimize-size Enable all size reduction options below"
+ echo " --minimal-archs Use virtual PTX for sm_75/sm_90, real SASS for sm_80/86/89"
+ echo " --no-iq-quants Exclude IQ quant MMQ template instances"
+ echo " --strip Strip the final shared library"
+ echo " --compress Pass --compress-mode=size to nvcc (requires CUDA >= 12.8)"
exit 0
;;
*)
@@ -128,6 +163,14 @@ echo " Source: $GGML_CUDA_DIR"
echo " Output: $OUTPUT"
echo " Build: $BUILD_DIR"
echo " Jobs: $JOBS"
+if [ "$MINIMAL_ARCHS" = "1" ] || [ "$NO_IQ_QUANTS" = "1" ] || [ "$STRIP" = "1" ] || [ "$COMPRESS" = "1" ]; then
+ echo " Size reduction:"
+ [ "$MINIMAL_ARCHS" = "1" ] && echo " - Minimal archs (PTX for sm_75/sm_90)"
+ [ "$NO_IQ_QUANTS" = "1" ] && echo " - No IQ quant templates"
+ [ "$STRIP" = "1" ] && echo " - Strip enabled"
+ [ "$COMPRESS" = "1" ] && echo " - Compress mode enabled"
+fi
+[ "$FA_ALL_QUANTS" = "1" ] && echo " FA all quants: all fattn-vec template instances included"
# Copy TinyBLAS files if needed
if [ "$USE_CUBLAS" = "0" ]; then
@@ -142,29 +185,50 @@ fi
# sm_86: Ampere (RTX 3000 series mobile)
# sm_89: Ada Lovelace (RTX 4000 series, L40S)
# sm_90: Hopper (H100)
-ARCH_FLAGS="\
+if [ "$MINIMAL_ARCHS" = "1" ]; then
+ # Virtual PTX for less-used archs (JIT-compiled and cached on first run),
+ # real SASS for the most popular archs (no JIT penalty).
+ ARCH_FLAGS="\
+ -gencode arch=compute_75,code=compute_75 \
+ -gencode arch=compute_80,code=sm_80 \
+ -gencode arch=compute_86,code=sm_86 \
+ -gencode arch=compute_89,code=sm_89 \
+ -gencode arch=compute_90,code=compute_90"
+else
+ ARCH_FLAGS="\
-gencode arch=compute_75,code=sm_75 \
-gencode arch=compute_80,code=sm_80 \
-gencode arch=compute_86,code=sm_86 \
-gencode arch=compute_89,code=sm_89 \
-gencode arch=compute_90,code=sm_90"
+fi
-# Enabling Blackwell native codegen requires CUDA 13.x
+# Detect CUDA version for Blackwell and compress support
CUDA_VERSION=$("$NVCC" --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p')
+CUDA_MAJOR="${CUDA_VERSION%%.*}"
+CUDA_MINOR="${CUDA_VERSION#*.}"
HOST_ARCH=$(uname -m)
# Blackwell aarch64 non-server platforms (sm_110: Jetson Thor & family, sm_121: DGX Spark GB10)
-if [ "$HOST_ARCH" = "aarch64" ] && [ "${CUDA_VERSION%%.*}" = "13" ]; then
+if [ "$HOST_ARCH" = "aarch64" ] && [ "$CUDA_MAJOR" = "13" ]; then
ARCH_FLAGS="\
-gencode arch=compute_110f,code=sm_110f \
- -gencode arch=compute_121a,code=sm_121a \
- --compress-mode=size"
+ -gencode arch=compute_121a,code=sm_121a"
# Blackwell GPUs: CUDA 13.x append sm_120 family GPU support (RTX 5000 series, RTX PRO Blackwell)
-elif [ "${CUDA_VERSION%%.*}" = "13" ]; then
+elif [ "$CUDA_MAJOR" = "13" ]; then
ARCH_FLAGS="$ARCH_FLAGS \
- -gencode arch=compute_120f,code=sm_120f \
- --compress-mode=size"
+ -gencode arch=compute_120f,code=sm_120f"
+fi
+
+# --compress-mode=size: opt-in via --compress (or --minimize-size). Requires CUDA >= 12.8.
+if [ "$COMPRESS" = "1" ]; then
+ if [ "$CUDA_MAJOR" -gt 12 ] 2>/dev/null || \
+ { [ "$CUDA_MAJOR" = "12" ] && [ "$CUDA_MINOR" -ge 8 ] 2>/dev/null; }; then
+ ARCH_FLAGS="$ARCH_FLAGS --compress-mode=size"
+ else
+ echo "Warning: --compress requested but CUDA $CUDA_VERSION < 12.8; ignoring."
+ fi
fi
# NVCC compiler flags
@@ -183,8 +247,15 @@ COMMON_FLAGS="\
-DGGML_MULTIPLATFORM \
$BLAS_DEFINE"
+if [ "$NO_IQ_QUANTS" = "1" ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_NO_IQ_QUANTS"
+fi
+if [ "$FA_ALL_QUANTS" = "1" ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_FA_ALL_QUANTS"
+fi
+
# Collect sources
-collect_gpu_sources "$GGML_CUDA_DIR" "$EXTRA_SOURCES"
+collect_gpu_sources "$GGML_CUDA_DIR" "$EXTRA_SOURCES" "$NO_IQ_QUANTS" "$FA_ALL_QUANTS"
echo " Sources: $NUM_SOURCES .cu files"
echo ""
@@ -203,6 +274,12 @@ compile_ggml_core "$LLAMA_CPP_DIR" "$BUILD_DIR"
# Link
link_shared_library "$NVCC" "--shared" "$ARCH_FLAGS" "$BUILD_DIR" "$OUTPUT" "$LINK_LIBS"
+# Strip
+if [ "$STRIP" = "1" ]; then
+ echo "Stripping $OUTPUT..."
+ strip --strip-unneeded "$OUTPUT"
+fi
+
# Done
if [ "$USE_CUBLAS" = "1" ]; then
print_build_summary "$OUTPUT" "$START_TIME" "Note: This library requires libcublas.so at runtime"
diff --git a/llamafile/cuda_parallel.bat b/llamafile/cuda_parallel.bat
index 65c63aec..a8891f0b 100644
--- a/llamafile/cuda_parallel.bat
+++ b/llamafile/cuda_parallel.bat
@@ -26,14 +26,38 @@ set "CLEAN=0"
set "OUTPUT="
set "JOBS=%NUMBER_OF_PROCESSORS%"
if not defined JOBS set "JOBS=4"
+set "MINIMAL_ARCHS=0"
+set "NO_IQ_QUANTS=0"
+set "STRIP=0"
+set "COMPRESS=0"
+set "FA_ALL_QUANTS=0"
:parse_args
if "%~1"=="" goto done_args
-if /i "%~1"=="--cublas" (set "USE_CUBLAS=1" & shift & goto parse_args)
-if /i "%~1"=="--clean" (set "CLEAN=1" & shift & goto parse_args)
-if /i "%~1"=="--output" (set "OUTPUT=%~2" & shift & shift & goto parse_args)
+if /i "%~1"=="--cublas" (set "USE_CUBLAS=1" & shift & goto parse_args)
+if /i "%~1"=="--clean" (set "CLEAN=1" & shift & goto parse_args)
+if /i "%~1"=="--output" (set "OUTPUT=%~2" & shift & shift & goto parse_args)
+if /i "%~1"=="--fa-all-quants" (set "FA_ALL_QUANTS=1" & shift & goto parse_args)
+if /i "%~1"=="--minimize-size" (set "MINIMAL_ARCHS=1" & set "NO_IQ_QUANTS=1" & set "STRIP=1" & set "COMPRESS=1" & shift & goto parse_args)
+if /i "%~1"=="--minimal-archs" (set "MINIMAL_ARCHS=1" & shift & goto parse_args)
+if /i "%~1"=="--no-iq-quants" (set "NO_IQ_QUANTS=1" & shift & goto parse_args)
+if /i "%~1"=="--strip" (set "STRIP=1" & shift & goto parse_args)
+if /i "%~1"=="--compress" (set "COMPRESS=1" & shift & goto parse_args)
if /i "%~1"=="--help" (
echo Usage: cuda_parallel.bat [-jN] [--clean] [--cublas] [--output PATH]
+ echo -jN Use N parallel jobs ^(default: NUMBER_OF_PROCESSORS^)
+ echo --clean Clean build directory before building
+ echo --cublas Use NVIDIA cuBLAS instead of TinyBLAS
+ echo --output Output path for shared library
+ echo --fa-all-quants Compile all flash-attention vec quant combos
+ echo ^(default: f16-f16, q4_0-q4_0, q8_0-q8_0, bf16-bf16 only^)
+ echo.
+ echo Size reduction options ^(all off by default^):
+ echo --minimize-size Enable all size reduction options below
+ echo --minimal-archs Use virtual PTX for sm_75/sm_90, real SASS for sm_80/86/89
+ echo --no-iq-quants Exclude IQ quant MMQ template instances
+ echo --strip No-op on Windows ^(debug info is in a separate .pdb^)
+ echo --compress Pass --compress-mode=size to nvcc ^(requires CUDA ^>= 12.8^)
exit /b 0
)
:: Handle -jN
@@ -119,28 +143,50 @@ if not exist "%PARALLEL%" (
echo.
)
-:: -------- detect CUDA version for Blackwell support --------
+:: -------- architecture flags --------
+:: Virtual PTX for less-used archs (JIT on first run), real SASS for popular archs.
set "ARCH_FLAGS="
-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_75,code=sm_75"
-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_80,code=sm_80"
-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_86,code=sm_86"
-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_89,code=sm_89"
-set "ARCH_FLAGS=%ARCH_FLAGS% -gencode arch=compute_90,code=sm_90"
-
-:: Check for CUDA 13.x to add Blackwell support
-for /f "tokens=*" %%v in ('nvcc --version 2^>nul ^| findstr /r "release [0-9]"') do (
- set "NVCC_VER_LINE=%%v"
+if "%MINIMAL_ARCHS%"=="1" (
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_75,code=compute_75"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_80,code=sm_80"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_86,code=sm_86"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_89,code=sm_89"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_90,code=compute_90"
+) else (
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_75,code=sm_75"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_80,code=sm_80"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_86,code=sm_86"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_89,code=sm_89"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_90,code=sm_90"
)
+
+:: Detect CUDA version for Blackwell and compress support
+:: nvcc prints e.g. "Cuda compilation tools, release 13.2, V13.2.51".
+:: Splitting on ',', ' ', and '.' produces tokens 4..6 = "release", "13", "2".
set "CUDA_MAJOR="
-if defined NVCC_VER_LINE (
- for /f "tokens=2 delims=," %%a in ("!NVCC_VER_LINE!") do (
- for /f "tokens=2" %%b in ("%%a") do (
- for /f "tokens=1 delims=." %%c in ("%%b") do set "CUDA_MAJOR=%%c"
- )
- )
+set "CUDA_MINOR="
+for /f "tokens=5,6 delims=, ." %%a in ('nvcc --version 2^>nul ^| findstr /r "release [0-9]"') do (
+ set "CUDA_MAJOR=%%a"
+ set "CUDA_MINOR=%%b"
)
if "%CUDA_MAJOR%"=="13" (
- set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_120f,code=sm_120f --compress-mode=size"
+ set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_120f,code=sm_120f"
+)
+
+:: Fall back to 0 if version parsing failed, to keep the numeric comparisons below valid.
+if not defined CUDA_MAJOR set "CUDA_MAJOR=0"
+if not defined CUDA_MINOR set "CUDA_MINOR=0"
+
+:: --compress-mode=size: opt-in via --compress (or --minimize-size). Requires CUDA >= 12.8.
+if "%COMPRESS%"=="1" (
+ set "COMPRESS_OK=0"
+ if !CUDA_MAJOR! gtr 12 set "COMPRESS_OK=1"
+ if !CUDA_MAJOR! equ 12 if !CUDA_MINOR! geq 8 set "COMPRESS_OK=1"
+ if "!COMPRESS_OK!"=="1" (
+ set "ARCH_FLAGS=!ARCH_FLAGS! --compress-mode=size"
+ ) else (
+ echo Warning: --compress requested but CUDA !CUDA_MAJOR!.!CUDA_MINOR! ^< 12.8; ignoring.
+ )
)
:: -------- copy TinyBLAS files if needed --------
@@ -160,6 +206,8 @@ set "COMMON_FLAGS=%COMMON_FLAGS% -Xcompiler="/nologo /EHsc /O2 /GR /MT /std:c++1
set "COMMON_FLAGS=%COMMON_FLAGS% -diag-suppress 177 -diag-suppress 221 -diag-suppress 550"
set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"
set "COMMON_FLAGS=%COMMON_FLAGS% %BLAS_DEFINE%"
+if "%NO_IQ_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_NO_IQ_QUANTS"
+if "%FA_ALL_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_FA_ALL_QUANTS"
:: -------- extract GGML version --------
set "GGML_VERSION=unknown"
@@ -188,6 +236,14 @@ echo Source: %GGML_CUDA_DIR%
echo Output: %OUTPUT%
echo Build: %BUILD_DIR%
echo Jobs: %JOBS%
+if "%MINIMAL_ARCHS%%NO_IQ_QUANTS%%STRIP%%COMPRESS%" neq "0000" (
+ echo Size reduction:
+ if "%MINIMAL_ARCHS%"=="1" echo - Minimal archs ^(PTX for sm_75/sm_90^)
+ if "%NO_IQ_QUANTS%"=="1" echo - No IQ quant templates
+ if "%STRIP%"=="1" echo - Strip ^(no-op on Windows; debug info is in a separate .pdb^)
+ if "%COMPRESS%"=="1" echo - Compress mode enabled
+)
+if "%FA_ALL_QUANTS%"=="1" echo FA all quants: all fattn-vec template instances included
echo.
:: -------- generate compilation command list --------
@@ -219,8 +275,11 @@ for %%f in ("%GGML_CUDA_DIR%\*.cu") do (
set "OBJ_FILES=!OBJ_FILES! "!OBJ!""
)
-:: Template-instances CUDA sources
-for %%f in ("%GGML_CUDA_DIR%\template-instances\*.cu") do (
+:: Template-instances: selectively included to mirror upstream CMake defaults.
+set "TI_DIR=%GGML_CUDA_DIR%\template-instances"
+
+:: 2. fattn-mma and fattn-tile instances (always included)
+for %%f in ("%TI_DIR%\fattn-mma-*.cu" "%TI_DIR%\fattn-tile-*.cu") do (
set "BASE=%%~nf"
set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"
if not exist "!OBJ!" (
@@ -230,6 +289,53 @@ for %%f in ("%GGML_CUDA_DIR%\template-instances\*.cu") do (
set "OBJ_FILES=!OBJ_FILES! "!OBJ!""
)
+:: 3. fattn-vec: only the 4 default quant combos unless --fa-all-quants.
+if "%FA_ALL_QUANTS%"=="1" (
+ set "FATTN_VEC_GLOB=%TI_DIR%\fattn-vec-instance-*.cu"
+) else (
+ set "FATTN_VEC_GLOB=%TI_DIR%\fattn-vec-instance-f16-f16.cu %TI_DIR%\fattn-vec-instance-q4_0-q4_0.cu %TI_DIR%\fattn-vec-instance-q8_0-q8_0.cu %TI_DIR%\fattn-vec-instance-bf16-bf16.cu"
+)
+for %%f in (!FATTN_VEC_GLOB!) do (
+ if exist "%%f" (
+ set "BASE=%%~nf"
+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"
+ if not exist "!OBJ!" (
+ echo ti-!BASE!.cu:::nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "%%f">> "%CMD_FILE%"
+ set /a CMD_COUNT+=1
+ )
+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""
+ )
+)
+
+:: 4. mmf instances (always included)
+for %%f in ("%TI_DIR%\mmf-*.cu") do (
+ set "BASE=%%~nf"
+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"
+ if not exist "!OBJ!" (
+ echo ti-!BASE!.cu:::nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "%%f">> "%CMD_FILE%"
+ set /a CMD_COUNT+=1
+ )
+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""
+)
+
+:: 5. mmq instances: all unless --no-iq-quants excludes mmq-instance-iq*
+for %%f in ("%TI_DIR%\mmq-*.cu") do (
+ set "BASE=%%~nf"
+ set "SKIP=0"
+ if "%NO_IQ_QUANTS%"=="1" (
+ echo !BASE! | findstr /b /i "mmq-instance-iq" >nul
+ if not errorlevel 1 set "SKIP=1"
+ )
+ if "!SKIP!"=="0" (
+ set "OBJ=%BUILD_DIR%\ti-!BASE!.obj"
+ if not exist "!OBJ!" (
+ echo ti-!BASE!.cu:::nvcc -c %ARCH_FLAGS% %COMMON_FLAGS% -o "!OBJ!" "%%f">> "%CMD_FILE%"
+ set /a CMD_COUNT+=1
+ )
+ set "OBJ_FILES=!OBJ_FILES! "!OBJ!""
+ )
+)
+
:: -------- compile .cu sources in parallel --------
if !CMD_COUNT! gtr 0 (
echo Compiling !CMD_COUNT! .cu files with %JOBS% parallel jobs...
@@ -295,6 +401,10 @@ if errorlevel 1 (
exit /b 1
)
+if "%STRIP%"=="1" (
+ echo Note: --strip is a no-op on Windows ^(debug info is in a separate .pdb^)
+)
+
echo.
echo Successfully built: %OUTPUT%
for %%f in ("%OUTPUT%") do echo Size: %%~zf bytes