Skip to content

Distribute and run LLMs with a single file.

Update llama.cpp to b10103 (c588c4f) (#1030)

Davide Eynard0ce2b877b223
diff --git a/docs/skills/llamafile/building.md b/docs/skills/llamafile/building.md
index 63a57c00..75565ad2 100644
--- a/docs/skills/llamafile/building.md
+++ b/docs/skills/llamafile/building.md
@@ -217,9 +217,11 @@ list in sync with the CMakeLists (it must probe all seven feature tests).
### CUDA `--minimize-size` drops IQ-quant kernels (CPU fallback)
The release CUDA dylib is built with `--minimize-size`, which (among the size
-options) passes `-DGGML_CUDA_NO_IQ_QUANTS`. That compiles out every IQ-quant
-CUDA path — the MMQ/MMVQ matmul kernels and the fp16/fp32 dequant converters
-in `convert.cu`, `mmq.cu`, `mmvq.cu`, and the `f32→iq4_nl` copy in `cpy.cu`.
+options) passes `-DGGML_CUDA_NO_IQ_QUANTS`. That compiles out most IQ-quant
+CUDA paths — the MMQ/MMVQ matmul kernels in `mmq.cu`/`mmvq.cu`, the bf16/fp16
+dequant converters in `convert.cu`, and the `f32→iq4_nl` copy in `cpy.cu`
+(`ggml_get_to_fp32_cuda`'s IQ cases stay unguarded, so the `float` dequant
+instantiations still compile — a minor size cost).
The payoff is a much smaller `.so`; the cost is that it cannot run IQ-quantized
tensors on the GPU.
diff --git a/llama.cpp b/llama.cpp
index 846e991e..c588c4f4 160000
--- a/llama.cpp
+++ b/llama.cpp
@@ -1 +1 @@
-Subproject commit 846e991ec3c7ccec49112ff2c5b00b710e5f551d
+Subproject commit c588c4f47683e73ad2d69f50480bec6cc85fd0f7
diff --git a/llama.cpp.patches/README.md b/llama.cpp.patches/README.md
index 4cd5b656..0dfd2150 100644
--- a/llama.cpp.patches/README.md
+++ b/llama.cpp.patches/README.md
@@ -122,11 +122,11 @@ Llamafile uses TinyBLAS as a lightweight replacement for cuBLAS, enabling GPU su
### Optional IQ-Quant Exclusion (CUDA)
-The IQ ("importance") quantization formats (`IQ1_S`, `IQ2_XXS`/`XS`/`S`, `IQ3_S`/`XXS`, `IQ4_NL`/`XS`) pull in a large amount of CUDA template instantiation that inflates compile time and binary size. These patches gate every IQ code path behind `#ifndef GGML_CUDA_NO_IQ_QUANTS`, so a build can compile them out by defining `GGML_CUDA_NO_IQ_QUANTS`. When the macro is undefined (the default), behavior is unchanged.
+The IQ ("importance") quantization formats (`IQ1_S`, `IQ2_XXS`/`XS`/`S`, `IQ3_S`/`XXS`, `IQ4_NL`/`XS`) pull in a large amount of CUDA template instantiation that inflates compile time and binary size. These patches gate the IQ code paths behind `#ifndef GGML_CUDA_NO_IQ_QUANTS` — the MMQ/MMVQ matmul kernels, the `f32 → IQ4_NL` copy, and the IQ dequant cases in `ggml_get_to_bf16_cuda`/`ggml_get_to_fp16_cuda` — so a build can compile them out by defining `GGML_CUDA_NO_IQ_QUANTS`. (`ggml_get_to_fp32_cuda`'s IQ cases are not guarded, so the `float` dequant-template instantiations still compile — a minor size cost, harmless because `ggml_backend_cuda_device_supports_op` gates the same IQ ops, so those tensors fall back to CPU in a minimized build regardless.) When the macro is undefined (the default), behavior is unchanged.
| Patch | Description |
|-------|-------------|
-| `ggml_src_ggml-cuda_convert.cu.patch` | Guards IQ dequantization cases in `ggml_get_to_fp16_cuda` and `ggml_get_to_fp32_cuda` |
+| `ggml_src_ggml-cuda_convert.cu.patch` | Guards IQ dequantization cases in `ggml_get_to_bf16_cuda` and `ggml_get_to_fp16_cuda` |
| `ggml_src_ggml-cuda_cpy.cu.patch` | Guards the `f32 → IQ4_NL` copy helper and its dispatch case |
| `ggml_src_ggml-cuda_mmq.cu.patch` | Guards IQ cases in `ggml_cuda_mul_mat_q_switch_type` and in the `ggml_cuda_should_use_mmq` support/heuristic switches |
| `ggml_src_ggml-cuda_mmq.cuh.patch` | Guards the `extern DECL_MMQ_CASE(...)` declarations for IQ types |
diff --git a/llama.cpp.patches/llamafile-files/BUILD.mk b/llama.cpp.patches/llamafile-files/BUILD.mk
index f392bceb..532781e1 100644
--- a/llama.cpp.patches/llamafile-files/BUILD.mk
+++ b/llama.cpp.patches/llamafile-files/BUILD.mk
@@ -119,6 +119,7 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/jina-bert-v2.cpp \
llama.cpp/src/models/jina-bert-v3.cpp \
llama.cpp/src/models/kimi-linear.cpp \
+ llama.cpp/src/models/laguna.cpp \
llama.cpp/src/models/lfm2.cpp \
llama.cpp/src/models/lfm2moe.cpp \
llama.cpp/src/models/llada-moe.cpp \
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch
index c76fb7fb..6c50add6 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
--- a/llama.cpp/ggml/src/ggml-cuda/convert.cu
+++ b/llama.cpp/ggml/src/ggml-cuda/convert.cu
-@@ -730,6 +730,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {
+@@ -479,6 +479,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {
return dequantize_row_q5_K_cuda;
case GGML_TYPE_Q6_K:
return dequantize_row_q6_K_cuda;
@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
case GGML_TYPE_IQ2_XXS:
return dequantize_row_iq2_xxs_cuda;
case GGML_TYPE_IQ2_XS:
-@@ -748,6 +749,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {
+@@ -497,6 +498,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {
return dequantize_row_iq4_xs_cuda;
case GGML_TYPE_IQ3_S:
return dequantize_row_iq3_s_cuda;
@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
case GGML_TYPE_MXFP4:
return dequantize_row_mxfp4_cuda;
case GGML_TYPE_NVFP4:
-@@ -788,6 +790,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
+@@ -537,6 +539,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
return dequantize_row_q5_K_cuda;
case GGML_TYPE_Q6_K:
return dequantize_row_q6_K_cuda;
@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
case GGML_TYPE_IQ2_XXS:
return dequantize_row_iq2_xxs_cuda;
case GGML_TYPE_IQ2_XS:
-@@ -806,6 +809,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
+@@ -555,6 +558,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
return dequantize_row_iq4_xs_cuda;
case GGML_TYPE_IQ3_S:
return dequantize_row_iq3_s_cuda;
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
index 2cbee3cb..36a9f9be 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
@@ -420,7 +420,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
case GGML_TYPE_BF16:
return true;
default:
-@@ -4914,9 +4947,14 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
+@@ -4932,9 +4965,14 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
if (src0_type == GGML_TYPE_Q5_1 && src1_type == GGML_TYPE_F32) {
return true;
}
@@ -435,7 +435,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_I32) {
return true;
}
-@@ -5142,7 +5180,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
+@@ -5160,7 +5198,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
}
}
@@ -444,7 +444,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
const bool integrated = ggml_cuda_info().devices[dev_ctx->device].integrated;
return (ggml_backend_buft_is_cuda(buft) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft));
-@@ -5163,13 +5201,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
+@@ -5181,13 +5219,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
}
}
@@ -460,7 +460,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
#ifdef GGML_CUDA_NO_PEER_COPY
return nullptr;
#else
-@@ -5187,14 +5225,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_
+@@ -5205,14 +5243,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_
#endif
}
@@ -477,7 +477,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
GGML_UNUSED(dev);
CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));
}
-@@ -5223,23 +5261,23 @@ struct ggml_backend_cuda_reg_context {
+@@ -5241,23 +5279,23 @@ struct ggml_backend_cuda_reg_context {
std::vector<ggml_backend_dev_t> devices;
};
@@ -505,7 +505,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
static std::vector<ggml_backend_feature> features = []() {
std::vector<ggml_backend_feature> features;
#define _STRINGIFY(...) #__VA_ARGS__
-@@ -5296,7 +5334,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t
+@@ -5314,7 +5352,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t
GGML_UNUSED(reg);
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch
index 1e15f36e..5743045a 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch
@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
default:
GGML_ABORT("fatal error");
break;
-@@ -258,6 +260,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+@@ -271,6 +273,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q5_K:
case GGML_TYPE_Q6_K:
@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
case GGML_TYPE_IQ2_XXS:
case GGML_TYPE_IQ2_XS:
case GGML_TYPE_IQ2_S:
-@@ -266,6 +269,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+@@ -279,6 +282,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
case GGML_TYPE_IQ1_S:
case GGML_TYPE_IQ4_XS:
case GGML_TYPE_IQ4_NL:
@@ -33,7 +33,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
mmq_supported = true;
break;
default:
-@@ -328,9 +332,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+@@ -341,9 +345,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
return ne11 <= 128;
case GGML_TYPE_Q6_K:
return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch
index 19f8357e..995c52dc 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
--- a/llama.cpp/ggml/src/ggml-cuda/mmq.cuh
+++ b/llama.cpp/ggml/src/ggml-cuda/mmq.cuh
-@@ -1478,6 +1478,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);
+@@ -1549,6 +1549,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);
extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);
extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);
extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);
@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);
-@@ -1486,6 +1487,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);
+@@ -1557,6 +1558,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);
extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);
extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);