Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Update llama.cpp to b10103 (c588c4f) (#1030)
Davide Eynard0ce2b877b223
diff --git a/docs/skills/llamafile/building.md b/docs/skills/llamafile/building.mdindex 63a57c00..75565ad2 100644--- a/docs/skills/llamafile/building.md+++ b/docs/skills/llamafile/building.md@@ -217,9 +217,11 @@ list in sync with the CMakeLists (it must probe all seven feature tests).### CUDA `--minimize-size` drops IQ-quant kernels (CPU fallback)The release CUDA dylib is built with `--minimize-size`, which (among the size-options) passes `-DGGML_CUDA_NO_IQ_QUANTS`. That compiles out every IQ-quant-CUDA path — the MMQ/MMVQ matmul kernels and the fp16/fp32 dequant converters-in `convert.cu`, `mmq.cu`, `mmvq.cu`, and the `f32→iq4_nl` copy in `cpy.cu`.+options) passes `-DGGML_CUDA_NO_IQ_QUANTS`. That compiles out most IQ-quant+CUDA paths — the MMQ/MMVQ matmul kernels in `mmq.cu`/`mmvq.cu`, the bf16/fp16+dequant converters in `convert.cu`, and the `f32→iq4_nl` copy in `cpy.cu`+(`ggml_get_to_fp32_cuda`'s IQ cases stay unguarded, so the `float` dequant+instantiations still compile — a minor size cost).The payoff is a much smaller `.so`; the cost is that it cannot run IQ-quantizedtensors on the GPU.diff --git a/llama.cpp b/llama.cppindex 846e991e..c588c4f4 160000--- a/llama.cpp+++ b/llama.cpp@@ -1 +1 @@-Subproject commit 846e991ec3c7ccec49112ff2c5b00b710e5f551d+Subproject commit c588c4f47683e73ad2d69f50480bec6cc85fd0f7diff --git a/llama.cpp.patches/README.md b/llama.cpp.patches/README.mdindex 4cd5b656..0dfd2150 100644--- a/llama.cpp.patches/README.md+++ b/llama.cpp.patches/README.md@@ -122,11 +122,11 @@ Llamafile uses TinyBLAS as a lightweight replacement for cuBLAS, enabling GPU su### Optional IQ-Quant Exclusion (CUDA)-The IQ ("importance") quantization formats (`IQ1_S`, `IQ2_XXS`/`XS`/`S`, `IQ3_S`/`XXS`, `IQ4_NL`/`XS`) pull in a large amount of CUDA template instantiation that inflates compile time and binary size. These patches gate every IQ code path behind `#ifndef GGML_CUDA_NO_IQ_QUANTS`, so a build can compile them out by defining `GGML_CUDA_NO_IQ_QUANTS`. When the macro is undefined (the default), behavior is unchanged.+The IQ ("importance") quantization formats (`IQ1_S`, `IQ2_XXS`/`XS`/`S`, `IQ3_S`/`XXS`, `IQ4_NL`/`XS`) pull in a large amount of CUDA template instantiation that inflates compile time and binary size. These patches gate the IQ code paths behind `#ifndef GGML_CUDA_NO_IQ_QUANTS` — the MMQ/MMVQ matmul kernels, the `f32 → IQ4_NL` copy, and the IQ dequant cases in `ggml_get_to_bf16_cuda`/`ggml_get_to_fp16_cuda` — so a build can compile them out by defining `GGML_CUDA_NO_IQ_QUANTS`. (`ggml_get_to_fp32_cuda`'s IQ cases are not guarded, so the `float` dequant-template instantiations still compile — a minor size cost, harmless because `ggml_backend_cuda_device_supports_op` gates the same IQ ops, so those tensors fall back to CPU in a minimized build regardless.) When the macro is undefined (the default), behavior is unchanged.| Patch | Description ||-------|-------------|-| `ggml_src_ggml-cuda_convert.cu.patch` | Guards IQ dequantization cases in `ggml_get_to_fp16_cuda` and `ggml_get_to_fp32_cuda` |+| `ggml_src_ggml-cuda_convert.cu.patch` | Guards IQ dequantization cases in `ggml_get_to_bf16_cuda` and `ggml_get_to_fp16_cuda` || `ggml_src_ggml-cuda_cpy.cu.patch` | Guards the `f32 → IQ4_NL` copy helper and its dispatch case || `ggml_src_ggml-cuda_mmq.cu.patch` | Guards IQ cases in `ggml_cuda_mul_mat_q_switch_type` and in the `ggml_cuda_should_use_mmq` support/heuristic switches || `ggml_src_ggml-cuda_mmq.cuh.patch` | Guards the `extern DECL_MMQ_CASE(...)` declarations for IQ types |diff --git a/llama.cpp.patches/llamafile-files/BUILD.mk b/llama.cpp.patches/llamafile-files/BUILD.mkindex f392bceb..532781e1 100644--- a/llama.cpp.patches/llamafile-files/BUILD.mk+++ b/llama.cpp.patches/llamafile-files/BUILD.mk@@ -119,6 +119,7 @@ LLAMA_SRCS_CPP := \llama.cpp/src/models/jina-bert-v2.cpp \llama.cpp/src/models/jina-bert-v3.cpp \llama.cpp/src/models/kimi-linear.cpp \+ llama.cpp/src/models/laguna.cpp \llama.cpp/src/models/lfm2.cpp \llama.cpp/src/models/lfm2moe.cpp \llama.cpp/src/models/llada-moe.cpp \diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patchindex c76fb7fb..6c50add6 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu--- a/llama.cpp/ggml/src/ggml-cuda/convert.cu+++ b/llama.cpp/ggml/src/ggml-cuda/convert.cu-@@ -730,6 +730,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {+@@ -479,6 +479,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {return dequantize_row_q5_K_cuda;case GGML_TYPE_Q6_K:return dequantize_row_q6_K_cuda;@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cucase GGML_TYPE_IQ2_XXS:return dequantize_row_iq2_xxs_cuda;case GGML_TYPE_IQ2_XS:-@@ -748,6 +749,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {+@@ -497,6 +498,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {return dequantize_row_iq4_xs_cuda;case GGML_TYPE_IQ3_S:return dequantize_row_iq3_s_cuda;@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cucase GGML_TYPE_MXFP4:return dequantize_row_mxfp4_cuda;case GGML_TYPE_NVFP4:-@@ -788,6 +790,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {+@@ -537,6 +539,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {return dequantize_row_q5_K_cuda;case GGML_TYPE_Q6_K:return dequantize_row_q6_K_cuda;@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cucase GGML_TYPE_IQ2_XXS:return dequantize_row_iq2_xxs_cuda;case GGML_TYPE_IQ2_XS:-@@ -806,6 +809,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {+@@ -555,6 +558,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {return dequantize_row_iq4_xs_cuda;case GGML_TYPE_IQ3_S:return dequantize_row_iq3_s_cuda;diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patchindex 2cbee3cb..36a9f9be 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch@@ -420,7 +420,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cucase GGML_TYPE_BF16:return true;default:-@@ -4914,9 +4947,14 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g+@@ -4932,9 +4965,14 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gif (src0_type == GGML_TYPE_Q5_1 && src1_type == GGML_TYPE_F32) {return true;}@@ -435,7 +435,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuif (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_I32) {return true;}-@@ -5142,7 +5180,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g+@@ -5160,7 +5198,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g}}@@ -444,7 +444,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;const bool integrated = ggml_cuda_info().devices[dev_ctx->device].integrated;return (ggml_backend_buft_is_cuda(buft) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft));-@@ -5163,13 +5201,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {+@@ -5181,13 +5219,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {}}@@ -460,7 +460,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu#ifdef GGML_CUDA_NO_PEER_COPYreturn nullptr;#else-@@ -5187,14 +5225,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_+@@ -5205,14 +5243,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_#endif}@@ -477,7 +477,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuGGML_UNUSED(dev);CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));}-@@ -5223,23 +5261,23 @@ struct ggml_backend_cuda_reg_context {+@@ -5241,23 +5279,23 @@ struct ggml_backend_cuda_reg_context {std::vector<ggml_backend_dev_t> devices;};@@ -505,7 +505,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.custatic std::vector<ggml_backend_feature> features = []() {std::vector<ggml_backend_feature> features;#define _STRINGIFY(...) #__VA_ARGS__-@@ -5296,7 +5334,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t+@@ -5314,7 +5352,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_tGGML_UNUSED(reg);}diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patchindex 1e15f36e..5743045a 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cudefault:GGML_ABORT("fatal error");break;-@@ -258,6 +260,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t+@@ -271,6 +273,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_tcase GGML_TYPE_Q4_K:case GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cucase GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:case GGML_TYPE_IQ2_S:-@@ -266,6 +269,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t+@@ -279,6 +282,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_tcase GGML_TYPE_IQ1_S:case GGML_TYPE_IQ4_XS:case GGML_TYPE_IQ4_NL:@@ -33,7 +33,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cummq_supported = true;break;default:-@@ -328,9 +332,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t+@@ -341,9 +345,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_treturn ne11 <= 128;case GGML_TYPE_Q6_K:return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patchindex 19f8357e..995c52dc 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh--- a/llama.cpp/ggml/src/ggml-cuda/mmq.cuh+++ b/llama.cpp/ggml/src/ggml-cuda/mmq.cuh-@@ -1478,6 +1478,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);+@@ -1549,6 +1549,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuhextern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);-@@ -1486,6 +1487,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);+@@ -1557,6 +1558,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);