Skip to content

Distribute and run LLMs with a single file.

Update llama.cpp submodule to dbe9c0c (+ embed real web UI) (#983)

Davide Eynard7bb2d8801871
diff --git a/.llamafile_plugin/.claude-plugin/marketplace.json b/.llamafile_plugin/.claude-plugin/marketplace.json
index 59e25978..145d3bc0 100644
--- a/.llamafile_plugin/.claude-plugin/marketplace.json
+++ b/.llamafile_plugin/.claude-plugin/marketplace.json
@@ -9,7 +9,7 @@
{
"name": "llamafile",
"description": "Build guidance and commands for the llamafile project",
- "version": "0.1.1",
+ "version": "0.1.2",
"author": {
"name": "Mozilla AI",
"email": "[email protected]"
diff --git a/.llamafile_plugin/.claude-plugin/plugin.json b/.llamafile_plugin/.claude-plugin/plugin.json
index 60ea39a7..52f49d78 100644
--- a/.llamafile_plugin/.claude-plugin/plugin.json
+++ b/.llamafile_plugin/.claude-plugin/plugin.json
@@ -1,5 +1,5 @@
{
"name": "llamafile",
- "version": "0.1.1",
+ "version": "0.1.2",
"description": "Build guidance and commands for the llamafile project"
}
diff --git a/docs/skills/llamafile/SKILL.md b/docs/skills/llamafile/SKILL.md
index 10955619..7532ed00 100644
--- a/docs/skills/llamafile/SKILL.md
+++ b/docs/skills/llamafile/SKILL.md
@@ -1,7 +1,7 @@
---
name: llamafile
description: This skill should be used when the user asks to "build llamafile", "rebuild llamafile", "run llamafile", "run llamafile tests", "debug llamafile", "set up llamafile", "update patches", "fix patch conflict", "update llama.cpp", "pull latest llama.cpp", "sync upstream llama.cpp", "reset submodules", "write a test for llamafile", "how does llamafile work", "llamafile architecture", or needs guidance on the llamafile build system, patch workflow, submodule integration, cosmocc toolchain, or development practices.
-version: 0.1.2
+version: 0.1.3
---
# Llamafile Development Guide
diff --git a/docs/skills/llamafile/update_llamacpp.md b/docs/skills/llamafile/update_llamacpp.md
index 2e830a01..3feb996f 100644
--- a/docs/skills/llamafile/update_llamacpp.md
+++ b/docs/skills/llamafile/update_llamacpp.md
@@ -4,7 +4,18 @@ llamafile relies on llama.cpp for many of its functionalities. Keeping it up-to-
with the latest version upstream is generally a good practice, as it brings both
bugfixes and support for recent models and features.
-This document describes the steps to keep llamafile updated with upstream.
+This document describes the steps to keep llamafile updated with upstream. At a high-level,
+a llama.cpp consists in the following tasks
+
+1. Updating the submodule
+2. Verifying and updating the current patches (important: never create patches manually,
+ but run `../tools/generate-patches.sh --output-dir ../llama.cpp.patches` from the
+ llama.cpp directory)
+3. Updating build dependencies, to make sure new/changed deps are taken into account
+4. Updating llamafile code: llamafile is built on top of llama.cpp and you want to
+ make sure its code still works with the new, updated submodule. NOTE that this also
+ includes GPU acceleration libs (cuda, rocm, vulkan .sh and .bat scripts in the
+ `llamafile/` directory)
## Step 1: Update the submodule
@@ -78,6 +89,8 @@ target.
- Check if the llamafile code that calls llama.cpp server/main needs updates
- Review `llamafile/` for any API changes in llama.cpp that need to be reflected
- Pay attention to changes in `llama.cpp/include/` for API modifications
+- Also check GPU acceleration libraries code (cuda, rocm, vulkan .sh and .bat
+ scripts in the `llamafile/` directory)
At the end of this step, you should be able to build all targets in this repo,
i.e. the following verification step should return a successful result
diff --git a/llama.cpp b/llama.cpp
index 7b8443ac..dbe9c0c8 160000
--- a/llama.cpp
+++ b/llama.cpp
@@ -1 +1 @@
-Subproject commit 7b8443ac786c06438e0f407b7adaa72c220b5099
+Subproject commit dbe9c0c8ce65354c372f5d4ab507e5424a755e9f
diff --git a/llama.cpp.patches/README.md b/llama.cpp.patches/README.md
index 3be219ce..54341f27 100644
--- a/llama.cpp.patches/README.md
+++ b/llama.cpp.patches/README.md
@@ -52,6 +52,7 @@ The `GGML_CALL` macro (defined as `__attribute__((__ms_abi__))` when `GGML_MULTI
| `ggml_src_ggml-cuda_ggml-cuda.cu.patch` | Adds `GGML_CALL` to all CUDA backend callback implementations (60+ functions); also adds `free_struct` and TinyBLAS BF16 guard (see below) |
| `ggml_src_ggml-metal_ggml-metal.cpp.patch` | Adds `GGML_CALL` to all Metal backend callback implementations (62 functions); also adds `free_struct` (see below) |
| `ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch` | Adds `GGML_CALL` to all Vulkan backend callback implementations; also adds `free_struct` and a heap memory underflow fix (see below) |
+| `ggml_src_ggml-backend-meta.cpp.patch` | Adds `GGML_CALL` to all meta-device, meta-buffer-type, meta-buffer, and meta-backend callback implementations (the meta backend aggregates several simple backends behind one interface, so its callbacks are reached through the same function-pointer structs) |
### Cross-Module Memory Management
@@ -97,6 +98,28 @@ Llamafile uses TinyBLAS as a lightweight replacement for cuBLAS, enabling GPU su
| `ggml_src_ggml-cuda_common.cuh.patch` | Disables BF16 MMA when using TinyBLAS (TinyBLAS would incorrectly interpret BF16 as FP16) |
| `ggml_src_ggml-cuda_ggml-cuda.cu.patch` | Disables BF16 in `ggml_cuda_op_mul_mat_cublas` when using TinyBLAS |
+### Optional IQ-Quant Exclusion (CUDA)
+
+The IQ ("importance") quantization formats (`IQ1_S`, `IQ2_XXS`/`XS`/`S`, `IQ3_S`/`XXS`, `IQ4_NL`/`XS`) pull in a large amount of CUDA template instantiation that inflates compile time and binary size. These patches gate every IQ code path behind `#ifndef GGML_CUDA_NO_IQ_QUANTS`, so a build can compile them out by defining `GGML_CUDA_NO_IQ_QUANTS`. When the macro is undefined (the default), behavior is unchanged.
+
+| Patch | Description |
+|-------|-------------|
+| `ggml_src_ggml-cuda_convert.cu.patch` | Guards IQ dequantization cases in `ggml_get_to_fp16_cuda` and `ggml_get_to_fp32_cuda` |
+| `ggml_src_ggml-cuda_cpy.cu.patch` | Guards the `f32 → IQ4_NL` copy helper and its dispatch case |
+| `ggml_src_ggml-cuda_mmq.cu.patch` | Guards IQ cases in `ggml_cuda_mul_mat_q_switch_type` and in the `ggml_cuda_should_use_mmq` support/heuristic switches |
+| `ggml_src_ggml-cuda_mmq.cuh.patch` | Guards the `extern DECL_MMQ_CASE(...)` declarations for IQ types |
+| `ggml_src_ggml-cuda_mmvq.cu.patch` | Guards IQ cases in `get_vec_dot_q_cuda` and `get_vdr_mmvq` |
+
+### CPU Performance Optimizations (llamafile #975)
+
+These patches restore llamafile's optimized CPU kernels (TinyBLAS matmul, AVX-512 flash-attention helpers) on top of upstream's CPU backend, and tune CPU-only defaults. The hooks call into symbols exported from `llamafile/sgemm.cpp` and are compiled only when `GGML_USE_LLAMAFILE` is defined.
+
+| Patch | Description |
+|-------|-------------|
+| `ggml_src_ggml-cpu_ggml-cpu.c.patch` | Routes MoE matmul (`ggml_compute_forward_mul_mat_id`) through `llamafile_mixmul` / `llamafile_mixmul_iqk`, mirroring the dense-matmul `llamafile_sgemm` hook; reserves work-buffer space for the MoE kernel in `ggml_graph_plan` via `llamafile_mixmul_needs` |
+| `ggml_src_ggml-cpu_ops.cpp.patch` | Routes flash-attention inner loops through llamafile's AVX-512 helpers (`llamafile_fa_vec_dot_f16`, `llamafile_fa_fp16_to_fp32_row`, `llamafile_fa_simd_gemm`) in both the one-chunk and tiled FA paths; also accumulates VKQ in f32 on CPUs lacking native f16 FMA (avoiding costly f16↔f32 round-trips per KV step) |
+| `src_llama-context.cpp.patch` | Defaults `-fa auto` to **off** on CPU-only setups (no GPU devices), since the CPU flash-attention path is slower than the non-FA path on x86; users can still force `-fa on` for memory savings on long contexts |
+
### Llamafile File Handling
These patches integrate llamafile's file handling APIs for loading models from bundled zip archives and `.llamafile` containers.
@@ -111,7 +134,22 @@ These patches integrate llamafile's file handling APIs for loading models from b
| Patch | Description |
|-------|-------------|
-| `tools_server_server.cpp.patch` | Renames `main()` to `server_main()` with `on_ready`/`on_shutdown_available` callbacks for combined TUI+server mode; adds Metal/GPU backend trigger before `common_init()`; adds Cosmopolitan-specific standalone `main()` with `cosmo_args`, verbose flag handling, and GPU pre-initialization; handles `LLAMAFILE_TUI` exit to avoid Metal cleanup crashes |
+| `tools_server_server.cpp.patch` | Renames upstream's `llama_server()` to `server_main()` and adds `on_ready`/`on_shutdown_available` callbacks for combined TUI+server mode; adds Metal/GPU backend trigger before `common_init()`; adds Cosmopolitan-specific standalone `main()` with `cosmo_args`, verbose flag handling, and GPU pre-initialization; handles `LLAMAFILE_TUI` exit to avoid Metal cleanup crashes |
+
+The web UI moved upstream from prebuilt `tools/server/public/*` assets to
+a Svelte project under `tools/ui/`, embedded at CMake time via
+`tools/ui/embed.cpp`. cosmocc has no JS toolchain, so `apply-patches.sh`
+(run by `make setup`) downloads the prebuilt Svelte bundle from the
+`ggml-org/llama-ui` Hugging Face bucket into `llama.cpp/tools/ui/dist/`.
+At build time, `llama.cpp/BUILD.mk` compiles `tools/ui/embed.cpp` with
+`cosmoc++` (its APE output runs on the host) and runs it against the
+downloaded assets to emit `o/$(MODE)/llama.cpp/tools/ui/ui.{cpp,h}`,
+which is then compiled like any other C++ source and linked into
+`llama-server` and `llamafile`. If the download fails (offline, version
+not yet on HF) `apply-patches.sh` warns and continues with `dist/`
+empty — `embed.cpp` then emits a no-op `llama_ui_find_asset`, and
+`server-http.cpp` skips UI route registration via its
+`LLAMA_UI_HAS_ASSETS` guard so the API still works.
### Bug Fixes
@@ -119,6 +157,7 @@ These patches integrate llamafile's file handling APIs for loading models from b
|-------|-------------|
| `ggml_src_ggml-backend-reg.cpp.patch` | Suppresses debug log noise for non-existent backend search paths (irrelevant for llamafile's DSO loading approach) |
| `ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch` | Fixes unsigned integer underflow in `ggml_backend_vk_get_device_memory` where Vulkan's `heapUsage` can exceed `heapBudget` (clamps to zero instead of wrapping) |
+| `src_models_t5.cpp.patch` | Forward-declares the `graph<false>`/`graph<true>` explicit specializations before `build_arch_graph` so clang's `-std=gnu++23` doesn't reject them as specializations after implicit instantiation |
## Creating New Patches
diff --git a/llama.cpp.patches/apply-patches.sh b/llama.cpp.patches/apply-patches.sh
index 2594f94b..e50d6c63 100755
--- a/llama.cpp.patches/apply-patches.sh
+++ b/llama.cpp.patches/apply-patches.sh
@@ -37,6 +37,9 @@ for patch_file in "$PATCHES_DIR"/*.patch; do
fi
done
+# Fetch the prebuilt web UI assets (see fetch-ui-assets.sh for details).
+"$SCRIPT_DIR/fetch-ui-assets.sh"
+
echo ""
echo "Patches applied successfully!"
echo "Note: These changes are not committed to the submodule."
diff --git a/llama.cpp.patches/fetch-ui-assets.sh b/llama.cpp.patches/fetch-ui-assets.sh
new file mode 100755
index 00000000..e0d80561
--- /dev/null
+++ b/llama.cpp.patches/fetch-ui-assets.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+# Fetch the prebuilt llama.cpp web UI assets from Hugging Face.
+#
+# Upstream's tools/ui/scripts/ui-assets.cmake pulls the Svelte build outputs
+# from the ggml-org/llama-ui HF bucket. We do the same here so the cosmocc
+# build never has to run a JS toolchain. If the fetch fails (no network,
+# version not yet published, HF down) we leave tools/ui/dist empty; BUILD.mk's
+# embed step then generates a no-asset ui.cpp and the server still works, just
+# without the web UI.
+#
+# Run by apply-patches.sh (i.e. `make setup`); also safe to run standalone to
+# re-fetch the UI without re-applying patches.
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LLAMA_DIR="$SCRIPT_DIR/../llama.cpp"
+
+HF_BUCKET="${LLAMAFILE_UI_HF_BUCKET:-llama-ui}"
+HF_BASE="https://huggingface.co/buckets/ggml-org/${HF_BUCKET}/resolve"
+HF_TREE_API="https://huggingface.co/api/buckets/ggml-org/${HF_BUCKET}/tree"
+UI_DIST="$LLAMA_DIR/tools/ui/dist"
+UI_ASSETS=(bundle.css bundle.js index.html loading.html)
+
+# Echo the highest bNNNN tag in the bucket whose build number is <= $1, or
+# nothing. The tree API returns directories in ascending order, 100 per page,
+# with a `Link: <...>; rel="next"` header for the following page.
+pick_ui_tag() {
+ local cur="$1"
+ local url="${HF_TREE_API}?limit=100&recursive=false"
+ local hdrs best="" n saw_newer guard=0 body
+ hdrs="$(mktemp)"
+ while [ -n "$url" ] && [ "$guard" -lt 100 ]; do
+ guard=$((guard + 1))
+ body="$(curl -fsSL --max-time 30 -D "$hdrs" "$url" 2>/dev/null)" || break
+ saw_newer=0
+ for n in $(printf '%s' "$body" | grep -oE '"path":"b[0-9]+"' \
+ | grep -oE '[0-9]+'); do
+ if [ "$n" -le "$cur" ]; then
+ if [ -z "$best" ] || [ "$n" -gt "$best" ]; then
+ best="$n"
+ fi
+ else
+ saw_newer=1
+ fi
+ done
+ # Once a page holds a tag newer than us, all later pages are newer too,
+ # so there is nothing better to find.
+ if [ "$saw_newer" -eq 1 ]; then
+ break
+ fi
+ url="$(grep -i '^link:' "$hdrs" \
+ | grep -oE '<[^>]+>; *rel="next"' | grep -oE 'https?://[^>]+' | head -1)"
+ done
+ rm -f "$hdrs"
+ [ -n "$best" ] && printf 'b%s' "$best"
+}
+
+echo ""
+echo "Fetching prebuilt web UI assets from Hugging Face..."
+
+# Pick the version to download. Upstream's CMake just tries the exact build
+# number and then "latest", but for llamafile that's fragile: the exact tag for
+# our pinned commit is often not published yet, and "latest" can be built
+# against a newer backend than the llama.cpp we've pinned. Instead we enumerate
+# the tags actually present in the bucket and pick the newest one that is <= our
+# build number. "latest" is kept only as a last-resort fallback (enumeration
+# failed, e.g. offline / API change, or our commit predates every published
+# tag) so the build can still get *some* UI rather than none.
+UI_CUR_BUILD="$(cd "$LLAMA_DIR" && git describe --tags --always 2>/dev/null \
+ | grep -oE '^b[0-9]+' | grep -oE '[0-9]+' || true)"
+
+UI_CANDIDATES=()
+if [ -n "$UI_CUR_BUILD" ]; then
+ echo " resolving newest UI tag <= b$UI_CUR_BUILD ..."
+ UI_BEST_TAG="$(pick_ui_tag "$UI_CUR_BUILD")"
+ if [ -n "$UI_BEST_TAG" ]; then
+ echo " selected UI tag $UI_BEST_TAG"
+ UI_CANDIDATES+=("$UI_BEST_TAG")
+ else
+ echo " no UI tag <= b$UI_CUR_BUILD found in bucket; will try 'latest'"
+ fi
+fi
+UI_CANDIDATES+=("latest")
+
+mkdir -p "$UI_DIST"
+ui_ok=false
+for v in "${UI_CANDIDATES[@]}"; do
+ echo " trying $HF_BASE/$v ..."
+ fail=false
+ for asset in "${UI_ASSETS[@]}" checksums.txt; do
+ if ! curl -fsSL --max-time 60 -o "$UI_DIST/$asset" \
+ "$HF_BASE/$v/$asset?download=true"; then
+ fail=true
+ break
+ fi
+ done
+ if $fail; then
+ continue
+ fi
+
+ # Best-effort sha256 verification against checksums.txt (one "<hash> <name>"
+ # line per asset). Skip if shasum/sha256sum isn't around.
+ if command -v shasum >/dev/null 2>&1; then
+ sha_cmd="shasum -a 256"
+ elif command -v sha256sum >/dev/null 2>&1; then
+ sha_cmd="sha256sum"
+ else
+ sha_cmd=""
+ fi
+ if [ -n "$sha_cmd" ] && [ -f "$UI_DIST/checksums.txt" ]; then
+ bad=false
+ for asset in "${UI_ASSETS[@]}"; do
+ want=$(awk -v a="$asset" '$2 == a { print $1 }' "$UI_DIST/checksums.txt")
+ got=$($sha_cmd "$UI_DIST/$asset" | awk '{print $1}')
+ if [ -z "$want" ] || [ "$want" != "$got" ]; then
+ echo " checksum mismatch for $asset (want=$want got=$got)"
+ bad=true
+ break
+ fi
+ done
+ if $bad; then
+ continue
+ fi
+ fi
+
+ echo " fetched UI assets from $v"
+ ui_ok=true
+ break
+done
+
+if ! $ui_ok; then
+ echo " warning: could not download UI assets; server will build without the web UI"
+ rm -f "$UI_DIST"/*
+fi
diff --git a/llama.cpp.patches/llamafile-files/BUILD.mk b/llama.cpp.patches/llamafile-files/BUILD.mk
index d4489c24..15500421 100644
--- a/llama.cpp.patches/llamafile-files/BUILD.mk
+++ b/llama.cpp.patches/llamafile-files/BUILD.mk
@@ -67,12 +67,13 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/chatglm.cpp \
llama.cpp/src/models/codeshell.cpp \
llama.cpp/src/models/cogvlm.cpp \
- llama.cpp/src/models/cohere2-iswa.cpp \
+ llama.cpp/src/models/cohere2.cpp \
llama.cpp/src/models/command-r.cpp \
llama.cpp/src/models/dbrx.cpp \
llama.cpp/src/models/deci.cpp \
llama.cpp/src/models/deepseek.cpp \
llama.cpp/src/models/deepseek2.cpp \
+ llama.cpp/src/models/deepseek2ocr.cpp \
llama.cpp/src/models/delta-net-base.cpp \
llama.cpp/src/models/dots1.cpp \
llama.cpp/src/models/dream.cpp \
@@ -86,52 +87,66 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/falcon.cpp \
llama.cpp/src/models/gemma-embedding.cpp \
llama.cpp/src/models/gemma.cpp \
- llama.cpp/src/models/gemma2-iswa.cpp \
+ llama.cpp/src/models/gemma2.cpp \
llama.cpp/src/models/gemma3.cpp \
- llama.cpp/src/models/gemma3n-iswa.cpp \
- llama.cpp/src/models/gemma4-iswa.cpp \
+ llama.cpp/src/models/gemma3n.cpp \
+ llama.cpp/src/models/gemma4.cpp \
+ llama.cpp/src/models/glm-dsa.cpp \
llama.cpp/src/models/glm4-moe.cpp \
llama.cpp/src/models/glm4.cpp \
llama.cpp/src/models/gpt2.cpp \
llama.cpp/src/models/gptneox.cpp \
llama.cpp/src/models/granite-hybrid.cpp \
+ llama.cpp/src/models/granite-moe.cpp \
llama.cpp/src/models/granite.cpp \
llama.cpp/src/models/mamba-base.cpp \
llama.cpp/src/models/grok.cpp \
llama.cpp/src/models/grovemoe.cpp \
llama.cpp/src/models/hunyuan-dense.cpp \
llama.cpp/src/models/hunyuan-moe.cpp \
+ llama.cpp/src/models/hunyuan-vl.cpp \
llama.cpp/src/models/internlm2.cpp \
llama.cpp/src/models/jais.cpp \
llama.cpp/src/models/jais2.cpp \
llama.cpp/src/models/jamba.cpp \
+ llama.cpp/src/models/jina-bert-v2.cpp \
+ llama.cpp/src/models/jina-bert-v3.cpp \
llama.cpp/src/models/kimi-linear.cpp \
llama.cpp/src/models/lfm2.cpp \
+ llama.cpp/src/models/lfm2moe.cpp \
llama.cpp/src/models/llada-moe.cpp \
llama.cpp/src/models/llada.cpp \
+ llama.cpp/src/models/llama-embed.cpp \
llama.cpp/src/models/llama4.cpp \
llama.cpp/src/models/llama.cpp \
llama.cpp/src/models/maincoder.cpp \
llama.cpp/src/models/mamba.cpp \
- llama.cpp/src/models/mimo2-iswa.cpp \
+ llama.cpp/src/models/mamba2.cpp \
+ llama.cpp/src/models/mimo2.cpp \
+ llama.cpp/src/models/minicpm.cpp \
llama.cpp/src/models/minicpm3.cpp \
llama.cpp/src/models/minimax-m2.cpp \
llama.cpp/src/models/mistral3.cpp \
+ llama.cpp/src/models/mistral4.cpp \
llama.cpp/src/models/modern-bert.cpp \
llama.cpp/src/models/mpt.cpp \
+ llama.cpp/src/models/nemotron-h-moe.cpp \
llama.cpp/src/models/nemotron-h.cpp \
llama.cpp/src/models/nemotron.cpp \
llama.cpp/src/models/neo-bert.cpp \
+ llama.cpp/src/models/nomic-bert-moe.cpp \
+ llama.cpp/src/models/nomic-bert.cpp \
llama.cpp/src/models/olmo.cpp \
llama.cpp/src/models/olmo2.cpp \
llama.cpp/src/models/olmoe.cpp \
- llama.cpp/src/models/openai-moe-iswa.cpp \
+ llama.cpp/src/models/openai-moe.cpp \
llama.cpp/src/models/openelm.cpp \
llama.cpp/src/models/orion.cpp \
llama.cpp/src/models/paddleocr.cpp \
- llama.cpp/src/models/pangu-embedded.cpp \
+ llama.cpp/src/models/pangu-embed.cpp \
llama.cpp/src/models/phi2.cpp \
llama.cpp/src/models/phi3.cpp \
+ llama.cpp/src/models/phimoe.cpp \
llama.cpp/src/models/plamo.cpp \
llama.cpp/src/models/plamo2.cpp \
llama.cpp/src/models/plamo3.cpp \
@@ -145,8 +160,8 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/qwen3next.cpp \
llama.cpp/src/models/qwen35.cpp \
llama.cpp/src/models/qwen35moe.cpp \
- llama.cpp/src/models/qwen3vl-moe.cpp \
llama.cpp/src/models/qwen3vl.cpp \
+ llama.cpp/src/models/qwen3vlmoe.cpp \
llama.cpp/src/models/refact.cpp \
llama.cpp/src/models/rnd1.cpp \
llama.cpp/src/models/rwkv6-base.cpp \
@@ -159,10 +174,11 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/smollm3.cpp \
llama.cpp/src/models/stablelm.cpp \
llama.cpp/src/models/starcoder.cpp \
- llama.cpp/src/models/step35-iswa.cpp \
llama.cpp/src/models/starcoder2.cpp \
+ llama.cpp/src/models/step35.cpp \
llama.cpp/src/models/t5.cpp \
llama.cpp/src/models/t5encoder.cpp \
+ llama.cpp/src/models/talkie.cpp \
llama.cpp/src/models/wavtokenizer-dec.cpp \
llama.cpp/src/models/xverse.cpp \
llama.cpp/src/llama-adapter.cpp \
@@ -291,12 +307,14 @@ MTMD_SRCS_CPP := \
llama.cpp/tools/mtmd/models/gemma4a.cpp \
llama.cpp/tools/mtmd/models/gemma4v.cpp \
llama.cpp/tools/mtmd/models/glm4v.cpp \
- llama.cpp/tools/mtmd/models/hunyuanocr.cpp \
+ llama.cpp/tools/mtmd/models/granite-speech.cpp \
+ llama.cpp/tools/mtmd/models/hunyuanvl.cpp \
llama.cpp/tools/mtmd/models/internvl.cpp \
llama.cpp/tools/mtmd/models/kimik25.cpp \
llama.cpp/tools/mtmd/models/kimivl.cpp \
llama.cpp/tools/mtmd/models/llama4.cpp \
llama.cpp/tools/mtmd/models/llava.cpp \
+ llama.cpp/tools/mtmd/models/mimovl.cpp \
llama.cpp/tools/mtmd/models/minicpmv.cpp \
llama.cpp/tools/mtmd/models/mobilenetv5.cpp \
llama.cpp/tools/mtmd/models/nemotron-v2-vl.cpp \
@@ -321,24 +339,59 @@ HTTPLIB_SRCS := llama.cpp/vendor/cpp-httplib/httplib.cpp
HTTPLIB_OBJS := $(HTTPLIB_SRCS:%.cpp=o/$(MODE)/%.cpp.o)
# ==============================================================================
-# Server Assets (convert HTML to C++ headers)
+# Web UI assets
# ==============================================================================
-
-# Generate .hpp files from binary assets using xxd-like conversion
-o/$(MODE)/llama.cpp/tools/server/%.hpp: llama.cpp/tools/server/public/%
- @mkdir -p $(dir $@)
- $(eval VARNAME := $(shell echo "$(notdir $*)" | sed 's/[.-]/_/g'))
- @echo 'unsigned char $(VARNAME)[] = {' > $@
- @od -An -tx1 -v $< | awk '{for(i=1;i<=NF;i++){if(NR>1||i>1)printf", "; printf"0x%s",$$i}}' >> $@
- @echo >> $@
- @echo '};' >> $@
- @echo 'unsigned int $(VARNAME)_len = sizeof($(VARNAME));' >> $@
-
-SERVER_ASSETS := \
- o/$(MODE)/llama.cpp/tools/server/index.html.hpp \
- o/$(MODE)/llama.cpp/tools/server/bundle.js.hpp \
- o/$(MODE)/llama.cpp/tools/server/bundle.css.hpp \
- o/$(MODE)/llama.cpp/tools/server/loading.html.hpp
+#
+# Upstream switched from prebuilt bundles in tools/server/public/ to a
+# Svelte project under tools/ui/, embedded at CMake time by
+# tools/ui/embed.cpp into a generated ui.cpp + ui.h. cosmocc has no JS
+# toolchain, so apply-patches.sh (run by `make setup`) downloads the
+# prebuilt Svelte bundle from the ggml-org/llama-ui Hugging Face bucket
+# into llama.cpp/tools/ui/dist/. Here we compile embed.cpp with cosmocc
+# (its APE output runs on the host) and run it against whatever's in
+# dist/ to emit ui.cpp/ui.h. With assets present, ui.h defines
+# LLAMA_UI_HAS_ASSETS and server-http.cpp wires up /, /bundle.js,
+# /bundle.css; without them, embed.cpp emits a no-op llama_ui_find_asset
+# and the routes stay unregistered.
+
+UI_DIST := llama.cpp/tools/ui/dist
+UI_GEN_DIR := o/$(MODE)/llama.cpp/tools/ui
+UI_EMBED_SRC := llama.cpp/tools/ui/embed.cpp
+UI_EMBED_TOOL := $(UI_GEN_DIR)/llama-ui-embed
+UI_CPP_GEN := $(UI_GEN_DIR)/ui.cpp
+UI_H_GEN := $(UI_GEN_DIR)/ui.h
+
+# Assets we ask embed to bundle if they exist. wildcard returns "" for
+# missing files, which lets the build proceed UI-less when offline.
+UI_ASSETS_BUNDLE_CSS := $(wildcard $(UI_DIST)/bundle.css)
+UI_ASSETS_BUNDLE_JS := $(wildcard $(UI_DIST)/bundle.js)
+UI_ASSETS_INDEX_HTML := $(wildcard $(UI_DIST)/index.html)
+UI_ASSETS_LOADING_HTML := $(wildcard $(UI_DIST)/loading.html)
+UI_ASSETS_FILES := \
+ $(UI_ASSETS_BUNDLE_CSS) \
+ $(UI_ASSETS_BUNDLE_JS) \
+ $(UI_ASSETS_INDEX_HTML) \
+ $(UI_ASSETS_LOADING_HTML)
+
+# Build embed.cpp standalone (no llamafile flags, no llama.cpp includes).
+# cosmoc++ produces an APE that runs on the build host, so we don't need
+# a separate system compiler. Compiled with stock C++17 so the source
+# isn't entangled with -DCOSMOCC or the GGML defines.
+$(UI_EMBED_TOOL): $(UI_EMBED_SRC) $(COSMOCC)
+ @mkdir -p $(@D)
+ $(CXX) -O2 -std=gnu++17 -o $@ $<
+
+# Generate ui.cpp/ui.h. Re-run whenever the embed tool or any dist asset
+# changes. The recipe passes <name> <path> pairs only for assets that
+# actually exist on disk; missing files are silently skipped, matching
+# the behaviour of embed.cpp's "no assets -> stub" mode.
+$(UI_CPP_GEN) $(UI_H_GEN) &: $(UI_EMBED_TOOL) $(UI_ASSETS_FILES)
+ @mkdir -p $(UI_GEN_DIR)
+ $(UI_EMBED_TOOL) $(UI_CPP_GEN) $(UI_H_GEN) \
+ $(if $(UI_ASSETS_BUNDLE_CSS),bundle.css $(UI_ASSETS_BUNDLE_CSS)) \
+ $(if $(UI_ASSETS_BUNDLE_JS),bundle.js $(UI_ASSETS_BUNDLE_JS)) \
+ $(if $(UI_ASSETS_INDEX_HTML),index.html $(UI_ASSETS_INDEX_HTML)) \
+ $(if $(UI_ASSETS_LOADING_HTML),loading.html $(UI_ASSETS_LOADING_HTML))
# ==============================================================================
# Tools (in tools/ directory)
@@ -367,6 +420,9 @@ TOOL_IMATRIX_OBJS := $(TOOL_IMATRIX_SRCS:%.cpp=o/$(MODE)/%.cpp.o)
TOOL_PERPLEXITY_OBJS := $(TOOL_PERPLEXITY_SRCS:%.cpp=o/$(MODE)/%.cpp.o)
TOOL_BENCH_OBJS := $(TOOL_BENCH_SRCS:%.cpp=o/$(MODE)/%.cpp.o)
TOOL_SERVER_OBJS := $(TOOL_SERVER_SRCS:%.cpp=o/$(MODE)/%.cpp.o)
+# ui.cpp is generated by embed; placed under o/$(MODE)/ and built via the
+# generic %.cpp -> %.cpp.o rule.
+UI_GEN_OBJ := $(UI_CPP_GEN:%.cpp=%.cpp.o)
# llamafile objects are used to add dynamic GPU support (Metal, CUDA, ROCm, Vulkan)
TOOL_LLAMAFILE_OBJS := \
o/$(MODE)/llamafile/llamafile.o \
@@ -375,8 +431,9 @@ TOOL_LLAMAFILE_OBJS := \
o/$(MODE)/llamafile/vulkan.o \
o/$(MODE)/llamafile/zip.o
-# Server objects depend on generated assets
-$(TOOL_SERVER_OBJS): $(SERVER_ASSETS) llamafile/llamafile.h
+# Server objects depend on the llamafile bridge header and on the
+# generated ui.h (server-http.cpp #includes it directly).
+$(TOOL_SERVER_OBJS): llamafile/llamafile.h $(UI_H_GEN)
# ==============================================================================
# Compiler flags
@@ -393,12 +450,18 @@ $(TOOL_PERPLEXITY_OBJS) $(TOOL_BENCH_OBJS) $(TOOL_SERVER_OBJS) $(MTMD_OBJS): \
-iquote llama.cpp/ggml/src/ggml-cpu \
-iquote llama.cpp/src \
-iquote llama.cpp/tools/mtmd \
- -iquote o/$(MODE)/llama.cpp/tools/server \
+ -iquote $(UI_GEN_DIR) \
-isystem llama.cpp/vendor
-# Server needs llamafile headers for Metal support and web UI
+# Server needs llamafile headers for Metal support.
+# The generated ui.h sits in $(UI_GEN_DIR); the -iquote above lets every
+# server source resolve `#include "ui.h"`.
$(TOOL_SERVER_OBJS): private CPPFLAGS += -iquote llamafile
-$(TOOL_SERVER_OBJS): private CCFLAGS += -DLLAMA_BUILD_WEBUI
+
+# Compile the generated ui.cpp with the same llama.cpp CPPFLAGS so
+# stddef.h and friends resolve via cosmocc's libc.
+$(UI_GEN_OBJ): private CPPFLAGS += -iquote $(UI_GEN_DIR)
+$(UI_GEN_OBJ): $(UI_H_GEN)
# Version definitions
$(GGML_OBJS): private CCFLAGS += \
@@ -501,14 +564,14 @@ o/$(MODE)/llama.cpp/llama-bench/llama-bench: \
o/$(MODE)/llama.cpp/server/llama-server: \
$(TOOL_SERVER_OBJS) \
+ $(UI_GEN_OBJ) \
$(MTMD_OBJS) \
$(HTTPLIB_OBJS) \
$(TOOL_LLAMAFILE_OBJS) \
$$(TINYBLAS_CPU_OBJS) \
- o/$(MODE)/llama.cpp/llama.cpp.a \
- $(SERVER_ASSETS)
+ o/$(MODE)/llama.cpp/llama.cpp.a
@mkdir -p $(dir $@)
- $(LINK.o) $(TOOL_SERVER_OBJS) $(MTMD_OBJS) $(HTTPLIB_OBJS) $(TOOL_LLAMAFILE_OBJS) $(TINYBLAS_CPU_OBJS) o/$(MODE)/llama.cpp/llama.cpp.a $(LOADLIBES) $(LDLIBS) -o $@
+ $(LINK.o) $(TOOL_SERVER_OBJS) $(UI_GEN_OBJ) $(MTMD_OBJS) $(HTTPLIB_OBJS) $(TOOL_LLAMAFILE_OBJS) $(TINYBLAS_CPU_OBJS) o/$(MODE)/llama.cpp/llama.cpp.a $(LOADLIBES) $(LDLIBS) -o $@
# ==============================================================================
# Dependencies
diff --git a/llama.cpp.patches/patches/common_arg.cpp.patch b/llama.cpp.patches/patches/common_arg.cpp.patch
index 153f0ac0..4626cffa 100644
--- a/llama.cpp.patches/patches/common_arg.cpp.patch
+++ b/llama.cpp.patches/patches/common_arg.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/common/arg.cpp b/common/arg.cpp
--- a/llama.cpp/common/arg.cpp
+++ b/llama.cpp/common/arg.cpp
-@@ -38,6 +38,8 @@
+@@ -37,6 +37,8 @@
#ifndef __EMSCRIPTEN__
#ifdef __linux__
#include <linux/limits.h>
diff --git a/llama.cpp.patches/patches/common_common.cpp.patch b/llama.cpp.patches/patches/common_common.cpp.patch
index 7e5a3001..33e2cae0 100644
--- a/llama.cpp.patches/patches/common_common.cpp.patch
+++ b/llama.cpp.patches/patches/common_common.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/common/common.cpp b/common/common.cpp
--- a/llama.cpp/common/common.cpp
+++ b/llama.cpp/common/common.cpp
-@@ -33,6 +33,10 @@
+@@ -34,6 +34,10 @@
#include <sys/sysctl.h>
#endif
@@ -12,7 +12,7 @@ diff --git a/common/common.cpp b/common/common.cpp
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#ifndef NOMINMAX
-@@ -71,7 +75,38 @@ common_time_meas::~common_time_meas() {
+@@ -72,7 +76,38 @@ common_time_meas::~common_time_meas() {
//
int32_t common_cpu_get_num_physical_cores() {
@@ -52,7 +52,7 @@ diff --git a/common/common.cpp b/common/common.cpp
// enumerate the set of thread siblings, num entries is num cores
std::unordered_set<std::string> siblings;
for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {
-@@ -972,6 +1007,16 @@ std::string fs_get_cache_directory() {
+@@ -1012,6 +1047,16 @@ std::string fs_get_cache_directory() {
cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");
#elif defined(_WIN32)
cache_directory = std::getenv("LOCALAPPDATA");
@@ -69,10 +69,10 @@ diff --git a/common/common.cpp b/common/common.cpp
#elif defined(__EMSCRIPTEN__)
GGML_ABORT("not implemented on this platform");
#else
-@@ -1148,10 +1193,31 @@ common_init_result::common_init_result(common_params & params) :
-
+@@ -1189,10 +1234,31 @@ common_init_result::common_init_result(common_params & params, bool model_only)
if (params.fit_params) {
- LOG_INF("%s: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on\n", __func__);
+ LOG_INF("%s: fitting params to device memory ...\n", __func__);
+ LOG_INF("%s: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n", __func__);
+
+ // if a multimodal projection model is specified and will be loaded on GPU,
+ // add its estimated size to the fit margins so the fit leaves enough room for it
@@ -100,5 +100,5 @@ diff --git a/common/common.cpp b/common/common.cpp
- params.fit_params_target.data(),
+ fit_params_target.data(),
params.fit_params_min_ctx,
- params.verbosity >= 4 ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
+ params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
}
diff --git a/llama.cpp.patches/patches/common_log.cpp.patch b/llama.cpp.patches/patches/common_log.cpp.patch
index b99c8094..99499705 100644
--- a/llama.cpp.patches/patches/common_log.cpp.patch
+++ b/llama.cpp.patches/patches/common_log.cpp.patch
@@ -9,19 +9,25 @@ diff --git a/common/log.cpp b/common/log.cpp
#endif // defined(_WIN32)
int common_log_verbosity_thold = LOG_DEFAULT_LLAMA;
-@@ -261,10 +262,27 @@ public:
+@@ -260,10 +261,33 @@ public:
running = true;
thrd = std::thread([this]() {
+#if !defined(_WIN32)
-+ // Block SIGINT and SIGTERM on this thread to prevent EINTR during
-+ // condition_variable::wait(). In Cosmopolitan libc, condition_variable
-+ // throws std::system_error(EINTR) when interrupted by signals, and this
-+ // exception cannot be caught (goes directly to std::terminate).
++ // Block every signal on this thread so pthread_cond_timedwait()
++ // can't be interrupted with EINTR (libcxx rethrows EINTR as
++ // std::system_error which would std::terminate the process).
++ // Keep synchronous fault signals deliverable so real crashes
++ // still surface on whichever thread caused them.
+ sigset_t block_mask;
-+ sigemptyset(&block_mask);
-+ sigaddset(&block_mask, SIGINT);
-+ sigaddset(&block_mask, SIGTERM);
++ sigfillset(&block_mask);
++ sigdelset(&block_mask, SIGSEGV);
++ sigdelset(&block_mask, SIGBUS);
++ sigdelset(&block_mask, SIGILL);
++ sigdelset(&block_mask, SIGFPE);
++ sigdelset(&block_mask, SIGABRT);
++ sigdelset(&block_mask, SIGSYS);
++ sigdelset(&block_mask, SIGTRAP);
+ pthread_sigmask(SIG_BLOCK, &block_mask, nullptr);
+#endif
while (true) {
@@ -35,6 +41,6 @@ diff --git a/common/log.cpp b/common/log.cpp
+ while (head == tail) {
+ cv.wait_for(lock, std::chrono::seconds(30));
+ }
-
cur = entries[head];
+ head = (head + 1) % entries.size();
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch
index 2821ecb5..e780df58 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
--- a/llama.cpp/ggml/src/ggml-backend-meta.cpp
+++ b/llama.cpp/ggml/src/ggml-backend-meta.cpp
-@@ -83,19 +83,19 @@ struct ggml_backend_meta_device_context {
+@@ -84,19 +84,19 @@ struct ggml_backend_meta_device_context {
static bool ggml_backend_dev_is_meta(ggml_backend_dev_t dev);
@@ -24,7 +24,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_ASSERT(ggml_backend_dev_is_meta(dev));
const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
*free = 0;
-@@ -108,13 +108,13 @@ static void ggml_backend_meta_device_get_memory(ggml_backend_dev_t dev, size_t *
+@@ -109,13 +109,13 @@ static void ggml_backend_meta_device_get_memory(ggml_backend_dev_t dev, size_t *
}
}
@@ -40,7 +40,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_ASSERT(ggml_backend_dev_is_meta(dev));
const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
-@@ -142,20 +142,20 @@ static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_back
+@@ -143,20 +143,20 @@ static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_back
}
}
@@ -66,7 +66,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_ASSERT(ggml_backend_dev_is_meta(dev));
ggml_backend_dev_t dev_buft = ggml_backend_buft_get_device(buft);
if (!ggml_backend_dev_is_meta(dev_buft)) {
-@@ -272,7 +272,7 @@ static size_t ggml_backend_meta_buft_n_bufts(ggml_backend_buffer_type_t meta_buf
+@@ -273,7 +273,7 @@ static size_t ggml_backend_meta_buft_n_bufts(ggml_backend_buffer_type_t meta_buf
return meta_buft_ctx->simple_bufts.size();
}
@@ -75,7 +75,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_ASSERT(ggml_backend_buft_is_meta(buft));
const ggml_backend_meta_buffer_type_context * meta_buft_ctx = (const ggml_backend_meta_buffer_type_context *) buft->context;
return meta_buft_ctx->name.c_str();
-@@ -285,9 +285,9 @@ static ggml_backend_buffer_type_t ggml_backend_meta_buft_simple_buft(ggml_backen
+@@ -286,9 +286,9 @@ static ggml_backend_buffer_type_t ggml_backend_meta_buft_simple_buft(ggml_backen
return meta_buft_ctx->simple_bufts[index];
}
@@ -87,7 +87,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
size_t max_alignment = 1;
for (size_t i = 0; i < n_simple_bufts; i++) {
-@@ -298,7 +298,7 @@ static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_ty
+@@ -299,7 +299,7 @@ static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_ty
return max_alignment;
}
@@ -96,7 +96,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
size_t max_size = SIZE_MAX;
for (size_t i = 0; i < n_simple_bufts; i++) {
-@@ -307,7 +307,7 @@ static size_t ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_typ
+@@ -308,7 +308,7 @@ static size_t ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_typ
return max_size;
}
@@ -105,7 +105,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
size_t max_alloc_size = 0;
for (size_t i = 0; i < n_simple_bufts; i++) {
-@@ -317,7 +317,7 @@ static size_t ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_t
+@@ -318,7 +318,7 @@ static size_t ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_t
return max_alloc_size;
}
@@ -114,7 +114,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
for (size_t i = 0; i < n_simple_bufts; i++) {
if (!ggml_backend_buft_is_host(ggml_backend_meta_buft_simple_buft(buft, i))) {
-@@ -340,7 +340,7 @@ bool ggml_backend_buft_is_meta(ggml_backend_buffer_type_t buft) {
+@@ -341,7 +341,7 @@ bool ggml_backend_buft_is_meta(ggml_backend_buffer_type_t buft) {
return buft != nullptr && buft->iface.get_name == ggml_backend_meta_buffer_type_iface.get_name;
}
@@ -123,7 +123,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
static std::map<ggml_backend_dev_t, struct ggml_backend_buffer_type> meta_bufts;
GGML_ASSERT(ggml_backend_dev_is_meta(dev));
{
-@@ -367,7 +367,7 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_
+@@ -368,7 +368,7 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_
return &result.first->second;
}
@@ -132,7 +132,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_ASSERT(ggml_backend_dev_is_meta(dev));
const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
-@@ -414,7 +414,7 @@ struct ggml_backend_meta_buffer_context {
+@@ -451,7 +451,7 @@ struct ggml_backend_meta_buffer_context {
}
};
@@ -140,9 +140,9 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
+static void GGML_CALL ggml_backend_meta_buffer_free_buffer(ggml_backend_buffer_t buffer) {
GGML_ASSERT(ggml_backend_buffer_is_meta(buffer));
ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;
- for (auto & [ctx, buf] : buf_ctx->buf_configs) {
-@@ -1077,12 +1077,12 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co
- return ret;
+ delete buf_ctx;
+@@ -1122,7 +1122,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co
+ return ggml_backend_meta_get_split_state(buf_ctx->get_simple_tensor_container(tensor), tensor, assume_sync);
}
-static void * ggml_backend_meta_buffer_get_base(ggml_backend_buffer_t buffer) {
@@ -150,14 +150,16 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_UNUSED(buffer);
return (void *) 0x1000000000000000; // FIXME
}
+@@ -1242,14 +1242,14 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m
+ return GGML_STATUS_SUCCESS;
+ }
-static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
+static enum ggml_status GGML_CALL ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
GGML_ASSERT(ggml_backend_buffer_is_meta(buffer));
ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;
- const size_t n_simple_bufs = ggml_backend_meta_buffer_n_bufs(buffer);
-@@ -1197,7 +1197,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer
- return GGML_STATUS_SUCCESS;
+ buf_ctx->stc_compute_index = buf_ctx->stc_compute_index_next;
+ return ggml_backend_meta_buffer_init_tensor_impl(buf_ctx->get_simple_tensor_container(tensor), tensor);
}
-static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
@@ -165,7 +167,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);
GGML_ASSERT(ggml_is_contiguous(tensor));
-@@ -1304,7 +1304,7 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
+@@ -1359,7 +1359,7 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
}
}
@@ -174,7 +176,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);
GGML_ASSERT(ggml_is_contiguous(tensor));
-@@ -1397,14 +1397,14 @@ static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, co
+@@ -1455,14 +1455,14 @@ static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, co
}
}
@@ -188,10 +190,10 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
-static void ggml_backend_meta_buffer_reset(ggml_backend_buffer_t buffer) {
+static void GGML_CALL ggml_backend_meta_buffer_reset(ggml_backend_buffer_t buffer) {
- const size_t n_buffers = ggml_backend_meta_buffer_n_bufs(buffer);
- for (size_t i = 0; i < n_buffers; i++) {
- ggml_backend_buffer_reset(ggml_backend_meta_buffer_simple_buffer(buffer, i));
-@@ -1429,7 +1429,7 @@ bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) {
+ GGML_ASSERT(ggml_backend_buffer_is_meta(buffer));
+ ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;
+ for (size_t i = 0; i < buf_ctx->bufs.size(); i++) {
+@@ -1488,7 +1488,7 @@ bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) {
return buf != nullptr && buf->iface.free_buffer == ggml_backend_meta_buffer_iface.free_buffer;
}
@@ -199,8 +201,8 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
+static ggml_backend_buffer_t GGML_CALL ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
- ggml_init_params params = {
-@@ -1567,20 +1567,20 @@ struct ggml_backend_meta_context {
+ const ggml_init_params params = {
+@@ -1655,20 +1655,20 @@ struct ggml_backend_meta_context {
}
};
@@ -224,7 +226,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_backends = ggml_backend_meta_n_backends(backend);
GGML_ASSERT(offset == 0);
GGML_ASSERT(ggml_is_contiguous(tensor));
-@@ -1621,7 +1621,7 @@ static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tens
+@@ -1712,7 +1712,7 @@ static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tens
}
}
@@ -233,7 +235,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_backends = ggml_backend_meta_n_backends(backend);
GGML_ASSERT(offset == 0);
GGML_ASSERT(ggml_is_contiguous(tensor));
-@@ -1662,14 +1662,14 @@ static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggm
+@@ -1756,14 +1756,14 @@ static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggm
}
}
@@ -250,7 +252,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_ASSERT(cgraph->grads == nullptr);
const size_t n_backends = ggml_backend_meta_n_backends(backend);
ggml_backend_meta_context * backend_ctx = (ggml_backend_meta_context *) backend->context;
-@@ -2101,7 +2101,7 @@ bool ggml_backend_is_meta(ggml_backend_t backend) {
+@@ -2232,7 +2232,7 @@ bool ggml_backend_is_meta(ggml_backend_t backend) {
return backend != nullptr && backend->iface.get_name == ggml_backend_meta_i.get_name;
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patch
index 329927a8..878666c4 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c
-@@ -1520,6 +1520,26 @@ static void ggml_compute_forward_mul_mat_id(
+@@ -1530,6 +1530,26 @@ static void ggml_compute_forward_mul_mat_id(
const struct ggml_tensor * src1 = dst->src[1];
const struct ggml_tensor * ids = dst->src[2];
@@ -28,7 +28,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
GGML_TENSOR_BINARY_OP_LOCALS
const int ith = params->ith;
-@@ -1639,6 +1659,24 @@ static void ggml_compute_forward_mul_mat_id(
+@@ -1649,6 +1669,24 @@ static void ggml_compute_forward_mul_mat_id(
const int64_t nr0 = ne01;
const int64_t nr1 = cne1;
@@ -53,7 +53,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
int chunk_size = 16;
if (nr0 == 1 || nr1 == 1) {
chunk_size = 64;
-@@ -2827,6 +2865,19 @@ struct ggml_cplan ggml_graph_plan(
+@@ -2837,6 +2875,19 @@ struct ggml_cplan ggml_graph_plan(
cur += n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping) + sizeof(int64_t);
// atomic_current_chunk
cur += CACHE_LINE_SIZE*n_as + CACHE_LINE_SIZE;
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patch
index 443fc909..95080b18 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patch
@@ -20,7 +20,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
// ggml_compute_forward_dup
static void ggml_compute_forward_dup_same_cont(
-@@ -8247,6 +8259,15 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
+@@ -8293,6 +8305,15 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
GGML_ASSERT(( q_to_vec_dot) && "fattn: unsupported K-type");
GGML_ASSERT((v->type == GGML_TYPE_F32 || v_to_float ) && "fattn: unsupported V-type");
@@ -36,7 +36,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
int ith = params->ith;
for (int ir = ir0; ir < ir1; ++ir) {
-@@ -8266,10 +8287,22 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
+@@ -8312,10 +8333,22 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*DV); // (temporary) FP16 VKQ accumulator
ggml_fp16_t * Q_q = (ggml_fp16_t *) (VKQ32 + 2*DV); // (temporary) buffer for Q converted to quantized/FP16
@@ -62,7 +62,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
}
const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]) : NULL;
-@@ -8298,7 +8331,13 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
+@@ -8344,7 +8377,13 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
float s; // KQ value
const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);
@@ -76,7 +76,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
s = s*scale; // scale KQ value
-@@ -8315,21 +8354,36 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
+@@ -8361,21 +8400,36 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));
@@ -116,7 +116,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
} else {
if (s > M) {
// s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f
-@@ -8337,26 +8391,20 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
+@@ -8383,26 +8437,20 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
ms = expf(Mold - M);
// V = V*expf(Mold - M)
@@ -146,7 +146,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
for (int64_t d = 0; d < DV; ++d) {
VKQ32[d] = GGML_CPU_FP16_TO_FP32(VKQ16[d]);
}
-@@ -8577,8 +8625,24 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
+@@ -8623,8 +8671,24 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;
if (kv_type == GGML_TYPE_F16) {
const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data;
@@ -173,7 +173,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
}
} else {
const float * k_f32_src = (const float *)k_data;
-@@ -8588,7 +8652,18 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
+@@ -8634,7 +8698,18 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
}
}
memset(KQ, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));
@@ -192,7 +192,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, scale);
// Set padded KQ entries to -inf so softmax gives them zero weight
-@@ -8641,7 +8716,15 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
+@@ -8687,7 +8762,15 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
for (int tk = 0; tk < kv_tile; tk++) {
const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
if (kv_type == GGML_TYPE_F16) {
@@ -208,7 +208,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
} else {
memcpy(V32 + tk * DV, v_data, DV * sizeof(float));
}
-@@ -8651,7 +8734,16 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
+@@ -8697,7 +8780,16 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
memset(KQ + tq * KV_TILE_SZ, 0, KV_TILE_SZ * sizeof(float));
}
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch
index 84db581e..a3d3bec0 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh
--- a/llama.cpp/ggml/src/ggml-cuda/common.cuh
+++ b/llama.cpp/ggml/src/ggml-cuda/common.cuh
-@@ -294,9 +294,15 @@ static bool fp16_mma_hardware_available(const int cc) {
+@@ -316,9 +316,15 @@ static bool fp16_mma_hardware_available(const int cc) {
}
static bool bf16_mma_hardware_available(const int cc) {
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch
index 36166384..13e4a5b9 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu
--- a/llama.cpp/ggml/src/ggml-cuda/cpy.cu
+++ b/llama.cpp/ggml/src/ggml-cuda/cpy.cu
-@@ -360,6 +360,7 @@ static void ggml_cpy_q5_1_f32_cuda(
+@@ -368,6 +368,7 @@ static void ggml_cpy_q5_1_f32_cuda(
ne10, ne11, ne12, nb10, nb11, nb12, nb13);
}
@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu
static void ggml_cpy_f32_iq4_nl_cuda(
const char * cx, char * cdst, const int64_t ne,
const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,
-@@ -371,6 +372,7 @@ static void ggml_cpy_f32_iq4_nl_cuda(
+@@ -379,6 +380,7 @@ static void ggml_cpy_f32_iq4_nl_cuda(
cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, 1, 0, stream>>>
(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
}
@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu
void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1) {
const int64_t ne = ggml_nelements(src0);
-@@ -465,9 +467,11 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
+@@ -473,9 +475,11 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
} else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_F32) {
ggml_cpy_q5_0_f32_cuda
(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
index bf0329de..989a89e5 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
-@@ -631,7 +631,7 @@ struct ggml_backend_cuda_buffer_context {
+@@ -637,7 +637,7 @@ struct ggml_backend_cuda_buffer_context {
}
};
@@ -10,7 +10,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
delete ctx;
}
-@@ -640,12 +640,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {
+@@ -646,12 +646,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {
return buffer->iface.free_buffer == ggml_backend_cuda_buffer_free_buffer;
}
@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
if (tensor->view_src != NULL) {
-@@ -666,7 +666,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer
+@@ -672,7 +672,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer
return GGML_STATUS_SUCCESS;
}
@@ -34,7 +34,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -674,7 +674,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,
+@@ -680,7 +680,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -43,7 +43,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -682,7 +682,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
+@@ -688,7 +688,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -52,7 +52,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -690,7 +690,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co
+@@ -696,7 +696,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -61,7 +61,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
-@@ -700,7 +700,7 @@ static void ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer,
+@@ -706,7 +706,7 @@ static void ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer,
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -70,7 +70,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
-@@ -710,7 +710,7 @@ static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,
+@@ -716,7 +716,7 @@ static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -79,7 +79,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (ggml_backend_buffer_is_cuda(src->buffer)) {
ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;
ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context;
-@@ -731,7 +731,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co
+@@ -737,7 +737,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co
GGML_UNUSED(buffer);
}
@@ -88,7 +88,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -739,6 +739,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t
+@@ -745,6 +745,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -103,7 +103,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
/* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,
/* .get_base = */ ggml_backend_cuda_buffer_get_base,
-@@ -751,6 +759,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
+@@ -757,6 +765,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
/* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,
/* .clear = */ ggml_backend_cuda_buffer_clear,
/* .reset = */ NULL,
@@ -111,7 +111,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
};
// cuda buffer type
-@@ -759,7 +768,7 @@ struct ggml_backend_cuda_buffer_type_context {
+@@ -765,7 +774,7 @@ struct ggml_backend_cuda_buffer_type_context {
std::string name;
};
@@ -120,7 +120,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
return ctx->name.c_str();
-@@ -769,7 +778,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {
+@@ -775,7 +784,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {
return buft->iface.get_name == ggml_backend_cuda_buffer_type_get_name;
}
@@ -129,7 +129,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
ggml_cuda_set_device(buft_ctx->device);
-@@ -788,13 +797,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac
+@@ -794,13 +803,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac
return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);
}
@@ -145,7 +145,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
size_t size = ggml_nbytes(tensor);
int64_t ne0 = tensor->ne[0];
-@@ -908,19 +917,19 @@ struct ggml_backend_cuda_split_buffer_context {
+@@ -914,19 +923,19 @@ struct ggml_backend_cuda_split_buffer_context {
};
@@ -168,7 +168,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported
GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors");
-@@ -970,7 +979,7 @@ static enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_
+@@ -976,7 +985,7 @@ static enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_
return GGML_STATUS_SUCCESS;
}
@@ -177,7 +177,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
// split tensors must always be set in their entirety at once
GGML_ASSERT(offset == 0);
GGML_ASSERT(size == ggml_nbytes(tensor));
-@@ -1009,7 +1018,7 @@ static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buff
+@@ -1015,7 +1024,7 @@ static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buff
}
}
@@ -186,7 +186,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
// split tensors must always be set in their entirety at once
GGML_ASSERT(offset == 0);
GGML_ASSERT(size == ggml_nbytes(tensor));
-@@ -1048,7 +1057,7 @@ static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buff
+@@ -1054,7 +1063,7 @@ static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buff
}
}
@@ -195,7 +195,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
GGML_UNUSED(buffer);
GGML_UNUSED(value);
}
-@@ -1065,11 +1074,12 @@ static const ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {
+@@ -1071,11 +1080,12 @@ static const ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {
/* .cpy_tensor = */ NULL,
/* .clear = */ ggml_backend_cuda_split_buffer_clear,
/* .reset = */ NULL,
@@ -209,7 +209,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;
return ctx->name.c_str();
-@@ -1079,7 +1089,7 @@ static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {
+@@ -1085,7 +1095,7 @@ static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {
return buft->iface.get_name == ggml_backend_cuda_split_buffer_type_get_name;
}
@@ -218,7 +218,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
// since we don't know the exact split after rounding, we cannot allocate the device buffers at this point
// instead, we allocate them for each tensor separately in init_tensor
// however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated,
-@@ -1089,13 +1099,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(gg
+@@ -1095,13 +1105,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(gg
return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size);
}
@@ -234,7 +234,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;
GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors");
-@@ -1123,7 +1133,7 @@ static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_bu
+@@ -1129,7 +1139,7 @@ static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_bu
return total_size;
}
@@ -243,8 +243,8 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
return false;
GGML_UNUSED(buft);
-@@ -1280,7 +1290,7 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg
- #endif // GGML_USE_NCCL
+@@ -1433,7 +1443,7 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg
+ return comm_ctx->try_allreduce(comm_ctx, tensors);
}
-ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split) {
@@ -252,7 +252,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
static std::mutex mutex;
std::lock_guard<std::mutex> lock(mutex);
-@@ -1324,7 +1334,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device,
+@@ -1477,7 +1487,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device,
// host buffer type
@@ -261,7 +261,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
return GGML_CUDA_NAME "_Host";
GGML_UNUSED(buft);
-@@ -1334,7 +1344,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {
+@@ -1487,7 +1497,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {
return buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name;
}
@@ -270,7 +270,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
CUDA_CHECK(cudaFreeHost(buffer->context));
}
-@@ -1356,17 +1366,25 @@ static void * ggml_cuda_host_malloc(size_t size) {
+@@ -1509,17 +1519,25 @@ static void * ggml_cuda_host_malloc(size_t size) {
return ptr;
}
@@ -298,7 +298,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
return buffer;
}
-@@ -1494,8 +1512,13 @@ static void ggml_cuda_op_mul_mat_cublas(
+@@ -1647,8 +1665,13 @@ static void ggml_cuda_op_mul_mat_cublas(
const int cc = ggml_cuda_info().devices[id].cc;
@@ -312,7 +312,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
const bool use_fp16 =
src0->type != GGML_TYPE_NVFP4 &&
-@@ -2969,20 +2992,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
+@@ -3127,20 +3150,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
// backend
@@ -336,7 +336,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -2991,7 +3014,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens
+@@ -3149,7 +3172,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens
CUDA_CHECK(cudaMemcpyAsync((char *) tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));
}
@@ -345,7 +345,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -3000,7 +3023,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm
+@@ -3158,7 +3181,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm
CUDA_CHECK(cudaMemcpyAsync(data, (const char *) tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));
}
@@ -354,7 +354,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -3011,7 +3034,7 @@ static void ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct
+@@ -3169,7 +3192,7 @@ static void ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct
(char *) tensor->data + offset, stride_tensor, data, stride_data, size, n_copies, cudaMemcpyHostToDevice, cuda_ctx->stream()));
}
@@ -363,7 +363,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -3022,7 +3045,7 @@ static void ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, const
+@@ -3180,7 +3203,7 @@ static void ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, const
data, stride_data, (const char *) tensor->data + offset, stride_tensor, size, n_copies, cudaMemcpyDeviceToHost, cuda_ctx->stream()));
}
@@ -372,7 +372,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;
ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;
-@@ -3077,7 +3100,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_
+@@ -3235,7 +3258,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_
return true;
}
@@ -381,7 +381,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
CUDA_CHECK(cudaStreamSynchronize(cuda_ctx->stream()));
-@@ -4215,7 +4238,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co
+@@ -4457,7 +4480,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co
}
#endif // USE_CUDA_GRAPH
@@ -390,7 +390,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_cuda_set_device(cuda_ctx->device);
-@@ -4274,13 +4297,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
+@@ -4516,13 +4539,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
return GGML_STATUS_SUCCESS;
}
@@ -406,7 +406,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
if (ggml_backend_is_cuda(backend)) {
-@@ -4299,7 +4322,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
+@@ -4541,7 +4564,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
}
}
@@ -415,7 +415,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
#ifdef USE_CUDA_GRAPH
-@@ -4587,7 +4610,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot
+@@ -4829,7 +4852,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot
CUDA_CHECK(cudaMemGetInfo(free, total));
}
@@ -424,7 +424,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
return false;
}
-@@ -4610,7 +4633,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
+@@ -4852,7 +4875,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
#endif // CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA)
}
@@ -433,7 +433,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
return;
}
-@@ -4633,12 +4656,12 @@ struct ggml_backend_cuda_device_context {
+@@ -4875,12 +4898,12 @@ struct ggml_backend_cuda_device_context {
int op_offload_min_batch_size;
};
@@ -448,7 +448,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
return ctx->description.c_str();
}
-@@ -4719,7 +4742,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k
+@@ -4961,7 +4984,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k
}
#endif // defined(__linux__)
@@ -457,7 +457,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
ggml_cuda_set_device(ctx->device);
CUDA_CHECK(cudaMemGetInfo(free, total));
-@@ -4749,12 +4772,12 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
+@@ -4991,12 +5014,12 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
}
@@ -472,7 +472,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
props->name = ggml_backend_cuda_device_get_name(dev);
-@@ -4778,24 +4801,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
+@@ -5020,24 +5043,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
};
}
@@ -501,7 +501,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
// split buffers can only be used with GGML_OP_MUL_MAT
-@@ -5179,7 +5202,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
+@@ -5424,7 +5447,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
}
}
@@ -510,7 +510,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
const bool integrated = ggml_cuda_info().devices[dev_ctx->device].integrated;
return (((ggml_backend_buft_is_cuda(buft) || ggml_backend_buft_is_cuda_split(buft)) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft)));
-@@ -5200,13 +5223,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
+@@ -5445,13 +5468,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
}
}
@@ -526,7 +526,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
#ifdef GGML_CUDA_NO_PEER_COPY
return nullptr;
#else
-@@ -5224,14 +5247,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_
+@@ -5469,14 +5492,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_
#endif
}
@@ -543,7 +543,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
GGML_UNUSED(dev);
CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));
}
-@@ -5260,23 +5283,23 @@ struct ggml_backend_cuda_reg_context {
+@@ -5505,23 +5528,23 @@ struct ggml_backend_cuda_reg_context {
std::vector<ggml_backend_dev_t> devices;
};
@@ -571,7 +571,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
static std::vector<ggml_backend_feature> features = []() {
std::vector<ggml_backend_feature> features;
#define _STRINGIFY(...) #__VA_ARGS__
-@@ -5337,7 +5360,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t
+@@ -5582,7 +5605,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t
GGML_UNUSED(reg);
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch
index c54c0e0f..159312ad 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch
@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
default:
GGML_ABORT("fatal error");
break;
-@@ -286,6 +288,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+@@ -285,6 +287,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q5_K:
case GGML_TYPE_Q6_K:
@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
case GGML_TYPE_IQ2_XXS:
case GGML_TYPE_IQ2_XS:
case GGML_TYPE_IQ2_S:
-@@ -294,6 +297,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+@@ -293,6 +296,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
case GGML_TYPE_IQ1_S:
case GGML_TYPE_IQ4_XS:
case GGML_TYPE_IQ4_NL:
@@ -33,7 +33,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
mmq_supported = true;
break;
default:
-@@ -356,9 +360,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+@@ -355,9 +359,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
return ne11 <= 128;
case GGML_TYPE_Q6_K:
return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch
index c52d0de0..37a3c63b 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
--- a/llama.cpp/ggml/src/ggml-cuda/mmq.cuh
+++ b/llama.cpp/ggml/src/ggml-cuda/mmq.cuh
-@@ -4088,6 +4088,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);
+@@ -4152,6 +4152,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);
extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);
extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);
extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);
@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);
-@@ -4096,6 +4097,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);
+@@ -4160,6 +4161,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);
extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);
extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
index 1c1f68cb..2d44eee0 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp
--- a/llama.cpp/ggml/src/ggml-metal/ggml-metal.cpp
+++ b/llama.cpp/ggml/src/ggml-metal/ggml-metal.cpp
-@@ -23,7 +23,7 @@ static int g_devices = 1;
+@@ -26,7 +26,7 @@ static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer);
// shared buffer
@@ -10,7 +10,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
-@@ -31,7 +31,7 @@ static void ggml_backend_metal_buffer_shared_free_buffer(ggml_backend_buffer_t b
+@@ -34,7 +34,7 @@ static void ggml_backend_metal_buffer_shared_free_buffer(ggml_backend_buffer_t b
ggml_metal_buffer_free(ctx);
}
@@ -19,7 +19,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
-@@ -39,7 +39,7 @@ static void * ggml_backend_metal_buffer_shared_get_base(ggml_backend_buffer_t bu
+@@ -42,7 +42,7 @@ static void * ggml_backend_metal_buffer_shared_get_base(ggml_backend_buffer_t bu
return ggml_metal_buffer_get_base(ctx);
}
@@ -28,7 +28,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
-@@ -47,7 +47,7 @@ static void ggml_backend_metal_buffer_shared_memset_tensor(ggml_backend_buffer_t
+@@ -50,7 +50,7 @@ static void ggml_backend_metal_buffer_shared_memset_tensor(ggml_backend_buffer_t
ggml_metal_buffer_memset_tensor(ctx, tensor, value, offset, size);
}
@@ -37,7 +37,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
-@@ -55,7 +55,7 @@ static void ggml_backend_metal_buffer_shared_set_tensor(ggml_backend_buffer_t bu
+@@ -58,7 +58,7 @@ static void ggml_backend_metal_buffer_shared_set_tensor(ggml_backend_buffer_t bu
ggml_metal_buffer_set_tensor(ctx, tensor, data, offset, size);
}
@@ -46,7 +46,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
-@@ -63,7 +63,7 @@ static void ggml_backend_metal_buffer_shared_get_tensor(ggml_backend_buffer_t bu
+@@ -66,7 +66,7 @@ static void ggml_backend_metal_buffer_shared_get_tensor(ggml_backend_buffer_t bu
ggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size);
}
@@ -55,8 +55,8 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
-@@ -75,7 +75,7 @@ static bool ggml_backend_metal_buffer_shared_cpy_tensor(ggml_backend_buffer_t bu
- return false;
+@@ -78,7 +78,7 @@ static bool ggml_backend_metal_buffer_shared_cpy_tensor(ggml_backend_buffer_t bu
+ return ggml_metal_buffer_cpy_tensor(ctx, src, dst);
}
-static void ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer, uint8_t value) {
@@ -64,7 +64,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
-@@ -83,6 +83,14 @@ static void ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer,
+@@ -86,6 +86,14 @@ static void ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer,
ggml_metal_buffer_clear(ctx, value);
}
@@ -77,13 +77,13 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
+}
+
static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {
- /* .free_buffer = */ ggml_backend_metal_buffer_shared_free_buffer,
- /* .get_base = */ ggml_backend_metal_buffer_shared_get_base,
-@@ -95,11 +103,12 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {
- /* .cpy_tensor = */ ggml_backend_metal_buffer_shared_cpy_tensor,
- /* .clear = */ ggml_backend_metal_buffer_shared_clear,
- /* .reset = */ NULL,
-+ /* .free_struct = */ ggml_backend_metal_buffer_free_struct,
+ /* .free_buffer = */ ggml_backend_metal_buffer_shared_free_buffer,
+ /* .get_base = */ ggml_backend_metal_buffer_shared_get_base,
+@@ -98,11 +106,12 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {
+ /* .cpy_tensor = */ ggml_backend_metal_buffer_shared_cpy_tensor,
+ /* .clear = */ ggml_backend_metal_buffer_shared_clear,
+ /* .reset = */ NULL,
++ /* .free_struct = */ ggml_backend_metal_buffer_free_struct,
};
// private buffer
@@ -93,7 +93,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -107,7 +116,7 @@ static void ggml_backend_metal_buffer_private_free_buffer(ggml_backend_buffer_t
+@@ -110,7 +119,7 @@ static void ggml_backend_metal_buffer_private_free_buffer(ggml_backend_buffer_t
ggml_metal_buffer_free(ctx);
}
@@ -102,7 +102,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -115,7 +124,7 @@ static void * ggml_backend_metal_buffer_private_get_base(ggml_backend_buffer_t b
+@@ -118,7 +127,7 @@ static void * ggml_backend_metal_buffer_private_get_base(ggml_backend_buffer_t b
return ggml_metal_buffer_get_base(ctx);
}
@@ -111,7 +111,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -123,7 +132,7 @@ static void ggml_backend_metal_buffer_private_memset_tensor(ggml_backend_buffer_
+@@ -126,7 +135,7 @@ static void ggml_backend_metal_buffer_private_memset_tensor(ggml_backend_buffer_
ggml_metal_buffer_memset_tensor(ctx, tensor, value, offset, size);
}
@@ -120,7 +120,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -131,7 +140,7 @@ static void ggml_backend_metal_buffer_private_set_tensor(ggml_backend_buffer_t b
+@@ -134,7 +143,7 @@ static void ggml_backend_metal_buffer_private_set_tensor(ggml_backend_buffer_t b
ggml_metal_buffer_set_tensor(ctx, tensor, data, offset, size);
}
@@ -129,7 +129,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -139,7 +148,7 @@ static void ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t b
+@@ -142,7 +151,7 @@ static void ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t b
ggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size);
}
@@ -138,8 +138,8 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -151,7 +160,7 @@ static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t b
- return false;
+@@ -154,7 +163,7 @@ static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t b
+ return ggml_metal_buffer_cpy_tensor(ctx, src, dst);
}
-static void ggml_backend_metal_buffer_private_clear(ggml_backend_buffer_t buffer, uint8_t value) {
@@ -147,15 +147,15 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -171,6 +180,7 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = {
- /* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor,
- /* .clear = */ ggml_backend_metal_buffer_private_clear,
- /* .reset = */ NULL,
-+ /* .free_struct = */ ggml_backend_metal_buffer_free_struct,
+@@ -174,6 +183,7 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = {
+ /* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor,
+ /* .clear = */ ggml_backend_metal_buffer_private_clear,
+ /* .reset = */ NULL,
++ /* .free_struct = */ ggml_backend_metal_buffer_free_struct,
};
static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer) {
-@@ -243,33 +253,33 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
+@@ -246,33 +256,33 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
// default (shared) buffer type
@@ -195,7 +195,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return false;
GGML_UNUSED(buft);
-@@ -319,33 +329,33 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi
+@@ -322,33 +332,33 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi
// default (private) buffer type
@@ -235,7 +235,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return false;
GGML_UNUSED(buft);
-@@ -394,34 +404,34 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev
+@@ -397,34 +407,34 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev
// mapped buffer type
@@ -276,7 +276,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return false;
GGML_UNUSED(buft);
-@@ -472,13 +482,13 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi
+@@ -475,13 +485,13 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi
// backend
@@ -292,7 +292,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_t ctx = (ggml_metal_t)backend->context;
// wait for any ongoing async operations to finish
-@@ -489,25 +499,25 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {
+@@ -492,25 +502,25 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {
free(backend);
}
@@ -322,7 +322,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
if (!ggml_backend_is_metal(backend_src) || !ggml_backend_is_metal(backend_dst)) {
return false;
}
-@@ -528,27 +538,27 @@ static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml
+@@ -531,27 +541,27 @@ static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml
return ggml_metal_cpy_tensor_async(ctx_src, ctx_dst, src, dst);
}
@@ -354,7 +354,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_t ctx = (ggml_metal_t)backend->context;
ggml_metal_graph_optimize(ctx, cgraph);
-@@ -640,7 +650,7 @@ void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {
+@@ -643,7 +653,7 @@ void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {
// backend device
@@ -363,7 +363,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);
-@@ -648,25 +658,25 @@ static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {
+@@ -651,25 +661,25 @@ static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {
return props_dev->name;
}
@@ -393,7 +393,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
props->name = ggml_backend_metal_device_get_name(dev);
props->description = ggml_backend_metal_device_get_description(dev);
props->type = ggml_backend_metal_device_get_type(dev);
-@@ -681,7 +691,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac
+@@ -684,7 +694,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac
};
}
@@ -402,7 +402,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_t ctx = ggml_metal_init(ctx_dev);
-@@ -706,7 +716,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t
+@@ -709,7 +719,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t
GGML_UNUSED(params);
}
@@ -411,7 +411,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);
-@@ -714,7 +724,7 @@ static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggml
+@@ -717,7 +727,7 @@ static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggml
return props_dev->use_shared_buffers ? ggml_backend_metal_buffer_type_shared(props_dev->device) : ggml_backend_metal_buffer_type_private(props_dev->device);
}
@@ -420,7 +420,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_buffer_t res = ggml_metal_buffer_map(ctx_dev, ptr, size, max_tensor_size);
-@@ -724,13 +734,13 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen
+@@ -727,13 +737,13 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen
return ggml_backend_buffer_init(ggml_backend_metal_buffer_type_mapped(props_dev->device), ggml_backend_metal_buffer_shared_i, res, size);
}
@@ -436,7 +436,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return
buft->device == dev && (
buft->iface.get_name == ggml_backend_metal_buffer_type_shared_get_name ||
-@@ -751,7 +761,7 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
+@@ -754,7 +764,7 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
}
}
@@ -445,7 +445,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
return (op->op == GGML_OP_MUL_MAT ||
-@@ -759,7 +769,7 @@ static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const g
+@@ -762,7 +772,7 @@ static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const g
get_op_batch_size(op) >= ggml_metal_device_get_props(ctx_dev)->op_offload_min_batch_size;
}
@@ -454,7 +454,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_event_t event = ggml_metal_device_event_init(ctx_dev);
-@@ -773,7 +783,7 @@ static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_dev
+@@ -776,7 +786,7 @@ static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_dev
return ev;
}
@@ -463,7 +463,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_event_t ev = (ggml_metal_event_t)event->context;
-@@ -783,7 +793,7 @@ static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_ba
+@@ -786,7 +796,7 @@ static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_ba
delete event;
}
@@ -472,7 +472,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_event_t evt = (ggml_metal_event_t)event->context;
-@@ -835,18 +845,18 @@ struct ggml_backend_metal_reg_deleter {
+@@ -838,18 +848,18 @@ struct ggml_backend_metal_reg_deleter {
typedef std::unique_ptr<struct ggml_backend_metal_reg, ggml_backend_metal_reg_deleter> ggml_backend_metal_reg_ptr;
@@ -494,7 +494,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_backend_metal_reg_t ctx = (ggml_backend_metal_reg_t)reg->context;
GGML_ASSERT(index < ctx->devices.size());
return ctx->devices[index];
-@@ -859,13 +869,13 @@ static ggml_backend_feature g_ggml_backend_metal_features[] = {
+@@ -862,13 +872,13 @@ static ggml_backend_feature g_ggml_backend_metal_features[] = {
{ NULL, NULL },
};
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
index 53a01fbc..0ae19742 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
--- a/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp
-@@ -252,11 +252,11 @@ struct vk_queue {
+@@ -249,11 +249,11 @@ struct vk_queue {
}
};
@@ -18,7 +18,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
static ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = {
/* .get_name = */ ggml_backend_vk_buffer_type_name,
/* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer,
-@@ -2089,7 +2089,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph *
+@@ -2116,7 +2116,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph *
typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
@@ -27,7 +27,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
static VkDeviceSize ggml_vk_get_max_buffer_range(const ggml_backend_vk_context * ctx, const vk_buffer &buf, const VkDeviceSize offset) {
const VkDeviceSize range = std::min(VkDeviceSize{buf->size - offset},
-@@ -3469,20 +3469,15 @@ static void ggml_vk_load_shaders(vk_device& device) {
+@@ -3634,20 +3634,15 @@ static void ggml_vk_load_shaders(vk_device& device) {
if (!pipeline->needed || pipeline->compiled) {
continue;
}
@@ -54,7 +54,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
}
};
-@@ -13569,20 +13564,28 @@ static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {
+@@ -13871,20 +13866,28 @@ static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {
return buffer->buft->iface.get_name == ggml_backend_vk_buffer_type_name;
}
@@ -86,7 +86,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_init_tensor(" << buffer << " (" << buffer->context << "), " << tensor << ")");
if (tensor->view_src != nullptr) {
GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);
-@@ -13590,7 +13593,7 @@ static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t
+@@ -13892,7 +13895,7 @@ static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t
return GGML_STATUS_SUCCESS;
}
@@ -95,7 +95,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_memset_tensor(" << buffer << ", " << tensor << ", " << value << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
vk_buffer buf = buf_ctx->dev_buffer;
-@@ -13603,7 +13606,7 @@ static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, g
+@@ -13905,7 +13908,7 @@ static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, g
ggml_vk_buffer_memset(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, val32, size);
}
@@ -104,25 +104,43 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
vk_buffer buf = buf_ctx->dev_buffer;
-@@ -13615,7 +13618,7 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml
+@@ -13917,7 +13920,7 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml
ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);
}
+-static void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset,
++static void GGML_CALL ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset,
+ size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
+ VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " <<
+ n_copies << ", " << stride_tensor << ", " << stride_data << ")");
+@@ -13931,7 +13934,7 @@ static void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, g
+ ggml_vk_buffer_write_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_data, stride_tensor, size, n_copies);
+ }
+
-static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
-@@ -13628,7 +13631,7 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, cons
+@@ -13944,7 +13947,7 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, cons
ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);
}
+-static void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset,
++static void GGML_CALL ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset,
+ size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
+ VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " <<
+ n_copies << ", " << stride_tensor << ", " << stride_data << ")");
+@@ -13959,7 +13962,7 @@ static void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, c
+ ggml_vk_buffer_read_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_tensor, stride_data, size, n_copies);
+ }
+
-static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
+static bool GGML_CALL ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
if (ggml_nbytes(src) == 0) {
return true;
}
-@@ -13649,7 +13652,7 @@ static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, cons
+@@ -13980,7 +13983,7 @@ static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, cons
UNUSED(buffer);
}
@@ -131,7 +149,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context;
ggml_vk_buffer_memset(ctx->dev_buffer, 0, value, buffer->size);
-@@ -13667,16 +13670,17 @@ static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
+@@ -13998,16 +14001,17 @@ static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
/* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,
/* .clear = */ ggml_backend_vk_buffer_clear,
/* .reset = */ NULL,
@@ -151,7 +169,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_MEMORY("ggml_backend_vk_buffer_type_alloc_buffer(" << size << ")");
ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;
-@@ -13692,17 +13696,17 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backe
+@@ -14023,17 +14027,17 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backe
return ggml_backend_buffer_init(buft, ggml_backend_vk_buffer_interface, bufctx, size);
}
@@ -172,7 +190,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
return ggml_nbytes(tensor);
UNUSED(buft);
-@@ -13720,24 +13724,24 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {
+@@ -14051,24 +14055,24 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {
// host buffer type
@@ -201,7 +219,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_MEMORY("ggml_backend_vk_host_buffer_type_alloc_buffer(" << size << ")");
size += 32; // Behave like the CPU buffer type
-@@ -13753,19 +13757,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_
+@@ -14084,19 +14088,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_
ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
buffer->buft = buft;
buffer->iface.free_buffer = ggml_backend_vk_host_buffer_free_buffer;
@@ -224,7 +242,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
return vk_instance.devices[0]->suballocation_block_size;
UNUSED(buft);
-@@ -13797,13 +13802,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {
+@@ -14128,13 +14133,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {
// backend
@@ -240,26 +258,38 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");
-@@ -13819,7 +13824,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_b
+@@ -14150,7 +14155,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_b
return &ctx->device->buffer_type;
}
+-static void ggml_backend_vk_set_tensor_2d_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset,
++static void GGML_CALL ggml_backend_vk_set_tensor_2d_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset,
+ size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
+ VK_LOG_DEBUG("ggml_backend_vk_set_tensor_2d_async(" << size << ", " << n_copies << ")");
+ ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
+@@ -14214,12 +14219,12 @@ static void ggml_backend_vk_set_tensor_2d_async(ggml_backend_t backend, ggml_ten
+ }
+ }
+
-static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_vk_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
VK_LOG_DEBUG("ggml_backend_vk_set_tensor_async(" << size << ")");
+ ggml_backend_vk_set_tensor_2d_async(backend, tensor, data, offset, size, 1, size, size);
+ }
+
+-static void ggml_backend_vk_get_tensor_2d_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset,
++static void GGML_CALL ggml_backend_vk_get_tensor_2d_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset,
+ size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
+ VK_LOG_DEBUG("ggml_backend_vk_get_tensor_2d_async(" << size << ", " << n_copies << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
- GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_get_default_buffer_type(backend) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type");
-@@ -13866,7 +13871,7 @@ static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, ggml_tensor
+@@ -14270,12 +14275,12 @@ static void ggml_backend_vk_get_tensor_2d_async(ggml_backend_t backend, const gg
}
}
-static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_vk_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
VK_LOG_DEBUG("ggml_backend_vk_get_tensor_async(" << size << ")");
- ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
- GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_get_default_buffer_type(backend) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type");
-@@ -13900,7 +13905,7 @@ static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, const ggml_
- }
+ ggml_backend_vk_get_tensor_2d_async(backend, tensor, data, offset, size, 1, size, size);
}
-static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {
@@ -267,7 +297,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_cpy_tensor_async(" << src << " -> " << dst << ", size=" << ggml_nbytes(src) << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend_dst->context;
-@@ -14025,7 +14030,7 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
+@@ -14400,7 +14405,7 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
}
}
@@ -276,7 +306,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_synchronize()");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -14446,7 +14451,7 @@ static int32_t find_first_set(uint32_t x) {
+@@ -14936,7 +14941,7 @@ static int32_t find_first_set(uint32_t x) {
return ret;
}
@@ -285,7 +315,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -14828,7 +14833,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
+@@ -15339,7 +15344,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
}
// Sort the graph for improved parallelism.
@@ -294,7 +324,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
{
VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -15066,7 +15071,7 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
+@@ -15596,7 +15601,7 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
}
}
@@ -303,7 +333,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_event_record(backend=" << backend << ", event=" << event << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
vk_event *vkev = (vk_event *)event->context;
-@@ -15104,7 +15109,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_ev
+@@ -15634,7 +15639,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_ev
ctx->compute_ctx.reset();
}
@@ -312,7 +342,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_event_wait(backend=" << backend << ", event=" << event << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
vk_event *vkev = (vk_event *)event->context;
-@@ -15201,7 +15206,10 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total
+@@ -15731,7 +15736,10 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total
*total += heap.size;
if (membudget_supported && i < budgetprops.heapUsage.size()) {
@@ -324,7 +354,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
} else {
*free += heap.size;
}
-@@ -15269,38 +15277,38 @@ struct ggml_backend_vk_device_context {
+@@ -15799,38 +15807,38 @@ struct ggml_backend_vk_device_context {
int op_offload_min_batch_size;
};
@@ -370,7 +400,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
props->name = ggml_backend_vk_device_get_name(dev);
-@@ -15316,13 +15324,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml
+@@ -15846,13 +15854,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml
};
}
@@ -386,7 +416,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
const vk_device& device = ggml_vk_get_device(ctx->device);
-@@ -15868,7 +15876,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
+@@ -16384,7 +16392,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
UNUSED(dev);
}
@@ -395,7 +425,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
if (buft->iface.get_name != ggml_backend_vk_buffer_type_name) {
return false;
}
-@@ -15894,13 +15902,13 @@ static int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {
+@@ -16410,13 +16418,13 @@ static int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {
}
}
@@ -411,7 +441,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -15923,7 +15931,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t
+@@ -16439,7 +16447,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t
};
}
@@ -420,7 +450,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -15943,7 +15951,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backe
+@@ -16459,7 +16467,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backe
delete event;
}
@@ -429,7 +459,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_device_event_synchronize(backend=" << dev << ", event=" << event << ")");
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -16000,7 +16008,7 @@ static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, si
+@@ -16516,7 +16524,7 @@ static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, si
return buf;
}
@@ -438,7 +468,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")");
GGML_UNUSED(max_tensor_size);
-@@ -16038,17 +16046,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {
+@@ -16554,17 +16562,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {
/* .event_synchronize = */ ggml_backend_vk_device_event_synchronize,
};
diff --git a/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch b/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch
index 73195679..a9a46b47 100644
--- a/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch
@@ -12,180 +12,62 @@ diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp
#include <cinttypes>
#include <cstddef>
#include <cstdint>
-@@ -358,18 +362,107 @@ struct gguf_reader {
- return nread == size;
- }
-
-+ size_t tell() const {
-+ return gguf_ftell(file);
-+ }
-+
-+ bool seek(size_t offset, int whence) const {
-+ return gguf_fseek(file, offset, whence) == 0;
-+ }
-+
- private:
- FILE * file;
-
- mutable uint64_t nbytes_remain;
- };
+@@ -976,7 +980,54 @@ struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, stru
+ return gguf_init_from_reader(gr, params);
+ }
+#ifdef COSMOCC
-+// [llamafile] Reader that uses llamafile API for both regular files and memory-mapped content
++// [llamafile] Reader callback that reads through the llamafile API so we can
++// handle plain GGUF, /zip/ paths, [email protected], and .llamafile containers.
+struct gguf_llamafile_reader {
+ struct llamafile * lfile;
++ uint64_t offset;
++};
+
-+ gguf_llamafile_reader(struct llamafile * lfile) : lfile(lfile) {}
-+
-+ template <typename T>
-+ bool read(T & dst) const {
-+ return llamafile_read(lfile, &dst, sizeof(dst)) == (long)sizeof(dst);
-+ }
-+
-+ template <typename T>
-+ bool read(std::vector<T> & dst, const size_t n) const {
-+ dst.resize(n);
-+ for (size_t i = 0; i < dst.size(); ++i) {
-+ if constexpr (std::is_same<T, bool>::value) {
-+ bool tmp;
-+ if (!read(tmp)) {
-+ return false;
-+ }
-+ dst[i] = tmp;
-+ } else {
-+ if (!read(dst[i])) {
-+ return false;
-+ }
-+ }
-+ }
-+ return true;
-+ }
-+
-+ bool read(bool & dst) const {
-+ int8_t tmp = -1;
-+ if (!read(tmp)) {
-+ return false;
-+ }
-+ dst = tmp != 0;
-+ return true;
-+ }
-+
-+ bool read(enum ggml_type & dst) const {
-+ int32_t tmp = -1;
-+ if (!read(tmp)) {
-+ return false;
-+ }
-+ dst = ggml_type(tmp);
-+ return true;
-+ }
++static size_t gguf_llamafile_reader_callback(void * userdata, void * output, uint64_t offset, size_t len) {
++ GGML_ASSERT(len > 0);
+
-+ bool read(enum gguf_type & dst) const {
-+ int32_t tmp = -1;
-+ if (!read(tmp)) {
-+ return false;
-+ }
-+ dst = gguf_type(tmp);
-+ return true;
-+ }
++ gguf_llamafile_reader & reader = *static_cast<gguf_llamafile_reader *>(userdata);
+
-+ bool read(std::string & dst) const {
-+ uint64_t size = 0;
-+ if (!read(size)) {
-+ return false;
++ if (reader.offset != offset) {
++ if (!llamafile_seek(reader.lfile, static_cast<size_t>(offset), SEEK_SET)) {
++ return 0;
+ }
-+ dst.resize(size);
-+ return llamafile_read(lfile, dst.data(), dst.length()) == (long)dst.length();
++ reader.offset = offset;
+ }
+
-+ bool read(void * dst, const size_t size) const {
-+ return llamafile_read(lfile, dst, size) == (long)size;
++ const long nread = llamafile_read(reader.lfile, output, len);
++ if (nread <= 0) {
++ return 0;
+ }
-+
-+ size_t tell() const {
-+ return llamafile_tell(lfile);
-+ }
-+
-+ bool seek(size_t offset, int whence) const {
-+ return llamafile_seek(lfile, offset, whence);
-+ }
-+};
-+#endif
-+
- struct gguf_context * gguf_init_empty(void) {
- return new gguf_context;
- }
-
--template<typename T>
--bool gguf_read_emplace_helper(const struct gguf_reader & gr, std::vector<struct gguf_kv> & kv, const std::string & key, const bool is_array, const size_t n) {
-+template<typename T, typename Reader>
-+bool gguf_read_emplace_helper(const Reader & gr, std::vector<struct gguf_kv> & kv, const std::string & key, const bool is_array, const size_t n) {
- if (is_array) {
- std::vector<T> value;
- try {
-@@ -394,12 +487,8 @@ bool gguf_read_emplace_helper(const struct gguf_reader & gr, std::vector<struct
- return true;
- }
-
--struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params) {
-- if (!file) {
-- return nullptr;
-- }
--
-- const struct gguf_reader gr(file);
-+template<typename Reader>
-+struct gguf_context * gguf_init_from_reader_impl(const Reader & gr, struct gguf_init_params params) {
- struct gguf_context * ctx = new gguf_context;
-
- bool ok = true;
-@@ -700,14 +789,14 @@ struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_para
- GGML_ASSERT(int64_t(ctx->info.size()) == n_tensors);
-
- // we require the data section to be aligned, so take into account any padding
-- if (gguf_fseek(file, GGML_PAD(gguf_ftell(file), ctx->alignment), SEEK_SET) != 0) {
-+ if (!gr.seek(GGML_PAD(gr.tell(), ctx->alignment), SEEK_SET)) {
- GGML_LOG_ERROR("%s: failed to seek to beginning of data section\n", __func__);
- gguf_free(ctx);
- return nullptr;
- }
-
- // store the current file offset - this is where the data section starts
-- ctx->offset = gguf_ftell(file);
-+ ctx->offset = gr.tell();
-
- // compute the total size of the data section, taking into account the alignment
- {
-@@ -844,7 +933,30 @@ struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_para
- return ctx;
- }
-
-+// Wrapper for FILE*-based reading
-+struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params) {
-+ if (!file) {
-+ return nullptr;
-+ }
-+ const struct gguf_reader gr(file);
-+ return gguf_init_from_reader_impl(gr, params);
++ reader.offset += static_cast<uint64_t>(nread);
++ return static_cast<size_t>(nread);
+}
++#endif
+
struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params) {
+#ifdef COSMOCC
-+ // [llamafile] Use llamafile API for all file types: plain GGUF, /zip/ paths,
-+ // [email protected], .llamafile containers. The llamafile API handles both
-+ // FILE*-backed and memory-mapped content transparently.
++ // [llamafile] Route all GGUF file loads through the llamafile API so that
++ // /zip/ paths and .llamafile containers work transparently.
+ struct llamafile * lfile = llamafile_open_gguf(fname, "rb");
+ if (!lfile) {
+ GGML_LOG_ERROR("%s: failed to open GGUF file '%s': %s\n", __func__, fname, strerror(errno));
+ return nullptr;
+ }
-+ const struct gguf_llamafile_reader gr(lfile);
-+ struct gguf_context * result = gguf_init_from_reader_impl(gr, params);
++
++ gguf_llamafile_reader reader = {
++ /*.lfile = */ lfile,
++ /*.offset = */ 0,
++ };
++ const struct gguf_reader gr(gguf_llamafile_reader_callback, &reader, SIZE_MAX, 0, llamafile_size(lfile));
++ struct gguf_context * result = gguf_init_from_reader(gr, params);
+ llamafile_close(lfile);
+ return result;
+#else
FILE * file = ggml_fopen(fname, "rb");
if (!file) {
-@@ -855,6 +967,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p
+@@ -987,6 +1038,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p
struct gguf_context * result = gguf_init_from_file_ptr(file, params);
fclose(file);
return result;
diff --git a/llama.cpp.patches/patches/src_llama-context.cpp.patch b/llama.cpp.patches/patches/src_llama-context.cpp.patch
index 2d46b892..6f5eec20 100644
--- a/llama.cpp.patches/patches/src_llama-context.cpp.patch
+++ b/llama.cpp.patches/patches/src_llama-context.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
--- a/llama.cpp/src/llama-context.cpp
+++ b/llama.cpp/src/llama-context.cpp
-@@ -426,6 +426,16 @@ void llama_context::sched_reserve() {
+@@ -448,6 +448,16 @@ void llama_context::sched_reserve() {
LLAMA_LOG_DEBUG("%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n", __func__, n_tokens, n_seqs, n_outputs);
// resolve automatic Flash Attention use
diff --git a/llama.cpp.patches/patches/src_llama-mmap.cpp.patch b/llama.cpp.patches/patches/src_llama-mmap.cpp.patch
index a51c2b7b..a15e763b 100644
--- a/llama.cpp.patches/patches/src_llama-mmap.cpp.patch
+++ b/llama.cpp.patches/patches/src_llama-mmap.cpp.patch
@@ -12,7 +12,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
#include <cstring>
#include <climits>
#include <stdexcept>
-@@ -173,6 +177,15 @@ struct llama_file::impl {
+@@ -181,6 +185,15 @@ struct llama_file::impl {
}
#else
impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {
@@ -28,7 +28,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
#ifdef __linux__
// Try unbuffered I/O for read only
if (use_direct_io && std::strcmp(mode, "rb") == 0) {
-@@ -225,6 +238,9 @@ struct llama_file::impl {
+@@ -233,6 +246,9 @@ struct llama_file::impl {
}
size_t tell() const {
@@ -36,9 +36,9 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
+ return llamafile_tell(lfile);
+#endif
if (fd == -1) {
- long ret = std::ftell(fp);
+ off_t ret = llama_mmap_ftell(fp);
if (ret == -1) {
-@@ -242,6 +258,11 @@ struct llama_file::impl {
+@@ -250,6 +266,11 @@ struct llama_file::impl {
}
void seek(size_t offset, int whence) const {
@@ -49,8 +49,8 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
+#else
off_t ret = 0;
if (fd == -1) {
- ret = std::fseek(fp, (long) offset, whence);
-@@ -251,12 +272,22 @@ struct llama_file::impl {
+ ret = llama_mmap_fseek(fp, offset, whence);
+@@ -259,12 +280,22 @@ struct llama_file::impl {
if (ret == -1) {
throw std::runtime_error(format("seek error: %s", strerror(errno)));
}
@@ -73,7 +73,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
errno = 0;
if (fd == -1) {
const size_t curr_off = tell();
-@@ -306,6 +337,7 @@ struct llama_file::impl {
+@@ -314,6 +345,7 @@ struct llama_file::impl {
bytes_read += (size_t) ret;
}
}
@@ -81,7 +81,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
}
void read_aligned_chunk(void * dest, size_t size) {
-@@ -366,13 +398,20 @@ struct llama_file::impl {
+@@ -374,13 +406,20 @@ struct llama_file::impl {
}
~impl() {
@@ -102,7 +102,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
std::string fname;
#endif
-@@ -401,6 +440,10 @@ size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
+@@ -409,6 +448,10 @@ size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }
int llama_file::file_id() const {
@@ -113,7 +113,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
#ifdef _WIN32
return _fileno(pimpl->fp);
#else
-@@ -408,9 +451,10 @@ int llama_file::file_id() const {
+@@ -416,9 +459,10 @@ int llama_file::file_id() const {
return pimpl->fd;
}
#if defined(fileno)
@@ -126,7 +126,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
#endif
#endif
}
-@@ -428,14 +472,56 @@ uint32_t llama_file::read_u32() { return pimpl->read_u32(); }
+@@ -436,14 +480,56 @@ uint32_t llama_file::read_u32() { return pimpl->read_u32(); }
void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }
void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
@@ -183,7 +183,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
int fd = file->file_id();
int flags = MAP_SHARED;
if (numa) { prefetch = 0; }
-@@ -480,6 +566,10 @@ struct llama_mmap::impl {
+@@ -488,6 +574,10 @@ struct llama_mmap::impl {
}
void unmap_fragment(size_t first, size_t last) {
@@ -194,7 +194,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
int page_size = sysconf(_SC_PAGESIZE);
align_range(&first, &last, page_size);
size_t len = last - first;
-@@ -516,6 +606,14 @@ struct llama_mmap::impl {
+@@ -524,6 +614,14 @@ struct llama_mmap::impl {
}
~impl() {
diff --git a/llama.cpp.patches/patches/src_models_t5.cpp.patch b/llama.cpp.patches/patches/src_models_t5.cpp.patch
new file mode 100644
index 00000000..40e2a98a
--- /dev/null
+++ b/llama.cpp.patches/patches/src_models_t5.cpp.patch
@@ -0,0 +1,16 @@
+diff --git a/src/models/t5.cpp b/src/models/t5.cpp
+--- a/llama.cpp/src/models/t5.cpp
++++ b/llama.cpp/src/models/t5.cpp
+@@ -106,6 +106,12 @@ void llama_model_t5::load_arch_tensors(llama_model_loader &) {
+ }
+ }
+
++// Forward-declare the explicit specializations so that the std::make_unique
++// calls below see them before triggering implicit instantiation. Required
++// when compiling with -std=gnu++23 on clang.
++template <> llama_model_t5::graph<false>::graph(const llama_model & model, const llm_graph_params & params);
++template <> llama_model_t5::graph<true>::graph(const llama_model & model, const llm_graph_params & params);
++
+ std::unique_ptr<llm_graph_context> llama_model_t5::build_arch_graph(const llm_graph_params & params) const {
+ switch (params.gtype) {
+ case LLM_GRAPH_TYPE_ENCODER:
diff --git a/llama.cpp.patches/patches/tools_server_server-models.cpp.patch b/llama.cpp.patches/patches/tools_server_server-models.cpp.patch
index 52d68c2b..7f1b70c6 100644
--- a/llama.cpp.patches/patches/tools_server_server-models.cpp.patch
+++ b/llama.cpp.patches/patches/tools_server_server-models.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
--- a/llama.cpp/tools/server/server-models.cpp
+++ b/llama.cpp/tools/server/server-models.cpp
-@@ -30,6 +30,7 @@
+@@ -31,6 +31,7 @@
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
@@ -9,7 +9,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
extern char **environ;
#endif
-@@ -523,9 +524,12 @@ void server_models::unload_lru() {
+@@ -715,9 +716,12 @@ void server_models::unload_lru() {
// wait for unload to complete
{
std::unique_lock<std::mutex> lk(mutex);
@@ -25,22 +25,30 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
}
}
}
-@@ -627,6 +631,14 @@ void server_models::load(const std::string & name) {
+@@ -826,6 +830,22 @@ void server_models::load(const std::string & name) {
});
std::thread stopping_thread([&]() {
-+ // Block SIGINT/SIGTERM on this thread to prevent EINTR during
-+ // condition_variable waits. Signals are handled by the main thread.
++ // Block every signal on this thread so pthread_cond_timedwait()
++ // can't be interrupted with EINTR (libcxx rethrows EINTR as
++ // std::system_error and std::terminates the process). Keep
++ // synchronous fault signals deliverable so real crashes still
++ // surface.
+ sigset_t block_mask;
-+ sigemptyset(&block_mask);
-+ sigaddset(&block_mask, SIGINT);
-+ sigaddset(&block_mask, SIGTERM);
++ sigfillset(&block_mask);
++ sigdelset(&block_mask, SIGSEGV);
++ sigdelset(&block_mask, SIGBUS);
++ sigdelset(&block_mask, SIGILL);
++ sigdelset(&block_mask, SIGFPE);
++ sigdelset(&block_mask, SIGABRT);
++ sigdelset(&block_mask, SIGSYS);
++ sigdelset(&block_mask, SIGTRAP);
+ pthread_sigmask(SIG_BLOCK, &block_mask, nullptr);
+
// thread to monitor stopping signal OR child crash
auto is_stopping = [this, &name]() {
return this->stopping_models.find(name) != this->stopping_models.end();
-@@ -636,7 +648,10 @@ void server_models::load(const std::string & name) {
+@@ -835,7 +855,10 @@ void server_models::load(const std::string & name) {
};
{
std::unique_lock<std::mutex> lk(this->mutex);
@@ -52,7 +60,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
}
// child may have already exited (e.g. crashed) — skip shutdown sequence
if (!subprocess_alive(child_proc.get())) {
-@@ -762,13 +777,14 @@ void server_models::update_status(const std::string & name, server_model_status
+@@ -984,13 +1007,14 @@ void server_models::update_loaded_info(const std::string & name, std::string & r
void server_models::wait_until_loading_finished(const std::string & name) {
std::unique_lock<std::mutex> lk(mutex);
diff --git a/llama.cpp.patches/patches/tools_server_server-queue.cpp.patch b/llama.cpp.patches/patches/tools_server_server-queue.cpp.patch
index 43040106..e8a6e25c 100644
--- a/llama.cpp.patches/patches/tools_server_server-queue.cpp.patch
+++ b/llama.cpp.patches/patches/tools_server_server-queue.cpp.patch
@@ -28,26 +28,29 @@ diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp
}
}
-@@ -123,6 +130,18 @@ void server_queue::terminate() {
+@@ -123,6 +130,21 @@ void server_queue::terminate() {
}
void server_queue::start_loop(int64_t idle_sleep_ms) {
-+ // Block SIGINT and SIGTERM on this thread to prevent EINTR during
-+ // condition_variable::wait(). In Cosmopolitan libc, condition_variable
-+ // throws std::system_error(EINTR) when interrupted by signals, and this
-+ // exception cannot be caught (goes directly to std::terminate).
-+ // The main thread handles these signals, sets running=false, and calls
-+ // notify_all() to wake us up cleanly.
++ // Block every signal on this thread so pthread_cond_timedwait() can't
++ // be interrupted with EINTR (libcxx rethrows EINTR as
++ // std::system_error and std::terminates the process). Keep synchronous
++ // fault signals deliverable so real crashes still surface.
+ sigset_t block_mask;
-+ sigemptyset(&block_mask);
-+ sigaddset(&block_mask, SIGINT);
-+ sigaddset(&block_mask, SIGTERM);
++ sigfillset(&block_mask);
++ sigdelset(&block_mask, SIGSEGV);
++ sigdelset(&block_mask, SIGBUS);
++ sigdelset(&block_mask, SIGILL);
++ sigdelset(&block_mask, SIGFPE);
++ sigdelset(&block_mask, SIGABRT);
++ sigdelset(&block_mask, SIGSYS);
++ sigdelset(&block_mask, SIGTRAP);
+ pthread_sigmask(SIG_BLOCK, &block_mask, nullptr);
+
running = true;
time_last_task = ggml_time_ms();
-@@ -181,9 +200,13 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
+@@ -181,9 +203,13 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
callback_sleeping_state(true);
req_stop_sleeping = false;
// wait until we are requested to exit sleeping state
@@ -64,7 +67,7 @@ diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp
if (!running) { // may changed during sleep
break; // terminate
}
-@@ -266,13 +289,17 @@ void server_response::remove_waiting_task_ids(const std::unordered_set<int> & id
+@@ -266,13 +292,17 @@ void server_response::remove_waiting_task_ids(const std::unordered_set<int> & id
server_task_result_ptr server_response::recv(const std::unordered_set<int> & id_tasks) {
while (true) {
std::unique_lock<std::mutex> lock(mutex_results);
diff --git a/llama.cpp.patches/patches/tools_server_server.cpp.patch b/llama.cpp.patches/patches/tools_server_server.cpp.patch
index 8a48a3ff..f292b9d0 100644
--- a/llama.cpp.patches/patches/tools_server_server.cpp.patch
+++ b/llama.cpp.patches/patches/tools_server_server.cpp.patch
@@ -21,17 +21,24 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
static std::function<void(int)> shutdown_handler;
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
-@@ -71,12 +77,22 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
+@@ -71,15 +77,26 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
};
}
--int main(int argc, char ** argv) {
+-// satisfies -Wmissing-declarations
+-int llama_server(int argc, char ** argv);
+-
+-int llama_server(int argc, char ** argv) {
+// Core server logic - can be called from llamafile main.cpp or standalone main()
+// Optional callbacks for combined mode:
+// on_ready: called when server is fully loaded and accepting requests
+// on_shutdown_available: provides a function to trigger server shutdown
+int server_main(int argc, char ** argv,
+ std::function<void(const std::string &)> on_ready,
++ std::function<void(std::function<void()>)> on_shutdown_available);
++
++int server_main(int argc, char ** argv,
++ std::function<void(const std::string &)> on_ready,
+ std::function<void(std::function<void()>)> on_shutdown_available) {
std::setlocale(LC_NUMERIC, "C");
@@ -45,7 +52,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {
-@@ -295,6 +311,13 @@ int main(int argc, char ** argv) {
+@@ -309,6 +326,13 @@ int llama_server(int argc, char ** argv) {
// this will unblock start_loop()
ctx_server.terminate();
};
@@ -59,9 +66,9 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
}
// TODO: refactor in common/console
-@@ -326,6 +349,11 @@ int main(int argc, char ** argv) {
- LOG_INF("%s: server is listening on %s\n", __func__, ctx_http.listening_address.c_str());
- LOG_INF("%s: starting the main loop...\n", __func__);
+@@ -339,6 +363,11 @@ int llama_server(int argc, char ** argv) {
+ } else {
+ SRV_INF("server is listening on %s\n", ctx_http.listening_address.c_str());
+ // Notify caller that server is ready (for combined mode TUI startup)
+ if (on_ready) {
@@ -71,7 +78,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
// optionally, notify router server that this instance is ready
std::thread monitor_thread;
if (server_models::is_child_server()) {
-@@ -349,5 +377,48 @@ int main(int argc, char ** argv) {
+@@ -363,5 +392,48 @@ int llama_server(int argc, char ** argv) {
}
}
diff --git a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
index 8c350166..bd1e2919 100644
--- a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
+++ b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
@@ -1,23 +1,30 @@
diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp
--- a/llama.cpp/vendor/cpp-httplib/httplib.cpp
+++ b/llama.cpp/vendor/cpp-httplib/httplib.cpp
-@@ -6257,6 +6257,15 @@ void ThreadPool::cleanup_finished_threads() {
+@@ -6291,6 +6291,22 @@ void ThreadPool::cleanup_finished_threads() {
}
void ThreadPool::worker(bool is_dynamic) {
+#ifndef _WIN32
-+ // Block SIGINT/SIGTERM on pool threads to prevent EINTR during
-+ // condition_variable waits. Signals are handled by the main thread.
++ // Block every signal on this thread so pthread_cond_timedwait() can't
++ // be interrupted with EINTR (libcxx rethrows EINTR as std::system_error
++ // and std::terminates the process). Keep synchronous fault signals
++ // deliverable so real crashes still surface.
+ sigset_t block_mask;
-+ sigemptyset(&block_mask);
-+ sigaddset(&block_mask, SIGINT);
-+ sigaddset(&block_mask, SIGTERM);
++ sigfillset(&block_mask);
++ sigdelset(&block_mask, SIGSEGV);
++ sigdelset(&block_mask, SIGBUS);
++ sigdelset(&block_mask, SIGILL);
++ sigdelset(&block_mask, SIGFPE);
++ sigdelset(&block_mask, SIGABRT);
++ sigdelset(&block_mask, SIGSYS);
++ sigdelset(&block_mask, SIGTRAP);
+ pthread_sigmask(SIG_BLOCK, &block_mask, nullptr);
+#endif
for (;;) {
std::function<void()> fn;
{
-@@ -6274,7 +6283,13 @@ void ThreadPool::worker(bool is_dynamic) {
+@@ -6308,7 +6324,13 @@ void ThreadPool::worker(bool is_dynamic) {
break;
}
} else {
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index eb2dc2bd..00d48f21 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -223,7 +223,9 @@ LLAMAFILE_OBJS := \
LLAMAFILE_HIGHLIGHT_GPERF_FILES := $(wildcard llamafile/highlight/*.gperf)
LLAMAFILE_HIGHLIGHT_KEYWORDS := $(LLAMAFILE_HIGHLIGHT_GPERF_FILES:%.gperf=o/$(MODE)/%.o)
-# Server objects for llamafile
+# Server objects for llamafile. server-http.cpp references
+# llama_ui_find_asset, which lives in the generated ui.cpp produced by
+# llama.cpp/BUILD.mk's web UI block (see UI_GEN_OBJ).
LLAMAFILE_SERVER_SUPPORT_OBJS := \
o/$(MODE)/llama.cpp/tools/server/server-chat.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-common.cpp.o \
@@ -232,7 +234,8 @@ LLAMAFILE_SERVER_SUPPORT_OBJS := \
o/$(MODE)/llama.cpp/tools/server/server-models.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-queue.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-task.cpp.o \
- o/$(MODE)/llama.cpp/tools/server/server-tools.cpp.o
+ o/$(MODE)/llama.cpp/tools/server/server-tools.cpp.o \
+ $(UI_GEN_OBJ)
# Metal source files to embed in the executable (for runtime compilation on macOS)
# These are extracted at runtime and compiled into ggml-metal.dylib
@@ -293,7 +296,7 @@ LLAMAFILE_SERVER_INCS := \
-iquote o/$(MODE)/llama.cpp/tools/server
# Compile server.cpp
-o/$(MODE)/llamafile/server.cpp.o: llama.cpp/tools/server/server.cpp $(SERVER_ASSETS)
+o/$(MODE)/llamafile/server.cpp.o: llama.cpp/tools/server/server.cpp
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) $(LLAMAFILE_CPPFLAGS) $(LLAMAFILE_SERVER_INCS) -DLLAMA_BUILD_WEBUI -c -o $@ $<
@@ -310,8 +313,7 @@ o/$(MODE)/llamafile/llamafile: \
o/$(MODE)/llamafile/main.o \
o/$(MODE)/llamafile/server.cpp.o \
$(LLAMAFILE_OBJS) \
- $(LLAMAFILE_DEPS) \
- $(SERVER_ASSETS)
+ $(LLAMAFILE_DEPS)
@mkdir -p $(@D)
$(CXX) $(LDFLAGS) -o $@ $(filter %.o,$^) $(LDLIBS)
diff --git a/llamafile/cuda.bat b/llamafile/cuda.bat
index ba940c77..6c773259 100644
--- a/llamafile/cuda.bat
+++ b/llamafile/cuda.bat
@@ -137,12 +137,22 @@ if "%MINIMAL_ARCHS%"=="1" (
set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_90,code=sm_90"
)
-:: Detect CUDA version for Blackwell and compress support
-:: nvcc prints e.g. "Cuda compilation tools, release 13.2, V13.2.51".
-:: Splitting on ',', ' ', and '.' produces tokens 4..6 = "release", "13", "2".
-set "CUDA_MAJOR="
-set "CUDA_MINOR="
-for /f "tokens=5,6 delims=, ." %%a in ('nvcc --version 2^>nul ^| findstr /r "release [0-9]"') do (
+:: Detect CUDA version for Blackwell and compress support.
+:: nvcc emits e.g. "Cuda compilation tools, release 13.2, V13.2.51".
+:: Parse in flat single-char-delim steps. A previous attempt that used
+:: `delims=, .` (comma+space+dot together, with `.` right before the
+:: closing quote) errors with `." was unexpected at this time.` on the
+:: cmd.exe shipped with at least some Windows builds, so the loop never
+:: runs and the version falls back to 0. `/b /c:` anchors findstr to the
+:: literal version line — without /c:, the space in the pattern would
+:: split it into two OR'd regex patterns.
+set "CUDA_MAJOR=0"
+set "CUDA_MINOR=0"
+set "_REL_LINE="
+set "_REL_VER="
+for /f "tokens=2 delims=," %%a in ('nvcc --version 2^>nul ^| findstr /b /c:"Cuda compilation tools"') do set "_REL_LINE=%%a"
+for /f "tokens=2" %%a in ("!_REL_LINE!") do set "_REL_VER=%%a"
+for /f "tokens=1,2 delims=." %%a in ("!_REL_VER!") do (
set "CUDA_MAJOR=%%a"
set "CUDA_MINOR=%%b"
)
@@ -150,10 +160,6 @@ if "%CUDA_MAJOR%"=="13" (
set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_120f,code=sm_120f"
)
-:: Fall back to 0 if version parsing failed, to keep the numeric comparisons below valid.
-if not defined CUDA_MAJOR set "CUDA_MAJOR=0"
-if not defined CUDA_MINOR set "CUDA_MINOR=0"
-
:: --compress-mode=size: opt-in via --compress (or --minimize-size). Requires CUDA >= 12.8.
if "%COMPRESS%"=="1" (
set "COMPRESS_OK=0"
diff --git a/llamafile/cuda_parallel.bat b/llamafile/cuda_parallel.bat
index a8891f0b..104fbfdc 100644
--- a/llamafile/cuda_parallel.bat
+++ b/llamafile/cuda_parallel.bat
@@ -160,12 +160,22 @@ if "%MINIMAL_ARCHS%"=="1" (
set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_90,code=sm_90"
)
-:: Detect CUDA version for Blackwell and compress support
-:: nvcc prints e.g. "Cuda compilation tools, release 13.2, V13.2.51".
-:: Splitting on ',', ' ', and '.' produces tokens 4..6 = "release", "13", "2".
-set "CUDA_MAJOR="
-set "CUDA_MINOR="
-for /f "tokens=5,6 delims=, ." %%a in ('nvcc --version 2^>nul ^| findstr /r "release [0-9]"') do (
+:: Detect CUDA version for Blackwell and compress support.
+:: nvcc emits e.g. "Cuda compilation tools, release 13.2, V13.2.51".
+:: Parse in flat single-char-delim steps. A previous attempt that used
+:: `delims=, .` (comma+space+dot together, with `.` right before the
+:: closing quote) errors with `." was unexpected at this time.` on the
+:: cmd.exe shipped with at least some Windows builds, so the loop never
+:: runs and the version falls back to 0. `/b /c:` anchors findstr to the
+:: literal version line — without /c:, the space in the pattern would
+:: split it into two OR'd regex patterns.
+set "CUDA_MAJOR=0"
+set "CUDA_MINOR=0"
+set "_REL_LINE="
+set "_REL_VER="
+for /f "tokens=2 delims=," %%a in ('nvcc --version 2^>nul ^| findstr /b /c:"Cuda compilation tools"') do set "_REL_LINE=%%a"
+for /f "tokens=2" %%a in ("!_REL_LINE!") do set "_REL_VER=%%a"
+for /f "tokens=1,2 delims=." %%a in ("!_REL_VER!") do (
set "CUDA_MAJOR=%%a"
set "CUDA_MINOR=%%b"
)
@@ -173,10 +183,6 @@ if "%CUDA_MAJOR%"=="13" (
set "ARCH_FLAGS=!ARCH_FLAGS! -gencode arch=compute_120f,code=sm_120f"
)
-:: Fall back to 0 if version parsing failed, to keep the numeric comparisons below valid.
-if not defined CUDA_MAJOR set "CUDA_MAJOR=0"
-if not defined CUDA_MINOR set "CUDA_MINOR=0"
-
:: --compress-mode=size: opt-in via --compress (or --minimize-size). Requires CUDA >= 12.8.
if "%COMPRESS%"=="1" (
set "COMPRESS_OK=0"
diff --git a/llamafile/llamafile.c b/llamafile/llamafile.c
index 90eb3521..48c3f272 100644
--- a/llamafile/llamafile.c
+++ b/llamafile/llamafile.c
@@ -16,6 +16,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#define _COSMO_SOURCE // exposes cosmo extensions like makedirs()
+
#include "llamafile.h"
#include "version.h"
#include "zip.h"
@@ -648,46 +650,6 @@ bool llamafile_file_exists(const char *path) {
return !stat(path, &st);
}
-/**
- * Creates directories recursively, like `mkdir -p`.
- * Returns 0 on success, -1 on error.
- */
-int llamafile_makedirs(const char *path, int mode) {
- char tmp[PATH_MAX];
- char *p = NULL;
- size_t len;
-
- snprintf(tmp, sizeof(tmp), "%s", path);
- len = strlen(tmp);
-
- if (tmp[len - 1] == '/')
- tmp[len - 1] = '\0';
-
- if (mkdir(tmp, mode) == 0)
- return 0;
-
- if (errno == EEXIST) {
- struct stat st;
- if (stat(tmp, &st) == 0 && S_ISDIR(st.st_mode))
- return 0;
- return -1;
- }
-
- if (errno != ENOENT)
- return -1;
-
- for (p = tmp + 1; *p; p++) {
- if (*p == '/') {
- *p = '\0';
- if (mkdir(tmp, mode) != 0 && errno != EEXIST)
- return -1;
- *p = '/';
- }
- }
-
- return mkdir(tmp, mode);
-}
-
// Probe `path`, and if it exists, hand it to link_fn. Logs both the probe
// and the success at INFO level. Returns true only if link_fn loaded it.
static bool try_link_dso_from(const char *path, const char *backend_name,
@@ -739,7 +701,12 @@ bool llamafile_try_load_prebuilt_dso(const char *name, const char *backend_name,
snprintf(dso, PATH_MAX, "/zip/%s", name);
if (llamafile_file_exists(dso)) {
llamafile_get_app_dir(app_dir, PATH_MAX);
- if (llamafile_makedirs(app_dir, 0755) != 0) {
+ // Use cosmo's makedirs(), which recurses back-to-front via dirname()
+ // and stops at the first existing ancestor. That matters on Windows: a
+ // front-to-back mkdir -p of e.g. "/C/Users/Shadow/..." would try to
+ // mkdir("/C") (the drive root C:\), which fails with EACCES and looks
+ // like a spurious "Permission denied" when extracting bundled GPU DSOs.
+ if (makedirs(app_dir, 0755) != 0) {
perror(app_dir);
return false;
}
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index b3b45410..6a91438e 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -83,7 +83,6 @@ int llamafile_is_file_newer_than(const char *, const char *);
// Common utilities for GPU backend loaders (defined in llamafile.c)
const char *llamafile_get_dso_extension(void);
bool llamafile_file_exists(const char *);
-int llamafile_makedirs(const char *, int);
// Link function type for TryLoadPrebuiltDso
typedef bool (*llamafile_link_dso_fn)(const char *dso_path);
diff --git a/llamafile/main.cpp b/llamafile/main.cpp
index bfdb0bd3..c5fb79b5 100644
--- a/llamafile/main.cpp
+++ b/llamafile/main.cpp
@@ -35,6 +35,7 @@
#include "version.h"
#include <cstdio>
+#include <cstdlib>
#include <cstring>
#include <functional>
#include <mutex>
@@ -42,10 +43,59 @@
#include <vector>
#include <condition_variable>
+#include <signal.h>
+#include <string.h>
+#include <unistd.h>
+
#ifdef COSMOCC
#include <cosmo.h>
#endif
+namespace {
+
+// Diagnostic signal logger. Installed before any worker thread spawns so
+// we can see what signal is being delivered when something interrupts a
+// cv.wait_for() and trips libcxx's EINTR throw. Async-signal-safe:
+// touches only signal-safe writes and a fixed-size buffer on the stack.
+void diag_signal_log(int sig, siginfo_t * /*info*/, void * /*ctx*/) {
+ char buf[128];
+ const char *name = strsignal(sig);
+ int n = snprintf(buf, sizeof(buf),
+ "[diag-signal] sig=%d (%s) tid=%lu\n",
+ sig, name ? name : "?", (unsigned long)pthread_self());
+ if (n > 0) {
+ if ((size_t)n > sizeof(buf)) n = sizeof(buf);
+ (void)write(STDERR_FILENO, buf, (size_t)n);
+ }
+}
+
+void install_diag_signal_handlers() {
+ struct sigaction sa;
+ memset(&sa, 0, sizeof(sa));
+ sa.sa_sigaction = diag_signal_log;
+ sa.sa_flags = SA_SIGINFO | SA_RESTART;
+ sigemptyset(&sa.sa_mask);
+
+ // Every async signal we plausibly receive but never installed a
+ // handler for. SIGINT/SIGTERM are intentionally left alone — those
+ // already have a graceful-shutdown handler upstream and we don't
+ // want to clobber it.
+ int sigs[] = {
+ SIGCHLD, SIGPIPE, SIGURG, SIGIO, SIGALRM,
+ SIGHUP, SIGUSR1, SIGUSR2, SIGWINCH,
+#ifdef SIGINFO
+ SIGINFO,
+#endif
+ SIGCONT, SIGTSTP, SIGTTIN, SIGTTOU,
+ SIGXCPU, SIGXFSZ, SIGVTALRM, SIGPROF,
+ };
+ for (size_t i = 0; i < sizeof(sigs)/sizeof(sigs[0]); ++i) {
+ sigaction(sigs[i], &sa, nullptr);
+ }
+}
+
+} // namespace
+
// Forward declarations
extern int server_main(int argc, char **argv,
std::function<void(const std::string &)> on_ready,
@@ -232,6 +282,16 @@ int main(int argc, char **argv) {
return 0;
}
+ // Diagnostic: when LLAMAFILE_TRACE_SIGNALS=1 is set, log every
+ // unexpected async signal we receive. Useful for tracking down
+ // future EINTR-style crashes (see the widened pthread_sigmask in
+ // common/log.cpp, server-queue.cpp, server-models.cpp, and
+ // vendor/cpp-httplib/httplib.cpp). Off by default so normal runs
+ // stay quiet on terminal resizes (SIGWINCH) and similar.
+ if (getenv("LLAMAFILE_TRACE_SIGNALS")) {
+ install_diag_signal_handlers();
+ }
+
// Parse llamafile arguments and determine execution mode
// This also handles GPU initialization via llamafile_early_gpu_init()
lf::LlamafileArgs args = lf::parse_llamafile_args(argc, argv);
diff --git a/llamafile/metal.c b/llamafile/metal.c
index 0f1a7187..933ddaf0 100644
--- a/llamafile/metal.c
+++ b/llamafile/metal.c
@@ -30,6 +30,8 @@
// cannot resolve symbols from the parent process.
//
+#define _COSMO_SOURCE // exposes cosmo extensions like makedirs()
+
#include "llamafile.h"
#include <cosmo.h>
#include <dlfcn.h>
@@ -301,7 +303,7 @@ static bool BuildMetal(const char *dso) {
}
// Create app directory
- if (llamafile_makedirs(app_dir, 0755) != 0) {
+ if (makedirs(app_dir, 0755) != 0) {
perror(app_dir);
return false;
}
@@ -317,7 +319,7 @@ static bool BuildMetal(const char *dso) {
size_t parent_len = last_slash - src;
memcpy(parent_dir, src, parent_len);
parent_dir[parent_len] = '\0';
- llamafile_makedirs(parent_dir, 0755);
+ makedirs(parent_dir, 0755);
}
switch (llamafile_is_file_newer_than(metal_srcs[i].zip, src)) {
diff --git a/llamafile/tinyblas-compat.h b/llamafile/tinyblas-compat.h
index 7a9ee7d8..32b5085e 100644
--- a/llamafile/tinyblas-compat.h
+++ b/llamafile/tinyblas-compat.h
@@ -58,6 +58,7 @@
#define cublasSetStream tinyblasSetStream
#define cublasGetStream tinyblasGetStream
#define cublasSgemm tinyblasSgemm
+#define cublasSgemmStridedBatched tinyblasSgemmStridedBatched
#define cublasGemmEx tinyblasGemmEx
#define cublasGemmBatchedEx tinyblasGemmBatchedEx
#define cublasGemmStridedBatchedEx tinyblasGemmStridedBatchedEx
diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cu
index bbf12df5..758f68ad 100644
--- a/llamafile/tinyblas.cu
+++ b/llamafile/tinyblas.cu
@@ -471,6 +471,27 @@ tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t handle, tinyblasOperation_t tran
TINYBLAS_GEMM_DEFAULT);
}
+// Strided-batched SGEMM. There is no dedicated batched kernel: each batch is
+// an independent SGEMM at a fixed pointer stride, so launch batchCount of them
+// in sequence (serialized on the handle's stream). OUT_PROD is the only caller
+// (ggml-cuda/out-prod.cu) and is off the typical inference hot path, so the
+// per-matrix launch overhead is acceptable.
+tinyblasStatus_t tinyblasSgemmStridedBatched(tinyblasHandle_t handle, tinyblasOperation_t transa,
+ tinyblasOperation_t transb, int m, int n, int k,
+ const float *alpha, const float *A, int lda,
+ long long strideA, const float *B, int ldb,
+ long long strideB, const float *beta, float *C,
+ int ldc, long long strideC, int batchCount) {
+ for (int i = 0; i < batchCount; ++i) {
+ tinyblasStatus_t status =
+ tinyblasSgemm(handle, transa, transb, m, n, k, alpha, A + i * strideA, lda,
+ B + i * strideB, ldb, beta, C + i * strideC, ldc);
+ if (status != TINYBLAS_STATUS_SUCCESS)
+ return status;
+ }
+ return TINYBLAS_STATUS_SUCCESS;
+}
+
template <int CONFIG, int BM, int BN, int BK, int VE, int WM, int WN, int WNI, int TM, int TN,
int TT, typename WORD, typename SRC, typename DST>
static __global__ void __launch_bounds__(TT) tinyblasGE_entry(tinyblasOperation_t aT, //
diff --git a/llamafile/tinyblas.h b/llamafile/tinyblas.h
index 6fb6214a..54e189c6 100644
--- a/llamafile/tinyblas.h
+++ b/llamafile/tinyblas.h
@@ -60,6 +60,12 @@ tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t, tinyblasOperation_t, tinyblasOp
int, const float *, const float *, int, const float *, int,
const float *, float *, int);
+tinyblasStatus_t tinyblasSgemmStridedBatched(tinyblasHandle_t, tinyblasOperation_t,
+ tinyblasOperation_t, int, int, int, const float *,
+ const float *, int, long long, const float *, int,
+ long long, const float *, float *, int, long long,
+ int);
+
tinyblasStatus_t tinyblasStrsmBatched(tinyblasHandle_t, tinyblasSideMode_t, tinyblasFillMode_t,
tinyblasOperation_t, tinyblasDiagType_t, int, int,
const float *, const float *const[], int,