Skip to content

LLM inference in C/C++

C++ ggml

hexagon: support for multi-NPU devices (IQ9, IQ10) and fully asynchronous backend (#26501)

Max Krasnyansky192067b72d1b
diff --git a/docs/backend/snapdragon/CMakeUserPresets.json b/docs/backend/snapdragon/CMakeUserPresets.json
index 848d735f1c..afc7392323 100644
--- a/docs/backend/snapdragon/CMakeUserPresets.json
+++ b/docs/backend/snapdragon/CMakeUserPresets.json
@@ -8,7 +8,7 @@
"toolset": { "value": "host=x86_64", "strategy": "external" },
"cacheVariables": {
"ANDROID_ABI": "arm64-v8a",
- "ANDROID_PLATFORM": "android-31",
+ "ANDROID_PLATFORM": "android-34",
"CMAKE_TOOLCHAIN_FILE": "$env{ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake",
"CMAKE_C_FLAGS": "-march=armv8.7a+fp16+dotprod+i8mm -fvectorize -ffp-model=fast -fno-finite-math-only -flto -D_GNU_SOURCE",
"CMAKE_CXX_FLAGS": "-march=armv8.7a+fp16+dotprod+i8mm -fvectorize -ffp-model=fast -fno-finite-math-only -flto -D_GNU_SOURCE",
diff --git a/docs/backend/snapdragon/README.md b/docs/backend/snapdragon/README.md
index e9f0e21585..391c8bf230 100644
--- a/docs/backend/snapdragon/README.md
+++ b/docs/backend/snapdragon/README.md
@@ -2,39 +2,47 @@
## Setup
-### Android
+The cross-compilation toolchain images are provided by the
+[Qualcomm Snapdragon Toolchain registry](https://github.com/snapdragon-toolchain).
+These Docker images include the Android NDK, OpenCL SDK, Hexagon SDK, CMake, and the necessary cross-compilers:
-The easiest way to build llama.cpp for a Snapdragon-based Android device is using the toolchain Docker image (see github.com/snapdragon-toolchain).
-This image includes Android NDK, OpenCL SDK, Hexagon SDK, CMake, etc.
+* **Android toolchain**: `ghcr.io/snapdragon-toolchain/arm64-android:v0.7`
+* **Linux toolchain**: `ghcr.io/snapdragon-toolchain/arm64-linux:v0.7`
-This method works on Linux, macOS, and Windows. macOS and Windows users should install Docker Desktop.
+The unified build utility (`scripts/snapdragon/build.py`) automatically pulls
+and orchestrates these containers to perform target compilation.
+You only need to ensure that Docker (or Docker Desktop on macOS/Windows) is running on your host machine.
+Specific setup, build, and installation details for Linux and Windows on Snapdragon platforms are documented in:
+* [Linux on Snapdragon guide](linux.md)
+* [Windows on Snapdragon guide](windows.md)
-```
-~/src/llama.cpp$ docker run -it -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-android:v0.7
-[d]/> cd /workspace
-```
+## How to Build
+
+### Using build.py script (Recommended)
-Note: The rest of the **Android** build process assumes that you're running inside the toolchain container.
+The easiest way to build llama.cpp is by using the `scripts/snapdragon/build.py` script. It automatically copies the CMake presets,
+launches the correct compilation Docker container, builds the libraries and tools,
+installs them, and optionally pushes them to your ADB device.
-### Windows On Snapdragon
+Build and deploy for Android target (accepts `android` or `adb` alias):
+```
+$ ./scripts/snapdragon/build.py --target adb --push
+```
-Native Windows 11 arm64 builds has the following tools dependencies:
-- MS Visual Studio 2026 (Community Edition or Pro)
- - MSVC arm64 standard and runtime libraries
- - UCRT and Driver Kit
-- LLVM core libraries and Clang compiler (winget)
-- CMake, Git, Python (winget)
-- Hexagon SDK Community Edition 6.6 or later (see windows.md)
-- OpenCL SDK 2.3 or later (see windows.md)
+Build and deploy for Linux target (accepts `linux` or `lnx` alias):
+```
+$ ./scripts/snapdragon/build.py --target linux:user@host --push
+```
-Note: The rest of the **Windows** build process assumes that you're running natively in Powershell.
-Adapt below build commands accordingly.
+### Manual CMake Build
-## How to Build
+Alternatively, you can build llama.cpp manually by entering the cross-compilation Docker container and running the CMake commands:
-Let's build llama.cpp with CPU, OpenCL, and Hexagon backends via CMake presets:
+```bash
+# Start the cross-compilation container manually:
+~/src/llama.cpp$ docker run -it --rm -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-android:v0.7
-```
+# Inside the container, build the project using presets:
[d]/workspace> cp docs/backend/snapdragon/CMakeUserPresets.json .
[d]/workspace> cmake --preset arm64-android-snapdragon-release -B build-snapdragon
@@ -68,19 +76,19 @@ Preset CMake variables:
To generate an installable "package" simply use cmake --install:
```
-[d]/workspace> cmake --install build-snapdragon --prefix pkg-snapdragon/llama.cpp
+[d]/workspace> cmake --install build-snapdragon --prefix pkg-android/llama.cpp
-- Install configuration: "Release"
--- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-cpu.so
--- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-opencl.so
--- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-hexagon.so
--- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-htp-v73.so
--- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-htp-v75.so
--- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-htp-v79.so
--- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml-htp-v81.so
--- Installing: /workspace/pkg-snapdragon/llama.cpp/lib/libggml.so
+-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-cpu.so
+-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-opencl.so
+-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-hexagon.so
+-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-htp-v73.so
+-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-htp-v75.so
+-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-htp-v79.so
+-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml-htp-v81.so
+-- Installing: /workspace/pkg-android/llama.cpp/lib/libggml.so
...
--- Installing: /workspace/pkg-snapdragon/llama.cpp/bin/llama-bench
--- Installing: /workspace/pkg-snapdragon/llama.cpp/bin/llama-cli
+-- Installing: /workspace/pkg-android/llama.cpp/bin/llama-bench
+-- Installing: /workspace/pkg-android/llama.cpp/bin/llama-cli
...
```
@@ -91,14 +99,14 @@ To generate an installable "package" simply use cmake --install:
For this step, your device needs to be configured for on-device development.
Please see https://developer.android.com/studio/debug/dev-options for details.
-Once ADB is enabled, use `adb push` to install `pkg-snapdragon` on the device.
+Once ADB is enabled, use `adb push` to install `pkg-android` on the device.
**Note that the toolchain Docker image doesn't have ADB and doesn't set up the ADB bridge. Please use native ADB on the host.**
```
-~/src/llama.cpp$ adb push pkg-snapdragon/llama.cpp /data/local/tmp/
-pkg-snapdragon/llama.cpp/bin/: 67 files pushed, 0 skipped. 190.2 MB/s (919095042 bytes in 4.607s)
-pkg-snapdragon/llama.cpp/include/: 19 files pushed, 0 skipped. 20.5 MB/s (255173 bytes in 0.012s)
-pkg-snapdragon/llama.cpp/lib/: 16 files pushed, 0 skipped. 144.4 MB/s (43801382 bytes in 0.289s)
+~/src/llama.cpp$ adb push pkg-android/llama.cpp /data/local/tmp/
+pkg-android/llama.cpp/bin/: 67 files pushed, 0 skipped. 190.2 MB/s (919095042 bytes in 4.607s)
+pkg-android/llama.cpp/include/: 19 files pushed, 0 skipped. 20.5 MB/s (255173 bytes in 0.012s)
+pkg-android/llama.cpp/lib/: 16 files pushed, 0 skipped. 144.4 MB/s (43801382 bytes in 0.289s)
102 files pushed, 0 skipped. 186.9 MB/s (963151597 bytes in 4.914s)
```
@@ -115,24 +123,44 @@ Llama-3.2-1B-Instruct-Q4_0.gguf: 1 file pushed, 0 skipped. 38.3 MB/s (773025920
### Windows
-All artifacts are already installed in the `pkg-snapdragon` folder.
-To run, adapt below instructions to use Powershell scripts in `scripts/snapdragon/windows`.
+All artifacts are already installed in the `pkg-wos` folder.
+To run, you can use the `scripts/snapdragon/run.py` runner script (see details below).
## How to Run
-The easiest way to run llama.cpp cli tools is using provided wrapper scripts that properly set up all required environment variables.
+The easiest way to run llama.cpp cli tools is using the provided `scripts/snapdragon/run.py` wrapper script. This script automatically
+maps CLI options to environment variables, resolves executable paths, and runs the command locally, via ADB, or remotely via SSH on the
+target device.
-llama.cpp supports three backends on Snapdragon-based devices: CPU, Adreno GPU (GPUOpenCL), and Hexagon NPU (HTP0-4).
-You can select which backend to run the model on using the `D=` variable, which maps to the `--device` option.
+llama.cpp supports three backends on Snapdragon-based devices: CPU, Adreno GPU (GPUOpenCL), and Hexagon NPU.
+You can select which backend(s) to run the model on using the `--device` option of the tool (or `--devices` option in `run.py`).
Hexagon NPU behaves as a "GPU" device when it comes to `-ngl` and other offload-related options.
-Here are some examples of running various llama.cpp tools via ADB.
+Here are some examples of running various llama.cpp tools.
-Simple question for Llama-3.2-1B
+Generating a completion with Gemma on Android (relying on default `HTP0:0` device and default thread count `-t 6`):
```
-~/src/llama.cpp$ M=Llama-3.2-1B-Instruct-Q4_0.gguf D=HTP0 ./scripts/snapdragon/adb/run-completion.sh -p "what is the most popular cookie in the world?"
+~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb -- llama-completion -m models/gemma-2-2b-it-Q4_0.gguf -f prompts/sample_prompt_1024.txt --jinja -st
+...
+ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev 1
+ggml-hex: Hexagon Arch version v79
+ggml-hex: allocating new session: HTP0:0
+...
+load_tensors: offloading output layer to GPU
+load_tensors: offloaded 27/27 layers to GPU
+load_tensors: CPU model buffer size = 300.00 MiB
+load_tensors: HTP0:0 model buffer size = 1400.26 MiB
+...
+llama_perf_context_print: prompt eval time = 320.00 ms / 1024 tokens ( 0.31 ms per token, 3200.00 tokens per second)
+llama_perf_context_print: eval time = 2100.00 ms / 100 runs ( 21.00 ms per token, 47.62 tokens per second)
+```
+
+Simple question for Llama-3.2-1B:
+
+```
+~/src/llama.cpp$ ./scripts/snapdragon/run.py --target android --devices HTP0 -- llama-cli -m Llama-3.2-1B-Instruct-Q4_0.gguf -p "what is the most popular cookie in the world?"
...
ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev 1
ggml-hex: Hexagon Arch version v79
@@ -142,8 +170,7 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v
load_tensors: offloading output layer to GPU
load_tensors: offloaded 17/17 layers to GPU
load_tensors: CPU model buffer size = 225.49 MiB
-load_tensors: HTP0 model buffer size = 0.26 MiB
-load_tensors: HTP0-REPACK model buffer size = 504.00 MiB
+load_tensors: HTP0 model buffer size = 504.26 MiB
...
I hope this helps you understand the world's most popular cookies! [end of text]
...
@@ -156,60 +183,25 @@ llama_perf_context_print: graphs reused = 473
llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted |
llama_memory_breakdown_print: | - HTP0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 |
llama_memory_breakdown_print: | - Host | 439 = 225 + 136 + 77 |
-llama_memory_breakdown_print: | - HTP0-REPACK | 504 = 504 + 0 + 0 |
-```
-
-Summary request for OLMoE-1B-7B. This is a large model that requires two HTP sessions/devices
-
-```
-~/src/llama.cpp$ M=OLMoE-1B-7B-0125-Instruct-Q4_0.gguf NDEV=2 D=HTP0,HTP1 ./scripts/snapdragon/adb/run-completion.sh -f surfing.txt
-...
-ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev 1
-ggml-hex: Hexagon Arch version v81
-ggml-hex: allocating new session: HTP0
-ggml-hex: allocating new session: HTP1
-...
-load_tensors: offloading output layer to GPU
-load_tensors: offloaded 17/17 layers to GPU
-load_tensors: CPU model buffer size = 143.86 MiB
-load_tensors: HTP1 model buffer size = 0.23 MiB
-load_tensors: HTP1-REPACK model buffer size = 1575.00 MiB
-load_tensors: HTP0 model buffer size = 0.28 MiB
-load_tensors: HTP0-REPACK model buffer size = 2025.00 MiB
-...
-llama_context: CPU output buffer size = 0.19 MiB
-llama_kv_cache: HTP1 KV buffer size = 238.00 MiB
-llama_kv_cache: HTP0 KV buffer size = 306.00 MiB
-llama_kv_cache: size = 544.00 MiB ( 8192 cells, 16 layers, 1/1 seqs), K (q8_0): 272.00 MiB, V (q8_0): 272.00 MiB
-llama_context: HTP0 compute buffer size = 15.00 MiB
-llama_context: HTP1 compute buffer size = 15.00 MiB
-llama_context: CPU compute buffer size = 24.56 MiB
-...
-llama_perf_context_print: prompt eval time = 1730.57 ms / 212 tokens ( 8.16 ms per token, 122.50 tokens per second)
-llama_perf_context_print: eval time = 5624.75 ms / 257 runs ( 21.89 ms per token, 45.69 tokens per second)
-llama_perf_context_print: total time = 7377.33 ms / 469 tokens
-llama_perf_context_print: graphs reused = 255
-llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted |
-llama_memory_breakdown_print: | - HTP0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 |
-llama_memory_breakdown_print: | - HTP1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 |
-llama_memory_breakdown_print: | - Host | 742 = 144 + 544 + 54 |
-llama_memory_breakdown_print: | - HTP1-REPACK | 1575 = 1575 + 0 + 0 |
-llama_memory_breakdown_print: | - HTP0-REPACK | 2025 = 2025 + 0 + 0 |
```
-Op test for MUL_MAT
+Op test for MUL_MAT:
```
-~/src/llama.cpp$ HB=0 ./scripts/snapdragon/adb/run-tool.sh test-backend-ops -b HTP0 -o MUL_MAT
+~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --hex-hostbuf 0 --devices HTP0:0 -- test-backend-ops -b HTP0:0 -o MUL_MAT
...
-Backend 2/3: HTP0
+Backend 2/3: HTP0:0
Device description: Hexagon
Device memory: 2048 MB (2048 MB free)
MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1): OK
MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1): OK
MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1): OK
+```
+
+Llama benchmark:
-~/src/llama.cpp-hexagon$ M=Llama-3.2-1B-Instruct-Q4_0.gguf ./scripts/snapdragon/adb/run-bench.sh -p 128 -n 64
+```
+~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --devices HTP0 -- llama-bench -p 128 -n 64 -m Llama-3.2-1B-Instruct-Q4_0.gguf
...
ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev 1
ggml-hex: Hexagon Arch version v79
@@ -219,15 +211,20 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v
| ---------------| ---------: | -----: | ---------- | --: | ------: | ------: | ---: | ----: | ------------: |
| llama 1B Q4_0 | 729.75 MiB | 1.24 B | HTP | 99 | 4 | 128 | 0 | pp128 | 169.42 ± 1.75 |
| llama 1B Q4_0 | 729.75 MiB | 1.24 B | HTP | 99 | 4 | 128 | 0 | tg64 | 51.54 ± 1.13 |
-
-build: 6a8cf8914 (6733)
```
## Environment variables
-- `GGML_HEXAGON_NDEV=1`
- Controls the number of devices/sessions to allocate. The default is 1.
- Most quantized models under 4B fit into a single session; an 8B model needs two, and a 20B model needs four.
+- `GGML_HEXAGON_DEVICES` (default: not set, defaults to HTP0 session)
+ Controls which NPU devices and sessions to allocate. Can be configured as:
+ - A single integer `N`: Allocates `N` sessions named `HTP0`, `HTP1`, ..., `HTP<N-1>` (behaves identically to `GGML_HEXAGON_NDEV=N`).
+ - A comma-separated list of device names in `HTP<physical_idx>:<virtual_idx>` format (or legacy `HTP<idx>` format). For example, `HTP0:0,HTP0:1` creates two virtual
+ sessions on the first physical NPU (useful for memory limits). `HTP0:0,HTP1:0` allocates one session on each of the two physical NPUs
+ on a dual-NPU device.
+
+- `GGML_HEXAGON_NDEV` (deprecated)
+ Replaced by `GGML_HEXAGON_DEVICES`. Controls the number of virtual sessions to allocate on physical NPU `0`.
+ Allocates sessions named `HTP0`, `HTP1`, etc.
- `GGML_HEXAGON_NHVX=0`
Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version).
@@ -255,26 +252,17 @@ build: 6a8cf8914 (6733)
- `2` Extended profile with per-op `usecs`, `cycles` and default PMU counter data
- `0x1,...,0x8` Extended profile with per-op `usecs`, `cycles` and custom PMU counter data
- The logging output can be either saved into a file for post-processing or it can be piped directly into the post-processing tool to generate the report.
+ The logging output can be either saved into a file for post-processing or it can be piped directly into the post-processing tool
+ to generate the report.
Examples:
- `GGML_HEXAGON_PROFILE=1 llama-completion ... |& ./scripts/snapdragon/ggml-hexagon-profile.py -`
-
-- `GGML_HEXAGON_OPSTAGE=0x0`
- Allows enabling specific stages of the Op processing pipeline:
-
- - `0x1` Enable Op Queue (i.e., queuing Ops into NPU)
- - `0x2` Enable Op Compute (MUL_MAT, etc.)
-
- Examples:
-
- `GGML_HEXAGON_OPSTAGE=0x1 llama-completion ...` - Ops are enqueued to the NPU but dma & compute are disabled
- `GGML_HEXAGON_OPSTAGE=0x3 llama-completion ...` - Full queuing and processing of Ops (default)
+ `GGML_HEXAGON_PROFILE=1 ./scripts/snapdragon/run.py --target adb -- llama-cli ... |& ./scripts/snapdragon/ggml-hexagon-profile.py -`
- `GGML_HEXAGON_OPFILTER=regex`
Allows filtering (disabling) Ops that match the regex pattern:
Examples:
- `GGML_HEXAGON_OPFILTER="FLASH_ATTN_EXT" llama-completion ...` - Disable Flash Attention on Hexagon (falls back to CPU or GPU)
- `GGML_HEXAGON_OPFILTER="ADD\|SUB" llama-completion ...` - Disable ADD and SUB on Hexagon (fall back to CPU or GPU)
+ `GGML_HEXAGON_OPFILTER="FLASH_ATTN_EXT" ./scripts/snapdragon/run.py --target adb -- llama-cli ...` - Disable Flash Attention on Hexagon (falls back to CPU or GPU)
+ `GGML_HEXAGON_OPFILTER="ADD\|SUB" ./scripts/snapdragon/run.py --target adb -- llama-cli ...` - Disable ADD and SUB on Hexagon (fall back to CPU or GPU)
+
diff --git a/docs/backend/snapdragon/developer.md b/docs/backend/snapdragon/developer.md
index 9d56638e3d..d7d9f2a279 100644
--- a/docs/backend/snapdragon/developer.md
+++ b/docs/backend/snapdragon/developer.md
@@ -39,22 +39,21 @@ the repacking.
## Large model handling
-Hexagon NPU session (aka Process Domain (PD) in the Hexagon docs) is limited to a memory mapping of around 3.5GB.
-In llama.cpp/GGML the Hexagon session is mapped to a single GGML backend device (HTP0, HTP1, etc).
+Hexagon NPU sessions (aka Process Domains (PD) in the Hexagon SDK) are limited to a maximum memory mapping window of around 3.5GB.
+In llama.cpp/GGML, each Hexagon session is mapped to a single GGML backend device (e.g., `HTP0:0`, `HTP0:1`, etc. when using
+`GGML_HEXAGON_DEVICES`, or `HTP0`, `HTP1` in legacy mode).
-In order to map models larger than 3.5GB we need to allocate multiple devices and split the model.
-For this we're taking advantage of the llama.cpp/GGML multi-GPU layer-splitting support.
-Each Hexagon device behaves like a GPU from the offload and model splitting perspective.
+To support running models larger than 3.5GB on a single device, the Hexagon backend dynamically maps and unmaps execution buffers
+during the graph execution cycle to stay within the Process Domain window. This enables large models to run successfully on a single
+NPU device.
-Here is an example of running GPT-OSS-20B model on a newer Snapdragon device with 16GB of DDR.
+Alternatively, users can choose to use standard llama.cpp/GGML layer-splitting mode to partition and split the model across
+multiple Hexagon devices or virtual sessions (which behave like multiple GPUs from the offload and splitting perspective).
+
+Here is an example of running GPT-OSS-20B model on a Snapdragon device using 4 virtual sessions on a single NPU (physical index 0).
```
-M=gpt-oss-20b-Q4_0.gguf NDEV=4 D=HTP0,HTP1,HTP2,HTP3 P=surfing.txt scripts/snapdragon/adb/run-completion.sh -f surfing.txt -n 32
-...
-LD_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib
-ADSP_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib
-GGML_HEXAGON_NDEV=4 ./bin/llama-cli --load-mode none -m /data/local/tmp/llama.cpp/../gguf/gpt-oss-20b-Q4_0.gguf
- -t 4 --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 --device HTP0,HTP1,HTP2,HTP3 -no-cnv -f surfing.txt
+~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --devices HTP0:0,HTP0:1,HTP0:2,HTP0:3 -- llama-cli --load-mode none -m /data/local/tmp/gguf/gpt-oss-20b-Q4_0.gguf -t 4 --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 -no-cnv -f surfing.txt
...
llama_model_loader: - type f32: 289 tensors
llama_model_loader: - type q4_0: 96 tensors
@@ -63,33 +62,29 @@ llama_model_loader: - type mxfp4: 72 tensors
...
load_tensors: offloaded 25/25 layers to GPU
load_tensors: CPU model buffer size = 1182.09 MiB
-load_tensors: HTP1 model buffer size = 6.64 MiB
-load_tensors: HTP1-REPACK model buffer size = 2505.94 MiB
-load_tensors: HTP3 model buffer size = 5.55 MiB
-load_tensors: HTP3-REPACK model buffer size = 2088.28 MiB
-load_tensors: HTP0 model buffer size = 7.75 MiB
-load_tensors: HTP0-REPACK model buffer size = 2923.59 MiB
-load_tensors: HTP2 model buffer size = 6.64 MiB
-load_tensors: HTP2-REPACK model buffer size = 2505.94 MiB
+load_tensors: HTP0:1 model buffer size = 2512.58 MiB
+load_tensors: HTP0:3 model buffer size = 2093.83 MiB
+load_tensors: HTP0:0 model buffer size = 2931.34 MiB
+load_tensors: HTP0:2 model buffer size = 2512.58 MiB
...
llama_context: n_ctx_per_seq (8192) < n_ctx_train (131072) -- the full capacity of the model will not be utilized
llama_context: CPU output buffer size = 0.77 MiB
llama_kv_cache_iswa: creating non-SWA KV cache, size = 8192 cells
-llama_kv_cache: HTP1 KV buffer size = 25.50 MiB
-llama_kv_cache: HTP3 KV buffer size = 25.50 MiB
-llama_kv_cache: HTP0 KV buffer size = 25.50 MiB
-llama_kv_cache: HTP2 KV buffer size = 25.50 MiB
+llama_kv_cache: HTP0:1 KV buffer size = 25.50 MiB
+llama_kv_cache: HTP0:3 KV buffer size = 25.50 MiB
+llama_kv_cache: HTP0:0 KV buffer size = 25.50 MiB
+llama_kv_cache: HTP0:2 KV buffer size = 25.50 MiB
llama_kv_cache: size = 102.00 MiB ( 8192 cells, 12 layers, 1/1 seqs), K (q8_0): 51.00 MiB, V (q8_0): 51.00 MiB
llama_kv_cache_iswa: creating SWA KV cache, size = 256 cells
-llama_kv_cache: HTP1 KV buffer size = 0.80 MiB
-llama_kv_cache: HTP3 KV buffer size = 0.53 MiB
-llama_kv_cache: HTP0 KV buffer size = 1.06 MiB
-llama_kv_cache: HTP2 KV buffer size = 0.80 MiB
+llama_kv_cache: HTP0:1 KV buffer size = 0.80 MiB
+llama_kv_cache: HTP0:3 KV buffer size = 0.53 MiB
+llama_kv_cache: HTP0:0 KV buffer size = 1.06 MiB
+llama_kv_cache: HTP0:2 KV buffer size = 0.80 MiB
llama_kv_cache: size = 3.19 MiB ( 256 cells, 12 layers, 1/1 seqs), K (q8_0): 1.59 MiB, V (q8_0): 1.59 MiB
-llama_context: HTP0 compute buffer size = 16.06 MiB
-llama_context: HTP1 compute buffer size = 16.06 MiB
-llama_context: HTP2 compute buffer size = 16.06 MiB
-llama_context: HTP3 compute buffer size = 16.06 MiB
+llama_context: HTP0:0 compute buffer size = 16.06 MiB
+llama_context: HTP0:1 compute buffer size = 16.06 MiB
+llama_context: HTP0:2 compute buffer size = 16.06 MiB
+llama_context: HTP0:3 compute buffer size = 16.06 MiB
llama_context: CPU compute buffer size = 98.19 MiB
...
llama_perf_context_print: prompt eval time = 3843.67 ms / 197 tokens ( 19.51 ms per token, 51.25 tokens per second)
@@ -97,13 +92,9 @@ llama_perf_context_print: eval time = 1686.13 ms / 31 runs ( 54.3
llama_perf_context_print: total time = 6266.30 ms / 228 tokens
llama_perf_context_print: graphs reused = 30
llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted |
-llama_memory_breakdown_print: | - HTP0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 |
-llama_memory_breakdown_print: | - HTP1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 |
-llama_memory_breakdown_print: | - HTP2 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 |
-llama_memory_breakdown_print: | - HTP3 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 |
+llama_memory_breakdown_print: | - HTP0:0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 |
+llama_memory_breakdown_print: | - HTP0:1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 |
+llama_memory_breakdown_print: | - HTP0:2 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 |
+llama_memory_breakdown_print: | - HTP0:3 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 |
llama_memory_breakdown_print: | - Host | 1476 = 1208 + 105 + 162 |
-llama_memory_breakdown_print: | - HTP1-REPACK | 2505 = 2505 + 0 + 0 |
-llama_memory_breakdown_print: | - HTP3-REPACK | 2088 = 2088 + 0 + 0 |
-llama_memory_breakdown_print: | - HTP0-REPACK | 2923 = 2923 + 0 + 0 |
-llama_memory_breakdown_print: | - HTP2-REPACK | 2505 = 2505 + 0 + 0 |
```
diff --git a/docs/backend/snapdragon/linux.md b/docs/backend/snapdragon/linux.md
index 90fdadb6c9..d4ecd9b1b1 100644
--- a/docs/backend/snapdragon/linux.md
+++ b/docs/backend/snapdragon/linux.md
@@ -1,25 +1,37 @@
# Snapdragon-based Linux devices
-## Docker Setup
+The cross-compilation is performed using the Snapdragon Linux Docker toolchain image (see
+[github.com/snapdragon-toolchain](https://github.com/snapdragon-toolchain)):
-The easiest way to build llama.cpp for a Snapdragon-based Linux device is using the toolchain Docker image (see [github.com/snapdragon-toolchain](https://github.com/snapdragon-toolchain)).
-This image includes OpenCL SDK, Hexagon SDK, CMake, and the ARM64 Linux cross-compilation toolchain.
+* **Linux toolchain**: `ghcr.io/snapdragon-toolchain/arm64-linux:v0.7`
-Cross-compilation is supported on **Linux X86** hosts. The resulting binaries are deployed to and run on the target **Qualcomm Snapdragon ARM64 Linux** device.
+The unified build utility (`scripts/snapdragon/build.py`) automatically pulls
+and orchestrates this container to perform target compilation. You only need to
+ensure that Docker is running on your host machine.
+
+## How to Build
+
+### Using build.py script (Recommended)
+
+The easiest way to build llama.cpp is by using the `scripts/snapdragon/build.py` script. It automatically copies the CMake presets,
+launches the correct compilation Docker container, builds the libraries and tools,
+installs them, and optionally pushes them to your target device.
+
+Build and deploy for a Linux target (using SSH deployment alias `lnx` or `linux`):
```
-~/src/llama.cpp$ docker run -it -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-linux:v0.1
-[d]/> cd /workspace
+$ ./scripts/snapdragon/build.py --target lnx:user@host --push
```
-Note: The rest of the **Linux** build process assumes that you're running inside the toolchain container.
+### Manual CMake Build
+Alternatively, you can build llama.cpp manually by entering the cross-compilation Docker container and running the CMake commands:
-## How to Build
+```bash
+# Start the cross-compilation container manually:
+~/src/llama.cpp$ docker run -it --rm -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-linux:v0.7
-Let's build llama.cpp with CPU, OpenCL, and Hexagon backends via CMake presets:
-
-```
+# Inside the container, build the project using presets:
[d]/workspace> cp docs/backend/snapdragon/CMakeUserPresets.json .
[d]/workspace> cmake --preset arm64-linux-snapdragon-release -B build-snapdragon
@@ -30,17 +42,19 @@ Let's build llama.cpp with CPU, OpenCL, and Hexagon backends via CMake presets:
To generate an installable "package" simply use cmake --install, then zip it:
```
-[d]/workspace> cmake --install build-snapdragon --prefix pkg-snapdragon
-[d]/workspace> zip -r pkg-snapdragon.zip pkg-snapdragon
+[d]/workspace> cmake --install build-snapdragon --prefix pkg-linux
+[d]/workspace> zip -r pkg-linux.zip pkg-linux
```
## How to Install
-For this step, you will deploy the built binaries and libraries to the target Linux device. Transfer `pkg-snapdragon.zip` to the target device, then unzip it and set up the environment variables:
+For this step, you will deploy the built binaries and libraries to the target
+Linux device. Transfer `pkg-linux.zip` to the target device, then unzip it
+and set up the environment variables:
```
-$ unzip pkg-snapdragon.zip
-$ cd pkg-snapdragon
+$ unzip pkg-linux.zip
+$ cd pkg-linux
$ export LD_LIBRARY_PATH=./lib
$ export ADSP_LIBRARY_PATH=./lib
```
@@ -52,7 +66,28 @@ $ wget https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF/resolve/main/
```
## How to Run
-Next, since we have setup the environment variables, we can run the llama-cli with the Hexagon backends:
+You can run locally on the Snapdragon Linux device:
+```
+$ ./scripts/snapdragon/run.py --devices HTP0 -- llama-cli -m Llama-3.2-3B-Instruct-Q4_0.gguf -ngl 99 -p "what is the most popular cookie in the world?"
+```
+
+Or run remotely from your host development machine using the SSH target option:
+```
+$ ./scripts/snapdragon/run.py --target lnx:user@host --devices HTP0 -- llama-cli -m Llama-3.2-3B-Instruct-Q4_0.gguf -ngl 99 -p "what is the most popular cookie in the world?"
+```
+
+For multi-NPU systems, you can run a tensor split completion command targeting a remote Linux system:
+```
+$ ./scripts/snapdragon/run.py --target ubuntu:[email protected] --device HTP0:0,HTP1:0 -- llama-completion -m models/gemma-2b-it-Q4_0.gguf -f prompts/sample_prompt_1024.txt --jinja -st --split-mode tensor --ctx-size 8192
+```
+
+This translates to the following command being executed remotely via SSH:
+```
++ ssh [email protected] "cd ~/llama.cpp && ulimit -c unlimited && LD_LIBRARY_PATH=./lib ADSP_LIBRARY_PATH=./lib GGML_HEXAGON_DEVICES=HTP0:0,HTP1:0 GGML_HEXAGON_OPPOLL=1 ./bin/llama-completion -m models/gemma-2b-it-Q4_0.gguf -f prompts/sample_prompt_1024.txt --jinja -st --split-mode tensor --ctx-size 8192 -v -n 16 --device HTP0:0,HTP1:0 -ngl 99 --ubatch-size 1024 -fa on -t 6"
+```
+
+Alternatively, you can run the binary directly on the device:
```
$ ./bin/llama-cli -m Llama-3.2-3B-Instruct-Q4_0.gguf --device HTP0 -ngl 99 -p "what is the most popular cookie in the world?"
```
+
diff --git a/docs/backend/snapdragon/windows.md b/docs/backend/snapdragon/windows.md
index aa731413c9..3f7d60dd95 100644
--- a/docs/backend/snapdragon/windows.md
+++ b/docs/backend/snapdragon/windows.md
@@ -1,3 +1,18 @@
+# Snapdragon-based Windows devices
+
+## Tool Dependencies
+
+Native Windows 11 arm64 builds have the following tool dependencies:
+- MS Visual Studio 2026 (Community Edition or Pro)
+ - MSVC arm64 standard and runtime libraries
+ - UCRT and Driver Kit
+- LLVM core libraries and Clang compiler (winget)
+- CMake, Git, Python (winget)
+- Hexagon SDK Community Edition 6.6 or later (see below)
+- OpenCL SDK 2.3 or later (see below)
+
+Note: The rest of the **Windows** build process assumes that you're running natively in Powershell.
+
## Overview
The document covers procedures for installing the latest GPU and NPU drivers, and OpenCL and Hexagon SDKs.
@@ -53,7 +68,8 @@ Download the driver from
https://softwarecenter.qualcomm.com/catalog/item/Qualcomm_HND
-After the automated installation and reboot please make sure that the Hexagon NPU device shows up in the `Device Manager` (under `Neural Processors`).
+After the automated installation and reboot please make sure that the Hexagon NPU device shows up in the `Device Manager`
+(under `Neural Processors`).
If the device is not available you can try installing all components (`qcnspmcdm8380`, `qcnspmcdm8380_ext`) manually.
The components are extracted into
@@ -130,12 +146,12 @@ However, additional settings are required for generating and signing HTP Ops lib
> cmake --preset arm64-windows-snapdragon-release -B build-wos
...
-> cmake --install build-wos --prefix pkg-snapdragon
+> cmake --install build-wos --prefix pkg-wos
```
Once the build is complete HTP ops libraries will be installed like this
```
-> dir pkg-snapdragon/lib
+> dir pkg-wos/lib
...
-a---- 1/22/2026 6:01 PM 187656 libggml-htp-v73.so
-a---- 1/22/2026 6:01 PM 191752 libggml-htp-v75.so
@@ -147,8 +163,8 @@ Once the build is complete HTP ops libraries will be installed like this
The .cat file, the signature and proper certificate installation can be verified with
```
-> signtool.exe verify /v /pa .\pkg-snapdragon\lib\libggml-htp.cat
-Verifying: .\pkg-snapdragon\lib\libggml-htp.cat
+> signtool.exe verify /v /pa .\pkg-wos\lib\libggml-htp.cat
+Verifying: .\pkg-wos\lib\libggml-htp.cat
Signature Index: 0 (Primary Signature)
Hash of file (sha256): 9820C664DA59D5EAE31DBB664127FCDAEF59CDC31502496BC567544EC2F401CF
@@ -156,6 +172,6 @@ Hash of file (sha256): 9820C664DA59D5EAE31DBB664127FCDAEF59CDC31502496BC567544EC
Signing Certificate Chain:
Issued to: GGML.HTP.v1
...
-Successfully verified: .\pkg-snapdragon\lib\libggml-htp.cat
+Successfully verified: .\pkg-wos\lib\libggml-htp.cat
...
```
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index e8a5009b38..c1e9f919d9 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -6,6 +6,7 @@
#include <time.h>
#include <atomic>
+#include <memory>
#include <chrono>
#include <mutex>
#include <thread>
@@ -18,6 +19,7 @@
#include <unordered_map>
#include <regex>
#include <queue>
+#include <deque>
#include <algorithm>
#ifdef _WIN32
@@ -52,6 +54,8 @@
#include "htp/matmul-ops.h"
#include "htp/flash-attn-ops.h"
#include "htp/unary-ops.h"
+#include "htp/get-rows-ops.h"
+#include "htp/set-rows-ops.h"
#include "htp_iface.h"
#include "htp-drv.h"
@@ -59,6 +63,36 @@ using intvec = std::vector<int>;
using uintvec = std::vector<unsigned int>;
using u32vec = std::vector<uint32_t>;
+#define GGML_HEXAGON_MAX_SESSIONS 16
+
+#define GGML_HEXAGON_FENCE_BUFFER_SIZE 8192
+#define GGML_HEXAGON_FENCE_SLOT_SIZE 128
+
+struct ggml_hexagon_device_config {
+ int physical_idx = 0;
+ int virtual_idx = 0;
+ std::string name;
+};
+
+static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS];
+
+static int get_domain_id(int physical_idx) {
+ switch (physical_idx) {
+ case 0: return 3; // CDSP0 (all devices)
+ case 1: return 4; // CDSP1 (IQ9, IQ10)
+ case 2: return 18; // CDSP2 (IQ10)
+ case 3: return 19; // CDSP3 (IQ10)
+ default: return CDSP_DOMAIN_ID + physical_idx;
+ }
+}
+
+static std::string get_domain_name(int physical_idx) {
+ if (physical_idx == 0) {
+ return CDSP_DOMAIN_NAME;
+ }
+ return std::string("cdsp") + std::to_string(physical_idx);
+}
+
static int opt_arch = 0; // autodetect
static size_t opt_ndev = 1;
static size_t opt_nhvx = 0; // use all
@@ -68,20 +102,19 @@ static size_t opt_mbuf = 1ul * 1024 * 1024 * 1024; // max buffer size
static int opt_etm = 0;
static int opt_verbose = 0;
static int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu)
-static int opt_hostbuf = 1; // hostbuf ON by default
+static bool opt_hostbuf = false;
static int opt_mm_select = 3; // 3 = HMX -> Tiled -> Flat -> CPU, 2 = Tiled -> Flat -> CPU, 1 = Flat -> CPU
static int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)
+static int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (DMA, default), 1 = unfused ALLREDUCE (DMA), 0 = fallback to CPY+FENCE
// Default PMU events, if profiling with PMU (mode=2) is enabled
// See https://docs.qualcomm.com/doc/80-N2040-60/topic/pmu-events.html
// https://docs.qualcomm.com/doc/80-N2040-61/topic/hvx-pmu-events.html
static u32vec opt_pmu_evt { 0x3, 0x111, 0x100, 0x105, 0x240, 0x256, 0x7D, 0x8C };
-// Enable all stages by default
-static int opt_opstage = HTP_OPSTAGE_QUEUE | HTP_OPSTAGE_COMPUTE;
static int opt_opbatch = 1024; // max number of ops in a batch
-static int opt_opqueue = 16; // max number of pending batches
+static int opt_opqueue = 64; // max number of pending batches
static int opt_optrace = 0; // trace buffer size per thread (0 means default)
static int opt_oppoll = 0; // polling for batch completions
static int opt_opfusion = 1; // enable/disable op fusion
@@ -121,7 +154,7 @@ static void ggml_hexagon_dump_op_exec(const std::string &sess_name, const htp_op
static void ggml_hexagon_dump_op_supp(const std::string &sess_name, const struct ggml_tensor * op, bool supp) {
if (!opt_verbose) return;
- htp_opformat fmt(htp_opformat(htp_opnode{const_cast<ggml_tensor*>(op), {}, HTP_OP_INVALID}));
+ htp_opformat fmt(htp_opformat(htp_opnode(HTP_OP_INVALID, const_cast<ggml_tensor*>(op))));
GGML_LOG_DEBUG("ggml-hex: %s supports-op %s|%s|%s|%s|%s|%s|%s\n", sess_name.c_str(),
ggml_op_desc(op), fmt.names, fmt.dims, fmt.types, fmt.strides, fmt.buffs, supp ? "yes" : "no");
}
@@ -144,6 +177,7 @@ static const char * htp_event_name(uint16_t id) {
case HTP_TRACE_EVT_L2FLUSH: return "L2FLUSH";
case HTP_TRACE_EVT_INIT: return "INIT";
case HTP_TRACE_EVT_BUFF: return "BUFF";
+ case HTP_TRACE_EVT_FENCE: return "FENCE";
default: return "UNKNOWN";
}
}
@@ -205,7 +239,12 @@ static void ggml_hexagon_dump_trace_events(const std::string & sess_name, const
}
}
-// **
+enum ggml_hexagon_tensor_flags {
+ GGML_HEXAGON_TENSOR_REPACK = (1 << 0),
+ GGML_HEXAGON_TENSOR_WEIGHT = (1 << 1),
+ GGML_HEXAGON_TENSOR_FENCE = (1 << 2),
+ GGML_HEXAGON_TENSOR_FUSEABLE = (1 << 3),
+};
static inline bool ggml_hexagon_is_repack_type(enum ggml_type type) {
return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 ||
@@ -227,6 +266,15 @@ static void ggml_hexagon_precompute_matmul_params(
struct htp_mm_kernel_params * kparams
);
+static void ggml_hexagon_precompute_fused_matmul_add_params(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * src2,
+ const struct ggml_tensor * dst,
+ struct htp_mm_kernel_params * kparams
+);
+
static void ggml_hexagon_precompute_unary_params(
const struct ggml_hexagon_session * sess,
uint32_t op,
@@ -236,25 +284,75 @@ static void ggml_hexagon_precompute_unary_params(
struct htp_unary_kernel_params * kparams
);
-static void ggml_hexagon_precompute_fused_qkv_params(
+static void ggml_hexagon_precompute_get_rows_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
- struct htp_mm_kernel_params * kparams
+ const struct ggml_tensor * dst,
+ struct htp_get_rows_kernel_params * kparams
);
-static void ggml_hexagon_precompute_fused_ffn_params(
+static void ggml_hexagon_precompute_set_rows_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ struct htp_set_rows_kernel_params * kparams
+);
+
+static void ggml_hexagon_precompute_fused_mmnx_params(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ int32_t n_weights,
struct htp_mm_kernel_params * kparams
);
+static bool ggml_hexagon_precompute_allreduce_params(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * dst,
+ uint32_t rank,
+ uint32_t n_ranks,
+ bool has_add,
+ bool is_row_bcast,
+ struct htp_allreduce_kernel_params * kparams
+);
+
+static bool mm_is_hmx_eligible(const ggml_tensor * t);
+static bool is_mergeable_mul_mat(const ggml_tensor * t);
+static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2);
+
// ** backend sessions
+struct ggml_hexagon_tensor_extra {
+ std::vector<uint8_t> shadow_buf;
+ size_t shadow_size { 0 };
+ uint32_t flags { 0 };
+};
+
+static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) {
+ if (!t || !t->extra) return false;
+ auto extra = (const struct ggml_hexagon_tensor_extra *) t->extra;
+ return (extra->flags & GGML_HEXAGON_TENSOR_FUSEABLE) != 0;
+}
+
+struct htp_opnode;
+
struct ggml_hexagon_opbatch;
struct ggml_hexagon_opqueue;
-struct htp_opnode;
+struct ggml_hexagon_shared_buffer;
+struct ggml_hexagon_session;
+
+struct ggml_backend_hexagon_comm_context {
+ std::vector<ggml_backend_t> backends;
+ size_t n_backends = 0;
+ uint32_t fence_seq = 0;
+};
+
+struct ggml_hexagon_event {
+ ggml_hexagon_session * sess = nullptr;
+ uint64_t seq = 0;
+};
struct ggml_hexagon_session {
std::string name;
@@ -264,6 +362,8 @@ struct ggml_hexagon_session {
uint32_t domain_id;
uint64_t queue_id;
int dev_id;
+ int phys_idx;
+ int virt_idx;
bool valid_session;
bool valid_handle;
bool valid_queue;
@@ -273,20 +373,24 @@ struct ggml_hexagon_session {
ggml_hexagon_opbatch* op_batch;
ggml_hexagon_opqueue* op_queue;
+ std::unordered_map<int, std::unique_ptr<ggml_hexagon_shared_buffer>> cloned_buffers;
+ std::unordered_set<ggml_hexagon_session *> sync_peers;
+
ggml_backend_buffer_type buffer_type = {};
- ggml_backend_buffer_type repack_buffer_type = {};
+ ggml_backend_buffer_type host_buffer_type = {};
- uint32_t n_threads = 0;
- uint32_t n_hvx = 0;
- uint32_t n_hmx = 0;
- uint64_t vtcm_size = 0;
- size_t max_vmem = 0;
+ uint32_t n_threads = 0;
+ uint32_t n_hvx = 0;
+ uint32_t n_hmx = 0;
+ uint64_t vtcm_size = 0;
+ size_t max_vmem = 0;
size_t max_bufsize = 0;
+ uint32_t fence_seq;
- struct {
- uint64_t uid = 0;
- std::vector<htp_opnode> htp_nodes;
- } cached_graph;
+ uint64_t cached_uid = 0;
+ std::vector<htp_opnode> cached_nodes;
+
+ mutable std::unordered_set<const ggml_tensor *> needs_repack;
ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false);
~ggml_hexagon_session() noexcept(true);
@@ -297,10 +401,31 @@ struct ggml_hexagon_session {
void release() noexcept(true);
void enqueue_op(const htp_opnode & node);
- void flush(bool all = true);
+ void enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor = nullptr, uint32_t fence_seq = 0);
+ void enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq = 0);
+ void enqueue_allreduce(const ggml_tensor * dst, const std::vector<const ggml_tensor *> & src_tensors, const std::vector<const ggml_tensor *> & sync_tensors, uint32_t rank, uint32_t n_ranks, uint32_t fence_seq_entry = 0, uint32_t fence_seq_exit = 0);
+ void flush(bool all = true);
void flush_pending(bool all = false);
- void flush_batch();
+ void flush_batch(size_t min_ops = 1);
+
+ uint64_t record_event();
+ void wait_event(uint64_t seq);
+
+ bool clone_buffer(const ggml_hexagon_shared_buffer*);
+
+ void add_sync_peer(ggml_hexagon_session * peer) {
+ sync_peers.insert(peer);
+ }
+
+ void flush_sync_peers() {
+ if (sync_peers.empty()) return;
+
+ for (auto * peer : sync_peers) {
+ peer->flush_batch();
+ }
+ sync_peers.clear();
+ }
};
// ** backend buffers
@@ -315,26 +440,68 @@ struct ggml_backend_hexagon_buffer_type_context {
std::string name;
};
+struct ggml_hexagon_rpcmem_block {
+ uint8_t * base = nullptr;
+ int fd = -1;
+ size_t size = 0;
+
+ ggml_hexagon_rpcmem_block(size_t size) {
+ base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size);
+ if (!base) {
+ throw std::runtime_error("ggml-hex: rpcmem_alloc failed");
+ }
+ fd = rpcmem_to_fd(base);
+ if (fd < 0) {
+ rpcmem_free(base);
+ throw std::runtime_error("ggml-hex: rpcmem_to_fd failed");
+ }
+ this->size = size;
+ }
+
+ ~ggml_hexagon_rpcmem_block() {
+ if (base) {
+ rpcmem_free(base);
+ }
+ }
+};
+
struct ggml_hexagon_shared_buffer {
- ggml_hexagon_session * sess;
- uint8_t * base;
- size_t size;
- int fd;
- bool mapped;
- bool pinned;
+ ggml_hexagon_session * sess;
+ std::shared_ptr<ggml_hexagon_rpcmem_block> mem;
+ std::vector<ggml_hexagon_tensor_extra *> tensor_extra;
+ uint32_t fence_head = 0;
+ size_t fences_size = 0;
+ bool mapped;
+ bool pinned;
+
+ const char * c_name() const { return sess->c_name(); }
+ uint8_t * base() const { return mem ? mem->base : nullptr; }
+ size_t size() const { return mem ? mem->size : 0; }
+ int fd() const { return mem ? mem->fd : -1; }
+
+ uint8_t * alloc_fence() {
+ if (fences_size == 0) return nullptr;
+ int max_slots = fences_size / GGML_HEXAGON_FENCE_SLOT_SIZE;
+ uint32_t slot = (fence_head++) % max_slots;
+
+ size_t guard_offset = size() - fences_size;
+ uint8_t * fence_ptr = base() + guard_offset + (size_t)slot * GGML_HEXAGON_FENCE_SLOT_SIZE;
+ return fence_ptr;
+ }
void mmap() {
+ if (!this->mem) return;
fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : FASTRPC_MAP_FD_DELAYED;
- int err = fastrpc_mmap(sess->domain_id, this->fd, (void *) this->base, 0, this->size, flags);
+ int err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags);
if (err != 0) {
GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(),
- sess->domain_id, this->size, this->fd, (unsigned) err);
+ sess->domain_id, size(), fd(), (unsigned) err);
throw std::runtime_error("ggml-hex: fastrpc_mmap failed (see log for details)");
}
HEX_VERBOSE("ggml-hex: %s mapped buffer: base %p size %zu fd %d pinned %u\n",
- sess->c_name(), (void *) this->base, this->size, this->fd, pinned);
+ sess->c_name(), (void *) base(), size(), fd(), pinned);
this->mapped = true;
}
@@ -342,66 +509,69 @@ struct ggml_hexagon_shared_buffer {
void unmap() {
if (!this->mapped) return;
- if (!this->pinned) {
+ if (!this->pinned && mem) {
// HTP might still hold a reference, tell it drop it
- htp_iface_munmap(sess->handle, this->fd);
+ htp_iface_munmap(sess->handle, fd());
}
- fastrpc_munmap(sess->domain_id, this->fd, (void *) this->base, this->size);
+ if (mem) {
+ fastrpc_munmap(sess->domain_id, fd(), (void *) base(), size());
+ }
HEX_VERBOSE("ggml-hex: %s unmapped buffer: base %p size %zu fd %d\n", sess->c_name(),
- (void *) this->base, size, this->fd);
+ (void *) base(), size(), fd());
this->mapped = false;
- this->fd = -1;
}
void alloc(size_t size) {
- if (this->base) return;
-
- this->base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size);
- if (!this->base) {
- GGML_LOG_ERROR("ggml-hex: %s failed to allocate buffer : size %zu\n", sess->c_name(), size);
- throw std::runtime_error("ggml-hex: rpcmem_alloc failed (see log for details)");
- }
+ if (this->mem) return;
- this->fd = rpcmem_to_fd(this->base);
- if (this->fd < 0) {
- GGML_LOG_ERROR("ggml-hex: %s failed to get FD for buffer %p\n", sess->c_name(), (void *) this->base);
- throw std::runtime_error("ggml-hex: rpcmem_to_fd failed (see log for details)");
- }
- this->size = size;
+ this->mem = std::make_shared<ggml_hexagon_rpcmem_block>(size);
HEX_VERBOSE("ggml-hex: %s allocated buffer: base %p size %zu fd %d pinned %d\n", sess->c_name(),
- (void *) this->base, this->size, this->fd, (int) pinned);
+ (void *) base(), this->size(), fd(), (int) pinned);
mmap();
}
void free() {
- if (!this->base) return;
-
unmap();
- rpcmem_free(this->base);
-
- HEX_VERBOSE("ggml-hex: %s freed buffer: base %p size %zu fd %d\n", sess->c_name(),
- (void *) this->base, size, this->fd);
+ // The memory is freed when the shared_ptr refcount drops to 0.
+ HEX_VERBOSE("ggml-hex: %s release ref on buffer: base %p size %zu fd %d\n", sess->c_name(),
+ (void *) base(), size(), fd());
+ this->mem = nullptr;
+ }
+
+ ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, size_t size, bool pinned = false, size_t fence_size = 0) {
+ this->sess = sess;
+ this->mapped = false;
+ this->pinned = pinned;
+ this->fences_size = fence_size;
+
+ // Size adjustment inside the buffer class
+ size_t guard_offset = (size + 4095) & ~4095;
+ size_t total_size = guard_offset;
+ if (fence_size > 0) {
+ total_size += 4096 + fence_size;
+ }
- this->base = NULL;
+ alloc(total_size);
}
- ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, size_t size, bool pinned = false) {
- this->sess = sess;
- this->size = 0;
- this->base = nullptr;
- this->fd = -1;
- this->mapped = false;
- this->pinned = pinned;
-
- alloc(size);
+ // Clone constructor for cross-session mapping
+ ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, const ggml_hexagon_shared_buffer & other) {
+ this->sess = sess;
+ this->mem = other.mem;
+ this->mapped = false;
+ this->pinned = other.pinned;
+ this->fences_size = other.fences_size;
}
~ggml_hexagon_shared_buffer() {
free();
+ for (auto * extra : tensor_extra) {
+ delete extra;
+ }
}
};
@@ -416,18 +586,25 @@ static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer
static void * ggml_backend_hexagon_buffer_get_base(ggml_backend_buffer_t buffer) {
auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(buffer->context);
- return sbuf->base;
+ return sbuf->base();
}
static enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(buffer->context);
auto sess = sbuf->sess;
- HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu usage %d\n", sess->c_name(),
- tensor->name, (void *) sbuf->base, tensor->data, ggml_nbytes(tensor), (int) buffer->usage);
+ HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu\n", sess->c_name(),
+ tensor->name, (void *) sbuf->base(), tensor->data, ggml_nbytes(tensor));
+
+ auto extra = new ggml_hexagon_tensor_extra();
+ sbuf->tensor_extra.push_back(extra);
- if (tensor->view_src != NULL && tensor->view_offs == 0) {
- return GGML_STATUS_SUCCESS; // nothing to do for the view
+ tensor->extra = extra;
+ if (ggml_hexagon_is_repack_type(tensor->type)) {
+ if (sess->needs_repack.count(tensor)) {
+ extra->flags |= GGML_HEXAGON_TENSOR_REPACK;
+ sess->needs_repack.erase(tensor);
+ }
}
return GGML_STATUS_SUCCESS;
@@ -499,7 +676,7 @@ static void pack_mxfp4_quants(block_mxfp4 * x, const uint8_t * qs, unsigned int
}
// repack q4_0 data into q4_0_tiled tensor
-static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t size) {
+static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
const block_q4_0 * src_matrix = (const block_q4_0 *) data;
int64_t ne0 = t->ne[0];
int64_t ne1 = t->ne[1];
@@ -513,46 +690,49 @@ static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t size) {
const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0;
const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size;
- for (int i3 = 0; i3 < ne3; i3++) {
- for (int i2 = 0; i2 < ne2; i2++) {
- const block_q4_0 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32));
- uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
+ size_t slice_size = ne1 * ggml_row_size(t->type, ne0);
+ int64_t start_slice = offset / slice_size;
+ int64_t end_slice = (offset + size + slice_size - 1) / slice_size;
+ if (end_slice > ne2 * ne3) {
+ end_slice = ne2 * ne3;
+ }
- for (int ct = 0; ct < n_col_tiles; ct++) {
- for (int kt = 0; kt < n_k_tiles; kt++) {
- uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size;
+ for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+ const block_q4_0 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32));
+ uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size;
- uint8_t tile_quants[32][32];
- for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- unpack_q4_0_quants(tile_quants[row], &src_expert[r * (ne0 / 32) + kt], 0);
- } else {
- memset(tile_quants[row], 8, 32);
- }
- }
+ for (int ct = 0; ct < n_col_tiles; ct++) {
+ for (int kt = 0; kt < n_k_tiles; kt++) {
+ uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size;
- for (int cp = 0; cp < 16; cp++) {
- for (int row = 0; row < 32; row++) {
- tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp];
- }
+ uint8_t tile_quants[32][32];
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ if (r < ne1 && kt < ne0 / 32) {
+ unpack_q4_0_quants(tile_quants[row], &src_slice[r * (ne0 / 32) + kt], 0);
+ } else {
+ memset(tile_quants[row], 8, 32);
}
+ }
- ggml_half * scale_dst = (ggml_half *)(tile_dst + 512);
+ for (int cp = 0; cp < 16; cp++) {
for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_expert[r * (ne0 / 32) + kt].d : 0;
+ tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp];
}
}
+
+ ggml_half * scale_dst = (ggml_half *)(tile_dst + 512);
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * (ne0 / 32) + kt].d : 0;
+ }
}
}
}
-
- GGML_UNUSED(size);
}
// repack q4_0_tiled tensor into q4_0 data
-static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t size) {
+static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t offset, size_t size) {
block_q4_0 * dst_matrix = (block_q4_0 *) data;
int64_t ne0 = t->ne[0];
int64_t ne1 = t->ne[1];
@@ -566,48 +746,65 @@ static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t size) {
const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0;
const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size;
- for (int i3 = 0; i3 < ne3; i3++) {
- for (int i2 = 0; i2 < ne2; i2++) {
- block_q4_0 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32));
- const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
-
- for (int ct = 0; ct < n_col_tiles; ct++) {
- for (int kt = 0; kt < n_k_tiles; kt++) {
- const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size;
-
- uint8_t tile_quants[32][32];
- for (int cp = 0; cp < 16; cp++) {
- for (int row = 0; row < 32; row++) {
- uint8_t val = tile_src[cp * 32 + row];
- tile_quants[row][2 * cp + 0] = val & 0x0F;
- tile_quants[row][2 * cp + 1] = val >> 4;
- }
- }
+ size_t slice_size = ne1 * ggml_row_size(t->type, ne0);
+ size_t row_size_bytes = ggml_row_size(t->type, ne0);
+ int64_t start_slice = offset / slice_size;
+ int64_t end_slice = (offset + size + slice_size - 1) / slice_size;
+ if (end_slice > ne2 * ne3) {
+ end_slice = ne2 * ne3;
+ }
+
+ for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+ size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size);
+ size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size);
+ size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size;
+ size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size;
+
+ int64_t start_row = slice_offset_start / row_size_bytes;
+ int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes;
+ end_row = (std::min)(end_row, ne1);
+
+ int start_ct = start_row / 32;
+ int end_ct = (end_row + 31) / 32;
+ end_ct = (std::min)(end_ct, n_col_tiles);
+
+ block_q4_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q4_0);
+ const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size;
+ for (int ct = start_ct; ct < end_ct; ct++) {
+ for (int kt = 0; kt < n_k_tiles; kt++) {
+ const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size;
+
+ uint8_t tile_quants[32][32];
+ for (int cp = 0; cp < 16; cp++) {
for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- pack_q4_0_quants(&dst_expert[r * (ne0 / 32) + kt], tile_quants[row], 0);
- }
+ uint8_t val = tile_src[cp * 32 + row];
+ tile_quants[row][2 * cp + 0] = val & 0x0F;
+ tile_quants[row][2 * cp + 1] = val >> 4;
}
+ }
- const ggml_half * scale_src = (const ggml_half *)(tile_src + 512);
- for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- dst_expert[r * (ne0 / 32) + kt].d = scale_src[row];
- }
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ if (r >= start_row && r < end_row && kt < ne0 / 32) {
+ pack_q4_0_quants(&dst_slice[(r - start_row) * (ne0 / 32) + kt], tile_quants[row], 0);
+ }
+ }
+
+ const ggml_half * scale_src = (const ggml_half *)(tile_src + 512);
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ if (r >= start_row && r < end_row && kt < ne0 / 32) {
+ dst_slice[(r - start_row) * (ne0 / 32) + kt].d = scale_src[row];
}
}
}
}
}
-
- GGML_UNUSED(size);
}
// repack q4_1 data into q4_1_tiled tensor
-static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t size) {
+static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
const block_q4_1 * src_matrix = (const block_q4_1 *) data;
int64_t ne0 = t->ne[0];
int64_t ne1 = t->ne[1];
@@ -621,52 +818,55 @@ static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t size) {
const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1;
const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size;
- for (int i3 = 0; i3 < ne3; i3++) {
- for (int i2 = 0; i2 < ne2; i2++) {
- const block_q4_1 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32));
- uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
+ size_t slice_size = ne1 * ggml_row_size(t->type, ne0);
+ int64_t start_slice = offset / slice_size;
+ int64_t end_slice = (offset + size + slice_size - 1) / slice_size;
+ if (end_slice > ne2 * ne3) {
+ end_slice = ne2 * ne3;
+ }
- for (int ct = 0; ct < n_col_tiles; ct++) {
- for (int kt = 0; kt < n_k_tiles; kt++) {
- uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size;
+ for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+ const block_q4_1 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32));
+ uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size;
- uint8_t tile_quants[32][32];
- for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- unpack_q4_1_quants(tile_quants[row], &src_expert[r * (ne0 / 32) + kt], 0);
- } else {
- memset(tile_quants[row], 0, 32);
- }
- }
+ for (int ct = 0; ct < n_col_tiles; ct++) {
+ for (int kt = 0; kt < n_k_tiles; kt++) {
+ uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size;
- for (int cp = 0; cp < 16; cp++) {
- for (int row = 0; row < 32; row++) {
- tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp];
- }
+ uint8_t tile_quants[32][32];
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ if (r < ne1 && kt < ne0 / 32) {
+ unpack_q4_1_quants(tile_quants[row], &src_slice[r * (ne0 / 32) + kt], 0);
+ } else {
+ memset(tile_quants[row], 0, 32);
}
+ }
- ggml_half * scale_dst = (ggml_half *)(tile_dst + 512);
+ for (int cp = 0; cp < 16; cp++) {
for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- scale_dst[2 * row + 0] = src_expert[r * (ne0 / 32) + kt].d;
- scale_dst[2 * row + 1] = src_expert[r * (ne0 / 32) + kt].m;
- } else {
- scale_dst[2 * row + 0] = 0;
- scale_dst[2 * row + 1] = 0;
- }
+ tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp];
+ }
+ }
+
+ ggml_half * scale_dst = (ggml_half *)(tile_dst + 512);
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ if (r < ne1 && kt < ne0 / 32) {
+ scale_dst[2 * row + 0] = src_slice[r * (ne0 / 32) + kt].d;
+ scale_dst[2 * row + 1] = src_slice[r * (ne0 / 32) + kt].m;
+ } else {
+ scale_dst[2 * row + 0] = 0;
+ scale_dst[2 * row + 1] = 0;
}
}
}
}
}
-
- GGML_UNUSED(size);
}
// repack q4_1_tiled tensor into q4_1 data
-static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t size) {
+static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t offset, size_t size) {
block_q4_1 * dst_matrix = (block_q4_1 *) data;
int64_t ne0 = t->ne[0];
int64_t ne1 = t->ne[1];
@@ -680,49 +880,66 @@ static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t size) {
const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1;
const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size;
- for (int i3 = 0; i3 < ne3; i3++) {
- for (int i2 = 0; i2 < ne2; i2++) {
- block_q4_1 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32));
- const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
-
- for (int ct = 0; ct < n_col_tiles; ct++) {
- for (int kt = 0; kt < n_k_tiles; kt++) {
- const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size;
-
- uint8_t tile_quants[32][32];
- for (int cp = 0; cp < 16; cp++) {
- for (int row = 0; row < 32; row++) {
- uint8_t val = tile_src[cp * 32 + row];
- tile_quants[row][2 * cp + 0] = val & 0x0F;
- tile_quants[row][2 * cp + 1] = val >> 4;
- }
- }
+ size_t slice_size = ne1 * ggml_row_size(t->type, ne0);
+ size_t row_size_bytes = ggml_row_size(t->type, ne0);
+ int64_t start_slice = offset / slice_size;
+ int64_t end_slice = (offset + size + slice_size - 1) / slice_size;
+ if (end_slice > ne2 * ne3) {
+ end_slice = ne2 * ne3;
+ }
+ for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+ size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size);
+ size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size);
+ size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size;
+ size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size;
+
+ int64_t start_row = slice_offset_start / row_size_bytes;
+ int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes;
+ end_row = (std::min)(end_row, ne1);
+
+ int start_ct = start_row / 32;
+ int end_ct = (end_row + 31) / 32;
+ end_ct = (std::min)(end_ct, n_col_tiles);
+
+ block_q4_1 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q4_1);
+ const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size;
+
+ for (int ct = start_ct; ct < end_ct; ct++) {
+ for (int kt = 0; kt < n_k_tiles; kt++) {
+ const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size;
+
+ uint8_t tile_quants[32][32];
+ for (int cp = 0; cp < 16; cp++) {
for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- pack_q4_1_quants(&dst_expert[r * (ne0 / 32) + kt], tile_quants[row], 0);
- }
+ uint8_t val = tile_src[cp * 32 + row];
+ tile_quants[row][2 * cp + 0] = val & 0x0F;
+ tile_quants[row][2 * cp + 1] = val >> 4;
}
+ }
- const ggml_half * scale_src = (const ggml_half *)(tile_src + 512);
- for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- dst_expert[r * (ne0 / 32) + kt].d = scale_src[2 * row];
- dst_expert[r * (ne0 / 32) + kt].m = scale_src[2 * row + 1];
- }
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ if (r >= start_row && r < end_row && kt < ne0 / 32) {
+ pack_q4_1_quants(&dst_slice[(r - start_row) * (ne0 / 32) + kt], tile_quants[row], 0);
+ }
+ }
+
+ const ggml_half * scale_src = (const ggml_half *)(tile_src + 512);
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ if (r >= start_row && r < end_row && kt < ne0 / 32) {
+ dst_slice[(r - start_row) * (ne0 / 32) + kt].d = scale_src[2 * row];
+ dst_slice[(r - start_row) * (ne0 / 32) + kt].m = scale_src[2 * row + 1];
}
}
}
}
}
-
- GGML_UNUSED(size);
}
// repack q8_0 data into q8_0_tiled tensor
-static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t size) {
+static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
const block_q8_0 * src_matrix = (const block_q8_0 *) data;
int64_t ne0 = t->ne[0];
int64_t ne1 = t->ne[1];
@@ -736,41 +953,44 @@ static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t size) {
const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q8_0;
const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size;
- for (int i3 = 0; i3 < ne3; i3++) {
- for (int i2 = 0; i2 < ne2; i2++) {
- const block_q8_0 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32));
- uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
-
- for (int ct = 0; ct < n_col_tiles; ct++) {
- for (int kt = 0; kt < n_k_tiles; kt++) {
- uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size;
-
- for (int cp = 0; cp < 16; cp++) {
- int col0 = cp * 2;
- int col1 = col0 + 1;
- for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- const block_q8_0 * b = (r < ne1 && kt < ne0 / 32) ? &src_expert[r * (ne0 / 32) + kt] : NULL;
- tile_dst[cp * 64 + 2 * row + 0] = b ? b->qs[col0] : 0;
- tile_dst[cp * 64 + 2 * row + 1] = b ? b->qs[col1] : 0;
- }
- }
+ size_t slice_size = ne1 * ggml_row_size(t->type, ne0);
+ int64_t start_slice = offset / slice_size;
+ int64_t end_slice = (offset + size + slice_size - 1) / slice_size;
+ if (end_slice > ne2 * ne3) {
+ end_slice = ne2 * ne3;
+ }
+
+ for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+ const block_q8_0 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32));
+ uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size;
- ggml_half * scale_dst = (ggml_half *)(tile_dst + 1024);
+ for (int ct = 0; ct < n_col_tiles; ct++) {
+ for (int kt = 0; kt < n_k_tiles; kt++) {
+ uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size;
+
+ for (int cp = 0; cp < 16; cp++) {
+ int col0 = cp * 2;
+ int col1 = col0 + 1;
for (int row = 0; row < 32; row++) {
int64_t r = ct * 32 + row;
- scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_expert[r * (ne0 / 32) + kt].d : 0;
+ const block_q8_0 * b = (r < ne1 && kt < ne0 / 32) ? &src_slice[r * (ne0 / 32) + kt] : NULL;
+ tile_dst[cp * 64 + 2 * row + 0] = b ? b->qs[col0] : 0;
+ tile_dst[cp * 64 + 2 * row + 1] = b ? b->qs[col1] : 0;
}
}
+
+ ggml_half * scale_dst = (ggml_half *)(tile_dst + 1024);
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * (ne0 / 32) + kt].d : 0;
+ }
}
}
}
-
- GGML_UNUSED(size);
}
// repack q8_0_tiled tensor into q8_0 data
-static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t size) {
+static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t offset, size_t size) {
block_q8_0 * dst_matrix = (block_q8_0 *) data;
int64_t ne0 = t->ne[0];
int64_t ne1 = t->ne[1];
@@ -784,45 +1004,62 @@ static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t size) {
const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q8_0;
const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size;
- for (int i3 = 0; i3 < ne3; i3++) {
- for (int i2 = 0; i2 < ne2; i2++) {
- block_q8_0 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32));
- const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
-
- for (int ct = 0; ct < n_col_tiles; ct++) {
- for (int kt = 0; kt < n_k_tiles; kt++) {
- const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size;
-
- for (int cp = 0; cp < 16; cp++) {
- int col0 = cp * 2;
- int col1 = col0 + 1;
- for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- block_q8_0 & b = dst_expert[r * (ne0 / 32) + kt];
- b.qs[col0] = tile_src[cp * 64 + 2 * row + 0];
- b.qs[col1] = tile_src[cp * 64 + 2 * row + 1];
- }
- }
- }
+ size_t slice_size = ne1 * ggml_row_size(t->type, ne0);
+ size_t row_size_bytes = ggml_row_size(t->type, ne0);
+ int64_t start_slice = offset / slice_size;
+ int64_t end_slice = (offset + size + slice_size - 1) / slice_size;
+ if (end_slice > ne2 * ne3) {
+ end_slice = ne2 * ne3;
+ }
+
+ for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+ size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size);
+ size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size);
+ size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size;
+ size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size;
+
+ int64_t start_row = slice_offset_start / row_size_bytes;
+ int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes;
+ end_row = (std::min)(end_row, ne1);
+
+ int start_ct = start_row / 32;
+ int end_ct = (end_row + 31) / 32;
+ end_ct = (std::min)(end_ct, n_col_tiles);
+
+ block_q8_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q8_0);
+ const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size;
- const ggml_half * scale_src = (const ggml_half *)(tile_src + 1024);
+ for (int ct = start_ct; ct < end_ct; ct++) {
+ for (int kt = 0; kt < n_k_tiles; kt++) {
+ const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size;
+
+ for (int cp = 0; cp < 16; cp++) {
+ int col0 = cp * 2;
+ int col1 = col0 + 1;
for (int row = 0; row < 32; row++) {
int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- dst_expert[r * (ne0 / 32) + kt].d = scale_src[row];
+ if (r >= start_row && r < end_row && kt < ne0 / 32) {
+ block_q8_0 & b = dst_slice[(r - start_row) * (ne0 / 32) + kt];
+ b.qs[col0] = tile_src[cp * 64 + 2 * row + 0];
+ b.qs[col1] = tile_src[cp * 64 + 2 * row + 1];
}
}
}
+
+ const ggml_half * scale_src = (const ggml_half *)(tile_src + 1024);
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ if (r >= start_row && r < end_row && kt < ne0 / 32) {
+ dst_slice[(r - start_row) * (ne0 / 32) + kt].d = scale_src[row];
+ }
+ }
}
}
}
-
- GGML_UNUSED(size);
}
// repack mxfp4 data into mxfp4_tiled tensor
-static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t size) {
+static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
const block_mxfp4 * src_matrix = (const block_mxfp4 *) data;
int64_t ne0 = t->ne[0];
int64_t ne1 = t->ne[1];
@@ -836,46 +1073,49 @@ static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t size)
const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_MXFP4;
const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size;
- for (int i3 = 0; i3 < ne3; i3++) {
- for (int i2 = 0; i2 < ne2; i2++) {
- const block_mxfp4 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32));
- uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
+ size_t slice_size = ne1 * ggml_row_size(t->type, ne0);
+ int64_t start_slice = offset / slice_size;
+ int64_t end_slice = (offset + size + slice_size - 1) / slice_size;
+ if (end_slice > ne2 * ne3) {
+ end_slice = ne2 * ne3;
+ }
- for (int ct = 0; ct < n_col_tiles; ct++) {
- for (int kt = 0; kt < n_k_tiles; kt++) {
- uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size;
+ for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+ const block_mxfp4 * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * (ne0 / 32));
+ uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size;
- uint8_t tile_quants[32][32];
- for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- unpack_mxfp4_quants(tile_quants[row], &src_expert[r * (ne0 / 32) + kt], 0);
- } else {
- memset(tile_quants[row], 0, 32);
- }
- }
+ for (int ct = 0; ct < n_col_tiles; ct++) {
+ for (int kt = 0; kt < n_k_tiles; kt++) {
+ uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size;
- for (int cp = 0; cp < 16; cp++) {
- for (int row = 0; row < 32; row++) {
- tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp];
- }
+ uint8_t tile_quants[32][32];
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ if (r < ne1 && kt < ne0 / 32) {
+ unpack_mxfp4_quants(tile_quants[row], &src_slice[r * (ne0 / 32) + kt], 0);
+ } else {
+ memset(tile_quants[row], 0, 32);
}
+ }
- uint8_t * scale_dst = tile_dst + 512;
+ for (int cp = 0; cp < 16; cp++) {
for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_expert[r * (ne0 / 32) + kt].e : 0;
+ tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp];
}
}
+
+ uint8_t * scale_dst = tile_dst + 512;
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * (ne0 / 32) + kt].e : 0;
+ }
}
}
}
-
- GGML_UNUSED(size);
}
// repack mxfp4_tiled tensor into mxfp4 data
-static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t size) {
+static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t offset, size_t size) {
block_mxfp4 * dst_matrix = (block_mxfp4 *) data;
int64_t ne0 = t->ne[0];
int64_t ne1 = t->ne[1];
@@ -889,133 +1129,179 @@ static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t size)
const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_MXFP4;
const size_t matrix_size = n_col_tiles * n_k_tiles * tile_size;
- for (int i3 = 0; i3 < ne3; i3++) {
- for (int i2 = 0; i2 < ne2; i2++) {
- block_mxfp4 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32));
- const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
-
- for (int ct = 0; ct < n_col_tiles; ct++) {
- for (int kt = 0; kt < n_k_tiles; kt++) {
- const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size;
-
- uint8_t tile_quants[32][32];
- for (int cp = 0; cp < 16; cp++) {
- for (int row = 0; row < 32; row++) {
- uint8_t val = tile_src[cp * 32 + row];
- tile_quants[row][2 * cp + 0] = val & 0x0F;
- tile_quants[row][2 * cp + 1] = val >> 4;
- }
- }
+ size_t slice_size = ne1 * ggml_row_size(t->type, ne0);
+ size_t row_size_bytes = ggml_row_size(t->type, ne0);
+ int64_t start_slice = offset / slice_size;
+ int64_t end_slice = (offset + size + slice_size - 1) / slice_size;
+ if (end_slice > ne2 * ne3) {
+ end_slice = ne2 * ne3;
+ }
+
+ for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+ size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size);
+ size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size);
+ size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size;
+ size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size;
+
+ int64_t start_row = slice_offset_start / row_size_bytes;
+ int64_t end_row = (slice_offset_end + row_size_bytes - 1) / row_size_bytes;
+ end_row = (std::min)(end_row, ne1);
+
+ int start_ct = start_row / 32;
+ int end_ct = (end_row + 31) / 32;
+ end_ct = (std::min)(end_ct, n_col_tiles);
+ block_mxfp4 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_mxfp4);
+ const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size;
+
+ for (int ct = start_ct; ct < end_ct; ct++) {
+ for (int kt = 0; kt < n_k_tiles; kt++) {
+ const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size;
+
+ uint8_t tile_quants[32][32];
+ for (int cp = 0; cp < 16; cp++) {
for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- pack_mxfp4_quants(&dst_expert[r * (ne0 / 32) + kt], tile_quants[row], 0);
- }
+ uint8_t val = tile_src[cp * 32 + row];
+ tile_quants[row][2 * cp + 0] = val & 0x0F;
+ tile_quants[row][2 * cp + 1] = val >> 4;
}
+ }
- const uint8_t * scale_src = tile_src + 512;
- for (int row = 0; row < 32; row++) {
- int64_t r = ct * 32 + row;
- if (r < ne1 && kt < ne0 / 32) {
- dst_expert[r * (ne0 / 32) + kt].e = scale_src[row];
- }
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ if (r >= start_row && r < end_row && kt < ne0 / 32) {
+ pack_mxfp4_quants(&dst_slice[(r - start_row) * (ne0 / 32) + kt], tile_quants[row], 0);
+ }
+ }
+
+ const uint8_t * scale_src = tile_src + 512;
+ for (int row = 0; row < 32; row++) {
+ int64_t r = ct * 32 + row;
+ if (r >= start_row && r < end_row && kt < ne0 / 32) {
+ dst_slice[(r - start_row) * (ne0 / 32) + kt].e = scale_src[row];
}
}
}
}
}
-
- GGML_UNUSED(size);
}
-static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer,
- ggml_tensor * tensor,
- const void * data,
- size_t offset,
- size_t size) {
- auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
- auto sess = sbuf->sess;
-
- HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu\n", sess->c_name(), tensor->name, data, offset, size);
-
+static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) {
switch (tensor->type) {
case GGML_TYPE_Q4_0:
- GGML_ASSERT(offset == 0);
- GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
- repack_q4_0_tiled(tensor, data, size);
+ repack_q4_0_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_Q4_1:
- GGML_ASSERT(offset == 0);
- GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
- repack_q4_1_tiled(tensor, data, size);
+ repack_q4_1_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_Q8_0:
- GGML_ASSERT(offset == 0);
- GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
- repack_q8_0_tiled(tensor, data, size);
+ repack_q8_0_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_IQ4_NL:
- GGML_ASSERT(offset == 0);
- GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
- // IQ4_NL has identical block layout to Q4_0 (ggml_half d + uint8_t qs[16])
- repack_q4_0_tiled(tensor, data, size);
+ repack_q4_0_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_MXFP4:
- GGML_ASSERT(offset == 0);
- GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
- repack_mxfp4_tiled(tensor, data, size);
+ repack_mxfp4_tiled(tensor, data, 0, size);
break;
default:
- memcpy((char *) tensor->data + offset, data, size);
break;
}
}
+static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer,
+ ggml_tensor * tensor,
+ const void * data,
+ size_t offset,
+ size_t size) {
+ auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;
+ auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
+ auto sess = sbuf->sess;
+
+ if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
+ extra->flags |= GGML_HEXAGON_TENSOR_WEIGHT;
+ if (ggml_hexagon_is_repack_type(tensor->type)) {
+ extra->flags |= GGML_HEXAGON_TENSOR_REPACK;
+ }
+ }
+
+ HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n",
+ sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags);
+
+ if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) {
+ memcpy((char *) tensor->data + offset, data, size);
+ return;
+ }
+
+ if (offset == 0 && size == ggml_nbytes(tensor) && extra->shadow_buf.empty()) {
+ repack_tensor_tiled(tensor, data, size);
+ return;
+ }
+
+ if (extra->shadow_buf.size() < ggml_nbytes(tensor)) {
+ extra->shadow_buf.resize(ggml_nbytes(tensor));
+ }
+ memcpy(extra->shadow_buf.data() + offset, data, size);
+ extra->shadow_size += size;
+
+ if (extra->shadow_size >= ggml_nbytes(tensor)) {
+ repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size());
+ extra->shadow_buf.clear();
+ extra->shadow_buf.shrink_to_fit();
+ extra->shadow_size = 0;
+ }
+}
+
static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,
const ggml_tensor * tensor,
void * data,
size_t offset,
size_t size) {
- auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
- auto sess = sbuf->sess;
+ auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;
+ auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
+ auto sess = sbuf->sess;
- HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu\n", sess->c_name(), tensor->name, data, offset, size);
+ HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n",
+ sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags);
+
+ if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) {
+ memcpy(data, (const char *) tensor->data + offset, size);
+ return;
+ }
switch (tensor->type) {
case GGML_TYPE_Q4_0:
GGML_ASSERT(offset == 0);
GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
- repack_tiled_q4_0(data, tensor, size);
+ repack_tiled_q4_0(data, tensor, offset, size);
break;
case GGML_TYPE_Q4_1:
GGML_ASSERT(offset == 0);
GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
- repack_tiled_q4_1(data, tensor, size);
+ repack_tiled_q4_1(data, tensor, offset, size);
break;
case GGML_TYPE_Q8_0:
GGML_ASSERT(offset == 0);
GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
- repack_tiled_q8_0(data, tensor, size);
+ repack_tiled_q8_0(data, tensor, offset, size);
break;
case GGML_TYPE_IQ4_NL:
GGML_ASSERT(offset == 0);
GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
- repack_tiled_q4_0(data, tensor, size);
+ repack_tiled_q4_0(data, tensor, offset, size);
break;
case GGML_TYPE_MXFP4:
GGML_ASSERT(offset == 0);
GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
- repack_tiled_mxfp4(data, tensor, size);
+ repack_tiled_mxfp4(data, tensor, offset, size);
break;
default:
@@ -1035,11 +1321,121 @@ static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t bu
GGML_UNUSED(dst);
}
+static void ggml_backend_hexagon_buffer_set_tensor_2d(ggml_backend_buffer_t buffer,
+ ggml_tensor * tensor,
+ const void * data,
+ size_t offset,
+ size_t size,
+ size_t n_copies,
+ size_t stride_tensor,
+ size_t stride_data) {
+ auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;
+ auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
+ auto sess = sbuf->sess;
+
+ if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
+ extra->flags |= GGML_HEXAGON_TENSOR_WEIGHT;
+ if (ggml_hexagon_is_repack_type(tensor->type)) {
+ extra->flags |= GGML_HEXAGON_TENSOR_REPACK;
+ }
+ }
+
+ HEX_VERBOSE("ggml-hex: %s set-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d flags 0x%x\n",
+ sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, extra->flags);
+
+ if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) {
+ for (size_t i = 0; i < n_copies; i++) {
+ memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size);
+ }
+ return;
+ }
+
+ if (extra->shadow_buf.size() < ggml_nbytes(tensor)) {
+ extra->shadow_buf.resize(ggml_nbytes(tensor));
+ }
+ for (size_t i = 0; i < n_copies; i++) {
+ memcpy(extra->shadow_buf.data() + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size);
+ }
+ extra->shadow_size += n_copies * size;
+
+ if (extra->shadow_size >= ggml_nbytes(tensor)) {
+ repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size());
+ extra->shadow_buf.clear();
+ extra->shadow_buf.shrink_to_fit();
+ extra->shadow_size = 0;
+ }
+}
+
+static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,
+ const ggml_tensor * tensor,
+ void * data,
+ size_t offset,
+ size_t size,
+ size_t n_copies,
+ size_t stride_tensor,
+ size_t stride_data) {
+ auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;
+ auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
+ auto sess = sbuf->sess;
+
+ HEX_VERBOSE("ggml-hex: %s get-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n",
+ sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage);
+
+ if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) == 0) {
+ for (size_t i = 0; i < n_copies; i++) {
+ memcpy((uint8_t *)data + i * stride_data, (const uint8_t *)tensor->data + offset + i * stride_tensor, size);
+ }
+ return;
+ }
+
+ size_t temp_size = n_copies > 0 ? (n_copies - 1) * stride_tensor + size : 0;
+ size_t slice_size = tensor->ne[1] * ggml_row_size(tensor->type, tensor->ne[0]);
+ size_t slice_offset = offset % slice_size;
+ size_t row_size_bytes = ggml_row_size(tensor->type, tensor->ne[0]);
+
+ GGML_ASSERT((slice_offset % row_size_bytes) == 0 && "offset must be aligned to row boundary");
+ GGML_ASSERT((temp_size % row_size_bytes) == 0 && "temp_size must be a multiple of row size");
+ GGML_ASSERT((slice_offset / row_size_bytes) % 32 == 0 && "offset must be aligned to tile size (32 rows)");
+ GGML_ASSERT((offset + temp_size) <= ggml_nbytes(tensor));
+
+ std::vector<uint8_t> temp_buf(temp_size);
+
+ switch (tensor->type) {
+ case GGML_TYPE_Q4_0:
+ repack_tiled_q4_0(temp_buf.data(), tensor, offset, temp_size);
+ break;
+
+ case GGML_TYPE_Q4_1:
+ repack_tiled_q4_1(temp_buf.data(), tensor, offset, temp_size);
+ break;
+
+ case GGML_TYPE_Q8_0:
+ repack_tiled_q8_0(temp_buf.data(), tensor, offset, temp_size);
+ break;
+
+ case GGML_TYPE_IQ4_NL:
+ repack_tiled_q4_0(temp_buf.data(), tensor, offset, temp_size);
+ break;
+
+ case GGML_TYPE_MXFP4:
+ repack_tiled_mxfp4(temp_buf.data(), tensor, offset, temp_size);
+ break;
+
+ default:
+ memcpy(temp_buf.data(), (const uint8_t *) tensor->data + offset, temp_size);
+ break;
+ }
+
+ for (size_t i = 0; i < n_copies; i++) {
+ memcpy((uint8_t *) data + i * stride_data, temp_buf.data() + i * stride_tensor, size);
+ }
+}
+
static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
- HEX_VERBOSE("ggml-hex: %s clear-buff base %p size %zu\n", sess->c_name(), (void *) sbuf->base, sbuf->size);
- memset(sbuf->base, value, sbuf->size);
+ HEX_VERBOSE("ggml-hex: %s clear-buff base %p size %zu\n", sess->c_name(), (void *) sbuf->base(), sbuf->size());
+ memset(sbuf->base(), value, sbuf->size());
}
static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = {
@@ -1049,6 +1445,40 @@ static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = {
/* .memset_tensor = */ NULL,
/* .set_tensor = */ ggml_backend_hexagon_buffer_set_tensor,
/* .get_tensor = */ ggml_backend_hexagon_buffer_get_tensor,
+ /* .set_tensor_2d = */ ggml_backend_hexagon_buffer_set_tensor_2d,
+ /* .get_tensor_2d = */ ggml_backend_hexagon_buffer_get_tensor_2d,
+ /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor,
+ /* .clear = */ ggml_backend_hexagon_buffer_clear,
+ /* .reset = */ NULL,
+};
+
+// ** backend buffer type
+
+static void ggml_backend_hexagon_host_buffer_set_tensor(ggml_backend_buffer_t buffer,
+ ggml_tensor * tensor,
+ const void * data,
+ size_t offset,
+ size_t size) {
+ memcpy((char *) tensor->data + offset, data, size);
+ GGML_UNUSED(buffer);
+}
+
+static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t buffer,
+ const ggml_tensor * tensor,
+ void * data,
+ size_t offset,
+ size_t size) {
+ memcpy(data, (const char *) tensor->data + offset, size);
+ GGML_UNUSED(buffer);
+}
+
+static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = {
+ /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer,
+ /* .get_base = */ ggml_backend_hexagon_buffer_get_base,
+ /* .init_tensor = */ ggml_backend_hexagon_buffer_init_tensor,
+ /* .memset_tensor = */ NULL,
+ /* .set_tensor = */ ggml_backend_hexagon_host_buffer_set_tensor,
+ /* .get_tensor = */ ggml_backend_hexagon_host_buffer_get_tensor,
/* .set_tensor_2d = */ NULL,
/* .get_tensor_2d = */ NULL,
/* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor,
@@ -1066,24 +1496,22 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer(
ggml_backend_buffer_type_t buffer_type, size_t size) {
auto sess = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->sess;
try {
- size += 4 * 1024; // guard page
- ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size);
+ ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE);
return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size);
} catch (const std::exception & exc) {
- GGML_LOG_ERROR("ggml-hex: %s failed to allocate buffer context (host): %s\n", sess->c_name(), exc.what());
+ GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", sess->c_name(), exc.what());
return nullptr;
}
}
-static ggml_backend_buffer_t ggml_backend_hexagon_repack_buffer_type_alloc_buffer(
+static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer(
ggml_backend_buffer_type_t buffer_type, size_t size) {
auto sess = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->sess;
try {
- size += 4 * 1024; // guard page
- ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size);
- return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size);
+ ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE);
+ return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_host_buffer_interface, sbuf, size);
} catch (const std::exception & exc) {
- GGML_LOG_ERROR("ggml-hex: %s failed to allocate buffer context (repack): %s\n", sess->c_name(), exc.what());
+ GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", sess->c_name(), exc.what());
return nullptr;
}
}
@@ -1094,7 +1522,7 @@ static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer
}
static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) {
- if (t->type == GGML_TYPE_Q4_0 || t->type == GGML_TYPE_Q4_1 || t->type == GGML_TYPE_Q8_0 || t->type == GGML_TYPE_IQ4_NL || t->type == GGML_TYPE_MXFP4) {
+ if (ggml_hexagon_is_repack_type(t->type)) {
int64_t ne0 = hex_round_up(t->ne[0], 32);
int64_t ne1 = hex_round_up(t->ne[1], 32);
int64_t ne2 = t->ne[2];
@@ -1112,14 +1540,12 @@ static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_
}
static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
- return opt_hostbuf;
-
+ return false;
GGML_UNUSED(buft);
}
-static bool ggml_backend_hexagon_repack_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
- return false;
-
+static bool ggml_backend_hexagon_host_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
+ return true;
GGML_UNUSED(buft);
}
@@ -1132,26 +1558,19 @@ static ggml_backend_buffer_type_i ggml_backend_hexagon_buffer_type_interface = {
/* .is_host = */ ggml_backend_hexagon_buffer_type_is_host,
};
-static ggml_backend_buffer_type_i ggml_backend_hexagon_repack_buffer_type_interface = {
+static ggml_backend_buffer_type_i ggml_backend_hexagon_host_buffer_type_interface = {
/* .get_name = */ ggml_backend_hexagon_buffer_type_name,
- /* .alloc_buffer = */ ggml_backend_hexagon_repack_buffer_type_alloc_buffer,
+ /* .alloc_buffer = */ ggml_backend_hexagon_host_buffer_type_alloc_buffer,
/* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment,
/* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size,
/* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size,
- /* .is_host = */ ggml_backend_hexagon_repack_buffer_type_is_host,
+ /* .is_host = */ ggml_backend_hexagon_host_buffer_type_is_host,
};
static bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) {
return b->buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment;
}
-static inline bool ggml_backend_buffer_is_hexagon_repack(const struct ggml_backend_buffer * b) {
- if (!opt_hostbuf) {
- return ggml_backend_buffer_is_hexagon(b);
- }
- return b->buft->iface.alloc_buffer == ggml_backend_hexagon_repack_buffer_type_alloc_buffer;
-}
-
struct ggml_hexagon_opbatch {
ggml_hexagon_session* sess;
@@ -1165,8 +1584,6 @@ struct ggml_hexagon_opbatch {
std::unordered_map<const ggml_tensor*, int> t_map; // tensor ptr to index
std::unordered_multimap<void*, int> d_map; // tensor data to index
-
-
unsigned int n_bufs; // num buffers in the batch
unsigned int n_tens; // num tensors ...
unsigned int n_ops; // num ops ...
@@ -1186,6 +1603,7 @@ struct ggml_hexagon_opbatch {
b_map.clear();
t_map.clear();
d_map.clear();
+ ops.resize(n_ops_max);
}
ggml_hexagon_opbatch(ggml_hexagon_session *sess, size_t batch_size, size_t max_vmem) {
@@ -1218,39 +1636,39 @@ struct ggml_hexagon_opbatch {
// add buffer and return its index
int add_buffer(ggml_hexagon_shared_buffer * sbuf) {
// Lookup by fd
- auto it = b_map.find(sbuf->fd);
+ auto it = b_map.find(sbuf->fd());
if (it != b_map.end()) { return it->second; }
// Add new buffer to the batch
int bi = n_bufs++;
GGML_ASSERT(n_bufs < HTP_OP_MAX_BUFS);
- b_map.insert({sbuf->fd, bi});
+ b_map.insert({sbuf->fd(), bi});
htp_buf_desc &b = h_bufs[bi];
- b.base = (uint64_t) sbuf->base;
- b.fd = sbuf->fd;
- b.size = sbuf->size;
+ b.base = (uint64_t) sbuf->base();
+ b.fd = sbuf->fd();
+ b.size = sbuf->size();
b_vmem += b.size;
- HEX_VERBOSE("ggml-hex: %s add-buffer #%u : fd %d base %p size %zu : vmem %zu\n", sess->c_name(), bi, b.fd, (void*) sbuf->base, (size_t) b.size, b_vmem);
+ HEX_VERBOSE("ggml-hex: %s add-buffer #%u : fd %d base %p size %zu : vmem %zu\n", sess->c_name(), bi, b.fd, (void*) sbuf->base(), (size_t) b.size, b_vmem);
return bi;
}
-
-
bool same_shape(const htp_tensor * h, const ggml_tensor * t) const {
+ auto extra = (ggml_hexagon_tensor_extra *) t->extra;
+
int64_t ne0 = t->ne[0];
int64_t ne1 = t->ne[1];
- const bool is_repack = ggml_backend_buffer_is_hexagon_repack(t->buffer) && ggml_hexagon_is_repack_type(t->type);
+ const bool is_repack = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0;
if (is_repack) {
ne0 = hex_round_up(ne0, 32);
ne1 = hex_round_up(ne1, 32);
}
int64_t nb1 = is_repack ? ggml_row_size(t->type, ne0) : t->nb[1];
- int64_t nb2 = is_repack ? nb1 * ne1 : t->nb[2];
+ int64_t nb2 = is_repack ? nb1 * ne1 : t->nb[2];
int64_t nb3 = is_repack ? nb2 * t->ne[2] : t->nb[3];
return (h->type == t->type) &&
@@ -1260,7 +1678,8 @@ struct ggml_hexagon_opbatch {
// add tensor and return its index
int add_tensor(const ggml_tensor * t) {
- auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
+ auto extra = (ggml_hexagon_tensor_extra *) t->extra;
+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
// First lookup by tensor data
auto range = d_map.equal_range(t->data);
@@ -1280,7 +1699,7 @@ struct ggml_hexagon_opbatch {
t_map.insert({t, ti});
d_map.insert({t->data, ti});
- uint64_t t_offset = (uint8_t *) t->data - sbuf->base;
+ uint64_t t_offset = (uint8_t *) t->data - sbuf->base();
size_t t_size = ggml_nbytes(t);
htp_tensor &h = h_tens[ti];
@@ -1289,7 +1708,7 @@ struct ggml_hexagon_opbatch {
h.data = t_offset;
h.type = t->type;
- const bool is_repack = ggml_backend_buffer_is_hexagon_repack(t->buffer) && ggml_hexagon_is_repack_type(t->type);
+ const bool is_repack = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0;
if (is_repack) {
h.ne[0] = hex_round_up(t->ne[0], 32);
h.ne[1] = hex_round_up(t->ne[1], 32);
@@ -1308,11 +1727,15 @@ struct ggml_hexagon_opbatch {
h.nb[0] = t->nb[0]; h.nb[1] = t->nb[1]; h.nb[2] = t->nb[2]; h.nb[3] = t->nb[3];
}
-
-
h.flags = 0;
- if (ggml_backend_buffer_get_usage(t->buffer) != GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
- h.flags |= HTP_TENSOR_COMPUTE;
+ if ((extra->flags & GGML_HEXAGON_TENSOR_WEIGHT) != 0) {
+ h.flags |= HTP_TENSOR_WEIGHT;
+ }
+ if ((extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0) {
+ h.flags |= HTP_TENSOR_REPACK;
+ }
+ if ((extra->flags & GGML_HEXAGON_TENSOR_FENCE) != 0) {
+ h.flags |= HTP_TENSOR_FENCE;
}
HEX_VERBOSE("ggml-hex: %s add-tensor #%u %s : bi %d data %p offset %zu size %zu flags 0x%x : %zu:%zu:%zu:%zu\n", sess->c_name(),
@@ -1336,8 +1759,8 @@ struct ggml_hexagon_opbatch {
extra_tens++;
auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
- if (!b_map.count(sbuf->fd)) {
- extra_vmem += sbuf->size;
+ if (!b_map.count(sbuf->fd())) {
+ extra_vmem += sbuf->size();
extra_bufs += 1;
}
}
@@ -1372,32 +1795,497 @@ struct ggml_hexagon_opbatch {
o.opcode = node.opcode;
o.flags = 0;
- if (!(opt_opstage & HTP_OPSTAGE_COMPUTE)) {
- o.flags |= HTP_OPFLAGS_SKIP_COMPUTE;
- }
+ ggml_hexagon_dump_op_exec(sess->c_name(), ops[n], o.flags);
+
+ auto inputs = node.get_inputs();
+ for (unsigned int i=0; i < HTP_OP_MAX_INPUTS; i++) {
+ o.src[i] = (i < inputs.size() && inputs[i]) ? add_tensor(inputs[i]) : 0xffff;
+ }
+
+ auto outputs = node.get_outputs();
+ for (unsigned int i=0; i < HTP_OP_MAX_OUTPUTS; i++) {
+ o.dst[i] = (i < outputs.size() && outputs[i]) ? add_tensor(outputs[i]) : 0xffff;
+ }
+ }
+
+ bool try_fuse_allreduce_add(const htp_opnode & node) {
+ if (n_ops == 0 || opt_ar_select != 2) return false;
+ if (node.opcode != HTP_OP_ADD) return false;
+
+ htp_opnode & last_node = ops[n_ops - 1];
+ if (last_node.opcode != HTP_OP_ALLREDUCE) return false;
+
+ auto * ar_kparams = (struct htp_allreduce_kernel_params *) last_node.kernel_params;
+ const uint32_t rank = (uint32_t) ar_kparams->rank;
+ const ggml_tensor * ar_local = (rank < last_node.inputs.size()) ? last_node.inputs[rank] : nullptr;
+ const ggml_tensor * add_src0 = node.src0();
+ const ggml_tensor * add_src1 = node.src1();
+
+ if (!add_src0 || !add_src1 || !ar_local) return false;
+ if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false;
+
+ const ggml_tensor * res_tensor = nullptr;
+ if (add_src0 == ar_local || add_src0->data == ar_local->data) {
+ res_tensor = add_src1;
+ } else if (add_src1 == ar_local || add_src1->data == ar_local->data) {
+ res_tensor = add_src0;
+ } else {
+ return false;
+ }
+
+ if (!res_tensor || !res_tensor->data) return false;
+
+ if (ar_local->type != res_tensor->type) return false;
+
+ const bool is_same_shape = (ar_local->ne[0] == res_tensor->ne[0] && ar_local->ne[1] == res_tensor->ne[1] &&
+ ar_local->ne[2] == res_tensor->ne[2] && ar_local->ne[3] == res_tensor->ne[3]);
+ const bool is_row_bcast = (ar_local->ne[0] == res_tensor->ne[0] &&
+ res_tensor->ne[1] == 1 && res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1);
+
+ if (!is_same_shape && !is_row_bcast) return false;
+
+ if (is_same_shape) {
+ if (ar_local->nb[1] != res_tensor->nb[1] || ar_local->nb[2] != res_tensor->nb[2] ||
+ ar_local->nb[3] != res_tensor->nb[3]) {
+ return false;
+ }
+ if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(res_tensor)) {
+ return false;
+ }
+ }
+ if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(node.dst())) {
+ return false;
+ }
+
+ struct htp_allreduce_kernel_params new_kparams;
+ if (!ggml_hexagon_precompute_allreduce_params(
+ sess, node.dst(), (uint32_t) ar_kparams->rank, (uint32_t) ar_kparams->n_ranks, true, is_row_bcast, &new_kparams
+ )) {
+ HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", sess->c_name());
+ return false;
+ }
+
+ size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0;
+ auto fit_t = [&](const ggml_tensor * t) {
+ if (!t) return;
+ if (!t_map.count(t)) {
+ extra_tens++;
+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
+ if (!b_map.count(sbuf->fd())) {
+ extra_vmem += sbuf->size();
+ extra_bufs += 1;
+ }
+ }
+ };
+ fit_t(res_tensor);
+ fit_t(node.dst());
+ if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) {
+ return false;
+ }
+
+ last_node.opcode = HTP_OP_ALLREDUCE_ADD;
+ last_node.name = "ALLREDUCE+ADD";
+ last_node.inputs.push_back(res_tensor);
+ last_node.outputs.clear();
+ last_node.outputs.push_back(node.dst());
+ last_node.fused.push_back(node.node);
+ memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams));
+
+ htp_op_desc & o = h_ops[n_ops - 1];
+ o.opcode = HTP_OP_ALLREDUCE_ADD;
+ memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams));
+
+ const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks;
+ o.src[2 * n_ranks] = add_tensor(res_tensor);
+ o.dst[0] = add_tensor(node.dst());
+ for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) {
+ o.dst[d] = 0xffff;
+ }
+
+ HEX_VERBOSE("ggml-hex: %s fused ALLREDUCE+ADD (#%u)\n", sess->c_name(), n_ops - 1);
+ return true;
+ }
+
+ bool try_fuse_rms_norm_mul(const htp_opnode & node) {
+ if (n_ops == 0) return false;
+ if (node.opcode != HTP_OP_MUL) return false;
+
+ htp_opnode & last_node = ops[n_ops - 1];
+ if (last_node.opcode != HTP_OP_RMS_NORM) return false;
+
+ const ggml_tensor * mul_src0 = node.src0();
+ const ggml_tensor * mul_src1 = node.src1();
+ const ggml_tensor * rms_out = last_node.dst();
+
+ if (!mul_src0 || !mul_src1 || !rms_out) return false;
+ if (!ggml_hexagon_tensor_is_fuseable(rms_out)) return false;
+
+ const ggml_tensor * weight = nullptr;
+ if (mul_src0 == rms_out || mul_src0->data == rms_out->data) {
+ weight = mul_src1;
+ } else if (mul_src1 == rms_out || mul_src1->data == rms_out->data) {
+ weight = mul_src0;
+ } else {
+ return false;
+ }
+
+ if (!weight || !weight->data) return false;
+
+ const ggml_tensor * src0 = last_node.src0();
+ if (!src0 || !src0->data) return false;
+
+ if (src0->ne[0] != weight->ne[0] || src0->ne[0] != node.dst()->ne[0]) {
+ return false;
+ }
+
+ const bool is_row_bcast = (weight->ne[1] == 1 && weight->ne[2] == 1 && weight->ne[3] == 1);
+ const bool is_same_shape = (src0->ne[0] == weight->ne[0] && src0->ne[1] == weight->ne[1] &&
+ src0->ne[2] == weight->ne[2] && src0->ne[3] == weight->ne[3]);
+ if (!is_row_bcast && !is_same_shape) return false;
+
+ if (!ggml_are_same_shape(src0, node.dst())) {
+ return false;
+ }
+ if (ggml_is_contiguous(src0) != ggml_is_contiguous(node.dst())) {
+ return false;
+ }
+
+ struct htp_unary_kernel_params new_kparams;
+ ggml_hexagon_precompute_unary_params(
+ sess, HTP_OP_RMS_NORM_MUL, src0, weight, node.dst(), &new_kparams
+ );
+
+ if ((size_t) new_kparams.vtcm_size > sess->vtcm_size) {
+ HEX_VERBOSE("ggml-hex: %s skip RMS_NORM_MUL fusion: VTCM needed (%d) > budget (%zu)\n",
+ sess->c_name(), new_kparams.vtcm_size, sess->vtcm_size);
+ return false;
+ }
+
+ size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0;
+ auto fit_t = [&](const ggml_tensor * t) {
+ if (!t) return;
+ if (!t_map.count(t)) {
+ extra_tens++;
+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
+ if (!b_map.count(sbuf->fd())) {
+ extra_vmem += sbuf->size();
+ extra_bufs += 1;
+ }
+ }
+ };
+ fit_t(weight);
+ fit_t(node.dst());
+ if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) {
+ return false;
+ }
+
+ last_node.opcode = HTP_OP_RMS_NORM_MUL;
+ last_node.name = "RMS_NORM+MUL";
+ last_node.inputs.clear();
+ last_node.inputs.push_back(src0);
+ last_node.inputs.push_back(weight);
+ last_node.outputs.clear();
+ last_node.outputs.push_back(node.dst());
+ last_node.fused.push_back(node.node);
+ memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams));
+
+ htp_op_desc & o = h_ops[n_ops - 1];
+ o.opcode = HTP_OP_RMS_NORM_MUL;
+ memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams));
+
+ o.src[0] = add_tensor(src0);
+ o.src[1] = add_tensor(weight);
+ for (uint32_t s = 2; s < HTP_OP_MAX_INPUTS; s++) {
+ o.src[s] = 0xffff;
+ }
+ o.dst[0] = add_tensor(node.dst());
+ for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) {
+ o.dst[d] = 0xffff;
+ }
+
+ HEX_VERBOSE("ggml-hex: %s fused RMS_NORM+MUL (#%u)\n", sess->c_name(), n_ops - 1);
+ return true;
+ }
+
+ bool try_fuse_mul_mat_add(const htp_opnode & node) {
+ if (n_ops == 0) return false;
+ if (node.opcode != HTP_OP_ADD) return false;
+
+ htp_opnode & last_node = ops[n_ops - 1];
+ if (last_node.opcode != HTP_OP_MUL_MAT) return false;
+
+ const ggml_tensor * add_src0 = node.src0();
+ const ggml_tensor * add_src1 = node.src1();
+ const ggml_tensor * mm_out = last_node.dst();
+
+ if (!add_src0 || !add_src1 || !mm_out) return false;
+ if (!ggml_hexagon_tensor_is_fuseable(mm_out)) return false;
+
+ const ggml_tensor * src2 = nullptr;
+ if (add_src0 == mm_out || add_src0->data == mm_out->data) {
+ src2 = add_src1;
+ } else if (add_src1 == mm_out || add_src1->data == mm_out->data) {
+ src2 = add_src0;
+ } else {
+ return false;
+ }
+
+ if (!src2 || !src2->data) return false;
+
+ const ggml_tensor * src0 = last_node.src0();
+ const ggml_tensor * src1 = last_node.src1();
+ if (!src0 || !src1) return false;
+
+ struct htp_mm_kernel_params kparams;
+ ggml_hexagon_precompute_fused_matmul_add_params(sess, src0, src1, src2, node.dst(), &kparams);
+ const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3];
+ const bool can_fuse = (kparams.n_hmx > 0) || (src1_nrows == 1);
+ if (!can_fuse) return false;
+
+ if ((size_t) kparams.vtcm_size > sess->vtcm_size) {
+ HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: VTCM needed (%d) > budget (%zu)\n",
+ sess->c_name(), kparams.vtcm_size, sess->vtcm_size);
+ return false;
+ }
+
+ size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0;
+ auto fit_t = [&](const ggml_tensor * t) {
+ if (!t) return;
+ if (!t_map.count(t)) {
+ extra_tens++;
+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
+ if (!b_map.count(sbuf->fd())) {
+ extra_vmem += sbuf->size();
+ extra_bufs += 1;
+ }
+ }
+ };
+ fit_t(src2);
+ fit_t(node.dst());
+ if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) {
+ return false;
+ }
+
+ last_node.opcode = HTP_OP_MUL_MAT_ADD;
+ last_node.name = "MUL_MAT+ADD";
+ last_node.inputs.clear();
+ last_node.inputs.push_back(src0);
+ last_node.inputs.push_back(src1);
+ last_node.inputs.push_back(src2);
+ last_node.outputs.clear();
+ last_node.outputs.push_back(node.dst());
+ last_node.fused.push_back(node.node);
+ memcpy(last_node.kernel_params, &kparams, sizeof(kparams));
+
+ htp_op_desc & o = h_ops[n_ops - 1];
+ o.opcode = HTP_OP_MUL_MAT_ADD;
+ memcpy(o.kernel_params, &kparams, sizeof(kparams));
+
+ o.src[0] = add_tensor(src0);
+ o.src[1] = add_tensor(src1);
+ o.src[2] = add_tensor(src2);
+ for (uint32_t s = 3; s < HTP_OP_MAX_INPUTS; s++) {
+ o.src[s] = 0xffff;
+ }
+ o.dst[0] = add_tensor(node.dst());
+ for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) {
+ o.dst[d] = 0xffff;
+ }
+
+ HEX_VERBOSE("ggml-hex: %s fused MUL_MAT+ADD (#%u)\n", sess->c_name(), n_ops - 1);
+ return true;
+ }
+
+ bool try_fuse_mul_mat_nx(const htp_opnode & node) {
+ if (n_ops == 0 || node.opcode != HTP_OP_MUL_MAT) return false;
+ if (!is_mergeable_mul_mat(node.node)) return false;
+
+ const ggml_tensor * w_in = node.src0();
+ const ggml_tensor * x_in = node.src1();
+ const ggml_tensor * d_in = node.dst();
+ if (!w_in || !x_in || !d_in) return false;
+
+ htp_opnode & last_node = ops[n_ops - 1];
+
+ // Case 1: last_node is already MUL_MAT_NX
+ if (last_node.opcode == HTP_OP_MUL_MAT_NX) {
+ const uint32_t curr_n = (uint32_t) last_node.outputs.size();
+ if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) {
+ return false;
+ }
+
+ const ggml_tensor * w0 = last_node.inputs[0];
+ const ggml_tensor * x = last_node.inputs[curr_n];
+
+ if (x_in != x || w_in->type != w0->type || w_in->ne[0] != w0->ne[0]) {
+ return false;
+ }
+
+ struct htp_mm_kernel_params kparams;
+ ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, curr_n + 1, &kparams);
+ if ((size_t) kparams.vtcm_size > sess->vtcm_size) {
+ HEX_VERBOSE("ggml-hex: %s skip NX fusion: VTCM needed (%d) > budget (%zu)\n",
+ sess->c_name(), kparams.vtcm_size, sess->vtcm_size);
+ return false;
+ }
+
+ size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0;
+ auto fit_t = [&](const ggml_tensor * t) {
+ if (!t) return;
+ if (!t_map.count(t)) {
+ extra_tens++;
+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
+ if (!b_map.count(sbuf->fd())) {
+ extra_vmem += sbuf->size();
+ extra_bufs += 1;
+ }
+ }
+ };
+ fit_t(w_in);
+ fit_t(d_in);
+ if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) {
+ return false;
+ }
+
+ last_node.inputs[curr_n] = w_in;
+ last_node.inputs.push_back(x);
+ last_node.outputs.push_back(d_in);
+ last_node.fused.push_back(node.node);
+ memcpy(last_node.kernel_params, &kparams, sizeof(kparams));
+
+ htp_op_desc & o = h_ops[n_ops - 1];
+ memcpy(o.kernel_params, &kparams, sizeof(kparams));
+
+ for (uint32_t s = 0; s <= curr_n + 1; s++) {
+ o.src[s] = add_tensor(last_node.inputs[s]);
+ }
+ for (uint32_t s = curr_n + 2; s < HTP_OP_MAX_INPUTS; s++) {
+ o.src[s] = 0xffff;
+ }
+ for (uint32_t d = 0; d <= curr_n; d++) {
+ o.dst[d] = add_tensor(last_node.outputs[d]);
+ }
+ for (uint32_t d = curr_n + 1; d < HTP_OP_MAX_OUTPUTS; d++) {
+ o.dst[d] = 0xffff;
+ }
+
+ HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_NX (N=%u, #%u)\n", sess->c_name(), curr_n + 1, n_ops - 1);
+ return true;
+ }
+
+ // Case 2: last_node is single MUL_MAT
+ if (last_node.opcode == HTP_OP_MUL_MAT) {
+ if (!is_mergeable_mul_mat_pair(last_node.node, node.node)) {
+ return false;
+ }
+
+ const ggml_tensor * w0 = last_node.src0();
+ const ggml_tensor * x = last_node.src1();
+ const ggml_tensor * w1 = node.src0();
+ if (!w0 || !x || !w1) return false;
- ggml_hexagon_dump_op_exec(sess->c_name(), ops[n], o.flags);
+ struct htp_mm_kernel_params kparams;
+ ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, 2, &kparams);
+ if ((size_t) kparams.vtcm_size > sess->vtcm_size) {
+ HEX_VERBOSE("ggml-hex: %s skip NX fusion: VTCM needed (%d) > budget (%zu)\n",
+ sess->c_name(), kparams.vtcm_size, sess->vtcm_size);
+ return false;
+ }
- auto inputs = node.get_inputs();
- for (unsigned int i=0; i < HTP_OP_MAX_INPUTS; i++) {
- o.src[i] = (i < inputs.size() && inputs[i]) ? add_tensor(inputs[i]) : 0xffff;
- }
+ size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0;
+ auto fit_t = [&](const ggml_tensor * t) {
+ if (!t) return;
+ if (!t_map.count(t)) {
+ extra_tens++;
+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
+ if (!b_map.count(sbuf->fd())) {
+ extra_vmem += sbuf->size();
+ extra_bufs += 1;
+ }
+ }
+ };
+ fit_t(w1);
+ fit_t(node.dst());
+ if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) {
+ return false;
+ }
- auto outputs = node.get_outputs();
- for (unsigned int i=0; i < HTP_OP_MAX_OUTPUTS; i++) {
- o.dst[i] = (i < outputs.size() && outputs[i]) ? add_tensor(outputs[i]) : 0xffff;
+ const ggml_tensor * dst_0 = last_node.dst();
+ const ggml_tensor * dst_1 = node.dst();
+
+ last_node.opcode = HTP_OP_MUL_MAT_NX;
+ last_node.name = "MUL_MAT_NX";
+ last_node.inputs.clear();
+ last_node.inputs.push_back(w0);
+ last_node.inputs.push_back(w1);
+ last_node.inputs.push_back(x);
+ last_node.outputs.clear();
+ last_node.outputs.push_back(dst_0);
+ last_node.outputs.push_back(dst_1);
+ last_node.fused.push_back(node.node);
+ memcpy(last_node.kernel_params, &kparams, sizeof(kparams));
+
+ htp_op_desc & o = h_ops[n_ops - 1];
+ o.opcode = HTP_OP_MUL_MAT_NX;
+ memcpy(o.kernel_params, &kparams, sizeof(kparams));
+
+ o.src[0] = add_tensor(w0);
+ o.src[1] = add_tensor(w1);
+ o.src[2] = add_tensor(x);
+ for (uint32_t s = 3; s < HTP_OP_MAX_INPUTS; s++) {
+ o.src[s] = 0xffff;
+ }
+ o.dst[0] = add_tensor(dst_0);
+ o.dst[1] = add_tensor(dst_1);
+ for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) {
+ o.dst[d] = 0xffff;
+ }
+
+ HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_NX (N=2, #%u)\n", sess->c_name(), n_ops - 1);
+ return true;
}
+
+ return false;
}
- void finalize_ranges() {
+enum ggml_hexagon_fusion_flags {
+ GGML_HEXAGON_FUSE_ALLREDUCE_ADD = (1 << 1), // 2
+ GGML_HEXAGON_FUSE_RMS_NORM_MUL = (1 << 2), // 4
+ GGML_HEXAGON_FUSE_MUL_MAT_ADD = (1 << 3), // 8
+ GGML_HEXAGON_FUSE_MUL_MAT_NX = (1 << 4), // 16
+};
+
+static inline bool ggml_hexagon_is_fusion_enabled(int flag) {
+ if (opt_opfusion <= 0) return false;
+ if (opt_opfusion == 1) return true; // 1 enables all
+ return (opt_opfusion & flag) != 0;
+}
+
+ bool try_fuse(const htp_opnode & node) {
+ if (!opt_opfusion) return false;
+ if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_ALLREDUCE_ADD) && try_fuse_allreduce_add(node)) return true;
+ if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_RMS_NORM_MUL) && try_fuse_rms_norm_mul(node)) return true;
+ if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_ADD) && try_fuse_mul_mat_add(node)) return true;
+ if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_NX) && try_fuse_mul_mat_nx(node)) return true;
+ return false;
}
};
+struct ggml_hexagon_registry {
+ ggml_hexagon_registry(ggml_backend_reg_t reg);
+ ~ggml_hexagon_registry();
+
+ ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS];
+};
+
struct ggml_hexagon_opqueue {
// Shared buffer for storing batches
ggml_hexagon_shared_buffer *shm_buf;
size_t shm_blk_size;
+ uint64_t req_seq = 0;
+ uint64_t rsp_seq = 0;
+
using opvec = std::vector<htp_opnode>;
std::queue<unsigned int> done; // completed batch ids
@@ -1429,8 +2317,8 @@ struct ggml_hexagon_opqueue {
for (unsigned int i = 0; i < depth; i++) { done.push(i); }
if (opt_verbose) {
- GGML_LOG_INFO("ggml-hex: %s allocated op-queue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n",
- sess->c_name(), batch_size, depth, shm_buf->size, shm_blk_size);
+ GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n",
+ sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size);
}
}
@@ -1453,8 +2341,9 @@ struct ggml_hexagon_opqueue {
req.n_bufs = op_batch->n_bufs;
req.n_tensors = op_batch->n_tens;
req.n_ops = op_batch->n_ops;
+ req.seq = ++req_seq;
- op_cache[req.id] = op_batch->ops;
+ op_cache[req.id] = std::move(op_batch->ops);
start_usec[req.id] = ggml_time_us();
const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs;
@@ -1470,10 +2359,10 @@ struct ggml_hexagon_opqueue {
req.n_traces = 0;
}
- dbuf.ptr = shm_buf->base + (req.id * shm_blk_size);
- dbuf.fd = shm_buf->fd;
+ dbuf.ptr = shm_buf->base() + (req.id * shm_blk_size);
+ dbuf.fd = shm_buf->fd();
dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;
- dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base;
+ dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base();
dbuf.size = b_size + t_size + o_size + p_size + tr_size;
GGML_ASSERT(dbuf.size <= shm_blk_size);
@@ -1487,7 +2376,7 @@ struct ggml_hexagon_opqueue {
memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size);
memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size);
- HEX_VERBOSE("ggml-hex: %s op-queue push batch #%u : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n",
+ HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%u : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n",
shm_buf->sess->c_name(), req.id, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem,
b_size, t_size, o_size, (size_t) dbuf.size);
@@ -1530,33 +2419,40 @@ struct ggml_hexagon_opqueue {
const size_t m_size = b_size + t_size + o_size + p_size + tr_size;
GGML_ASSERT(m_size <= shm_blk_size);
- HEX_VERBOSE("ggml-hex: %s op-queue pop batch #%u : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n",
+ HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%u : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n",
shm_buf->sess->c_name(), rsp.id, rsp.n_bufs, rsp.n_tensors, rsp.n_ops,
(size_t) dbuf.size, b_size, t_size, o_size);
uint8_t * m_ptr = (uint8_t*) dbuf.ptr;
uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size);
- if (opt_profile && rsp.n_ops > 0) {
+ if (rsp.n_ops > 0) {
auto & ops = op_cache[rsp.id];
-
GGML_ASSERT(rsp.n_ops <= ops.size());
const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr;
-
const htp_trace_desc * trace_events = nullptr;
-
if (opt_profile == 3) {
trace_events = (const htp_trace_desc *) (p_ptr + p_size);
}
- ggml_hexagon_dump_batch_prof(shm_buf->sess->name, rsp);
+ if (opt_profile) {
+ ggml_hexagon_dump_batch_prof(shm_buf->sess->name, rsp);
+ }
for (uint32_t i = 0; i < rsp.n_ops; i++) {
- ggml_hexagon_dump_op_prof(shm_buf->sess->name, ops[i], pd[i]);
+ if (opt_profile) {
+ ggml_hexagon_dump_op_prof(shm_buf->sess->name, ops[i], pd[i]);
+ }
}
- ggml_hexagon_dump_trace_events(shm_buf->sess->name, rsp, trace_events, n_traces);
+ if (opt_profile) {
+ ggml_hexagon_dump_trace_events(shm_buf->sess->name, rsp, trace_events, n_traces);
+ }
+ }
+
+ if (rsp.seq > rsp_seq) {
+ rsp_seq = rsp.seq;
}
}
};
@@ -1601,10 +2497,8 @@ void ggml_hexagon_session::flush_pending(bool all) {
}
}
-void ggml_hexagon_session::flush_batch() {
- if (op_batch->empty()) { return; }
-
- op_batch->finalize_ranges();
+void ggml_hexagon_session::flush_batch(size_t min_ops) {
+ if (op_batch->n_ops < min_ops) { return; }
htp_opbatch_req req {};
dspqueue_buffer dbuf{};
@@ -1625,17 +2519,267 @@ void ggml_hexagon_session::flush_batch() {
}
}
+void ggml_hexagon_session::flush(bool all) {
+ flush_sync_peers();
+ flush_batch();
+ flush_pending(all);
+}
+
void ggml_hexagon_session::enqueue_op(const htp_opnode & node) {
+ for (auto t : node.get_inputs()) {
+ if (t && t->buffer && ggml_backend_buffer_is_hexagon(t->buffer)) {
+ if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != this) {
+ this->clone_buffer(static_cast<const ggml_hexagon_shared_buffer *>(t->buffer->context));
+ }
+ }
+ }
+ for (auto t : node.get_outputs()) {
+ if (t && t->buffer && ggml_backend_buffer_is_hexagon(t->buffer)) {
+ if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != this) {
+ this->clone_buffer(static_cast<const ggml_hexagon_shared_buffer *>(t->buffer->context));
+ }
+ }
+ }
+
+ if (opt_opfusion && op_batch->try_fuse(node)) {
+ return;
+ }
+
if (!op_batch->fit_op(node)) {
flush_batch();
}
op_batch->add_op(node);
}
-// Flush HTP response queue i.e wait for all outstanding requests to complete
-void ggml_hexagon_session::flush(bool all) {
+void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) {
+ htp_opnode cpy_node(HTP_OP_CPY);
+
+ ggml_tensor* node = cpy_node.add_dummy(*dst);
+ node->op = GGML_OP_CPY;
+ node->src[0] = const_cast<ggml_tensor *>(src);
+ node->src[1] = sync_tensor ? cpy_node.add_dummy(*sync_tensor) : nullptr;
+ if (sync_tensor) {
+ node->op_params[0] = (int32_t) fence_seq;
+ }
+
+ cpy_node.init(node);
+ if (sync_tensor) {
+ cpy_node.name = "CPY+FENCE";
+ }
+ this->enqueue_op(cpy_node);
+}
+
+void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq) {
+ htp_opnode sync_node(HTP_OP_FENCE);
+
+ ggml_tensor* node = sync_node.add_dummy(*sync_tensor);
+ node->op = GGML_OP_NONE;
+ node->src[0] = node;
+ node->op_params[0] = (int32_t) fence_seq;
+
+ sync_node.init(node);
+ sync_node.name = "FENCE";
+ this->enqueue_op(sync_node);
+}
+
+static bool ggml_hexagon_precompute_allreduce_params(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * dst,
+ uint32_t rank,
+ uint32_t n_ranks,
+ bool has_add,
+ bool is_row_bcast,
+ struct htp_allreduce_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+ kparams->rank = (int32_t) rank;
+ kparams->n_ranks = (int32_t) n_ranks;
+ kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0;
+
+ const uint32_t n_bufs = n_ranks + 1 + (has_add ? 1 : 0);
+ const uint32_t nelem = (uint32_t) ggml_nelements(dst);
+ const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);
+ const bool is_contiguous = ggml_is_contiguous(dst);
+
+ const uint32_t ne0 = (uint32_t) dst->ne[0];
+ const uint32_t ne1 = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
+ kparams->ne0 = (int32_t) ne0;
+ kparams->ne1 = (int32_t) ne1;
+
+ const bool use_1d = is_contiguous && !(has_add && is_row_bcast && ne1 > 1);
+
+ if (has_add) {
+ kparams->n_dsts = 1;
+ if (use_1d) {
+ kparams->rank_elem_start = 0;
+ kparams->rank_nelem = (int32_t) nelem;
+ } else {
+ kparams->rank_elem_start = 0;
+ kparams->rank_nelem = (int32_t) ne1;
+ }
+ } else {
+ kparams->n_dsts = (int32_t) n_ranks;
+ if (use_1d) {
+ const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128);
+ const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem);
+ const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem);
+ const uint32_t rank_nelem = rank_elem_end - rank_elem_start;
+ kparams->rank_elem_start = (int32_t) rank_elem_start;
+ kparams->rank_nelem = (int32_t) rank_nelem;
+ } else {
+ const uint32_t rank_chunk_rows = (ne1 + n_ranks - 1) / n_ranks;
+ const uint32_t rank_r0 = (std::min)(rank * rank_chunk_rows, ne1);
+ const uint32_t rank_r1 = (std::min)(rank_r0 + rank_chunk_rows, ne1);
+ const uint32_t rank_nrows = rank_r1 - rank_r0;
+ kparams->rank_elem_start = (int32_t) rank_r0;
+ kparams->rank_nelem = (int32_t) rank_nrows;
+ }
+ }
+
+ if (use_1d) {
+ const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem;
+ const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nelem / 128));
+ kparams->n_threads = n_threads;
+
+ uint32_t block_elems = 65536;
+ if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) {
+ block_elems = hex_round_up(rank_nelem / (n_threads * 2), 128);
+ }
+ block_elems = (std::max)(128u, block_elems);
+
+ kparams->block_elems = block_elems;
+ kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;
+ kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread;
+
+ while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_elems > 128) {
+ const size_t max_bytes_per_buf = sess->vtcm_size / (n_threads * n_bufs * 2);
+ block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128);
+ if (block_elems < 128) break;
+ kparams->block_elems = block_elems;
+ kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;
+ kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread;
+ }
+
+ if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) {
+ HEX_VERBOSE("ggml-hex: %s allreduce 1D solver failed to fit VTCM (%d > %zu)\n",
+ sess->c_name(), kparams->vtcm_size, sess->vtcm_size);
+ return false;
+ }
+
+ kparams->elems_per_thread = hex_round_up((rank_nelem + n_threads - 1) / n_threads, block_elems);
+ kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D;
+ return true;
+ } else {
+ const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem;
+ const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nrows));
+ kparams->n_threads = n_threads;
+
+ const uint32_t row_bytes = ne0 * elem_size;
+ const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128);
+ kparams->row_size_aligned = row_size_aligned;
+
+ const uint32_t nrows_per_thread = (rank_nrows + n_threads - 1) / n_threads;
+ uint32_t block_rows = (std::min)(128u, nrows_per_thread);
+ block_rows = (std::max)(1u, block_rows);
+ kparams->block_elems = block_rows;
+
+ kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);
+ kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread;
+
+ while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_rows > 1) {
+ const size_t max_rows_per_buf = sess->vtcm_size / (n_threads * n_bufs * 2 * row_size_aligned);
+ block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf);
+ kparams->block_elems = block_rows;
+ kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);
+ kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread;
+ if (max_rows_per_buf == 0) break;
+ }
+
+ if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_rows < 1) {
+ HEX_VERBOSE("ggml-hex: %s allreduce 2D solver failed to fit VTCM (%d > %zu)\n",
+ sess->c_name(), kparams->vtcm_size, sess->vtcm_size);
+ return false;
+ }
+
+ kparams->elems_per_thread = nrows_per_thread;
+ kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D;
+ return true;
+ }
+}
+
+void ggml_hexagon_session::enqueue_allreduce(
+ const ggml_tensor * dst,
+ const std::vector<const ggml_tensor *> & src_tensors,
+ const std::vector<const ggml_tensor *> & sync_tensors,
+ uint32_t rank,
+ uint32_t n_ranks,
+ uint32_t fence_seq_entry,
+ uint32_t fence_seq_exit
+) {
+ htp_opnode ar_node(HTP_OP_ALLREDUCE);
+
+ ggml_tensor* node = ar_node.add_dummy(*dst);
+ node->op = GGML_OP_NONE;
+ node->op_params[0] = (int32_t) fence_seq_entry;
+ node->op_params[1] = (int32_t) fence_seq_exit;
+
+ ar_node.init(node);
+
+ ar_node.inputs.clear();
+ for (size_t i = 0; i < src_tensors.size(); i++) {
+ ar_node.inputs.push_back(src_tensors[i]);
+ }
+ for (size_t i = 0; i < sync_tensors.size(); i++) {
+ ar_node.inputs.push_back(ar_node.add_dummy(*sync_tensors[i]));
+ }
+
+ ar_node.outputs.clear();
+ for (size_t i = 0; i < src_tensors.size(); i++) {
+ ar_node.outputs.push_back(src_tensors[i]);
+ }
+
+ ggml_hexagon_precompute_allreduce_params(
+ this, dst, rank, n_ranks, false, false,
+ (struct htp_allreduce_kernel_params *) ar_node.kernel_params
+ );
+
+ ar_node.name = "ALLREDUCE";
+ this->enqueue_op(ar_node);
+}
+
+void ggml_hexagon_session::wait_event(uint64_t seq) {
+ flush_sync_peers();
+ HEX_VERBOSE("ggml-hex: %s opqueue-wait start: seq %llu, current rsp-seq %llu, pending %d\n",
+ this->name.c_str(), (unsigned long long)seq, (unsigned long long)op_queue->rsp_seq, (int)this->op_pending);
+ while (op_queue->rsp_seq < seq && this->op_pending > 0) {
+ this->flush_pending(false);
+ }
+ HEX_VERBOSE("ggml-hex: %s opqueue-wait end: seq %llu, current rsp-seq %llu, pending %d\n",
+ this->name.c_str(), (unsigned long long)seq, (unsigned long long)op_queue->rsp_seq, (int)this->op_pending);
+}
+
+uint64_t ggml_hexagon_session::record_event() {
flush_batch();
- flush_pending(all);
+ return op_queue->req_seq;
+}
+
+bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer *sbuf)
+{
+ if (this->cloned_buffers.find(sbuf->fd()) != this->cloned_buffers.end()) return true;
+
+ HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(),
+ sbuf->c_name(), sbuf->base(), sbuf->size(), sbuf->fd());
+
+ auto clone = std::make_unique<ggml_hexagon_shared_buffer>(this, *sbuf);
+ try {
+ clone->mmap();
+ } catch (const std::exception & exc) {
+ GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what());
+ return false;
+ }
+
+ this->cloned_buffers[sbuf->fd()] = std::move(clone);
+ return true;
}
static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) {
@@ -1668,37 +2812,44 @@ static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) {
}
void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
+ const auto & config = opt_device_configs[dev_id];
+ int phys_idx = config.physical_idx;
+ int virt_idx = config.virtual_idx;
+
this->valid_session = false;
this->valid_handle = false;
this->valid_queue = false;
this->valid_iface = false;
- this->domain_id = 3; // Default for CDSP, updated after the session is created
- this->session_id = 0; // Default for CDSP, updated after the session is created
+ this->phys_idx = phys_idx;
+ this->virt_idx = virt_idx;
+ this->domain_id = get_domain_id(phys_idx);
+ this->session_id = 0;
this->dev_id = dev_id;
- this->name = std::string("HTP") + std::to_string(dev_id);
-
- this->op_pending = 0;
+ this->name = config.name;
+ this->op_pending = 0;
GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str());
domain * my_domain = htpdrv_get_domain(this->domain_id);
if (my_domain == NULL) {
- GGML_LOG_ERROR("ggml-hex: unable to get domain struct for CDSP\n");
+ GGML_LOG_ERROR("ggml-hex: unable to get domain struct for CDSP (domain_id %d)\n", this->domain_id);
throw std::runtime_error("ggml-hex: failed to get CDSP domain (see log for details)");
}
- // Create new session
- if (dev_id != 0) {
+ std::string dom_name = get_domain_name(phys_idx);
+
+ // Create new session if virtual_idx > 0
+ if (virt_idx > 0) {
struct remote_rpc_reserve_new_session n;
- n.domain_name_len = strlen(CDSP_DOMAIN_NAME);
- n.domain_name = const_cast<char *>(CDSP_DOMAIN_NAME);
+ n.domain_name_len = dom_name.size();
+ n.domain_name = const_cast<char *>(dom_name.c_str());
n.session_name = const_cast<char *>(this->name.c_str());
n.session_name_len = this->name.size();
int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n));
if (err != AEE_SUCCESS) {
- GGML_LOG_ERROR("ggml-hex: failed to reserve new session %d : error 0x%x\n", dev_id, err);
+ GGML_LOG_ERROR("ggml-hex: failed to reserve new session %d (physical %d, virtual %d) : error 0x%x\n", dev_id, phys_idx, virt_idx, err);
throw std::runtime_error("ggml-hex: remote_session_control(new-sess) failed (see log for details)");
}
@@ -1717,8 +2868,8 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
struct remote_rpc_get_uri u = {};
u.session_id = this->session_id;
- u.domain_name = const_cast<char *>(CDSP_DOMAIN_NAME);
- u.domain_name_len = strlen(CDSP_DOMAIN_NAME);
+ u.domain_name = const_cast<char *>(dom_name.c_str());
+ u.domain_name_len = dom_name.size();
u.module_uri = const_cast<char *>(htp_uri);
u.module_uri_len = strlen(htp_uri);
u.uri = session_uri;
@@ -1731,7 +2882,7 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
snprintf(session_uri, htp_URI_domain_len, "%s%s", htp_uri, my_domain->uri);
- GGML_LOG_WARN("ggml-hex: failed to get URI for session %d : error 0x%x. Falling back to single session URI: %s\n", dev_id, err, session_uri);
+ GGML_LOG_WARN("ggml-hex: failed to get URI for session %d (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n", dev_id, phys_idx, virt_idx, err, session_uri);
}
}
@@ -1899,14 +3050,17 @@ void ggml_hexagon_session::release() noexcept(true) {
if (this->valid_handle) {
htp_iface_close(this->handle);
}
+
+ this->cloned_buffers.clear();
}
ggml_hexagon_session::ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false) {
- buffer_type.device = dev;
- repack_buffer_type.device = dev;
+ buffer_type.device = dev;
+ host_buffer_type.device = dev;
op_batch = nullptr;
op_queue = nullptr;
+ fence_seq = ((uintptr_t)this) & 0xFFFF;
try {
allocate(dev_id);
@@ -1914,8 +3068,8 @@ ggml_hexagon_session::ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) n
buffer_type.iface = ggml_backend_hexagon_buffer_type_interface;
buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name, this);
- repack_buffer_type.iface = ggml_backend_hexagon_repack_buffer_type_interface;
- repack_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name + "-REPACK", this);
+ host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface;
+ host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name + "-HOST", this);
} catch (const std::exception & exc) {
release();
throw;
@@ -1926,7 +3080,7 @@ ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) {
release();
delete static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type.context);
- delete static_cast<ggml_backend_hexagon_buffer_type_context *>(repack_buffer_type.context);
+ delete static_cast<ggml_backend_hexagon_buffer_type_context *>(host_buffer_type.context);
}
// ** backend interface
@@ -1946,7 +3100,8 @@ static bool ggml_hexagon_flash_attn_is_hmx_eligible(
return false;
}
- if (k->type != GGML_TYPE_F16 || v->type != GGML_TYPE_F16) {
+ if ((k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) ||
+ (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) {
return false;
}
@@ -2098,8 +3253,10 @@ static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_sess
const struct ggml_tensor * src4 = op->src[4];
const struct ggml_tensor * dst = op;
- // Check for F16 support only as requested
- if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) || src1->type != GGML_TYPE_F16 || src2->type != GGML_TYPE_F16) {
+ // Check for F16/Q8_0 support
+ if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) ||
+ (src1->type != GGML_TYPE_F16 && src1->type != GGML_TYPE_Q8_0) ||
+ (src2->type != GGML_TYPE_F16 && src2->type != GGML_TYPE_Q8_0)) {
return false;
}
@@ -2352,7 +3509,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
htp_mm_hvx_vtcm_layout_build(
&L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,
- 0, src0->nb[1], 0, src2_row_size, d, true, false, false
+ 0, src0->nb[1], 0, src2_row_size, d, true, false
);
if (L.total_bytes <= vtcm_budget) {
best_n_prefetch = d;
@@ -2362,7 +3519,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
if (best_n_prefetch == 2 && L.total_bytes > vtcm_budget) {
htp_mm_hvx_vtcm_layout_build(
&L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,
- 0, src0->nb[1], 0, src2_row_size, 2, true, false, false
+ 0, src0->nb[1], 0, src2_row_size, 2, true, false
);
}
kparams->n_prefetch = best_n_prefetch;
@@ -2386,7 +3543,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
htp_mm_hvx_vtcm_layout_build(
&L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false, false
+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false
);
if (L.total_bytes <= vtcm_budget) {
best_n_prefetch = d;
@@ -2396,7 +3553,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
if (best_n_prefetch == 2 && L.total_bytes > vtcm_budget) {
htp_mm_hvx_vtcm_layout_build(
&L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 2, false, false, false
+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 2, false, false
);
}
@@ -2420,7 +3577,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(
&L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false, false
+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false
);
kparams->n_prefetch = 16;
@@ -2440,7 +3597,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(
&L, HTP_MM_KERNEL_HVX_F16_F16_VTCM, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false, false
+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false
);
if (!is_batched && !is_permuted && L.total_bytes <= vtcm_budget) {
@@ -2460,7 +3617,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
kparams->src1_row_size = src1->nb[1];
htp_mm_hvx_vtcm_layout_build(
&L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false, false
+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false
);
kparams->vtcm_size = L.total_bytes;
kparams->vtcm_src0_size = L.src0_bytes;
@@ -2476,7 +3633,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(
&L, HTP_MM_KERNEL_HVX_F32_F32_VTCM, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false, false
+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false
);
if (!is_batched && !is_permuted && L.total_bytes <= vtcm_budget) {
@@ -2492,7 +3649,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
kparams->src1_row_size = src1->nb[1];
htp_mm_hvx_vtcm_layout_build(
&L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,
- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false, false
+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, false, false
);
kparams->vtcm_size = L.total_bytes;
kparams->vtcm_src0_size = L.src0_bytes;
@@ -2642,80 +3799,99 @@ static void ggml_hexagon_precompute_unary_params(
kparams->div_tpr = init_fastdiv_values(tiles_per_row);
}
-static void ggml_hexagon_precompute_fused_qkv_params(
+static void ggml_hexagon_precompute_get_rows_params(
const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0, // Wk
- const struct ggml_tensor * src1, // x
- struct htp_mm_kernel_params * kparams
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1,
+ const struct ggml_tensor * dst,
+ struct htp_get_rows_kernel_params * kparams
) {
memset(kparams, 0, sizeof(*kparams));
- const int wtype = src0->type;
- const bool is_repack = ggml_hexagon_is_repack_type((ggml_type) wtype);
+ const uint32_t ne00 = src0->ne[0];
+ const uint32_t ne02 = src0->ne[2];
+ const uint32_t ne03 = src0->ne[3];
- const int ne10 = src1->ne[0];
- const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3];
- const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
- const size_t src0_row_size = src0->nb[1];
+ const uint32_t ne10 = src1->ne[0];
+ const uint32_t ne11 = src1->ne[1];
+ const uint32_t ne12 = src1->ne[2];
+ const uint32_t nr = ne10 * ne11 * ne12;
- uint32_t best_n_prefetch = 16;
+ const size_t nb01 = src0->nb[1];
+ const size_t nb1 = dst->nb[1];
- if (is_repack) {
- const uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;
- best_n_prefetch = 2;
- for (uint32_t d = max_prefetch; d >= 2; d /= 2) {
- struct htp_mm_hvx_vtcm_layout L;
- htp_mm_hvx_vtcm_layout_build(
- &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads,
- 0, src0_row_size, src1_row_size, 0, d, false, true, false
- );
- if (L.total_bytes <= sess->vtcm_size) {
- best_n_prefetch = d;
- break;
+ const bool can_use_dma = (src0->type == dst->type) && (nb01 == nb1);
+ const bool use_dma = can_use_dma && (ne00 >= 2048);
+
+ kparams->use_dma = use_dma ? 1 : 0;
+
+ uint32_t chunks_per_row = 1;
+ uint32_t chunk_size = ne00;
+ uint32_t total_tasks = nr;
+
+ if (use_dma) {
+ kparams->n_threads = (std::min)((uint32_t)sess->n_threads, nr);
+ kparams->tasks_per_thread = (nr + kparams->n_threads - 1) / kparams->n_threads;
+ } else {
+ if (src0->type == GGML_TYPE_F32 && nr < sess->n_threads) {
+ const uint32_t min_chunk_size = 1024;
+ uint32_t max_chunks = ne00 / min_chunk_size;
+ if (max_chunks == 0) {
+ max_chunks = 1;
}
+ chunks_per_row = (std::min)((sess->n_threads + nr - 1) / nr, max_chunks);
+ chunk_size = (ne00 + chunks_per_row - 1) / chunks_per_row;
+ total_tasks = nr * chunks_per_row;
}
+ kparams->n_threads = (std::min)(total_tasks, (uint32_t)sess->n_threads);
+ kparams->tasks_per_thread = (total_tasks + kparams->n_threads - 1) / kparams->n_threads;
}
- struct htp_mm_hvx_vtcm_layout L;
- bool try_tiled = (opt_mm_select >= 2);
+ kparams->chunks_per_row = chunks_per_row;
+ kparams->chunk_size = chunk_size;
+ kparams->total_tasks = total_tasks;
- // Test tiled first
- htp_mm_hvx_vtcm_layout_build(
- &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads,
- 0, src0_row_size, src1_row_size, 0, best_n_prefetch, false, true, false
- );
+ kparams->div_ne10 = init_fastdiv_values(ne10);
+ kparams->div_ne10_ne11 = init_fastdiv_values(ne10 * ne11);
+ kparams->div_chunks_per_row = init_fastdiv_values(chunks_per_row);
+ kparams->div_ne02 = init_fastdiv_values(ne02);
+ kparams->div_ne03 = init_fastdiv_values(ne03);
- if (try_tiled && L.total_bytes <= sess->vtcm_size) {
- kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW;
- kparams->vtcm_src0_size = L.src0_bytes;
- kparams->vtcm_src1_size = L.src1_bytes;
- kparams->vtcm_src2_size = L.src2_bytes;
- kparams->vtcm_src3_size = L.src3_bytes;
- kparams->vtcm_dst_size = L.dst_bytes;
- kparams->vtcm_size = L.total_bytes;
- kparams->n_prefetch = best_n_prefetch;
- } else {
- kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT;
- size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
+ struct htp_get_rows_vtcm_layout vtcm_layout;
+ htp_get_rows_vtcm_layout_build(&vtcm_layout, src0->type, ne00, kparams->n_threads);
+ kparams->vtcm_size = vtcm_layout.total_bytes;
+}
- htp_mm_hvx_vtcm_layout_build(
- &L, HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT, wtype, ne10, src1_nrows, sess->n_threads,
- 0, src0_row_size, flat_src1_row_size, 0, best_n_prefetch, false, true, false
- );
- kparams->vtcm_src0_size = L.src0_bytes;
- kparams->vtcm_src1_size = L.src1_bytes;
- kparams->vtcm_src2_size = L.src2_bytes;
- kparams->vtcm_src3_size = L.src3_bytes;
- kparams->vtcm_dst_size = L.dst_bytes;
- kparams->vtcm_size = L.total_bytes;
- kparams->n_prefetch = best_n_prefetch;
- }
+static void ggml_hexagon_precompute_set_rows_params(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0, // values
+ const struct ggml_tensor * src1, // indices
+ const struct ggml_tensor * dst, // destination
+ struct htp_set_rows_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ const uint32_t nr = src0->ne[1];
+
+ kparams->n_threads = (std::min)((uint32_t)sess->n_threads, nr);
+ kparams->tasks_per_thread = (nr + kparams->n_threads - 1) / kparams->n_threads;
+ kparams->total_tasks = nr;
+
+ kparams->div_ne11 = init_fastdiv_values(src1->ne[1]);
+ kparams->div_ne12 = init_fastdiv_values(src1->ne[2]);
+ kparams->div_tasks_per_thread = init_fastdiv_values(kparams->tasks_per_thread);
+ kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);
+
+ struct htp_set_rows_vtcm_layout vtcm_layout;
+ htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_threads);
+ kparams->vtcm_size = vtcm_layout.total_bytes;
}
-static void ggml_hexagon_precompute_fused_ffn_params(
+static void ggml_hexagon_precompute_fused_mmnx_params(
const struct ggml_hexagon_session * sess,
- const struct ggml_tensor * src0, // Wgate
- const struct ggml_tensor * src1, // y
+ const struct ggml_tensor * src0, // W0
+ const struct ggml_tensor * src1, // x
+ int32_t n_weights,
struct htp_mm_kernel_params * kparams
) {
memset(kparams, 0, sizeof(*kparams));
@@ -2737,7 +3913,7 @@ static void ggml_hexagon_precompute_fused_ffn_params(
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(
&L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads,
- 0, src0_row_size, src1_row_size, 0, d, false, false, true
+ 0, src0_row_size, src1_row_size, 0, d, false, true
);
if (L.total_bytes <= sess->vtcm_size) {
best_n_prefetch = d;
@@ -2752,34 +3928,42 @@ static void ggml_hexagon_precompute_fused_ffn_params(
// Test tiled first
htp_mm_hvx_vtcm_layout_build(
&L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads,
- 0, src0_row_size, src1_row_size, 0, best_n_prefetch, false, false, true
+ 0, src0_row_size, src1_row_size, 0, best_n_prefetch, false, true
);
if (try_tiled && L.total_bytes <= sess->vtcm_size) {
kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW;
kparams->vtcm_src0_size = L.src0_bytes;
kparams->vtcm_src1_size = L.src1_bytes;
- kparams->vtcm_src2_size = L.src2_bytes;
kparams->vtcm_dst_size = L.dst_bytes;
kparams->vtcm_size = L.total_bytes;
kparams->n_prefetch = best_n_prefetch;
+ kparams->n_weights = n_weights;
} else {
kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT;
size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
htp_mm_hvx_vtcm_layout_build(
&L, HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT, wtype, ne10, src1_nrows, sess->n_threads,
- 0, src0_row_size, flat_src1_row_size, 0, best_n_prefetch, false, false, true
+ 0, src0_row_size, flat_src1_row_size, 0, best_n_prefetch, false, true
);
kparams->vtcm_src0_size = L.src0_bytes;
kparams->vtcm_src1_size = L.src1_bytes;
- kparams->vtcm_src2_size = L.src2_bytes;
kparams->vtcm_dst_size = L.dst_bytes;
kparams->vtcm_size = L.total_bytes;
kparams->n_prefetch = best_n_prefetch;
+ kparams->n_weights = n_weights;
}
}
+static bool ggml_hexagon_tensor_is_host(const struct ggml_hexagon_session * sess, const struct ggml_tensor * t) {
+ return t && t->buffer && t->buffer->buft == &sess->host_buffer_type;
+}
+
+static bool ggml_hexagon_tensor_is_non_host(const struct ggml_hexagon_session * sess, const struct ggml_tensor * t) {
+ return t && t->buffer && t->buffer->buft != &sess->host_buffer_type;
+}
+
static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
@@ -2811,9 +3995,8 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s
return false; // no broadcasting (for now)
}
- // src0 (weights) must be repacked
- if (src0->buffer && !ggml_backend_buffer_is_hexagon_repack(src0->buffer)) {
- return false;
+ if (!src0->buffer) {
+ sess->needs_repack.insert(src0);
}
break;
@@ -2872,9 +4055,8 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session
return false;
}
- // src0 (weights) must be repacked
- if (src0->buffer && !ggml_backend_buffer_is_hexagon_repack(src0->buffer)) {
- return false;
+ if (!src0->buffer) {
+ sess->needs_repack.insert(src0);
}
break;
@@ -2970,7 +4152,7 @@ static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * ses
if (dst->type != GGML_TYPE_F32) {
return false;
}
- if (ggml_is_permuted(src0)) {
+ if (!ggml_is_contiguous_rows(src0)) {
return false;
}
if (!ggml_are_same_shape(src0, dst)) {
@@ -3114,7 +4296,11 @@ static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * s
static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
const struct ggml_tensor * src0 = op->src[0]; // values
const struct ggml_tensor * src1 = op->src[1]; // indices
- const struct ggml_tensor * dst = op;
+ const struct ggml_tensor * dst = op->src[2] ? op->src[2] : op;
+
+ if (dst->type == GGML_TYPE_Q8_0 && src0->ne[0] < 32) {
+ return false;
+ }
if (src0->type != GGML_TYPE_F32) {
return false;
@@ -3124,7 +4310,7 @@ static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session *
return false;
}
- if (dst->type != GGML_TYPE_F16) {
+ if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_Q8_0) {
return false;
}
@@ -3138,7 +4324,11 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
const struct ggml_tensor * src1 = op->src[1]; // indices
const struct ggml_tensor * dst = op;
- if (src0->type != GGML_TYPE_F32) {
+ if (src0->type != GGML_TYPE_F32 && src0->ne[0] < 32) {
+ return false;
+ }
+
+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_Q8_0) {
return false;
}
@@ -3521,133 +4711,22 @@ static bool is_mergeable_mul_mat(const ggml_tensor * t) {
if (!t || t->op != GGML_OP_MUL_MAT) return false;
if (t->src[1]->type != GGML_TYPE_F32) return false;
return ggml_is_quantized(t->src[0]->type) && !mm_is_hmx_eligible(t);
-}
-
-static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2) {
- if (!is_mergeable_mul_mat(n1) || !is_mergeable_mul_mat(n2)) {
- return false;
- }
- if (n1->src[1] != n2->src[1]) {
- return false;
- }
- if (n1->src[0]->ne[0] != n2->src[0]->ne[0] ||
- n1->src[0]->ne[1] != n2->src[0]->ne[1]) {
- return false;
- }
- if (n1->src[0]->type != n2->src[0]->type) {
- return false;
- }
- return true;
-}
-
-static bool is_qkv_mergeable(const ggml_tensor * n_q, const ggml_tensor * n_k, const ggml_tensor * n_v) {
- if (!is_mergeable_mul_mat(n_q) || !is_mergeable_mul_mat(n_k) || !is_mergeable_mul_mat(n_v)) {
- return false;
- }
- if (n_q->src[1] != n_k->src[1] || n_q->src[1] != n_v->src[1]) {
- return false;
- }
- if (n_q->src[0]->type != n_k->src[0]->type || n_q->src[0]->type != n_v->src[0]->type) {
- return false;
- }
- if (n_k->src[0]->ne[0] != n_v->src[0]->ne[0] ||
- n_k->src[0]->ne[1] != n_v->src[0]->ne[1]) {
- return false;
- }
- if (n_q->src[0]->ne[0] != n_k->src[0]->ne[0]) {
- return false;
- }
- return true;
-}
-
-static bool try_fuse_node(const ggml_hexagon_session * sess, const ggml_cgraph * graph, int & i, std::vector<htp_opnode> & nodes) {
- if (!opt_opfusion) {
- return false;
- }
-
- ggml_tensor * n = graph->nodes[i];
- ggml_tensor * next_node = (i + 1 < graph->n_nodes) ? graph->nodes[i + 1] : nullptr;
-
- if (n->op == GGML_OP_RMS_NORM && next_node) {
- if (next_node->op == GGML_OP_MUL && op_is_compute(next_node) && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {
- htp_opnode node(n, {}, HTP_OP_RMS_NORM_MUL);
- node.add_fused(next_node);
-
- auto inputs = node.get_inputs();
- const struct ggml_tensor * src0 = inputs[0];
- const struct ggml_tensor * src1 = inputs.size() > 1 ? inputs[1] : nullptr;
- ggml_hexagon_precompute_unary_params(sess,
- node.opcode, src0, src1, node.dst(),
- (struct htp_unary_kernel_params *)node.kernel_params
- );
-
- nodes.push_back(std::move(node));
- i++; // skip the fused MUL node
- return true;
- }
- }
-
- if (is_mergeable_mul_mat(n)) {
- ggml_tensor * n1 = (i + 1 < graph->n_nodes) ? graph->nodes[i + 1] : nullptr;
- ggml_tensor * n2 = (i + 2 < graph->n_nodes) ? graph->nodes[i + 2] : nullptr;
- if (is_qkv_mergeable(n, n1, n2)) {
- struct htp_mm_kernel_params kparams;
- ggml_hexagon_precompute_fused_qkv_params(sess, n1->src[0], n1->src[1], &kparams);
- if ((size_t)kparams.vtcm_size <= sess->vtcm_size) {
- // Reorder to KVQ: K (n1), V (n2), Q (n)
- htp_opnode node(n1, {}, HTP_OP_MUL_MAT_QKV);
- node.add_fused(n2, true);
- node.add_fused(n, true);
- memcpy(node.kernel_params, &kparams, sizeof(kparams));
- nodes.push_back(std::move(node));
- i += 2;
- return true;
- } else {
- HEX_VERBOSE("ggml-hex: skip QKV fusion because VTCM needed (%d) > budget (%zu)\n",
- kparams.vtcm_size, sess->vtcm_size);
- }
- }
- if (is_mergeable_mul_mat_pair(n, n1)) {
- struct htp_mm_kernel_params kparams;
- ggml_hexagon_precompute_fused_ffn_params(sess, n->src[0], n->src[1], &kparams);
- if ((size_t)kparams.vtcm_size <= sess->vtcm_size) {
- htp_opnode node(n, {}, HTP_OP_MUL_MAT_FFN);
- node.add_fused(n1, true);
- memcpy(node.kernel_params, &kparams, sizeof(kparams));
- nodes.push_back(std::move(node));
- i += 1;
- return true;
- } else {
- HEX_VERBOSE("ggml-hex: skip FFN fusion because VTCM needed (%d) > budget (%zu)\n",
- kparams.vtcm_size, sess->vtcm_size);
- }
- }
- }
+}
- if (n->op == GGML_OP_MUL_MAT && next_node) {
- if (next_node->op == GGML_OP_ADD && op_is_compute(next_node) && ggml_can_fuse(graph, i, { GGML_OP_MUL_MAT, GGML_OP_ADD })) {
- if (next_node->src[0] == n || next_node->src[1] == n) {
- const struct ggml_tensor * src2 = (next_node->src[0] == n) ? next_node->src[1] : next_node->src[0];
- struct htp_mm_kernel_params kparams;
- ggml_hexagon_precompute_fused_matmul_add_params(sess, n->src[0], n->src[1], src2, next_node, &kparams);
- const int src1_nrows = n->src[1]->ne[1] * n->src[1]->ne[2] * n->src[1]->ne[3];
- const bool can_fuse = (kparams.n_hmx > 0) || (src1_nrows == 1);
- if (can_fuse && (size_t)kparams.vtcm_size <= sess->vtcm_size) {
- htp_opnode node(n, {}, HTP_OP_MUL_MAT_ADD);
- node.add_fused(next_node);
- memcpy(node.kernel_params, &kparams, sizeof(kparams));
- nodes.push_back(std::move(node));
- i += 1;
- return true;
- } else if (can_fuse) {
- HEX_VERBOSE("ggml-hex: skip MUL_MAT_ADD fusion because VTCM needed (%d) > budget (%zu)\n",
- kparams.vtcm_size, sess->vtcm_size);
- }
- }
- }
+static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2) {
+ if (!is_mergeable_mul_mat(n1) || !is_mergeable_mul_mat(n2)) {
+ return false;
}
-
- return false;
+ if (n1->src[1] != n2->src[1]) {
+ return false;
+ }
+ if (n1->src[0]->ne[0] != n2->src[0]->ne[0]) {
+ return false;
+ }
+ if (n1->src[0]->type != n2->src[0]->type) {
+ return false;
+ }
+ return true;
}
static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) {
@@ -3659,24 +4738,34 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
std::vector<htp_opnode> computed_nodes;
// Check for cache hit
- bool cache_hit = (graph->uid != 0 && sess->cached_graph.uid == graph->uid);
+ bool cache_hit = (graph->uid != 0 && sess->cached_uid == graph->uid);
if (cache_hit) {
- nodes_ptr = &sess->cached_graph.htp_nodes;
+ nodes_ptr = &sess->cached_nodes;
} else {
+ // Tag fusable tensors in graph
+ for (int i = 0; i < graph->n_nodes; i++) {
+ auto * extra = (ggml_hexagon_tensor_extra *) graph->nodes[i]->extra;
+ if (!extra) continue;
+
+ if (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {
+ extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;
+ } else if (graph->nodes[i]->op == GGML_OP_MUL_MAT) {
+ if ((i + 1 < graph->n_nodes && graph->nodes[i + 1]->op == GGML_OP_ADD && ggml_can_fuse(graph, i, { GGML_OP_MUL_MAT, GGML_OP_ADD })) ||
+ ggml_node_has_n_uses(graph, i, 1)) {
+ extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;
+ }
+ }
+ }
+
computed_nodes.reserve(graph->n_nodes);
- // Fuse and finalize
for (int i = 0; i < graph->n_nodes; ++i) {
ggml_tensor * n = graph->nodes[i];
if (!op_is_compute(n)) {
continue;
}
- if (try_fuse_node(sess, graph, i, computed_nodes)) {
- continue;
- }
-
- htp_opnode node(n, {}, HTP_OP_INVALID);
+ htp_opnode node(HTP_OP_INVALID, n);
node.opcode = op_remap_to_htp(n);
if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID) {
ggml_hexagon_precompute_matmul_params(sess,
@@ -3696,29 +4785,34 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
node.opcode, src0, src1, node.dst(),
(struct htp_unary_kernel_params *)node.kernel_params
);
+ } else if (node.opcode == HTP_OP_GET_ROWS) {
+ ggml_hexagon_precompute_get_rows_params(sess,
+ node.node->src[0], node.node->src[1], node.dst(),
+ (struct htp_get_rows_kernel_params *)node.kernel_params
+ );
+ } else if (node.opcode == HTP_OP_SET_ROWS) {
+ ggml_hexagon_precompute_set_rows_params(sess,
+ node.node->src[0], node.node->src[1], node.dst(),
+ (struct htp_set_rows_kernel_params *)node.kernel_params
+ );
}
computed_nodes.push_back(std::move(node));
}
if (graph->uid != 0) {
- sess->cached_graph.uid = graph->uid;
- sess->cached_graph.htp_nodes = std::move(computed_nodes);
- nodes_ptr = &sess->cached_graph.htp_nodes;
+ sess->cached_uid = graph->uid;
+ sess->cached_nodes = std::move(computed_nodes);
+ nodes_ptr = &sess->cached_nodes;
} else {
nodes_ptr = &computed_nodes;
}
}
// Queue and execute
- if (opt_opstage & HTP_OPSTAGE_QUEUE) {
- for (const auto & node : *nodes_ptr) {
- sess->enqueue_op(node);
- }
+ for (const auto & node : *nodes_ptr) {
+ sess->enqueue_op(node);
}
- // Wait until all pending ops complete
- sess->flush();
-
return GGML_STATUS_SUCCESS;
}
@@ -3731,6 +4825,106 @@ static void ggml_backend_hexagon_synchronize(ggml_backend_t backend) {
sess->flush();
}
+enum ggml_hexagon_mem_range_type {
+ HEXAGON_MEM_RANGE_TYPE_SRC,
+ HEXAGON_MEM_RANGE_TYPE_DST,
+};
+
+struct ggml_hexagon_mem_range {
+ uint64_t pb;
+ uint64_t p0;
+ uint64_t p1;
+ ggml_hexagon_mem_range_type pt;
+};
+
+struct ggml_hexagon_mem_ranges {
+ std::vector<ggml_hexagon_mem_range> ranges;
+
+ void reset() {
+ ranges.clear();
+ }
+
+ void add(const ggml_hexagon_mem_range & mr) {
+ ranges.push_back(mr);
+ }
+
+ bool check(const ggml_hexagon_mem_range & mr) const {
+ for (const auto & cmp : ranges) {
+ if (mr.pb != cmp.pb) {
+ continue;
+ }
+ if (mr.pt == HEXAGON_MEM_RANGE_TYPE_SRC && cmp.pt == HEXAGON_MEM_RANGE_TYPE_SRC) {
+ continue;
+ }
+ if (mr.p0 < cmp.p1 && mr.p1 > cmp.p0) {
+ return false;
+ }
+ }
+ return true;
+ }
+};
+
+static ggml_hexagon_mem_range ggml_hexagon_mem_range_from_tensor(const ggml_tensor * tensor, ggml_hexagon_mem_range_type pt) {
+ const ggml_tensor * base = tensor->view_src ? tensor->view_src : tensor;
+ ggml_hexagon_mem_range mr;
+ if (tensor->buffer) {
+ mr = {
+ /*.pb =*/ (uint64_t) tensor->buffer,
+ /*.p0 =*/ (uint64_t) tensor->data,
+ /*.p1 =*/ (uint64_t) tensor->data + ggml_backend_buft_get_alloc_size(tensor->buffer->buft, tensor),
+ /*.pt =*/ pt,
+ };
+ } else {
+ mr = {
+ /*.pb =*/ (uint64_t) base,
+ /*.p0 =*/ 0,
+ /*.p1 =*/ 1024,
+ /*.pt =*/ pt,
+ };
+ }
+ return mr;
+}
+
+static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, const htp_opnode & node) {
+ if (node.is_empty()) return;
+
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ if (node.node->src[i]) {
+ mrs.add(ggml_hexagon_mem_range_from_tensor(node.node->src[i], HEXAGON_MEM_RANGE_TYPE_SRC));
+ }
+ }
+ for (const auto * fused : node.fused) {
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ if (fused->src[i]) {
+ mrs.add(ggml_hexagon_mem_range_from_tensor(fused->src[i], HEXAGON_MEM_RANGE_TYPE_SRC));
+ }
+ }
+ }
+ mrs.add(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST));
+}
+
+static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & mrs, const htp_opnode & node) {
+ if (node.is_empty()) return true;
+
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ if (node.node->src[i]) {
+ if (!mrs.check(ggml_hexagon_mem_range_from_tensor(node.node->src[i], HEXAGON_MEM_RANGE_TYPE_SRC))) {
+ return false;
+ }
+ }
+ }
+ for (const auto * fused : node.fused) {
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
+ if (fused->src[i]) {
+ if (!mrs.check(ggml_hexagon_mem_range_from_tensor(fused->src[i], HEXAGON_MEM_RANGE_TYPE_SRC))) {
+ return false;
+ }
+ }
+ }
+ }
+ return mrs.check(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST));
+}
+
static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<htp_opnode> & nodes) {
const int n = nodes.size();
@@ -3739,28 +4933,32 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<ht
std::vector<bool> used(n, false);
- // The main goal here is to stack the MUL_MAT ops with the same src1 input.
- // This allows use to reuse dynamically quantized src1 in VTCM.
+ ggml_hexagon_mem_ranges mrs;
- // TODO: the current version might do incorrect reordering in cases where quantized src0
- // input is an output of another Op.
+ // The main goal here is to stack the MUL_MAT ops with the same src1 input.
+ // This allows us to reuse dynamically quantized src1 in VTCM.
for (int i0 = 0; i0 < n; i0++) {
if (used[i0]) {
continue;
}
- res.push_back(i0);
-
const auto & node0 = nodes[i0];
if (!node0.stackable()) {
+ res.push_back(i0);
+ used[i0] = true;
continue;
}
// that many nodes forward to search for stackable nodes that can reuse VTCM
constexpr int N_FORWARD = 16;
+ std::vector<int> stack;
+ stack.push_back(i0);
+
+ mrs.reset();
+
for (int i1 = i0 + 1; i1 < i0 + N_FORWARD && i1 < n; i1++) {
if (used[i1]) {
continue;
@@ -3768,11 +4966,17 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<ht
const auto & node1 = nodes[i1];
- if (node1.stackable() && node1.same_input(node0)) {
- res.push_back(i1);
- used[i1] = true;
+ if (node1.stackable() && node1.same_input(node0) && ggml_hexagon_mem_ranges_check_node(mrs, node1)) {
+ stack.push_back(i1);
+ } else {
+ ggml_hexagon_mem_ranges_add_node(mrs, node1);
}
}
+
+ for (int idx : stack) {
+ res.push_back(idx);
+ used[idx] = true;
+ }
}
return res;
@@ -3788,14 +4992,9 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr
std::vector<htp_opnode> nodes;
nodes.reserve(gf->n_nodes);
- // fuse nodes:
- // we don't want to make reorders that break fusing, so we first pack all fusable tensors
- // and perform the reorder over the fused nodes. after the reorder is done, we unfuse
+ // Pack nodes for reordering
for (int i = 0; i < n; i++) {
- htp_opnode node = {
- /*.node =*/gf->nodes[i],
- /*.fused =*/{},
- };
+ htp_opnode node(HTP_OP_INVALID, gf->nodes[i]);
// fuse only ops that start with these operations
// can be expanded when needed
@@ -3856,22 +5055,200 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr
GGML_UNUSED(backend);
}
+static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {
+ auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
+ auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
+ auto sbuf_dst = (ggml_hexagon_shared_buffer *) dst->buffer->context;
+
+ if (sess_dst->fence_seq == 0) sess_dst->fence_seq = 1;
+ uint32_t fence_seq = sess_dst->fence_seq++;
+ if (sess_dst->fence_seq == 0) sess_dst->fence_seq = 1;
+
+ volatile uint32_t * fence = (volatile uint32_t *) sbuf_dst->alloc_fence();
+
+ HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq %u\n",
+ sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src), fence_seq);
+
+ // dummy extra (must be static)
+ static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE };
+
+ ggml_tensor fence_tensor {};
+ fence_tensor.buffer = dst->buffer;
+ fence_tensor.extra = &fence_extra;
+ fence_tensor.data = (void *) fence;
+ fence_tensor.type = GGML_TYPE_I32;
+ fence_tensor.ne[0] = 1;
+ fence_tensor.ne[1] = 1;
+ fence_tensor.ne[2] = 1;
+ fence_tensor.ne[3] = 1;
+ fence_tensor.nb[0] = sizeof(int32_t);
+ fence_tensor.nb[1] = sizeof(int32_t);
+ fence_tensor.nb[2] = sizeof(int32_t);
+ fence_tensor.nb[3] = sizeof(int32_t);
+ fence_tensor.op = GGML_OP_NONE;
+
+ sess_src->enqueue_cpy(src, dst, &fence_tensor, fence_seq);
+ sess_dst->enqueue_fence(&fence_tensor, fence_seq);
+
+ sess_dst->add_sync_peer(sess_src);
+
+ return true;
+}
+
+static bool ggml_hexagon_cpy_tensor_async_virt(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {
+ auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
+ auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
+ auto sbuf_dst = (ggml_hexagon_shared_buffer *) dst->buffer->context;
+
+ if (!sess_src->clone_buffer(sbuf_dst)) { return false; }
+
+ HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n",
+ sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src));
+
+ sess_src->enqueue_cpy(src, dst);
+ sess_src->flush(true);
+
+ return true;
+}
+
+static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {
+ if (!ggml_backend_is_hexagon(backend_src) || !ggml_backend_is_hexagon(backend_dst)) {
+ return false;
+ }
+
+ *(ggml_hexagon_tensor_extra *) dst->extra = *(const ggml_hexagon_tensor_extra *) src->extra;
+
+ auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
+ auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
+
+ if (sess_src == sess_dst) {
+ HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src));
+ sess_src->enqueue_cpy(src, dst);
+ sess_src->flush_batch();
+ return true;
+ }
+
+ if (sess_src->phys_idx != sess_dst->phys_idx)
+ return ggml_hexagon_cpy_tensor_async_phys(backend_src, backend_dst, src, dst);
+
+ return ggml_hexagon_cpy_tensor_async_virt(backend_src, backend_dst, src, dst);
+}
+
+static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_dev_t dev) {
+ ggml_hexagon_event * hex_event = new ggml_hexagon_event();
+ HEX_VERBOSE("ggml-hex: %s event-new : event %p\n", ggml_backend_dev_name(dev), (void *)hex_event);
+
+ return new ggml_backend_event {
+ /* .device = */ dev,
+ /* .context = */ hex_event,
+ };
+}
+
+static void ggml_backend_hexagon_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) {
+ GGML_UNUSED(dev);
+
+ if (event == nullptr) {
+ return;
+ }
+
+ ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context;
+ HEX_VERBOSE("ggml-hex: %s event-free : event %p\n", ggml_backend_dev_name(dev), (void *)hex_event);
+ delete hex_event;
+ delete event;
+}
+
+static void ggml_backend_hexagon_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) {
+ GGML_UNUSED(dev);
+
+ ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context;
+ HEX_VERBOSE("ggml-hex: %s event-synchronize : event %p seq %llu\n",
+ ggml_backend_dev_name(dev), (void *)hex_event, (unsigned long long)hex_event->seq);
+ if (hex_event->sess != nullptr) {
+ hex_event->sess->wait_event(hex_event->seq);
+ }
+}
+
+static void ggml_backend_hexagon_event_record(ggml_backend_t backend, ggml_backend_event_t event) {
+ auto sess = static_cast<ggml_hexagon_session *>(backend->context);
+ ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context;
+
+ hex_event->sess = sess;
+ hex_event->seq = sess->record_event();
+ HEX_VERBOSE("ggml-hex: %s event-record : event %p seq %llu\n",
+ sess->c_name(), (void *)hex_event, (unsigned long long)hex_event->seq);
+}
+
+static void ggml_backend_hexagon_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {
+ GGML_UNUSED(backend);
+
+ ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context;
+ if (hex_event->sess != nullptr) {
+ HEX_VERBOSE("ggml-hex: %s event-wait : event %p seq %llu\n",
+ hex_event->sess->c_name(), (void *)hex_event, (unsigned long long)hex_event->seq);
+ hex_event->sess->wait_event(hex_event->seq);
+ }
+}
+
+static void ggml_backend_hexagon_set_tensor_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+ auto sess = static_cast<ggml_hexagon_session *>(backend->context);
+ HEX_VERBOSE("ggml-hex: %s set-tensor-async %s : data %p offset %zu size %zu usage %d\n",
+ sess->c_name(), tensor->name, data, offset, size, tensor->buffer ? (int) tensor->buffer->usage : -1);
+ ggml_backend_tensor_set(tensor, data, offset, size);
+}
+
+static void ggml_backend_hexagon_get_tensor_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+ auto sess = static_cast<ggml_hexagon_session *>(backend->context);
+ HEX_VERBOSE("ggml-hex: %s get-tensor-async %s : data %p offset %zu size %zu usage %d\n",
+ sess->c_name(), tensor->name, data, offset, size, tensor->buffer ? (int) tensor->buffer->usage : -1);
+ sess->flush(true);
+ ggml_backend_tensor_get(tensor, data, offset, size);
+}
+
+static void ggml_backend_hexagon_set_tensor_2d_async(ggml_backend_t backend,
+ struct ggml_tensor * tensor,
+ const void * data,
+ size_t offset,
+ size_t size,
+ size_t n_copies,
+ size_t stride_tensor,
+ size_t stride_data) {
+ auto sess = static_cast<ggml_hexagon_session *>(backend->context);
+ HEX_VERBOSE("ggml-hex: %s set-tensor-2d-async %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n",
+ sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, tensor->buffer ? (int) tensor->buffer->usage : -1);
+ ggml_backend_tensor_set_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data);
+}
+
+static void ggml_backend_hexagon_get_tensor_2d_async(ggml_backend_t backend,
+ const struct ggml_tensor * tensor,
+ void * data,
+ size_t offset,
+ size_t size,
+ size_t n_copies,
+ size_t stride_tensor,
+ size_t stride_data) {
+ auto sess = static_cast<ggml_hexagon_session *>(backend->context);
+ HEX_VERBOSE("ggml-hex: %s get-tensor-2d-async %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n",
+ sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, tensor->buffer ? (int) tensor->buffer->usage : -1);
+ sess->flush(true);
+ ggml_backend_tensor_get_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data);
+}
+
static struct ggml_backend_i hexagon_backend_i = {
/* .get_name = */ ggml_backend_hexagon_name,
/* .free = */ ggml_backend_hexagon_free,
- /* .set_tensor_async = */ NULL,
- /* .get_tensor_async = */ NULL,
- /* .set_tensor_2d_async = */ NULL,
- /* .get_tensor_2d_async = */ NULL,
- /* .cpy_tensor_async = */ NULL,
+ /* .set_tensor_async = */ ggml_backend_hexagon_set_tensor_async,
+ /* .get_tensor_async = */ ggml_backend_hexagon_get_tensor_async,
+ /* .set_tensor_2d_async = */ ggml_backend_hexagon_set_tensor_2d_async,
+ /* .get_tensor_2d_async = */ ggml_backend_hexagon_get_tensor_2d_async,
+ /* .cpy_tensor_async = */ ggml_backend_hexagon_cpy_tensor_async,
/* .synchronize = */ ggml_backend_hexagon_synchronize,
/* .graph_plan_create = */ NULL,
/* .graph_plan_free = */ NULL,
/* .graph_plan_update = */ NULL,
/* .graph_plan_compute = */ NULL,
/* .graph_compute = */ ggml_backend_hexagon_graph_compute,
- /* .event_record = */ NULL,
- /* .event_wait = */ NULL,
+ /* .event_record = */ ggml_backend_hexagon_event_record,
+ /* .event_wait = */ ggml_backend_hexagon_event_wait,
/* .graph_optimize = */ ggml_backend_hexagon_graph_optimize,
};
@@ -3932,9 +5309,9 @@ static void ggml_backend_hexagon_device_get_props(ggml_backend_dev_t dev, struct
ggml_backend_hexagon_device_get_memory(dev, &props->memory_free, &props->memory_total);
props->caps = {
/* .async = */ true,
- /* .host_buffer = */ (bool) opt_hostbuf,
+ /* .host_buffer = */ false,
/* .buffer_from_host_ptr = */ false,
- /* .events = */ false,
+ /* .events = */ true,
/* .mmap_support = */ false,
};
}
@@ -3944,32 +5321,12 @@ static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_buffer_type(gg
return &sess->buffer_type;
}
-static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_repack_buffer_type(ggml_backend_dev_t dev) {
- auto sess = static_cast<ggml_hexagon_session *>(dev->context);
- return &sess->repack_buffer_type;
-}
-
-static bool ggml_hexagon_supported_buffer(ggml_hexagon_session *sess, const struct ggml_tensor * t) {
- if (t && t->buffer) {
- if (ggml_backend_buffer_is_hexagon(t->buffer) == false) return false; // not our buffer
- if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != sess) return false; // wrong session
- }
- return true;
-}
-
-static bool ggml_hexagon_supported_buffers(ggml_hexagon_session *sess, const struct ggml_tensor * t) {
- // all srcs & dsts must be mapped to the same session
- if (!ggml_hexagon_supported_buffer(sess, t)) {
- return false;
- }
-
- for (int i = 0; i < GGML_MAX_SRC; i++) {
- if (!ggml_hexagon_supported_buffer(sess, t->src[i])) {
- return false;
- }
+static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_host_buffer_type(ggml_backend_dev_t dev) {
+ if (!opt_hostbuf) {
+ return NULL;
}
-
- return true;
+ auto sess = static_cast<ggml_hexagon_session *>(dev->context);
+ return &sess->host_buffer_type;
}
static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -4067,12 +5424,6 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
return false;
}
- // all srcs & dsts must be mapped to the same session
- if (!ggml_hexagon_supported_buffers(sess, op)) {
- ggml_hexagon_dump_op_supp(sess->name, op, false);
- return false;
- }
-
bool supp = false;
switch (op->op) {
case GGML_OP_NONE:
@@ -4233,31 +5584,20 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
}
static bool ggml_backend_hexagon_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
- if (buft->iface.get_alignment != ggml_backend_hexagon_buffer_type_get_alignment) {
- return false;
- }
-
- auto s0 = static_cast<ggml_hexagon_session *>(dev->context);
- auto s1 = static_cast<ggml_backend_hexagon_buffer_type_context *>(buft->context)->sess;
-
- // Need session/domain-id for buffers to be compatible
- bool supp = (s0->session_id == s1->session_id);
+ auto sess = static_cast<ggml_hexagon_session *>(dev->context);
- HEX_VERBOSE("ggml-hex: %s device-supports-buft %s (%d)\n", s0->name.c_str(), s1->name.c_str(), (int) supp);
+ // Technically we can clone hexagon buffers from any session but for some reason the output is garbled with layer-split,
+ // tensor-split works correctly, so it needs mode debugging and investigation. For now accept only our own buffers.
+#if 0
+ bool supp = (buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment);
+#else
+ bool supp = (buft == &sess->host_buffer_type) || (buft == &sess->buffer_type);
+#endif
+ HEX_VERBOSE("ggml-hex: %s device-supports-buft %s %s\n", sess->name.c_str(), ggml_backend_buft_name(buft), supp ? "yes" : "no");
return supp;
}
-static ggml_backend_buffer_type_t * ggml_backend_hexagon_device_get_extra_buffers_type(ggml_backend_dev_t dev) {
- auto s0 = static_cast<ggml_hexagon_session *>(dev->context);
- HEX_VERBOSE("ggml-hex: device-get-extra-buft : %s \n", s0->name.c_str());
-
- static ggml_backend_buffer_type_t bufts[2];
- bufts[0] = ggml_backend_hexagon_device_get_repack_buffer_type(dev);
- bufts[1] = NULL;
- return bufts;
-}
-
static const struct ggml_backend_device_i ggml_backend_hexagon_device_i = {
/* .get_name = */ ggml_backend_hexagon_device_get_name,
/* .get_description = */ ggml_backend_hexagon_device_get_description,
@@ -4266,27 +5606,18 @@ static const struct ggml_backend_device_i ggml_backend_hexagon_device_i = {
/* .get_props = */ ggml_backend_hexagon_device_get_props,
/* .init_backend = */ ggml_backend_hexagon_device_init,
/* .get_buffer_type = */ ggml_backend_hexagon_device_get_buffer_type,
- /* .get_host_buffer_type = */ NULL, // ggml_backend_hexagon_device_get_host_buffer_type,
+ /* .get_host_buffer_type = */ ggml_backend_hexagon_device_get_host_buffer_type,
/* .buffer_from_host_ptr = */ NULL, // ggml_backend_hexagon_device_buffer_from_ptr,
/* .supports_op = */ ggml_backend_hexagon_device_supports_op,
/* .supports_buft = */ ggml_backend_hexagon_device_supports_buft,
/* .offload_op = */ NULL, // ggml_backend_hexagon_device_offload_op,
- /* .event_new = */ NULL,
- /* .event_free = */ NULL,
- /* .event_synchronize = */ NULL,
+ /* .event_new = */ ggml_backend_hexagon_device_event_new,
+ /* .event_free = */ ggml_backend_hexagon_device_event_free,
+ /* .event_synchronize = */ ggml_backend_hexagon_device_event_synchronize,
};
//** backend registry
-#define GGML_HEXAGON_MAX_SESSIONS 16
-
-struct ggml_hexagon_registry {
- ggml_hexagon_registry(ggml_backend_reg_t reg);
- ~ggml_hexagon_registry();
-
- ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS];
-};
-
ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
GGML_LOG_INFO("ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev %zu\n", opt_ndev);
@@ -4303,6 +5634,7 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
devices[i].context = nullptr;
}
}
+
}
ggml_hexagon_registry::~ggml_hexagon_registry() {
@@ -4335,14 +5667,129 @@ static ggml_backend_dev_t ggml_backend_hexagon_reg_get_device(ggml_backend_reg_t
return &hreg->devices[index];
}
-static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, const char * name) {
- if (strcmp(name, "ggml_backend_dev_get_extra_bufts") == 0 && opt_hostbuf) {
- ggml_backend_dev_get_extra_bufts_t fct = ggml_backend_hexagon_device_get_extra_buffers_type;
- return (void *) fct;
+// ** communication context for tensor-split allreduce
+
+static void * ggml_backend_hexagon_comm_init(ggml_backend_t * backends, size_t n_backends) {
+ if (n_backends < 2 || n_backends > 4) {
+ return nullptr;
}
- return NULL;
+ for (size_t i = 0; i < n_backends; ++i) {
+ if (!ggml_backend_is_hexagon(backends[i])) {
+ return nullptr;
+ }
+ }
+
+ auto * ctx = new ggml_backend_hexagon_comm_context();
+ ctx->backends.assign(backends, backends + n_backends);
+ ctx->n_backends = n_backends;
+ ctx->fence_seq = (((uintptr_t) ctx) & 0xFFFF) | 1;
+
+ return ctx;
+}
+
+static void ggml_backend_hexagon_comm_free(void * comm_ctx_v) {
+ if (!comm_ctx_v) return;
+ delete static_cast<ggml_backend_hexagon_comm_context *>(comm_ctx_v);
+}
+
+static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) {
+ if (opt_ar_select == 0 || !comm_ctx_v) return false;
+ auto * comm_ctx = static_cast<ggml_backend_hexagon_comm_context *>(comm_ctx_v);
+ const size_t n_backends = comm_ctx->n_backends;
+
+ if (n_backends < 2 || n_backends > 4) return false;
+
+ for (size_t i = 0; i < n_backends; i++) {
+ if (!tensors[i] || !tensors[i]->buffer || !ggml_backend_buffer_is_hexagon(tensors[i]->buffer)) {
+ return false;
+ }
+ if (tensors[i]->type != tensors[0]->type) {
+ return false;
+ }
+ if (!ggml_is_contiguous(tensors[i])) {
+ return false;
+ }
+ if (ggml_nelements(tensors[i]) != ggml_nelements(tensors[0])) {
+ return false;
+ }
+ }
+
+ if (tensors[0]->type != GGML_TYPE_F16 && tensors[0]->type != GGML_TYPE_F32) {
+ return false;
+ }
+
+ for (size_t r = 0; r < n_backends; r++) {
+ auto sess = static_cast<ggml_hexagon_session *>(comm_ctx->backends[r]->context);
+ struct htp_allreduce_kernel_params kparams;
+ if (!ggml_hexagon_precompute_allreduce_params(sess, tensors[r], (uint32_t) r, (uint32_t) n_backends, false, false, &kparams)) {
+ return false;
+ }
+ }
+
+ if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1;
+ uint32_t fence_seq_entry = comm_ctx->fence_seq++;
+ if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1;
+ uint32_t fence_seq_exit = comm_ctx->fence_seq++;
+ if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1;
+
+ volatile uint32_t * fences[GGML_HEXAGON_MAX_SESSIONS];
+ for (size_t i = 0; i < n_backends; i++) {
+ auto sbuf = (ggml_hexagon_shared_buffer *) tensors[i]->buffer->context;
+ fences[i] = (volatile uint32_t *) sbuf->alloc_fence();
+ }
+
+ static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE };
+ ggml_tensor fence_tensors[GGML_HEXAGON_MAX_SESSIONS];
+ for (size_t i = 0; i < n_backends; i++) {
+ fence_tensors[i] = {};
+ fence_tensors[i].buffer = tensors[i]->buffer;
+ fence_tensors[i].extra = &fence_extra;
+ fence_tensors[i].data = (void *) fences[i];
+ fence_tensors[i].type = GGML_TYPE_I32;
+ fence_tensors[i].ne[0] = 4;
+ fence_tensors[i].ne[1] = 1;
+ fence_tensors[i].ne[2] = 1;
+ fence_tensors[i].ne[3] = 1;
+ fence_tensors[i].nb[0] = sizeof(int32_t);
+ fence_tensors[i].nb[1] = sizeof(int32_t);
+ fence_tensors[i].nb[2] = sizeof(int32_t);
+ fence_tensors[i].nb[3] = sizeof(int32_t);
+ fence_tensors[i].op = GGML_OP_NONE;
+ }
+
+ std::vector<const ggml_tensor *> data_tensors(n_backends);
+ std::vector<const ggml_tensor *> sync_tensors(n_backends);
+ for (size_t i = 0; i < n_backends; i++) {
+ data_tensors[i] = tensors[i];
+ sync_tensors[i] = &fence_tensors[i];
+ }
+
+ for (size_t r = 0; r < n_backends; r++) {
+ auto sess = static_cast<ggml_hexagon_session *>(comm_ctx->backends[r]->context);
+ sess->enqueue_allreduce(tensors[r], data_tensors, sync_tensors, (uint32_t) r, (uint32_t) n_backends, fence_seq_entry, fence_seq_exit);
+ for (size_t j = 0; j < n_backends; j++) {
+ if (r != j) {
+ sess->add_sync_peer(static_cast<ggml_hexagon_session *>(comm_ctx->backends[j]->context));
+ }
+ }
+ }
+
+ return true;
+}
+
+static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, const char * name) {
GGML_UNUSED(reg);
+ if (strcmp(name, "ggml_backend_comm_init") == 0) {
+ return (void *) ggml_backend_hexagon_comm_init;
+ }
+ if (strcmp(name, "ggml_backend_comm_free") == 0) {
+ return (void *) ggml_backend_hexagon_comm_free;
+ }
+ if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) {
+ return (void *) ggml_backend_hexagon_comm_allreduce_tensor;
+ }
+ return NULL;
}
template<typename T> std::vector<T> str_to_vec(const char* str) {
@@ -4379,8 +5826,6 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
"please update hexagon_type to match ggml_type");
const char * str_verbose = getenv("GGML_HEXAGON_VERBOSE");
- const char * str_hostbuf = getenv("GGML_HEXAGON_HOSTBUF");
- const char * str_opstage = getenv("GGML_HEXAGON_OPSTAGE");
const char * str_opbatch = getenv("GGML_HEXAGON_OPBATCH");
const char * str_opqueue = getenv("GGML_HEXAGON_OPQUEUE");
const char * str_oppoll = getenv("GGML_HEXAGON_OPPOLL");
@@ -4389,15 +5834,16 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
const char * str_profile = getenv("GGML_HEXAGON_PROFILE");
const char * str_etm = getenv("GGML_HEXAGON_ETM");
const char * str_nhvx = getenv("GGML_HEXAGON_NHVX");
- const char * str_use_hmx = getenv("GGML_HEXAGON_USE_HMX");
const char * str_nhmx = getenv("GGML_HEXAGON_NHMX");
const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT");
const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT");
+ const char * str_ar_select = getenv("GGML_HEXAGON_AR_SELECT");
const char * str_ndev = getenv("GGML_HEXAGON_NDEV");
const char * str_arch = getenv("GGML_HEXAGON_ARCH");
const char * str_vmem = getenv("GGML_HEXAGON_VMEM");
const char * str_mbuf = getenv("GGML_HEXAGON_MBUF");
const char * str_optrace = getenv("GGML_HEXAGON_OPTRACE");
+ const char * str_hostbuf = getenv("GGML_HEXAGON_HOSTBUF");
// Init Arch first since it affects other defaults
if (!str_arch) {
@@ -4430,8 +5876,6 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
opt_opfilter = str_opfilter ? new std::regex(str_opfilter, RE_ICASE) : NULL;
opt_verbose = str_verbose ? atoi(str_verbose) : 0;
- opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) : opt_hostbuf;
- opt_opstage = str_opstage ? strtoul(str_opstage, NULL, 0) : opt_opstage;
opt_opbatch = str_opbatch ? strtoul(str_opbatch, NULL, 0) : opt_opbatch;
opt_opqueue = str_opqueue ? strtoul(str_opqueue, NULL, 0) : opt_opqueue;
opt_optrace = str_optrace ? strtoul(str_optrace, NULL, 0) : (opt_opbatch * 256);
@@ -4440,16 +5884,90 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
opt_profile = str_profile ? atoi(str_profile) : 0;
opt_etm = str_etm ? atoi(str_etm) : 0;
opt_nhvx = str_nhvx ? strtoul(str_nhvx, NULL, 0) : opt_nhvx;
- opt_nhmx = str_nhmx ? atoi(str_nhmx) : (str_use_hmx ? atoi(str_use_hmx) : opt_nhmx);
+ opt_nhmx = str_nhmx ? atoi(str_nhmx) : opt_nhmx;
opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select;
opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select;
- opt_ndev = str_ndev ? strtoul(str_ndev, NULL, 0) : opt_ndev;
- opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) : opt_hostbuf;
+ opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select;
opt_mbuf = str_mbuf ? strtoul(str_mbuf, NULL, 0) * MiB : opt_mbuf;
opt_vmem = str_vmem ? strtoul(str_vmem, NULL, 0) * MiB : opt_vmem;
+ opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) != 0 : opt_hostbuf;
+
+ // Parse device configuration
+ const char * str_devices = getenv("GGML_HEXAGON_DEVICES");
+ if (!str_devices && str_ndev && str_ndev[0] != '\0') {
+ GGML_LOG_WARN("DEPRECATED: GGML_HEXAGON_NDEV is deprecated. use GGML_HEXAGON_DEVICES instead\n");
+ str_devices = str_ndev;
+ }
+
+ if (str_devices && str_devices[0] != '\0') {
+ bool is_single_number = true;
+ for (int i = 0; str_devices[i] != '\0'; i++) {
+ if (!isdigit((unsigned char)str_devices[i])) {
+ is_single_number = false;
+ break;
+ }
+ }
+ if (is_single_number) {
+ int n = atoi(str_devices);
+ if (n < 1) n = 1;
+ if (n > GGML_HEXAGON_MAX_SESSIONS) n = GGML_HEXAGON_MAX_SESSIONS;
+ opt_ndev = n;
+ for (size_t i = 0; i < opt_ndev; i++) {
+ opt_device_configs[i].physical_idx = 0;
+ opt_device_configs[i].virtual_idx = (int)i;
+ opt_device_configs[i].name = "HTP" + std::to_string(i);
+ }
+ } else {
+ std::string s_devices(str_devices);
+ std::stringstream ss(s_devices);
+ std::string item;
+ opt_ndev = 0;
+ while (std::getline(ss, item, ',')) {
+ size_t start = item.find_first_not_of(" \t\r\n");
+ size_t end = item.find_last_not_of(" \t\r\n");
+ if (start == std::string::npos) {
+ continue;
+ }
+ item = item.substr(start, end - start + 1);
+
+ if (item.rfind("HTP", 0) == 0) {
+ std::string rest = item.substr(3);
+ size_t colon_pos = rest.find(':');
+ int phys = 0;
+ int virt = 0;
+ try {
+ if (colon_pos == std::string::npos) {
+ phys = std::stoi(rest);
+ virt = 0;
+ } else {
+ phys = std::stoi(rest.substr(0, colon_pos));
+ virt = std::stoi(rest.substr(colon_pos + 1));
+ }
+ } catch (...) {
+ GGML_LOG_WARN("ggml-hex: failed to parse device index in '%s'\n", item.c_str());
+ continue;
+ }
- if (opt_ndev > GGML_HEXAGON_MAX_SESSIONS) {
- opt_ndev = GGML_HEXAGON_MAX_SESSIONS;
+ if (opt_ndev < GGML_HEXAGON_MAX_SESSIONS) {
+ opt_device_configs[opt_ndev].physical_idx = phys;
+ opt_device_configs[opt_ndev].virtual_idx = virt;
+ opt_device_configs[opt_ndev].name = colon_pos == std::string::npos
+ ? "HTP" + std::to_string(phys)
+ : "HTP" + std::to_string(phys) + ":" + std::to_string(virt);
+ opt_ndev++;
+ } else {
+ GGML_LOG_WARN("ggml-hex: max sessions limit reached (%d), ignoring device %s\n", GGML_HEXAGON_MAX_SESSIONS, item.c_str());
+ }
+ } else {
+ GGML_LOG_WARN("ggml-hex: invalid device name format '%s', must start with HTP\n", item.c_str());
+ }
+ }
+ }
+ } else {
+ opt_ndev = 1;
+ opt_device_configs[0].physical_idx = 0;
+ opt_device_configs[0].virtual_idx = 0;
+ opt_device_configs[0].name = "HTP0";
}
#if defined(__ANDROID__)
diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h
index b0c859dacf..741b5e04eb 100644
--- a/ggml/src/ggml-hexagon/htp-opnode.h
+++ b/ggml/src/ggml-hexagon/htp-opnode.h
@@ -8,60 +8,107 @@
#include <algorithm>
#include <string>
#include <vector>
+#include <memory>
#include <stdio.h>
#include "htp-ops.h"
#include "htp/matmul-ops.h"
#include "htp/flash-attn-ops.h"
#include "htp/unary-ops.h"
+#include "htp/allreduce-ops.h"
struct htp_opnode {
- ggml_tensor * node = nullptr;
-
- std::vector<ggml_tensor *> fused;
-
- htp_op_code opcode = HTP_OP_INVALID;
+ ggml_tensor * node { nullptr };
+ htp_op_code opcode { HTP_OP_INVALID };
+ int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] {0};
+
+ std::vector<ggml_tensor *> fused;
+ std::vector<std::shared_ptr<ggml_tensor>> dummy;
+
+ std::vector<const ggml_tensor *> inputs;
+ std::vector<const ggml_tensor *> outputs;
+ std::string name;
+
+ int n_active_src(const ggml_tensor * t) const {
+ if (!t) return 0;
+ for (int i = GGML_MAX_SRC - 1; i >= 0; i--) {
+ if (t->src[i]) {
+ return i + 1;
+ }
+ }
+ return 0;
+ }
- std::vector<ggml_tensor *> extra_dsts;
+ void init(ggml_tensor * node) {
+ this->node = node;
+ if (this->node) {
+ this->name = ggml_op_desc(this->node);
- int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] = {0};
+ // Build inputs (preserving optional nullptrs)
+ int n_inputs = n_active_src(this->node);
+ this->inputs.resize(n_inputs, nullptr);
+ for (int i = 0; i < n_inputs; i++) {
+ this->inputs[i] = this->node->src[i];
+ }
- htp_opnode(ggml_tensor * node = nullptr, std::vector<ggml_tensor *> fused = {}, htp_op_code opcode = HTP_OP_INVALID, std::vector<ggml_tensor *> extra_dsts = {})
- : node(node), fused(std::move(fused)), opcode(opcode), extra_dsts(std::move(extra_dsts)) {}
+ // Build outputs
+ this->outputs.push_back(this->dst());
+ }
+ }
- ggml_op op() const {
- return node->op;
+ htp_opnode(htp_op_code opcode = HTP_OP_INVALID, ggml_tensor * node = nullptr) : opcode(opcode) {
+ init(node);
}
- const ggml_tensor * dst() const {
- return fused.empty() ? node : fused.back();
+ ggml_op op() const { return node->op; }
+ const ggml_tensor * src0() const { return node->src[0]; }
+ const ggml_tensor * src1() const { return node->src[1]; }
+ const ggml_tensor * dst() const { return outputs.empty() ? node : outputs.back(); }
+
+ ggml_tensor * add_dummy(const ggml_tensor & t) {
+ dummy.push_back(std::make_shared<ggml_tensor>(t));
+ return dummy.back().get();
}
void add_fused(ggml_tensor * t, bool extra_dst = false) {
fused.push_back(t);
+
+ name += "+";
+ name += ggml_op_desc(t);
+
if (extra_dst) {
- extra_dsts.push_back(t);
+ outputs.push_back(t);
+ } else {
+ outputs.clear();
+ outputs.push_back(t);
}
- }
- std::vector<const ggml_tensor *> get_outputs() const {
- std::vector<const ggml_tensor *> res;
- if (extra_dsts.empty()) {
- res.push_back(dst());
- } else {
- res.push_back(node);
- for (const auto * x : extra_dsts) {
- res.push_back(x);
+ // Remove the newly fused intermediate output tensor t from inputs (if it was there)
+ inputs.erase(std::remove(inputs.begin(), inputs.end(), t), inputs.end());
+
+ // Append new inputs from t, preserving middle nullptrs
+ int n_inputs = n_active_src(t);
+ for (int i = 0; i < n_inputs; i++) {
+ const auto * src = t->src[i];
+ if (!src) {
+ inputs.push_back(nullptr);
+ } else if (src != node &&
+ std::find(fused.begin(), fused.end(), src) == fused.end() &&
+ std::find(inputs.begin(), inputs.end(), src) == inputs.end()) {
+ inputs.push_back(src);
}
}
- return res;
}
- const ggml_tensor * src0() const {
- return node->src[0];
+ const std::vector<const ggml_tensor *> & get_inputs() const {
+ return inputs;
}
- const ggml_tensor * src1() const {
- return node->src[1];
+ const std::vector<const ggml_tensor *> & get_outputs() const {
+ return outputs;
+ }
+
+ std::string op_name() const {
+ return name;
}
bool is_empty() const {
@@ -81,75 +128,6 @@ struct htp_opnode {
bool same_input(const htp_opnode& n) const {
return n.src1() == this->src1();
}
-
- std::vector<const ggml_tensor *> get_inputs() const {
- if (fused.empty()) {
- int last_non_null = -1;
- for (int i = 0; i < GGML_MAX_SRC; i++) {
- if (node->src[i]) {
- last_non_null = i;
- }
- }
- std::vector<const ggml_tensor *> inputs(last_non_null + 1, nullptr);
- for (int i = 0; i <= last_non_null; i++) {
- inputs[i] = node->src[i];
- }
- return inputs;
- }
-
- std::vector<const ggml_tensor *> inputs(GGML_MAX_SRC, nullptr);
- std::vector<const ggml_tensor *> outputs;
- outputs.push_back(node);
- for (const auto * f : fused) {
- outputs.push_back(f);
- }
-
- auto contains = [&](const std::vector<const ggml_tensor *> & vec, const ggml_tensor * t) {
- for (const auto * x : vec) {
- if (x == t) return true;
- }
- return false;
- };
-
- int count = 0;
- auto add_input = [&](const ggml_tensor * t) {
- if (t && !contains(outputs, t) && !contains(inputs, t)) {
- if (count < (int)inputs.size()) {
- inputs[count++] = t;
- } else {
- inputs.push_back(t);
- }
- }
- };
-
- for (int i = 0; i < GGML_MAX_SRC; i++) {
- if (node->src[i]) {
- add_input(node->src[i]);
- }
- }
- for (const auto * f : fused) {
- for (int i = 0; i < GGML_MAX_SRC; i++) {
- if (f->src[i]) {
- add_input(f->src[i]);
- }
- }
- }
-
- inputs.resize(count);
- return inputs;
- }
-
- std::string op_name() const {
- if (fused.empty()) {
- return ggml_op_desc(node);
- }
- std::string name = ggml_op_desc(node);
- for (const auto * f : fused) {
- name += "+";
- name += ggml_op_desc(f);
- }
- return name;
- }
};
struct htp_opformat {
@@ -337,8 +315,7 @@ struct htp_opformat {
}
void format_kernel_params(char * str, size_t max_size, const htp_opnode & node) {
if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID ||
- node.opcode == HTP_OP_MUL_MAT_QKV || node.opcode == HTP_OP_MUL_MAT_FFN ||
- node.opcode == HTP_OP_MUL_MAT_ADD) {
+ node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ADD) {
const auto * kparams = (const struct htp_mm_kernel_params *) node.kernel_params;
const char * path = "unknown";
int32_t type = kparams->kernel_type;
diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt
index b00aa2bc94..77f3ee39dd 100644
--- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt
+++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt
@@ -43,6 +43,7 @@ add_library(${HTP_LIB} SHARED
pad-ops.c
argsort-ops.c
im2col-ops.c
+ allreduce-ops.c
)
target_compile_definitions(${HTP_LIB} PRIVATE
diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c
index 9973c088dd..0a8bf84e38 100644
--- a/ggml/src/ggml-hexagon/htp/act-ops.c
+++ b/ggml/src/ggml-hexagon/htp/act-ops.c
@@ -183,6 +183,53 @@ static void swiglu_oai_f32(const float * restrict src0,
static const float GELU_COEF_A = 0.044715f;
static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
+static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) {
+ v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));
+ v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3));
+
+ HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));
+ HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));
+ HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x);
+
+ HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2));
+ v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));
+
+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1));
+ v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx);
+ v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x);
+
+ HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));
+ v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);
+
+ HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));
+ HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));
+
+ // Newton-Raphson with 2 iterations
+ HVX_Vector two_sf = hvx_vec_splat_f32(2.0f);
+ HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5);
+ HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf(
+ i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5)))));
+ r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(
+ r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5))));
+ HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf);
+
+ res = Q6_Vqf32_vmpy_VsfVsf(v3, res);
+
+ return Q6_Vsf_equals_Vqf32(res);
+}
+
+static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v,
+ HVX_Vector one,
+ HVX_Vector max_exp,
+ HVX_Vector min_exp) {
+ const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v);
+ const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp);
+
+ HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v);
+ out = Q6_V_vmux_QVV(pred_max, out, one);
+ return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero());
+}
+
static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src0 % 128 == 0);
@@ -200,20 +247,13 @@ static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * rest
const HVX_Vector v_coef_a_times_sqrt = hvx_vec_splat_f32(GELU_COEF_A_TIMES_SQRT);
const HVX_Vector v_sqrt_2_pi = hvx_vec_splat_f32(SQRT_2_OVER_PI);
- const HVX_Vector v_half = hvx_vec_splat_f32(0.5f);
const HVX_Vector v_one = hvx_vec_splat_f32(1.0f);
- const HVX_Vector v_two = hvx_vec_splat_f32(2.0f);
-
- // Hoisted fast sigmoid / inverse constants to avoid loop-internal overhead
- const HVX_Vector v_log2f = Q6_V_vsplat_R(FAST_SIGMOID_LOG2F);
- const HVX_Vector v_c1 = Q6_V_vsplat_R(FAST_SIGMOID_C1);
- const HVX_Vector v_c2 = Q6_V_vsplat_R(FAST_SIGMOID_C2);
- const HVX_Vector v_inv_aprox = Q6_V_vsplat_R(0x7EEEEBB3);
const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f);
const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f);
uint32_t i = 0;
+ _Pragma("unroll(4)")
for (; i < nvec; i++) {
HVX_Vector x = vsrc0[i];
HVX_Vector g = vsrc1[i];
@@ -223,56 +263,13 @@ static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * rest
coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi);
HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef);
- // y2 = 2 * inner
- HVX_Vector y2 = hvx_vec_mul_f32_f32(inner, v_two);
-
- // Sigmoid guard check predicates
- HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(v_max_exp, y2);
- HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(y2, v_min_exp);
-
- // Fast sigmoid approximation
- HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(y2, v_log2f);
- v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), v_half);
-
- HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));
- HVX_Vector x_sig = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));
- HVX_Vector xx_sig = Q6_Vqf32_vmpy_Vqf32Vqf32(x_sig, x_sig);
-
- HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx_sig), v_c2);
- v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, v_log2f);
+ // y2 = 2 * inner = inner + inner
+ HVX_Vector y2 = hvx_vec_add_f32_f32(inner, inner);
- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x_sig), v_c1);
- v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx_sig);
- v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x_sig);
-
- HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));
- v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);
-
- HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));
- HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));
-
- // Fast division (Newton-Raphson with 2 iterations)
- HVX_Vector i_sf = Q6_Vw_vsub_VwVw(v_inv_aprox, v5);
- HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf(
- i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5)))));
- r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(
- r_qf, Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5))));
- HVX_Vector res_inv = Q6_Vsf_equals_Vqf32(r_qf);
-
- HVX_Vector sig2y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, res_inv));
-
- // Sigmoid guards
- sig2y = Q6_V_vmux_QVV(pred_max, sig2y, v_one);
- sig2y = Q6_V_vmux_QVV(pred_min, sig2y, Q6_V_vzero());
-
- // tanh(inner) = 2 * sigmoid(2 * inner) - 1
- HVX_Vector tanh_val = hvx_vec_mul_f32_f32(sig2y, v_two);
- tanh_val = hvx_vec_sub_f32_f32(tanh_val, v_one);
-
- HVX_Vector tanh_plus_one = hvx_vec_add_f32_f32(tanh_val, v_one);
- HVX_Vector half_x = hvx_vec_mul_f32_f32(x, v_half);
- HVX_Vector gelu_x = hvx_vec_mul_f32_f32(half_x, tanh_plus_one);
+ // Fast sigmoid approximation (2 iterations)
+ HVX_Vector sig2y = hvx_vec_fast_sigmoid_f32_guard_2it(y2, v_one, v_max_exp, v_min_exp);
+ HVX_Vector gelu_x = hvx_vec_mul_f32_f32(x, sig2y);
vdst[i] = hvx_vec_mul_f32_f32(gelu_x, g);
}
@@ -285,50 +282,11 @@ static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * rest
coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi);
HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef);
- HVX_Vector y2 = hvx_vec_mul_f32_f32(inner, v_two);
-
- HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(v_max_exp, y2);
- HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(y2, v_min_exp);
-
- HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(y2, v_log2f);
- v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), v_half);
-
- HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));
- HVX_Vector x_sig = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));
- HVX_Vector xx_sig = Q6_Vqf32_vmpy_Vqf32Vqf32(x_sig, x_sig);
-
- HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx_sig), v_c2);
- v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, v_log2f);
-
- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x_sig), v_c1);
- v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx_sig);
- v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x_sig);
-
- HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));
- v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);
-
- HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));
- HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));
-
- HVX_Vector i_sf = Q6_Vw_vsub_VwVw(v_inv_aprox, v5);
- HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf(
- i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5)))));
- r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(
- r_qf, Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5))));
- HVX_Vector res_inv = Q6_Vsf_equals_Vqf32(r_qf);
-
- HVX_Vector sig2y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, res_inv));
-
- sig2y = Q6_V_vmux_QVV(pred_max, sig2y, v_one);
- sig2y = Q6_V_vmux_QVV(pred_min, sig2y, Q6_V_vzero());
-
- HVX_Vector tanh_val = hvx_vec_mul_f32_f32(sig2y, v_two);
- tanh_val = hvx_vec_sub_f32_f32(tanh_val, v_one);
+ HVX_Vector y2 = hvx_vec_add_f32_f32(inner, inner);
- HVX_Vector tanh_plus_one = hvx_vec_add_f32_f32(tanh_val, v_one);
- HVX_Vector half_x = hvx_vec_mul_f32_f32(x, v_half);
- HVX_Vector gelu_x = hvx_vec_mul_f32_f32(half_x, tanh_plus_one);
+ HVX_Vector sig2y = hvx_vec_fast_sigmoid_f32_guard_2it(y2, v_one, v_max_exp, v_min_exp);
+ HVX_Vector gelu_x = hvx_vec_mul_f32_f32(x, sig2y);
HVX_Vector res = hvx_vec_mul_f32_f32(gelu_x, g);
hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), res);
}
diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.c b/ggml/src/ggml-hexagon/htp/allreduce-ops.c
new file mode 100644
index 0000000000..d35f685a6d
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.c
@@ -0,0 +1,398 @@
+#pragma clang diagnostic ignored "-Wunused-variable"
+#pragma clang diagnostic ignored "-Wunused-function"
+#pragma clang diagnostic ignored "-Wunused-but-set-variable"
+
+#include <HAP_farf.h>
+#include <HAP_perf.h>
+#include <stdatomic.h>
+#include <math.h>
+#include <string.h>
+
+#define GGML_COMMON_DECL_C
+#include "ggml-common.h"
+#include "htp-ctx.h"
+#include "htp-ops.h"
+#include "hvx-utils.h"
+#include "htp-tensor.h"
+#include "hex-dma.h"
+#include "hex-profile.h"
+#include "allreduce-ops.h"
+
+struct htp_allreduce_context {
+ struct htp_ops_context * octx;
+ uint32_t n_ranks;
+ uint32_t n_dsts;
+ uint32_t nelem;
+ uint32_t ne0;
+ uint32_t ne1;
+ uint32_t row_size_aligned;
+ uint32_t rank_elem_start;
+ uint32_t rank_nelem;
+ uint32_t elems_per_thread;
+ uint32_t block_elems;
+ uint32_t vtcm_size_per_thread;
+ bool is_row_bcast;
+ uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS];
+ uint8_t * dst_spad_base;
+ uint8_t * res_spad_base;
+};
+
+#define DEFINE_ALLREDUCE_THREAD_DMA_1D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD) \
+static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
+ struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \
+ struct htp_ops_context * octx = actx->octx; \
+ \
+ const uint32_t n_ranks = actx->n_ranks; \
+ const uint32_t n_dsts = actx->n_dsts; \
+ const uint32_t block_elems = actx->block_elems; \
+ \
+ const uint32_t dr = actx->elems_per_thread; \
+ const uint32_t ir0 = actx->rank_elem_start + dr * ith; \
+ const uint32_t ir1 = MIN(ir0 + dr, actx->rank_elem_start + actx->rank_nelem); \
+ if (ir0 >= ir1) return; \
+ \
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
+ dma_queue * q = octx->ctx->dma[ith]; \
+ \
+ uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; \
+ for (uint32_t s = 0; s < n_ranks; s++) { \
+ src_spad_base[s] = actx->src_spad_base[s] + (ith * actx->vtcm_size_per_thread); \
+ } \
+ uint8_t * dst_spad_base = actx->dst_spad_base + (ith * actx->vtcm_size_per_thread); \
+ uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + (ith * actx->vtcm_size_per_thread)) : NULL; \
+ \
+ const size_t spad_half = actx->vtcm_size_per_thread / 2; \
+ uint32_t ir_prefetch = ir0; \
+ int spad_idx = 0; \
+ \
+ for (int k = 0; k < 2 && ir_prefetch < ir1; k++) { \
+ uint32_t cur_elems = MIN(block_elems, ir1 - ir_prefetch); \
+ size_t cur_bytes = cur_elems * sizeof(TYPE); \
+ uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \
+ for (uint32_t d = 0; d < n_dsts; d++) { \
+ uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + ir_prefetch * sizeof(TYPE); \
+ dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 0); \
+ } \
+ for (uint32_t s = 0; s < n_ranks; s++) { \
+ uint8_t * s_spad = src_spad_base[s] + spad_idx * spad_half; \
+ const uint8_t * s_ddr = (const uint8_t *) octx->src[s]->data + ir_prefetch * sizeof(TYPE); \
+ dma_queue_push(q, dma_make_ptr(s_spad, s_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \
+ } \
+ if (HAS_ADD) { \
+ uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \
+ const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \
+ dma_queue_push(q, dma_make_ptr(r_spad, r_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \
+ } \
+ ir_prefetch += cur_elems; \
+ spad_idx ^= 1; \
+ } \
+ \
+ for (uint32_t ir = ir0; ir < ir1; ) { \
+ uint32_t cur_elems = MIN(block_elems, ir1 - ir); \
+ size_t cur_bytes = cur_elems * sizeof(TYPE); \
+ uint8_t * d_spad = NULL; \
+ for (uint32_t d = 0; d < n_dsts; d++) { \
+ d_spad = (uint8_t *) dma_queue_pop(q).src; \
+ } \
+ uint8_t * s_spad[HTP_ALLREDUCE_MAX_RANKS]; \
+ for (uint32_t s = 0; s < n_ranks; s++) { \
+ s_spad[s] = (uint8_t *) dma_queue_pop(q).dst; \
+ } \
+ uint8_t * r_spad = HAS_ADD ? (uint8_t *) dma_queue_pop(q).dst : NULL; \
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \
+ HVX_ADD_FN(d_spad, s_spad[0], s_spad[1], cur_elems); \
+ for (uint32_t s = 2; s < n_ranks; s++) { \
+ HVX_ADD_FN(d_spad, d_spad, s_spad[s], cur_elems); \
+ } \
+ if (HAS_ADD) { \
+ HVX_ADD_FN(d_spad, d_spad, r_spad, cur_elems); \
+ } \
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \
+ for (uint32_t d = 0; d < n_dsts; d++) { \
+ uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + ir * sizeof(TYPE); \
+ dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 1); \
+ } \
+ if (ir_prefetch < ir1) { \
+ uint32_t next_elems = MIN(block_elems, ir1 - ir_prefetch); \
+ size_t next_bytes = next_elems * sizeof(TYPE); \
+ for (uint32_t s = 0; s < n_ranks; s++) { \
+ const uint8_t * s_next = (const uint8_t *) octx->src[s]->data + ir_prefetch * sizeof(TYPE); \
+ dma_queue_push(q, dma_make_ptr(s_spad[s], s_next), next_bytes, next_bytes, next_bytes, 1); \
+ } \
+ if (HAS_ADD) { \
+ const uint8_t * r_next = (const uint8_t *) octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \
+ dma_queue_push(q, dma_make_ptr(r_spad, r_next), next_bytes, next_bytes, next_bytes, 1); \
+ } \
+ ir_prefetch += next_elems; \
+ } \
+ ir += cur_elems; \
+ } \
+ dma_queue_flush(q); \
+}
+
+DEFINE_ALLREDUCE_THREAD_DMA_1D(f16, __fp16, hvx_add_f16_aaa, 0)
+DEFINE_ALLREDUCE_THREAD_DMA_1D(f32, float, hvx_add_f32_aaa, 0)
+DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f16, __fp16, hvx_add_f16_aaa, 1)
+DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f32, float, hvx_add_f32_aaa, 1)
+
+#define DEFINE_ALLREDUCE_THREAD_DMA_2D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD, IS_ROW_BCAST) \
+static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
+ struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \
+ struct htp_ops_context * octx = actx->octx; \
+ \
+ const uint32_t n_ranks = actx->n_ranks; \
+ const uint32_t n_dsts = actx->n_dsts; \
+ const uint32_t ne0 = actx->ne0; \
+ const uint32_t block_rows = actx->block_elems; \
+ const uint32_t row_size_aligned = actx->row_size_aligned; \
+ const uint32_t row_bytes = ne0 * sizeof(TYPE); \
+ \
+ const uint32_t dr = actx->elems_per_thread; \
+ const uint32_t r0 = actx->rank_elem_start + dr * ith; \
+ const uint32_t r1 = MIN(r0 + dr, actx->rank_elem_start + actx->rank_nelem); \
+ if (r0 >= r1) return; \
+ \
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
+ dma_queue * q = octx->ctx->dma[ith]; \
+ \
+ uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; \
+ for (uint32_t s = 0; s < n_ranks; s++) { \
+ src_spad_base[s] = actx->src_spad_base[s] + (ith * actx->vtcm_size_per_thread); \
+ } \
+ uint8_t * dst_spad_base = actx->dst_spad_base + (ith * actx->vtcm_size_per_thread); \
+ uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + (ith * actx->vtcm_size_per_thread))) : NULL; \
+ \
+ const size_t spad_half = actx->vtcm_size_per_thread / 2; \
+ uint32_t r_prefetch = r0; \
+ int spad_idx = 0; \
+ \
+ for (int k = 0; k < 2 && r_prefetch < r1; k++) { \
+ uint32_t cur_rows = MIN(block_rows, r1 - r_prefetch); \
+ uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \
+ for (uint32_t d = 0; d < n_dsts; d++) { \
+ uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + r_prefetch * octx->dsts[d]->nb[1]; \
+ dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, 0); \
+ } \
+ for (uint32_t s = 0; s < n_ranks; s++) { \
+ uint8_t * s_spad = src_spad_base[s] + spad_idx * spad_half; \
+ const uint8_t * s_ddr = (const uint8_t *) octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \
+ dma_queue_push(q, dma_make_ptr(s_spad, s_ddr), row_size_aligned, octx->src[s]->nb[1], row_bytes, cur_rows); \
+ } \
+ if (HAS_ADD && !IS_ROW_BCAST) { \
+ uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \
+ const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \
+ dma_queue_push(q, dma_make_ptr(r_spad, r_ddr), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, cur_rows); \
+ } \
+ r_prefetch += cur_rows; \
+ spad_idx ^= 1; \
+ } \
+ \
+ for (uint32_t r = r0; r < r1; ) { \
+ uint32_t cur_rows = MIN(block_rows, r1 - r); \
+ uint8_t * d_spad = NULL; \
+ for (uint32_t d = 0; d < n_dsts; d++) { \
+ d_spad = (uint8_t *) dma_queue_pop(q).src; \
+ } \
+ uint8_t * s_spad[HTP_ALLREDUCE_MAX_RANKS]; \
+ for (uint32_t s = 0; s < n_ranks; s++) { \
+ s_spad[s] = (uint8_t *) dma_queue_pop(q).dst; \
+ } \
+ uint8_t * r_spad = (HAS_ADD && !IS_ROW_BCAST) ? (uint8_t *) dma_queue_pop(q).dst : NULL; \
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
+ for (uint32_t row = 0; row < cur_rows; row++) { \
+ uint8_t * d_row = d_spad + row * row_size_aligned; \
+ const uint8_t * s0_row = s_spad[0] + row * row_size_aligned; \
+ const uint8_t * s1_row = s_spad[1] + row * row_size_aligned; \
+ HVX_ADD_FN(d_row, s0_row, s1_row, ne0); \
+ for (uint32_t s = 2; s < n_ranks; s++) { \
+ const uint8_t * ss_row = s_spad[s] + row * row_size_aligned; \
+ HVX_ADD_FN(d_row, d_row, ss_row, ne0); \
+ } \
+ if (HAS_ADD) { \
+ const uint8_t * res_row = IS_ROW_BCAST ? res_spad_base : (r_spad + row * row_size_aligned); \
+ HVX_ADD_FN(d_row, d_row, res_row, ne0); \
+ } \
+ } \
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
+ for (uint32_t d = 0; d < n_dsts; d++) { \
+ uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + r * octx->dsts[d]->nb[1]; \
+ dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, cur_rows); \
+ } \
+ if (r_prefetch < r1) { \
+ uint32_t next_rows = MIN(block_rows, r1 - r_prefetch); \
+ for (uint32_t s = 0; s < n_ranks; s++) { \
+ const uint8_t * s_next = (const uint8_t *) octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \
+ dma_queue_push(q, dma_make_ptr(s_spad[s], s_next), row_size_aligned, octx->src[s]->nb[1], row_bytes, next_rows); \
+ } \
+ if (HAS_ADD && !IS_ROW_BCAST) { \
+ const uint8_t * r_next = (const uint8_t *) octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \
+ dma_queue_push(q, dma_make_ptr(r_spad, r_next), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, next_rows); \
+ } \
+ r_prefetch += next_rows; \
+ } \
+ r += cur_rows; \
+ } \
+ dma_queue_flush(q); \
+}
+
+DEFINE_ALLREDUCE_THREAD_DMA_2D(f16, __fp16, hvx_add_f16_aaa, 0, 0)
+DEFINE_ALLREDUCE_THREAD_DMA_2D(f32, float, hvx_add_f32_aaa, 0, 0)
+DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f16, __fp16, hvx_add_f16_aaa, 1, 0)
+DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f32, float, hvx_add_f32_aaa, 1, 0)
+DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f16, __fp16, hvx_add_f16_aaa, 1, 1)
+DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f32, float, hvx_add_f32_aaa, 1, 1)
+
+int op_allreduce(struct htp_ops_context * octx) {
+ const struct htp_allreduce_kernel_params * kparams = (const struct htp_allreduce_kernel_params *) octx->kernel_params;
+ const struct htp_tensor * dst = octx->dst;
+
+ const uint32_t rank = (uint32_t) kparams->rank;
+ const uint32_t n_ranks = (uint32_t) kparams->n_ranks;
+
+ if (n_ranks < 2 || n_ranks > HTP_ALLREDUCE_MAX_RANKS || rank >= n_ranks) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
+ if (dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_F32) {
+ return HTP_STATUS_NO_SUPPORT;
+ }
+
+ const uint32_t nelem = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3];
+ const uint32_t fence_seq_entry = (uint32_t) octx->op_params[0];
+ const uint32_t fence_seq_exit = (uint32_t) octx->op_params[1];
+
+ // 1. Entry Barrier: Synchronize all ranks before reading
+ struct htp_thread_trace * tr0 = &octx->ctx->trace[0];
+ htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry);
+
+ const struct htp_tensor * my_sync = octx->src[n_ranks + rank];
+ atomic_uint * my_fence = (atomic_uint *) my_sync->data;
+
+ atomic_store(&my_fence[0], fence_seq_entry);
+ asm volatile ("syncht" : : : "memory");
+ Q6_dccleaninva_A((void *) my_fence);
+
+ for (uint32_t j = 0; j < n_ranks; j++) {
+ if (j == rank) continue;
+ const struct htp_tensor * peer_sync = octx->src[n_ranks + j];
+ atomic_uint * peer_fence = (atomic_uint *) peer_sync->data;
+ uint64_t spins = 0;
+ while (1) {
+ Q6_dccleaninva_A((void *) peer_fence);
+ uint32_t val = atomic_load(&peer_fence[0]);
+ if (val == fence_seq_entry || val == fence_seq_exit) {
+ break;
+ }
+ if (++spins > HTP_FENCE_TIMEOUT) {
+ FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_entry);
+ return HTP_STATUS_INTERNAL_ERR;
+ }
+ hex_pause();
+ }
+ }
+ asm volatile ("syncht" : : : "memory");
+
+ htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry);
+
+ // 2. Multi-threaded Reduction across assigned rank chunk
+ if (nelem > 0) {
+ const uint32_t n_threads = (uint32_t) kparams->n_threads;
+ const uint32_t block_elems = (uint32_t) kparams->block_elems;
+ const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread;
+ const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread;
+
+ const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD);
+
+ struct htp_allreduce_context actx;
+ actx.octx = octx;
+ actx.n_ranks = n_ranks;
+ actx.n_dsts = (uint32_t) kparams->n_dsts ? (uint32_t) kparams->n_dsts : n_ranks;
+ actx.nelem = nelem;
+ actx.ne0 = (uint32_t) kparams->ne0;
+ actx.ne1 = (uint32_t) kparams->ne1;
+ actx.row_size_aligned = (uint32_t) kparams->row_size_aligned;
+ actx.rank_elem_start = (uint32_t) kparams->rank_elem_start;
+ actx.rank_nelem = (uint32_t) kparams->rank_nelem;
+ actx.elems_per_thread = elems_per_thread;
+ actx.block_elems = block_elems;
+ actx.vtcm_size_per_thread = vtcm_size_per_thread;
+ actx.is_row_bcast = (kparams->is_row_bcast != 0);
+
+ work_queue_func_t reduce_fun = NULL;
+ switch (kparams->kernel_type) {
+ case HTP_ALLREDUCE_KERNEL_DMA_1D:
+ if (has_add) {
+ reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_1d_add_f16 : allreduce_thread_dma_1d_add_f32;
+ } else {
+ reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_1d_f16 : allreduce_thread_dma_1d_f32;
+ }
+ break;
+ case HTP_ALLREDUCE_KERNEL_DMA_2D:
+ if (has_add) {
+ if (kparams->is_row_bcast) {
+ reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_add_bcast_f16 : allreduce_thread_dma_2d_add_bcast_f32;
+ } else {
+ reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_add_f16 : allreduce_thread_dma_2d_add_f32;
+ }
+ } else {
+ reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_f16 : allreduce_thread_dma_2d_f32;
+ }
+ break;
+ default:
+ return HTP_STATUS_NO_SUPPORT;
+ }
+
+ uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base;
+ for (uint32_t s = 0; s < n_ranks; s++) {
+ actx.src_spad_base[s] = vtcm_ptr;
+ vtcm_ptr += n_threads * vtcm_size_per_thread;
+ }
+ actx.dst_spad_base = vtcm_ptr;
+ vtcm_ptr += n_threads * vtcm_size_per_thread;
+ if (has_add) {
+ actx.res_spad_base = vtcm_ptr;
+ vtcm_ptr += (actx.is_row_bcast ? 1 : n_threads) * vtcm_size_per_thread;
+ }
+
+ if (has_add && actx.is_row_bcast) {
+ const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data;
+ const uint32_t row_bytes = actx.ne0 * (dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float));
+ dma_queue * q = octx->ctx->dma[0];
+ dma_queue_push(q, dma_make_ptr(actx.res_spad_base, r_ddr), actx.row_size_aligned, 0, row_bytes, 1);
+ dma_queue_pop(q);
+ }
+
+ work_queue_run(octx->ctx->work_queue, reduce_fun, &actx, n_threads);
+ }
+
+ // 4. Exit Barrier: Synchronize all ranks after writing
+ htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
+
+ atomic_store(&my_fence[0], fence_seq_exit);
+ asm volatile ("syncht" : : : "memory");
+ Q6_dccleaninva_A((void *) my_fence);
+
+ for (uint32_t j = 0; j < n_ranks; j++) {
+ if (j == rank) continue;
+ const struct htp_tensor * peer_sync = octx->src[n_ranks + j];
+ atomic_uint * peer_fence = (atomic_uint *) peer_sync->data;
+ uint64_t spins = 0;
+ while (1) {
+ Q6_dccleaninva_A((void *) peer_fence);
+ uint32_t val = atomic_load(&peer_fence[0]);
+ if (val == fence_seq_exit) {
+ break;
+ }
+ if (++spins > HTP_FENCE_TIMEOUT) {
+ FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_exit);
+ return HTP_STATUS_INTERNAL_ERR;
+ }
+ hex_pause();
+ }
+ }
+ asm volatile ("syncht" : : : "memory");
+
+ htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
+
+ return HTP_STATUS_OK;
+}
diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.h
new file mode 100644
index 0000000000..de447d87e9
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h
@@ -0,0 +1,40 @@
+#ifndef ALLREDUCE_OPS_H
+#define ALLREDUCE_OPS_H
+
+#include <stdint.h>
+
+#define HTP_ALLREDUCE_MAX_RANKS 4
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum htp_allreduce_kernel_type {
+ HTP_ALLREDUCE_KERNEL_UNSUPPORTED = 0,
+ HTP_ALLREDUCE_KERNEL_DMA_1D,
+ HTP_ALLREDUCE_KERNEL_DMA_2D,
+};
+
+struct htp_allreduce_kernel_params {
+ int32_t rank;
+ int32_t n_ranks;
+ int32_t n_threads;
+ int32_t block_elems; // 1D: block_elems, 2D: block_rows
+ int32_t elems_per_thread; // 1D: nelem_per_thread, 2D: nrows_per_thread
+ int32_t vtcm_size_per_thread;
+ int32_t vtcm_size;
+ int32_t kernel_type;
+ int32_t ne0;
+ int32_t ne1;
+ int32_t row_size_aligned;
+ int32_t rank_elem_start;
+ int32_t rank_nelem;
+ int32_t n_dsts;
+ int32_t is_row_bcast;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ALLREDUCE_OPS_H */
diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c
index ae507effa5..15bc8dc244 100644
--- a/ggml/src/ggml-hexagon/htp/cpy-ops.c
+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c
@@ -4,6 +4,7 @@
#include <HAP_farf.h>
#include <HAP_perf.h>
+#include <qurt_memory.h>
#include <math.h>
#include <string.h>
@@ -14,6 +15,7 @@
#include "htp-ops.h"
#include "htp-ops.h"
#include "hvx-utils.h"
+#include "htp-tensor.h"
struct htp_copy_context {
struct htp_ops_context * octx;
@@ -78,7 +80,7 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
} \
}
-DEFINE_CPY_SAMESHAPE(f32, float, 4)
+DEFINE_CPY_SAMESHAPE(f32, float, 4)
DEFINE_CPY_SAMESHAPE(f16, __fp16, 2)
#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
@@ -179,7 +181,7 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void
} \
}
-DEFINE_CPY_RESHAPE(f32, float, 4)
+DEFINE_CPY_RESHAPE(f32, float, 4)
DEFINE_CPY_RESHAPE(f16, __fp16, 2)
static void cpy_thread_f16_f32_sameshape(unsigned int nth, unsigned int ith, void * data) {
@@ -232,6 +234,41 @@ static void cpy_thread_f32_f16_sameshape(unsigned int nth, unsigned int ith, voi
}
}
+static inline void cpy_dma_sametype_sameshape(
+ struct htp_ops_context * octx,
+ const struct htp_tensor * dst,
+ const struct htp_tensor * src0,
+ uint32_t elem_size,
+ uint32_t ne00, uint32_t ne01, uint32_t ne02, uint32_t ne03,
+ uint32_t nb01, uint32_t nb02, uint32_t nb03,
+ uint32_t nb1, uint32_t nb2, uint32_t nb3
+) {
+ const bool contiguous_outer =
+ (ne02 == 1 || (nb02 == ne01 * nb01 && nb2 == ne01 * nb1)) &&
+ (ne03 == 1 || (nb03 == ne02 * nb02 && nb3 == ne02 * nb2));
+
+ dma_queue * q = octx->ctx->dma[0];
+
+ if (contiguous_outer) {
+ dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03);
+ dma_queue_pop(q);
+ return;
+ }
+
+ for (uint32_t i03 = 0; i03 < ne03; i03++) {
+ for (uint32_t i02 = 0; i02 < ne02; i02++) {
+ uint8_t* dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3;
+ uint8_t* src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03;
+ if (!dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01)) {
+ dma_queue_flush(q);
+ dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01);
+ }
+ }
+ }
+
+ dma_queue_flush(q);
+}
+
int op_cpy(struct htp_ops_context * octx) {
cpy_preamble;
@@ -264,14 +301,11 @@ int op_cpy(struct htp_ops_context * octx) {
ct.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads;
- worker_callback_t copy_fun;
+ worker_callback_t copy_fun = NULL;
+ bool use_dma = false;
if (sametype && sameshape) {
- if (src0->type == HTP_TYPE_F32) {
- copy_fun = cpy_thread_f32_sameshape;
- } else {
- copy_fun = cpy_thread_f16_sameshape;
- }
+ use_dma = true;
} else if (sameshape) {
/**/ if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32)
copy_fun = cpy_thread_f16_f32_sameshape;
@@ -289,7 +323,28 @@ int op_cpy(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- worker_pool_run_func(octx->ctx->worker_pool, copy_fun, &ct, n_threads);
+ if (use_dma) {
+ cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3);
+ } else {
+ worker_pool_run_func(octx->ctx->worker_pool, copy_fun, &ct, n_threads);
+ }
+
+ const struct htp_tensor *sync = octx->src[1];
+ if (sync) {
+ if (!use_dma) {
+ // htp_tensor_flush_all(octx->ctx, octx->dsts, 1);
+ qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
+ }
+
+ atomic_uint * sync_fence = (atomic_uint *) sync->data;
+ const uint32_t seq = (uint32_t) octx->op_params[0];
+
+ atomic_store(&sync_fence[0], seq);
+ asm volatile ("syncht" : : : "memory");
+ Q6_dccleaninva_A((void *) sync_fence);
+
+ FARF(HIGH, "ggml-hex: sync-release : fence %p seq %u\n", sync_fence, seq);
+ }
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h
index 264284bda8..190ca3a9b9 100644
--- a/ggml/src/ggml-hexagon/htp/dma-queue.h
+++ b/ggml/src/ggml-hexagon/htp/dma-queue.h
@@ -244,17 +244,18 @@ static inline dma_ptr dma_queue_pop(dma_queue * q) {
return dptr;
}
- dma_descriptor_2d * desc = &r->desc[r->pop_idx];
+ dptr = r->dptr[r->pop_idx];
+
+ volatile dma_descriptor_2d * desc = &r->desc[r->pop_idx];
// Wait for desc to complete
if (!desc->done) {
+ // FARF(ALWAYS, "dma-poll: idx %u dst %p src %p", r->pop_idx, dptr.dst, dptr.src);
while (!desc->done) {
dmpoll();
}
}
- dptr = r->dptr[r->pop_idx];
-
htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx);
r->pop_idx = (r->pop_idx + 1) & r->idx_mask;
diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
index 8176562904..c76b4d3a3a 100644
--- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
+++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
@@ -30,6 +30,8 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
+#include "htp-tensor.h"
+#include "hvx-quant.h"
#include "flash-attn-ops.h"
#include "hvx-fa-kernels.h"
@@ -85,12 +87,17 @@ struct htp_fa_context {
uint8_t * spad_m;
uint8_t * spad_a;
+ const struct htp_tensor * k;
+ const struct htp_tensor * v;
+
uint64_t t_start;
};
struct hmx_fa_context {
const struct htp_ops_context * octx;
const struct htp_tensor * sinks; // attention sinks (src[4]), NULL if absent
+ const struct htp_tensor * k;
+ const struct htp_tensor * v;
bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2
uint32_t n_threads;
@@ -214,8 +221,8 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
const uint32_t DV = nev0;
const size_t size_q_row = DK * ((q->type == HTP_TYPE_F32) ? 4 : 2);
- const size_t size_k_row = DK * sizeof(__fp16);
- const size_t size_v_row = DV * sizeof(__fp16);
+ const size_t size_k_row = htp_tensor_get_row_size(k->type, DK);
+ const size_t size_v_row = htp_tensor_get_row_size(v->type, DV);
// Scratchpad buffers for Q, K, V, Mask, and VKQ32 accumulator
uint8_t * spad_q = factx->spad_q + factx->size_q_block * ith;
@@ -364,6 +371,23 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
uint8_t * v_base = dma_queue_pop(dma).dst; // V
__fp16 * m_base = mask ? dma_queue_pop(dma).dst : NULL; // M
+ if (factx->k->type == HTP_TYPE_Q8_0) {
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir);
+ for (uint32_t r = 0; r < current_block_size; ++r) {
+ __fp16 * row_k = (__fp16 *)(k_base + r * factx->size_k_row_padded);
+ hvx_dequantize_row_q8_0_f16(row_k, row_k, DK);
+ }
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir);
+ }
+ if (factx->v->type == HTP_TYPE_Q8_0) {
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir);
+ for (uint32_t r = 0; r < current_block_size; ++r) {
+ __fp16 * row_v = (__fp16 *)(v_base + r * factx->size_v_row_padded);
+ hvx_dequantize_row_q8_0_f16(row_v, row_v, DV);
+ }
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir);
+ }
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_QK, ir);
// Inner loop processing the block from VTCM
@@ -625,6 +649,12 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data)
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start));
+ if (factx->k->type == HTP_TYPE_Q8_0) {
+ for (uint32_t r = start; r < end; ++r) {
+ __fp16 * row_k = (__fp16 *)((char *)args->curr_k + r * args->src_stride * sizeof(__fp16));
+ hvx_dequantize_row_q8_0_f16(row_k, row_k, factx->DK);
+ }
+ }
hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles[args->buf_idx], (const __fp16 *) args->curr_k, total_rows, factx->DK,
args->src_stride, start, end);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start));
@@ -673,6 +703,12 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data)
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start));
+ if (factx->v->type == HTP_TYPE_Q8_0) {
+ for (uint32_t r = start; r < end; ++r) {
+ __fp16 * row_v = (__fp16 *)((char *)args->v_src + r * args->src_stride * sizeof(__fp16));
+ hvx_dequantize_row_q8_0_f16(row_v, row_v, factx->DV);
+ }
+ }
hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV,
args->src_stride, (uint32_t) args->n_col_tiles, start, end);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start));
@@ -1809,6 +1845,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
memset(&factx, 0, sizeof(factx));
factx.octx = octx;
factx.sinks = octx->src[4]; // NULL if this op has no attention sinks
+ factx.k = k;
+ factx.v = v;
factx.n_threads = kparams->n_threads;
factx.DK = DK;
factx.DV = DV;
@@ -1853,10 +1891,10 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
// ======== VTCM allocation (GQA-aware) ========
// K/V row sizes drive the DMA descriptors (not the VTCM layout) and are used
// throughout the KV loop below.
- const size_t size_k_row = DK * sizeof(__fp16);
- const size_t size_v_row = DV * sizeof(__fp16);
- const size_t size_k_row_padded = hex_round_up(size_k_row, 128);
- const size_t size_v_row_padded = hex_round_up(size_v_row, 128);
+ const size_t size_k_row = htp_tensor_get_row_size(k->type, DK);
+ const size_t size_v_row = htp_tensor_get_row_size(v->type, DV);
+ const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128);
+ const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128);
// Build the VTCM layout once (shared with the host estimator) and place every
// scratch buffer at its computed offset.
@@ -2348,7 +2386,9 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
const struct htp_tensor * dst = octx->dst;
// Check support
- if ((q->type != HTP_TYPE_F16 && q->type != HTP_TYPE_F32) || k->type != HTP_TYPE_F16 || v->type != HTP_TYPE_F16) {
+ if ((q->type != HTP_TYPE_F16 && q->type != HTP_TYPE_F32) ||
+ (k->type != HTP_TYPE_F16 && k->type != HTP_TYPE_Q8_0) ||
+ (v->type != HTP_TYPE_F16 && v->type != HTP_TYPE_Q8_0)) {
return HTP_STATUS_NO_SUPPORT;
}
@@ -2364,6 +2404,8 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
struct htp_fa_context factx;
factx.octx = octx;
+ factx.k = k;
+ factx.v = v;
factx.t_start = HAP_perf_get_qtimer_count();
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
index bf7063e988..05769d17f7 100644
--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
@@ -12,18 +12,17 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
-#include "htp-ops.h"
+#include "htp-tensor.h"
#include "hvx-utils.h"
+#include "hvx-quant.h"
+#include "get-rows-ops.h"
+#include "work-queue.h"
struct get_rows_context {
struct htp_ops_context * octx;
- uint32_t tasks_per_thread;
- uint32_t total_tasks;
- uint32_t chunks_per_row;
- uint32_t chunk_size;
- struct fastdiv_values get_rows_div_ne10;
- struct fastdiv_values get_rows_div_ne10_ne11;
- struct fastdiv_values get_rows_div_chunks_per_row;
+ const struct htp_get_rows_kernel_params * kparams;
+ struct htp_get_rows_vtcm_layout vtcm_layout;
+ uint8_t * vtcm_base;
};
#define get_rows_preamble \
@@ -56,102 +55,161 @@ struct get_rows_context {
\
const uint32_t nr = ne10 * ne11 * ne12;
-static void get_rows_thread_f32_f32_dma(unsigned int nth, unsigned int ith, void *data) {
- struct get_rows_context * grctx = (struct get_rows_context *)data;
- struct htp_ops_context * octx = grctx->octx;
- get_rows_preamble;
-
- uint64_t qt = HAP_perf_get_qtimer_count();
-
- const uint32_t dr = grctx->tasks_per_thread;
- const uint32_t ir0 = dr * ith;
- if (ir0 >= grctx->total_tasks) {
- return;
- }
- const uint32_t ir1 = MIN(ir0 + dr, grctx->total_tasks);
-
- const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
-
- dma_queue * dma_queue = octx->ctx->dma[ith];
- for (uint32_t i = ir0; i < ir1; ++i) {
- const uint32_t i12 = fastdiv(i, &grctx->get_rows_div_ne10_ne11);
- const uint32_t rem = i - i12 * ne11 * ne10;
- const uint32_t i11 = fastdiv(rem, &grctx->get_rows_div_ne10);
- const uint32_t i10 = rem - i11 * ne10;
-
- const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
- uint32_t i01 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;
-
- if (i01 >= ne01) {
- continue;
- }
-
- const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i11*nb02 + i12*nb03;
- const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3;
-
- while (!dma_queue_push(dma_queue, dma_make_ptr((void *)dst_ptr, (const void *)src0_ptr), nb1, nb01, ne00 * sizeof(float), 1)) {
- dma_queue_pop(dma_queue);
- }
- }
- dma_queue_flush(dma_queue);
-
- qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt);
- FARF(HIGH, "get-rows-f32-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
- ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt);
+#define GET_ROWS_THREAD_ST_FN(IDX_TYPE) \
+static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \
+ struct get_rows_context * grctx = (struct get_rows_context *)data; \
+ struct htp_ops_context * octx = grctx->octx; \
+ const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \
+ get_rows_preamble; \
+ const uint32_t dr = kparams->tasks_per_thread; \
+ const uint32_t ir0 = dr * ith; \
+ if (ir0 >= kparams->total_tasks) { \
+ return; \
+ } \
+ const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \
+ const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00); \
+ dma_queue * dma_queue = octx->ctx->dma[ith]; \
+ for (uint32_t i = ir0; i < ir1; ++i) { \
+ const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); \
+ const uint32_t rem = i - i12 * ne11 * ne10; \
+ const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \
+ const uint32_t i10 = rem - i11 * ne10; \
+ const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \
+ const uint32_t i01 = (uint32_t)*src1_ptr; \
+ assert(i01 < ne01); \
+ const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \
+ const uint32_t i02 = i11 - q02 * ne02; \
+ const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \
+ const uint32_t i03 = i12 - q03 * ne03; \
+ const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; \
+ const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; \
+ while (!dma_queue_push(dma_queue, dma_make_ptr((void *)dst_ptr, (const void *)src0_ptr), nb1, nb01, \
+ row_size_bytes, 1)) { \
+ dma_queue_pop(dma_queue); \
+ } \
+ } \
+ dma_queue_flush(dma_queue); \
}
-static void get_rows_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void *data) {
- struct get_rows_context * grctx = (struct get_rows_context *)data;
- struct htp_ops_context * octx = grctx->octx;
- get_rows_preamble;
-
- uint64_t qt = HAP_perf_get_qtimer_count();
-
- const uint32_t dr = grctx->tasks_per_thread;
- const uint32_t ir0 = dr * ith;
- if (ir0 >= grctx->total_tasks) {
- return;
- }
- const uint32_t ir1 = MIN(ir0 + dr, grctx->total_tasks);
-
- const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
+GET_ROWS_THREAD_ST_FN(int32_t)
+GET_ROWS_THREAD_ST_FN(int64_t)
+
+#define GET_ROWS_THREAD_DT_FN(TYPE_NAME, SRC0_SIZE_EXPR, IDX_TYPE, COMPUTE_EXPR) \
+static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \
+ struct get_rows_context * grctx = (struct get_rows_context *)data; \
+ struct htp_ops_context * octx = grctx->octx; \
+ const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \
+ get_rows_preamble; \
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
+ const uint32_t dr = kparams->tasks_per_thread; \
+ const uint32_t ir0 = dr * ith; \
+ if (ir0 >= kparams->total_tasks) { \
+ return; \
+ } \
+ const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \
+ const uint32_t chunks_per_row = kparams->chunks_per_row; \
+ const uint32_t chunk_size = kparams->chunk_size; \
+ dma_queue * dma_queue = octx->ctx->dma[ith]; \
+ const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; \
+ uint8_t * vtcm_src0 = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \
+ uint8_t * vtcm_dst = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \
+ for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) { \
+ const uint32_t i = ir0 + step; \
+ const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \
+ const uint32_t chunk_idx = i - row_idx * chunks_per_row; \
+ const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \
+ const uint32_t rem = row_idx - i12 * ne11 * ne10; \
+ const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \
+ const uint32_t i10 = rem - i11 * ne10; \
+ const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \
+ const uint32_t i01 = (uint32_t)*src1_ptr; \
+ assert(i01 < ne01); \
+ const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \
+ const uint32_t i02 = i11 - q02 * ne02; \
+ const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \
+ const uint32_t i03 = i12 - q03 * ne03; \
+ const uint32_t offset = chunk_idx * chunk_size; \
+ const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \
+ const uint32_t cur_src0_bytes = SRC0_SIZE_EXPR(cur_elems); \
+ const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \
+ const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03 + SRC0_SIZE_EXPR(offset); \
+ dma_queue_push(dma_queue, \
+ dma_make_ptr((void *)(uintptr_t)octx->dst->data, \
+ vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \
+ cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 0); \
+ dma_queue_push(dma_queue, \
+ dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \
+ (const void *)src0_ptr), \
+ vtcm_layout->src0_spad_half_size, cur_src0_bytes, cur_src0_bytes, 1); \
+ } \
+ for (uint32_t step = 0; step < ir1 - ir0; ++step) { \
+ const uint32_t i = ir0 + step; \
+ void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \
+ void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \
+ const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \
+ const uint32_t chunk_idx = i - row_idx * chunks_per_row; \
+ const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \
+ const uint32_t rem = row_idx - i12 * ne11 * ne10; \
+ const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \
+ const uint32_t i10 = rem - i11 * ne10; \
+ const uint32_t offset = chunk_idx * chunk_size; \
+ const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \
+ const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, i); \
+ COMPUTE_EXPR; \
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, i); \
+ const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float); \
+ dma_queue_push(dma_queue, \
+ dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \
+ cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 1); \
+ const uint32_t next_step = step + 2; \
+ if (next_step < ir1 - ir0) { \
+ const uint32_t pi = ir0 + next_step; \
+ const uint32_t prow_idx = fastdiv(pi, &kparams->div_chunks_per_row); \
+ const uint32_t pchunk_idx = pi - prow_idx * chunks_per_row; \
+ const uint32_t pi12 = fastdiv(prow_idx, &kparams->div_ne10_ne11); \
+ const uint32_t prem = prow_idx - pi12 * ne11 * ne10; \
+ const uint32_t pi11 = fastdiv(prem, &kparams->div_ne10); \
+ const uint32_t pi10 = prem - pi11 * ne10; \
+ const IDX_TYPE * psrc1_ptr = (const IDX_TYPE *)(octx->src[1]->data + pi10*nb10 + pi11*nb11 + pi12*nb12); \
+ const uint32_t pi01 = (uint32_t)*psrc1_ptr; \
+ assert(pi01 < ne01); \
+ const uint32_t pq02 = fastdiv(pi11, &kparams->div_ne02); \
+ const uint32_t pi02 = pi11 - pq02 * ne02; \
+ const uint32_t pq03 = fastdiv(pi12, &kparams->div_ne03); \
+ const uint32_t pi03 = pi12 - pq03 * ne03; \
+ const uint32_t poffset = pchunk_idx * chunk_size; \
+ const uint32_t pcur_elems = (poffset < ne00) ? MIN(chunk_size, ne00 - poffset) : 0; \
+ const uint32_t pcur_src0_bytes = SRC0_SIZE_EXPR(pcur_elems); \
+ const uintptr_t psrc0_ptr = \
+ octx->src[0]->data + pi01*nb01 + pi02*nb02 + pi03*nb03 + SRC0_SIZE_EXPR(poffset); \
+ dma_queue_push(dma_queue, \
+ dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \
+ vtcm_layout->src0_spad_half_size, pcur_src0_bytes, pcur_src0_bytes, 1); \
+ } \
+ } \
+ dma_queue_flush(dma_queue); \
+}
- const uint32_t chunks_per_row = grctx->chunks_per_row;
- const uint32_t chunk_size = grctx->chunk_size;
- for (uint32_t i = ir0; i < ir1; ++i) {
- const uint32_t row_idx = fastdiv(i, &grctx->get_rows_div_chunks_per_row);
- const uint32_t chunk_idx = i - row_idx * chunks_per_row;
+#define F32_BYTES(n) ((n) * sizeof(float))
+#define F16_BYTES(n) ((n) * sizeof(__fp16))
+#define Q8_0_BYTES(n) (((n) / 32) * sizeof(block_q8_0))
- const uint32_t i12 = fastdiv(row_idx, &grctx->get_rows_div_ne10_ne11);
- const uint32_t rem = row_idx - i12 * ne11 * ne10;
- const uint32_t i11 = fastdiv(rem, &grctx->get_rows_div_ne10);
- const uint32_t i10 = rem - i11 * ne10;
+GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int32_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })
+GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int64_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })
- const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
- uint32_t i01 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;
+GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })
+GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })
- if (i01 >= ne01) {
- continue;
- }
-
- const uint32_t offset = chunk_idx * chunk_size;
- if (offset < ne00) {
- const uint32_t copy_size = MIN(chunk_size, ne00 - offset);
- const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i11*nb02 + i12*nb03 + offset * sizeof(float);
- const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float);
- hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, copy_size);
- }
- }
-
- qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt);
- FARF(HIGH, "get-rows-f32-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
- ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt);
-}
+GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })
+GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })
int op_get_rows(struct htp_ops_context * octx) {
- get_rows_preamble;
+ const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params;
- if (octx->src[0]->type != HTP_TYPE_F32) {
+ if (octx->src[0]->type != HTP_TYPE_F32 &&
+ octx->src[0]->type != HTP_TYPE_F16 &&
+ octx->src[0]->type != HTP_TYPE_Q8_0) {
return HTP_STATUS_NO_SUPPORT;
}
@@ -167,52 +225,28 @@ int op_get_rows(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
- const uint32_t nb00 = octx->src[0]->nb[0];
- const uint32_t nb0 = octx->dst->nb[0];
-
- const bool can_use_dma = (nb00 == sizeof(float)) && (nb0 == sizeof(float));
- const bool use_dma = can_use_dma && (ne00 >= 2048);
-
struct get_rows_context grctx;
grctx.octx = octx;
- grctx.get_rows_div_ne10 = init_fastdiv_values(octx->src[1]->ne[0]);
- grctx.get_rows_div_ne10_ne11 = init_fastdiv_values(octx->src[1]->ne[0] * octx->src[1]->ne[1]);
+ grctx.kparams = kparams;
+ grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
- if (use_dma) {
- grctx.chunks_per_row = 1;
- grctx.chunk_size = ne00;
- grctx.total_tasks = nr;
- grctx.get_rows_div_chunks_per_row = init_fastdiv_values(1);
+ const uint32_t ne00 = octx->src[0]->ne[0];
+ htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, kparams->n_threads);
- const uint32_t n_threads = MIN(nr, octx->n_threads);
- grctx.tasks_per_thread = (nr + n_threads - 1) / n_threads;
+ const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
- worker_pool_run_func(octx->ctx->worker_pool, get_rows_thread_f32_f32_dma, &grctx, n_threads);
+ work_queue_func_t q_func = NULL;
+ if (kparams->use_dma) {
+ q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t);
} else {
- uint32_t chunks_per_row = 1;
- uint32_t chunk_size = ne00;
- uint32_t total_tasks = nr;
-
- if (nr < octx->n_threads) {
- const uint32_t min_chunk_size = 1024;
- uint32_t max_chunks = ne00 / min_chunk_size;
- if (max_chunks == 0) {
- max_chunks = 1;
- }
- chunks_per_row = MIN((octx->n_threads + nr - 1) / nr, max_chunks);
- chunk_size = (ne00 + chunks_per_row - 1) / chunks_per_row;
- total_tasks = nr * chunks_per_row;
+ switch (octx->src[0]->type) {
+ case HTP_TYPE_F32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f32_int32_t : get_rows_thread_f32_int64_t); break;
+ case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break;
+ case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break;
+ default: return HTP_STATUS_NO_SUPPORT;
}
-
- grctx.chunks_per_row = chunks_per_row;
- grctx.chunk_size = chunk_size;
- grctx.total_tasks = total_tasks;
- grctx.get_rows_div_chunks_per_row = init_fastdiv_values(chunks_per_row);
-
- const uint32_t n_threads = MIN(total_tasks, octx->n_threads);
- grctx.tasks_per_thread = (total_tasks + n_threads - 1) / n_threads;
-
- worker_pool_run_func(octx->ctx->worker_pool, get_rows_thread_f32_f32_hvx, &grctx, n_threads);
}
+
+ work_queue_run(octx->ctx->work_queue, q_func, &grctx, kparams->n_threads);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
new file mode 100644
index 0000000000..0e7c2ca8cf
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
@@ -0,0 +1,77 @@
+#ifndef HTP_GET_ROWS_OPS_H
+#define HTP_GET_ROWS_OPS_H
+
+#include "hex-fastdiv.h"
+
+struct htp_get_rows_kernel_params {
+ int32_t n_threads;
+ int32_t use_dma;
+ int32_t chunks_per_row;
+ int32_t chunk_size;
+ int32_t total_tasks;
+ int32_t tasks_per_thread;
+ int32_t vtcm_size;
+
+ // Fastdiv helpers
+ struct fastdiv_values div_ne10;
+ struct fastdiv_values div_ne10_ne11;
+ struct fastdiv_values div_chunks_per_row;
+ struct fastdiv_values div_ne02;
+ struct fastdiv_values div_ne03;
+};
+
+struct htp_get_rows_vtcm_layout {
+ size_t total_bytes;
+ size_t off_src0;
+ size_t off_dst;
+
+ size_t src0_bytes_per_thread;
+ size_t dst_bytes_per_thread;
+
+ size_t src0_spad_half_size;
+ size_t dst_spad_half_size;
+};
+
+static inline void htp_get_rows_vtcm_layout_build(
+ struct htp_get_rows_vtcm_layout * vtcm_layout,
+ int type,
+ uint32_t ne00,
+ uint32_t n_threads) {
+
+ uint32_t src0_row_size = 0;
+ switch (type) {
+ case 0: // HTP_TYPE_F32
+ src0_row_size = ne00 * 4;
+ break;
+ case 1: // HTP_TYPE_F16
+ src0_row_size = ne00 * 2;
+ break;
+ case 8: // HTP_TYPE_Q8_0
+ src0_row_size = (ne00 / 32) * 34;
+ break;
+ default:
+ src0_row_size = 0;
+ break;
+ }
+
+ size_t src0_row_size_aligned = (src0_row_size + 255) & ~255;
+ size_t dst_row_size_aligned = (ne00 * sizeof(float) + 255) & ~255;
+
+ vtcm_layout->src0_spad_half_size = src0_row_size_aligned;
+ vtcm_layout->dst_spad_half_size = dst_row_size_aligned;
+
+ vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2;
+ vtcm_layout->dst_bytes_per_thread = dst_row_size_aligned * 2;
+
+ vtcm_layout->off_src0 = 0;
+ vtcm_layout->off_dst = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads;
+ vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;
+}
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob");
+#endif
+
+#endif // HTP_GET_ROWS_OPS_H
diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.h
index 93e87efcb4..1b39650300 100644
--- a/ggml/src/ggml-hexagon/htp/hex-utils.h
+++ b/ggml/src/ggml-hexagon/htp/hex-utils.h
@@ -39,17 +39,22 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) {
#define HEX_L2_LINE_SIZE 128
#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration)
+#define HEX_L2_FLUSH_IL_THRESHOLD 1024 // inline flush threshold
#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024)
#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024)
static inline void hex_l2flush(void * addr, size_t size) {
const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1);
const uint32_t e = (((uint32_t) addr) + size + HEX_L2_LINE_SIZE - 1) & ~(HEX_L2_LINE_SIZE - 1);
- for (uint32_t i = s; i < e; i += HEX_L2_BLOCK_SIZE) {
- Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0);
- Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1);
- Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2);
- Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3);
+ const uint32_t eb = s + ((e - s) & ~(HEX_L2_BLOCK_SIZE - 1));
+ for (uint32_t i = s; i < eb; i += HEX_L2_BLOCK_SIZE) {
+ Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 0));
+ Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 1));
+ Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 2));
+ Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 3));
+ }
+ for (uint32_t i = eb; i < e; i += HEX_L2_LINE_SIZE) {
+ Q6_dccleaninva_A((void *) i);
}
}
diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h
index e0f9a0c40d..88ecf144b9 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ctx.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h
@@ -117,8 +117,7 @@ struct htp_context {
int op_matmul(struct htp_ops_context * octx);
int op_matmul_id(struct htp_ops_context * octx);
-int op_matmul_qkv(struct htp_ops_context * octx);
-int op_matmul_ffn(struct htp_ops_context * octx);
+int op_matmul_nx(struct htp_ops_context * octx);
int op_binary(struct htp_ops_context * octx);
int op_unary(struct htp_ops_context * octx);
int op_sum_rows(struct htp_ops_context * octx);
@@ -141,5 +140,6 @@ int op_solve_tri(struct htp_ops_context * octx);
int op_gated_delta_net(struct htp_ops_context * octx);
int op_pad(struct htp_ops_context * octx);
int op_im2col(struct htp_ops_context * octx);
+int op_allreduce(struct htp_ops_context * octx);
#endif /* HTP_CTX_H */
diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h
index a138f062aa..b4023b34d3 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ops.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h
@@ -43,13 +43,6 @@ enum htp_data_type {
-// Mask to enable various stages of the Ops.
-// Used for debugging and profiling.
-enum htp_op_stage {
- HTP_OPSTAGE_QUEUE = (1 << 0), // Enable Queueing (ie calls into NPU)
- HTP_OPSTAGE_COMPUTE = (1 << 1), // Enable Compute
-};
-
// Do not reorder first 4 (used as an index)
enum htp_op_code {
HTP_OP_MUL = 0,
@@ -58,8 +51,7 @@ enum htp_op_code {
HTP_OP_DIV = 3,
HTP_OP_MUL_MAT,
HTP_OP_MUL_MAT_ID,
- HTP_OP_MUL_MAT_QKV,
- HTP_OP_MUL_MAT_FFN,
+ HTP_OP_MUL_MAT_NX,
HTP_OP_MUL_MAT_ADD,
HTP_OP_RMS_NORM,
HTP_OP_RMS_NORM_MUL,
@@ -99,12 +91,15 @@ enum htp_op_code {
HTP_OP_CONCAT,
HTP_OP_CLAMP,
HTP_OP_IM2COL,
+ HTP_OP_FENCE,
+ HTP_OP_ALLREDUCE,
+ HTP_OP_ALLREDUCE_ADD,
HTP_OP_INVALID
};
#define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS
-#define HTP_OP_MAX_INPUTS 6 // aka GGML_MAX_SRCS
+#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS
#define HTP_OP_MAX_OUTPUTS 4
#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS
#define HTP_OP_MAX_KERN_PARAMS 32
@@ -112,13 +107,16 @@ enum htp_op_code {
#define HTP_OP_MAX_BUFS 16
#define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16)
+#define HTP_FENCE_TIMEOUT (1000000000ULL)
+
#define HTP_OP_MAX_VMEM_DEFAULT (3355443200u)
#define HTP_MMAP_MAX_VMEM (2147483648u)
enum htp_tensor_flags {
- HTP_TENSOR_COMPUTE = (1U << 0), // Tensor buffer temporal compute data (not weights)
- HTP_TENSOR_DIRTY = (1U << 1) // Tensor buffer is dirty and needs to be flushed
+ HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute)
+ HTP_TENSOR_REPACK = (1U << 1), // Tensor is in repacked tiled format
+ HTP_TENSOR_FENCE = (1U << 2) // Tensor is synchronization fence (explicitly managed)
};
// Tensor descriptor
@@ -175,6 +173,7 @@ enum htp_trace_event_id {
HTP_TRACE_EVT_L2FLUSH = 1,
HTP_TRACE_EVT_INIT = 2,
HTP_TRACE_EVT_BUFF = 3,
+ HTP_TRACE_EVT_FENCE = 4,
HTP_TRACE_EVT_HVX_COMP = 20,
HTP_TRACE_EVT_HVX_A_QUANT = 21,
@@ -215,6 +214,7 @@ struct htp_opbatch_req {
uint32_t n_ops; // Number of ops
uint32_t n_traces; // Number of trace descriptors per thread
uint32_t pad; // unused
+ uint64_t seq; // Sequence number
// struct htp_buf_desc bufs[]; -- dspqueue buf 0
// struct htp_tensor tensors[]; -- dspqueue buf 0
// struct htp_op_desc ops[]; -- dspqueue buf 0
@@ -231,6 +231,7 @@ struct htp_opbatch_rsp {
uint32_t pad; // align to 8 bytes
uint64_t cycles_start; // Start cycle counter
uint64_t cycles_stop; // Stop cycle counter
+ uint64_t seq; // Sequence number
// struct htp_prof_desc profs[]; -- dspqueue buf 0
};
diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.c b/ggml/src/ggml-hexagon/htp/htp-tensor.c
index 39436e26df..ae377c9221 100644
--- a/ggml/src/ggml-hexagon/htp/htp-tensor.c
+++ b/ggml/src/ggml-hexagon/htp/htp-tensor.c
@@ -79,7 +79,14 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co
for (uint32_t i = 0; i < n; i++) {
const struct htp_tensor * t = tensors[i];
- if (!t) continue;
+ if (!t || (t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE))) {
+ continue;
+ }
+
+ if (t->size <= HEX_L2_FLUSH_IL_THRESHOLD) {
+ hex_l2flush((void *) (uintptr_t) t->data, t->size);
+ continue;
+ }
uint32_t t_start = t->data;
uint32_t t_end = t_start + t->size;
@@ -242,7 +249,7 @@ void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * co
for (uint32_t i = 0; i < n; i++) {
const struct htp_tensor * t = tensors[i];
- if (t && (t->flags & HTP_TENSOR_COMPUTE) && is_tensor_dirty(ctx, t)) {
+ if (t && !(t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE)) && is_tensor_dirty(ctx, t)) {
dirty_tensors[n_dirty++] = t;
total_dirty += t->size;
}
diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.h b/ggml/src/ggml-hexagon/htp/htp-tensor.h
index 2c3fc54c74..c9cadbae3f 100644
--- a/ggml/src/ggml-hexagon/htp/htp-tensor.h
+++ b/ggml/src/ggml-hexagon/htp/htp-tensor.h
@@ -13,6 +13,15 @@ static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) {
return (uint32_t *) &t->flags;
}
+static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) {
+ switch (type) {
+ case HTP_TYPE_F32: return ne00 * 4;
+ case HTP_TYPE_F16: return ne00 * 2;
+ case HTP_TYPE_Q8_0: return (ne00 / 32) * 34;
+ default: return 0;
+ }
+}
+
struct htp_context;
void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h
index 82e3416970..c8d0003ab5 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-arith.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h
@@ -17,9 +17,9 @@
#define hvx_arith_loop_body(dst_type, src0_type, src1_type, elem_size, vec_store, vec_op) \
do { \
- dst_type * restrict vdst = (dst_type *) dst; \
- src0_type * restrict vsrc0 = (src0_type *) src0; \
- src1_type * restrict vsrc1 = (src1_type *) src1; \
+ dst_type * vdst = (dst_type *) dst; \
+ src0_type * vsrc0 = (src0_type *) src0; \
+ src1_type * vsrc1 = (src1_type *) src1; \
\
const uint32_t epv = 128 / (elem_size); \
const uint32_t nvec = n / epv; \
@@ -57,40 +57,40 @@
// Generic macro to define alignment permutations for an op
#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \
-static inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
+static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
-static inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
+static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
-static inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
+static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
-static inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
+static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
-static inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
+static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
-static inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
+static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
-static inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
+static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
-static inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
+static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
diff --git a/ggml/src/ggml-hexagon/htp/hvx-quant.h b/ggml/src/ggml-hexagon/htp/hvx-quant.h
new file mode 100644
index 0000000000..6b172cd63c
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/hvx-quant.h
@@ -0,0 +1,165 @@
+#ifndef HVX_QUANT_H
+#define HVX_QUANT_H
+
+#include <math.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "hvx-arith.h"
+#include "hvx-base.h"
+#include "hvx-reduce.h"
+#include "hvx-repl.h"
+#include "hvx-utils.h"
+
+#ifndef GGML_COMMON_DECL_C
+#define GGML_COMMON_DECL_C
+#endif
+#include "ggml-common.h"
+#include "ggml-impl.h"
+
+static inline void hvx_quantize_row_q8_0_f32(void * restrict dst_ptr, const float * restrict src_ptr, int n) {
+ const int nb = n / QK8_0;
+ block_q8_0 * dst = (block_q8_0 *) dst_ptr;
+ HVX_Vector zero = Q6_V_vzero();
+
+ int i = 0;
+ for (; i + 3 < nb; i += 4) {
+ HVX_Vector * vx = (HVX_Vector *) (src_ptr + i * QK8_0);
+
+ HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0]));
+ HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1]));
+ HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2]));
+ HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3]));
+
+ HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero);
+ HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero);
+ HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero);
+ HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero);
+
+ HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero);
+ HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero);
+ HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero);
+ HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero);
+
+ HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf)));
+ HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf)));
+
+ HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));
+ HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));
+
+ HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0
+ HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0
+ HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16);
+ HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16);
+
+ HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);
+ HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf);
+ vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));
+ vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));
+
+ HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf);
+ HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf);
+ HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16);
+
+ hvx_vec_store_u(&dst[i + 0].d, 2, vd01_hf);
+ hvx_vec_store_u(dst[i + 0].qs, 32, vx_i8);
+
+ hvx_vec_store_u(&dst[i + 1].d, 2, Q6_V_vror_VR(vd01_hf, 64));
+ hvx_vec_store_u(dst[i + 1].qs, 32, Q6_V_vror_VR(vx_i8, 32));
+
+ hvx_vec_store_u(&dst[i + 2].d, 2, vd23_hf);
+ hvx_vec_store_u(dst[i + 2].qs, 32, Q6_V_vror_VR(vx_i8, 64));
+
+ hvx_vec_store_u(&dst[i + 3].d, 2, Q6_V_vror_VR(vd23_hf, 64));
+ hvx_vec_store_u(dst[i + 3].qs, 32, Q6_V_vror_VR(vx_i8, 96));
+ }
+
+ for (; i < nb; i++) {
+ const float * block_src = src_ptr + i * QK8_0;
+ HVX_Vector vx = *(const HVX_UVector *) block_src;
+ HVX_Vector v_abs = hvx_vec_abs_f32(vx);
+ HVX_Vector v_max = hvx_vec_reduce_max_f32(v_abs);
+ float amax = hvx_vec_get_f32(v_max);
+
+ const float d = amax / 127.0f;
+ const float id = d ? (1.0f / d) : 0.0f;
+ dst[i].d = GGML_FP32_TO_FP16(d);
+
+ HVX_Vector vid = hvx_vec_splat_f32(id);
+ HVX_Vector v_scaled = hvx_vec_mul_f32_f32(vx, vid);
+ HVX_Vector v_scaled_qf = Q6_Vqf32_vsub_VsfVsf(v_scaled, zero);
+ HVX_Vector v_scaled_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(zero, v_scaled_qf)));
+ HVX_Vector v_i16 = hvx_vec_i16_from_hf_rnd_sat(v_scaled_hf);
+ HVX_Vector v_i8 = Q6_Vb_vpack_VhVh_sat(zero, v_i16);
+
+ hvx_vec_store_u(dst[i].qs, 32, v_i8);
+ }
+}
+
+static inline void hvx_dequantize_row_q8_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) {
+ const int nb = n / QK8_0;
+ const block_q8_0 * src = (const block_q8_0 *) src_ptr;
+
+ for (int i = 0; i < nb; i++) {
+ HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
+ HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16);
+ HVX_Vector vd = Q6_V_lo_W(vp_f32);
+
+ HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs;
+
+ HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq_i8);
+ HVX_Vector v_i16 = Q6_V_lo_W(p16);
+ HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16);
+ HVX_Vector v_i32 = Q6_V_lo_W(p32);
+
+ HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32);
+ HVX_Vector res = hvx_vec_mul_f32_f32(v_f32, vd);
+
+ float * block_dst = dst_ptr + i * QK8_0;
+ hvx_vmem(block_dst) = res;
+ }
+}
+
+static inline void hvx_dequantize_row_q8_0_f16(__fp16 * restrict dst_ptr, const void * restrict src_ptr, int n) {
+ const int nb = n / QK8_0;
+ const block_q8_0 * src = (const block_q8_0 *) src_ptr;
+
+ for (int i = nb - 1; i >= 0; i--) {
+ HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
+ HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16);
+ HVX_Vector vd = Q6_V_lo_W(vp_f32);
+
+ HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs;
+
+ HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq_i8);
+ HVX_Vector v_i16 = Q6_V_lo_W(p16);
+ HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16);
+ HVX_Vector v_i32 = Q6_V_lo_W(p32);
+
+ HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32);
+ HVX_Vector res_f32 = hvx_vec_mul_f32_f32(v_f32, vd);
+
+ HVX_Vector res_f16 = hvx_vec_f32_to_f16(res_f32, Q6_V_vzero());
+
+ __fp16 * block_dst = dst_ptr + i * QK8_0;
+ hvx_vec_store_u(block_dst, QK8_0 * sizeof(__fp16), res_f16);
+ }
+}
+
+static inline void hvx_dequantize_row_f16_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) {
+ const int nb = n / 32;
+ const _Float16 * src = (const _Float16 *) src_ptr;
+
+ for (int i = 0; i < nb; i++) {
+ HVX_Vector v_f16 = *(const HVX_UVector *) (src + i * 32);
+ HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16);
+ HVX_Vector res = Q6_V_lo_W(vp_f32);
+
+ float * block_dst = dst_ptr + i * 32;
+ hvx_vmem(block_dst) = res;
+ }
+}
+
+
+
+#endif // HVX_QUANT_H
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index 880e20c995..975ba0c7af 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -18,6 +18,7 @@
#include <qurt_memory.h>
#include <remote.h>
#include <string.h>
+#include <stdatomic.h>
#include "hex-utils.h"
#include "hex-dma.h"
@@ -32,6 +33,7 @@
#include "htp_iface.h"
#include "work-queue.h"
#include "hex-profile.h"
+#include "allreduce-ops.h"
#define HMX_QUEUE_CAPACITY 16
#define HMX_QUEUE_STACK_SIZE 16384
@@ -46,6 +48,36 @@ struct htp_handle {
struct htp_context * ctx;
};
+static inline void * htp_mmap(uint32_t fd, uint32_t size) {
+ void * va = (void *)-1;
+ for (int retry = 0; retry < 2; retry++) {
+#if __HVX_ARCH__ > 73
+ va = HAP_mmap2(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0);
+#else
+ if (size > HTP_MMAP_MAX_VMEM) {
+ FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) size);
+ abort();
+ }
+ va = HAP_mmap(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0);
+#endif
+ if (va != (void *)-1 && va != NULL) {
+ return va;
+ }
+ if (retry == 0) {
+ FARF(HIGH, "mmap failed first try (va %p fd %u size %u), retrying...", va, fd, size);
+ }
+ }
+ return NULL;
+}
+
+static inline void htp_munmap(void * va, uint32_t size) {
+#if __HVX_ARCH__ > 73
+ HAP_munmap2(va, size);
+#else
+ HAP_munmap(va, size);
+#endif
+}
+
AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) {
(void) uri;
struct htp_handle * h = calloc(1, sizeof(*h));
@@ -127,11 +159,7 @@ AEEResult htp_iface_close(remote_handle64 handle) {
// release the mmaps (if any)
for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
if (ctx->mmap[i].size) {
-#if __HVX_ARCH__ > 73
- HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size);
-#else
- HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size);
-#endif
+ htp_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size);
ctx->mmap[i].size = 0;
ctx->mmap[i].base = NULL;
ctx->mmap[i].fd = -1;
@@ -175,18 +203,9 @@ AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) {
struct htp_mmap *m = &ctx->mmap[i];
if (!m->size) {
FARF(HIGH, "mmap : fd %u size %u", fd, size);
-#if __HVX_ARCH__ > 73
- void *va = HAP_mmap2(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0);
-#else
- if (size > HTP_MMAP_MAX_VMEM) { // HAP_mmap has a size limit of 2GB
- FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) size);
- abort(); // can't do much else at this point
- }
-
- void *va = HAP_mmap(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0);
-#endif
- if (va == (void*)-1) {
- FARF(ERROR, "mmap failed : va %p fd %u size %u", va, fd, (uint32_t) size);
+ void *va = htp_mmap(fd, size);
+ if (va == NULL) {
+ FARF(ERROR, "mmap failed : fd %u size %u", fd, (uint32_t) size);
return AEE_EFAILED;
}
@@ -212,11 +231,7 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) {
struct htp_mmap *m = &ctx->mmap[i];
if (fd < 0 || m->fd == fd) {
FARF(HIGH, "unmmap : base %p fd %u size %u", (void*) m->base, m->fd, (uint32_t) m->size);
-#if __HVX_ARCH__ > 73
- HAP_munmap2((void *) m->base, m->size);
-#else
- HAP_munmap((void *) m->base, m->size);
-#endif
+ htp_munmap((void *) m->base, m->size);
m->size = 0;
m->base = NULL;
m->fd = -1;
@@ -228,7 +243,7 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) {
static void vtcm_acquire(struct htp_context * ctx) {
if (!ctx->vtcm_valid) {
- int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 1000000u);
+ int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 10000000u);
if (err != 0) {
FARF(ERROR, "ggml-hex: failed to acquire VTCM: 0x%08x", (unsigned)err);
abort();
@@ -692,8 +707,45 @@ static inline void profile_stop(uint32_t mode, struct profile_data * d) {
}
}
+static int op_fence(struct htp_ops_context * octx) {
+ struct htp_context *ctx = octx->ctx;
+ struct htp_thread_trace * tr = &ctx->trace[0];
+ const uint32_t seq = (uint32_t) octx->op_params[0];
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
+
+ const struct htp_tensor * sync = octx->src[0];
+ atomic_uint * sync_fence = (atomic_uint *) sync->data;
+ uint64_t spins = 0;
+ while (1) {
+ Q6_dccleaninva_A((void *) sync_fence);
+ asm volatile ("syncht" : : : "memory");
+ uint32_t val = atomic_load(&sync_fence[0]);
+ if ((int32_t)(val - seq) >= 0) {
+ break;
+ }
+ if (++spins > HTP_FENCE_TIMEOUT) {
+ FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq %u\n", sync_fence, spins, seq);
+ break;
+ }
+ hex_pause();
+ }
+
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
+
+ FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq %u\n", sync_fence, spins, seq);
+ return HTP_STATUS_OK;
+}
+
static int execute_op(struct htp_ops_context * octx) {
switch (octx->op) {
+ case HTP_OP_FENCE:
+ return op_fence(octx);
+
+ case HTP_OP_ALLREDUCE:
+ case HTP_OP_ALLREDUCE_ADD:
+ return op_allreduce(octx);
+
case HTP_OP_MUL_MAT:
case HTP_OP_MUL_MAT_ADD:
return op_matmul(octx);
@@ -701,11 +753,8 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_MUL_MAT_ID:
return op_matmul_id(octx);
- case HTP_OP_MUL_MAT_QKV:
- return op_matmul_qkv(octx);
-
- case HTP_OP_MUL_MAT_FFN:
- return op_matmul_ffn(octx);
+ case HTP_OP_MUL_MAT_NX:
+ return op_matmul_nx(octx);
case HTP_OP_MUL:
case HTP_OP_ADD:
@@ -818,12 +867,8 @@ static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct
static inline void drop_mmap(struct htp_context *ctx, struct htp_mmap *m) {
if (m->size) {
- FARF(HIGH, "unmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size);
-#if __HVX_ARCH__ > 73
- HAP_munmap2((void *) m->base, m->size);
-#else
- HAP_munmap((void *) m->base, m->size);
-#endif
+ FARF(ALWAYS, "unmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size);
+ htp_munmap((void *) m->base, m->size);
m->size = 0;
m->base = 0;
m->fd = -1;
@@ -837,18 +882,9 @@ static inline void mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) {
for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) {
struct htp_mmap *m = &ctx->mmap[i];
if (!m->size) {
-#if __HVX_ARCH__ > 73
- void *va = HAP_mmap2(NULL, b->size, HAP_PROT_READ | HAP_PROT_WRITE, 0, b->fd, 0);
-#else
- if (b->size > HTP_MMAP_MAX_VMEM) { // HAP_mmap has a size limit of 2GB
- FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) b->size);
- abort(); // can't do much else at this point
- }
-
- void *va = HAP_mmap(NULL, b->size, HAP_PROT_READ | HAP_PROT_WRITE, 0, b->fd, 0);
-#endif
- if (va == (void*)-1) {
- FARF(ERROR, "mmap failed : va %p fd %u size %u", va, b->fd, (uint32_t) b->size);
+ void *va = htp_mmap(b->fd, b->size);
+ if (va == NULL) {
+ FARF(ERROR, "mmap failed : fd %u size %u", b->fd, (uint32_t) b->size);
abort(); // can't do much else at this point
}
@@ -856,10 +892,13 @@ static inline void mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) {
m->fd = b->fd;
m->size = b->size;
- FARF(HIGH, "mmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size);
+ FARF(ALWAYS, "mmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size);
return;
}
}
+
+ FARF(ERROR, "mmap failed : exceeded mapping capacity limit of %u", HTP_MAX_MMAPS);
+ abort();
}
static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uint32_t n_bufs) {
@@ -1081,6 +1120,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
rsp.usecs = batch_prof.usecs;
rsp.cycles_start = batch_prof.cycles_start;
rsp.cycles_stop = batch_prof.cycles_stop;
+ rsp.seq = req->seq;
if (ctx->profiler == HTP_PROF_TRACE) {
for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c
index 9d385469ae..a6adc0e61f 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.c
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c
@@ -64,6 +64,7 @@ typedef struct {
struct htp_mm_context {
const char * type;
struct htp_ops_context * octx;
+ const struct htp_tensor * act;
void (*vec_dot_1x1)(const uint32_t n, float * restrict s0,
const void * restrict vx0,
@@ -478,7 +479,7 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \
DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, NULL); \
- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \
\
if (push_ct < ct_end) { \
dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile, src0_row + push_ct * tile_row_stride), \
@@ -502,150 +503,67 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void
} \
}
-#define MATMUL_QKV_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \
-static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
+#define MATMUL_NX_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \
+static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
struct htp_mm_context * mmctx = data; \
struct htp_ops_context * octx = mmctx->octx; \
+ const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \
+ const uint32_t n_weights = kparams->n_weights; \
\
- const struct htp_tensor * restrict src0 = octx->src[0]; /* Wk */ \
- const struct htp_tensor * restrict src1 = octx->src[1]; /* x */ \
- const struct htp_tensor * restrict src2 = octx->src[2]; /* Wv */ \
- const struct htp_tensor * restrict src3 = octx->src[3]; /* Wq */ \
- const struct htp_tensor * restrict dst_k = octx->dsts[0]; \
- const struct htp_tensor * restrict dst_v = octx->dsts[1]; \
- const struct htp_tensor * restrict dst_q = octx->dsts[2]; \
- \
- const uint32_t ne00 = src0->ne[0]; \
- const uint32_t ne10 = src1->ne[0]; \
- const uint32_t src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; \
- \
- const size_t dst_k_row_size = dst_k->nb[1]; /* K and V share output width */ \
- const size_t dst_q_row_size = dst_q->nb[1]; /* Q may be wider (GQA) */ \
+ const struct htp_tensor * restrict act = octx->src[n_weights]; /* x */ \
+ const uint32_t ne10 = act->ne[0]; \
+ const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3]; \
const size_t src1_stride = mmctx->vtcm_src1_stride; \
\
- uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \
- uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith; \
- uint8_t * restrict vtcm_src3_ptr = mmctx->vtcm_src3 + mmctx->vtcm_src3_size_per_thread * ith; \
- uint8_t * restrict src1_data = mmctx->vtcm_src1; \
+ uint8_t * restrict vtcm_weight_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \
+ uint8_t * restrict src1_data = mmctx->vtcm_src1; \
\
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
- \
- const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \
const uint32_t n_prefetch = kparams->n_prefetch; \
assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \
\
- const uint8_t * restrict src0_row = (const uint8_t *) src0->data; \
- const uint8_t * restrict src2_row = (const uint8_t *) src2->data; \
- const uint8_t * restrict src3_row = (const uint8_t *) src3->data; \
- \
const uint32_t tile_size = TILE_SIZE; \
const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \
- \
- uint32_t n_k_tiles_w = ne00 / 32; \
uint32_t n_k_tiles_a = ne10 / 32; \
- uint32_t tile_row_stride = n_k_tiles_w * tile_size; \
uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \
\
dma_queue * dma_queue = octx->ctx->dma[ith]; \
\
- /* 1. Process K and V together */ \
- const uint32_t src0_nrows_kv = src0->ne[1] * src0->ne[2] * src0->ne[3]; /* src0 is Wk */ \
- uint32_t src0_nrows_per_thread_kv = (src0_nrows_kv + nth - 1) / nth; \
- src0_nrows_per_thread_kv = hex_round_up(src0_nrows_per_thread_kv, 32); \
- \
- const uint32_t start_row_kv = src0_nrows_per_thread_kv * ith; \
- const uint32_t end_row_kv = MIN(start_row_kv + src0_nrows_per_thread_kv, src0_nrows_kv); \
- \
- uint32_t ct_start_kv = start_row_kv / 32; \
- uint32_t ct_end_kv = (end_row_kv + 31) / 32; \
- \
- uint32_t push_ct = ct_start_kv; \
- if (start_row_kv < end_row_kv) { \
- for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end_kv; d++, push_ct++) { \
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \
- src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + d * tile_row_transfer_size_aligned, \
- src2_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \
- } \
- } \
- \
hvx_mm_run_quant_task(mmctx, ith); \
\
- if (start_row_kv < end_row_kv) { \
- \
- for (uint32_t ct = ct_start_kv; ct < ct_end_kv; ct++) { \
- const uint8_t * w_tile_k = dma_queue_pop(dma_queue).dst; \
- const uint8_t * w_tile_v = dma_queue_pop(dma_queue).dst; \
- \
- int valid_rows = (int)src0->ne[1] - (int)(ct * 32); \
- valid_rows = MIN(32, MAX(0, valid_rows)); \
- \
- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ith); \
- uint32_t ir1 = 0; \
- for (; ir1 + 1 < src1_nrows; ir1 += 2) { \
- const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); \
- const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); \
- \
- float * restrict dst_row0_k = (float *) (dst_k->data + ((ir1+0) * dst_k_row_size)); \
- float * restrict dst_row1_k = (float *) (dst_k->data + ((ir1+1) * dst_k_row_size)); \
- float * dst_ptr0_k = &dst_row0_k[ct * 32]; \
- float * dst_ptr1_k = &dst_row1_k[ct * 32]; \
- \
- float * restrict dst_row0_v = (float *) (dst_v->data + ((ir1+0) * dst_k_row_size)); \
- float * restrict dst_row1_v = (float *) (dst_v->data + ((ir1+1) * dst_k_row_size)); \
- float * dst_ptr0_v = &dst_row0_v[ct * 32]; \
- float * dst_ptr1_v = &dst_row1_v[ct * 32]; \
- \
- DOT_2X2(ne10, dst_ptr0_k, dst_ptr1_k, w_tile_k, src1_col0, src1_col1, valid_rows, NULL, NULL); \
- DOT_2X2(ne10, dst_ptr0_v, dst_ptr1_v, w_tile_v, src1_col0, src1_col1, valid_rows, NULL, NULL); \
- } \
+ for (uint32_t widx = 0; widx < n_weights; widx++) { \
+ const struct htp_tensor * restrict src_w = octx->src[widx]; \
+ const struct htp_tensor * restrict dst = octx->dsts[widx]; \
+ if (!src_w || !dst) continue; \
\
- for (; ir1 < src1_nrows; ++ir1) { \
- const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); \
- \
- float * restrict dst_row_k = (float *) (dst_k->data + (ir1 * dst_k_row_size)); \
- float * dst_ptr_k = &dst_row_k[ct * 32]; \
- \
- float * restrict dst_row_v = (float *) (dst_v->data + (ir1 * dst_k_row_size)); \
- float * dst_ptr_v = &dst_row_v[ct * 32]; \
- \
- DOT_2X1(ne10, dst_ptr_k, w_tile_k, src1_col, valid_rows, NULL); \
- DOT_2X1(ne10, dst_ptr_v, w_tile_v, src1_col, valid_rows, NULL); \
- } \
- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ith); \
+ const uint32_t ne00 = src_w->ne[0]; \
+ const uint32_t ne01 = src_w->ne[1]; \
+ const size_t dst_row_size = dst->nb[1]; \
+ const uint8_t * restrict src_w_row = (const uint8_t *) src_w->data; \
\
- if (push_ct < ct_end_kv) { \
- dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile_k, src0_row + push_ct * tile_row_stride), \
- aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \
- dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile_v, src2_row + push_ct * tile_row_stride), \
- aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \
- push_ct++; \
- } \
- } \
- } \
+ uint32_t n_k_tiles_w = ne00 / 32; \
+ uint32_t tile_row_stride = n_k_tiles_w * tile_size; \
\
- /* 2. Process Q separately */ \
- const uint32_t src0_nrows_q = src3->ne[1] * src3->ne[2] * src3->ne[3]; /* src3 is Wq */ \
- uint32_t src0_nrows_per_thread_q = (src0_nrows_q + nth - 1) / nth; \
- src0_nrows_per_thread_q = hex_round_up(src0_nrows_per_thread_q, 32); \
+ const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3]; \
+ uint32_t src0_nrows_per_thread = (src0_nrows + nth - 1) / nth; \
+ src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); \
\
- const uint32_t start_row_q = src0_nrows_per_thread_q * ith; \
- const uint32_t end_row_q = MIN(start_row_q + src0_nrows_per_thread_q, src0_nrows_q); \
+ const uint32_t start_row = src0_nrows_per_thread * ith; \
+ const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_nrows); \
+ if (start_row >= end_row) continue; \
\
- if (start_row_q < end_row_q) { \
- uint32_t ct_start_q = start_row_q / 32; \
- uint32_t ct_end_q = (end_row_q + 31) / 32; \
+ uint32_t ct_start = start_row / 32; \
+ uint32_t ct_end = (end_row + 31) / 32; \
\
- uint32_t push_ct = ct_start_q; \
- for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end_q; d++, push_ct++) { \
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src3_ptr + d * tile_row_transfer_size_aligned, \
- src3_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \
+ uint32_t push_ct = ct_start; \
+ for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \
+ dma_queue_push(dma_queue, dma_make_ptr(vtcm_weight_ptr + d * tile_row_transfer_size_aligned, \
+ src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \
} \
\
- for (uint32_t ct = ct_start_q; ct < ct_end_q; ct++) { \
- const uint8_t * w_tile_q = dma_queue_pop(dma_queue).dst; \
- \
- int valid_rows = (int)src3->ne[1] - (int)(ct * 32); \
+ for (uint32_t ct = ct_start; ct < ct_end; ct++) { \
+ const uint8_t * w_tile = dma_queue_pop(dma_queue).dst; \
+ int valid_rows = (int)ne01 - (int)(ct * 32); \
valid_rows = MIN(32, MAX(0, valid_rows)); \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \
@@ -654,26 +572,24 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith,
const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); \
const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); \
\
- float * restrict dst_row0_q = (float *) (dst_q->data + ((ir1+0) * dst_q_row_size)); \
- float * restrict dst_row1_q = (float *) (dst_q->data + ((ir1+1) * dst_q_row_size)); \
- float * dst_ptr0_q = &dst_row0_q[ct * 32]; \
- float * dst_ptr1_q = &dst_row1_q[ct * 32]; \
+ float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size)); \
+ float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size)); \
+ float * dst_ptr0 = &dst_row0[ct * 32]; \
+ float * dst_ptr1 = &dst_row1[ct * 32]; \
\
- DOT_2X2(ne10, dst_ptr0_q, dst_ptr1_q, w_tile_q, src1_col0, src1_col1, valid_rows, NULL, NULL); \
+ DOT_2X2(ne10, dst_ptr0, dst_ptr1, w_tile, src1_col0, src1_col1, valid_rows, NULL, NULL); \
} \
\
for (; ir1 < src1_nrows; ++ir1) { \
const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); \
- \
- float * restrict dst_row_q = (float *) (dst_q->data + (ir1 * dst_q_row_size)); \
- float * dst_ptr_q = &dst_row_q[ct * 32]; \
- \
- DOT_2X1(ne10, dst_ptr_q, w_tile_q, src1_col, valid_rows, NULL); \
+ float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); \
+ float * dst_ptr = &dst_row[ct * 32]; \
+ DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, NULL); \
} \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \
\
- if (push_ct < ct_end_q) { \
- dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile_q, src3_row + push_ct * tile_row_stride), \
+ if (push_ct < ct_end) { \
+ dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile, src_w_row + push_ct * tile_row_stride), \
aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \
push_ct++; \
} \
@@ -681,121 +597,6 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith,
} \
}
-#define MATMUL_FFN_2D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \
-static void hvx_mm_ffn_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
- struct htp_mm_context * mmctx = data; \
- struct htp_ops_context * octx = mmctx->octx; \
- \
- const struct htp_tensor * restrict src0 = octx->src[0]; /* Wgate */ \
- const struct htp_tensor * restrict src1 = octx->src[1]; /* y */ \
- const struct htp_tensor * restrict src2 = octx->src[2]; /* Wup */ \
- const struct htp_tensor * restrict dst_gate = octx->dsts[0]; \
- const struct htp_tensor * restrict dst_up = octx->dsts[1]; \
- \
- const uint32_t ne00 = src0->ne[0]; \
- const uint32_t ne01 = src0->ne[1]; \
- const uint32_t ne10 = src1->ne[0]; \
- const uint32_t src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; \
- \
- const size_t dst_row_size = dst_gate->nb[1]; \
- const size_t src1_stride = mmctx->vtcm_src1_stride; \
- \
- uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \
- uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith; \
- uint8_t * restrict src1_data = mmctx->vtcm_src1; \
- \
- struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
- \
- const uint8_t * restrict src0_row = (const uint8_t *) src0->data; \
- const uint8_t * restrict src2_row = (const uint8_t *) src2->data; \
- \
- const uint32_t tile_size = TILE_SIZE; \
- const uint32_t aligned_tile_size = hex_align_up(tile_size, 128); \
- \
- const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \
- const uint32_t n_prefetch = kparams->n_prefetch; \
- assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); \
- \
- uint32_t n_k_tiles_w = ne00 / 32; \
- uint32_t n_k_tiles_a = ne10 / 32; \
- uint32_t tile_row_stride = n_k_tiles_w * tile_size; \
- uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; \
- dma_queue * dma_queue = octx->ctx->dma[ith]; \
- \
- const uint32_t src0_nrows = ne01 * src0->ne[2] * src0->ne[3]; \
- const uint32_t src0_start_row = mmctx->src0_nrows_per_thread * ith; \
- const uint32_t src0_end_row = MIN(src0_start_row + mmctx->src0_nrows_per_thread, src0_nrows); \
- \
- uint32_t ct_start = src0_start_row / 32; \
- uint32_t ct_end = (src0_end_row + 31) / 32; \
- \
- uint32_t push_ct = ct_start; \
- if (src0_start_row < src0_end_row) { \
- for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \
- src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + d * tile_row_transfer_size_aligned, \
- src2_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \
- } \
- } \
- \
- hvx_mm_run_quant_task(mmctx, ith); \
- \
- if (src0_start_row >= src0_end_row) { \
- return; \
- } \
- \
- for (uint32_t ct = ct_start; ct < ct_end; ct++) { \
- const uint8_t * w_tile_gate = dma_queue_pop(dma_queue).dst; \
- const uint8_t * w_tile_up = dma_queue_pop(dma_queue).dst; \
- \
- int valid_rows = (int)ne01 - (int)(ct * 32); \
- valid_rows = MIN(32, MAX(0, valid_rows)); \
- \
- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \
- uint32_t ir1 = 0; \
- for (; ir1 + 1 < src1_nrows; ir1 += 2) { \
- const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); \
- const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); \
- \
- float * restrict dst_row0_gate = (float *) (dst_gate->data + ((ir1+0) * dst_row_size)); \
- float * restrict dst_row1_gate = (float *) (dst_gate->data + ((ir1+1) * dst_row_size)); \
- float * dst_ptr0_gate = &dst_row0_gate[ct * 32]; \
- float * dst_ptr1_gate = &dst_row1_gate[ct * 32]; \
- \
- float * restrict dst_row0_up = (float *) (dst_up->data + ((ir1+0) * dst_row_size)); \
- float * restrict dst_row1_up = (float *) (dst_up->data + ((ir1+1) * dst_row_size)); \
- float * dst_ptr0_up = &dst_row0_up[ct * 32]; \
- float * dst_ptr1_up = &dst_row1_up[ct * 32]; \
- \
- DOT_2X2(ne10, dst_ptr0_gate, dst_ptr1_gate, w_tile_gate, src1_col0, src1_col1, valid_rows, NULL, NULL); \
- DOT_2X2(ne10, dst_ptr0_up, dst_ptr1_up, w_tile_up, src1_col0, src1_col1, valid_rows, NULL, NULL); \
- } \
- \
- for (; ir1 < src1_nrows; ++ir1) { \
- const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); \
- \
- float * restrict dst_row_gate = (float *) (dst_gate->data + (ir1 * dst_row_size)); \
- float * dst_ptr_gate = &dst_row_gate[ct * 32]; \
- \
- float * restrict dst_row_up = (float *) (dst_up->data + (ir1 * dst_row_size)); \
- float * dst_ptr_up = &dst_row_up[ct * 32]; \
- \
- DOT_2X1(ne10, dst_ptr_gate, w_tile_gate, src1_col, valid_rows, NULL); \
- DOT_2X1(ne10, dst_ptr_up, w_tile_up, src1_col, valid_rows, NULL); \
- } \
- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \
- \
- if (push_ct < ct_end) { \
- dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile_gate, src0_row + push_ct * tile_row_stride), \
- aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \
- dma_queue_push(dma_queue, dma_make_ptr((uint8_t *)w_tile_up, src2_row + push_ct * tile_row_stride), \
- aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \
- push_ct++; \
- } \
- } \
-}
-
MATMUL_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1)
MATMUL_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1)
MATMUL_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1)
@@ -812,7 +613,7 @@ MATMUL_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot
static void name(unsigned int nth, unsigned int ith, void * data) { \
struct htp_mm_context * mmctx = data; \
struct htp_ops_context * octx = mmctx->octx; \
- const struct htp_tensor * src = octx->src[1]; \
+ const struct htp_tensor * src = mmctx->act; \
const uint32_t ne0 = src->ne[0]; \
const uint32_t ne1 = src->ne[1]; \
const uint32_t ne2 = src->ne[2]; \
@@ -854,7 +655,7 @@ static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, vo
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
- const struct htp_tensor * src = octx->src[1];
+ const struct htp_tensor * src = mmctx->act;
quantize_f32_q8_0_tiled_block_kernel(
(const float *) src->data,
@@ -878,7 +679,7 @@ static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, vo
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
- const struct htp_tensor * src = octx->src[1];
+ const struct htp_tensor * src = mmctx->act;
quantize_f32_q8_1_tiled_block_kernel(
(const float *) src->data,
@@ -909,30 +710,17 @@ MATVEC_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x1)
MATVEC_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x1)
-MATMUL_QKV_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1)
-MATMUL_QKV_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1)
-MATMUL_QKV_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1)
-MATMUL_QKV_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1)
-MATMUL_QKV_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1)
-MATMUL_QKV_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x2, flat_vec_dot_q4_0_32x1)
-MATMUL_QKV_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x2, flat_vec_dot_q4_1_32x1)
-MATMUL_QKV_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x2, flat_vec_dot_q8_0_32x1)
-MATMUL_QKV_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x2, flat_vec_dot_iq4nl_32x1)
-MATMUL_QKV_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot_mxfp4_32x1)
-
-
-MATMUL_FFN_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1)
-MATMUL_FFN_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1)
-MATMUL_FFN_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1)
-MATMUL_FFN_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1)
-MATMUL_FFN_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1)
-
-MATMUL_FFN_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x2, flat_vec_dot_q4_0_32x1)
-MATMUL_FFN_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x2, flat_vec_dot_q4_1_32x1)
-MATMUL_FFN_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x2, flat_vec_dot_q8_0_32x1)
-MATMUL_FFN_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x2, flat_vec_dot_iq4nl_32x1)
-MATMUL_FFN_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot_mxfp4_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x2, flat_vec_dot_q4_0_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x2, flat_vec_dot_q4_1_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x2, flat_vec_dot_q8_0_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x2, flat_vec_dot_iq4nl_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot_mxfp4_32x1)
static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) {
htp_matmul_preamble;
@@ -1353,6 +1141,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
struct htp_mm_context mmctx_struct = {0};
struct htp_mm_context * mmctx = &mmctx_struct;
mmctx->octx = octx;
+ mmctx->act = src1;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
@@ -1528,7 +1317,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads,
- dst_row_size, src0_row_size, src1_row_size, src2 ? src2->nb[1] : 0, kparams->n_prefetch, false, false, false);
+ dst_row_size, src0_row_size, src1_row_size, src2 ? src2->nb[1] : 0, kparams->n_prefetch, false, false);
if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM ||
kparams->kernel_type == HTP_MM_KERNEL_HVX_F32_F32_VTCM ||
@@ -1587,297 +1376,97 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
-static void hvx_mm_qkv_2d(unsigned int nth, unsigned int ith, void * data) {
+static void hvx_mm_nx_2d(unsigned int nth, unsigned int ith, void * data) {
struct htp_mm_context * mmctx = data;
struct htp_ops_context * octx = mmctx->octx;
+ const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
+ const uint32_t n_weights = kparams->n_weights;
- const struct htp_tensor * restrict src0 = octx->src[0]; // Wk
- const struct htp_tensor * restrict src1 = octx->src[1]; // x
- const struct htp_tensor * restrict src2 = octx->src[2]; // Wv
- const struct htp_tensor * restrict src3 = octx->src[3]; // Wq
- const struct htp_tensor * restrict dst_k = octx->dsts[0];
- const struct htp_tensor * restrict dst_v = octx->dsts[1];
- const struct htp_tensor * restrict dst_q = octx->dsts[2];
-
- const uint32_t ne00 = src0->ne[0];
- const uint32_t ne01 = src0->ne[1];
- const uint32_t ne02 = src0->ne[2];
- const uint32_t ne03 = src0->ne[3];
-
- const uint32_t ne11 = src1->ne[1];
- const uint32_t ne12 = src1->ne[2];
- const uint32_t ne13 = src1->ne[3];
-
- const uint32_t src0_nrows = ne01 * ne02 * ne03;
- const uint32_t src1_nrows = ne11 * ne12 * ne13;
-
- const uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread;
- const uint32_t src0_start_row = src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
- const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);
-
- const size_t dst_k_row_size = dst_k->nb[1]; // K and V share output width
- const size_t dst_q_row_size = dst_q->nb[1]; // Q may be wider (GQA)
- const size_t src0_row_size = src0->nb[1];
- const size_t src2_row_size = src2->nb[1];
- const size_t src3_row_size = src3->nb[1];
-
- const size_t src0_stride = mmctx->vtcm_src0_stride;
- const size_t src2_stride = mmctx->vtcm_src2_stride;
- const size_t src3_stride = mmctx->vtcm_src3_stride;
+ const struct htp_tensor * restrict act = octx->src[n_weights];
+ const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3];
const size_t src1_stride = mmctx->vtcm_src1_stride;
uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
- uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith;
- uint8_t * restrict vtcm_src3_ptr = mmctx->vtcm_src3 + mmctx->vtcm_src3_size_per_thread * ith;
- uint8_t * restrict src1_data = mmctx->vtcm_src1;
+ uint8_t * restrict src1_data = mmctx->vtcm_src1;
dma_queue * dma_queue = octx->ctx->dma[ith];
-
- const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const uint32_t n_prefetch = kparams->n_prefetch;
assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0);
const uint32_t prefetch_mask = n_prefetch - 1;
- const uint8_t * restrict src0_row = (const uint8_t *) src0->data;
- const uint8_t * restrict src2_row = (const uint8_t *) src2->data;
- const uint8_t * restrict src3_row = (const uint8_t *) src3->data;
-
- // Prefill spad with src0, src2, src3 rows
- if (src0_start_row < src0_end_row) {
- for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {
- const int is0 = (ir0 - src0_start_row);
- if (is0 >= (int)n_prefetch) {
- break;
- }
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size),
- src0_stride, src0_row_size, src0_row_size, 2);
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size),
- src2_stride, src2_row_size, src2_row_size, 2);
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src3_ptr + is0 * src3_stride, src3_row + ir0 * src3_row_size),
- src3_stride, src3_row_size, src3_row_size, 2);
- }
- }
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
hvx_mm_run_quant_task(mmctx, ith);
- if (src0_start_row >= src0_end_row) {
- return;
- }
-
- // Process rows
- for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {
- const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;
- const uint8_t * ss2 = dma_queue_pop(dma_queue).dst;
- const uint8_t * ss3 = dma_queue_pop(dma_queue).dst;
-
- // Process src1 columns in pairs (2×2 tiling)
- uint32_t ir1 = 0;
- for (; ir1 + 1 < src1_nrows; ir1 += 2) {
- const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride);
- const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride);
-
- float * restrict dst_row0_k = (float *) (dst_k->data + ((ir1+0) * dst_k_row_size));
- float * restrict dst_row1_k = (float *) (dst_k->data + ((ir1+1) * dst_k_row_size));
- mmctx->vec_dot_2x2(ne00, &dst_row0_k[ir0], &dst_row1_k[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1);
-
- float * restrict dst_row0_v = (float *) (dst_v->data + ((ir1+0) * dst_k_row_size));
- float * restrict dst_row1_v = (float *) (dst_v->data + ((ir1+1) * dst_k_row_size));
- mmctx->vec_dot_2x2(ne00, &dst_row0_v[ir0], &dst_row1_v[ir0], ss2, ss2 + src2_stride, src1_col0, src1_col1);
-
- float * restrict dst_row0_q = (float *) (dst_q->data + ((ir1+0) * dst_q_row_size));
- float * restrict dst_row1_q = (float *) (dst_q->data + ((ir1+1) * dst_q_row_size));
- mmctx->vec_dot_2x2(ne00, &dst_row0_q[ir0], &dst_row1_q[ir0], ss3, ss3 + src3_stride, src1_col0, src1_col1);
- }
-
- // Handle remaining src1 rows (fallback to 2×1)
- for (; ir1 < src1_nrows; ++ir1) {
- const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride);
-
- float * restrict dst_row_k = (float *) (dst_k->data + (ir1 * dst_k_row_size));
- mmctx->vec_dot_2x1(ne00, &dst_row_k[ir0], ss0, ss0 + src0_stride, src1_col);
-
- float * restrict dst_row_v = (float *) (dst_v->data + (ir1 * dst_k_row_size));
- mmctx->vec_dot_2x1(ne00, &dst_row_v[ir0], ss2, ss2 + src2_stride, src1_col);
-
- float * restrict dst_row_q = (float *) (dst_q->data + (ir1 * dst_q_row_size));
- mmctx->vec_dot_2x1(ne00, &dst_row_q[ir0], ss3, ss3 + src3_stride, src1_col);
- }
-
- // Prefetch next (n + vtcm_nrows) rows
- const int pr0 = (ir0 + n_prefetch);
- const int is0 = (pr0 - src0_start_row) & prefetch_mask;
- if (pr0 < src0_end_row_x2) {
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size),
- src0_stride, src0_row_size, src0_row_size, 2);
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + pr0 * src2_row_size),
- src2_stride, src2_row_size, src2_row_size, 2);
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src3_ptr + is0 * src3_stride, src3_row + pr0 * src3_row_size),
- src3_stride, src3_row_size, src3_row_size, 2);
- }
- }
-
- // Process last row (if any)
- if (src0_end_row != src0_end_row_x2) {
- uint32_t ir0 = src0_end_row_x2;
- const int is0 = (ir0 - src0_start_row) & prefetch_mask;
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size),
- src0_stride, src0_row_size, src0_row_size, 1);
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size),
- src2_stride, src2_row_size, src2_row_size, 1);
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src3_ptr + is0 * src3_stride, src3_row + ir0 * src3_row_size),
- src3_stride, src3_row_size, src3_row_size, 1);
-
- const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;
- const uint8_t * ss2 = dma_queue_pop(dma_queue).dst;
- const uint8_t * ss3 = dma_queue_pop(dma_queue).dst;
-
- for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) {
- const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride);
-
- float * restrict dst_row_k = (float *) (dst_k->data + (ir1 * dst_k_row_size));
- mmctx->vec_dot_1x1(ne00, &dst_row_k[ir0], ss0, src1_col);
+ for (uint32_t widx = 0; widx < n_weights; widx++) {
+ const struct htp_tensor * restrict src_w = octx->src[widx];
+ const struct htp_tensor * restrict dst = octx->dsts[widx];
+ if (!src_w || !dst) continue;
- float * restrict dst_row_v = (float *) (dst_v->data + (ir1 * dst_k_row_size));
- mmctx->vec_dot_1x1(ne00, &dst_row_v[ir0], ss2, src1_col);
+ const uint32_t ne00 = src_w->ne[0];
+ const uint32_t ne01 = src_w->ne[1];
+ const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3];
- float * restrict dst_row_q = (float *) (dst_q->data + (ir1 * dst_q_row_size));
- mmctx->vec_dot_1x1(ne00, &dst_row_q[ir0], ss3, src1_col);
- }
- }
-}
-
-static void hvx_mm_ffn_2d(unsigned int nth, unsigned int ith, void * data) {
- struct htp_mm_context * mmctx = data;
- struct htp_ops_context * octx = mmctx->octx;
+ uint32_t src0_nrows_per_thread = (src0_nrows + nth - 1) / nth;
+ src0_nrows_per_thread += (src0_nrows_per_thread & 1);
- const struct htp_tensor * restrict src0 = octx->src[0]; // Wgate
- const struct htp_tensor * restrict src1 = octx->src[1]; // y
- const struct htp_tensor * restrict src2 = octx->src[2]; // Wup
- const struct htp_tensor * restrict dst_gate = octx->dsts[0];
- const struct htp_tensor * restrict dst_up = octx->dsts[1];
+ const uint32_t src0_start_row = src0_nrows_per_thread * ith;
+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
+ const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);
+ if (src0_start_row >= src0_end_row) continue;
- const uint32_t ne00 = src0->ne[0];
- const uint32_t ne01 = src0->ne[1];
- const uint32_t ne02 = src0->ne[2];
- const uint32_t ne03 = src0->ne[3];
+ const size_t dst_row_size = dst->nb[1];
+ const size_t src0_row_size = src_w->nb[1];
+ const size_t src0_stride = hex_round_up(src0_row_size, 128);
- const uint32_t ne11 = src1->ne[1];
- const uint32_t ne12 = src1->ne[2];
- const uint32_t ne13 = src1->ne[3];
+ const uint8_t * restrict src0_row = (const uint8_t *) src_w->data;
- const uint32_t src0_nrows = ne01 * ne02 * ne03;
- const uint32_t src1_nrows = ne11 * ne12 * ne13;
-
- const uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread;
- const uint32_t src0_start_row = src0_nrows_per_thread * ith;
- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
- const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);
-
- const size_t dst_row_size = dst_gate->nb[1];
- const size_t src0_row_size = src0->nb[1];
- const size_t src2_row_size = src2->nb[1];
-
- const size_t src0_stride = mmctx->vtcm_src0_stride;
- const size_t src2_stride = mmctx->vtcm_src2_stride;
- const size_t src1_stride = mmctx->vtcm_src1_stride;
-
- uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith;
- uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith;
- uint8_t * restrict src1_data = mmctx->vtcm_src1;
-
- dma_queue * dma_queue = octx->ctx->dma[ith];
-
- const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
- const uint32_t n_prefetch = kparams->n_prefetch;
- assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0);
- const uint32_t prefetch_mask = n_prefetch - 1;
-
- const uint8_t * restrict src0_row = (const uint8_t *) src0->data;
- const uint8_t * restrict src2_row = (const uint8_t *) src2->data;
-
- // Prefill spad with src0, src2 rows
- if (src0_start_row < src0_end_row) {
for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {
const int is0 = (ir0 - src0_start_row);
- if (is0 >= (int)n_prefetch) {
- break;
- }
+ if (is0 >= (int)n_prefetch) break;
dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size),
src0_stride, src0_row_size, src0_row_size, 2);
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size),
- src2_stride, src2_row_size, src2_row_size, 2);
- }
- }
-
- hvx_mm_run_quant_task(mmctx, ith);
-
- if (src0_start_row >= src0_end_row) {
- return;
- }
-
- // Process rows
- for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {
- const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;
- const uint8_t * ss2 = dma_queue_pop(dma_queue).dst;
-
- // Process src1 columns in pairs (2×2 tiling)
- uint32_t ir1 = 0;
- for (; ir1 + 1 < src1_nrows; ir1 += 2) {
- const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride);
- const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride);
-
- float * restrict dst_row0_gate = (float *) (dst_gate->data + ((ir1+0) * dst_row_size));
- float * restrict dst_row1_gate = (float *) (dst_gate->data + ((ir1+1) * dst_row_size));
- mmctx->vec_dot_2x2(ne00, &dst_row0_gate[ir0], &dst_row1_gate[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1);
-
- float * restrict dst_row0_up = (float *) (dst_up->data + ((ir1+0) * dst_row_size));
- float * restrict dst_row1_up = (float *) (dst_up->data + ((ir1+1) * dst_row_size));
- mmctx->vec_dot_2x2(ne00, &dst_row0_up[ir0], &dst_row1_up[ir0], ss2, ss2 + src2_stride, src1_col0, src1_col1);
}
- // Handle remaining src1 rows (fallback to 2×1)
- for (; ir1 < src1_nrows; ++ir1) {
- const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride);
-
- float * restrict dst_row_gate = (float *) (dst_gate->data + (ir1 * dst_row_size));
- mmctx->vec_dot_2x1(ne00, &dst_row_gate[ir0], ss0, ss0 + src0_stride, src1_col);
-
- float * restrict dst_row_up = (float *) (dst_up->data + (ir1 * dst_row_size));
- mmctx->vec_dot_2x1(ne00, &dst_row_up[ir0], ss2, ss2 + src2_stride, src1_col);
- }
+ for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) {
+ const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0);
+ uint32_t ir1 = 0;
+ for (; ir1 + 1 < src1_nrows; ir1 += 2) {
+ const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride);
+ const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride);
+ float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size));
+ float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size));
+ mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1);
+ }
+ for (; ir1 < src1_nrows; ++ir1) {
+ const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride);
+ float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size));
+ mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, src1_col);
+ }
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0);
- // Prefetch next rows
- const int pr0 = (ir0 + n_prefetch);
- const int is0 = (pr0 - src0_start_row) & prefetch_mask;
- if (pr0 < src0_end_row_x2) {
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size),
- src0_stride, src0_row_size, src0_row_size, 2);
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + pr0 * src2_row_size),
- src2_stride, src2_row_size, src2_row_size, 2);
+ const int pr0 = (ir0 + n_prefetch);
+ const int is0 = (pr0 - src0_start_row) & prefetch_mask;
+ if (pr0 < src0_end_row_x2) {
+ dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size),
+ src0_stride, src0_row_size, src0_row_size, 2);
+ }
}
- }
-
- // Process last row (if any)
- if (src0_end_row != src0_end_row_x2) {
- uint32_t ir0 = src0_end_row_x2;
- const int is0 = (ir0 - src0_start_row) & prefetch_mask;
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size),
- src0_stride, src0_row_size, src0_row_size, 1);
- dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size),
- src2_stride, src2_row_size, src2_row_size, 1);
- const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;
- const uint8_t * ss2 = dma_queue_pop(dma_queue).dst;
-
- for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) {
- const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride);
-
- float * restrict dst_row_gate = (float *) (dst_gate->data + (ir1 * dst_row_size));
- mmctx->vec_dot_1x1(ne00, &dst_row_gate[ir0], ss0, src1_col);
-
- float * restrict dst_row_up = (float *) (dst_up->data + (ir1 * dst_row_size));
- mmctx->vec_dot_1x1(ne00, &dst_row_up[ir0], ss2, src1_col);
+ if (src0_end_row != src0_end_row_x2) {
+ uint32_t ir0 = src0_end_row_x2;
+ const int is0 = (ir0 - src0_start_row) & prefetch_mask;
+ dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size),
+ src0_stride, src0_row_size, src0_row_size, 1);
+ const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0);
+ for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) {
+ const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride);
+ float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size));
+ mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col);
+ }
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0);
}
}
}
@@ -3485,7 +3074,7 @@ static int hvx_mm_matmul_id(
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads,
- 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false, false);
+ 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false);
size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes;
@@ -3610,6 +3199,7 @@ int op_matmul_id(struct htp_ops_context * octx) {
struct htp_mm_context mmctx_struct = {0};
struct htp_mm_context * mmctx = &mmctx_struct;
mmctx->octx = octx;
+ mmctx->act = src1;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
@@ -3690,18 +3280,15 @@ int op_matmul_id(struct htp_ops_context * octx) {
return s;
}
-
-int op_matmul_qkv(struct htp_ops_context * octx) {
+int op_matmul_nx(struct htp_ops_context * octx) {
struct htp_thread_trace * tr = &octx->ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
- const struct htp_tensor * restrict src0 = octx->src[0]; // Wk
- const struct htp_tensor * restrict src1 = octx->src[1]; // x
- const struct htp_tensor * restrict src2 = octx->src[2]; // Wv
- const struct htp_tensor * restrict src3 = octx->src[3]; // Wq
- const struct htp_tensor * restrict dst_k = octx->dsts[0];
- const struct htp_tensor * restrict dst_v = octx->dsts[1];
- const struct htp_tensor * restrict dst_q = octx->dsts[2];
+ const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
+ const uint32_t n_weights = kparams->n_weights;
+
+ const struct htp_tensor * restrict src0 = octx->src[0]; // first weight
+ const struct htp_tensor * restrict act = octx->src[n_weights]; // activation x
bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 ||
src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL ||
@@ -3710,19 +3297,9 @@ int op_matmul_qkv(struct htp_ops_context * octx) {
struct htp_mm_context mmctx_struct = {0};
struct htp_mm_context * mmctx = &mmctx_struct;
mmctx->octx = octx;
+ mmctx->act = act;
- const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
-
- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3];
-
- // Compute src0_nrows_per_thread
- mmctx->src0_nrows_per_thread = (src0_nrows + octx->n_threads - 1) / octx->n_threads;
- if (is_repacked) {
- mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32);
- } else {
- mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even
- }
+ const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3];
const size_t src0_row_size = src0->nb[1];
const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);
@@ -3732,7 +3309,7 @@ int op_matmul_qkv(struct htp_ops_context * octx) {
}
const uint32_t qk = QK_Q8_0_TILED;
- const uint32_t nb = (src1->ne[0] + qk - 1) / qk;
+ const uint32_t nb = (act->ne[0] + qk - 1) / qk;
const uint32_t total_nb = src1_nrows * nb;
worker_callback_t quant_task_func;
@@ -3758,185 +3335,39 @@ int op_matmul_qkv(struct htp_ops_context * octx) {
size_t src1_row_size;
if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {
- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(src1->ne[0]) : htp_mm_q8_0_flat_row_size(src1->ne[0]);
+ src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(act->ne[0]) : htp_mm_q8_0_flat_row_size(act->ne[0]);
} else {
- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(src1->ne[0]) : htp_mm_q8_0_tiled_row_size(src1->ne[0]);
+ src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]);
}
struct htp_mm_hvx_vtcm_layout L;
- htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, src1->ne[0], src1_nrows, octx->n_threads,
- 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, false, true, false);
+ htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads,
+ 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, false, true);
size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes;
if (octx->ctx->vtcm_size < vtcm_size) {
- FARF(ERROR, "matmul-qkv: current VTCM reservation %zu is too small, needed %zu\n",
+ FARF(ERROR, "matmul-nx: current VTCM reservation %zu is too small, needed %zu\n",
octx->ctx->vtcm_size, vtcm_size);
return HTP_STATUS_VTCM_TOO_SMALL;
}
uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;
- mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);
mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);
- mmctx->vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src2);
- mmctx->vtcm_src3 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src3);
- mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);
-
- octx->src1_spad.src = NULL;
- octx->src0_spad.src = NULL;
- octx->src2_spad.src = NULL;
- octx->src3_spad.src = NULL;
- octx->dst_spad.src = NULL;
-
- mmctx->vtcm_src0_stride = is_repacked ? 0 : src0_row_size_padded;
- mmctx->vtcm_src2_stride = is_repacked ? 0 : src0_row_size_padded;
- mmctx->vtcm_src3_stride = is_repacked ? 0 : src0_row_size_padded;
- mmctx->vtcm_src1_stride = src1_row_size;
-
- mmctx->vtcm_src0_size_per_thread = L.src0_bytes / octx->n_threads;
- mmctx->vtcm_src1_size_per_thread = L.src1_bytes;
- mmctx->vtcm_src2_size_per_thread = L.src2_bytes / octx->n_threads;
- mmctx->vtcm_src3_size_per_thread = L.src3_bytes / octx->n_threads;
- mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads;
-
- mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks;
- mmctx->quant_task_func = quant_task_func;
- mmctx->n_quant_tasks = n_quant_tasks;
- atomic_init(&mmctx->quant_barrier, n_quant_tasks);
-
- // Run fused matmul
- const uint32_t n_matmul_jobs = octx->n_threads;
- worker_callback_t matmul_job_func;
- if (is_repacked) {
- if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {
- switch (src0->type) {
- case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_qkv_2d_repacked_q4_0_flat; break;
- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_qkv_2d_repacked_q4_1_flat; break;
- case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_qkv_2d_repacked_q8_0_flat; break;
- case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_qkv_2d_repacked_iq4nl_flat; break;
- case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_qkv_2d_repacked_mxfp4_flat; break;
- default: return HTP_STATUS_NO_SUPPORT;
- }
- } else {
- switch (src0->type) {
- case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_qkv_2d_repacked_q4_0; break;
- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_qkv_2d_repacked_q4_1; break;
- case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_qkv_2d_repacked_q8_0; break;
- case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_qkv_2d_repacked_iq4nl; break;
- case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_qkv_2d_repacked_mxfp4; break;
- default: return HTP_STATUS_NO_SUPPORT;
- }
- }
- } else {
- matmul_job_func = hvx_mm_qkv_2d;
- }
-
- htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
-
- worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs);
-
- return HTP_STATUS_OK;
-}
-
-int op_matmul_ffn(struct htp_ops_context * octx) {
- struct htp_thread_trace * tr = &octx->ctx->trace[0];
- htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
-
- const struct htp_tensor * restrict src0 = octx->src[0]; // Wgate
- const struct htp_tensor * restrict src1 = octx->src[1]; // y
- const struct htp_tensor * restrict src2 = octx->src[2]; // Wup
- const struct htp_tensor * restrict dst_gate = octx->dsts[0];
- const struct htp_tensor * restrict dst_up = octx->dsts[1];
-
- bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 ||
- src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL ||
- src0->type == HTP_TYPE_MXFP4);
-
- struct htp_mm_context mmctx_struct = {0};
- struct htp_mm_context * mmctx = &mmctx_struct;
- mmctx->octx = octx;
-
- const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
-
- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
- const uint32_t src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3];
-
- // Compute src0_nrows_per_thread
- mmctx->src0_nrows_per_thread = (src0_nrows + octx->n_threads - 1) / octx->n_threads;
- if (is_repacked) {
- mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32);
- } else {
- mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even
- }
-
- const size_t src0_row_size = src0->nb[1];
- const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);
-
- if (hvx_mm_init_vec_dot(mmctx, src0->type) != 0) {
- return HTP_STATUS_NO_SUPPORT;
- }
-
- const uint32_t qk = QK_Q8_0_TILED;
- const uint32_t nb = (src1->ne[0] + qk - 1) / qk;
- const uint32_t total_nb = src1_nrows * nb;
-
- worker_callback_t quant_task_func;
- uint32_t n_quant_tasks = 1;
- if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {
- n_quant_tasks = MIN(src1_nrows, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat;
- } else if (src1_nrows < octx->n_threads) {
- n_quant_tasks = MIN(total_nb, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;
- for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {
- uint32_t ib_first = (total_nb * (ith + 0)) / n_quant_tasks;
- uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks;
- mmctx->quant_ib_first[ith] = ib_first;
- mmctx->quant_ib_last[ith] = ib_last;
- mmctx->quant_r[ith] = ib_first / nb;
- mmctx->quant_c[ith] = ib_first % nb;
- }
- } else {
- n_quant_tasks = MIN(src1_nrows, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;
- }
-
- size_t src1_row_size;
- if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {
- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(src1->ne[0]) : htp_mm_q8_0_flat_row_size(src1->ne[0]);
- } else {
- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(src1->ne[0]) : htp_mm_q8_0_tiled_row_size(src1->ne[0]);
- }
-
- struct htp_mm_hvx_vtcm_layout L;
- htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, src1->ne[0], src1_nrows, octx->n_threads,
- 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, false, false, true);
-
- size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes;
-
- if (octx->ctx->vtcm_size < vtcm_size) {
- FARF(ERROR, "matmul-ffn: current VTCM reservation %zu is too small, needed %zu\n", octx->ctx->vtcm_size, vtcm_size);
- return HTP_STATUS_VTCM_TOO_SMALL;
- }
-
- uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;
mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);
- mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);
- mmctx->vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src2);
mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);
- octx->src1_spad.src = NULL;
octx->src0_spad.src = NULL;
+ octx->src1_spad.src = NULL;
octx->src2_spad.src = NULL;
+ octx->src3_spad.src = NULL;
octx->dst_spad.src = NULL;
mmctx->vtcm_src0_stride = is_repacked ? 0 : src0_row_size_padded;
- mmctx->vtcm_src2_stride = is_repacked ? 0 : src0_row_size_padded;
mmctx->vtcm_src1_stride = src1_row_size;
mmctx->vtcm_src0_size_per_thread = L.src0_bytes / octx->n_threads;
mmctx->vtcm_src1_size_per_thread = L.src1_bytes;
- mmctx->vtcm_src2_size_per_thread = L.src2_bytes / octx->n_threads;
mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads;
mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks;
@@ -3950,25 +3381,25 @@ int op_matmul_ffn(struct htp_ops_context * octx) {
if (is_repacked) {
if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {
switch (src0->type) {
- case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_ffn_2d_repacked_q4_0_flat; break;
- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_ffn_2d_repacked_q4_1_flat; break;
- case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_ffn_2d_repacked_q8_0_flat; break;
- case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_ffn_2d_repacked_iq4nl_flat; break;
- case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_ffn_2d_repacked_mxfp4_flat; break;
+ case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0_flat; break;
+ case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1_flat; break;
+ case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0_flat; break;
+ case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl_flat; break;
+ case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4_flat; break;
default: return HTP_STATUS_NO_SUPPORT;
}
} else {
switch (src0->type) {
- case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_ffn_2d_repacked_q4_0; break;
- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_ffn_2d_repacked_q4_1; break;
- case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_ffn_2d_repacked_q8_0; break;
- case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_ffn_2d_repacked_iq4nl; break;
- case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_ffn_2d_repacked_mxfp4; break;
+ case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0; break;
+ case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1; break;
+ case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0; break;
+ case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl; break;
+ case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4; break;
default: return HTP_STATUS_NO_SUPPORT;
}
}
} else {
- matmul_job_func = hvx_mm_ffn_2d;
+ matmul_job_func = hvx_mm_nx_2d;
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h
index 6c393664c6..dbc8e35909 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.h
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h
@@ -88,6 +88,7 @@ struct htp_mm_kernel_params {
int32_t vtcm_src2_size; // src2 scratchpad size in VTCM (fused only)
int32_t vtcm_src3_size; // src3 scratchpad size in VTCM (fused only)
int32_t vtcm_dst_size; // dst scratchpad size in VTCM
+ int32_t n_weights; // Number of weights for fused NX
// Precomputed division values
struct fastdiv_values div_ne12_ne1;
@@ -463,8 +464,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
size_t src2_row_size,
uint32_t n_prefetch,
bool is_matmul_id,
- bool is_fused_qkv,
- bool is_fused_ffn
+ bool is_fused_nx
) {
size_t src0_sz = 0;
size_t src1_sz = 0;
@@ -476,44 +476,33 @@ static inline void htp_mm_hvx_vtcm_layout_build(
wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL ||
wtype == HTP_TYPE_MXFP4);
- if (is_fused_qkv || is_fused_ffn) {
+ if (is_fused_nx) {
const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);
const size_t quant_scratch_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)) * n_threads;
- size_t src0_sz_per_thread = 0;
- size_t src2_sz_per_thread = 0;
- size_t src3_sz_per_thread = 0;
+ size_t weight_sz_per_thread = 0;
if (is_repack) {
uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype);
uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32;
uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
- src0_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
- src2_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
- if (is_fused_qkv) {
- src3_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
- }
+ weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
} else {
- src0_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128);
- src2_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128);
- if (is_fused_qkv) {
- src3_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128);
- }
+ weight_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128);
}
- size_t flat_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
- size_t tiled_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
+ size_t flat_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
+ size_t tiled_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
- if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {
- src1_sz = hex_round_up(flat_src1_row_size * src1_nrows, 128);
- } else {
- src1_sz = hex_round_up(tiled_src1_row_size * src1_nrows, 128);
- }
+ size_t act_sz = (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT)
+ ? hex_round_up(flat_act_row_size * src1_nrows, 128)
+ : hex_round_up(tiled_act_row_size * src1_nrows, 128);
- src0_sz = src0_sz_per_thread * n_threads;
- src2_sz = src2_sz_per_thread * n_threads;
- src3_sz = src3_sz_per_thread * n_threads;
+ src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch buffer
+ src1_sz = act_sz; // quantized activation buffer
+ src2_sz = 0;
+ src3_sz = 0;
dst_sz = quant_scratch_size;
} else if (is_matmul_id) {
const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128);
@@ -616,8 +605,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(
}
size_t off = 0;
- VTCM_LAYOUT_ALLOC(off, off_src1, src1_sz);
VTCM_LAYOUT_ALLOC(off, off_src0, src0_sz);
+ VTCM_LAYOUT_ALLOC(off, off_src1, src1_sz);
VTCM_LAYOUT_ALLOC(off, off_src2, src2_sz);
VTCM_LAYOUT_ALLOC(off, off_src3, src3_sz);
VTCM_LAYOUT_ALLOC(off, off_dst, dst_sz);
diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.c
index 58c54967db..fa14bf0ef6 100644
--- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c
+++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c
@@ -8,14 +8,20 @@
#include <math.h>
#include <string.h>
-#include "hex-dma.h"
+#include "dma-queue.h"
+#include "work-queue.h"
#include "hvx-utils.h"
+#include "hex-utils.h"
+#include "hvx-copy.h"
+#include "hvx-quant.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
+
#include "htp-ctx.h"
#include "htp-ops.h"
-#include "htp-ops.h"
+#include "htp-tensor.h"
+#include "htp/set-rows-ops.h"
#define set_rows_preamble \
const uint32_t ne00 = octx->src[0]->ne[0]; \
@@ -47,116 +53,142 @@
\
const uint32_t nr = ne01;
-struct htp_set_rows_context {
+struct set_rows_context {
struct htp_ops_context * octx;
- struct fastdiv_values div_ne12;
- struct fastdiv_values div_ne11;
- uint32_t src0_nrows_per_thread;
+ const struct htp_set_rows_kernel_params * kparams;
+ struct htp_set_rows_vtcm_layout vtcm_layout;
+ uint8_t * vtcm_base;
};
-static void set_rows_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) {
- struct htp_set_rows_context * srctx = (struct htp_set_rows_context *)data;
- struct htp_ops_context * octx = srctx->octx;
-
- set_rows_preamble;
-
- uint64_t qt = HAP_perf_get_qtimer_count();
-
- // parallelize by rows of src0
- const uint32_t dr = srctx->src0_nrows_per_thread;
- const uint32_t ir0 = dr * ith;
- if (ir0 >= nr) {
- return;
- }
- const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr;
-
- const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
-
- for (uint32_t i03 = 0; i03 < ne03; ++i03) {
- for (uint32_t i02 = 0; i02 < ne02; ++i02) {
- for (uint32_t i = ir0; i < ir1; ++i) {
- const uint32_t i12 = fastmodulo(i03, ne12, &srctx->div_ne12);
- const uint32_t i11 = fastmodulo(i02, ne11, &srctx->div_ne11);
- const uint32_t i10 = i;
-
- const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
-
- uint32_t i1 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;
- if (i1 >= ne1) {
- // ignore invalid indices
- continue;
- }
-
- const uintptr_t src0_ptr = octx->src[0]->data + i*nb01 + i02*nb02 + i03*nb03;
- const uintptr_t dst_ptr = octx->dst->data + i1*nb1 + i02*nb2 + i03*nb3;
-
- // copy row
- hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, ne00);
- }
- }
- }
-
- qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt);
- FARF(HIGH, "set-rows-f32-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
- ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt);
+#define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \
+static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \
+ struct set_rows_context * srctx = (struct set_rows_context *)data; \
+ struct htp_ops_context * octx = srctx->octx; \
+ const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \
+ set_rows_preamble; \
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
+ const uint32_t dr = kparams->tasks_per_thread; \
+ const uint32_t ir0 = dr * ith; \
+ if (ir0 >= kparams->total_tasks) { \
+ return; \
+ } \
+ const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \
+ dma_queue * dma_queue = octx->ctx->dma[ith]; \
+ const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \
+ uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \
+ uint8_t * vtcm_dst = srctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \
+ const uint32_t src0_row_size = ne00 * sizeof(float); \
+ const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \
+ const uint32_t nrows_per_thread = ir1 - ir0; \
+ const uint32_t total_steps = ne03 * ne02 * nrows_per_thread; \
+ uint32_t pi_step = 0; \
+ uint32_t pi02 = 0; \
+ uint32_t pi03 = 0; \
+ for (uint32_t step = 0, spad_idx = 0; step < total_steps && spad_idx < 2; ++step, spad_idx++) { \
+ uint32_t i = ir0 + pi_step; \
+ const uintptr_t src0_ptr = octx->src[0]->data + i*nb01 + pi02*nb02 + pi03*nb03; \
+ dma_queue_push(dma_queue, \
+ dma_make_ptr((void *)octx->dst->data, \
+ vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \
+ dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \
+ dma_queue_push(dma_queue, \
+ dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \
+ (const void *)src0_ptr), \
+ vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \
+ pi_step++; \
+ if (pi_step == nrows_per_thread) { \
+ pi_step = 0; \
+ pi02++; \
+ if (pi02 == ne02) { \
+ pi02 = 0; \
+ pi03++; \
+ } \
+ } \
+ } \
+ uint32_t ci_step = 0; \
+ uint32_t ci02 = 0; \
+ uint32_t ci03 = 0; \
+ uint32_t ci11_base = 0; \
+ uint32_t ci12_base = 0; \
+ for (uint32_t step = 0; step < total_steps; ++step) { \
+ void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \
+ void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \
+ uint32_t i = ir0 + ci_step; \
+ const uintptr_t src1_addr = octx->src[1]->data + i*nb10 + ci11_base*nb11 + ci12_base*nb12; \
+ const IDX_TYPE i1 = *(const IDX_TYPE *)src1_addr; \
+ const bool valid_i1 = ((uint64_t)i1 < (uint64_t)ne1); \
+ const uint32_t target_i1 = (uint32_t)i1; \
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, step); \
+ if (valid_i1) { \
+ COMPUTE_EXPR; \
+ } \
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, step); \
+ if (valid_i1) { \
+ const uintptr_t dst_ptr = octx->dst->data + target_i1*nb1 + ci02*nb2 + ci03*nb3; \
+ dma_queue_push(dma_queue, \
+ dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \
+ dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 1); \
+ } else { \
+ dma_queue_push(dma_queue, \
+ dma_make_ptr((void *)octx->dst->data, (const void *)dst_spad), \
+ dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \
+ } \
+ const uint32_t next_step = step + 2; \
+ if (next_step < total_steps) { \
+ uint32_t ni = ir0 + pi_step; \
+ const uintptr_t psrc0_ptr = octx->src[0]->data + ni*nb01 + pi02*nb02 + pi03*nb03; \
+ dma_queue_push(dma_queue, \
+ dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \
+ vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \
+ pi_step++; \
+ if (pi_step == nrows_per_thread) { \
+ pi_step = 0; \
+ pi02++; \
+ if (pi02 == ne02) { \
+ pi02 = 0; \
+ pi03++; \
+ } \
+ } \
+ } \
+ ci_step++; \
+ if (ci_step == nrows_per_thread) { \
+ ci_step = 0; \
+ ci02++; \
+ ci11_base++; \
+ if (ci11_base == ne11) { \
+ ci11_base = 0; \
+ } \
+ if (ci02 == ne02) { \
+ ci02 = 0; \
+ ci03++; \
+ ci12_base++; \
+ if (ci12_base == ne12) { \
+ ci12_base = 0; \
+ } \
+ } \
+ } \
+ } \
+ dma_queue_flush(dma_queue); \
}
-static void set_rows_thread_f16_f32(unsigned int nth, unsigned int ith, void *data) {
- struct htp_set_rows_context * srctx = (struct htp_set_rows_context *)data;
- struct htp_ops_context * octx = srctx->octx;
-
- set_rows_preamble;
+SET_ROWS_THREAD_DMA_FN(f32, int32_t, { hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
+SET_ROWS_THREAD_DMA_FN(f32, int64_t, { hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
- uint64_t qt = HAP_perf_get_qtimer_count();
+SET_ROWS_THREAD_DMA_FN(f16, int32_t, { hvx_copy_f16_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
+SET_ROWS_THREAD_DMA_FN(f16, int64_t, { hvx_copy_f16_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
- // parallelize by rows of src0
- const uint32_t dr = srctx->src0_nrows_per_thread;
- const uint32_t ir0 = dr * ith;
- if (ir0 >= nr) {
- return;
- }
- const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr;
-
- const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
-
- for (uint32_t i03 = 0; i03 < ne03; ++i03) {
- for (uint32_t i02 = 0; i02 < ne02; ++i02) {
- for (uint32_t i = ir0; i < ir1; ++i) {
- const uint32_t i12 = fastmodulo(i03, ne12, &srctx->div_ne12);
- const uint32_t i11 = fastmodulo(i02, ne11, &srctx->div_ne11);
- const uint32_t i10 = i;
-
- const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
-
- uint32_t i1 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;
- if (i1 >= ne1) {
- // ignore invalid indices
- continue;
- }
-
- const uint8_t* src0_ptr = (const uint8_t *) octx->src[0]->data + i*nb01 + i02*nb02 + i03*nb03;
- uint8_t* dst_ptr = (uint8_t *) octx->dst->data + i1*nb1 + i02*nb2 + i03*nb3;
-
- hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00);
- }
- }
- }
-
- qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt);
- FARF(HIGH, "set-rows-f16-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
- ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt);
-}
+SET_ROWS_THREAD_DMA_FN(q8_0, int32_t, { hvx_quantize_row_q8_0_f32(dst_spad, (const float *)src_spad, ne00); })
+SET_ROWS_THREAD_DMA_FN(q8_0, int64_t, { hvx_quantize_row_q8_0_f32(dst_spad, (const float *)src_spad, ne00); })
int op_set_rows(struct htp_ops_context * octx) {
+ const struct htp_set_rows_kernel_params * kparams = (const struct htp_set_rows_kernel_params *)octx->kernel_params;
set_rows_preamble;
- const uint32_t n_threads = MIN(nr, octx->n_threads);
-
if (octx->src[0]->type != HTP_TYPE_F32) {
return HTP_STATUS_NO_SUPPORT;
}
- if (octx->dst->type != HTP_TYPE_F32 && octx->dst->type != HTP_TYPE_F16) {
+ if (octx->dst->type != HTP_TYPE_F32 && octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_Q8_0) {
return HTP_STATUS_NO_SUPPORT;
}
@@ -164,27 +196,27 @@ int op_set_rows(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
- if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
- return HTP_STATUS_OK;
- }
+ // l2fetch the src1 (indices) tensor in the main thread
+ hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]);
- struct htp_set_rows_context srctx;
+ struct set_rows_context srctx;
srctx.octx = octx;
- srctx.div_ne12 = init_fastdiv_values(ne12);
- srctx.div_ne11 = init_fastdiv_values(ne11);
-
- srctx.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads;
-
- switch(octx->dst->type) {
- case HTP_TYPE_F32:
- worker_pool_run_func(octx->ctx->worker_pool, set_rows_thread_f32_f32, &srctx, n_threads);
- break;
- case HTP_TYPE_F16:
- worker_pool_run_func(octx->ctx->worker_pool, set_rows_thread_f16_f32, &srctx, n_threads);
- break;
- default:
- return HTP_STATUS_NO_SUPPORT;
+ srctx.kparams = kparams;
+
+ htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, kparams->n_threads);
+ srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
+
+ work_queue_func_t q_func = NULL;
+ const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
+
+ switch (octx->dst->type) {
+ case HTP_TYPE_F32: q_func = is_i32 ? set_rows_thread_dma_f32_int32_t : set_rows_thread_dma_f32_int64_t; break;
+ case HTP_TYPE_F16: q_func = is_i32 ? set_rows_thread_dma_f16_int32_t : set_rows_thread_dma_f16_int64_t; break;
+ case HTP_TYPE_Q8_0: q_func = is_i32 ? set_rows_thread_dma_q8_0_int32_t : set_rows_thread_dma_q8_0_int64_t; break;
+ default: return HTP_STATUS_NO_SUPPORT;
}
+ work_queue_run(octx->ctx->work_queue, q_func, &srctx, kparams->n_threads);
+
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.h b/ggml/src/ggml-hexagon/htp/set-rows-ops.h
new file mode 100644
index 0000000000..5e98d2cb55
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.h
@@ -0,0 +1,74 @@
+#ifndef HTP_SET_ROWS_OPS_H
+#define HTP_SET_ROWS_OPS_H
+
+#include "hex-fastdiv.h"
+
+struct htp_set_rows_kernel_params {
+ int32_t n_threads;
+ int32_t total_tasks;
+ int32_t tasks_per_thread;
+ int32_t vtcm_size;
+
+ // Fastdiv helpers
+ struct fastdiv_values div_ne11;
+ struct fastdiv_values div_ne12;
+ struct fastdiv_values div_tasks_per_thread;
+ struct fastdiv_values div_ne02;
+};
+
+struct htp_set_rows_vtcm_layout {
+ size_t total_bytes;
+ size_t off_src0;
+ size_t off_dst;
+
+ size_t src0_bytes_per_thread;
+ size_t dst_bytes_per_thread;
+
+ size_t src0_spad_half_size;
+ size_t dst_spad_half_size;
+};
+
+static inline void htp_set_rows_vtcm_layout_build(
+ struct htp_set_rows_vtcm_layout * vtcm_layout,
+ int dst_type,
+ uint32_t ne00,
+ uint32_t n_threads) {
+
+ size_t src0_row_size = ne00 * 4;
+ size_t dst_row_size = 0;
+ switch (dst_type) {
+ case 0: // HTP_TYPE_F32
+ dst_row_size = ne00 * 4;
+ break;
+ case 1: // HTP_TYPE_F16
+ dst_row_size = ne00 * 2;
+ break;
+ case 8: // HTP_TYPE_Q8_0
+ dst_row_size = (ne00 / 32) * 34;
+ break;
+ default:
+ dst_row_size = 0;
+ break;
+ }
+
+ size_t src0_row_size_aligned = (src0_row_size + 255) & ~255;
+ size_t dst_row_size_aligned = (dst_row_size + 255) & ~255;
+
+ vtcm_layout->src0_spad_half_size = src0_row_size_aligned;
+ vtcm_layout->dst_spad_half_size = dst_row_size_aligned;
+
+ vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2;
+ vtcm_layout->dst_bytes_per_thread = dst_row_size_aligned * 2;
+
+ vtcm_layout->off_src0 = 0;
+ vtcm_layout->off_dst = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads;
+ vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;
+}
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_set_rows_kernel_params) <= 128, "htp_set_rows_kernel_params is too large for kernel_params blob");
+#else
+_Static_assert(sizeof(struct htp_set_rows_kernel_params) <= 128, "htp_set_rows_kernel_params is too large for kernel_params blob");
+#endif
+
+#endif // HTP_SET_ROWS_OPS_H
diff --git a/scripts/snapdragon/adb/run-bench.sh b/scripts/snapdragon/adb/run-bench.sh
deleted file mode 100755
index eaae80a77d..0000000000
--- a/scripts/snapdragon/adb/run-bench.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/sh
-#
-
-# Basedir on device
-basedir=/data/local/tmp/llama.cpp
-
-branch=.
-[ "$B" != "" ] && branch=$B
-
-adbserial=
-[ "$S" != "" ] && adbserial="-s $S"
-
-adbhost=
-[ "$H" != "" ] && adbhost="-H $H"
-
-model="Llama-3.2-3B-Instruct-Q4_0.gguf"
-[ "$M" != "" ] && model="$M"
-
-device="HTP0"
-[ "$D" != "" ] && device="$D"
-
-verbose=
-[ "$V" != "" ] && verbose="GGML_HEXAGON_VERBOSE=$V" cli_opts="$cli_opts -v"
-
-profile=
-[ "$PROF" != "" ] && profile="GGML_HEXAGON_PROFILE=$PROF" cli_opts="$cli_opts -v"
-
-opmask=
-[ "$OPSTAGE" != "" ] && opmask="GGML_HEXAGON_OPSTAGE=$OPSTAGE"
-
-nhvx=
-[ "$NHVX" != "" ] && nhvx="GGML_HEXAGON_NHVX=$NHVX"
-
-ndev=
-[ "$NDEV" != "" ] && ndev="GGML_HEXAGON_NDEV=$NDEV"
-
-hb=
-[ "$HB" != "" ] && hb="GGML_HEXAGON_HOSTBUF=$HB"
-
-set -x
-
-adb $adbserial $adbhost shell " \
- cd $basedir; \
- LD_LIBRARY_PATH=$basedir/$branch/lib \
- ADSP_LIBRARY_PATH=$basedir/$branch/lib \
- $ndev $nhvx $opmask $verbose $profile $hb ./$branch/bin/llama-bench --device $device --load-mode none -m $basedir/../gguf/$model \
- --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \
- --ubatch-size 1024 -fa 1 -ngl 99 $cli_opts $@ \
-"
diff --git a/scripts/snapdragon/adb/run-cli.sh b/scripts/snapdragon/adb/run-cli.sh
deleted file mode 100755
index 27a4a14195..0000000000
--- a/scripts/snapdragon/adb/run-cli.sh
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/bin/sh
-#
-
-# Basedir on device
-basedir=/data/local/tmp/llama.cpp
-
-cli_opts=
-
-branch=.
-[ "$B" != "" ] && branch=$B
-
-adbserial=
-[ "$S" != "" ] && adbserial="-s $S"
-
-adbhost=
-[ "$H" != "" ] && adbhost="-H $H"
-
-model="Llama-3.2-3B-Instruct-Q4_0.gguf"
-[ "$M" != "" ] && model="$M"
-
-device="HTP0"
-[ "$D" != "" ] && device="$D"
-
-verbose=
-[ "$V" != "" ] && verbose="GGML_HEXAGON_VERBOSE=$V" cli_opts="$cli_opts -v"
-
-sched=
-[ "$SCHED" != "" ] && sched="GGML_SCHED_DEBUG=2" cli_opts="$cli_opts -v"
-
-profile=
-[ "$PROF" != "" ] && profile="GGML_HEXAGON_PROFILE=$PROF" cli_opts="$cli_opts -v"
-
-opmask=
-[ "$OPSTAGE" != "" ] && opmask="GGML_HEXAGON_OPSTAGE=$OPSTAGE"
-
-nhvx=
-[ "$NHVX" != "" ] && nhvx="GGML_HEXAGON_NHVX=$NHVX"
-
-hmx=
-[ "$HMX" != "" ] && hmx="GGML_HEXAGON_USE_HMX=$HMX"
-
-ndev=
-[ "$NDEV" != "" ] && ndev="GGML_HEXAGON_NDEV=$NDEV"
-
-hb=
-[ "$HB" != "" ] && hb="GGML_HEXAGON_HOSTBUF=$HB"
-
-opbatch=
-[ "$OB" != "" ] && opbatch="GGML_HEXAGON_OPBATCH=$OB"
-
-opqueue=
-[ "$OQ" != "" ] && opqueue="GGML_HEXAGON_OPQUEUE=$OQ"
-
-opflt=
-[ "$OF" != "" ] && opflt="GGML_HEXAGON_OPFILTER=$OF"
-
-vmem=
-[ "$VM" != "" ] && opflt="GGML_HEXAGON_VMEM=$VM"
-
-mbuf=
-[ "$MB" != "" ] && opflt="GGML_HEXAGON_MBUF=$MB"
-vmem=
-[ "$VM" != "" ] && vmem="GGML_HEXAGON_VMEM=$VM"
-
-mbuf=
-[ "$MB" != "" ] && mbuf="GGML_HEXAGON_MBUF=$MB"
-set -x
-
-adb $adbserial $adbhost shell " \
- cd $basedir; ulimit -c unlimited; \
- LD_LIBRARY_PATH=$basedir/$branch/lib \
- ADSP_LIBRARY_PATH=$basedir/$branch/lib \
- $verbose $sched $opmask $profile $nhvx $hmx $ndev $hb $opbatch $opqueue $opflt $vmem $mbuf \
- ./$branch/bin/llama-cli --load-mode none -m $basedir/../gguf/$model \
- --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \
- --ctx-size 8192 --ubatch-size 1024 -fa on \
- -ngl 99 --device $device $cli_opts $@ \
-"
diff --git a/scripts/snapdragon/adb/run-completion.sh b/scripts/snapdragon/adb/run-completion.sh
deleted file mode 100755
index 30893ed293..0000000000
--- a/scripts/snapdragon/adb/run-completion.sh
+++ /dev/null
@@ -1,86 +0,0 @@
-#!/bin/sh
-#
-
-# Basedir on device
-basedir=/data/local/tmp/llama.cpp
-
-cli_opts=
-
-branch=.
-[ "$B" != "" ] && branch=$B
-
-adbserial=
-[ "$S" != "" ] && adbserial="-s $S"
-
-adbhost=
-[ "$H" != "" ] && adbhost="-H $H"
-
-model="Llama-3.2-3B-Instruct-Q4_0.gguf"
-[ "$M" != "" ] && model="$M"
-
-device="HTP0"
-[ "$D" != "" ] && device="$D"
-
-verbose=
-[ "$V" != "" ] && verbose="GGML_HEXAGON_VERBOSE=$V" cli_opts="$cli_opts -v"
-
-sched=
-[ "$SCHED" != "" ] && sched="GGML_SCHED_DEBUG=2" cli_opts="$cli_opts -v"
-
-profile=
-[ "$PROF" != "" ] && profile="GGML_HEXAGON_PROFILE=$PROF" cli_opts="$cli_opts -v"
-
-opmask=
-[ "$OPSTAGE" != "" ] && opmask="GGML_HEXAGON_OPSTAGE=$OPSTAGE"
-
-nhvx=
-[ "$NHVX" != "" ] && nhvx="GGML_HEXAGON_NHVX=$NHVX"
-
-hmx=
-[ "$HMX" != "" ] && hmx="GGML_HEXAGON_USE_HMX=$HMX"
-
-ndev=
-[ "$NDEV" != "" ] && ndev="GGML_HEXAGON_NDEV=$NDEV"
-
-hb=
-[ "$HB" != "" ] && hb="GGML_HEXAGON_HOSTBUF=$HB"
-
-opbatch=
-[ "$OB" != "" ] && opbatch="GGML_HEXAGON_OPBATCH=$OB"
-
-opqueue=
-[ "$OQ" != "" ] && opqueue="GGML_HEXAGON_OPQUEUE=$OQ"
-
-oppoll=
-[ "$OP" != "" ] && oppoll="GGML_HEXAGON_OPPOLL=$OP"
-
-opflt=
-[ "$OF" != "" ] && opflt="GGML_HEXAGON_OPFILTER=$OF"
-
-opfuse=
-[ "$OC" != "" ] && opfuse="GGML_HEXAGON_OPFUSION=$OC"
-
-vmem=
-[ "$VM" != "" ] && vmem="GGML_HEXAGON_VMEM=$VM"
-
-mbuf=
-[ "$MB" != "" ] && mbuf="GGML_HEXAGON_MBUF=$MB"
-
-mmsel=
-[ "$MM" != "" ] && mmsel="GGML_HEXAGON_MM_SELECT=$MM"
-
-fasel=
-[ "$FA" != "" ] && fasel="GGML_HEXAGON_FA_SELECT=$FA"
-
-set -x
-
-adb $adbserial $adbhost shell " \
- cd $basedir; ulimit -c unlimited; \
- LD_LIBRARY_PATH=$basedir/$branch/lib \
- ADSP_LIBRARY_PATH=$basedir/$branch/lib \
- $verbose $sched $opmask $profile $nhvx $hmx $ndev $hb $opbatch $opqueue $oppoll $opflt $opfuse $vmem $mbuf $mmsel $fasel \
- ./$branch/bin/llama-completion --load-mode none -m $basedir/../gguf/$model \
- --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \
- --ctx-size 8192 --ubatch-size 1024 -fa on \
- -ngl 99 --device $device $cli_opts $@ \
-"
diff --git a/scripts/snapdragon/adb/run-mtmd.sh b/scripts/snapdragon/adb/run-mtmd.sh
deleted file mode 100755
index 65dd6ec59e..0000000000
--- a/scripts/snapdragon/adb/run-mtmd.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/sh
-#
-
-# Basedir on device
-basedir=/data/local/tmp/llama.cpp
-
-cli_opts=
-
-branch=.
-[ "$B" != "" ] && branch=$B
-
-adbserial=
-[ "$S" != "" ] && adbserial="-s $S"
-
-adbhost=
-[ "$H" != "" ] && adbhost="-H $H"
-
-model="gemma-3-4b-it-Q4_0.gguf"
-[ "$M" != "" ] && model="$M"
-
-mmproj="mmproj-F16.gguf"
-[ "$MMPROJ" != "" ] && mmproj="$MMPROJ"
-
-image=
-[ "$IMG" != "" ] && image="$IMG"
-
-device="HTP0"
-[ "$D" != "" ] && device="$D"
-
-verbose=
-[ "$V" != "" ] && verbose="GGML_HEXAGON_VERBOSE=$V"
-
-experimental="GGML_HEXAGON_EXPERIMENTAL=1"
-[ "$E" != "" ] && experimental="GGML_HEXAGON_EXPERIMENTAL=$E"
-
-sched=
-[ "$SCHED" != "" ] && sched="GGML_SCHED_DEBUG=2" cli_opts="$cli_opts -v"
-
-profile=
-[ "$PROF" != "" ] && profile="GGML_HEXAGON_PROFILE=$PROF"
-
-opmask=
-[ "$OPSTAGE" != "" ] && opmask="GGML_HEXAGON_OPSTAGE=$OPSTAGE"
-
-nhvx=
-[ "$NHVX" != "" ] && nhvx="GGML_HEXAGON_NHVX=$NHVX"
-
-hmx=
-[ "$HMX" != "" ] && hmx="GGML_HEXAGON_USE_HMX=$HMX"
-
-ndev=
-[ "$NDEV" != "" ] && ndev="GGML_HEXAGON_NDEV=$NDEV"
-
-# MTMD backend device for vision model (defaults to CPU if not set)
-mtmd_backend=
-[ "$MTMD_DEVICE" != "" ] && mtmd_backend="MTMD_BACKEND_DEVICE=$MTMD_DEVICE"
-
-set -x
-
-adb $adbserial $adbhost shell " \
- cd $basedir; ulimit -c unlimited; \
- LD_LIBRARY_PATH=$basedir/$branch/lib \
- ADSP_LIBRARY_PATH=$basedir/$branch/lib \
- $verbose $experimental $sched $opmask $profile $hmx $nhvx $ndev $mtmd_backend \
- ./$branch/bin/llama-mtmd-cli --load-mode none -m $basedir/../gguf/$model \
- --mmproj $basedir/../gguf/$mmproj \
- --image $basedir/../gguf/$image \
- --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \
- --ctx-size 8192 --ubatch-size 1024 -fa on \
- -ngl 99 --device $device -v $cli_opts $@ \
-"
diff --git a/scripts/snapdragon/adb/run-tool.sh b/scripts/snapdragon/adb/run-tool.sh
deleted file mode 100755
index 8bf7ba3a54..0000000000
--- a/scripts/snapdragon/adb/run-tool.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/sh
-#
-
-# Basedir on device
-basedir=/data/local/tmp/llama.cpp
-
-cli_opts=
-
-branch=.
-[ "$B" != "" ] && branch=$B
-
-adbserial=
-[ "$S" != "" ] && adbserial="-s $S"
-
-adbhost=
-[ "$H" != "" ] && adbhost="-H $H"
-
-device="HTP0"
-[ "$D" != "" ] && device="$D"
-
-verbose=
-[ "$V" != "" ] && verbose="GGML_HEXAGON_VERBOSE=$V"
-
-sched=
-[ "$SCHED" != "" ] && sched="GGML_SCHED_DEBUG=2" cli_opts="$cli_opts -v"
-
-profile=
-[ "$PROF" != "" ] && profile="GGML_HEXAGON_PROFILE=$PROF"
-
-opmask=
-[ "$OPSTAGE" != "" ] && opmask="GGML_HEXAGON_OPSTAGE=$OPSTAGE"
-
-nhvx=
-[ "$NHVX" != "" ] && nhvx="GGML_HEXAGON_NHVX=$NHVX"
-
-hmx=
-[ "$HMX" != "" ] && hmx="GGML_HEXAGON_USE_HMX=$HMX"
-
-ndev=
-[ "$NDEV" != "" ] && ndev="GGML_HEXAGON_NDEV=$NDEV"
-
-hb=
-[ "$HB" != "" ] && hb="GGML_HEXAGON_HOSTBUF=$HB"
-
-opbatch=
-[ "$OB" != "" ] && opbatch="GGML_HEXAGON_OPBATCH=$OB"
-
-opqueue=
-[ "$OQ" != "" ] && opqueue="GGML_HEXAGON_OPQUEUE=$OQ"
-
-oppoll=
-[ "$OP" != "" ] && oppoll="GGML_HEXAGON_OPPOLL=$OP"
-
-opfuse=
-[ "$OC" != "" ] && opfuse="GGML_HEXAGON_OPFUSION=$OC"
-
-mmsel=
-[ "$MM" != "" ] && mmsel="GGML_HEXAGON_MM_SELECT=$MM"
-
-fasel=
-[ "$FA" != "" ] && fasel="GGML_HEXAGON_FA_SELECT=$FA"
-
-set -x
-
-tool=$1; shift
-
-adb $adbserial $adbhost shell " \
- cd $basedir; ulimit -c unlimited; \
- LD_LIBRARY_PATH=$basedir/$branch/lib \
- ADSP_LIBRARY_PATH=$basedir/$branch/lib \
- $verbose $sched $opmask $profile $nhvx $hmx $ndev $hb $opbatch $opqueue $oppoll $opfuse $mmsel $fasel ./$branch/bin/$tool $@ \
-"
diff --git a/scripts/snapdragon/build.py b/scripts/snapdragon/build.py
new file mode 100755
index 0000000000..02dcb930cb
--- /dev/null
+++ b/scripts/snapdragon/build.py
@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+#
+# Build llama.cpp for Snapdragon (via Docker or natively) and push to device.
+#
+
+import sys
+import os
+import argparse
+import subprocess
+import platform
+import shutil
+import logging
+
+logger = logging.getLogger("build")
+
+
+def parse_target(target_str):
+ if not target_str:
+ return None, None
+ if target_str.startswith("adb") or target_str.startswith("android"):
+ parts = target_str.split(":", 1)
+ serial = parts[1] if len(parts) > 1 else None
+ return "android", serial
+ elif target_str.startswith("lnx") or target_str.startswith("linux") or target_str.startswith("ubuntu"):
+ parts = target_str.split(":", 1)
+ host = parts[1] if len(parts) > 1 else None
+ return "linux", host
+ elif target_str in ("wos", "windows"):
+ return "windows", None
+ else:
+ return None, None
+
+
+def get_uid_gid():
+ if platform.system() != "Windows":
+ return [f"{os.getuid()}:{os.getgid()}"]
+ return []
+
+
+def main():
+ logging.basicConfig(level=logging.INFO, format='%(message)s')
+ parser = argparse.ArgumentParser(
+ description="Build llama.cpp for Snapdragon using cross-compilation docker containers or natively."
+ )
+ parser.add_argument("--target", default="android", help="Compilation target and deployment definition (e.g. android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, windows/wos) (default: android)")
+ parser.add_argument("--build-dir", help="Build directory name (defaults to build-TARGET[-dbg], e.g. build-android)")
+ parser.add_argument("--install-dir", help="Install directory name (defaults to pkg-TARGET[-dbg], e.g. pkg-android)")
+ parser.add_argument("--jobs", "-j", type=int, help="Number of build jobs (defaults to CPU thread count)")
+ parser.add_argument("--no-docker", action="store_true", help="Build natively on the host instead of in a docker container")
+ parser.add_argument("--preset", help="Override the CMake preset to use")
+ parser.add_argument("--debug", action="store_true", help="Build in debug mode (uses -debug presets instead of -release)")
+
+ # Push options
+ parser.add_argument("--push", action="store_true", help="Push built package to the target device via ADB or SSH/SCP")
+ parser.add_argument("--target-dir", help="Target directory on the device (default: /data/local/tmp/llama.cpp for Android, ~/llama.cpp for Linux)")
+
+ # Toolchain options
+ parser.add_argument("--toolchain-version", default="v0.7", help="Docker toolchain image version/tag (default: v0.7)")
+ parser.add_argument("--toolchain-url", default="ghcr.io/snapdragon-toolchain", help="Docker toolchain registry URL/namespace (default: ghcr.io/snapdragon-toolchain)")
+
+ args = parser.parse_args()
+
+ target_type, target_val = parse_target(args.target)
+ if not target_type:
+ logger.error(f"Error: Invalid target format '{args.target}'. Must be android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, or windows/wos.")
+ sys.exit(1)
+
+ # Determine preset and check if it's debug
+ preset = args.preset
+ if preset:
+ is_debug = args.debug or ("debug" in preset.lower())
+ else:
+ is_debug = args.debug
+ config_type = "debug" if is_debug else "release"
+ if args.no_docker:
+ if target_type == "windows" or platform.system() == "Windows":
+ preset = f"arm64-windows-snapdragon-{config_type}"
+ elif target_type == "linux":
+ preset = f"arm64-linux-snapdragon-{config_type}"
+ else:
+ preset = f"arm64-android-snapdragon-{config_type}"
+ else:
+ preset = f"arm64-linux-snapdragon-{config_type}" if target_type == "linux" else f"arm64-android-snapdragon-{config_type}"
+
+ target_prefix = args.target.split(":", 1)[0]
+ suffix = "-dbg" if is_debug else ""
+
+ build_dir = args.build_dir
+ if not build_dir:
+ build_dir = f"build-{target_prefix}{suffix}"
+
+ install_dir = args.install_dir
+ if not install_dir:
+ install_dir = f"pkg-{target_prefix}{suffix}"
+
+ repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+
+ # Ensure CMakeUserPresets.json is in the workspace root, update if docs version is newer
+ preset_src = os.path.join(repo_root, "docs", "backend", "snapdragon", "CMakeUserPresets.json")
+ preset_dst = os.path.join(repo_root, "CMakeUserPresets.json")
+ if os.path.exists(preset_src):
+ should_copy = False
+ if not os.path.exists(preset_dst):
+ should_copy = True
+ else:
+ # Check modification times
+ src_mtime = os.path.getmtime(preset_src)
+ dst_mtime = os.path.getmtime(preset_dst)
+ if src_mtime > dst_mtime:
+ preset_bak = preset_dst + ".bak"
+ logger.info(f"Docs CMakeUserPresets.json is newer. Backing up existing {preset_dst} to {preset_bak}")
+ shutil.copy2(preset_dst, preset_bak)
+ should_copy = True
+
+ if should_copy:
+ logger.info(f"Copying CMakeUserPresets.json from {preset_src} to {preset_dst}")
+ shutil.copy2(preset_src, preset_dst)
+ else:
+ logger.warning("Warning: CMakeUserPresets.json not found in docs/backend/snapdragon/.")
+
+ jobs = args.jobs if args.jobs else os.cpu_count() or 4
+
+ if target_type == "windows":
+ logger.info("Windows target selected. Forcing native compilation...")
+ args.no_docker = True
+ if platform.system() != "Windows":
+ logger.warning("Warning: Windows compilation is intended to run on Windows arm64 hosts.")
+
+ if args.no_docker:
+ # Native/local host build
+ logger.info("Running native/local CMake build...")
+ install_prefix = os.path.join(repo_root, install_dir, "llama.cpp")
+
+ # Configure
+ configure_cmd = ["cmake", f"--preset={preset}", "-B", build_dir]
+ logger.info(f"+ {' '.join(configure_cmd)}")
+ res = subprocess.run(configure_cmd, cwd=repo_root)
+ if res.returncode != 0:
+ logger.error("CMake configuration failed.")
+ sys.exit(res.returncode)
+
+ # Build
+ build_cmd = ["cmake", "--build", build_dir, "-j", str(jobs)]
+ logger.info(f"+ {' '.join(build_cmd)}")
+ res = subprocess.run(build_cmd, cwd=repo_root)
+ if res.returncode != 0:
+ logger.error("CMake build failed.")
+ sys.exit(res.returncode)
+
+ # Install
+ install_cmd = ["cmake", "--install", build_dir, "--prefix", install_prefix]
+ logger.info(f"+ {' '.join(install_cmd)}")
+ res = subprocess.run(install_cmd, cwd=repo_root)
+ if res.returncode != 0:
+ logger.error("CMake install failed.")
+ sys.exit(res.returncode)
+ else:
+ # Docker-based build
+ logger.info("Running Docker-based cross-compilation build...")
+ image_name = "arm64-linux" if target_type == "linux" else "arm64-android"
+ image = f"{args.toolchain_url}/{image_name}:{args.toolchain_version}"
+
+ install_prefix_container = f"/workspace/{install_dir}/llama.cpp"
+
+ build_sh_cmd = (
+ f"cmake --preset {preset} -B /workspace/{build_dir} && "
+ f"cmake --build /workspace/{build_dir} -j {jobs} && "
+ f"cmake --install /workspace/{build_dir} --prefix {install_prefix_container}"
+ )
+
+ docker_cmd = [
+ "docker", "run", "--rm",
+ "--volume", f"{repo_root}:/workspace",
+ "--workdir", "/workspace",
+ "--platform", "linux/amd64"
+ ]
+ uid_gid = get_uid_gid()
+ if uid_gid:
+ docker_cmd += ["-u", uid_gid[0]]
+
+ docker_cmd += [image, "bash", "-c", build_sh_cmd]
+
+ logger.info(f"+ {' '.join(docker_cmd)}")
+ res = subprocess.run(docker_cmd, cwd=repo_root)
+ if res.returncode != 0:
+ logger.error("Docker-based build failed.")
+ sys.exit(res.returncode)
+
+ logger.info("\nBuild and installation completed successfully!")
+
+ # Push/deploy if requested
+ if args.push:
+ src_path = os.path.join(repo_root, install_dir, "llama.cpp")
+ if not os.path.exists(src_path):
+ logger.error(f"Error: installation directory {src_path} does not exist. Cannot deploy.")
+ sys.exit(1)
+
+ # Resolve target directory on device
+ target_dir = args.target_dir
+ if not target_dir:
+ target_dir = "/data/local/tmp/llama.cpp" if target_type == "android" else "~/llama.cpp"
+ target_dir = target_dir.rstrip("/")
+
+ sub_items = [item for item in os.listdir(src_path) if not item.startswith(".")]
+
+ if target_type == "android":
+ logger.info("\nPushing built artifacts to Android device via ADB...")
+ adb_cmd = ["adb"]
+ if target_val: # serial
+ adb_cmd += ["-s", target_val]
+
+ # Clean stale package files on device
+ if sub_items:
+ clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
+ clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"]
+ logger.info(f"+ {' '.join(clean_cmd)}")
+ subprocess.run(clean_cmd)
+
+ # Android destination directory is target_dir
+ push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir]
+ logger.info(f"+ {' '.join(push_cmd)}")
+ res = subprocess.run(push_cmd)
+ if res.returncode != 0:
+ logger.error("ADB push failed.")
+ sys.exit(res.returncode)
+ logger.info("ADB push completed successfully!")
+
+ elif target_type == "linux":
+ ssh_host = target_val
+ if not ssh_host:
+ logger.error("Error: SSH host not specified in target (e.g. use linux:user@host, lnx:user@host, or ubuntu:user@host). Cannot deploy.")
+ sys.exit(1)
+ logger.info(f"\nDeploying built artifacts to Linux device {ssh_host} via SSH/SCP...")
+
+ # Clean stale package files on remote host
+ if sub_items:
+ clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
+ clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"]
+ logger.info(f"+ {' '.join(clean_cmd)}")
+ subprocess.run(clean_cmd)
+
+ # Deploy to target_dir
+ deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"]
+ logger.info(f"+ {' '.join(deploy_cmd)}")
+ res = subprocess.run(deploy_cmd)
+ if res.returncode != 0:
+ logger.error("SSH/SCP deploy failed.")
+ sys.exit(res.returncode)
+ logger.info("SSH/SCP deploy completed successfully!")
+
+ elif target_type == "windows":
+ logger.info("\nPush for Windows on Snapdragon (windows) target is currently a stub.")
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except KeyboardInterrupt:
+ logger.info("\nInterrupted by user.")
+ sys.exit(130)
diff --git a/scripts/snapdragon/ggml-hexagon-profile.py b/scripts/snapdragon/ggml-hexagon-profile.py
index 97a3acd26c..038d92fb5c 100755
--- a/scripts/snapdragon/ggml-hexagon-profile.py
+++ b/scripts/snapdragon/ggml-hexagon-profile.py
@@ -34,6 +34,26 @@ trace_pattern = re.compile(
r"trace-evt\s+(?P<event>[A-Z_0-9\-]+):\s+thread\s+(?P<thread>\d+)\s+info\s+(?P<info>\d+)\s+(?P<state>start|stop)\s+(?P<cycles>\d+)"
)
+device_pattern = re.compile(r"\b(HTP\d+(?::\d+)?)\s+(?:profile-op|trace-evt)\b")
+
+
+def extract_device(line):
+ m = device_pattern.search(line)
+ if m:
+ return m.group(1)
+ return "HTP0"
+
+
+def device_matches(record_device, target_device):
+ targets = [t.strip() for t in target_device.split(',')]
+ for target in targets:
+ if record_device == target:
+ return True
+ if record_device.startswith(target + ":"):
+ return True
+ return False
+
+
logger = logging.getLogger("ggml-hexagon-profile")
@@ -72,7 +92,7 @@ class CycleUnwrapper:
return raw + self.high_part
-def parse_log(file_path, pmu_index=None):
+def parse_log(file_path, pmu_index=None, limit=None, device_filter=None, op_filter_re=None):
try:
if file_path != "-":
f = open(file_path, 'r', encoding='utf-8', errors='ignore')
@@ -85,13 +105,22 @@ def parse_log(file_path, pmu_index=None):
all_ops: List[Dict[str, Any]] = []
all_traces: List[Dict[str, Any]] = []
current_op: Optional[Dict[str, Any]] = None
+ ops_count_per_device = {}
+ if device_filter is not None:
+ for target in device_filter.split(','):
+ ops_count_per_device[target.strip()] = 0
+ limit_reached = False
- timestamp_pattern = re.compile(r"^(?P<min>\d+)\.(?P<sec>\d+)\.(?P<ms>\d+)\.(?P<us>\d+)\s+[A-Z]\s+")
- unwrapper = None
- trace_unwrapper = None
+ timestamp_pattern = re.compile(r"(?P<min>\d+)\.(?P<sec>\d+)\.(?P<ms>\d+)\.(?P<us>\d+)\s+[A-Z]\s+")
+ unwrappers = {}
+ last_batch_start = {}
+ trace_unwrappers = {}
for line in f:
- ts_match = timestamp_pattern.match(line)
+ if "profile-op" not in line and "trace-evt" not in line:
+ continue
+
+ ts_match = timestamp_pattern.search(line)
abs_usec = 0
if ts_match:
abs_usec = (
@@ -100,8 +129,11 @@ def parse_log(file_path, pmu_index=None):
+ int(ts_match.group('us'))
)
- if "|" in line and "profile-op" in line:
- parts = [p.strip() for p in line.split("|")]
+ device = extract_device(line)
+
+ idx = line.find("profile-op")
+ if idx != -1 and "|" in line[idx:]:
+ parts = [p.strip() for p in line[idx:].split("|")]
prefix = parts[0]
prefix_match = re.search(r"profile-op\s+(?P<op_name>[A-Z_0-9+]+)", prefix)
if not prefix_match:
@@ -145,7 +177,6 @@ def parse_log(file_path, pmu_index=None):
except (ValueError, IndexError):
pmu_val = None
- evt_val = None
evt_val = None
if types.startswith("evt-cnt "):
try:
@@ -158,14 +189,18 @@ def parse_log(file_path, pmu_index=None):
if op_name == "OPBATCH":
if cycles_start_raw:
unwrapped_cycles_start = int(cycles_start_raw)
- unwrapper = CycleUnwrapper(unwrapped_cycles_start)
- trace_unwrapper = CycleUnwrapper(unwrapped_cycles_start)
+ unwrappers[device] = CycleUnwrapper(unwrapped_cycles_start)
+ last_batch_start[device] = unwrapped_cycles_start
+ for k in list(trace_unwrappers.keys()):
+ if k[0] == device:
+ del trace_unwrappers[k]
else:
- if cycles_start_raw and unwrapper is not None:
- unwrapped_cycles_start = unwrapper.unwrap(int(cycles_start_raw))
+ if cycles_start_raw:
+ device_unwrapper = unwrappers.get(device)
+ if device_unwrapper is not None:
+ unwrapped_cycles_start = device_unwrapper.unwrap(int(cycles_start_raw))
- idx = line.find("profile-op ")
- op_text = line[idx + 11:].strip() if idx != -1 else line.strip()
+ op_text = re.sub(r"^profile-op\s+", "", line[idx:]).strip() if idx != -1 else line.strip()
current_op = {
'name': op_name,
@@ -180,24 +215,58 @@ def parse_log(file_path, pmu_index=None):
'pmu_val': pmu_val,
'evt_val': evt_val,
'abs_usec': abs_usec,
- 'trace_events': []
+ 'trace_events': [],
+ 'device': device
}
all_ops.append(current_op)
+
+ # Check if matching early exit criteria
+ matched = False
+ matched_target = None
+ if device_filter is not None:
+ targets = [t.strip() for t in device_filter.split(',')]
+ for target in targets:
+ if device == target or device.startswith(target + ":"):
+ matched = True
+ matched_target = target
+ break
+ else:
+ matched = True
+ matched_target = device
+
+ if op_filter_re is not None and not op_filter_re.search(op_text):
+ matched = False
+
+ if matched:
+ if matched_target not in ops_count_per_device:
+ ops_count_per_device[matched_target] = 0
+ ops_count_per_device[matched_target] += 1
+
+ if limit is not None and len(ops_count_per_device) > 0 and all(count >= limit for count in ops_count_per_device.values()):
+ limit_reached = True
+
+ if limit_reached and op_name == "OPBATCH":
+ break
continue
trace_match = trace_pattern.search(line)
if trace_match:
+ thread = int(trace_match.group('thread'))
raw_cyc = int(trace_match.group('cycles'))
unwrapped_cyc = None
- if trace_unwrapper is not None:
- unwrapped_cyc = trace_unwrapper.unwrap(raw_cyc)
+ th_key = (device, thread)
+ if th_key not in trace_unwrappers:
+ batch_start = last_batch_start.get(device)
+ trace_unwrappers[th_key] = CycleUnwrapper(batch_start)
+ unwrapped_cyc = trace_unwrappers[th_key].unwrap(raw_cyc)
all_traces.append({
- 'thread': int(trace_match.group('thread')),
+ 'thread': thread,
'event': trace_match.group('event'),
'info': int(trace_match.group('info')),
'cycles': raw_cyc,
'unwrapped_cycles': unwrapped_cyc,
- 'state': trace_match.group('state')
+ 'state': trace_match.group('state'),
+ 'device': device
})
f.close()
@@ -207,39 +276,45 @@ def parse_log(file_path, pmu_index=None):
op['start_cycles'] = op['unwrapped_cycles_start']
op['end_cycles'] = op['start_cycles'] + op['cycles'] if op['start_cycles'] is not None else None
- # Filter ops with valid start_cycles
- valid_ops = [op for op in all_ops if op['start_cycles'] is not None and op['end_cycles'] is not None]
+ # Group ops by device
+ valid_ops_by_dev = defaultdict(list)
+ for op in all_ops:
+ if op['start_cycles'] is not None and op['end_cycles'] is not None:
+ valid_ops_by_dev[op['device']].append(op)
+
+ # Group trace events by device
+ traces_by_dev = defaultdict(list)
+ for e in all_traces:
+ if e['unwrapped_cycles'] is not None:
+ traces_by_dev[e['device']].append(e)
- # Separate OPBATCH ops from other ops
- opbatch_ops = [op for op in valid_ops if op['name'] == "OPBATCH"]
- other_ops = [op for op in valid_ops if op['name'] != "OPBATCH"]
+ for device, dev_ops in valid_ops_by_dev.items():
+ opbatch_ops = [op for op in dev_ops if op['name'] == "OPBATCH"]
+ other_ops = [op for op in dev_ops if op['name'] != "OPBATCH"]
- # Sort them by start_cycles to enable binary search
- opbatch_ops.sort(key=lambda op: op['start_cycles'])
- other_ops.sort(key=lambda op: op['start_cycles'])
+ opbatch_ops.sort(key=lambda op: op['start_cycles'])
+ other_ops.sort(key=lambda op: op['start_cycles'])
- opbatch_starts = [op['start_cycles'] for op in opbatch_ops]
- other_starts = [op['start_cycles'] for op in other_ops]
+ opbatch_starts = [op['start_cycles'] for op in opbatch_ops]
+ other_starts = [op['start_cycles'] for op in other_ops]
- # Map trace events to any operator whose cycles contain them
- for e in all_traces:
- cyc = e['unwrapped_cycles']
- if cyc is None:
- continue
+ dev_traces = traces_by_dev.get(device, [])
+ for e in dev_traces:
+ cyc = e['unwrapped_cycles']
- # Map to OPBATCH
- idx = bisect.bisect_right(opbatch_starts, cyc) - 1
- if idx >= 0:
- op = opbatch_ops[idx]
- if op['start_cycles'] <= cyc <= op['end_cycles']:
- op['trace_events'].append(e)
+ # Map to OPBATCH
+ idx = bisect.bisect_right(opbatch_starts, cyc) - 1
+ if idx >= 0:
+ op = opbatch_ops[idx]
+ if op['start_cycles'] <= cyc <= op['end_cycles']:
+ op['trace_events'].append(e)
- # Map to other ops
- idx = bisect.bisect_right(other_starts, cyc) - 1
- if idx >= 0:
- op = other_ops[idx]
- if op['start_cycles'] <= cyc <= op['end_cycles']:
- op['trace_events'].append(e)
+ # Map to other ops
+ idx = bisect.bisect_right(other_starts, cyc) - 1
+ if idx >= 0:
+ op = other_ops[idx]
+ if op['start_cycles'] <= cyc <= op['end_cycles']:
+ op['trace_events'].append(e)
return all_ops
@@ -563,6 +638,7 @@ def main():
parser.add_argument("--timeline", type=str, nargs='?', const='summary', choices=["summary", "bubbles"],
help="Output ASCII art event summary or thread idle bubble analysis (default: summary)")
parser.add_argument("--filter", type=str, help="Regex filter matching against the original profile-op line")
+ parser.add_argument("--device", type=str, help="Device to filter by (e.g. HTP0, HTP0:0) or 'split' to generate separate reports per device")
group = parser.add_mutually_exclusive_group()
group.add_argument("--head", type=int, help="Limit to first N ops")
@@ -586,29 +662,84 @@ def main():
logger.warning(f"Invalid width format '{w}'")
final_pmu_name = (args.pmu_name or f"#{args.pmu_index}") if args.pmu_index is not None else None
- ops = parse_log(args.logfile, pmu_index=args.pmu_index)
+ op_filter_re = None
if args.filter:
try:
- filter_re = re.compile(args.filter)
+ op_filter_re = re.compile(args.filter)
except re.error as e:
logger.error(f"Invalid regex filter: {e}")
sys.exit(1)
- ops = [op for op in ops if filter_re.search(op['op_text'])]
-
- if args.head is not None:
- ops = ops[:args.head]
- elif args.tail is not None:
- ops = ops[-args.tail:]
-
- if args.timeline:
- for op in ops:
- if args.timeline == "summary":
- print_ascii_summary(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events'])
- elif args.timeline == "bubbles":
- print_bubbles_timeline(op)
+
+ limit = args.head if args.head is not None else None
+ device_filter = args.device if (args.device and args.device != "split") else None
+ ops = parse_log(args.logfile, pmu_index=args.pmu_index, limit=limit, device_filter=device_filter, op_filter_re=op_filter_re)
+
+ if args.device and args.device != "split":
+ ops = [op for op in ops if device_matches(op['device'], args.device)]
+
+ if args.device == "split":
+ unique_devices = sorted(list(set(op['device'] for op in ops)))
+ for dev in unique_devices:
+ dev_ops = [op for op in ops if device_matches(op['device'], dev)]
+
+ if args.filter:
+ try:
+ filter_re = re.compile(args.filter)
+ except re.error as e:
+ logger.error(f"Invalid regex filter: {e}")
+ sys.exit(1)
+ dev_ops = [op for op in dev_ops if filter_re.search(op['op_text'])]
+
+ if args.head is not None:
+ dev_ops = dev_ops[:args.head]
+ elif args.tail is not None:
+ dev_ops = dev_ops[-args.tail:]
+
+ logger.info("\n=========================================")
+ logger.info(f" Device: {dev}")
+ logger.info("=========================================")
+
+ if args.timeline:
+ for op in dev_ops:
+ if args.timeline == "summary":
+ print_ascii_summary(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events'])
+ elif args.timeline == "bubbles":
+ print_bubbles_timeline(op)
+ else:
+ generate_report(dev_ops, args.top, overrides, args.sort, pmu_name=final_pmu_name)
else:
- generate_report(ops, args.top, overrides, args.sort, pmu_name=final_pmu_name)
+ if args.filter:
+ try:
+ filter_re = re.compile(args.filter)
+ except re.error as e:
+ logger.error(f"Invalid regex filter: {e}")
+ sys.exit(1)
+ ops = [op for op in ops if filter_re.search(op['op_text'])]
+
+ if args.head is not None or args.tail is not None:
+ ops_by_dev = defaultdict(list)
+ for op in ops:
+ ops_by_dev[op['device']].append(op)
+
+ filtered_ops = []
+ for dev in sorted(ops_by_dev.keys()):
+ dev_ops = ops_by_dev[dev]
+ if args.head is not None:
+ dev_ops = dev_ops[:args.head]
+ elif args.tail is not None:
+ dev_ops = dev_ops[-args.tail:]
+ filtered_ops.extend(dev_ops)
+ ops = filtered_ops
+
+ if args.timeline:
+ for op in ops:
+ if args.timeline == "summary":
+ print_ascii_summary(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events'])
+ elif args.timeline == "bubbles":
+ print_bubbles_timeline(op)
+ else:
+ generate_report(ops, args.top, overrides, args.sort, pmu_name=final_pmu_name)
if __name__ == "__main__":
diff --git a/scripts/snapdragon/ggml-hexagon-trace.py b/scripts/snapdragon/ggml-hexagon-trace.py
index 4755adfa13..99bf771b85 100755
--- a/scripts/snapdragon/ggml-hexagon-trace.py
+++ b/scripts/snapdragon/ggml-hexagon-trace.py
@@ -20,6 +20,31 @@ trace_pattern = re.compile(
r"trace-evt\s+(?P<event>[A-Z_0-9\-]+):\s+thread\s+(?P<thread>\d+)\s+info\s+(?P<info>\d+)\s+(?P<state>start|stop)\s+(?P<cycles>\d+)"
)
+device_pattern = re.compile(r"\b(HTP\d+(?::\d+)?)\s+(?:profile-op|trace-evt)\b")
+
+
+def extract_device(line):
+ m = device_pattern.search(line)
+ if m:
+ return m.group(1)
+ return "HTP0"
+
+
+def device_matches(record_device, target_device):
+ targets = [t.strip() for t in target_device.split(',')]
+ for target in targets:
+ if record_device == target:
+ return True
+ if record_device.startswith(target + ":"):
+ return True
+ return False
+
+
+def get_split_output_path(base_path, device_name):
+ safe_device = device_name.replace(':', '_')
+ root, ext = os.path.splitext(base_path)
+ return f"{root}-{safe_device}{ext}"
+
def normalize_event_name(evt_type, info=0):
if evt_type == "HVX_COMP":
@@ -54,7 +79,79 @@ class CycleUnwrapper:
return raw + self.high_part
-def parse_log(file_path):
+class DeviceTimeMapper:
+ def __init__(self, dev, ops):
+ self.dev = dev
+ self.batches = []
+ for op in ops:
+ if op.get('device') == dev and op.get('name') == 'OPBATCH' and op.get('unwrapped_cycles_start') is not None:
+ cycles = op.get('cycles', 0)
+ usec = op.get('usec', 0)
+ start_cyc = op['unwrapped_cycles_start']
+ freq = (cycles / usec) if usec > 0 and cycles > 0 else 1000.0
+ if freq <= 0:
+ freq = 1000.0
+ self.batches.append({
+ 'start_cycles': start_cyc,
+ 'cycles': cycles,
+ 'end_cycles': start_cyc + cycles,
+ 'usec': usec,
+ 'dur_ns': usec * 1000,
+ 'freq_mhz': freq,
+ })
+
+ self.batches.sort(key=lambda b: b['start_cycles'])
+
+ for i, b in enumerate(self.batches):
+ if i == 0:
+ b['start_time_ns'] = 0
+ else:
+ prev = self.batches[i - 1]
+ idle_cyc = max(0, b['start_cycles'] - prev['end_cycles'])
+ idle_ns = int(round((idle_cyc / prev['freq_mhz']) * 1000))
+ b['start_time_ns'] = prev['start_time_ns'] + prev['dur_ns'] + idle_ns
+
+ self.batch_starts = [b['start_cycles'] for b in self.batches]
+
+ valid_starts = [op['unwrapped_cycles_start'] for op in ops if op.get('device') == dev and op.get('unwrapped_cycles_start') is not None]
+ self.min_cyc = min(valid_starts) if valid_starts else 0
+ if self.batches:
+ self.default_freq = self.batches[0]['freq_mhz']
+ else:
+ freqs = [op['cycles'] / op['usec'] for op in ops if op.get('device') == dev and op.get('usec', 0) > 0 and op.get('cycles', 0) > 0]
+ self.default_freq = statistics.mean(freqs) if freqs else 1000.0
+
+ def get_batch(self, cyc):
+ if not self.batches:
+ return None
+ idx = bisect.bisect_right(self.batch_starts, cyc) - 1
+ if idx >= 0:
+ return self.batches[idx]
+ return self.batches[0]
+
+ def get_freq(self, cyc=None):
+ if cyc is not None:
+ b = self.get_batch(cyc)
+ if b is not None:
+ return b['freq_mhz']
+ return self.default_freq
+
+ def cycle_to_ns(self, cyc):
+ if cyc is None:
+ return 0
+ b = self.get_batch(cyc)
+ if b is not None:
+ return b['start_time_ns'] + int(round(((cyc - b['start_cycles']) / b['freq_mhz']) * 1000))
+ return int(round(((cyc - self.min_cyc) / self.default_freq) * 1000))
+
+ def dur_cycles_to_ns(self, cyc_start, cyc_dur):
+ if cyc_dur is None:
+ return 0
+ freq = self.get_freq(cyc_start)
+ return int(round((cyc_dur / freq) * 1000))
+
+
+def parse_log(file_path, limit=None, device_filter=None, op_filter_re=None):
try:
if file_path != "-":
f = open(file_path, 'r', encoding='utf-8', errors='ignore')
@@ -67,14 +164,25 @@ def parse_log(file_path):
all_ops: List[Dict[str, Any]] = []
all_traces: List[Dict[str, Any]] = []
current_op: Optional[Dict[str, Any]] = None
- unwrapper = None
- trace_unwrapper = None
+ ops_count_per_device = {}
+ if device_filter is not None:
+ for target in device_filter.split(','):
+ ops_count_per_device[target.strip()] = 0
+ limit_reached = False
+ unwrappers = {}
+ last_batch_start = {}
+ trace_unwrappers = {}
line_idx = 0
for line in f:
line_idx += 1
- if "|" in line and "profile-op" in line:
- parts = [p.strip() for p in line.split("|")]
+ if "profile-op" not in line and "trace-evt" not in line:
+ continue
+ device = extract_device(line)
+
+ idx = line.find("profile-op")
+ if idx != -1 and "|" in line[idx:]:
+ parts = [p.strip() for p in line[idx:].split("|")]
prefix = parts[0]
prefix_match = re.search(r"profile-op\s+(?P<op_name>[A-Z_0-9+]+)", prefix)
if not prefix_match:
@@ -115,14 +223,18 @@ def parse_log(file_path):
if op_name == "OPBATCH":
if cycles_start_raw:
unwrapped_cycles_start = int(cycles_start_raw)
- unwrapper = CycleUnwrapper(unwrapped_cycles_start)
- trace_unwrapper = CycleUnwrapper(unwrapped_cycles_start)
+ unwrappers[device] = CycleUnwrapper(unwrapped_cycles_start)
+ last_batch_start[device] = unwrapped_cycles_start
+ for k in list(trace_unwrappers.keys()):
+ if k[0] == device:
+ del trace_unwrappers[k]
else:
- if cycles_start_raw and unwrapper is not None:
- unwrapped_cycles_start = unwrapper.unwrap(int(cycles_start_raw))
+ if cycles_start_raw:
+ device_unwrapper = unwrappers.get(device)
+ if device_unwrapper is not None:
+ unwrapped_cycles_start = device_unwrapper.unwrap(int(cycles_start_raw))
- idx = line.find("profile-op ")
- op_text = line[idx + 11:].strip() if idx != -1 else line.strip()
+ op_text = re.sub(r"^profile-op\s+", "", line[idx:]).strip() if idx != -1 else line.strip()
evt_str = None
if types.startswith("evt-cnt "):
@@ -142,24 +254,59 @@ def parse_log(file_path):
'cycles_start': int(cycles_start_raw) if cycles_start_raw else None,
'unwrapped_cycles_start': unwrapped_cycles_start,
'trace_events': [],
- 'line_num': line_idx
+ 'line_num': line_idx,
+ 'device': device
}
all_ops.append(current_op)
+
+ # Check if matching early exit criteria
+ matched = False
+ matched_target = None
+ if device_filter is not None:
+ targets = [t.strip() for t in device_filter.split(',')]
+ for target in targets:
+ if device == target or device.startswith(target + ":"):
+ matched = True
+ matched_target = target
+ break
+ else:
+ matched = True
+ matched_target = device
+
+ if op_filter_re is not None and not op_filter_re.search(op_text):
+ matched = False
+
+ if matched:
+ if matched_target not in ops_count_per_device:
+ ops_count_per_device[matched_target] = 0
+ ops_count_per_device[matched_target] += 1
+
+ if limit is not None and len(ops_count_per_device) > 0 and all(count >= limit for count in ops_count_per_device.values()):
+ limit_reached = True
+
+ if limit_reached and op_name == "OPBATCH":
+ break
continue
trace_match = trace_pattern.search(line)
if trace_match:
+ thread = int(trace_match.group('thread'))
raw_cyc = int(trace_match.group('cycles'))
unwrapped_cyc = None
- if trace_unwrapper is not None:
- unwrapped_cyc = trace_unwrapper.unwrap(raw_cyc)
+ th_key = (device, thread)
+ if th_key not in trace_unwrappers:
+ batch_start = last_batch_start.get(device)
+ trace_unwrappers[th_key] = CycleUnwrapper(batch_start)
+ unwrapped_cyc = trace_unwrappers[th_key].unwrap(raw_cyc)
all_traces.append({
- 'thread': int(trace_match.group('thread')),
+ 'thread': thread,
'event': trace_match.group('event'),
'info': int(trace_match.group('info')),
'cycles': raw_cyc,
'unwrapped_cycles': unwrapped_cyc,
- 'state': trace_match.group('state')
+ 'state': trace_match.group('state'),
+ 'line_num': line_idx,
+ 'device': device
})
f.close()
@@ -274,27 +421,24 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
logger.warning("No operators found after filtering.")
return
- # Compute average frequency
- frequencies = []
- for op in filtered_ops:
- if op['usec'] > 0 and op['cycles'] > 0:
- frequencies.append(op['cycles'] / op['usec'])
- avg_freq_mhz = statistics.mean(frequencies) if frequencies else 1000.0
- if avg_freq_mhz <= 0:
- avg_freq_mhz = 1000.0
-
# Assign start and end cycles to each operator
for op in filtered_ops:
op['start_cycles'] = op['unwrapped_cycles_start']
- op['end_cycles'] = op['start_cycles'] + op['cycles']
+ op['end_cycles'] = op['start_cycles'] + op['cycles'] if op['start_cycles'] is not None else None
- global_min_cyc = min(op['start_cycles'] for op in filtered_ops if op['start_cycles'] is not None)
+ # Get list of unique devices present in the operations
+ unique_devices = sorted(list(set(op['device'] for op in filtered_ops)))
+ device_to_idx = {dev: idx for idx, dev in enumerate(unique_devices)}
+ time_mappers = {dev: DeviceTimeMapper(dev, filtered_ops) for dev in unique_devices}
# Process events
completed_events = []
if trace_events:
trace_events = sorted(trace_events, key=lambda e: e['unwrapped_cycles'])
- one_usec_cycles = max(avg_freq_mhz, 1.0)
+
+ one_usec_cycles = {}
+ for dev in unique_devices:
+ one_usec_cycles[dev] = max(time_mappers[dev].get_freq(), 1.0)
active_starts = {}
for e in trace_events:
@@ -303,31 +447,36 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
info = e['info']
state = e['state']
cyc = e['unwrapped_cycles']
+ dev = e['device']
- key = (t, evt, info)
+ key = (dev, t, evt, info)
if state == 'start':
# Handle missing stop (start followed by another start)
if key in active_starts:
- prev_start = active_starts[key]
+ prev_e = active_starts[key]
completed_events.append({
'thread': t,
'event': evt,
'info': info,
- 'start_cyc': prev_start,
- 'end_cyc': prev_start + one_usec_cycles,
+ 'start_cyc': prev_e['unwrapped_cycles'],
+ 'end_cyc': prev_e['unwrapped_cycles'] + one_usec_cycles.get(dev, 1000.0),
+ 'line_num': prev_e.get('line_num'),
'missing_stop': True,
+ 'device': dev
})
- active_starts[key] = cyc
+ active_starts[key] = e
elif state == 'stop':
if key in active_starts:
- start_cyc = active_starts[key]
+ prev_e = active_starts[key]
del active_starts[key]
completed_events.append({
'thread': t,
'event': evt,
'info': info,
- 'start_cyc': start_cyc,
+ 'start_cyc': prev_e['unwrapped_cycles'],
'end_cyc': cyc,
+ 'line_num': prev_e.get('line_num'),
+ 'device': dev
})
else:
# Handle missing start (stop without start)
@@ -335,31 +484,36 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
'thread': t,
'event': evt,
'info': info,
- 'start_cyc': cyc - one_usec_cycles,
+ 'start_cyc': cyc - one_usec_cycles.get(dev, 1000.0),
'end_cyc': cyc,
+ 'line_num': e.get('line_num'),
'missing_start': True,
+ 'device': dev
})
# Clear remaining unmatched starts
- for key, start_cyc in active_starts.items():
- t, evt, info = key
+ for key, prev_e in active_starts.items():
+ dev, t, evt, info = key
completed_events.append({
'thread': t,
'event': evt,
'info': info,
- 'start_cyc': start_cyc,
- 'end_cyc': start_cyc + one_usec_cycles,
+ 'start_cyc': prev_e['unwrapped_cycles'],
+ 'end_cyc': prev_e['unwrapped_cycles'] + one_usec_cycles.get(dev, 1000.0),
+ 'line_num': prev_e.get('line_num'),
'missing_stop': True,
+ 'device': dev
})
completed_events.sort(key=lambda e: e['start_cyc'])
- # Convert event times to microseconds and apply clamp rounded to 1ns resolution (3 decimals)
+ # Convert event times to nanoseconds using per-device / per-batch time mapper
for e in completed_events:
- start_us = (e['start_cyc'] - global_min_cyc) / avg_freq_mhz
- dur_us = (e['end_cyc'] - e['start_cyc']) / avg_freq_mhz
- e['ts_ns'] = int(round(start_us * 1000))
- e['dur_ns'] = int(round(max(dur_us, 0.1) * 1000))
+ dev = e['device']
+ tm = time_mappers[dev]
+ e['ts_ns'] = tm.cycle_to_ns(e['start_cyc'])
+ dur_ns = tm.dur_cycles_to_ns(e['start_cyc'], e['end_cyc'] - e['start_cyc'])
+ e['dur_ns'] = max(dur_ns, 100)
# Allocate slots (sub-tracks) to prevent overlaps on same virtual track
active_slots = defaultdict(list)
@@ -368,14 +522,15 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
evt = e['event']
ts = e['ts_ns']
dur = e['dur_ns']
+ dev = e['device']
norm_evt = normalize_event_name(evt, e['info'])
if norm_evt == "DMA":
- track_key = (t, "DMA")
+ track_key = (dev, t, "DMA")
elif t == 10:
- track_key = (t, "HMX")
+ track_key = (dev, t, "HMX")
else:
- track_key = (t, "HVX")
+ track_key = (dev, t, "HVX")
slots = active_slots[track_key]
allocated_slot = -1
@@ -395,6 +550,7 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
t = e['thread']
evt = e['event']
slot = e['slot']
+ dev = e['device']
norm_evt = normalize_event_name(evt, e['info'])
if norm_evt == "DMA":
@@ -408,56 +564,69 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
evt_id = 2
t_sort = 1 if t == 10 else t + 2
+ dev_idx = device_to_idx[dev]
+
# Unique UUID for each sub-track
if t == 10:
- uuid = 20 # HMX thread track UUID
+ uuid = dev_idx * 10000000 + 20 # HMX thread track UUID
else:
- uuid = int(t_sort * 1000000 + evt_id * 1000 + slot)
+ uuid = int(dev_idx * 10000000 + t_sort * 1000000 + evt_id * 1000 + slot)
e['uuid'] = uuid
- used_tracks[uuid] = (t, track_evt, slot)
+ used_tracks[uuid] = (dev, t, track_evt, slot)
with open(output_path, "wb") as f:
- # Define Process with EXPLICIT child sorting
- proc_desc = make_process_descriptor(1, "HTP NPU")
- proc_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(1, process=proc_desc, child_ordering=3))
- write_trace_packet_to_file(f, proc_packet)
-
- # Define Operators Track (UUID = 2) as a thread track at rank 1, tid 8
- op_thread_desc = make_thread_descriptor(1, 8, "Ops", sort_index=1)
- op_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(2, parent_uuid=1, thread=op_thread_desc))
- write_trace_packet_to_file(f, op_packet)
-
- # Define HMX Thread Track (UUID = 20) at rank 2, tid 9
- hmx_thread_desc = make_thread_descriptor(1, 9, "HMX", sort_index=2)
- hmx_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(20, parent_uuid=1, thread=hmx_thread_desc))
- write_trace_packet_to_file(f, hmx_packet)
-
- # Define Thread Tracks (T0, T1, ..., T9)
- unique_threads = sorted(list(set(t for (t, _, _) in used_tracks.values() if t != 10)))
- for t in unique_threads:
- thread_uuid = 10 + t
- thread_name = f"T{t}"
- # Sort order starts from index 3 (T0 -> 3, T1 -> 4, etc.)
- sort_index = 3 + t
- tid = 10 + t
- thread_desc = make_thread_descriptor(1, tid, thread_name, sort_index=sort_index)
- thread_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(
- thread_uuid,
- parent_uuid=1,
- thread=thread_desc,
- sibling_order_rank=sort_index,
- child_ordering=3 # Explicit child sorting for sub-tracks
- ))
- write_trace_packet_to_file(f, thread_packet)
+ for dev in unique_devices:
+ dev_idx = device_to_idx[dev]
+ pid = dev_idx + 1
+ proc_uuid = dev_idx * 10000000 + 1
+
+ # Define Process with EXPLICIT child sorting
+ proc_name = dev
+ proc_desc = make_process_descriptor(pid, proc_name)
+ proc_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(proc_uuid, process=proc_desc, child_ordering=3))
+ write_trace_packet_to_file(f, proc_packet)
+
+ # Define Operators Track as a thread track
+ op_track_uuid = dev_idx * 10000000 + 2
+ op_tid = pid * 100 + 8
+ op_thread_desc = make_thread_descriptor(pid, op_tid, "Ops", sort_index=1)
+ op_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(op_track_uuid, parent_uuid=proc_uuid, thread=op_thread_desc))
+ write_trace_packet_to_file(f, op_packet)
+
+ # Define HMX Thread Track at rank 2
+ hmx_track_uuid = dev_idx * 10000000 + 20
+ hmx_tid = pid * 100 + 9
+ hmx_thread_desc = make_thread_descriptor(pid, hmx_tid, "HMX", sort_index=2)
+ hmx_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(hmx_track_uuid, parent_uuid=proc_uuid, thread=hmx_thread_desc))
+ write_trace_packet_to_file(f, hmx_packet)
+
+ # Define Thread Tracks (T0, T1, ..., T9) for this device
+ dev_used_tracks = {uuid: val for uuid, val in used_tracks.items() if val[0] == dev}
+ unique_threads = sorted(list(set(t for (_, t, _, _) in dev_used_tracks.values() if t != 10)))
+ for t in unique_threads:
+ thread_uuid = dev_idx * 10000000 + 10 + t
+ thread_name = f"T{t}"
+ sort_index = 3 + t
+ tid = pid * 100 + 10 + t
+ thread_desc = make_thread_descriptor(pid, tid, thread_name, sort_index=sort_index)
+ thread_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(
+ thread_uuid,
+ parent_uuid=proc_uuid,
+ thread=thread_desc,
+ sibling_order_rank=sort_index,
+ child_ordering=3 # Explicit child sorting for sub-tracks
+ ))
+ write_trace_packet_to_file(f, thread_packet)
# Define Track descriptors for sub-tracks parented to thread tracks
for uuid in sorted(used_tracks.keys()):
- if uuid == 20:
+ dev, t, evt, slot = used_tracks[uuid]
+ dev_idx = device_to_idx[dev]
+ if t == 10:
continue
- t, evt, slot = used_tracks[uuid]
name = f"T{t} {evt}"
rank = 0 if evt == "HVX" else 1
- parent_thread_uuid = 10 + t
+ parent_thread_uuid = dev_idx * 10000000 + 10 + t
# Sibling merge behavior: 1 (SIBLING_MERGE_BEHAVIOR_BY_TRACK_NAME)
track_desc = make_track_descriptor(
uuid=uuid,
@@ -470,15 +639,18 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
write_trace_packet_to_file(f, track_packet)
# Emit Operators
- last_op_end_ns = 0
+ last_op_end_ns = defaultdict(int)
for op in filtered_ops:
- op_start_ns = int(round(((op['start_cycles'] - global_min_cyc) / avg_freq_mhz) * 1000))
- op_dur_ns = int(round((op['cycles'] / avg_freq_mhz) * 1000))
+ dev = op['device']
+ dev_idx = device_to_idx[dev]
+ tm = time_mappers[dev]
+ op_start_ns = tm.cycle_to_ns(op['start_cycles'])
+ op_dur_ns = tm.dur_cycles_to_ns(op['start_cycles'], op['cycles'])
if op['name'] != "OPBATCH":
- if op_start_ns < last_op_end_ns:
- op_start_ns = last_op_end_ns
+ if op_start_ns < last_op_end_ns[dev]:
+ op_start_ns = last_op_end_ns[dev]
clamped_dur = max(op_dur_ns, 100) # Clamp to 100ns (0.1us)
- last_op_end_ns = op_start_ns + clamped_dur
+ last_op_end_ns[dev] = op_start_ns + clamped_dur
else:
clamped_dur = max(op_dur_ns, 100)
@@ -495,24 +667,41 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path):
if 'evt' in op and op['evt']:
debug_annots.append(make_debug_annotation("evt", string_val=op['evt']))
+ op_track_uuid = dev_idx * 10000000 + 2
+
# Slice Begin
- evt_begin = make_track_event(1, 2, name=f"{op['name']} ({op['dims']})", category="operator", debug_annotations=debug_annots)
+ evt_begin = make_track_event(1, op_track_uuid, name=f"{op['name']} ({op['dims']})", category="operator", debug_annotations=debug_annots)
packet_begin = make_trace_packet(op_start_ns, track_event=evt_begin)
write_trace_packet_to_file(f, packet_begin)
# Slice End
- evt_end = make_track_event(2, 2)
+ evt_end = make_track_event(2, op_track_uuid)
packet_end = make_trace_packet(op_start_ns + clamped_dur, track_event=evt_end)
write_trace_packet_to_file(f, packet_end)
# Emit Thread Trace Events
for e in completed_events:
norm_name = normalize_event_name(e['event'], e['info'])
- name = f"DMA {e['info']}" if norm_name == "DMA" else norm_name
+ if norm_name == "DMA":
+ name = f"DMA {e['info']}"
+ elif norm_name == "FENCE":
+ name = f"FENCE {e['info']}" if e.get('info') is not None and e['info'] != 0 else "FENCE"
+ else:
+ name = norm_name
+
if e.get('missing_start') or e.get('missing_stop'):
name += "!"
debug_annots = []
+ if 'line_num' in e and e['line_num'] is not None:
+ debug_annots.append(make_debug_annotation("line", int_val=e['line_num']))
+ if norm_name == "FENCE" and e.get('info') is not None:
+ debug_annots.append(make_debug_annotation("seq", int_val=e['info']))
+ elif norm_name == "DMA" and e.get('info') is not None:
+ debug_annots.append(make_debug_annotation("channel", int_val=e['info']))
+ elif e.get('info') is not None and e['info'] != 0:
+ debug_annots.append(make_debug_annotation("info", int_val=e['info']))
+
if e.get('missing_start'):
debug_annots.append(make_debug_annotation("missing_start", string_val="true"))
if e.get('missing_stop'):
@@ -536,6 +725,7 @@ def main():
parser.add_argument("logfile", help="Path to hex-log profile file")
parser.add_argument("-o", "--output", default="optrace.perfetto-trace", help="Output trace file path (default: optrace.perfetto-trace)")
parser.add_argument("--filter", type=str, help="Regex filter matching against the original profile-op line")
+ parser.add_argument("--device", type=str, help="Device to filter by (e.g. HTP0, HTP0:0) or 'split' to generate separate files per device")
group = parser.add_mutually_exclusive_group()
group.add_argument("--head", type=int, help="Limit to first N ops")
@@ -544,7 +734,21 @@ def main():
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format='%(message)s')
- ops, traces = parse_log(args.logfile)
+ op_filter_re = None
+ if args.filter:
+ try:
+ op_filter_re = re.compile(args.filter)
+ except re.error as e:
+ logger.error(f"Invalid regex filter: {e}")
+ sys.exit(1)
+
+ limit = args.head if args.head is not None else None
+ device_filter = args.device if (args.device and args.device != "split") else None
+ ops, traces = parse_log(args.logfile, limit=limit, device_filter=device_filter, op_filter_re=op_filter_re)
+
+ if args.device and args.device != "split":
+ ops = [op for op in ops if device_matches(op['device'], args.device)]
+ traces = [t for t in traces if device_matches(t['device'], args.device)]
if args.filter:
try:
@@ -554,35 +758,60 @@ def main():
sys.exit(1)
ops = [op for op in ops if filter_re.search(op['op_text'])]
- if args.head is not None:
- ops = ops[:args.head]
- elif args.tail is not None:
- ops = ops[-args.tail:]
+ if args.head is not None or args.tail is not None:
+ ops_by_dev = defaultdict(list)
+ for op in ops:
+ ops_by_dev[op['device']].append(op)
+
+ filtered_ops = []
+ for dev in sorted(ops_by_dev.keys()):
+ dev_ops = ops_by_dev[dev]
+ if args.head is not None:
+ dev_ops = dev_ops[:args.head]
+ elif args.tail is not None:
+ dev_ops = dev_ops[-args.tail:]
+ filtered_ops.extend(dev_ops)
+ ops = filtered_ops
if args.filter or args.head is not None or args.tail is not None:
- valid_ranges = []
+ # Group valid ranges by device
+ valid_ranges_by_dev = defaultdict(list)
for op in ops:
start_cyc = op['unwrapped_cycles_start']
end_cyc = start_cyc + op['cycles'] if start_cyc is not None else None
if start_cyc is not None and end_cyc is not None:
- valid_ranges.append((start_cyc, end_cyc))
+ valid_ranges_by_dev[op['device']].append((start_cyc, end_cyc))
- valid_ranges.sort(key=lambda r: r[0])
- range_starts = [r[0] for r in valid_ranges]
+ for dev in valid_ranges_by_dev:
+ valid_ranges_by_dev[dev].sort(key=lambda r: r[0])
+
+ range_starts_by_dev = {dev: [r[0] for r in ranges] for dev, ranges in valid_ranges_by_dev.items()}
filtered_traces = []
for e in traces:
cyc = e['unwrapped_cycles']
if cyc is None:
continue
+ dev = e['device']
+ range_starts = range_starts_by_dev.get(dev)
+ if not range_starts:
+ continue
idx = bisect.bisect_right(range_starts, cyc) - 1
if idx >= 0:
- start, end = valid_ranges[idx]
+ start, end = valid_ranges_by_dev[dev][idx]
if start <= cyc <= end:
filtered_traces.append(e)
traces = filtered_traces
- generate_perfetto_trace(ops, traces, args.output)
+ if args.device == "split":
+ unique_devices = sorted(list(set(op['device'] for op in ops)))
+ for dev in unique_devices:
+ dev_ops = [op for op in ops if device_matches(op['device'], dev)]
+ dev_traces = [t for t in traces if device_matches(t['device'], dev)]
+ out_path = get_split_output_path(args.output, dev)
+ generate_perfetto_trace(dev_ops, dev_traces, out_path)
+ else:
+ generate_perfetto_trace(ops, traces, args.output)
if __name__ == "__main__":
diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.py
new file mode 100755
index 0000000000..d639b16b82
--- /dev/null
+++ b/scripts/snapdragon/run.py
@@ -0,0 +1,405 @@
+#!/usr/bin/env python3
+#
+# Run llama.cpp tools on Snapdragon devices (natively, via ADB, or SSH).
+#
+
+import sys
+import os
+import argparse
+import subprocess
+import platform
+import shlex
+import logging
+
+logger = logging.getLogger("run")
+
+
+def parse_target(target_str):
+ if not target_str:
+ return None, None
+ if target_str.startswith("adb") or target_str.startswith("android"):
+ parts = target_str.split(":", 1)
+ serial = parts[1] if len(parts) > 1 else None
+ return "android", serial
+ elif target_str.startswith("lnx") or target_str.startswith("linux") or target_str.startswith("ubuntu"):
+ parts = target_str.split(":", 1)
+ host = parts[1] if len(parts) > 1 else None
+ return "linux", host
+ elif target_str in ("wos", "windows"):
+ return "windows", None
+ else:
+ return None, None
+
+
+def shlex_join(args_list):
+ if hasattr(shlex, 'join'):
+ return shlex.join(args_list)
+ import pipes
+ return " ".join(pipes.quote(x) for x in args_list)
+
+
+def main():
+ logging.basicConfig(level=logging.INFO, format='%(message)s')
+ # Split arguments at '--'
+ if '--' in sys.argv:
+ idx = sys.argv.index('--')
+ run_args = sys.argv[1:idx]
+ cmd_args = sys.argv[idx + 1:]
+ else:
+ run_args = sys.argv[1:]
+ cmd_args = []
+
+ parser = argparse.ArgumentParser(
+ description="Unified runner for llama.cpp tools on Snapdragon (natively, via ADB, or via SSH)."
+ )
+ parser.add_argument("--target", help="Execution target (e.g. android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, windows/wos) (default: local run)")
+ parser.add_argument("--target-dir", help="Target directory on the device (default: /data/local/tmp/llama.cpp for Android, ~/llama.cpp for Linux)")
+ parser.add_argument("--install-dir", help="Install directory name (defaults to pkg-TARGET or pkg-TARGET-dbg prefix based on target)")
+ parser.add_argument("--debug", action="store_true", help="Use debug build (defaults to pkg-TARGET-dbg folder)")
+ parser.add_argument("--devices", "--device", "-d", help="Select execution devices (split into NPU and OpenCL GPUs automatically, default: HTP0)")
+ parser.add_argument("--verbose", help="Verbose level (enables both Hexagon and OpenCL kernel cache debugging)")
+ parser.add_argument("--profile", help="Profiling flag (enables Hexagon profiling and OpenCL autotuning)")
+ parser.add_argument("--sched-debug", action="store_true", help="Enable GGML/llama.cpp scheduler debug output (GGML_SCHED_DEBUG=2)")
+ parser.add_argument("--mtmd-device", help="Specify the backend device ID for Multi-Threaded Multi-Device setup (MTMD_BACKEND_DEVICE)")
+
+ # Hexagon specific parameters
+ parser.add_argument("--hex-verbose", help="Enable verbose logging (GGML_HEXAGON_VERBOSE)")
+ parser.add_argument("--hex-profile", help="Enable NPU/Hexagon profiling and performance metrics print (GGML_HEXAGON_PROFILE)")
+ parser.add_argument("--hex-nhvx", help="Number of HVX units to use (GGML_HEXAGON_NHVX)")
+ parser.add_argument("--hex-nhmx", help="Number of HMX units to use. 0 disables HMX power-up (GGML_HEXAGON_NHMX)")
+ parser.add_argument("--hex-hostbuf", help="Enable host buffers (GGML_HEXAGON_HOSTBUF)")
+ parser.add_argument("--hex-opbatch", help="Maximum number of operations to batch into a single HTP execution (GGML_HEXAGON_OPBATCH)")
+ parser.add_argument("--hex-opqueue", help="Size of the asynchronous NPU operation queue (GGML_HEXAGON_OPQUEUE)")
+ parser.add_argument("--hex-oppoll", default="1", help="Enable (1) or Disable (0) polling for NPU opbatch completion (GGML_HEXAGON_OPPOLL) (default: 1)")
+ parser.add_argument("--hex-opfilter", help="Regex pattern to filter/select which operators are offloaded to NPU (GGML_HEXAGON_OPFILTER)")
+ parser.add_argument("--hex-opfusion", help="NPU graph node fusion optimization level (0: disabled, 1: enabled) (GGML_HEXAGON_OPFUSION)")
+ parser.add_argument("--hex-vmem", help="Maximum NPU VMEM size limit in MB to allocate (GGML_HEXAGON_VMEM)")
+ parser.add_argument("--hex-mbuf", help="Maximum host buffer size limit in MB to allocate (GGML_HEXAGON_MBUF)")
+ parser.add_argument("--hex-mm-select", help="Select MUL_MAT and MUL_MAT_ID kernel (GGML_HEXAGON_MM_SELECT) 3:HMX,2:HVX-tiled,1:HVX-flat,0:disable")
+ parser.add_argument("--hex-fa-select", help="Select Flash Attention kernel (GGML_HEXAGON_FA_SELECT) 2:HMX,1:HVX,0:disable")
+ parser.add_argument("--hex-ar-select", help="Select All-Reduce kernel (GGML_HEXAGON_AR_SELECT) 1:enable,0:disable")
+ parser.add_argument("--hex-etm", help="Enable Embedded Trace Macrocell hardware tracing / trace logging (GGML_HEXAGON_ETM)")
+ parser.add_argument("--hex-arch", help="Target Hexagon NPU architecture version override (v73, v75, v79, v81, etc.) (GGML_HEXAGON_ARCH)")
+ parser.add_argument("--hex-optrace", help="Trace buffer size in number of records (GGML_HEXAGON_OPTRACE)")
+
+ # OpenCL specific parameters
+ parser.add_argument("--cl-platform", help="Select OpenCL platform name/regex (e.g. Qualified Qualcomm OpenCL platform) (GGML_OPENCL_PLATFORM)")
+ parser.add_argument("--cl-device", help="Select OpenCL device name/regex (e.g. Adreno GPU) (GGML_OPENCL_DEVICE)")
+ parser.add_argument("--cl-opfilter", help="Regex pattern to filter/select which operators are offloaded to OpenCL (GGML_OPENCL_OPFILTER)")
+ parser.add_argument("--cl-disable-fusion", action="store_true", help="Disable OpenCL kernel fusion optimizations (GGML_OPENCL_DISABLE_FUSION)")
+ parser.add_argument("--cl-cache-dir", help="Directory path to store compiled OpenCL program binaries (GGML_OPENCL_KERNEL_CACHE_DIR)")
+ parser.add_argument("--cl-cache-debug", help="Enable verbose debugging logs for the kernel caching system (GGML_OPENCL_KERNEL_CACHE_DEBUG)")
+ parser.add_argument("--cl-fa-tune", action="store_true", help="Enable automatic Flash Attention kernel autotuning (GGML_OPENCL_FA_TUNE)")
+ parser.add_argument("--cl-adreno-xmem", action="store_true", help="Enforce matmul using texture/image (xmem) memory paths on Adreno GPUs (GGML_OPENCL_ADRENO_XMEM_GEMM)")
+ parser.add_argument("--cl-adreno-large-buffer", action="store_true", help="Allow allocating larger buffer sizes on Adreno GPUs (GGML_OPENCL_ADRENO_USE_LARGE_BUFFER)")
+
+ args = parser.parse_args(run_args)
+
+ if not cmd_args:
+ parser.print_help()
+ logger.error("\nError: No command specified after '--'")
+ sys.exit(1)
+
+ target_type = None
+ target_val = None
+ target_prefix = None
+ if args.target:
+ target_type, target_val = parse_target(args.target)
+ if not target_type:
+ logger.error(f"Error: Invalid target format '{args.target}'. Must be android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, or windows/wos.")
+ sys.exit(1)
+ target_prefix = args.target.split(":", 1)[0]
+
+ # Resolve install directory
+ install_dir = args.install_dir
+ if not install_dir:
+ if target_prefix:
+ suffix = "-dbg" if args.debug else ""
+ install_dir = f"pkg-{target_prefix}{suffix}"
+ else:
+ # Smart branch folder detection for local run if default is not set
+ prefixes = ("wos", "windows", "lnx", "linux", "ubuntu", "adb", "android")
+ suffixes = ("-dbg", "") if args.debug else ("", "-dbg")
+ found = False
+ for suffix in suffixes:
+ for prefix in prefixes:
+ test_path = f"./pkg-{prefix}{suffix}/llama.cpp"
+ if os.path.exists(test_path):
+ install_dir = f"pkg-{prefix}{suffix}"
+ found = True
+ break
+ if found:
+ break
+ if not install_dir:
+ install_dir = "pkg-android" # Fallback default
+
+ # Host side package path
+ package_path = os.path.join(install_dir, "llama.cpp")
+
+ # Environment variables to map
+ env_vars = {}
+
+ def set_env(env_name, opt_val):
+ if opt_val is not None:
+ env_vars[env_name] = str(opt_val)
+ elif env_name in os.environ:
+ env_vars[env_name] = os.environ[env_name]
+
+ # Resolve and filter devices (HTP vs OpenCL)
+ devices_val = args.devices if args.devices is not None else "HTP0"
+ if devices_val.isdigit():
+ hex_devices = devices_val
+ cl_device = ""
+ else:
+ parts = [p.strip() for p in devices_val.split(",")]
+ # Any device containing "htp" is Hexagon, rest is OpenCL
+ hex_parts = [p for p in parts if "htp" in p.lower()]
+ cl_parts = [p for p in parts if "htp" not in p.lower()]
+ hex_devices = ",".join(hex_parts)
+ cl_device = ",".join(cl_parts)
+
+ # Set Hexagon devices
+ if hex_devices:
+ env_vars["GGML_HEXAGON_DEVICES"] = hex_devices
+ elif "GGML_HEXAGON_DEVICES" in os.environ:
+ env_vars["GGML_HEXAGON_DEVICES"] = os.environ["GGML_HEXAGON_DEVICES"]
+
+ # Set OpenCL device (unless overridden by --cl-device)
+ final_cl_device = args.cl_device if args.cl_device is not None else cl_device
+ if final_cl_device:
+ env_vars["GGML_OPENCL_DEVICE"] = final_cl_device
+ elif "GGML_OPENCL_DEVICE" in os.environ:
+ env_vars["GGML_OPENCL_DEVICE"] = os.environ["GGML_OPENCL_DEVICE"]
+
+ # Map shared & backend-specific parameters with correct overrides
+
+ # Verbose logging mapping
+ hex_verbose_val = args.hex_verbose if args.hex_verbose is not None else args.verbose
+ set_env("GGML_HEXAGON_VERBOSE", hex_verbose_val)
+
+ cl_cache_debug_val = args.cl_cache_debug if args.cl_cache_debug is not None else args.verbose
+ set_env("GGML_OPENCL_KERNEL_CACHE_DEBUG", cl_cache_debug_val)
+
+ # Profiling mapping
+ hex_profile_val = args.hex_profile if args.hex_profile is not None else args.profile
+ set_env("GGML_HEXAGON_PROFILE", hex_profile_val)
+
+ if args.cl_fa_tune or args.profile is not None:
+ env_vars["GGML_OPENCL_FA_TUNE"] = "1"
+ elif "GGML_OPENCL_FA_TUNE" in os.environ:
+ env_vars["GGML_OPENCL_FA_TUNE"] = os.environ["GGML_OPENCL_FA_TUNE"]
+
+ # Other Hexagon environment variables
+ set_env("GGML_HEXAGON_NHVX", args.hex_nhvx)
+ set_env("GGML_HEXAGON_NHMX", args.hex_nhmx)
+ set_env("GGML_HEXAGON_HOSTBUF", args.hex_hostbuf)
+ set_env("GGML_HEXAGON_OPBATCH", args.hex_opbatch)
+ set_env("GGML_HEXAGON_OPQUEUE", args.hex_opqueue)
+ set_env("GGML_HEXAGON_OPPOLL", args.hex_oppoll)
+ set_env("GGML_HEXAGON_OPFILTER", args.hex_opfilter)
+ set_env("GGML_HEXAGON_OPFUSION", args.hex_opfusion)
+ set_env("GGML_HEXAGON_VMEM", args.hex_vmem)
+ set_env("GGML_HEXAGON_MBUF", args.hex_mbuf)
+ set_env("GGML_HEXAGON_MM_SELECT", args.hex_mm_select)
+ set_env("GGML_HEXAGON_FA_SELECT", args.hex_fa_select)
+ set_env("GGML_HEXAGON_AR_SELECT", args.hex_ar_select)
+ set_env("GGML_HEXAGON_ETM", args.hex_etm)
+ set_env("GGML_HEXAGON_ARCH", args.hex_arch)
+ set_env("GGML_HEXAGON_OPTRACE", args.hex_optrace)
+ set_env("MTMD_BACKEND_DEVICE", args.mtmd_device)
+
+ # OpenCL environment variables
+ set_env("GGML_OPENCL_PLATFORM", args.cl_platform)
+ set_env("GGML_OPENCL_OPFILTER", args.cl_opfilter)
+ set_env("GGML_OPENCL_KERNEL_CACHE_DIR", args.cl_cache_dir)
+
+ if args.cl_disable_fusion:
+ env_vars["GGML_OPENCL_DISABLE_FUSION"] = "1"
+ elif "GGML_OPENCL_DISABLE_FUSION" in os.environ:
+ env_vars["GGML_OPENCL_DISABLE_FUSION"] = os.environ["GGML_OPENCL_DISABLE_FUSION"]
+
+ if args.cl_adreno_xmem:
+ env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = "1"
+ elif "GGML_OPENCL_ADRENO_XMEM_GEMM" in os.environ:
+ env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = os.environ["GGML_OPENCL_ADRENO_XMEM_GEMM"]
+
+ if args.cl_adreno_large_buffer:
+ env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = "1"
+ elif "GGML_OPENCL_ADRENO_USE_LARGE_BUFFER" in os.environ:
+ env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = os.environ["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"]
+
+ if args.sched_debug:
+ env_vars["GGML_SCHED_DEBUG"] = "2"
+
+ # Resolve executable path
+ executable = cmd_args[0]
+ known_binaries = ["llama-cli", "llama-bench", "llama-completion", "llama-mtmd-cli", "test-backend-ops"]
+ if executable in known_binaries:
+ if target_type in ("android", "linux"):
+ resolved_exec = f"./bin/{executable}"
+ else:
+ if platform.system() == "Windows":
+ resolved_exec = os.path.normpath(os.path.join(package_path, "bin", f"{executable}.exe"))
+ else:
+ resolved_exec = os.path.normpath(os.path.join(package_path, "bin", executable))
+ cmd_args[0] = resolved_exec
+
+ # Infer device string to pass to the tool
+ basename = os.path.basename(executable)
+ if basename.endswith(".exe"):
+ basename = basename[:-4]
+
+ device_val = None
+ if basename == "test-backend-ops":
+ for i in range(len(cmd_args)):
+ if cmd_args[i] in ("-p", "--params") and i + 1 < len(cmd_args):
+ val = cmd_args[i + 1]
+ new_val = ""
+ for j, char in enumerate(val):
+ if char in ('[', ']'):
+ if j > 0 and val[j - 1] == '\\':
+ new_val += char
+ else:
+ new_val += '\\' + char
+ else:
+ new_val += char
+ cmd_args[i + 1] = new_val
+
+ has_b = any(arg == "-b" for arg in cmd_args)
+ if not has_b:
+ if args.devices:
+ if args.devices.isdigit():
+ n = int(args.devices)
+ device_val = ",".join(f"HTP{i}" for i in range(n))
+ else:
+ device_val = args.devices
+ elif "D" in os.environ:
+ device_val = os.environ["D"]
+ elif "DEVICE" in os.environ:
+ device_val = os.environ["DEVICE"]
+ else:
+ device_val = "HTP0"
+ if device_val:
+ cmd_args += ["-b", device_val]
+ else:
+ has_device = any(arg.startswith("--device") for arg in cmd_args)
+ if not has_device:
+ if args.devices:
+ if args.devices.isdigit():
+ n = int(args.devices)
+ device_val = ",".join(f"HTP{i}" for i in range(n))
+ else:
+ device_val = args.devices
+ elif "D" in os.environ:
+ device_val = os.environ["D"]
+ elif "DEVICE" in os.environ:
+ device_val = os.environ["DEVICE"]
+ else:
+ device_val = "HTP0"
+ if device_val:
+ cmd_args += ["--device", device_val]
+
+ # Automatically add -v to known llama tools if sched-debug, verbose, or profile are set
+ verbose_trigger = (
+ args.sched_debug
+ or args.verbose is not None
+ or args.profile is not None
+ or args.hex_verbose is not None
+ or args.hex_profile is not None
+ or args.hex_optrace is not None
+ )
+ if verbose_trigger and basename in ("llama-cli", "llama-completion", "llama-bench", "llama-server", "llama-mtmd-cli"):
+ if "-v" not in cmd_args and "--verbose" not in cmd_args:
+ cmd_args.append("-v")
+
+ # Inject defaults for llama-cli, llama-completion, and llama-server if not overridden by the user
+ if basename in ("llama-cli", "llama-completion", "llama-server"):
+ if "-ngl" not in cmd_args and "--n-gpu-layers" not in cmd_args:
+ cmd_args += ["-ngl", "99"]
+ if "--ubatch-size" not in cmd_args and "-ub" not in cmd_args:
+ cmd_args += ["--ubatch-size", "1024"]
+ if "-fa" not in cmd_args and "--flash-attn" not in cmd_args:
+ cmd_args += ["-fa", "on"]
+
+ if basename in ("llama-cli", "llama-completion", "llama-server", "llama-bench"):
+ if "-t" not in cmd_args and "--threads" not in cmd_args:
+ cmd_args += ["-t", "6"]
+
+ # Resolve target directory on device
+ target_dir = args.target_dir
+ if not target_dir:
+ target_dir = "/data/local/tmp/llama.cpp" if target_type == "android" else "~/llama.cpp"
+
+ if target_type == "android":
+ # Run via ADB
+ adb_base = ["adb"]
+ if target_val: # serial
+ adb_base += ["-s", target_val]
+
+ env_parts = [
+ "LD_LIBRARY_PATH=./lib",
+ "ADSP_LIBRARY_PATH=./lib"
+ ]
+ for k, v in env_vars.items():
+ env_parts.append(f"{k}={v}")
+ env_str = " ".join(env_parts)
+
+ cmd_str = shlex_join(cmd_args)
+ adb_shell_cmd = f"cd {target_dir} && ulimit -c unlimited && {env_str} {cmd_str}"
+ full_cmd = adb_base + ["shell", adb_shell_cmd]
+
+ logger.info(f"+ {' '.join(full_cmd)}")
+ res = subprocess.run(full_cmd)
+ sys.exit(res.returncode)
+
+ elif target_type == "linux":
+ ssh_host = target_val
+ if not ssh_host:
+ logger.error("Error: SSH host not specified in target (e.g. use linux:user@host, lnx:user@host, or ubuntu:user@host). Cannot execute.")
+ sys.exit(1)
+
+ # Linux remote run via SSH
+ env_parts = [
+ "LD_LIBRARY_PATH=./lib",
+ "ADSP_LIBRARY_PATH=./lib"
+ ]
+ for k, v in env_vars.items():
+ env_parts.append(f"{k}={v}")
+ env_str = " ".join(env_parts)
+
+ cmd_str = shlex_join(cmd_args)
+ ssh_shell_cmd = f"cd {target_dir} && ulimit -c unlimited && {env_str} {cmd_str}"
+ full_cmd = ["ssh", ssh_host, ssh_shell_cmd]
+
+ logger.info(f"+ {' '.join(full_cmd)}")
+ res = subprocess.run(full_cmd)
+ sys.exit(res.returncode)
+
+ elif target_type == "windows":
+ logger.info("Windows target execution is currently a stub.")
+ sys.exit(0)
+
+ else:
+ # Run locally
+ local_env = os.environ.copy()
+ lib_dir = os.path.normpath(os.path.join(package_path, "lib"))
+ local_env["ADSP_LIBRARY_PATH"] = lib_dir
+ if platform.system() == "Windows":
+ local_env["PATH"] = lib_dir + os.path.pathsep + local_env.get("PATH", "")
+ else:
+ local_env["LD_LIBRARY_PATH"] = lib_dir + os.path.pathsep + local_env.get("LD_LIBRARY_PATH", "")
+
+ for k, v in env_vars.items():
+ local_env[k] = v
+
+ logger.info(f"+ {shlex_join(cmd_args)}")
+ res = subprocess.run(cmd_args, env=local_env)
+ sys.exit(res.returncode)
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except KeyboardInterrupt:
+ logger.info("\nInterrupted by user.")
+ sys.exit(130)
diff --git a/scripts/snapdragon/windows/run-bench.ps1 b/scripts/snapdragon/windows/run-bench.ps1
deleted file mode 100644
index 6eb656e66d..0000000000
--- a/scripts/snapdragon/windows/run-bench.ps1
+++ /dev/null
@@ -1,48 +0,0 @@
-
-#!/usr/bin/env pwsh
-
-# Basedir on device
-$basedir=".\pkg-snapdragon"
-
-$cli_opts=$args
-
-$model="Llama-3.2-3B-Instruct-Q4_0.gguf"
-if ($null -ne $env:M) {
- $model=$env:M
-}
-
-$device="HTP0"
-if ($null -ne $env:D) {
- $device=$env:D
-}
-
-if ($null -ne $env:V) {
- $env:GGML_HEXAGON_VERBOSE=$env:V
-}
-
-if ($null -ne $env:PROF) {
- $env:GGML_HEXAGON_PROFILE=$env:PROF
-}
-
-if ($null -ne $env:OPSTAGE) {
- $env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE
-}
-
-if ($null -ne $env:NHVX) {
- $env:GGML_HEXAGON_NHVX=$env:NHVX
-}
-
-if ($null -ne $env:NDEV) {
- $env:GGML_HEXAGON_NDEV=$env:NDEV
-}
-
-if ($null -ne $env:HB) {
- $env:GGML_HEXAGON_HOSTBUF=$env:HB
-}
-
-$env:ADSP_LIBRARY_PATH="$basedir\lib"
-
-& "$basedir\bin\llama-bench.exe" `
- --load-mode none -m $basedir\..\..\gguf\$model `
- --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 `
- --ubatch-size 1024 -ngl 99 --device $device $cli_opts
diff --git a/scripts/snapdragon/windows/run-cli.ps1 b/scripts/snapdragon/windows/run-cli.ps1
deleted file mode 100644
index 5da8bff33e..0000000000
--- a/scripts/snapdragon/windows/run-cli.ps1
+++ /dev/null
@@ -1,53 +0,0 @@
-
-#!/usr/bin/env pwsh
-
-# Basedir on device
-$basedir=".\pkg-snapdragon"
-
-$cli_opts=$args
-
-$model="Llama-3.2-3B-Instruct-Q4_0.gguf"
-if ($null -ne $env:M) {
- $model=$env:M
-}
-
-$device="HTP0"
-if ($null -ne $env:D) {
- $device=$env:D
-}
-
-if ($null -ne $env:V) {
- $env:GGML_HEXAGON_VERBOSE=$env:V
-}
-
-if ($null -ne $env:SCHED) {
- $env:GGML_SCHED_DEBUG=$env:SCHED; $cli_opts="$cli_opts -v"
-}
-
-if ($null -ne $env:PROF) {
- $env:GGML_HEXAGON_PROFILE=$env:PROF
-}
-
-if ($null -ne $env:OPSTAGE) {
- $env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE
-}
-
-if ($null -ne $env:NHVX) {
- $env:GGML_HEXAGON_NHVX=$env:NHVX
-}
-
-if ($null -ne $env:NDEV) {
- $env:GGML_HEXAGON_NDEV=$env:NDEV
-}
-
-if ($null -ne $env:HB) {
- $env:GGML_HEXAGON_HOSTBUF=$env:HB
-}
-
-$env:ADSP_LIBRARY_PATH="$basedir\lib"
-
-& "$basedir\bin\llama-cli.exe" `
- --load-mode none -m $basedir\..\..\gguf\$model `
- --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 `
- --ctx-size 8192 --ubatch-size 1024 -fa on `
- -ngl 99 --device $device $cli_opts
diff --git a/scripts/snapdragon/windows/run-completion.ps1 b/scripts/snapdragon/windows/run-completion.ps1
deleted file mode 100644
index 08ef139b7e..0000000000
--- a/scripts/snapdragon/windows/run-completion.ps1
+++ /dev/null
@@ -1,53 +0,0 @@
-
-#!/usr/bin/env pwsh
-
-# Basedir on device
-$basedir=".\pkg-snapdragon"
-
-$cli_opts=$args
-
-$model="Llama-3.2-3B-Instruct-Q4_0.gguf"
-if ($null -ne $env:M) {
- $model=$env:M
-}
-
-$device="HTP0"
-if ($null -ne $env:D) {
- $device=$env:D
-}
-
-if ($null -ne $env:V) {
- $env:GGML_HEXAGON_VERBOSE=$env:V
-}
-
-if ($null -ne $env:SCHED) {
- $env:GGML_SCHED_DEBUG=$env:SCHED; $cli_opts="$cli_opts -v"
-}
-
-if ($null -ne $env:PROF) {
- $env:GGML_HEXAGON_PROFILE=$env:PROF
-}
-
-if ($null -ne $env:OPSTAGE) {
- $env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE
-}
-
-if ($null -ne $env:NHVX) {
- $env:GGML_HEXAGON_NHVX=$env:NHVX
-}
-
-if ($null -ne $env:NDEV) {
- $env:GGML_HEXAGON_NDEV=$env:NDEV
-}
-
-if ($null -ne $env:HB) {
- $env:GGML_HEXAGON_HOSTBUF=$env:HB
-}
-
-$env:ADSP_LIBRARY_PATH="$basedir\lib"
-
-& "$basedir\bin\llama-completion.exe" `
- --load-mode none -m $basedir\..\..\gguf\$model `
- --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 `
- --ctx-size 8192 --ubatch-size 1024 -fa on `
- -ngl 99 -no-cnv --device $device $cli_opts
diff --git a/scripts/snapdragon/windows/run-mtmd.ps1 b/scripts/snapdragon/windows/run-mtmd.ps1
deleted file mode 100644
index 6e270ec90b..0000000000
--- a/scripts/snapdragon/windows/run-mtmd.ps1
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/usr/bin/env pwsh
-
-# Basedir on device
-$basedir=".\pkg-snapdragon"
-
-$cli_opts=$args
-
-$model="gemma-3-4b-it-Q4_0.gguf"
-if ($null -ne $env:M) {
- $model=$env:M
-}
-
-$mmproj="mmproj-F16.gguf"
-if ($null -ne $env:MMPROJ) {
- $mmproj=$env:MMPROJ
-}
-
-$image=""
-if ($null -ne $env:IMG) {
- $image=$env:IMG
-}
-
-$device="HTP0"
-if ($null -ne $env:D) {
- $device=$env:D
-}
-
-if ($null -ne $env:V) {
- $env:GGML_HEXAGON_VERBOSE=$env:V
-}
-
-if ($null -ne $env:SCHED) {
- $env:GGML_SCHED_DEBUG=$env:SCHED; $cli_opts="$cli_opts -v"
-}
-
-if ($null -ne $env:PROF) {
- $env:GGML_HEXAGON_PROFILE=$env:PROF
-}
-
-if ($null -ne $env:OPSTAGE) {
- $env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE
-}
-
-if ($null -ne $env:NHVX) {
- $env:GGML_HEXAGON_NHVX=$env:NHVX
-}
-
-if ($null -ne $env:NDEV) {
- $env:GGML_HEXAGON_NDEV=$env:NDEV
-}
-
-if ($null -ne $env:HB) {
- $env:GGML_HEXAGON_HOSTBUF=$env:HB
-}
-
-if ($null -ne $env:MTMD_DEVICE) {
- $env:MTMD_BACKEND_DEVICE=$env:MTMD_DEVICE
-}
-
-$env:ADSP_LIBRARY_PATH="$basedir\lib"
-
-& "$basedir\bin\llama-mtmd-cli.exe" `
- --load-mode none -m $basedir\..\..\gguf\$model `
- --mmproj $basedir\..\..\gguf\$mmproj `
- --image $basedir\..\..\gguf\$image `
- --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 `
- --ctx-size 8192 --ubatch-size 1024 -fa on `
- -ngl 99 --device $device -v $cli_opts
diff --git a/scripts/snapdragon/windows/run-tool.ps1 b/scripts/snapdragon/windows/run-tool.ps1
deleted file mode 100644
index 15c880f2db..0000000000
--- a/scripts/snapdragon/windows/run-tool.ps1
+++ /dev/null
@@ -1,56 +0,0 @@
-
-#!/usr/bin/env pwsh
-
-# Basedir on device
-$basedir=".\pkg-snapdragon"
-
-if ($args.Count -eq 0) {
- Write-Host "No arguments provided.Expected the tool and argument to run."
- exit -1
-}
-
-$tool=$args[0]
-$cli_opts=@()
-
-if ($args.Count -gt 1) {
- $cli_opts=$args[1..($args.Count - 1)]
- $remainingArgs = $args[1..($args.Count - 1)]
-}
-
-$device="HTP0"
-if ($null -ne $env:D) {
- $device=$env:D
-}
-
-if ($null -ne $env:V) {
- $env:GGML_HEXAGON_VERBOSE=$env:V
-}
-
-if ($null -ne $env:SCHED) {
- $env:GGML_SCHED_DEBUG=$env:SCHED; $cli_opts="$cli_opts -v"
-}
-
-if ($null -ne $env:PROF) {
- $env:GGML_HEXAGON_PROFILE=$env:PROF
-}
-
-if ($null -ne $env:OPSTAGE) {
- $env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE
-}
-
-if ($null -ne $env:NHVX) {
- $env:GGML_HEXAGON_NHVX=$env:NHVX
-}
-
-if ($null -ne $env:NDEV) {
- $env:GGML_HEXAGON_NDEV=$env:NDEV
-}
-
-if ($null -ne $env:HB) {
- $env:GGML_HEXAGON_HOSTBUF=$env:HB
-}
-
-$env:ADSP_LIBRARY_PATH="$basedir\lib"
-
-& "$basedir\bin\$tool" `
- $cli_opts
diff --git a/scripts/snapdragon/windows/setup-build.ps1 b/scripts/snapdragon/windows/setup-build.ps1
deleted file mode 100644
index d8ef24d441..0000000000
--- a/scripts/snapdragon/windows/setup-build.ps1
+++ /dev/null
@@ -1,105 +0,0 @@
-# Requires Run as Administrator is NOT strictly necessary for User-scope env vars,
-# but recommended for creating directories in C:\ root if permissions are restricted.
-
-$ErrorActionPreference = "Stop"
-
-# --- Configuration ---
-$BaseDir = "C:\Qualcomm"
-
-# SDK 1: Hexagon
-$HexagonUrl = "https://github.com/snapdragon-toolchain/hexagon-sdk/releases/download/v6.6.0.0/hexagon-sdk-v6.6.0.0-arm64-wos.tar.xz"
-$HexagonParent = Join-Path $BaseDir "Hexagon_SDK"
-$HexagonSdkVersion = "6.6.0.0"
-$HexagonToolsVersion = "19.0.07"
-$HexagonSdkTarget = Join-Path $HexagonParent $HexagonSdkVersion
-$HexagonToolsTarget = Join-Path $HexagonSdkTarget "\tools\HEXAGON_Tools\$HexagonToolsVersion"
-
-# SDK 2: OpenCL
-$OpenCLUrl = "https://github.com/snapdragon-toolchain/opencl-sdk/releases/download/v2.3.2/adreno-opencl-sdk-v2.3.2-arm64-wos.tar.xz"
-$OpenCLParent = Join-Path $BaseDir "OpenCL_SDK"
-$OpenCLVersion = "2.3.2"
-$OpenCLTarget = Join-Path $OpenCLParent $OpenCLVersion
-
-# --- Helper Function ---
-function Install-QualcommSDK {
- param (
- [string]$Url,
- [string]$ParentDir,
- [string]$TargetDir,
- [string]$Name
- )
-
- # 1. Create Parent Directory
- if (-not (Test-Path -Path $ParentDir)) {
- Write-Host "Creating directory: $ParentDir" -ForegroundColor Cyan
- New-Item -Path $ParentDir -ItemType Directory -Force | Out-Null
- }
-
- # 2. Check for Specific Version Directory
- if (Test-Path -Path $TargetDir) {
- Write-Host "$Name ($TargetDir) already exists. Skipping download." -ForegroundColor Green
- }
- else {
- Write-Host "$Name not found. preparing to download..." -ForegroundColor Yellow
-
- # Create the target directory to extract into
- New-Item -Path $TargetDir -ItemType Directory -Force | Out-Null
-
- # Define temporary archive path
- $TempFile = Join-Path $ParentDir "temp_sdk.tar.xz"
-
- try {
- # Download
- Write-Host "Downloading from: $Url"
- Invoke-WebRequest -Uri $Url -OutFile $TempFile
-
- # Untar
- # Note: We assume Windows includes tar.exe (Win 10 build 17063+)
- Write-Host "Extracting archive to $TargetDir..."
-
- # We use -C to extract contents INTO the target directory created above
- tar -xJvf $TempFile -C $TargetDir\..
-
- Write-Host "Extraction complete." -ForegroundColor Green
- }
- catch {
- Write-Error "Failed to download or extract $Name. Error: $_"
- # Cleanup target dir if failed so script tries again next time
- Remove-Item -Path $TargetDir -Recurse -Force -ErrorAction SilentlyContinue
- }
- finally {
- # Cleanup Archive
- if (Test-Path $TempFile) { Remove-Item $TempFile -Force }
- }
- }
-}
-
-# --- Execution ---
-
-# 1. Ensure Base C:\Qualcomm exists
-if (-not (Test-Path $BaseDir)) {
- New-Item -Path $BaseDir -ItemType Directory -Force | Out-Null
-}
-
-# 2. Run Install Logic
-Install-QualcommSDK -Url $HexagonUrl -ParentDir $HexagonParent -TargetDir $HexagonSdkTarget -Name "Hexagon SDK"
-Install-QualcommSDK -Url $OpenCLUrl -ParentDir $OpenCLParent -TargetDir $OpenCLTarget -Name "OpenCL SDK"
-
-# --- Environment Variables ---
-
-Write-Host "`nSetting Environment Variables..." -ForegroundColor Cyan
-
-# Set OPENCL_SDK_ROOT
-[System.Environment]::SetEnvironmentVariable('OPENCL_SDK_ROOT', $OpenCLTarget, [System.EnvironmentVariableTarget]::User)
-$env:OPENCL_SDK_ROOT = $OpenCLTarget # Set for current session as well
-Write-Host "OPENCL_SDK_ROOT set to: $OpenCLTarget"
-
-# Set HEXAGON_SDK_ROOT
-[System.Environment]::SetEnvironmentVariable('HEXAGON_SDK_ROOT', $HexagonSdkTarget, [System.EnvironmentVariableTarget]::User)
-$env:HEXAGON_SDK_ROOT = $HexagonSdkTarget # Set for current session as well
-Write-Host "HEXAGON_SDK_ROOT set to: $HexagonSdkTarget"
-
-# Set HEXAGON_SDK_ROOT
-[System.Environment]::SetEnvironmentVariable('HEXAGON_TOOLS_ROOT', $HexagonToolsTarget, [System.EnvironmentVariableTarget]::User)
-$env:HEXAGON_TOOLS_ROOT = $HexagonToolsTarget # Set for current session as well
-Write-Host "HEXAGON_TOOLS_ROOT set to: $HexagonToolsTarget"
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 53e93a1448..b1d86a4e21 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -2469,8 +2469,13 @@ struct test_set_rows : public test_case {
// See dicussion here: https://github.com/ggml-org/llama.cpp/pull/23760#issuecomment-4566312209
double max_nmse_err(ggml_backend_t backend) override {
ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
- if (type_dst == GGML_TYPE_Q8_0 && strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) {
- return std::max(test_case::max_nmse_err(backend), 2e-7);
+ if (type_dst == GGML_TYPE_Q8_0) {
+ if (strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) {
+ return std::max(test_case::max_nmse_err(backend), 2e-7);
+ }
+ if (strcmp(ggml_backend_reg_name(reg), "HTP") == 0) {
+ return std::max(test_case::max_nmse_err(backend), 5e-6);
+ }
}
return test_case::max_nmse_err(backend);
}