37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
Develop 2026-10-25: a whole model on a card with memory of its own, a prompt on the CPU as tiles - on a discrete GPU every pass whole, one submission each, a mixture's experts in a tier of the card's memory and the CPU computing the others, the plan on the GPU when the card holds them all; every fma() of the shaders one operation (tools/spv_fma.c); the GPU checked when it opens; on the CPU a prompt's products as tiles of eight rows for every weight type (Qwen3-4B Q4_K_M 57 to 79 tokens/s with AVX2, 71 to 103 with AVX-VNNI; Qwen3.8-27B UD-Q4_K_XL 9.0 to 15.6), rows ending inside a group of 256 by rows; the same bits
camauridbc9a72a07be
diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 4a19a94..67dae2e 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -27,6 +27,7 @@ env:VULKAN_HEADERS_TAG: vulkan-sdk-1.4.363.0VULKAN_HEADERS_COMMIT: 6802bb4733b63ed5efd3adb308a6c885ef180ea1PACKAGE: janas-linux-x86_64+ DIAG: janas-diag-linux-x86_64jobs:linux-x86_64:@@ -111,6 +112,23 @@ jobs:sha256sum "$PACKAGE.tar.gz" > "$PACKAGE.tar.gz.sha256"cat "$PACKAGE.tar.gz.sha256"+ - name: Pack the diagnostics+ run: |+ set -euo pipefail+ # vkcheck and gpu_check alone, for a tester who only has to tell+ # whether the GPU works: no model, no engine, a few megabytes+ B=bin/x86_64-linux+ D="$DIAG/janas-diag"+ mkdir -p "$D"+ install -m 755 "$B/vkcheck" "$B/gpu_check" "$D/"+ install -m 644 dist/README-diag.md "$D/README.md"+ install -m 644 LICENSE "$D/"+ (cd "$D" && find . -type f ! -name SHA256SUMS -print0 | sort -z |+ xargs -0 sha256sum > SHA256SUMS)+ tar -C "$DIAG" -czf "$DIAG.tar.gz" janas-diag+ sha256sum "$DIAG.tar.gz" > "$DIAG.tar.gz.sha256"+ cat "$DIAG.tar.gz.sha256"+- name: Keep the package (run by hand)if: github.event_name == 'workflow_dispatch'uses: actions/upload-artifact@v7@@ -119,6 +137,8 @@ jobs:path: |${{ env.PACKAGE }}.tar.gz${{ env.PACKAGE }}.tar.gz.sha256+ ${{ env.DIAG }}.tar.gz+ ${{ env.DIAG }}.tar.gz.sha256- name: Publish the release (a tag)if: startsWith(github.ref, 'refs/tags/v')@@ -127,8 +147,9 @@ jobs:run: |gh release create "$GITHUB_REF_NAME" \"$PACKAGE.tar.gz" "$PACKAGE.tar.gz.sha256" \+ "$DIAG.tar.gz" "$DIAG.tar.gz.sha256" \--title "Janas $GITHUB_REF_NAME" \- --notes "Janas for Linux x86-64 (Ubuntu 22.04, Debian 12 and later): janas-linux-x86_64.tar.gz; for Windows 10 and 11 x86-64: janas-windows-x86_64.zip. Unpack it and start: janas-get list, then janas-chat <model>. What changed: ChangeLog.md. The SHA-256 of every file: SHA256SUMS in the package."+ --notes "Janas for Linux x86-64 (Ubuntu 22.04, Debian 12 and later): janas-linux-x86_64.tar.gz; for Windows 10 and 11 x86-64: janas-windows-x86_64.zip. Only the GPU checks (vkcheck, gpu_check), to tell whether a GPU computes what the CPU does: janas-diag-linux-x86_64.tar.gz, janas-diag-windows-x86_64.zip. Unpack it and start: janas-get list, then janas-chat <model>. What changed: ChangeLog.md. The SHA-256 of every file: SHA256SUMS in the package."# Windows x86-64: the same sources built with MinGW-w64 on Linux (a# newer Ubuntu: the C library does not matter for Windows, the MinGW@@ -208,6 +229,7 @@ jobs:timeout-minutes: 20env:WPACKAGE: janas-windows-x86_64+ WDIAG: janas-diag-windows-x86_64steps:- uses: actions/checkout@v7@@ -248,6 +270,17 @@ jobs:(cd "$WPACKAGE" && zip -q -r -X "../$WPACKAGE.zip" janas)sha256sum "$WPACKAGE.zip" > "$WPACKAGE.zip.sha256"cat "$WPACKAGE.zip.sha256"+ # the diagnostics alone, as for Linux+ D="$WDIAG/janas-diag"+ mkdir -p "$D"+ install -m 755 "$B/vkcheck.exe" "$B/gpu_check.exe" "$D/"+ install -m 644 dist/README-diag.md "$D/README.md"+ install -m 644 LICENSE "$D/"+ (cd "$D" && find . -type f ! -name SHA256SUMS -print0 | sort -z |+ xargs -0 sha256sum > SHA256SUMS)+ (cd "$WDIAG" && zip -q -r -X "../$WDIAG.zip" janas-diag)+ sha256sum "$WDIAG.zip" > "$WDIAG.zip.sha256"+ cat "$WDIAG.zip.sha256"- name: Keep the package (run by hand)if: github.event_name == 'workflow_dispatch'@@ -257,6 +290,8 @@ jobs:path: |${{ env.WPACKAGE }}.zip${{ env.WPACKAGE }}.zip.sha256+ ${{ env.WDIAG }}.zip+ ${{ env.WDIAG }}.zip.sha256- name: Add it to the release (a tag)if: startsWith(github.ref, 'refs/tags/v')@@ -264,7 +299,8 @@ jobs:GH_TOKEN: ${{ github.token }}run: |gh release upload "$GITHUB_REF_NAME" \- "$WPACKAGE.zip" "$WPACKAGE.zip.sha256"+ "$WPACKAGE.zip" "$WPACKAGE.zip.sha256" \+ "$WDIAG.zip" "$WDIAG.zip.sha256"# The wiki, made from the same pages as the repository: README.md becomes# Home, each docs/*.md a page named after its title. It is never editeddiff --git a/ChangeLog.md b/ChangeLog.mdindex 3965b16..2781f11 100644--- a/ChangeLog.md+++ b/ChangeLog.md@@ -2,6 +2,47 @@> Curated, user-facing summary of completed work, newest first.+## [2026-10-24] - A card with memory of its own runs the whole model++- **Every pass whole on a discrete GPU**: on a card with memory of its own the model's passes - a prompt's blocks, a chain of drafts to verify, single tokens - go whole to the GPU, one submission each, where before every product's rows were shared with the CPU and the activations crossed the bus at each one (an Arc B580 read a prompt at 15.8 tokens/s, issue #43). The token's state - each attention layer's KV rows and scales, the Gated DeltaNet states and their log - is copied to the card's memory and has one owner at a time: the GPU after its pass, the CPU after its own, copied over only when the passes change hands (`janas_gpu_sync`), only the rows the GPU's layers use, so that an MTP block's and Gemma 4's assistant's own stay the CPU's. `JANAS_GPU_WHOLE=0` shares the products as before. Checked as a card of its own on the development laptop's Arc (`JANAS_GPU_COPY=1`), functional runs only: Qwen3.5-2B, Qwen3-4B and Qwen3.5-9B greedy, with and without drafts the same tokens, the text sensible; Qwen3.5-2B's 64 tokens 141 single tokens, 27 draft checks and 2 blocks whole on the GPU, the state sent three times and never read back; a prompt on the GPU and its tokens on the CPU (`JANAS_GPU_WHOLE=0`) right too. As on an integrated GPU, the whole pass's norms and attention sum in the GPU's order, so the text is not the CPU's to the bit. - **The whole pass's working buffers in the card's own memory**: on a card with memory of its own every intermediate of a pass is in its memory, where the shaders read and write it at its speed, and only what the host writes or reads - a block's inputs and position, the parameters once, the logits and hidden states - goes through a host-visible mirror (`janas_gt_sync_range` copies those ranges); before, every layer's activations crossed the bus. The same tokens to the bit as with the buffer in host memory (Qwen3.5-2B, Qwen3-4B, Qwen3.5-9B, as a card of its own; a prompt on the GPU and its tokens on the CPU too). Mixtures too: their routing, read and written by the host between layers, goes through the mirror as the rest; Qwen3-30B-A3B's prompt blocks whole on the GPU gave the same tokens through the mirror as without, also under the validation layer (`JANAS_GPU_TOK_MIRROR=1` puts the buffer in the GPU's memory behind a mirror on an integrated GPU too, to test it with the weights read in place; the verbose line says where the buffer is). `JANAS_GPU_TOK_HOSTBUF=1` keeps the host-visible buffer for any model. The staging buffers and a card's copies of the weights now declare the copies they take part in (transfer usage), which a driver may enforce. Not yet: mixtures larger than the card, Qwen3.8-Flash-Next on a card of its own.+- **A dense model's pass in one submission with its copies**: the copies of a pass's inputs (the embeddings, the rotations, the position words) to the card and of its logits and hidden states back to the mirror are recorded in the pass's own commands, with their barriers, instead of four submissions of their own around it, each waited for; the same tokens to the bit (Qwen3.5-2B with and without MTP drafts, Qwen3-4B against the host-visible buffer). Under the Khronos validation layer with synchronization checks, gpu_check on every type and Qwen3.5-2B's generation with drafts, on the shared memory and as a card of its own: no error.+- **A mixture larger than the card: a tier of experts in its memory, the CPU computing the others**: on a card with memory of its own a mixture's whole pass no longer needs every expert copied to the card. It keeps a tier of slots in the card's memory - the room it has left, 85% of its memory less what Janas took and 1 GiB for the rest (`JANAS_GPU_TIER_MB=<n>` asks for n MiB), in pieces of 1 GiB that any driver allocates - and between two layers, where the CPU already sorts the router's choice, each chosen expert is either in the tier, or read into it in place of the one used least among sixteen (never one this layer counts on), or computed by the CPU from the layer's normalized input into its own rows, which the GPU then sums with the others in the router's order (`janas_m_expert_rows`, the CPU's expert kernels). Only the experts not in the tier are asked of the cache, so a tier hit never reads the disk. The arena is not copied to such a card any more, so the tier has its memory (before, a cache holding every expert was copied whole, and a copy that did not fit could leave pieces behind). `JANAS_GPU_TIER=0` turns the tier off. Checked as a card of its own on the development laptop's Arc, functional runs only: Qwen3-30B-A3B UD-Q3_K_XL with a tier of 1 GiB (480 of its 6,144 experts) and of 300 MB (140, a slot given up at nearly every layer) - sensible text, every pass whole on the GPU, no error under the validation layer with synchronization checks. The text is not the same to the bit with every tier: the two tiers' first 15 tokens are the same, then they part, as the run with drafts parts from the one without after 8-9 tokens - an expert computed by the CPU or by the GPU rounds differently (the GPU's expert products sum in their own order), and which computes it depends on what the tier holds.+- **Every fma() of the shaders one operation, as on the CPU: a mixture's tier no longer changes its answer**: glslang decorates a `precise` variable's multiplies and adds NoContraction but not its `fma()` calls, and GLSL.std.450's Fma without it may be done as a multiply and an add, two roundings - the Arc's driver did split some: the experts' activation (`expert_act.comp`, its e^x and reciprocal) came out a unit in the last place off the CPU's in about 1.5% of its values, which moved a block's scale in one expert in eighty. The shaders' SPIR-V now goes through `tools/spv_fma.c`, which decorates every Fma (built by `build.sh` with the build machine's compiler, `HOSTCC`, and writing the C array that `--vn` wrote). With that, and the CPU's experts quantizing the GPU's input with the same deterministic operations (`janas_q8k_quantize_det`), an expert computed by the CPU gives the GPU's bits: Qwen3-30B-A3B's logits on 160 tokens are the same to the bit with a tier of 300 MB, 1 GiB and every expert. Found with a probe that recomputed each GPU expert on the CPU (input, gate and up the same to the bit, the activation not). Every block's experts now go through the matvecs by default (`JANAS_GPU_MVID=16` as before: a prompt's blocks in `*_mmid`'s tiles), Q6_K_P's too (`q6kp_matvec.comp` with `ID`, its planes and level from the table): a prompt's GPU experts went through tiles that are not the CPU's bits and its other experts through the CPU, so the tokens with drafts and without parted after some ten (Qwen3-30B-A3B with a tier of 300 MB at the 11th, Qwen3.6-35B-A3B with one of 1 GiB at the 40th); now they are the same, 48 of 48 on both. The prompt as a card of its own on the laptop's Arc: 7% faster with every expert in the tier, 3% slower with 300 MB.+- **A mixture's whole pass on a card of its own 2.2 times as fast on a single token**: measured as a card of its own on the laptop's Arc (Qwen3-30B-A3B UD-Q3_K_XL, every expert in the tier, 160 tokens of fixed text one at a time, the second half): from 4.7 to 10.5 tokens/s; with a tier of 300 MB, the other experts on the CPU, from 3.1 to 8.5; Qwen3.6-35B-A3B Q4_K_M with a tier of 1 GiB 7.0 tokens/s. Four causes, found with the GPU's own timestamps: (1) between two layers the CPU's plan was read and written in six or seven copies of their own, each a submission waited for - now the choice of experts (and, with a tier, the experts' input) goes to the mirror at the end of a layer's commands and the tables come back at the start of the next one's, and the experts read into the tier and the CPU's rows go in one submission together (`janas_g_copies`); (2) the router's product went through `dense_mm`'s tiles of 64 rows by 64 vectors, three workgroups on a token - now `dense_mv.comp`, a workgroup a row, on up to 8 vectors (also the DeltaNet's beta and alpha, and qwen4exp's mixes, these not tried: the whole pass refuses them on a card of its own); (3) `moe_topk.comp` had one invocation a token search 128 logits nine times - now a workgroup a token, each search a reduction; the router from 1.07 to 0.11 ms a layer; (4) the experts' products on a few vectors went through `*_mmid`'s tiles of 16 vectors, nearly empty on a token (9 GB/s) - now through the matvec shaders of Q3_K, Q4_K and Q6_K with a constant `ID` that reads the weights, inputs and outputs from the same table (blocks of 16 tokens or fewer, `JANAS_GPU_MVID=<n>`, 0 never): gate and up from 55 to 26 ms a token. Every step checked: logits the same to the bit before and after the copies were moved; the same agreement with the CPU (145-150 of 160 tokens' most likely token, as Qwen3-4B's whole pass, 154); no error under the validation layer with synchronization checks; prompts as fast or faster (512 tokens 111.9 to 112.9 tokens/s, 120.7 with every block through the matvecs). With a tier holding every expert the tokens are the same with drafts and without; with a partial one they part after some tokens, as before: the CPU's experts do not round as the GPU's (open).+- **Q2_K and the IQ types with grids as tiles** (IQ3_S, IQ3_XXS, IQ2_XXS, IQ2_XS, IQ2_S, IQ1_S, IQ1_M): through Q6_K's tiles, a group of 16 weights a scale; the IQ types' weights as signed bytes (level times sign; with AVX-VNNI plus 128, the offset taken off with the scales' sums of activations, with AVX2 by sign and abs), their scales 1 + 2 s, the block's d with its factor; Q2_K's values 0 to 3, its mins' sums apart. The same bits (`test_quant`'s 11 rows to 65 vectors with all eight, AVX2 and AVX-VNNI; mutations of the offset, of abs and of Q2_K's dmin caught). From 12 vectors; on 64 (12 threads) 1.6 to 2.6 times as fast: IQ1_M 0.6 to 1.6 GB/s with AVX-VNNI, IQ3_XXS 1.2 to 2.8, Q2_K 1.5 to 2.6; with AVX2 1.5 to 2.2 times.+- **Q2_0 as tiles, and rows that end inside a group of 256 by rows**: Q2_0's two-bit values (the weight plus 1) go through the tiles of the types in blocks of 32, a block of 64 two of them with its d in both, from 24 vectors with AVX-VNNI (on 64, 12 threads, 1.7 to 2.1 GB/s); with AVX2 its products on reordered activations stay, the tiles never faster. Rows that end inside a group of 256 - Qwen3.8-Flash-Next's down experts, 640 columns - no longer go one by one on many vectors: the whole groups of all the rows, then the tails padded 16 rows at a time, both through the rows' products and their tiles (`janas_dot_multi_rows_n`); the down matrix (2560 rows of 640, one core) 1.5 times as fast on 8 vectors, 2.4 on 64 with AVX-VNNI, 1.7 with AVX2. The same bits (`test_row_tails` on 33 vectors and 37 rows; mutations of Q2_0's offset and of the tails' sum caught).+- **IQ4_XS as tiles too**: its signed values from the IQ4 table transposed, a signed scale a 32-weight group; with AVX-VNNI the values plus 128 by vpdpbusd, a multiply by the scales, 128 times the scales' sums of activations taken off; with AVX2 sign and abs, the scales by madd; one fma a block as before, the same bits. From 12 vectors; on 64 with AVX-VNNI 2.4 times as fast (12 threads), with AVX2 39%. Qwen3.8-27B UD-Q4_K_XL (IQ4_XS where its rows are many), prompt of 256 tokens with every tile type, AVX-VNNI: 9.0 to 15.6 tokens/s. The tiles' scales are made with vectors (an 8 x 8 transpose of the rows' scale pairs): Q3_K's tiles 19% faster on 24 vectors.+- **The types in blocks of 32 as tiles too** (Q8_0, IQ4_NL, Q4_0, MXFP4, Q5_0, Q4_1, Q5_1): each block's values unpacked to bytes with an offset (Q4_0's 8 and Q5_0's 16 as stored, MXFP4's 12, Q8_0's and IQ4_NL's 128 with AVX-VNNI) taken off with the block's activation sum; with AVX2 the small values' maddubs summed in int16 (eight for Q4_0 and Q4_1, four for the others), Q8_0 and IQ4_NL by sign and abs; the floats theirs, a sum per block position added as janas_hsum8, the same bits. On 64 vectors with AVX2 (12 threads) from 28% (Q8_0, IQ4_NL) to 70% (MXFP4) faster; with AVX-VNNI, whose rows' products had no reduction tree, from 24 vectors, 8% (Q8_0) to 33% (Q5_1). Gemma-4-26B-A4B UD-Q4_K_M's prompt of 512 tokens with every tile type, AVX2: 45.3 to 58.4 tokens/s.+- **Q3_K as tiles too**: its three-bit values (the weight plus 4) through Q6_K's tiles, four maddubs summed in int16 before a madd (AVX2), the -4 folded as Q6_K's -32; the same bits. Qwen3-30B-A3B UD-Q3_K_XL (its experts Q3_K), prompt of 512 tokens: 57.9 to 67.5 tokens/s with AVX2, 69.0 to 78.6 with AVX-VNNI.+- **Q5_K as tiles too**: five-bit values unpacked to bytes and transposed, four maddubs summed in int16 and one madd by the scales, twice a 32-weight piece (AVX2), or eight vpdpbusd and one multiply (AVX-VNNI); the mins and floats Q4_K's, the same bits. Qwen3.5-9B Q4_K_M (its attention and recurrent projections Q5_K), prompt of 512 tokens with every tile type: 30.7 to 43.3 tokens/s with AVX2, 38.3 to 57.5 with AVX-VNNI.+- **Q6_K and Q6_K_P as tiles too**: the eight rows' six-bit values unpacked to bytes (from Q6_K, or from Q6_K_P's planes at its level) and transposed the same way, a 16-weight group four dwords: two maddubs summed in int16 and one madd by the scales (AVX2), or four vpdpbusd and one multiply (AVX-VNNI); the same bits (`test_q6k_tiles`: 11 rows, 9 blocks, 1 to 65 vectors, every level, also under Wine). With Q4_K's tiles, Qwen3-4B Q4_K_M's prompt of 512 tokens: 58.7 to 79.1 tokens/s with AVX2 (1.33 times llama.cpp's on the same machine), 73.0 to 103 with AVX-VNNI; Qwen3.5-9B Q4_K_M, the model of issue #45, 30.7 to 40.1 with AVX2. `JANAS_Q4K_GEMM` is now `JANAS_CPU_TILES`.+- **A prompt's Q4_K products on the CPU as tiles of eight rows**: on eight or more tokens the Q4_K rows are taken eight at a time, transposed four bytes at a time into a small buffer once for 64 tokens, and multiplied by tiles of tokens with every lane a row - a 32-weight piece eight maddubs summed in int16 and one madd by the scales (AVX2), or eight vpdpbusd and one multiply (AVX-VNNI), and no horizontal sums; the same integer sums and float operations, so the results are the other kernels' to the bit (`test_quant` on 1 to 65 vectors and 11 rows, also under Wine). Qwen3-4B Q4_K_M's prompt of 512 tokens on the laptop's CPU, 12 threads: 57.1 to 72.0 tokens/s with AVX2 (llama.cpp on the same machine: 56.0, 59.4 on 6 threads), 71.2 to 92.2 with AVX-VNNI; Qwen3-30B-A3B Q4_K_M 54.6 to 60.7 (AVX2) and 63.9 to 70.2 (AVX-VNNI). `JANAS_CPU_TILES=0` keeps the rows' products. `JANAS_ROW_PIECE_KB` (off by default, not measured where it may count) takes a chunk's rows in pieces that stay in the level-2 cache, for CPUs with little of it (issue #45).+- **A mixture whose experts the card holds all: its pass in one submission, the plan on the GPU**: when the tier has a slot for every expert, every one is read into it at the first pass (layer by layer, the next one read ahead) and `moe_plan.comp` writes after each router what the CPU's plan wrote between two submissions - each row's token, each pair's row, the products' tables and tiles, the dispatches' sizes - the same words in the same places; the 49 submissions a token, each waited for, become one. Qwen3-30B-A3B UD-Q3_K_XL as a card of its own on the laptop's Arc, every expert in the tier: one token at a time 10.5 to 13.3 tokens/s, blocks of 16 50.2 to 53.9; the logits the same to the bit as with the CPU's plan, token by token and in blocks; drafts and none the same tokens; no error under the validation layer. The time between segments, measured past the tier's filling, was 16 ms of 88 a token, 2 of them the CPU's plan and the rest the submissions' round trip. `JANAS_GPU_GPLAN=0` keeps the CPU's plan; a watched router (`janas_llm_model_route_observer`) keeps it too.+- **MXFP4 and Q2_0 experts the same bits on the CPU and the card**: on a card of its own a mixture's MXFP4 and Q2_0 experts go through `b32_matvec.comp`'s rows reading the plan's table, as Q3_K, Q4_K and Q6_K's already did, instead of `b32_mmid.comp`'s tiles; Qwen3.6-35B-A3B MXFP4_MOE as a card of its own on the laptop's Arc: the logits the same to the bit with tiers of 1 and 2 GiB, drafts and none the same tokens, no error under the validation layer, one token at a time 4.1 to 4.5 tokens/s with a tier of 1 GiB. Q2_0's rows of 640 take the tail of `b32_matvec.comp`; Qwen3.8-Flash-Next is not yet taken whole by a card of its own, so that path is compiled, not run.+- **The CPU's plan in the token profile**: `JANAS_GPU_TOKEN_PROFILE=1` prints the time of a mixture's plans on the CPU beside the time between two segments; on Qwen3-30B-A3B with every expert in the tier, past the tier's filling, 2.1 ms of the 16 ms a token between segments: the rest is the submissions' round trip.+- **The GPU's timestamps of a mixture's layers**: `JANAS_GPU_TOKEN_PROFILE=1` now times a mixture's segments too, with the router, the experts' sum and the time between two segments (the CPU's plan, the submission's round trip) as phases of their own.+- **The state read back right from a cached stage**: the Y stage may be cached and not coherent (it is on the laptop's Arc): a read-back is invalidated, an upload flushed - without it the CPU read stale state and wrote only token 0.+- `llm_gen`: `LLM_GEN_DUMP=<path>` writes the first run's tokens.++## [2026-10-24] - The GPU checked when it opens, and gpu_check diagnoses by itself++- **gpu_check finds the faulty shader by itself**: every product runs twice, all its rows on the GPU and then shared with the CPU as the engine does, and nothing stops at the first error (it stopped after ten, which on an Arc B580 were all one Q6_K shape). Every wrong product is counted under the shader that computed it (`janas_gpu_last_shader`: name, vectors a workgroup, subgroup size), with its worst difference in units in the last place - a few: the driver rounds a float operation differently; many: wrong values - and the first wrong row with both values in hexadecimal (the printed ones looked equal on a Radeon 890M). Then it runs each wrong product again with the other shaders the engine could take for it (`janas_gpu_set_path`: the subgroup-per-row shaders and the ones with the lanes on the vectors off) and with every other subgroup size the device allows, and prints a summary to paste into an issue. The device line names the driver, its version and the float controls. `GPU_CHECK_TYPES=` checks some types only.+- **The engine checks the GPU when it opens it**: a product of every type the GPU takes, on the vector counts that pick each of its shaders, against the CPU's, bit for bit (small random matrices, 110 ms on the Arc of the development laptop, 270 ms as a card of its own). A shader this driver gets wrong is left out - the subgroup-per-row ones, the ones with the lanes on the vectors, q4k_mv, each family with another shader for the same products - and a type still wrong without them goes to the CPU; the status line says so (`GPU present (active, ...; the subgroup-per-row shaders (wrong on this driver))`). On the Arc B580 of issue #43 a Q6_K shader gave text with no sense; with the check the answers would be right from the first run. The result is kept in `~/.cache/janas/gpu_selfcheck` under the device, its driver's version, the subgroup size and Janas's version: once per driver. `JANAS_GPU_SELFCHECK=0` leaves the check out, `=2` runs it again. Tested with a Q6_K row shader broken on purpose at subgroups of 32, as a card of its own: found in 312 ms and left out, and the next opening read the decision from the cache.+- **`janas-diag` packages**: `vkcheck` and `gpu_check` alone, without the engine and with a page of their own, for Linux and Windows in every release (under 1 MB on Linux).+- **The sums of a row's tail kept apart from its products on any GPU**: in `q8_0_matvec.comp` and `b32_matvec.comp` the accumulators and their sums are `precise`, so that no compiler may fuse a sum with the product before it - the CPU rounds both - where a tail's accumulators restart from zero inside the loop (Q8_0 2560x640 one bit off on a Radeon 890M, issue #46, the likely cause). The same bits on the Arc of the development laptop, in place and as a card of its own, subgroups of 8 and 32.++## [2026-10-24] - Faster kernels on the CPU: Qwen3.8-Flash-Next's Q2_0 experts, IQ2 and IQ1, Q2_K, prompts of Q4_K and of the types in blocks of 32++- **A reply 13% faster, a prompt 7.5% faster, the same bits**: on the CPU (Core Ultra 9 185H, 12 threads, a 12 GiB expert cache), 1,024 tokens of Italian prose and then 200 generated, two runs each alternated with the release of 24 October, a reply went from 10.7-11.0 to 12.3-12.5 tokens/s and the prompt from 24.2 to 26.1; the experts of a reply from 30 ms a token to 19.8.+- **Q2_0 rows against activations reordered once for all the rows**: a Q2_0 byte holds four weights one apart, so each row was unpacked into the activations' order with shuffles, and the eight sums of 32 of each group of 256 reduced with six hadd. Now the activations are laid out as the weights' bit planes once per call (`q2_0_xperm`), each row's 16 bytes become two registers with two variable shifts and two ands, and two hadd are enough. One vector: 7.7 GB/s on a core and 34 on twelve threads before, 19 and 69 now; two, four and eight vectors (a prompt's experts, `janas_dot_multi_rows`): 1.8, 1.7 and 1.5 times as fast. Sums of integers in another order and the same float operations: equal to the scalar reference to the bit.+- **The IQ2 and IQ1 kernels 2.5 to 4.5 times as fast, Q6_K's on 2-8 vectors 8-12%**: the loops over a group's four sub-groups (IQ2_XXS/XS/S, IQ1_S/M) and over Q6_K's two halves were left rolled by GCC, their grid entries, signs and scales kept in memory; unrolled, one vector on a core went from 1.0 to 3.0 GB/s (IQ2_XXS), 1.2 to 5.4 (IQ2_S), 0.9 to 3.0 (IQ1_S), 0.6 to 2.3 (IQ1_M), and on twelve threads from 7.0 to 14.1, 8.4 to 20.9, 5.7 to 13.4 and 4.0 to 11.1. IQ1_M's scale is converted by F16C as the others', not by a call in the loop. The same bits.+- **Q2_K a third faster on one vector**: a block's sixteen scales are spread to its sub-groups by one shuffle each from a register, not loaded and broadcast one by one, and a vector's two sums reduced together. One vector from 8.6 to 12.9 GB/s on a core and from 37 to 49.5 on twelve threads, 2-8 vectors 8-20% (the UD-Q2_K_XL quantizations). The same bits.+- **Q4_K on 5-8 vectors 9-13% faster, Q5_K 6%** (`janas_dot_multi_rows`): what the products need of the vectors alone - their scales, their sums of 32 two by two laid out one int32 lane a vector - is made once for all the rows of a call, so that a row's mins on eight vectors are four madd where they were eight and six hadd, and its scales one load where they were eight. On eight vectors 5.7 to 6.45 GB/s on a core, 23.1 to 25.1 on twelve threads. A prompt of 512 tokens on the CPU against the release of 24 October, with the IQ and Q6_K changes above: Qwen3-4B Q4_K_M 61.4-62.5 to 73.2-73.4 tokens/s, Qwen3-30B-A3B 59.0-59.7 to 63.0-63.8; replies the same or 1-2% faster. The same bits.+- **The types in blocks of 32 about twice as fast on 2-8 vectors with AVX-VNNI** (Q8_0, Q4_0, IQ4_NL, MXFP4, Q5_0, Q4_1, Q5_1: a prompt's products): each group's eight blocks transposed in registers and the vectors' activations transposed once for all the rows of a call, so that eight `vpdpbusd` leave each block's sum in its own lane - no sign tricks and no tree of six hadd a vector. The signed weights go unsigned with an offset (128, or Q4_0's 8 and Q5_0's 16) and the offset times each block's sum of activations is taken off, exactly. On twelve threads and eight vectors Q8_0 from 25.5 to 52 GB/s, Q4_0 14.0 to 28.4, IQ4_NL 14.0 to 26.7, MXFP4 14.2 to 25.7, Q5_0 17.2 to 29.1, Q4_1 16.2 to 26.5, Q5_1 17.0 to 26.2; on two vectors Q4_0 40 to 68, Q8_0 78.5 to 87.6. Without AVX-VNNI, and on one vector, as before. The same bits.+- **IQ3_XXS 40-48% faster, IQ2_XXS and IQ2_XS 30-55%**: the signs of a 7-bit index, with the parity that completes them, read as eight bytes of multipliers from a table of 128 made at start (as ggml's ksigns), four at a time, instead of worked out and spread to bytes in the loop. One vector on a core IQ3_XXS from 4.65 to 6.9 GB/s, IQ2_XXS 3.0 to 4.8, IQ2_XS 3.45 to 4.8; on twelve threads 17.5 to 24.5, 14.2 to 19.6, 14.6 to 19.0. IQ2_S, whose signs are stored as bytes with no parity, keeps its way. The same bits.+- `tests/bench_kernels` measures every type the CPU multiplies (Q5_0, Q2_0, Q2_K, IQ3_XXS, IQ2_XXS/XS/S, IQ1_S/M added), with the IQ types' block scales normal fp16 numbers as in real weights (random bytes made some subnormal, and the float products slow), and the types named alone with `BENCH_TYPES=Q4_K,Q2_0`.+- `test_quant` checks the rows of Q2_0 against 1-17 vectors at once, and the rows of Q2_0, the K types and the types in blocks of 32 on 2, 48 and 130 groups of 256 (the vectors made ready a few at a time, and not at all), against the single products and the reference.+## [2026-10-24] - The chat's text on a Windows console in any language, gpu_check in the packages, the NPU made ready sooner- **Fixed: janas-chat on a Windows console printed every character past ASCII as mojibake** under a locale whose code page is not UTF-8 (issue #44, Ukrainian: "в–„" for "▄"). The chat asked for the user's locale, which on Windows it never used - the widths of characters are counted there without it - and msvcrt then translated what was written to the console from that locale's code page. Not asked for on Windows any more; the rest of the systems as before.diff --git a/README.md b/README.mdindex 3836b47..f5e1fb6 100644--- a/README.md+++ b/README.md@@ -17,7 +17,7 @@| Model | reading 1,024 tokens: Janas-LLM | how | llama.cpp | writing: Janas-LLM | how | llama.cpp ||---|---:|---|---:|---:|---|---:|-| Qwen3.8-Flash-Next (66.5 GB, 177B, more than the memory) | 32.7 tok/s | GPU, 12 threads | 4.82 tok/s‡ | 10.95 tok/s | CPU, 12 threads | 3.12 tok/s‡ |+| Qwen3.8-Flash-Next (66.5 GB, 177B, more than the memory) | 33.0 tok/s | GPU, 12 threads | 4.82 tok/s‡ | 10.95 tok/s | CPU, 12 threads | 3.12 tok/s‡ || Qwen3-Next-80B-A3B (48.4 GB, more than the memory) | 68.4 tok/s | GPU, 20 threads | 17.7 tok/s | 26.4 tok/s | CPU, 20 threads | 3.0 tok/s || Qwen3-Coder-Next (48.4 GB, more than the memory) | 71.4 tok/s | GPU, 12 threads | 14.0 tok/s | 26.5 tok/s | CPU, 20 threads | 2.9 tok/s || Qwen3.6-35B-A3B | 136 tok/s | GPU, 12 threads | 81.8 tok/s | 28.0 tok/s | CPU, 20 threads | 12.1 tok/s |diff --git a/build.sh b/build.shindex 5a236c5..46fad60 100755--- a/build.sh+++ b/build.sh@@ -270,20 +270,33 @@ elsefiecho '#include <vulkan/vulkan.h>' | "$CC" "${VK_INC[@]}" -E -x c - >/dev/null 2>&1 ||GPU_MISSING="${GPU_MISSING:+$GPU_MISSING and }the Vulkan headers (libvulkan-dev, vulkan-headers)"+# every fma() of a shader decorated NoContraction, one operation as on the+# CPU (tools/spv_fma.c: glslang leaves them free to be split), by a tool+# built with the compiler of the machine that builds (HOSTCC, cc)+SPV_FMA="$SPV/spv_fma"+if [ -n "$GLSLANG" ] &&+ { [ ! -x "$SPV_FMA" ] || [ tools/spv_fma.c -nt "$SPV_FMA" ]; }; then+ if ! "${HOSTCC:-cc}" -O2 -o "$SPV_FMA" tools/spv_fma.c; then+ GLSLANG=""+ GPU_MISSING="${GPU_MISSING:+$GPU_MISSING and }a C compiler for the build machine (HOSTCC) to build tools/spv_fma.c"+ fi+fiif [ -n "$GLSLANG" ]; thenfor comp in src/*/shaders/*.comp; do[ -f "$comp" ] || continuename=$(basename "${comp%.comp}")out="$SPV/$name.h"- if [ ! -f "$out" ] || [ "$comp" -nt "$out" ]; then+ if [ ! -f "$out" ] || [ "$comp" -nt "$out" ] || [ "$SPV_FMA" -nt "$out" ]; thenecho "SPV $comp"if ! glslangValidator -V --target-env vulkan1.3 \- --vn "janas_spv_$name" "$comp" -o "$out" >/dev/null; then+ "$comp" -o "$SPV/$name.spv" >/dev/null ||+ ! "$SPV_FMA" "$SPV/$name.spv" "janas_spv_$name" "$out"; thenrm -f "$out"GLSLANG=""GPU_MISSING="${GPU_MISSING:+$GPU_MISSING and }a glslangValidator that compiles $comp"breakfi+ rm -f "$SPV/$name.spv"fidonefidiff --git a/dist/README-diag.md b/dist/README-diag.mdnew file mode 100644index 0000000..04d1f48--- /dev/null+++ b/dist/README-diag.md@@ -0,0 +1,35 @@+# Janas diagnostics++Two programs that tell whether a GPU can run Janas-LLM's GPU backend, and whether it computes what the CPU does, without installing Janas or downloading a model. A few megabytes; Linux x86-64 (Ubuntu 22.04, Debian 12 and later) and Windows 10 and 11 x86-64.++## vkcheck++Lists what the GPU backend needs with the answer of the GPU's driver, the decision the engine would take, and three short tests on the device: zero-copy, the int8 dot product, the float operations that keep the GPU's results the CPU's to the bit. A few seconds.++## gpu_check++Runs every kind of product the GPU takes on the GPU Janas would use - each weight type, rows that end inside a group, several vector counts and strides, the BF16 products - and compares each with the CPU's, bit for bit. Each product runs twice: all its rows on the GPU, and shared with the CPU as the engine does. A card with memory of its own gets the weights copied there, as in the engine. Under a minute on most machines.++Nothing stops at the first error. For every wrong product the report says:++- which shader computed it;+- how far its results are from the CPU's, in units in the last place: a few mean the driver rounds a float operation differently, many mean wrong values;+- the first wrong row, with both values in hexadecimal.++Then gpu_check isolates the culprits by itself:++- it runs each wrong product again with the other shaders the engine could take for it;+- it runs it again with every other subgroup size the device allows.++The last line says `ok` or `FAILED`.++Run it in a terminal from this folder, `./gpu_check` on Linux and `.\gpu_check` in PowerShell on Windows. It opens the GPU even where `JANAS_GPU=0` keeps the engine off it. Other variables:++- `GPU_CHECK_TYPES=Q6_K,Q8_0` checks those types only;+- `JANAS_GPU_DEVICE=discrete` or `integrated` picks the GPU when there are both.++## Sending the results++If either program reports a problem, paste its whole output into an issue at <https://github.com/prabanta-dev/janas/issues>, with the GPU's driver version if the output does not show it.++The SHA-256 of every file is in SHA256SUMS.diff --git a/docs/hardware.md b/docs/hardware.mdindex 1538193..89a70e1 100644--- a/docs/hardware.md+++ b/docs/hardware.md@@ -38,12 +38,14 @@ The [hardware report form](https://github.com/prabanta-dev/janas/issues/new?temp**Whether a GPU can run the GPU backend**, and why not: `bin/x86_64-linux/vkcheck` lists what the backend needs with the answer of the GPU's driver, the decision the engine would take, and three short tests on the device (zero-copy, the int8 dot product, the float operations that keep the GPU's results the CPU's to the bit). It builds alone where Janas does not build yet, macOS with MoltenVK included: `cc -O2 tools/vkcheck.c -o vkcheck -ldl -lm` (on macOS without `-ldl -lm`, with `-I/opt/homebrew/include`). Its output is welcome in a hardware report.-**Whether the GPU computes what the CPU does**: `gpu_check`, in the packages from the release of 24 October 2026, runs every kind of product the GPU takes - each weight type, the rows that end inside a group, the BF16 products - on the GPU Janas would use and compares each with the CPU's, as a card with memory of its own works where it is one; its last line says `ok` or `FAILED`, with the shapes that differ above it. A minute or two. With wrong answers from the GPU and right ones with `JANAS_GPU=0`, its output says whether the products are where it goes wrong.+**The engine checks the GPU by itself**: when it opens the GPU it runs a product of every type against the CPU's and leaves out any shader the driver gets wrong (the status line says which; once per driver, kept in `~/.cache/janas/gpu_selfcheck`; `JANAS_GPU_SELFCHECK=0` turns it off).++**Whether the GPU computes what the CPU does**: `gpu_check`, in the packages from the release of 24 October 2026, runs every kind of product the GPU takes - each weight type, the rows that end inside a group, several vector counts, the BF16 products - on the GPU Janas would use, all its rows on the GPU and then shared with the CPU as in the engine, and compares each with the CPU's bit for bit, as a card with memory of its own works where it is one. Nothing stops at the first error: the report names the shader of every wrong product, how far its results are (a few units in the last place: the driver rounds differently; many: wrong values), the first wrong row in hexadecimal, and then runs the wrong products again with the other shaders the engine could take and with every other subgroup size the device allows, so that it isolates the culprit by itself. Its last line says `ok` or `FAILED`. Under a minute on most machines. `vkcheck` and `gpu_check` also come alone, without the engine, in `janas-diag-linux-x86_64.tar.gz` and `janas-diag-windows-x86_64.zip` (a few megabytes, no model needed); their whole output is what an issue about the GPU needs.**What is most wanted, in order.**1. **CPUs without AVX-VNNI**, and **AMD** — the arithmetic has a path for them and it is the least exercised of all.-2. **Discrete GPUs** (NVIDIA, AMD, Intel Arc). The GPU support is correct where it was checked - bit for bit on a reply's tokens, and on a prompt as close to the references as the CPU - and it pays where it was measured: on the development laptop's integrated GPU, with the fast prompt, Qwen3-4B reads a prompt 2.6 times faster than without the GPU (85 to 220 tokens/s; before the fast prompt, 83 to 107), and on an RTX 5080 (issue #15) Qwen3-4B read 13% faster and, with the GPU on single tokens too, replied 11-16% faster. From the release of 10 October to that of 13 October a discrete card's results stayed in its own memory, where the CPU never read them (issue #17, corrupted output on a Quadro RTX 5000); fixed since, and checked here on the path such a card takes, not yet on one. Every other card is unknown, and the fast prompt has not been seen working on a discrete one yet (`JANAS_GPU_FAST_PROMPT=0` turns it off if it misbehaves). **Careful:** a GPU driver reset takes the desktop with it. Start with `janas-bench`, not with a long chat.+2. **Discrete GPUs** (NVIDIA, AMD, Intel Arc). The GPU support is correct where it was checked - bit for bit on a reply's tokens, and on a prompt as close to the references as the CPU - and it pays where it was measured: on the development laptop's integrated GPU, with the fast prompt, Qwen3-4B reads a prompt 2.6 times faster than without the GPU (85 to 220 tokens/s; before the fast prompt, 83 to 107), and on an RTX 5080 (issue #15) Qwen3-4B read 13% faster and, with the GPU on single tokens too, replied 11-16% faster. From the release of 10 October to that of 13 October a discrete card's results stayed in its own memory, where the CPU never read them (issue #17, corrupted output on a Quadro RTX 5000); fixed since, and checked here on the path such a card takes, not yet on one. On the develop branch after 24 October 2026 a card with memory of its own runs every pass of the model whole - a prompt's blocks, draft checks, single tokens - with the model's state and working buffers in its memory, instead of sharing each product's rows with the CPU (`JANAS_GPU_WHOLE=0`: as before); a mixture keeps in the card's memory a tier of the experts it uses most, as many as fit, and the CPU computes the others between two layers (`JANAS_GPU_TIER=0`: not; `JANAS_GPU_TIER_MB=<n>` for n MiB). This was checked only on the development laptop's integrated GPU acting as such a card (`JANAS_GPU_COPY=1`), with functional runs - Qwen3.5-2B, Qwen3-4B, Qwen3.5-9B, Qwen3-30B-A3B with tiers from 300 MB to every expert, Qwen3.6-35B-A3B with a tier of 1 GB - and speeds measured only there, which say nothing of a discrete card's: Qwen3-30B-A3B one token at a time 13.3 tokens/s with every expert in the tier (the plan on the GPU, a pass in one submission), 8.5 with 300 MB (the CPU alone: 29.8). An expert the CPU computes gives the same bits as one the card computes (Q3_K, Q4_K, Q6_K, Q6_K_P and MXFP4 experts), so the tier's size does not change the answer and drafts give the same tokens as without them; with MXFP4, Q2_0, IQ or Q5_K experts the card's results still round differently from the CPU's. Every other card is unknown, and the fast prompt has not been seen working on a discrete one yet (`JANAS_GPU_FAST_PROMPT=0` turns it off if it misbehaves). **Careful:** a GPU driver reset takes the desktop with it. Start with `janas-bench`, not with a long chat.3. **Other NPUs.** Only the first Core Ultra's NPU (Meteor Lake) has been tried; Lunar Lake's and Arrow Lake's should take the same graphs and have not been seen doing it. AMD's and Qualcomm's NPUs are not supported yet.4. **Machines with 16 GB or less.** The engine is supposed to trade quality for memory on its own — fewer bits per weight, light experts not in memory left out — and the thresholds for that were measured on 32 GB.5. **Slower disks.** Everything assumes an NVMe SSD; on a SATA disk the engine should still work and simply wait more, but nobody has watched it do so.diff --git a/src/common/env_names.c b/src/common/env_names.cindex f7ec2be..3eff4f3 100644--- a/src/common/env_names.c+++ b/src/common/env_names.c@@ -56,6 +56,8 @@ const char *const JANAS_ENV_NAMES[] = {"JANAS_ALPHAVANTAGE_KEY","JANAS_GPU_MOE","JANAS_GPU_MV","JANAS_GPU_MVD",+ "JANAS_GPU_MVID",+ "JANAS_GPU_GPLAN","JANAS_GPU_ARENA_INPLACE","JANAS_GPU_EXPERT_MIN","JANAS_GPU_EXPERT_SHARE",@@ -64,10 +66,16 @@ const char *const JANAS_ENV_NAMES[] = {"JANAS_ALPHAVANTAGE_KEY","JANAS_GPU_REPACK","JANAS_GPU_REPEATABLE","JANAS_GPU_ROW_NV",+ "JANAS_GPU_SELFCHECK","JANAS_GPU_SG",+ "JANAS_GPU_TIER",+ "JANAS_GPU_TIER_MB","JANAS_GPU_TOKEN","JANAS_GPU_TOKEN_PROFILE",+ "JANAS_GPU_TOK_HOSTBUF",+ "JANAS_GPU_TOK_MIRROR","JANAS_GPU_VERBOSE",+ "JANAS_GPU_WHOLE","JANAS_GPU_XCOH","JANAS_HC_MIX","JANAS_HEAD_BITS",@@ -75,6 +83,8 @@ const char *const JANAS_ENV_NAMES[] = {"JANAS_ALPHAVANTAGE_KEY","JANAS_JANET_REC","JANAS_JANET_REC_LAYERS","JANAS_KERNELS",+ "JANAS_ROW_PIECE_KB",+ "JANAS_CPU_TILES","JANAS_KV","JANAS_LOCATION","JANAS_METRICS",diff --git a/src/common/sysinfo.c b/src/common/sysinfo.cindex af063cc..4b61c46 100644--- a/src/common/sysinfo.c+++ b/src/common/sysinfo.c@@ -300,4 +300,28 @@ uint64_t janas_rss_bytes(void)return ok && page > 0 ? (uint64_t)resident * (uint64_t)page : 0;}+size_t janas_l2_bytes(void)+{+ for (int i = 0; i < 8; i++) {+ char path[96], buf[32];+ snprintf(path, sizeof(path),+ "/sys/devices/system/cpu/cpu0/cache/index%d/level", i);+ if (read_long(path) != 2)+ continue;+ snprintf(path, sizeof(path),+ "/sys/devices/system/cpu/cpu0/cache/index%d/size", i);+ FILE *f = fopen(path, "r");+ if (!f)+ return 0;+ char *ok = fgets(buf, sizeof(buf), f);+ fclose(f);+ char *end;+ unsigned long v = ok ? strtoul(buf, &end, 10) : 0;+ if (!ok || end == buf)+ return 0;+ return *end == 'K' ? v << 10 : *end == 'M' ? v << 20 : v;+ }+ return 0;+}+#endif /* !_WIN32 */diff --git a/src/common/sysinfo.h b/src/common/sysinfo.hindex b9e0bce..70b88b1 100644--- a/src/common/sysinfo.h+++ b/src/common/sysinfo.h@@ -61,4 +61,7 @@ uint64_t janas_mem_total(void);/* The process's resident memory now, in bytes (0 if unknown). */uint64_t janas_rss_bytes(void);+/* The level-2 cache of the first CPU, in bytes (0 if unknown). */+size_t janas_l2_bytes(void);+#endifdiff --git a/src/common/sysinfo_win.c b/src/common/sysinfo_win.cindex f060e7d..357628b 100644--- a/src/common/sysinfo_win.c+++ b/src/common/sysinfo_win.c@@ -300,4 +300,16 @@ uint64_t janas_rss_bytes(void): 0;}+size_t janas_l2_bytes(void)+{+ SYSTEM_LOGICAL_PROCESSOR_INFORMATION info[256];+ DWORD len = sizeof(info);+ if (!GetLogicalProcessorInformation(info, &len))+ return 0;+ for (DWORD i = 0; i < len / sizeof(info[0]); i++)+ if (info[i].Relationship == RelationCache && info[i].Cache.Level == 2)+ return info[i].Cache.Size;+ return 0;+}+#endif /* _WIN32 */diff --git a/src/llm/api.c b/src/llm/api.cindex 7122bcc..325427e 100644--- a/src/llm/api.c+++ b/src/llm/api.c@@ -27,6 +27,7 @@#include "llm/attention.h"#include "llm/expert_cache.h"#include "llm/generate.h"+#include "llm/gpu.h"#include "llm/jns.h"#include "llm/model.h"#include "llm/quant.h"@@ -962,7 +963,7 @@ int32_t janas_llm_describe(const janas_llm *llm, char *buf, int32_t cap,* runs. The same goes for the attention, which the context decides.*/janas_llm *w = (janas_llm *)llm;- char gpubuf[224];+ char gpubuf[448];snprintf(gpubuf, sizeof(gpubuf), ", no GPU (%s)",janas_llm_model_gpu_why(llm->m));const char *gpu = gpubuf;@@ -985,6 +986,12 @@ int32_t janas_llm_describe(const janas_llm *llm, char *buf, int32_t cap,list);gpu = gpubuf;}+ /* what the check on opening left out on this driver */+ const char *note = janas_gpu_check_note(janas_llm_model_gpu(llm->m));+ if (note[0] && gpu == gpubuf) {+ size_t n = strlen(gpubuf);+ snprintf(gpubuf + n - 1, sizeof(gpubuf) - n + 1, "; %s)", note);+ }}char attn[64] = "";if (janas_llm_model_attn_scores(llm->m) == JANAS_LLM_ATTN_FAST)diff --git a/src/llm/gpu.c b/src/llm/gpu.cindex bf4a5e6..c1ea063 100644--- a/src/llm/gpu.c+++ b/src/llm/gpu.c@@ -45,6 +45,7 @@#include "q80_mmq.h" /* janas_spv_q80_mmq */#include "q5k_mmq.h" /* janas_spv_q5k_mmq */#include "dense_mm.h" /* janas_spv_dense_mm */+#include "dense_mv.h" /* janas_spv_dense_mv */#include "q3k_mmid.h" /* janas_spv_q3k_mmid */#include "q4k_mmid.h" /* janas_spv_q4k_mmid */#include "q6kp_mmid.h" /* janas_spv_q6kp_mmid */@@ -111,7 +112,9 @@ int janas_g_make_stage(struct janas_gpu *g, struct stage *s, size_t bytes,.usage =VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT |- VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT};+ VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT |+ VK_BUFFER_USAGE_TRANSFER_SRC_BIT |+ VK_BUFFER_USAGE_TRANSFER_DST_BIT};if (g->vkCreateBuffer(g->dev, &bi, NULL, &s->buf) != VK_SUCCESS)return -1;VkMemoryRequirements req;@@ -200,9 +203,13 @@ static int pick_device(struct janas_gpu *g, uint32_t *family, char *err,continue;if (pp.apiVersion < VK_API_VERSION_1_3)REJECT("its driver offers Vulkan below 1.3")+ VkPhysicalDeviceVulkan12Properties p12 = {+ .sType =+ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_PROPERTIES};VkPhysicalDeviceVulkan13Properties p13 = {.sType =- VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_PROPERTIES};+ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_PROPERTIES,+ .pNext = &p12};VkPhysicalDeviceVulkan11Properties p11 = {.sType =VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES,@@ -286,6 +293,32 @@ static int pick_device(struct janas_gpu *g, uint32_t *family, char *err,if (qf[f].queueFlags & VK_QUEUE_COMPUTE_BIT) {*family = f;snprintf(g->name, sizeof(g->name), "%s", pp.deviceName);+ g->vendor_id = pp.vendorID;+ g->device_id = pp.deviceID;+ g->driver_version = pp.driverVersion;+ /* what a report needs to tell one device and driver+ from another, and whether its float operations can+ be the CPU's */+ snprintf(+ g->about, sizeof(g->about),+ "%.96s, %s; driver %.64s %.64s (%#x), Vulkan %u.%u.%u; "+ "vendor %#x device %#x; subgroups %u-%u%s, "+ "running %u; float32 round to nearest even %s, "+ "denormals preserved %s, signed zero/inf/nan "+ "preserved %s",+ pp.deviceName, g->discrete ? "discrete" : "integrated",+ p12.driverName, p12.driverInfo,+ (unsigned)pp.driverVersion,+ VK_API_VERSION_MAJOR(pp.apiVersion),+ VK_API_VERSION_MINOR(pp.apiVersion),+ VK_API_VERSION_PATCH(pp.apiVersion),+ (unsigned)pp.vendorID, (unsigned)pp.deviceID,+ p13.minSubgroupSize, p13.maxSubgroupSize,+ control ? " (chosen)" : "", sg,+ p12.shaderRoundingModeRTEFloat32 ? "yes" : "no",+ p12.shaderDenormPreserveFloat32 ? "yes" : "no",+ p12.shaderSignedZeroInfNanPreserveFloat32 ? "yes"+ : "no");return 0;}REJECT("no compute queue")@@ -408,8 +441,12 @@ static VkResult make_pipeline(struct janas_gpu *g, const uint32_t *code,.pName = "main",.pSpecializationInfo = &si},.layout = g->pl};- return g->vkCreateComputePipelines(g->dev, VK_NULL_HANDLE, 1, &cpi, NULL,- pipe);+ r = g->vkCreateComputePipelines(g->dev, VK_NULL_HANDLE, 1, &cpi, NULL,+ pipe);+ if (r == VK_SUCCESS && g->n_made < MAX_MADE)+ g->made[g->n_made++] = (struct made){+ .p = *pipe, .code = code, .nv = nv, .sg = sg, .kind = kind};+ return r;}VkResult janas_g_make_pipeline_nv(struct janas_gpu *g, const uint32_t *code,@@ -443,6 +480,16 @@ struct janas_gpu *janas_gpu_create(char *err, size_t err_len)const char *nve = getenv("JANAS_GPU_NV");g->nvg = nve && atoi(nve) > 0 && atoi(nve) <= 16 ? (uint32_t)atoi(nve): NV_PER_GROUP;+ /* the products' paths (janas_gpu_set_path): JANAS_GPU_NO_ROW+ leaves out the subgroup-per-row shaders, JANAS_GPU_ROW_NV (0 to+ 8, 4 by default) the vectors they take at most, JANAS_GPU_MVD=0+ the shaders with the lanes on the vectors, JANAS_GPU_MV=1 takes+ q4k_mv (being measured) */+ const char *e;+ g->row_on = !getenv("JANAS_GPU_NO_ROW");+ g->row_nv = (e = getenv("JANAS_GPU_ROW_NV")) ? atoi(e) : 4;+ g->mvd_on = !(e = getenv("JANAS_GPU_MVD")) || atoi(e) > 0;+ g->mv_on = (e = getenv("JANAS_GPU_MV")) && atoi(e) > 0;}if (!g) {snprintf(err, err_len, "out of memory");@@ -668,6 +715,27 @@ struct janas_gpu *janas_gpu_create(char *err, size_t err_len)iqk[i].kind, 1) != VK_SUCCESS)g->pp1_iqk[i] = VK_NULL_HANDLE;}+ /* dense_mm's products on a few vectors: without, dense_mm's */+ for (int i = 0; i < 3; i++)+ if (janas_g_make_pipeline(g, janas_spv_dense_mv,+ sizeof(janas_spv_dense_mv), &g->sm_dv[i],+ &g->pipe_dv[i], i) != VK_SUCCESS)+ g->pipe_dv[i] = VK_NULL_HANDLE;+ /* the experts' products on a few vectors (gpu_token_moe.c) */+ const uint32_t *mc[6] = {janas_spv_q3k_matvec, janas_spv_q4k_matvec,+ janas_spv_q6k_matvec, janas_spv_q6kp_matvec,+ janas_spv_b32_matvec, janas_spv_b32_matvec};+ size_t mb[6] = {+ sizeof(janas_spv_q3k_matvec), sizeof(janas_spv_q4k_matvec),+ sizeof(janas_spv_q6k_matvec), sizeof(janas_spv_q6kp_matvec),+ sizeof(janas_spv_b32_matvec), sizeof(janas_spv_b32_matvec)};+ /* constant 2: ID, b32_matvec's in bit 8 beside its KIND */+ uint32_t mk[6] = {1, 1, 1, 1, 256 | 4, 256 | 6};+ for (int i = 0; i < 6; i++)+ if (janas_g_make_pipeline_nv(g, mc[i], mb[i], &g->sm_mvid[i],+ &g->pipe_mvid[i], mk[i],+ MVID_NV) != VK_SUCCESS)+ g->pipe_mvid[i] = VK_NULL_HANDLE;/* their tiles for the fast prompt */for (int i = 0; i < N_IQK; i++)if (make_pipeline(g, janas_spv_iqk_mmq, sizeof(janas_spv_iqk_mmq),@@ -849,6 +917,7 @@ struct janas_gpu *janas_gpu_create(char *err, size_t err_len)*bw = getenv("JANAS_GPU_BLOCKWAIT");g->nospin = ns ? atoi(ns) != 0 : janas_gpu_fast_prompt();g->block_wait = bw ? atoi(bw) != 0 : janas_gpu_fast_prompt();+ janas_g_selfcheck(g);return g;missing:@@ -1047,11 +1116,21 @@ void janas_gpu_destroy(struct janas_gpu *g)g->vkDestroyPipeline(g->dev, g->pipe_mm5, NULL);if (g->sm_mm5)g->vkDestroyShaderModule(g->dev, g->sm_mm5, NULL);+ for (int i = 0; i < 6; i++) {+ if (g->pipe_mvid[i])+ g->vkDestroyPipeline(g->dev, g->pipe_mvid[i], NULL);+ if (g->sm_mvid[i])+ g->vkDestroyShaderModule(g->dev, g->sm_mvid[i], NULL);+ }for (int i = 0; i < 3; i++) {if (g->pipe_dm[i])g->vkDestroyPipeline(g->dev, g->pipe_dm[i], NULL);if (g->sm_dm[i])g->vkDestroyShaderModule(g->dev, g->sm_dm[i], NULL);+ if (g->pipe_dv[i])+ g->vkDestroyPipeline(g->dev, g->pipe_dv[i], NULL);+ if (g->sm_dv[i])+ g->vkDestroyShaderModule(g->dev, g->sm_dv[i], NULL);}for (int i = 0; i < 5; i++) {VkPipeline pp = i == 4 ? g->pipe_id3@@ -1140,6 +1219,68 @@ unsigned janas_gpu_subgroup(const struct janas_gpu *g)return g->sg;}+/* The product shaders' names, for janas_gpu_last_shader. */+static const struct {+ const uint32_t *code;+ const char *name;+} spv_names[] = {+#define N(x) {janas_spv_##x, #x}+ N(q4k_matvec), N(q6k_matvec), N(q6kp_matvec), N(q5k_matvec),+ N(q8_0_matvec), N(b32_matvec), N(iq4xs_matvec), N(q3k_matvec),+ N(q4k_mmq), N(q6kp_mmq), N(q6k_mmq), N(q3k_mmq),+ N(iq4xs_mmq), N(iq3s_mmq), N(iq4nl_mmq), N(mxfp4_mmid),+ N(b32_mmid), N(bf16_matvec), N(iq1_matvec), N(iq2_matvec),+ N(iq3xxs_matvec), N(iqk_mmid), N(iqk_mmq), N(q2k_matvec),+ N(q80_mmq), N(q5k_mmq), N(dense_mm), N(q3k_mmid),+ N(q4k_mmid), N(q6kp_mmid), N(q6k_mmid), N(iq3s_matvec),+ N(expert_act), N(q4k_row), N(q6k_row), N(q6kp_row),+ N(q6ka_row), N(q4k_mv), N(q4k_mvd), N(q6k_mvd),+ N(q5k_mvd), N(iq4xs_mvd)+#undef N+};++size_t janas_gpu_last_shader(const struct janas_gpu *g, char *buf, size_t len)+{+ buf[0] = 0;+ for (int i = 0; i < g->n_made; i++)+ if (g->made[i].p == g->last_pipe) {+ const char *name = "a shader";+ for (size_t k = 0; k < sizeof(spv_names) / sizeof(*spv_names); k++)+ if (spv_names[k].code == g->made[i].code)+ name = spv_names[k].name;+ int n = snprintf(buf, len, "%s NV %u SG %u", name, g->made[i].nv,+ g->made[i].sg);+ if (g->made[i].kind && n > 0 && (size_t)n < len)+ n += snprintf(buf + n, len - (size_t)n, " kind %u",+ g->made[i].kind);+ return n > 0 ? (size_t)n : 0;+ }+ return 0;+}++int janas_gpu_set_path(struct janas_gpu *g, enum janas_gpu_path path, int value)+{+ int *f = path == JANAS_GPU_PATH_ROW ? &g->row_on+ : path == JANAS_GPU_PATH_ROW_NV ? &g->row_nv+ : path == JANAS_GPU_PATH_MVD ? &g->mvd_on+ : &g->mv_on;+ int was = *f;+ *f = value;+ return was;+}++void janas_gpu_subgroup_range(const struct janas_gpu *g, unsigned *min,+ unsigned *max)+{+ *min = g->sg_control ? g->sg_min : g->sg;+ *max = g->sg_control ? g->sg_max : g->sg;+}++const char *janas_gpu_about(const struct janas_gpu *g)+{+ return g->about;+}+size_t janas_gpu_min_n(const struct janas_gpu *g){return g->min_n;@@ -1186,6 +1327,7 @@ static int copy_to_device(struct janas_gpu *g, const void *p, size_t bytes).usage =VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT |+ VK_BUFFER_USAGE_TRANSFER_SRC_BIT |VK_BUFFER_USAGE_TRANSFER_DST_BIT};if (g->vkCreateBuffer(g->dev, &bi, NULL, &rg->buf) != VK_SUCCESS)return -1;@@ -1211,6 +1353,7 @@ static int copy_to_device(struct janas_gpu *g, const void *p, size_t bytes)}if (g->vkBindBufferMemory(g->dev, rg->buf, rg->mem, 0) != VK_SUCCESS)goto fail;+ g->local_bytes += req.size;/* staging: 64 MiB of host memory, one chunk at a time */struct stage st = {0};size_t chunk = (size_t)64 << 20;@@ -1273,6 +1416,136 @@ fail:return -1;}+int janas_g_make_local(struct janas_gpu *g, struct stage *s, size_t bytes)+{+ memset(s, 0, sizeof(*s));+ VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,+ .size = bytes,+ .usage =+ VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |+ VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT |+ VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT |+ VK_BUFFER_USAGE_TRANSFER_SRC_BIT |+ VK_BUFFER_USAGE_TRANSFER_DST_BIT};+ if (g->vkCreateBuffer(g->dev, &bi, NULL, &s->buf) != VK_SUCCESS)+ return -1;+ VkMemoryRequirements req;+ g->vkGetBufferMemoryRequirements(g->dev, s->buf, &req);+ int t = local_type(g, req.memoryTypeBits);+ VkMemoryAllocateFlagsInfo fl = {+ .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,+ .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT};+ VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,+ .pNext = &fl,+ .allocationSize = req.size,+ .memoryTypeIndex = (uint32_t)t};+ if (t < 0 ||+ g->vkAllocateMemory(g->dev, &ai, NULL, &s->mem) != VK_SUCCESS ||+ g->vkBindBufferMemory(g->dev, s->buf, s->mem, 0) != VK_SUCCESS) {+ g->vkDestroyBuffer(g->dev, s->buf, NULL);+ if (s->mem)+ g->vkFreeMemory(g->dev, s->mem, NULL);+ memset(s, 0, sizeof(*s));+ return -1;+ }+ g->local_bytes += req.size;+ s->addr = janas_g_buffer_address(g, s->buf);+ s->bytes = bytes;+ return 0;+}++int janas_g_copies(struct janas_gpu *g, const struct janas_g_cp *c, uint32_t n)+{+ VkCommandBufferBeginInfo bb = {+ .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,+ .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};+ VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,+ .commandBufferCount = 1,+ .pCommandBuffers = &g->cb};+ int ok = g->vkResetCommandBuffer(g->cb, 0) == VK_SUCCESS &&+ g->vkBeginCommandBuffer(g->cb, &bb) == VK_SUCCESS;+ for (uint32_t i = 0; ok && i < n; i++) {+ VkBufferCopy cp = {+ .srcOffset = c[i].so, .dstOffset = c[i].dof, .size = c[i].bytes};+ g->vkCmdCopyBuffer(g->cb, c[i].src, c[i].dst, 1, &cp);+ }+ ok = ok && g->vkEndCommandBuffer(g->cb) == VK_SUCCESS;+ pthread_mutex_lock(&g->qlock);+ ok = ok && g->vkQueueSubmit(g->queue, 1, &sub, g->fence) == VK_SUCCESS;+ pthread_mutex_unlock(&g->qlock);+ ok = ok && g->vkWaitForFences(g->dev, 1, &g->fence, VK_TRUE, UINT64_MAX) ==+ VK_SUCCESS;+ g->vkResetFences(g->dev, 1, &g->fence);+ return ok ? 0 : -1;+}++int janas_g_copy(struct janas_gpu *g, VkBuffer src, VkDeviceSize so,+ VkBuffer dst, VkDeviceSize dof, VkDeviceSize bytes)+{+ struct janas_g_cp c = {+ .src = src, .dst = dst, .so = so, .dof = dof, .bytes = bytes};+ return janas_g_copies(g, &c, 1);+}++int janas_gpu_sync(struct janas_gpu *g, void *p, size_t bytes, int to_device)+{+ if (bytes == 0)+ return 0;+ const struct region *rg = janas_g_find_region(g, p, bytes);+ if (!rg || g->broken || g->busy)+ return -1;+ if (rg->host_shared)+ return 0; /* the GPU reads and writes the host's memory itself */+ VkDeviceSize at = (VkDeviceSize)((const uint8_t *)p - rg->host);+ /* through the Y stage, idle between passes, a piece at a time */+ for (size_t done = 0; done < bytes;) {+ size_t n = bytes - done < g->ys.bytes ? bytes - done : g->ys.bytes;+ /* the stage may be cached, not coherent: written bytes flushed+ before the copy, read ones invalidated after it */+ VkMappedMemoryRange mr = {+ .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,+ .memory = g->ys.mem,+ .size = (n + g->atom - 1) / g->atom * g->atom < g->ys.bytes+ ? (n + g->atom - 1) / g->atom * g->atom+ : VK_WHOLE_SIZE};+ if (to_device) {+ memcpy(g->ys.map, (const uint8_t *)p + done, n);+ if (!g->ys.coherent)+ g->vkFlushMappedMemoryRanges(g->dev, 1, &mr);+ }+ VkCommandBufferBeginInfo bb = {+ .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,+ .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};+ VkBufferCopy cp = {.srcOffset = to_device ? 0 : at + done,+ .dstOffset = to_device ? at + done : 0,+ .size = n};+ VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,+ .commandBufferCount = 1,+ .pCommandBuffers = &g->cb};+ int ok = g->vkResetCommandBuffer(g->cb, 0) == VK_SUCCESS &&+ g->vkBeginCommandBuffer(g->cb, &bb) == VK_SUCCESS;+ if (ok)+ g->vkCmdCopyBuffer(g->cb, to_device ? g->ys.buf : rg->buf,+ to_device ? rg->buf : g->ys.buf, 1, &cp);+ ok = ok && g->vkEndCommandBuffer(g->cb) == VK_SUCCESS;+ pthread_mutex_lock(&g->qlock);+ ok = ok && g->vkQueueSubmit(g->queue, 1, &sub, g->fence) == VK_SUCCESS;+ pthread_mutex_unlock(&g->qlock);+ ok = ok && g->vkWaitForFences(g->dev, 1, &g->fence, VK_TRUE,+ UINT64_MAX) == VK_SUCCESS;+ g->vkResetFences(g->dev, 1, &g->fence);+ if (!ok)+ return -1;+ if (!to_device) {+ if (!g->ys.coherent)+ g->vkInvalidateMappedMemoryRanges(g->dev, 1, &mr);+ memcpy((uint8_t *)p + done, g->ys.map, n);+ }+ done += n;+ }+ return 0;+}+/* an import refused, and where, with JANAS_GPU_VERBOSE */static int import_refused(const struct janas_gpu *g, const void *p,size_t bytes, const char *why, int r)@@ -1350,6 +1623,16 @@ int janas_gpu_import(struct janas_gpu *g, const void *p, size_t bytes)return 0;}+void janas_g_drop_region(struct janas_gpu *g, const void *host)+{+ if (g->n_reg == 0 || g->reg[g->n_reg - 1].host != host)+ return;+ struct region *rg = &g->reg[--g->n_reg];+ g->vkDestroyBuffer(g->dev, rg->buf, NULL);+ g->vkFreeMemory(g->dev, rg->mem, NULL);+ memset(rg, 0, sizeof(*rg));+}+void *janas_gpu_alloc(struct janas_gpu *g, size_t bytes){bytes = (bytes + 4095) / 4096 * 4096;@@ -1529,7 +1812,7 @@ int janas_gpu_can(const struct janas_gpu *g, const struct janas_matvec_task *t)(plane_level(t) < 3 || plane_region(g, t->plane[1], t->rows, nb)));int ik = iqk_index(t->type);typed |= ik >= 0 && g->pipe_iqk[ik] && (uintptr_t)t->w % 4 == 0;- return !g->broken && typed && tail_ok(t) &&+ return !g->broken && typed && tail_ok(t) && !janas_g_type_off(g, t->type) &&nv * nb * GPU_BLOCK <= X_STAGE_BYTES &&t->rows * nv * sizeof(float) <= Y_STAGE_BYTES &&janas_g_find_region(g, t->w, read_bytes(t, t->rows));@@ -1767,22 +2050,16 @@ static int mm_on(void)/* q4k_mv for Q4_K on 2 to JANAS_GPU_FAST_N - 1 vectors (JANAS_GPU_MV=1;off by default while it is measured) */-static int mv_on(void)+static int mv_on(const struct janas_gpu *g){- static int on = -1;- if (on < 0)- on = getenv("JANAS_GPU_MV") && atoi(getenv("JANAS_GPU_MV")) > 0;- return on;+ return g->mv_on;}/* q4k_mvd, q6k_mvd, q5k_mvd and iq4xs_mvd for Q4_K, Q6_K, Q5_K and IQ4_XSon 2 to 8 vectors (JANAS_GPU_MVD=0 leaves them out) */-static int mvd_on(void)+static int mvd_on(const struct janas_gpu *g){- static int on = -1;- if (on < 0)- on = !getenv("JANAS_GPU_MVD") || atoi(getenv("JANAS_GPU_MVD")) > 0;- return on;+ return g->mvd_on;}/* The shader of a type with the lanes on the vectors (2 to 8 vectors), or@@ -1798,7 +2075,7 @@ static VkPipeline mvd_pipe(const struct janas_gpu *g, int type)size_t janas_gpu_few_max(const struct janas_gpu *g){- return g->pipe_mvd && mvd_on() ? 8 : 0;+ return g->pipe_mvd && mvd_on(g) ? 8 : 0;}/* Whether bytes from a lie within one 4 GB of the address space (the high@@ -1879,10 +2156,7 @@ int janas_g_record_task(struct janas_gpu *g, VkPipeline *bound,/* up to row_nv vectors (JANAS_GPU_ROW_NV, 0 to 8; 4 by default) a rowshader takes them, its variant for 2, 4 or 8 (kv), where theshaders reading a row per invocation are slower; more, those */- static int row_nv = -1;- if (row_nv < 0)- row_nv =- getenv("JANAS_GPU_ROW_NV") ? atoi(getenv("JANAS_GPU_ROW_NV")) : 4;+ int row_nv = g->row_nv;int kv = nv <= 2 ? 0 : nv <= 4 ? 1 : 2;const struct region *rg = janas_g_find_region(g, t->w, read_bytes(t, rows));if (!rg)@@ -1926,24 +2200,23 @@ int janas_g_record_task(struct janas_gpu *g, VkPipeline *bound,want = rp;nvg = nv == 1 ? 1 : 2u << kv;rows_wg = 64 / g->sg;- } else if (nv > 1 && nv <= 8 && mvd_pipe(g, t->type) && mvd_on() &&+ } else if (nv > 1 && nv <= 8 && mvd_pipe(g, t->type) && mvd_on(g) &&in_4g(wa, read_bytes(t, rows)) &&in_4g(xa, nv * nb * GPU_BLOCK)) {want = mvd_pipe(g, t->type);nvg = 8;rows_wg = MVD_ROWS;} else if (t->type == JANAS_Q4_K && nv > 1 && nv < JANAS_GPU_FAST_N &&- g->pipe_mv && mv_on() && in_4g(wa, read_bytes(t, rows)) &&+ g->pipe_mv && mv_on(g) && in_4g(wa, read_bytes(t, rows)) &&in_4g(xa, nv * nb * GPU_BLOCK)) {want = g->pipe_mv;nvg = MV_NV;- } else if (nv == 1 && pk >= 0 && g->rp[pk] && nb <= 128 &&- !getenv("JANAS_GPU_NO_ROW")) {+ } else if (nv == 1 && pk >= 0 && g->rp[pk] && nb <= 128 && g->row_on) {want = g->rp[pk];nvg = 1;rows_wg = 64 / g->sg; /* a subgroup per row */} else if (nv > 1 && (int)nv <= row_nv && pk >= 0 && pk < 3 &&- g->rpv[kv][pk] && nb <= 128 && !getenv("JANAS_GPU_NO_ROW")) {+ g->rpv[kv][pk] && nb <= 128 && g->row_on) {want = g->rpv[kv][pk];nvg = 2u << kv;rows_wg = 64 / g->sg;@@ -2006,6 +2279,7 @@ int janas_g_record_task(struct janas_gpu *g, VkPipeline *bound,g->vkCmdBindPipeline(g->cb, VK_PIPELINE_BIND_POINT_COMPUTE, want);*bound = want;}+ g->last_pipe = want;struct push pc = {.rows = (uint32_t)rows,.nb = (uint32_t)nb,.nv = (uint32_t)nv,@@ -2966,6 +3240,44 @@ unsigned janas_gpu_subgroup(const struct janas_gpu *g)return 0;}+size_t janas_gpu_last_shader(const struct janas_gpu *g, char *buf, size_t len)+{+ (void)g;+ if (len)+ buf[0] = 0;+ return 0;+}++int janas_gpu_set_path(struct janas_gpu *g, enum janas_gpu_path path, int value)+{+ (void)g;+ (void)path;+ (void)value;+ return 0;+}++void janas_gpu_subgroup_range(const struct janas_gpu *g, unsigned *min,+ unsigned *max)+{+ (void)g;+ *min = *max = 0;+}++const char *janas_gpu_about(const struct janas_gpu *g)+{+ (void)g;+ return "";+}++int janas_gpu_sync(struct janas_gpu *g, void *p, size_t bytes, int to_device)+{+ (void)g;+ (void)p;+ (void)bytes;+ (void)to_device;+ return -1;+}+size_t janas_gpu_min_n(const struct janas_gpu *g){(void)g;diff --git a/src/llm/gpu.h b/src/llm/gpu.hindex f87fdcc..bd62001 100644--- a/src/llm/gpu.h+++ b/src/llm/gpu.h@@ -168,6 +168,31 @@ void janas_gpu_keep_awake(struct janas_gpu *g, int on);/* The device's name, and the subgroup size the shaders run with. */const char *janas_gpu_name(const struct janas_gpu *g);unsigned janas_gpu_subgroup(const struct janas_gpu *g);+/* The subgroup sizes the device lets the shaders be given (the running one+ twice where it cannot be chosen): JANAS_GPU_SG picks one at creation. */+void janas_gpu_subgroup_range(const struct janas_gpu *g, unsigned *min,+ unsigned *max);+/* One line on the device and its driver for a report: name, kind, driver+ and its version, Vulkan, subgroups, the float controls that keep its+ results the CPU's. */+const char *janas_gpu_about(const struct janas_gpu *g);+/* What the check of the GPU's products against the CPU's on opening left+ out on this driver ("" if nothing; gpu_selfcheck.c). */+const char *janas_gpu_check_note(const struct janas_gpu *g);++/* For diagnosis (gpu_check): the shader that recorded the last product,+ as "q6k_row NV 1 SG 32" (its name, vectors a workgroup, subgroup size,+ kind where it serves several types), "" if none; and the products'+ paths switched on or off, the value before returned. */+size_t janas_gpu_last_shader(const struct janas_gpu *g, char *buf, size_t len);+enum janas_gpu_path {+ JANAS_GPU_PATH_ROW, /* the subgroup-per-row shaders (*_row.comp) */+ JANAS_GPU_PATH_ROW_NV, /* ... on up to this many vectors (4) */+ JANAS_GPU_PATH_MVD, /* the lanes on the vectors (*_mvd.comp) */+ JANAS_GPU_PATH_MV /* q4k_mv.comp (off) */+};+int janas_gpu_set_path(struct janas_gpu *g, enum janas_gpu_path path,+ int value);/* The fewest vectors a product needs to be given to the GPU: 1 on adiscrete GPU (single tokens too), 2 on an integrated one;JANAS_GPU_MIN_N sets it. */@@ -186,4 +211,13 @@ size_t janas_gpu_few_max(const struct janas_gpu *g);On a card with memory of its own an import is a copy. */int janas_gpu_shares_host(const struct janas_gpu *g);+/* Bytes of a card's own memory still free for Janas (85% of it, less what+ it took); 0 on a GPU that shares the host's memory. */+size_t janas_gpu_local_room(const struct janas_gpu *g);+/* Bytes of host memory and the GPU's copy of them made the same: to the+ device (to_device 1) or back to the host (0), for buffers both write on+ a card with memory of its own; nothing where the GPU shares the host's+ memory. 0, or -1 (not imported, or the GPU busy or broken). */+int janas_gpu_sync(struct janas_gpu *g, void *p, size_t bytes, int to_device);+#endifdiff --git a/src/llm/gpu_impl.h b/src/llm/gpu_impl.hindex dede18b..cb29921 100644--- a/src/llm/gpu_impl.h+++ b/src/llm/gpu_impl.h@@ -57,6 +57,7 @@#define GPU_BLOCK 320 /* a Q8_K block in the shader's layout */#define SHARE_SLOTS 64#define MAX_GROUP 64+#define MAX_MADE 384 /* pipelines a device gets (some 200) *//* instance-level functions, then device-level ones */#define VK_INSTANCE_FUNCS(X) \@@ -144,6 +145,9 @@ struct pending {size_t off; /* in the Y stage */};+/* The vectors a workgroup of pipe_mvid takes */+#define MVID_NV 4+/* The push constants of the shaders (level and the bare planes: Q6_K_Ponly). */struct push {@@ -181,13 +185,19 @@ struct janas_gpu {VkPipeline pipex4, pipe3, pipei3; /* IQ4_XS, Q3_K, IQ3_S *//* Q4_K on many vectors in tiles (q4k_mm.comp), MM_ROWS x MM_VECS */VkShaderModule sm_mm, sm_mm6, sm_mm6s, sm_mm80, sm_mm5, sm_mm3, sm_mmx4,- sm_mmi3, sm_mmn4, sm_mmmx, sm_mm50, sm_mm20,- sm_dm[3]; /* sm_mmx4 IQ4_XS, mmi3 IQ3_S, mmn4 IQ4_NL, mmmx MXFP4,- mm50 Q5_0, mm20 Q2_0; dm f32, bf16 and Q8_0 weights on- float vectors */+ sm_mmi3, sm_mmn4, sm_mmmx, sm_mm50, sm_mm20, sm_dm[3],+ sm_dv[3]; /* sm_mmx4 IQ4_XS, mmi3 IQ3_S, mmn4 IQ4_NL,+ mmmx MXFP4, mm50 Q5_0, mm20 Q2_0; dm f32,+ bf16 and Q8_0 weights on float vectors, dv+ the same on a few (optional) */VkPipeline pipe_mm, pipe_mm6, pipe_mm6s, pipe_mm80, pipe_mm5, pipe_mm3,pipe_mmx4, pipe_mmi3, pipe_mmn4, pipe_mmmx, pipe_mm50, pipe_mm20,- pipe_dm[3];+ pipe_dm[3], pipe_dv[3];+ /* q3k, q4k, q6k, q6kp_matvec and b32_matvec's MXFP4 and Q2_0 with ID:+ a mixture's experts on a few vectors, MVID_NV a workgroup, from+ *_mmid.comp's table (optional) */+ VkShaderModule sm_mvid[6];+ VkPipeline pipe_mvid[6];int only_fast; /* janas_gpu_set_only_fast */uint32_t pass_n; /* janas_gpu_set_pass; UINT32_MAX until said */uint32_t nvg; /* vectors a product's workgroup takes (NV_PER_GROUP) */@@ -244,10 +254,11 @@ struct janas_gpu {size_t n_pend;int busy, broken;char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];- uint32_t sg; /* subgroup size the shader runs with */- int discrete; /* weights copied to its memory, not shared */- int host_import; /* VK_EXT_external_memory_host available */- int sg_control; /* the size is required (else the device's fixed one) */+ uint32_t sg; /* subgroup size the shader runs with */+ int discrete; /* weights copied to its memory, not shared */+ VkDeviceSize local_bytes; /* taken of its own memory, never given back */+ int host_import; /* VK_EXT_external_memory_host available */+ int sg_control; /* the size is required (else the device's fixed one) */uint32_t sg_min, sg_max; /* the sizes it may be given (sg_control) *//* the GPU's share of the rows, per product shape */struct share {@@ -281,7 +292,46 @@ struct janas_gpu {unsigned keep_us;VkCommandBuffer ka_cb;VkFence ka_fence;+ /* every pipeline made, for janas_gpu_last_shader: its SPIR-V, vectors+ a workgroup, subgroup size and kind */+ struct made {+ VkPipeline p;+ const uint32_t *code;+ uint32_t nv, sg, kind;+ } made[MAX_MADE];+ int n_made;+ VkPipeline last_pipe; /* the last product janas_g_record_task recorded */+ /* the paths a product may take (janas_gpu_set_path; at creation from+ JANAS_GPU_NO_ROW, JANAS_GPU_ROW_NV, JANAS_GPU_MVD, JANAS_GPU_MV) */+ int row_on, row_nv, mvd_on, mv_on;+ char about[768]; /* janas_gpu_about */+ uint32_t vendor_id, device_id, driver_version;+ /* what the check on opening left out (gpu_selfcheck.c): types the GPU+ got wrong whatever the shader, and a line for the user */+ int off_types[16], n_off;+ char check_note[256];+};++/* The products checked against the CPU's when the GPU opens, and the+ shaders or types this driver gets wrong left out (gpu_selfcheck.c). */+void janas_g_selfcheck(struct janas_gpu *g);+/* Whether the check left the type out. */+int janas_g_type_off(const struct janas_gpu *g, int type);+/* A buffer in the device's own memory (no host mapping), for shaders and+ copies; 0, or -1. */+int janas_g_make_local(struct janas_gpu *g, struct stage *s, size_t bytes);+/* Copies bytes between two buffers and waits: 0, or -1. */+int janas_g_copy(struct janas_gpu *g, VkBuffer src, VkDeviceSize so,+ VkBuffer dst, VkDeviceSize dof, VkDeviceSize bytes);+/* A copy of janas_g_copies'. */+struct janas_g_cp {+ VkBuffer src, dst;+ VkDeviceSize so, dof, bytes;};+/* Copies c[0..n) in one submission and waits: 0, or -1. */+int janas_g_copies(struct janas_gpu *g, const struct janas_g_cp *c, uint32_t n);+/* Forgets the last region imported, at host (the check's own weights). */+void janas_g_drop_region(struct janas_gpu *g, const void *host);/* Helpers shared by gpu.c and gpu_token.c (defined in gpu.c). */int janas_g_mem_type(const struct janas_gpu *g, uint32_t bits,diff --git a/src/llm/gpu_selfcheck.c b/src/llm/gpu_selfcheck.cnew file mode 100644index 0000000..10787e7--- /dev/null+++ b/src/llm/gpu_selfcheck.c@@ -0,0 +1,435 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * gpu_selfcheck.c - when the GPU opens, a product of every type it takes,+ * on the vector counts that pick each of its shaders, against the CPU's,+ * bit for bit (a fraction of a second, small random matrices). A shader+ * this device's driver gets wrong is left out: the subgroup-per-row ones,+ * the ones with the lanes on the vectors, q4k_mv (each family has another+ * shader for the same products); a type still wrong without them goes to+ * the CPU. On an Arc B580 a Q6_K shader gave text with no sense, issue+ * #43: with the check the engine answers right from the first run. The+ * result is kept in ~/.cache/janas/gpu_selfcheck under the device, its+ * driver's version, the subgroup size and Janas's version, so that the+ * check runs once per driver. JANAS_GPU_SELFCHECK=0 leaves it out+ * (gpu_check does, to see every shader), =2 runs it even when the cache+ * has the answer; JANAS_GPU_VERBOSE prints what it did. Not the fast+ * prompt's tiles (64 vectors and more, rounded otherwise: gpu_fast_check).+ */+#include "gpu.h"++#if defined(JANAS_GPU_SPV) && __has_include(<vulkan/vulkan.h>)+#include "gpu_impl.h"++#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <time.h>++#include "common/aligned.h"+#include "common/pool.h"+#include "matvec.h"+#include "quant.h"+#include "tuner.h" /* janas_cache_dir */++#ifndef JANAS_VERSION+#define JANAS_VERSION "dev"+#endif++enum { ROWS = 128, MAXC = 640, NV_MAX = 9, SLOT = 128 << 10 };++/* the types the products take, their block's bytes and where its scales+ are (offset of d, of a second scale or -1); 640 wide only for the types+ that sum a tail */+static const struct kind {+ int type;+ unsigned block, step; /* bytes a block of 256 weights, a block's */+ int d, m;+ int tail;+} kinds[] = {+ {JANAS_Q4_K, 144, 144, 0, 2, 0}, {JANAS_Q5_K, 176, 176, 0, 2, 0},+ {JANAS_Q6_K, 210, 210, 208, -1, 0}, {JANAS_Q6_K_P, 82, 82, 80, -1, 0},+ {JANAS_Q8_0, 272, 34, 0, -1, 1}, {JANAS_Q4_0, 144, 18, 0, -1, 1},+ {JANAS_IQ4_NL, 144, 18, 0, -1, 1}, {JANAS_Q4_1, 160, 20, 0, 2, 1},+ {JANAS_Q5_1, 192, 24, 0, 2, 1}, {JANAS_MXFP4, 136, 17, -1, -1, 1},+ {JANAS_Q5_0, 176, 22, 0, -1, 1}, {JANAS_Q2_0, 72, 18, 0, -1, 1},+ {JANAS_IQ4_XS, 136, 136, 0, -1, 0}, {JANAS_Q3_K, 110, 110, 108, -1, 0},+ {JANAS_IQ3_S, 110, 110, 0, -1, 0}, {JANAS_Q2_K, 84, 84, 80, 82, 0},+ {JANAS_IQ2_XXS, 66, 66, 0, -1, 0}, {JANAS_IQ2_XS, 74, 74, 0, -1, 0},+ {JANAS_IQ2_S, 82, 82, 0, -1, 0}, {JANAS_IQ3_XXS, 98, 98, 0, -1, 0},+ {JANAS_IQ1_S, 50, 50, 0, -1, 0}, {JANAS_IQ1_M, 56, 56, -2, -1, 0}};+enum { N_KINDS = sizeof(kinds) / sizeof(kinds[0]) };++/* vector counts: one (the row shaders, or the one-vector ones), 2 and 3+ (the row shaders' variants, the lanes on the vectors), 5 and 9 (the+ shaders of many vectors) */+static const size_t nvs[] = {1, 2, 3, 5, 9};+enum { N_NVS = sizeof(nvs) / sizeof(nvs[0]) };++static uint64_t rng = 0x9e3779b97f4a7c15ull;++static uint32_t rnd(void)+{+ rng ^= rng << 13;+ rng ^= rng >> 7;+ rng ^= rng << 17;+ return (uint32_t)rng;+}++/* random weights of kind k, scales small and normal */+static void fill(const struct kind *k, uint8_t *w, size_t bytes)+{+ for (size_t i = 0; i < bytes; i++)+ w[i] = (uint8_t)rnd();+ for (size_t b = 0; b + k->step <= bytes; b += k->step) {+ uint16_t d = janas_fp32_to_fp16(0.0005f + (rnd() % 1000) * 1e-6f);+ if (k->type == JANAS_MXFP4) {+ w[b] = (uint8_t)(118 + rnd() % 8);+ } else if (k->d == -2) {+ /* IQ1_M: d in the top nibbles of its four scale words */+ for (int s = 0; s < 4; s++) {+ uint16_t sc;+ memcpy(&sc, w + b + 48 + 2 * s, 2);+ sc = (uint16_t)((sc & 0x0fff) | ((d >> 4 * s & 15) << 12));+ memcpy(w + b + 48 + 2 * s, &sc, 2);+ }+ } else {+ memcpy(w + b + k->d, &d, 2);+ }+ if (k->m >= 0) {+ uint16_t m =+ k->type == JANAS_Q4_1 || k->type == JANAS_Q5_1+ ? janas_fp32_to_fp16(-0.01f + (rnd() % 1000) * 2e-5f)+ : janas_fp32_to_fp16(0.0002f + (rnd() % 1000) * 1e-6f);+ memcpy(w + b + k->m, &m, 2);+ }+ }+}++/* one product wrong: kind, width, vectors and the shader that ran it */+struct wrong {+ int k;+ size_t cols, nv;+ char shader[96];+};++/* Runs kind k's product of cols columns on nv vectors on the GPU, all+ its rows, and compares it with the CPU's: 1 right, 0 wrong (shader+ named), -1 not taken by the GPU. */+static int check_one(struct janas_gpu *g, struct janas_pool *pool,+ const uint8_t *w, const struct janas_block_q8k *x,+ float *ya, float *yb, int k, size_t cols, size_t nv,+ char *shader, size_t len)+{+ size_t nbx = (cols + JANAS_QK - 1) / JANAS_QK;+ struct janas_matvec_task t = {.type = kinds[k].type,+ .w = w,+ .x = x,+ .rows = ROWS,+ .cols = cols,+ .n_vec = nv,+ .x_stride = nbx};+ t.y = ya;+ memset(ya, 0, ROWS * nv * sizeof(float));+ janas_matvec_q4k_group(pool, &t, 1);+ if (!janas_gpu_can(g, &t))+ return -1;+ t.y = yb;+ memset(yb, 0, ROWS * nv * sizeof(float));+ size_t rows = ROWS;+ int waited;+ if (janas_gpu_begin(g, &t, &rows, 1) != 0)+ return -1;+ if (janas_gpu_finish(g, &waited) != 0)+ return 0;+ janas_gpu_last_shader(g, shader, len);+ return memcmp(ya, yb, ROWS * nv * sizeof(float)) == 0;+}++/* The cache's key for this device and driver, and its file. */+static void cache_key(const struct janas_gpu *g, char *key, size_t len)+{+ snprintf(key, len, "%04x:%04x:%08x:sg%u:%s:%s", g->vendor_id, g->device_id,+ g->driver_version, g->sg,+ janas_gpu_shares_host(g) ? "shared" : "copy", JANAS_VERSION);+}++/* What a saved check says: 1 found (switches and types applied). */+static int load(struct janas_gpu *g, const char *key, int *flags)+{+ char path[512];+ if (janas_cache_dir(path, sizeof(path) - 32, 0) != 0)+ return 0;+ strcat(path, "/gpu_selfcheck");+ FILE *f = fopen(path, "r");+ if (!f)+ return 0;+ char line[512];+ int found = 0, row = 1, mvd = 1, mv = 1, off[16], n_off = 0;+ while (fgets(line, sizeof(line), f)) {+ char k[160], types[256] = "";+ int r, d, v;+ if (sscanf(line, "%159s row=%d mvd=%d mv=%d off=%255s", k, &r, &d, &v,+ types) < 4 ||+ strcmp(k, key) != 0)+ continue;+ /* the last line of the key counts */+ found = 1;+ row = r;+ mvd = d;+ mv = v;+ n_off = 0;+ for (char *p = types; *p && n_off < 16;) {+ off[n_off++] = atoi(p);+ p = strchr(p, ',');+ if (!p)+ break;+ p++;+ }+ if (strcmp(types, "-") == 0)+ n_off = 0;+ }+ fclose(f);+ if (!found)+ return 0;+ g->row_on &= row;+ g->mvd_on &= mvd;+ g->mv_on &= mv;+ flags[0] = row;+ flags[1] = mvd;+ flags[2] = mv;+ for (int i = 0; i < n_off; i++)+ g->off_types[g->n_off++] = off[i];+ return 1;+}++static void save(const struct janas_gpu *g, const char *key, int row, int mvd,+ int mv)+{+ char path[512];+ if (janas_cache_dir(path, sizeof(path) - 32, 1) != 0)+ return;+ strcat(path, "/gpu_selfcheck");+ FILE *f = fopen(path, "a");+ if (!f)+ return;+ fprintf(f, "%s row=%d mvd=%d mv=%d off=", key, row, mvd, mv);+ for (int i = 0; i < g->n_off; i++)+ fprintf(f, "%s%d", i ? "," : "", g->off_types[i]);+ fprintf(f, "%s\n", g->n_off ? "" : "-");+ fclose(f);+}++int janas_g_type_off(const struct janas_gpu *g, int type)+{+ for (int i = 0; i < g->n_off; i++)+ if (g->off_types[i] == type)+ return 1;+ return 0;+}++static const char *type_name(int type)+{+ switch (type) {+ case JANAS_Q4_K:+ return "Q4_K";+ case JANAS_Q5_K:+ return "Q5_K";+ case JANAS_Q6_K:+ return "Q6_K";+ case JANAS_Q6_K_P:+ return "Q6_K_P";+ case JANAS_Q8_0:+ return "Q8_0";+ case JANAS_Q4_0:+ return "Q4_0";+ case JANAS_IQ4_NL:+ return "IQ4_NL";+ case JANAS_Q4_1:+ return "Q4_1";+ case JANAS_Q5_1:+ return "Q5_1";+ case JANAS_MXFP4:+ return "MXFP4";+ case JANAS_Q5_0:+ return "Q5_0";+ case JANAS_Q2_0:+ return "Q2_0";+ case JANAS_IQ4_XS:+ return "IQ4_XS";+ case JANAS_Q3_K:+ return "Q3_K";+ case JANAS_IQ3_S:+ return "IQ3_S";+ case JANAS_Q2_K:+ return "Q2_K";+ case JANAS_IQ2_XXS:+ return "IQ2_XXS";+ case JANAS_IQ2_XS:+ return "IQ2_XS";+ case JANAS_IQ2_S:+ return "IQ2_S";+ case JANAS_IQ3_XXS:+ return "IQ3_XXS";+ case JANAS_IQ1_S:+ return "IQ1_S";+ case JANAS_IQ1_M:+ return "IQ1_M";+ }+ return "?";+}++/* The line for the user: what the check left out (row, mvd, mv: 0 where+ it turned that family off). */+static void make_note(struct janas_gpu *g, int row, int mvd, int mv)+{+ char *p = g->check_note;+ size_t len = sizeof(g->check_note);+ int n = 0;+ p[0] = 0;+ const char *fam[3] = {"the subgroup-per-row shaders",+ "the lanes-on-vectors shaders", "q4k_mv"};+ int off[3] = {!row, !mvd, !mv};+ for (int i = 0; i < 3; i++)+ if (off[i] && (size_t)n < len)+ n +=+ snprintf(p + n, len - (size_t)n, "%s%s", n ? ", " : "", fam[i]);+ for (int i = 0; i < g->n_off && (size_t)n < len; i++)+ n += snprintf(p + n, len - (size_t)n, "%s%s",+ i ? " "+ : n ? "; on the CPU: "+ : "on the CPU: ",+ type_name(g->off_types[i]));+ if (p[0] && (size_t)n < len)+ snprintf(p + n, len - (size_t)n, " (wrong on this driver)");+}++/* Every product once; the wrong ones in bad (up to max), their count+ returned. Only the cases in only (n_only of them), where given. */+static int run_all(struct janas_gpu *g, struct janas_pool *pool, uint8_t *w,+ const struct janas_block_q8k *x, float *ya, float *yb,+ const struct wrong *only, int n_only, struct wrong *bad,+ int max)+{+ int n = 0;+ for (int k = 0; k < N_KINDS; k++)+ for (int c = 0; c < 1 + kinds[k].tail; c++)+ for (int v = 0; v < N_NVS; v++) {+ size_t cols = c ? MAXC : 512, nv = nvs[v];+ int want = !only;+ for (int i = 0; only && i < n_only; i++)+ want |= only[i].k == k && only[i].cols == cols &&+ only[i].nv == nv;+ if (!want)+ continue;+ char sh[96];+ if (check_one(g, pool, w + (size_t)k * SLOT, x, ya, yb, k, cols,+ nv, sh, sizeof(sh)) == 0) {+ if (n < max)+ bad[n] = (struct wrong){.k = k, .cols = cols, .nv = nv};+ if (n < max)+ snprintf(bad[n].shader, sizeof(bad[n].shader), "%s",+ sh);+ n++;+ }+ }+ return n < max ? n : max;+}++void janas_g_selfcheck(struct janas_gpu *g)+{+ const char *e = getenv("JANAS_GPU_SELFCHECK");+ int mode = e ? atoi(e) : 1;+ if (mode == 0 || g->broken)+ return;+ char key[160];+ cache_key(g, key, sizeof(key));+ int flags[3];+ if (mode == 1 && load(g, key, flags)) {+ make_note(g, flags[0], flags[1], flags[2]);+ return;+ }+ struct timespec t0, t1;+ clock_gettime(CLOCK_MONOTONIC, &t0);+ size_t wbytes = (size_t)N_KINDS * SLOT;+ uint8_t *w = janas_aligned_alloc(4096, wbytes);+ size_t nbx = (MAXC + JANAS_QK - 1) / JANAS_QK;+ struct janas_block_q8k *x = malloc(NV_MAX * nbx * sizeof(*x));+ float *ya = malloc(ROWS * NV_MAX * sizeof(float)),+ *yb = malloc(ROWS * NV_MAX * sizeof(float)), xf[3 * JANAS_QK];+ struct janas_pool *pool = janas_pool_create(2, NULL);+ if (!w || !x || !ya || !yb || !pool)+ goto done;+ for (int k = 0; k < N_KINDS; k++)+ fill(&kinds[k], w + (size_t)k * SLOT, SLOT);+ for (size_t v = 0; v < NV_MAX; v++) {+ /* the activations of a row ending inside a group: its last group+ whole, past the row's end zero (model_ffn.c) */+ for (size_t i = 0; i < nbx * JANAS_QK; i++)+ xf[i] =+ i < MAXC ? (float)((int)(rnd() % 2001) - 1000) / 500.0f : 0.0f;+ janas_q8k_quantize(xf, x + v * nbx, nbx * JANAS_QK);+ }+ if (janas_gpu_import(g, w, wbytes) != 0)+ goto done;+ int was_row = g->row_on, was_mvd = g->mvd_on, was_mv = g->mv_on;+ enum { MAXW = 64 };+ struct wrong bad[MAXW];+ int n = run_all(g, pool, w, x, ya, yb, NULL, 0, bad, MAXW);+ if (n) {+ /* the families with another shader for the same products off, as+ far as one of theirs was wrong; then again */+ for (int i = 0; i < n; i++) {+ if (strstr(bad[i].shader, "_row "))+ g->row_on = 0;+ if (strstr(bad[i].shader, "_mvd "))+ g->mvd_on = 0;+ if (strstr(bad[i].shader, "q4k_mv "))+ g->mv_on = 0;+ }+ struct wrong still[MAXW];+ int m = run_all(g, pool, w, x, ya, yb, bad, n, still, MAXW);+ /* what is still wrong: the type to the CPU */+ for (int i = 0; i < m; i++)+ if (!janas_g_type_off(g, kinds[still[i].k].type) && g->n_off < 16)+ g->off_types[g->n_off++] = kinds[still[i].k].type;+ make_note(g, g->row_on || !was_row, g->mvd_on || !was_mvd,+ g->mv_on || !was_mv);+ fprintf(stderr,+ "gpu: %d of the GPU's products differ from the CPU's on this "+ "driver (first: %s %s, %zu wide, %zu vectors); left out: "+ "%s (JANAS_GPU_SELFCHECK=0 keeps them, gpu_check tells "+ "more)\n",+ n, bad[0].shader, type_name(kinds[bad[0].k].type), bad[0].cols,+ bad[0].nv, g->check_note);+ }+ save(g, key, g->row_on || !was_row, g->mvd_on || !was_mvd,+ g->mv_on || !was_mv);+ janas_g_drop_region(g, w);+ clock_gettime(CLOCK_MONOTONIC, &t1);+ if (g->verbose || getenv("JANAS_GPU_VERBOSE"))+ fprintf(stderr, "gpu self-check: %d wrong, %.0f ms (%s)\n", n,+ (t1.tv_sec - t0.tv_sec) * 1e3 + (t1.tv_nsec - t0.tv_nsec) / 1e6,+ g->check_note[0] ? g->check_note : "all right");+done:+ janas_pool_destroy(pool);+ janas_aligned_free(w);+ free(x);+ free(ya);+ free(yb);+}++const char *janas_gpu_check_note(const struct janas_gpu *g)+{+ return g->check_note;+}++#else++const char *janas_gpu_check_note(const struct janas_gpu *g)+{+ (void)g;+ return "";+}++#endifdiff --git a/src/llm/gpu_token.c b/src/llm/gpu_token.cindex 644d9dc..64350d3 100644--- a/src/llm/gpu_token.c+++ b/src/llm/gpu_token.c@@ -34,6 +34,7 @@#include "moe_topk.h" /* janas_spv_moe_topk */#include "moe_gather.h" /* janas_spv_moe_gather */#include "moe_combine.h" /* janas_spv_moe_combine */+#include "moe_plan.h" /* janas_spv_moe_plan */#include "silu_mul.h" /* janas_spv_silu_mul *//* The attention's first step in subgroups of 16 where the size may be@@ -52,13 +53,29 @@ static const char *const ph_name[PH_N] = {"norm+quant", "q k v","down", "output","ssm q k v z", "ssm beta+alpha","ssm conv", "ssm recurrence",- "ssm gate"};+ "ssm gate", "router",+ "experts' sum", "between segs"};int janas_gpu_shares_host(const struct janas_gpu *g){return g && !g->broken && !g->discrete && g->host_import &&!getenv("JANAS_GPU_COPY");}+size_t janas_gpu_local_room(const struct janas_gpu *g)+{+ if (!g || g->broken || !g->discrete)+ return 0;+ /* the largest heap of the card's own memory, 85% of it: the desktop+ and other programs keep the rest */+ VkDeviceSize heap = 0;+ for (uint32_t i = 0; i < g->mp.memoryHeapCount; i++)+ if ((g->mp.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) &&+ g->mp.memoryHeaps[i].size > heap)+ heap = g->mp.memoryHeaps[i].size;+ VkDeviceSize keep = heap / 100 * 85;+ return keep > g->local_bytes ? (size_t)(keep - g->local_bytes) : 0;+}+/* The GPU's address of host memory in one of its regions, 0 if none. */VkDeviceAddress janas_gt_addr_of(const struct janas_gpu *g, const void *p,size_t bytes)@@ -284,14 +301,45 @@ static int check_model(const struct janas_gpu *g,return 0;}+/* The token's buffer, bytes long: host-visible where the GPU shares the+ host's memory; on a card with memory of its own in that memory, where+ the shaders read and write it at its speed, the host's few writes (the+ block's inputs, the parameters) and reads (logits, hidden states, a+ mixture's routing) through a mirror (JANAS_GPU_TOK_HOSTBUF=1: as on an+ integrated GPU, across the bus; JANAS_GPU_TOK_MIRROR=1: the mirror on an+ integrated one too, to test it with weights read in place). */+static int make_buf(struct janas_gpu *g, struct janas_gpu_tok *t, size_t bytes)+{+ if ((janas_gpu_shares_host(g) && !getenv("JANAS_GPU_TOK_MIRROR")) ||+ getenv("JANAS_GPU_TOK_HOSTBUF"))+ return janas_g_make_stage(g, &t->buf, bytes, 1);+ struct stage d;+ if (janas_g_make_local(g, &d, bytes) != 0)+ return janas_g_make_stage(g, &t->buf, bytes, 1);+ if (janas_g_make_stage(g, &t->mirror, bytes, 1) != 0) {+ g->vkDestroyBuffer(g->dev, d.buf, NULL);+ g->vkFreeMemory(g->dev, d.mem, NULL);+ return -1;+ }+ t->buf = d;+ t->buf.map = t->mirror.map;+ t->buf.coherent = 0; /* every range through janas_gt_sync_range */+ t->local = 1;+ if (getenv("JANAS_GPU_VERBOSE"))+ fprintf(stderr,+ "gpu token buffer: %zu MiB in the GPU's memory, through a "+ "mirror\n",+ bytes >> 20);+ return 0;+}+struct janas_gpu_tok *janas_gpu_tok_create(struct janas_gpu *g,const struct janas_gpu_tok_model *md,char *err, size_t err_len){- if (!janas_gpu_shares_host(g)) {- fail(err, err_len, "the GPU does not read the host's memory in place");- return NULL;- }+ /* on a card with memory of its own the caller keeps the state the+ same in both places (model_gpu.c sync_state): every buffer named in+ md needs a region all the same, which check_model sees */if (check_model(g, md, err, err_len) != 0)return NULL;struct janas_gpu_tok *t = calloc(1, sizeof(*t));@@ -408,8 +456,23 @@ struct janas_gpu_tok *janas_gpu_tok_create(struct janas_gpu *g,t->o_inv = take(&at, np * 4);t->tiles1 = 3 * 2 * ne;t->tiles2 = 3 * ne;- t->o_tab1 = take(&at, (t->tiles1 + 2 * (np / 16 + ne)) * 16);- t->o_tab2 = take(&at, (t->tiles2 + np / 16 + ne) * 16);+ /* blocks of mvid_n tokens or fewer in tiles of MVID_NV vectors: by+ default all of them, so that an expert's bits are the CPU's in+ every block - a prompt with a tier of 300 MB went through *_mmid+ (not the CPU's bits) on the GPU's experts and through the CPU on+ the others, the split set by what the tier held, and with drafts+ and without the tokens parted at the 11th; on the laptop's Arc+ as a card of its own the prompt was 7% faster with every expert+ in the tier, 3% slower with 300 MB (24 Oct 2026) */+ const char *mv = getenv("JANAS_GPU_MVID");+ t->mvid_n = mv ? (uint32_t)atoi(mv) : (uint32_t)nb;+ if (t->mvid_n > nb)+ t->mvid_n = (uint32_t)nb;+ size_t nt = np / 16, nm = (size_t)t->mvid_n * md->k / MVID_NV;+ if (nm > nt)+ nt = nm;+ t->o_tab1 = take(&at, (t->tiles1 + 2 * (nt + ne)) * 16);+ t->o_tab2 = take(&at, (t->tiles2 + nt + ne) * 16);t->o_ind = take(&at, 32);t->cnt = calloc(ne + 1, sizeof(uint32_t));t->fill = calloc(ne, sizeof(uint32_t));@@ -450,7 +513,7 @@ struct janas_gpu_tok *janas_gpu_tok_create(struct janas_gpu *g,/* cached for the host (the logits are read on the CPU: 2.5 ms a tokenfrom uncached memory on the development laptop), flushed andinvalidated by hand where it is not coherent */- if (janas_g_make_stage(g, &t->buf, at, 1) != 0 ||+ if (make_buf(g, t, at) != 0 ||janas_g_make_pipeline(g, janas_spv_norm_quant,sizeof(janas_spv_norm_quant), &t->sm_nq, &t->p_nq,0) != VK_SUCCESS ||@@ -497,6 +560,11 @@ struct janas_gpu_tok *janas_gpu_tok_create(struct janas_gpu *g,janas_gpu_tok_destroy(g, t);return NULL;}+ /* the plan on the GPU when the tier holds every expert (optional) */+ if (ne &&+ janas_g_make_pipeline(g, janas_spv_moe_plan, sizeof(janas_spv_moe_plan),+ &t->sm_pl, &t->p_pl, 0) != VK_SUCCESS)+ t->p_pl = VK_NULL_HANDLE;/* zero: the padding of the feed-forwards' widths is never written */memset(t->buf.map, 0, at);janas_gt_sync_range(g, t, 0, at, 1);@@ -550,6 +618,20 @@ struct janas_gpu_tok *janas_gpu_tok_create(struct janas_gpu *g,if (g->vkCreateQueryPool(g->dev, &qi, NULL, &t->qp) != VK_SUCCESS)t->qp = VK_NULL_HANDLE;}+ /* a mixture larger than the card: its tier of experts */+ if (ne && md->tier_bytes) {+ if (!md->fetch || !md->cpu_rows || !md->slot_bytes ||+ janas_gt_tier_create(g, t, md) != 0) {+ fail(err, err_len, "no room for a tier of experts on the card");+ janas_gpu_tok_destroy(g, t);+ return NULL;+ }+ if (getenv("JANAS_GPU_VERBOSE"))+ fprintf(stderr,+ "gpu experts: a tier of %u slots (%zu MiB) in the "+ "card's memory\n",+ t->n_slot, (t->n_slot * t->slot_bytes) >> 20);+ }return t;}@@ -570,6 +652,9 @@ void janas_gpu_tok_destroy(struct janas_gpu *g, struct janas_gpu_tok *t)(double)t->pass_tokens / t->tokens, all / t->tokens * 1e-6,t->wall_ns / t->tokens * 1e-6,t->copy_ns / t->tokens * 1e-6);+ if (t->plan_ns > 0)+ fprintf(stderr, " (the CPU's plans %.3f ms a pass)\n",+ t->plan_ns / t->tokens * 1e-6);for (int i = 0; i < PH_N; i++)fprintf(stderr, " %-14s %7.3f ms %5.1f%%\n", ph_name[i],t->ph_ns[i] / t->tokens * 1e-6,@@ -584,20 +669,22 @@ void janas_gpu_tok_destroy(struct janas_gpu *g, struct janas_gpu_tok *t)g->vkFreeCommandBuffers(g->dev, g->cp, 1, &t->cb[n][a][h]);janas_gt_free_segments(g, t, &t->seg_cb[n][a][h]);}- VkPipeline p[15] = {t->p_nq, t->p_qk, t->p_at, t->p_am, t->p_gc,- t->p_gs, t->p_gl, t->p_gr, t->p_gg, t->p_tk,- t->p_ga, t->p_cb, t->p_sm, t->p_gb, t->p_gx};- VkShaderModule s[15] = {t->sm_nq, t->sm_qk, t->sm_at, t->sm_am, t->sm_gc,- t->sm_gs, t->sm_gl, t->sm_gr, t->sm_gg, t->sm_tk,- t->sm_ga, t->sm_cb, t->sm_sm, t->sm_gb, t->sm_gx};+ VkPipeline p[16] = {t->p_nq, t->p_qk, t->p_at, t->p_am, t->p_gc, t->p_gs,+ t->p_gl, t->p_gr, t->p_gg, t->p_tk, t->p_ga, t->p_cb,+ t->p_sm, t->p_gb, t->p_gx, t->p_pl};+ VkShaderModule s[16] = {t->sm_nq, t->sm_qk, t->sm_at, t->sm_am,+ t->sm_gc, t->sm_gs, t->sm_gl, t->sm_gr,+ t->sm_gg, t->sm_tk, t->sm_ga, t->sm_cb,+ t->sm_sm, t->sm_gb, t->sm_gx, t->sm_pl};janas_gt_hc_destroy(g, t);janas_gt_qsa_destroy(g, t);+ janas_gt_tier_destroy(g, t);free(t->par);free(t->cnt);free(t->fill);free(t->ids);free(t->slots);- for (int i = 0; i < 15; i++) {+ for (int i = 0; i < 16; i++) {if (p[i])g->vkDestroyPipeline(g->dev, p[i], NULL);if (s[i])@@ -607,14 +694,52 @@ void janas_gpu_tok_destroy(struct janas_gpu *g, struct janas_gpu_tok *t)g->vkDestroyBuffer(g->dev, t->buf.buf, NULL);if (t->buf.mem)g->vkFreeMemory(g->dev, t->buf.mem, NULL);+ if (t->mirror.buf)+ g->vkDestroyBuffer(g->dev, t->mirror.buf, NULL);+ if (t->mirror.mem)+ g->vkFreeMemory(g->dev, t->mirror.mem, NULL);free(t);}+/* Bytes [off, off + bytes) of the mirror out of the CPU's caches (flush)+ or reloaded from memory (!flush), where it is not coherent. */+void janas_gt_mirror_cache(struct janas_gpu *g, struct janas_gpu_tok *t,+ size_t off, size_t bytes, int flush)+{+ if (t->mirror.coherent)+ return;+ VkDeviceSize a = off / g->atom * g->atom,+ e = (off + bytes + g->atom - 1) / g->atom * g->atom;+ VkMappedMemoryRange mr = {.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,+ .memory = t->mirror.mem,+ .offset = a,+ .size =+ e >= t->mirror.bytes ? VK_WHOLE_SIZE : e - a};+ if (flush)+ g->vkFlushMappedMemoryRanges(g->dev, 1, &mr);+ else+ g->vkInvalidateMappedMemoryRanges(g->dev, 1, &mr);+}+/* Bytes [off, off + bytes) of the token's buffer out of the CPU's caches- (flush) or reloaded from memory (!flush), where it is not coherent. */+ (flush) or reloaded from memory (!flush), where it is not coherent; on a+ card with memory of its own copied too, a submission each. */void janas_gt_sync_range(struct janas_gpu *g, struct janas_gpu_tok *t,size_t off, size_t bytes, int flush){+ if (t->local) {+ /* the mirror's own cache lines, then the copy; or the other way */+ if (flush) {+ janas_gt_mirror_cache(g, t, off, bytes, 1);+ if (janas_g_copy(g, t->mirror.buf, off, t->buf.buf, off, bytes))+ g->broken = 1;+ } else {+ if (janas_g_copy(g, t->buf.buf, off, t->mirror.buf, off, bytes))+ g->broken = 1;+ janas_gt_mirror_cache(g, t, off, bytes, 0);+ }+ return;+ }if (t->buf.coherent)return;VkDeviceSize a = off / g->atom * g->atom;@@ -753,11 +878,63 @@ static int record(struct janas_gpu *g, struct janas_gpu_tok *t,if (g->vkBeginCommandBuffer(g->cb, &bb) != VK_SUCCESS)return -1;VkPipeline bound = VK_NULL_HANDLE;- t->prof = seg < 0; /* a pass in one submission, of any length */- if (t->qp && t->prof) {+ /* a pass in one submission, of any length; or a mixture's segments,+ recorded one after the other the first time a block runs, their+ timestamps one series */+ t->prof = 1;+ if (t->qp && seg <= 0) {t->n_ts = 0;g->vkCmdResetQueryPool(g->cb, t->qp, 0, MAX_TS);janas_gt_stamp(g, t, PH_N); /* the start */+ } else if (t->qp) {+ janas_gt_stamp(g, t, PH_BETWEEN);+ }+ /* on a card with memory of its own: the pass's inputs from the mirror,+ in its own commands (the sizes fixed by n, the values read at run+ time) - a mixture's in its first submission, its outputs in its last,+ and between the two what the CPU's plan writes and reads */+ int last = seg < 0 || seg == (int)md->n_layer;+ if (t->local && seg <= 0) {+ VkBufferCopy in[3] = {+ {.srcOffset = t->o_x,+ .dstOffset = t->o_x,+ .size = (VkDeviceSize)n * md->dm * sizeof(float)},+ {.srcOffset = t->o_rope,+ .dstOffset = t->o_rope,+ .size = (VkDeviceSize)n * md->n_rot * sizeof(float)},+ {.srcOffset = t->o_pos,+ .dstOffset = t->o_pos,+ .size = JANAS_GT_POS_BYTES}};+ g->vkCmdCopyBuffer(g->cb, t->mirror.buf, t->buf.buf, 3, in);+ VkMemoryBarrier ib = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,+ .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,+ .dstAccessMask = VK_ACCESS_SHADER_READ_BIT |+ VK_ACCESS_SHADER_WRITE_BIT};+ g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_TRANSFER_BIT,+ VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &ib,+ 0, NULL, 0, NULL);+ }+ if (seg > 0 && md->n_expert) {+ /* the previous layer's plan (gpu_token_moe.c): its rows' tokens, its+ pairs' rows, its tables and dispatches' sizes, laid out one after+ the other; and the experts read into the tier, or the CPU's rows,+ copied in a submission of their own before this one */+ if (t->local) {+ VkBufferCopy tb = {.srcOffset = t->o_gt,+ .dstOffset = t->o_gt,+ .size = t->o_ind + 32 - t->o_gt};+ g->vkCmdCopyBuffer(g->cb, t->mirror.buf, t->buf.buf, 1, &tb);+ }+ VkMemoryBarrier pb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,+ .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,+ .dstAccessMask =+ VK_ACCESS_SHADER_READ_BIT |+ VK_ACCESS_SHADER_WRITE_BIT |+ VK_ACCESS_INDIRECT_COMMAND_READ_BIT};+ g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_TRANSFER_BIT,+ VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |+ VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,+ 0, 1, &pb, 0, NULL, 0, NULL);}uint32_t nl = md->n_layer;if (md->hc) {@@ -798,6 +975,9 @@ static int record(struct janas_gpu *g, struct janas_gpu_tok *t,: janas_gt_rec_ffn(g, t, &bound, &ly->gate, &ly->up,&ly->down, n, YD)) != 0)return -1;+ /* a mixture's pass in one submission: the plan on the GPU */+ if (seg < 0 && md->n_expert)+ janas_gt_rec_plan(g, t, md, l, &bound, n);}/* the last feed-forward added, the final norm, the output head fortokens first .. n - 1 (none: the residual left in X, the next block's@@ -822,9 +1002,48 @@ static int record(struct janas_gpu *g, struct janas_gpu_tok *t,VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,.dstAccessMask = VK_ACCESS_HOST_READ_BIT};- g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,- VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &mb, 0, NULL, 0,- NULL);+ VkPipelineStageFlags from = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;+ if (t->local) {+ /* the logits and the hidden states back to the mirror (qwen4exp's:+ the streams after the last layer); before, layer seg's choice of+ experts for the plan, and with a tier the experts' input for the+ CPU's */+ VkBufferCopy out[2];+ uint32_t no = 0;+ size_t ho = md->hc ? t->o_res : t->o_hid,+ hb = (size_t)n * (md->hc ? md->hc : 1) * md->dm * 4;+ if (last && first < n)+ out[no++] = (VkBufferCopy){.srcOffset = t->o_logits,+ .dstOffset = t->o_logits,+ .size = (VkDeviceSize)(n - first) *+ md->n_vocab * 4};+ if (last && hid)+ out[no++] =+ (VkBufferCopy){.srcOffset = ho, .dstOffset = ho, .size = hb};+ if (!last) {+ out[no++] = (VkBufferCopy){.srcOffset = t->o_pe,+ .dstOffset = t->o_pe,+ .size = (VkDeviceSize)n * md->k * 4};+ if (t->n_slot)+ out[no++] =+ (VkBufferCopy){.srcOffset = t->o_xnf,+ .dstOffset = t->o_xnf,+ .size = (VkDeviceSize)n * md->dm * 4};+ }+ if (no) {+ VkMemoryBarrier ob = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,+ .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,+ .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT};+ g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,+ VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &ob,+ 0, NULL, 0, NULL);+ g->vkCmdCopyBuffer(g->cb, t->buf.buf, t->mirror.buf, no, out);+ mb.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;+ from = VK_PIPELINE_STAGE_TRANSFER_BIT;+ }+ }+ g->vkCmdPipelineBarrier(g->cb, from, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &mb,+ 0, NULL, 0, NULL);if (g->vkEndCommandBuffer(g->cb) != VK_SUCCESS)return -1;return 0;@@ -936,18 +1155,28 @@ int janas_gpu_tok_run(struct janas_gpu *g, struct janas_gpu_tok *t,write, the tokens to replay, the log's row: read by the shaders atrun time */uint32_t pw[5] = {pos, src, dst, replay, log_row};+ _Static_assert(sizeof(pw) == JANAS_GT_POS_BYTES, "the pass's words");memcpy(t->buf.map + t->o_pos, pw, sizeof(pw));- janas_gt_sync_range(g, t, t->o_x, (size_t)n * md->dm * sizeof(float), 1);- janas_gt_sync_range(g, t, t->o_rope, (size_t)n * md->n_rot * sizeof(float),- 1);- janas_gt_sync_range(g, t, t->o_pos, sizeof(pw), 1);+ /* the pass copies them itself (record: a mixture's first and last+ submissions) */+ void (*in)(struct janas_gpu *, struct janas_gpu_tok *, size_t, size_t,+ int) = t->local ? janas_gt_mirror_cache : janas_gt_sync_range;+ in(g, t, t->o_x, (size_t)n * md->dm * sizeof(float), 1);+ in(g, t, t->o_rope, (size_t)n * md->n_rot * sizeof(float), 1);+ in(g, t, t->o_pos, sizeof(pw), 1);int ai = all == 1 && n == 1 ? 0 : all, hi = hidden != NULL;struct timespec w0, w1;clock_gettime(CLOCK_MONOTONIC, &w0);/* a failure once the GPU has started: with a replay the base itupdates in place is no longer the state the CPU would replay on */int failed = replay ? -2 : -1;- if (md->n_expert) {+ /* a mixture whose experts the tier holds all: read in at its first+ pass, then its passes whole, unless the router's choices are+ watched */+ if (md->n_expert && t->n_slot && !t->gplan_tried &&+ janas_gt_tier_fill(g, t, md) != 0 && g->broken)+ return -1;+ if (md->n_expert && (!t->gplan || (md->watched && md->watched(md->ctx)))) {if (janas_gt_run_segments(g, t, md, n, first, ai, hi, nch) != 0)return failed;} else {@@ -968,14 +1197,14 @@ int janas_gpu_tok_run(struct janas_gpu *g, struct janas_gpu_tok *t,clock_gettime(CLOCK_MONOTONIC, &w1);size_t lb = (size_t)(n - first) * md->n_vocab * 4;if (lb) {- janas_gt_sync_range(g, t, t->o_logits, lb, 0);+ in(g, t, t->o_logits, lb, 0);memcpy(logits, t->buf.map + t->o_logits, lb);}if (hidden) {/* qwen4exp's: the streams after the last layer, hc dm a token */size_t hb = (size_t)n * (md->hc ? md->hc : 1) * md->dm * 4,ho = md->hc ? t->o_res : t->o_hid;- janas_gt_sync_range(g, t, ho, hb, 0);+ in(g, t, ho, hb, 0);memcpy(hidden, t->buf.map + ho, hb);}struct timespec w2;@@ -1064,4 +1293,10 @@ int janas_gpu_shares_host(const struct janas_gpu *g)return 0;}+size_t janas_gpu_local_room(const struct janas_gpu *g)+{+ (void)g;+ return 0;+}+#endifdiff --git a/src/llm/gpu_token.h b/src/llm/gpu_token.hindex 2de43fb..2534ffe 100644--- a/src/llm/gpu_token.h+++ b/src/llm/gpu_token.h@@ -118,6 +118,22 @@ struct janas_gpu_tok_model {caller goes on (or NULL): a block of many tokens chooses nearly allof them, so none is read for nothing */void (*prefetch)(void *ctx, uint32_t layer);+ /* whether routed must see each layer's choice (or NULL): then a layer+ at a time even when the card holds every expert */+ int (*watched)(void *ctx);+ /* a card whose memory does not hold all the experts (tier_bytes > 0):+ a tier of slots of slot_bytes in its memory keeps those used most,+ read into it from the slots fetch gives; the others computed by+ cpu_rows, layer's experts ids[0..nd) of slots[], their rows+ [ra[i], rb[i]) of out (dm floats each, before their weights), row r+ a pair of token tok[r], from xn (n tokens of dm floats, the+ feed-forward's normalized input): 0, or -1 (the block goes to the+ CPU) */+ size_t tier_bytes, slot_bytes;+ int (*cpu_rows)(void *ctx, uint32_t layer, uint32_t n, const float *xn,+ uint32_t nd, const uint32_t *ids,+ const uint8_t *const *slots, const uint32_t *ra,+ const uint32_t *rb, const uint32_t *tok, float *out);void *ctx;/* qwen4exp: hc streams of the residual (0: one, the others), themixes' rank, the output's mix (no inject); ng_layer the n-gramdiff --git a/src/llm/gpu_token_hc.c b/src/llm/gpu_token_hc.cindex 5f31ea9..29821a8 100644--- a/src/llm/gpu_token_hc.c+++ b/src/llm/gpu_token_hc.c@@ -40,9 +40,14 @@ void janas_gt_rec_dense(struct janas_gpu *g, VkPipeline *bound, int kind,janas_g_push_addr(pd.w, w);janas_g_push_addr(pd.x, x);janas_g_push_addr(pd.y, y);- janas_gt_bind(g, bound, g->pipe_dm[kind]);+ /* a workgroup a row on a few vectors (dense_mv.comp), tiles else */+ int mv = n <= DENSE_MV_MAX && g->pipe_dv[kind];+ janas_gt_bind(g, bound, mv ? g->pipe_dv[kind] : g->pipe_dm[kind]);janas_gt_push(g, &pd, sizeof(pd));- g->vkCmdDispatch(g->cb, (rows + 63) / 64, (n + 63) / 64, 1);+ if (mv)+ g->vkCmdDispatch(g->cb, rows, (n + 3) / 4, 1);+ else+ g->vkCmdDispatch(g->cb, (rows + 63) / 64, (n + 63) / 64, 1);}/* x rounded to bf16 into xb (silu 0), or SiLU(x) in place and that intodiff --git a/src/llm/gpu_token_impl.h b/src/llm/gpu_token_impl.hindex b590ef8..5dc0fc5 100644--- a/src/llm/gpu_token_impl.h+++ b/src/llm/gpu_token_impl.h@@ -97,6 +97,11 @@ struct push_cb {uint32_t dout[2], pw[2], inv[2], sh[2], shw[2], y[2];uint32_t dm, k, has_sh;};+struct push_pl { /* moe_plan.comp; offsets in bytes from b */+ uint32_t at[2], b[2];+ uint32_t pe, gt, inv, t1, t2, ind, xg, egu, eh, dout;+ uint32_t n, k, ne, nbd, nbf, ep, dm, eff, s1, s2, tiles1, tiles2;+};struct push_id {uint32_t t[2];uint32_t rows, nb, tiles;@@ -158,6 +163,7 @@ _Static_assert(sizeof(struct push_at) <= JANAS_G_PUSH_BYTES, "push size");shared memory takes: the query heads of a KV head times hd, and times arun. */#define ATT_RUN 128+#define JANAS_GT_POS_BYTES 20 /* the pass's words at o_pos */#define ATT_MAXQ 2048#define ATT_MAXS 1024/* ... larger where a KV head has more than 8 query heads (attn_decode.comp@@ -189,7 +195,9 @@ _Static_assert(sizeof(struct push_cb) <= JANAS_G_PUSH_BYTES, "push size");#define TOK_PART_BUDGET ((size_t)64 << 20)/* JANAS_GPU_TOKEN_PROFILE=1: the GPU's own timestamps after each phase of- the token, summed per phase and printed when the model is closed */+ the token, summed per phase and printed when the model is closed; a+ mixture's segments too, with the time between two of them (the CPU's+ plan, the submission's round trip) as a phase of its own */enum {PH_NORM,PH_QKV,@@ -209,6 +217,11 @@ enum {PH_SSM_CONV,PH_SSM_REC,PH_SSM_GATE,+ /* a mixture's: the router's choice, the experts' weighted sum, the+ time from a segment's last timestamp to the next one's start */+ PH_ROUTER,+ PH_COMBINE,+ PH_BETWEEN,PH_N};#define MAX_TS 1024@@ -223,14 +236,15 @@ struct janas_gpu_tok {/* [n][all][hidden wanted] */VkCommandBuffer cb[TOK_MAX_BLOCK + 1][3][2];uint32_t cb_nch[TOK_MAX_BLOCK + 1][3]- [2]; /* the runs each was recorded for */- int prof; /* timestamps in the commands being recorded (n = 1) */- VkQueryPool qp; /* timestamps, if profiling */- uint32_t n_ts; /* written this token */+ [2]; /* the runs each was recorded for */+ int prof; /* timestamps in the commands being recorded */+ VkQueryPool qp; /* timestamps, if profiling */+ uint32_t n_ts; /* written this token */uint8_t ph_of[MAX_TS]; /* the phase each timestamp closes */double ph_ns[PH_N];double wall_ns; /* submission to the fence, as the CPU sees it */double copy_ns; /* the logits copied out */+ double plan_ns; /* a mixture's plans, on the CPU */uint64_t tokens; /* with profiling: passes measured */uint64_t pass_tokens; /* ... and their tokens */VkShaderModule sm_nq, sm_qk, sm_at, sm_am;@@ -241,6 +255,11 @@ struct janas_gpu_tok {VkPipeline p_gc, p_gs, p_gl, p_gr, p_gg, p_gb, p_gx;size_t *par; /* per layer: its parameters' offsets in buf (PAR_*) */struct stage buf; /* every intermediate, one allocation */+ /* on a card with memory of its own buf is in that memory (its map this+ mirror's, host-visible): janas_gt_sync_range copies the ranges the+ host writes or reads between the two */+ struct stage mirror;+ int local;/* offsets into buf, each nb tokens' worth */uint32_t nch; /* the attention's runs of positions, for the cache */size_t part_floats; /* room for the runs of a block, floats */@@ -272,11 +291,44 @@ struct janas_gpu_tok {blocks of 32, where a down matrix is in blocks of 32 */size_t o_hf, o_h32;uint32_t tiles1, tiles2;+ /* blocks of this many tokens or fewer (by default all): the experts'+ products whose types have one through the matvec shaders (gpu.c+ pipe_mvid, bit for bit with the CPU's), not *_mmid's tiles+ (JANAS_GPU_MVID=<n>, 0 never) */+ uint32_t mvid_n;uint32_t n_seg; /* n_layer + 1 *//* the CPU's side: the experts' counts and fill, those chosen, theirslots */uint32_t *cnt, *fill, *ids;const uint8_t **slots;+ /* a card whose memory does not hold the experts (gpu_token_moe.c): a+ tier of n_slot slots of slot_bytes in it with the experts used most,+ slot_of[layer * n_expert + e] (or -1), owner (layer * n_expert + e,+ or UINT32_MAX), uses[layer * n_expert + e] counted at each block,+ stamp the plan that last counted on a slot (plan_no: that plan's);+ the experts read into it through up (a stage of UP_SLOTS slots); per+ block the experts of a layer that the GPU computes (gpu ids, their+ tier slot), those that the CPU does, the misses asked of fetch */+#define TIER_PIECES 64+ struct stage tier[TIER_PIECES], up; /* the tier in pieces of per slots */+ uint32_t n_slot, n_free, scan, plan_no, per;+ size_t slot_bytes;+ int32_t *slot_of;+ uint32_t *owner, *uses, *stamp, *on_gpu, *gpu_slot, *on_cpu, *miss, *ra,+ *rb;+ const uint8_t **cpu_slots;+ /* a plan's copies in one submission: the last experts read into the+ tier, the CPU's rows */+ struct janas_g_cp *cps;+ uint64_t tier_hits, tier_loads, cpu_experts;+ /* every expert in the tier (janas_gt_tier_fill, tried once): the plan+ on the GPU, moe_plan.comp after each router, a mixture's pass in one+ submission; at: the experts' addresses in the tier, 3 uvec4 each,+ n_expert a layer */+ int gplan, gplan_tried;+ struct stage at;+ VkShaderModule sm_pl;+ VkPipeline p_pl;/* the feed-forwards' widths padded to whole groups of 256: theexperts' (effp) and the shared expert's or dense one's (ffp) */uint32_t effp, ffp;@@ -309,6 +361,11 @@ VkDeviceAddress janas_gt_addr_of(const struct janas_gpu *g, const void *p,size_t bytes);/* Bytes [off, off + bytes) of the token's buffer out of the CPU's caches(flush) or reloaded from memory (!flush), where it is not coherent. */+/* Bytes [off, off + bytes) of the mirror out of the CPU's caches (flush)+ or reloaded from memory (!flush), where it is not coherent: the copies+ between it and buf are in the pass's own commands. */+void janas_gt_mirror_cache(struct janas_gpu *g, struct janas_gpu_tok *t,+ size_t off, size_t bytes, int flush);void janas_gt_sync_range(struct janas_gpu *g, struct janas_gpu_tok *t,size_t off, size_t bytes, int flush);/* A command buffer for cb, allocated, and the commands of segment seg of a@@ -374,7 +431,9 @@ void janas_gt_rec_bf16(struct janas_gpu *g, VkPipeline *bound,VkDeviceAddress w, VkDeviceAddress x, VkDeviceAddress y,uint32_t rows, uint32_t cols, uint32_t ys, uint32_t n,float scale);-/* the same for w of dense_mm.comp's kind: 1 bf16, 2 Q8_0 */+/* the same for w of dense_mm.comp's kind: 0 f32, 1 bf16, 2 Q8_0; on+ DENSE_MV_MAX vectors or fewer a workgroup a row (dense_mv.comp) */+#define DENSE_MV_MAX 8void janas_gt_rec_dense(struct janas_gpu *g, VkPipeline *bound, int kind,VkDeviceAddress w, VkDeviceAddress x, VkDeviceAddress y,uint32_t rows, uint32_t cols, uint32_t ys, uint32_t n,@@ -400,6 +459,18 @@ void janas_gt_rec_experts(struct janas_gpu *g, struct janas_gpu_tok *t,const struct janas_gpu_tok_model *md,const struct janas_gpu_tok_layer *ly,VkPipeline *bound, VkDeviceAddress YD, uint32_t n);+int janas_gt_tier_create(struct janas_gpu *g, struct janas_gpu_tok *t,+ const struct janas_gpu_tok_model *md);+void janas_gt_tier_destroy(struct janas_gpu *g, struct janas_gpu_tok *t);+/* Every expert read into a tier that holds them all, their addresses+ written for moe_plan.comp, t->gplan set: 0, or -1 (the plan stays the+ CPU's). */+int janas_gt_tier_fill(struct janas_gpu *g, struct janas_gpu_tok *t,+ const struct janas_gpu_tok_model *md);+/* Layer l's plan on the GPU from its router's choice (t->gplan) */+void janas_gt_rec_plan(struct janas_gpu *g, struct janas_gpu_tok *t,+ const struct janas_gpu_tok_model *md, uint32_t l,+ VkPipeline *bound, uint32_t n);void janas_gt_free_segments(struct janas_gpu *g, struct janas_gpu_tok *t,VkCommandBuffer **seg);int janas_gt_run_segments(struct janas_gpu *g, struct janas_gpu_tok *t,diff --git a/src/llm/gpu_token_moe.c b/src/llm/gpu_token_moe.cindex 7646075..fafe5d1 100644--- a/src/llm/gpu_token_moe.c+++ b/src/llm/gpu_token_moe.c@@ -7,8 +7,10 @@*/#include "gpu_token_impl.h"+#include <stdio.h>#include <stdlib.h>#include <string.h>+#include <time.h>#include "quant.h"@@ -42,17 +44,9 @@ int janas_gt_rec_router(struct janas_gpu *g, struct janas_gpu_tok *t,janas_gt_rec_ffn(g, t, bound, &ly->sh_gate_w, &ly->sh_up, &ly->sh_down,n, B + t->o_sh) != 0)return -1;- struct push_dm pd = {.rows = ne + (ly->sh_gate ? 1 : 0),- .cols = md->dm,- .nv = n,- .ys = ne + 1,- .scale = 1.0f};- janas_g_push_addr(pd.w, B + t->par[(size_t)l * PAR_N + PAR_R]);- janas_g_push_addr(pd.x, B + t->o_xnf);- janas_g_push_addr(pd.y, B + t->o_r);- janas_gt_bind(g, bound, g->pipe_dm[0]);- janas_gt_push(g, &pd, sizeof(pd));- g->vkCmdDispatch(g->cb, (pd.rows + 63) / 64, (n + 63) / 64, 1);+ janas_gt_rec_dense(g, bound, 0, B + t->par[(size_t)l * PAR_N + PAR_R],+ B + t->o_xnf, B + t->o_r, ne + (ly->sh_gate ? 1 : 0),+ md->dm, ne + 1, n, 1.0f);janas_g_compute_barrier(g);struct push_tk pk = {.ne = ne, .rs = ne + 1, .k = md->k, .n = n, .sh = ly->sh_gate != NULL};@@ -62,11 +56,58 @@ int janas_gt_rec_router(struct janas_gpu *g, struct janas_gpu_tok *t,janas_g_push_addr(pk.shw, B + t->o_shw);janas_gt_bind(g, bound, t->p_tk);janas_gt_push(g, &pk, sizeof(pk));- g->vkCmdDispatch(g->cb, (n + 63) / 64, 1, 1);+ g->vkCmdDispatch(g->cb, n, 1, 1); /* a workgroup a token */janas_g_compute_barrier(g);+ janas_gt_stamp(g, t, PH_ROUTER);return 0;}+/* The matvec with ID for an expert's matrix of type ty on a block of n+ tokens (gpu.c pipe_mvid: a few vectors an expert, read at the weights'+ pace and bit for bit with the CPU), or NULL: *_mmid's tiles, 64 rows by+ 16 vectors, most of them empty then. */+static VkPipeline by_rows(const struct janas_gpu *g,+ const struct janas_gpu_tok *t, int ty, uint32_t n)+{+ int i = ty == JANAS_Q3_K ? 0+ : ty == JANAS_Q4_K ? 1+ : ty == JANAS_Q6_K ? 2+ : ty == JANAS_Q6_K_P ? 3+ : ty == JANAS_MXFP4 ? 4+ : ty == JANAS_Q2_0 ? 5+ : -1;+ return n <= t->mvid_n && i >= 0 ? g->pipe_mvid[i] : VK_NULL_HANDLE;+}++/* An expert product's dispatch, indirect: a matvec's (rows in the+ workgroups' y) or a *_mmid's (in x), the table at tab, its tiles from+ uvec4 tiles on; cols the rows' weights, nb their groups of 256 with the+ one a row may end inside */+static void experts_product(struct janas_gpu *g, struct janas_gpu_tok *t,+ VkPipeline *bound, VkPipeline mv, VkPipeline id,+ VkDeviceAddress tab, uint32_t rows, uint32_t cols,+ uint32_t nb, uint32_t tiles, uint32_t ngroups,+ size_t ind)+{+ if (mv) {+ /* the whole groups and the tail's blocks of 32 (b32_matvec's) */+ struct push pm = {.rows = rows,+ .nb = cols / JANAS_QK,+ .nv = tiles,+ .tail = cols % JANAS_QK / 32};+ janas_g_push_addr(pm.w, tab);+ janas_gt_bind(g, bound, mv);+ janas_gt_push(g, &pm, sizeof(pm));+ } else {+ struct push_id pi = {+ .rows = rows, .nb = nb, .tiles = tiles, .ngroups = ngroups};+ janas_g_push_addr(pi.t, tab);+ janas_gt_bind(g, bound, id);+ janas_gt_push(g, &pi, sizeof(pi));+ }+ g->vkCmdDispatchIndirect(g->cb, t->buf.buf, ind);+}+/* A mixture's experts on the block whose router chose them (plan wrotetheir tables and sizes): the activations gathered in the experts'order, gate and up, the activation, down, then each token's weighted@@ -90,17 +131,15 @@ void janas_gt_rec_experts(struct janas_gpu *g, struct janas_gpu_tok *t,janas_gt_push(g, &pg, sizeof(pg));g->vkCmdDispatch(g->cb, np, 1, 1);janas_g_compute_barrier(g);- struct push_id pi = {- .rows = md->eff, .nb = nbd, .tiles = t->tiles1, .ngroups = md->dm / 32};- janas_g_push_addr(pi.t, B + t->o_tab1);- janas_gt_bind(g, bound,- gtp == JANAS_MXFP4 ? g->pipe_idmx- : gtp == JANAS_Q3_K ? g->pipe_id3- : gtp == JANAS_Q2_0 ? g->pipe_idb[2]- : g->pipe_id4);- janas_gt_push(g, &pi, sizeof(pi));- g->vkCmdDispatchIndirect(g->cb, t->buf.buf, t->o_ind);+ experts_product(g, t, bound, by_rows(g, t, gtp, n),+ gtp == JANAS_MXFP4 ? g->pipe_idmx+ : gtp == JANAS_Q3_K ? g->pipe_id3+ : gtp == JANAS_Q2_0 ? g->pipe_idb[2]+ : g->pipe_id4,+ B + t->o_tab1, md->eff, md->dm, nbd, t->tiles1, md->dm / 32,+ t->o_ind);janas_g_compute_barrier(g);+ janas_gt_stamp(g, t, PH_GATEUP);struct push_ac ac = {.blocks = np * nbf};janas_g_push_addr(ac.g, B + t->o_egu);janas_g_push_addr(ac.u, B + t->o_egu + (size_t)np * t->effp * 4);@@ -109,19 +148,18 @@ void janas_gt_rec_experts(struct janas_gpu *g, struct janas_gpu_tok *t,janas_gt_push(g, &ac, sizeof(ac));g->vkCmdDispatch(g->cb, np * nbf, 1, 1);janas_g_compute_barrier(g);- pi = (struct push_id){- .rows = md->dm, .nb = nbf, .tiles = t->tiles2, .ngroups = md->eff / 32};- janas_g_push_addr(pi.t, B + t->o_tab2);- janas_gt_bind(g, bound,- dt == JANAS_Q4_K ? g->pipe_id4- : dt == JANAS_Q6_K ? g->pipe_id6s- : dt == JANAS_MXFP4 ? g->pipe_idmx- : dt == JANAS_Q3_K ? g->pipe_id3- : dt == JANAS_Q2_0 ? g->pipe_idb[2]- : g->pipe_id6);- janas_gt_push(g, &pi, sizeof(pi));- g->vkCmdDispatchIndirect(g->cb, t->buf.buf, t->o_ind + 16);+ janas_gt_stamp(g, t, PH_ACT);+ experts_product(g, t, bound, by_rows(g, t, dt, n),+ dt == JANAS_Q4_K ? g->pipe_id4+ : dt == JANAS_Q6_K ? g->pipe_id6s+ : dt == JANAS_MXFP4 ? g->pipe_idmx+ : dt == JANAS_Q3_K ? g->pipe_id3+ : dt == JANAS_Q2_0 ? g->pipe_idb[2]+ : g->pipe_id6,+ B + t->o_tab2, md->dm, md->eff, nbf, t->tiles2,+ md->eff / 32, t->o_ind + 16);janas_g_compute_barrier(g);+ janas_gt_stamp(g, t, PH_DOWN);struct push_cb pc = {.dm = md->dm, .k = md->k, .has_sh = ly->sh_gate_w.type != 0};janas_g_push_addr(pc.dout, B + t->o_dout);@@ -134,6 +172,7 @@ void janas_gt_rec_experts(struct janas_gpu *g, struct janas_gpu_tok *t,janas_gt_push(g, &pc, sizeof(pc));g->vkCmdDispatch(g->cb, md->dm / JANAS_QK, n, 1);janas_g_compute_barrier(g);+ janas_gt_stamp(g, t, PH_COMBINE);}/* An entry of a *_mmid table: weights, activations, outputs (ys floats@@ -151,6 +190,300 @@ static void table_entry(uint32_t *e, VkDeviceAddress w, VkDeviceAddress x,janas_g_push_addr(e + 10, p1);}+/* The experts read into the tier a stage at a time */+#define UP_SLOTS 8+/* The tier's slots looked at for the one to give up, from where the last+ search stopped: the one of the expert used least among them */+#define TIER_SCAN 16+/* The tier in allocations of 1 GiB at most: some drivers take no larger */+#define TIER_PIECE ((size_t)1 << 30)++static VkDeviceAddress tier_addr(const struct janas_gpu_tok *t, uint32_t s)+{+ return t->tier[s / t->per].addr ++ (VkDeviceAddress)(s % t->per) * t->slot_bytes;+}++int janas_gt_tier_create(struct janas_gpu *g, struct janas_gpu_tok *t,+ const struct janas_gpu_tok_model *md)+{+ size_t sb = (md->slot_bytes + 255) / 256 * 256,+ all = (size_t)md->n_layer * md->n_expert;+ /* what the caller asks, within the room the card has left */+ size_t want = md->tier_bytes, room = janas_gpu_local_room(g);+ if (room && want > room)+ want = room;+ size_t per = TIER_PIECE / sb, n = want / sb;+ if (n > all)+ n = all;+ if (per == 0 || n > per * TIER_PIECES)+ n = per * TIER_PIECES;+ if (n < md->n_expert) /* not even a layer's: no tier, nor this path */+ return -1;+ for (size_t at = 0, i = 0; at < n; at += per, i++)+ if (janas_g_make_local(g, &t->tier[i],+ (n - at < per ? n - at : per) * sb) != 0)+ return -1;+ t->per = (uint32_t)per;+ if (janas_g_make_stage(g, &t->up, UP_SLOTS * sb, 1) != 0)+ return -1;+ t->slot_of = malloc(all * sizeof(*t->slot_of));+ t->uses = calloc(all, sizeof(*t->uses));+ t->owner = malloc(n * sizeof(*t->owner));+ t->stamp = calloc(n, sizeof(*t->stamp));+ t->ra = malloc(md->n_expert * sizeof(*t->ra));+ t->rb = malloc(md->n_expert * sizeof(*t->rb));+ t->on_gpu = malloc(md->n_expert * sizeof(*t->on_gpu));+ t->gpu_slot = malloc(md->n_expert * sizeof(*t->gpu_slot));+ t->on_cpu = malloc(md->n_expert * sizeof(*t->on_cpu));+ t->miss = malloc(md->n_expert * sizeof(*t->miss));+ t->cpu_slots = malloc(md->n_expert * sizeof(*t->cpu_slots));+ t->cps = malloc((UP_SLOTS + md->n_expert) * sizeof(*t->cps));+ if (!t->slot_of || !t->uses || !t->owner || !t->stamp || !t->ra || !t->rb ||+ !t->on_gpu || !t->gpu_slot || !t->on_cpu || !t->miss || !t->cpu_slots ||+ !t->cps)+ return -1;+ for (size_t i = 0; i < all; i++)+ t->slot_of[i] = -1;+ for (size_t i = 0; i < n; i++)+ t->owner[i] = UINT32_MAX;+ t->n_slot = t->n_free = (uint32_t)n;+ t->slot_bytes = sb;+ return 0;+}++void janas_gt_tier_destroy(struct janas_gpu *g, struct janas_gpu_tok *t)+{+ for (int i = 0; i <= TIER_PIECES + 1; i++) {+ struct stage *st = i < TIER_PIECES ? &t->tier[i]+ : i == TIER_PIECES ? &t->up+ : &t->at;+ if (st->buf)+ g->vkDestroyBuffer(g->dev, st->buf, NULL);+ if (st->mem)+ g->vkFreeMemory(g->dev, st->mem, NULL);+ }+ free(t->slot_of);+ free(t->uses);+ free(t->owner);+ free(t->stamp);+ free(t->ra);+ free(t->rb);+ free(t->on_gpu);+ free(t->gpu_slot);+ free(t->on_cpu);+ free(t->miss);+ free(t->cpu_slots);+ free(t->cps);+}++/* A slot of the tier for expert key (layer * n_expert + e): a free one,+ or among TIER_SCAN the one of the expert used least, if less than key+ and no expert of this plan's (stamp); -1 if none. */+static int32_t tier_take(struct janas_gpu_tok *t, uint32_t key)+{+ int32_t s = -1;+ if (t->n_free) {+ s = (int32_t)(t->n_slot - t->n_free--);+ } else {+ uint32_t best = t->uses[key];+ for (uint32_t i = 0; i < TIER_SCAN && i < t->n_slot; i++) {+ uint32_t c = (t->scan + i) % t->n_slot, o = t->owner[c];+ if (t->stamp[c] != t->plan_no && t->uses[o] < best) {+ best = t->uses[o];+ s = (int32_t)c;+ }+ }+ t->scan = (t->scan + TIER_SCAN) % t->n_slot;+ if (s < 0)+ return -1;+ t->slot_of[t->owner[s]] = -1;+ }+ t->owner[s] = key;+ t->slot_of[key] = s;+ t->stamp[s] = t->plan_no;+ return s;+}++/* Experts slots[0..n) read into the tier's slots dst[0..n), through the+ stage, a submission for each UP_SLOTS of them but the last, whose copies+ are left in t->cps (their number): or -1. */+static int tier_load(struct janas_gpu *g, struct janas_gpu_tok *t,+ const uint8_t *const *slots, const int32_t *dst,+ uint32_t n, size_t bytes)+{+ uint32_t c = 0;+ for (uint32_t i0 = 0; i0 < n; i0 += UP_SLOTS) {+ c = n - i0 < UP_SLOTS ? n - i0 : UP_SLOTS;+ for (uint32_t i = 0; i < c; i++)+ memcpy(t->up.map + (size_t)i * t->slot_bytes, slots[i0 + i], bytes);+ if (!t->up.coherent) {+ VkMappedMemoryRange mr = {.sType =+ VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,+ .memory = t->up.mem,+ .offset = 0,+ .size = VK_WHOLE_SIZE};+ g->vkFlushMappedMemoryRanges(g->dev, 1, &mr);+ }+ for (uint32_t i = 0; i < c; i++)+ t->cps[i] = (struct janas_g_cp){+ .src = t->up.buf,+ .dst = t->tier[dst[i0 + i] / t->per].buf,+ .so = (VkDeviceSize)i * t->slot_bytes,+ .dof = (VkDeviceSize)(dst[i0 + i] % t->per) * t->slot_bytes,+ .bytes = bytes};+ if (i0 + c < n && janas_g_copies(g, t->cps, c) != 0)+ return -1;+ }+ t->tier_loads += n;+ return (int)c;+}++/* The experts of a layer moe_plan.comp takes at most (its MAXE) */+#define GPLAN_MAX_EXPERTS 512++int janas_gt_tier_fill(struct janas_gpu *g, struct janas_gpu_tok *t,+ const struct janas_gpu_tok_model *md)+{+ uint32_t ne = md->n_expert, nl = md->n_layer;+ if (t->gplan_tried)+ return t->gplan ? 0 : -1;+ t->gplan_tried = 1;+ const char *ev = getenv("JANAS_GPU_GPLAN");+ if ((ev && atoi(ev) == 0) || !t->p_pl || ne > GPLAN_MAX_EXPERTS ||+ t->n_slot < (size_t)nl * ne)+ return -1;+ size_t ab = (size_t)nl * ne * 48;+ if (janas_g_make_local(g, &t->at, ab) != 0)+ return -1;+ /* layer by layer, the next one read ahead, UP_SLOTS experts a stage */+ uint32_t ids[UP_SLOTS];+ const uint8_t *src[UP_SLOTS], *keep[UP_SLOTS];+ int32_t dst[UP_SLOTS];+ if (md->prefetch)+ md->prefetch(md->ctx, 0);+ for (uint32_t l = 0; l < nl; l++) {+ if (md->prefetch && l + 1 < nl)+ md->prefetch(md->ctx, l + 1);+ for (uint32_t e0 = 0; e0 < ne; e0 += UP_SLOTS) {+ uint32_t c = ne - e0 < UP_SLOTS ? ne - e0 : UP_SLOTS, m = 0;+ for (uint32_t i = 0; i < c; i++)+ ids[i] = e0 + i;+ if (md->fetch(md->ctx, l, ids, c, src) != 0)+ return -1;+ for (uint32_t i = 0; i < c; i++) {+ uint32_t key = l * ne + ids[i];+ if (t->slot_of[key] >= 0)+ continue; /* there from an earlier plan */+ if ((dst[m] = tier_take(t, key)) < 0)+ return -1;+ keep[m++] = src[i];+ }+ int ncp = m ? tier_load(g, t, keep, dst, m, md->slot_bytes) : 0;+ if (ncp < 0 || (ncp && janas_g_copies(g, t->cps, (uint32_t)ncp)))+ return -1;+ }+ }+ /* the addresses, through the stage */+ uint32_t *w = (uint32_t *)t->up.map;+ size_t per = t->up.bytes / 48; /* experts a copy */+ for (size_t k0 = 0; k0 < (size_t)nl * ne; k0 += per) {+ size_t c = (size_t)nl * ne - k0 < per ? (size_t)nl * ne - k0 : per;+ for (size_t i = 0; i < c; i++) {+ uint32_t key = (uint32_t)(k0 + i), l = key / ne;+ const struct janas_gpu_tok_layer *ly = &md->layers[l];+ VkDeviceAddress s = tier_addr(t, (uint32_t)t->slot_of[key]), p0 = 0,+ p1 = 0;+ if (ly->down.type == JANAS_Q6_K_P) {+ p0 = ly->poff[0] ? s + ly->poff[0] : 0;+ p1 = ly->poff[1] ? s + ly->poff[1] : 0;+ }+ uint32_t *e = w + 12 * i;+ janas_g_push_addr(e, s + ly->off[0]);+ janas_g_push_addr(e + 2, s + ly->off[1]);+ janas_g_push_addr(e + 4, s + ly->off[2]);+ janas_g_push_addr(e + 6, p0);+ janas_g_push_addr(e + 8, p1);+ e[10] = e[11] = 0;+ }+ if (!t->up.coherent) {+ VkMappedMemoryRange mr = {.sType =+ VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,+ .memory = t->up.mem,+ .offset = 0,+ .size = VK_WHOLE_SIZE};+ g->vkFlushMappedMemoryRanges(g->dev, 1, &mr);+ }+ if (janas_g_copy(g, t->up.buf, 0, t->at.buf, k0 * 48, c * 48) != 0)+ return -1;+ }+ t->gplan = 1;+ if (getenv("JANAS_GPU_VERBOSE"))+ fprintf(stderr,+ "gpu experts: every one in the tier, the plan on the GPU, "+ "a pass in one submission\n");+ return 0;+}++void janas_gt_rec_plan(struct janas_gpu *g, struct janas_gpu_tok *t,+ const struct janas_gpu_tok_model *md, uint32_t l,+ VkPipeline *bound, uint32_t n)+{+ const struct janas_gpu_tok_layer *ly = &md->layers[l];+ int r1 = by_rows(g, t, ly->gate.type, n) != VK_NULL_HANDLE,+ r2 = by_rows(g, t, ly->down.type, n) != VK_NULL_HANDLE;+ struct push_pl pp = {.pe = (uint32_t)t->o_pe,+ .gt = (uint32_t)t->o_gt,+ .inv = (uint32_t)t->o_inv,+ .t1 = (uint32_t)t->o_tab1,+ .t2 = (uint32_t)t->o_tab2,+ .ind = (uint32_t)t->o_ind,+ .xg = (uint32_t)t->o_xg,+ .egu = (uint32_t)t->o_egu,+ .eh = (uint32_t)t->o_eh,+ .dout = (uint32_t)t->o_dout,+ .n = n,+ .k = md->k,+ .ne = md->n_expert,+ .nbd = md->dm / JANAS_QK,+ .nbf = t->effp / JANAS_QK,+ .ep = t->effp,+ .dm = md->dm,+ .eff = md->eff,+ .s1 = r1 ? MVID_NV : 16,+ .s2 = r2 ? MVID_NV : 16,+ .tiles1 = t->tiles1,+ .tiles2 = t->tiles2};+ janas_g_push_addr(pp.at,+ t->at.addr + (VkDeviceAddress)l * md->n_expert * 48);+ janas_g_push_addr(pp.b, t->buf.addr);+ janas_gt_bind(g, bound, t->p_pl);+ janas_gt_push(g, &pp, sizeof(pp));+ g->vkCmdDispatch(g->cb, 1, 1, 1);+ /* the tables read by the products, their sizes by the dispatches */+ VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,+ .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,+ .dstAccessMask = VK_ACCESS_SHADER_READ_BIT |+ VK_ACCESS_INDIRECT_COMMAND_READ_BIT};+ g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,+ VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |+ VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,+ 0, 1, &mb, 0, NULL, 0, NULL);+ janas_gt_stamp(g, t, PH_BETWEEN);+}++/* The host's side of bytes [off, off + bytes) of buf: on a card with+ memory of its own the mirror's cache lines (the pass's commands copy+ them), else buf's own. */+static void host_side(struct janas_gpu *g, struct janas_gpu_tok *t, size_t off,+ size_t bytes, int flush)+{+ if (t->local)+ janas_gt_mirror_cache(g, t, off, bytes, flush);+ else+ janas_gt_sync_range(g, t, off, bytes, flush);+}+/** Between two of a mixture's submissions, on the CPU: layer l's choice (n* tokens, k experts each) read back and sorted by expert, as model_ffn.c@@ -164,7 +497,7 @@ static int plan(struct janas_gpu *g, struct janas_gpu_tok *t,const struct janas_gpu_tok_layer *ly = &md->layers[l];uint32_t k = md->k, ne = md->n_expert, np = n * k, nd = 0;uint32_t nbd = md->dm / JANAS_QK, nbf = t->effp / JANAS_QK, ep = t->effp;- janas_gt_sync_range(g, t, t->o_pe, (size_t)np * 4, 0);+ host_side(g, t, t->o_pe, (size_t)np * 4, 0);const uint32_t *pe = (const uint32_t *)(t->buf.map + t->o_pe);uint32_t *cnt = t->cnt, *gt = (uint32_t *)(t->buf.map + t->o_gt),*inv = (uint32_t *)(t->buf.map + t->o_inv);@@ -187,30 +520,96 @@ static int plan(struct janas_gpu *g, struct janas_gpu_tok *t,gt[r] = p / k;inv[p] = r;}- if (md->fetch(md->ctx, l, t->ids, nd, t->slots) != 0)- return -1;+ /* the experts the GPU computes, and where it reads them: all of them,+ as the cache holds them; or, with a tier, those in it or read into+ it now, the others computed by the CPU into their rows */+ uint32_t ng = 0, nc = 0, nm = 0;+ int ncp = 0; /* copies left for one submission */+ if (!t->n_slot) {+ if (md->fetch(md->ctx, l, t->ids, nd, t->slots) != 0)+ return -1;+ for (uint32_t i = 0; i < nd; i++)+ t->ids[ng++] = t->ids[i];+ } else {+ uint32_t base = l * ne;+ t->plan_no++;+ for (uint32_t i = 0; i < nd; i++) {+ uint32_t e = t->ids[i];+ t->uses[base + e]++;+ if (t->slot_of[base + e] >= 0) {+ t->stamp[t->slot_of[base + e]] = t->plan_no;+ t->on_gpu[ng] = e;+ t->gpu_slot[ng++] = (uint32_t)t->slot_of[base + e];+ t->tier_hits++;+ } else {+ t->miss[nm++] = e;+ }+ }+ if (nm && md->fetch(md->ctx, l, t->miss, nm, t->cpu_slots) != 0)+ return -1;+ /* misses into the tier where it gives up an expert used less;+ the rest to the CPU */+ int32_t *dst = (int32_t *)t->fill; /* free until the sort below */+ const uint8_t **src = t->slots;+ uint32_t nl = 0;+ for (uint32_t i = 0; i < nm; i++) {+ uint32_t e = t->miss[i];+ int32_t s2 = tier_take(t, base + e);+ if (s2 >= 0) {+ dst[nl] = s2;+ src[nl++] = t->cpu_slots[i];+ t->on_gpu[ng] = e;+ t->gpu_slot[ng++] = (uint32_t)s2;+ } else {+ t->on_cpu[nc] = e;+ t->cpu_slots[nc++] = t->cpu_slots[i];+ }+ }+ if (nl && (ncp = tier_load(g, t, src, dst, nl, md->slot_bytes)) < 0)+ return -1;+ for (uint32_t i = 0; i < ng; i++)+ t->ids[i] = t->on_gpu[i];+ t->cpu_experts += nc;+ }+ /* the fill of each expert again (t->fill held the slots above) */+ for (uint32_t e = 0; e < ne; e++)+ t->fill[e] = cnt[e];VkDeviceAddress B = t->buf.addr;uint32_t *t1 = (uint32_t *)(t->buf.map + t->o_tab1),*t2 = (uint32_t *)(t->buf.map + t->o_tab2);uint32_t n1 = 0, n2 = 0; /* tiles */+ /* vectors a tile: a matvec's workgroup's, or a *_mmid's */+ int r1 = by_rows(g, t, ly->gate.type, n) != VK_NULL_HANDLE,+ r2 = by_rows(g, t, ly->down.type, n) != VK_NULL_HANDLE;+ uint32_t s1 = r1 ? MVID_NV : 16, s2 = r2 ? MVID_NV : 16;/* the matrices' bytes; a down matrix's rows may end inside a group */size_t gb = (size_t)md->eff * janas_qtype_row_bytes(ly->gate.type, md->dm),db = (size_t)md->dm * janas_qtype_row_bytes(ly->down.type, md->eff) +4,pb = (size_t)md->dm * nbf * JANAS_Q6KP_PLANE;- for (uint32_t i = 0; i < nd; i++) {+ for (uint32_t i = 0; i < ng; i++) {uint32_t e = t->ids[i], a = cnt[e], nv = cnt[e + 1] - a;- const uint8_t *s = t->slots[i];- VkDeviceAddress wg = janas_gt_addr_of(g, s + ly->off[0], gb),- wu = janas_gt_addr_of(g, s + ly->off[1], gb),- wd = janas_gt_addr_of(g, s + ly->off[2], db), p0 = 0,- p1 = 0;- /* the planes the cache holds (none, one or both: level 1 to 3) */- if (ly->down.type == JANAS_Q6_K_P &&- ((ly->poff[0] &&- !(p0 = janas_gt_addr_of(g, s + ly->poff[0], pb))) ||- (ly->poff[1] && !(p1 = janas_gt_addr_of(g, s + ly->poff[1], pb)))))- return -1;+ VkDeviceAddress wg, wu, wd, p0 = 0, p1 = 0;+ if (t->n_slot) {+ VkDeviceAddress s = tier_addr(t, t->gpu_slot[i]);+ wg = s + ly->off[0], wu = s + ly->off[1], wd = s + ly->off[2];+ if (ly->down.type == JANAS_Q6_K_P) {+ p0 = ly->poff[0] ? s + ly->poff[0] : 0;+ p1 = ly->poff[1] ? s + ly->poff[1] : 0;+ }+ } else {+ const uint8_t *s = t->slots[i];+ wg = janas_gt_addr_of(g, s + ly->off[0], gb);+ wu = janas_gt_addr_of(g, s + ly->off[1], gb);+ wd = janas_gt_addr_of(g, s + ly->off[2], db);+ /* the planes the cache holds (none, one or both: level 1-3) */+ if (ly->down.type == JANAS_Q6_K_P &&+ ((ly->poff[0] &&+ !(p0 = janas_gt_addr_of(g, s + ly->poff[0], pb))) ||+ (ly->poff[1] &&+ !(p1 = janas_gt_addr_of(g, s + ly->poff[1], pb)))))+ return -1;+ }if (!wg || !wu || !wd)return -1;VkDeviceAddress x = B + t->o_xg + (VkDeviceAddress)a * nbd * 320;@@ -224,23 +623,51 @@ static int plan(struct janas_gpu *g, struct janas_gpu_tok *t,B + t->o_eh + (VkDeviceAddress)a * nbf * 320,B + t->o_dout + (VkDeviceAddress)a * md->dm * 4, nv, md->dm,p0, p1);- for (uint32_t v = 0; v < nv; v += 16) {+ for (uint32_t v = 0; v < nv; v += s1) {uint32_t *u = t1 + 4 * (t->tiles1 + n1);u[0] = 2 * i, u[1] = v, u[2] = u[3] = 0;u[4] = 2 * i + 1, u[5] = v, u[6] = u[7] = 0;n1 += 2;- u = t2 + 4 * (t->tiles2 + n2++);+ }+ for (uint32_t v = 0; v < nv; v += s2) {+ uint32_t *u = t2 + 4 * (t->tiles2 + n2++);u[0] = i, u[1] = v, u[2] = u[3] = 0;}}+ /* a matvec's rows in its workgroups' y, a *_mmid's in x */uint32_t *ind = (uint32_t *)(t->buf.map + t->o_ind);- ind[0] = (md->eff + 63) / 64, ind[1] = 1, ind[2] = n1;- ind[4] = (md->dm + 63) / 64, ind[5] = 1, ind[6] = n2;- janas_gt_sync_range(g, t, t->o_gt, (size_t)np * 4, 1);- janas_gt_sync_range(g, t, t->o_inv, (size_t)np * 4, 1);- janas_gt_sync_range(g, t, t->o_tab1, ((size_t)t->tiles1 + n1) * 16, 1);- janas_gt_sync_range(g, t, t->o_tab2, ((size_t)t->tiles2 + n2) * 16, 1);- janas_gt_sync_range(g, t, t->o_ind, 32, 1);+ ind[0] = r1 ? 1 : (md->eff + 63) / 64, ind[2] = n1;+ ind[1] = r1 ? (md->eff + 63) / 64 : 1;+ ind[4] = r2 ? 1 : (md->dm + 63) / 64, ind[6] = n2;+ ind[5] = r2 ? (md->dm + 63) / 64 : 1;+ /* o_gt to o_ind are one range (gpu_token.c's layout), copied by the+ next submission */+ host_side(g, t, t->o_gt, t->o_ind + 32 - t->o_gt, 1);+ if (nc) {+ /* the CPU's experts, from the layer's normalized input (copied to+ the mirror by the last submission), into their rows */+ for (uint32_t i = 0; i < nc; i++)+ t->ra[i] = cnt[t->on_cpu[i]], t->rb[i] = cnt[t->on_cpu[i] + 1];+ host_side(g, t, t->o_xnf, (size_t)n * md->dm * 4, 0);+ if (md->cpu_rows(md->ctx, l, n, (const float *)(t->buf.map + t->o_xnf),+ nc, t->on_cpu, t->cpu_slots, t->ra, t->rb, gt,+ (float *)(t->buf.map + t->o_dout)) != 0)+ return -1;+ for (uint32_t i = 0; i < nc; i++) {+ size_t o = t->o_dout + (size_t)t->ra[i] * md->dm * 4,+ b = (size_t)(t->rb[i] - t->ra[i]) * md->dm * 4;+ host_side(g, t, o, b, 1);+ if (t->local)+ t->cps[ncp++] = (struct janas_g_cp){.src = t->mirror.buf,+ .dst = t->buf.buf,+ .so = o,+ .dof = o,+ .bytes = b};+ }+ }+ /* the last experts read into the tier and the CPU's rows, together */+ if (ncp && janas_g_copies(g, t->cps, (uint32_t)ncp) != 0)+ return -1;return 0;}@@ -263,8 +690,16 @@ int janas_gt_run_segments(struct janas_gpu *g, struct janas_gpu_tok *t,few that no token takes are under 2% (e^-4) */int ahead = md->prefetch && (size_t)n * md->k >= 4 * (size_t)md->n_expert;for (uint32_t s = 0; s < t->n_seg; s++) {+ struct timespec p0, p1;+ if (t->qp)+ clock_gettime(CLOCK_MONOTONIC, &p0);if (s && plan(g, t, md, s - 1, n) != 0)return -1;+ if (t->qp) {+ clock_gettime(CLOCK_MONOTONIC, &p1);+ t->plan_ns += (double)(p1.tv_sec - p0.tv_sec) * 1e9 ++ (double)(p1.tv_nsec - p0.tv_nsec);+ }/* layer s's experts read while the GPU computes up to its router'schoice, as the CPU's path reads the next layer's (model_ffn.cjanas_m_prefetch_block): without, the GPU waited for eachdiff --git a/src/llm/gpu_token_rec.c b/src/llm/gpu_token_rec.cindex 23cc0f2..c9d0081 100644--- a/src/llm/gpu_token_rec.c+++ b/src/llm/gpu_token_rec.c@@ -139,17 +139,8 @@ int janas_gt_rec_gdn(struct janas_gpu *g, struct janas_gpu_tok *t,g->vkCmdDispatch(g->cb, (2 * md->nv + GBA_RW - 1) / GBA_RW, (n + 7) / 8,1);} else {- struct push_dm pd = {.rows = 2 * md->nv,- .cols = md->dm,- .nv = n,- .ys = 2 * md->nv,- .scale = 1.0f};- janas_g_push_addr(pd.w, B + pr[PAR_BA]);- janas_g_push_addr(pd.x, B + t->o_xnf);- janas_g_push_addr(pd.y, BA);- janas_gt_bind(g, bound, g->pipe_dm[0]);- janas_gt_push(g, &pd, sizeof(pd));- g->vkCmdDispatch(g->cb, (2 * md->nv + 63) / 64, (n + 63) / 64, 1);+ janas_gt_rec_dense(g, bound, 0, B + pr[PAR_BA], B + t->o_xnf, BA,+ 2 * md->nv, md->dm, 2 * md->nv, n, 1.0f);janas_g_compute_barrier(g);struct push_gact px = {.nv = md->nv, .n = n};janas_g_push_addr(px.ba, BA);diff --git a/src/llm/matvec.c b/src/llm/matvec.cindex a3e2d18..cd1fc22 100644--- a/src/llm/matvec.c+++ b/src/llm/matvec.c@@ -4,6 +4,7 @@* matvec.c - quantized matrix-vector product on a thread pool (see matvec.h).*/#include "matvec.h"+#include "common/sysinfo.h"#include <stdatomic.h>#include <stdlib.h>@@ -46,6 +47,37 @@ static inline size_t claim(atomic_size_t *next, size_t total, size_t div,return g0;}+/*+ * A chunk's rows on many vectors are taken in pieces whose weights stay in+ * the level-2 cache while every group of eight vectors goes over them: the+ * kernels run the groups outside and the rows inside, so a chunk larger+ * than the cache was read from memory again for each group of eight - the+ * whole matrix once every eight tokens of a prompt. A piece is a quarter of+ * the cache (the rest for the vectors), a cache of 256 KB where the+ * machine does not say. Not by default: on the laptop's Core Ultra (2 MB of+ * L2) a chunk already fits and pieces of 128 KB cost 6% of a prompt (Qwen3-+ * 4B, 52.9 against 56.1 tokens/s, AVX2), and on a Ryzen 5 5500 (512 KB, a+ * prompt 2.2 times slower than llama.cpp's, issue #45) it is not measured+ * yet. JANAS_ROW_PIECE_KB=<n> takes pieces of n KB, -1 a quarter of the+ * L2, 0 or unset the chunk whole.+ */+static size_t row_piece_bytes(void)+{+ static _Atomic size_t bytes; /* 0: not yet known */+ size_t b = atomic_load_explicit(&bytes, memory_order_relaxed);+ if (b)+ return b;+ const char *e = getenv("JANAS_ROW_PIECE_KB");+ if (e && atoi(e) < 0) {+ size_t l2 = janas_l2_bytes();+ b = (l2 ? l2 : (size_t)256 << 10) / 4;+ } else {+ b = e && atoi(e) > 0 ? (size_t)atoi(e) << 10 : SIZE_MAX;+ }+ atomic_store_explicit(&bytes, b, memory_order_relaxed);+ return b;+}+struct matvec_job {const struct janas_block_q4k *w;const struct janas_block_q8k *x;@@ -123,7 +155,13 @@ static void group_worker(void *arg, int tid, int n_threads)size_t ys = k->y_stride ? k->y_stride : k->rows;if (k->type == JANAS_Q6_K_P) {size_t pb = nb * JANAS_Q6KP_PLANE;- for (size_t r = r0; r < r0 + nr; r++) {+ /* eight rows at a time as tiles, where they serve */+ const uint8_t *pl1 = k->plane[0], *pl0 = k->plane[1];+ size_t r8 = janas_q6k_tile_rows(+ k->type, w0, row_bytes, pl1 ? pl1 + r0 * pb : NULL,+ pl0 ? pl0 + r0 * pb : NULL, pb, k->x, xs, nv, nb, nr,+ k->y + r0, ys);+ for (size_t r = r0 + r8; r < r0 + nr; r++) {const void *row = (const char *)k->w + r * row_bytes;const uint8_t *p1 =k->plane[0] ? (const uint8_t *)k->plane[0] + r * pb@@ -138,15 +176,18 @@ static void group_worker(void *arg, int tid, int n_threads)janas_dot_rows_n(k->type, w0, row_bytes, k->x, k->cols, nr,k->y + r0);} else if (tail) {- for (size_t i = 0; i < nr; i++)- janas_dot_multi_n(k->type, w0 + i * row_bytes, k->x, xs, nv,- k->cols, k->y + r0 + i, ys);+ janas_dot_multi_rows_n(k->type, w0, row_bytes, k->x, xs, nv,+ k->cols, nr, k->y + r0, ys);} else if (nv == 1) {janas_dot_rows(k->type, w0, row_bytes, k->x, nb, nr, k->y + r0);} else {- for (size_t i = 0; i < nr; i++)- janas_dot_multi(k->type, w0 + i * row_bytes, k->x, xs, nv,- nb, k->y + r0 + i, ys);+ size_t piece = row_piece_bytes() / row_bytes;+ if (nv <= 8 || piece < 1)+ piece = nv <= 8 ? nr : 1;+ for (size_t i = 0; i < nr; i += piece)+ janas_dot_multi_rows(+ k->type, w0 + i * row_bytes, row_bytes, k->x, xs, nv,+ nb, nr - i < piece ? nr - i : piece, k->y + r0 + i, ys);}g0 = end;}diff --git a/src/llm/model.c b/src/llm/model.cindex 4e91ee3..2fd4a4b 100644--- a/src/llm/model.c+++ b/src/llm/model.c@@ -434,6 +434,10 @@ static int forward(struct janas_llm_model *m, const int32_t *tokens, uint32_t n,for (uint32_t j = 0; j < n; j++)if (tokens[j] < 0 || (uint32_t)tokens[j] >= m->n_vocab)return -1;+ /* from the start: nothing of the GPU's copy of the state matters, the+ host's (the recurrent states zeroed below) is the one */+ if (m->gtok_dev && pos0 == 0)+ m->gtok_where = JANAS_GT_HOST;if (m->a->rec) {if (pos0 == 0) {m->src = 0;@@ -482,6 +486,8 @@ static int forward(struct janas_llm_model *m, const int32_t *tokens, uint32_t n,if (gpu == -2)return -1; /* the GPU failed after replaying on the base in place:no state to go on from (base_pos stays invalid) */+ if (gpu != 0) /* the CPU's pass: the state the GPU made newer back */+ janas_m_gtok_host(m, 1);if (gpu == 0) {m->ring_hi = pos0 + n;if (m->a->rec && n <= JANAS_LLM_MAX_BLOCK) {diff --git a/src/llm/model_assist.c b/src/llm/model_assist.cindex 7ccea68..55661c9 100644--- a/src/llm/model_assist.c+++ b/src/llm/model_assist.c@@ -394,6 +394,7 @@ int janas_m_assist_draft(struct janas_llm_model *m, int32_t token,uint32_t dm = m->d_model, d = a->d;if (p + 1 >= m->n_ctx || token < 0 || (uint32_t)token >= m->n_vocab)return -1;+ janas_m_gtok_host(m, 0); /* it reads the model's KV cache */double t0 = now();/* [embedding of the token, as the main model scales it; h] */janas_m_embed(m, token, a->cat);diff --git a/src/llm/model_ffn.c b/src/llm/model_ffn.cindex 69082ae..45605a7 100644--- a/src/llm/model_ffn.c+++ b/src/llm/model_ffn.c@@ -366,6 +366,54 @@ static void run_experts(struct janas_llm_model *m,}}+int janas_m_expert_rows(struct janas_llm_model *m, uint32_t l, uint32_t n,+ const float *xn, uint32_t nd, const uint32_t *ids,+ const uint8_t *const *slots, const uint32_t *ra,+ const uint32_t *rb, const uint32_t *tok, float *out)+{+ uint32_t dm = m->d_model, nbd = dm / JANAS_QK;+ if (n > m->blk)+ return -1;+ /* as the GPU quantizes the same values (norm_quant.comp): then an+ expert computed here gives the GPU's bits */+ for (uint32_t j = 0; j < n; j++)+ janas_q8k_quantize_det(xn + (size_t)j * dm, m->xq + j * nbd, dm);+ const struct janas_jns_layer *jl = &m->j.layers[l];+ /* the CPU's products only: the GPU waits for these between two of its+ submissions */+ int off = m->gpu_off;+ m->gpu_off = 1;+ for (uint32_t c0 = 0; c0 < nd; c0 += EXPERT_CHUNK) {+ struct expert_run runs[EXPERT_CHUNK];+ uint32_t cn = nd - c0 < EXPERT_CHUNK ? nd - c0 : EXPERT_CHUNK;+ for (uint32_t c = 0; c < cn; c++) {+ uint32_t i = c0 + c;+ if (rb[i] > m->blk * (m->k + 1)) {+ m->gpu_off = off;+ return -1;+ }+ for (uint32_t r = ra[i]; r < rb[i]; r++)+ m->pair_tok[r] = tok[r], m->order[r] = r;+ const uint8_t *pl[2] = {NULL, NULL};+ for (int q = 0; q < m->exp_level - 1; q++)+ if (jl->plane[q].type)+ pl[q] = slots[i] + jl->plane[q].offset;+ runs[c] = (struct expert_run){{slots[i], slots[i], slots[i]},+ {pl[0], pl[1]},+ jl->m,+ ra[i],+ rb[i],+ (int)ids[i]};+ }+ run_experts(m, runs, cn);+ }+ m->gpu_off = off;+ for (uint32_t i = 0; i < nd; i++)+ memcpy(out + (size_t)ra[i] * dm, m->dout + (size_t)ra[i] * dm,+ (size_t)(rb[i] - ra[i]) * dm * sizeof(float));+ return 0;+}+int janas_m_route_rank(struct janas_llm_model *m, uint32_t l, uint32_t n,uint32_t k, uint32_t *ids){diff --git a/src/llm/model_gpu.c b/src/llm/model_gpu.cindex 8e930ca..0617fc5 100644--- a/src/llm/model_gpu.c+++ b/src/llm/model_gpu.c@@ -27,11 +27,33 @@#include "model_impl.h"+int janas_m_gpu_whole(void)+{+ static int whole = -1;+ if (whole < 0) {+ const char *w = getenv("JANAS_GPU_WHOLE");+ whole = w ? atoi(w) > 0 : 1;+ }+ return whole;+}++int janas_m_gpu_tier(const struct janas_llm_model *m)+{+ return m->n_expert > 1 && m->gpu && !m->gpu_off &&+ !janas_gpu_shares_host(m->gpu) && janas_m_gpu_whole() &&+ !(getenv("JANAS_GPU_TIER") && atoi(getenv("JANAS_GPU_TIER")) == 0);+}+/* Why a model is not for the token on the GPU, NULL if it is. */static const char *not_for_gpu(const struct janas_llm_model *m){- if (!m->gpu || m->gpu_off || !janas_gpu_shares_host(m->gpu))- return "no GPU reading the host's memory";+ if (!m->gpu || m->gpu_off)+ return "no GPU";+ /* a card with memory of its own: the state copied there and back+ (sync_state); not yet qwen4exp's rings and indexer, which the CPU's+ passes read in between */+ if (!janas_gpu_shares_host(m->gpu) && (m->a->hc || m->ix_heads))+ return "qwen4exp's streams on a card with memory of its own";/* qwen3.5: its recurrent layers and gated queries; an MTP block or adrafting model gets the final hidden states */int q35 = m->a->rec == JANAS_REC_Q35 && m->a->attn_gate;@@ -77,7 +99,10 @@ static const char *not_for_gpu(const struct janas_llm_model *m)held = janas_expert_cache_slots(m->cache);if (e && atoi(e) <= 0)return "a mixture, JANAS_GPU_MOE=0";- if (!e && held * 2 < all) {+ /* a card with memory of its own: its tier of experts, the CPU+ computing those not in it */+ int tier = janas_m_gpu_tier(m);+ if (!e && !tier && held * 2 < all) {static char why[128];snprintf(why, sizeof why,"a mixture whose cache holds %.0f%% of its experts, "@@ -88,7 +113,7 @@ static const char *not_for_gpu(const struct janas_llm_model *m)/* nor experts whose down matrices are plain Q6_K (q6k_mmid.comp):Qwen3-Coder-30B-A3B read a prompt of 1,024 tokens at 116.9tokens/s instead of 131.5 (10 Oct 2026) */- if (!e && m->j.layers[0].m[JANAS_JNS_DOWN].type == JANAS_Q6_K)+ if (!e && !tier && m->j.layers[0].m[JANAS_JNS_DOWN].type == JANAS_Q6_K)return "a mixture whose experts' down matrices are plain Q6_K, ""slower whole on the GPU (JANAS_GPU_MOE=1 takes it)";/* a mixture: the router's choice whole (no expert left out for@@ -102,7 +127,7 @@ static const char *not_for_gpu(const struct janas_llm_model *m)if (m->layers[l].exp_scale || !m->layers[l].router16)return "a mixture the token's shaders do not take yet";}- if (m->arena_gpu != 1 || m->arena_stale)+ if (!janas_m_gpu_tier(m) && (m->arena_gpu != 1 || m->arena_stale))return "the feed-forward weights are not on the GPU";for (uint32_t l = 0; l < m->n_layer; l++) {const struct layer *ly = &m->layers[l];@@ -295,6 +320,24 @@ static void gtok_routed(void *ctx, uint32_t layer, uint32_t n, uint32_t k,/* ... and the experts of a layer read ahead, all of them (a prompt's blockasks for nearly all), where the cache can read ahead */+/* ... the experts not in the card's tier, on the CPU (gpu_token.h+ cpu_rows) ... */+static int gtok_cpu_rows(void *ctx, uint32_t layer, uint32_t n, const float *xn,+ uint32_t nd, const uint32_t *ids,+ const uint8_t *const *slots, const uint32_t *ra,+ const uint32_t *rb, const uint32_t *tok, float *out)+{+ return janas_m_expert_rows(ctx, layer, n, xn, nd, ids, slots, ra, rb, tok,+ out);+}++/* ... whether someone watches the router's choices */+static int gtok_watched(void *ctx)+{+ const struct janas_llm_model *m = ctx;+ return m->route_fn != NULL;+}+static void gtok_prefetch(void *ctx, uint32_t layer){struct janas_llm_model *m = ctx;@@ -380,6 +423,78 @@ static int hc_build(struct janas_llm_model *m, char *err, size_t len)return 0;}+/* The token's state between the host and the GPU's copy (to_device 1:+ there; 0: back): each attention layer's KV rows and scales, the+ recurrent states and their log. 0, or -1. */+static int sync_state(struct janas_llm_model *m, int to_device)+{+ struct janas_gpu *g = m->gpu;+ for (uint32_t l = 0; l < m->n_layer; l++) {+ const struct layer *ly = &m->layers[l];+ if (ly->rec)+ continue;+ uint32_t s = ly->slot;+ size_t kb = m->kv_off[s + 1] - m->kv_off[s],+ sb = (m->ks_off[s + 1] - m->ks_off[s]) * sizeof(float);+ if (janas_gpu_sync(g, m->kcache + m->kv_off[s], kb, to_device) != 0 ||+ janas_gpu_sync(g, m->vcache + m->kv_off[s], kb, to_device) != 0 ||+ janas_gpu_sync(g, m->kscale + m->ks_off[s], sb, to_device) != 0 ||+ janas_gpu_sync(g, m->vscale + m->ks_off[s], sb, to_device) != 0)+ return -1;+ }+ if (m->a->rec) {+ size_t L = JANAS_LLM_MAX_BLOCK,+ sb = 2 * (size_t)m->n_rec * m->n_vh * m->ds * m->ds *+ sizeof(uint16_t),+ cb = 2 * (size_t)m->n_rec * m->conv_ch * (m->d_conv - 1) *+ sizeof(float),+ lm = (size_t)m->n_rec * L * m->conv_ch * sizeof(float),+ lb = (size_t)m->n_rec * L * m->n_vh * sizeof(float);+ if (janas_gpu_sync(g, m->ssm_buf, sb, to_device) != 0 ||+ janas_gpu_sync(g, m->conv_buf, cb, to_device) != 0 ||+ janas_gpu_sync(g, m->log_mixed, lm, to_device) != 0 ||+ janas_gpu_sync(g, m->log_beta, lb, to_device) != 0 ||+ janas_gpu_sync(g, m->log_decay, lb, to_device) != 0)+ return -1;+ }+ return 0;+}++void janas_m_gtok_host(struct janas_llm_model *m, int write)+{+ if (!m->gtok_dev)+ return;+ if (m->gtok_where == JANAS_GT_DEV) {+ m->gtok_pull++;+ if (sync_state(m, 0) != 0) {+ /* the GPU's copy cannot come back: the CPU's state is not+ this conversation's any more; the GPU left out from here */+ fprintf(stderr, "janas: the GPU's state could not be read back\n");+ m->gtok_state = -1;+ }+ m->gtok_where = JANAS_GT_SAME;+ }+ if (write)+ m->gtok_where = JANAS_GT_HOST;+}++/* A mixture's tier of experts in the card's memory (0: none): the room the+ card has left, less some for the token's buffers made before it;+ JANAS_GPU_TIER_MB=<n> asks for n MiB (on an integrated GPU as a card of+ its own, JANAS_GPU_COPY=1, 1 GiB unless asked) */+static size_t tier_bytes(const struct janas_llm_model *m)+{+ if (!janas_m_gpu_tier(m))+ return 0;+ const char *e = getenv("JANAS_GPU_TIER_MB");+ if (e && atol(e) > 0)+ return (size_t)atol(e) << 20;+ size_t room = janas_gpu_local_room(m->gpu), keep = (size_t)1 << 30;+ if (!room)+ return keep;+ return room > keep ? room - keep : 0;+}+/* The layers described for the GPU, the KV cache imported: 0, or -1. */static int build(struct janas_llm_model *m, char *err, size_t len){@@ -395,6 +510,35 @@ static int build(struct janas_llm_model *m, char *err, size_t len)}m->gtok_kv = 1;}+ /* on a card with memory of its own the recurrent states and their log+ too (an integrated GPU was given them with the engine's buffers),+ and the state is the host's until the first pass on the GPU */+ if (!janas_gpu_shares_host(m->gpu) && !m->gtok_dev) {+ if (m->a->rec && !m->gtok_rec_in) {+ size_t L = JANAS_LLM_MAX_BLOCK,+ sb = 2 * (size_t)m->n_rec * m->n_vh * m->ds * m->ds *+ sizeof(uint16_t),+ cb = 2 * (size_t)m->n_rec * m->conv_ch * (m->d_conv - 1) *+ sizeof(float),+ lm = (size_t)m->n_rec * L * m->conv_ch * sizeof(float),+ lb = (size_t)m->n_rec * L * m->n_vh * sizeof(float);+#define PAGES(b) (((b) + 4095) & ~(size_t)4095)+ if (janas_gpu_import(m->gpu, m->ssm_buf, PAGES(sb)) != 0 ||+ janas_gpu_import(m->gpu, m->conv_buf, PAGES(cb)) != 0 ||+ janas_gpu_import(m->gpu, m->log_mixed, PAGES(lm)) != 0 ||+ janas_gpu_import(m->gpu, m->log_beta, PAGES(lb)) != 0 ||+ janas_gpu_import(m->gpu, m->log_decay, PAGES(lb)) != 0) {+ snprintf(err, len,+ "the recurrent states cannot be copied "+ "to the GPU");+ return -1;+ }+#undef PAGES+ m->gtok_rec_in = 1;+ }+ m->gtok_dev = 1;+ m->gtok_where = JANAS_GT_HOST;+ }free(m->gtok_layers);m->gtok_layers = calloc(m->n_layer, sizeof(*m->gtok_layers));if (!m->gtok_layers) {@@ -563,6 +707,10 @@ static int build(struct janas_llm_model *m, char *err, size_t len).prefetch = moe && janas_expert_cache_can_prefetch(m->cache)? gtok_prefetch: NULL,+ .tier_bytes = tier_bytes(m),+ .slot_bytes = janas_expert_cache_slot_bytes(m->cache),+ .cpu_rows = gtok_cpu_rows,+ .watched = moe ? gtok_watched : NULL,.ctx = m,.ng_layer = m->n_layer};if (m->a->hc && hc_build(m, err, len) != 0)@@ -663,7 +811,13 @@ int janas_m_gpu_token(struct janas_llm_model *m, uint32_t pos, uint32_t n,attention sum in its own order; 13 Oct 2026). */int short_pass = shorts_on() && m->gpu_use && n >= 2 &&n <= JANAS_LLM_MAX_BLOCK && m->n_expert <= 1;- if (!prompt && !short_pass && !(tok && (n == 1 || blocks > 0)))+ /* a card with memory of its own, the model on it: every pass there,+ tokens, drafts and blocks, the CPU's work and the copies over the+ bus between them gone (an Arc B580 read a prompt at 15.8 tokens/s+ with the products shared with the CPU, issue #43); JANAS_GPU_WHOLE=0+ as on an integrated GPU */+ int whole = janas_m_gpu_whole() && m->gpu && !janas_gpu_shares_host(m->gpu);+ if (!prompt && !short_pass && !whole && !(tok && (n == 1 || blocks > 0)))return -1;/* a recurrent model: the GPU replays first what forward() asks (them->replay tokens of the last call that were kept, at most@@ -702,6 +856,13 @@ int janas_m_gpu_token(struct janas_llm_model *m, uint32_t pos, uint32_t n,uint32_t half = m->n_rot / 2;if (all && n > janas_gpu_tok_block_at(m->gtok, &m->gtok_md, pos))return -1;+ /* the state the CPU changed last: to the GPU's copy first */+ if (m->gtok_dev && m->gtok_where == JANAS_GT_HOST) {+ if (sync_state(m, 1) != 0)+ return -1;+ m->gtok_push++;+ m->gtok_where = JANAS_GT_SAME;+ }/* the recurrent state: the first piece from half src to the other,the later ones on that one in place; each piece at least theconvolution's rows of state long. The first replays, every piece of@@ -741,8 +902,15 @@ int janas_m_gpu_token(struct janas_llm_model *m, uint32_t pos, uint32_t n,"janas: the whole token on the GPU failed at position ""%u (%u tokens), the CPU's\n",pos + j, k);+ /* the GPU's copy is not the host's any more: sent again+ before the next pass there */+ if (m->gtok_dev)+ m->gtok_where = JANAS_GT_HOST;return rc == -2 || (j && m->replay) ? -2 : -1;}+ if (m->gtok_dev)+ m->gtok_where = JANAS_GT_DEV;+ m->gtok_runs[n == 1 ? 0 : n <= 16 ? 1 : 2]++;/* qwen4exp: the indexer's keys of the piece's positions, from eachattention layer's input the GPU kept */for (uint32_t l = 0; m->gtok_att_in && l < m->n_layer; l++)diff --git a/src/llm/model_impl.h b/src/llm/model_impl.hindex fbce1a3..d066a90 100644--- a/src/llm/model_impl.h+++ b/src/llm/model_impl.h@@ -205,6 +205,14 @@ struct janas_llm_model {with a GPU copy */int gtok_state, gtok_kv;uint64_t gtok_loads;+ /* on a card with memory of its own (gtok_dev): the token's state - the+ KV rows of the layers it takes, the recurrent states and their log -+ is in two places, newer where gtok_where says (JANAS_GT_*), made the+ same when the passes change hands (janas_m_gtok_host) */+ int gtok_dev, gtok_where, gtok_rec_in;+ /* passes whole on the GPU by length (1, 2-16, more), the state's+ copies there and back (JANAS_GPU_VERBOSE prints them at the end) */+ uint64_t gtok_runs[3], gtok_push, gtok_pull;/* qwen4exp on the GPU: the pass's tokens (its n-gram embeddings), eachattention layer's input at bf16 as the GPU made it (n_attn x blk xd_model, for the indexer's keys), the ring of the n-gram conv@@ -595,6 +603,13 @@ void janas_m_arena_to_gpu(struct janas_llm_model *m);int janas_m_gpu_token(struct janas_llm_model *m, uint32_t pos, uint32_t n,int all, float *logits);void janas_m_apply_candidate(struct janas_llm_model *m, int cand, uint32_t nv);+/* The token's state for the CPU, on a card with memory of its own: what+ the GPU made newer copied back first; write: the CPU is about to change+ it (the GPU's copy older from then on). Nothing elsewhere. */+#define JANAS_GT_SAME 0+#define JANAS_GT_DEV 1+#define JANAS_GT_HOST 2+void janas_m_gtok_host(struct janas_llm_model *m, int write);/* Whether the whole token is asked of the GPU (JANAS_GPU_TOKEN=1). */int janas_m_gpu_token_asked(void);/* model_npu.c: the NPU for the products of blocks (JANAS_NPU=1), opened@@ -796,6 +811,19 @@ int janas_m_route_rank(struct janas_llm_model *m, uint32_t l, uint32_t n,void janas_m_prefetch(struct janas_llm_model *m, uint32_t l);/* The same for a prefill block of n tokens (JANAS_PREFETCH_PREFILL). */void janas_m_prefetch_block(struct janas_llm_model *m, uint32_t l, uint32_t n);+/* Whether a card with memory of its own runs every pass whole+ (JANAS_GPU_WHOLE, on unless 0), and whether a mixture's experts go+ through its tier there (JANAS_GPU_TIER=0: not), model_gpu.c. */+int janas_m_gpu_whole(void);+int janas_m_gpu_tier(const struct janas_llm_model *m);+/* Experts ids[0..nd) of layer l (their slots) on the pairs of rows+ [ra[i], rb[i]) of out, row r token tok[r] of xn (n tokens of d_model+ floats, normalized), on the CPU: their down products before the+ router's weights, for the GPU's tier (gpu_token.h cpu_rows). 0, or -1. */+int janas_m_expert_rows(struct janas_llm_model *m, uint32_t l, uint32_t n,+ const float *xn, uint32_t nd, const uint32_t *ids,+ const uint8_t *const *slots, const uint32_t *ra,+ const uint32_t *rb, const uint32_t *tok, float *out);int janas_m_moe_block(struct janas_llm_model *m, const struct layer *ly,struct janas_expert_cache *cache,const struct janas_jns_layer *jl, uint32_t l, uint32_t n,diff --git a/src/llm/model_load.c b/src/llm/model_load.cindex 1938e2c..5617e99 100644--- a/src/llm/model_load.c+++ b/src/llm/model_load.c@@ -1441,6 +1441,17 @@ void janas_llm_model_free(struct janas_llm_model *m){if (janas_m_exp_tracing())janas_m_exp_report();+ if (m && getenv("JANAS_GPU_VERBOSE") &&+ (m->gtok_runs[0] | m->gtok_runs[1] | m->gtok_runs[2]))+ fprintf(stderr,+ "janas: whole on the GPU %llu single tokens, %llu passes of "+ "2-16, %llu blocks; the state %s (%llu there, %llu back)\n",+ (unsigned long long)m->gtok_runs[0],+ (unsigned long long)m->gtok_runs[1],+ (unsigned long long)m->gtok_runs[2],+ m->gtok_dev ? "copied to the card's memory" : "shared",+ (unsigned long long)m->gtok_push,+ (unsigned long long)m->gtok_pull);if (m && m->skip_asked && (getenv("JANAS_MISS_SKIP") || m->trace))fprintf(stderr,"janas: experts not in RAM below %.3f left out: %llu of %llu "@@ -1833,6 +1844,16 @@ void janas_m_arena_to_gpu(struct janas_llm_model *m){if (!m->gpu || m->gpu_off || !m->cache || m->arena_gpu < 0)return;+ /* a mixture on a card with memory of its own: its experts through the+ tier the whole pass keeps there (gpu_token_moe.c), not a copy of the+ arena beside it */+ if (m->arena_gpu == 0 && janas_m_gpu_tier(m)) {+ m->arena_gpu = -1;+ if (getenv("JANAS_GPU_VERBOSE"))+ fprintf(stderr, "arena: not copied, the experts through the "+ "card's tier\n");+ return;+ }if (m->arena_gpu == 1) {if (!m->arena_stale && !m->arena_in_place &&janas_expert_cache_loads(m->cache) != m->arena_loads)diff --git a/src/llm/model_state.c b/src/llm/model_state.cindex b62aaa1..806edf3 100644--- a/src/llm/model_state.c+++ b/src/llm/model_state.c@@ -232,6 +232,7 @@ struct janas_llm_state *janas_llm_model_state_save(struct janas_llm_model *m,{if (n > m->n_ctx)return NULL;+ janas_m_gtok_host(m, 0); /* what the GPU made newer, read here *//* a ring keeps the window before n only while nothing past n + (itscapacity - the window) has been written over it */for (uint32_t sl = 0; sl < m->n_attn; sl++)@@ -314,6 +315,7 @@ struct janas_llm_state *janas_llm_model_state_save(struct janas_llm_model *m,int janas_llm_model_state_load(struct janas_llm_model *m,const struct janas_llm_state *st){+ janas_m_gtok_host(m, 1); /* the state written here *//* the rows copied do not depend on the context opened, only on itsbeing long enough */if (st->n > m->n_ctx || st->n_slots != m->n_attn + (m->mtp ? 1u : 0u) ||diff --git a/src/llm/quant.c b/src/llm/quant.cindex 20ecdd6..cf08c9e 100644--- a/src/llm/quant.c+++ b/src/llm/quant.c@@ -1244,14 +1244,19 @@ float janas_iq2_dot_ref(int type, const void *w,return acc;}-/* IQ1_M: the block's d, from the top nibbles of its four scale words */-static inline float iq1m_d(const struct janas_block_iq1m *w)+/* IQ1_M: the block's d as fp16, from the top nibbles of its four scale+ words (the kernels convert it with F16C: a call in their loop costs) */+static inline uint16_t iq1m_d16(const struct janas_block_iq1m *w){uint16_t sc[4];memcpy(sc, w->scales, 8);- uint16_t h = (uint16_t)((sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) |- ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000));- return janas_fp16_to_fp32(h);+ return (uint16_t)((sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) |+ ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000));+}++static inline float iq1m_d(const struct janas_block_iq1m *w)+{+ return janas_fp16_to_fp32(iq1m_d16(w));}/* sub-group l (0-31) of an IQ1_M block: its grid entry, whether its shift@@ -1465,22 +1470,68 @@ static inline void unpack_scales_u32(const uint8_t *q, uint32_t out[4])slower than IQ4_NL on 12 threads). Filled before main. */static float e8m0_tab[256];+/* The signs of an IQ2 and IQ3 7-bit index (iq_signs) as eight bytes, 1 or+ -1 (0xff): weight j's multiplier in byte j, for the kernels to read four+ at a time (the parity and the bits spread to bytes took some thirty+ instructions a group of 32). */+static uint64_t iq_sign64[128];+/* the same for a byte of eight signs as it is (IQ2_S) */+static uint64_t iq_byte64[256];++/* iq2_sub with the signs as eight bytes of multipliers (iq_sign64) */+static inline void iq2_sub_mask(int type, const void *blk, int l,+ uint64_t *grid, uint64_t *mask, int *scale)+{+ int g = l / 4, k = l % 4;+ if (type == JANAS_IQ2_XXS) {+ const struct janas_block_iq2xxs *w = blk;+ uint32_t aux[2];+ memcpy(aux, w->qs + 4 * g, 8);+ *grid = iq2xxs_grid[aux[0] >> 8 * k & 255];+ *mask = iq_sign64[aux[1] >> 7 * k & 127];+ *scale = 1 + 2 * (int)(aux[1] >> 28);+ } else if (type == JANAS_IQ2_XS) {+ const struct janas_block_iq2xs *w = blk;+ uint32_t q = w->qs[4 * g + k];+ *grid = iq2xs_grid[q & 511];+ *mask = iq_sign64[q >> 9];+ *scale = 1 + 2 * (w->scales[g] >> 4 * (k / 2) & 15);+ } else { /* IQ2_S: its kernel spreads the sign bytes itself */+ const struct janas_block_iq2s *w = blk;+ *grid = iq2s_grid[w->qs[4 * g + k] |+ ((uint32_t)w->qh[g] << (8 - 2 * k) & 0x300)];+ *mask = iq_byte64[w->qs[32 + 4 * g + k]];+ *scale = 1 + 2 * (w->scales[g] >> 4 * (k / 2) & 15);+ }+}+/* The kernels proper: quant_dotn.h, once for AVX2 and once for AVX-VNNI. */#define DOTN(name) name##_avx2#define DOTN_TARGET __attribute__((target("avx2,fma,f16c")))#define DOTN_ACC(acc, p, s) _mm256_add_epi32(acc, _mm256_madd_epi16(p, s))+#define DOTN_US2(u0, s0, u1, s1) \+ _mm256_madd_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(u0, s0), \+ _mm256_maddubs_epi16(u1, s1)), \+ _mm256_set1_epi16(1))#include "quant_dotn.h"#undef DOTN#undef DOTN_TARGET#undef DOTN_ACC+#undef DOTN_US2+#define DOTN_HAS_VNNI#define DOTN(name) name##_vnni#define DOTN_TARGET __attribute__((target("avx2,fma,f16c,avxvnni")))#define DOTN_ACC(acc, p, s) _mm256_dpwssd_avx_epi32(acc, p, s)+#define DOTN_US2(u0, s0, u1, s1) \+ _mm256_dpbusd_avx_epi32( \+ _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), u0, s0), u1, s1)#include "quant_dotn.h"#undef DOTN#undef DOTN_TARGET#undef DOTN_ACC+#undef DOTN_US2+#undef DOTN_HAS_VNNI#endiftypedef float (*dot_fn)(const struct janas_block_q4k *,@@ -1547,12 +1598,48 @@ typedef void (*dotn6p_fn)(const struct janas_block_q6kp *, const uint8_t *,static dotn6p_fn dot6p_impl; /* NULL: the scalar reference */static const char *dot_name = "ref";static int use_vnni;+/* Q3_K, Q4_K, Q5_K, Q6_K and Q6_K_P on many vectors through the tiles+ (quant_dotn.h q4k_gemm, q5k_gemm, q6k_gemm); JANAS_CPU_TILES=0 the rows'+ products */+static int use_tiles = 1;+/* The vectors from which the tiles pay: their buffer is filled once for+ the vectors of a call, so on few the rows' products win. bench_kernels+ (BENCH_NV, 2048 columns, 12 threads, 25 Oct 2026): Q4_K, Q5_K and Q6_K+ lose at 8 vectors (Q4_K 28.1 against 23.7 GB/s with AVX-VNNI, 21.0+ against 18.8 with AVX2) and win from 12 (17.5 against 19.8, 13.4+ against 14.8); Q3_K, whose filling costs more, wins from 24 (5.8+ against 6.5) and loses at 16 (8.9 against 8.1). */+#define TILES_MIN_NV 12+#define TILES_MIN_NV_Q3K 20+/* the types in blocks of 32 (BENCH_NV, 12 threads): with AVX2 the tiles+ win from 12 vectors (Q8_0 18.3 against 20.9 GB/s), with AVX-VNNI, whose+ rows' products have no reduction tree either, from 24 (Q8_0 19.6+ against 20.4, at 16 29.3 against 27.8) */+#define TILES_MIN_NV_B32 (use_vnni ? 24 : 12)+/* Q2_0, against its rows' products on reordered activations (BENCH_NV,+ 12 threads): with AVX-VNNI the tiles win from 24 vectors (4.6 against+ 5.3 GB/s, at 64 1.7 against 2.1), with AVX2 never (at 12 8.0 against+ 6.6, at 32 and 64 even), its two-bit values unpacked as cheaply */+#define TILES_MIN_NV_Q20 (use_vnni ? (size_t)24 : SIZE_MAX)++/* Q2_K and the IQ types with grids (BENCH_NV, 12 threads): from 12+ vectors, Q2_K even there (8.6 against 8.2 GB/s with AVX-VNNI), the IQ+ types ahead (IQ3_S 8.8 against 6.4); on 64 1.6 to 2.6 times as fast */+#define TILES_MIN_NV_IQK 12/* Resolved before main: the kernels are first called from pool threads. */__attribute__((constructor)) static void dot_kernels_init(void){for (int e = 0; e < 256; e++)e8m0_tab[e] = janas_e8m0_half((uint8_t)e);+ for (uint32_t i = 0; i < 256; i++) {+ uint64_t m = 0;+ for (int j = 0; j < 8; j++)+ m |= (uint64_t)(i >> j & 1 ? 0xff : 0x01) << 8 * j;+ iq_byte64[i] = m;+ }+ for (uint32_t i = 0; i < 128; i++)+ iq_sign64[i] = iq_byte64[iq_signs(i)];#if defined(__x86_64__)__builtin_cpu_init();if (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma") &&@@ -1578,6 +1665,8 @@ __attribute__((constructor)) static void dot_kernels_init(void)dotis_impl = iq3s_dot_avx2;dot6p_impl = q6kp_dotn_avx2;dot_name = "avx2";+ const char *qg = getenv("JANAS_CPU_TILES");+ use_tiles = !(qg && atoi(qg) == 0);const char *k = getenv("JANAS_KERNELS"); /* avx2: no VNNI */if (__builtin_cpu_supports("avxvnni") && !(k && !strcmp(k, "avx2"))) {use_vnni = 1;@@ -1743,6 +1832,35 @@ float janas_q6kp_dot(const struct janas_block_q6kp *w, const uint8_t *p1,return janas_q6kp_dot_ref(w, p1, p0, x, nb);}+size_t janas_q6k_tile_rows(int type, const void *w, size_t row_bytes,+ const uint8_t *p1, const uint8_t *p0, size_t pb,+ const struct janas_block_q8k *x, size_t x_stride,+ size_t nv, size_t nb, size_t rows, float *y,+ size_t y_stride)+{+#if defined(__x86_64__)+ size_t r8 = rows / 8 * 8;+ if (!use_tiles || !dot6p_impl ||+ nv < (type == JANAS_Q3_K ? TILES_MIN_NV_Q3K : TILES_MIN_NV) || !r8 ||+ (type != JANAS_Q6_K && type != JANAS_Q6_K_P && type != JANAS_Q3_K))+ return 0;+ /* q6k_gemm's L: 0 Q6_K, 1-3 Q6_K_P's level, 4 Q3_K */+ int level = type == JANAS_Q6_K ? 0+ : type == JANAS_Q3_K ? 4+ : p0 ? 3+ : p1 ? 2+ : 1;+ (use_vnni ? q6k_gemm_vnni : q6k_gemm_avx2)(+ w, row_bytes, p1, p0, pb, x, x_stride, nv, nb, r8, y, y_stride, level);+ return r8;+#else+ (void)type, (void)w, (void)row_bytes, (void)p1, (void)p0, (void)pb;+ (void)x, (void)x_stride, (void)nv, (void)nb, (void)rows, (void)y;+ (void)y_stride;+ return 0;+#endif+}+void janas_q6kp_dot_multi(const struct janas_block_q6kp *w, const uint8_t *p1,const uint8_t *p0, const struct janas_block_q8k *x,size_t x_stride, size_t nv, size_t nb, float *out,@@ -2095,6 +2213,92 @@ void janas_dot_rows(int type, const void *w, size_t row_bytes,y[r] = janas_dot(type, (const char *)w + r * row_bytes, x, nb);}+void janas_dot_multi_rows(int type, const void *w, size_t row_bytes,+ const struct janas_block_q8k *x, size_t x_stride,+ size_t nv, size_t nb, size_t rows, float *y,+ size_t y_stride)+{+#if defined(__x86_64__)+ if (type == JANAS_Q2_0 && use_avx2_fma && __builtin_cpu_supports("f16c")) {+ size_t r8 = use_tiles && nv >= TILES_MIN_NV_Q20 ? rows / 8 * 8 : 0;+ if (r8)+ (use_vnni ? b32_gemm_vnni : b32_gemm_avx2)(+ type, w, row_bytes, x, x_stride, nv, nb, r8, y, y_stride);+ if (rows > r8)+ (use_vnni ? q2_0_multi_rows_vnni : q2_0_multi_rows_avx2)(+ (const char *)w + r8 * row_bytes, row_bytes, x, x_stride, nv,+ nb, rows - r8, y + r8, y_stride);+ return;+ }+ if (use_tiles && use_avx2_fma && nv >= (size_t)TILES_MIN_NV_B32 &&+ rows >= 8 && __builtin_cpu_supports("f16c")) {+ size_t r8 = rows / 8 * 8;+ if ((use_vnni ? b32_gemm_vnni : b32_gemm_avx2)(+ type, w, row_bytes, x, x_stride, nv, nb, r8, y, y_stride)) {+ w = (const char *)w + r8 * row_bytes;+ y += r8;+ rows -= r8;+ if (!rows)+ return;+ }+ }+ if (use_vnni && b32_multi_rows_vnni(type, w, row_bytes, x, x_stride, nv, nb,+ rows, y, y_stride))+ return;+ if (type == JANAS_Q4_K && use_avx2_fma && __builtin_cpu_supports("f16c")) {+ size_t r8 = use_tiles && nv >= TILES_MIN_NV ? rows / 8 * 8 : 0;+ if (r8)+ (use_vnni ? q4k_gemm_vnni : q4k_gemm_avx2)(+ w, row_bytes, x, x_stride, nv, nb, r8, y, y_stride);+ if (rows > r8)+ (use_vnni ? q4k_multi_rows_vnni : q4k_multi_rows_avx2)(+ (const char *)w + r8 * row_bytes, row_bytes, x, x_stride, nv,+ nb, rows - r8, y + r8, y_stride);+ return;+ }+ if (type == JANAS_Q5_K && use_avx2_fma && __builtin_cpu_supports("f16c")) {+ size_t r8 = use_tiles && nv >= TILES_MIN_NV ? rows / 8 * 8 : 0;+ if (r8)+ (use_vnni ? q5k_gemm_vnni : q5k_gemm_avx2)(+ w, row_bytes, x, x_stride, nv, nb, r8, y, y_stride);+ if (rows > r8)+ (use_vnni ? q5k_multi_rows_vnni : q5k_multi_rows_avx2)(+ (const char *)w + r8 * row_bytes, row_bytes, x, x_stride, nv,+ nb, rows - r8, y + r8, y_stride);+ return;+ }+ if (type == JANAS_IQ4_XS && use_tiles && use_avx2_fma &&+ nv >= TILES_MIN_NV && rows >= 8 && __builtin_cpu_supports("f16c")) {+ size_t r8 = rows / 8 * 8;+ (use_vnni ? iq4xs_gemm_vnni : iq4xs_gemm_avx2)(+ w, row_bytes, x, x_stride, nv, nb, r8, y, y_stride);+ w = (const char *)w + r8 * row_bytes;+ y += r8;+ rows -= r8;+ }+ if (use_tiles && use_avx2_fma && nv >= (size_t)TILES_MIN_NV_IQK &&+ rows >= 8 && __builtin_cpu_supports("f16c")) {+ size_t r8 = rows / 8 * 8;+ if ((use_vnni ? iqk_gemm_vnni : iqk_gemm_avx2)(+ type, w, row_bytes, x, x_stride, nv, nb, r8, y, y_stride)) {+ w = (const char *)w + r8 * row_bytes;+ y += r8;+ rows -= r8;+ }+ }+ if (type == JANAS_Q6_K || type == JANAS_Q3_K) {+ size_t r8 = janas_q6k_tile_rows(type, w, row_bytes, NULL, NULL, 0, x,+ x_stride, nv, nb, rows, y, y_stride);+ w = (const char *)w + r8 * row_bytes;+ y += r8;+ rows -= r8;+ }+#endif+ for (size_t r = 0; r < rows; r++)+ janas_dot_multi(type, (const char *)w + r * row_bytes, x, x_stride, nv,+ nb, y + r, y_stride);+}+void janas_dot_multi(int type, const void *w, const struct janas_block_q8k *x,size_t x_stride, size_t nv, size_t nb, float *out,size_t out_stride)@@ -2291,6 +2495,44 @@ void janas_dot_rows_n(int type, const void *w, size_t row_bytes,}}+/* the whole groups of all the rows, then the tails CHUNK rows and VEC+ vectors at a time: janas_dot_multi_rows gives them the tiles */+void janas_dot_multi_rows_n(int type, const void *w, size_t row_bytes,+ const struct janas_block_q8k *x, size_t x_stride,+ size_t nv, size_t n, size_t rows, float *y,+ size_t y_stride)+{+ size_t nb = n / JANAS_QK;+ if (n % JANAS_QK == 0) {+ janas_dot_multi_rows(type, w, row_bytes, x, x_stride, nv, nb, rows, y,+ y_stride);+ return;+ }+ enum { CHUNK = 16, VEC = 64 };+ size_t gb = janas_qtype_block_size(type), full = nb * gb;+ _Alignas(64) uint8_t pad[CHUNK * TAIL_GROUP];+ float t[CHUNK * VEC];+ if (nb)+ janas_dot_multi_rows(type, w, row_bytes, x, x_stride, nv, nb, rows, y,+ y_stride);+ for (size_t r0 = 0; r0 < rows; r0 += CHUNK) {+ size_t c = rows - r0 < CHUNK ? rows - r0 : CHUNK;+ for (size_t i = 0; i < c; i++)+ tail_group(type, (const uint8_t *)w + (r0 + i) * row_bytes, full,+ row_bytes, pad + i * gb);+ for (size_t v0 = 0; v0 < nv; v0 += VEC) {+ size_t g = nv - v0 < VEC ? nv - v0 : VEC;+ janas_dot_multi_rows(type, pad, gb, x + v0 * x_stride + nb,+ x_stride, g, 1, c, t, c);+ for (size_t v = 0; v < g; v++)+ for (size_t i = 0; i < c; i++) {+ float *o = y + r0 + i + (v0 + v) * y_stride;+ *o = (nb ? *o : 0.0f) + t[i + v * c];+ }+ }+ }+}+void janas_dot_multi_n(int type, const void *w, const struct janas_block_q8k *x,size_t x_stride, size_t nv, size_t n, float *out,size_t out_stride)diff --git a/src/llm/quant.h b/src/llm/quant.hindex e23a80f..f2002f5 100644--- a/src/llm/quant.h+++ b/src/llm/quant.h@@ -368,6 +368,12 @@ void janas_dot_rows_n(int type, const void *w, size_t row_bytes,void janas_dot_multi_n(int type, const void *w, const struct janas_block_q8k *x,size_t x_stride, size_t nv, size_t n, float *out,size_t out_stride);+/* janas_dot_multi_n on rows rows of row_bytes: y[r + v * y_stride], the+ same bits */+void janas_dot_multi_rows_n(int type, const void *w, size_t row_bytes,+ const struct janas_block_q8k *x, size_t x_stride,+ size_t nv, size_t n, size_t rows, float *y,+ size_t y_stride);/** Dot product of nb blocks of a supported type with nb Q8_K blocks. Aborts on@@ -403,6 +409,14 @@ void janas_dot_multi(int type, const void *w, const struct janas_block_q8k *x,size_t x_stride, size_t nv, size_t nb, float *out,size_t out_stride);+/* janas_dot_multi on rows rows of row_bytes from w: y[r + v * y_stride].+ The same bits; Q2_0 reorders the vectors' activations once for all the+ rows. */+void janas_dot_multi_rows(int type, const void *w, size_t row_bytes,+ const struct janas_block_q8k *x, size_t x_stride,+ size_t nv, size_t nb, size_t rows, float *y,+ size_t y_stride);+/* Float helpers, AVX2 when available: dot product and y += a * x. */float janas_dot_f32(const float *a, const float *b, size_t n);void janas_axpy_f32(float *y, float a, const float *x, size_t n);@@ -528,6 +542,16 @@ float janas_q6kp_dot(const struct janas_block_q6kp *w, const uint8_t *p1,float janas_q6kp_dot_ref(const struct janas_block_q6kp *w, const uint8_t *p1,const uint8_t *p0, const struct janas_block_q8k *x,size_t nb);+/* Q6_K or Q3_K (p1, p0 NULL) or Q6_K_P (each row's planes pb bytes apart from+ p1 and p0, NULL past its level) rows against nv vectors, y[r + v ys], eight+ rows at a time as tiles (quant_dotn.h q6k_gemm): the rows done, a+ multiple of eight (0 where the tiles do not serve: few vectors, no+ AVX2, JANAS_CPU_TILES=0). The same bits as janas_dot_multi. */+size_t janas_q6k_tile_rows(int type, const void *w, size_t row_bytes,+ const uint8_t *p1, const uint8_t *p0, size_t pb,+ const struct janas_block_q8k *x, size_t x_stride,+ size_t nv, size_t nb, size_t rows, float *y,+ size_t y_stride);/* As janas_dot_multi, for the plane form: one row against nv vectors. */void janas_q6kp_dot_multi(const struct janas_block_q6kp *w, const uint8_t *p1,const uint8_t *p0, const struct janas_block_q8k *x,diff --git a/src/llm/quant_dotn.h b/src/llm/quant_dotn.hindex f80e7e6..95022a3 100644--- a/src/llm/quant_dotn.h+++ b/src/llm/quant_dotn.h@@ -13,6 +13,10 @@* DOTN_ACC(acc, p, s) acc + the int16 pairs p times the int16 scales s,* summed pairwise into int32 lanes: madd + add on* AVX2, a single vpdpwssd on AVX-VNNI+ * DOTN_US2(u0, s0, u1, s1) the unsigned bytes u0 times the signed s0+ * plus u1 times s1, four by four into int32 lanes:+ * maddubs, add, madd on AVX2 (the products must fit+ * int16 two by two), two vpdpbusd on AVX-VNNI* Integer sums are exact, so every int32 lane holds the same value with* either definition, and the float operations are the same, in the same* order: all variants, single or multi-vector, give bit-identical results@@ -30,6 +34,18 @@ DOTN(hsum_epi32)(__m256i v)return _mm_cvtsi128_si32(s);}+/* The eight lanes of a and of b summed (exact), together: *ra and *rb. */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(hsum2_epi32)(__m256i a, __m256i b, int *ra, int *rb)+{+ __m256i h = _mm256_hadd_epi32(a, b);+ __m128i s = _mm_add_epi32(_mm256_castsi256_si128(h),+ _mm256_extracti128_si256(h, 1));+ s = _mm_hadd_epi32(s, s);+ *ra = _mm_cvtsi128_si32(s);+ *rb = _mm_extract_epi32(s, 1);+}+static inline DOTN_TARGET __attribute__((always_inline)) intDOTN(hsum128_epi32)(__m128i s){@@ -70,10 +86,54 @@ DOTN(prefetch_ahead)(const void *at, size_t size)* bit. From three vectors on, the sums of all the vectors are reduced* together and their accumulators share a register, one lane per vector.*/+/* What the products of a row on 5-8 vectors need of the vectors alone,+ made once for all the rows (q4k_multi_rows): for each group of 256 the+ vectors' scales, then their bsums2 two by two, int16 pairs with one+ int32 lane a vector, so that the eight vectors' sums of mins x+ activations are four madd and not eight and six hadd. */+#define Q4K_PRE 160 /* bytes a group of 256: 8 floats, 4 x 8 pairs */++static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q4k_pre)(const struct janas_block_q8k *x, size_t xs, int nv, size_t nb,+ uint8_t *pre)+{+ for (size_t b = 0; b < nb; b++) {+ float *d8 = (float *)(pre + Q4K_PRE * b);+ int16_t *pr = (int16_t *)(pre + Q4K_PRE * b + 32);+ for (int v = 0; v < 8; v++) {+ const struct janas_block_q8k *xb = x + (size_t)v * xs + b;+ d8[v] = v < nv ? xb->d : 0.0f;+ for (int i = 0; i < 4; i++)+ for (int h = 0; h < 2; h++)+ pr[16 * i + 2 * v + h] =+ v < nv ? xb->bsums2[2 * i + h] : (int16_t)0;+ }+ }+}++/* From the vectors made ready (q4k_pre), group b's: their scales and the+ sums of the block's eight mins times their activations, a lane each. */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(k_pre_mins)(const uint8_t *pb, __m128i mins16, __m256 *d8, __m256i *im8)+{+ const __m256i *pr = (const __m256i *)(pb + 32);+ __m256i mm = _mm256_broadcastsi128_si256(mins16);+ *d8 = _mm256_loadu_ps((const float *)pb);+ *im8 = _mm256_add_epi32(+ _mm256_add_epi32(_mm256_madd_epi16(_mm256_loadu_si256(pr),+ _mm256_shuffle_epi32(mm, 0x00)),+ _mm256_madd_epi16(_mm256_loadu_si256(pr + 1),+ _mm256_shuffle_epi32(mm, 0x55))),+ _mm256_add_epi32(_mm256_madd_epi16(_mm256_loadu_si256(pr + 2),+ _mm256_shuffle_epi32(mm, 0xaa)),+ _mm256_madd_epi16(_mm256_loadu_si256(pr + 3),+ _mm256_shuffle_epi32(mm, 0xff))));+}+static inline DOTN_TARGET __attribute__((always_inline)) voidDOTN(q4k_dotn_body)(const struct janas_block_q4k *w,const struct janas_block_q8k *x, size_t xs, size_t nb,- float *out, const int nv)+ float *out, const int nv, const uint8_t *pre){const __m256i low4 = _mm256_set1_epi8(0x0f);/* up to 2 vectors: lanes acc0, acc1, accm0, accm1; 3 or 4: one lane@@ -129,7 +189,7 @@ DOTN(q4k_dotn_body)(const struct janas_block_q4k *w,_Pragma("GCC unroll 8") for (int v = 0; v < 8; v++){pm[v] = _mm_setzero_si128();- if (v < nv) {+ if (v < nv && !(pre && nv > 4)) {const struct janas_block_q8k *xb = x + (size_t)v * xs + b;pm[v] = _mm_madd_epi16(mins16, _mm_loadu_si128((const __m128i *)xb->bsums2));@@ -173,15 +233,21 @@ DOTN(q4k_dotn_body)(const struct janas_block_q4k *w,__m256i is8 =_mm256_add_epi32(_mm256_permute2x128_si256(t0, t1, 0x20),_mm256_permute2x128_si256(t0, t1, 0x31));- __m128i m03 = _mm_hadd_epi32(_mm_hadd_epi32(pm[0], pm[1]),- _mm_hadd_epi32(pm[2], pm[3]));- __m128i m47 = _mm_hadd_epi32(_mm_hadd_epi32(pm[4], pm[5]),- _mm_hadd_epi32(pm[6], pm[7]));- __m256i im8 = _mm256_set_m128i(m47, m03);+ __m256i im8;+ __m256 d8;+ if (pre) {+ DOTN(k_pre_mins)(pre + Q4K_PRE * b, mins16, &d8, &im8);+ } else {+ __m128i m03 = _mm_hadd_epi32(_mm_hadd_epi32(pm[0], pm[1]),+ _mm_hadd_epi32(pm[2], pm[3]));+ __m128i m47 = _mm_hadd_epi32(_mm_hadd_epi32(pm[4], pm[5]),+ _mm_hadd_epi32(pm[6], pm[7]));+ im8 = _mm256_set_m128i(m47, m03);#define D8(v) (v < nv ? x[(size_t)(v) * xs + b].d : 0.0f)- __m256 d8 = _mm256_setr_ps(D8(0), D8(1), D8(2), D8(3), D8(4), D8(5),- D8(6), D8(7));+ d8 = _mm256_setr_ps(D8(0), D8(1), D8(2), D8(3), D8(4), D8(5),+ D8(6), D8(7));#undef D8+ }acc8 = _mm256_fmadd_ps(_mm256_mul_ps(_mm256_set1_ps(dw), d8),_mm256_cvtepi32_ps(is8), acc8);accm8 = _mm256_fmadd_ps(_mm256_mul_ps(_mm256_set1_ps(dmw), d8),@@ -208,7 +274,7 @@ DOTN(q4k_dotn_body)(const struct janas_block_q4k *w,static inline DOTN_TARGET __attribute__((always_inline)) voidDOTN(q5k_dotn_body)(const struct janas_block_q5k *w,const struct janas_block_q8k *x, size_t xs, size_t nb,- float *out, const int nv)+ float *out, const int nv, const uint8_t *pre){const __m256i low4 = _mm256_set1_epi8(0x0f);const __m256i one = _mm256_set1_epi8(1);@@ -273,7 +339,7 @@ DOTN(q5k_dotn_body)(const struct janas_block_q5k *w,_Pragma("GCC unroll 8") for (int v = 0; v < 8; v++){pm[v] = _mm_setzero_si128();- if (v < nv) {+ if (v < nv && !(pre && nv > 4)) {const struct janas_block_q8k *xb = x + (size_t)v * xs + b;pm[v] = _mm_madd_epi16(mins16, _mm_loadu_si128((const __m128i *)xb->bsums2));@@ -303,15 +369,21 @@ DOTN(q5k_dotn_body)(const struct janas_block_q5k *w,__m256i is8 =_mm256_add_epi32(_mm256_permute2x128_si256(t0, t1, 0x20),_mm256_permute2x128_si256(t0, t1, 0x31));- __m128i m03 = _mm_hadd_epi32(_mm_hadd_epi32(pm[0], pm[1]),- _mm_hadd_epi32(pm[2], pm[3]));- __m128i m47 = _mm_hadd_epi32(_mm_hadd_epi32(pm[4], pm[5]),- _mm_hadd_epi32(pm[6], pm[7]));- __m256i im8 = _mm256_set_m128i(m47, m03);+ __m256i im8;+ __m256 d8;+ if (pre && nv > 4) {+ DOTN(k_pre_mins)(pre + Q4K_PRE * b, mins16, &d8, &im8);+ } else {+ __m128i m03 = _mm_hadd_epi32(_mm_hadd_epi32(pm[0], pm[1]),+ _mm_hadd_epi32(pm[2], pm[3]));+ __m128i m47 = _mm_hadd_epi32(_mm_hadd_epi32(pm[4], pm[5]),+ _mm_hadd_epi32(pm[6], pm[7]));+ im8 = _mm256_set_m128i(m47, m03);#define D8(v) (v < nv ? x[(size_t)(v) * xs + b].d : 0.0f)- __m256 d8 = _mm256_setr_ps(D8(0), D8(1), D8(2), D8(3), D8(4), D8(5),- D8(6), D8(7));+ d8 = _mm256_setr_ps(D8(0), D8(1), D8(2), D8(3), D8(4), D8(5),+ D8(6), D8(7));#undef D8+ }acc8 = _mm256_fmadd_ps(_mm256_mul_ps(_mm256_set1_ps(dw), d8),_mm256_cvtepi32_ps(is8), acc8);accm8 = _mm256_fmadd_ps(_mm256_mul_ps(_mm256_set1_ps(dmw), d8),@@ -436,7 +508,8 @@ DOTN(q6k_dotn_body)(const struct janas_block_q6k *w,__m256i sumi[8];_Pragma("GCC unroll 8") for (int v = 0; v < nv; v++) sumi[v] =_mm256_setzero_si256();- for (int n = 0; n < 2; n++) {+ _Pragma("GCC unroll 2") for (int n = 0; n < 2; n++)+ {__m256i ql0 =_mm256_loadu_si256((const __m256i *)(w[b].ql + 64 * n));__m256i ql1 =@@ -528,28 +601,28 @@ static DOTN_TARGET void DOTN(q4k_dotn)(const struct janas_block_q4k *w,{switch (nv) {case 8:- DOTN(q4k_dotn_body)(w, x, xs, nb, out, 8);+ DOTN(q4k_dotn_body)(w, x, xs, nb, out, 8, NULL);break;case 7:- DOTN(q4k_dotn_body)(w, x, xs, nb, out, 7);+ DOTN(q4k_dotn_body)(w, x, xs, nb, out, 7, NULL);break;case 6:- DOTN(q4k_dotn_body)(w, x, xs, nb, out, 6);+ DOTN(q4k_dotn_body)(w, x, xs, nb, out, 6, NULL);break;case 5:- DOTN(q4k_dotn_body)(w, x, xs, nb, out, 5);+ DOTN(q4k_dotn_body)(w, x, xs, nb, out, 5, NULL);break;case 4:- DOTN(q4k_dotn_body)(w, x, xs, nb, out, 4);+ DOTN(q4k_dotn_body)(w, x, xs, nb, out, 4, NULL);break;case 3:- DOTN(q4k_dotn_body)(w, x, xs, nb, out, 3);+ DOTN(q4k_dotn_body)(w, x, xs, nb, out, 3, NULL);break;case 2:- DOTN(q4k_dotn_body)(w, x, xs, nb, out, 2);+ DOTN(q4k_dotn_body)(w, x, xs, nb, out, 2, NULL);break;default:- DOTN(q4k_dotn_body)(w, x, xs, nb, out, 1);+ DOTN(q4k_dotn_body)(w, x, xs, nb, out, 1, NULL);}}@@ -589,7 +662,7 @@ static DOTN_TARGET float DOTN(q4k_dot)(const struct janas_block_q4k *w,size_t nb){float r;- DOTN(q4k_dotn_body)(w, x, 0, nb, &r, 1);+ DOTN(q4k_dotn_body)(w, x, 0, nb, &r, 1, NULL);return r;}@@ -608,28 +681,28 @@ static DOTN_TARGET void DOTN(q5k_dotn)(const struct janas_block_q5k *w,{switch (nv) {case 8:- DOTN(q5k_dotn_body)(w, x, xs, nb, out, 8);+ DOTN(q5k_dotn_body)(w, x, xs, nb, out, 8, NULL);break;case 7:- DOTN(q5k_dotn_body)(w, x, xs, nb, out, 7);+ DOTN(q5k_dotn_body)(w, x, xs, nb, out, 7, NULL);break;case 6:- DOTN(q5k_dotn_body)(w, x, xs, nb, out, 6);+ DOTN(q5k_dotn_body)(w, x, xs, nb, out, 6, NULL);break;case 5:- DOTN(q5k_dotn_body)(w, x, xs, nb, out, 5);+ DOTN(q5k_dotn_body)(w, x, xs, nb, out, 5, NULL);break;case 4:- DOTN(q5k_dotn_body)(w, x, xs, nb, out, 4);+ DOTN(q5k_dotn_body)(w, x, xs, nb, out, 4, NULL);break;case 3:- DOTN(q5k_dotn_body)(w, x, xs, nb, out, 3);+ DOTN(q5k_dotn_body)(w, x, xs, nb, out, 3, NULL);break;case 2:- DOTN(q5k_dotn_body)(w, x, xs, nb, out, 2);+ DOTN(q5k_dotn_body)(w, x, xs, nb, out, 2, NULL);break;default:- DOTN(q5k_dotn_body)(w, x, xs, nb, out, 1);+ DOTN(q5k_dotn_body)(w, x, xs, nb, out, 1, NULL);}}@@ -665,12 +738,328 @@ static DOTN_TARGET void DOTN(q8_0_dotn)(const struct janas_block_q8_0 *w,}}+/* Q4_K and Q5_K rows against nv vectors, y[r + v ys]: eight vectors at a+ time with what they need of themselves made once for all the rows+ (q4k_pre); four or fewer as janas_dot_multi takes them. The same bits. */+#define Q4K_PRE_MAX 128 /* groups of 256 on the stack: 20 KB */+#define K_MULTI_ROWS(name, T) \+ static DOTN_TARGET void DOTN(name##_multi_rows)( \+ const void *w, size_t row_bytes, const struct janas_block_q8k *x, \+ size_t xs, size_t nv, size_t nb, size_t rows, float *y, size_t ys) \+ { \+ _Alignas(32) uint8_t pre[Q4K_PRE_MAX * Q4K_PRE]; \+ for (size_t v0 = 0; v0 < nv; v0 += 8) { \+ int n = nv - v0 < 8 ? (int)(nv - v0) : 8; \+ const struct janas_block_q8k *xv = x + v0 * xs; \+ int with = n > 4 && rows > 1 && nb <= Q4K_PRE_MAX; \+ if (with) \+ DOTN(q4k_pre)(xv, xs, n, nb, pre); \+ for (size_t r = 0; r < rows; r++) { \+ const T *wr = (const void *)((const char *)w + r * row_bytes); \+ float o[8]; \+ if (!with) \+ DOTN(name##_dotn)(wr, xv, xs, n, nb, o); \+ else if (n == 8) \+ DOTN(name##_dotn_body)(wr, xv, xs, nb, o, 8, pre); \+ else if (n == 7) \+ DOTN(name##_dotn_body)(wr, xv, xs, nb, o, 7, pre); \+ else if (n == 6) \+ DOTN(name##_dotn_body)(wr, xv, xs, nb, o, 6, pre); \+ else \+ DOTN(name##_dotn_body)(wr, xv, xs, nb, o, 5, pre); \+ for (int i = 0; i < n; i++) \+ y[r + (v0 + (size_t)i) * ys] = o[i]; \+ } \+ } \+ }+K_MULTI_ROWS(q4k, struct janas_block_q4k)+K_MULTI_ROWS(q5k, struct janas_block_q5k)+#undef K_MULTI_ROWS++/*+ * Q4_K as a product of tiles, for a prompt's blocks: eight rows by T+ * tokens. The products above hold a row's 32-weight piece in one register+ * against a vector's 32 activations, so every maddubs needs its own madd to+ * weigh the piece by its scale, and the two share the same ports: 32+ * products a cycle at best, about 15 measured on a core (Core Ultra 7 155H,+ * AVX2). Here the eight rows are transposed four bytes at a time into a+ * small buffer (q4kg_blk, a chunk of blocks at a time, once for every 64+ * tokens), so that a register holds four weights of each of eight rows and+ * the token's four activations are broadcast against it: a 32-weight+ * piece is eight maddubs, summed in int16 (8 x 2 x 15 x 128 = 30720, no+ * overflow), then one madd by the eight rows' scales - and every lane is a+ * row, so no horizontal sums. The integer sums are the same exact numbers+ * and the floats the same operations in the same order (d x d8 times the+ * sum, fused into the accumulator, the mins' apart, their difference at+ * the end), so the result is the other kernels' to the bit.+ */+#ifndef Q4KG_DEFINED+#define Q4KG_DEFINED+#define Q4KG_CHUNK 16 /* blocks transposed at a time: 23 KB */+#define Q4KG_TOK 64 /* tokens whose sums are held at a time */+struct q4kg_blk {+ uint32_t qs[4][8][8]; /* [piece][dword][row] */+ int16_t sc[8][16]; /* [sub-block][2 row + i]: its scale, twice */+ int16_t mn[4][16]; /* [k][2 row + i]: mins 2k and 2k + 1 */+ int32_t sc32[8][8]; /* [sub-block][row]: the scale (AVX-VNNI's) */+ float d[8], dmin[8];+};+#endif++/* The eight scales and eight mins of a Q4_K or Q5_K block of each of the+ eight rows (s[r]: scales 0-7, mins 0-7, as unpack_scales_u32 lays them)+ into the tiles' tables: an 8 x 8 transpose of their pairs gives the+ mins' pairs (mn), and the scales' bytes, zero extended, sc32 (AVX-VNNI)+ or sc twice a row */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(kmin_scales8)(int16_t (*sc)[16], int32_t (*sc32)[8], int16_t (*mn)[16],+ const __m128i *s, const int sgn)+{+ __m128i b[8], c[8], t[8];+ for (int r = 0; r < 8; r += 2) {+ b[r] = _mm_unpacklo_epi16(s[r], s[r + 1]);+ b[r + 1] = _mm_unpackhi_epi16(s[r], s[r + 1]);+ }+ for (int h = 0; h < 8; h += 4) {+ c[h] = _mm_unpacklo_epi32(b[h], b[h + 2]);+ c[h + 1] = _mm_unpackhi_epi32(b[h], b[h + 2]);+ c[h + 2] = _mm_unpacklo_epi32(b[h + 1], b[h + 3]);+ c[h + 3] = _mm_unpackhi_epi32(b[h + 1], b[h + 3]);+ }+ for (int q = 0; q < 4; q++) {+ t[2 * q] = _mm_unpacklo_epi64(c[q], c[q + 4]);+ t[2 * q + 1] = _mm_unpackhi_epi64(c[q], c[q + 4]);+ }+ /* t[kk]: word r is row r's scales 2kk and 2kk + 1, t[4 + kk] its mins */+ const __m128i split =+ _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);+ for (int kk = 0; kk < 4; kk++) {+ _mm256_storeu_si256((__m256i *)mn[kk],+ sgn ? _mm256_cvtepi8_epi16(t[4 + kk])+ : _mm256_cvtepu8_epi16(t[4 + kk]));+ __m128i e = _mm_shuffle_epi8(t[kk], split);+#ifdef DOTN_HAS_VNNI+ (void)sc;+ __m128i e1 = _mm_srli_si128(e, 8);+ _mm256_storeu_si256((__m256i *)sc32[2 * kk],+ sgn ? _mm256_cvtepi8_epi32(e)+ : _mm256_cvtepu8_epi32(e));+ _mm256_storeu_si256((__m256i *)sc32[2 * kk + 1],+ sgn ? _mm256_cvtepi8_epi32(e1)+ : _mm256_cvtepu8_epi32(e1));+#else+ (void)sc32;+ for (int h = 0; h < 2; h++) {+ __m128i eh = h ? _mm_srli_si128(e, 8) : e;+ __m128i v = sgn ? _mm_cvtepi8_epi16(eh) : _mm_cvtepu8_epi16(eh);+ _mm256_storeu_si256((__m256i *)sc[2 * kk + h],+ _mm256_set_m128i(_mm_unpackhi_epi16(v, v),+ _mm_unpacklo_epi16(v, v)));+ }+#endif+ }+}++/* Blocks [b0, b0 + n) of rows w[0..7] into bk */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q4kg_fill)(const struct janas_block_q4k *const *w, size_t b0, int n,+ struct q4kg_blk *bk)+{+ for (int i = 0; i < n; i++) {+ struct q4kg_blk *k = bk + i;+ for (int c = 0; c < 4; c++) {+ __m256i r[8], t[8], u[8];+ for (int j = 0; j < 8; j++)+ r[j] = _mm256_loadu_si256(+ (const __m256i *)(w[j][b0 + i].qs + 32 * c));+ for (int j = 0; j < 8; j += 2) {+ t[j] = _mm256_unpacklo_epi32(r[j], r[j + 1]);+ t[j + 1] = _mm256_unpackhi_epi32(r[j], r[j + 1]);+ }+ for (int j = 0; j < 8; j += 4) {+ u[j] = _mm256_unpacklo_epi64(t[j], t[j + 2]);+ u[j + 1] = _mm256_unpackhi_epi64(t[j], t[j + 2]);+ u[j + 2] = _mm256_unpacklo_epi64(t[j + 1], t[j + 3]);+ u[j + 3] = _mm256_unpackhi_epi64(t[j + 1], t[j + 3]);+ }+ for (int q = 0; q < 4; q++) {+ _mm256_storeu_si256(+ (__m256i *)k->qs[c][q],+ _mm256_permute2x128_si256(u[q], u[q + 4], 0x20));+ _mm256_storeu_si256(+ (__m256i *)k->qs[c][q + 4],+ _mm256_permute2x128_si256(u[q], u[q + 4], 0x31));+ }+ }+ __m128i sv[8];+ for (int j = 0; j < 8; j++) {+ const struct janas_block_q4k *blk = &w[j][b0 + i];+ uint32_t u[4];+ unpack_scales_u32(blk->scales, u);+ sv[j] = _mm_loadu_si128((const __m128i *)u);+ k->d[j] = _cvtsh_ss(blk->d);+ k->dmin[j] = _cvtsh_ss(blk->dmin);+ }+ DOTN(kmin_scales8)(k->sc, k->sc32, k->mn, sv, 0);+ }+}++/* T tokens x[0..T) (xs blocks apart) on n transposed blocks from block b0:+ the eight rows' sums added into af and am (8 floats a token) */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q4kg_tile)(const struct q4kg_blk *bk, int n, size_t b0,+ const struct janas_block_q8k *x, size_t xs, float *af,+ float *am, const int T)+{+ const __m256i low4 = _mm256_set1_epi8(0x0f);+ for (int i = 0; i < n; i++) {+ const struct q4kg_blk *k = bk + i;+ size_t b = b0 + (size_t)i;+ __m256i is[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) is[t] =+ _mm256_setzero_si256();+ _Pragma("GCC unroll 4") for (int c = 0; c < 4; c++)+ {+ _Pragma("GCC unroll 2") for (int h = 0; h < 2; h++)+ {+#ifdef DOTN_HAS_VNNI+ /* four products summed into the row's int32 lane at+ once, the piece's scale by one multiply */+ __m256i a32[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) a32[t] =+ _mm256_setzero_si256();+ _Pragma("GCC unroll 8") for (int q = 0; q < 8; q++)+ {+ __m256i wv =+ _mm256_loadu_si256((const __m256i *)k->qs[c][q]);+ wv = h ? _mm256_and_si256(_mm256_srli_epi16(wv, 4), low4)+ : _mm256_and_si256(wv, low4);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ int32_t aw;+ memcpy(&aw,+ x[(size_t)t * xs + b].qs + 64 * c + 32 * h ++ 4 * q,+ 4);+ a32[t] = _mm256_dpbusd_avx_epi32(a32[t], wv,+ _mm256_set1_epi32(aw));+ }+ }+ __m256i sc =+ _mm256_loadu_si256((const __m256i *)k->sc32[2 * c + h]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) is[t] =+ _mm256_add_epi32(is[t], _mm256_mullo_epi32(a32[t], sc));+#else+ __m256i a16[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) a16[t] =+ _mm256_setzero_si256();+ _Pragma("GCC unroll 8") for (int q = 0; q < 8; q++)+ {+ __m256i wv =+ _mm256_loadu_si256((const __m256i *)k->qs[c][q]);+ wv = h ? _mm256_and_si256(_mm256_srli_epi16(wv, 4), low4)+ : _mm256_and_si256(wv, low4);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ int32_t aw;+ memcpy(&aw,+ x[(size_t)t * xs + b].qs + 64 * c + 32 * h ++ 4 * q,+ 4);+ a16[t] = _mm256_add_epi16(+ a16[t],+ _mm256_maddubs_epi16(wv, _mm256_set1_epi32(aw)));+ }+ }+ __m256i sc =+ _mm256_loadu_si256((const __m256i *)k->sc[2 * c + h]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) is[t] =+ DOTN_ACC(is[t], a16[t], sc);+#endif+ }+ }+ __m256 dw = _mm256_loadu_ps(k->d), dmw = _mm256_loadu_ps(k->dmin);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ const struct janas_block_q8k *xb = x + (size_t)t * xs + b;+ __m256i im = _mm256_setzero_si256();+ _Pragma("GCC unroll 4") for (int kk = 0; kk < 4; kk++)+ {+ int32_t bw;+ memcpy(&bw, xb->bsums2 + 2 * kk, 4);+ im = _mm256_add_epi32(+ im, _mm256_madd_epi16(+ _mm256_loadu_si256((const __m256i *)k->mn[kk]),+ _mm256_set1_epi32(bw)));+ }+ __m256 d8 = _mm256_set1_ps(xb->d);+ _mm256_storeu_ps(af + 8 * t,+ _mm256_fmadd_ps(_mm256_mul_ps(dw, d8),+ _mm256_cvtepi32_ps(is[t]),+ _mm256_loadu_ps(af + 8 * t)));+ _mm256_storeu_ps(am + 8 * t,+ _mm256_fmadd_ps(_mm256_mul_ps(dmw, d8),+ _mm256_cvtepi32_ps(im),+ _mm256_loadu_ps(am + 8 * t)));+ }+ }+}++#ifndef Q4KG_T+#define Q4KG_T 6 /* tokens a tile */+#endif++/* rows (a multiple of 8) of Q4_K against nv vectors, y[r + v ys] */+static DOTN_TARGET void DOTN(q4k_gemm)(const void *w, size_t row_bytes,+ const struct janas_block_q8k *x,+ size_t xs, size_t nv, size_t nb,+ size_t rows, float *y, size_t ys)+{+ _Alignas(32) struct q4kg_blk bk[Q4KG_CHUNK];+ _Alignas(32) float af[Q4KG_TOK * 8], am[Q4KG_TOK * 8];+ for (size_t g = 0; g < rows; g += 8) {+ const struct janas_block_q4k *wr[8];+ for (int j = 0; j < 8; j++)+ wr[j] = (const void *)((const char *)w + (g + j) * row_bytes);+ for (size_t v0 = 0; v0 < nv; v0 += Q4KG_TOK) {+ size_t nt = nv - v0 < Q4KG_TOK ? nv - v0 : Q4KG_TOK;+ memset(af, 0, nt * 8 * sizeof(float));+ memset(am, 0, nt * 8 * sizeof(float));+ for (size_t b0 = 0; b0 < nb; b0 += Q4KG_CHUNK) {+ int n = nb - b0 < Q4KG_CHUNK ? (int)(nb - b0) : Q4KG_CHUNK;+ DOTN(q4kg_fill)(wr, b0, n, bk);+ size_t t0 = 0;+ for (; t0 + Q4KG_T <= nt; t0 += Q4KG_T)+ DOTN(q4kg_tile)(bk, n, b0, x + (v0 + t0) * xs, xs,+ af + 8 * t0, am + 8 * t0, Q4KG_T);+ /* the rest: a tile of 4, then 1 to 3 */+ for (; t0 + 4 <= nt; t0 += 4)+ DOTN(q4kg_tile)(bk, n, b0, x + (v0 + t0) * xs, xs,+ af + 8 * t0, am + 8 * t0, 4);+ const struct janas_block_q8k *xt = x + (v0 + t0) * xs;+ if (nt - t0 == 3)+ DOTN(q4kg_tile)(bk, n, b0, xt, xs, af + 8 * t0, am + 8 * t0,+ 3);+ else if (nt - t0 == 2)+ DOTN(q4kg_tile)(bk, n, b0, xt, xs, af + 8 * t0, am + 8 * t0,+ 2);+ else if (nt - t0 == 1)+ DOTN(q4kg_tile)(bk, n, b0, xt, xs, af + 8 * t0, am + 8 * t0,+ 1);+ }+ for (size_t t = 0; t < nt; t++)+ for (int j = 0; j < 8; j++)+ y[g + j + (v0 + t) * ys] = af[8 * t + j] - am[8 * t + j];+ }+ }+}+static DOTN_TARGET float DOTN(q5k_dot)(const struct janas_block_q5k *w,const struct janas_block_q8k *x,size_t nb){float r;- DOTN(q5k_dotn_body)(w, x, 0, nb, &r, 1);+ DOTN(q5k_dotn_body)(w, x, 0, nb, &r, 1, NULL);return r;}@@ -952,7 +1341,8 @@ DOTN(q6kp_dotn_body)(const struct janas_block_q6kp *w, const uint8_t *p1,__m256i sumi[8];_Pragma("GCC unroll 8") for (int v = 0; v < nv; v++) sumi[v] =_mm256_setzero_si256();- for (int n = 0; n < 2; n++) {+ _Pragma("GCC unroll 2") for (int n = 0; n < 2; n++)+ {__m256i h = _mm256_loadu_si256((const __m256i *)(w[b].q + 32 * n));__m256i m1 =level >= 2@@ -1037,55 +1427,1444 @@ DOTN(q6kp_dotn_body)(const struct janas_block_q6kp *w, const uint8_t *p1,_mm256_cvtepi32_ps(is8), acc8);}}- float r[8];- if (nv <= 2)- _mm_storeu_ps(r, acc4);- else- _mm256_storeu_ps(r, acc8);- _Pragma("GCC unroll 8") for (int v = 0; v < nv; v++) out[v] = r[v];+ float r[8];+ if (nv <= 2)+ _mm_storeu_ps(r, acc4);+ else+ _mm256_storeu_ps(r, acc8);+ _Pragma("GCC unroll 8") for (int v = 0; v < nv; v++) out[v] = r[v];+}++#define Q6KP_LEVEL(L) \+ switch (nv) { \+ case 8: \+ DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 8, L); \+ break; \+ case 7: \+ DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 7, L); \+ break; \+ case 6: \+ DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 6, L); \+ break; \+ case 5: \+ DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 5, L); \+ break; \+ case 4: \+ DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 4, L); \+ break; \+ case 3: \+ DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 3, L); \+ break; \+ case 2: \+ DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 2, L); \+ break; \+ default: \+ DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 1, L); \+ }++static DOTN_TARGET void DOTN(q6kp_dotn)(const struct janas_block_q6kp *w,+ const uint8_t *p1, const uint8_t *p0,+ const struct janas_block_q8k *x,+ size_t xs, int nv, size_t nb,+ float *out)+{+ if (p0)+ Q6KP_LEVEL(3)+ else if (p1)+ Q6KP_LEVEL(2)+ else+ Q6KP_LEVEL(1)+}+#undef Q6KP_LEVEL++/*+ * Q6_K and Q6_K_P as products of tiles (see q4k_gemm): the eight rows'+ * six-bit values unpacked to bytes and transposed four at a time+ * (q6kg_fill), so that a register holds four weights of each row. A+ * scale covers 16 weights, four dwords: with six bits two maddubs fit in+ * int16 (2 x 2 x 63 x 128 = 32256), so one madd by the scales every two+ * (AVX2), or four vpdpbusd and one multiply (AVX-VNNI). The offset of the+ * weights (-32, or the levels' centres) and the float operations are the+ * products' above, so the same bits: Q6_K's isum = sumi - 32 sum(scale x+ * group sums), dw = d; Q6_K_P's 2 isum = (sumi << shk) - c sum(...), dw =+ * d / 2.+ */+#ifndef Q6KG_DEFINED+#define Q6KG_DEFINED+/* AVX-VNNI: a group by four vpdpbusd and a multiply by the scales (1),+ or by maddubs summed in int16 and one vpdpwssd (0), fewer instructions+ on the multipliers' ports but slower (bench_kernels, 12 threads, 32+ vectors: Q6_K 12.1 against 10.3 GB/s, Q3_K 5.3 against 4.5; 25 Oct+ 2026) */+#ifndef Q6KG_DP+#define Q6KG_DP(L) 1+#endif+#define Q6KG_CHUNK 8 /* blocks transposed at a time: 26 KB */+struct q6kg_blk {+ uint32_t qs[64][8]; /* [dword][row] */+ int16_t sc16[16][16]; /* [group][2 row + i]: its scale, twice */+ int16_t scp[8][16]; /* [k][2 row + i]: scales 2k and 2k + 1 */+ int32_t sc32[16][8]; /* [group][row] */+ float d[8], dmin[8]; /* dmin: Q2_K's */+};+#endif++/* The 32 bytes of q (a quarter of a block of eight rows, j's in r[j])+ transposed into qs[8 q + 0..7] */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(gemm_t8)(__m256i r[8], uint32_t (*qs)[8])+{+ __m256i t[8], u[8];+ for (int j = 0; j < 8; j += 2) {+ t[j] = _mm256_unpacklo_epi32(r[j], r[j + 1]);+ t[j + 1] = _mm256_unpackhi_epi32(r[j], r[j + 1]);+ }+ for (int j = 0; j < 8; j += 4) {+ u[j] = _mm256_unpacklo_epi64(t[j], t[j + 2]);+ u[j + 1] = _mm256_unpackhi_epi64(t[j], t[j + 2]);+ u[j + 2] = _mm256_unpacklo_epi64(t[j + 1], t[j + 3]);+ u[j + 3] = _mm256_unpackhi_epi64(t[j + 1], t[j + 3]);+ }+ for (int q = 0; q < 4; q++) {+ _mm256_storeu_si256((__m256i *)qs[q],+ _mm256_permute2x128_si256(u[q], u[q + 4], 0x20));+ _mm256_storeu_si256((__m256i *)qs[q + 4],+ _mm256_permute2x128_si256(u[q], u[q + 4], 0x31));+ }+}++/* an 8 x 8 transpose of the rows' pairs of int8: t[kk], word r is row r's+ bytes 2kk and 2kk + 1 */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q6kg_pairs8)(const __m128i *s, __m128i *t)+{+ __m128i b[8], c[8];+ for (int r = 0; r < 8; r += 2) {+ b[r] = _mm_unpacklo_epi16(s[r], s[r + 1]);+ b[r + 1] = _mm_unpackhi_epi16(s[r], s[r + 1]);+ }+ for (int h = 0; h < 8; h += 4) {+ c[h] = _mm_unpacklo_epi32(b[h], b[h + 2]);+ c[h + 1] = _mm_unpackhi_epi32(b[h], b[h + 2]);+ c[h + 2] = _mm_unpacklo_epi32(b[h + 1], b[h + 3]);+ c[h + 3] = _mm_unpackhi_epi32(b[h + 1], b[h + 3]);+ }+ for (int q = 0; q < 4; q++) {+ t[2 * q] = _mm_unpacklo_epi64(c[q], c[q + 4]);+ t[2 * q + 1] = _mm_unpackhi_epi64(c[q], c[q + 4]);+ }+}++/* The sixteen int8 scales of each of the eight rows (s[r]) into k: their+ pairs give scp (or, m not NULL, the pairs of m, Q2_K's mins), and the+ pairs' bytes, sign extended, the tiles' sc32 (AVX-VNNI) or sc16 */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q6kg_scales8)(struct q6kg_blk *k, const __m128i *s, const __m128i *m)+{+ __m128i t[8], tm[8];+ DOTN(q6kg_pairs8)(s, t);+ if (m)+ DOTN(q6kg_pairs8)(m, tm);+ const __m128i split =+ _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);+ for (int kk = 0; kk < 8; kk++) {+ _mm256_storeu_si256((__m256i *)k->scp[kk],+ _mm256_cvtepi8_epi16(m ? tm[kk] : t[kk]));+ __m128i e = _mm_shuffle_epi8(t[kk], split); /* group 2kk, 2kk + 1 */+#ifdef DOTN_HAS_VNNI+ if (Q6KG_DP(0)) {+ _mm256_storeu_si256((__m256i *)k->sc32[2 * kk],+ _mm256_cvtepi8_epi32(e));+ _mm256_storeu_si256((__m256i *)k->sc32[2 * kk + 1],+ _mm256_cvtepi8_epi32(_mm_srli_si128(e, 8)));+ continue;+ }+#endif+ for (int h = 0; h < 2; h++) {+ __m128i v = _mm_cvtepi8_epi16(h ? _mm_srli_si128(e, 8) : e);+ _mm256_storeu_si256((__m256i *)k->sc16[2 * kk + h],+ _mm256_set_m128i(_mm_unpackhi_epi16(v, v),+ _mm_unpacklo_epi16(v, v)));+ }+ }+}++/* Q3_K's sixteen scales less 32 (defined below) */+static inline DOTN_TARGET+ __attribute__((always_inline)) __m128i DOTN(q3k_scales16)(const uint8_t *q);++/* Q6_K blocks [b0, b0 + n) of rows w[0..7] into bk */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q6kg_fill_q6k)(const struct janas_block_q6k *const *w, size_t b0, int n,+ struct q6kg_blk *bk)+{+ const __m256i low4 = _mm256_set1_epi8(0x0f), low2 = _mm256_set1_epi8(0x03);+ for (int i = 0; i < n; i++) {+ struct q6kg_blk *k = bk + i;+ for (int hn = 0; hn < 2; hn++) {+ __m256i q[4][8];+ for (int j = 0; j < 8; j++) {+ const struct janas_block_q6k *bl = &w[j][b0 + i];+ __m256i ql0 =+ _mm256_loadu_si256((const __m256i *)(bl->ql + 64 * hn));+ __m256i ql1 = _mm256_loadu_si256(+ (const __m256i *)(bl->ql + 64 * hn + 32));+ __m256i qh =+ _mm256_loadu_si256((const __m256i *)(bl->qh + 32 * hn));+ q[0][j] = _mm256_or_si256(+ _mm256_and_si256(ql0, low4),+ _mm256_slli_epi16(_mm256_and_si256(qh, low2), 4));+ q[1][j] = _mm256_or_si256(+ _mm256_and_si256(ql1, low4),+ _mm256_slli_epi16(+ _mm256_and_si256(_mm256_srli_epi16(qh, 2), low2), 4));+ q[2][j] = _mm256_or_si256(+ _mm256_and_si256(_mm256_srli_epi16(ql0, 4), low4),+ _mm256_slli_epi16(+ _mm256_and_si256(_mm256_srli_epi16(qh, 4), low2), 4));+ q[3][j] = _mm256_or_si256(+ _mm256_and_si256(_mm256_srli_epi16(ql1, 4), low4),+ _mm256_slli_epi16(+ _mm256_and_si256(_mm256_srli_epi16(qh, 6), low2), 4));+ }+ for (int part = 0; part < 4; part++)+ DOTN(gemm_t8)(q[part], k->qs + 8 * (4 * hn + part));+ }+ __m128i sc[8];+ for (int j = 0; j < 8; j++) {+ sc[j] = _mm_loadu_si128((const __m128i *)w[j][b0 + i].scales);+ k->d[j] = _cvtsh_ss(w[j][b0 + i].d);+ }+ DOTN(q6kg_scales8)(k, sc, NULL);+ }+}++/* Q6_K_P blocks [b0, b0 + n) of rows w[0..7] (planes p1[j], p0[j]) at+ level L into bk */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q6kg_fill_q6kp)(const struct janas_block_q6kp *const *w,+ const uint8_t *const *p1, const uint8_t *const *p0,+ size_t b0, int n, struct q6kg_blk *bk, const int L)+{+ const __m256i low2 = _mm256_set1_epi8(0x03);+ for (int i = 0; i < n; i++) {+ struct q6kg_blk *k = bk + i;+ size_t b = b0 + (size_t)i;+ for (int hn = 0; hn < 2; hn++) {+ __m256i q[4][8];+ for (int j = 0; j < 8; j++) {+ __m256i h =+ _mm256_loadu_si256((const __m256i *)(w[j][b].q + 32 * hn));+ __m256i m1 =+ L >= 2 ? _mm256_loadu_si256((+ const __m256i *)(p1[j] + b * JANAS_Q6KP_PLANE ++ 32 * hn))+ : _mm256_setzero_si256();+ __m256i m0 =+ L >= 3 ? _mm256_loadu_si256((+ const __m256i *)(p0[j] + b * JANAS_Q6KP_PLANE ++ 32 * hn))+ : _mm256_setzero_si256();+ for (int part = 0; part < 4; part++) {+#define Q6KG_BITS(reg) _mm256_and_si256(_mm256_srli_epi16(reg, 2 * part), low2)+ q[part][j] =+ L == 3 ? _mm256_or_si256(+ _mm256_or_si256(+ _mm256_slli_epi16(Q6KG_BITS(h), 4),+ _mm256_slli_epi16(Q6KG_BITS(m1), 2)),+ Q6KG_BITS(m0))+ : L == 2 ? _mm256_or_si256(+ _mm256_slli_epi16(Q6KG_BITS(h), 2),+ Q6KG_BITS(m1))+ : Q6KG_BITS(h);+#undef Q6KG_BITS+ }+ }+ for (int part = 0; part < 4; part++)+ DOTN(gemm_t8)(q[part], k->qs + 8 * (4 * hn + part));+ }+ __m128i sc[8];+ for (int j = 0; j < 8; j++) {+ sc[j] = _mm_loadu_si128((const __m128i *)w[j][b].scales);+ k->d[j] = _cvtsh_ss(w[j][b].d) * 0.5f;+ }+ DOTN(q6kg_scales8)(k, sc, NULL);+ }+}++/* Q3_K blocks [b0, b0 + n) of rows w[0..7] into bk: the values 0 to 7+ (the weight plus 4), the sixteen scales less 32 */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q6kg_fill_q3k)(const struct janas_block_q3k *const *w, size_t b0, int n,+ struct q6kg_blk *bk)+{+ const __m256i three = _mm256_set1_epi8(3), one = _mm256_set1_epi8(1);+ for (int i = 0; i < n; i++) {+ struct q6kg_blk *k = bk + i;+ size_t b = b0 + (size_t)i;+ for (int kk = 0; kk < 8; kk++) {+ const int hn = kk / 4, j = kk % 4;+ __m256i u[8];+ for (int r = 0; r < 8; r++) {+ __m256i qs =+ _mm256_loadu_si256((const __m256i *)(w[r][b].qs + 32 * hn));+ __m256i hm = _mm256_loadu_si256((const __m256i *)w[r][b].hmask);+ u[r] = _mm256_or_si256(+ _mm256_and_si256(_mm256_srli_epi16(qs, 2 * j), three),+ _mm256_slli_epi16(+ _mm256_and_si256(_mm256_srli_epi16(hm, j + 4 * hn),+ one),+ 2));+ }+ DOTN(gemm_t8)(u, k->qs + 8 * kk);+ }+ __m128i sc[8];+ for (int r = 0; r < 8; r++) {+ sc[r] = DOTN(q3k_scales16)(w[r][b].scales);+ k->d[r] = _cvtsh_ss(w[r][b].d);+ }+ DOTN(q6kg_scales8)(k, sc, NULL);+ }+}++/* T tokens on n transposed blocks from b0, the sums added into af (8+ floats a token); L 0 for Q6_K, else Q6_K_P's level */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q6kg_tile)(const struct q6kg_blk *bk, int n, size_t b0,+ const struct janas_block_q8k *x, size_t xs, float *af,+ float *am, const int T, const int L)+{+ /* L 4: Q3_K, the weights' -4 and d whole; L 5: the IQ types' signed+ values, plus 128 with AVX-VNNI (sign and abs with AVX2, no offset);+ L 6: Q2_K's values 0 to 3, scp its mins, their sums into am */+ const int shk = L == 0 || L >= 4 ? 0 : L == 3 ? 1 : L == 2 ? 3 : 5;+ const __m256i cv = _mm256_set1_epi32(L == 0 ? 32+ : L == 4 ? 4+ : L == 5 ? 128+ : L == 3 ? 64+ : L == 2 ? 61+ : 49);+#ifdef DOTN_HAS_VNNI+ const int off = L != 6;+#else+ const int off = L != 5 && L != 6;+#endif+ for (int i = 0; i < n; i++) {+ const struct q6kg_blk *k = bk + i;+ size_t b = b0 + (size_t)i;+ __m256i is[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) is[t] =+ _mm256_setzero_si256();+ for (int g = 0; g < 16; g++) {+#ifdef DOTN_HAS_VNNI+ if (Q6KG_DP(L) || L == 5) {+ __m256i a32[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) a32[t] =+ _mm256_setzero_si256();+ _Pragma("GCC unroll 4") for (int q = 0; q < 4; q++)+ {+ __m256i wv =+ _mm256_loadu_si256((const __m256i *)k->qs[4 * g + q]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ int32_t aw;+ memcpy(&aw, x[(size_t)t * xs + b].qs + 16 * g + 4 * q,+ 4);+ a32[t] = _mm256_dpbusd_avx_epi32(a32[t], wv,+ _mm256_set1_epi32(aw));+ }+ }+ __m256i sc = _mm256_loadu_si256((const __m256i *)k->sc32[g]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) is[t] =+ _mm256_add_epi32(is[t], _mm256_mullo_epi32(a32[t], sc));+ continue;+ }+#endif+ __m256i sc = _mm256_loadu_si256((const __m256i *)k->sc16[g]);+ /* maddubs summed in int16 before a madd: two of six bits (or+ the IQ types' values, up to 62), four of three (4 x 2 x 7 x+ 128) or two */+ const int G = L == 4 || L == 6 ? 4 : 2;+ _Pragma("GCC unroll 2") for (int hq = 0; hq < 4 / G; hq++)+ {+ __m256i a16[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) a16[t] =+ _mm256_setzero_si256();+ _Pragma("GCC unroll 4") for (int q = G * hq; q < G * hq + G;+ q++)+ {+ __m256i wv =+ _mm256_loadu_si256((const __m256i *)k->qs[4 * g + q]);+ __m256i wa = L == 5 ? _mm256_abs_epi8(wv) : wv;+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ int32_t aw;+ memcpy(&aw, x[(size_t)t * xs + b].qs + 16 * g + 4 * q,+ 4);+ __m256i av = _mm256_set1_epi32(aw);+ if (L == 5)+ av = _mm256_sign_epi8(av, wv);+ a16[t] = _mm256_add_epi16(a16[t],+ _mm256_maddubs_epi16(wa, av));+ }+ }+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) is[t] =+ DOTN_ACC(is[t], a16[t], sc);+ }+ }+ __m256 dw = _mm256_loadu_ps(k->d);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ const struct janas_block_q8k *xb = x + (size_t)t * xs + b;+ __m256i mn = _mm256_setzero_si256();+ if (off || L == 6)+ _Pragma("GCC unroll 8") for (int kk = 0; kk < 8; kk++)+ {+ int32_t bw;+ memcpy(&bw, xb->bsums + 2 * kk, 4);+ mn = _mm256_add_epi32(+ mn, _mm256_madd_epi16(+ _mm256_loadu_si256((const __m256i *)k->scp[kk]),+ _mm256_set1_epi32(bw)));+ }+ __m256i si = off ? _mm256_sub_epi32(_mm256_slli_epi32(is[t], shk),+ _mm256_mullo_epi32(mn, cv))+ : is[t];+ __m256 d8 = _mm256_set1_ps(xb->d);+ _mm256_storeu_ps(af + 8 * t,+ _mm256_fmadd_ps(_mm256_mul_ps(dw, d8),+ _mm256_cvtepi32_ps(si),+ _mm256_loadu_ps(af + 8 * t)));+ if (L == 6)+ _mm256_storeu_ps(+ am + 8 * t,+ _mm256_fmadd_ps(_mm256_mul_ps(_mm256_loadu_ps(k->dmin), d8),+ _mm256_cvtepi32_ps(mn),+ _mm256_loadu_ps(am + 8 * t)));+ }+ }+}++#ifndef Q6KG_T+#define Q6KG_T 6+#endif++/* every token of af's tiles on bk (see q4k_gemm) */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q6kg_tiles)(const struct q6kg_blk *bk, int n, size_t b0,+ const struct janas_block_q8k *x, size_t xs, size_t nt,+ float *af, float *am, const int L)+{+ size_t t0 = 0;+#define Q6KG_TILE(T) \+ DOTN(q6kg_tile)(bk, n, b0, x + t0 * xs, xs, af + 8 * t0, am + 8 * t0, T, L)+ for (; t0 + Q6KG_T <= nt; t0 += Q6KG_T)+ Q6KG_TILE(Q6KG_T);+ for (; t0 + 4 <= nt; t0 += 4)+ Q6KG_TILE(4);+ if (nt - t0 == 3)+ Q6KG_TILE(3);+ else if (nt - t0 == 2)+ Q6KG_TILE(2);+ else if (nt - t0 == 1)+ Q6KG_TILE(1);+#undef Q6KG_TILE+}++/* rows (a multiple of 8) of Q6_K (p1 NULL and pb 0) or Q6_K_P (rows' planes+ pb bytes apart from p1 and p0, NULL past its level) against nv vectors,+ y[r + v ys] */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q6k_gemm_body)(const void *w, size_t row_bytes, const uint8_t *p1,+ const uint8_t *p0, size_t pb,+ const struct janas_block_q8k *x, size_t xs, size_t nv,+ size_t nb, size_t rows, float *y, size_t ys, const int L)+{+ struct q6kg_blk bk[Q6KG_CHUNK];+ float af[Q4KG_TOK * 8];+ for (size_t g = 0; g < rows; g += 8) {+ const void *wr[8];+ const uint8_t *r1[8], *r0[8];+ for (int j = 0; j < 8; j++) {+ wr[j] = (const char *)w + (g + j) * row_bytes;+ r1[j] = p1 ? p1 + (g + j) * pb : NULL;+ r0[j] = p0 ? p0 + (g + j) * pb : NULL;+ }+ for (size_t v0 = 0; v0 < nv; v0 += Q4KG_TOK) {+ size_t nt = nv - v0 < Q4KG_TOK ? nv - v0 : Q4KG_TOK;+ memset(af, 0, nt * 8 * sizeof(float));+ for (size_t b0 = 0; b0 < nb; b0 += Q6KG_CHUNK) {+ int n = nb - b0 < Q6KG_CHUNK ? (int)(nb - b0) : Q6KG_CHUNK;+ if (L == 0)+ DOTN(q6kg_fill_q6k)(+ (const struct janas_block_q6k *const *)wr, b0, n, bk);+ else if (L == 4)+ DOTN(q6kg_fill_q3k)(+ (const struct janas_block_q3k *const *)wr, b0, n, bk);+ else+ DOTN(q6kg_fill_q6kp)(+ (const struct janas_block_q6kp *const *)wr, r1, r0, b0,+ n, bk, L);+ DOTN(q6kg_tiles)(bk, n, b0, x + v0 * xs, xs, nt, af, af, L);+ }+ for (size_t t = 0; t < nt; t++)+ for (int j = 0; j < 8; j++)+ y[g + j + (v0 + t) * ys] = af[8 * t + j];+ }+ }+}++static DOTN_TARGET void+DOTN(q6k_gemm)(const void *w, size_t row_bytes, const uint8_t *p1,+ const uint8_t *p0, size_t pb, const struct janas_block_q8k *x,+ size_t xs, size_t nv, size_t nb, size_t rows, float *y,+ size_t ys, int level)+{+ switch (level) {+ case 0:+ DOTN(q6k_gemm_body)(w, row_bytes, p1, p0, pb, x, xs, nv, nb, rows, y,+ ys, 0);+ break;+ case 1:+ DOTN(q6k_gemm_body)(w, row_bytes, p1, p0, pb, x, xs, nv, nb, rows, y,+ ys, 1);+ break;+ case 2:+ DOTN(q6k_gemm_body)(w, row_bytes, p1, p0, pb, x, xs, nv, nb, rows, y,+ ys, 2);+ break;+ case 4:+ DOTN(q6k_gemm_body)(w, row_bytes, p1, p0, pb, x, xs, nv, nb, rows, y,+ ys, 4);+ break;+ default:+ DOTN(q6k_gemm_body)(w, row_bytes, p1, p0, pb, x, xs, nv, nb, rows, y,+ ys, 3);+ }+}++/*+ * Q2_K and the IQ types with grids (IQ3_S, IQ3_XXS, IQ2_XXS, IQ2_XS, IQ2_S,+ * IQ1_S, IQ1_M) as Q6_K's tiles: a group of 16 weights a scale. Q2_K's+ * values 0 to 3 (level 6), its mins' sums apart as its products do; the IQ+ * types' weights as signed bytes, level times sign (level 5), their scales+ * 1 + 2 s, a 32-weight group's in both its groups of 16, and the block's d+ * with its factor (1/4, 1/8) as their products take it. The integer sums+ * are exact, one fma a block (two for Q2_K): the same bits.+ */+static inline DOTN_TARGET __attribute__((always_inline)) size_t+DOTN(iqkg_size)(const int TY)+{+ return TY == JANAS_IQ3_S ? sizeof(struct janas_block_iq3s)+ : TY == JANAS_IQ3_XXS ? sizeof(struct janas_block_iq3xxs)+ : TY == JANAS_IQ1_S ? sizeof(struct janas_block_iq1s)+ : TY == JANAS_IQ1_M ? sizeof(struct janas_block_iq1m)+ : TY == JANAS_Q2_K ? sizeof(struct janas_block_q2k)+ : iq2_size(TY);+}++/* weights 32 g to 32 g + 31 of an IQ block as signed bytes, the scales of+ its two groups of 16 into s[2 g] and s[2 g + 1] */+static inline DOTN_TARGET __attribute__((always_inline)) __m256i+DOTN(iqkg_group)(const int TY, const uint8_t *blk, int g, int8_t *s)+{+ if (TY == JANAS_IQ3_S) {+ const struct janas_block_iq3s *w = (const void *)blk;+ const __m256i shifts = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);+ const __m256i sel =+ _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2,+ 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3);+ const __m256i bit =+ _mm256_set1_epi64x((long long)0x8040201008040201ull);+ __m128i q8 = _mm_loadl_epi64((const __m128i *)(w->qs + 8 * g));+ __m256i nine = _mm256_slli_epi32(+ _mm256_and_si256(+ _mm256_srlv_epi32(_mm256_set1_epi32(w->qh[g]), shifts),+ _mm256_set1_epi32(1)),+ 8);+ __m256i lv = _mm256_i32gather_epi32(+ (const int *)iq3s_grid,+ _mm256_or_si256(_mm256_cvtepu8_epi32(q8), nine), 4);+ uint32_t sb;+ memcpy(&sb, w->signs + 4 * g, 4);+ __m256i m = _mm256_shuffle_epi8(_mm256_set1_epi32((int)sb), sel);+ m = _mm256_cmpeq_epi8(_mm256_and_si256(m, bit), bit);+ s[2 * g] = s[2 * g + 1] = (int8_t)iq3s_scale(w, g);+ return _mm256_sign_epi8(lv, _mm256_or_si256(m, _mm256_set1_epi8(1)));+ }+ if (TY == JANAS_IQ3_XXS) {+ const struct janas_block_iq3xxs *w = (const void *)blk;+ __m128i q8 = _mm_loadl_epi64((const __m128i *)(w->qs + 8 * g));+ __m256i lv = _mm256_i32gather_epi32((const int *)iq3xxs_grid,+ _mm256_cvtepu8_epi32(q8), 4);+ uint32_t aux = iq3xxs_aux(w, g);+ s[2 * g] = s[2 * g + 1] = (int8_t)(1 + 2 * (aux >> 28));+ return _mm256_sign_epi8(+ lv, _mm256_setr_epi64x((long long)iq_sign64[aux & 127],+ (long long)iq_sign64[aux >> 7 & 127],+ (long long)iq_sign64[aux >> 14 & 127],+ (long long)iq_sign64[aux >> 21 & 127]));+ }+ uint64_t gr[4], sm[4];+ int sl[4];+ if (TY == JANAS_IQ1_S || TY == JANAS_IQ1_M) {+ _Pragma("GCC unroll 4") for (int k = 0; k < 4; k++)+ {+ int neg;+ if (TY == JANAS_IQ1_S)+ iq1s_sub((const void *)blk, 4 * g + k, &gr[k], &neg, &sl[k]);+ else+ iq1m_sub((const void *)blk, 4 * g + k, &gr[k], &neg, &sl[k]);+ sm[k] = neg ? 0xffffffffffffffffull : 0x0101010101010101ull;+ }+ __m256i lev = _mm256_setr_epi64x((long long)gr[0], (long long)gr[1],+ (long long)gr[2], (long long)gr[3]);+ s[2 * g] = (int8_t)sl[0];+ s[2 * g + 1] = (int8_t)sl[2];+ return _mm256_add_epi8(+ _mm256_and_si256(_mm256_slli_epi16(lev, 3),+ _mm256_set1_epi8((char)0xf8)),+ _mm256_setr_epi64x((long long)sm[0], (long long)sm[1],+ (long long)sm[2], (long long)sm[3]));+ }+ if (TY == JANAS_IQ2_S) {+ _Pragma("GCC unroll 4") for (int k = 0; k < 4; k++)+ {+ uint32_t s8;+ iq2_sub(TY, blk, 4 * g + k, &gr[k], &s8, &sl[k]);+ sm[k] = iq_byte64[s8];+ }+ } else {+ _Pragma("GCC unroll 4") for (int k = 0; k < 4; k++)+ iq2_sub_mask(TY, blk, 4 * g + k, &gr[k], &sm[k], &sl[k]);+ }+ s[2 * g] = (int8_t)sl[0];+ s[2 * g + 1] = (int8_t)sl[2];+ return _mm256_sign_epi8(+ _mm256_setr_epi64x((long long)gr[0], (long long)gr[1], (long long)gr[2],+ (long long)gr[3]),+ _mm256_setr_epi64x((long long)sm[0], (long long)sm[1], (long long)sm[2],+ (long long)sm[3]));+}++/* an IQ block's d as its products take it */+static inline DOTN_TARGET __attribute__((always_inline)) float+DOTN(iqkg_d)(const int TY, const uint8_t *blk)+{+ uint16_t h;+ if (TY == JANAS_IQ1_M)+ h = iq1m_d16((const void *)blk);+ else+ memcpy(&h, blk, 2);+ float d = _cvtsh_ss(h);+ return TY == JANAS_IQ3_S ? d : TY == JANAS_IQ3_XXS ? d * 0.25f : d * 0.125f;+}++/* blocks [b0, b0 + n) of rows w[0..7] of an IQ type or Q2_K into bk */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(iqkg_fill)(const int TY, const uint8_t *const *w, size_t b0, int n,+ struct q6kg_blk *bk)+{+ const size_t size = DOTN(iqkg_size)(TY);+ for (int i = 0; i < n; i++) {+ struct q6kg_blk *k = bk + i;+ size_t b = b0 + (size_t)i;+ if (TY == JANAS_Q2_K) {+ const __m256i three = _mm256_set1_epi8(3);+ for (int kk = 0; kk < 8; kk++) {+ __m256i u[8];+ for (int r = 0; r < 8; r++) {+ const struct janas_block_q2k *bl =+ (const struct janas_block_q2k *)(w[r] + b * size);+ __m256i q = _mm256_loadu_si256(+ (const __m256i *)(bl->qs + 32 * (kk / 4)));+ u[r] = _mm256_and_si256(_mm256_srli_epi16(q, 2 * (kk % 4)),+ three);+ }+ DOTN(gemm_t8)(u, k->qs + 8 * kk);+ }+ __m128i sc[8], mn[8];+ for (int r = 0; r < 8; r++) {+ const struct janas_block_q2k *bl =+ (const struct janas_block_q2k *)(w[r] + b * size);+ __m128i m8 = _mm_loadu_si128((const __m128i *)bl->scales);+ sc[r] = _mm_and_si128(m8, _mm_set1_epi8(15));+ mn[r] = _mm_and_si128(_mm_srli_epi16(m8, 4), _mm_set1_epi8(15));+ k->d[r] = _cvtsh_ss(bl->d);+ k->dmin[r] = _cvtsh_ss(bl->dmin);+ }+ DOTN(q6kg_scales8)(k, sc, mn);+ continue;+ }+ int8_t s[8][16];+ for (int kk = 0; kk < 8; kk++) {+ __m256i u[8];+ for (int r = 0; r < 8; r++) {+ u[r] = DOTN(iqkg_group)(TY, w[r] + b * size, kk, s[r]);+#ifdef DOTN_HAS_VNNI+ u[r] = _mm256_xor_si256(u[r], _mm256_set1_epi8((char)0x80));+#endif+ }+ DOTN(gemm_t8)(u, k->qs + 8 * kk);+ }+ __m128i sc[8];+ for (int r = 0; r < 8; r++) {+ sc[r] = _mm_loadu_si128((const __m128i *)s[r]);+ k->d[r] = DOTN(iqkg_d)(TY, w[r] + b * size);+ }+ DOTN(q6kg_scales8)(k, sc, NULL);+ }+}++static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(iqk_gemm_t)(const int TY, const void *w, size_t row_bytes,+ const struct janas_block_q8k *x, size_t xs, size_t nv,+ size_t nb, size_t rows, float *y, size_t ys)+{+ const int L = TY == JANAS_Q2_K ? 6 : 5;+ struct q6kg_blk bk[Q6KG_CHUNK];+ float af[Q4KG_TOK * 8], am[Q4KG_TOK * 8];+ for (size_t g = 0; g < rows; g += 8) {+ const uint8_t *wr[8];+ for (int j = 0; j < 8; j++)+ wr[j] = (const uint8_t *)w + (g + j) * row_bytes;+ for (size_t v0 = 0; v0 < nv; v0 += Q4KG_TOK) {+ size_t nt = nv - v0 < Q4KG_TOK ? nv - v0 : Q4KG_TOK;+ memset(af, 0, nt * 8 * sizeof(float));+ if (L == 6)+ memset(am, 0, nt * 8 * sizeof(float));+ for (size_t b0 = 0; b0 < nb; b0 += Q6KG_CHUNK) {+ int n = nb - b0 < Q6KG_CHUNK ? (int)(nb - b0) : Q6KG_CHUNK;+ DOTN(iqkg_fill)(TY, wr, b0, n, bk);+ DOTN(q6kg_tiles)(bk, n, b0, x + v0 * xs, xs, nt, af, am, L);+ }+ for (size_t t = 0; t < nt; t++)+ for (int j = 0; j < 8; j++)+ y[g + j + (v0 + t) * ys] =+ L == 6 ? af[8 * t + j] - am[8 * t + j] : af[8 * t + j];+ }+ }+}++/* rows (a multiple of 8) of Q2_K or an IQ type with a grid against nv+ vectors, y[r + v ys]: 1, or 0 for another type */+static DOTN_TARGET int DOTN(iqk_gemm)(int type, const void *w, size_t row_bytes,+ const struct janas_block_q8k *x,+ size_t xs, size_t nv, size_t nb,+ size_t rows, float *y, size_t ys)+{+ switch (type) {+#define IQKG(T) \+ case T: \+ DOTN(iqk_gemm_t)(T, w, row_bytes, x, xs, nv, nb, rows, y, ys); \+ return 1;+ IQKG(JANAS_Q2_K)+ IQKG(JANAS_IQ3_S)+ IQKG(JANAS_IQ3_XXS)+ IQKG(JANAS_IQ2_XXS)+ IQKG(JANAS_IQ2_XS)+ IQKG(JANAS_IQ2_S)+ IQKG(JANAS_IQ1_S)+ IQKG(JANAS_IQ1_M)+#undef IQKG+ }+ return 0;+}++/*+ * Q5_K as tiles (see q4k_gemm and q6k_gemm): the eight rows' five-bit+ * values unpacked to bytes and transposed; a 32-weight piece is eight+ * dwords, of which four maddubs fit in int16 (4 x 2 x 31 x 128 = 31744):+ * two madds by the scales a piece (AVX2), or eight vpdpbusd and one+ * multiply (AVX-VNNI); the mins and the floats Q4_K's, the same bits.+ */+#ifndef Q5KG_DEFINED+#define Q5KG_DEFINED+#define Q5KG_CHUNK 8+struct q5kg_blk {+ uint32_t qs[64][8]; /* [dword][row] */+ int16_t sc[8][16]; /* [sub-block][2 row + i] */+ int16_t mn[4][16]; /* [k][2 row + i]: mins 2k and 2k + 1 */+ int32_t sc32[8][8]; /* [sub-block][row] */+ float d[8], dmin[8];+};+#endif++static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q5kg_fill)(const struct janas_block_q5k *const *w, size_t b0, int n,+ struct q5kg_blk *bk)+{+ const __m256i low4 = _mm256_set1_epi8(0x0f), one = _mm256_set1_epi8(1);+ for (int i = 0; i < n; i++) {+ struct q5kg_blk *k = bk + i;+ size_t b = b0 + (size_t)i;+ for (int c = 0; c < 4; c++) {+ __m256i lo[8], hi[8];+ for (int j = 0; j < 8; j++) {+ __m256i q4 =+ _mm256_loadu_si256((const __m256i *)(w[j][b].qs + 32 * c));+ __m256i qh = _mm256_loadu_si256((const __m256i *)w[j][b].qh);+ lo[j] = _mm256_or_si256(+ _mm256_and_si256(q4, low4),+ _mm256_slli_epi16(+ _mm256_and_si256(_mm256_srli_epi16(qh, 2 * c), one),+ 4));+ hi[j] = _mm256_or_si256(+ _mm256_and_si256(_mm256_srli_epi16(q4, 4), low4),+ _mm256_slli_epi16(+ _mm256_and_si256(_mm256_srli_epi16(qh, 2 * c + 1), one),+ 4));+ }+ DOTN(gemm_t8)(lo, k->qs + 8 * (2 * c));+ DOTN(gemm_t8)(hi, k->qs + 8 * (2 * c + 1));+ }+ __m128i sv[8];+ for (int j = 0; j < 8; j++) {+ const struct janas_block_q5k *blk = &w[j][b];+ uint32_t u[4];+ unpack_scales_u32(blk->scales, u);+ sv[j] = _mm_loadu_si128((const __m128i *)u);+ k->d[j] = _cvtsh_ss(blk->d);+ k->dmin[j] = _cvtsh_ss(blk->dmin);+ }+ DOTN(kmin_scales8)(k->sc, k->sc32, k->mn, sv, 0);+ }+}++static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q5kg_tile)(const struct q5kg_blk *bk, int n, size_t b0,+ const struct janas_block_q8k *x, size_t xs, float *af,+ float *am, const int T)+{+ for (int i = 0; i < n; i++) {+ const struct q5kg_blk *k = bk + i;+ size_t b = b0 + (size_t)i;+ __m256i is[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) is[t] =+ _mm256_setzero_si256();+ for (int sb = 0; sb < 8; sb++) {+#ifdef DOTN_HAS_VNNI+ __m256i a32[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) a32[t] =+ _mm256_setzero_si256();+ _Pragma("GCC unroll 8") for (int q = 0; q < 8; q++)+ {+ __m256i wv =+ _mm256_loadu_si256((const __m256i *)k->qs[8 * sb + q]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ int32_t aw;+ memcpy(&aw, x[(size_t)t * xs + b].qs + 32 * sb + 4 * q, 4);+ a32[t] = _mm256_dpbusd_avx_epi32(a32[t], wv,+ _mm256_set1_epi32(aw));+ }+ }+ __m256i sc = _mm256_loadu_si256((const __m256i *)k->sc32[sb]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) is[t] =+ _mm256_add_epi32(is[t], _mm256_mullo_epi32(a32[t], sc));+#else+ __m256i sc = _mm256_loadu_si256((const __m256i *)k->sc[sb]);+ _Pragma("GCC unroll 2") for (int hq = 0; hq < 2; hq++)+ {+ __m256i a16[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) a16[t] =+ _mm256_setzero_si256();+ _Pragma("GCC unroll 4") for (int q = 4 * hq; q < 4 * hq + 4;+ q++)+ {+ __m256i wv =+ _mm256_loadu_si256((const __m256i *)k->qs[8 * sb + q]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ int32_t aw;+ memcpy(&aw, x[(size_t)t * xs + b].qs + 32 * sb + 4 * q,+ 4);+ a16[t] = _mm256_add_epi16(+ a16[t],+ _mm256_maddubs_epi16(wv, _mm256_set1_epi32(aw)));+ }+ }+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) is[t] =+ DOTN_ACC(is[t], a16[t], sc);+ }+#endif+ }+ __m256 dw = _mm256_loadu_ps(k->d), dmw = _mm256_loadu_ps(k->dmin);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ const struct janas_block_q8k *xb = x + (size_t)t * xs + b;+ __m256i im = _mm256_setzero_si256();+ _Pragma("GCC unroll 4") for (int kk = 0; kk < 4; kk++)+ {+ int32_t bw;+ memcpy(&bw, xb->bsums2 + 2 * kk, 4);+ im = _mm256_add_epi32(+ im, _mm256_madd_epi16(+ _mm256_loadu_si256((const __m256i *)k->mn[kk]),+ _mm256_set1_epi32(bw)));+ }+ __m256 d8 = _mm256_set1_ps(xb->d);+ _mm256_storeu_ps(af + 8 * t,+ _mm256_fmadd_ps(_mm256_mul_ps(dw, d8),+ _mm256_cvtepi32_ps(is[t]),+ _mm256_loadu_ps(af + 8 * t)));+ _mm256_storeu_ps(am + 8 * t,+ _mm256_fmadd_ps(_mm256_mul_ps(dmw, d8),+ _mm256_cvtepi32_ps(im),+ _mm256_loadu_ps(am + 8 * t)));+ }+ }+}++/* rows (a multiple of 8) of Q5_K against nv vectors, y[r + v ys] */+static DOTN_TARGET void DOTN(q5k_gemm)(const void *w, size_t row_bytes,+ const struct janas_block_q8k *x,+ size_t xs, size_t nv, size_t nb,+ size_t rows, float *y, size_t ys)+{+ struct q5kg_blk bk[Q5KG_CHUNK];+ float af[Q4KG_TOK * 8], am[Q4KG_TOK * 8];+ for (size_t g = 0; g < rows; g += 8) {+ const struct janas_block_q5k *wr[8];+ for (int j = 0; j < 8; j++)+ wr[j] = (const void *)((const char *)w + (g + j) * row_bytes);+ for (size_t v0 = 0; v0 < nv; v0 += Q4KG_TOK) {+ size_t nt = nv - v0 < Q4KG_TOK ? nv - v0 : Q4KG_TOK;+ memset(af, 0, nt * 8 * sizeof(float));+ memset(am, 0, nt * 8 * sizeof(float));+ for (size_t b0 = 0; b0 < nb; b0 += Q5KG_CHUNK) {+ int n = nb - b0 < Q5KG_CHUNK ? (int)(nb - b0) : Q5KG_CHUNK;+ DOTN(q5kg_fill)(wr, b0, n, bk);+ const struct janas_block_q8k *xv = x + v0 * xs;+ size_t t0 = 0;+ for (; t0 + Q4KG_T <= nt; t0 += Q4KG_T)+ DOTN(q5kg_tile)(bk, n, b0, xv + t0 * xs, xs, af + 8 * t0,+ am + 8 * t0, Q4KG_T);+ for (; t0 + 4 <= nt; t0 += 4)+ DOTN(q5kg_tile)(bk, n, b0, xv + t0 * xs, xs, af + 8 * t0,+ am + 8 * t0, 4);+ if (nt - t0 == 3)+ DOTN(q5kg_tile)(bk, n, b0, xv + t0 * xs, xs, af + 8 * t0,+ am + 8 * t0, 3);+ else if (nt - t0 == 2)+ DOTN(q5kg_tile)(bk, n, b0, xv + t0 * xs, xs, af + 8 * t0,+ am + 8 * t0, 2);+ else if (nt - t0 == 1)+ DOTN(q5kg_tile)(bk, n, b0, xv + t0 * xs, xs, af + 8 * t0,+ am + 8 * t0, 1);+ }+ for (size_t t = 0; t < nt; t++)+ for (int j = 0; j < 8; j++)+ y[g + j + (v0 + t) * ys] = af[8 * t + j] - am[8 * t + j];+ }+ }+}++/*+ * The types in blocks of 32 (Q8_0, IQ4_NL, Q4_0, MXFP4, Q5_0, Q4_1, Q5_1)+ * as tiles of eight rows (see q4k_gemm): each block's 32 values unpacked+ * to bytes, as w + o (o: Q4_0's 8 and Q5_0's 16 as stored, MXFP4's 12,+ * Q4_1's and Q5_1's 0, Q8_0's and IQ4_NL's 128 with AVX-VNNI, Q2_0's 1:+ * a block of 64 two blocks here, its d in both), transposed+ * four at a time, and o times the block's activation sum taken off the+ * exact integer sum. With AVX2 the small values' maddubs are summed in+ * int16 before one madd (Q4_0, Q4_1, Q2_0: eight, 8 x 2 x 15 x 128; Q5_0, Q5_1,+ * MXFP4: four), Q8_0's and IQ4_NL's signed bytes go through sign and abs as+ * in the products above. The floats are theirs: an accumulator for each+ * block position, fma(d x d32, sum), Q4_1's and Q5_1's mins apart, the+ * positions summed as janas_hsum8 does: the same bits.+ */+#ifndef B32G_DEFINED+#define B32G_DEFINED+#define B32G_CHUNK 8+struct b32g_blk {+ uint32_t qs[8][8][8]; /* [block][dword][row] */+ float d[8][8], m[8][8]; /* [block][row] */+};+#endif++static inline DOTN_TARGET __attribute__((always_inline)) void+ DOTN(q2_0_unpack)(const uint8_t *qs, __m256i *lo, __m256i *hi);++/* The offset added to a type's values (see above) */+static inline DOTN_TARGET __attribute__((always_inline)) int+DOTN(b32g_off)(const int TY)+{+#ifdef DOTN_HAS_VNNI+ const int sgn = 128;+#else+ const int sgn = 0;+#endif+ return TY == JANAS_Q4_0 ? 8+ : TY == JANAS_Q2_0 ? 1+ : TY == JANAS_Q5_0 ? 16+ : TY == JANAS_MXFP4 ? 12+ : TY == JANAS_Q4_1 || TY == JANAS_Q5_1 ? 0+ : sgn;+}++static inline DOTN_TARGET __attribute__((always_inline)) size_t+DOTN(b32g_size)(const int TY)+{+ return TY == JANAS_Q8_0 ? sizeof(struct janas_block_q8_0)+ : TY == JANAS_MXFP4 ? sizeof(struct janas_block_mxfp4)+ : TY == JANAS_Q5_0 ? sizeof(struct janas_block_q5_0)+ : TY == JANAS_Q4_1 ? sizeof(struct janas_block_q4_1)+ : TY == JANAS_Q5_1 ? sizeof(struct janas_block_q5_1)+ : TY == JANAS_IQ4_NL ? sizeof(struct janas_block_iq4nl)+ : sizeof(struct janas_block_q4_0);+}++/* A block's 32 values plus the type's offset, weights 0-31 in order */+static inline DOTN_TARGET __attribute__((always_inline)) __m256i+DOTN(b32g_unpack)(const int TY, const uint8_t *blk)+{+ const __m256i low4 = _mm256_set1_epi8(0x0f);+ const int o = DOTN(b32g_off)(TY);+ if (TY == JANAS_Q8_0)+ return _mm256_add_epi8(_mm256_loadu_si256((const __m256i *)(blk + 2)),+ _mm256_set1_epi8((char)o));+ if (TY == JANAS_IQ4_NL || TY == JANAS_MXFP4) {+ const int8_t *t =+ TY == JANAS_MXFP4 ? janas_mxfp4_values : janas_iq4_values;+ __m256i lev =+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)t));+ __m128i b = _mm_loadu_si128(+ (const __m128i *)(blk + (TY == JANAS_MXFP4 ? 1 : 2)));+ __m256i q = _mm256_shuffle_epi8(+ lev, _mm256_set_m128i(+ _mm_and_si128(_mm_srli_epi16(b, 4), _mm_set1_epi8(0x0f)),+ _mm_and_si128(b, _mm_set1_epi8(0x0f))));+ return _mm256_add_epi8(q, _mm256_set1_epi8((char)o));+ }+ /* the nibbles, low ones weights 0-15 (Q4_0, Q5_0, Q4_1, Q5_1) */+ size_t at = TY == JANAS_Q4_0 ? 2+ : TY == JANAS_Q5_0 ? 6+ : TY == JANAS_Q4_1 ? 4+ : 8;+ __m128i b16 = _mm_loadu_si128((const __m128i *)(blk + at));+ __m256i q =+ _mm256_and_si256(_mm256_set_m128i(_mm_srli_epi16(b16, 4), b16), low4);+ if (TY == JANAS_Q5_0 || TY == JANAS_Q5_1) {+ /* byte l takes byte l / 8 of qh, then keeps bit l % 8 of it */+ const __m256i sel =+ _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2,+ 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3);+ const __m256i bit =+ _mm256_set1_epi64x((long long)0x8040201008040201ull);+ uint32_t qh;+ memcpy(&qh, blk + (TY == JANAS_Q5_0 ? 2 : 4), 4);+ __m256i h = _mm256_shuffle_epi8(_mm256_set1_epi32((int)qh), sel);+ h = _mm256_and_si256(_mm256_cmpeq_epi8(_mm256_and_si256(h, bit), bit),+ _mm256_set1_epi8(0x10));+ q = _mm256_or_si256(q, h);+ }+ return q;+}++/* Groups [b0, b0 + n) of rows w[0..7] (each row's groups gs bytes apart:+ eight blocks of the type) into bk */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(b32g_fill)(const int TY, const uint8_t *const *w, size_t b0, int n,+ struct b32g_blk *bk)+{+ const size_t size = DOTN(b32g_size)(TY);+ const int dm = TY == JANAS_Q4_1 || TY == JANAS_Q5_1;+ for (int i = 0; i < n; i++) {+ struct b32g_blk *k = bk + i;+ if (TY == JANAS_Q2_0) {+ /* a block of 64: positions 2 kb2 and 2 kb2 + 1 */+ for (int kb2 = 0; kb2 < 4; kb2++) {+ __m256i lo[8], hi[8];+ for (int j = 0; j < 8; j++) {+ const struct janas_block_q2_0 *blk =+ (const struct janas_block_q2_0 *)w[j] + (b0 + i) * 4 ++ kb2;+ DOTN(q2_0_unpack)(blk->qs, &lo[j], &hi[j]);+ k->d[2 * kb2][j] = k->d[2 * kb2 + 1][j] = _cvtsh_ss(blk->d);+ }+ DOTN(gemm_t8)(lo, k->qs[2 * kb2]);+ DOTN(gemm_t8)(hi, k->qs[2 * kb2 + 1]);+ }+ continue;+ }+ for (int kb = 0; kb < 8; kb++) {+ __m256i r[8];+ for (int j = 0; j < 8; j++) {+ const uint8_t *blk = w[j] + ((b0 + i) * 8 + kb) * size;+ r[j] = DOTN(b32g_unpack)(TY, blk);+ if (TY == JANAS_MXFP4) {+ k->d[kb][j] = e8m0_tab[blk[0]];+ } else {+ uint16_t d, m = 0;+ memcpy(&d, blk, 2);+ if (dm)+ memcpy(&m, blk + 2, 2);+ k->d[kb][j] = _cvtsh_ss(d);+ k->m[kb][j] = dm ? _cvtsh_ss(m) : 0.0f;+ }+ }+ DOTN(gemm_t8)(r, k->qs[kb]);+ }+ }+}++/* T tokens on n groups from b0: the eight positions' sums added into af and+ (Q4_1, Q5_1) am, 64 floats a token: [position][row] */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(b32g_tile)(const int TY, const struct b32g_blk *bk, int n, size_t b0,+ const struct janas_block_q8k *x, size_t xs, float *af,+ float *am, const int T)+{+ const int o = DOTN(b32g_off)(TY);+ const int dm = TY == JANAS_Q4_1 || TY == JANAS_Q5_1;+#ifndef DOTN_HAS_VNNI+ const __m256i ones = _mm256_set1_epi16(1);+ /* signed bytes (Q8_0, IQ4_NL): sign and abs; else the maddubs summed+ in int16, G at a time */+ const int sg = TY == JANAS_Q8_0 || TY == JANAS_IQ4_NL;+ const int G =+ TY == JANAS_Q4_0 || TY == JANAS_Q4_1 || TY == JANAS_Q2_0 ? 8 : 4;+#endif+ for (int i = 0; i < n; i++) {+ const struct b32g_blk *k = bk + i;+ size_t b = b0 + (size_t)i;+ for (int kb = 0; kb < 8; kb++) {+ __m256i s[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) s[t] =+ _mm256_setzero_si256();+#ifdef DOTN_HAS_VNNI+ _Pragma("GCC unroll 8") for (int q = 0; q < 8; q++)+ {+ __m256i wv = _mm256_loadu_si256((const __m256i *)k->qs[kb][q]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ int32_t aw;+ memcpy(&aw, x[(size_t)t * xs + b].qs32 + 32 * kb + 4 * q,+ 4);+ s[t] = _mm256_dpbusd_avx_epi32(s[t], wv,+ _mm256_set1_epi32(aw));+ }+ }+#else+ if (sg) {+ _Pragma("GCC unroll 8") for (int q = 0; q < 8; q++)+ {+ __m256i wv =+ _mm256_loadu_si256((const __m256i *)k->qs[kb][q]);+ __m256i aw8 = _mm256_abs_epi8(wv);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ int32_t aw;+ memcpy(&aw,+ x[(size_t)t * xs + b].qs32 + 32 * kb + 4 * q, 4);+ s[t] = _mm256_add_epi32(+ s[t], _mm256_madd_epi16(+ _mm256_maddubs_epi16(+ aw8, _mm256_sign_epi8(+ _mm256_set1_epi32(aw), wv)),+ ones));+ }+ }+ } else {+ _Pragma("GCC unroll 2") for (int h = 0; h < 8 / G; h++)+ {+ __m256i a16[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) a16[t] =+ _mm256_setzero_si256();+ _Pragma("GCC unroll 8") for (int q = G * h; q < G * h + G;+ q++)+ {+ __m256i wv =+ _mm256_loadu_si256((const __m256i *)k->qs[kb][q]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ int32_t aw;+ memcpy(&aw,+ x[(size_t)t * xs + b].qs32 + 32 * kb + 4 * q,+ 4);+ a16[t] = _mm256_add_epi16(+ a16[t], _mm256_maddubs_epi16(+ wv, _mm256_set1_epi32(aw)));+ }+ }+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) s[t] =+ _mm256_add_epi32(s[t], _mm256_madd_epi16(a16[t], ones));+ }+ }+#endif+ __m256 dv = _mm256_loadu_ps(k->d[kb]);+ __m256 mv = _mm256_loadu_ps(k->m[kb]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ const struct janas_block_q8k *xb = x + (size_t)t * xs + b;+ int asum = xb->bsums32[2 * kb] + xb->bsums32[2 * kb + 1];+ __m256 d8 = _mm256_set1_ps(xb->d32[kb]);+ float *pa = af + 64 * t + 8 * kb;+ __m256i is =+ o ? _mm256_sub_epi32(s[t], _mm256_set1_epi32(o * asum))+ : s[t];+ _mm256_storeu_ps(pa, _mm256_fmadd_ps(_mm256_mul_ps(dv, d8),+ _mm256_cvtepi32_ps(is),+ _mm256_loadu_ps(pa)));+ if (dm) {+ float *pm = am + 64 * t + 8 * kb;+ _mm256_storeu_ps(+ pm, _mm256_fmadd_ps(_mm256_mul_ps(mv, d8),+ _mm256_set1_ps((float)asum),+ _mm256_loadu_ps(pm)));+ }+ }+ }+ }+}++/* The eight positions of a token's sums, row by row, as janas_hsum8 */+static inline DOTN_TARGET __attribute__((always_inline)) __m256+DOTN(b32g_hsum8)(const float *a)+{+ __m256 p0 = _mm256_loadu_ps(a), p1 = _mm256_loadu_ps(a + 8),+ p2 = _mm256_loadu_ps(a + 16), p3 = _mm256_loadu_ps(a + 24),+ p4 = _mm256_loadu_ps(a + 32), p5 = _mm256_loadu_ps(a + 40),+ p6 = _mm256_loadu_ps(a + 48), p7 = _mm256_loadu_ps(a + 56);+ return _mm256_add_ps(+ _mm256_add_ps(_mm256_add_ps(p0, p4), _mm256_add_ps(p2, p6)),+ _mm256_add_ps(_mm256_add_ps(p1, p5), _mm256_add_ps(p3, p7)));+}++#define B32G_TOK 32 /* tokens whose sums are held at a time: 16 KB */++static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(b32_gemm_t)(const int TY, const void *w, size_t row_bytes,+ const struct janas_block_q8k *x, size_t xs, size_t nv,+ size_t nb, size_t rows, float *y, size_t ys)+{+ struct b32g_blk bk[B32G_CHUNK];+ float af[B32G_TOK * 64], am[B32G_TOK * 64];+ const int dm = TY == JANAS_Q4_1 || TY == JANAS_Q5_1;+ for (size_t g = 0; g < rows; g += 8) {+ const uint8_t *wr[8];+ for (int j = 0; j < 8; j++)+ wr[j] = (const uint8_t *)w + (g + j) * row_bytes;+ for (size_t v0 = 0; v0 < nv; v0 += B32G_TOK) {+ size_t nt = nv - v0 < B32G_TOK ? nv - v0 : B32G_TOK;+ memset(af, 0, nt * 64 * sizeof(float));+ if (dm)+ memset(am, 0, nt * 64 * sizeof(float));+ for (size_t b0 = 0; b0 < nb; b0 += B32G_CHUNK) {+ int n = nb - b0 < B32G_CHUNK ? (int)(nb - b0) : B32G_CHUNK;+ DOTN(b32g_fill)(TY, wr, b0, n, bk);+ const struct janas_block_q8k *xv = x + v0 * xs;+ size_t t0 = 0;+ for (; t0 + 4 <= nt; t0 += 4)+ DOTN(b32g_tile)(TY, bk, n, b0, xv + t0 * xs, xs,+ af + 64 * t0, am + 64 * t0, 4);+ if (nt - t0 == 3)+ DOTN(b32g_tile)(TY, bk, n, b0, xv + t0 * xs, xs,+ af + 64 * t0, am + 64 * t0, 3);+ else if (nt - t0 == 2)+ DOTN(b32g_tile)(TY, bk, n, b0, xv + t0 * xs, xs,+ af + 64 * t0, am + 64 * t0, 2);+ else if (nt - t0 == 1)+ DOTN(b32g_tile)(TY, bk, n, b0, xv + t0 * xs, xs,+ af + 64 * t0, am + 64 * t0, 1);+ }+ for (size_t t = 0; t < nt; t++) {+ float r[8], rm[8];+ _mm256_storeu_ps(r, DOTN(b32g_hsum8)(af + 64 * t));+ if (dm)+ _mm256_storeu_ps(rm, DOTN(b32g_hsum8)(am + 64 * t));+ for (int j = 0; j < 8; j++)+ y[g + j + (v0 + t) * ys] = dm ? r[j] + rm[j] : r[j];+ }+ }+ }+}++/* rows (a multiple of 8) of a type in blocks of 32 against nv vectors,+ y[r + v ys]: 1, or 0 for another type */+static DOTN_TARGET int DOTN(b32_gemm)(int type, const void *w, size_t row_bytes,+ const struct janas_block_q8k *x,+ size_t xs, size_t nv, size_t nb,+ size_t rows, float *y, size_t ys)+{+ switch (type) {+#define B32G(T) \+ case T: \+ DOTN(b32_gemm_t)(T, w, row_bytes, x, xs, nv, nb, rows, y, ys); \+ return 1;+ B32G(JANAS_Q8_0)+ B32G(JANAS_IQ4_NL)+ B32G(JANAS_Q4_0)+ B32G(JANAS_MXFP4)+ B32G(JANAS_Q5_0)+ B32G(JANAS_Q4_1)+ B32G(JANAS_Q5_1)+ B32G(JANAS_Q2_0)+#undef B32G+ }+ return 0;+}++/*+ * IQ4_XS as tiles (see q4k_gemm): its values from the IQ4 table, signed,+ * transposed four bytes at a time, its eight 32-weight groups each with a+ * signed scale. AVX-VNNI: the values plus 128 by vpdpbusd, a multiply by+ * the scales a group, and 128 times the scales' sums of activations (made+ * by madd from the mins' table, here the scales) taken off; AVX2: sign and+ * abs, the scales by madd. The integer sums are exact, and the float one+ * fma a block, d x d8 times the sum, as the products above: the same bits.+ */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(iq4xsg_fill)(const struct janas_block_iq4xs *const *w, size_t b0, int n,+ struct q5kg_blk *bk)+{+ const __m256i levels = DOTN(iq4_levels)();+ for (int i = 0; i < n; i++) {+ struct q5kg_blk *k = bk + i;+ size_t b = b0 + (size_t)i;+ for (int g = 0; g < 8; g++) {+ __m256i r[8];+ for (int j = 0; j < 8; j++) {+ r[j] = DOTN(iq4_unpack32)(w[j][b].qs + 16 * g, levels);+#ifdef DOTN_HAS_VNNI+ r[j] = _mm256_xor_si256(r[j], _mm256_set1_epi8((char)0x80));+#endif+ }+ DOTN(gemm_t8)(r, k->qs + 8 * g);+ }+ __m128i sv[8];+ for (int j = 0; j < 8; j++) {+ int8_t ls[16];+ for (int g = 0; g < 8; g++)+ ls[g] = ls[8 + g] = (int8_t)iq4xs_scale(&w[j][b], g);+ sv[j] = _mm_loadu_si128((const __m128i *)ls);+ k->d[j] = _cvtsh_ss(w[j][b].d);+ }+ DOTN(kmin_scales8)(k->sc, k->sc32, k->mn, sv, 1);+ }}-#define Q6KP_LEVEL(L) \- switch (nv) { \- case 8: \- DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 8, L); \- break; \- case 7: \- DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 7, L); \- break; \- case 6: \- DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 6, L); \- break; \- case 5: \- DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 5, L); \- break; \- case 4: \- DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 4, L); \- break; \- case 3: \- DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 3, L); \- break; \- case 2: \- DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 2, L); \- break; \- default: \- DOTN(q6kp_dotn_body)(w, p1, p0, x, xs, nb, out, 1, L); \+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(iq4xsg_tile)(const struct q5kg_blk *bk, int n, size_t b0,+ const struct janas_block_q8k *x, size_t xs, float *af,+ const int T)+{+ for (int i = 0; i < n; i++) {+ const struct q5kg_blk *k = bk + i;+ size_t b = b0 + (size_t)i;+ __m256i is[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) is[t] =+ _mm256_setzero_si256();+ for (int sb = 0; sb < 8; sb++) {+#ifdef DOTN_HAS_VNNI+ __m256i a32[8];+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) a32[t] =+ _mm256_setzero_si256();+ _Pragma("GCC unroll 8") for (int q = 0; q < 8; q++)+ {+ __m256i wv =+ _mm256_loadu_si256((const __m256i *)k->qs[8 * sb + q]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ int32_t aw;+ memcpy(&aw, x[(size_t)t * xs + b].qs + 32 * sb + 4 * q, 4);+ a32[t] = _mm256_dpbusd_avx_epi32(a32[t], wv,+ _mm256_set1_epi32(aw));+ }+ }+ __m256i sc = _mm256_loadu_si256((const __m256i *)k->sc32[sb]);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++) is[t] =+ _mm256_add_epi32(is[t], _mm256_mullo_epi32(a32[t], sc));+#else+ __m256i sc = _mm256_loadu_si256((const __m256i *)k->sc[sb]);+ _Pragma("GCC unroll 8") for (int q = 0; q < 8; q++)+ {+ __m256i wv =+ _mm256_loadu_si256((const __m256i *)k->qs[8 * sb + q]);+ __m256i aw8 = _mm256_abs_epi8(wv);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ int32_t aw;+ memcpy(&aw, x[(size_t)t * xs + b].qs + 32 * sb + 4 * q, 4);+ is[t] = DOTN_ACC(+ is[t],+ _mm256_maddubs_epi16(+ aw8, _mm256_sign_epi8(_mm256_set1_epi32(aw), wv)),+ sc);+ }+ }+#endif+ }+ __m256 dw = _mm256_loadu_ps(k->d);+ _Pragma("GCC unroll 8") for (int t = 0; t < T; t++)+ {+ const struct janas_block_q8k *xb = x + (size_t)t * xs + b;+ __m256i si = is[t];+#ifdef DOTN_HAS_VNNI+ __m256i cr = _mm256_setzero_si256();+ _Pragma("GCC unroll 4") for (int kk = 0; kk < 4; kk++)+ {+ int32_t bw;+ memcpy(&bw, xb->bsums2 + 2 * kk, 4);+ cr = _mm256_add_epi32(+ cr, _mm256_madd_epi16(+ _mm256_loadu_si256((const __m256i *)k->mn[kk]),+ _mm256_set1_epi32(bw)));+ }+ si = _mm256_sub_epi32(si, _mm256_slli_epi32(cr, 7));+#endif+ _mm256_storeu_ps(+ af + 8 * t,+ _mm256_fmadd_ps(_mm256_mul_ps(dw, _mm256_set1_ps(xb->d)),+ _mm256_cvtepi32_ps(si),+ _mm256_loadu_ps(af + 8 * t)));+ }}+}-static DOTN_TARGET void DOTN(q6kp_dotn)(const struct janas_block_q6kp *w,- const uint8_t *p1, const uint8_t *p0,- const struct janas_block_q8k *x,- size_t xs, int nv, size_t nb,- float *out)+/* rows (a multiple of 8) of IQ4_XS against nv vectors, y[r + v ys] */+static DOTN_TARGET void DOTN(iq4xs_gemm)(const void *w, size_t row_bytes,+ const struct janas_block_q8k *x,+ size_t xs, size_t nv, size_t nb,+ size_t rows, float *y, size_t ys){- if (p0)- Q6KP_LEVEL(3)- else if (p1)- Q6KP_LEVEL(2)- else- Q6KP_LEVEL(1)+ struct q5kg_blk bk[Q5KG_CHUNK];+ float af[Q4KG_TOK * 8];+ for (size_t g = 0; g < rows; g += 8) {+ const struct janas_block_iq4xs *wr[8];+ for (int j = 0; j < 8; j++)+ wr[j] = (const void *)((const char *)w + (g + j) * row_bytes);+ for (size_t v0 = 0; v0 < nv; v0 += Q4KG_TOK) {+ size_t nt = nv - v0 < Q4KG_TOK ? nv - v0 : Q4KG_TOK;+ memset(af, 0, nt * 8 * sizeof(float));+ for (size_t b0 = 0; b0 < nb; b0 += Q5KG_CHUNK) {+ int n = nb - b0 < Q5KG_CHUNK ? (int)(nb - b0) : Q5KG_CHUNK;+ DOTN(iq4xsg_fill)(wr, b0, n, bk);+ const struct janas_block_q8k *xv = x + v0 * xs;+ size_t t0 = 0;+ for (; t0 + Q4KG_T <= nt; t0 += Q4KG_T)+ DOTN(iq4xsg_tile)(bk, n, b0, xv + t0 * xs, xs, af + 8 * t0,+ Q4KG_T);+ for (; t0 + 4 <= nt; t0 += 4)+ DOTN(iq4xsg_tile)(bk, n, b0, xv + t0 * xs, xs, af + 8 * t0,+ 4);+ if (nt - t0 == 3)+ DOTN(iq4xsg_tile)(bk, n, b0, xv + t0 * xs, xs, af + 8 * t0,+ 3);+ else if (nt - t0 == 2)+ DOTN(iq4xsg_tile)(bk, n, b0, xv + t0 * xs, xs, af + 8 * t0,+ 2);+ else if (nt - t0 == 1)+ DOTN(iq4xsg_tile)(bk, n, b0, xv + t0 * xs, xs, af + 8 * t0,+ 1);+ }+ for (size_t t = 0; t < nt; t++)+ for (int j = 0; j < 8; j++)+ y[g + j + (v0 + t) * ys] = af[8 * t + j];+ }+ }}-#undef Q6KP_LEVEL/* Q4_0: IQ4_NL's kernel with the evenly spaced levels. */static DOTN_TARGET void DOTN(q4_0_dotn)(const struct janas_block_q4_0 *w,@@ -1493,6 +3272,290 @@ DOTN(q5_0_dotn_body)(const struct janas_block_q5_0 *w,DOTN(hsum8_ps)(acc[v]);}+#ifdef DOTN_HAS_VNNI+/*+ * The types in blocks of 32 (Q8_0, IQ4_NL, Q4_0, MXFP4, Q5_0, Q4_1, Q5_1)+ * on 2-8 vectors with no reduction tree (AVX-VNNI): each group's eight+ * blocks, unpacked to bytes, transposed in registers - register r holding+ * bytes 4r..4r+3 of block j in int32 lane j - and the vectors' activations+ * transposed the same way once for all the rows (b32_xt), so that eight+ * vpdpbusd leave block j's sum in lane j. vpdpbusd takes unsigned weights:+ * the signed ones go as w + o (o = 128, or Q4_0's 8 and Q5_0's 16 as they+ * are stored) and o times each block's sum of activations, made ready with+ * them, is taken off: sum (w + o) a - o sum a = sum w a, exactly. A+ * vector's products were eight sign, maddubs and madd and a tree of six+ * hadd; the float operations are each type's own, the same: the same bits.+ * Q8_0 on eight vectors from 6.6 to 14.7 GB/s on a core and from 25 to 55+ * on twelve threads (24 Oct 2026).+ */+#define B32XT 288 /* bytes a vector a group: 256 transposed, 8 int32 sums */+#define B32XT_MAX (48 * 1024)++/* The eight registers' 8 x 8 int32 transposed: lane j of r[i] to lane i of+ r[j]. */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(t8x8_epi32)(__m256i *r)+{+ __m256i a0 = _mm256_unpacklo_epi32(r[0], r[1]);+ __m256i a1 = _mm256_unpackhi_epi32(r[0], r[1]);+ __m256i a2 = _mm256_unpacklo_epi32(r[2], r[3]);+ __m256i a3 = _mm256_unpackhi_epi32(r[2], r[3]);+ __m256i a4 = _mm256_unpacklo_epi32(r[4], r[5]);+ __m256i a5 = _mm256_unpackhi_epi32(r[4], r[5]);+ __m256i a6 = _mm256_unpacklo_epi32(r[6], r[7]);+ __m256i a7 = _mm256_unpackhi_epi32(r[6], r[7]);+ __m256i b0 = _mm256_unpacklo_epi64(a0, a2);+ __m256i b1 = _mm256_unpackhi_epi64(a0, a2);+ __m256i b2 = _mm256_unpacklo_epi64(a1, a3);+ __m256i b3 = _mm256_unpackhi_epi64(a1, a3);+ __m256i b4 = _mm256_unpacklo_epi64(a4, a6);+ __m256i b5 = _mm256_unpackhi_epi64(a4, a6);+ __m256i b6 = _mm256_unpacklo_epi64(a5, a7);+ __m256i b7 = _mm256_unpackhi_epi64(a5, a7);+ r[0] = _mm256_permute2x128_si256(b0, b4, 0x20);+ r[1] = _mm256_permute2x128_si256(b1, b5, 0x20);+ r[2] = _mm256_permute2x128_si256(b2, b6, 0x20);+ r[3] = _mm256_permute2x128_si256(b3, b7, 0x20);+ r[4] = _mm256_permute2x128_si256(b0, b4, 0x31);+ r[5] = _mm256_permute2x128_si256(b1, b5, 0x31);+ r[6] = _mm256_permute2x128_si256(b2, b6, 0x31);+ r[7] = _mm256_permute2x128_si256(b3, b7, 0x31);+}++/* nv vectors' activations transposed (block j's dword r in lane j of+ register r) and their blocks' sums: vector v's group b at xt + B32XT (v+ nb + b). */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(b32_xt)(const struct janas_block_q8k *x, size_t xs, size_t nv, size_t nb,+ uint8_t *xt)+{+ const __m256i ones = _mm256_set1_epi16(1);+ for (size_t v = 0; v < nv; v++)+ for (size_t b = 0; b < nb; b++) {+ const struct janas_block_q8k *xb = x + v * xs + b;+ uint8_t *o = xt + B32XT * (v * nb + b);+ __m256i r[8];+ _Pragma("GCC unroll 8") for (int j = 0; j < 8; j++) r[j] =+ _mm256_loadu_si256((const __m256i *)(xb->qs32 + 32 * j));+ DOTN(t8x8_epi32)(r);+ _Pragma("GCC unroll 8") for (int j = 0; j < 8; j++)+ _mm256_storeu_si256((__m256i *)(o + 32 * j), r[j]);+ _mm256_storeu_si256(+ (__m256i *)(o + 256),+ _mm256_madd_epi16(+ _mm256_loadu_si256((const __m256i *)xb->bsums32), ones));+ }+}++/* A block's bytes and the unsigned 32 values of its weights, the offset o+ added (see above); its d (and m) at its start, MXFP4's E8M0 byte. */+static inline DOTN_TARGET __attribute__((always_inline)) size_t+DOTN(b32_size)(const int type)+{+ return type == JANAS_Q8_0 ? sizeof(struct janas_block_q8_0)+ : type == JANAS_MXFP4 ? sizeof(struct janas_block_mxfp4)+ : type == JANAS_Q5_0 ? sizeof(struct janas_block_q5_0)+ : type == JANAS_Q4_1 ? sizeof(struct janas_block_q4_1)+ : type == JANAS_Q5_1 ? sizeof(struct janas_block_q5_1)+ : type == JANAS_IQ4_NL ? sizeof(struct janas_block_iq4nl)+ : sizeof(struct janas_block_q4_0);+}++static inline DOTN_TARGET __attribute__((always_inline)) __m256i+DOTN(b32_unpack)(const int type, const uint8_t *blk)+{+ const __m256i flip = _mm256_set1_epi8((char)0x80),+ low4 = _mm256_set1_epi8(0x0f);+ if (type == JANAS_Q8_0)+ return _mm256_xor_si256(_mm256_loadu_si256((const __m256i *)(blk + 2)),+ flip);+ if (type == JANAS_IQ4_NL || type == JANAS_MXFP4) {+ const int8_t *t =+ type == JANAS_MXFP4 ? janas_mxfp4_values : janas_iq4_values;+ __m256i lev =+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)t));+ return _mm256_xor_si256(+ DOTN(iq4_unpack32)(blk + (type == JANAS_MXFP4 ? 1 : 2), lev), flip);+ }+ /* the nibbles, low ones weights 0-15 (Q4_0, Q5_0, Q4_1, Q5_1) */+ size_t at = type == JANAS_Q4_0 ? 2+ : type == JANAS_Q5_0 ? 6+ : type == JANAS_Q4_1 ? 4+ : 8;+ __m128i b16 = _mm_loadu_si128((const __m128i *)(blk + at));+ __m256i q =+ _mm256_and_si256(_mm256_set_m128i(_mm_srli_epi16(b16, 4), b16), low4);+ if (type == JANAS_Q5_0 || type == JANAS_Q5_1) {+ /* as dm_dotn_body: byte l of a lane takes byte l / 8 of qh, then+ keeps bit l % 8 of it */+ const __m256i sel =+ _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2,+ 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3);+ const __m256i bit =+ _mm256_set1_epi64x((long long)0x8040201008040201ull);+ uint32_t qh;+ memcpy(&qh, blk + (type == JANAS_Q5_0 ? 2 : 4), 4);+ __m256i h = _mm256_shuffle_epi8(_mm256_set1_epi32((int)qh), sel);+ h = _mm256_and_si256(_mm256_cmpeq_epi8(_mm256_and_si256(h, bit), bit),+ _mm256_set1_epi8(0x10));+ q = _mm256_or_si256(q, h);+ }+ return q;+}++static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(b32_xt_body)(const int type, const uint8_t *w,+ const struct janas_block_q8k *x, size_t xs, const uint8_t *xt,+ size_t nb, float *out, const int nv)+{+ const size_t size = DOTN(b32_size)(type);+ /* log2 of the offset o; Q4_1 and Q5_1 have none, but their mins */+ const int lo = type == JANAS_Q4_0 ? 3 : type == JANAS_Q5_0 ? 4 : 7;+ const int dm = type == JANAS_Q4_1 || type == JANAS_Q5_1;+ __m256 acc[8], accm[8];+ _Pragma("GCC unroll 8") for (int v = 0; v < nv; v++)+ {+ acc[v] = _mm256_setzero_ps();+ accm[v] = _mm256_setzero_ps();+ }+ for (size_t b = 0; b < nb; b++) {+ const uint8_t *wb = w + 8 * b * size;+ if (nv <= 2)+ DOTN(prefetch_ahead)(wb, 8 * size);+ __m256 dk, mk = _mm256_setzero_ps();+ if (type == JANAS_MXFP4) {+ dk = _mm256_setr_ps(e8m0_tab[wb[0]], e8m0_tab[wb[size]],+ e8m0_tab[wb[2 * size]], e8m0_tab[wb[3 * size]],+ e8m0_tab[wb[4 * size]], e8m0_tab[wb[5 * size]],+ e8m0_tab[wb[6 * size]], e8m0_tab[wb[7 * size]]);+ } else {+ uint16_t d[8], m[8];+ _Pragma("GCC unroll 8") for (int k = 0; k < 8; k++)+ {+ memcpy(&d[k], wb + k * size, 2);+ if (dm)+ memcpy(&m[k], wb + k * size + 2, 2);+ }+ dk = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)d));+ if (dm)+ mk = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)m));+ }+ __m256i q[8];+ _Pragma("GCC unroll 8") for (int k = 0; k < 8; k++) q[k] =+ DOTN(b32_unpack)(type, wb + k * size);+ DOTN(t8x8_epi32)(q);+ _Pragma("GCC unroll 8") for (int v = 0; v < nv; v++)+ {+ const __m256i *a =+ (const __m256i *)(xt + B32XT * ((size_t)v * nb + b));+ __m256i s = _mm256_setzero_si256();+ _Pragma("GCC unroll 8") for (int k = 0; k < 8; k++) s =+ _mm256_dpbusd_avx_epi32(s, q[k], _mm256_loadu_si256(a + k));+ __m256i as8 = _mm256_loadu_si256(a + 8);+ const struct janas_block_q8k *xb = x + (size_t)v * xs + b;+ __m256 dx = _mm256_loadu_ps(xb->d32);+ if (dm) {+ acc[v] = _mm256_fmadd_ps(_mm256_mul_ps(dk, dx),+ _mm256_cvtepi32_ps(s), acc[v]);+ accm[v] = _mm256_fmadd_ps(_mm256_mul_ps(mk, dx),+ _mm256_cvtepi32_ps(as8), accm[v]);+ } else {+ __m256i is8 = _mm256_sub_epi32(s, _mm256_slli_epi32(as8, lo));+ acc[v] = _mm256_fmadd_ps(_mm256_mul_ps(dk, dx),+ _mm256_cvtepi32_ps(is8), acc[v]);+ }+ }+ }+ _Pragma("GCC unroll 8") for (int v = 0; v < nv; v++) out[v] =+ dm ? DOTN(hsum8_ps)(acc[v]) + DOTN(hsum8_ps)(accm[v])+ : DOTN(hsum8_ps)(acc[v]);+}++static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(b32_xt_n)(const int type, const uint8_t *w,+ const struct janas_block_q8k *x, size_t xs, const uint8_t *xt,+ size_t nb, float *o, int n)+{+ switch (n) {+ case 8:+ DOTN(b32_xt_body)(type, w, x, xs, xt, nb, o, 8);+ break;+ case 7:+ DOTN(b32_xt_body)(type, w, x, xs, xt, nb, o, 7);+ break;+ case 6:+ DOTN(b32_xt_body)(type, w, x, xs, xt, nb, o, 6);+ break;+ case 5:+ DOTN(b32_xt_body)(type, w, x, xs, xt, nb, o, 5);+ break;+ case 4:+ DOTN(b32_xt_body)(type, w, x, xs, xt, nb, o, 4);+ break;+ case 3:+ DOTN(b32_xt_body)(type, w, x, xs, xt, nb, o, 3);+ break;+ case 2:+ DOTN(b32_xt_body)(type, w, x, xs, xt, nb, o, 2);+ break;+ default:+ DOTN(b32_xt_body)(type, w, x, xs, xt, nb, o, 1);+ }+}++/* rows rows of one of these types against nv vectors, y[r + v ys]: up to+ eight vectors transposed at a time, as many as fit in B32XT_MAX bytes.+ 0, nothing done, for one vector, one row, or rows too long. */+static inline DOTN_TARGET __attribute__((always_inline)) int+DOTN(b32_multi_rows_t)(const int type, const void *w, size_t row_bytes,+ const struct janas_block_q8k *x, size_t xs, size_t nv,+ size_t nb, size_t rows, float *y, size_t ys)+{+ size_t g = nb ? B32XT_MAX / (B32XT * nb) : 0;+ if (g > 8)+ g = 8;+ if (g < 2 || nv < 2 || rows < 2)+ return 0;+ _Alignas(32) uint8_t xt[B32XT_MAX];+ for (size_t v0 = 0; v0 < nv; v0 += g) {+ int n = nv - v0 < g ? (int)(nv - v0) : (int)g;+ const struct janas_block_q8k *xv = x + v0 * xs;+ DOTN(b32_xt)(xv, xs, (size_t)n, nb, xt);+ for (size_t r = 0; r < rows; r++) {+ float o[8];+ DOTN(b32_xt_n)+ (type, (const uint8_t *)w + r * row_bytes, xv, xs, xt, nb, o, n);+ for (int i = 0; i < n; i++)+ y[r + (v0 + (size_t)i) * ys] = o[i];+ }+ }+ return 1;+}++static DOTN_TARGET int DOTN(b32_multi_rows)(int type, const void *w,+ size_t row_bytes,+ const struct janas_block_q8k *x,+ size_t xs, size_t nv, size_t nb,+ size_t rows, float *y, size_t ys)+{+ switch (type) {+#define B32(T) \+ case T: \+ return DOTN(b32_multi_rows_t)(T, w, row_bytes, x, xs, nv, nb, rows, y, \+ ys);+ B32(JANAS_Q8_0)+ B32(JANAS_IQ4_NL)+ B32(JANAS_Q4_0)+ B32(JANAS_MXFP4)+ B32(JANAS_Q5_0)+ B32(JANAS_Q4_1)+ B32(JANAS_Q5_1)+#undef B32+ }+ return 0;+}+#endif+/* The 64 two-bit values of a Q2_0 block in their order, 0-3: the fourplanes of qs (byte i of plane j is weight 4i + j) interleaved, eight bitsthen sixteen. lo holds weights 0-31, hi 32-63. */@@ -1640,6 +3703,232 @@ static DOTN_TARGET float DOTN(q2_0_dot)(const struct janas_block_q2_0 *w,return r;}+/*+ * Q2_0 rows against vectors whose activations were reordered once for all+ * the rows: the experts of Qwen3.8-Flash-Next, on one vector while it+ * answers and on a few while it reads. Byte i of a block holds weights+ * 4i + j in its bits 2j; two variable shifts of the 16 bytes, read into+ * both halves of a register, give the bit planes 0 | 2 and 1 | 3, the 16+ * bytes of plane j being weights 4i + j, and the activations are laid out+ * the same way (q2_0_xperm64). In each half, int32 lanes 0-1 are weights+ * 0-31 of the block and lanes 2-3 weights 32-63, so two hadd and one add of+ * the halves leave the eight sums of 32 in order, where Q8_0's tree takes+ * six hadd. Sums of integers, exact in any order, then the float operations+ * of the scalar reference: the same bits. Unpacking each row in the order+ * of the activations held one vector to 7.7 GB/s on a core and 34 on+ * twelve threads, against Q4_K's 24 and 85; this way 19 and 69 (24 Oct+ * 2026).+ */+#define Q2_0_XP_MAX 128 /* groups of 256 reordered on the stack: 32 KB */++/* The 64 activations of a Q2_0 block as its planes: 64 bytes, planes 0 | 2+ then 1 | 3, 16 bytes each, plane j the activations 4i + j. */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q2_0_xperm64)(const int8_t *a, int8_t *out)+{+ /* in each 16 bytes, the four bytes j apart side by side: dword j of+ lane m is activations 16m + 4i + j, i = 0-3 */+ const __m256i g =+ _mm256_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15,+ 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);+ __m256i b0 = _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i *)a), g);+ __m256i b1 =+ _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i *)(a + 32)), g);+ /* lane m of b0 / b1 is D(m, 0..3); gathered across the lanes: plane j+ is D(0, j), D(1, j), D(2, j), D(3, j) */+ __m256i e0 = _mm256_shuffle_epi32(_mm256_permute4x64_epi64(b0, 0xd8), 0xd8);+ __m256i e1 = _mm256_shuffle_epi32(_mm256_permute4x64_epi64(b1, 0xd8), 0xd8);+ _mm256_storeu_si256((__m256i *)out, _mm256_unpacklo_epi64(e0, e1));+ _mm256_storeu_si256((__m256i *)(out + 32), _mm256_unpackhi_epi64(e0, e1));+}++/* nv vectors reordered into xp, vector v's group b at xp + 256 (v nb + b). */+static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q2_0_xperm)(const struct janas_block_q8k *x, size_t xs, size_t nv,+ size_t nb, int8_t *xp)+{+ for (size_t v = 0; v < nv; v++)+ for (size_t b = 0; b < nb; b++)+ _Pragma("GCC unroll 4") for (int k = 0; k < 4; k++)+ DOTN(q2_0_xperm64)(x[v * xs + b].qs32 + 64 * k,+ xp + 256 * (v * nb + b) + 64 * k);+}++static inline DOTN_TARGET __attribute__((always_inline)) void+DOTN(q2_0_xp_body)(const struct janas_block_q2_0 *w,+ const struct janas_block_q8k *x, size_t xs, const int8_t *xp,+ size_t nb, float *out, const int nv)+{+ const __m256i m3 = _mm256_set1_epi8(3), ones = _mm256_set1_epi16(1),+ c02 = _mm256_setr_epi64x(0, 0, 4, 4),+ c13 = _mm256_setr_epi64x(2, 2, 6, 6);+ __m256 acc[8];+ _Pragma("GCC unroll 8") for (int v = 0; v < nv; v++) acc[v] =+ _mm256_setzero_ps();+ for (size_t b = 0; b < nb; b++) {+ if (nv <= 2)+ DOTN(prefetch_ahead)(w + 4 * b, 4 * sizeof(*w));+ const struct janas_block_q2_0 *wb = w + 4 * b;+ uint64_t d4 = (uint64_t)wb[0].d | (uint64_t)wb[1].d << 16 |+ (uint64_t)wb[2].d << 32 | (uint64_t)wb[3].d << 48;+ __m128i h = _mm_cvtsi64_si128((long long)d4);+ __m256 dk = _mm256_cvtph_ps(_mm_unpacklo_epi16(h, h));+ __m256i p02[4], p13[4];+ _Pragma("GCC unroll 4") for (int k = 0; k < 4; k++)+ {+ __m256i q = _mm256_broadcastsi128_si256(+ _mm_loadu_si128((const __m128i *)wb[k].qs));+ p02[k] = _mm256_and_si256(_mm256_srlv_epi64(q, c02), m3);+ p13[k] = _mm256_and_si256(_mm256_srlv_epi64(q, c13), m3);+ }+ _Pragma("GCC unroll 8") for (int v = 0; v < nv; v++)+ {+ const struct janas_block_q8k *xb = x + (size_t)v * xs + b;+ const int8_t *a = xp + 256 * ((size_t)v * nb + b);+ __m256i s[4];+ _Pragma("GCC unroll 4") for (int k = 0; k < 4; k++) s[k] = DOTN_US2(+ p02[k], _mm256_loadu_si256((const __m256i *)(a + 64 * k)),+ p13[k], _mm256_loadu_si256((const __m256i *)(a + 64 * k + 32)));+ __m256i h01 = _mm256_hadd_epi32(s[0], s[1]);+ __m256i h23 = _mm256_hadd_epi32(s[2], s[3]);+ __m256i t =+ _mm256_add_epi32(_mm256_permute2x128_si256(h01, h23, 0x20),+ _mm256_permute2x128_si256(h01, h23, 0x31));+ __m256i as8 = _mm256_madd_epi16(+ _mm256_loadu_si256((const __m256i *)xb->bsums32), ones);+ acc[v] = _mm256_fmadd_ps(+ _mm256_mul_ps(dk, _mm256_loadu_ps(xb->d32)),+ _mm256_cvtepi32_ps(_mm256_sub_epi32(t, as8)), acc[v]);+ }+ }+ _Pragma("GCC unroll 8") for (int v = 0; v < nv; v++) out[v] =+ DOTN(hsum8_ps)(acc[v]);+}++/* rows Q2_0 rows of row_bytes from w against nv vectors (vector v at x + v+ xs): y[r + v ys]. Up to eight vectors at a time, as many as their+ reordered activations fit in Q2_0_XP_MAX groups; with no room, or a+ single row, the kernels on activations as they are. */+static DOTN_TARGET void DOTN(q2_0_multi_rows)(const void *w, size_t row_bytes,+ const struct janas_block_q8k *x,+ size_t xs, size_t nv, size_t nb,+ size_t rows, float *y, size_t ys)+{+ size_t g = nb ? Q2_0_XP_MAX / nb : 0;+ if (g > 8)+ g = 8;+ if (g == 0 || rows < 2) {+ for (size_t r = 0; r < rows; r++) {+ const struct janas_block_q2_0 *wr =+ (const void *)((const char *)w + r * row_bytes);+ for (size_t v = 0; v < nv; v += 8) {+ int n = nv - v < 8 ? (int)(nv - v) : 8;+ float o[8];+ DOTN(q2_0_dotn)(wr, x + v * xs, xs, n, nb, o);+ for (int i = 0; i < n; i++)+ y[r + (v + (size_t)i) * ys] = o[i];+ }+ }+ return;+ }+ _Alignas(32) int8_t xp[Q2_0_XP_MAX * 256];+ for (size_t v0 = 0; v0 < nv; v0 += g) {+ int n = nv - v0 < g ? (int)(nv - v0) : (int)g;+ const struct janas_block_q8k *xv = x + v0 * xs;+ DOTN(q2_0_xperm)(xv, xs, (size_t)n, nb, xp);+ for (size_t r = 0; r < rows; r++) {+ const struct janas_block_q2_0 *wr =+ (const void *)((const char *)w + r * row_bytes);+ float o[8];+ switch (n) {+ case 8:+ DOTN(q2_0_xp_body)(wr, xv, xs, xp, nb, o, 8);+ break;+ case 7:+ DOTN(q2_0_xp_body)(wr, xv, xs, xp, nb, o, 7);+ break;+ case 6:+ DOTN(q2_0_xp_body)(wr, xv, xs, xp, nb, o, 6);+ break;+ case 5:+ DOTN(q2_0_xp_body)(wr, xv, xs, xp, nb, o, 5);+ break;+ case 4:+ DOTN(q2_0_xp_body)(wr, xv, xs, xp, nb, o, 4);+ break;+ case 3:+ DOTN(q2_0_xp_body)(wr, xv, xs, xp, nb, o, 3);+ break;+ case 2:+ DOTN(q2_0_xp_body)(wr, xv, xs, xp, nb, o, 2);+ break;+ default:+ DOTN(q2_0_xp_body)(wr, xv, xs, xp, nb, o, 1);+ }+ for (int i = 0; i < n; i++)+ y[r + (v0 + (size_t)i) * ys] = o[i];+ }+ }+}++/* one vector: the body above with nv = 1 let the kernel slip from 19 to+ 16 GB/s on a core, its weights held through the loop over vectors */+static inline DOTN_TARGET __attribute__((always_inline)) float+DOTN(q2_0_dot_xp)(const struct janas_block_q2_0 *w,+ const struct janas_block_q8k *x, const int8_t *xp, size_t nb)+{+ const __m256i m3 = _mm256_set1_epi8(3), ones = _mm256_set1_epi16(1),+ c02 = _mm256_setr_epi64x(0, 0, 4, 4),+ c13 = _mm256_setr_epi64x(2, 2, 6, 6);+ __m256 acc = _mm256_setzero_ps();+ for (size_t b = 0; b < nb; b++) {+ DOTN(prefetch_ahead)(w + 4 * b, 4 * sizeof(*w));+ const struct janas_block_q2_0 *wb = w + 4 * b;+ uint64_t d4 = (uint64_t)wb[0].d | (uint64_t)wb[1].d << 16 |+ (uint64_t)wb[2].d << 32 | (uint64_t)wb[3].d << 48;+ __m128i h = _mm_cvtsi64_si128((long long)d4);+ __m256 dk = _mm256_cvtph_ps(_mm_unpacklo_epi16(h, h));+ const int8_t *a = xp + 256 * b;+ __m256i s[4];+ _Pragma("GCC unroll 4") for (int k = 0; k < 4; k++)+ {+ __m256i v = _mm256_broadcastsi128_si256(+ _mm_loadu_si128((const __m128i *)wb[k].qs));+ __m256i p02 = _mm256_and_si256(_mm256_srlv_epi64(v, c02), m3);+ __m256i p13 = _mm256_and_si256(_mm256_srlv_epi64(v, c13), m3);+ s[k] = DOTN_US2(+ p02, _mm256_loadu_si256((const __m256i *)(a + 64 * k)), p13,+ _mm256_loadu_si256((const __m256i *)(a + 64 * k + 32)));+ }+ __m256i h01 = _mm256_hadd_epi32(s[0], s[1]);+ __m256i h23 = _mm256_hadd_epi32(s[2], s[3]);+ __m256i t = _mm256_add_epi32(_mm256_permute2x128_si256(h01, h23, 0x20),+ _mm256_permute2x128_si256(h01, h23, 0x31));+ __m256i as8 = _mm256_madd_epi16(+ _mm256_loadu_si256((const __m256i *)x[b].bsums32), ones);+ __m256i is8 = _mm256_sub_epi32(t, as8);+ acc = _mm256_fmadd_ps(_mm256_mul_ps(dk, _mm256_loadu_ps(x[b].d32)),+ _mm256_cvtepi32_ps(is8), acc);+ }+ return DOTN(hsum8_ps)(acc);+}++static DOTN_TARGET void DOTN(q2_0_rows)(const void *w, size_t row_bytes,+ const struct janas_block_q8k *x,+ size_t nb, size_t rows, float *y)+{+ if (nb > Q2_0_XP_MAX || rows < 2) {+ DOTN(q2_0_multi_rows)(w, row_bytes, x, 0, 1, nb, rows, y, 0);+ return;+ }+ _Alignas(32) int8_t xp[Q2_0_XP_MAX * 256];+ DOTN(q2_0_xperm)(x, 0, 1, nb, xp);+ for (size_t r = 0; r < rows; r++)+ y[r] = DOTN(q2_0_dot_xp)(+ (const struct janas_block_q2_0 *)((const char *)w + r * row_bytes),+ x, xp, nb);+}+/* The 16 scales of a Q3_K block, -32..31, as q3k_scales() gives them,unpacked in a register: the low and high nibbles of the first eightbytes, and the two bits of the last four each group takes. */@@ -1837,36 +4126,62 @@ DOTN(iq2_dotn_body)(const int type, const void *w,const struct janas_block_q8k *x, size_t xs, size_t nb,float *out, const int nv){- const __m256i sel =- _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2,- 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3);- const __m256i bit = _mm256_set1_epi64x((long long)0x8040201008040201ull);- const __m256i one = _mm256_set1_epi8(1);const size_t size = iq2_size(type);float acc[8] = {0};for (size_t b = 0; b < nb; b++) {const uint8_t *blk = (const uint8_t *)w + b * size;if (nv <= 2)DOTN(prefetch_ahead)(blk, size);- __m256i lv[8], sg[8], sc[8];+ /* each group's weights used at once by every vector, the vectors'+ integer sums in registers: the 24 registers of a block's+ unpacked weights, kept for the vectors, went to the stack and+ back (1.2 GB/s on a core, 7.6 on twelve threads: 24 Oct 2026) */+ __m256i sum[8];+ _Pragma("GCC unroll 8") for (int v = 0; v < nv; v++) sum[v] =+ _mm256_setzero_si256();_Pragma("GCC unroll 8") for (int g = 0; g < 8; g++){- uint64_t gr[4];- uint32_t sb = 0;+ uint64_t gr[4], sm[4];int sl[4];- for (int k = 0; k < 4; k++) {- uint32_t s8;- iq2_sub(type, blk, 4 * g + k, &gr[k], &s8, &sl[k]);- sb |= s8 << 8 * k;+ __m256i sg;+ if (type == JANAS_IQ2_S) {+ /* signs bytes as they are: spread to the bytes in the+ register, cheaper than four reads of iq_byte64 */+ uint32_t sb = 0;+ _Pragma("GCC unroll 4") for (int k = 0; k < 4; k++)+ {+ uint32_t s8;+ iq2_sub(type, blk, 4 * g + k, &gr[k], &s8, &sl[k]);+ sb |= s8 << 8 * k;+ }+ const __m256i sel = _mm256_setr_epi8(+ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2,+ 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3);+ const __m256i bit =+ _mm256_set1_epi64x((long long)0x8040201008040201ull);+ __m256i m =+ _mm256_shuffle_epi8(_mm256_set1_epi32((int)sb), sel);+ m = _mm256_cmpeq_epi8(_mm256_and_si256(m, bit), bit);+ sg = _mm256_or_si256(m, _mm256_set1_epi8(1)); /* -1: set */+ } else {+ /* 1 or -1 a weight (iq_sign64): no parity to work out */+ _Pragma("GCC unroll 4") for (int k = 0; k < 4; k++)+ iq2_sub_mask(type, blk, 4 * g + k, &gr[k], &sm[k], &sl[k]);+ sg = _mm256_setr_epi64x((long long)sm[0], (long long)sm[1],+ (long long)sm[2], (long long)sm[3]);}- lv[g] = _mm256_setr_epi64x((long long)gr[0], (long long)gr[1],- (long long)gr[2], (long long)gr[3]);- __m256i m = _mm256_shuffle_epi8(_mm256_set1_epi32((int)sb), sel);- m = _mm256_cmpeq_epi8(_mm256_and_si256(m, bit), bit);- sg[g] = _mm256_or_si256(m, one); /* -1 where the sign bit is set */+ __m256i lv = _mm256_setr_epi64x((long long)gr[0], (long long)gr[1],+ (long long)gr[2], (long long)gr[3]);/* the products' lanes 0-7 are weights 0-15, sub-groups 0 and 1 */- sc[g] = _mm256_set_m128i(_mm_set1_epi16((short)sl[2]),- _mm_set1_epi16((short)sl[0]));+ __m256i sc = _mm256_set_m128i(_mm_set1_epi16((short)sl[2]),+ _mm_set1_epi16((short)sl[0]));+ _Pragma("GCC unroll 8") for (int v = 0; v < nv; v++)+ {+ __m256i a = _mm256_loadu_si256(+ (const __m256i *)(x[(size_t)v * xs + b].qs + 32 * g));+ __m256i p = _mm256_maddubs_epi16(lv, _mm256_sign_epi8(a, sg));+ sum[v] = DOTN_ACC(sum[v], p, sc);+ }}uint16_t h;memcpy(&h, blk, 2);@@ -1874,16 +4189,7 @@ DOTN(iq2_dotn_body)(const int type, const void *w,_Pragma("GCC unroll 8") for (int v = 0; v < nv; v++){const struct janas_block_q8k *xb = x + (size_t)v * xs + b;- __m256i sum = _mm256_setzero_si256();- _Pragma("GCC unroll 8") for (int g = 0; g < 8; g++)- {- __m256i a =- _mm256_loadu_si256((const __m256i *)(xb->qs + 32 * g));- __m256i p =- _mm256_maddubs_epi16(lv[g], _mm256_sign_epi8(a, sg[g]));- sum = DOTN_ACC(sum, p, sc[g]);- }- acc[v] = fmaf(d * xb->d, (float)DOTN(hsum_epi32)(sum), acc[v]);+ acc[v] = fmaf(d * xb->d, (float)DOTN(hsum_epi32)(sum[v]), acc[v]);}}_Pragma("GCC unroll 8") for (int v = 0; v < nv; v++) out[v] = acc[v];@@ -1991,7 +4297,8 @@ DOTN(iq1_dotn_body)(const int iq1s, const void *w,{uint64_t gr[4], sh[4];int sl[4];- for (int k = 0; k < 4; k++) {+ _Pragma("GCC unroll 4") for (int k = 0; k < 4; k++)+ {int neg;if (iq1s)iq1s_sub(&ws[b], 4 * g + k, &gr[k], &neg, &sl[k]);@@ -2012,7 +4319,7 @@ DOTN(iq1_dotn_body)(const int iq1s, const void *w,sc[g] = _mm256_set_m128i(_mm_set1_epi16((short)sl[2]),_mm_set1_epi16((short)sl[0]));}- float d = (iq1s ? _cvtsh_ss(ws[b].d) : iq1m_d(&wm[b])) * 0.125f;+ float d = (_cvtsh_ss(iq1s ? ws[b].d : iq1m_d16(&wm[b]))) * 0.125f;_Pragma("GCC unroll 8") for (int v = 0; v < nv; v++){const struct janas_block_q8k *xb = x + (size_t)v * xs + b;@@ -2128,6 +4435,12 @@ DOTN(q2k_dotn_body)(const struct janas_block_q2k *w,if (nv <= 2)DOTN(prefetch_ahead)(&w[b], sizeof(w[b]));__m256i q[8], sc[8];+ __m128i m8 = _mm_loadu_si128((const __m128i *)w[b].scales);+ /* the sixteen scales in both lanes; sub-group k's two (weights+ 32k..32k+15 and the next 16) spread over a lane each, as int16,+ by one shuffle: sixteen scalar loads and broadcasts before */+ __m256i s16 =+ _mm256_broadcastsi128_si256(_mm_and_si128(m8, _mm_set1_epi8(15)));_Pragma("GCC unroll 2") for (int n = 0; n < 2; n++){__m256i bytes =@@ -2136,13 +4449,20 @@ DOTN(q2k_dotn_body)(const struct janas_block_q2k *w,{q[4 * n + j] = _mm256_and_si256(_mm256_srli_epi16(bytes, (int)(2 * j)), three);- int g = 8 * n + 2 * j;- sc[4 * n + j] = _mm256_set_m128i(- _mm_set1_epi16((short)(w[b].scales[g + 1] & 15)),- _mm_set1_epi16((short)(w[b].scales[g] & 15)));+ const int g = 8 * n + 2 * j;+ sc[4 * n + j] = _mm256_shuffle_epi8(+ s16,+ _mm256_setr_epi16(+ (short)(0x8000 | g), (short)(0x8000 | g),+ (short)(0x8000 | g), (short)(0x8000 | g),+ (short)(0x8000 | g), (short)(0x8000 | g),+ (short)(0x8000 | g), (short)(0x8000 | g),+ (short)(0x8000 | (g + 1)), (short)(0x8000 | (g + 1)),+ (short)(0x8000 | (g + 1)), (short)(0x8000 | (g + 1)),+ (short)(0x8000 | (g + 1)), (short)(0x8000 | (g + 1)),+ (short)(0x8000 | (g + 1)), (short)(0x8000 | (g + 1))));}}- __m128i m8 = _mm_loadu_si128((const __m128i *)w[b].scales);__m256i mins = _mm256_cvtepu8_epi16(_mm_srli_epi16(_mm_and_si128(m8, _mm_set1_epi8((char)0xf0)), 4));float d = _cvtsh_ss(w[b].d), dmin = _cvtsh_ss(w[b].dmin);@@ -2158,8 +4478,10 @@ DOTN(q2k_dotn_body)(const struct janas_block_q2k *w,}__m256i mn = _mm256_madd_epi16(mins, _mm256_loadu_si256((const __m256i *)xb->bsums));- acc[v] = fmaf(d * xb->d, (float)DOTN(hsum_epi32)(sum), acc[v]);- accm[v] = fmaf(dmin * xb->d, (float)DOTN(hsum_epi32)(mn), accm[v]);+ int is, im;+ DOTN(hsum2_epi32)(sum, mn, &is, &im);+ acc[v] = fmaf(d * xb->d, (float)is, acc[v]);+ accm[v] = fmaf(dmin * xb->d, (float)im, accm[v]);}}_Pragma("GCC unroll 8") for (int v = 0; v < nv; v++) out[v] =@@ -2213,11 +4535,6 @@ DOTN(iq3xxs_dotn_body)(const struct janas_block_iq3xxs *w,const struct janas_block_q8k *x, size_t xs, size_t nb,float *out, const int nv){- const __m256i sel =- _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2,- 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3);- const __m256i bit = _mm256_set1_epi64x((long long)0x8040201008040201ull);- const __m256i one = _mm256_set1_epi8(1);float acc[8] = {0};for (size_t b = 0; b < nb; b++) {if (nv <= 2)@@ -2229,12 +4546,10 @@ DOTN(iq3xxs_dotn_body)(const struct janas_block_iq3xxs *w,lv[g] = _mm256_i32gather_epi32((const int *)iq3xxs_grid,_mm256_cvtepu8_epi32(q8), 4);uint32_t aux = iq3xxs_aux(&w[b], g);- uint32_t sb = iq_signs(aux & 127) | iq_signs(aux >> 7 & 127) << 8 |- iq_signs(aux >> 14 & 127) << 16 |- iq_signs(aux >> 21 & 127) << 24;- __m256i m = _mm256_shuffle_epi8(_mm256_set1_epi32((int)sb), sel);- m = _mm256_cmpeq_epi8(_mm256_and_si256(m, bit), bit);- sg[g] = _mm256_or_si256(m, one); /* -1 where the sign bit is set */+ sg[g] = _mm256_setr_epi64x((long long)iq_sign64[aux & 127],+ (long long)iq_sign64[aux >> 7 & 127],+ (long long)iq_sign64[aux >> 14 & 127],+ (long long)iq_sign64[aux >> 21 & 127]);sc[g] = _mm256_set1_epi16((short)(1 + 2 * (aux >> 28)));}float d = _cvtsh_ss(w[b].d) * 0.25f;@@ -2367,7 +4682,6 @@ DOTN_ROWS(q4_0, struct janas_block_q4_0)DOTN_ROWS(q5_1, struct janas_block_q5_1)DOTN_ROWS(q4_1, struct janas_block_q4_1)DOTN_ROWS(q5_0, struct janas_block_q5_0)-DOTN_ROWS(q2_0, struct janas_block_q2_0)DOTN_ROWS(iq3xxs, struct janas_block_iq3xxs)DOTN_ROWS(iq1m, struct janas_block_iq1m)DOTN_ROWS(iq1s, struct janas_block_iq1s)diff --git a/src/llm/shaders/b32_matvec.comp b/src/llm/shaders/b32_matvec.compindex aa1d6b6..6ad8876 100644--- a/src/llm/shaders/b32_matvec.comp+++ b/src/llm/shaders/b32_matvec.comp@@ -29,6 +29,9 @@// realigned by the byte, the others' by the half word. An E8M0 byte of 0 or// 1 is a subnormal scale, which a GPU may flush to zero where the CPU does// not: such blocks (weights under 2^-125) differ from the CPU's.+// With ID (bit 8 of constant 2) a mixture's experts, the table of+// *_mmid.comp (gpu_token_moe.c) in w, as q4k_matvec.comp reads it: x's+// vectors nb blocks of 256 apart, one more with a tail.#extension GL_EXT_integer_dot_product : require#extension GL_KHR_shader_subgroup_basic : require#extension GL_KHR_shader_subgroup_ballot : require@@ -38,7 +41,9 @@layout(local_size_x = 64) in;layout(constant_id = 0) const uint NV = 8;layout(constant_id = 1) const uint SG = 8;-layout(constant_id = 2) const uint KIND = 0;+layout(constant_id = 2) const uint KID = 0;+const uint KIND = KID & 255u;+const bool ID = (KID & 256u) != 0u;const uint NX = (20u + SG - 1u) / SG;/* bytes of a block of 32 weights (Q2_0: half a block), and of a group ofeight in words */@@ -104,9 +109,12 @@ uint iq4(uint w)(T[(w >> 24) & 15u] << 24);}+/* precise, as the accumulators below: no add fused with a product (a+ tail's blocks are single products from zero, which a compiler may see) */float hsum8(float a[8]){- return ((a[0] + a[4]) + (a[2] + a[6])) + ((a[1] + a[5]) + (a[3] + a[7]));+ precise float s = ((a[0] + a[4]) + (a[2] + a[6])) + ((a[1] + a[5]) + (a[3] + a[7]));+ return s;}/* the word of wg at byte b, realigned by the half word */@@ -121,20 +129,32 @@ void main()T[gl_LocalInvocationID.x] =uint(KIND == 4u ? MX[gl_LocalInvocationID.x] : IQ4[gl_LocalInvocationID.x]) & 0xffu;barrier();- U1 W = U1(p.w);- U4 X = U4(p.x);- F Y = F(p.y);+ uvec2 pw = p.w, px = p.x, py = p.y;+ uint nv = p.nv, xs = p.xs, ys = p.ys, v0 = gl_WorkGroupID.x * NV;+ if (ID) {+ U4 Tb = U4(p.w);+ uvec4 tl = Tb.v[p.nv + gl_WorkGroupID.z];+ uvec4 t0 = Tb.v[3u * tl.x], t1 = Tb.v[3u * tl.x + 1u];+ pw = t0.xy, px = t0.zw, py = t1.xy;+ nv = t1.z, xs = p.nb + (p.tail != 0u ? 1u : 0u), ys = t1.w, v0 = tl.y;+ }+ U1 W = U1(pw);+ U4 X = U4(px);+ F Y = F(py);uint lane = gl_SubgroupInvocationID;/* workgroup y: 64 rows; x: a group of NV vectors, so that the groupsof the same rows go out together and the weights the first readsfrom memory the others find in the L3 */uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;uint row = min(grow, p.rows - 1u);- uint v0 = gl_WorkGroupID.x * NV;uint xo[NV];for (uint v = 0; v < NV; v++)- xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;- float acc[NV][8], accm[NV][8], res[NV];+ xo[v] = min(v0 + v, nv - 1u) * xs * 20u;+ /* precise: the CPU rounds every product and every sum apart; without+ it a compiler may fuse a sum with a product before it (the tail's+ accumulators start from zero in the loop, a known value: AMD's,+ issue #46) */+ precise float acc[NV][8], accm[NV][8], res[NV];for (uint v = 0; v < NV; v++) {res[v] = 0.0;for (uint k = 0; k < 8u; k++) {@@ -238,12 +258,12 @@ void main()}if (grow < p.rows)for (uint v = 0; v < NV; v++)- if (v0 + v < p.nv) {- float r = hsum8(acc[v]);+ if (v0 + v < nv) {+ precise float r = hsum8(acc[v]);if (DM)r = r + hsum8(accm[v]);if (p.tail != 0u)r = res[v] + r;- Y.v[(v0 + v) * p.ys + row] = r;+ Y.v[(v0 + v) * ys + row] = r;}}diff --git a/src/llm/shaders/dense_mv.comp b/src/llm/shaders/dense_mv.compnew file mode 100644index 0000000..2cc7ff6--- /dev/null+++ b/src/llm/shaders/dense_mv.comp@@ -0,0 +1,77 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+#version 450+// dense_mm.comp's products (weights left as they are, f32 (KIND 0), bf16+// (1) or Q8_0 (2), on float vectors) for a few vectors: a workgroup a row+// and VW vectors (the second dimension), its 64 invocations each a stride+// of the columns, the sums added up in shared memory. dense_mm's tiles of+// 64 rows by 64 vectors left a router's 129 rows on one token 3+// workgroups, its 2048 columns 128 steps of a barrier each (1.1 ms a+// layer on the laptop's Arc, 24 Oct 2026). y[v * ys + r] = scale x the dot+// product of row r with vector v (vectors cols floats apart). NOT bit for+// bit with the CPU: summed in another order.+#extension GL_EXT_control_flow_attributes : require+#extension GL_EXT_buffer_reference : require+#extension GL_EXT_buffer_reference_uvec2 : require++layout(local_size_x = 64) in;+layout(constant_id = 0) const uint NV = 8; // unused+layout(constant_id = 1) const uint SG = 8; // unused+layout(constant_id = 2) const uint KIND = 0;++const uint VW = 4u;++layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer F1 { float v[]; };+layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };+layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer FO { float v[]; };++layout(push_constant) uniform P { uvec2 w, x, y; uint rows, cols, nv, ys; float scale; } p;++shared float red[64 * VW];++// weight k of row r+float wt(uint r, uint k)+{+ uint i = r * p.cols + k;+ if (KIND == 0u)+ return F1(p.w).v[i];+ if (KIND == 2u) {+ /* the block's first byte: its scale, then its quants */+ uint b = (r * (p.cols / 32u) + k / 32u) * 34u;+ U1 W = U1(p.w);+ uint dw = W.v[b >> 2];+ float d = unpackHalf2x16((b & 2u) != 0u ? dw >> 16 : dw & 0xffffu).x;+ uint qb = b + 2u + k % 32u;+ int q = bitfieldExtract(int(W.v[qb >> 2]), int(8u * (qb & 3u)), 8);+ return d * float(q);+ }+ uint u = U1(p.w).v[i >> 1];+ return uintBitsToFloat((i & 1u) != 0u ? u & 0xffff0000u : u << 16);+}++void main()+{+ F1 X = F1(p.x);+ FO Y = FO(p.y);+ uint id = gl_LocalInvocationID.x;+ uint r = gl_WorkGroupID.x, v0 = gl_WorkGroupID.y * VW;+ uint nv = min(p.nv - v0, VW);+ float acc[VW];+ [[unroll]] for (uint j = 0; j < VW; j++) acc[j] = 0.0;+ for (uint k = id; k < p.cols; k += 64u) {+ float w = wt(r, k);+ [[unroll]] for (uint j = 0; j < VW; j++)+ if (j < nv)+ acc[j] += w * X.v[(v0 + j) * p.cols + k];+ }+ [[unroll]] for (uint j = 0; j < VW; j++) red[j * 64u + id] = acc[j];+ barrier();+ [[unroll]] for (uint s = 32u; s > 0u; s >>= 1) {+ if (id < s)+ [[unroll]] for (uint j = 0; j < VW; j++)+ red[j * 64u + id] += red[j * 64u + id + s];+ barrier();+ }+ if (id < nv)+ Y.v[(v0 + id) * p.ys + r] = red[id * 64u] * p.scale;+}diff --git a/src/llm/shaders/moe_plan.comp b/src/llm/shaders/moe_plan.compnew file mode 100644index 0000000..de723de--- /dev/null+++ b/src/llm/shaders/moe_plan.comp@@ -0,0 +1,137 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+#version 450+// A mixture's plan on the GPU (gpu_token_moe.c), when the card's tier+// holds every expert: what plan() writes on the CPU between two+// submissions, the same words in the same places. The router's choice pe+// (n tokens, k experts each) sorted by expert - each row's token gt[r] and+// each pair's row inv[p], an expert's rows in its pairs' order - the+// tables of the gate and up products and of the down ones with their tiles+// of s1 and s2 vectors, and the sizes of the two indirect dispatches. at:+// the layer's experts in the tier, 3 uvec4 each: (gate, up), (down, p0),+// (p1, 0). One workgroup: the sort, a few hundred steps, by its first+// lane, the tables by all.+#extension GL_EXT_buffer_reference : require+#extension GL_EXT_buffer_reference_uvec2 : require++layout(local_size_x = 64) in;++layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };+layout(buffer_reference, std430, buffer_reference_align = 4) buffer U1 { uint v[]; };++layout(push_constant) uniform P {+ uvec2 at, b;+ uint pe, gt, inv, t1, t2, ind, xg, egu, eh, dout; /* bytes from b */+ uint n, k, ne, nbd, nbf, ep, dm, eff, s1, s2, tiles1, tiles2;+} p;++const uint MAXE = 512u; /* experts a layer at most (gpu_token_moe.c) */+/* the experts' first rows (cnt[e], rows up to cnt[e + 1]), the experts+ chosen in order, the next row of each, the first tile of each chosen */+shared uint cnt[MAXE + 1u], ids[MAXE], fill[MAXE], tile1[MAXE + 1u],+ tile2[MAXE + 1u];+shared uint nd;++/* a + o, 64 bits as two words */+uvec2 add(uvec2 a, uint o)+{+ uint c, lo = uaddCarry(a.x, o, c);+ return uvec2(lo, a.y + c);+}++/* gpu_token_moe.c table_entry: weights, activations, outputs, vectors, ys+ floats apart, p0, p1 - word e on of T */+void entry(U1 T, uint e, uvec2 w, uvec2 x, uvec2 y, uint nv, uint ys, uvec2 p0,+ uvec2 p1)+{+ T.v[e] = w.x;+ T.v[e + 1u] = w.y;+ T.v[e + 2u] = x.x;+ T.v[e + 3u] = x.y;+ T.v[e + 4u] = y.x;+ T.v[e + 5u] = y.y;+ T.v[e + 6u] = nv;+ T.v[e + 7u] = ys;+ T.v[e + 8u] = p0.x;+ T.v[e + 9u] = p0.y;+ T.v[e + 10u] = p1.x;+ T.v[e + 11u] = p1.y;+}++void main()+{+ uint lane = gl_LocalInvocationID.x, np = p.n * p.k;+ U1 PE = U1(add(p.b, p.pe)), GT = U1(add(p.b, p.gt)),+ INV = U1(add(p.b, p.inv)), T1 = U1(add(p.b, p.t1)),+ T2 = U1(add(p.b, p.t2)), IND = U1(add(p.b, p.ind));+ U4 AT = U4(p.at);+ for (uint e = lane; e <= p.ne; e += 64u)+ cnt[e] = 0u;+ barrier();+ if (lane == 0u) {+ /* the counts, their running sums, the experts chosen in order */+ for (uint q = 0u; q < np; q++)+ cnt[min(PE.v[q], p.ne - 1u) + 1u]++;+ uint m = 0u, n1 = 0u, n2 = 0u;+ for (uint e = 0u; e < p.ne; e++) {+ uint c = cnt[e + 1u];+ if (c != 0u) {+ tile1[m] = n1;+ tile2[m] = n2;+ n1 += 2u * ((c + p.s1 - 1u) / p.s1);+ n2 += (c + p.s2 - 1u) / p.s2;+ ids[m++] = e;+ }+ cnt[e + 1u] += cnt[e];+ fill[e] = cnt[e];+ }+ nd = m;+ tile1[m] = n1;+ tile2[m] = n2;+ for (uint q = 0u; q < np; q++) {+ uint r = fill[min(PE.v[q], p.ne - 1u)]++;+ GT.v[r] = q / p.k;+ INV.v[q] = r;+ }+ /* a matvec's rows in its workgroups' y, a *_mmid's in x */+ bool r1 = p.s1 != 16u, r2 = p.s2 != 16u;+ uint g1 = (p.eff + 63u) / 64u, g2 = (p.dm + 63u) / 64u;+ IND.v[0] = r1 ? 1u : g1;+ IND.v[1] = r1 ? g1 : 1u;+ IND.v[2] = n1;+ IND.v[4] = r2 ? 1u : g2;+ IND.v[5] = r2 ? g2 : 1u;+ IND.v[6] = n2;+ }+ barrier();+ /* an expert chosen a lane: its three entries and its tiles */+ for (uint i = lane; i < nd; i += 64u) {+ uint e = ids[i], a = cnt[e], nv = cnt[e + 1u] - a;+ uvec4 w0 = AT.v[3u * e], w1 = AT.v[3u * e + 1u], w2 = AT.v[3u * e + 2u];+ uvec2 x = add(p.b, p.xg + a * p.nbd * 320u), z = uvec2(0u);+ entry(T1, 12u * (2u * i), w0.xy, x, add(p.b, p.egu + a * p.ep * 4u), nv,+ p.ep, z, z);+ entry(T1, 12u * (2u * i + 1u), w0.zw, x,+ add(p.b, p.egu + (np + a) * p.ep * 4u), nv, p.ep, z, z);+ entry(T2, 12u * i, w1.xy, add(p.b, p.eh + a * p.nbf * 320u),+ add(p.b, p.dout + a * p.dm * 4u), nv, p.dm, w1.zw, w2.xy);+ uint u = 4u * (p.tiles1 + tile1[i]);+ for (uint v = 0u; v < nv; v += p.s1, u += 8u) {+ T1.v[u] = 2u * i;+ T1.v[u + 1u] = v;+ T1.v[u + 2u] = 0u;+ T1.v[u + 3u] = 0u;+ T1.v[u + 4u] = 2u * i + 1u;+ T1.v[u + 5u] = v;+ T1.v[u + 6u] = 0u;+ T1.v[u + 7u] = 0u;+ }+ u = 4u * (p.tiles2 + tile2[i]);+ for (uint v = 0u; v < nv; v += p.s2, u += 4u) {+ T2.v[u] = i;+ T2.v[u + 1u] = v;+ T2.v[u + 2u] = 0u;+ T2.v[u + 3u] = 0u;+ }+ }+}diff --git a/src/llm/shaders/moe_topk.comp b/src/llm/shaders/moe_topk.compindex 9e59d04..2472549 100644--- a/src/llm/shaders/moe_topk.comp+++ b/src/llm/shaders/moe_topk.comp@@ -5,7 +5,10 @@// model_ffn.c route_tok: per token, the softmax of its ne router logits,// the k largest (the first of equals), each weight divided by the sum of// the k; logit ne, the shared expert's gate (if sh), through a sigmoid, 1-// without one. One invocation a token. NOT bit for bit with the CPU: the+// without one. A workgroup a token, its logits in shared memory, each+// search a reduction over the 64 invocations (one invocation a token read+// some 1300 logits one after the other: 0.5 ms a layer on a single token+// on the laptop's Arc, 24 Oct 2026). NOT bit for bit with the CPU: the// GPU's exp, the router's sums in another order.#extension GL_EXT_buffer_reference : require#extension GL_EXT_buffer_reference_uvec2 : require@@ -20,45 +23,91 @@ layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer UO// j k + i; shw: the shared expert's weight of each tokenlayout(push_constant) uniform P { uvec2 r, pe, pw, shw; uint ne, rs, k, n, sh; } p;-const uint MAXK = 16u, MAXW = 16u; // k <= 16, ne <= 512+const uint MAXK = 16u, MAXE = 512u; // k <= 16, ne <= 512+const uint NONE = 0xffffffffu;++shared float sv[MAXE];+shared float rv[64];+shared uint ri[64];+shared uint taken[MAXE / 32u];+shared uint ids[MAXK];+shared float w[MAXK];void main(){- uint j = gl_GlobalInvocationID.x;- if (j >= p.n)- return;+ uint j = gl_WorkGroupID.x, id = gl_LocalInvocationID.x;F R = F(p.r);uint base = j * p.rs;float mx = R.v[base];- for (uint e = 1u; e < p.ne; e++)- mx = max(mx, R.v[base + e]);+ for (uint e = id; e < p.ne; e += 64u) {+ float v = R.v[base + e];+ sv[e] = v;+ mx = max(mx, v);+ }+ if (id < MAXE / 32u)+ taken[id] = 0u;+ rv[id] = mx;+ barrier();+ for (uint s = 32u; s > 0u; s >>= 1) {+ if (id < s)+ rv[id] = max(rv[id], rv[id + s]);+ barrier();+ }+ mx = rv[0];+ barrier();float sum = 0.0;- for (uint e = 0u; e < p.ne; e++)- sum += exp(R.v[base + e] - mx);- uint taken[MAXW];- for (uint i = 0u; i < MAXW; i++)- taken[i] = 0u;- uint ids[MAXK];- float w[MAXK], wsum = 0.0;+ for (uint e = id; e < p.ne; e += 64u)+ sum += exp(sv[e] - mx);+ rv[id] = sum;+ barrier();+ for (uint s = 32u; s > 0u; s >>= 1) {+ if (id < s)+ rv[id] += rv[id + s];+ barrier();+ }+ sum = rv[0];+ barrier();for (uint i = 0u; i < p.k; i++) {- uint best = 0u;+ /* each invocation's largest not taken (the first of equals: e+ grows), then the largest of those, the lower index of equals */float bv = 0.0;- bool any = false;- for (uint e = 0u; e < p.ne; e++) {+ uint be = NONE;+ for (uint e = id; e < p.ne; e += 64u) {if ((taken[e / 32u] & (1u << (e % 32u))) != 0u)continue;- float v = R.v[base + e];- if (!any || v > bv) {- bv = v;- best = e;- any = true;+ if (be == NONE || sv[e] > bv) {+ bv = sv[e];+ be = e;}}- taken[best / 32u] |= 1u << (best % 32u);- ids[i] = best;- w[i] = exp(bv - mx) / sum;- wsum += w[i];+ rv[id] = bv;+ ri[id] = be;+ barrier();+ for (uint s = 32u; s > 0u; s >>= 1) {+ if (id < s) {+ uint oe = ri[id + s], me = ri[id];+ float ov = rv[id + s], mv = rv[id];+ if (oe != NONE &&+ (me == NONE || ov > mv || (ov == mv && oe < me))) {+ rv[id] = ov;+ ri[id] = oe;+ }+ }+ barrier();+ }+ if (id == 0u) {+ uint best = ri[0];+ taken[best / 32u] |= 1u << (best % 32u);+ ids[i] = best;+ w[i] = exp(rv[0] - mx) / sum;+ }+ barrier();}+ if (id != 0u)+ return;+ float wsum = 0.0;+ for (uint i = 0u; i < p.k; i++)+ wsum += w[i];UO PE = UO(p.pe);FO PW = FO(p.pw);for (uint i = 0u; i < p.k; i++) {diff --git a/src/llm/shaders/q3k_matvec.comp b/src/llm/shaders/q3k_matvec.compindex 5b2e68b..3aa531e 100644--- a/src/llm/shaders/q3k_matvec.comp+++ b/src/llm/shaders/q3k_matvec.comp@@ -22,12 +22,17 @@layout(local_size_x = 64) in;layout(constant_id = 0) const uint NV = 8;layout(constant_id = 1) const uint SG = 8;+layout(constant_id = 2) const uint ID = 0;const uint NX = (19u + SG - 1u) / SG;layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };+// With ID (constant 2) a mixture's experts, the table of *_mmid.comp+// (gpu_token_moe.c) in w: 3 uvec4 an expert, (w, x), (y, nv, ys), (p1,+// p0), then from uvec4 nv on one a workgroup's z, (expert, first vector);+// x's vectors nb blocks apart.layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys; } p;int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }@@ -36,19 +41,27 @@ uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }void main(){- U1 W = U1(p.w);- U4 X = U4(p.x);- F Y = F(p.y);+ uvec2 pw = p.w, px = p.x, py = p.y;+ uint nv = p.nv, xs = p.xs, ys = p.ys, v0 = gl_WorkGroupID.x * NV;+ if (ID != 0u) {+ U4 T = U4(p.w);+ uvec4 tl = T.v[p.nv + gl_WorkGroupID.z];+ uvec4 t0 = T.v[3u * tl.x], t1 = T.v[3u * tl.x + 1u];+ pw = t0.xy, px = t0.zw, py = t1.xy;+ nv = t1.z, xs = p.nb, ys = t1.w, v0 = tl.y;+ }+ U1 W = U1(pw);+ U4 X = U4(px);+ F Y = F(py);uint lane = gl_SubgroupInvocationID;/* workgroup y: 64 rows; x: a group of NV vectors, so that the groupsof the same rows go out together and the weights the first readsfrom memory the others find in the L3 */uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;uint row = min(grow, p.rows - 1u);- uint v0 = gl_WorkGroupID.x * NV;uint xo[NV];for (uint v = 0; v < NV; v++)- xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;+ xo[v] = min(v0 + v, nv - 1u) * xs * 20u;float acc[NV];for (uint v = 0; v < NV; v++)acc[v] = 0.0;@@ -104,6 +117,6 @@ void main()}if (grow < p.rows)for (uint v = 0; v < NV; v++)- if (v0 + v < p.nv)- Y.v[(v0 + v) * p.ys + row] = acc[v];+ if (v0 + v < nv)+ Y.v[(v0 + v) * ys + row] = acc[v];}diff --git a/src/llm/shaders/q4k_matvec.comp b/src/llm/shaders/q4k_matvec.compindex cdba94d..504e2da 100644--- a/src/llm/shaders/q4k_matvec.comp+++ b/src/llm/shaders/q4k_matvec.comp@@ -19,6 +19,7 @@layout(local_size_x = 64) in;layout(constant_id = 0) const uint NV = 8;layout(constant_id = 1) const uint SG = 8;+layout(constant_id = 2) const uint ID = 0;const uint NX = (19u + SG - 1u) / SG;layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };@@ -26,6 +27,10 @@ layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F// w: the first row; x: vector v, block b at uvec4 (v * xs + b) * 20;// y: row r of vector v at y[v * ys + r]+// With ID (constant 2) a mixture's experts, the table of *_mmid.comp+// (gpu_token_moe.c) in w: 3 uvec4 an expert, (w, x), (y, nv, ys), (p1,+// p0), then from uvec4 nv on one a workgroup's z, (expert, first vector);+// x's vectors nb blocks apart.layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys; } p;int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }@@ -34,18 +39,26 @@ uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }void main(){- U4 W = U4(p.w), X = U4(p.x);- F Y = F(p.y);+ uvec2 pw = p.w, px = p.x, py = p.y;+ uint nv = p.nv, xs = p.xs, ys = p.ys, v0 = gl_WorkGroupID.x * NV;+ if (ID != 0u) {+ U4 T = U4(p.w);+ uvec4 tl = T.v[p.nv + gl_WorkGroupID.z];+ uvec4 t0 = T.v[3u * tl.x], t1 = T.v[3u * tl.x + 1u];+ pw = t0.xy, px = t0.zw, py = t1.xy;+ nv = t1.z, xs = p.nb, ys = t1.w, v0 = tl.y;+ }+ U4 W = U4(pw), X = U4(px);+ F Y = F(py);uint lane = gl_SubgroupInvocationID;/* workgroup y: 64 rows; x: a group of NV vectors, so that the groupsof the same rows go out together and the weights the first readsfrom memory the others find in the L3 */uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;uint row = min(grow, p.rows - 1u);- uint v0 = gl_WorkGroupID.x * NV;uint xo[NV];for (uint v = 0; v < NV; v++)- xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;+ xo[v] = min(v0 + v, nv - 1u) * xs * 20u;float acc[NV], accm[NV];for (uint v = 0; v < NV; v++) { acc[v] = 0.0; accm[v] = 0.0; }uint wo = row * p.nb * 9u;@@ -89,6 +102,6 @@ void main()}if (grow < p.rows)for (uint v = 0; v < NV; v++)- if (v0 + v < p.nv)- Y.v[(v0 + v) * p.ys + row] = acc[v] - accm[v];+ if (v0 + v < nv)+ Y.v[(v0 + v) * ys + row] = acc[v] - accm[v];}diff --git a/src/llm/shaders/q6k_matvec.comp b/src/llm/shaders/q6k_matvec.compindex 9f24489..fa5fcbd 100644--- a/src/llm/shaders/q6k_matvec.comp+++ b/src/llm/shaders/q6k_matvec.comp@@ -19,12 +19,17 @@layout(local_size_x = 64) in;layout(constant_id = 0) const uint NV = 8;layout(constant_id = 1) const uint SG = 8;+layout(constant_id = 2) const uint ID = 0;const uint NX = (19u + SG - 1u) / SG;layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };+// With ID (constant 2) a mixture's experts, the table of *_mmid.comp+// (gpu_token_moe.c) in w: 3 uvec4 an expert, (w, x), (y, nv, ys), (p1,+// p0), then from uvec4 nv on one a workgroup's z, (expert, first vector);+// x's vectors nb blocks apart.layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys; } p;int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }@@ -32,18 +37,26 @@ uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }void main(){- U4 X = U4(p.x);- F Y = F(p.y);+ uvec2 pw = p.w, px = p.x, py = p.y;+ uint nv = p.nv, xs = p.xs, ys = p.ys, v0 = gl_WorkGroupID.x * NV;+ if (ID != 0u) {+ U4 T = U4(p.w);+ uvec4 tl = T.v[p.nv + gl_WorkGroupID.z];+ uvec4 t0 = T.v[3u * tl.x], t1 = T.v[3u * tl.x + 1u];+ pw = t0.xy, px = t0.zw, py = t1.xy;+ nv = t1.z, xs = p.nb, ys = t1.w, v0 = tl.y;+ }+ U4 X = U4(px);+ F Y = F(py);uint lane = gl_SubgroupInvocationID;/* workgroup y: 64 rows; x: a group of NV vectors, so that the groupsof the same rows go out together and the weights the first readsfrom memory the others find in the L3 */uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;uint row = min(grow, p.rows - 1u);- uint v0 = gl_WorkGroupID.x * NV;uint xo[NV];for (uint v = 0; v < NV; v++)- xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;+ xo[v] = min(v0 + v, nv - 1u) * xs * 20u;float acc[NV];for (uint v = 0; v < NV; v++)acc[v] = 0.0;@@ -52,8 +65,8 @@ void main()/* the block's 52 words (208 bytes) and d, realigned: the 16-byteboundary below it, s bytes before (even: 210 and the rows' startare), and 14 uvec4 from there */- uint lo = p.w.x + off;- uvec2 at = uvec2(lo & ~15u, p.w.y + (lo < p.w.x ? 1u : 0u));+ uint lo = pw.x + off;+ uvec2 at = uvec2(lo & ~15u, pw.y + (lo < pw.x ? 1u : 0u));uint s = lo & 15u;U4 B = U4(at);uint rw[57];@@ -117,6 +130,6 @@ void main()}if (grow < p.rows)for (uint v = 0; v < NV; v++)- if (v0 + v < p.nv)- Y.v[(v0 + v) * p.ys + row] = acc[v];+ if (v0 + v < nv)+ Y.v[(v0 + v) * ys + row] = acc[v];}diff --git a/src/llm/shaders/q6kp_matvec.comp b/src/llm/shaders/q6kp_matvec.compindex 79e4eb4..c9e1500 100644--- a/src/llm/shaders/q6kp_matvec.comp+++ b/src/llm/shaders/q6kp_matvec.comp@@ -22,6 +22,7 @@layout(local_size_x = 64) in;layout(constant_id = 0) const uint NV = 8;layout(constant_id = 1) const uint SG = 8;+layout(constant_id = 2) const uint ID = 0;const uint NX = (19u + SG - 1u) / SG;layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };@@ -30,6 +31,10 @@ layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F// level 3, 2 or 1: the planes present; p1 (level >= 2) and p0 (level 3) the// bare planes of the first row, nb * 64 bytes a row+// With ID (constant 2) a mixture's experts, the table of *_mmid.comp+// (gpu_token_moe.c) in w: 3 uvec4 an expert, (w, x), (y, nv, ys), (the+// bare planes p1, p0: the level they make), then from uvec4 nv on one a+// workgroup's z, (expert, first vector); x's vectors nb blocks apart.layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys, level; uvec2 p1, p0; } p;int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }@@ -37,31 +42,42 @@ uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }void main(){- U4 P1 = U4(p.p1), P0 = U4(p.p0);- U4 X = U4(p.x);- F Y = F(p.y);+ uvec2 wa = p.w, px = p.x, py = p.y, a1 = p.p1, a0 = p.p0;+ uint nv = p.nv, xs = p.xs, ys = p.ys, level = p.level,+ v0 = gl_WorkGroupID.x * NV;+ if (ID != 0u) {+ U4 T = U4(p.w);+ uvec4 tl = T.v[p.nv + gl_WorkGroupID.z];+ uvec4 t0 = T.v[3u * tl.x], t1 = T.v[3u * tl.x + 1u],+ t2 = T.v[3u * tl.x + 2u];+ wa = t0.xy, px = t0.zw, py = t1.xy, a1 = t2.xy, a0 = t2.zw;+ nv = t1.z, xs = p.nb, ys = t1.w, v0 = tl.y;+ level = a0 != uvec2(0u) ? 3u : a1 != uvec2(0u) ? 2u : 1u;+ }+ U4 P1 = U4(a1), P0 = U4(a0);+ U4 X = U4(px);+ F Y = F(py);uint lane = gl_SubgroupInvocationID;/* workgroup y: 64 rows; x: a group of NV vectors, so that the groupsof the same rows go out together and the weights the first readsfrom memory the others find in the L3 */uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;uint row = min(grow, p.rows - 1u);- uint v0 = gl_WorkGroupID.x * NV;uint xo[NV];for (uint v = 0; v < NV; v++)- xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;+ xo[v] = min(v0 + v, nv - 1u) * xs * 20u;float acc[NV];for (uint v = 0; v < NV; v++)acc[v] = 0.0;- uint shk = p.level == 3u ? 1u : p.level == 2u ? 3u : 5u;- int c = p.level == 3u ? 64 : p.level == 2u ? 61 : 49;+ uint shk = level == 3u ? 1u : level == 2u ? 3u : 5u;+ int c = level == 3u ? 64 : level == 2u ? 61 : 49;uint off = row * p.nb * 82u; /* bytes */uint pw = row * p.nb * 4u; /* uvec4 of the bare planes */for (uint b = 0; b < p.nb; b++, off += 82u, pw += 4u) {/* the block's 20 words (80 bytes) and d, realigned from the 16-byteboundary s bytes below it (s even) */- uint lo = p.w.x + off;- U4 B = U4(uvec2(lo & ~15u, p.w.y + (lo < p.w.x ? 1u : 0u)));+ uint lo = wa.x + off;+ U4 B = U4(uvec2(lo & ~15u, wa.y + (lo < wa.x ? 1u : 0u)));uint s = lo & 15u;uint rw[25];for (uint k = 0; k < 6u; k++) {@@ -85,8 +101,8 @@ void main()sc[4u * k + j] = bitfieldExtract(int(wd[16u + k]), int(8u * j), 8);uint w1[16], w0[16];for (uint k = 0; k < 4u; k++) {- uvec4 a1 = p.level >= 2u ? P1.v[pw + k] : uvec4(0u);- uvec4 a0 = p.level >= 3u ? P0.v[pw + k] : uvec4(0u);+ uvec4 a1 = level >= 2u ? P1.v[pw + k] : uvec4(0u);+ uvec4 a0 = level >= 3u ? P0.v[pw + k] : uvec4(0u);w1[4u * k] = a1.x; w1[4u * k + 1u] = a1.y; w1[4u * k + 2u] = a1.z; w1[4u * k + 3u] = a1.w;w0[4u * k] = a0.x; w0[4u * k + 1u] = a0.y; w0[4u * k + 2u] = a0.z; w0[4u * k + 3u] = a0.w;}@@ -106,9 +122,9 @@ void main()for (uint k = 0; k < 8u; k++) {uint i = 8u * n + k;uint q = (wd[i] >> hsh) & 0x03030303u;- if (p.level >= 2u)+ if (level >= 2u)q = (q << 2) | ((w1[i] >> hsh) & 0x03030303u);- if (p.level >= 3u)+ if (level >= 3u)q = (q << 2) | ((w0[i] >> hsh) & 0x03030303u);if (k < 4u)dot0 += dp(q, a[k]);@@ -132,6 +148,6 @@ void main()}if (grow < p.rows)for (uint v = 0; v < NV; v++)- if (v0 + v < p.nv)- Y.v[(v0 + v) * p.ys + row] = acc[v];+ if (v0 + v < nv)+ Y.v[(v0 + v) * ys + row] = acc[v];}diff --git a/src/llm/shaders/q8_0_matvec.comp b/src/llm/shaders/q8_0_matvec.compindex ce9e400..3638aa4 100644--- a/src/llm/shaders/q8_0_matvec.comp+++ b/src/llm/shaders/q8_0_matvec.comp@@ -56,7 +56,11 @@ void main()uint xo[NV];for (uint v = 0; v < NV; v++)xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;- float acc[NV][8], res[NV];+ /* precise: the CPU rounds every product and every sum apart; without+ it a compiler may fuse a sum with a product before it (the tail's+ accumulators start from zero in the loop, a known value: AMD's,+ issue #46) */+ precise float acc[NV][8], res[NV];for (uint v = 0; v < NV; v++) {res[v] = 0.0;for (uint k = 0; k < 8u; k++)@@ -117,7 +121,7 @@ void main()if (grow < p.rows)for (uint v = 0; v < NV; v++)if (v0 + v < p.nv) {- float r = ((acc[v][0] + acc[v][4]) + (acc[v][2] + acc[v][6])) ++ precise float r = ((acc[v][0] + acc[v][4]) + (acc[v][2] + acc[v][6])) +((acc[v][1] + acc[v][5]) + (acc[v][3] + acc[v][7]));if (p.tail != 0u)r = res[v] + r;diff --git a/tests/bench_kernels.c b/tests/bench_kernels.cindex a2d8303..edf303b 100644--- a/tests/bench_kernels.c+++ b/tests/bench_kernels.c@@ -14,6 +14,9 @@* (the memory a GPU sharing the host's reads fastest), BENCH_MEM=huge2m in* ordinary memory aligned to 2 MB and collapsed into huge pages: the CPU's* speed on each, against malloc's (13 Oct 2026).+ * BENCH_TYPES=Q4_K,Q2_0 measures the types named alone; BENCH_NV=8,16,64+ * the vectors a product takes (1,2,4,8 by default, 64 at most): where the+ * prompt's tiles (JANAS_CPU_TILES) pay against the rows' products.* Defaults: 256 MiB of weights per type, 12 threads on CPUs 0-11; the* single core is first_cpu (default 2).* The weights are random bytes below 0x7c, so that every fp16 scale is a@@ -63,11 +66,34 @@ static unsigned char *weights(size_t bytes)static const struct {const char *name;int type;-} TYPES[] = {- {"Q4_K", JANAS_Q4_K}, {"Q5_K", JANAS_Q5_K}, {"Q6_K", JANAS_Q6_K},- {"Q3_K", JANAS_Q3_K}, {"Q8_0", JANAS_Q8_0}, {"Q4_0", JANAS_Q4_0},- {"Q4_1", JANAS_Q4_1}, {"Q5_1", JANAS_Q5_1}, {"IQ4_NL", JANAS_IQ4_NL},- {"IQ4_XS", JANAS_IQ4_XS}, {"IQ3_S", JANAS_IQ3_S}, {"MXFP4", JANAS_MXFP4}};+} TYPES[] = {{"Q4_K", JANAS_Q4_K}, {"Q5_K", JANAS_Q5_K},+ {"Q6_K", JANAS_Q6_K}, {"Q3_K", JANAS_Q3_K},+ {"Q8_0", JANAS_Q8_0}, {"Q4_0", JANAS_Q4_0},+ {"Q4_1", JANAS_Q4_1}, {"Q5_1", JANAS_Q5_1},+ {"IQ4_NL", JANAS_IQ4_NL}, {"IQ4_XS", JANAS_IQ4_XS},+ {"IQ3_S", JANAS_IQ3_S}, {"MXFP4", JANAS_MXFP4},+ {"Q5_0", JANAS_Q5_0}, {"Q2_0", JANAS_Q2_0},+ {"Q2_K", JANAS_Q2_K}, {"IQ3_XXS", JANAS_IQ3_XXS},+ {"IQ2_XXS", JANAS_IQ2_XXS}, {"IQ2_XS", JANAS_IQ2_XS},+ {"IQ2_S", JANAS_IQ2_S}, {"IQ1_S", JANAS_IQ1_S},+ {"IQ1_M", JANAS_IQ1_M}};++/* BENCH_TYPES=Q4_K,Q2_0 measures those types alone (all when unset). */+static int wanted(const char *name)+{+ const char *list = getenv("BENCH_TYPES");+ if (!list || !*list)+ return 1;+ size_t n = strlen(name);+ for (const char *p = list; *p;) {+ const char *e = strchr(p, ',');+ size_t len = e ? (size_t)(e - p) : strlen(p);+ if (len == n && !strncmp(p, name, n))+ return 1;+ p += len + (e != NULL);+ }+ return 0;+}/* Seconds per product, the best of passes run for at least 0.3 s. */static double timed(struct janas_pool *pool, struct janas_matvec_task *t)@@ -116,13 +142,32 @@ int main(int argc, char **argv)fprintf(stderr, "bench_kernels: no thread pool\n");return 1;}- const size_t cols = 2048, nb = cols / JANAS_QK, nvs[] = {1, 2, 4, 8};- struct janas_block_q8k *x = calloc(8 * nb, sizeof(*x));- float *v = malloc(8 * cols * sizeof(float));+ enum { MAX_NV = 64 };+ size_t nvs[8] = {1, 2, 4, 8}, n_nv = 4;+ const char *ev = getenv("BENCH_NV");+ if (ev) {+ n_nv = 0;+ for (const char *c = ev; *c && n_nv < 8;) {+ long n = strtol(c, (char **)&c, 10);+ if (n >= 1 && n <= MAX_NV)+ nvs[n_nv++] = (size_t)n;+ while (*c == ',' || *c == ' ')+ c++;+ if (*c && (*c < '0' || *c > '9'))+ break;+ }+ if (!n_nv) {+ fprintf(stderr, "bench_kernels: BENCH_NV lists 1 to 64\n");+ return 1;+ }+ }+ const size_t cols = 2048, nb = cols / JANAS_QK;+ struct janas_block_q8k *x = calloc(MAX_NV * nb, sizeof(*x));+ float *v = malloc(MAX_NV * cols * sizeof(float));srand(1);- for (size_t i = 0; i < 8 * cols; i++)+ for (size_t i = 0; i < MAX_NV * cols; i++)v[i] = (float)rand() / RAND_MAX - 0.5f;- for (int k = 0; k < 8; k++)+ for (int k = 0; k < MAX_NV; k++)janas_q8k_quantize(v + k * cols, x + k * nb, cols);printf("GB/s of weights, %zu MiB a type, 2048 columns; 1 core (CPU %d) ""and %d threads; weights in %s\n",@@ -130,13 +175,22 @@ int main(int argc, char **argv)mem_kind == 1 ? "the GPU driver's memory": mem_kind == 2 ? "2 MB pages": "malloc's memory");- printf("%-7s %8s %8s %8s %8s | %8s %8s %8s %8s\n", "type", "1x1", "1x2",- "1x4", "1x8", "Nx1", "Nx2", "Nx4", "Nx8");+ printf("%-7s", "type");+ for (int p = 0; p < 2; p++) {+ for (size_t i = 0; i < n_nv; i++) {+ char h[16];+ snprintf(h, sizeof(h), "%sx%zu", p ? "N" : "1", nvs[i]);+ printf(" %8s", h);+ }+ printf(p ? "\n" : " |");+ }for (size_t k = 0; k < sizeof TYPES / sizeof *TYPES; k++) {+ if (!wanted(TYPES[k].name))+ continue;size_t bs = janas_qtype_block_size(TYPES[k].type);size_t rows = (mib << 20) / (nb * bs), bytes = rows * nb * bs;unsigned char *w = weights(bytes);- float *y = malloc(8 * rows * sizeof(float));+ float *y = malloc(MAX_NV * rows * sizeof(float));if (!w || !y) {fprintf(stderr, "bench_kernels: out of memory\n");return 1;@@ -148,13 +202,27 @@ int main(int argc, char **argv)if (TYPES[k].type == JANAS_MXFP4)for (size_t i = 0; i < bytes; i += sizeof(struct janas_block_mxfp4))w[i] = (unsigned char)(118 + rand() % 12);+ /* the IQ types' scale of a block, at its start (IQ1_M: the top+ nibbles of its four scale words), a normal fp16 as in real+ weights: a subnormal one sends every float product of the block+ down the slow path (IQ1_M read 0.3 GB/s on random bytes) */+ int t = TYPES[k].type;+ if (t == JANAS_IQ2_XXS || t == JANAS_IQ2_XS || t == JANAS_IQ2_S ||+ t == JANAS_IQ3_XXS || t == JANAS_IQ1_S)+ for (size_t i = 0; i < bytes; i += bs) {+ uint16_t d = (uint16_t)(0x2000 + rand() % 0x1000);+ memcpy(w + i, &d, 2);+ }+ if (t == JANAS_IQ1_M)+ for (size_t i = 0; i < bytes; i += bs)+ w[i + bs - 1] = (unsigned char)(0x20 | (w[i + bs - 1] & 0x0f));#if defined(__linux__)if (mem_kind == 2)madvise(w, bytes, MADV_COLLAPSE);#endifprintf("%-7s", TYPES[k].name);for (int p = 0; p < 2; p++) {- for (size_t i = 0; i < 4; i++) {+ for (size_t i = 0; i < n_nv; i++) {struct janas_matvec_task t = {.type = TYPES[k].type,.w = w,.x = x,diff --git a/tests/gpu_check.c b/tests/gpu_check.cindex 6244f32..e962bde 100644--- a/tests/gpu_check.c+++ b/tests/gpu_check.c@@ -1,23 +1,40 @@/* SPDX-License-Identifier: GPL-3.0-or-laterCopyright (C) 2026 Maurizio Cammalleri *//*- * gpu_check.c - products split between the GPU and the CPU must equal the- * CPU's bit for bit: random Q4_K, Q5_K, Q6_K, Q6_K_P (bit planes, at its- * three levels), Q8_0 and the blocks of 32 or 64 (Q4_0, IQ4_NL, Q4_1, Q5_1,- * MXFP4, Q5_0, Q2_0) matrices in shared memory, and the GPU's aligned copy- * of Q6_K (on the GPU alone, against the CPU's Q6_K), several shapes, vector- * counts and strides; for the blocks of 32 or 64 and Q8_0 also rows that- * end inside a group of 256 (qwen4exp's 640, its shared experts' down- * matrices Q8_0); Q2_K, IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS,- * IQ1_S and IQ1_M (Qwen3.8-Flash-Next's other files, unsloth's 1- and 2-bit- * ones). Then bf16 weights on float vectors- * (janas_gpu_bf16_mv) against the CPU's eight lanes (bf16.h), with x and y- * where the GPU shares them and through its stages (JANAS_GPU_COPY=1: the- * weights too, as on a card with memory of its own). Skips (exit 0) when- * no GPU is usable.+ * gpu_check.c - products on the GPU must equal the CPU's bit for bit, and+ * where they do not, the report says which shader is at fault: random Q4_K,+ * Q5_K, Q6_K, Q6_K_P (bit planes, at its three levels), Q8_0 and the blocks+ * of 32 or 64 (Q4_0, IQ4_NL, Q4_1, Q5_1, MXFP4, Q5_0, Q2_0) matrices, the+ * GPU's aligned copy of Q6_K (against the CPU's Q6_K), Q2_K, IQ2_XXS,+ * IQ2_XS, IQ2_S, IQ3_XXS, IQ1_S and IQ1_M (Qwen3.8-Flash-Next's other+ * files, unsloth's 1- and 2-bit ones), several shapes, vector counts and+ * strides; for the blocks of 32 or 64 and Q8_0 also rows that end inside a+ * group of 256 (qwen4exp's 640, its shared experts' down matrices Q8_0).+ * Then bf16 weights on float vectors (janas_gpu_bf16_mv) against the CPU's+ * eight lanes (bf16.h), with x and y where the GPU shares them and through+ * its stages.+ *+ * Every product runs twice: all its rows on the GPU, with the name of the+ * shader that computed them, and shared with the CPU as the engine does+ * (the GPU's share adapting over four calls). Nothing stops at the first+ * error: every wrong product is counted under its shader, with the worst+ * difference in units in the last place (a few: the driver rounds a float+ * operation differently, fused or reordered; many: wrong values). Then the+ * tool isolates the culprits by itself: each wrong product again with the+ * other shaders the engine could take for it, and with every other+ * subgroup size the device allows. The report at the end is meant to be+ * pasted whole into an issue. Skips (exit 0) when no GPU is usable.** Usage: gpu_check+ * GPU_CHECK_TYPES=Q6_K,Q8_0 checks those types only; GPU_CHECK_ENGINE=1+ * the engine's pool (12 threads on CPUs 0-11, GPU_CHECK_PIN=1 the caller+ * on CPU 0); GPU_CHECK_BIG_GB=n touches n GiB first, like the engine's+ * expert cache; GPU_CHECK_GAP_US=n CPU-only work between the shared calls.+ * JANAS_GPU_COPY=1 makes an integrated GPU work as a card with memory of+ * its own.*/+#include <math.h>+#include <stdint.h>#include <stdio.h>#include <stdlib.h>#include <string.h>@@ -29,6 +46,10 @@#include "llm/matvec.h"#include "llm/quant.h"+#ifndef JANAS_VERSION+#define JANAS_VERSION "dev"+#endif+static uint64_t rng = 88172645463325252ull;static uint32_t rnd(void)@@ -39,50 +60,86 @@ static uint32_t rnd(void)return (uint32_t)rng;}-int main(void)+enum { MAXR = 4096, MAXC = 4096, MAXV = 64 };++/* the weights, made once and imported into every GPU tried */+static uint8_t *w, *w6, *wa, *wp, *w5, *w8, *w32[7], *wq[7], *wx, *w3[2];+static size_t wbytes, w6bytes, wabytes, wpbytes, w5bytes, w8bytes, w32bytes[7],+ wqbytes[7], wxbytes, w3bytes;+static const uint8_t *pl1, *pl0;+static struct janas_block_q8k *x;+static float *ya, *yb;+static size_t ysz;+static struct janas_pool *pool;++/* blocks of 32 with four or five bits: Q4_0 and IQ4_NL (18 bytes: d, 16 of+ nibbles), Q4_1 (20: d, m, ...), Q5_1 (24: d, m, qh, ...), MXFP4 (17: the+ E8M0 byte, 16 of nibbles), Q5_0 (22: d, qh, ...), and Q2_0's blocks of 64+ (18: d, two bits a weight; 9 bytes for 32 below) */+static const int t32[7] = {JANAS_Q4_0, JANAS_IQ4_NL, JANAS_Q4_1, JANAS_Q5_1,+ JANAS_MXFP4, JANAS_Q5_0, JANAS_Q2_0};+static const size_t s32[7] = {18, 18, 20, 24, 17, 22, 9};+/* the 256-block types of iq2, iq3xxs, iq1 and q2k_matvec.comp */+static const int tq[7] = {JANAS_Q2_K, JANAS_IQ2_XXS, JANAS_IQ2_XS,+ JANAS_IQ2_S, JANAS_IQ3_XXS, JANAS_IQ1_S,+ JANAS_IQ1_M};++/* the last three: rows ending inside a group of 256, for the blocks of 32+ or 64 and Q8_0 only (640: qwen4exp's experts' down matrices) */+static const size_t shapes[][2] = {{4096, 2048}, {1000, 256}, {128, 4096},+ {2048, 512}, {4096, 4096}, {777, 1024},+ {1500, 768}, /* odd blocks per row */+ {2560, 640}, {300, 896}, {77, 128}};+static const size_t nvs[] = {1, 2, 3, 5, 8, 9, 16, 17, 33, 64};+enum {+ N_SHAPES = sizeof(shapes) / sizeof(shapes[0]),+ N_NVS = sizeof(nvs) / sizeof(nvs[0]),+ N_KINDS = 25+};+/* 0 Q4_K, 1 Q6_K, 2-4 Q6_K_P with three, two and one planes, 5 Q5_K, 6+ Q8_0, 7-10 Q4_0, IQ4_NL, Q4_1, Q5_1, 11 IQ4_XS, 12 Q3_K, 13 IQ3_S, 14 the+ aligned Q6_K, 15 MXFP4, 16 Q5_0, 17 Q2_0, 18-24 Q2_K, IQ2_XXS, IQ2_XS,+ IQ2_S, IQ3_XXS, IQ1_S, IQ1_M */+static const char *names[N_KINDS] = {+ "Q4_K", "Q6_K", "Q6_K_P/3", "Q6_K_P/2", "Q6_K_P/1",+ "Q5_K", "Q8_0", "Q4_0", "IQ4_NL", "Q4_1",+ "Q5_1", "IQ4_XS", "Q3_K", "IQ3_S", "Q6_K_A",+ "MXFP4", "Q5_0", "Q2_0", "Q2_K", "IQ2_XXS",+ "IQ2_XS", "IQ2_S", "IQ3_XXS", "IQ1_S", "IQ1_M"};++static void fill(uint8_t *p, size_t n){- /* the exact path: the fast prompt (on by default) rounds otherwise */- setenv("JANAS_GPU_FAST_PROMPT", "0", 1);- /* GPU_CHECK_BIG_GB=n: n GiB of touched memory, like the engine's- expert cache */- const char *bg = getenv("GPU_CHECK_BIG_GB");- if (bg) {- size_t big = (size_t)atoi(bg) << 30;- uint8_t *b = janas_aligned_alloc(4096, big);- if (b)- memset(b, 1, big);- }- char err[256];- struct janas_gpu *g = janas_gpu_create(err, sizeof(err));- if (!g) {- printf("gpu_check: no GPU (%s), skipped\n", err);- return 0;- }- enum { MAXR = 4096, MAXC = 4096, MAXV = 64 };- size_t wbytes = (size_t)MAXR * (MAXC / JANAS_QK) * 144;- wbytes = (wbytes + 4095) / 4096 * 4096;- uint8_t *w = janas_aligned_alloc(4096, wbytes);- for (size_t i = 0; i < wbytes; i++)- w[i] = (uint8_t)rnd();+ for (size_t i = 0; i < n; i++)+ p[i] = (uint8_t)rnd();+}++static size_t page(size_t n)+{+ return (n + 4095) / 4096 * 4096;+}++static void make_weights(void)+{+ wbytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 144);+ w = janas_aligned_alloc(4096, wbytes);+ fill(w, wbytes);struct janas_block_q4k *wb = (struct janas_block_q4k *)w;for (size_t b = 0; b < wbytes / 144; b++) {wb[b].d = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);wb[b].dmin = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);}/* Q6_K weights too, with small scales */- size_t w6bytes = (size_t)MAXR * (MAXC / JANAS_QK) * 210 + 4096;- w6bytes = (w6bytes + 4095) / 4096 * 4096;- uint8_t *w6 = janas_aligned_alloc(4096, w6bytes);- for (size_t i = 0; i < w6bytes; i++)- w6[i] = (uint8_t)rnd();+ w6bytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 210 + 4096);+ w6 = janas_aligned_alloc(4096, w6bytes);+ fill(w6, w6bytes);struct janas_block_q6k *w6b = (struct janas_block_q6k *)w6;for (size_t b = 0; b < (w6bytes - 4096) / 210; b++)w6b[b].d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);/* the same Q6_K in the GPU's aligned copy (gpu.h JANAS_Q6KA_BLOCK):ql, qh, scales, d at byte 208 */size_t nb6 = (w6bytes - 4096) / 210;- size_t wabytes = (nb6 * JANAS_Q6KA_BLOCK + 4095) / 4096 * 4096;- uint8_t *wa = janas_aligned_alloc(4096, wabytes);+ wabytes = page(nb6 * JANAS_Q6KA_BLOCK);+ wa = janas_aligned_alloc(4096, wabytes);memset(wa, 0, wabytes);for (size_t b = 0; b < nb6; b++) {uint8_t *d = wa + b * JANAS_Q6KA_BLOCK;@@ -93,79 +150,57 @@ int main(void)}/* Q6_K_P: the high planes (82-byte blocks), then the two bare planes(64 bytes a block), row after row as in an expert slot */- size_t pbase = (size_t)MAXR * (MAXC / JANAS_QK) * 82 + 4096;- pbase = (pbase + 4095) / 4096 * 4096;+ size_t pbase = page((size_t)MAXR * (MAXC / JANAS_QK) * 82 + 4096);size_t pplane = (size_t)MAXR * (MAXC / JANAS_QK) * JANAS_Q6KP_PLANE;- size_t wpbytes = pbase + 2 * pplane;- uint8_t *wp = janas_aligned_alloc(4096, wpbytes);- for (size_t i = 0; i < wpbytes; i++)- wp[i] = (uint8_t)rnd();+ wpbytes = pbase + 2 * pplane;+ wp = janas_aligned_alloc(4096, wpbytes);+ fill(wp, wpbytes);struct janas_block_q6kp *wpb = (struct janas_block_q6kp *)wp;for (size_t b = 0; b < (pbase - 4096) / 82; b++)wpb[b].d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);- const uint8_t *pl1 = wp + pbase, *pl0 = wp + pbase + pplane;+ pl1 = wp + pbase;+ pl0 = wp + pbase + pplane;/* Q5_K */- size_t w5bytes = (size_t)MAXR * (MAXC / JANAS_QK) * 176;- w5bytes = (w5bytes + 4095) / 4096 * 4096;- uint8_t *w5 = janas_aligned_alloc(4096, w5bytes);- for (size_t i = 0; i < w5bytes; i++)- w5[i] = (uint8_t)rnd();+ w5bytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 176);+ w5 = janas_aligned_alloc(4096, w5bytes);+ fill(w5, w5bytes);struct janas_block_q5k *w5b = (struct janas_block_q5k *)w5;for (size_t b = 0; b < w5bytes / 176; b++) {w5b[b].d = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);w5b[b].dmin = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);}/* Q8_0: blocks of 32, eight to 256 weights */- size_t w8bytes = (size_t)MAXR * (MAXC / JANAS_QK) * 272;- w8bytes = (w8bytes + 4095) / 4096 * 4096;- uint8_t *w8 = janas_aligned_alloc(4096, w8bytes);- for (size_t i = 0; i < w8bytes; i++)- w8[i] = (uint8_t)rnd();+ w8bytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 272);+ w8 = janas_aligned_alloc(4096, w8bytes);+ fill(w8, w8bytes);struct janas_block_q8_0 *w8b = (struct janas_block_q8_0 *)w8;for (size_t b = 0; b < w8bytes / 34; b++)w8b[b].d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);- /* blocks of 32 with four or five bits: Q4_0 and IQ4_NL (18 bytes: d,- 16 of nibbles), Q4_1 (20: d, m, ...), Q5_1 (24: d, m, qh, ...), MXFP4- (17: the E8M0 byte, 16 of nibbles), Q5_0 (22: d, qh, ...), and Q2_0's- blocks of 64 (18: d, two bits a weight; 9 bytes for 32 below); random- bytes with d (and m) set to small scales, MXFP4's byte to powers of- two from 2^-10 to 2^-3 (a subnormal one, under 2^-125, a GPU may- flush) */- static const int t32[7] = {JANAS_Q4_0, JANAS_IQ4_NL, JANAS_Q4_1,- JANAS_Q5_1, JANAS_MXFP4, JANAS_Q5_0,- JANAS_Q2_0};/* Q3_K (d at byte 108) and IQ3_S (d at 0): 110-byte blocks */- uint8_t *w3[2];- size_t w3bytes = (size_t)MAXR * (MAXC / JANAS_QK) * 110 + 4096;- w3bytes = (w3bytes + 4095) / 4096 * 4096;+ w3bytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 110 + 4096);for (int k = 0; k < 2; k++) {w3[k] = janas_aligned_alloc(4096, w3bytes);- for (size_t i = 0; i < w3bytes; i++)- w3[k][i] = (uint8_t)rnd();+ fill(w3[k], w3bytes);for (size_t b = 0; b + 110 <= w3bytes; b += 110) {uint16_t d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);memcpy(w3[k] + b + (k == 0 ? 108 : 0), &d, 2);}}/* IQ4_XS: 136-byte blocks, d, scales_h, scales_l, nibbles */- size_t wxbytes = (size_t)MAXR * (MAXC / JANAS_QK) * 136;- wxbytes = (wxbytes + 4095) / 4096 * 4096;- uint8_t *wx = janas_aligned_alloc(4096, wxbytes);- for (size_t i = 0; i < wxbytes; i++)- wx[i] = (uint8_t)rnd();+ wxbytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 136);+ wx = janas_aligned_alloc(4096, wxbytes);+ fill(wx, wxbytes);for (size_t b = 0; b + 136 <= wxbytes; b += 136) {uint16_t d = janas_fp32_to_fp16(0.00005f + (rnd() % 1000) * 1e-7f);memcpy(wx + b, &d, 2);}- static const size_t s32[7] = {18, 18, 20, 24, 17, 22, 9};- uint8_t *w32[7];- size_t w32bytes[7];+ /* the blocks of 32 or 64: random bytes with d (and m) set to small+ scales, MXFP4's byte to powers of two from 2^-10 to 2^-3 (a subnormal+ one, under 2^-125, a GPU may flush) */for (int k = 0; k < 7; k++) {- w32bytes[k] = (size_t)MAXR * (MAXC / JANAS_QK) * 8 * s32[k];- w32bytes[k] = (w32bytes[k] + 4095) / 4096 * 4096;+ w32bytes[k] = page((size_t)MAXR * (MAXC / JANAS_QK) * 8 * s32[k]);w32[k] = janas_aligned_alloc(4096, w32bytes[k]);- for (size_t i = 0; i < w32bytes[k]; i++)- w32[k][i] = (uint8_t)rnd();+ fill(w32[k], w32bytes[k]);size_t step = t32[k] == JANAS_Q2_0 ? 18 : s32[k];for (size_t b = 0; b + step <= w32bytes[k]; b += step) {if (t32[k] == JANAS_MXFP4) {@@ -181,21 +216,14 @@ int main(void)}}}- /* the 256-block types of iq2, iq3xxs, iq1 and q2k_matvec.comp: random- bytes, the scales small - d at byte 0, Q2_K's d and dmin at 80 and- 82, IQ1_M's d in the top nibbles of its four scale words (48-55) */- static const int tq[7] = {JANAS_Q2_K, JANAS_IQ2_XXS, JANAS_IQ2_XS,- JANAS_IQ2_S, JANAS_IQ3_XXS, JANAS_IQ1_S,- JANAS_IQ1_M};- uint8_t *wq[7];- size_t wqbytes[7];+ /* the 256-block types: random bytes, the scales small - d at byte 0,+ Q2_K's d and dmin at 80 and 82, IQ1_M's d in the top nibbles of its+ four scale words (48-55) */for (int k = 0; k < 7; k++) {size_t bs = janas_qtype_block_size(tq[k]);- wqbytes[k] = (size_t)MAXR * (MAXC / JANAS_QK) * bs + 4096;- wqbytes[k] = (wqbytes[k] + 4095) / 4096 * 4096;+ wqbytes[k] = page((size_t)MAXR * (MAXC / JANAS_QK) * bs + 4096);wq[k] = janas_aligned_alloc(4096, wqbytes[k]);- for (size_t i = 0; i < wqbytes[k]; i++)- wq[k][i] = (uint8_t)rnd();+ fill(wq[k], wqbytes[k]);for (size_t b = 0; b + bs <= wqbytes[k] - 4096; b += bs) {uint16_t d = janas_fp32_to_fp16(0.0005f + (rnd() % 1000) * 1e-6f);if (tq[k] == JANAS_Q2_K) {@@ -215,265 +243,633 @@ int main(void)}}}- for (int k = 0; k < 7; k++)- if (janas_gpu_import(g, wq[k], wqbytes[k]) != 0) {- printf("gpu_check: import failed\n");- return 1;- }- if (janas_gpu_import(g, w, wbytes) != 0 ||- janas_gpu_import(g, w6, w6bytes) != 0 ||- janas_gpu_import(g, wa, wabytes) != 0 ||- janas_gpu_import(g, wp, wpbytes) != 0 ||- janas_gpu_import(g, w5, w5bytes) != 0 ||- janas_gpu_import(g, w8, w8bytes) != 0 ||- janas_gpu_import(g, w32[0], w32bytes[0]) != 0 ||- janas_gpu_import(g, w32[1], w32bytes[1]) != 0 ||- janas_gpu_import(g, w32[2], w32bytes[2]) != 0 ||- janas_gpu_import(g, w32[3], w32bytes[3]) != 0 ||- janas_gpu_import(g, w32[4], w32bytes[4]) != 0 ||- janas_gpu_import(g, w32[5], w32bytes[5]) != 0 ||- janas_gpu_import(g, w32[6], w32bytes[6]) != 0 ||- janas_gpu_import(g, wx, wxbytes) != 0 ||- janas_gpu_import(g, w3[0], w3bytes) != 0 ||- janas_gpu_import(g, w3[1], w3bytes) != 0) {- printf("gpu_check: import failed\n");- return 1;- }size_t xblocks = (size_t)MAXV * 2 * (MAXC / JANAS_QK);- struct janas_block_q8k *x = malloc(xblocks * sizeof(*x));+ x = malloc(xblocks * sizeof(*x));float *xf = malloc(MAXC * sizeof(float));for (size_t v = 0; v < (size_t)MAXV * 2; v++) {for (int i = 0; i < MAXC; i++)xf[i] = (float)((int)(rnd() % 2001) - 1000) / 500.0f;janas_q8k_quantize(xf, x + v * (MAXC / JANAS_QK), MAXC);}- /* GPU_CHECK_ENGINE=1: the engine's pool (12 threads on CPUs 0-11,- the caller pinned to CPU 0), else 4 unpinned threads */+ free(xf);+ ysz = (size_t)MAXR * MAXV * 2;+ ya = malloc(ysz * sizeof(float));+ yb = malloc(ysz * sizeof(float));+}++static int import_all(struct janas_gpu *g)+{+ for (int k = 0; k < 7; k++)+ if (janas_gpu_import(g, wq[k], wqbytes[k]) != 0 ||+ janas_gpu_import(g, w32[k], w32bytes[k]) != 0)+ return -1;+ return janas_gpu_import(g, w, wbytes) != 0 ||+ janas_gpu_import(g, w6, w6bytes) != 0 ||+ janas_gpu_import(g, wa, wabytes) != 0 ||+ janas_gpu_import(g, wp, wpbytes) != 0 ||+ janas_gpu_import(g, w5, w5bytes) != 0 ||+ janas_gpu_import(g, w8, w8bytes) != 0 ||+ janas_gpu_import(g, wx, wxbytes) != 0 ||+ janas_gpu_import(g, w3[0], w3bytes) != 0 ||+ janas_gpu_import(g, w3[1], w3bytes) != 0+ ? -1+ : 0;+}++/* a product: weight kind, shape, vector count, strided or not */+struct kase {+ int q6, s, k, strided;+};++/* The task of a case, for the CPU (gpu 0) or the GPU alone (1: the+ aligned copy where the kind is Q6_K_A), and the stride of its results;+ 0 for a shape the kind does not take (a tail: blocks of 32 or 64, Q8_0+ only). */+static int make_task(const struct kase *c, int gpu, struct janas_matvec_task *t,+ size_t *ys)+{+ int q6 = c->q6;+ /* the blocks of 32 or 64: their table's index */+ int b32 = q6 >= 7 && q6 <= 10 ? q6 - 7+ : q6 == 15 ? 4+ : q6 == 16 || q6 == 17 ? q6 - 11+ : -1;+ int iq = q6 >= 18 ? q6 - 18 : -1; /* the 256-block kinds */+ size_t rows = shapes[c->s][0], cols = shapes[c->s][1];+ if (cols % JANAS_QK && b32 < 0 && q6 != 6)+ return 0;+ /* the activations of a tail row: its group whole */+ size_t nb = cols / JANAS_QK, nv = nvs[c->k],+ nbx = (cols + JANAS_QK - 1) / JANAS_QK;+ size_t skip = (size_t)c->s % 2; /* a row further in, every other shape */+ size_t rb32 = b32 >= 0 ? janas_qtype_row_bytes(t32[b32], cols) : 0;+ *t = (struct janas_matvec_task){+ .type = iq >= 0 ? tq[iq]+ : b32 >= 0 ? t32[b32]+ : q6 == 14 ? (gpu ? JANAS_Q6_K_A : JANAS_Q6_K)+ : q6 == 13 ? JANAS_IQ3_S+ : q6 == 12 ? JANAS_Q3_K+ : q6 == 11 ? JANAS_IQ4_XS+ : q6 == 6 ? JANAS_Q8_0+ : q6 == 5 ? JANAS_Q5_K+ : q6 >= 2 ? JANAS_Q6_K_P+ : q6 == 1 ? JANAS_Q6_K+ : JANAS_Q4_K,+ .w = iq >= 0 ? wq[iq] + skip * janas_qtype_block_size(tq[iq]) * nb+ : b32 >= 0 ? w32[b32] + skip * rb32+ : q6 == 14 ? (gpu ? wa + skip * JANAS_Q6KA_BLOCK * nb+ : w6 + skip * 210 * nb)+ : q6 >= 12 ? w3[q6 - 12] + skip * 110 * nb+ : q6 == 11 ? wx + skip * 136 * nb+ : q6 == 6 ? w8 + skip * janas_qtype_row_bytes(JANAS_Q8_0, cols)+ : q6 == 5 ? w5 + skip * 176 * nb+ : q6 >= 2 ? wp + skip * 82 * nb+ : q6 == 1 ? w6 + skip * 210 * nb+ : w + skip * 144 * nb,+ .plane = {q6 == 2 || q6 == 3 ? pl1 + skip * JANAS_Q6KP_PLANE * nb+ : NULL,+ q6 == 2 ? pl0 + skip * JANAS_Q6KP_PLANE * nb : NULL},+ .x = x,+ .rows = rows,+ .cols = cols,+ .n_vec = nv,+ .x_stride = c->strided ? 2 * nbx : 0,+ .y_stride = c->strided ? rows + 5 : 0};+ *ys = c->strided ? rows + 5 : rows;+ return 1;+}++/* Units in the last place between two floats (NaN: the most). */+static uint64_t ulps(float a, float b)+{+ if (isnan(a) || isnan(b))+ return UINT64_MAX;+ int32_t i, j;+ memcpy(&i, &a, 4);+ memcpy(&j, &b, 4);+ int64_t oi = i >= 0 ? i : (int64_t)INT32_MIN - i,+ oj = j >= 0 ? j : (int64_t)INT32_MIN - j;+ return (uint64_t)(oi > oj ? oi - oj : oj - oi);+}++/* rows that differ from the CPU's, of all; the worst; the first that+ differs (vector, row, the two values) */+struct diff {+ size_t bad, all;+ uint64_t ulp;+ size_t v, r;+ float a, b;+};++static void compare(size_t nv, size_t rows, size_t ys, struct diff *d)+{+ memset(d, 0, sizeof(*d));+ d->all = nv * rows;+ for (size_t v = 0; v < nv; v++)+ for (size_t r = 0; r < rows; r++) {+ float a = ya[v * ys + r], b = yb[v * ys + r];+ if (memcmp(&a, &b, 4) == 0)+ continue;+ if (!d->bad++) {+ d->v = v;+ d->r = r;+ d->a = a;+ d->b = b;+ }+ uint64_t u = ulps(a, b);+ if (u > d->ulp)+ d->ulp = u;+ }+}++static void run_cpu(const struct kase *c)+{+ struct janas_matvec_task t;+ size_t ys;+ make_task(c, 0, &t, &ys);+ memset(ya, 0, ysz * sizeof(float));+ t.y = ya;+ janas_matvec_q4k_group(pool, &t, 1);+}++/* Every row on the GPU: 0 and the shader's name, -1 where the GPU does not+ take the product (the CPU does it in the engine), -2 where the call+ failed. */+static int run_gpu(struct janas_gpu *g, const struct kase *c, struct diff *d,+ char *shader, size_t len)+{+ struct janas_matvec_task t;+ size_t ys;+ make_task(c, 1, &t, &ys);+ memset(yb, 0, ysz * sizeof(float));+ t.y = yb;+ size_t rows = t.rows;+ int waited;+ shader[0] = 0;+ if (janas_gpu_begin(g, &t, &rows, 1) != 0)+ return -1;+ if (janas_gpu_finish(g, &waited) != 0)+ return -2;+ janas_gpu_last_shader(g, shader, len);+ compare(t.n_vec, t.rows, ys, d);+ return 0;+}++/* As the engine does: the rows shared between GPU and CPU, the share+ adapting over several calls; results must not change. */+static void run_shared(struct janas_gpu *g, const struct kase *c,+ struct diff *d)+{+ struct janas_matvec_task t;+ size_t ys;+ make_task(c, 0, &t, &ys);+ memset(yb, 0, ysz * sizeof(float));+ t.y = yb;+ const char *gap = getenv("GPU_CHECK_GAP_US");+ for (int n = 0; n < 4; n++) {+ janas_gpu_matvec_group(g, pool, &t, 1);+ /* CPU-only work between calls, as in the engine */+ if (gap) {+ struct timespec a, b;+ clock_gettime(CLOCK_MONOTONIC, &a);+ do+ clock_gettime(CLOCK_MONOTONIC, &b);+ while ((b.tv_sec - a.tv_sec) * 1000000 ++ (b.tv_nsec - a.tv_nsec) / 1000 <+ atoi(gap));+ }+ }+ compare(t.n_vec, t.rows, ys, d);+}++/* the products of a kind and a shader (or "shared with the CPU": right on+ the GPU alone, wrong when shared), with up to MAX_EX wrong ones kept */+enum { MAX_GROUPS = 512, MAX_EX = 3 };+struct group {+ int q6;+ char shader[96];+ int runs, wrong;+ size_t rows_bad, rows_all;+ uint64_t ulp;+ int nex;+ struct kase ex[MAX_EX];+ struct diff dex[MAX_EX];+};+static struct group groups[MAX_GROUPS];+static int n_groups;++static struct group *group_of(int q6, const char *shader)+{+ for (int i = 0; i < n_groups; i++)+ if (groups[i].q6 == q6 && strcmp(groups[i].shader, shader) == 0)+ return &groups[i];+ if (n_groups == MAX_GROUPS)+ return &groups[MAX_GROUPS - 1];+ struct group *gr = &groups[n_groups++];+ gr->q6 = q6;+ snprintf(gr->shader, sizeof(gr->shader), "%s", shader);+ return gr;+}++static void count(struct group *gr, const struct kase *c, const struct diff *d)+{+ gr->runs++;+ gr->rows_all += d->all;+ if (!d->bad)+ return;+ gr->wrong++;+ gr->rows_bad += d->bad;+ if (d->ulp > gr->ulp)+ gr->ulp = d->ulp;+ if (gr->nex < MAX_EX) {+ gr->ex[gr->nex] = *c;+ gr->dex[gr->nex++] = *d;+ }+}++static void print_case(const struct kase *c)+{+ printf("%zux%zu nv %zu%s", shapes[c->s][0], shapes[c->s][1], nvs[c->k],+ c->strided ? " strided" : "");+}++static void print_ulp(uint64_t u)+{+ if (u == UINT64_MAX)+ printf("NaN");+ else+ printf("%llu ulp", (unsigned long long)u);+}++/* a few units in the last place: rounding, not a wrong computation */+#define ROUNDING_ULP 64++static void print_diff(const struct diff *d)+{+ uint32_t ua, ub;+ memcpy(&ua, &d->a, 4);+ memcpy(&ub, &d->b, 4);+ printf("vector %zu row %zu: CPU %.9g (%08x) GPU %.9g (%08x), ", d->v, d->r,+ d->a, ua, d->b, ub);+ print_ulp(ulps(d->a, d->b));+}++static const char *verdict(uint64_t u)+{+ return u <= ROUNDING_ULP+ ? "the last bits only: the driver rounds a float operation "+ "differently (fused, reordered or flushed)"+ : "wrong values: a shader or a driver bug";+}++/* Each wrong product again under one setting of the paths: what ran and+ whether it was right. */+static void try_paths(struct janas_gpu *g, const struct group *gr,+ const char *what, int row, int mvd)+{+ int was_row = janas_gpu_set_path(g, JANAS_GPU_PATH_ROW, row),+ was_mvd = janas_gpu_set_path(g, JANAS_GPU_PATH_MVD, mvd);+ for (int e = 0; e < gr->nex; e++) {+ struct diff d;+ char sh[96];+ run_cpu(&gr->ex[e]);+ int rc = run_gpu(g, &gr->ex[e], &d, sh, sizeof(sh));+ if (rc == 0 && strcmp(sh, gr->shader) == 0)+ continue; /* the setting changes nothing here */+ printf(" %s, ", what);+ print_case(&gr->ex[e]);+ if (rc != 0)+ printf(": %s\n",+ rc == -1 ? "the GPU does not take it" : "the call failed");+ else if (!d.bad)+ printf(": %s right\n", sh);+ else {+ printf(": %s wrong, %zu of %zu rows, worst ", sh, d.bad, d.all);+ print_ulp(d.ulp);+ printf("\n");+ }+ }+ janas_gpu_set_path(g, JANAS_GPU_PATH_ROW, was_row);+ janas_gpu_set_path(g, JANAS_GPU_PATH_MVD, was_mvd);+}++static int wanted(int q6)+{+ const char *f = getenv("GPU_CHECK_TYPES");+ if (!f || !*f)+ return 1;+ size_t n = strlen(names[q6]);+ for (const char *p = f; (p = strstr(p, names[q6])) != NULL; p += n)+ if ((p == f || p[-1] == ',') && (p[n] == 0 || p[n] == ','))+ return 1;+ return 0;+}++/* bf16 products: Qwen3.8-Flash-Next's mixes (10240 wide, 320 wide), a tail+ of three past the last eight, a row shorter than eight; single vectors+ too (as on a card with memory of its own). The count of wrong shapes;+ *runs and *refused set. */+static int check_bf16(struct janas_gpu *g, int *runs, int *refused,+ uint16_t **w16p, float **bxp, float **byp)+{+ *runs = *refused = 0;+ if (!janas_bf16_lanes())+ return 0;+ static const size_t bsh[][2] = {{320, 10240}, {10240, 320}, {2560, 2560},+ {512, 2560}, {100, 1003}, {37, 13}};+ static const size_t bnv[] = {1, 2, 3, 4, 5, 8, 17, 63};+ enum { BW = 2560 * 2560, BX = 63 * 10240 };+ size_t xbytes = page((size_t)BX * sizeof(float));+ uint16_t *w16 = janas_aligned_alloc(4096, (size_t)BW * 2);+ float *bx = janas_aligned_alloc(4096, xbytes),+ *by = janas_aligned_alloc(4096, xbytes);+ *w16p = w16;+ *bxp = bx;+ *byp = by;+ float *bxs = malloc(xbytes), *bys = malloc(xbytes), *bref = malloc(xbytes);+ for (size_t i = 0; i < BW; i++) {+ float f = (float)((int)(rnd() % 20001) - 10000) * 1e-5f;+ uint32_t u;+ memcpy(&u, &f, 4);+ w16[i] = (uint16_t)(u >> 16);+ }+ for (size_t i = 0; i < BX; i++)+ bx[i] = bxs[i] = (float)((int)(rnd() % 2001) - 1000) / 500.0f;+ janas_gpu_set_min_n(g, 1); /* single vectors too */+ int bad = 0;+ if (janas_gpu_import(g, w16, (size_t)BW * 2) != 0 ||+ janas_gpu_import(g, bx, xbytes) != 0 ||+ janas_gpu_import(g, by, xbytes) != 0) {+ printf(" bf16: the import failed\n");+ bad++;+ }+ for (size_t s = 0; !bad && s < sizeof(bsh) / sizeof(*bsh); s++)+ for (size_t k = 0; k < sizeof(bnv) / sizeof(*bnv); k++)+ for (int staged = 0; staged < 2; staged++) {+ size_t rows = bsh[s][0], cols = bsh[s][1], nv = bnv[k];+ float scale = staged ? 0.25f : 1.0f;+ const float *xin = staged ? bxs : bx;+ float *yout = staged ? bys : by;+ for (size_t v = 0; v < nv; v++)+ for (size_t r = 0; r < rows; r++)+ bref[v * rows + r] =+ janas_bf16_dot(w16 + r * cols, xin + v * cols,+ (uint32_t)cols) *+ scale;+ memset(yout, 0, nv * rows * sizeof(float));+ if (janas_gpu_bf16_mv(g, w16, xin, yout, rows, cols, nv,+ scale) != 0) {+ (*refused)++;+ continue;+ }+ (*runs)++;+ size_t nbad = 0, first = 0;+ uint64_t worst = 0;+ for (size_t i = 0; i < nv * rows; i++)+ if (memcmp(&bref[i], &yout[i], 4) != 0) {+ if (!nbad++)+ first = i;+ uint64_t u = ulps(bref[i], yout[i]);+ worst = u > worst ? u : worst;+ }+ if (nbad) {+ printf(" bf16 %zux%zu nv %zu %s: %zu of %zu wrong, "+ "worst ",+ rows, cols, nv,+ staged ? "through the stages" : "in place", nbad,+ nv * rows);+ print_ulp(worst);+ printf(", e.g. %zu: CPU %.9g GPU %.9g\n", first,+ bref[first], yout[first]);+ bad++;+ }+ }+ free(bxs);+ free(bys);+ free(bref);+ return bad;+}++int main(void)+{+ /* the exact path: the fast prompt (on by default) rounds otherwise */+ setenv("JANAS_GPU_FAST_PROMPT", "0", 1);+ /* every shader as it is: the engine's check on opening would leave out+ the ones this driver gets wrong */+ setenv("JANAS_GPU_SELFCHECK", "0", 1);+ const char *bg = getenv("GPU_CHECK_BIG_GB");+ if (bg) {+ size_t big = (size_t)atoi(bg) << 30;+ uint8_t *b = janas_aligned_alloc(4096, big);+ if (b)+ memset(b, 1, big);+ }+ printf("gpu_check %s\n", JANAS_VERSION);+ const char *off = getenv("JANAS_GPU");+ if (off && strcmp(off, "0") == 0)+ printf("(JANAS_GPU=0 turns the GPU off for the engine; gpu_check "+ "opens it anyway, to check it)\n");+ char err[256];+ struct janas_gpu *g = janas_gpu_create(err, sizeof(err));+ if (!g) {+ printf("gpu_check: no GPU (%s), skipped\n", err);+ return 0;+ }+ printf("device: %s\n", janas_gpu_about(g));+ printf("weights in the %s\n", janas_gpu_shares_host(g)+ ? "host's memory, read in place"+ : "card's own memory (a copy)");+ make_weights();+ if (import_all(g) != 0) {+ printf("gpu_check: import failed\n");+ return 1;+ }int cpus[12] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};int engine = getenv("GPU_CHECK_ENGINE") != NULL;- struct janas_pool *pool =- engine ? janas_pool_create(12, cpus) : janas_pool_create(4, NULL);+ pool = engine ? janas_pool_create(12, cpus) : janas_pool_create(4, NULL);if (engine && getenv("GPU_CHECK_PIN"))janas_pin_current_thread(0);- size_t ysz = (size_t)MAXR * MAXV * 2;- float *ya = malloc(ysz * sizeof(float)), *yb = malloc(ysz * sizeof(float));- /* the last three: rows ending inside a group of 256, for the blocks- of 32 or 64 and Q8_0 only (640: qwen4exp's experts' down- matrices) */- const size_t shapes[][2] = {{4096, 2048}, {1000, 256}, {128, 4096},- {2048, 512}, {4096, 4096}, {777, 1024},- {1500, 768}, /* odd blocks per row */- {2560, 640}, {300, 896}, {77, 128}};- const size_t nvs[] = {1, 2, 3, 5, 8, 9, 16, 17, 33, 64};- int bad = 0, runs = 0;- /* 0 Q4_K, 1 Q6_K, 2-4 Q6_K_P with three, two and one planes, 5 Q5_K,- 6 Q8_0, 7-10 Q4_0, IQ4_NL, Q4_1, Q5_1, 11 IQ4_XS,- 12 Q3_K, 13 IQ3_S, 14 the aligned Q6_K, 15 MXFP4, 16 Q5_0, 17 Q2_0,- 18-24 Q2_K, IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ1_S, IQ1_M */- static const char *names[] = {- "Q4_K", "Q6_K", "Q6_K_P/3", "Q6_K_P/2", "Q6_K_P/1",- "Q5_K", "Q8_0", "Q4_0", "IQ4_NL", "Q4_1",- "Q5_1", "IQ4_XS", "Q3_K", "IQ3_S", "Q6_K_A",- "MXFP4", "Q5_0", "Q2_0", "Q2_K", "IQ2_XXS",- "IQ2_XS", "IQ2_S", "IQ3_XXS", "IQ1_S", "IQ1_M"};- for (int q6 = 0; q6 < 25; q6++)- for (size_t s = 0; s < sizeof(shapes) / sizeof(shapes[0]); s++)- for (size_t k = 0; k < sizeof(nvs) / sizeof(nvs[0]); k++)++ /* every product, all its rows on the GPU, then shared with the CPU */+ printf("\nproducts (each all on the GPU, then shared with the CPU):\n");+ int runs = 0, wrong = 0, refused = 0, failed = 0;+ for (int q6 = 0; q6 < N_KINDS; q6++) {+ if (!wanted(q6))+ continue;+ int k_runs = 0, k_wrong = 0, k_refused = 0, k_shared = 0;+ char used[512] = "";+ for (int s = 0; s < N_SHAPES; s++)+ for (int k = 0; k < N_NVS; k++)for (int strided = 0; strided < 2; strided++) {- /* the blocks of 32 or 64: their table's index */- int b32 = q6 >= 7 && q6 <= 10 ? q6 - 7- : q6 == 15 ? 4- : q6 == 16 || q6 == 17 ? q6 - 11- : -1;- int iq = q6 >= 18 ? q6 - 18 : -1; /* the 256-block kinds */- size_t rows = shapes[s][0], cols = shapes[s][1];- if (cols % JANAS_QK && b32 < 0 && q6 != 6)- continue; /* a tail: blocks of 32 or 64, Q8_0 */- /* the activations of a tail row: its group whole */- size_t nb = cols / JANAS_QK, nv = nvs[k],- nbx = (cols + JANAS_QK - 1) / JANAS_QK;- size_t skip = s % 2; /* a row further in, every other- shape */- size_t rb32 =- b32 >= 0 ? janas_qtype_row_bytes(t32[b32], cols) : 0;- struct janas_matvec_task t = {- .type = iq >= 0 ? tq[iq]- : b32 >= 0 ? t32[b32]- : q6 == 14 ? JANAS_Q6_K- : q6 == 13 ? JANAS_IQ3_S- : q6 == 12 ? JANAS_Q3_K- : q6 == 11 ? JANAS_IQ4_XS- : q6 == 6 ? JANAS_Q8_0- : q6 == 5 ? JANAS_Q5_K- : q6 >= 2 ? JANAS_Q6_K_P- : q6 == 1 ? JANAS_Q6_K- : JANAS_Q4_K,- .w = iq >= 0- ? wq[iq] + skip *- janas_qtype_block_size(tq[iq]) *- nb- : b32 >= 0 ? w32[b32] + skip * rb32- : q6 == 14 ? w6 + skip * 210 * nb- : q6 >= 12 ? w3[q6 - 12] + skip * 110 * nb- : q6 == 11 ? wx + skip * 136 * nb- : q6 == 6 ? w8 + skip * janas_qtype_row_bytes(- JANAS_Q8_0, cols)- : q6 == 5 ? w5 + skip * 176 * nb- : q6 >= 2 ? wp + skip * 82 * nb- : q6 == 1 ? w6 + skip * 210 * nb- : w + skip * 144 * nb,- .plane = {q6 == 2 || q6 == 3- ? pl1 + skip * JANAS_Q6KP_PLANE * nb- : NULL,- q6 == 2 ? pl0 + skip * JANAS_Q6KP_PLANE * nb- : NULL},- .x = x,- .rows = rows,- .cols = cols,- .n_vec = nv,- .x_stride = strided ? 2 * nbx : 0,- .y_stride = strided ? rows + 5 : 0};- size_t ys = strided ? rows + 5 : rows;- memset(ya, 0, ysz * sizeof(float));- memset(yb, 0, ysz * sizeof(float));- t.y = ya;- janas_matvec_q4k_group(pool, &t, 1);- t.y = yb;- if (q6 == 14) {- /* the CPU has no aligned copy: all on the GPU */- t.type = JANAS_Q6_K_A;- t.w = wa + skip * JANAS_Q6KA_BLOCK * nb;- }- if (getenv("GPU_ONLY") || q6 == 14) {- /* every row on the GPU, nothing on the CPU */- int waited;- if (janas_gpu_begin(g, &t, &rows, 1) != 0 ||- janas_gpu_finish(g, &waited) != 0)- printf("gpu call failed\n");+ struct kase c = {q6, s, k, strided};+ struct janas_matvec_task t;+ size_t ys;+ if (!make_task(&c, 0, &t, &ys))+ continue;+ run_cpu(&c);+ struct diff d;+ char sh[96];+ int rc = run_gpu(g, &c, &d, sh, sizeof(sh));+ k_runs++;+ if (rc == -1) {+ k_refused++;+ } else if (rc == -2) {+ failed++;+ printf(" %s ", names[q6]);+ print_case(&c);+ printf(": the GPU's call failed\n");} else {- /* several calls: the share adapts, results must not */- for (int c = 0; c < 4; c++) {- janas_gpu_matvec_group(g, pool, &t, 1);- /* GPU_CHECK_GAP_US: CPU-only work between calls,- as in the engine */- const char *gap = getenv("GPU_CHECK_GAP_US");- if (gap) {- struct timespec a, b;- clock_gettime(CLOCK_MONOTONIC, &a);- do- clock_gettime(CLOCK_MONOTONIC, &b);- while ((b.tv_sec - a.tv_sec) * 1000000 +- (b.tv_nsec - a.tv_nsec) / 1000 <- atoi(gap));- }+ struct group *gr = group_of(q6, sh);+ count(gr, &c, &d);+ k_wrong += d.bad != 0;+ if (!strstr(used, sh) &&+ strlen(used) + strlen(sh) + 3 < sizeof(used)) {+ if (used[0])+ strcat(used, ", ");+ strcat(used, sh);}}- runs++;- for (size_t v = 0; v < nv && bad < 10; v++)- for (size_t r = 0; r < rows; r++)- if (memcmp(&ya[v * ys + r], &yb[v * ys + r], 4)) {- size_t nbad = 0;- for (size_t q = 0; q < rows; q++)- nbad += memcmp(&ya[v * ys + q],- &yb[v * ys + q], 4) != 0;- printf("[%zu wrong rows] ", nbad);- printf(- "%s %zux%zu nv %zu strided %d: v %zu row %zu: "- "%g vs %g\n",- names[q6], rows, cols, nv, strided, v, r,- ya[v * ys + r], yb[v * ys + r]);- for (size_t u = 0;- getenv("GPU_CHECK_DEBUG") && u < 2 * MAXV;- u++)- if (janas_q4k_dot(- (const struct janas_block_q4k *)- t.w +- r * nb,- x + u * nb, nb) == yb[v * ys + r])- printf(" = row %zu times vector %zu\n",- r, u);- bad++;- break;- }+ /* the aligned copy has no CPU counterpart to share+ with; a product the GPU got wrong alone is not+ counted twice */+ if (q6 == 14 || (rc == 0 && d.bad))+ continue;+ run_shared(g, &c, &d);+ if (d.bad) {+ count(group_of(q6, "shared with the CPU"), &c, &d);+ k_shared++;+ }}- /* bf16 products: Qwen3.8-Flash-Next's mixes (10240 wide, 320 wide), a- tail of three past the last eight, a row shorter than eight; single- vectors too (as on a card with memory of its own) */- int b16_runs = 0, b16_refused = 0;- /* freed after the GPU, which imported them */+ runs += k_runs;+ wrong += k_wrong + k_shared;+ refused += k_refused;+ printf(" %-9s %-6s %3d products", names[q6],+ k_wrong || k_shared ? "WRONG" : "ok", k_runs);+ if (k_wrong)+ printf(", %d wrong on the GPU", k_wrong);+ if (k_shared)+ printf(", %d wrong shared with the CPU", k_shared);+ if (k_refused)+ printf(", %d left to the CPU", k_refused);+ printf(" [%s]\n", used[0] ? used : "no GPU shader");+ fflush(stdout);+ }++ /* the bf16 products */+ int b16_runs, b16_refused;uint16_t *w16 = NULL;float *bx = NULL, *by = NULL;- if (janas_bf16_lanes()) {- static const size_t bsh[][2] = {{320, 10240}, {10240, 320},- {2560, 2560}, {512, 2560},- {100, 1003}, {37, 13}};- static const size_t bnv[] = {1, 2, 3, 4, 5, 8, 17, 63};- enum { BW = 2560 * 2560, BX = 63 * 10240 };- size_t xbytes = ((size_t)BX * sizeof(float) + 4095) / 4096 * 4096;- w16 = janas_aligned_alloc(4096, (size_t)BW * 2);- bx = janas_aligned_alloc(4096, xbytes);- by = janas_aligned_alloc(4096, xbytes);- float *bxs = malloc(xbytes), *bys = malloc(xbytes),- *bref = malloc(xbytes);- for (size_t i = 0; i < BW; i++) {- float f = (float)((int)(rnd() % 20001) - 10000) * 1e-5f;- uint32_t u;- memcpy(&u, &f, 4);- w16[i] = (uint16_t)(u >> 16);+ int b16_bad = check_bf16(g, &b16_runs, &b16_refused, &w16, &bx, &by);+ printf(" %-9s %-6s %3d products", "bf16",+ b16_bad || b16_refused ? "WRONG" : "ok", b16_runs);+ if (b16_bad)+ printf(", %d wrong", b16_bad);+ if (b16_refused)+ printf(", %d refused", b16_refused);+ printf("\n");++ /* the wrong ones, by shader */+ int culprits = 0;+ for (int i = 0; i < n_groups; i++) {+ struct group *gr = &groups[i];+ if (!gr->wrong)+ continue;+ if (!culprits++)+ printf("\nwrong products, by shader:\n");+ printf(" %s on %s: %d of %d products wrong, %zu of %zu rows, "+ "worst ",+ gr->shader, names[gr->q6], gr->wrong, gr->runs, gr->rows_bad,+ gr->rows_all);+ print_ulp(gr->ulp);+ printf(" - %s\n", verdict(gr->ulp));+ for (int e = 0; e < gr->nex; e++) {+ printf(" e.g. ");+ print_case(&gr->ex[e]);+ printf(": %zu of %zu rows, ", gr->dex[e].bad, gr->dex[e].all);+ print_diff(&gr->dex[e]);+ printf("\n");}- for (size_t i = 0; i < BX; i++)- bx[i] = bxs[i] = (float)((int)(rnd() % 2001) - 1000) / 500.0f;- janas_gpu_set_min_n(g, 1); /* single vectors too */- if (janas_gpu_import(g, w16, (size_t)BW * 2) != 0 ||- janas_gpu_import(g, bx, xbytes) != 0 ||- janas_gpu_import(g, by, xbytes) != 0) {- printf("gpu_check: bf16 import failed\n");- bad++;+ }++ /* isolating them: the other paths the engine could take */+ if (culprits) {+ printf("\nthe same products by other ways:\n");+ for (int i = 0; i < n_groups; i++) {+ struct group *gr = &groups[i];+ if (!gr->wrong || strcmp(gr->shader, "shared with the CPU") == 0)+ continue;+ printf(" %s on %s:\n", gr->shader, names[gr->q6]);+ try_paths(g, gr, "without the subgroup-per-row shaders", 0, 1);+ try_paths(g, gr, "without the lanes-on-vectors shaders", 1, 0);+ try_paths(g, gr, "without both", 0, 0);}- for (size_t s = 0; !bad && s < sizeof(bsh) / sizeof(*bsh); s++)- for (size_t k = 0; k < sizeof(bnv) / sizeof(*bnv); k++)- for (int staged = 0; staged < 2; staged++) {- size_t rows = bsh[s][0], cols = bsh[s][1], nv = bnv[k];- float scale = staged ? 0.25f : 1.0f;- const float *xin = staged ? bxs : bx;- float *yout = staged ? bys : by;- for (size_t v = 0; v < nv; v++)- for (size_t r = 0; r < rows; r++)- bref[v * rows + r] =- janas_bf16_dot(w16 + r * cols, xin + v * cols,- (uint32_t)cols) *- scale;- memset(yout, 0, nv * rows * sizeof(float));- if (janas_gpu_bf16_mv(g, w16, xin, yout, rows, cols, nv,- scale) != 0) {- b16_refused++;- continue;- }- b16_runs++;- size_t nbad = 0;- for (size_t i = 0; i < nv * rows; i++)- nbad += memcmp(&bref[i], &yout[i], 4) != 0;- if (nbad && bad < 10) {- printf("[%zu wrong of %zu] bf16 %zux%zu nv %zu %s: "- "%g vs %g\n",- nbad, nv * rows, rows, cols, nv,- staged ? "through the stages" : "in place",- bref[0], yout[0]);- bad++;- }- }- free(bxs);- free(bys);- free(bref);}- printf("gpu_check (%s, subgroups of %u): %d shapes, bf16 %d (%d "- "refused), %s\n",- janas_gpu_name(g), janas_gpu_subgroup(g), runs, b16_runs,- b16_refused, bad || b16_refused ? "FAILED" : "ok");+ unsigned sg = janas_gpu_subgroup(g), sg_min, sg_max;+ janas_gpu_subgroup_range(g, &sg_min, &sg_max);+ char name[256];+ snprintf(name, sizeof(name), "%s", janas_gpu_name(g));janas_gpu_destroy(g);janas_aligned_free(w16);janas_aligned_free(bx);janas_aligned_free(by);++ /* ... and with every other subgroup size the device allows */+ if (culprits && sg_min < sg_max) {+ printf("\nthe same products with other subgroup sizes:\n");+ for (unsigned z = sg_min; z <= sg_max; z *= 2) {+ if (z == sg)+ continue;+ char zs[16];+ snprintf(zs, sizeof(zs), "%u", z);+ setenv("JANAS_GPU_SG", zs, 1);+ struct janas_gpu *g2 = janas_gpu_create(err, sizeof(err));+ if (!g2 || import_all(g2) != 0) {+ printf(" subgroups of %u: the GPU did not open (%s)\n", z,+ g2 ? "import failed" : err);+ if (g2)+ janas_gpu_destroy(g2);+ continue;+ }+ for (int i = 0; i < n_groups; i++) {+ struct group *gr = &groups[i];+ if (!gr->wrong ||+ strcmp(gr->shader, "shared with the CPU") == 0)+ continue;+ for (int e = 0; e < gr->nex; e++) {+ struct diff d;+ char sh[96];+ run_cpu(&gr->ex[e]);+ int rc = run_gpu(g2, &gr->ex[e], &d, sh, sizeof(sh));+ printf(" subgroups of %u, %s ", z, names[gr->q6]);+ print_case(&gr->ex[e]);+ if (rc != 0)+ printf(": %s\n", rc == -1 ? "the GPU does not take it"+ : "the call failed");+ else if (!d.bad)+ printf(": %s right\n", sh);+ else {+ printf(": %s wrong, %zu of %zu rows, worst ", sh, d.bad,+ d.all);+ print_ulp(d.ulp);+ printf("\n");+ }+ }+ }+ janas_gpu_destroy(g2);+ }+ }++ int bad = wrong || failed || b16_bad || b16_refused;+ printf("\nsummary: %d products, %d wrong, %d left to the CPU, %d calls "+ "failed; bf16 %d, %d wrong, %d refused; %d shader%s at fault\n",+ runs, wrong, refused, failed, b16_runs, b16_bad, b16_refused,+ culprits, culprits == 1 ? "" : "s");+ if (bad)+ printf("Please paste everything above into an issue at "+ "https://github.com/prabanta-dev/janas/issues\n");+ printf("gpu_check (%s, subgroups of %u): %d shapes, bf16 %d (%d "+ "refused), %s\n",+ name, sg, runs, b16_runs, b16_refused, bad ? "FAILED" : "ok");janas_pool_destroy(pool);janas_aligned_free(w);janas_aligned_free(w6);@@ -481,16 +877,15 @@ int main(void)janas_aligned_free(wp);janas_aligned_free(w5);janas_aligned_free(w8);- for (int k = 0; k < 7; k++)+ for (int k = 0; k < 7; k++) {janas_aligned_free(w32[k]);- for (int k = 0; k < 7; k++)janas_aligned_free(wq[k]);+ }janas_aligned_free(wx);janas_aligned_free(w3[0]);janas_aligned_free(w3[1]);free(x);- free(xf);free(ya);free(yb);- return bad || b16_refused ? 1 : 0;+ return bad ? 1 : 0;}diff --git a/tests/llm_gen.c b/tests/llm_gen.cindex d152730..cbf0033 100644--- a/tests/llm_gen.c+++ b/tests/llm_gen.c@@ -12,6 +12,7 @@* With mtp= the drafts come from the model's MTP block instead of lookup.* With cpus=<list> (e.g. 0-11 or 18,19) the compute threads run on those* CPUs, one each (to run it beside another process on other cores).+ * LLM_GEN_DUMP=<path> writes the first run's tokens there (int32).*/#include <stdio.h>#include <stdlib.h>@@ -240,6 +241,16 @@ int main(int argc, char **argv)fprintf(stderr, "generation failed\n");return 1;}+ /* LLM_GEN_DUMP=<path>: the first run's tokens there (int32), to read+ or compare against another run's */+ const char *dump = getenv("LLM_GEN_DUMP");+ if (dump) {+ FILE *f = fopen(dump, "wb");+ if (!f || fwrite(a, sizeof(int32_t), (size_t)na, f) != (size_t)na)+ fprintf(stderr, "%s: not written\n", dump);+ if (f)+ fclose(f);+ }printf("first tokens:");for (int i = 0; i < 12 && i < na; i++)printf(" %d", a[i]);diff --git a/tests/test_quant.c b/tests/test_quant.cindex da5468e..4d512c8 100644--- a/tests/test_quant.c+++ b/tests/test_quant.c@@ -254,6 +254,70 @@ static void test_q6k_planes(void)free(pf);}+/*+ * Q6_K and Q6_K_P as tiles of eight rows (janas_q6k_tile_rows): 11 rows, 9+ * blocks (past the eight transposed at a time), 1 to 65 vectors (past the+ * 64 tokens at a time), Q6_K_P at every level; each result janas_q6k_dot's+ * or janas_q6kp_dot's to the bit, and the rows done a multiple of eight.+ */+static void test_q6k_tiles(void)+{+ enum { ROWS = 11, NB = 9, NV = 65, N = NB * JANAS_QK };+ size_t pb = NB * JANAS_Q6KP_PLANE;+ struct janas_block_q6k *w = malloc(ROWS * NB * sizeof(*w));+ struct janas_block_q6kp *base = malloc(ROWS * NB * sizeof(*base));+ uint8_t *p1 = malloc(ROWS * pb), *p0 = malloc(ROWS * pb);+ struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));+ float *xf = malloc(N * sizeof(float));+ float *out = malloc(ROWS * NV * sizeof(float));+ unsigned char *p = (unsigned char *)w;+ for (size_t i = 0; i < ROWS * NB * sizeof(*w); i++)+ p[i] = (unsigned char)rnd();+ for (int b = 0; b < ROWS * NB; b++)+ w[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));+ for (int r = 0; r < ROWS; r++)+ janas_q6k_to_planes(w + r * NB, NB, base + r * NB, p1 + r * pb,+ p0 + r * pb);+ for (int v = 0; v < NV; v++) {+ for (int i = 0; i < N; i++)+ xf[i] = rndf(-1.0f, 1.0f);+ janas_q8k_quantize(xf, x + v * NB, N);+ }+ for (int level = 0; level <= 3; level++) {+ const uint8_t *a1 = level >= 2 ? p1 : NULL;+ const uint8_t *a0 = level >= 3 ? p0 : NULL;+ int t = level ? JANAS_Q6_K_P : JANAS_Q6_K;+ const void *wr = level ? (const void *)base : (const void *)w;+ size_t rb = level ? NB * sizeof(*base) : NB * sizeof(*w);+ for (int nv = 1; nv <= NV; nv += nv < 9 ? 1 : 8) {+ for (int i = 0; i < ROWS * NV; i++)+ out[i] = -12345.0f;+ size_t done = janas_q6k_tile_rows(t, wr, rb, a1, a0, pb, x, NB,+ (size_t)nv, NB, ROWS, out, ROWS);+ CHECK(done == (nv >= 12 ? 8u : 0u), "q6k tiles: %zu rows", done);+ for (size_t r = 0; r < done; r++)+ for (int v = 0; v < nv; v++) {+ float one = level+ ? janas_q6kp_dot(base + r * NB,+ a1 ? a1 + r * pb : NULL,+ a0 ? a0 + r * pb : NULL,+ x + v * NB, NB)+ : janas_q6k_dot(w + r * NB, x + v * NB, NB);+ CHECK(out[r + ROWS * v] == one,+ "q6k tiles level %d nv %d r %zu v %d: %g vs %g",+ level, nv, r, v, out[r + ROWS * v], one);+ }+ }+ }+ free(w);+ free(base);+ free(p1);+ free(p0);+ free(x);+ free(xf);+ free(out);+}+/* janas_q6kpn_axpy at every level: with h = 1 on zeros the very floatsjanas_q6kp_dequantize gives, whatever planes lie below the level; withany h on any y, y plus h times them. */@@ -1066,6 +1130,115 @@ static void random_blocks(int type, unsigned char *w, size_t n)}}+/*+ * janas_dot_multi_rows on Q2_0 and the K types, on long rows: the vectors+ * reordered (Q2_0) or their scales and sums made ready (Q4_K, Q5_K, Q6_K) a few+ * at a time, or not at all where the rows are too long (130 groups), 1-65+ * vectors; each result equal to janas_dot's to the bit, and Q2_0's to the+ * reference.+ */+static void test_multi_rows_long(void)+{+ /* 11 rows: Q4_K's tiles take eight (q4k_gemm), the products the+ rest; 65 vectors: past the tiles' 64 tokens at a time */+ enum { ROWS = 11, NV = 65 };+ static const size_t lens[] = {2, 48, 130};+ static const int types[] = {+ JANAS_Q2_0, JANAS_Q4_K, JANAS_Q5_K, JANAS_Q6_K, JANAS_Q8_0,+ JANAS_Q4_0, JANAS_Q5_0, JANAS_IQ4_NL, JANAS_MXFP4, JANAS_Q4_1,+ JANAS_Q5_1, JANAS_Q3_K, JANAS_IQ4_XS, JANAS_Q2_K, JANAS_IQ3_S,+ JANAS_IQ3_XXS, JANAS_IQ2_XXS, JANAS_IQ2_XS, JANAS_IQ2_S, JANAS_IQ1_S,+ JANAS_IQ1_M};+ for (unsigned ti = 0; ti < sizeof(types) / sizeof(*types); ti++)+ for (unsigned li = 0; li < sizeof(lens) / sizeof(*lens); li++) {+ int t = types[ti];+ size_t nb = lens[li], n = nb * JANAS_QK;+ size_t rb = janas_qtype_row_bytes(t, n);+ unsigned char *w = malloc(ROWS * rb);+ struct janas_block_q8k *x = malloc(NV * nb * sizeof(*x));+ float *xf = malloc(n * sizeof(float)), out[ROWS * NV];+ for (int r = 0; r < ROWS; r++) {+ if (t == JANAS_Q2_0 || t == JANAS_Q8_0 || t == JANAS_Q4_0 ||+ t == JANAS_Q5_0 || t == JANAS_IQ4_NL)+ random_blocks(t, w + r * rb, n);+ else if (t == JANAS_MXFP4 || t == JANAS_Q4_1 ||+ t == JANAS_Q5_1) {+ /* blocks of 32: MXFP4's power of two, Q4_1's and+ Q5_1's d and m, finite */+ size_t bs = janas_qtype_row_bytes(t, 32);+ unsigned char *q = w + r * rb;+ for (size_t i = 0; i < rb; i++)+ q[i] = (unsigned char)rnd();+ for (size_t i = 0; i < rb; i += bs) {+ if (t == JANAS_MXFP4) {+ q[i] = (unsigned char)(118 + rnd() % 12);+ continue;+ }+ uint16_t d[2] = {+ janas_fp32_to_fp16(rndf(-0.01f, 0.01f)),+ janas_fp32_to_fp16(rndf(-0.01f, 0.01f))};+ memcpy(q + i, d, 4);+ }+ } else+ for (size_t b = 0; b < nb; b++) {+ /* Q4_K and Q5_K: d and dmin first, the IQ types d;+ Q6_K and Q3_K: d last, Q2_K d and dmin; IQ1_M: d+ in the top nibbles of its four scale words */+ size_t bs = janas_qtype_block_size(t);+ unsigned char *q = w + r * rb + b * bs;+ for (size_t i = 0; i < bs; i++)+ q[i] = (unsigned char)rnd();+ uint16_t d[2] = {+ janas_fp32_to_fp16(rndf(-0.01f, 0.01f)),+ janas_fp32_to_fp16(rndf(-0.01f, 0.01f))};+ if (t == JANAS_Q6_K || t == JANAS_Q3_K)+ memcpy(q + bs - 2, d, 2);+ else if (t == JANAS_Q2_K)+ memcpy(q + bs - 4, d, 4);+ else if (t == JANAS_IQ1_M) {+ uint16_t sc[4];+ memcpy(sc, q + bs - 8, 8);+ for (int k = 0; k < 4; k++)+ sc[k] = (uint16_t)((sc[k] & 0x0fff) |+ (d[0] >> 4 * k & 15) << 12);+ memcpy(q + bs - 8, sc, 8);+ } else+ memcpy(q, d, 4);+ }+ }+ for (int v = 0; v < NV; v++) {+ for (size_t i = 0; i < n; i++)+ xf[i] = rndf(-1.0f, 1.0f);+ janas_q8k_quantize(xf, x + v * nb, n);+ }+ for (int nv = 1; nv <= NV; nv += nv < 9 ? 1 : 8) {+ janas_dot_multi_rows(t, w, rb, x, nb, (size_t)nv, nb, ROWS, out,+ ROWS);+ for (int v = 0; v < nv; v++)+ for (int r = 0; r < ROWS; r++) {+ float one = janas_dot(t, w + r * rb, x + v * nb, nb);+ CHECK(out[r + ROWS * v] == one,+ "type %d %zu groups nv %d r %d v %d: %g vs %g", t,+ nb, nv, r, v, out[r + ROWS * v], one);+ if (t == JANAS_Q2_0) {+ float ref = janas_q2_0_dot_ref(+ (const void *)(w + r * rb), x + v * nb, nb);+ CHECK(one == ref, "q2_0 %zu groups: %g vs %g", nb,+ one, ref);+ }+ }+ }+ float one[ROWS];+ janas_dot_rows(t, w, rb, x, nb, ROWS, one);+ for (int r = 0; r < ROWS; r++)+ CHECK(one[r] == janas_dot(t, w + r * rb, x, nb),+ "type %d %zu groups rows %d", t, nb, r);+ free(w);+ free(x);+ free(xf);+ }+}+/** Q5_0 and Q2_0: the reference against a float product over the* dequantized weights, the kernels (single, 1-17 vectors, rows) equal to it@@ -1126,6 +1299,20 @@ static void test_dot_q5_0_q2_0(void)CHECK(rows[r] == one, "%s rows %d: %g vs %g", name, r, rows[r],one);}+ /* the same four rows on 1-17 vectors at once */+ for (int nv = 1; nv <= NV; nv++) {+ float out[4 * NV];+ janas_dot_multi_rows(t, w, 2 * gb, x, NB, (size_t)nv, 2, 4, out,+ 4);+ for (int r = 0; r < 4; r++)+ for (int v = 0; v < nv; v++) {+ float one =+ janas_dot(t, w + 2 * gb * (size_t)r, x + v * NB, 2);+ CHECK(out[r + 4 * v] == one,+ "%s multi rows nv %d r %d v %d: %g vs %g", name,+ nv, r, v, out[r + 4 * v], one);+ }+ }}/* by hand. Q5_0: weight 17 takes bit 17 of qh and the high nibble ofbyte 1, weight 1 the low nibble alone, less 16 */@@ -1163,7 +1350,9 @@ static void test_dot_q5_0_q2_0(void)*/static void test_row_tails(void){- enum { NMAX = 768, NBX = NMAX / JANAS_QK, NV = 17, ROWS = 37 };+ /* 33 vectors and 37 rows: the tiles take 32 of them (from 24 vectors),+ the tails 16 rows at a time */+ enum { NMAX = 768, NBX = NMAX / JANAS_QK, NV = 33, ROWS = 37 };static const int types[] = {JANAS_Q2_0, JANAS_Q5_0, JANAS_IQ4_NL,JANAS_Q4_0, JANAS_Q8_0};static const size_t ns[] = {640, 64, 320, 576, 704};@@ -1229,8 +1418,23 @@ static void test_row_tails(void)n, v, nv, out[v], e);}}- /* the grouped products: one vector and five */- for (int nv = 1; pool && nv <= 5; nv += 4) {+ for (int nv = 1; nv <= NV; nv += 8) {+ janas_dot_multi_rows_n(t, w, rb, x, nbx, (size_t)nv, n, ROWS, y,+ ROWS);+ for (int v = 0; v < nv; v++)+ for (int r = 0; r < ROWS; r++) {+ float e =+ janas_dot_n(t, w + (size_t)r * rb, x + v * nbx, n);+ CHECK(y[v * ROWS + r] == e,+ "type %d n %zu multi rows nv %d v %d row %d: %g "+ "vs %g",+ t, n, nv, v, r, y[v * ROWS + r], e);+ }+ }+ /* the grouped products: one vector, five and NV */+ for (int nv = 1; pool && nv <= NV; nv = nv == 1 ? 5+ : nv == 5 ? NV+ : NV + 1) {struct janas_matvec_task task = {.type = t,.w = w,.x = x,@@ -2008,6 +2212,8 @@ int main(void)test_dot_mxfp4();test_dot_q4_0_q5_1();test_dot_q5_0_q2_0();+ test_multi_rows_long();+ test_q6k_tiles();test_row_tails();test_dot_q3k();test_dot_iq3s();diff --git a/tools/spv_fma.c b/tools/spv_fma.cnew file mode 100644index 0000000..0832e2e--- /dev/null+++ b/tools/spv_fma.c@@ -0,0 +1,115 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * spv_fma.c - a shader's SPIR-V with every fma() decorated NoContraction,+ * written as the C array build.sh includes (glslangValidator --vn's form).+ *+ * Usage: spv_fma <in.spv> <array name> <out.h>+ *+ * GLSL.std.450's Fma may be done as a multiply and an add, two roundings,+ * unless it is decorated NoContraction; glslang decorates a precise+ * variable's multiplies and adds but not its fma() calls, and the Arc's+ * driver did split some of them: the experts' activation (expert_act.comp)+ * came out a bit off the CPU's in a few hundredths of its values, enough+ * to move a block's scale in one expert in eighty (24 Oct 2026). The+ * shaders write fma() where the CPU fuses, so every one of them is meant+ * as one operation.+ */+#include <stdint.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>++enum {+ OP_EXT_INST_IMPORT = 11,+ OP_EXT_INST = 12,+ OP_DECORATE = 71,+ OP_TYPE_FIRST = 19, /* OpTypeVoid .. OpTypeForwardPointer: after the */+ OP_TYPE_LAST = 39, /* annotations, so the new ones go before them */+ GLSL_FMA = 50,+ DEC_NO_CONTRACTION = 42,+};++int main(int argc, char **argv)+{+ if (argc != 4) {+ fprintf(stderr, "usage: spv_fma <in.spv> <array name> <out.h>\n");+ return 2;+ }+ FILE *f = fopen(argv[1], "rb");+ if (!f) {+ perror(argv[1]);+ return 1;+ }+ fseek(f, 0, SEEK_END);+ long bytes = ftell(f);+ fseek(f, 0, SEEK_SET);+ size_t n = bytes > 0 ? (size_t)bytes / 4 : 0;+ uint32_t *w = malloc((n ? n : 1) * 4);+ if (!w || fread(w, 4, n, f) != n || n < 5 || w[0] != 0x07230203u) {+ fprintf(stderr, "%s: not SPIR-V\n", argv[1]);+ return 1;+ }+ fclose(f);+ /* the GLSL.std.450 set, the Fma results, those already decorated, and+ where the types start */+ uint32_t glsl = 0, *fma = malloc(n * 4), nf = 0;+ uint8_t *done = calloc(w[3], 1); /* w[3]: the ids' bound */+ size_t types = 0;+ if (!fma || !done)+ return 1;+ for (size_t i = 5; i < n;) {+ uint32_t op = w[i] & 0xffff, wc = w[i] >> 16;+ if (wc == 0 || i + wc > n) {+ fprintf(stderr, "%s: bad instruction at word %zu\n", argv[1], i);+ return 1;+ }+ if (op == OP_EXT_INST_IMPORT && wc >= 3 &&+ strncmp((const char *)&w[i + 2], "GLSL.std.450", 13) == 0)+ glsl = w[i + 1];+ if (op == OP_EXT_INST && wc >= 5 && w[i + 3] == glsl &&+ w[i + 4] == GLSL_FMA)+ fma[nf++] = w[i + 2];+ if (op == OP_DECORATE && wc >= 3 && w[i + 2] == DEC_NO_CONTRACTION &&+ w[i + 1] < w[3])+ done[w[i + 1]] = 1;+ if (!types && op >= OP_TYPE_FIRST && op <= OP_TYPE_LAST)+ types = i;+ i += wc;+ }+ if (nf && !types) {+ fprintf(stderr, "%s: no types\n", argv[1]);+ return 1;+ }+ FILE *o = fopen(argv[3], "w");+ if (!o) {+ perror(argv[3]);+ return 1;+ }+ fprintf(o,+ "#pragma once\n/* %s, its %u fma() NoContraction (spv_fma) "+ "*/\nconst uint32_t %s[] = {\n",+ argv[1], nf, argv[2]);+ size_t col = 0;+#define PUT(x) fprintf(o, "%s0x%08x,", col++ % 8 ? "" : "\n\t", (unsigned)(x))+ for (size_t i = 0; i < n; i++) {+ if (i == types)+ for (uint32_t k = 0; k < nf; k++)+ if (!done[fma[k]]) {+ done[fma[k]] = 1;+ PUT((3u << 16) | OP_DECORATE);+ PUT(fma[k]);+ PUT(DEC_NO_CONTRACTION);+ }+ PUT(w[i]);+ }+ fprintf(o, "\n};\n");+ if (fclose(o) != 0) {+ perror(argv[3]);+ return 1;+ }+ free(w);+ free(fma);+ free(done);+ return 0;+}