Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Gyro: the Hadamard row kernels go up in slices of 65535 rows (gridDim.y limit): the FP16 prompt path rotates T*K rows, which a 16K-token chunk takes past it (invalid configuration argument on a build without the fork's MMQ)
Niko122102bbf51fb7e9
diff --git a/docs/MODELS.md b/docs/MODELS.mdindex ebfeb1001..9427d7b1e 100644--- a/docs/MODELS.md+++ b/docs/MODELS.md@@ -144,6 +144,26 @@ speed bound and NVIDIA/AMD/Intel details: [QUANTS.md](QUANTS.md); the files: [UDSTART-HERE.bat --setup --family unsloth --model UD-Q5_K_XL```+<a id="gyro"></a>++### Gyro-S and Gyro-M (agentionai)++The fifth version in setup's menu (`--family gyro`) is **agentionai's Gyro quantizations** of the same model: trellis-coded+experts in a rotated basis. They are the smallest options and never setup's default. The download includes the+28 GB n-gram table; the memory figure is VRAM and RAM together.++| Size | Download | Memory it needs (VRAM + RAM) | In plain words |+| --- | ---: | ---: | --- |+| **Gyro-S** | 58.5 GB | 27.6 GB | the smallest option, for PCs with little memory; lower quality |+| **Gyro-M** | 92.0 GB | 35 GB | about IQ3_XXS quality with about 20% less GPU memory; decodes about 20% slower than Q2_0 on 12 GB cards |++NVIDIA, and AMD (experimental); not Intel Arc or the Mac. Images are off. Details, measurements and credit:+[GYRO.md](GYRO.md).++```+START-HERE.bat --setup --family gyro --model Gyro-S+```+### OrcaRouter Uncensored IQ3_XXSFor **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](ORCA.md). It needs andiff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cuindex e291fb2f6..fd0e180ef 100644--- a/src/kernels/cuda/iq_kernels.cu+++ b/src/kernels/cuda/iq_kernels.cu@@ -19,6 +19,7 @@#include "ggml-common-tq.h" // APR trellis types (agention llama.cpp fork)#include "tq_lut_i8.cuh" // int8 copy of the trellis codebook (one global scale) for the dp4a dot+#include <algorithm>#include <cmath>#include <cstdio>#include <utility>@@ -3150,6 +3151,7 @@ void hadamard_bad_block(const char* what, int block, int64_t width) {"(hadamard.txt's block and the expert widths disagree)\n", what, block, (long long) width);std::exit(1);}+constexpr int64_t kHadamardMaxRows = 65535; // gridDim.ybool hadamard_block_ok(int block, int64_t width) {return block >= 2 && block <= 1024 && (block & (block - 1)) == 0 && width % block == 0;}@@ -3158,17 +3160,25 @@ bool hadamard_block_ok(int block, int64_t width) {void hadamard_rows(const float* x, float* y, int64_t rows, int64_t width, int block, const float* signs, void* stream) {if (rows <= 0) return;if (!hadamard_block_ok(block, width)) hadamard_bad_block("hadamard_rows", block, width);- hadamard_rows_kernel<float><<<dim3((unsigned) (width / block), (unsigned) rows), block, block * sizeof(float),- (cudaStream_t) stream>>>(x, y, width, signs, 1.0f / std::sqrt((float) block));+ // gridDim.y is limited to 65535: a prompt chunk's T * K rows (16K tokens x 10 experts) go up in slices+ for (int64_t r0 = 0; r0 < rows; r0 += kHadamardMaxRows) {+ const int64_t n = std::min<int64_t>(kHadamardMaxRows, rows - r0);+ hadamard_rows_kernel<float><<<dim3((unsigned) (width / block), (unsigned) n), block, block * sizeof(float),+ (cudaStream_t) stream>>>(x + r0 * width, y + r0 * width, width, signs,+ 1.0f / std::sqrt((float) block));+ }check("hadamard_rows");}void hadamard_rows_f16(uint16_t* x, int64_t rows, int64_t width, int block, const float* signs, void* stream) {if (rows <= 0) return;if (!hadamard_block_ok(block, width)) hadamard_bad_block("hadamard_rows_f16", block, width);- hadamard_rows_kernel<__half><<<dim3((unsigned) (width / block), (unsigned) rows), block, block * sizeof(float),- (cudaStream_t) stream>>>((const __half*) x, (__half*) x, width, signs,- 1.0f / std::sqrt((float) block));+ for (int64_t r0 = 0; r0 < rows; r0 += kHadamardMaxRows) { // gridDim.y <= 65535, as in hadamard_rows+ const int64_t n = std::min<int64_t>(kHadamardMaxRows, rows - r0);+ hadamard_rows_kernel<__half><<<dim3((unsigned) (width / block), (unsigned) n), block, block * sizeof(float),+ (cudaStream_t) stream>>>((const __half*) x + r0 * width, (__half*) x + r0 * width,+ width, signs, 1.0f / std::sqrt((float) block));+ }check("hadamard_rows_f16");}