Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Gyro: the Hadamard row kernels go up in slices of 65535 rows (gridDim.y limit): the FP16 prompt path rotates T*K rows, which a 16K-token chunk takes past it (invalid configuration argument on a build without the fork's MMQ)

Niko122102bbf51fb7e9
diff --git a/docs/MODELS.md b/docs/MODELS.md
index ebfeb1001..9427d7b1e 100644
--- a/docs/MODELS.md
+++ b/docs/MODELS.md
@@ -144,6 +144,26 @@ speed bound and NVIDIA/AMD/Intel details: [QUANTS.md](QUANTS.md); the files: [UD
START-HERE.bat --setup --family unsloth --model UD-Q5_K_XL
```
+<a id="gyro"></a>
+
+### Gyro-S and Gyro-M (agentionai)
+
+The fifth version in setup's menu (`--family gyro`) is **agentionai's Gyro quantizations** of the same model: trellis-coded
+experts in a rotated basis. They are the smallest options and never setup's default. The download includes the
+28 GB n-gram table; the memory figure is VRAM and RAM together.
+
+| Size | Download | Memory it needs (VRAM + RAM) | In plain words |
+| --- | ---: | ---: | --- |
+| **Gyro-S** | 58.5 GB | 27.6 GB | the smallest option, for PCs with little memory; lower quality |
+| **Gyro-M** | 92.0 GB | 35 GB | about IQ3_XXS quality with about 20% less GPU memory; decodes about 20% slower than Q2_0 on 12 GB cards |
+
+NVIDIA, and AMD (experimental); not Intel Arc or the Mac. Images are off. Details, measurements and credit:
+[GYRO.md](GYRO.md).
+
+```
+START-HERE.bat --setup --family gyro --model Gyro-S
+```
+
### OrcaRouter Uncensored IQ3_XXS
For **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](ORCA.md). It needs an
diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cu
index e291fb2f6..fd0e180ef 100644
--- a/src/kernels/cuda/iq_kernels.cu
+++ b/src/kernels/cuda/iq_kernels.cu
@@ -19,6 +19,7 @@
#include "ggml-common-tq.h" // APR trellis types (agention llama.cpp fork)
#include "tq_lut_i8.cuh" // int8 copy of the trellis codebook (one global scale) for the dp4a dot
+#include <algorithm>
#include <cmath>
#include <cstdio>
#include <utility>
@@ -3150,6 +3151,7 @@ void hadamard_bad_block(const char* what, int block, int64_t width) {
"(hadamard.txt's block and the expert widths disagree)\n", what, block, (long long) width);
std::exit(1);
}
+constexpr int64_t kHadamardMaxRows = 65535; // gridDim.y
bool hadamard_block_ok(int block, int64_t width) {
return block >= 2 && block <= 1024 && (block & (block - 1)) == 0 && width % block == 0;
}
@@ -3158,17 +3160,25 @@ bool hadamard_block_ok(int block, int64_t width) {
void hadamard_rows(const float* x, float* y, int64_t rows, int64_t width, int block, const float* signs, void* stream) {
if (rows <= 0) return;
if (!hadamard_block_ok(block, width)) hadamard_bad_block("hadamard_rows", block, width);
- hadamard_rows_kernel<float><<<dim3((unsigned) (width / block), (unsigned) rows), block, block * sizeof(float),
- (cudaStream_t) stream>>>(x, y, width, signs, 1.0f / std::sqrt((float) block));
+ // gridDim.y is limited to 65535: a prompt chunk's T * K rows (16K tokens x 10 experts) go up in slices
+ for (int64_t r0 = 0; r0 < rows; r0 += kHadamardMaxRows) {
+ const int64_t n = std::min<int64_t>(kHadamardMaxRows, rows - r0);
+ hadamard_rows_kernel<float><<<dim3((unsigned) (width / block), (unsigned) n), block, block * sizeof(float),
+ (cudaStream_t) stream>>>(x + r0 * width, y + r0 * width, width, signs,
+ 1.0f / std::sqrt((float) block));
+ }
check("hadamard_rows");
}
void hadamard_rows_f16(uint16_t* x, int64_t rows, int64_t width, int block, const float* signs, void* stream) {
if (rows <= 0) return;
if (!hadamard_block_ok(block, width)) hadamard_bad_block("hadamard_rows_f16", block, width);
- hadamard_rows_kernel<__half><<<dim3((unsigned) (width / block), (unsigned) rows), block, block * sizeof(float),
- (cudaStream_t) stream>>>((const __half*) x, (__half*) x, width, signs,
- 1.0f / std::sqrt((float) block));
+ for (int64_t r0 = 0; r0 < rows; r0 += kHadamardMaxRows) { // gridDim.y <= 65535, as in hadamard_rows
+ const int64_t n = std::min<int64_t>(kHadamardMaxRows, rows - r0);
+ hadamard_rows_kernel<__half><<<dim3((unsigned) (width / block), (unsigned) n), block, block * sizeof(float),
+ (cudaStream_t) stream>>>((const __half*) x + r0 * width, (__half*) x + r0 * width,
+ width, signs, 1.0f / std::sqrt((float) block));
+ }
check("hadamard_rows_f16");
}