Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: the first dsv41 kernel ported and bit-exact - the Q8 activation quantiser

Niko12216bfcfa777f1e
diff --git a/metal/tests/dsv41/q8_quant_parity.mm b/metal/tests/dsv41/q8_quant_parity.mm
new file mode 100644
index 000000000..d46a86fff
--- /dev/null
+++ b/metal/tests/dsv41/q8_quant_parity.mm
@@ -0,0 +1,105 @@
+// q8probe.mm - the first dsv41 kernel in MSL: the activation quantiser at ds4_gguf.cu:201.
+//
+// CUDA's q8 path: per 32 values, d = max|x| / 127 (a half), then q = round(x / d) clamped to [-128, 127], stored as int8
+// with the f16 scale. This compiles the MSL version at run time, runs it on the M5 Max GPU, and compares it against the
+// same arithmetic in C++ - if the two agree, this function is ready to become a Pipeline in metal/src/engine.
+// clang++ -O2 -fobjc-arc -framework Metal -framework Foundation -o q8probe q8probe.mm && ./q8probe
+#import <Foundation/Foundation.h>
+#import <Metal/Metal.h>
+#include <cstdio>
+#include <cstdlib>
+#include <cmath>
+#include <vector>
+#include <cstring>
+
+static const char* kSrc = R"MSL(
+#include <metal_stdlib>
+using namespace metal;
+
+// one threadgroup of 32 threads per block of 32 values (CUDA's layout), one block per 32 inputs
+kernel void q8_quant(device const float* x [[buffer(0)]], constant uint& ldx [[buffer(1)]],
+ device char* q [[buffer(2)]], device half* d [[buffer(3)]],
+ uint gid [[thread_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) {
+ const uint b = gid / 32u; // the block this thread belongs to
+ const int base = int(b) * int(ldx);
+ float v = x[base + int(lane)];
+ float amax = fabs(v);
+ amax = max(amax, simd_shuffle_xor(amax, 16));
+ amax = max(amax, simd_shuffle_xor(amax, 8));
+ amax = max(amax, simd_shuffle_xor(amax, 4));
+ amax = max(amax, simd_shuffle_xor(amax, 2));
+ amax = max(amax, simd_shuffle_xor(amax, 1));
+ const float scale = amax / 127.0f;
+ const float inv = scale > 0.0f ? 1.0f / scale : 0.0f;
+ int iq = int(rint(v * inv));
+ iq = clamp(iq, -128, 127);
+ q[int(b) * 32 + int(lane)] = char(iq);
+ if (lane == 0) d[int(b)] = half(scale);
+}
+)MSL";
+
+int main() {
+ @autoreleasepool {
+ const int n = 32 * 4096;
+ std::vector<float> x(n);
+ srand(7);
+ for (int i = 0; i < n; ++i) x[i] = ((rand() % 20001) - 10000) / 997.0f;
+
+ id<MTLDevice> dev = MTLCreateSystemDefaultDevice();
+ NSError* err = nil;
+ id<MTLLibrary> lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kSrc] options:nil error:&err];
+ if (!lib) { printf("MSL compile failed: %s\n", [[err localizedDescription] UTF8String]); return 1; }
+ id<MTLComputePipelineState> pso = [dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"q8_quant"] error:&err];
+ if (!pso) { printf("pipeline failed: %s\n", [[err localizedDescription] UTF8String]); return 1; }
+
+ id<MTLBuffer> bx = [dev newBufferWithBytes:x.data() length:n * sizeof(float) options:MTLResourceStorageModeShared];
+ id<MTLBuffer> bq = [dev newBufferWithLength:n options:MTLResourceStorageModeShared];
+ id<MTLBuffer> bd = [dev newBufferWithLength:(n / 32) * 2 options:MTLResourceStorageModeShared];
+ id<MTLCommandQueue> qq = [dev newCommandQueue];
+ id<MTLCommandBuffer> cb = [qq commandBuffer];
+ id<MTLComputeCommandEncoder> ce = [cb computeCommandEncoder];
+ [ce setComputePipelineState:pso];
+ [ce setBuffer:bx offset:0 atIndex:0];
+ unsigned ldx = 32; // the stride between blocks of 32 values
+ [ce setBytes:&ldx length:4 atIndex:1];
+ [ce setBuffer:bq offset:0 atIndex:1 + 1];
+ [ce setBuffer:bd offset:0 atIndex:2 + 1];
+ [ce dispatchThreads:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
+ [ce endEncoding];
+ [cb commit];
+ [cb waitUntilCompleted];
+
+ // the same arithmetic on the CPU
+ const signed char* gq = (const signed char*) [bq contents];
+ const _Float16* gd = (const _Float16*) [bd contents];
+ // first block, side by side
+ {
+ float amax = 0; for (int i = 0; i < 32; ++i) amax = fmaxf(amax, fabsf(x[i]));
+ const float scale = amax / 127.0f, inv = scale > 0 ? 1.0f / scale : 0.0f;
+ printf("block0: gpu d=%.6f cpu d=%.6f\n", (double) gd[0], (double) scale);
+ printf(" gpu q:"); for (int i = 0; i < 8; ++i) printf(" %d", (int) gq[i]);
+ printf("\n cpu q:"); for (int i = 0; i < 8; ++i) printf(" %d", (int) rintf(x[i] * inv));
+ printf("\nblock1: gpu d=%.6f gpu q:", (double) gd[1]); for (int i = 32; i < 40; ++i) printf(" %d", (int) gq[i]);
+ float am1 = 0; for (int i = 32; i < 64; ++i) am1 = fmaxf(am1, fabsf(x[i]));
+ printf("\n cpu d=%.6f cpu q:", (double) (am1 / 127.0f)); for (int i = 32; i < 40; ++i) printf(" %d", (int) rintf(x[i] * (127.0f / am1)));
+ printf("\n");
+ } int bad = 0; double worst = 0;
+ for (int b = 0; b < n / 32; ++b) {
+ float amax = 0;
+ for (int i = 0; i < 32; ++i) amax = fmaxf(amax, fabsf(x[b * 32 + i]));
+ const float scale = amax / 127.0f;
+ const float inv = scale > 0 ? 1.0f / scale : 0.0f;
+ const double dd = fabs((double) gd[b] - (double) scale) / (scale > 0 ? scale : 1);
+ worst = fmax(worst, dd);
+ for (int i = 0; i < 32; ++i) {
+ int iq = (int) rintf(x[b * 32 + i] * inv);
+ iq = iq < -128 ? -128 : (iq > 127 ? 127 : iq);
+ if (iq != gq[b * 32 + i]) ++bad;
+ }
+ }
+ printf("blocks %d, values %d\n", n / 32, n);
+ printf("int8 differ: %d worst scale rel err %.3e\n", bad, worst);
+ printf("%s\n", bad == 0 && worst < 1e-3 ? "PASS" : "FAIL");
+ }
+ return 0;
+}