A high-performance inference engine for AI models
quant_matmul fix
Aleksei Savin21391ac3ade1
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalindex 999a8c2d6..89772b581 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal@@ -89,10 +89,11 @@ inline void qouter(static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");if (bits == 4) {- U s[2] = {scale, scale / 16.0f};+ U s0 = scale;+ U s1 = scale / 16.0f;for (int i = 0; i < (values_per_thread / 2); i++) {- result[2 * i] += x * (s[0] * (w[i] & 0x0f) + bias);- result[2 * i + 1] += x * (s[1] * (w[i] & 0xf0) + bias);+ result[2 * i] += x * (s0 * (w[i] & 0x0f) + bias);+ result[2 * i + 1] += x * (s1 * (w[i] & 0xf0) + bias);}} else if (bits == 8) {for (int i = 0; i < values_per_thread; i++) {@@ -227,10 +228,11 @@ inline void dequantize(static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");if (bits == 4) {- U s[2] = {scale, scale / static_cast<U>(16.0f)};+ U s0 = scale;+ U s1 = scale / static_cast<U>(16.0f);for (int i = 0; i < (N / 2); i++) {- w_local[2 * i] = s[0] * (w[i] & 0x0f) + bias;- w_local[2 * i + 1] = s[1] * (w[i] & 0xf0) + bias;+ w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;+ w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;}} else if (bits == 8) {for (int i = 0; i < N; i++) {