Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

quant_matmul fix

Aleksei Savin21391ac3ade1
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
index 999a8c2d6..89772b581 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
@@ -89,10 +89,11 @@ inline void qouter(
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
if (bits == 4) {
- U s[2] = {scale, scale / 16.0f};
+ U s0 = scale;
+ U s1 = scale / 16.0f;
for (int i = 0; i < (values_per_thread / 2); i++) {
- result[2 * i] += x * (s[0] * (w[i] & 0x0f) + bias);
- result[2 * i + 1] += x * (s[1] * (w[i] & 0xf0) + bias);
+ result[2 * i] += x * (s0 * (w[i] & 0x0f) + bias);
+ result[2 * i + 1] += x * (s1 * (w[i] & 0xf0) + bias);
}
} else if (bits == 8) {
for (int i = 0; i < values_per_thread; i++) {
@@ -227,10 +228,11 @@ inline void dequantize(
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
if (bits == 4) {
- U s[2] = {scale, scale / static_cast<U>(16.0f)};
+ U s0 = scale;
+ U s1 = scale / static_cast<U>(16.0f);
for (int i = 0; i < (N / 2); i++) {
- w_local[2 * i] = s[0] * (w[i] & 0x0f) + bias;
- w_local[2 * i + 1] = s[1] * (w[i] & 0xf0) + bias;
+ w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;
+ w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;
}
} else if (bits == 8) {
for (int i = 0; i < N; i++) {