Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

unified gemv

eugene7b2614e86b20
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h
similarity index 62%
rename from crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.h
rename to crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h
index b76b4a5eb..7f1b2f305 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h
@@ -3,33 +3,41 @@
#include <metal_stdlib>
#include "../../common/defines.h"
-#include "../../gemm/common/quant_unpack.h"
+#include "quant_unpack.h"
using namespace metal;
namespace uzu {
namespace gemm {
+// Pre-scales each activation lane k by 2^-(BITS*k). qdot then reads the
+// matching weight nibble/byte in place (value = q * 2^(BITS*k)) without
+// shifting it down to the low bits; the positional 2^(BITS*k) factor cancels
+// the 2^-(BITS*k) here, so the dot product is unchanged. All factors are powers
+// of two, so this is bit-exact.
template <typename T, typename U, int VALUES_PER_THREAD, int BITS>
-inline U load_vector(const device T* x, thread U* x_thread) {
- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
-
+METAL_FUNC U load_vector(const device T* x, thread U* x_thread) {
using U4 = vec<U, 4>;
+ const U4 inv =
+ U4(U(1),
+ U(1) / U(1u << BITS),
+ U(1) / U(1u << (2u * BITS)),
+ U(1) / U(1u << (3u * BITS)));
U sum = 0;
thread U4* x_vec4 = reinterpret_cast<thread U4*>(x_thread);
+ METAL_PRAGMA_UNROLL
for (int i = 0; i < VALUES_PER_THREAD / 4; i++) {
U4 v = U4(x[4 * i], x[4 * i + 1], x[4 * i + 2], x[4 * i + 3]);
sum += v[0] + v[1] + v[2] + v[3];
- x_vec4[i] = v;
+ x_vec4[i] = v * inv;
}
return sum;
}
-template <typename T, typename U, int VALUES_PER_THREAD, int BITS>
-inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {
- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
-
+template <typename T, typename U, int VALUES_PER_THREAD>
+METAL_FUNC U load_vector_safe(const device T* x, thread U* x_thread, int N) {
U sum = 0;
+ METAL_PRAGMA_UNROLL
for (int i = 0; i < VALUES_PER_THREAD; ++i) {
x_thread[i] = 0;
}
@@ -42,30 +50,7 @@ inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {
}
template <typename U, int VALUES_PER_THREAD, int BITS>
-inline void qouter(
- const thread uint8_t* w,
- U x,
- U scale,
- U bias,
- thread U* result
-) {
- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
-
- if (BITS == 4) {
- for (int i = 0; i < (VALUES_PER_THREAD / 2); i++) {
- result[2 * i] += x * (scale * uint_to_fp<U>(w[i] & 0x0fu) + bias);
- result[2 * i + 1] +=
- x * (scale * uint_to_fp<U>((w[i] >> 4) & 0x0fu) + bias);
- }
- } else if (BITS == 8) {
- for (int i = 0; i < VALUES_PER_THREAD; i++) {
- result[i] += x * (scale * w[i] + bias);
- }
- }
-}
-
-template <typename U, int VALUES_PER_THREAD, int BITS>
-inline U qdot(
+METAL_FUNC U qdot(
const device uint8_t* w,
const thread U* x_thread,
U scale,
@@ -75,33 +60,35 @@ inline U qdot(
static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
U accumulator = 0;
- if (BITS == 4) {
+ if constexpr (BITS == 4) {
using U4 = vec<U, 4>;
const device ushort* weight_words =
reinterpret_cast<const device ushort*>(w);
const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);
+ METAL_PRAGMA_UNROLL
for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {
- uint weight_word = weight_words[i];
- U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(
- weight_word,
- weight_word >> 4,
- weight_word >> 8,
- weight_word >> 12
- ));
+ // Mask each nibble in place (no shifts); value of lane k is n_k << (4*k),
+ // i.e. n_k * 16^k, which is < 2^23 so the magic-number convert is valid.
+ // The matching x lane was pre-divided by 16^k in load_vector.
+ const uint4 lanes =
+ uint4(weight_words[i]) & uint4(0x000fu, 0x00f0u, 0x0f00u, 0xf000u);
+ const U4 weight_vec4 =
+ U4(as_type<float4>(lanes | uint4(0x4b000000u)) - float4(8388608.0f));
accumulator += dot(x_vec4[i], weight_vec4);
}
- } else if (BITS == 8) {
+ } else if constexpr (BITS == 8) {
using U4 = vec<U, 4>;
const device uint* weight_words = reinterpret_cast<const device uint*>(w);
const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);
+ METAL_PRAGMA_UNROLL
for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {
- uint weight_word = weight_words[i];
- U4 weight_vec4 = uint4_to_fp4<U, 8>(uint4(
- weight_word,
- weight_word >> 8,
- weight_word >> 16,
- weight_word >> 24
- ));
+ // Mask each byte in place (no shifts); lane k value is b_k * 256^k. This
+ // exceeds the magic-number range for k=3, so use the hardware convert
+ // (exact for b_k * 256^k). x lane k was pre-divided by 256^k.
+ const uint4 lanes =
+ uint4(weight_words[i]) &
+ uint4(0x000000ffu, 0x0000ff00u, 0x00ff0000u, 0xff000000u);
+ const U4 weight_vec4 = U4(float4(lanes));
accumulator += dot(x_vec4[i], weight_vec4);
}
}
@@ -109,7 +96,7 @@ inline U qdot(
}
template <typename U, int VALUES_PER_THREAD, int BITS>
-inline U qdot_safe(
+METAL_FUNC U qdot_safe(
const device uint8_t* w,
const thread U* x_thread,
U scale,
@@ -120,7 +107,7 @@ inline U qdot_safe(
static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
U accumulator = 0;
- if (BITS == 4) {
+ if constexpr (BITS == 4) {
using U4 = vec<U, 4>;
const device uint16_t* weight_words =
reinterpret_cast<const device uint16_t*>(w);
@@ -150,7 +137,7 @@ inline U qdot_safe(
accumulator +=
x_thread[base_index + 2] * uint_to_fp<U>((weight_word >> 8) & 0xf);
}
- } else if (BITS == 8) {
+ } else if constexpr (BITS == 8) {
for (int i = 0; i < N; i++) {
accumulator += x_thread[i] * w[i];
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_pack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_pack.h
new file mode 100644
index 000000000..b1a960ca9
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_pack.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <metal_stdlib>
+
+namespace uzu {
+namespace gemm {
+
+template <int bits, int word_size_bits = 8>
+METAL_FUNC constexpr short get_pack_factor() {
+ return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : word_size_bits / bits);
+}
+
+template <int bits, int word_size_bits = 8>
+METAL_FUNC constexpr short get_bytes_per_pack() {
+ constexpr int is_power_of_two_bits = (bits & (bits - 1)) == 0;
+ return is_power_of_two_bits ? (word_size_bits / 8) : (bits == 5 ? 5 : 3);
+}
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h
new file mode 100644
index 000000000..4735576f3
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h
@@ -0,0 +1,110 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../common/defines.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+template <typename U>
+METAL_FUNC U uint_to_fp(uint32_t x) {
+ return static_cast<U>(as_type<float>(x | 0x4B000000u) - 8388608.0f);
+}
+
+template <>
+METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {
+ return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);
+}
+
+template <int BITS>
+METAL_FUNC float4 _uint4_to_fp4_float(uint4 n) {
+ static_assert(BITS > 0 && BITS <= 23, "BITS must fit in float23 mantissa");
+ constexpr uint mask = (1u << BITS) - 1u;
+ n &= uint4(mask);
+ return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);
+}
+
+template <typename U, int BITS>
+METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);
+
+template <>
+METAL_FUNC float4 uint4_to_fp4<float, 4>(uint4 n) {
+ return _uint4_to_fp4_float<4>(n);
+}
+
+template <>
+METAL_FUNC float4 uint4_to_fp4<float, 8>(uint4 n) {
+ return _uint4_to_fp4_float<8>(n);
+}
+
+template <>
+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 4>(uint4 n) {
+ return bfloat4(_uint4_to_fp4_float<4>(n));
+}
+
+template <>
+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {
+ return bfloat4(_uint4_to_fp4_float<8>(n));
+}
+
+template <typename U, int N, int bits>
+METAL_FUNC void dequantize(
+ const device uint8_t* w,
+ U scale,
+ U bias,
+ threadgroup U* w_local
+) {
+ static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
+
+ if constexpr (bits == 4) {
+ U s0 = scale;
+ U s1 = scale / static_cast<U>(16.0f);
+ METAL_PRAGMA_UNROLL
+ for (int i = 0; i < (N / 2); i++) {
+ w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;
+ w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;
+ }
+ } else if constexpr (bits == 8) {
+ METAL_PRAGMA_UNROLL
+ for (int i = 0; i < N; i++) {
+ w_local[i] = scale * w[i] + bias;
+ }
+ }
+}
+
+template <>
+METAL_FUNC void dequantize<bfloat, 8, 4>(
+ const device uint8_t* w,
+ bfloat scale,
+ bfloat bias,
+ threadgroup bfloat* w_local
+) {
+ const device uint32_t* w_ptr = reinterpret_cast<const device uint32_t*>(w);
+ uint32_t packed = *w_ptr;
+
+ bfloat4 v0, v1;
+
+ v0.x = static_cast<bfloat>(packed & 0xF);
+ v0.y = static_cast<bfloat>((packed >> 4) & 0xF);
+ v0.z = static_cast<bfloat>((packed >> 8) & 0xF);
+ v0.w = static_cast<bfloat>((packed >> 12) & 0xF);
+
+ v1.x = static_cast<bfloat>((packed >> 16) & 0xF);
+ v1.y = static_cast<bfloat>((packed >> 20) & 0xF);
+ v1.z = static_cast<bfloat>((packed >> 24) & 0xF);
+ v1.w = static_cast<bfloat>((packed >> 28) & 0xF);
+
+ v0 = v0 * scale + bias;
+ v1 = v1 * scale + bias;
+
+ threadgroup bfloat4* out_ptr =
+ reinterpret_cast<threadgroup bfloat4*>(w_local);
+ out_ptr[0] = v0;
+ out_ptr[1] = v1;
+}
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h
new file mode 100644
index 000000000..a9fa82495
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h
@@ -0,0 +1,237 @@
+#pragma once
+
+#include "../../common/qdot.h"
+#include "../../common/quant_pack.h"
+#include "gemv_common.h"
+#include "quant_row_offsets.h"
+
+namespace uzu {
+namespace gemm {
+
+template <
+ typename BT,
+ typename AT,
+ typename U,
+ GemmBPrologueKind B_PROLOGUE,
+ uint GROUP_SIZE,
+ uint BITS,
+ uint K_SPLIT,
+ bool INPUT_ALIGNED>
+struct BSource {
+ static METAL_FUNC void accumulate(
+ thread U (&result)[RESULTS_PER_SIMDGROUP],
+ const device uint32_t* b,
+ const device BT* scales,
+ const device uint8_t* zero_points,
+ const device BT* biases,
+ const device AT* a,
+ uint in_vec_size,
+ uint out_row,
+ uint batch_idx,
+ uint simd_lane,
+ uint k_slice
+ ) {
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {
+ constexpr uint values_per_thread = 4;
+ constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;
+ typedef vec<BT, 4> W4;
+ typedef vec<AT, 4> I4;
+ const uint k_stride = K_SPLIT * block_size;
+ const uint k_start = k_slice * block_size;
+ const device BT* w = reinterpret_cast<const device BT*>(b);
+ w += out_row * in_vec_size + simd_lane * values_per_thread + k_start;
+ const device AT* in =
+ a + batch_idx * in_vec_size + simd_lane * values_per_thread + k_start;
+
+ uint k = k_start;
+ for (; k + block_size <= in_vec_size; k += k_stride) {
+ float4 xv =
+ static_cast<float4>(*reinterpret_cast<const device I4*>(in));
+ result[0] +=
+ dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w)),
+ xv);
+ result[1] +=
+ dot(static_cast<float4>(
+ *reinterpret_cast<const device W4*>(w + in_vec_size)
+ ),
+ xv);
+ result[2] +=
+ dot(static_cast<float4>(
+ *reinterpret_cast<const device W4*>(w + 2 * in_vec_size)
+ ),
+ xv);
+ result[3] +=
+ dot(static_cast<float4>(
+ *reinterpret_cast<const device W4*>(w + 3 * in_vec_size)
+ ),
+ xv);
+ w += k_stride;
+ in += k_stride;
+ }
+
+ if constexpr (K_SPLIT == 1 && !INPUT_ALIGNED) {
+ const uint thread_offset = simd_lane * values_per_thread;
+ const int remaining =
+ (k + thread_offset < in_vec_size)
+ ? min(static_cast<int>(in_vec_size - k - thread_offset),
+ static_cast<int>(values_per_thread))
+ : 0;
+ if (remaining > 0) {
+ const device BT* w0 = w;
+ const device BT* w1 = w + in_vec_size;
+ const device BT* w2 = w + 2 * in_vec_size;
+ const device BT* w3 = w + 3 * in_vec_size;
+ for (int j = 0; j < remaining; j++) {
+ U x = static_cast<U>(in[j]);
+ result[0] += static_cast<U>(w0[j]) * x;
+ result[1] += static_cast<U>(w1[j]) * x;
+ result[2] += static_cast<U>(w2[j]) * x;
+ result[3] += static_cast<U>(w3[j]) * x;
+ }
+ }
+ }
+ } else {
+ constexpr uint packs_per_thread = BITS == 2 ? 1 : 2;
+ constexpr uint pack_factor = get_pack_factor<BITS, 32>();
+ constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();
+ constexpr uint values_per_thread = pack_factor * packs_per_thread;
+ constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;
+ constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;
+ const device uint8_t* ws = (const device uint8_t*)b;
+ thread U x_thread[values_per_thread];
+
+ const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
+ const uint in_vec_size_g = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;
+ ws += out_row * in_vec_size_w +
+ simd_lane * packs_per_thread * bytes_per_pack;
+
+ QuantRowOffsets<BT, U, B_PROLOGUE, BITS> prep;
+ prep.group_stride = in_vec_size_g;
+ const uint g_offset = simd_lane / scale_step_per_thread;
+ prep.scales = scales + out_row * in_vec_size_g + g_offset;
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
+ prep.biases = biases + out_row * in_vec_size_g + g_offset;
+ } else if constexpr (
+ B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant
+ ) {
+ if (BITS == 4) {
+ prep.zp_stride = (in_vec_size_g + 1) / 2;
+ prep.zps = zero_points + out_row * prep.zp_stride + g_offset / 2;
+ prep.high_nibble = (g_offset & 1);
+ } else {
+ prep.zp_stride = in_vec_size_g;
+ prep.zps = zero_points + out_row * prep.zp_stride + g_offset;
+ }
+ }
+
+ const device AT* in =
+ a + batch_idx * in_vec_size + simd_lane * values_per_thread;
+
+ uint k = 0;
+ for (; k + block_size <= in_vec_size; k += block_size) {
+ U sum = load_vector<AT, U, values_per_thread, BITS>(in, x_thread);
+
+ const device uint8_t* wl0 = ws;
+ const device uint8_t* wl1 = ws + in_vec_size_w;
+ const device uint8_t* wl2 = ws + 2 * in_vec_size_w;
+ const device uint8_t* wl3 = ws + 3 * in_vec_size_w;
+
+ U scale[4];
+ U offset[4];
+ prep.load(scale, offset);
+ result[0] += qdot<U, values_per_thread, BITS>(
+ wl0,
+ x_thread,
+ scale[0],
+ offset[0],
+ sum
+ );
+ result[1] += qdot<U, values_per_thread, BITS>(
+ wl1,
+ x_thread,
+ scale[1],
+ offset[1],
+ sum
+ );
+ result[2] += qdot<U, values_per_thread, BITS>(
+ wl2,
+ x_thread,
+ scale[2],
+ offset[2],
+ sum
+ );
+ result[3] += qdot<U, values_per_thread, BITS>(
+ wl3,
+ x_thread,
+ scale[3],
+ offset[3],
+ sum
+ );
+
+ ws += block_size * bytes_per_pack / pack_factor;
+ prep.advance(block_size / GROUP_SIZE);
+ in += block_size;
+ }
+
+ if constexpr (!INPUT_ALIGNED) {
+ const uint thread_offset = simd_lane * values_per_thread;
+ const int remaining =
+ (k + thread_offset < in_vec_size)
+ ? min(static_cast<int>(in_vec_size - k - thread_offset),
+ static_cast<int>(values_per_thread))
+ : 0;
+ if (remaining > 0) {
+ U sum = load_vector_safe<AT, U, values_per_thread>(
+ in,
+ x_thread,
+ remaining
+ );
+
+ const device uint8_t* wl0 = ws;
+ const device uint8_t* wl1 = ws + in_vec_size_w;
+ const device uint8_t* wl2 = ws + 2 * in_vec_size_w;
+ const device uint8_t* wl3 = ws + 3 * in_vec_size_w;
+
+ U scale[4];
+ U offset[4];
+ prep.load(scale, offset);
+ result[0] += qdot_safe<U, values_per_thread, BITS>(
+ wl0,
+ x_thread,
+ scale[0],
+ offset[0],
+ sum,
+ remaining
+ );
+ result[1] += qdot_safe<U, values_per_thread, BITS>(
+ wl1,
+ x_thread,
+ scale[1],
+ offset[1],
+ sum,
+ remaining
+ );
+ result[2] += qdot_safe<U, values_per_thread, BITS>(
+ wl2,
+ x_thread,
+ scale[2],
+ offset[2],
+ sum,
+ remaining
+ );
+ result[3] += qdot_safe<U, values_per_thread, BITS>(
+ wl3,
+ x_thread,
+ scale[3],
+ offset[3],
+ sum,
+ remaining
+ );
+ }
+ }
+ }
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
new file mode 100644
index 000000000..1babf6c5a
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
@@ -0,0 +1,85 @@
+#pragma once
+
+#include "../../../hadamard_transform/hadamard_transform.h"
+#include "gemv_common.h"
+
+namespace uzu {
+namespace gemm {
+
+template <typename BT, typename DT, typename U>
+struct Epilogue {
+ static METAL_FUNC void store(
+ thread U (&result)[RESULTS_PER_SIMDGROUP],
+ device DT* d,
+ const device BT* output_bias,
+ const device int32_t* hadamard_factors,
+ threadgroup U* shared_results,
+ float ab_scale,
+ GemmDTransform output_transform,
+ uint out_row,
+ uint out_vec_size,
+ uint out_block_idx,
+ uint simd_group,
+ uint simd_lane,
+ bool writer
+ ) {
+ const bool is_scale = output_transform.contains(GemmDTransform::SCALE);
+ const bool is_accumulate =
+ output_transform.contains(GemmDTransform::ACCUMULATE);
+ const bool is_bias = output_transform.contains(GemmDTransform::BIAS);
+ const bool use_hadamard = output_transform.contains(GemmDTransform::RHT);
+
+ if (writer && simd_lane == 0) {
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
+ U value = result[row];
+ if (is_scale) {
+ value = static_cast<U>(ab_scale) * value;
+ }
+ const uint global_row = out_row + row;
+ if (is_accumulate && global_row < out_vec_size) {
+ value += static_cast<U>(d[row]);
+ }
+ if (is_bias && global_row < out_vec_size) {
+ value += static_cast<U>(output_bias[global_row]);
+ }
+ result[row] = value;
+ }
+ }
+
+ if (use_hadamard) {
+ if (simd_lane == 0) {
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
+ shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] =
+ result[row];
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (simd_group == 0) {
+ uint global_out_idx = out_block_idx * 32 + simd_lane;
+ if (global_out_idx < out_vec_size) {
+ d[simd_lane] = simdgroup_output_random_hadamard_transform(
+ static_cast<ushort>(simd_lane),
+ static_cast<DT>(shared_results[simd_lane]),
+ hadamard_factors[global_out_idx]
+ );
+ }
+ }
+ } else {
+ if (writer && simd_lane == 0) {
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
+ if (out_row + row < out_vec_size) {
+ d[row] = static_cast<DT>(result[row]);
+ }
+ }
+ }
+ }
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/gemv_common.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/gemv_common.h
new file mode 100644
index 000000000..f83f418d8
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/gemv_common.h
@@ -0,0 +1,16 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../common/defines.h"
+#include "../../../generated/gemm.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+METAL_CONST uint RESULTS_PER_SIMDGROUP = 4;
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h
new file mode 100644
index 000000000..6a28cbbb5
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h
@@ -0,0 +1,36 @@
+#pragma once
+
+#include "gemv_common.h"
+
+namespace uzu {
+namespace gemm {
+
+template <uint K_SPLIT, uint NUM_SIMDGROUPS>
+struct OutputTile {
+ uint out_row;
+ uint row_group;
+ uint k_slice;
+ bool writer;
+
+ static METAL_FUNC OutputTile
+ make(uint out_block_idx, uint simd_group, uint out_vec_size) {
+ constexpr uint rows_per_threadgroup =
+ (NUM_SIMDGROUPS / K_SPLIT) * RESULTS_PER_SIMDGROUP;
+ const uint row_group = simd_group / K_SPLIT;
+ const uint k_slice = simd_group % K_SPLIT;
+ uint out_row = out_block_idx * rows_per_threadgroup +
+ row_group * RESULTS_PER_SIMDGROUP;
+ out_row = out_vec_size > RESULTS_PER_SIMDGROUP
+ ? min(out_row, out_vec_size - RESULTS_PER_SIMDGROUP)
+ : 0u;
+ OutputTile tile;
+ tile.out_row = out_row;
+ tile.row_group = row_group;
+ tile.k_slice = k_slice;
+ tile.writer = (K_SPLIT == 1) || (k_slice == 0);
+ return tile;
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h
new file mode 100644
index 000000000..0965132c0
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h
@@ -0,0 +1,76 @@
+#pragma once
+
+#include "../../common/quant_pack.h"
+#include "gemv_common.h"
+
+namespace uzu {
+namespace gemm {
+
+template <typename BT, typename U, GemmBPrologueKind B_PROLOGUE, uint BITS>
+struct QuantRowOffsets {
+ static_assert(
+ BITS == 4 || BITS == 8,
+ "QuantRowOffsets supports 4- and 8-bit only"
+ );
+ const device BT* scales = nullptr;
+ const device BT* biases = nullptr;
+ const device uint8_t* zps = nullptr;
+ uint group_stride = 0;
+ uint zp_stride = 0;
+ bool high_nibble = false;
+
+ void load(thread U* scale, thread U* offset) const {
+ scale[0] = static_cast<U>(scales[0]);
+ scale[1] = static_cast<U>(scales[group_stride]);
+ scale[2] = static_cast<U>(scales[2 * group_stride]);
+ scale[3] = static_cast<U>(scales[3 * group_stride]);
+
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
+ offset[0] = static_cast<U>(biases[0]);
+ offset[1] = static_cast<U>(biases[group_stride]);
+ offset[2] = static_cast<U>(biases[2 * group_stride]);
+ offset[3] = static_cast<U>(biases[3 * group_stride]);
+ } else if constexpr (
+ B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant
+ ) {
+ uchar4 zp_bytes = uchar4(
+ zps[0],
+ zps[zp_stride],
+ zps[2 * zp_stride],
+ zps[3 * zp_stride]
+ );
+ uchar4 zp_nibbles;
+ if constexpr (BITS == 4) {
+ const uint8_t shift = high_nibble ? 4u : 0u;
+ zp_nibbles = (zp_bytes >> shift) & uchar4(0x0F);
+ } else {
+ zp_nibbles = zp_bytes;
+ }
+ offset[0] = -scale[0] * static_cast<U>(zp_nibbles.x);
+ offset[1] = -scale[1] * static_cast<U>(zp_nibbles.y);
+ offset[2] = -scale[2] * static_cast<U>(zp_nibbles.z);
+ offset[3] = -scale[3] * static_cast<U>(zp_nibbles.w);
+ } else {
+ constexpr U midpoint = U(1u << (BITS - 1));
+ offset[0] = -scale[0] * midpoint;
+ offset[1] = -scale[1] * midpoint;
+ offset[2] = -scale[2] * midpoint;
+ offset[3] = -scale[3] * midpoint;
+ }
+ }
+
+ void advance(uint groups) {
+ scales += groups;
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
+ biases += groups;
+ } else if constexpr (
+ B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant
+ ) {
+ constexpr uint zps_per_byte = get_pack_factor<BITS, 8>();
+ zps += groups / zps_per_byte;
+ }
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h
new file mode 100644
index 000000000..d08ad70bb
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h
@@ -0,0 +1,49 @@
+#pragma once
+
+#include "gemv_common.h"
+
+namespace uzu {
+namespace gemm {
+
+template <typename U, uint K_SPLIT, uint NUM_SIMDGROUPS>
+struct Reduce {
+ static METAL_FUNC void run(
+ thread U (&result)[RESULTS_PER_SIMDGROUP],
+ threadgroup U* shared_results,
+ uint simd_group,
+ uint simd_lane,
+ uint row_group,
+ uint k_slice
+ ) {
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
+ result[row] = simd_sum(result[row]);
+ }
+
+ if constexpr (K_SPLIT > 1) {
+ if (simd_lane == 0) {
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
+ shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] =
+ result[row];
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (k_slice == 0 && simd_lane == 0) {
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
+ U acc = 0;
+ METAL_PRAGMA_UNROLL
+ for (uint s = 0; s < K_SPLIT; s++) {
+ acc += shared_results
+ [(row_group * K_SPLIT + s) * RESULTS_PER_SIMDGROUP + row];
+ }
+ result[row] = acc;
+ }
+ }
+ }
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs
deleted file mode 100644
index ea0f28a20..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-use super::{kernel::GemvKernel, spec::GemvSpecialization};
-use crate::backends::{
- common::{
- AsBufferRangeRef, Buffer, Encoder,
- kernel::matmul::{MatmulArguments, MatmulB, MatmulError},
- },
- metal::Metal,
-};
-
-pub(crate) fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
- kernel: &mut GemvKernel,
- encoder: &mut Encoder<Metal>,
- arguments: MatmulArguments<'a, Metal, TB>,
-) -> Result<(), MatmulError<Metal>> {
- let ab_scale = arguments.d_transform.ab_scale;
- let output_bias = arguments.d_transform.bias;
- let is_accumulate = arguments.d_transform.accumulate;
-
- let MatmulArguments {
- a,
- a_offset,
- b,
- b_offset,
- b_leading_dimension,
- b_transpose,
- d,
- m,
- n,
- k,
- ..
- } = arguments;
- let weights = match b {
- MatmulB::FullPrecision {
- b: w,
- } => w,
- _ => panic!("FP gemv requires FullPrecision B"),
- };
- assert!(b_transpose, "encode_gemv does not support b_transpose=false");
- assert!(b_offset == 0, "encode_gemv does not support nonzero b_offset");
- assert!(b_leading_dimension.is_none_or(|ld| ld == k), "encode_gemv does not support custom b_leading_dimension");
-
- let specialization = GemvSpecialization::select(k, n, is_accumulate, output_bias.is_some());
-
- kernel.get_or_create(encoder.context(), specialization)?.encode(
- weights,
- (a, a_offset),
- output_bias,
- d,
- k,
- n,
- k,
- ab_scale,
- m,
- specialization.output_rows_per_threadgroup(),
- encoder,
- );
-
- Ok(())
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
index e7bd34d05..0aa6c8cf5 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -1,318 +1,117 @@
-#include <metal_stdlib>
-
-#include "../../common/defines.h"
#include "../../common/dsl.h"
-#include "../../common/thread_context.h"
-
-#include <metal_simdgroup>
+#include "../../generated/gemm.h"
+#include "common/b_source.h"
+#include "common/epilogue.h"
+#include "common/output_tile.h"
+#include "common/reduce.h"
using namespace metal;
-
-// Upper bound for threadgroup memory (in floats).
-// Max config: tg_simd_cols=16, output_rows_per_tg=16, thread_out_rows=4
-// → 16 * (16 + 4) = 320. Unused when tg_simd_cols == 1.
-#define GEMV_MAX_THREADGROUP_MEMORY 320
-
-template <typename WeightT, typename InputT, typename OutputT>
-VARIANTS(WeightT, bfloat, float)
-VARIANTS(InputT, bfloat, float)
-VARIANTS(OutputT, bfloat, float)
-KERNEL(MatmulGemv)(
- const device WeightT* matrix,
- const device InputT* input_vector,
- const device WeightT* output_bias OPTIONAL(is_bias),
- device OutputT* output_vector,
- const constant uint& input_dimension,
- const constant uint& output_dimension,
- const constant uint& matrix_leading_dimension,
- const constant float& output_scale,
- const constant uint& batch_rows,
- const constant uint& output_rows_per_threadgroup,
- threadgroup float threadgroup_memory[GEMV_MAX_THREADGROUP_MEMORY],
- const uint tg_simd_rows SPECIALIZE,
- const uint tg_simd_cols SPECIALIZE,
- const uint sg_thread_rows SPECIALIZE,
- const uint sg_thread_cols SPECIALIZE,
- const uint thread_out_rows SPECIALIZE,
- const uint thread_out_cols SPECIALIZE,
- const bool is_accumulate SPECIALIZE,
- const bool is_bias SPECIALIZE,
- const uint threadgroup_index_x GROUPS((output_dimension + output_rows_per_threadgroup - 1) / output_rows_per_threadgroup),
- const uint threadgroup_index_y GROUPS(batch_rows),
- const uint thread_index_x THREADS(32),
- const uint thread_index_y THREADS(16),
- const uint thread_index_z THREADS(1),
- const ThreadContext thread_context
+using namespace uzu::gemm;
+
+template <
+ typename AT,
+ typename BT,
+ typename DT,
+ GemmBPrologueKind B_PROLOGUE,
+ uint GROUP_SIZE,
+ uint BITS,
+ uint K_SPLIT,
+ bool INPUT_ALIGNED,
+ uint NUM_SIMDGROUPS>
+VARIANTS(AT, bfloat, float)
+VARIANTS(BT, bfloat, float)
+VARIANTS(DT, bfloat, float)
+CONSTRAINT(BT != "float" || (AT == "float" && DT == "float"))
+VARIANTS(
+ B_PROLOGUE,
+ GemmBPrologueKind::FullPrecision,
+ GemmBPrologueKind::ScaleBiasDequant,
+ GemmBPrologueKind::ScaleZeroPointDequant,
+ GemmBPrologueKind::ScaleSymmetricDequant)
+VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)
+VARIANTS(BITS, 0, 4, 8)
+VARIANTS(K_SPLIT, 1, 2, 4, 8)
+VARIANTS(INPUT_ALIGNED, false, true)
+VARIANTS(NUM_SIMDGROUPS, 2, 8)
+CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))
+CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))
+CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")
+CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || K_SPLIT == 1)
+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::FullPrecision || NUM_SIMDGROUPS == 8)
+KERNEL(Gemv)(
+ const device uint32_t* b,
+ const device BT* scales
+ OPTIONAL(B_PROLOGUE != GemmBPrologueKind::FullPrecision),
+ const device uint8_t* zero_points
+ OPTIONAL(B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant),
+ const device BT* biases
+ OPTIONAL(B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant),
+ const device AT* a,
+ device DT* d,
+ const device BT* output_bias
+ OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),
+ const device int32_t* hadamard_factors
+ OPTIONAL(output_transform.contains(GemmDTransform::RHT)),
+ const constant uint& in_vec_size,
+ const constant uint& out_vec_size,
+ const constant uint& batch_size,
+ const constant float& ab_scale,
+ const constant uint& group_count_x,
+ const GemmDTransform output_transform SPECIALIZE,
+ threadgroup float shared_results[NUM_SIMDGROUPS * 4],
+ const uint batch_idx GROUPS(batch_size),
+ const uint out_block_idx GROUPS(group_count_x),
+ const uint simd_lane THREADS(32),
+ const uint simd_group THREADS(NUM_SIMDGROUPS)
) {
- if (output_rows_per_threadgroup <= 0) {
- return;
- }
-
- // Simdgroup guard — kill excess simdgroups beyond what this tile uses
- if (thread_context.simdgroup_index >= tg_simd_rows * tg_simd_cols) {
- return;
- }
-
- // Derived constants from SPECIALIZE params
- const uint threads_per_threadgroup_col = tg_simd_cols * sg_thread_cols;
- const uint input_columns_per_threadgroup =
- threads_per_threadgroup_col * thread_out_cols;
-
- const uint batch_row = static_cast<uint>(threadgroup_index_y);
- if (batch_row >= batch_rows) {
- return;
- }
-
- // Thread local accumulation results
- thread float accumulated_values[4] = {0.0f, 0.0f, 0.0f, 0.0f};
- thread WeightT matrix_values[4];
- thread float vector_coefficients[4];
-
- const uint thread_row_in_simdgroup =
- sg_thread_cols != 32
- ? uint(thread_context.simd_lane_id) / uint(sg_thread_cols)
- : 0;
- const int thread_col_in_simdgroup =
- sg_thread_cols != 32
- ? uint(thread_context.simd_lane_id) % uint(sg_thread_cols)
- : uint(thread_context.simd_lane_id);
-
- const int simdgroup_column_index =
- tg_simd_cols != 1 ? uint(thread_context.simdgroup_index % tg_simd_cols)
- : 0;
-
- const uint simdgroup_row_thread_base =
- tg_simd_cols != 1
- ? uint(sg_thread_rows) *
- uint(thread_context.simdgroup_index / tg_simd_cols)
- : uint(sg_thread_rows) * uint(thread_context.simdgroup_index);
- const uint simdgroup_col_thread_base =
- tg_simd_cols != 1
- ? uint(sg_thread_cols) *
- uint(thread_context.simdgroup_index % tg_simd_cols)
- : 0;
-
- uint output_block_row_offset =
- (simdgroup_row_thread_base + thread_row_in_simdgroup) *
- uint(thread_out_rows);
- uint input_block_col_offset =
- (simdgroup_col_thread_base + thread_col_in_simdgroup) *
- uint(thread_out_cols);
-
- // Block position
- uint output_row_start =
- uint(threadgroup_index_x) * uint(output_rows_per_threadgroup) +
- output_block_row_offset;
-
- // Exit simdgroup if rows out of bound
- if (output_row_start >= output_dimension)
- return;
-
- // Adjust tail simdgroup to ensure in bound reads
- output_row_start =
- output_row_start + uint(thread_out_rows) <= output_dimension
- ? output_row_start
- : output_dimension - uint(thread_out_rows);
-
- const device WeightT* thread_matrix =
- matrix + output_row_start * matrix_leading_dimension;
-
- const uniform<uint> input_block_stride =
- make_uniform(uint(input_columns_per_threadgroup));
- const uniform<uint> input_vector_length = make_uniform(input_dimension);
- const uniform<uint> full_input_blocks =
- input_vector_length / input_block_stride;
- const uniform<uint> remaining_input_columns =
- input_vector_length - input_block_stride * full_input_blocks;
-
- // Loop over input_vector in blocks of input_columns_per_threadgroup
- for (uint input_block_index = 0; input_block_index < full_input_blocks;
- ++input_block_index) {
- // Load vector coefficients (unchecked)
- {
- const device InputT* input_vector_row =
- input_vector + batch_row * input_dimension;
- METAL_PRAGMA_UNROLL
- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
- input_col_offset++) {
- vector_coefficients[input_col_offset] = static_cast<float>(
- input_vector_row[input_block_col_offset + input_col_offset]
- );
- }
- }
-
- // Per thread work loop
- uint matrix_row_offset = 0;
- METAL_PRAGMA_UNROLL
- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
- output_row_offset++) {
- // Load matrix row (unchecked)
- METAL_PRAGMA_UNROLL
- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
- input_col_offset++) {
- matrix_values[input_col_offset] = thread_matrix
- [matrix_row_offset + input_block_col_offset + input_col_offset];
- }
-
- // Accumulate results
- METAL_PRAGMA_UNROLL
- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
- input_col_offset++) {
- accumulated_values[output_row_offset] +=
- static_cast<float>(matrix_values[input_col_offset]) *
- vector_coefficients[input_col_offset];
- }
-
- matrix_row_offset += matrix_leading_dimension;
- }
-
- input_block_col_offset += input_columns_per_threadgroup;
- }
-
- if (remaining_input_columns > 0) {
- // Load vector coefficients (checked)
- {
- const device InputT* input_vector_row =
- input_vector + batch_row * input_dimension;
- if (input_block_col_offset + uint(thread_out_cols) <=
- input_vector_length) {
- METAL_PRAGMA_UNROLL
- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
- input_col_offset++) {
- vector_coefficients[input_col_offset] = static_cast<float>(
- input_vector_row[input_block_col_offset + input_col_offset]
- );
- }
- } else {
- METAL_PRAGMA_UNROLL
- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
- input_col_offset++) {
- vector_coefficients[input_col_offset] =
- input_block_col_offset + uint(input_col_offset) <
- input_vector_length
- ? static_cast<float>(
- input_vector_row
- [input_block_col_offset + input_col_offset]
- )
- : 0.0f;
- }
- }
- }
-
- // Per thread work loop
- METAL_PRAGMA_UNROLL
- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
- output_row_offset++) {
- // Load matrix row (checked)
- if (input_block_col_offset + uint(thread_out_cols) <=
- input_vector_length) {
- METAL_PRAGMA_UNROLL
- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
- input_col_offset++) {
- matrix_values[input_col_offset] = thread_matrix
- [output_row_offset * matrix_leading_dimension +
- input_block_col_offset + input_col_offset];
- }
- } else {
- METAL_PRAGMA_UNROLL
- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
- input_col_offset++) {
- matrix_values[input_col_offset] =
- input_block_col_offset + uint(input_col_offset) <
- input_vector_length
- ? thread_matrix
- [output_row_offset * matrix_leading_dimension +
- input_block_col_offset + input_col_offset]
- : WeightT(0);
- }
- }
-
- // Accumulate results
- METAL_PRAGMA_UNROLL
- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
- input_col_offset++) {
- accumulated_values[output_row_offset] +=
- static_cast<float>(matrix_values[input_col_offset]) *
- vector_coefficients[input_col_offset];
- }
- }
- }
-
- // Simdgroup accumulations
- METAL_PRAGMA_UNROLL
- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
- output_row_offset++) {
- METAL_PRAGMA_UNROLL
- for (ushort simd_shuffle_offset = (sg_thread_cols / 2);
- simd_shuffle_offset >= 1;
- simd_shuffle_offset /= 2) {
- accumulated_values[output_row_offset] += simd_shuffle_down(
- accumulated_values[output_row_offset],
- simd_shuffle_offset
+ typedef float U;
+ thread U result[RESULTS_PER_SIMDGROUP] = {0};
+
+ OutputTile<K_SPLIT, NUM_SIMDGROUPS> tile =
+ OutputTile<K_SPLIT, NUM_SIMDGROUPS>::make(
+ out_block_idx,
+ simd_group,
+ out_vec_size
+ );
+ d += batch_idx * out_vec_size + tile.out_row;
+
+ BSource<BT, AT, U, B_PROLOGUE, GROUP_SIZE, BITS, K_SPLIT, INPUT_ALIGNED>::
+ accumulate(
+ result,
+ b,
+ scales,
+ zero_points,
+ biases,
+ a,
+ in_vec_size,
+ tile.out_row,
+ batch_idx,
+ simd_lane,
+ tile.k_slice
);
- }
- }
-
- // Threadgroup reduction (only when tg_simd_cols > 1)
- if (tg_simd_cols > 1) {
- const uint computed_output_rows_per_tg =
- uint(tg_simd_rows) * uint(sg_thread_rows) * uint(thread_out_rows);
-
- if (thread_col_in_simdgroup == 0) {
- threadgroup float* threadgroup_partial_accumulations =
- threadgroup_memory +
- simdgroup_column_index *
- (computed_output_rows_per_tg + uint(thread_out_rows)) +
- output_block_row_offset;
- METAL_PRAGMA_UNROLL
- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
- output_row_offset++) {
- threadgroup_partial_accumulations[output_row_offset] =
- accumulated_values[output_row_offset];
- }
-
- threadgroup_barrier(mem_flags::mem_none);
-
- if (simdgroup_column_index == 0) {
- threadgroup float* base_partial =
- threadgroup_memory + output_block_row_offset;
- for (uint reduction_simdgroup_col = 1;
- reduction_simdgroup_col < tg_simd_cols;
- reduction_simdgroup_col++) {
- METAL_PRAGMA_UNROLL
- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
- output_row_offset++) {
- accumulated_values[output_row_offset] += base_partial
- [reduction_simdgroup_col *
- (computed_output_rows_per_tg + uint(thread_out_rows)) +
- output_row_offset];
- }
- }
- }
- }
- }
- // Write outputs
- if (simdgroup_col_thread_base == 0 && thread_col_in_simdgroup == 0) {
- device OutputT* output_row_values =
- output_vector + batch_row * output_dimension;
- METAL_PRAGMA_UNROLL
- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
- output_row_offset++) {
- float accumulated_c =
- output_scale * accumulated_values[output_row_offset];
- if (is_accumulate) {
- accumulated_c += static_cast<float>(
- output_row_values[output_row_start + output_row_offset]
- );
- }
- if (is_bias) {
- accumulated_c += static_cast<float>(
- output_bias[output_row_start + output_row_offset]
- );
- }
- output_row_values[output_row_start + output_row_offset] =
- static_cast<OutputT>(accumulated_c);
- }
- }
+ Reduce<U, K_SPLIT, NUM_SIMDGROUPS>::run(
+ result,
+ shared_results,
+ simd_group,
+ simd_lane,
+ tile.row_group,
+ tile.k_slice
+ );
+
+ Epilogue<BT, DT, U>::store(
+ result,
+ d,
+ output_bias,
+ hadamard_factors,
+ shared_results,
+ ab_scale,
+ output_transform,
+ tile.out_row,
+ out_vec_size,
+ out_block_idx,
+ simd_group,
+ simd_lane,
+ tile.writer
+ );
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 502e4adab..11b6f572b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -1,65 +1,324 @@
-use std::collections::{HashMap, hash_map::Entry};
+use std::{
+ collections::{HashMap, hash_map::Entry},
+ sync::OnceLock,
+};
-use super::spec::GemvSpecialization;
use crate::{
backends::{
- common::kernel::matmul::MatmulError,
- metal::{Metal, context::MetalContext, kernel::MatmulGemvMetalKernel},
+ common::{
+ Allocation, AsBufferRangeRef, Buffer, Encoder,
+ gpu_types::{
+ QuantizationMethod,
+ gemm::{GemmBPrologueKind, GemmDTransform},
+ },
+ kernel::matmul::{MatmulArguments, MatmulB, MatmulError, MatmulQuantCombo},
+ },
+ metal::{Metal, context::MetalContext, kernel::GemvMetalKernel},
},
data_type::DataType,
};
-pub(crate) struct GemvKernel {
+const FP_BLOCK: u32 = 128;
+const QMV_RESULTS_PER_SIMDGROUP: u32 = 4;
+
+const DEFAULT_GEMV_MAX_BATCH: u32 = 8;
+static GEMV_MAX_BATCH: OnceLock<u32> = OnceLock::new();
+
+fn max_gemv_batch_threshold() -> u32 {
+ *GEMV_MAX_BATCH.get_or_init(|| {
+ std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)
+ })
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub(crate) struct GemvSpecialization {
+ b_prologue: GemmBPrologueKind,
+ group_size: u32,
+ bits: u32,
+ output_transform: GemmDTransform,
+ input_aligned: bool,
+ k_split: u32,
+ num_simdgroups: u32,
+}
+
+impl GemvSpecialization {
+ pub(crate) fn select<TB: AsBufferRangeRef>(
+ args: &MatmulArguments<Metal, TB>,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
+ ) -> Option<GemvSpecialization> {
+ if !args.b_transpose || args.b_offset != 0 {
+ return None;
+ }
+ let is_quant = !matches!(args.b, MatmulB::FullPrecision { .. });
+ let bad_leading_dimension = if is_quant {
+ args.b_leading_dimension.is_some()
+ } else {
+ args.b_leading_dimension.is_some_and(|ld| ld != args.k)
+ };
+ if bad_leading_dimension {
+ return None;
+ }
+ if args.d_transform.accumulate && !args.n.is_multiple_of(32) {
+ return None;
+ }
+ if args.d_transform.rht_factors.is_some() && !args.n.is_multiple_of(32) {
+ return None;
+ }
+ if is_quant {
+ if args.n < QMV_RESULTS_PER_SIMDGROUP || args.m >= 5 {
+ return None;
+ }
+ } else {
+ let mixed_precision = weights_data_type == DataType::F32
+ && (input_data_type != DataType::F32 || output_data_type != DataType::F32);
+ if mixed_precision || args.n < QMV_RESULTS_PER_SIMDGROUP || args.m > max_gemv_batch_threshold() {
+ return None;
+ }
+ }
+
+ let bits = args.b.bits_per_b().unwrap_or(0);
+ let block_size = if !is_quant {
+ FP_BLOCK
+ } else if bits == 4 {
+ 512
+ } else {
+ 256
+ };
+ let input_aligned = args.k.is_multiple_of(block_size);
+ let has_rht = args.d_transform.rht_factors.is_some();
+ let num_simdgroups = 8;
+ let k_split = if is_quant || has_rht {
+ 1
+ } else {
+ fp_k_split(args.n, args.k, input_aligned)
+ };
+ Some(GemvSpecialization {
+ b_prologue: args.b.b_prologue(),
+ group_size: args.b.group_size().unwrap_or(0),
+ bits,
+ output_transform: args.d_transform.mask(),
+ input_aligned,
+ k_split,
+ num_simdgroups,
+ })
+ }
+
+ fn quant_combo_specs(combo: MatmulQuantCombo) -> Vec<GemvSpecialization> {
+ let bits = DataType::from(combo.mode).size_in_bits() as u32;
+ let group_size = combo.group_size;
+ let b_prologue = match combo.method {
+ QuantizationMethod::ScaleBias => GemmBPrologueKind::ScaleBiasDequant,
+ QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,
+ QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,
+ };
+ let mut out = Vec::new();
+ for output_transform in [
+ GemmDTransform::empty(),
+ GemmDTransform::BIAS,
+ GemmDTransform::RHT,
+ GemmDTransform::BIAS | GemmDTransform::RHT,
+ ] {
+ for input_aligned in [true, false] {
+ out.push(GemvSpecialization {
+ b_prologue,
+ group_size,
+ bits,
+ output_transform,
+ input_aligned,
+ k_split: 1,
+ num_simdgroups: 8,
+ });
+ }
+ }
+ out
+ }
+}
+
+fn rows_per_threadgroup(
+ k_split: u32,
+ num_simdgroups: u32,
+) -> u32 {
+ (num_simdgroups / k_split) * QMV_RESULTS_PER_SIMDGROUP
+}
+
+fn fp_k_split(
+ n: u32,
+ k: u32,
+ input_aligned: bool,
+) -> u32 {
+ if !input_aligned || n >= 4096 {
+ 1
+ } else if k >= 16 * n || n <= 512 {
+ 8
+ } else if n <= 1024 || k >= 3072 {
+ 4
+ } else {
+ 2
+ }
+}
+
+pub(crate) struct GemvDispatch {
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
- pipelines: HashMap<GemvSpecialization, MatmulGemvMetalKernel>,
+ pipelines: HashMap<GemvSpecialization, GemvMetalKernel>,
}
-impl GemvKernel {
+impl GemvDispatch {
pub(crate) fn new(
- context: &MetalContext,
+ _context: &MetalContext,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
) -> Result<Self, MatmulError<Metal>> {
- let mut kernel = Self {
+ Ok(Self {
weights_data_type,
input_data_type,
output_data_type,
pipelines: HashMap::new(),
- };
- for &config in GemvSpecialization::precompile_configs(weights_data_type) {
- kernel.get_or_create(context, config)?;
+ })
+ }
+
+ pub(crate) fn preheat_quant_combo(
+ &mut self,
+ context: &MetalContext,
+ combo: MatmulQuantCombo,
+ ) -> Result<(), MatmulError<Metal>> {
+ if self.weights_data_type != DataType::BF16 {
+ return Ok(());
+ }
+ for specialization in GemvSpecialization::quant_combo_specs(combo) {
+ self.get_or_create(context, specialization)?;
}
- Ok(kernel)
+ Ok(())
}
- pub(crate) fn get_or_create(
+ fn get_or_create(
&mut self,
context: &MetalContext,
specialization: GemvSpecialization,
- ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {
+ ) -> Result<&GemvMetalKernel, MatmulError<Metal>> {
match self.pipelines.entry(specialization) {
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
- let kernel = MatmulGemvMetalKernel::new(
+ let kernel = GemvMetalKernel::new(
context,
- self.weights_data_type,
self.input_data_type,
+ self.weights_data_type,
self.output_data_type,
- specialization.threadgroup_rows,
- specialization.threadgroup_cols,
- specialization.threads_per_simdgroup_row,
- specialization.threads_per_simdgroup_col,
- specialization.elements_per_thread_row,
- specialization.elements_per_thread_col,
- specialization.is_accumulate,
- specialization.is_bias,
+ specialization.b_prologue,
+ specialization.group_size,
+ specialization.bits,
+ specialization.k_split,
+ specialization.input_aligned,
+ specialization.num_simdgroups,
+ specialization.output_transform,
)
.map_err(MatmulError::BackendError)?;
Ok(entry.insert(kernel))
},
}
}
+
+ pub(crate) fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
+ &mut self,
+ arguments: MatmulArguments<'a, Metal, TB>,
+ specialization: GemvSpecialization,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<(), MatmulError<Metal>> {
+ let ab_scale = arguments.d_transform.ab_scale;
+ let output_bias = arguments.d_transform.bias;
+ let rht_factors = arguments.d_transform.rht_factors;
+
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ d,
+ m,
+ n,
+ k,
+ ..
+ } = arguments;
+
+ let group_count_x = n.div_ceil(rows_per_threadgroup(specialization.k_split, specialization.num_simdgroups));
+
+ let context = encoder.context();
+ let pipeline = self.get_or_create(context, specialization)?;
+
+ match b {
+ MatmulB::FullPrecision {
+ b: weights,
+ } => {
+ pipeline.encode(
+ weights,
+ None::<&Allocation<Metal>>,
+ None::<&Allocation<Metal>>,
+ None::<&Allocation<Metal>>,
+ (a, a_offset),
+ &mut *d,
+ output_bias,
+ rht_factors,
+ k,
+ n,
+ m,
+ ab_scale,
+ group_count_x,
+ encoder,
+ );
+ },
+ quant_b @ (MatmulB::ScaleBiasDequant {
+ ..
+ }
+ | MatmulB::ScaleZeroPointDequant {
+ ..
+ }
+ | MatmulB::ScaleSymmetricDequant {
+ ..
+ }) => {
+ let (weights, scales, zero_points, biases) = match quant_b {
+ MatmulB::ScaleBiasDequant {
+ b: w,
+ scales,
+ biases,
+ ..
+ } => (w, scales, None, Some(biases)),
+ MatmulB::ScaleZeroPointDequant {
+ b: w,
+ scales,
+ zero_points,
+ ..
+ } => (w, scales, Some(zero_points), None),
+ MatmulB::ScaleSymmetricDequant {
+ b: w,
+ scales,
+ ..
+ } => (w, scales, None, None),
+ MatmulB::FullPrecision {
+ ..
+ } => unreachable!(),
+ };
+ pipeline.encode(
+ weights,
+ Some(scales),
+ zero_points,
+ biases,
+ (a, a_offset),
+ &mut *d,
+ output_bias,
+ rht_factors,
+ k,
+ n,
+ m,
+ ab_scale,
+ group_count_x,
+ encoder,
+ );
+ },
+ }
+
+ Ok(())
+ }
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
index d396315d2..a14cbb82f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
@@ -1,7 +1,3 @@
-pub(crate) mod fp;
mod kernel;
-mod quant_kernel;
-mod spec;
-pub(crate) use kernel::GemvKernel;
-pub use quant_kernel::QuantGemvKernel;
+pub(crate) use kernel::{GemvDispatch, GemvSpecialization};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
deleted file mode 100644
index 7fbc08019..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
+++ /dev/null
@@ -1,309 +0,0 @@
-#include <metal_stdlib>
-#include "../../common/dsl.h"
-#include "../../generated/quantization_method.h"
-#include "../gemm/common/quant_pack.h"
-#include "common/qmv_ops.h"
-
-using namespace uzu::quantization_method;
-using namespace uzu::gemm;
-
-template <
- typename WeightT,
- typename InputT,
- typename OutputT,
- uint GROUP_SIZE,
- uint BITS>
-VARIANTS(WeightT, float, bfloat)
-VARIANTS(InputT, float, bfloat)
-VARIANTS(OutputT, float, bfloat)
-VARIANTS(GROUP_SIZE, 16, 32, 64, 128)
-VARIANTS(BITS, 4, 8)
-KERNEL(QuantizedMatmulQmv)(
- const device uint32_t* weights,
- const device WeightT* scales,
- const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),
- const device WeightT* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),
- const device InputT* input,
- device OutputT* output,
- const constant uint& in_vec_size,
- const constant uint& out_vec_size,
- const constant uint& batch_size,
- const QuantizationMethod quant_method SPECIALIZE,
- const uint batch_idx GROUPS(batch_size),
- const uint out_block_idx GROUPS(out_vec_size.div_ceil(8)),
- const uint simd_lane THREADS(32),
- const uint simd_group THREADS(2)
-) {
- constexpr uint num_simdgroups = 2;
- constexpr uint results_per_simdgroup = 4;
- constexpr uint packs_per_thread = 1;
- constexpr uint pack_factor = get_pack_factor<BITS, 32>();
- constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();
-
- constexpr uint values_per_thread = pack_factor * packs_per_thread;
- constexpr uint block_size = values_per_thread * 32;
- constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;
-
- const device uint8_t* ws = (const device uint8_t*)weights;
- typedef float U;
- thread U x_thread[values_per_thread];
- thread U result[results_per_simdgroup] = {0};
-
- const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
- const uint in_vec_size_g = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;
- const device WeightT* scales_base = scales;
- const uint out_row =
- out_block_idx * (num_simdgroups * results_per_simdgroup) +
- simd_group * results_per_simdgroup;
- const uint used_out_row = min(out_vec_size - results_per_simdgroup, out_row);
-
- if (out_row >= out_vec_size) {
- return;
- }
-
- if (out_vec_size < (num_simdgroups * results_per_simdgroup)) {
- ws +=
- out_row * in_vec_size_w + simd_lane * packs_per_thread * bytes_per_pack;
- scales += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;
-
- const uint zp_stride =
- BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;
- const device uint8_t* zps_row_base = nullptr;
- const device WeightT* biases_row_base = nullptr;
- if (quant_method == QuantizationMethod::ScaleBias) {
- biases_row_base = biases + out_row * in_vec_size_g;
- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
- zps_row_base = zero_points + out_row * zp_stride;
- }
-
- input += batch_idx * in_vec_size + simd_lane * values_per_thread;
- output += batch_idx * out_vec_size + out_row;
-
- uint k = 0;
- for (; k + block_size <= in_vec_size; k += block_size) {
- U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);
-
- for (uint row = 0;
- out_row + row < out_vec_size && row < results_per_simdgroup;
- row++) {
- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
- const uint row_idx = out_row + row;
- const device WeightT* sr = scales_base + row_idx * in_vec_size_g;
-
- uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
- U s = static_cast<U>(sr[g]);
- if (quant_method == QuantizationMethod::ScaleBias) {
- const device WeightT* bl = biases_row_base + row * in_vec_size_g;
- U b = static_cast<U>(bl[g]);
- result[row] +=
- qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
- const device uint8_t* zl = zps_row_base + row * zp_stride;
- U zp;
- if (BITS == 4) {
- uint8_t zp_b = zl[g >> 1];
- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
- } else {
- zp = static_cast<U>(zl[g]);
- }
- result[row] +=
- qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
- } else {
- constexpr U midpoint = U(1u << (BITS - 1));
- result[row] += qdot<U, values_per_thread, BITS>(
- wl,
- x_thread,
- s,
- -s * midpoint,
- sum
- );
- }
- }
-
- ws += block_size * bytes_per_pack / pack_factor;
- scales += block_size / GROUP_SIZE;
- input += block_size;
- }
- const uint thread_offset = simd_lane * values_per_thread;
- const int remaining =
- (k + thread_offset < in_vec_size)
- ? min(in_vec_size - k - thread_offset, values_per_thread)
- : 0;
- if (remaining > 0) {
- U sum = load_vector_safe<InputT, U, values_per_thread, BITS>(
- input,
- x_thread,
- remaining
- );
-
- for (uint row = 0;
- out_row + row < out_vec_size && row < results_per_simdgroup;
- row++) {
- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
- const uint row_idx = out_row + row;
- const device WeightT* sr = scales_base + row_idx * in_vec_size_g;
-
- uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
- U s = static_cast<U>(sr[g]);
- if (quant_method == QuantizationMethod::ScaleBias) {
- const device WeightT* bl = biases_row_base + row * in_vec_size_g;
- U b = static_cast<U>(bl[g]);
- result[row] +=
- qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
- const device uint8_t* zl = zps_row_base + row * zp_stride;
- U zp;
- if (BITS == 4) {
- uint8_t zp_b = zl[g >> 1];
- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
- } else {
- zp = static_cast<U>(zl[g]);
- }
- result[row] +=
- qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
- } else {
- constexpr U midpoint = U(1u << (BITS - 1));
- result[row] += qdot<U, values_per_thread, BITS>(
- wl,
- x_thread,
- s,
- -s * midpoint,
- sum
- );
- }
- }
- }
-
- for (uint row = 0;
- out_row + row < out_vec_size && row < results_per_simdgroup;
- row++) {
- result[row] = simd_sum(result[row]);
- if (simd_lane == 0) {
- output[row] = static_cast<OutputT>(result[row]);
- }
- }
- } else {
- ws += used_out_row * in_vec_size_w +
- simd_lane * packs_per_thread * bytes_per_pack;
- scales += used_out_row * in_vec_size_g + simd_lane / scale_step_per_thread;
-
- const uint zp_stride =
- BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;
- const device uint8_t* zps_row_base = nullptr;
- const device WeightT* biases_row_base = nullptr;
- if (quant_method == QuantizationMethod::ScaleBias) {
- biases_row_base = biases + used_out_row * in_vec_size_g;
- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
- zps_row_base = zero_points + used_out_row * zp_stride;
- }
-
- input += batch_idx * in_vec_size + simd_lane * values_per_thread;
- output += batch_idx * out_vec_size + used_out_row;
-
- uint k = 0;
- for (; k + block_size <= in_vec_size; k += block_size) {
- U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);
-
- for (uint row = 0; row < results_per_simdgroup; row++) {
- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
- const uint row_idx = used_out_row + row;
- const device WeightT* sr = scales_base + row_idx * in_vec_size_g;
-
- uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
- U s = static_cast<U>(sr[g]);
- if (quant_method == QuantizationMethod::ScaleBias) {
- const device WeightT* bl = biases_row_base + row * in_vec_size_g;
- U b = static_cast<U>(bl[g]);
- result[row] +=
- qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
- const device uint8_t* zl = zps_row_base + row * zp_stride;
- U zp;
- if (BITS == 4) {
- uint8_t zp_b = zl[g >> 1];
- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
- } else {
- zp = static_cast<U>(zl[g]);
- }
- result[row] +=
- qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
- } else {
- constexpr U midpoint = U(1u << (BITS - 1));
- result[row] += qdot<U, values_per_thread, BITS>(
- wl,
- x_thread,
- s,
- -s * midpoint,
- sum
- );
- }
- }
-
- ws += block_size * bytes_per_pack / pack_factor;
- scales += block_size / GROUP_SIZE;
- input += block_size;
- }
- const uint thread_offset = simd_lane * values_per_thread;
- const int remaining =
- (k + thread_offset < in_vec_size)
- ? min(in_vec_size - k - thread_offset, values_per_thread)
- : 0;
-
- if (remaining > 0) {
- U sum = load_vector_safe<InputT, U, values_per_thread, BITS>(
- input,
- x_thread,
- remaining
- );
-
- for (uint row = 0; row < results_per_simdgroup; row++) {
- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
- const uint row_idx = used_out_row + row;
- const device WeightT* sr = scales_base + row_idx * in_vec_size_g;
-
- uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
- U s = static_cast<U>(sr[g]);
- if (quant_method == QuantizationMethod::ScaleBias) {
- const device WeightT* bl = biases_row_base + row * in_vec_size_g;
- U b = static_cast<U>(bl[g]);
- result[row] += qdot_safe<U, values_per_thread, BITS>(
- wl,
- x_thread,
- s,
- b,
- sum,
- remaining
- );
- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
- const device uint8_t* zl = zps_row_base + row * zp_stride;
- U zp;
- if (BITS == 4) {
- uint8_t zp_b = zl[g >> 1];
- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
- } else {
- zp = static_cast<U>(zl[g]);
- }
- result[row] +=
- qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
- } else {
- constexpr U midpoint = U(1u << (BITS - 1));
- result[row] += qdot_safe<U, values_per_thread, BITS>(
- wl,
- x_thread,
- s,
- -s * midpoint,
- sum,
- remaining
- );
- }
- }
- }
-
- for (uint row = 0; row < results_per_simdgroup; row++) {
- result[row] = simd_sum(result[row]);
- if (simd_lane == 0) {
- output[row] = static_cast<OutputT>(result[row]);
- }
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
deleted file mode 100644
index 8fe649a8a..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
+++ /dev/null
@@ -1,231 +0,0 @@
-#include <metal_stdlib>
-#include "../../common/dsl.h"
-#include "../../hadamard_transform/hadamard_transform.h"
-#include "../../generated/quantization_method.h"
-#include "../gemm/common/quant_pack.h"
-#include "common/qmv_ops.h"
-
-using namespace uzu::quantization_method;
-using namespace uzu::gemm;
-
-template <
- typename WeightT,
- typename InputT,
- typename OutputT,
- uint GROUP_SIZE,
- uint BITS>
-VARIANTS(WeightT, float, bfloat)
-VARIANTS(InputT, float, bfloat)
-VARIANTS(OutputT, float, bfloat)
-VARIANTS(GROUP_SIZE, 16, 32, 64, 128)
-VARIANTS(BITS, 4, 8)
-KERNEL(QuantizedMatmulQmvFast)(
- const device uint32_t* weights,
- const device WeightT* scales,
- const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),
- const device WeightT* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),
- const device InputT* input,
- device OutputT* output,
- const device int32_t* hadamard_factors OPTIONAL(use_hadamard),
- const constant uint& in_vec_size,
- const constant uint& out_vec_size,
- const constant uint& batch_size,
- const QuantizationMethod quant_method SPECIALIZE,
- const bool use_hadamard SPECIALIZE,
- threadgroup float shared_results[METAL_SIMD_SIZE],
- const uint batch_idx GROUPS(batch_size),
- const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),
- const uint simd_lane THREADS(32),
- const uint simd_group THREADS(8)
-) {
- constexpr uint packs_per_thread = BITS == 2 ? 1 : 2;
- constexpr uint num_simdgroups = 8;
- constexpr uint results_per_simdgroup = 4;
- constexpr uint pack_factor = get_pack_factor<BITS, 32>();
- constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();
- constexpr uint values_per_thread = pack_factor * packs_per_thread;
- constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;
- constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;
- const device uint8_t* ws = (const device uint8_t*)weights;
- typedef float U;
- thread U x_thread[values_per_thread];
- thread U result[results_per_simdgroup] = {0};
-
- const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
- const uint in_vec_size_g = in_vec_size / GROUP_SIZE;
- const uint out_row =
- out_block_idx * (num_simdgroups * results_per_simdgroup) +
- simd_group * results_per_simdgroup;
- ws += out_row * in_vec_size_w + simd_lane * packs_per_thread * bytes_per_pack;
- scales += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;
-
- uint zp_stride = 0;
- const device uint8_t* zps = nullptr;
- bool high_nibble = false;
-
- if (quant_method == QuantizationMethod::ScaleBias) {
- biases += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;
- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
- if (BITS == 4) {
- zp_stride = (in_vec_size_g + 1) / 2;
- zps = zero_points + out_row * zp_stride;
- uint g_offset = simd_lane / scale_step_per_thread;
- zps += g_offset / 2;
- high_nibble = (g_offset & 1);
- } else {
- zp_stride = in_vec_size_g;
- zps = zero_points + out_row * zp_stride;
- zps += simd_lane / scale_step_per_thread;
- }
- }
-
- input += batch_idx * in_vec_size + simd_lane * values_per_thread;
- output += batch_idx * out_vec_size + out_row;
-
- for (uint k = 0; k < in_vec_size; k += block_size) {
- U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);
-
- {
- auto wl0 = (const device uint8_t*)(ws);
- auto wl1 = (const device uint8_t*)(ws + in_vec_size_w);
- auto wl2 = (const device uint8_t*)(ws + 2 * in_vec_size_w);
- auto wl3 = (const device uint8_t*)(ws + 3 * in_vec_size_w);
-
- U s0 = static_cast<U>(scales[0]);
- U s1 = static_cast<U>(scales[in_vec_size_g]);
- U s2 = static_cast<U>(scales[2 * in_vec_size_g]);
- U s3 = static_cast<U>(scales[3 * in_vec_size_g]);
-
- if (quant_method == QuantizationMethod::ScaleBias) {
- U b0 = static_cast<U>(biases[0]);
- U b1 = static_cast<U>(biases[in_vec_size_g]);
- U b2 = static_cast<U>(biases[2 * in_vec_size_g]);
- U b3 = static_cast<U>(biases[3 * in_vec_size_g]);
- result[0] +=
- qdot<U, values_per_thread, BITS>(wl0, x_thread, s0, b0, sum);
- result[1] +=
- qdot<U, values_per_thread, BITS>(wl1, x_thread, s1, b1, sum);
- result[2] +=
- qdot<U, values_per_thread, BITS>(wl2, x_thread, s2, b2, sum);
- result[3] +=
- qdot<U, values_per_thread, BITS>(wl3, x_thread, s3, b3, sum);
- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
- uchar4 zp_bytes = uchar4(
- zps[0],
- zps[zp_stride],
- zps[2 * zp_stride],
- zps[3 * zp_stride]
- );
- uchar4 zp_nibbles;
- if (BITS == 4) {
- const uint8_t shift = high_nibble ? 4u : 0u;
- zp_nibbles = (zp_bytes >> shift) & uchar4(0x0F);
- } else {
- zp_nibbles = zp_bytes;
- }
- result[0] += qdot<U, values_per_thread, BITS>(
- wl0,
- x_thread,
- s0,
- -s0 * static_cast<U>(zp_nibbles.x),
- sum
- );
- result[1] += qdot<U, values_per_thread, BITS>(
- wl1,
- x_thread,
- s1,
- -s1 * static_cast<U>(zp_nibbles.y),
- sum
- );
- result[2] += qdot<U, values_per_thread, BITS>(
- wl2,
- x_thread,
- s2,
- -s2 * static_cast<U>(zp_nibbles.z),
- sum
- );
- result[3] += qdot<U, values_per_thread, BITS>(
- wl3,
- x_thread,
- s3,
- -s3 * static_cast<U>(zp_nibbles.w),
- sum
- );
- } else {
- constexpr U midpoint = U(1u << (BITS - 1));
- result[0] += qdot<U, values_per_thread, BITS>(
- wl0,
- x_thread,
- s0,
- -s0 * midpoint,
- sum
- );
- result[1] += qdot<U, values_per_thread, BITS>(
- wl1,
- x_thread,
- s1,
- -s1 * midpoint,
- sum
- );
- result[2] += qdot<U, values_per_thread, BITS>(
- wl2,
- x_thread,
- s2,
- -s2 * midpoint,
- sum
- );
- result[3] += qdot<U, values_per_thread, BITS>(
- wl3,
- x_thread,
- s3,
- -s3 * midpoint,
- sum
- );
- }
- }
-
- ws += block_size * bytes_per_pack / pack_factor;
- scales += block_size / GROUP_SIZE;
- if (quant_method == QuantizationMethod::ScaleBias) {
- biases += block_size / GROUP_SIZE;
- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {
- if (BITS == 4) {
- zps += (block_size / GROUP_SIZE) / 2;
- } else {
- zps += block_size / GROUP_SIZE;
- }
- }
- input += block_size;
- }
-
- for (uint row = 0; row < results_per_simdgroup; row++) {
- result[row] = simd_sum(result[row]);
- }
-
- if (use_hadamard) {
- if (simd_lane == 0) {
- for (uint row = 0; row < results_per_simdgroup; row++) {
- shared_results[simd_group * results_per_simdgroup + row] = result[row];
- }
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (simd_group == 0) {
- uint global_out_idx = out_block_idx * 32 + simd_lane;
- if (global_out_idx < out_vec_size) {
- output[simd_lane] = simdgroup_output_random_hadamard_transform(
- static_cast<ushort>(simd_lane),
- OutputT(shared_results[simd_lane]),
- hadamard_factors[global_out_idx]
- );
- }
- }
- } else {
- if (simd_lane == 0) {
- for (uint row = 0; row < results_per_simdgroup; row++) {
- output[row] = static_cast<OutputT>(result[row]);
- }
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
deleted file mode 100644
index 9e456cd82..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
+++ /dev/null
@@ -1,192 +0,0 @@
-use std::collections::{HashMap, hash_map::Entry};
-
-use crate::{
- backends::{
- common::{
- AsBufferRangeRef, Buffer, Encoder,
- gpu_types::{QuantizationMethod, QuantizationMode, gemm::GemmDTransform},
- kernel::matmul::{MatmulArguments, MatmulB, MatmulError},
- },
- metal::{
- Metal,
- context::MetalContext,
- kernel::{QuantizedMatmulQmvFastMetalKernel, QuantizedMatmulQmvMetalKernel},
- },
- },
- data_type::DataType,
-};
-
-const PATH: &str = "QuantGemv";
-
-#[derive(PartialEq, Eq, Hash, Clone, Copy)]
-struct QmvKey {
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
- use_hadamard: bool,
-}
-
-pub struct QuantGemvKernel {
- weights_data_type: DataType,
- input_data_type: DataType,
- output_data_type: DataType,
- qmv: HashMap<QmvKey, QuantizedMatmulQmvMetalKernel>,
- qmv_fast: HashMap<QmvKey, QuantizedMatmulQmvFastMetalKernel>,
-}
-
-impl QuantGemvKernel {
- pub fn new(
- _context: &MetalContext,
- weights_data_type: DataType,
- input_data_type: DataType,
- output_data_type: DataType,
- ) -> Self {
- Self {
- weights_data_type,
- input_data_type,
- output_data_type,
- qmv: HashMap::new(),
- qmv_fast: HashMap::new(),
- }
- }
-
- pub fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
- &mut self,
- encoder: &mut Encoder<Metal>,
- arguments: MatmulArguments<'a, Metal, TB>,
- ) -> Result<(), MatmulError<Metal>> {
- let mask = arguments.d_transform.mask();
- if mask.contains(GemmDTransform::SCALE) {
- return Err(MatmulError::UnsupportedDOp {
- bit: GemmDTransform::SCALE,
- path: PATH,
- });
- }
- if mask.contains(GemmDTransform::ACCUMULATE) {
- return Err(MatmulError::UnsupportedDOp {
- bit: GemmDTransform::ACCUMULATE,
- path: PATH,
- });
- }
- if !arguments.b_transpose || arguments.b_leading_dimension.is_some() {
- return Err(MatmulError::UnsupportedLayout {
- path: PATH,
- });
- }
-
- let hadamard_factors = arguments.d_transform.rht_factors;
-
- let MatmulArguments {
- a,
- a_offset,
- b,
- d,
- m,
- n,
- k,
- ..
- } = arguments;
- let (weights, scales, zp_or_bias, method, mode, group_size) = match b {
- MatmulB::ScaleBiasDequant {
- b: w,
- scales,
- biases,
- mode,
- group_size,
- } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),
- MatmulB::ScaleZeroPointDequant {
- b: w,
- scales,
- zero_points,
- mode,
- group_size,
- } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),
- MatmulB::ScaleSymmetricDequant {
- b: w,
- scales,
- mode,
- group_size,
- } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),
- MatmulB::FullPrecision {
- ..
- } => panic!("QuantGemvKernel requires quantized B"),
- };
-
- let bits = match mode {
- QuantizationMode::U4 => 4u32,
- QuantizationMode::I8 | QuantizationMode::U8 => 8u32,
- };
- let use_fast = n % 8 == 0 && k % 512 == 0;
-
- let (zero_points, biases) = match method {
- QuantizationMethod::ScaleBias => {
- (None, Some(zp_or_bias.expect("ScaleBias quantized matmul requires biases")))
- },
- QuantizationMethod::ScaleZeroPoint => {
- (Some(zp_or_bias.expect("ScaleZeroPoint quantized matmul requires zero_points")), None)
- },
- QuantizationMethod::ScaleSymmetric => (None, None),
- };
-
- if use_fast {
- let key = QmvKey {
- group_size,
- bits,
- quant_method: method,
- use_hadamard: hadamard_factors.is_some(),
- };
- let context = encoder.context();
- let kernel = match self.qmv_fast.entry(key) {
- Entry::Occupied(entry) => entry.into_mut(),
- Entry::Vacant(entry) => {
- let kernel = QuantizedMatmulQmvFastMetalKernel::new(
- context,
- self.weights_data_type,
- self.input_data_type,
- self.output_data_type,
- group_size,
- bits,
- method,
- hadamard_factors.is_some(),
- )
- .map_err(MatmulError::BackendError)?;
- entry.insert(kernel)
- },
- };
- kernel.encode(weights, scales, zero_points, biases, (a, a_offset), d, hadamard_factors, k, n, m, encoder);
- } else {
- if hadamard_factors.is_some() {
- return Err(MatmulError::UnsupportedDOp {
- bit: GemmDTransform::RHT,
- path: PATH,
- });
- }
- let key = QmvKey {
- group_size,
- bits,
- quant_method: method,
- use_hadamard: false,
- };
- let context = encoder.context();
- let kernel = match self.qmv.entry(key) {
- Entry::Occupied(entry) => entry.into_mut(),
- Entry::Vacant(entry) => {
- let kernel = QuantizedMatmulQmvMetalKernel::new(
- context,
- self.weights_data_type,
- self.input_data_type,
- self.output_data_type,
- group_size,
- bits,
- method,
- )
- .map_err(MatmulError::BackendError)?;
- entry.insert(kernel)
- },
- };
- kernel.encode(weights, scales, zero_points, biases, (a, a_offset), d, k, n, m, encoder);
- }
-
- Ok(())
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
deleted file mode 100644
index 1cc022959..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
+++ /dev/null
@@ -1,142 +0,0 @@
-use crate::data_type::DataType;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct GemvSpecialization {
- pub threadgroup_rows: u32,
- pub threadgroup_cols: u32,
- pub threads_per_simdgroup_row: u32,
- pub threads_per_simdgroup_col: u32,
- pub elements_per_thread_row: u32,
- pub elements_per_thread_col: u32,
- pub is_accumulate: bool,
- pub is_bias: bool,
-}
-
-impl GemvSpecialization {
- pub fn precompile_configs(weights_data_type: DataType) -> &'static [Self] {
- match weights_data_type {
- DataType::F32 => &[
- Self {
- threadgroup_rows: 4,
- threadgroup_cols: 1,
- threads_per_simdgroup_row: 1,
- threads_per_simdgroup_col: 32,
- elements_per_thread_row: 4,
- elements_per_thread_col: 4,
- is_accumulate: false,
- is_bias: false,
- },
- Self {
- threadgroup_rows: 4,
- threadgroup_cols: 1,
- threads_per_simdgroup_row: 1,
- threads_per_simdgroup_col: 32,
- elements_per_thread_row: 4,
- elements_per_thread_col: 4,
- is_accumulate: false,
- is_bias: true,
- },
- ],
- DataType::BF16 => &[
- Self {
- threadgroup_rows: 4,
- threadgroup_cols: 1,
- threads_per_simdgroup_row: 1,
- threads_per_simdgroup_col: 32,
- elements_per_thread_row: 4,
- elements_per_thread_col: 4,
- is_accumulate: false,
- is_bias: false,
- },
- Self {
- threadgroup_rows: 4,
- threadgroup_cols: 1,
- threads_per_simdgroup_row: 1,
- threads_per_simdgroup_col: 32,
- elements_per_thread_row: 4,
- elements_per_thread_col: 4,
- is_accumulate: false,
- is_bias: true,
- },
- Self {
- threadgroup_rows: 8,
- threadgroup_cols: 1,
- threads_per_simdgroup_row: 1,
- threads_per_simdgroup_col: 32,
- elements_per_thread_row: 4,
- elements_per_thread_col: 4,
- is_accumulate: false,
- is_bias: false,
- },
- Self {
- threadgroup_rows: 8,
- threadgroup_cols: 1,
- threads_per_simdgroup_row: 1,
- threads_per_simdgroup_col: 32,
- elements_per_thread_row: 4,
- elements_per_thread_col: 4,
- is_accumulate: false,
- is_bias: true,
- },
- ],
- _ => &[],
- }
- }
-
- pub fn output_rows_per_threadgroup(&self) -> u32 {
- self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row
- }
-
- pub fn select(
- input_dimension: u32,
- output_dimension: u32,
- is_accumulate: bool,
- is_bias: bool,
- ) -> Self {
- let (threadgroup_rows, threadgroup_cols);
- let (threads_per_simdgroup_row, threads_per_simdgroup_col);
- let (elements_per_thread_row, elements_per_thread_col);
-
- let threadgroup_simd_rows;
- let mut simdgroup_thread_rows = 1;
- let mut simdgroup_thread_cols = 32;
- let mut threadgroup_simd_cols = 1;
-
- if input_dimension <= 64 {
- threadgroup_simd_rows = 1;
- simdgroup_thread_rows = 8;
- simdgroup_thread_cols = 4;
- } else if input_dimension >= 16 * output_dimension {
- threadgroup_simd_rows = 1;
- threadgroup_simd_cols = 8;
- } else if output_dimension >= 4096 {
- threadgroup_simd_rows = 8;
- } else {
- threadgroup_simd_rows = 4;
- }
-
- let thread_output_rows = if output_dimension < 4 {
- 1
- } else {
- 4
- };
-
- threadgroup_rows = threadgroup_simd_rows;
- threadgroup_cols = threadgroup_simd_cols;
- threads_per_simdgroup_row = simdgroup_thread_rows;
- threads_per_simdgroup_col = simdgroup_thread_cols;
- elements_per_thread_row = thread_output_rows;
- elements_per_thread_col = 4;
-
- Self {
- threadgroup_rows,
- threadgroup_cols,
- threads_per_simdgroup_row,
- threads_per_simdgroup_col,
- elements_per_thread_row,
- elements_per_thread_col,
- is_accumulate,
- is_bias,
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index 982c80ca7..b66dca74d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -1,22 +1,13 @@
pub mod gemm;
pub mod gemv;
-use std::sync::OnceLock;
-
-use self::gemv::GemvKernel;
-pub use self::{
- gemm::{GemmDispatchPath, GemmKernel},
- gemv::QuantGemvKernel,
-};
+pub use self::gemm::{GemmDispatchPath, GemmKernel};
+use self::gemv::{GemvDispatch, GemvSpecialization};
use crate::{
backends::{
common::{
- AsBufferRangeRef, Backend, Buffer, Encoder,
- gpu_types::gemm::GemmDTransform,
- kernel::{
- HadamardTransformKernel, TensorAddBiasKernel,
- matmul::{MatmulArguments, MatmulB, MatmulError, MatmulKernel, MatmulQuantCombo},
- },
+ AsBufferRangeRef, Buffer, Encoder,
+ kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulQuantCombo},
},
metal::{Metal, context::MetalContext, error::MetalError},
},
@@ -24,118 +15,11 @@ use crate::{
};
pub struct MatmulMetalKernel {
- gemv: GemvKernel,
- pub quant_gemv: QuantGemvKernel,
+ gemv: GemvDispatch,
pub gemm: GemmKernel,
-}
-
-const DEFAULT_GEMV_MAX_BATCH: u32 = 8;
-static GEMV_MAX_BATCH: OnceLock<u32> = OnceLock::new();
-
-fn max_gemv_batch_threshold() -> u32 {
- *GEMV_MAX_BATCH.get_or_init(|| {
- std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)
- })
-}
-
-impl MatmulMetalKernel {
- fn dispatch_gemv<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
- &mut self,
- arguments: MatmulArguments<'a, Metal, TB>,
- encoder: &mut Encoder<Metal>,
- ) -> Result<(), MatmulError<Metal>> {
- let post_rht = arguments.d_transform.rht_factors;
-
- let MatmulArguments {
- a,
- a_offset,
- b,
- b_offset,
- b_leading_dimension,
- b_transpose,
- d,
- d_transform,
- m,
- n,
- k,
- } = arguments;
-
- // A temopary hack, will port the post ops to the gemv later.
- let d_transform = d_transform.without(GemmDTransform::RHT);
-
- gemv::fp::encode(
- &mut self.gemv,
- encoder,
- MatmulArguments {
- a,
- a_offset,
- b,
- b_offset,
- b_leading_dimension,
- b_transpose,
- d: &mut *d,
- d_transform,
- m,
- n,
- k,
- },
- )?;
-
- if let Some(factors) = post_rht {
- self.gemm.hadamard.encode(d, factors, n, m, encoder);
- }
- Ok(())
- }
-
- fn dispatch_quant_gemv<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
- &mut self,
- arguments: MatmulArguments<'a, Metal, TB>,
- encoder: &mut Encoder<Metal>,
- ) -> Result<(), MatmulError<Metal>> {
- if arguments.b_offset != 0 {
- return Err(MatmulError::UnsupportedLayout {
- path: "QuantGemv",
- });
- }
- let post_bias = arguments.d_transform.bias;
-
- let MatmulArguments {
- a,
- a_offset,
- b,
- b_offset,
- b_leading_dimension,
- b_transpose,
- d,
- d_transform,
- m,
- n,
- k,
- } = arguments;
- let d_transform = d_transform.without(GemmDTransform::BIAS);
-
- self.quant_gemv.encode(
- encoder,
- MatmulArguments {
- a,
- a_offset,
- b,
- b_offset,
- b_leading_dimension,
- b_transpose,
- d: &mut *d,
- d_transform,
- m,
- n,
- k,
- },
- )?;
-
- if let Some(bias) = post_bias {
- self.gemm.bias_add.encode(None::<&<Metal as Backend>::DenseBuffer>, bias, d, n, m * n, encoder);
- }
- Ok(())
- }
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
}
impl MatmulKernel for MatmulMetalKernel {
@@ -154,13 +38,15 @@ impl MatmulKernel for MatmulMetalKernel {
}
let gemm = GemmKernel::new(context, weights_data_type, input_data_type, output_data_type)?;
- let gemv = GemvKernel::new(context, weights_data_type, input_data_type, output_data_type)?;
- let quant_gemv = QuantGemvKernel::new(context, weights_data_type, input_data_type, output_data_type);
+ let gemv = GemvDispatch::new(context, weights_data_type, input_data_type, output_data_type)
+ .map_err(MetalError::from)?;
Ok(Self {
gemv,
- quant_gemv,
gemm,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
})
}
@@ -169,25 +55,14 @@ impl MatmulKernel for MatmulMetalKernel {
arguments: MatmulArguments<Metal, TB>,
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
- let is_quant = !matches!(arguments.b, MatmulB::FullPrecision { .. });
- let gemv_eligible = if is_quant {
- arguments.m < 5 || arguments.n == 1
- } else {
- arguments.b_transpose
- && arguments.b_offset == 0
- && arguments.b_leading_dimension.is_none_or(|ld| ld == arguments.k)
- && arguments.m <= max_gemv_batch_threshold()
- };
-
- if gemv_eligible {
- if is_quant {
- self.dispatch_quant_gemv(arguments, encoder)?;
- } else {
- self.dispatch_gemv(arguments, encoder)?;
- }
- Ok(())
- } else {
- self.gemm.encode(arguments, encoder)
+ match GemvSpecialization::select(
+ &arguments,
+ self.weights_data_type,
+ self.input_data_type,
+ self.output_data_type,
+ ) {
+ Some(spec) => self.gemv.encode(arguments, spec, encoder).map_err(MetalError::from),
+ None => self.gemm.encode(arguments, encoder),
}
}
@@ -196,6 +71,7 @@ impl MatmulKernel for MatmulMetalKernel {
context: &MetalContext,
combo: MatmulQuantCombo,
) -> Result<(), MetalError> {
+ self.gemv.preheat_quant_combo(context, combo).map_err(MetalError::from)?;
self.gemm.preheat_quant_combo(context, combo)
}
}
diff --git a/crates/backend-uzu/tests/common/matmul/shape.rs b/crates/backend-uzu/tests/common/matmul/shape.rs
index 33e7d58ef..d0760c519 100644
--- a/crates/backend-uzu/tests/common/matmul/shape.rs
+++ b/crates/backend-uzu/tests/common/matmul/shape.rs
@@ -72,7 +72,7 @@ pub fn bench_quant_gemv_shapes(bits: u32) -> impl Iterator<Item = Shape> {
256
};
let nk: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];
- let ms = &[1usize, 2, 4, 8];
+ let ms = &[1usize, 2, 4];
nk.iter()
.filter(move |&&(n, k)| n % 8 == 0 && k % block_size == 0)
.flat_map(move |&(n, k)| ms.iter().map(move |&m| Shape::new(m, k, n)))
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs
index 85b56a68b..109ff2d45 100644
--- a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs
@@ -1,14 +1,9 @@
-#![cfg(metal_backend)]
-
use backend_uzu::{
array::ArrayElement,
- backends::{
- common::{
- Backend,
- gpu_types::QuantizationMethod,
- kernel::{Kernels, matmul::MatmulKernel},
- },
- metal::{Metal, MetalContext},
+ backends::common::{
+ Backend, Context,
+ gpu_types::QuantizationMethod,
+ kernel::{Kernels, matmul::MatmulKernel},
},
};
use criterion::{BenchmarkId, Criterion, Throughput};
@@ -23,21 +18,21 @@ use crate::{
uzu_bench,
};
-fn bench_quant_gemv_typed<T: ArrayElement + Float>(
- context: &MetalContext,
+fn bench_gemv_typed<B: Backend, T: ArrayElement + Float>(
c: &mut Criterion,
+ context: &B::Context,
label: &str,
group_size: u32,
bits: u32,
quant_method: QuantizationMethod,
) {
- let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/QuantGemv/{}", type_short_name::<Metal>(), label));
+ let mut group = c.benchmark_group(format!("{}/Kernel/Gemv/{}", type_short_name::<B>(), label));
for shape in bench_quant_gemv_shapes(bits) {
let (m, k, n) = (shape.m, shape.k, shape.n);
let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);
- let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);
- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ let mut buffers = QuantBuffers::<B, T>::allocate(context, &input);
+ let mut matmul = <<B as Backend>::Kernels as Kernels>::MatmulKernel::new(
context,
T::data_type(),
T::data_type(),
@@ -47,18 +42,24 @@ fn bench_quant_gemv_typed<T: ArrayElement + Float>(
group.throughput(Throughput::Elements((m * n * k) as u64));
group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {
- iter_encode_loop::<Metal, _>(context, b, |encoder| {
- matmul.quant_gemv.encode(encoder, quant_arguments(&mut buffers, &input)).expect("encode quant gemv");
+ iter_encode_loop::<B, _>(context, b, |encoder| {
+ let args = quant_arguments(&mut buffers, &input);
+ matmul.encode(args, encoder).expect("encode failed");
});
});
}
}
#[uzu_bench]
-fn bench_quant_gemv(c: &mut Criterion) {
- let context = crate::common::shared_metal_context();
- bench_quant_gemv_typed::<bf16>(&context, c, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
- bench_quant_gemv_typed::<bf16>(&context, c, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- bench_quant_gemv_typed::<bf16>(&context, c, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
- bench_quant_gemv_typed::<bf16>(&context, c, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
+fn bench_gemv(c: &mut Criterion) {
+ for_each_backend!(|B| {
+ let context = <B as Backend>::Context::new().unwrap();
+ bench_gemv_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);
+ bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_gemv_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
+ bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_gemv_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
+ bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);
+ });
}
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
index a42f5d912..15951e0af 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
@@ -132,3 +132,46 @@ fn bias_parity_bf16() {
}
}
}
+
+#[test]
+fn gemv_fp_partial_output_block_bf16() {
+ use crate::common::matmul::Shape;
+ let context = MetalContext::new().expect("Metal context");
+ let mut kernel = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("MatmulKernel");
+ // Small m routes to GEMV; n that is not a multiple of the 32-row threadgroup
+ // block exercises the tail-row clamp (must not read weight rows past B).
+ let shapes = [Shape::new(1, 128, 33), Shape::new(2, 256, 65), Shape::new(4, 384, 100)];
+ for shape in shapes {
+ check_case::<bf16>(&context, &mut kernel, None, Case::new(shape), 1.0);
+ }
+}
+
+#[test]
+fn gemv_fp_output_transforms_bf16() {
+ use crate::common::matmul::Shape;
+ let context = MetalContext::new().expect("Metal context");
+ let mut kernel = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("MatmulKernel");
+ // Small m routes to GEMV; n % 32 == 0 keeps RHT/accumulate in the GEMV path.
+ let shape = Shape::new(2, 256, 64);
+ let cases = [
+ Case::new(shape).with_bias(true).with_rht(true),
+ Case::new(shape).with_accumulate(true),
+ Case::new(shape).with_accumulate(true).with_bias(true),
+ Case::new(shape).with_ab_scale(2.0).with_rht(true),
+ ];
+ for case in cases {
+ check_case::<bf16>(&context, &mut kernel, None, case, 1.0);
+ }
+}
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
index 0d1e0fbd6..87bca2467 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
@@ -16,6 +16,7 @@ use backend_uzu::{
matmul::{MatmulDOps, MatmulError, MatmulKernel},
},
},
+ cpu::Cpu,
metal::{GemmDispatchPath, Metal, MetalContext},
},
};
@@ -114,6 +115,54 @@ fn parity_bf16(
run_parity::<bf16>(m, k, n, gs, bits, method, 0.05, 0.4);
}
+fn run_parity_gemv<T: ArrayElement + Float + Debug + Display>(
+ m: usize,
+ k: usize,
+ n: usize,
+ group_size: u32,
+ bits: u32,
+ quant_method: QuantizationMethod,
+ rel_tol: f64,
+ abs_tol: f64,
+) {
+ let context = MetalContext::new().expect("Metal context");
+ let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 0);
+ let gemv = run_quant_metal::<T>(&context, &input, None);
+ let reference = run_quant_cpu::<T>(&input);
+ assert_parity::<T>(
+ &format!(
+ "gemv m={m} k={k} n={n} gs={group_size} bits={bits} method={quant_method:?} dtype={}",
+ std::any::type_name::<T>()
+ ),
+ &reference,
+ &gemv,
+ rel_tol,
+ abs_tol,
+ );
+}
+
+#[rstest]
+#[case::m1_gs32_4bit_mlx(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
+#[case::m1_gs64_4bit_mlx(1, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]
+#[case::m1_gs128_4bit_mlx(1, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]
+#[case::m1_gs32_4bit_zp(1, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]
+#[case::m1_gs64_8bit_zp(1, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
+#[case::m1_gs128_8bit_mlx(1, 256, 64, 128, 8, QuantizationMethod::ScaleBias)]
+#[case::m1_gs32_4bit_sym(1, 256, 64, 32, 4, QuantizationMethod::ScaleSymmetric)]
+#[case::m1_gs64_8bit_sym(1, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]
+#[case::m2_gs32_4bit_mlx(2, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
+#[case::m4_gs32_4bit_zp(4, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]
+fn parity_gemv_bf16(
+ #[case] m: usize,
+ #[case] k: usize,
+ #[case] n: usize,
+ #[case] gs: u32,
+ #[case] bits: u32,
+ #[case] method: QuantizationMethod,
+) {
+ run_parity_gemv::<bf16>(m, k, n, gs, bits, method, 0.05, 0.4);
+}
+
#[uzu_test]
fn parity_bf16_gs32_4bit_mlx_with_bias() {
let context = MetalContext::new().expect("Metal context");
@@ -154,6 +203,213 @@ fn parity_bf16_gs32_4bit_mlx_with_bias() {
assert_parity::<bf16>("with_bias", &reference, &actual, 0.05, 0.4);
}
+#[uzu_test]
+fn parity_bf16_gemv_qmv_fused_scale_bias() {
+ let context = MetalContext::new().expect("Metal context");
+ let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
+
+ let scale = 2.0_f32;
+ let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.25 + 0.1 * (j % 7) as f32).collect();
+ let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();
+
+ let raw = run_quant_cpu::<bf16>(&input);
+ let reference: Vec<bf16> = (0..input.m as usize)
+ .flat_map(|i| {
+ (0..input.n as usize).map(move |j| {
+ let idx = i * input.n as usize + j;
+ (idx, j)
+ })
+ })
+ .map(|(idx, j)| bf16::from_f32(scale * raw[idx].to_f32() + bias_f32[j]))
+ .collect();
+
+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
+ let bias_buf = crate::common::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);
+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("MatmulMetalKernel");
+
+ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
+ let mut args = quant_arguments(&mut buffers, &input);
+ args.d_transform = MatmulDOps {
+ ab_scale: scale,
+ accumulate: false,
+ bias: Some(&bias_buf),
+ rht_factors: None,
+ };
+ matmul.encode(args, &mut encoder).expect("encode quant gemv with scale+bias");
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);
+
+ assert_parity::<bf16>("gemv_qmv_scale_bias", &reference, &actual, 0.05, 0.4);
+}
+
+#[rstest]
+#[case::gs64_4bit(1, 96, 64, 64, 4, QuantizationMethod::ScaleBias)]
+#[case::gs64_4bit_zp(2, 96, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs128_8bit(1, 192, 64, 128, 8, QuantizationMethod::ScaleBias)]
+fn parity_gemv_partial_group_bf16(
+ #[case] m: usize,
+ #[case] k: usize,
+ #[case] n: usize,
+ #[case] gs: u32,
+ #[case] bits: u32,
+ #[case] method: QuantizationMethod,
+) {
+ // k is not a multiple of group_size, so the per-row scale/zp stride must use
+ // ceil(k / group_size) groups.
+ run_parity_gemv::<bf16>(m, k, n, gs, bits, method, 0.05, 0.6);
+}
+
+#[rstest]
+#[case::n12_gs32_4bit(1, 256, 12, 32, 4, QuantizationMethod::ScaleBias)]
+#[case::n20_gs64_4bit_zp(2, 256, 20, 64, 4, QuantizationMethod::ScaleZeroPoint)]
+#[case::n36_gs32_8bit(1, 256, 36, 32, 8, QuantizationMethod::ScaleBias)]
+fn parity_gemv_unaligned_width_bf16(
+ #[case] m: usize,
+ #[case] k: usize,
+ #[case] n: usize,
+ #[case] gs: u32,
+ #[case] bits: u32,
+ #[case] method: QuantizationMethod,
+) {
+ // n is not a multiple of 8; the output-tail clamp must cover the partial block.
+ run_parity_gemv::<bf16>(m, k, n, gs, bits, method, 0.05, 0.6);
+}
+
+#[uzu_test]
+fn parity_bf16_gemv_quant_rht_with_bias() {
+ let context = MetalContext::new().expect("Metal context");
+ let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
+ let rht: Vec<i32> = (0..input.n as usize)
+ .map(|i| {
+ if i % 2 == 0 {
+ 1
+ } else {
+ -1
+ }
+ })
+ .collect();
+ let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.3 + 0.05 * (j % 4) as f32).collect();
+ let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();
+
+ // CPU reference applies bias before RHT (matmul loop adds bias, then the
+ // Hadamard pass transforms the result) — the same order GEMV uses.
+ let cpu_context = <Cpu as Backend>::Context::new().expect("Cpu context");
+ let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);
+ let cpu_rht = crate::common::helpers::alloc_allocation_with_data::<Cpu, i32>(&cpu_context, &rht);
+ let cpu_bias = crate::common::helpers::alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, &bias_t);
+ let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &cpu_context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("MatmulCpuKernel");
+ let mut cpu_encoder = Encoder::<Cpu>::new(&cpu_context).expect("cpu encoder");
+ let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);
+ cpu_args.d_transform = MatmulDOps {
+ ab_scale: 1.0,
+ accumulate: false,
+ bias: Some(&cpu_bias),
+ rht_factors: Some(&cpu_rht),
+ };
+ cpu_matmul.encode(cpu_args, &mut cpu_encoder).expect("cpu encode quant+rht+bias");
+ cpu_encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let reference = allocation_to_vec::<Cpu, bf16>(&cpu_buffers.y);
+
+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
+ let metal_rht = crate::common::helpers::alloc_allocation_with_data::<Metal, i32>(&context, &rht);
+ let metal_bias = crate::common::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);
+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("MatmulMetalKernel");
+ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
+ let mut args = quant_arguments(&mut buffers, &input);
+ args.d_transform = MatmulDOps {
+ ab_scale: 1.0,
+ accumulate: false,
+ bias: Some(&metal_bias),
+ rht_factors: Some(&metal_rht),
+ };
+ matmul.encode(args, &mut encoder).expect("encode quant gemv with rht+bias");
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);
+
+ assert_parity::<bf16>("gemv_quant_rht_bias", &reference, &actual, 0.05, 0.6);
+}
+
+#[uzu_test]
+fn parity_bf16_gemv_quant_rht() {
+ let context = MetalContext::new().expect("Metal context");
+ // n % 32 == 0 so the output RHT covers whole 32-element blocks; m = 1 routes to GEMV.
+ let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
+ let rht: Vec<i32> = (0..input.n as usize)
+ .map(|i| {
+ if i % 2 == 0 {
+ 1
+ } else {
+ -1
+ }
+ })
+ .collect();
+
+ // CPU reference applies the RHT through the CPU matmul kernel.
+ let cpu_context = <Cpu as Backend>::Context::new().expect("Cpu context");
+ let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);
+ let cpu_rht = crate::common::helpers::alloc_allocation_with_data::<Cpu, i32>(&cpu_context, &rht);
+ let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &cpu_context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("MatmulCpuKernel");
+ let mut cpu_encoder = Encoder::<Cpu>::new(&cpu_context).expect("cpu encoder");
+ let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);
+ cpu_args.d_transform = MatmulDOps {
+ ab_scale: 1.0,
+ accumulate: false,
+ bias: None,
+ rht_factors: Some(&cpu_rht),
+ };
+ cpu_matmul.encode(cpu_args, &mut cpu_encoder).expect("cpu encode quant+rht");
+ cpu_encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let reference = allocation_to_vec::<Cpu, bf16>(&cpu_buffers.y);
+
+ // Metal GEMV: m = 1 quant routes to GEMV, RHT selects the 8-simdgroup (32-row) layout.
+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
+ let metal_rht = crate::common::helpers::alloc_allocation_with_data::<Metal, i32>(&context, &rht);
+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("MatmulMetalKernel");
+ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
+ let mut args = quant_arguments(&mut buffers, &input);
+ args.d_transform = MatmulDOps {
+ ab_scale: 1.0,
+ accumulate: false,
+ bias: None,
+ rht_factors: Some(&metal_rht),
+ };
+ matmul.encode(args, &mut encoder).expect("encode quant gemv with rht");
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);
+
+ assert_parity::<bf16>("gemv_quant_rht", &reference, &actual, 0.05, 0.6);
+}
+
#[uzu_test]
fn quant_gemm_nonzero_b_offset_returns_unsupported_layout() {
let context = MetalContext::new().expect("Metal context");