A high-performance inference engine for AI models
unified gemv
eugene7b2614e86b20
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.hsimilarity index 62%rename from crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.hrename to crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.hindex b76b4a5eb..7f1b2f305 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/qmv_ops.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h@@ -3,33 +3,41 @@#include <metal_stdlib>#include "../../common/defines.h"-#include "../../gemm/common/quant_unpack.h"+#include "quant_unpack.h"using namespace metal;namespace uzu {namespace gemm {+// Pre-scales each activation lane k by 2^-(BITS*k). qdot then reads the+// matching weight nibble/byte in place (value = q * 2^(BITS*k)) without+// shifting it down to the low bits; the positional 2^(BITS*k) factor cancels+// the 2^-(BITS*k) here, so the dot product is unchanged. All factors are powers+// of two, so this is bit-exact.template <typename T, typename U, int VALUES_PER_THREAD, int BITS>-inline U load_vector(const device T* x, thread U* x_thread) {- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");-+METAL_FUNC U load_vector(const device T* x, thread U* x_thread) {using U4 = vec<U, 4>;+ const U4 inv =+ U4(U(1),+ U(1) / U(1u << BITS),+ U(1) / U(1u << (2u * BITS)),+ U(1) / U(1u << (3u * BITS)));U sum = 0;thread U4* x_vec4 = reinterpret_cast<thread U4*>(x_thread);+ METAL_PRAGMA_UNROLLfor (int i = 0; i < VALUES_PER_THREAD / 4; i++) {U4 v = U4(x[4 * i], x[4 * i + 1], x[4 * i + 2], x[4 * i + 3]);sum += v[0] + v[1] + v[2] + v[3];- x_vec4[i] = v;+ x_vec4[i] = v * inv;}return sum;}-template <typename T, typename U, int VALUES_PER_THREAD, int BITS>-inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");-+template <typename T, typename U, int VALUES_PER_THREAD>+METAL_FUNC U load_vector_safe(const device T* x, thread U* x_thread, int N) {U sum = 0;+ METAL_PRAGMA_UNROLLfor (int i = 0; i < VALUES_PER_THREAD; ++i) {x_thread[i] = 0;}@@ -42,30 +50,7 @@ inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {}template <typename U, int VALUES_PER_THREAD, int BITS>-inline void qouter(- const thread uint8_t* w,- U x,- U scale,- U bias,- thread U* result-) {- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");-- if (BITS == 4) {- for (int i = 0; i < (VALUES_PER_THREAD / 2); i++) {- result[2 * i] += x * (scale * uint_to_fp<U>(w[i] & 0x0fu) + bias);- result[2 * i + 1] +=- x * (scale * uint_to_fp<U>((w[i] >> 4) & 0x0fu) + bias);- }- } else if (BITS == 8) {- for (int i = 0; i < VALUES_PER_THREAD; i++) {- result[i] += x * (scale * w[i] + bias);- }- }-}--template <typename U, int VALUES_PER_THREAD, int BITS>-inline U qdot(+METAL_FUNC U qdot(const device uint8_t* w,const thread U* x_thread,U scale,@@ -75,33 +60,35 @@ inline U qdot(static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");U accumulator = 0;- if (BITS == 4) {+ if constexpr (BITS == 4) {using U4 = vec<U, 4>;const device ushort* weight_words =reinterpret_cast<const device ushort*>(w);const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);+ METAL_PRAGMA_UNROLLfor (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {- uint weight_word = weight_words[i];- U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(- weight_word,- weight_word >> 4,- weight_word >> 8,- weight_word >> 12- ));+ // Mask each nibble in place (no shifts); value of lane k is n_k << (4*k),+ // i.e. n_k * 16^k, which is < 2^23 so the magic-number convert is valid.+ // The matching x lane was pre-divided by 16^k in load_vector.+ const uint4 lanes =+ uint4(weight_words[i]) & uint4(0x000fu, 0x00f0u, 0x0f00u, 0xf000u);+ const U4 weight_vec4 =+ U4(as_type<float4>(lanes | uint4(0x4b000000u)) - float4(8388608.0f));accumulator += dot(x_vec4[i], weight_vec4);}- } else if (BITS == 8) {+ } else if constexpr (BITS == 8) {using U4 = vec<U, 4>;const device uint* weight_words = reinterpret_cast<const device uint*>(w);const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);+ METAL_PRAGMA_UNROLLfor (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {- uint weight_word = weight_words[i];- U4 weight_vec4 = uint4_to_fp4<U, 8>(uint4(- weight_word,- weight_word >> 8,- weight_word >> 16,- weight_word >> 24- ));+ // Mask each byte in place (no shifts); lane k value is b_k * 256^k. This+ // exceeds the magic-number range for k=3, so use the hardware convert+ // (exact for b_k * 256^k). x lane k was pre-divided by 256^k.+ const uint4 lanes =+ uint4(weight_words[i]) &+ uint4(0x000000ffu, 0x0000ff00u, 0x00ff0000u, 0xff000000u);+ const U4 weight_vec4 = U4(float4(lanes));accumulator += dot(x_vec4[i], weight_vec4);}}@@ -109,7 +96,7 @@ inline U qdot(}template <typename U, int VALUES_PER_THREAD, int BITS>-inline U qdot_safe(+METAL_FUNC U qdot_safe(const device uint8_t* w,const thread U* x_thread,U scale,@@ -120,7 +107,7 @@ inline U qdot_safe(static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");U accumulator = 0;- if (BITS == 4) {+ if constexpr (BITS == 4) {using U4 = vec<U, 4>;const device uint16_t* weight_words =reinterpret_cast<const device uint16_t*>(w);@@ -150,7 +137,7 @@ inline U qdot_safe(accumulator +=x_thread[base_index + 2] * uint_to_fp<U>((weight_word >> 8) & 0xf);}- } else if (BITS == 8) {+ } else if constexpr (BITS == 8) {for (int i = 0; i < N; i++) {accumulator += x_thread[i] * w[i];}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_pack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_pack.hnew file mode 100644index 000000000..b1a960ca9--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_pack.h@@ -0,0 +1,20 @@+#pragma once++#include <metal_stdlib>++namespace uzu {+namespace gemm {++template <int bits, int word_size_bits = 8>+METAL_FUNC constexpr short get_pack_factor() {+ return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : word_size_bits / bits);+}++template <int bits, int word_size_bits = 8>+METAL_FUNC constexpr short get_bytes_per_pack() {+ constexpr int is_power_of_two_bits = (bits & (bits - 1)) == 0;+ return is_power_of_two_bits ? (word_size_bits / 8) : (bits == 5 ? 5 : 3);+}++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.hnew file mode 100644index 000000000..4735576f3--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h@@ -0,0 +1,110 @@+#pragma once++#include <metal_stdlib>++#include "../../common/defines.h"++using namespace metal;++namespace uzu {+namespace gemm {++template <typename U>+METAL_FUNC U uint_to_fp(uint32_t x) {+ return static_cast<U>(as_type<float>(x | 0x4B000000u) - 8388608.0f);+}++template <>+METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {+ return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);+}++template <int BITS>+METAL_FUNC float4 _uint4_to_fp4_float(uint4 n) {+ static_assert(BITS > 0 && BITS <= 23, "BITS must fit in float23 mantissa");+ constexpr uint mask = (1u << BITS) - 1u;+ n &= uint4(mask);+ return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);+}++template <typename U, int BITS>+METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);++template <>+METAL_FUNC float4 uint4_to_fp4<float, 4>(uint4 n) {+ return _uint4_to_fp4_float<4>(n);+}++template <>+METAL_FUNC float4 uint4_to_fp4<float, 8>(uint4 n) {+ return _uint4_to_fp4_float<8>(n);+}++template <>+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 4>(uint4 n) {+ return bfloat4(_uint4_to_fp4_float<4>(n));+}++template <>+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {+ return bfloat4(_uint4_to_fp4_float<8>(n));+}++template <typename U, int N, int bits>+METAL_FUNC void dequantize(+ const device uint8_t* w,+ U scale,+ U bias,+ threadgroup U* w_local+) {+ static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");++ if constexpr (bits == 4) {+ U s0 = scale;+ U s1 = scale / static_cast<U>(16.0f);+ METAL_PRAGMA_UNROLL+ for (int i = 0; i < (N / 2); i++) {+ w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;+ w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;+ }+ } else if constexpr (bits == 8) {+ METAL_PRAGMA_UNROLL+ for (int i = 0; i < N; i++) {+ w_local[i] = scale * w[i] + bias;+ }+ }+}++template <>+METAL_FUNC void dequantize<bfloat, 8, 4>(+ const device uint8_t* w,+ bfloat scale,+ bfloat bias,+ threadgroup bfloat* w_local+) {+ const device uint32_t* w_ptr = reinterpret_cast<const device uint32_t*>(w);+ uint32_t packed = *w_ptr;++ bfloat4 v0, v1;++ v0.x = static_cast<bfloat>(packed & 0xF);+ v0.y = static_cast<bfloat>((packed >> 4) & 0xF);+ v0.z = static_cast<bfloat>((packed >> 8) & 0xF);+ v0.w = static_cast<bfloat>((packed >> 12) & 0xF);++ v1.x = static_cast<bfloat>((packed >> 16) & 0xF);+ v1.y = static_cast<bfloat>((packed >> 20) & 0xF);+ v1.z = static_cast<bfloat>((packed >> 24) & 0xF);+ v1.w = static_cast<bfloat>((packed >> 28) & 0xF);++ v0 = v0 * scale + bias;+ v1 = v1 * scale + bias;++ threadgroup bfloat4* out_ptr =+ reinterpret_cast<threadgroup bfloat4*>(w_local);+ out_ptr[0] = v0;+ out_ptr[1] = v1;+}++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.hnew file mode 100644index 000000000..a9fa82495--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h@@ -0,0 +1,237 @@+#pragma once++#include "../../common/qdot.h"+#include "../../common/quant_pack.h"+#include "gemv_common.h"+#include "quant_row_offsets.h"++namespace uzu {+namespace gemm {++template <+ typename BT,+ typename AT,+ typename U,+ GemmBPrologueKind B_PROLOGUE,+ uint GROUP_SIZE,+ uint BITS,+ uint K_SPLIT,+ bool INPUT_ALIGNED>+struct BSource {+ static METAL_FUNC void accumulate(+ thread U (&result)[RESULTS_PER_SIMDGROUP],+ const device uint32_t* b,+ const device BT* scales,+ const device uint8_t* zero_points,+ const device BT* biases,+ const device AT* a,+ uint in_vec_size,+ uint out_row,+ uint batch_idx,+ uint simd_lane,+ uint k_slice+ ) {+ if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {+ constexpr uint values_per_thread = 4;+ constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;+ typedef vec<BT, 4> W4;+ typedef vec<AT, 4> I4;+ const uint k_stride = K_SPLIT * block_size;+ const uint k_start = k_slice * block_size;+ const device BT* w = reinterpret_cast<const device BT*>(b);+ w += out_row * in_vec_size + simd_lane * values_per_thread + k_start;+ const device AT* in =+ a + batch_idx * in_vec_size + simd_lane * values_per_thread + k_start;++ uint k = k_start;+ for (; k + block_size <= in_vec_size; k += k_stride) {+ float4 xv =+ static_cast<float4>(*reinterpret_cast<const device I4*>(in));+ result[0] +=+ dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w)),+ xv);+ result[1] +=+ dot(static_cast<float4>(+ *reinterpret_cast<const device W4*>(w + in_vec_size)+ ),+ xv);+ result[2] +=+ dot(static_cast<float4>(+ *reinterpret_cast<const device W4*>(w + 2 * in_vec_size)+ ),+ xv);+ result[3] +=+ dot(static_cast<float4>(+ *reinterpret_cast<const device W4*>(w + 3 * in_vec_size)+ ),+ xv);+ w += k_stride;+ in += k_stride;+ }++ if constexpr (K_SPLIT == 1 && !INPUT_ALIGNED) {+ const uint thread_offset = simd_lane * values_per_thread;+ const int remaining =+ (k + thread_offset < in_vec_size)+ ? min(static_cast<int>(in_vec_size - k - thread_offset),+ static_cast<int>(values_per_thread))+ : 0;+ if (remaining > 0) {+ const device BT* w0 = w;+ const device BT* w1 = w + in_vec_size;+ const device BT* w2 = w + 2 * in_vec_size;+ const device BT* w3 = w + 3 * in_vec_size;+ for (int j = 0; j < remaining; j++) {+ U x = static_cast<U>(in[j]);+ result[0] += static_cast<U>(w0[j]) * x;+ result[1] += static_cast<U>(w1[j]) * x;+ result[2] += static_cast<U>(w2[j]) * x;+ result[3] += static_cast<U>(w3[j]) * x;+ }+ }+ }+ } else {+ constexpr uint packs_per_thread = BITS == 2 ? 1 : 2;+ constexpr uint pack_factor = get_pack_factor<BITS, 32>();+ constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();+ constexpr uint values_per_thread = pack_factor * packs_per_thread;+ constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;+ constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;+ const device uint8_t* ws = (const device uint8_t*)b;+ thread U x_thread[values_per_thread];++ const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;+ const uint in_vec_size_g = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;+ ws += out_row * in_vec_size_w ++ simd_lane * packs_per_thread * bytes_per_pack;++ QuantRowOffsets<BT, U, B_PROLOGUE, BITS> prep;+ prep.group_stride = in_vec_size_g;+ const uint g_offset = simd_lane / scale_step_per_thread;+ prep.scales = scales + out_row * in_vec_size_g + g_offset;+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {+ prep.biases = biases + out_row * in_vec_size_g + g_offset;+ } else if constexpr (+ B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant+ ) {+ if (BITS == 4) {+ prep.zp_stride = (in_vec_size_g + 1) / 2;+ prep.zps = zero_points + out_row * prep.zp_stride + g_offset / 2;+ prep.high_nibble = (g_offset & 1);+ } else {+ prep.zp_stride = in_vec_size_g;+ prep.zps = zero_points + out_row * prep.zp_stride + g_offset;+ }+ }++ const device AT* in =+ a + batch_idx * in_vec_size + simd_lane * values_per_thread;++ uint k = 0;+ for (; k + block_size <= in_vec_size; k += block_size) {+ U sum = load_vector<AT, U, values_per_thread, BITS>(in, x_thread);++ const device uint8_t* wl0 = ws;+ const device uint8_t* wl1 = ws + in_vec_size_w;+ const device uint8_t* wl2 = ws + 2 * in_vec_size_w;+ const device uint8_t* wl3 = ws + 3 * in_vec_size_w;++ U scale[4];+ U offset[4];+ prep.load(scale, offset);+ result[0] += qdot<U, values_per_thread, BITS>(+ wl0,+ x_thread,+ scale[0],+ offset[0],+ sum+ );+ result[1] += qdot<U, values_per_thread, BITS>(+ wl1,+ x_thread,+ scale[1],+ offset[1],+ sum+ );+ result[2] += qdot<U, values_per_thread, BITS>(+ wl2,+ x_thread,+ scale[2],+ offset[2],+ sum+ );+ result[3] += qdot<U, values_per_thread, BITS>(+ wl3,+ x_thread,+ scale[3],+ offset[3],+ sum+ );++ ws += block_size * bytes_per_pack / pack_factor;+ prep.advance(block_size / GROUP_SIZE);+ in += block_size;+ }++ if constexpr (!INPUT_ALIGNED) {+ const uint thread_offset = simd_lane * values_per_thread;+ const int remaining =+ (k + thread_offset < in_vec_size)+ ? min(static_cast<int>(in_vec_size - k - thread_offset),+ static_cast<int>(values_per_thread))+ : 0;+ if (remaining > 0) {+ U sum = load_vector_safe<AT, U, values_per_thread>(+ in,+ x_thread,+ remaining+ );++ const device uint8_t* wl0 = ws;+ const device uint8_t* wl1 = ws + in_vec_size_w;+ const device uint8_t* wl2 = ws + 2 * in_vec_size_w;+ const device uint8_t* wl3 = ws + 3 * in_vec_size_w;++ U scale[4];+ U offset[4];+ prep.load(scale, offset);+ result[0] += qdot_safe<U, values_per_thread, BITS>(+ wl0,+ x_thread,+ scale[0],+ offset[0],+ sum,+ remaining+ );+ result[1] += qdot_safe<U, values_per_thread, BITS>(+ wl1,+ x_thread,+ scale[1],+ offset[1],+ sum,+ remaining+ );+ result[2] += qdot_safe<U, values_per_thread, BITS>(+ wl2,+ x_thread,+ scale[2],+ offset[2],+ sum,+ remaining+ );+ result[3] += qdot_safe<U, values_per_thread, BITS>(+ wl3,+ x_thread,+ scale[3],+ offset[3],+ sum,+ remaining+ );+ }+ }+ }+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.hnew file mode 100644index 000000000..1babf6c5a--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h@@ -0,0 +1,85 @@+#pragma once++#include "../../../hadamard_transform/hadamard_transform.h"+#include "gemv_common.h"++namespace uzu {+namespace gemm {++template <typename BT, typename DT, typename U>+struct Epilogue {+ static METAL_FUNC void store(+ thread U (&result)[RESULTS_PER_SIMDGROUP],+ device DT* d,+ const device BT* output_bias,+ const device int32_t* hadamard_factors,+ threadgroup U* shared_results,+ float ab_scale,+ GemmDTransform output_transform,+ uint out_row,+ uint out_vec_size,+ uint out_block_idx,+ uint simd_group,+ uint simd_lane,+ bool writer+ ) {+ const bool is_scale = output_transform.contains(GemmDTransform::SCALE);+ const bool is_accumulate =+ output_transform.contains(GemmDTransform::ACCUMULATE);+ const bool is_bias = output_transform.contains(GemmDTransform::BIAS);+ const bool use_hadamard = output_transform.contains(GemmDTransform::RHT);++ if (writer && simd_lane == 0) {+ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {+ U value = result[row];+ if (is_scale) {+ value = static_cast<U>(ab_scale) * value;+ }+ const uint global_row = out_row + row;+ if (is_accumulate && global_row < out_vec_size) {+ value += static_cast<U>(d[row]);+ }+ if (is_bias && global_row < out_vec_size) {+ value += static_cast<U>(output_bias[global_row]);+ }+ result[row] = value;+ }+ }++ if (use_hadamard) {+ if (simd_lane == 0) {+ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {+ shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] =+ result[row];+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (simd_group == 0) {+ uint global_out_idx = out_block_idx * 32 + simd_lane;+ if (global_out_idx < out_vec_size) {+ d[simd_lane] = simdgroup_output_random_hadamard_transform(+ static_cast<ushort>(simd_lane),+ static_cast<DT>(shared_results[simd_lane]),+ hadamard_factors[global_out_idx]+ );+ }+ }+ } else {+ if (writer && simd_lane == 0) {+ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {+ if (out_row + row < out_vec_size) {+ d[row] = static_cast<DT>(result[row]);+ }+ }+ }+ }+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/gemv_common.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/gemv_common.hnew file mode 100644index 000000000..f83f418d8--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/gemv_common.h@@ -0,0 +1,16 @@+#pragma once++#include <metal_stdlib>++#include "../../common/defines.h"+#include "../../../generated/gemm.h"++using namespace metal;++namespace uzu {+namespace gemm {++METAL_CONST uint RESULTS_PER_SIMDGROUP = 4;++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.hnew file mode 100644index 000000000..6a28cbbb5--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h@@ -0,0 +1,36 @@+#pragma once++#include "gemv_common.h"++namespace uzu {+namespace gemm {++template <uint K_SPLIT, uint NUM_SIMDGROUPS>+struct OutputTile {+ uint out_row;+ uint row_group;+ uint k_slice;+ bool writer;++ static METAL_FUNC OutputTile+ make(uint out_block_idx, uint simd_group, uint out_vec_size) {+ constexpr uint rows_per_threadgroup =+ (NUM_SIMDGROUPS / K_SPLIT) * RESULTS_PER_SIMDGROUP;+ const uint row_group = simd_group / K_SPLIT;+ const uint k_slice = simd_group % K_SPLIT;+ uint out_row = out_block_idx * rows_per_threadgroup ++ row_group * RESULTS_PER_SIMDGROUP;+ out_row = out_vec_size > RESULTS_PER_SIMDGROUP+ ? min(out_row, out_vec_size - RESULTS_PER_SIMDGROUP)+ : 0u;+ OutputTile tile;+ tile.out_row = out_row;+ tile.row_group = row_group;+ tile.k_slice = k_slice;+ tile.writer = (K_SPLIT == 1) || (k_slice == 0);+ return tile;+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.hnew file mode 100644index 000000000..0965132c0--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h@@ -0,0 +1,76 @@+#pragma once++#include "../../common/quant_pack.h"+#include "gemv_common.h"++namespace uzu {+namespace gemm {++template <typename BT, typename U, GemmBPrologueKind B_PROLOGUE, uint BITS>+struct QuantRowOffsets {+ static_assert(+ BITS == 4 || BITS == 8,+ "QuantRowOffsets supports 4- and 8-bit only"+ );+ const device BT* scales = nullptr;+ const device BT* biases = nullptr;+ const device uint8_t* zps = nullptr;+ uint group_stride = 0;+ uint zp_stride = 0;+ bool high_nibble = false;++ void load(thread U* scale, thread U* offset) const {+ scale[0] = static_cast<U>(scales[0]);+ scale[1] = static_cast<U>(scales[group_stride]);+ scale[2] = static_cast<U>(scales[2 * group_stride]);+ scale[3] = static_cast<U>(scales[3 * group_stride]);++ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {+ offset[0] = static_cast<U>(biases[0]);+ offset[1] = static_cast<U>(biases[group_stride]);+ offset[2] = static_cast<U>(biases[2 * group_stride]);+ offset[3] = static_cast<U>(biases[3 * group_stride]);+ } else if constexpr (+ B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant+ ) {+ uchar4 zp_bytes = uchar4(+ zps[0],+ zps[zp_stride],+ zps[2 * zp_stride],+ zps[3 * zp_stride]+ );+ uchar4 zp_nibbles;+ if constexpr (BITS == 4) {+ const uint8_t shift = high_nibble ? 4u : 0u;+ zp_nibbles = (zp_bytes >> shift) & uchar4(0x0F);+ } else {+ zp_nibbles = zp_bytes;+ }+ offset[0] = -scale[0] * static_cast<U>(zp_nibbles.x);+ offset[1] = -scale[1] * static_cast<U>(zp_nibbles.y);+ offset[2] = -scale[2] * static_cast<U>(zp_nibbles.z);+ offset[3] = -scale[3] * static_cast<U>(zp_nibbles.w);+ } else {+ constexpr U midpoint = U(1u << (BITS - 1));+ offset[0] = -scale[0] * midpoint;+ offset[1] = -scale[1] * midpoint;+ offset[2] = -scale[2] * midpoint;+ offset[3] = -scale[3] * midpoint;+ }+ }++ void advance(uint groups) {+ scales += groups;+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {+ biases += groups;+ } else if constexpr (+ B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant+ ) {+ constexpr uint zps_per_byte = get_pack_factor<BITS, 8>();+ zps += groups / zps_per_byte;+ }+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.hnew file mode 100644index 000000000..d08ad70bb--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h@@ -0,0 +1,49 @@+#pragma once++#include "gemv_common.h"++namespace uzu {+namespace gemm {++template <typename U, uint K_SPLIT, uint NUM_SIMDGROUPS>+struct Reduce {+ static METAL_FUNC void run(+ thread U (&result)[RESULTS_PER_SIMDGROUP],+ threadgroup U* shared_results,+ uint simd_group,+ uint simd_lane,+ uint row_group,+ uint k_slice+ ) {+ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {+ result[row] = simd_sum(result[row]);+ }++ if constexpr (K_SPLIT > 1) {+ if (simd_lane == 0) {+ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {+ shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] =+ result[row];+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (k_slice == 0 && simd_lane == 0) {+ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {+ U acc = 0;+ METAL_PRAGMA_UNROLL+ for (uint s = 0; s < K_SPLIT; s++) {+ acc += shared_results+ [(row_group * K_SPLIT + s) * RESULTS_PER_SIMDGROUP + row];+ }+ result[row] = acc;+ }+ }+ }+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rsdeleted file mode 100644index ea0f28a20..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs+++ /dev/null@@ -1,59 +0,0 @@-use super::{kernel::GemvKernel, spec::GemvSpecialization};-use crate::backends::{- common::{- AsBufferRangeRef, Buffer, Encoder,- kernel::matmul::{MatmulArguments, MatmulB, MatmulError},- },- metal::Metal,-};--pub(crate) fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(- kernel: &mut GemvKernel,- encoder: &mut Encoder<Metal>,- arguments: MatmulArguments<'a, Metal, TB>,-) -> Result<(), MatmulError<Metal>> {- let ab_scale = arguments.d_transform.ab_scale;- let output_bias = arguments.d_transform.bias;- let is_accumulate = arguments.d_transform.accumulate;-- let MatmulArguments {- a,- a_offset,- b,- b_offset,- b_leading_dimension,- b_transpose,- d,- m,- n,- k,- ..- } = arguments;- let weights = match b {- MatmulB::FullPrecision {- b: w,- } => w,- _ => panic!("FP gemv requires FullPrecision B"),- };- assert!(b_transpose, "encode_gemv does not support b_transpose=false");- assert!(b_offset == 0, "encode_gemv does not support nonzero b_offset");- assert!(b_leading_dimension.is_none_or(|ld| ld == k), "encode_gemv does not support custom b_leading_dimension");-- let specialization = GemvSpecialization::select(k, n, is_accumulate, output_bias.is_some());-- kernel.get_or_create(encoder.context(), specialization)?.encode(- weights,- (a, a_offset),- output_bias,- d,- k,- n,- k,- ab_scale,- m,- specialization.output_rows_per_threadgroup(),- encoder,- );-- Ok(())-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metalindex e7bd34d05..0aa6c8cf5 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -1,318 +1,117 @@-#include <metal_stdlib>--#include "../../common/defines.h"#include "../../common/dsl.h"-#include "../../common/thread_context.h"--#include <metal_simdgroup>+#include "../../generated/gemm.h"+#include "common/b_source.h"+#include "common/epilogue.h"+#include "common/output_tile.h"+#include "common/reduce.h"using namespace metal;--// Upper bound for threadgroup memory (in floats).-// Max config: tg_simd_cols=16, output_rows_per_tg=16, thread_out_rows=4-// → 16 * (16 + 4) = 320. Unused when tg_simd_cols == 1.-#define GEMV_MAX_THREADGROUP_MEMORY 320--template <typename WeightT, typename InputT, typename OutputT>-VARIANTS(WeightT, bfloat, float)-VARIANTS(InputT, bfloat, float)-VARIANTS(OutputT, bfloat, float)-KERNEL(MatmulGemv)(- const device WeightT* matrix,- const device InputT* input_vector,- const device WeightT* output_bias OPTIONAL(is_bias),- device OutputT* output_vector,- const constant uint& input_dimension,- const constant uint& output_dimension,- const constant uint& matrix_leading_dimension,- const constant float& output_scale,- const constant uint& batch_rows,- const constant uint& output_rows_per_threadgroup,- threadgroup float threadgroup_memory[GEMV_MAX_THREADGROUP_MEMORY],- const uint tg_simd_rows SPECIALIZE,- const uint tg_simd_cols SPECIALIZE,- const uint sg_thread_rows SPECIALIZE,- const uint sg_thread_cols SPECIALIZE,- const uint thread_out_rows SPECIALIZE,- const uint thread_out_cols SPECIALIZE,- const bool is_accumulate SPECIALIZE,- const bool is_bias SPECIALIZE,- const uint threadgroup_index_x GROUPS((output_dimension + output_rows_per_threadgroup - 1) / output_rows_per_threadgroup),- const uint threadgroup_index_y GROUPS(batch_rows),- const uint thread_index_x THREADS(32),- const uint thread_index_y THREADS(16),- const uint thread_index_z THREADS(1),- const ThreadContext thread_context+using namespace uzu::gemm;++template <+ typename AT,+ typename BT,+ typename DT,+ GemmBPrologueKind B_PROLOGUE,+ uint GROUP_SIZE,+ uint BITS,+ uint K_SPLIT,+ bool INPUT_ALIGNED,+ uint NUM_SIMDGROUPS>+VARIANTS(AT, bfloat, float)+VARIANTS(BT, bfloat, float)+VARIANTS(DT, bfloat, float)+CONSTRAINT(BT != "float" || (AT == "float" && DT == "float"))+VARIANTS(+ B_PROLOGUE,+ GemmBPrologueKind::FullPrecision,+ GemmBPrologueKind::ScaleBiasDequant,+ GemmBPrologueKind::ScaleZeroPointDequant,+ GemmBPrologueKind::ScaleSymmetricDequant)+VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)+VARIANTS(BITS, 0, 4, 8)+VARIANTS(K_SPLIT, 1, 2, 4, 8)+VARIANTS(INPUT_ALIGNED, false, true)+VARIANTS(NUM_SIMDGROUPS, 2, 8)+CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))+CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))+CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")+CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || K_SPLIT == 1)+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::FullPrecision || NUM_SIMDGROUPS == 8)+KERNEL(Gemv)(+ const device uint32_t* b,+ const device BT* scales+ OPTIONAL(B_PROLOGUE != GemmBPrologueKind::FullPrecision),+ const device uint8_t* zero_points+ OPTIONAL(B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant),+ const device BT* biases+ OPTIONAL(B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant),+ const device AT* a,+ device DT* d,+ const device BT* output_bias+ OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),+ const device int32_t* hadamard_factors+ OPTIONAL(output_transform.contains(GemmDTransform::RHT)),+ const constant uint& in_vec_size,+ const constant uint& out_vec_size,+ const constant uint& batch_size,+ const constant float& ab_scale,+ const constant uint& group_count_x,+ const GemmDTransform output_transform SPECIALIZE,+ threadgroup float shared_results[NUM_SIMDGROUPS * 4],+ const uint batch_idx GROUPS(batch_size),+ const uint out_block_idx GROUPS(group_count_x),+ const uint simd_lane THREADS(32),+ const uint simd_group THREADS(NUM_SIMDGROUPS)) {- if (output_rows_per_threadgroup <= 0) {- return;- }-- // Simdgroup guard — kill excess simdgroups beyond what this tile uses- if (thread_context.simdgroup_index >= tg_simd_rows * tg_simd_cols) {- return;- }-- // Derived constants from SPECIALIZE params- const uint threads_per_threadgroup_col = tg_simd_cols * sg_thread_cols;- const uint input_columns_per_threadgroup =- threads_per_threadgroup_col * thread_out_cols;-- const uint batch_row = static_cast<uint>(threadgroup_index_y);- if (batch_row >= batch_rows) {- return;- }-- // Thread local accumulation results- thread float accumulated_values[4] = {0.0f, 0.0f, 0.0f, 0.0f};- thread WeightT matrix_values[4];- thread float vector_coefficients[4];-- const uint thread_row_in_simdgroup =- sg_thread_cols != 32- ? uint(thread_context.simd_lane_id) / uint(sg_thread_cols)- : 0;- const int thread_col_in_simdgroup =- sg_thread_cols != 32- ? uint(thread_context.simd_lane_id) % uint(sg_thread_cols)- : uint(thread_context.simd_lane_id);-- const int simdgroup_column_index =- tg_simd_cols != 1 ? uint(thread_context.simdgroup_index % tg_simd_cols)- : 0;-- const uint simdgroup_row_thread_base =- tg_simd_cols != 1- ? uint(sg_thread_rows) *- uint(thread_context.simdgroup_index / tg_simd_cols)- : uint(sg_thread_rows) * uint(thread_context.simdgroup_index);- const uint simdgroup_col_thread_base =- tg_simd_cols != 1- ? uint(sg_thread_cols) *- uint(thread_context.simdgroup_index % tg_simd_cols)- : 0;-- uint output_block_row_offset =- (simdgroup_row_thread_base + thread_row_in_simdgroup) *- uint(thread_out_rows);- uint input_block_col_offset =- (simdgroup_col_thread_base + thread_col_in_simdgroup) *- uint(thread_out_cols);-- // Block position- uint output_row_start =- uint(threadgroup_index_x) * uint(output_rows_per_threadgroup) +- output_block_row_offset;-- // Exit simdgroup if rows out of bound- if (output_row_start >= output_dimension)- return;-- // Adjust tail simdgroup to ensure in bound reads- output_row_start =- output_row_start + uint(thread_out_rows) <= output_dimension- ? output_row_start- : output_dimension - uint(thread_out_rows);-- const device WeightT* thread_matrix =- matrix + output_row_start * matrix_leading_dimension;-- const uniform<uint> input_block_stride =- make_uniform(uint(input_columns_per_threadgroup));- const uniform<uint> input_vector_length = make_uniform(input_dimension);- const uniform<uint> full_input_blocks =- input_vector_length / input_block_stride;- const uniform<uint> remaining_input_columns =- input_vector_length - input_block_stride * full_input_blocks;-- // Loop over input_vector in blocks of input_columns_per_threadgroup- for (uint input_block_index = 0; input_block_index < full_input_blocks;- ++input_block_index) {- // Load vector coefficients (unchecked)- {- const device InputT* input_vector_row =- input_vector + batch_row * input_dimension;- METAL_PRAGMA_UNROLL- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;- input_col_offset++) {- vector_coefficients[input_col_offset] = static_cast<float>(- input_vector_row[input_block_col_offset + input_col_offset]- );- }- }-- // Per thread work loop- uint matrix_row_offset = 0;- METAL_PRAGMA_UNROLL- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;- output_row_offset++) {- // Load matrix row (unchecked)- METAL_PRAGMA_UNROLL- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;- input_col_offset++) {- matrix_values[input_col_offset] = thread_matrix- [matrix_row_offset + input_block_col_offset + input_col_offset];- }-- // Accumulate results- METAL_PRAGMA_UNROLL- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;- input_col_offset++) {- accumulated_values[output_row_offset] +=- static_cast<float>(matrix_values[input_col_offset]) *- vector_coefficients[input_col_offset];- }-- matrix_row_offset += matrix_leading_dimension;- }-- input_block_col_offset += input_columns_per_threadgroup;- }-- if (remaining_input_columns > 0) {- // Load vector coefficients (checked)- {- const device InputT* input_vector_row =- input_vector + batch_row * input_dimension;- if (input_block_col_offset + uint(thread_out_cols) <=- input_vector_length) {- METAL_PRAGMA_UNROLL- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;- input_col_offset++) {- vector_coefficients[input_col_offset] = static_cast<float>(- input_vector_row[input_block_col_offset + input_col_offset]- );- }- } else {- METAL_PRAGMA_UNROLL- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;- input_col_offset++) {- vector_coefficients[input_col_offset] =- input_block_col_offset + uint(input_col_offset) <- input_vector_length- ? static_cast<float>(- input_vector_row- [input_block_col_offset + input_col_offset]- )- : 0.0f;- }- }- }-- // Per thread work loop- METAL_PRAGMA_UNROLL- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;- output_row_offset++) {- // Load matrix row (checked)- if (input_block_col_offset + uint(thread_out_cols) <=- input_vector_length) {- METAL_PRAGMA_UNROLL- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;- input_col_offset++) {- matrix_values[input_col_offset] = thread_matrix- [output_row_offset * matrix_leading_dimension +- input_block_col_offset + input_col_offset];- }- } else {- METAL_PRAGMA_UNROLL- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;- input_col_offset++) {- matrix_values[input_col_offset] =- input_block_col_offset + uint(input_col_offset) <- input_vector_length- ? thread_matrix- [output_row_offset * matrix_leading_dimension +- input_block_col_offset + input_col_offset]- : WeightT(0);- }- }-- // Accumulate results- METAL_PRAGMA_UNROLL- for (uint input_col_offset = 0; input_col_offset < thread_out_cols;- input_col_offset++) {- accumulated_values[output_row_offset] +=- static_cast<float>(matrix_values[input_col_offset]) *- vector_coefficients[input_col_offset];- }- }- }-- // Simdgroup accumulations- METAL_PRAGMA_UNROLL- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;- output_row_offset++) {- METAL_PRAGMA_UNROLL- for (ushort simd_shuffle_offset = (sg_thread_cols / 2);- simd_shuffle_offset >= 1;- simd_shuffle_offset /= 2) {- accumulated_values[output_row_offset] += simd_shuffle_down(- accumulated_values[output_row_offset],- simd_shuffle_offset+ typedef float U;+ thread U result[RESULTS_PER_SIMDGROUP] = {0};++ OutputTile<K_SPLIT, NUM_SIMDGROUPS> tile =+ OutputTile<K_SPLIT, NUM_SIMDGROUPS>::make(+ out_block_idx,+ simd_group,+ out_vec_size+ );+ d += batch_idx * out_vec_size + tile.out_row;++ BSource<BT, AT, U, B_PROLOGUE, GROUP_SIZE, BITS, K_SPLIT, INPUT_ALIGNED>::+ accumulate(+ result,+ b,+ scales,+ zero_points,+ biases,+ a,+ in_vec_size,+ tile.out_row,+ batch_idx,+ simd_lane,+ tile.k_slice);- }- }-- // Threadgroup reduction (only when tg_simd_cols > 1)- if (tg_simd_cols > 1) {- const uint computed_output_rows_per_tg =- uint(tg_simd_rows) * uint(sg_thread_rows) * uint(thread_out_rows);-- if (thread_col_in_simdgroup == 0) {- threadgroup float* threadgroup_partial_accumulations =- threadgroup_memory +- simdgroup_column_index *- (computed_output_rows_per_tg + uint(thread_out_rows)) +- output_block_row_offset;- METAL_PRAGMA_UNROLL- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;- output_row_offset++) {- threadgroup_partial_accumulations[output_row_offset] =- accumulated_values[output_row_offset];- }-- threadgroup_barrier(mem_flags::mem_none);-- if (simdgroup_column_index == 0) {- threadgroup float* base_partial =- threadgroup_memory + output_block_row_offset;- for (uint reduction_simdgroup_col = 1;- reduction_simdgroup_col < tg_simd_cols;- reduction_simdgroup_col++) {- METAL_PRAGMA_UNROLL- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;- output_row_offset++) {- accumulated_values[output_row_offset] += base_partial- [reduction_simdgroup_col *- (computed_output_rows_per_tg + uint(thread_out_rows)) +- output_row_offset];- }- }- }- }- }- // Write outputs- if (simdgroup_col_thread_base == 0 && thread_col_in_simdgroup == 0) {- device OutputT* output_row_values =- output_vector + batch_row * output_dimension;- METAL_PRAGMA_UNROLL- for (uint output_row_offset = 0; output_row_offset < thread_out_rows;- output_row_offset++) {- float accumulated_c =- output_scale * accumulated_values[output_row_offset];- if (is_accumulate) {- accumulated_c += static_cast<float>(- output_row_values[output_row_start + output_row_offset]- );- }- if (is_bias) {- accumulated_c += static_cast<float>(- output_bias[output_row_start + output_row_offset]- );- }- output_row_values[output_row_start + output_row_offset] =- static_cast<OutputT>(accumulated_c);- }- }+ Reduce<U, K_SPLIT, NUM_SIMDGROUPS>::run(+ result,+ shared_results,+ simd_group,+ simd_lane,+ tile.row_group,+ tile.k_slice+ );++ Epilogue<BT, DT, U>::store(+ result,+ d,+ output_bias,+ hadamard_factors,+ shared_results,+ ab_scale,+ output_transform,+ tile.out_row,+ out_vec_size,+ out_block_idx,+ simd_group,+ simd_lane,+ tile.writer+ );}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 502e4adab..11b6f572b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -1,65 +1,324 @@-use std::collections::{HashMap, hash_map::Entry};+use std::{+ collections::{HashMap, hash_map::Entry},+ sync::OnceLock,+};-use super::spec::GemvSpecialization;use crate::{backends::{- common::kernel::matmul::MatmulError,- metal::{Metal, context::MetalContext, kernel::MatmulGemvMetalKernel},+ common::{+ Allocation, AsBufferRangeRef, Buffer, Encoder,+ gpu_types::{+ QuantizationMethod,+ gemm::{GemmBPrologueKind, GemmDTransform},+ },+ kernel::matmul::{MatmulArguments, MatmulB, MatmulError, MatmulQuantCombo},+ },+ metal::{Metal, context::MetalContext, kernel::GemvMetalKernel},},data_type::DataType,};-pub(crate) struct GemvKernel {+const FP_BLOCK: u32 = 128;+const QMV_RESULTS_PER_SIMDGROUP: u32 = 4;++const DEFAULT_GEMV_MAX_BATCH: u32 = 8;+static GEMV_MAX_BATCH: OnceLock<u32> = OnceLock::new();++fn max_gemv_batch_threshold() -> u32 {+ *GEMV_MAX_BATCH.get_or_init(|| {+ std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)+ })+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub(crate) struct GemvSpecialization {+ b_prologue: GemmBPrologueKind,+ group_size: u32,+ bits: u32,+ output_transform: GemmDTransform,+ input_aligned: bool,+ k_split: u32,+ num_simdgroups: u32,+}++impl GemvSpecialization {+ pub(crate) fn select<TB: AsBufferRangeRef>(+ args: &MatmulArguments<Metal, TB>,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,+ ) -> Option<GemvSpecialization> {+ if !args.b_transpose || args.b_offset != 0 {+ return None;+ }+ let is_quant = !matches!(args.b, MatmulB::FullPrecision { .. });+ let bad_leading_dimension = if is_quant {+ args.b_leading_dimension.is_some()+ } else {+ args.b_leading_dimension.is_some_and(|ld| ld != args.k)+ };+ if bad_leading_dimension {+ return None;+ }+ if args.d_transform.accumulate && !args.n.is_multiple_of(32) {+ return None;+ }+ if args.d_transform.rht_factors.is_some() && !args.n.is_multiple_of(32) {+ return None;+ }+ if is_quant {+ if args.n < QMV_RESULTS_PER_SIMDGROUP || args.m >= 5 {+ return None;+ }+ } else {+ let mixed_precision = weights_data_type == DataType::F32+ && (input_data_type != DataType::F32 || output_data_type != DataType::F32);+ if mixed_precision || args.n < QMV_RESULTS_PER_SIMDGROUP || args.m > max_gemv_batch_threshold() {+ return None;+ }+ }++ let bits = args.b.bits_per_b().unwrap_or(0);+ let block_size = if !is_quant {+ FP_BLOCK+ } else if bits == 4 {+ 512+ } else {+ 256+ };+ let input_aligned = args.k.is_multiple_of(block_size);+ let has_rht = args.d_transform.rht_factors.is_some();+ let num_simdgroups = 8;+ let k_split = if is_quant || has_rht {+ 1+ } else {+ fp_k_split(args.n, args.k, input_aligned)+ };+ Some(GemvSpecialization {+ b_prologue: args.b.b_prologue(),+ group_size: args.b.group_size().unwrap_or(0),+ bits,+ output_transform: args.d_transform.mask(),+ input_aligned,+ k_split,+ num_simdgroups,+ })+ }++ fn quant_combo_specs(combo: MatmulQuantCombo) -> Vec<GemvSpecialization> {+ let bits = DataType::from(combo.mode).size_in_bits() as u32;+ let group_size = combo.group_size;+ let b_prologue = match combo.method {+ QuantizationMethod::ScaleBias => GemmBPrologueKind::ScaleBiasDequant,+ QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,+ QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,+ };+ let mut out = Vec::new();+ for output_transform in [+ GemmDTransform::empty(),+ GemmDTransform::BIAS,+ GemmDTransform::RHT,+ GemmDTransform::BIAS | GemmDTransform::RHT,+ ] {+ for input_aligned in [true, false] {+ out.push(GemvSpecialization {+ b_prologue,+ group_size,+ bits,+ output_transform,+ input_aligned,+ k_split: 1,+ num_simdgroups: 8,+ });+ }+ }+ out+ }+}++fn rows_per_threadgroup(+ k_split: u32,+ num_simdgroups: u32,+) -> u32 {+ (num_simdgroups / k_split) * QMV_RESULTS_PER_SIMDGROUP+}++fn fp_k_split(+ n: u32,+ k: u32,+ input_aligned: bool,+) -> u32 {+ if !input_aligned || n >= 4096 {+ 1+ } else if k >= 16 * n || n <= 512 {+ 8+ } else if n <= 1024 || k >= 3072 {+ 4+ } else {+ 2+ }+}++pub(crate) struct GemvDispatch {weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,- pipelines: HashMap<GemvSpecialization, MatmulGemvMetalKernel>,+ pipelines: HashMap<GemvSpecialization, GemvMetalKernel>,}-impl GemvKernel {+impl GemvDispatch {pub(crate) fn new(- context: &MetalContext,+ _context: &MetalContext,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,) -> Result<Self, MatmulError<Metal>> {- let mut kernel = Self {+ Ok(Self {weights_data_type,input_data_type,output_data_type,pipelines: HashMap::new(),- };- for &config in GemvSpecialization::precompile_configs(weights_data_type) {- kernel.get_or_create(context, config)?;+ })+ }++ pub(crate) fn preheat_quant_combo(+ &mut self,+ context: &MetalContext,+ combo: MatmulQuantCombo,+ ) -> Result<(), MatmulError<Metal>> {+ if self.weights_data_type != DataType::BF16 {+ return Ok(());+ }+ for specialization in GemvSpecialization::quant_combo_specs(combo) {+ self.get_or_create(context, specialization)?;}- Ok(kernel)+ Ok(())}- pub(crate) fn get_or_create(+ fn get_or_create(&mut self,context: &MetalContext,specialization: GemvSpecialization,- ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {+ ) -> Result<&GemvMetalKernel, MatmulError<Metal>> {match self.pipelines.entry(specialization) {Entry::Occupied(entry) => Ok(entry.into_mut()),Entry::Vacant(entry) => {- let kernel = MatmulGemvMetalKernel::new(+ let kernel = GemvMetalKernel::new(context,- self.weights_data_type,self.input_data_type,+ self.weights_data_type,self.output_data_type,- specialization.threadgroup_rows,- specialization.threadgroup_cols,- specialization.threads_per_simdgroup_row,- specialization.threads_per_simdgroup_col,- specialization.elements_per_thread_row,- specialization.elements_per_thread_col,- specialization.is_accumulate,- specialization.is_bias,+ specialization.b_prologue,+ specialization.group_size,+ specialization.bits,+ specialization.k_split,+ specialization.input_aligned,+ specialization.num_simdgroups,+ specialization.output_transform,).map_err(MatmulError::BackendError)?;Ok(entry.insert(kernel))},}}++ pub(crate) fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(+ &mut self,+ arguments: MatmulArguments<'a, Metal, TB>,+ specialization: GemvSpecialization,+ encoder: &mut Encoder<Metal>,+ ) -> Result<(), MatmulError<Metal>> {+ let ab_scale = arguments.d_transform.ab_scale;+ let output_bias = arguments.d_transform.bias;+ let rht_factors = arguments.d_transform.rht_factors;++ let MatmulArguments {+ a,+ a_offset,+ b,+ d,+ m,+ n,+ k,+ ..+ } = arguments;++ let group_count_x = n.div_ceil(rows_per_threadgroup(specialization.k_split, specialization.num_simdgroups));++ let context = encoder.context();+ let pipeline = self.get_or_create(context, specialization)?;++ match b {+ MatmulB::FullPrecision {+ b: weights,+ } => {+ pipeline.encode(+ weights,+ None::<&Allocation<Metal>>,+ None::<&Allocation<Metal>>,+ None::<&Allocation<Metal>>,+ (a, a_offset),+ &mut *d,+ output_bias,+ rht_factors,+ k,+ n,+ m,+ ab_scale,+ group_count_x,+ encoder,+ );+ },+ quant_b @ (MatmulB::ScaleBiasDequant {+ ..+ }+ | MatmulB::ScaleZeroPointDequant {+ ..+ }+ | MatmulB::ScaleSymmetricDequant {+ ..+ }) => {+ let (weights, scales, zero_points, biases) = match quant_b {+ MatmulB::ScaleBiasDequant {+ b: w,+ scales,+ biases,+ ..+ } => (w, scales, None, Some(biases)),+ MatmulB::ScaleZeroPointDequant {+ b: w,+ scales,+ zero_points,+ ..+ } => (w, scales, Some(zero_points), None),+ MatmulB::ScaleSymmetricDequant {+ b: w,+ scales,+ ..+ } => (w, scales, None, None),+ MatmulB::FullPrecision {+ ..+ } => unreachable!(),+ };+ pipeline.encode(+ weights,+ Some(scales),+ zero_points,+ biases,+ (a, a_offset),+ &mut *d,+ output_bias,+ rht_factors,+ k,+ n,+ m,+ ab_scale,+ group_count_x,+ encoder,+ );+ },+ }++ Ok(())+ }}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rsindex d396315d2..a14cbb82f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs@@ -1,7 +1,3 @@-pub(crate) mod fp;mod kernel;-mod quant_kernel;-mod spec;-pub(crate) use kernel::GemvKernel;-pub use quant_kernel::QuantGemvKernel;+pub(crate) use kernel::{GemvDispatch, GemvSpecialization};diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metaldeleted file mode 100644index 7fbc08019..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal+++ /dev/null@@ -1,309 +0,0 @@-#include <metal_stdlib>-#include "../../common/dsl.h"-#include "../../generated/quantization_method.h"-#include "../gemm/common/quant_pack.h"-#include "common/qmv_ops.h"--using namespace uzu::quantization_method;-using namespace uzu::gemm;--template <- typename WeightT,- typename InputT,- typename OutputT,- uint GROUP_SIZE,- uint BITS>-VARIANTS(WeightT, float, bfloat)-VARIANTS(InputT, float, bfloat)-VARIANTS(OutputT, float, bfloat)-VARIANTS(GROUP_SIZE, 16, 32, 64, 128)-VARIANTS(BITS, 4, 8)-KERNEL(QuantizedMatmulQmv)(- const device uint32_t* weights,- const device WeightT* scales,- const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),- const device WeightT* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),- const device InputT* input,- device OutputT* output,- const constant uint& in_vec_size,- const constant uint& out_vec_size,- const constant uint& batch_size,- const QuantizationMethod quant_method SPECIALIZE,- const uint batch_idx GROUPS(batch_size),- const uint out_block_idx GROUPS(out_vec_size.div_ceil(8)),- const uint simd_lane THREADS(32),- const uint simd_group THREADS(2)-) {- constexpr uint num_simdgroups = 2;- constexpr uint results_per_simdgroup = 4;- constexpr uint packs_per_thread = 1;- constexpr uint pack_factor = get_pack_factor<BITS, 32>();- constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();-- constexpr uint values_per_thread = pack_factor * packs_per_thread;- constexpr uint block_size = values_per_thread * 32;- constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;-- const device uint8_t* ws = (const device uint8_t*)weights;- typedef float U;- thread U x_thread[values_per_thread];- thread U result[results_per_simdgroup] = {0};-- const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;- const uint in_vec_size_g = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;- const device WeightT* scales_base = scales;- const uint out_row =- out_block_idx * (num_simdgroups * results_per_simdgroup) +- simd_group * results_per_simdgroup;- const uint used_out_row = min(out_vec_size - results_per_simdgroup, out_row);-- if (out_row >= out_vec_size) {- return;- }-- if (out_vec_size < (num_simdgroups * results_per_simdgroup)) {- ws +=- out_row * in_vec_size_w + simd_lane * packs_per_thread * bytes_per_pack;- scales += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;-- const uint zp_stride =- BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;- const device uint8_t* zps_row_base = nullptr;- const device WeightT* biases_row_base = nullptr;- if (quant_method == QuantizationMethod::ScaleBias) {- biases_row_base = biases + out_row * in_vec_size_g;- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {- zps_row_base = zero_points + out_row * zp_stride;- }-- input += batch_idx * in_vec_size + simd_lane * values_per_thread;- output += batch_idx * out_vec_size + out_row;-- uint k = 0;- for (; k + block_size <= in_vec_size; k += block_size) {- U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);-- for (uint row = 0;- out_row + row < out_vec_size && row < results_per_simdgroup;- row++) {- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);- const uint row_idx = out_row + row;- const device WeightT* sr = scales_base + row_idx * in_vec_size_g;-- uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;- U s = static_cast<U>(sr[g]);- if (quant_method == QuantizationMethod::ScaleBias) {- const device WeightT* bl = biases_row_base + row * in_vec_size_g;- U b = static_cast<U>(bl[g]);- result[row] +=- qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {- const device uint8_t* zl = zps_row_base + row * zp_stride;- U zp;- if (BITS == 4) {- uint8_t zp_b = zl[g >> 1];- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));- } else {- zp = static_cast<U>(zl[g]);- }- result[row] +=- qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);- } else {- constexpr U midpoint = U(1u << (BITS - 1));- result[row] += qdot<U, values_per_thread, BITS>(- wl,- x_thread,- s,- -s * midpoint,- sum- );- }- }-- ws += block_size * bytes_per_pack / pack_factor;- scales += block_size / GROUP_SIZE;- input += block_size;- }- const uint thread_offset = simd_lane * values_per_thread;- const int remaining =- (k + thread_offset < in_vec_size)- ? min(in_vec_size - k - thread_offset, values_per_thread)- : 0;- if (remaining > 0) {- U sum = load_vector_safe<InputT, U, values_per_thread, BITS>(- input,- x_thread,- remaining- );-- for (uint row = 0;- out_row + row < out_vec_size && row < results_per_simdgroup;- row++) {- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);- const uint row_idx = out_row + row;- const device WeightT* sr = scales_base + row_idx * in_vec_size_g;-- uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;- U s = static_cast<U>(sr[g]);- if (quant_method == QuantizationMethod::ScaleBias) {- const device WeightT* bl = biases_row_base + row * in_vec_size_g;- U b = static_cast<U>(bl[g]);- result[row] +=- qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {- const device uint8_t* zl = zps_row_base + row * zp_stride;- U zp;- if (BITS == 4) {- uint8_t zp_b = zl[g >> 1];- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));- } else {- zp = static_cast<U>(zl[g]);- }- result[row] +=- qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);- } else {- constexpr U midpoint = U(1u << (BITS - 1));- result[row] += qdot<U, values_per_thread, BITS>(- wl,- x_thread,- s,- -s * midpoint,- sum- );- }- }- }-- for (uint row = 0;- out_row + row < out_vec_size && row < results_per_simdgroup;- row++) {- result[row] = simd_sum(result[row]);- if (simd_lane == 0) {- output[row] = static_cast<OutputT>(result[row]);- }- }- } else {- ws += used_out_row * in_vec_size_w +- simd_lane * packs_per_thread * bytes_per_pack;- scales += used_out_row * in_vec_size_g + simd_lane / scale_step_per_thread;-- const uint zp_stride =- BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;- const device uint8_t* zps_row_base = nullptr;- const device WeightT* biases_row_base = nullptr;- if (quant_method == QuantizationMethod::ScaleBias) {- biases_row_base = biases + used_out_row * in_vec_size_g;- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {- zps_row_base = zero_points + used_out_row * zp_stride;- }-- input += batch_idx * in_vec_size + simd_lane * values_per_thread;- output += batch_idx * out_vec_size + used_out_row;-- uint k = 0;- for (; k + block_size <= in_vec_size; k += block_size) {- U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);-- for (uint row = 0; row < results_per_simdgroup; row++) {- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);- const uint row_idx = used_out_row + row;- const device WeightT* sr = scales_base + row_idx * in_vec_size_g;-- uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;- U s = static_cast<U>(sr[g]);- if (quant_method == QuantizationMethod::ScaleBias) {- const device WeightT* bl = biases_row_base + row * in_vec_size_g;- U b = static_cast<U>(bl[g]);- result[row] +=- qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {- const device uint8_t* zl = zps_row_base + row * zp_stride;- U zp;- if (BITS == 4) {- uint8_t zp_b = zl[g >> 1];- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));- } else {- zp = static_cast<U>(zl[g]);- }- result[row] +=- qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);- } else {- constexpr U midpoint = U(1u << (BITS - 1));- result[row] += qdot<U, values_per_thread, BITS>(- wl,- x_thread,- s,- -s * midpoint,- sum- );- }- }-- ws += block_size * bytes_per_pack / pack_factor;- scales += block_size / GROUP_SIZE;- input += block_size;- }- const uint thread_offset = simd_lane * values_per_thread;- const int remaining =- (k + thread_offset < in_vec_size)- ? min(in_vec_size - k - thread_offset, values_per_thread)- : 0;-- if (remaining > 0) {- U sum = load_vector_safe<InputT, U, values_per_thread, BITS>(- input,- x_thread,- remaining- );-- for (uint row = 0; row < results_per_simdgroup; row++) {- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);- const uint row_idx = used_out_row + row;- const device WeightT* sr = scales_base + row_idx * in_vec_size_g;-- uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;- U s = static_cast<U>(sr[g]);- if (quant_method == QuantizationMethod::ScaleBias) {- const device WeightT* bl = biases_row_base + row * in_vec_size_g;- U b = static_cast<U>(bl[g]);- result[row] += qdot_safe<U, values_per_thread, BITS>(- wl,- x_thread,- s,- b,- sum,- remaining- );- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {- const device uint8_t* zl = zps_row_base + row * zp_stride;- U zp;- if (BITS == 4) {- uint8_t zp_b = zl[g >> 1];- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));- } else {- zp = static_cast<U>(zl[g]);- }- result[row] +=- qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);- } else {- constexpr U midpoint = U(1u << (BITS - 1));- result[row] += qdot_safe<U, values_per_thread, BITS>(- wl,- x_thread,- s,- -s * midpoint,- sum,- remaining- );- }- }- }-- for (uint row = 0; row < results_per_simdgroup; row++) {- result[row] = simd_sum(result[row]);- if (simd_lane == 0) {- output[row] = static_cast<OutputT>(result[row]);- }- }- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metaldeleted file mode 100644index 8fe649a8a..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal+++ /dev/null@@ -1,231 +0,0 @@-#include <metal_stdlib>-#include "../../common/dsl.h"-#include "../../hadamard_transform/hadamard_transform.h"-#include "../../generated/quantization_method.h"-#include "../gemm/common/quant_pack.h"-#include "common/qmv_ops.h"--using namespace uzu::quantization_method;-using namespace uzu::gemm;--template <- typename WeightT,- typename InputT,- typename OutputT,- uint GROUP_SIZE,- uint BITS>-VARIANTS(WeightT, float, bfloat)-VARIANTS(InputT, float, bfloat)-VARIANTS(OutputT, float, bfloat)-VARIANTS(GROUP_SIZE, 16, 32, 64, 128)-VARIANTS(BITS, 4, 8)-KERNEL(QuantizedMatmulQmvFast)(- const device uint32_t* weights,- const device WeightT* scales,- const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),- const device WeightT* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),- const device InputT* input,- device OutputT* output,- const device int32_t* hadamard_factors OPTIONAL(use_hadamard),- const constant uint& in_vec_size,- const constant uint& out_vec_size,- const constant uint& batch_size,- const QuantizationMethod quant_method SPECIALIZE,- const bool use_hadamard SPECIALIZE,- threadgroup float shared_results[METAL_SIMD_SIZE],- const uint batch_idx GROUPS(batch_size),- const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),- const uint simd_lane THREADS(32),- const uint simd_group THREADS(8)-) {- constexpr uint packs_per_thread = BITS == 2 ? 1 : 2;- constexpr uint num_simdgroups = 8;- constexpr uint results_per_simdgroup = 4;- constexpr uint pack_factor = get_pack_factor<BITS, 32>();- constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();- constexpr uint values_per_thread = pack_factor * packs_per_thread;- constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;- constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;- const device uint8_t* ws = (const device uint8_t*)weights;- typedef float U;- thread U x_thread[values_per_thread];- thread U result[results_per_simdgroup] = {0};-- const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;- const uint in_vec_size_g = in_vec_size / GROUP_SIZE;- const uint out_row =- out_block_idx * (num_simdgroups * results_per_simdgroup) +- simd_group * results_per_simdgroup;- ws += out_row * in_vec_size_w + simd_lane * packs_per_thread * bytes_per_pack;- scales += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;-- uint zp_stride = 0;- const device uint8_t* zps = nullptr;- bool high_nibble = false;-- if (quant_method == QuantizationMethod::ScaleBias) {- biases += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {- if (BITS == 4) {- zp_stride = (in_vec_size_g + 1) / 2;- zps = zero_points + out_row * zp_stride;- uint g_offset = simd_lane / scale_step_per_thread;- zps += g_offset / 2;- high_nibble = (g_offset & 1);- } else {- zp_stride = in_vec_size_g;- zps = zero_points + out_row * zp_stride;- zps += simd_lane / scale_step_per_thread;- }- }-- input += batch_idx * in_vec_size + simd_lane * values_per_thread;- output += batch_idx * out_vec_size + out_row;-- for (uint k = 0; k < in_vec_size; k += block_size) {- U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);-- {- auto wl0 = (const device uint8_t*)(ws);- auto wl1 = (const device uint8_t*)(ws + in_vec_size_w);- auto wl2 = (const device uint8_t*)(ws + 2 * in_vec_size_w);- auto wl3 = (const device uint8_t*)(ws + 3 * in_vec_size_w);-- U s0 = static_cast<U>(scales[0]);- U s1 = static_cast<U>(scales[in_vec_size_g]);- U s2 = static_cast<U>(scales[2 * in_vec_size_g]);- U s3 = static_cast<U>(scales[3 * in_vec_size_g]);-- if (quant_method == QuantizationMethod::ScaleBias) {- U b0 = static_cast<U>(biases[0]);- U b1 = static_cast<U>(biases[in_vec_size_g]);- U b2 = static_cast<U>(biases[2 * in_vec_size_g]);- U b3 = static_cast<U>(biases[3 * in_vec_size_g]);- result[0] +=- qdot<U, values_per_thread, BITS>(wl0, x_thread, s0, b0, sum);- result[1] +=- qdot<U, values_per_thread, BITS>(wl1, x_thread, s1, b1, sum);- result[2] +=- qdot<U, values_per_thread, BITS>(wl2, x_thread, s2, b2, sum);- result[3] +=- qdot<U, values_per_thread, BITS>(wl3, x_thread, s3, b3, sum);- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {- uchar4 zp_bytes = uchar4(- zps[0],- zps[zp_stride],- zps[2 * zp_stride],- zps[3 * zp_stride]- );- uchar4 zp_nibbles;- if (BITS == 4) {- const uint8_t shift = high_nibble ? 4u : 0u;- zp_nibbles = (zp_bytes >> shift) & uchar4(0x0F);- } else {- zp_nibbles = zp_bytes;- }- result[0] += qdot<U, values_per_thread, BITS>(- wl0,- x_thread,- s0,- -s0 * static_cast<U>(zp_nibbles.x),- sum- );- result[1] += qdot<U, values_per_thread, BITS>(- wl1,- x_thread,- s1,- -s1 * static_cast<U>(zp_nibbles.y),- sum- );- result[2] += qdot<U, values_per_thread, BITS>(- wl2,- x_thread,- s2,- -s2 * static_cast<U>(zp_nibbles.z),- sum- );- result[3] += qdot<U, values_per_thread, BITS>(- wl3,- x_thread,- s3,- -s3 * static_cast<U>(zp_nibbles.w),- sum- );- } else {- constexpr U midpoint = U(1u << (BITS - 1));- result[0] += qdot<U, values_per_thread, BITS>(- wl0,- x_thread,- s0,- -s0 * midpoint,- sum- );- result[1] += qdot<U, values_per_thread, BITS>(- wl1,- x_thread,- s1,- -s1 * midpoint,- sum- );- result[2] += qdot<U, values_per_thread, BITS>(- wl2,- x_thread,- s2,- -s2 * midpoint,- sum- );- result[3] += qdot<U, values_per_thread, BITS>(- wl3,- x_thread,- s3,- -s3 * midpoint,- sum- );- }- }-- ws += block_size * bytes_per_pack / pack_factor;- scales += block_size / GROUP_SIZE;- if (quant_method == QuantizationMethod::ScaleBias) {- biases += block_size / GROUP_SIZE;- } else if (quant_method == QuantizationMethod::ScaleZeroPoint) {- if (BITS == 4) {- zps += (block_size / GROUP_SIZE) / 2;- } else {- zps += block_size / GROUP_SIZE;- }- }- input += block_size;- }-- for (uint row = 0; row < results_per_simdgroup; row++) {- result[row] = simd_sum(result[row]);- }-- if (use_hadamard) {- if (simd_lane == 0) {- for (uint row = 0; row < results_per_simdgroup; row++) {- shared_results[simd_group * results_per_simdgroup + row] = result[row];- }- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (simd_group == 0) {- uint global_out_idx = out_block_idx * 32 + simd_lane;- if (global_out_idx < out_vec_size) {- output[simd_lane] = simdgroup_output_random_hadamard_transform(- static_cast<ushort>(simd_lane),- OutputT(shared_results[simd_lane]),- hadamard_factors[global_out_idx]- );- }- }- } else {- if (simd_lane == 0) {- for (uint row = 0; row < results_per_simdgroup; row++) {- output[row] = static_cast<OutputT>(result[row]);- }- }- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rsdeleted file mode 100644index 9e456cd82..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs+++ /dev/null@@ -1,192 +0,0 @@-use std::collections::{HashMap, hash_map::Entry};--use crate::{- backends::{- common::{- AsBufferRangeRef, Buffer, Encoder,- gpu_types::{QuantizationMethod, QuantizationMode, gemm::GemmDTransform},- kernel::matmul::{MatmulArguments, MatmulB, MatmulError},- },- metal::{- Metal,- context::MetalContext,- kernel::{QuantizedMatmulQmvFastMetalKernel, QuantizedMatmulQmvMetalKernel},- },- },- data_type::DataType,-};--const PATH: &str = "QuantGemv";--#[derive(PartialEq, Eq, Hash, Clone, Copy)]-struct QmvKey {- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,- use_hadamard: bool,-}--pub struct QuantGemvKernel {- weights_data_type: DataType,- input_data_type: DataType,- output_data_type: DataType,- qmv: HashMap<QmvKey, QuantizedMatmulQmvMetalKernel>,- qmv_fast: HashMap<QmvKey, QuantizedMatmulQmvFastMetalKernel>,-}--impl QuantGemvKernel {- pub fn new(- _context: &MetalContext,- weights_data_type: DataType,- input_data_type: DataType,- output_data_type: DataType,- ) -> Self {- Self {- weights_data_type,- input_data_type,- output_data_type,- qmv: HashMap::new(),- qmv_fast: HashMap::new(),- }- }-- pub fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(- &mut self,- encoder: &mut Encoder<Metal>,- arguments: MatmulArguments<'a, Metal, TB>,- ) -> Result<(), MatmulError<Metal>> {- let mask = arguments.d_transform.mask();- if mask.contains(GemmDTransform::SCALE) {- return Err(MatmulError::UnsupportedDOp {- bit: GemmDTransform::SCALE,- path: PATH,- });- }- if mask.contains(GemmDTransform::ACCUMULATE) {- return Err(MatmulError::UnsupportedDOp {- bit: GemmDTransform::ACCUMULATE,- path: PATH,- });- }- if !arguments.b_transpose || arguments.b_leading_dimension.is_some() {- return Err(MatmulError::UnsupportedLayout {- path: PATH,- });- }-- let hadamard_factors = arguments.d_transform.rht_factors;-- let MatmulArguments {- a,- a_offset,- b,- d,- m,- n,- k,- ..- } = arguments;- let (weights, scales, zp_or_bias, method, mode, group_size) = match b {- MatmulB::ScaleBiasDequant {- b: w,- scales,- biases,- mode,- group_size,- } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),- MatmulB::ScaleZeroPointDequant {- b: w,- scales,- zero_points,- mode,- group_size,- } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),- MatmulB::ScaleSymmetricDequant {- b: w,- scales,- mode,- group_size,- } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),- MatmulB::FullPrecision {- ..- } => panic!("QuantGemvKernel requires quantized B"),- };-- let bits = match mode {- QuantizationMode::U4 => 4u32,- QuantizationMode::I8 | QuantizationMode::U8 => 8u32,- };- let use_fast = n % 8 == 0 && k % 512 == 0;-- let (zero_points, biases) = match method {- QuantizationMethod::ScaleBias => {- (None, Some(zp_or_bias.expect("ScaleBias quantized matmul requires biases")))- },- QuantizationMethod::ScaleZeroPoint => {- (Some(zp_or_bias.expect("ScaleZeroPoint quantized matmul requires zero_points")), None)- },- QuantizationMethod::ScaleSymmetric => (None, None),- };-- if use_fast {- let key = QmvKey {- group_size,- bits,- quant_method: method,- use_hadamard: hadamard_factors.is_some(),- };- let context = encoder.context();- let kernel = match self.qmv_fast.entry(key) {- Entry::Occupied(entry) => entry.into_mut(),- Entry::Vacant(entry) => {- let kernel = QuantizedMatmulQmvFastMetalKernel::new(- context,- self.weights_data_type,- self.input_data_type,- self.output_data_type,- group_size,- bits,- method,- hadamard_factors.is_some(),- )- .map_err(MatmulError::BackendError)?;- entry.insert(kernel)- },- };- kernel.encode(weights, scales, zero_points, biases, (a, a_offset), d, hadamard_factors, k, n, m, encoder);- } else {- if hadamard_factors.is_some() {- return Err(MatmulError::UnsupportedDOp {- bit: GemmDTransform::RHT,- path: PATH,- });- }- let key = QmvKey {- group_size,- bits,- quant_method: method,- use_hadamard: false,- };- let context = encoder.context();- let kernel = match self.qmv.entry(key) {- Entry::Occupied(entry) => entry.into_mut(),- Entry::Vacant(entry) => {- let kernel = QuantizedMatmulQmvMetalKernel::new(- context,- self.weights_data_type,- self.input_data_type,- self.output_data_type,- group_size,- bits,- method,- )- .map_err(MatmulError::BackendError)?;- entry.insert(kernel)- },- };- kernel.encode(weights, scales, zero_points, biases, (a, a_offset), d, k, n, m, encoder);- }-- Ok(())- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rsdeleted file mode 100644index 1cc022959..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs+++ /dev/null@@ -1,142 +0,0 @@-use crate::data_type::DataType;--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct GemvSpecialization {- pub threadgroup_rows: u32,- pub threadgroup_cols: u32,- pub threads_per_simdgroup_row: u32,- pub threads_per_simdgroup_col: u32,- pub elements_per_thread_row: u32,- pub elements_per_thread_col: u32,- pub is_accumulate: bool,- pub is_bias: bool,-}--impl GemvSpecialization {- pub fn precompile_configs(weights_data_type: DataType) -> &'static [Self] {- match weights_data_type {- DataType::F32 => &[- Self {- threadgroup_rows: 4,- threadgroup_cols: 1,- threads_per_simdgroup_row: 1,- threads_per_simdgroup_col: 32,- elements_per_thread_row: 4,- elements_per_thread_col: 4,- is_accumulate: false,- is_bias: false,- },- Self {- threadgroup_rows: 4,- threadgroup_cols: 1,- threads_per_simdgroup_row: 1,- threads_per_simdgroup_col: 32,- elements_per_thread_row: 4,- elements_per_thread_col: 4,- is_accumulate: false,- is_bias: true,- },- ],- DataType::BF16 => &[- Self {- threadgroup_rows: 4,- threadgroup_cols: 1,- threads_per_simdgroup_row: 1,- threads_per_simdgroup_col: 32,- elements_per_thread_row: 4,- elements_per_thread_col: 4,- is_accumulate: false,- is_bias: false,- },- Self {- threadgroup_rows: 4,- threadgroup_cols: 1,- threads_per_simdgroup_row: 1,- threads_per_simdgroup_col: 32,- elements_per_thread_row: 4,- elements_per_thread_col: 4,- is_accumulate: false,- is_bias: true,- },- Self {- threadgroup_rows: 8,- threadgroup_cols: 1,- threads_per_simdgroup_row: 1,- threads_per_simdgroup_col: 32,- elements_per_thread_row: 4,- elements_per_thread_col: 4,- is_accumulate: false,- is_bias: false,- },- Self {- threadgroup_rows: 8,- threadgroup_cols: 1,- threads_per_simdgroup_row: 1,- threads_per_simdgroup_col: 32,- elements_per_thread_row: 4,- elements_per_thread_col: 4,- is_accumulate: false,- is_bias: true,- },- ],- _ => &[],- }- }-- pub fn output_rows_per_threadgroup(&self) -> u32 {- self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row- }-- pub fn select(- input_dimension: u32,- output_dimension: u32,- is_accumulate: bool,- is_bias: bool,- ) -> Self {- let (threadgroup_rows, threadgroup_cols);- let (threads_per_simdgroup_row, threads_per_simdgroup_col);- let (elements_per_thread_row, elements_per_thread_col);-- let threadgroup_simd_rows;- let mut simdgroup_thread_rows = 1;- let mut simdgroup_thread_cols = 32;- let mut threadgroup_simd_cols = 1;-- if input_dimension <= 64 {- threadgroup_simd_rows = 1;- simdgroup_thread_rows = 8;- simdgroup_thread_cols = 4;- } else if input_dimension >= 16 * output_dimension {- threadgroup_simd_rows = 1;- threadgroup_simd_cols = 8;- } else if output_dimension >= 4096 {- threadgroup_simd_rows = 8;- } else {- threadgroup_simd_rows = 4;- }-- let thread_output_rows = if output_dimension < 4 {- 1- } else {- 4- };-- threadgroup_rows = threadgroup_simd_rows;- threadgroup_cols = threadgroup_simd_cols;- threads_per_simdgroup_row = simdgroup_thread_rows;- threads_per_simdgroup_col = simdgroup_thread_cols;- elements_per_thread_row = thread_output_rows;- elements_per_thread_col = 4;-- Self {- threadgroup_rows,- threadgroup_cols,- threads_per_simdgroup_row,- threads_per_simdgroup_col,- elements_per_thread_row,- elements_per_thread_col,- is_accumulate,- is_bias,- }- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex 982c80ca7..b66dca74d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -1,22 +1,13 @@pub mod gemm;pub mod gemv;-use std::sync::OnceLock;--use self::gemv::GemvKernel;-pub use self::{- gemm::{GemmDispatchPath, GemmKernel},- gemv::QuantGemvKernel,-};+pub use self::gemm::{GemmDispatchPath, GemmKernel};+use self::gemv::{GemvDispatch, GemvSpecialization};use crate::{backends::{common::{- AsBufferRangeRef, Backend, Buffer, Encoder,- gpu_types::gemm::GemmDTransform,- kernel::{- HadamardTransformKernel, TensorAddBiasKernel,- matmul::{MatmulArguments, MatmulB, MatmulError, MatmulKernel, MatmulQuantCombo},- },+ AsBufferRangeRef, Buffer, Encoder,+ kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulQuantCombo},},metal::{Metal, context::MetalContext, error::MetalError},},@@ -24,118 +15,11 @@ use crate::{};pub struct MatmulMetalKernel {- gemv: GemvKernel,- pub quant_gemv: QuantGemvKernel,+ gemv: GemvDispatch,pub gemm: GemmKernel,-}--const DEFAULT_GEMV_MAX_BATCH: u32 = 8;-static GEMV_MAX_BATCH: OnceLock<u32> = OnceLock::new();--fn max_gemv_batch_threshold() -> u32 {- *GEMV_MAX_BATCH.get_or_init(|| {- std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)- })-}--impl MatmulMetalKernel {- fn dispatch_gemv<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(- &mut self,- arguments: MatmulArguments<'a, Metal, TB>,- encoder: &mut Encoder<Metal>,- ) -> Result<(), MatmulError<Metal>> {- let post_rht = arguments.d_transform.rht_factors;-- let MatmulArguments {- a,- a_offset,- b,- b_offset,- b_leading_dimension,- b_transpose,- d,- d_transform,- m,- n,- k,- } = arguments;-- // A temopary hack, will port the post ops to the gemv later.- let d_transform = d_transform.without(GemmDTransform::RHT);-- gemv::fp::encode(- &mut self.gemv,- encoder,- MatmulArguments {- a,- a_offset,- b,- b_offset,- b_leading_dimension,- b_transpose,- d: &mut *d,- d_transform,- m,- n,- k,- },- )?;-- if let Some(factors) = post_rht {- self.gemm.hadamard.encode(d, factors, n, m, encoder);- }- Ok(())- }-- fn dispatch_quant_gemv<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(- &mut self,- arguments: MatmulArguments<'a, Metal, TB>,- encoder: &mut Encoder<Metal>,- ) -> Result<(), MatmulError<Metal>> {- if arguments.b_offset != 0 {- return Err(MatmulError::UnsupportedLayout {- path: "QuantGemv",- });- }- let post_bias = arguments.d_transform.bias;-- let MatmulArguments {- a,- a_offset,- b,- b_offset,- b_leading_dimension,- b_transpose,- d,- d_transform,- m,- n,- k,- } = arguments;- let d_transform = d_transform.without(GemmDTransform::BIAS);-- self.quant_gemv.encode(- encoder,- MatmulArguments {- a,- a_offset,- b,- b_offset,- b_leading_dimension,- b_transpose,- d: &mut *d,- d_transform,- m,- n,- k,- },- )?;-- if let Some(bias) = post_bias {- self.gemm.bias_add.encode(None::<&<Metal as Backend>::DenseBuffer>, bias, d, n, m * n, encoder);- }- Ok(())- }+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,}impl MatmulKernel for MatmulMetalKernel {@@ -154,13 +38,15 @@ impl MatmulKernel for MatmulMetalKernel {}let gemm = GemmKernel::new(context, weights_data_type, input_data_type, output_data_type)?;- let gemv = GemvKernel::new(context, weights_data_type, input_data_type, output_data_type)?;- let quant_gemv = QuantGemvKernel::new(context, weights_data_type, input_data_type, output_data_type);+ let gemv = GemvDispatch::new(context, weights_data_type, input_data_type, output_data_type)+ .map_err(MetalError::from)?;Ok(Self {gemv,- quant_gemv,gemm,+ weights_data_type,+ input_data_type,+ output_data_type,})}@@ -169,25 +55,14 @@ impl MatmulKernel for MatmulMetalKernel {arguments: MatmulArguments<Metal, TB>,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {- let is_quant = !matches!(arguments.b, MatmulB::FullPrecision { .. });- let gemv_eligible = if is_quant {- arguments.m < 5 || arguments.n == 1- } else {- arguments.b_transpose- && arguments.b_offset == 0- && arguments.b_leading_dimension.is_none_or(|ld| ld == arguments.k)- && arguments.m <= max_gemv_batch_threshold()- };-- if gemv_eligible {- if is_quant {- self.dispatch_quant_gemv(arguments, encoder)?;- } else {- self.dispatch_gemv(arguments, encoder)?;- }- Ok(())- } else {- self.gemm.encode(arguments, encoder)+ match GemvSpecialization::select(+ &arguments,+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,+ ) {+ Some(spec) => self.gemv.encode(arguments, spec, encoder).map_err(MetalError::from),+ None => self.gemm.encode(arguments, encoder),}}@@ -196,6 +71,7 @@ impl MatmulKernel for MatmulMetalKernel {context: &MetalContext,combo: MatmulQuantCombo,) -> Result<(), MetalError> {+ self.gemv.preheat_quant_combo(context, combo).map_err(MetalError::from)?;self.gemm.preheat_quant_combo(context, combo)}}diff --git a/crates/backend-uzu/tests/common/matmul/shape.rs b/crates/backend-uzu/tests/common/matmul/shape.rsindex 33e7d58ef..d0760c519 100644--- a/crates/backend-uzu/tests/common/matmul/shape.rs+++ b/crates/backend-uzu/tests/common/matmul/shape.rs@@ -72,7 +72,7 @@ pub fn bench_quant_gemv_shapes(bits: u32) -> impl Iterator<Item = Shape> {256};let nk: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];- let ms = &[1usize, 2, 4, 8];+ let ms = &[1usize, 2, 4];nk.iter().filter(move |&&(n, k)| n % 8 == 0 && k % block_size == 0).flat_map(move |&(n, k)| ms.iter().map(move |&m| Shape::new(m, k, n)))diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rsindex 85b56a68b..109ff2d45 100644--- a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs@@ -1,14 +1,9 @@-#![cfg(metal_backend)]-use backend_uzu::{array::ArrayElement,- backends::{- common::{- Backend,- gpu_types::QuantizationMethod,- kernel::{Kernels, matmul::MatmulKernel},- },- metal::{Metal, MetalContext},+ backends::common::{+ Backend, Context,+ gpu_types::QuantizationMethod,+ kernel::{Kernels, matmul::MatmulKernel},},};use criterion::{BenchmarkId, Criterion, Throughput};@@ -23,21 +18,21 @@ use crate::{uzu_bench,};-fn bench_quant_gemv_typed<T: ArrayElement + Float>(- context: &MetalContext,+fn bench_gemv_typed<B: Backend, T: ArrayElement + Float>(c: &mut Criterion,+ context: &B::Context,label: &str,group_size: u32,bits: u32,quant_method: QuantizationMethod,) {- let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/QuantGemv/{}", type_short_name::<Metal>(), label));+ let mut group = c.benchmark_group(format!("{}/Kernel/Gemv/{}", type_short_name::<B>(), label));for shape in bench_quant_gemv_shapes(bits) {let (m, k, n) = (shape.m, shape.k, shape.n);let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);- let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(+ let mut buffers = QuantBuffers::<B, T>::allocate(context, &input);+ let mut matmul = <<B as Backend>::Kernels as Kernels>::MatmulKernel::new(context,T::data_type(),T::data_type(),@@ -47,18 +42,24 @@ fn bench_quant_gemv_typed<T: ArrayElement + Float>(group.throughput(Throughput::Elements((m * n * k) as u64));group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {- iter_encode_loop::<Metal, _>(context, b, |encoder| {- matmul.quant_gemv.encode(encoder, quant_arguments(&mut buffers, &input)).expect("encode quant gemv");+ iter_encode_loop::<B, _>(context, b, |encoder| {+ let args = quant_arguments(&mut buffers, &input);+ matmul.encode(args, encoder).expect("encode failed");});});}}#[uzu_bench]-fn bench_quant_gemv(c: &mut Criterion) {- let context = crate::common::shared_metal_context();- bench_quant_gemv_typed::<bf16>(&context, c, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);- bench_quant_gemv_typed::<bf16>(&context, c, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- bench_quant_gemv_typed::<bf16>(&context, c, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);- bench_quant_gemv_typed::<bf16>(&context, c, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);+fn bench_gemv(c: &mut Criterion) {+ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().unwrap();+ bench_gemv_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);+ bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);+ bench_gemv_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);+ bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);+ bench_gemv_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);+ bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);+ bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);+ });}diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rsindex a42f5d912..15951e0af 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs@@ -132,3 +132,46 @@ fn bias_parity_bf16() {}}}++#[test]+fn gemv_fp_partial_output_block_bf16() {+ use crate::common::matmul::Shape;+ let context = MetalContext::new().expect("Metal context");+ let mut kernel = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("MatmulKernel");+ // Small m routes to GEMV; n that is not a multiple of the 32-row threadgroup+ // block exercises the tail-row clamp (must not read weight rows past B).+ let shapes = [Shape::new(1, 128, 33), Shape::new(2, 256, 65), Shape::new(4, 384, 100)];+ for shape in shapes {+ check_case::<bf16>(&context, &mut kernel, None, Case::new(shape), 1.0);+ }+}++#[test]+fn gemv_fp_output_transforms_bf16() {+ use crate::common::matmul::Shape;+ let context = MetalContext::new().expect("Metal context");+ let mut kernel = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("MatmulKernel");+ // Small m routes to GEMV; n % 32 == 0 keeps RHT/accumulate in the GEMV path.+ let shape = Shape::new(2, 256, 64);+ let cases = [+ Case::new(shape).with_bias(true).with_rht(true),+ Case::new(shape).with_accumulate(true),+ Case::new(shape).with_accumulate(true).with_bias(true),+ Case::new(shape).with_ab_scale(2.0).with_rht(true),+ ];+ for case in cases {+ check_case::<bf16>(&context, &mut kernel, None, case, 1.0);+ }+}diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rsindex 0d1e0fbd6..87bca2467 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs@@ -16,6 +16,7 @@ use backend_uzu::{matmul::{MatmulDOps, MatmulError, MatmulKernel},},},+ cpu::Cpu,metal::{GemmDispatchPath, Metal, MetalContext},},};@@ -114,6 +115,54 @@ fn parity_bf16(run_parity::<bf16>(m, k, n, gs, bits, method, 0.05, 0.4);}+fn run_parity_gemv<T: ArrayElement + Float + Debug + Display>(+ m: usize,+ k: usize,+ n: usize,+ group_size: u32,+ bits: u32,+ quant_method: QuantizationMethod,+ rel_tol: f64,+ abs_tol: f64,+) {+ let context = MetalContext::new().expect("Metal context");+ let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 0);+ let gemv = run_quant_metal::<T>(&context, &input, None);+ let reference = run_quant_cpu::<T>(&input);+ assert_parity::<T>(+ &format!(+ "gemv m={m} k={k} n={n} gs={group_size} bits={bits} method={quant_method:?} dtype={}",+ std::any::type_name::<T>()+ ),+ &reference,+ &gemv,+ rel_tol,+ abs_tol,+ );+}++#[rstest]+#[case::m1_gs32_4bit_mlx(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]+#[case::m1_gs64_4bit_mlx(1, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]+#[case::m1_gs128_4bit_mlx(1, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]+#[case::m1_gs32_4bit_zp(1, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]+#[case::m1_gs64_8bit_zp(1, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]+#[case::m1_gs128_8bit_mlx(1, 256, 64, 128, 8, QuantizationMethod::ScaleBias)]+#[case::m1_gs32_4bit_sym(1, 256, 64, 32, 4, QuantizationMethod::ScaleSymmetric)]+#[case::m1_gs64_8bit_sym(1, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]+#[case::m2_gs32_4bit_mlx(2, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]+#[case::m4_gs32_4bit_zp(4, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]+fn parity_gemv_bf16(+ #[case] m: usize,+ #[case] k: usize,+ #[case] n: usize,+ #[case] gs: u32,+ #[case] bits: u32,+ #[case] method: QuantizationMethod,+) {+ run_parity_gemv::<bf16>(m, k, n, gs, bits, method, 0.05, 0.4);+}+#[uzu_test]fn parity_bf16_gs32_4bit_mlx_with_bias() {let context = MetalContext::new().expect("Metal context");@@ -154,6 +203,213 @@ fn parity_bf16_gs32_4bit_mlx_with_bias() {assert_parity::<bf16>("with_bias", &reference, &actual, 0.05, 0.4);}+#[uzu_test]+fn parity_bf16_gemv_qmv_fused_scale_bias() {+ let context = MetalContext::new().expect("Metal context");+ let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);++ let scale = 2.0_f32;+ let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.25 + 0.1 * (j % 7) as f32).collect();+ let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();++ let raw = run_quant_cpu::<bf16>(&input);+ let reference: Vec<bf16> = (0..input.m as usize)+ .flat_map(|i| {+ (0..input.n as usize).map(move |j| {+ let idx = i * input.n as usize + j;+ (idx, j)+ })+ })+ .map(|(idx, j)| bf16::from_f32(scale * raw[idx].to_f32() + bias_f32[j]))+ .collect();++ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);+ let bias_buf = crate::common::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("MatmulMetalKernel");++ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");+ let mut args = quant_arguments(&mut buffers, &input);+ args.d_transform = MatmulDOps {+ ab_scale: scale,+ accumulate: false,+ bias: Some(&bias_buf),+ rht_factors: None,+ };+ matmul.encode(args, &mut encoder).expect("encode quant gemv with scale+bias");+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);++ assert_parity::<bf16>("gemv_qmv_scale_bias", &reference, &actual, 0.05, 0.4);+}++#[rstest]+#[case::gs64_4bit(1, 96, 64, 64, 4, QuantizationMethod::ScaleBias)]+#[case::gs64_4bit_zp(2, 96, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]+#[case::gs128_8bit(1, 192, 64, 128, 8, QuantizationMethod::ScaleBias)]+fn parity_gemv_partial_group_bf16(+ #[case] m: usize,+ #[case] k: usize,+ #[case] n: usize,+ #[case] gs: u32,+ #[case] bits: u32,+ #[case] method: QuantizationMethod,+) {+ // k is not a multiple of group_size, so the per-row scale/zp stride must use+ // ceil(k / group_size) groups.+ run_parity_gemv::<bf16>(m, k, n, gs, bits, method, 0.05, 0.6);+}++#[rstest]+#[case::n12_gs32_4bit(1, 256, 12, 32, 4, QuantizationMethod::ScaleBias)]+#[case::n20_gs64_4bit_zp(2, 256, 20, 64, 4, QuantizationMethod::ScaleZeroPoint)]+#[case::n36_gs32_8bit(1, 256, 36, 32, 8, QuantizationMethod::ScaleBias)]+fn parity_gemv_unaligned_width_bf16(+ #[case] m: usize,+ #[case] k: usize,+ #[case] n: usize,+ #[case] gs: u32,+ #[case] bits: u32,+ #[case] method: QuantizationMethod,+) {+ // n is not a multiple of 8; the output-tail clamp must cover the partial block.+ run_parity_gemv::<bf16>(m, k, n, gs, bits, method, 0.05, 0.6);+}++#[uzu_test]+fn parity_bf16_gemv_quant_rht_with_bias() {+ let context = MetalContext::new().expect("Metal context");+ let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);+ let rht: Vec<i32> = (0..input.n as usize)+ .map(|i| {+ if i % 2 == 0 {+ 1+ } else {+ -1+ }+ })+ .collect();+ let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.3 + 0.05 * (j % 4) as f32).collect();+ let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();++ // CPU reference applies bias before RHT (matmul loop adds bias, then the+ // Hadamard pass transforms the result) — the same order GEMV uses.+ let cpu_context = <Cpu as Backend>::Context::new().expect("Cpu context");+ let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);+ let cpu_rht = crate::common::helpers::alloc_allocation_with_data::<Cpu, i32>(&cpu_context, &rht);+ let cpu_bias = crate::common::helpers::alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, &bias_t);+ let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &cpu_context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("MatmulCpuKernel");+ let mut cpu_encoder = Encoder::<Cpu>::new(&cpu_context).expect("cpu encoder");+ let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);+ cpu_args.d_transform = MatmulDOps {+ ab_scale: 1.0,+ accumulate: false,+ bias: Some(&cpu_bias),+ rht_factors: Some(&cpu_rht),+ };+ cpu_matmul.encode(cpu_args, &mut cpu_encoder).expect("cpu encode quant+rht+bias");+ cpu_encoder.end_encoding().submit().wait_until_completed().unwrap();+ let reference = allocation_to_vec::<Cpu, bf16>(&cpu_buffers.y);++ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);+ let metal_rht = crate::common::helpers::alloc_allocation_with_data::<Metal, i32>(&context, &rht);+ let metal_bias = crate::common::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("MatmulMetalKernel");+ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");+ let mut args = quant_arguments(&mut buffers, &input);+ args.d_transform = MatmulDOps {+ ab_scale: 1.0,+ accumulate: false,+ bias: Some(&metal_bias),+ rht_factors: Some(&metal_rht),+ };+ matmul.encode(args, &mut encoder).expect("encode quant gemv with rht+bias");+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);++ assert_parity::<bf16>("gemv_quant_rht_bias", &reference, &actual, 0.05, 0.6);+}++#[uzu_test]+fn parity_bf16_gemv_quant_rht() {+ let context = MetalContext::new().expect("Metal context");+ // n % 32 == 0 so the output RHT covers whole 32-element blocks; m = 1 routes to GEMV.+ let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);+ let rht: Vec<i32> = (0..input.n as usize)+ .map(|i| {+ if i % 2 == 0 {+ 1+ } else {+ -1+ }+ })+ .collect();++ // CPU reference applies the RHT through the CPU matmul kernel.+ let cpu_context = <Cpu as Backend>::Context::new().expect("Cpu context");+ let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);+ let cpu_rht = crate::common::helpers::alloc_allocation_with_data::<Cpu, i32>(&cpu_context, &rht);+ let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &cpu_context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("MatmulCpuKernel");+ let mut cpu_encoder = Encoder::<Cpu>::new(&cpu_context).expect("cpu encoder");+ let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);+ cpu_args.d_transform = MatmulDOps {+ ab_scale: 1.0,+ accumulate: false,+ bias: None,+ rht_factors: Some(&cpu_rht),+ };+ cpu_matmul.encode(cpu_args, &mut cpu_encoder).expect("cpu encode quant+rht");+ cpu_encoder.end_encoding().submit().wait_until_completed().unwrap();+ let reference = allocation_to_vec::<Cpu, bf16>(&cpu_buffers.y);++ // Metal GEMV: m = 1 quant routes to GEMV, RHT selects the 8-simdgroup (32-row) layout.+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);+ let metal_rht = crate::common::helpers::alloc_allocation_with_data::<Metal, i32>(&context, &rht);+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("MatmulMetalKernel");+ let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");+ let mut args = quant_arguments(&mut buffers, &input);+ args.d_transform = MatmulDOps {+ ab_scale: 1.0,+ accumulate: false,+ bias: None,+ rht_factors: Some(&metal_rht),+ };+ matmul.encode(args, &mut encoder).expect("encode quant gemv with rht");+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);++ assert_parity::<bf16>("gemv_quant_rht", &reference, &actual, 0.05, 0.6);+}+#[uzu_test]fn quant_gemm_nonzero_b_offset_returns_unsupported_layout() {let context = MetalContext::new().expect("Metal context");