A high-performance inference engine for AI models
Follow rust convention for NTTP name case (#227)
uuuvn555dae36c2c8
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metalindex e151c7d38..b599a39a2 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -5,9 +5,9 @@#define SEQUENCE_BLOCK_SIZE 32#define HEAD_BLOCK_SIZE 32-template <typename T, uint head_dim>+template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)-VARIANTS(head_dim, 64, 128, 256)+VARIANTS(HEAD_DIM, 64, 128, 256)KERNEL(AttentionSinglePass)(const device T* queries,const device T* keys,@@ -42,8 +42,8 @@ KERNEL(AttentionSinglePass)(const uint3 tpg = {num_heads, suffix_length, 1};constexpr bool query_transposed = false;- constexpr uint value_dim = head_dim;- constexpr uint qk_elements_per_thread = head_dim / HEAD_BLOCK_SIZE;+ constexpr uint value_dim = HEAD_DIM;+ constexpr uint qk_elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;constexpr uint value_elements_per_thread = value_dim / HEAD_BLOCK_SIZE;uint inner_k_stride = SEQUENCE_BLOCK_SIZE * int(k_seq_stride);uint inner_v_stride = SEQUENCE_BLOCK_SIZE * int(v_seq_stride);@@ -59,7 +59,7 @@ KERNEL(AttentionSinglePass)(const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx: head_idx * tpg.y + q_seq_idx;- queries += q_offset * head_dim + simd.lane_idx * qk_elements_per_thread;+ queries += q_offset * HEAD_DIM + simd.lane_idx * qk_elements_per_thread;keys += kv_head_idx * k_head_stride + simd.group_idx * k_seq_stride +simd.lane_idx * qk_elements_per_thread;values += kv_head_idx * v_head_stride + simd.group_idx * v_seq_stride +@@ -164,4 +164,4 @@ KERNEL(AttentionSinglePass)(out[i] = static_cast<T>(o[i]);}}-}\ No newline at end of file+}diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metalindex bb322b0fe..cb7ab22ce 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -7,9 +7,9 @@#define SEQUENCE_BLOCK_SIZE_2 32#define TOTAL_BLOCKS_COUNT 32-template <typename T, uint head_dim>+template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)-VARIANTS(head_dim, 64, 128, 256)+VARIANTS(HEAD_DIM, 64, 128, 256)KERNEL(AttentionTwoPass1)(const device T* queries,const device T* keys,@@ -47,8 +47,8 @@ KERNEL(AttentionTwoPass1)(const uint3 tpg = {num_heads, suffix_length, TOTAL_BLOCKS_COUNT};constexpr bool query_transposed = false;- constexpr uint value_dim = head_dim;- constexpr uint qk_elements_per_thread = head_dim / HEAD_BLOCK_SIZE;+ constexpr uint value_dim = HEAD_DIM;+ constexpr uint qk_elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;constexpr uint value_elements_per_thread = value_dim / HEAD_BLOCK_SIZE;uint inner_k_stride = SEQUENCE_BLOCK_SIZE_1 * k_seq_stride;uint inner_v_stride = SEQUENCE_BLOCK_SIZE_1 * v_seq_stride;@@ -66,7 +66,7 @@ KERNEL(AttentionTwoPass1)(: head_idx * tpg.y + q_seq_idx; // Consistent with single-passconst uint kv_head_idx = head_idx / gqa_factor;- queries += q_offset * head_dim + simd.lane_idx * qk_elements_per_thread;+ queries += q_offset * HEAD_DIM + simd.lane_idx * qk_elements_per_thread;keys += kv_head_idx * k_head_stride +(block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) * k_seq_stride +simd.lane_idx * qk_elements_per_thread;@@ -189,9 +189,9 @@ KERNEL(AttentionTwoPass1)(}}-template <typename T, uint head_dim>+template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)-VARIANTS(head_dim, 64, 128, 256)+VARIANTS(HEAD_DIM, 64, 128, 256)KERNEL(AttentionTwoPass2)(const device float* partials,const device float* sums,@@ -206,18 +206,18 @@ KERNEL(AttentionTwoPass2)(const uint tid THREADS(1024)) {const uint3 tpg = {num_heads, suffix_length, 1};- constexpr uint elements_per_thread = head_dim / HEAD_BLOCK_SIZE;+ constexpr uint elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;typedef float U;thread U o[elements_per_thread];const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout- partials += o_offset * TOTAL_BLOCKS_COUNT * head_dim +- simd.group_idx * head_dim + simd.lane_idx * elements_per_thread;+ partials += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM ++ simd.group_idx * HEAD_DIM + simd.lane_idx * elements_per_thread;sums += o_offset * TOTAL_BLOCKS_COUNT;maxs += o_offset * TOTAL_BLOCKS_COUNT;- out += o_offset * head_dim ++ out += o_offset * HEAD_DIM +simd.group_idx * elements_per_thread; // Our custom output layout// First everybody reads the max and sum_exp@@ -249,4 +249,4 @@ KERNEL(AttentionTwoPass2)(out[i] = static_cast<T>(o[i]);}}-}\ No newline at end of file+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metalindex 47ca57522..b005e80a5 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal@@ -2,10 +2,10 @@#include "../definitions.metal"#include "quant_matmul.metal"-template <typename T, int group_size, int bits>+template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)-VARIANTS(group_size, 32, 64, 128)-VARIANTS(bits, 4, 8)+VARIANTS(GROUP_SIZE, 32, 64, 128)+VARIANTS(BITS, 4, 8)KERNEL(QuantizedMatmulQmm)(const device uint32_t* w,const device T* scales,@@ -35,7 +35,7 @@ KERNEL(QuantizedMatmulQmm)(if (use_mlx_quant) {if (aligned_k) {- qmm_impl<T, group_size, bits, true, 32, 32, 32, true>(+ qmm_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, true>(w,scales,zero_points,@@ -53,7 +53,7 @@ KERNEL(QuantizedMatmulQmm)(simd_lid);} else {- qmm_impl<T, group_size, bits, false, 32, 32, 32, true>(+ qmm_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, true>(w,scales,zero_points,@@ -73,7 +73,7 @@ KERNEL(QuantizedMatmulQmm)(}} else {if (aligned_k) {- qmm_impl<T, group_size, bits, true, 32, 32, 32, false>(+ qmm_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, false>(w,scales,zero_points,@@ -91,7 +91,7 @@ KERNEL(QuantizedMatmulQmm)(simd_lid);} else {- qmm_impl<T, group_size, bits, false, 32, 32, 32, false>(+ qmm_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, false>(w,scales,zero_points,diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metalindex 111d3d0ed..f2b600578 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal@@ -2,10 +2,10 @@#include "../definitions.metal"#include "quant_matmul.metal"-template <typename T, int group_size, int bits>+template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)-VARIANTS(group_size, 32, 64, 128)-VARIANTS(bits, 4, 8)+VARIANTS(GROUP_SIZE, 32, 64, 128)+VARIANTS(BITS, 4, 8)KERNEL(QuantizedMatmulQmmTransposed)(const device uint32_t* w,const device T* scales,@@ -35,7 +35,7 @@ KERNEL(QuantizedMatmulQmmTransposed)(if (use_mlx_quant) {if (aligned_n) {- qmm_transposed_impl<T, group_size, bits, true, 32, 32, 32, true>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, true>(w,scales,zero_points,@@ -53,7 +53,7 @@ KERNEL(QuantizedMatmulQmmTransposed)(simd_lid);} else {- qmm_transposed_impl<T, group_size, bits, false, 32, 32, 32, true>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, true>(w,scales,zero_points,@@ -73,7 +73,7 @@ KERNEL(QuantizedMatmulQmmTransposed)(}} else {if (aligned_n) {- qmm_transposed_impl<T, group_size, bits, true, 32, 32, 32, false>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, false>(w,scales,zero_points,@@ -91,7 +91,7 @@ KERNEL(QuantizedMatmulQmmTransposed)(simd_lid);} else {- qmm_transposed_impl<T, group_size, bits, false, 32, 32, 32, false>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, false>(w,scales,zero_points,diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metalindex 9978484a2..f1508fa29 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal@@ -2,10 +2,10 @@#include "../definitions.metal"#include "quant_matmul.metal"-template <typename T, int group_size, int bits>+template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, bfloat)-VARIANTS(group_size, 64, 128)-VARIANTS(bits, 4, 8)+VARIANTS(GROUP_SIZE, 64, 128)+VARIANTS(BITS, 4, 8)KERNEL(QuantizedMatmulQmmTransposed64x64)(const device uint32_t* w,const device T* scales,@@ -33,7 +33,7 @@ KERNEL(QuantizedMatmulQmmTransposed64x64)(const uint simd_lid = tid_x;if (use_mlx_quant) {- qmm_transposed_impl<T, group_size, bits, true, 64, 64, 64, true>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, true>(w,scales,zero_points,@@ -51,7 +51,7 @@ KERNEL(QuantizedMatmulQmmTransposed64x64)(simd_lid);} else {- qmm_transposed_impl<T, group_size, bits, true, 64, 64, 64, false>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, false>(w,scales,zero_points,diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metalindex 78cb1be40..1729a2d54 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal@@ -2,10 +2,10 @@#include "../definitions.metal"#include "quant_matmul.metal"-template <typename T, int group_size, int bits>+template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)-VARIANTS(group_size, 32, 64, 128)-VARIANTS(bits, 4, 8)+VARIANTS(GROUP_SIZE, 32, 64, 128)+VARIANTS(BITS, 4, 8)KERNEL(QuantizedMatmulQmv)(const device uint32_t* w,const device T* scales,@@ -29,7 +29,7 @@ KERNEL(QuantizedMatmulQmv)(const uint simd_lid = tid_x;if (use_mlx_quant) {- qmv_impl<T, group_size, bits, true>(+ qmv_impl<T, GROUP_SIZE, BITS, true>(w,scales,zero_points,@@ -43,7 +43,7 @@ KERNEL(QuantizedMatmulQmv)(simd_lid);} else {- qmv_impl<T, group_size, bits, false>(+ qmv_impl<T, GROUP_SIZE, BITS, false>(w,scales,zero_points,diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metalindex f8fa503c3..fd34f0b56 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal@@ -2,10 +2,10 @@#include "../definitions.metal"#include "quant_matmul.metal"-template <typename T, int group_size, int bits>+template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)-VARIANTS(group_size, 32, 64, 128)-VARIANTS(bits, 4, 8)+VARIANTS(GROUP_SIZE, 32, 64, 128)+VARIANTS(BITS, 4, 8)KERNEL(QuantizedMatmulQmvFast)(const device uint32_t* w,const device T* scales,@@ -29,7 +29,7 @@ KERNEL(QuantizedMatmulQmvFast)(const uint simd_lid = tid_x;if (use_mlx_quant) {- qmv_fast_impl<T, group_size, bits, true>(+ qmv_fast_impl<T, GROUP_SIZE, BITS, true>(w,scales,zero_points,@@ -43,7 +43,7 @@ KERNEL(QuantizedMatmulQmvFast)(simd_lid);} else {- qmv_fast_impl<T, group_size, bits, false>(+ qmv_fast_impl<T, GROUP_SIZE, BITS, false>(w,scales,zero_points,diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metalindex 7e39d799d..85d5fa688 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal@@ -2,10 +2,10 @@#include "../definitions.metal"#include "quant_matmul.metal"-template <typename T, int group_size, int bits>+template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)-VARIANTS(group_size, 32, 64, 128)-VARIANTS(bits, 4, 8)+VARIANTS(GROUP_SIZE, 32, 64, 128)+VARIANTS(BITS, 4, 8)KERNEL(QuantizedMatmulQvm)(const device uint32_t* w,const device T* scales,@@ -29,7 +29,7 @@ KERNEL(QuantizedMatmulQvm)(const uint simd_lid = tid_x;if (use_mlx_quant) {- qvm_impl<T, group_size, bits, true>(+ qvm_impl<T, GROUP_SIZE, BITS, true>(w,scales,zero_points,@@ -43,7 +43,7 @@ KERNEL(QuantizedMatmulQvm)(simd_lid);} else {- qvm_impl<T, group_size, bits, false>(+ qvm_impl<T, GROUP_SIZE, BITS, false>(w,scales,zero_points,