Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Follow rust convention for NTTP name case (#227)

uuuvn555dae36c2c8
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
index e151c7d38..b599a39a2 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -5,9 +5,9 @@
#define SEQUENCE_BLOCK_SIZE 32
#define HEAD_BLOCK_SIZE 32
-template <typename T, uint head_dim>
+template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
-VARIANTS(head_dim, 64, 128, 256)
+VARIANTS(HEAD_DIM, 64, 128, 256)
KERNEL(AttentionSinglePass)(
const device T* queries,
const device T* keys,
@@ -42,8 +42,8 @@ KERNEL(AttentionSinglePass)(
const uint3 tpg = {num_heads, suffix_length, 1};
constexpr bool query_transposed = false;
- constexpr uint value_dim = head_dim;
- constexpr uint qk_elements_per_thread = head_dim / HEAD_BLOCK_SIZE;
+ constexpr uint value_dim = HEAD_DIM;
+ constexpr uint qk_elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;
constexpr uint value_elements_per_thread = value_dim / HEAD_BLOCK_SIZE;
uint inner_k_stride = SEQUENCE_BLOCK_SIZE * int(k_seq_stride);
uint inner_v_stride = SEQUENCE_BLOCK_SIZE * int(v_seq_stride);
@@ -59,7 +59,7 @@ KERNEL(AttentionSinglePass)(
const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx
: head_idx * tpg.y + q_seq_idx;
- queries += q_offset * head_dim + simd.lane_idx * qk_elements_per_thread;
+ queries += q_offset * HEAD_DIM + simd.lane_idx * qk_elements_per_thread;
keys += kv_head_idx * k_head_stride + simd.group_idx * k_seq_stride +
simd.lane_idx * qk_elements_per_thread;
values += kv_head_idx * v_head_stride + simd.group_idx * v_seq_stride +
@@ -164,4 +164,4 @@ KERNEL(AttentionSinglePass)(
out[i] = static_cast<T>(o[i]);
}
}
-}
\ No newline at end of file
+}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
index bb322b0fe..cb7ab22ce 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -7,9 +7,9 @@
#define SEQUENCE_BLOCK_SIZE_2 32
#define TOTAL_BLOCKS_COUNT 32
-template <typename T, uint head_dim>
+template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
-VARIANTS(head_dim, 64, 128, 256)
+VARIANTS(HEAD_DIM, 64, 128, 256)
KERNEL(AttentionTwoPass1)(
const device T* queries,
const device T* keys,
@@ -47,8 +47,8 @@ KERNEL(AttentionTwoPass1)(
const uint3 tpg = {num_heads, suffix_length, TOTAL_BLOCKS_COUNT};
constexpr bool query_transposed = false;
- constexpr uint value_dim = head_dim;
- constexpr uint qk_elements_per_thread = head_dim / HEAD_BLOCK_SIZE;
+ constexpr uint value_dim = HEAD_DIM;
+ constexpr uint qk_elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;
constexpr uint value_elements_per_thread = value_dim / HEAD_BLOCK_SIZE;
uint inner_k_stride = SEQUENCE_BLOCK_SIZE_1 * k_seq_stride;
uint inner_v_stride = SEQUENCE_BLOCK_SIZE_1 * v_seq_stride;
@@ -66,7 +66,7 @@ KERNEL(AttentionTwoPass1)(
: head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
const uint kv_head_idx = head_idx / gqa_factor;
- queries += q_offset * head_dim + simd.lane_idx * qk_elements_per_thread;
+ queries += q_offset * HEAD_DIM + simd.lane_idx * qk_elements_per_thread;
keys += kv_head_idx * k_head_stride +
(block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) * k_seq_stride +
simd.lane_idx * qk_elements_per_thread;
@@ -189,9 +189,9 @@ KERNEL(AttentionTwoPass1)(
}
}
-template <typename T, uint head_dim>
+template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
-VARIANTS(head_dim, 64, 128, 256)
+VARIANTS(HEAD_DIM, 64, 128, 256)
KERNEL(AttentionTwoPass2)(
const device float* partials,
const device float* sums,
@@ -206,18 +206,18 @@ KERNEL(AttentionTwoPass2)(
const uint tid THREADS(1024)
) {
const uint3 tpg = {num_heads, suffix_length, 1};
- constexpr uint elements_per_thread = head_dim / HEAD_BLOCK_SIZE;
+ constexpr uint elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;
typedef float U;
thread U o[elements_per_thread];
const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
- partials += o_offset * TOTAL_BLOCKS_COUNT * head_dim +
- simd.group_idx * head_dim + simd.lane_idx * elements_per_thread;
+ partials += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +
+ simd.group_idx * HEAD_DIM + simd.lane_idx * elements_per_thread;
sums += o_offset * TOTAL_BLOCKS_COUNT;
maxs += o_offset * TOTAL_BLOCKS_COUNT;
- out += o_offset * head_dim +
+ out += o_offset * HEAD_DIM +
simd.group_idx * elements_per_thread; // Our custom output layout
// First everybody reads the max and sum_exp
@@ -249,4 +249,4 @@ KERNEL(AttentionTwoPass2)(
out[i] = static_cast<T>(o[i]);
}
}
-}
\ No newline at end of file
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
index 47ca57522..b005e80a5 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
@@ -2,10 +2,10 @@
#include "../definitions.metal"
#include "quant_matmul.metal"
-template <typename T, int group_size, int bits>
+template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
-VARIANTS(group_size, 32, 64, 128)
-VARIANTS(bits, 4, 8)
+VARIANTS(GROUP_SIZE, 32, 64, 128)
+VARIANTS(BITS, 4, 8)
KERNEL(QuantizedMatmulQmm)(
const device uint32_t* w,
const device T* scales,
@@ -35,7 +35,7 @@ KERNEL(QuantizedMatmulQmm)(
if (use_mlx_quant) {
if (aligned_k) {
- qmm_impl<T, group_size, bits, true, 32, 32, 32, true>(
+ qmm_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, true>(
w,
scales,
zero_points,
@@ -53,7 +53,7 @@ KERNEL(QuantizedMatmulQmm)(
simd_lid
);
} else {
- qmm_impl<T, group_size, bits, false, 32, 32, 32, true>(
+ qmm_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, true>(
w,
scales,
zero_points,
@@ -73,7 +73,7 @@ KERNEL(QuantizedMatmulQmm)(
}
} else {
if (aligned_k) {
- qmm_impl<T, group_size, bits, true, 32, 32, 32, false>(
+ qmm_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, false>(
w,
scales,
zero_points,
@@ -91,7 +91,7 @@ KERNEL(QuantizedMatmulQmm)(
simd_lid
);
} else {
- qmm_impl<T, group_size, bits, false, 32, 32, 32, false>(
+ qmm_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, false>(
w,
scales,
zero_points,
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
index 111d3d0ed..f2b600578 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
@@ -2,10 +2,10 @@
#include "../definitions.metal"
#include "quant_matmul.metal"
-template <typename T, int group_size, int bits>
+template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
-VARIANTS(group_size, 32, 64, 128)
-VARIANTS(bits, 4, 8)
+VARIANTS(GROUP_SIZE, 32, 64, 128)
+VARIANTS(BITS, 4, 8)
KERNEL(QuantizedMatmulQmmTransposed)(
const device uint32_t* w,
const device T* scales,
@@ -35,7 +35,7 @@ KERNEL(QuantizedMatmulQmmTransposed)(
if (use_mlx_quant) {
if (aligned_n) {
- qmm_transposed_impl<T, group_size, bits, true, 32, 32, 32, true>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, true>(
w,
scales,
zero_points,
@@ -53,7 +53,7 @@ KERNEL(QuantizedMatmulQmmTransposed)(
simd_lid
);
} else {
- qmm_transposed_impl<T, group_size, bits, false, 32, 32, 32, true>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, true>(
w,
scales,
zero_points,
@@ -73,7 +73,7 @@ KERNEL(QuantizedMatmulQmmTransposed)(
}
} else {
if (aligned_n) {
- qmm_transposed_impl<T, group_size, bits, true, 32, 32, 32, false>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, false>(
w,
scales,
zero_points,
@@ -91,7 +91,7 @@ KERNEL(QuantizedMatmulQmmTransposed)(
simd_lid
);
} else {
- qmm_transposed_impl<T, group_size, bits, false, 32, 32, 32, false>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, false>(
w,
scales,
zero_points,
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
index 9978484a2..f1508fa29 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
@@ -2,10 +2,10 @@
#include "../definitions.metal"
#include "quant_matmul.metal"
-template <typename T, int group_size, int bits>
+template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, bfloat)
-VARIANTS(group_size, 64, 128)
-VARIANTS(bits, 4, 8)
+VARIANTS(GROUP_SIZE, 64, 128)
+VARIANTS(BITS, 4, 8)
KERNEL(QuantizedMatmulQmmTransposed64x64)(
const device uint32_t* w,
const device T* scales,
@@ -33,7 +33,7 @@ KERNEL(QuantizedMatmulQmmTransposed64x64)(
const uint simd_lid = tid_x;
if (use_mlx_quant) {
- qmm_transposed_impl<T, group_size, bits, true, 64, 64, 64, true>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, true>(
w,
scales,
zero_points,
@@ -51,7 +51,7 @@ KERNEL(QuantizedMatmulQmmTransposed64x64)(
simd_lid
);
} else {
- qmm_transposed_impl<T, group_size, bits, true, 64, 64, 64, false>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, false>(
w,
scales,
zero_points,
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
index 78cb1be40..1729a2d54 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
@@ -2,10 +2,10 @@
#include "../definitions.metal"
#include "quant_matmul.metal"
-template <typename T, int group_size, int bits>
+template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
-VARIANTS(group_size, 32, 64, 128)
-VARIANTS(bits, 4, 8)
+VARIANTS(GROUP_SIZE, 32, 64, 128)
+VARIANTS(BITS, 4, 8)
KERNEL(QuantizedMatmulQmv)(
const device uint32_t* w,
const device T* scales,
@@ -29,7 +29,7 @@ KERNEL(QuantizedMatmulQmv)(
const uint simd_lid = tid_x;
if (use_mlx_quant) {
- qmv_impl<T, group_size, bits, true>(
+ qmv_impl<T, GROUP_SIZE, BITS, true>(
w,
scales,
zero_points,
@@ -43,7 +43,7 @@ KERNEL(QuantizedMatmulQmv)(
simd_lid
);
} else {
- qmv_impl<T, group_size, bits, false>(
+ qmv_impl<T, GROUP_SIZE, BITS, false>(
w,
scales,
zero_points,
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
index f8fa503c3..fd34f0b56 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
@@ -2,10 +2,10 @@
#include "../definitions.metal"
#include "quant_matmul.metal"
-template <typename T, int group_size, int bits>
+template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
-VARIANTS(group_size, 32, 64, 128)
-VARIANTS(bits, 4, 8)
+VARIANTS(GROUP_SIZE, 32, 64, 128)
+VARIANTS(BITS, 4, 8)
KERNEL(QuantizedMatmulQmvFast)(
const device uint32_t* w,
const device T* scales,
@@ -29,7 +29,7 @@ KERNEL(QuantizedMatmulQmvFast)(
const uint simd_lid = tid_x;
if (use_mlx_quant) {
- qmv_fast_impl<T, group_size, bits, true>(
+ qmv_fast_impl<T, GROUP_SIZE, BITS, true>(
w,
scales,
zero_points,
@@ -43,7 +43,7 @@ KERNEL(QuantizedMatmulQmvFast)(
simd_lid
);
} else {
- qmv_fast_impl<T, group_size, bits, false>(
+ qmv_fast_impl<T, GROUP_SIZE, BITS, false>(
w,
scales,
zero_points,
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
index 7e39d799d..85d5fa688 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
@@ -2,10 +2,10 @@
#include "../definitions.metal"
#include "quant_matmul.metal"
-template <typename T, int group_size, int bits>
+template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
-VARIANTS(group_size, 32, 64, 128)
-VARIANTS(bits, 4, 8)
+VARIANTS(GROUP_SIZE, 32, 64, 128)
+VARIANTS(BITS, 4, 8)
KERNEL(QuantizedMatmulQvm)(
const device uint32_t* w,
const device T* scales,
@@ -29,7 +29,7 @@ KERNEL(QuantizedMatmulQvm)(
const uint simd_lid = tid_x;
if (use_mlx_quant) {
- qvm_impl<T, group_size, bits, true>(
+ qvm_impl<T, GROUP_SIZE, BITS, true>(
w,
scales,
zero_points,
@@ -43,7 +43,7 @@ KERNEL(QuantizedMatmulQvm)(
simd_lid
);
} else {
- qvm_impl<T, group_size, bits, false>(
+ qvm_impl<T, GROUP_SIZE, BITS, false>(
w,
scales,
zero_points,