Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dslify-rope (#169)

Aleksandr Golokoza338a45764fb
diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rs
index aae359ab9..490c67228 100644
--- a/crates/uzu/src/backends/metal/encodable_block/rope.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs
@@ -2,17 +2,19 @@
use super::{EncodableBlock, Metal};
use crate::{
- backends::metal::{
- KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, ProtocolObject,
- Retained,
- kernel::rope::{RopeError, RopeKernel, RopeKernelArguments},
+ backends::{
+ common::kernel::RopeKernel,
+ metal::{
+ KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError,
+ ProtocolObject, Retained, kernel::dsl::RopeMetalKernel,
+ },
},
encodable_block::EncodingParameters,
forward_pass::state::{ArrayId, ForwardPassState, RopeType},
};
pub struct Rope {
- kernel: RopeKernel,
+ kernel: RopeMetalKernel,
rope_type: RopeType,
}
@@ -21,10 +23,9 @@ impl Rope {
context: &MTLContext,
data_type: KernelDataType,
rope_type: RopeType,
- ) -> Result<Self, RopeError> {
- let kernel = RopeKernel::new(context, data_type)?;
+ ) -> Result<Self, MTLError> {
Ok(Self {
- kernel,
+ kernel: RopeMetalKernel::new(context, data_type.into())?,
rope_type,
})
}
@@ -101,21 +102,18 @@ impl EncodableBlock<Metal> for Rope {
let token_positions_offset = token_positions.offset();
self.kernel.encode(
+ qkv.buffer(),
+ rope_cosines.buffer(),
+ rope_sines.buffer(),
+ (token_positions.buffer(), token_positions_offset),
+ rotated_queries.buffer(),
+ rotated_keys.buffer(),
+ head_dim as u32,
+ num_heads as u32,
+ num_groups as u32,
+ suffix_length as u32,
+ rope_max_seq_len as u32,
compute_encoder,
- RopeKernelArguments {
- qkv_buffer: &qkv.buffer(),
- cosines_buffer: &rope_cosines.buffer(),
- sines_buffer: &rope_sines.buffer(),
- token_positions_buffer: &token_positions.buffer(),
- token_positions_offset,
- rotated_queries_buffer: &rotated_queries.buffer(),
- rotated_keys_buffer: &rotated_keys.buffer(),
- head_dim,
- num_heads,
- num_groups,
- suffix_length,
- max_sequence_length: rope_max_seq_len,
- },
- );
+ )
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 9cb03bdbd..52076d847 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -15,7 +15,6 @@ pub use mlp::{
pub mod mlp_fused;
pub mod moe;
pub mod quant_matmul;
-pub mod rope;
pub mod short_conv;
pub mod ssm;
pub mod token_copy;
@@ -32,7 +31,6 @@ pub use moe::{
MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,
MoeScatterWithMapArguments,
};
-pub use rope::{RopeError, RopeKernel, RopeKernelArguments};
pub use short_conv::{ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError, ShortConvPrefillArguments};
pub use ssm::{
Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, SSDPrefillArguments, SSDPrefillMode, SSMKernelError,
diff --git a/crates/uzu/src/backends/metal/kernel/rope/mod.rs b/crates/uzu/src/backends/metal/kernel/rope/mod.rs
deleted file mode 100644
index d11437399..000000000
--- a/crates/uzu/src/backends/metal/kernel/rope/mod.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-use thiserror::Error;
-
-use crate::backends::metal::{
- ComputeEncoderSetValue, KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,
- MTLError, MTLSize, ProtocolObject, Retained,
-};
-
-pub struct RopeKernel {
- pipeline_state: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
-}
-
-#[derive(Debug, Error)]
-pub enum RopeError {
- #[error("Metal error: {0}")]
- MetalError(#[from] MTLError),
- #[error("Function not found: {0}")]
- FunctionNotFound(String),
-}
-
-#[allow(dead_code)]
-pub struct RopeKernelArguments<'a> {
- pub qkv_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)
- pub cosines_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)
- pub sines_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)
- pub token_positions_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)
- pub token_positions_offset: usize, // byte offset into token_positions_buffer
- pub rotated_queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4)
- pub rotated_keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5)
- pub head_dim: usize,
- pub num_heads: usize,
- pub num_groups: usize,
- pub suffix_length: usize,
- pub max_sequence_length: usize,
-}
-
-impl RopeKernel {
- pub fn new(
- context: &MTLContext,
- data_type: KernelDataType,
- ) -> Result<Self, RopeError> {
- let function_name = format!("applyRope_{}", data_type.function_name_suffix());
-
- let pipeline_state = context.compute_pipeline_state(&function_name, None).map_err(|e| {
- eprintln!("Failed to create pipeline state for {}: {:?}", function_name, e);
- RopeError::MetalError(e)
- })?;
-
- Ok(Self {
- pipeline_state,
- })
- }
-
- pub fn encode(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: RopeKernelArguments,
- ) {
- compute_encoder.set_compute_pipeline_state(&self.pipeline_state);
-
- compute_encoder.set_buffer(Some(args.qkv_buffer), 0, 0);
- compute_encoder.set_buffer(Some(args.cosines_buffer), 0, 1);
- compute_encoder.set_buffer(Some(args.sines_buffer), 0, 2);
- compute_encoder.set_buffer(Some(args.token_positions_buffer), args.token_positions_offset as usize, 3);
- compute_encoder.set_buffer(Some(args.rotated_queries_buffer), 0, 4);
- compute_encoder.set_buffer(Some(args.rotated_keys_buffer), 0, 5);
-
- // Set constants
- let head_dim = args.head_dim as u32;
- let num_heads = args.num_heads as u32;
- let num_groups = args.num_groups as u32;
- let suffix_length = args.suffix_length as u32;
- let max_sequence_length = args.max_sequence_length as u32;
-
- compute_encoder.set_value(&head_dim, 6);
- compute_encoder.set_value(&num_heads, 7);
- compute_encoder.set_value(&num_groups, 8);
- compute_encoder.set_value(&suffix_length, 9);
- compute_encoder.set_value(&max_sequence_length, 10);
-
- let threads_per_threadgroup = MTLSize {
- width: 1,
- height: 1,
- depth: 32,
- };
- let total_threads = MTLSize {
- width: num_heads as usize,
- height: suffix_length as usize,
- depth: head_dim as usize,
- };
-
- compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
index 20f7f2244..3eab900a6 100644
--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal
+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
@@ -3,135 +3,104 @@
template <typename T>
inline T applyRopeTransform(
- TensorView3D<const T> qkvTensorView,
- uint tokenIdx,
- uint headIdx,
- uint dimIdx,
- uint halfDim,
- T cosVal,
- T sinVal
+ TensorView3D<const T> qkv_tensor_view,
+ uint token_idx,
+ uint head_idx,
+ uint dim_idx,
+ uint half_dim,
+ T cos_val,
+ T sin_val
) {
- T inputVal = qkvTensorView(tokenIdx, headIdx, dimIdx);
- T pairedVal = (dimIdx < halfDim)
- ? -qkvTensorView(tokenIdx, headIdx, dimIdx + halfDim)
- : qkvTensorView(tokenIdx, headIdx, dimIdx - halfDim);
- return inputVal * cosVal + pairedVal * sinVal;
+ T inputVal = qkv_tensor_view(token_idx, head_idx, dim_idx);
+ T pairedVal = (dim_idx < half_dim)
+ ? -qkv_tensor_view(token_idx, head_idx, dim_idx + half_dim)
+ : qkv_tensor_view(token_idx, head_idx, dim_idx - half_dim);
+ return inputVal * cos_val + pairedVal * sin_val;
}
template <typename T>
-void applyRope(
- device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]
- device const T* cosines, // [max_seq_len, head_dim]
- device const T* sines, // [max_seq_len, head_dim]
- device const int* tokenPositions, // [suffix_len] - actual token positions
- device T* rotatedQueries, // [num_heads, suffix_len, head_dim]
- device T* rotatedKeys, // [num_groups, suffix_len, head_dim]
- constant int& headDim,
- constant int& numHeads,
- constant int& numGroups,
- constant int& suffixLength,
- constant int& maxSequenceLength,
- const uint3 position // x: headIdx, y: tokenIdx, z: dimIdx
+VARIANTS(T, float, half, bfloat)
+KERNEL(Rope)(
+ device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]
+ device const T* cosines, // [max_seq_len, head_dim]
+ device const T* sines, // [max_seq_len, head_dim]
+ device const int* token_positions, // [suffix_len] - actual token positions
+ device T* rotated_queries, // [num_heads, suffix_len, head_dim]
+ device T* rotated_keys, // [num_heads, suffix_len, head_dim]
+ constant uint& head_dim,
+ constant uint& num_heads,
+ constant uint& num_groups,
+ constant uint& suffix_length,
+ constant uint& max_sequence_length,
+ const uint head_index AXIS(num_heads, 1),
+ const uint token_index AXIS(suffix_length, 1),
+ const uint dimension_index AXIS(head_dim, 32)
) {
- const uint headIndex = position.x;
- const uint tokenIndex = position.y;
- const uint dimensionIndex = position.z;
-
- if (headIndex >= numHeads || tokenIndex >= suffixLength ||
- dimensionIndex >= headDim)
+ if (head_index >= num_heads || token_index >= suffix_length ||
+ dimension_index >= head_dim)
+ return;
+ if (head_dim & 1)
return;
- if (headDim & 1)
- return; // head_dim must be even
- if (numHeads % numGroups != 0)
+ if (num_heads % num_groups != 0)
return;
- const uint groupIndex =
- headIndex / (numHeads / numGroups); // which KV group this head belongs to
- const uint totalHeads = numHeads + 2 * numGroups;
+ const uint group_index =
+ head_index /
+ (num_heads / num_groups); // which KV group this head belongs to
+ const uint total_heads = num_heads + 2 * num_groups;
// Use actual token position from buffer
- const uint rawPosition = tokenPositions[tokenIndex];
+ const uint raw_position = token_positions[token_index];
const uint absolutePosition =
- rawPosition > maxSequenceLength ? 0 : rawPosition;
+ raw_position > max_sequence_length ? 0 : raw_position;
- const uint halfDimension = headDim / 2;
+ const uint half_dimension = head_dim / 2;
- TensorView3D<const T> qkvTensorView =
- TensorView3D<const T>(qkv).shaped(suffixLength, totalHeads, headDim);
- TensorView2D<const T> cosinesTensorView =
- TensorView2D<const T>(cosines).shaped(maxSequenceLength, headDim);
- TensorView2D<const T> sinesTensorView =
- TensorView2D<const T>(sines).shaped(maxSequenceLength, headDim);
- TensorView3D<T> rotatedQueriesTensorView =
- TensorView3D<T>(rotatedQueries).shaped(numHeads, suffixLength, headDim);
- TensorView3D<T> rotatedKeysTensorView =
- TensorView3D<T>(rotatedKeys).shaped(numGroups, suffixLength, headDim);
+ TensorView3D<const T> qkv_tensor_view =
+ TensorView3D<const T>(qkv).shaped(suffix_length, total_heads, head_dim);
+ TensorView2D<const T> cosines_tensor_view =
+ TensorView2D<const T>(cosines).shaped(max_sequence_length, head_dim);
+ TensorView2D<const T> sines_tensor_view =
+ TensorView2D<const T>(sines).shaped(max_sequence_length, head_dim);
+ TensorView3D<T> rotated_queries_tensor_view =
+ TensorView3D<T>(rotated_queries)
+ .shaped(num_heads, suffix_length, head_dim);
+ TensorView3D<T> rotated_keys_tensor_view =
+ TensorView3D<T>(rotated_keys).shaped(num_groups, suffix_length, head_dim);
- const T cosVal = cosinesTensorView(absolutePosition, dimensionIndex);
- const T sinVal = sinesTensorView(absolutePosition, dimensionIndex);
+ const T cos_val = cosines_tensor_view(absolutePosition, dimension_index);
+ const T sin_val = sines_tensor_view(absolutePosition, dimension_index);
/* ---------- QUERIES ---------- */
T queryResult = applyRopeTransform(
- qkvTensorView,
- tokenIndex,
- headIndex,
- dimensionIndex,
- halfDimension,
- cosVal,
- sinVal
+ qkv_tensor_view,
+ token_index,
+ head_index,
+ dimension_index,
+ half_dimension,
+ cos_val,
+ sin_val
);
- rotatedQueriesTensorView(headIndex, tokenIndex, dimensionIndex) = queryResult;
+ rotated_queries_tensor_view(head_index, token_index, dimension_index) =
+ queryResult;
/* ---------- KEYS & VALUES (only first head of each group processes)
* ---------- */
- uint firstHeadInGroup = groupIndex * (numHeads / numGroups);
- if (headIndex == firstHeadInGroup) {
+ uint first_head_in_group = group_index * (num_heads / num_groups);
+ if (head_index == first_head_in_group) {
/* ---------- keys ---------- */
- uint keyHeadIndex =
- numHeads + groupIndex; // Keys start after all query heads
+ uint key_head_index =
+ num_heads + group_index; // Keys start after all query heads
T keyResult = applyRopeTransform(
- qkvTensorView,
- tokenIndex,
- keyHeadIndex,
- dimensionIndex,
- halfDimension,
- cosVal,
- sinVal
+ qkv_tensor_view,
+ token_index,
+ key_head_index,
+ dimension_index,
+ half_dimension,
+ cos_val,
+ sin_val
);
- rotatedKeysTensorView(groupIndex, tokenIndex, dimensionIndex) = keyResult;
+ rotated_keys_tensor_view(group_index, token_index, dimension_index) =
+ keyResult;
}
-}
-
-/* ---------- boiler-plate kernel wrappers ---------- */
-#define outerArguments(T) \
- (device const T* qkv [[buffer(0)]], \
- device const T* cosines [[buffer(1)]], \
- device const T* sines [[buffer(2)]], \
- device const int* tokenPositions [[buffer(3)]], \
- device T* rotatedQueries [[buffer(4)]], \
- device T* rotatedKeys [[buffer(5)]], \
- constant int& headDim [[buffer(6)]], \
- constant int& numHeads [[buffer(7)]], \
- constant int& numGroups [[buffer(8)]], \
- constant int& suffixLength [[buffer(9)]], \
- constant int& maxSequenceLength [[buffer(10)]], \
- const uint3 position [[thread_position_in_grid]])
-
-#define innerArguments \
- (qkv, \
- cosines, \
- sines, \
- tokenPositions, \
- rotatedQueries, \
- rotatedKeys, \
- headDim, \
- numHeads, \
- numGroups, \
- suffixLength, \
- maxSequenceLength, \
- position)
-
-generateKernels(32, applyRope)
-
-#undef outerArguments
-#undef innerArguments
+}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index 1c4a80f0f..8eb7dda33 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -21,7 +21,7 @@ pub use classifier_context::ClassifierContext;
pub use context::{DeviceArchitecture, DeviceClass, DeviceGeneration, MTLContext};
pub use encodable_block::Decoder;
pub use error::MTLError;
-pub use kernel::{KernelDataType, RopeKernel, dsl::MetalKernels};
+pub use kernel::{KernelDataType, dsl::MetalKernels};
pub use language_model_generator_context::LanguageModelGeneratorContext;
pub use media_utils::{ImagePreprocessingParams, ImagePreprocessingRequirements, MetalImagePreprocessor};
pub use metal::prelude::*;