A high-performance inference engine for AI models
dslify-rope (#169)
Aleksandr Golokoza338a45764fb
diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rsindex aae359ab9..490c67228 100644--- a/crates/uzu/src/backends/metal/encodable_block/rope.rs+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs@@ -2,17 +2,19 @@use super::{EncodableBlock, Metal};use crate::{- backends::metal::{- KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, ProtocolObject,- Retained,- kernel::rope::{RopeError, RopeKernel, RopeKernelArguments},+ backends::{+ common::kernel::RopeKernel,+ metal::{+ KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError,+ ProtocolObject, Retained, kernel::dsl::RopeMetalKernel,+ },},encodable_block::EncodingParameters,forward_pass::state::{ArrayId, ForwardPassState, RopeType},};pub struct Rope {- kernel: RopeKernel,+ kernel: RopeMetalKernel,rope_type: RopeType,}@@ -21,10 +23,9 @@ impl Rope {context: &MTLContext,data_type: KernelDataType,rope_type: RopeType,- ) -> Result<Self, RopeError> {- let kernel = RopeKernel::new(context, data_type)?;+ ) -> Result<Self, MTLError> {Ok(Self {- kernel,+ kernel: RopeMetalKernel::new(context, data_type.into())?,rope_type,})}@@ -101,21 +102,18 @@ impl EncodableBlock<Metal> for Rope {let token_positions_offset = token_positions.offset();self.kernel.encode(+ qkv.buffer(),+ rope_cosines.buffer(),+ rope_sines.buffer(),+ (token_positions.buffer(), token_positions_offset),+ rotated_queries.buffer(),+ rotated_keys.buffer(),+ head_dim as u32,+ num_heads as u32,+ num_groups as u32,+ suffix_length as u32,+ rope_max_seq_len as u32,compute_encoder,- RopeKernelArguments {- qkv_buffer: &qkv.buffer(),- cosines_buffer: &rope_cosines.buffer(),- sines_buffer: &rope_sines.buffer(),- token_positions_buffer: &token_positions.buffer(),- token_positions_offset,- rotated_queries_buffer: &rotated_queries.buffer(),- rotated_keys_buffer: &rotated_keys.buffer(),- head_dim,- num_heads,- num_groups,- suffix_length,- max_sequence_length: rope_max_seq_len,- },- );+ )}}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 9cb03bdbd..52076d847 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -15,7 +15,6 @@ pub use mlp::{pub mod mlp_fused;pub mod moe;pub mod quant_matmul;-pub mod rope;pub mod short_conv;pub mod ssm;pub mod token_copy;@@ -32,7 +31,6 @@ pub use moe::{MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,MoeScatterWithMapArguments,};-pub use rope::{RopeError, RopeKernel, RopeKernelArguments};pub use short_conv::{ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError, ShortConvPrefillArguments};pub use ssm::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, SSDPrefillArguments, SSDPrefillMode, SSMKernelError,diff --git a/crates/uzu/src/backends/metal/kernel/rope/mod.rs b/crates/uzu/src/backends/metal/kernel/rope/mod.rsdeleted file mode 100644index d11437399..000000000--- a/crates/uzu/src/backends/metal/kernel/rope/mod.rs+++ /dev/null@@ -1,93 +0,0 @@-use thiserror::Error;--use crate::backends::metal::{- ComputeEncoderSetValue, KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,- MTLError, MTLSize, ProtocolObject, Retained,-};--pub struct RopeKernel {- pipeline_state: Retained<ProtocolObject<dyn MTLComputePipelineState>>,-}--#[derive(Debug, Error)]-pub enum RopeError {- #[error("Metal error: {0}")]- MetalError(#[from] MTLError),- #[error("Function not found: {0}")]- FunctionNotFound(String),-}--#[allow(dead_code)]-pub struct RopeKernelArguments<'a> {- pub qkv_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)- pub cosines_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)- pub sines_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)- pub token_positions_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)- pub token_positions_offset: usize, // byte offset into token_positions_buffer- pub rotated_queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4)- pub rotated_keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5)- pub head_dim: usize,- pub num_heads: usize,- pub num_groups: usize,- pub suffix_length: usize,- pub max_sequence_length: usize,-}--impl RopeKernel {- pub fn new(- context: &MTLContext,- data_type: KernelDataType,- ) -> Result<Self, RopeError> {- let function_name = format!("applyRope_{}", data_type.function_name_suffix());-- let pipeline_state = context.compute_pipeline_state(&function_name, None).map_err(|e| {- eprintln!("Failed to create pipeline state for {}: {:?}", function_name, e);- RopeError::MetalError(e)- })?;-- Ok(Self {- pipeline_state,- })- }-- pub fn encode(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: RopeKernelArguments,- ) {- compute_encoder.set_compute_pipeline_state(&self.pipeline_state);-- compute_encoder.set_buffer(Some(args.qkv_buffer), 0, 0);- compute_encoder.set_buffer(Some(args.cosines_buffer), 0, 1);- compute_encoder.set_buffer(Some(args.sines_buffer), 0, 2);- compute_encoder.set_buffer(Some(args.token_positions_buffer), args.token_positions_offset as usize, 3);- compute_encoder.set_buffer(Some(args.rotated_queries_buffer), 0, 4);- compute_encoder.set_buffer(Some(args.rotated_keys_buffer), 0, 5);-- // Set constants- let head_dim = args.head_dim as u32;- let num_heads = args.num_heads as u32;- let num_groups = args.num_groups as u32;- let suffix_length = args.suffix_length as u32;- let max_sequence_length = args.max_sequence_length as u32;-- compute_encoder.set_value(&head_dim, 6);- compute_encoder.set_value(&num_heads, 7);- compute_encoder.set_value(&num_groups, 8);- compute_encoder.set_value(&suffix_length, 9);- compute_encoder.set_value(&max_sequence_length, 10);-- let threads_per_threadgroup = MTLSize {- width: 1,- height: 1,- depth: 32,- };- let total_threads = MTLSize {- width: num_heads as usize,- height: suffix_length as usize,- depth: head_dim as usize,- };-- compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);- }-}diff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metalindex 20f7f2244..3eab900a6 100644--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal@@ -3,135 +3,104 @@template <typename T>inline T applyRopeTransform(- TensorView3D<const T> qkvTensorView,- uint tokenIdx,- uint headIdx,- uint dimIdx,- uint halfDim,- T cosVal,- T sinVal+ TensorView3D<const T> qkv_tensor_view,+ uint token_idx,+ uint head_idx,+ uint dim_idx,+ uint half_dim,+ T cos_val,+ T sin_val) {- T inputVal = qkvTensorView(tokenIdx, headIdx, dimIdx);- T pairedVal = (dimIdx < halfDim)- ? -qkvTensorView(tokenIdx, headIdx, dimIdx + halfDim)- : qkvTensorView(tokenIdx, headIdx, dimIdx - halfDim);- return inputVal * cosVal + pairedVal * sinVal;+ T inputVal = qkv_tensor_view(token_idx, head_idx, dim_idx);+ T pairedVal = (dim_idx < half_dim)+ ? -qkv_tensor_view(token_idx, head_idx, dim_idx + half_dim)+ : qkv_tensor_view(token_idx, head_idx, dim_idx - half_dim);+ return inputVal * cos_val + pairedVal * sin_val;}template <typename T>-void applyRope(- device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]- device const T* cosines, // [max_seq_len, head_dim]- device const T* sines, // [max_seq_len, head_dim]- device const int* tokenPositions, // [suffix_len] - actual token positions- device T* rotatedQueries, // [num_heads, suffix_len, head_dim]- device T* rotatedKeys, // [num_groups, suffix_len, head_dim]- constant int& headDim,- constant int& numHeads,- constant int& numGroups,- constant int& suffixLength,- constant int& maxSequenceLength,- const uint3 position // x: headIdx, y: tokenIdx, z: dimIdx+VARIANTS(T, float, half, bfloat)+KERNEL(Rope)(+ device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]+ device const T* cosines, // [max_seq_len, head_dim]+ device const T* sines, // [max_seq_len, head_dim]+ device const int* token_positions, // [suffix_len] - actual token positions+ device T* rotated_queries, // [num_heads, suffix_len, head_dim]+ device T* rotated_keys, // [num_heads, suffix_len, head_dim]+ constant uint& head_dim,+ constant uint& num_heads,+ constant uint& num_groups,+ constant uint& suffix_length,+ constant uint& max_sequence_length,+ const uint head_index AXIS(num_heads, 1),+ const uint token_index AXIS(suffix_length, 1),+ const uint dimension_index AXIS(head_dim, 32)) {- const uint headIndex = position.x;- const uint tokenIndex = position.y;- const uint dimensionIndex = position.z;-- if (headIndex >= numHeads || tokenIndex >= suffixLength ||- dimensionIndex >= headDim)+ if (head_index >= num_heads || token_index >= suffix_length ||+ dimension_index >= head_dim)+ return;+ if (head_dim & 1)return;- if (headDim & 1)- return; // head_dim must be even- if (numHeads % numGroups != 0)+ if (num_heads % num_groups != 0)return;- const uint groupIndex =- headIndex / (numHeads / numGroups); // which KV group this head belongs to- const uint totalHeads = numHeads + 2 * numGroups;+ const uint group_index =+ head_index /+ (num_heads / num_groups); // which KV group this head belongs to+ const uint total_heads = num_heads + 2 * num_groups;// Use actual token position from buffer- const uint rawPosition = tokenPositions[tokenIndex];+ const uint raw_position = token_positions[token_index];const uint absolutePosition =- rawPosition > maxSequenceLength ? 0 : rawPosition;+ raw_position > max_sequence_length ? 0 : raw_position;- const uint halfDimension = headDim / 2;+ const uint half_dimension = head_dim / 2;- TensorView3D<const T> qkvTensorView =- TensorView3D<const T>(qkv).shaped(suffixLength, totalHeads, headDim);- TensorView2D<const T> cosinesTensorView =- TensorView2D<const T>(cosines).shaped(maxSequenceLength, headDim);- TensorView2D<const T> sinesTensorView =- TensorView2D<const T>(sines).shaped(maxSequenceLength, headDim);- TensorView3D<T> rotatedQueriesTensorView =- TensorView3D<T>(rotatedQueries).shaped(numHeads, suffixLength, headDim);- TensorView3D<T> rotatedKeysTensorView =- TensorView3D<T>(rotatedKeys).shaped(numGroups, suffixLength, headDim);+ TensorView3D<const T> qkv_tensor_view =+ TensorView3D<const T>(qkv).shaped(suffix_length, total_heads, head_dim);+ TensorView2D<const T> cosines_tensor_view =+ TensorView2D<const T>(cosines).shaped(max_sequence_length, head_dim);+ TensorView2D<const T> sines_tensor_view =+ TensorView2D<const T>(sines).shaped(max_sequence_length, head_dim);+ TensorView3D<T> rotated_queries_tensor_view =+ TensorView3D<T>(rotated_queries)+ .shaped(num_heads, suffix_length, head_dim);+ TensorView3D<T> rotated_keys_tensor_view =+ TensorView3D<T>(rotated_keys).shaped(num_groups, suffix_length, head_dim);- const T cosVal = cosinesTensorView(absolutePosition, dimensionIndex);- const T sinVal = sinesTensorView(absolutePosition, dimensionIndex);+ const T cos_val = cosines_tensor_view(absolutePosition, dimension_index);+ const T sin_val = sines_tensor_view(absolutePosition, dimension_index);/* ---------- QUERIES ---------- */T queryResult = applyRopeTransform(- qkvTensorView,- tokenIndex,- headIndex,- dimensionIndex,- halfDimension,- cosVal,- sinVal+ qkv_tensor_view,+ token_index,+ head_index,+ dimension_index,+ half_dimension,+ cos_val,+ sin_val);- rotatedQueriesTensorView(headIndex, tokenIndex, dimensionIndex) = queryResult;+ rotated_queries_tensor_view(head_index, token_index, dimension_index) =+ queryResult;/* ---------- KEYS & VALUES (only first head of each group processes)* ---------- */- uint firstHeadInGroup = groupIndex * (numHeads / numGroups);- if (headIndex == firstHeadInGroup) {+ uint first_head_in_group = group_index * (num_heads / num_groups);+ if (head_index == first_head_in_group) {/* ---------- keys ---------- */- uint keyHeadIndex =- numHeads + groupIndex; // Keys start after all query heads+ uint key_head_index =+ num_heads + group_index; // Keys start after all query headsT keyResult = applyRopeTransform(- qkvTensorView,- tokenIndex,- keyHeadIndex,- dimensionIndex,- halfDimension,- cosVal,- sinVal+ qkv_tensor_view,+ token_index,+ key_head_index,+ dimension_index,+ half_dimension,+ cos_val,+ sin_val);- rotatedKeysTensorView(groupIndex, tokenIndex, dimensionIndex) = keyResult;+ rotated_keys_tensor_view(group_index, token_index, dimension_index) =+ keyResult;}-}--/* ---------- boiler-plate kernel wrappers ---------- */-#define outerArguments(T) \- (device const T* qkv [[buffer(0)]], \- device const T* cosines [[buffer(1)]], \- device const T* sines [[buffer(2)]], \- device const int* tokenPositions [[buffer(3)]], \- device T* rotatedQueries [[buffer(4)]], \- device T* rotatedKeys [[buffer(5)]], \- constant int& headDim [[buffer(6)]], \- constant int& numHeads [[buffer(7)]], \- constant int& numGroups [[buffer(8)]], \- constant int& suffixLength [[buffer(9)]], \- constant int& maxSequenceLength [[buffer(10)]], \- const uint3 position [[thread_position_in_grid]])--#define innerArguments \- (qkv, \- cosines, \- sines, \- tokenPositions, \- rotatedQueries, \- rotatedKeys, \- headDim, \- numHeads, \- numGroups, \- suffixLength, \- maxSequenceLength, \- position)--generateKernels(32, applyRope)--#undef outerArguments-#undef innerArguments+}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex 1c4a80f0f..8eb7dda33 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -21,7 +21,7 @@ pub use classifier_context::ClassifierContext;pub use context::{DeviceArchitecture, DeviceClass, DeviceGeneration, MTLContext};pub use encodable_block::Decoder;pub use error::MTLError;-pub use kernel::{KernelDataType, RopeKernel, dsl::MetalKernels};+pub use kernel::{KernelDataType, dsl::MetalKernels};pub use language_model_generator_context::LanguageModelGeneratorContext;pub use media_utils::{ImagePreprocessingParams, ImagePreprocessingRequirements, MetalImagePreprocessor};pub use metal::prelude::*;