A high-performance inference engine for AI models
Sized constants (#358)
uuuvnc727f6d7802c
diff --git a/crates/backend-uzu/build/common/kernel.rs b/crates/backend-uzu/build/common/kernel.rsindex aa645e2c1..7e10ffec4 100644--- a/crates/backend-uzu/build/common/kernel.rs+++ b/crates/backend-uzu/build/common/kernel.rs@@ -8,7 +8,6 @@ pub enum KernelBufferAccess {pub enum KernelArgumentType {Buffer(KernelBufferAccess),Constant(Box<str>),- Scalar(Box<str>),}#[derive(PartialEq, Debug, Clone)]diff --git a/crates/backend-uzu/build/common/traitgen.rs b/crates/backend-uzu/build/common/traitgen.rsindex 1c110d652..58ba3b0a3 100644--- a/crates/backend-uzu/build/common/traitgen.rs+++ b/crates/backend-uzu/build/common/traitgen.rs@@ -51,10 +51,6 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {)},KernelArgumentType::Constant(ty) => {- let ty: Type = syn::parse_str(ty.as_ref()).unwrap();- (None, quote! { &[#ty] })- },- KernelArgumentType::Scalar(ty) => {let ty: Type = syn::parse_str(ty.as_ref()).unwrap();(None, quote! { #ty })},diff --git a/crates/backend-uzu/build/cpu/compiler.rs b/crates/backend-uzu/build/cpu/compiler.rsindex 9181ba8b8..234ba9164 100644--- a/crates/backend-uzu/build/cpu/compiler.rs+++ b/crates/backend-uzu/build/cpu/compiler.rs@@ -21,7 +21,7 @@ use crate::common::{#[derive(PartialEq, Debug)]pub enum FunctionArgumentType {Buffer(KernelBufferAccess),- Constant(Type),+ Constant(Type, Option<Expr>),Scalar(Type),Specialization(Type),}@@ -40,12 +40,20 @@ impl FunctionArgument {conditional: self.conditional.is_some(),ty: match &self.ty {FunctionArgumentType::Buffer(access) => KernelArgumentType::Buffer(access.clone()),- FunctionArgumentType::Constant(ty) => KernelArgumentType::Constant(+ FunctionArgumentType::Constant(ty, None) => KernelArgumentType::Constant(+ format!("&[{}]", ty.to_token_stream().to_string().replace(" :: ", "::")).into_boxed_str(),+ ),+ FunctionArgumentType::Constant(ty, Some(sz)) => KernelArgumentType::Constant(+ format!(+ "&[{}; {}]",+ ty.to_token_stream().to_string().replace(" :: ", "::"),+ sz.to_token_stream().to_string(),+ )+ .into_boxed_str(),+ ),+ FunctionArgumentType::Scalar(ty) => KernelArgumentType::Constant(ty.to_token_stream().to_string().replace(" :: ", "::").into_boxed_str(),),- FunctionArgumentType::Scalar(ty) => {- KernelArgumentType::Scalar(ty.to_token_stream().to_string().replace(" :: ", "::").into_boxed_str())- },FunctionArgumentType::Specialization(_) => {return None;},@@ -352,10 +360,6 @@ impl CpuCompiler {)},KernelArgumentType::Constant(ty) => {- let ty: Type = syn::parse_str(ty.as_ref()).context("cannot parse type")?;- (None, quote! { &[#ty] })- },- KernelArgumentType::Scalar(ty) => {let ty: Type = syn::parse_str(ty.as_ref()).context("cannot parse type")?;(None, quote! { #ty })},@@ -405,9 +409,12 @@ impl CpuCompiler {})}},- FunctionArgumentType::Constant(_) => {+ FunctionArgumentType::Constant(_, None) => {Some(quote! { let #argument_ident = #argument_ident.to_vec().into_boxed_slice(); })},+ FunctionArgumentType::Constant(_, Some(_)) => {+ Some(quote! { let #argument_ident = Box::new(*#argument_ident); })+ },FunctionArgumentType::Scalar(_) => None,FunctionArgumentType::Specialization(_) => {Some(quote! { let #argument_ident = self.#argument_ident; })@@ -434,7 +441,7 @@ impl CpuCompiler {quote! { #argument_ident.as_ptr() as _ }}},- FunctionArgumentType::Constant(_) => quote! { #argument_ident.as_ref() },+ FunctionArgumentType::Constant(_, _) => quote! { &*#argument_ident },FunctionArgumentType::Scalar(_) | FunctionArgumentType::Specialization(_) => {quote! { #argument_ident }},@@ -581,7 +588,8 @@ impl CpuCompiler {KernelBufferAccess::Read}),Type::Reference(ty) => match *ty.elem {- Type::Slice(ty) => FunctionArgumentType::Constant(*ty.elem),+ Type::Slice(ty) => FunctionArgumentType::Constant(*ty.elem, None),+ Type::Array(ty) => FunctionArgumentType::Constant(*ty.elem, Some(ty.len)),ty => bail!("unsupported reference type: {} ({:?})", ty.to_token_stream().to_string(), ty),},Type::Path(ty) => FunctionArgumentType::Scalar(Type::Path(ty)),diff --git a/crates/backend-uzu/build/metal/ast.rs b/crates/backend-uzu/build/metal/ast.rsindex d91893a63..7b72ccebd 100644--- a/crates/backend-uzu/build/metal/ast.rs+++ b/crates/backend-uzu/build/metal/ast.rs@@ -99,7 +99,7 @@ pub enum MetalBufferAccess {#[derive(Debug, Serialize, Deserialize)]pub enum MetalConstantType {Scalar,- Array,+ Array(Option<Box<str>>),}#[derive(Debug, Serialize, Deserialize)]@@ -267,9 +267,18 @@ impl MetalArgument {} else if let ["const", "constant", c_type_scalar, "*"] =self.c_type.split_whitespace().collect::<Vec<_>>().as_slice(){+ let size = if self.source.contains('[') && self.source.contains(']') {+ let lbracket = self.source.rfind('[').context("sized constant missing size bracket")? + 1;+ let rbracket = self.source.rfind(']').context("sized constant missing size bracket")?;+ let size_expr = &self.source[lbracket..rbracket];+ Some(size_expr.into())+ } else {+ None+ };+Ok(MetalArgumentType::Constant((Self::scalar_type_to_rust(c_type_scalar)?.into(),- MetalConstantType::Array,+ MetalConstantType::Array(size),)))} else if self.c_type.contains("threadgroup") && self.c_type.contains('*') {let lbracket = self.source.rfind('[').context("threadgroup missing size bracket")? + 1;@@ -397,13 +406,20 @@ impl MetalKernelInfo {Ok(MetalArgumentType::Constant((ty, MetalConstantType::Scalar))) => Some(KernelArgument {name: a.name.clone(),conditional: a.argument_condition().unwrap().is_some(),- ty: KernelArgumentType::Scalar(ty),+ ty: KernelArgumentType::Constant(ty),}),- Ok(MetalArgumentType::Constant((ty, MetalConstantType::Array))) => Some(KernelArgument {+ Ok(MetalArgumentType::Constant((ty, MetalConstantType::Array(None)))) => Some(KernelArgument {name: a.name.clone(),conditional: a.argument_condition().unwrap().is_some(),- ty: KernelArgumentType::Constant(ty),+ ty: KernelArgumentType::Constant(format!("&[{ty}]").into_boxed_str()),}),+ Ok(MetalArgumentType::Constant((ty, MetalConstantType::Array(Some(size))))) => {+ Some(KernelArgument {+ name: a.name.clone(),+ conditional: a.argument_condition().unwrap().is_some(),+ ty: KernelArgumentType::Constant(format!("&[{ty}; {size}]").into_boxed_str()),+ })+ },_ => None,}).collect(),diff --git a/crates/backend-uzu/build/metal/bindgen.rs b/crates/backend-uzu/build/metal/bindgen.rsindex b63e9ac1b..7eb32932d 100644--- a/crates/backend-uzu/build/metal/bindgen.rs+++ b/crates/backend-uzu/build/metal/bindgen.rs@@ -220,13 +220,23 @@ pub fn bindgen(quote! { compute_encoder.set_value(&#arg_name, #arg_count); },None,),- MetalConstantType::Array => (+ MetalConstantType::Array(None) => (quote! { &[#arg_dtype] },None,None,quote! { compute_encoder.set_slice(#arg_name, #arg_count); },None,),+ MetalConstantType::Array(Some(size)) => {+ let arg_size: Expr = syn::parse_str(&size).unwrap();+ (+ quote! { &[#arg_dtype; #arg_size] },+ None,+ None,+ quote! { compute_encoder.set_slice(#arg_name, #arg_count); },+ None,+ )+ },}},_ => unreachable!(),diff --git a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rsindex b14dfdb03..0f8af6ec8 100644--- a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs+++ b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs@@ -59,10 +59,10 @@ impl<B: Backend> SSDPrefillKernels<B> {args: SSDPrefillArguments<B>,mode: SSDPrefillMode,) {- let x_strides: Vec<u32> = args.x_strides.iter().map(|x| *x as u32).collect();- let dt_strides: Vec<u32> = args.dt_strides.iter().map(|x| *x as u32).collect();- let cb_strides: Vec<u32> = args.cb_strides.iter().map(|x| *x as u32).collect();- let state_strides: Vec<u32> = args.state_strides.iter().map(|x| *x as u32).collect();+ let x_strides: [u32; 3] = args.x_strides.map(|x| x as u32);+ let dt_strides: [u32; 2] = args.dt_strides.map(|x| x as u32);+ let cb_strides: [u32; 3] = args.cb_strides.map(|x| x as u32);+ let state_strides: [u32; 3] = args.state_strides.map(|x| x as u32);if mode == SSDPrefillMode::SinglePass {if args.state_size == 64 {@@ -78,10 +78,10 @@ impl<B: Backend> SSDPrefillKernels<B> {args.suffix_len as u32,args.group_size,args.state_size,- x_strides.as_slice(),- dt_strides.as_slice(),- cb_strides.as_slice(),- state_strides.as_slice(),+ &x_strides,+ &dt_strides,+ &cb_strides,+ &state_strides,args.channels as u32,args.head_dim as u32,encoder,@@ -99,10 +99,10 @@ impl<B: Backend> SSDPrefillKernels<B> {args.suffix_len as u32,args.group_size,args.state_size,- x_strides.as_slice(),- dt_strides.as_slice(),- cb_strides.as_slice(),- state_strides.as_slice(),+ &x_strides,+ &dt_strides,+ &cb_strides,+ &state_strides,args.channels as u32,args.head_dim as u32,encoder,@@ -121,10 +121,10 @@ impl<B: Backend> SSDPrefillKernels<B> {args.suffix_len as u32,args.group_size,args.state_size,- x_strides.as_slice(),- dt_strides.as_slice(),- cb_strides.as_slice(),- state_strides.as_slice(),+ &x_strides,+ &dt_strides,+ &cb_strides,+ &state_strides,args.channels as u32,args.head_dim as u32,encoder,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs b/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rsindex 3a673f923..674720e35 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs@@ -18,10 +18,10 @@ pub fn ssd_prefill64<T: ArrayElement + Float>(#[allow(unused)] suffix_len: u32,#[allow(unused)] group_size: u32,#[allow(unused)] state_size: u32,- #[allow(unused)] x_strides: &[u32],- #[allow(unused)] dt_strides: &[u32],- #[allow(unused)] cb_strides: &[u32],- #[allow(unused)] state_strides: &[u32],+ #[allow(unused)] x_strides: &[u32; 3],+ #[allow(unused)] dt_strides: &[u32; 2],+ #[allow(unused)] cb_strides: &[u32; 3],+ #[allow(unused)] state_strides: &[u32; 3],#[allow(unused)] num_heads: u32,#[allow(unused)] head_dim: u32,) {@@ -42,10 +42,10 @@ pub fn ssd_prefill<T: ArrayElement + Float>(#[allow(unused)] suffix_len: u32,#[allow(unused)] group_size: u32,#[allow(unused)] state_size: u32,- #[allow(unused)] x_strides: &[u32],- #[allow(unused)] dt_strides: &[u32],- #[allow(unused)] cb_strides: &[u32],- #[allow(unused)] state_strides: &[u32],+ #[allow(unused)] x_strides: &[u32; 3],+ #[allow(unused)] dt_strides: &[u32; 2],+ #[allow(unused)] cb_strides: &[u32; 3],+ #[allow(unused)] state_strides: &[u32; 3],#[allow(unused)] num_heads: u32,#[allow(unused)] head_dim: u32,) {@@ -66,10 +66,10 @@ pub fn ssd_prefill_sequential<T: ArrayElement + Float>(suffix_len: u32,group_size: u32,state_size: u32,- x_strides: &[u32],- dt_strides: &[u32],- cb_strides: &[u32],- state_strides: &[u32],+ x_strides: &[u32; 3],+ dt_strides: &[u32; 2],+ cb_strides: &[u32; 3],+ state_strides: &[u32; 3],channels: u32,head_dim: u32,) {diff --git a/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_update.rsindex 2de6ecea9..e4db4967f 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_update.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_update.rs@@ -18,10 +18,10 @@ pub fn ssd_update<T: ArrayElement + Float>(next_state: *mut T,group_size: u32,state_size: u32,- x_strides: &[u32],- dt_strides: &[u32],- cb_strides: &[u32],- state_strides: &[u32],+ x_strides: &[u32; 3],+ dt_strides: &[u32; 2],+ cb_strides: &[u32; 3],+ state_strides: &[u32; 4],b_size: u32,h_size: u32,dh_size: u32,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.hindex 2e0685009..37b5673bd 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h@@ -149,7 +149,7 @@ struct ThreadgroupGemm {const device T* a,const device T* b,device U* d,- const constant GemmParams* params,+ const constant GemmParams& params,const constant float& ab_scale,const bool is_accumulate,threadgroup T* a_shared,@@ -161,13 +161,13 @@ struct ThreadgroupGemm {) {(void)lid;- const int swizzle_stride = pow2(params->swizzle_log);+ const int swizzle_stride = pow2(params.swizzle_log);const int swizzled_row_id =tid.y * swizzle_stride + (tid.x % swizzle_stride);const int swizzled_col_id = tid.x / swizzle_stride;- if (params->threadgroups_per_row <= swizzled_col_id ||- params->threadgroups_per_column <= swizzled_row_id) {+ if (params.threadgroups_per_row <= swizzled_col_id ||+ params.threadgroups_per_column <= swizzled_row_id) {return;}@@ -179,21 +179,21 @@ struct ThreadgroupGemm {const size_t output_col_long = size_t(output_col);a += transpose_a ? output_row_long- : output_row_long * params->leading_dimension_a;- b += transpose_b ? output_col_long * params->leading_dimension_b+ : output_row_long * params.leading_dimension_a;+ b += transpose_b ? output_col_long * params.leading_dimension_b: output_col_long;- d += output_row_long * params->leading_dimension_d + output_col_long;+ d += output_row_long * params.leading_dimension_d + output_col_long;thread LoaderAType loader_a(a,- params->leading_dimension_a,+ params.leading_dimension_a,a_shared,simd_group_id,simd_lane_id);thread LoaderBType loader_b(b,- params->leading_dimension_b,+ params.leading_dimension_b,b_shared,simd_group_id,simd_lane_id@@ -206,7 +206,7 @@ struct ThreadgroupGemm {is_accumulate ? 1.0f : 0.0f);- int gemm_k_iterations = params->aligned_inner_iterations;+ int gemm_k_iterations = params.aligned_inner_iterations;///////////////////////////////////////////////////////////////////////////// MNK aligned loop@@ -228,7 +228,7 @@ struct ThreadgroupGemm {if (!K_aligned) {int leftover_block_depth =- params->K - params->aligned_inner_iterations * BLOCK_DEPTH;+ params.K - params.aligned_inner_iterations * BLOCK_DEPTH;short2 tile_dimensions_a =transpose_a ? short2(BLOCK_ROWS, leftover_block_depth): short2(leftover_block_depth, BLOCK_ROWS);@@ -245,19 +245,19 @@ struct ThreadgroupGemm {}threadgroup_tile- .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);- threadgroup_tile.store_result(d, params->leading_dimension_d);+ .apply_epilogue(d, params.leading_dimension_d, 1, epilogue);+ threadgroup_tile.store_result(d, params.leading_dimension_d);return;}///////////////////////////////////////////////////////////////////////////// MN unaligned loopelse {short threadgroup_block_rows =- min(BLOCK_ROWS, ((int)params->M) - output_row);+ min(BLOCK_ROWS, ((int)params.M) - output_row);short threadgroup_block_cols =- min(BLOCK_COLS, ((int)params->N) - output_col);+ min(BLOCK_COLS, ((int)params.N) - output_col);short leftover_block_depth =- params->K - params->aligned_inner_iterations * BLOCK_DEPTH;+ params.K - params.aligned_inner_iterations * BLOCK_DEPTH;if (threadgroup_block_rows == BLOCK_ROWS &&threadgroup_block_cols == BLOCK_COLS) {@@ -274,8 +274,8 @@ struct ThreadgroupGemm {);threadgroup_tile- .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);- threadgroup_tile.store_result(d, params->leading_dimension_d);+ .apply_epilogue(d, params.leading_dimension_d, 1, epilogue);+ threadgroup_tile.store_result(d, params.leading_dimension_d);return;} else if (threadgroup_block_cols == BLOCK_COLS) {@@ -293,14 +293,14 @@ struct ThreadgroupGemm {threadgroup_tile.apply_epilogue_safe(d,- params->leading_dimension_d,+ params.leading_dimension_d,1,short2(threadgroup_block_cols, threadgroup_block_rows),epilogue);threadgroup_tile.store_result_safe(d,- params->leading_dimension_d,+ params.leading_dimension_d,short2(threadgroup_block_cols, threadgroup_block_rows));return;@@ -320,14 +320,14 @@ struct ThreadgroupGemm {threadgroup_tile.apply_epilogue_safe(d,- params->leading_dimension_d,+ params.leading_dimension_d,1,short2(threadgroup_block_cols, threadgroup_block_rows),epilogue);threadgroup_tile.store_result_safe(d,- params->leading_dimension_d,+ params.leading_dimension_d,short2(threadgroup_block_cols, threadgroup_block_rows));return;@@ -347,14 +347,14 @@ struct ThreadgroupGemm {threadgroup_tile.apply_epilogue_safe(d,- params->leading_dimension_d,+ params.leading_dimension_d,1,short2(threadgroup_block_cols, threadgroup_block_rows),epilogue);threadgroup_tile.store_result_safe(d,- params->leading_dimension_d,+ params.leading_dimension_d,short2(threadgroup_block_cols, threadgroup_block_rows));return;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.hindex a275b3d00..e51142ad7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h@@ -103,7 +103,7 @@ struct ThreadgroupGemmMpp {const device T* left_matrix,const device T* right_matrix,device T* output_matrix,- const constant GemmParams* params,+ const constant GemmParams& params,const bool align_m,const bool align_n,const bool is_accumulate,@@ -115,7 +115,7 @@ struct ThreadgroupGemmMpp {uint2 threadgroup_position) {uint tile_id_x, tile_id_y;- if (params->use_morton) {+ if (params.use_morton) {uint linear_id = threadgroup_position.x;uint morton_x = linear_id;uint morton_y = linear_id >> 1;@@ -136,8 +136,8 @@ struct ThreadgroupGemmMpp {tile_id_y = threadgroup_position.y;}- if (tile_id_x >= params->threadgroups_per_row ||- tile_id_y >= params->threadgroups_per_column) {+ if (tile_id_x >= params.threadgroups_per_row ||+ tile_id_y >= params.threadgroups_per_column) {return;}@@ -147,9 +147,9 @@ struct ThreadgroupGemmMpp {const size_t block_col_start_long = size_t(block_col_start);const device T* left_block_ptr =- left_matrix + block_row_start_long * params->leading_dimension_a;+ left_matrix + block_row_start_long * params.leading_dimension_a;const device T* right_block_ptr =- right_matrix + block_col_start_long * params->leading_dimension_b;+ right_matrix + block_col_start_long * params.leading_dimension_b;ThreadgroupLoader<T,@@ -160,7 +160,7 @@ struct ThreadgroupGemmMpp {THREADGROUP_SIZE>left_loader(left_block_ptr,- params->leading_dimension_a,+ params.leading_dimension_a,left_shared,ushort(simd_group_id),ushort(simd_lane_id)@@ -174,7 +174,7 @@ struct ThreadgroupGemmMpp {THREADGROUP_SIZE>right_loader(right_block_ptr,- params->leading_dimension_b,+ params.leading_dimension_b,right_shared,ushort(simd_group_id),ushort(simd_lane_id)@@ -186,8 +186,8 @@ struct ThreadgroupGemmMpp {SIMDGROUP_BLOCK_N * (simd_group_id % SIMDGROUPS_PER_COLUMN);device T* output_ptr =- output_matrix + block_row_start_long * params->leading_dimension_d +- block_col_start_long + tile_row_offset * params->leading_dimension_d ++ output_matrix + block_row_start_long * params.leading_dimension_d ++ block_col_start_long + tile_row_offset * params.leading_dimension_d +tile_col_offset;const short simdgroup_limit_m =@@ -195,14 +195,14 @@ struct ThreadgroupGemmMpp {? SIMDGROUP_BLOCK_M: short(min(int(SIMDGROUP_BLOCK_M),- int(params->M) - int(block_row_start + tile_row_offset))+ int(params.M) - int(block_row_start + tile_row_offset)));const short simdgroup_limit_n =align_n? SIMDGROUP_BLOCK_N: short(min(int(SIMDGROUP_BLOCK_N),- int(params->N) - int(block_col_start + tile_col_offset))+ int(params.N) - int(block_col_start + tile_col_offset)));constexpr auto matmul_descriptor = mpp::tensor_ops::matmul2d_descriptor(@@ -234,20 +234,20 @@ struct ThreadgroupGemmMpp {decltype(right_tensor),AccumulatorType>();- const uint leading_dimension_d = params->leading_dimension_d;+ const uint leading_dimension_d = params.leading_dimension_d;AccumulatorType accumulator_storage[TILES_M * TILES_N][ACCUMULATOR_CAPACITY] = {{0}};- const uint full_prefetch_iterations = params->K / PREFETCH_K;- const uint k_remainder = params->K - full_prefetch_iterations * PREFETCH_K;+ const uint full_prefetch_iterations = params.K / PREFETCH_K;+ const uint k_remainder = params.K - full_prefetch_iterations * PREFETCH_K;const ushort actual_block_rows =align_m ? BLOCK_ROWS- : ushort(min(uint(BLOCK_ROWS), params->M - block_row_start));+ : ushort(min(uint(BLOCK_ROWS), params.M - block_row_start));const ushort actual_block_cols =align_n ? BLOCK_COLS- : ushort(min(uint(BLOCK_COLS), params->N - block_col_start));+ : ushort(min(uint(BLOCK_COLS), params.N - block_col_start));// Main loopfor (uint outer_k = 0; outer_k < full_prefetch_iterations; outer_k++) {diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metalindex f790479f0..69313b32a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal@@ -22,7 +22,7 @@ KERNEL(MatmulGemm)(const device T* a,const device T* b,device T* d,- const constant uzu::matmul::GemmParams* params,+ const constant uzu::matmul::GemmParams& params,const constant uint& group_count_x,const constant uint& group_count_y,const constant float& ab_scale,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metalindex e0a2a8b43..0a1c579a3 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal@@ -18,7 +18,7 @@ KERNEL(MatmulGemmMpp)(const device T* left_matrix,const device T* right_matrix,device T* output_matrix,- const constant uzu::matmul::GemmParams* params,+ const constant uzu::matmul::GemmParams& params,const constant uint& group_count_x,const constant uint& group_count_y,const constant float& ab_scale,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex f8f3a852a..c26498367 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -199,7 +199,7 @@ impl MatmulMetalKernel {(arguments.a, arguments.a_offset as usize),arguments.b,&mut *arguments.d,- std::slice::from_ref(¶ms),+ params,threadgroups_per_row,threadgroups_per_column,arguments.ab_scale,@@ -267,7 +267,7 @@ impl MatmulMetalKernel {(arguments.a, arguments.a_offset as usize),arguments.b,&mut *arguments.d,- std::slice::from_ref(¶ms),+ params,group_count_x,group_count_y,arguments.ab_scale,diff --git a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metalindex a5edac158..a49389ef8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal@@ -22,10 +22,10 @@ PUBLIC KERNEL(SSDPrefill64)(constant const uint& suffix_len,constant const uint& group_size,constant const uint& state_size,- constant const uint* x_strides,- constant const uint* dt_strides,- constant const uint* cb_strides,- constant const uint* state_strides,+ constant const uint x_strides[3],+ constant const uint dt_strides[2],+ constant const uint cb_strides[3],+ constant const uint state_strides[3],constant const uint& num_heads,constant const uint& head_dim,const ThreadContext thread_context,@@ -114,10 +114,10 @@ PUBLIC KERNEL(SSDPrefill)(constant const uint& suffix_len,constant const uint& group_size,constant const uint& state_size,- constant const uint* x_strides,- constant const uint* dt_strides,- constant const uint* cb_strides,- constant const uint* state_strides,+ constant const uint x_strides[3],+ constant const uint dt_strides[2],+ constant const uint cb_strides[3],+ constant const uint state_strides[3],constant const uint& num_heads,constant const uint& head_dim,const ThreadContext thread_context,@@ -224,10 +224,10 @@ PUBLIC KERNEL(SSDPrefillSequential)(constant const uint& suffix_len,constant const uint& group_size,constant const uint& state_size,- constant const uint* x_strides,- constant const uint* dt_strides,- constant const uint* cb_strides,- constant const uint* state_strides,+ constant const uint x_strides[3],+ constant const uint dt_strides[2],+ constant const uint cb_strides[3],+ constant const uint state_strides[3],constant const uint& channels,constant const uint& head_dim,const uint h_idx AXIS(channels, 32),diff --git a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs b/crates/backend-uzu/src/encodable_block/mamba_mixer.rsindex 0863028d5..b23c484b9 100644--- a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs@@ -343,10 +343,10 @@ impl<B: Backend> MambaMixer<B> {state_arr.buffer().borrow_mut().deref_mut(),group_size as u32,state_size as u32,- x_strides.as_slice(),- dt_strides.as_slice(),- cb_strides.as_slice(),- state_strides.as_slice(),+ &x_strides,+ &dt_strides,+ &cb_strides,+ &state_strides,suffix_length as u32,h,dh,diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rsindex da806262c..c2bc85e19 100644--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs@@ -111,10 +111,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(let state_array = context.create_array_from(&[input.state.len()], &input.state, "state");let y_array = context.create_array_uninitialized(&[total_x], T::data_type(), "y");- let x_strides: Vec<u32> = input.x_strides.iter().map(|&s| s as u32).collect();- let dt_strides: Vec<u32> = input.dt_strides.iter().map(|&s| s as u32).collect();- let cb_strides: Vec<u32> = input.cb_strides.iter().map(|&s| s as u32).collect();- let state_strides: Vec<u32> = input.state_strides.iter().map(|&s| s as u32).collect();+ let x_strides: [u32; 3] = input.x_strides.map(|s| s as u32);+ let dt_strides: [u32; 2] = input.dt_strides.map(|s| s as u32);+ let cb_strides: [u32; 3] = input.cb_strides.map(|s| s as u32);+ let state_strides: [u32; 3] = input.state_strides.map(|s| s as u32);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");match kernel_type {diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rsindex 077a32982..761ebcd2a 100644--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs@@ -121,10 +121,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tns_array.buffer().borrow_mut().deref_mut(),(h / g) as u32,input.n,- x_strides.as_slice(),- dt_strides.as_slice(),- cb_strides.as_slice(),- state_strides.as_slice(),+ &x_strides,+ &dt_strides,+ &cb_strides,+ &state_strides,input.bsz,input.h,input.dh,@@ -153,10 +153,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tns_array.buffer().borrow_mut().deref_mut(),(h / g) as u32,input.n,- x_strides.as_slice(),- dt_strides.as_slice(),- cb_strides.as_slice(),- state_strides.as_slice(),+ &x_strides,+ &dt_strides,+ &cb_strides,+ &state_strides,input.bsz,input.h,input.dh,