Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Sized constants (#358)

uuuvnc727f6d7802c
diff --git a/crates/backend-uzu/build/common/kernel.rs b/crates/backend-uzu/build/common/kernel.rs
index aa645e2c1..7e10ffec4 100644
--- a/crates/backend-uzu/build/common/kernel.rs
+++ b/crates/backend-uzu/build/common/kernel.rs
@@ -8,7 +8,6 @@ pub enum KernelBufferAccess {
pub enum KernelArgumentType {
Buffer(KernelBufferAccess),
Constant(Box<str>),
- Scalar(Box<str>),
}
#[derive(PartialEq, Debug, Clone)]
diff --git a/crates/backend-uzu/build/common/traitgen.rs b/crates/backend-uzu/build/common/traitgen.rs
index 1c110d652..58ba3b0a3 100644
--- a/crates/backend-uzu/build/common/traitgen.rs
+++ b/crates/backend-uzu/build/common/traitgen.rs
@@ -51,10 +51,6 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
)
},
KernelArgumentType::Constant(ty) => {
- let ty: Type = syn::parse_str(ty.as_ref()).unwrap();
- (None, quote! { &[#ty] })
- },
- KernelArgumentType::Scalar(ty) => {
let ty: Type = syn::parse_str(ty.as_ref()).unwrap();
(None, quote! { #ty })
},
diff --git a/crates/backend-uzu/build/cpu/compiler.rs b/crates/backend-uzu/build/cpu/compiler.rs
index 9181ba8b8..234ba9164 100644
--- a/crates/backend-uzu/build/cpu/compiler.rs
+++ b/crates/backend-uzu/build/cpu/compiler.rs
@@ -21,7 +21,7 @@ use crate::common::{
#[derive(PartialEq, Debug)]
pub enum FunctionArgumentType {
Buffer(KernelBufferAccess),
- Constant(Type),
+ Constant(Type, Option<Expr>),
Scalar(Type),
Specialization(Type),
}
@@ -40,12 +40,20 @@ impl FunctionArgument {
conditional: self.conditional.is_some(),
ty: match &self.ty {
FunctionArgumentType::Buffer(access) => KernelArgumentType::Buffer(access.clone()),
- FunctionArgumentType::Constant(ty) => KernelArgumentType::Constant(
+ FunctionArgumentType::Constant(ty, None) => KernelArgumentType::Constant(
+ format!("&[{}]", ty.to_token_stream().to_string().replace(" :: ", "::")).into_boxed_str(),
+ ),
+ FunctionArgumentType::Constant(ty, Some(sz)) => KernelArgumentType::Constant(
+ format!(
+ "&[{}; {}]",
+ ty.to_token_stream().to_string().replace(" :: ", "::"),
+ sz.to_token_stream().to_string(),
+ )
+ .into_boxed_str(),
+ ),
+ FunctionArgumentType::Scalar(ty) => KernelArgumentType::Constant(
ty.to_token_stream().to_string().replace(" :: ", "::").into_boxed_str(),
),
- FunctionArgumentType::Scalar(ty) => {
- KernelArgumentType::Scalar(ty.to_token_stream().to_string().replace(" :: ", "::").into_boxed_str())
- },
FunctionArgumentType::Specialization(_) => {
return None;
},
@@ -352,10 +360,6 @@ impl CpuCompiler {
)
},
KernelArgumentType::Constant(ty) => {
- let ty: Type = syn::parse_str(ty.as_ref()).context("cannot parse type")?;
- (None, quote! { &[#ty] })
- },
- KernelArgumentType::Scalar(ty) => {
let ty: Type = syn::parse_str(ty.as_ref()).context("cannot parse type")?;
(None, quote! { #ty })
},
@@ -405,9 +409,12 @@ impl CpuCompiler {
})
}
},
- FunctionArgumentType::Constant(_) => {
+ FunctionArgumentType::Constant(_, None) => {
Some(quote! { let #argument_ident = #argument_ident.to_vec().into_boxed_slice(); })
},
+ FunctionArgumentType::Constant(_, Some(_)) => {
+ Some(quote! { let #argument_ident = Box::new(*#argument_ident); })
+ },
FunctionArgumentType::Scalar(_) => None,
FunctionArgumentType::Specialization(_) => {
Some(quote! { let #argument_ident = self.#argument_ident; })
@@ -434,7 +441,7 @@ impl CpuCompiler {
quote! { #argument_ident.as_ptr() as _ }
}
},
- FunctionArgumentType::Constant(_) => quote! { #argument_ident.as_ref() },
+ FunctionArgumentType::Constant(_, _) => quote! { &*#argument_ident },
FunctionArgumentType::Scalar(_) | FunctionArgumentType::Specialization(_) => {
quote! { #argument_ident }
},
@@ -581,7 +588,8 @@ impl CpuCompiler {
KernelBufferAccess::Read
}),
Type::Reference(ty) => match *ty.elem {
- Type::Slice(ty) => FunctionArgumentType::Constant(*ty.elem),
+ Type::Slice(ty) => FunctionArgumentType::Constant(*ty.elem, None),
+ Type::Array(ty) => FunctionArgumentType::Constant(*ty.elem, Some(ty.len)),
ty => bail!("unsupported reference type: {} ({:?})", ty.to_token_stream().to_string(), ty),
},
Type::Path(ty) => FunctionArgumentType::Scalar(Type::Path(ty)),
diff --git a/crates/backend-uzu/build/metal/ast.rs b/crates/backend-uzu/build/metal/ast.rs
index d91893a63..7b72ccebd 100644
--- a/crates/backend-uzu/build/metal/ast.rs
+++ b/crates/backend-uzu/build/metal/ast.rs
@@ -99,7 +99,7 @@ pub enum MetalBufferAccess {
#[derive(Debug, Serialize, Deserialize)]
pub enum MetalConstantType {
Scalar,
- Array,
+ Array(Option<Box<str>>),
}
#[derive(Debug, Serialize, Deserialize)]
@@ -267,9 +267,18 @@ impl MetalArgument {
} else if let ["const", "constant", c_type_scalar, "*"] =
self.c_type.split_whitespace().collect::<Vec<_>>().as_slice()
{
+ let size = if self.source.contains('[') && self.source.contains(']') {
+ let lbracket = self.source.rfind('[').context("sized constant missing size bracket")? + 1;
+ let rbracket = self.source.rfind(']').context("sized constant missing size bracket")?;
+ let size_expr = &self.source[lbracket..rbracket];
+ Some(size_expr.into())
+ } else {
+ None
+ };
+
Ok(MetalArgumentType::Constant((
Self::scalar_type_to_rust(c_type_scalar)?.into(),
- MetalConstantType::Array,
+ MetalConstantType::Array(size),
)))
} else if self.c_type.contains("threadgroup") && self.c_type.contains('*') {
let lbracket = self.source.rfind('[').context("threadgroup missing size bracket")? + 1;
@@ -397,13 +406,20 @@ impl MetalKernelInfo {
Ok(MetalArgumentType::Constant((ty, MetalConstantType::Scalar))) => Some(KernelArgument {
name: a.name.clone(),
conditional: a.argument_condition().unwrap().is_some(),
- ty: KernelArgumentType::Scalar(ty),
+ ty: KernelArgumentType::Constant(ty),
}),
- Ok(MetalArgumentType::Constant((ty, MetalConstantType::Array))) => Some(KernelArgument {
+ Ok(MetalArgumentType::Constant((ty, MetalConstantType::Array(None)))) => Some(KernelArgument {
name: a.name.clone(),
conditional: a.argument_condition().unwrap().is_some(),
- ty: KernelArgumentType::Constant(ty),
+ ty: KernelArgumentType::Constant(format!("&[{ty}]").into_boxed_str()),
}),
+ Ok(MetalArgumentType::Constant((ty, MetalConstantType::Array(Some(size))))) => {
+ Some(KernelArgument {
+ name: a.name.clone(),
+ conditional: a.argument_condition().unwrap().is_some(),
+ ty: KernelArgumentType::Constant(format!("&[{ty}; {size}]").into_boxed_str()),
+ })
+ },
_ => None,
})
.collect(),
diff --git a/crates/backend-uzu/build/metal/bindgen.rs b/crates/backend-uzu/build/metal/bindgen.rs
index b63e9ac1b..7eb32932d 100644
--- a/crates/backend-uzu/build/metal/bindgen.rs
+++ b/crates/backend-uzu/build/metal/bindgen.rs
@@ -220,13 +220,23 @@ pub fn bindgen(
quote! { compute_encoder.set_value(&#arg_name, #arg_count); },
None,
),
- MetalConstantType::Array => (
+ MetalConstantType::Array(None) => (
quote! { &[#arg_dtype] },
None,
None,
quote! { compute_encoder.set_slice(#arg_name, #arg_count); },
None,
),
+ MetalConstantType::Array(Some(size)) => {
+ let arg_size: Expr = syn::parse_str(&size).unwrap();
+ (
+ quote! { &[#arg_dtype; #arg_size] },
+ None,
+ None,
+ quote! { compute_encoder.set_slice(#arg_name, #arg_count); },
+ None,
+ )
+ },
}
},
_ => unreachable!(),
diff --git a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs
index b14dfdb03..0f8af6ec8 100644
--- a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs
@@ -59,10 +59,10 @@ impl<B: Backend> SSDPrefillKernels<B> {
args: SSDPrefillArguments<B>,
mode: SSDPrefillMode,
) {
- let x_strides: Vec<u32> = args.x_strides.iter().map(|x| *x as u32).collect();
- let dt_strides: Vec<u32> = args.dt_strides.iter().map(|x| *x as u32).collect();
- let cb_strides: Vec<u32> = args.cb_strides.iter().map(|x| *x as u32).collect();
- let state_strides: Vec<u32> = args.state_strides.iter().map(|x| *x as u32).collect();
+ let x_strides: [u32; 3] = args.x_strides.map(|x| x as u32);
+ let dt_strides: [u32; 2] = args.dt_strides.map(|x| x as u32);
+ let cb_strides: [u32; 3] = args.cb_strides.map(|x| x as u32);
+ let state_strides: [u32; 3] = args.state_strides.map(|x| x as u32);
if mode == SSDPrefillMode::SinglePass {
if args.state_size == 64 {
@@ -78,10 +78,10 @@ impl<B: Backend> SSDPrefillKernels<B> {
args.suffix_len as u32,
args.group_size,
args.state_size,
- x_strides.as_slice(),
- dt_strides.as_slice(),
- cb_strides.as_slice(),
- state_strides.as_slice(),
+ &x_strides,
+ &dt_strides,
+ &cb_strides,
+ &state_strides,
args.channels as u32,
args.head_dim as u32,
encoder,
@@ -99,10 +99,10 @@ impl<B: Backend> SSDPrefillKernels<B> {
args.suffix_len as u32,
args.group_size,
args.state_size,
- x_strides.as_slice(),
- dt_strides.as_slice(),
- cb_strides.as_slice(),
- state_strides.as_slice(),
+ &x_strides,
+ &dt_strides,
+ &cb_strides,
+ &state_strides,
args.channels as u32,
args.head_dim as u32,
encoder,
@@ -121,10 +121,10 @@ impl<B: Backend> SSDPrefillKernels<B> {
args.suffix_len as u32,
args.group_size,
args.state_size,
- x_strides.as_slice(),
- dt_strides.as_slice(),
- cb_strides.as_slice(),
- state_strides.as_slice(),
+ &x_strides,
+ &dt_strides,
+ &cb_strides,
+ &state_strides,
args.channels as u32,
args.head_dim as u32,
encoder,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs b/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs
index 3a673f923..674720e35 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs
@@ -18,10 +18,10 @@ pub fn ssd_prefill64<T: ArrayElement + Float>(
#[allow(unused)] suffix_len: u32,
#[allow(unused)] group_size: u32,
#[allow(unused)] state_size: u32,
- #[allow(unused)] x_strides: &[u32],
- #[allow(unused)] dt_strides: &[u32],
- #[allow(unused)] cb_strides: &[u32],
- #[allow(unused)] state_strides: &[u32],
+ #[allow(unused)] x_strides: &[u32; 3],
+ #[allow(unused)] dt_strides: &[u32; 2],
+ #[allow(unused)] cb_strides: &[u32; 3],
+ #[allow(unused)] state_strides: &[u32; 3],
#[allow(unused)] num_heads: u32,
#[allow(unused)] head_dim: u32,
) {
@@ -42,10 +42,10 @@ pub fn ssd_prefill<T: ArrayElement + Float>(
#[allow(unused)] suffix_len: u32,
#[allow(unused)] group_size: u32,
#[allow(unused)] state_size: u32,
- #[allow(unused)] x_strides: &[u32],
- #[allow(unused)] dt_strides: &[u32],
- #[allow(unused)] cb_strides: &[u32],
- #[allow(unused)] state_strides: &[u32],
+ #[allow(unused)] x_strides: &[u32; 3],
+ #[allow(unused)] dt_strides: &[u32; 2],
+ #[allow(unused)] cb_strides: &[u32; 3],
+ #[allow(unused)] state_strides: &[u32; 3],
#[allow(unused)] num_heads: u32,
#[allow(unused)] head_dim: u32,
) {
@@ -66,10 +66,10 @@ pub fn ssd_prefill_sequential<T: ArrayElement + Float>(
suffix_len: u32,
group_size: u32,
state_size: u32,
- x_strides: &[u32],
- dt_strides: &[u32],
- cb_strides: &[u32],
- state_strides: &[u32],
+ x_strides: &[u32; 3],
+ dt_strides: &[u32; 2],
+ cb_strides: &[u32; 3],
+ state_strides: &[u32; 3],
channels: u32,
head_dim: u32,
) {
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_update.rs
index 2de6ecea9..e4db4967f 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_update.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/ssm/ssd_update.rs
@@ -18,10 +18,10 @@ pub fn ssd_update<T: ArrayElement + Float>(
next_state: *mut T,
group_size: u32,
state_size: u32,
- x_strides: &[u32],
- dt_strides: &[u32],
- cb_strides: &[u32],
- state_strides: &[u32],
+ x_strides: &[u32; 3],
+ dt_strides: &[u32; 2],
+ cb_strides: &[u32; 3],
+ state_strides: &[u32; 4],
b_size: u32,
h_size: u32,
dh_size: u32,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h
index 2e0685009..37b5673bd 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h
@@ -149,7 +149,7 @@ struct ThreadgroupGemm {
const device T* a,
const device T* b,
device U* d,
- const constant GemmParams* params,
+ const constant GemmParams& params,
const constant float& ab_scale,
const bool is_accumulate,
threadgroup T* a_shared,
@@ -161,13 +161,13 @@ struct ThreadgroupGemm {
) {
(void)lid;
- const int swizzle_stride = pow2(params->swizzle_log);
+ const int swizzle_stride = pow2(params.swizzle_log);
const int swizzled_row_id =
tid.y * swizzle_stride + (tid.x % swizzle_stride);
const int swizzled_col_id = tid.x / swizzle_stride;
- if (params->threadgroups_per_row <= swizzled_col_id ||
- params->threadgroups_per_column <= swizzled_row_id) {
+ if (params.threadgroups_per_row <= swizzled_col_id ||
+ params.threadgroups_per_column <= swizzled_row_id) {
return;
}
@@ -179,21 +179,21 @@ struct ThreadgroupGemm {
const size_t output_col_long = size_t(output_col);
a += transpose_a ? output_row_long
- : output_row_long * params->leading_dimension_a;
- b += transpose_b ? output_col_long * params->leading_dimension_b
+ : output_row_long * params.leading_dimension_a;
+ b += transpose_b ? output_col_long * params.leading_dimension_b
: output_col_long;
- d += output_row_long * params->leading_dimension_d + output_col_long;
+ d += output_row_long * params.leading_dimension_d + output_col_long;
thread LoaderAType loader_a(
a,
- params->leading_dimension_a,
+ params.leading_dimension_a,
a_shared,
simd_group_id,
simd_lane_id
);
thread LoaderBType loader_b(
b,
- params->leading_dimension_b,
+ params.leading_dimension_b,
b_shared,
simd_group_id,
simd_lane_id
@@ -206,7 +206,7 @@ struct ThreadgroupGemm {
is_accumulate ? 1.0f : 0.0f
);
- int gemm_k_iterations = params->aligned_inner_iterations;
+ int gemm_k_iterations = params.aligned_inner_iterations;
///////////////////////////////////////////////////////////////////////////
// MNK aligned loop
@@ -228,7 +228,7 @@ struct ThreadgroupGemm {
if (!K_aligned) {
int leftover_block_depth =
- params->K - params->aligned_inner_iterations * BLOCK_DEPTH;
+ params.K - params.aligned_inner_iterations * BLOCK_DEPTH;
short2 tile_dimensions_a =
transpose_a ? short2(BLOCK_ROWS, leftover_block_depth)
: short2(leftover_block_depth, BLOCK_ROWS);
@@ -245,19 +245,19 @@ struct ThreadgroupGemm {
}
threadgroup_tile
- .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);
- threadgroup_tile.store_result(d, params->leading_dimension_d);
+ .apply_epilogue(d, params.leading_dimension_d, 1, epilogue);
+ threadgroup_tile.store_result(d, params.leading_dimension_d);
return;
}
///////////////////////////////////////////////////////////////////////////
// MN unaligned loop
else {
short threadgroup_block_rows =
- min(BLOCK_ROWS, ((int)params->M) - output_row);
+ min(BLOCK_ROWS, ((int)params.M) - output_row);
short threadgroup_block_cols =
- min(BLOCK_COLS, ((int)params->N) - output_col);
+ min(BLOCK_COLS, ((int)params.N) - output_col);
short leftover_block_depth =
- params->K - params->aligned_inner_iterations * BLOCK_DEPTH;
+ params.K - params.aligned_inner_iterations * BLOCK_DEPTH;
if (threadgroup_block_rows == BLOCK_ROWS &&
threadgroup_block_cols == BLOCK_COLS) {
@@ -274,8 +274,8 @@ struct ThreadgroupGemm {
);
threadgroup_tile
- .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);
- threadgroup_tile.store_result(d, params->leading_dimension_d);
+ .apply_epilogue(d, params.leading_dimension_d, 1, epilogue);
+ threadgroup_tile.store_result(d, params.leading_dimension_d);
return;
} else if (threadgroup_block_cols == BLOCK_COLS) {
@@ -293,14 +293,14 @@ struct ThreadgroupGemm {
threadgroup_tile.apply_epilogue_safe(
d,
- params->leading_dimension_d,
+ params.leading_dimension_d,
1,
short2(threadgroup_block_cols, threadgroup_block_rows),
epilogue
);
threadgroup_tile.store_result_safe(
d,
- params->leading_dimension_d,
+ params.leading_dimension_d,
short2(threadgroup_block_cols, threadgroup_block_rows)
);
return;
@@ -320,14 +320,14 @@ struct ThreadgroupGemm {
threadgroup_tile.apply_epilogue_safe(
d,
- params->leading_dimension_d,
+ params.leading_dimension_d,
1,
short2(threadgroup_block_cols, threadgroup_block_rows),
epilogue
);
threadgroup_tile.store_result_safe(
d,
- params->leading_dimension_d,
+ params.leading_dimension_d,
short2(threadgroup_block_cols, threadgroup_block_rows)
);
return;
@@ -347,14 +347,14 @@ struct ThreadgroupGemm {
threadgroup_tile.apply_epilogue_safe(
d,
- params->leading_dimension_d,
+ params.leading_dimension_d,
1,
short2(threadgroup_block_cols, threadgroup_block_rows),
epilogue
);
threadgroup_tile.store_result_safe(
d,
- params->leading_dimension_d,
+ params.leading_dimension_d,
short2(threadgroup_block_cols, threadgroup_block_rows)
);
return;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
index a275b3d00..e51142ad7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
@@ -103,7 +103,7 @@ struct ThreadgroupGemmMpp {
const device T* left_matrix,
const device T* right_matrix,
device T* output_matrix,
- const constant GemmParams* params,
+ const constant GemmParams& params,
const bool align_m,
const bool align_n,
const bool is_accumulate,
@@ -115,7 +115,7 @@ struct ThreadgroupGemmMpp {
uint2 threadgroup_position
) {
uint tile_id_x, tile_id_y;
- if (params->use_morton) {
+ if (params.use_morton) {
uint linear_id = threadgroup_position.x;
uint morton_x = linear_id;
uint morton_y = linear_id >> 1;
@@ -136,8 +136,8 @@ struct ThreadgroupGemmMpp {
tile_id_y = threadgroup_position.y;
}
- if (tile_id_x >= params->threadgroups_per_row ||
- tile_id_y >= params->threadgroups_per_column) {
+ if (tile_id_x >= params.threadgroups_per_row ||
+ tile_id_y >= params.threadgroups_per_column) {
return;
}
@@ -147,9 +147,9 @@ struct ThreadgroupGemmMpp {
const size_t block_col_start_long = size_t(block_col_start);
const device T* left_block_ptr =
- left_matrix + block_row_start_long * params->leading_dimension_a;
+ left_matrix + block_row_start_long * params.leading_dimension_a;
const device T* right_block_ptr =
- right_matrix + block_col_start_long * params->leading_dimension_b;
+ right_matrix + block_col_start_long * params.leading_dimension_b;
ThreadgroupLoader<
T,
@@ -160,7 +160,7 @@ struct ThreadgroupGemmMpp {
THREADGROUP_SIZE>
left_loader(
left_block_ptr,
- params->leading_dimension_a,
+ params.leading_dimension_a,
left_shared,
ushort(simd_group_id),
ushort(simd_lane_id)
@@ -174,7 +174,7 @@ struct ThreadgroupGemmMpp {
THREADGROUP_SIZE>
right_loader(
right_block_ptr,
- params->leading_dimension_b,
+ params.leading_dimension_b,
right_shared,
ushort(simd_group_id),
ushort(simd_lane_id)
@@ -186,8 +186,8 @@ struct ThreadgroupGemmMpp {
SIMDGROUP_BLOCK_N * (simd_group_id % SIMDGROUPS_PER_COLUMN);
device T* output_ptr =
- output_matrix + block_row_start_long * params->leading_dimension_d +
- block_col_start_long + tile_row_offset * params->leading_dimension_d +
+ output_matrix + block_row_start_long * params.leading_dimension_d +
+ block_col_start_long + tile_row_offset * params.leading_dimension_d +
tile_col_offset;
const short simdgroup_limit_m =
@@ -195,14 +195,14 @@ struct ThreadgroupGemmMpp {
? SIMDGROUP_BLOCK_M
: short(
min(int(SIMDGROUP_BLOCK_M),
- int(params->M) - int(block_row_start + tile_row_offset))
+ int(params.M) - int(block_row_start + tile_row_offset))
);
const short simdgroup_limit_n =
align_n
? SIMDGROUP_BLOCK_N
: short(
min(int(SIMDGROUP_BLOCK_N),
- int(params->N) - int(block_col_start + tile_col_offset))
+ int(params.N) - int(block_col_start + tile_col_offset))
);
constexpr auto matmul_descriptor = mpp::tensor_ops::matmul2d_descriptor(
@@ -234,20 +234,20 @@ struct ThreadgroupGemmMpp {
decltype(right_tensor),
AccumulatorType>();
- const uint leading_dimension_d = params->leading_dimension_d;
+ const uint leading_dimension_d = params.leading_dimension_d;
AccumulatorType accumulator_storage[TILES_M * TILES_N]
[ACCUMULATOR_CAPACITY] = {{0}};
- const uint full_prefetch_iterations = params->K / PREFETCH_K;
- const uint k_remainder = params->K - full_prefetch_iterations * PREFETCH_K;
+ const uint full_prefetch_iterations = params.K / PREFETCH_K;
+ const uint k_remainder = params.K - full_prefetch_iterations * PREFETCH_K;
const ushort actual_block_rows =
align_m ? BLOCK_ROWS
- : ushort(min(uint(BLOCK_ROWS), params->M - block_row_start));
+ : ushort(min(uint(BLOCK_ROWS), params.M - block_row_start));
const ushort actual_block_cols =
align_n ? BLOCK_COLS
- : ushort(min(uint(BLOCK_COLS), params->N - block_col_start));
+ : ushort(min(uint(BLOCK_COLS), params.N - block_col_start));
// Main loop
for (uint outer_k = 0; outer_k < full_prefetch_iterations; outer_k++) {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal
index f790479f0..69313b32a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal
@@ -22,7 +22,7 @@ KERNEL(MatmulGemm)(
const device T* a,
const device T* b,
device T* d,
- const constant uzu::matmul::GemmParams* params,
+ const constant uzu::matmul::GemmParams& params,
const constant uint& group_count_x,
const constant uint& group_count_y,
const constant float& ab_scale,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal
index e0a2a8b43..0a1c579a3 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal
@@ -18,7 +18,7 @@ KERNEL(MatmulGemmMpp)(
const device T* left_matrix,
const device T* right_matrix,
device T* output_matrix,
- const constant uzu::matmul::GemmParams* params,
+ const constant uzu::matmul::GemmParams& params,
const constant uint& group_count_x,
const constant uint& group_count_y,
const constant float& ab_scale,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index f8f3a852a..c26498367 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -199,7 +199,7 @@ impl MatmulMetalKernel {
(arguments.a, arguments.a_offset as usize),
arguments.b,
&mut *arguments.d,
- std::slice::from_ref(&params),
+ params,
threadgroups_per_row,
threadgroups_per_column,
arguments.ab_scale,
@@ -267,7 +267,7 @@ impl MatmulMetalKernel {
(arguments.a, arguments.a_offset as usize),
arguments.b,
&mut *arguments.d,
- std::slice::from_ref(&params),
+ params,
group_count_x,
group_count_y,
arguments.ab_scale,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
index a5edac158..a49389ef8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
@@ -22,10 +22,10 @@ PUBLIC KERNEL(SSDPrefill64)(
constant const uint& suffix_len,
constant const uint& group_size,
constant const uint& state_size,
- constant const uint* x_strides,
- constant const uint* dt_strides,
- constant const uint* cb_strides,
- constant const uint* state_strides,
+ constant const uint x_strides[3],
+ constant const uint dt_strides[2],
+ constant const uint cb_strides[3],
+ constant const uint state_strides[3],
constant const uint& num_heads,
constant const uint& head_dim,
const ThreadContext thread_context,
@@ -114,10 +114,10 @@ PUBLIC KERNEL(SSDPrefill)(
constant const uint& suffix_len,
constant const uint& group_size,
constant const uint& state_size,
- constant const uint* x_strides,
- constant const uint* dt_strides,
- constant const uint* cb_strides,
- constant const uint* state_strides,
+ constant const uint x_strides[3],
+ constant const uint dt_strides[2],
+ constant const uint cb_strides[3],
+ constant const uint state_strides[3],
constant const uint& num_heads,
constant const uint& head_dim,
const ThreadContext thread_context,
@@ -224,10 +224,10 @@ PUBLIC KERNEL(SSDPrefillSequential)(
constant const uint& suffix_len,
constant const uint& group_size,
constant const uint& state_size,
- constant const uint* x_strides,
- constant const uint* dt_strides,
- constant const uint* cb_strides,
- constant const uint* state_strides,
+ constant const uint x_strides[3],
+ constant const uint dt_strides[2],
+ constant const uint cb_strides[3],
+ constant const uint state_strides[3],
constant const uint& channels,
constant const uint& head_dim,
const uint h_idx AXIS(channels, 32),
diff --git a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
index 0863028d5..b23c484b9 100644
--- a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
@@ -343,10 +343,10 @@ impl<B: Backend> MambaMixer<B> {
state_arr.buffer().borrow_mut().deref_mut(),
group_size as u32,
state_size as u32,
- x_strides.as_slice(),
- dt_strides.as_slice(),
- cb_strides.as_slice(),
- state_strides.as_slice(),
+ &x_strides,
+ &dt_strides,
+ &cb_strides,
+ &state_strides,
suffix_length as u32,
h,
dh,
diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs
index da806262c..c2bc85e19 100644
--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs
@@ -111,10 +111,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
let state_array = context.create_array_from(&[input.state.len()], &input.state, "state");
let y_array = context.create_array_uninitialized(&[total_x], T::data_type(), "y");
- let x_strides: Vec<u32> = input.x_strides.iter().map(|&s| s as u32).collect();
- let dt_strides: Vec<u32> = input.dt_strides.iter().map(|&s| s as u32).collect();
- let cb_strides: Vec<u32> = input.cb_strides.iter().map(|&s| s as u32).collect();
- let state_strides: Vec<u32> = input.state_strides.iter().map(|&s| s as u32).collect();
+ let x_strides: [u32; 3] = input.x_strides.map(|s| s as u32);
+ let dt_strides: [u32; 2] = input.dt_strides.map(|s| s as u32);
+ let cb_strides: [u32; 3] = input.cb_strides.map(|s| s as u32);
+ let state_strides: [u32; 3] = input.state_strides.map(|s| s as u32);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
match kernel_type {
diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs
index 077a32982..761ebcd2a 100644
--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs
@@ -121,10 +121,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
ns_array.buffer().borrow_mut().deref_mut(),
(h / g) as u32,
input.n,
- x_strides.as_slice(),
- dt_strides.as_slice(),
- cb_strides.as_slice(),
- state_strides.as_slice(),
+ &x_strides,
+ &dt_strides,
+ &cb_strides,
+ &state_strides,
input.bsz,
input.h,
input.dh,
@@ -153,10 +153,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
ns_array.buffer().borrow_mut().deref_mut(),
(h / g) as u32,
input.n,
- x_strides.as_slice(),
- dt_strides.as_slice(),
- cb_strides.as_slice(),
- state_strides.as_slice(),
+ &x_strides,
+ &dt_strides,
+ &cb_strides,
+ &state_strides,
input.bsz,
input.h,
input.dh,