Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Optional threadgroup (#458)

CCYeh2746ee450393
diff --git a/crates/backend-uzu/build/metal/ast.rs b/crates/backend-uzu/build/metal/ast.rs
index d6db93130..ba24dba9d 100644
--- a/crates/backend-uzu/build/metal/ast.rs
+++ b/crates/backend-uzu/build/metal/ast.rs
@@ -130,6 +130,38 @@ pub struct MetalArgument {
pub source: Box<str>,
}
+pub fn shared_element_type(c_type: &str) -> &str {
+ c_type.split(['*', '&', '(']).next().unwrap_or_default().trim_end()
+}
+
+pub fn shared_element_byte_size(c_type: &str) -> anyhow::Result<usize> {
+ let element_type = shared_element_type(c_type).rsplit(' ').next().unwrap_or_default();
+
+ // Split the lane digit off vectors: "float4" -> "float", 4.
+ let (scalar_name, lanes) = match element_type.chars().last() {
+ Some(lane_digit @ '2'..='4') => {
+ (element_type.strip_suffix(lane_digit).unwrap(), lane_digit.to_digit(10).unwrap() as usize)
+ },
+ _ => (element_type, 1),
+ };
+
+ let scalar_size = match scalar_name {
+ "bool" | "char" | "uchar" => 1,
+ "short" | "ushort" | "half" | "bfloat" => 2,
+ "int" | "uint" | "float" => 4,
+ "long" | "ulong" => 8,
+ other => bail!("unsupported OPTIONAL threadgroup element type `{other}`"),
+ };
+
+ // MSL pads 3-component vectors to 4 lanes, so `float3` is 16 bytes, not 12.
+ let padded_lanes = if lanes == 3 {
+ 4
+ } else {
+ lanes
+ };
+ Ok(scalar_size * padded_lanes)
+}
+
impl MetalArgument {
fn scalar_type_to_rust(c_type: &str) -> anyhow::Result<Box<str>> {
let mut tokens: Vec<_> = c_type.split_whitespace().collect();
@@ -203,6 +235,10 @@ impl MetalArgument {
_ => None,
}
}
+
+ pub fn is_optional_shared(&self) -> bool {
+ matches!(self.argument_type, MetalArgumentType::Shared(_)) && self.condition.is_some()
+ }
}
fn parse_argument_annotation(
@@ -217,8 +253,11 @@ fn parse_argument_annotation(
bail!("dsl.optional takes 1 argument, found {}", annotation.len() - 1);
}
let argument_type = parse_argument_type(c_type, source, None)?;
- if !matches!(argument_type, MetalArgumentType::Buffer(_) | MetalArgumentType::Constant(_)) {
- bail!("Only a buffer or a constant can be optional");
+ if !matches!(
+ argument_type,
+ MetalArgumentType::Buffer(_) | MetalArgumentType::Constant(_) | MetalArgumentType::Shared(_)
+ ) {
+ bail!("Only a buffer, a constant or a threadgroup argument can be optional");
}
return Ok((argument_type, Some(annotation[1].clone())));
}
diff --git a/crates/backend-uzu/build/metal/bindgen/arguments.rs b/crates/backend-uzu/build/metal/bindgen/arguments.rs
index 6df125c15..53faf9a4c 100644
--- a/crates/backend-uzu/build/metal/bindgen/arguments.rs
+++ b/crates/backend-uzu/build/metal/bindgen/arguments.rs
@@ -3,15 +3,22 @@ use proc_macro2::{Span, TokenStream};
use quote::{format_ident, quote};
use syn::{Expr, Ident, Lifetime, Type};
-use super::super::{
- ast::{MetalArgument, MetalArgumentType, MetalBufferAccess, MetalConstantType, MetalGroupsType, MetalKernelInfo},
- enum_path_rewrite::rewrite_for_rust,
+use super::{
+ super::{
+ ast::{
+ MetalArgument, MetalArgumentType, MetalBufferAccess, MetalConstantType, MetalGroupsType, MetalKernelInfo,
+ shared_element_byte_size,
+ },
+ enum_path_rewrite::rewrite_for_rust,
+ },
+ variant_path_rewriter::VariantPathRewriter,
};
use crate::common::{enum_paths::EnumPaths, utils::get_generic_name_stream};
pub enum ArgumentEmission {
Buffer(BufferArgument),
Constant(ConstantArgument),
+ Shared(SharedArgument),
IndirectDispatch(IndirectDispatchArgument),
}
@@ -44,14 +51,22 @@ struct ArgumentCondition {
rust_expression: TokenStream,
}
+pub struct SharedArgument {
+ condition: ArgumentCondition,
+ threadgroup_index: usize,
+ length_expression: TokenStream,
+}
+
pub struct IndirectDispatchArgument;
pub fn parse(
kernel: &MetalKernelInfo,
enum_paths: &EnumPaths,
+ variant_path_rewriter: &mut VariantPathRewriter,
) -> Result<Vec<ArgumentEmission>> {
let mut emissions = Vec::new();
let mut next_buffer_index = 0usize;
+ let mut next_threadgroup_index = 0usize;
let mut indirect_dispatch_emitted = false;
for argument in kernel.arguments.iter() {
@@ -67,6 +82,17 @@ pub fn parse(
emissions.push(ArgumentEmission::Constant(constant));
next_buffer_index += 1;
},
+ MetalArgumentType::Shared(dimensions) if argument.condition.is_some() => {
+ let shared = parse_shared_argument(
+ argument,
+ dimensions.as_deref(),
+ next_threadgroup_index,
+ enum_paths,
+ variant_path_rewriter,
+ )?;
+ emissions.push(ArgumentEmission::Shared(shared));
+ next_threadgroup_index += 1;
+ },
MetalArgumentType::Groups(MetalGroupsType::Indirect) if !indirect_dispatch_emitted => {
emissions.push(ArgumentEmission::IndirectDispatch(IndirectDispatchArgument));
indirect_dispatch_emitted = true;
@@ -78,6 +104,33 @@ pub fn parse(
Ok(emissions)
}
+fn parse_shared_argument(
+ argument: &MetalArgument,
+ dimensions: Option<&str>,
+ threadgroup_index: usize,
+ enum_paths: &EnumPaths,
+ variant_path_rewriter: &mut VariantPathRewriter,
+) -> Result<SharedArgument> {
+ let condition = parse_argument_condition(argument, enum_paths)?
+ .context("optional threadgroup argument must carry a condition")?;
+ let dimensions = dimensions.context("optional threadgroup argument must have explicit array dimensions")?;
+ let element_size = shared_element_byte_size(&argument.c_type)?;
+ let dimension_factors = dimensions
+ .split("][")
+ .map(|dimension| variant_path_rewriter.rewrite(dimension))
+ .collect::<Result<Vec<TokenStream>>>()?;
+ let element_count = dimension_factors
+ .into_iter()
+ .reduce(|left, right| quote! { (#left) * (#right) })
+ .expect("split always yields at least one dimension");
+ let length_expression = quote! { (((#element_count) as usize) * #element_size).div_ceil(16) * 16 };
+ Ok(SharedArgument {
+ condition,
+ threadgroup_index,
+ length_expression,
+ })
+}
+
fn parse_buffer_argument(
argument: &MetalArgument,
access: MetalBufferAccess,
@@ -160,15 +213,16 @@ impl ArgumentEmission {
Some(quote! { #field_name: #rust_expression })
}
- pub fn encode_argument_definition(&self) -> TokenStream {
+ pub fn encode_argument_definition(&self) -> Option<TokenStream> {
match self {
- ArgumentEmission::Buffer(buffer) => emit_buffer_argument_definition(buffer),
- ArgumentEmission::Constant(constant) => emit_constant_argument_definition(constant),
- ArgumentEmission::IndirectDispatch(_) => quote! {
+ ArgumentEmission::Buffer(buffer) => Some(emit_buffer_argument_definition(buffer)),
+ ArgumentEmission::Constant(constant) => Some(emit_constant_argument_definition(constant)),
+ ArgumentEmission::Shared(_) => None,
+ ArgumentEmission::IndirectDispatch(_) => Some(quote! {
__dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<
'__dsl_indirect_dispatch_buffer, TDslIndirectDispatchBuffer
>
- },
+ }),
}
}
@@ -179,7 +233,7 @@ impl ArgumentEmission {
Some(quote! { #lifetime })
},
ArgumentEmission::IndirectDispatch(_) => Some(quote! { '__dsl_indirect_dispatch_buffer }),
- ArgumentEmission::Constant(_) => None,
+ ArgumentEmission::Constant(_) | ArgumentEmission::Shared(_) => None,
}
}
@@ -196,7 +250,7 @@ impl ArgumentEmission {
ArgumentEmission::IndirectDispatch(_) => Some(quote! {
let __dsl_indirect_dispatch_buffer = __dsl_indirect_dispatch_buffer.into_parts();
}),
- ArgumentEmission::Constant(_) => None,
+ ArgumentEmission::Constant(_) | ArgumentEmission::Shared(_) => None,
}
}
@@ -211,7 +265,7 @@ impl ArgumentEmission {
flags: crate::backends::common::AccessFlags::compute_read(),
})
}),
- ArgumentEmission::Constant(_) => None,
+ ArgumentEmission::Constant(_) | ArgumentEmission::Shared(_) => None,
}
}
@@ -219,6 +273,7 @@ impl ArgumentEmission {
match self {
ArgumentEmission::Buffer(buffer) => emit_buffer_set(buffer),
ArgumentEmission::Constant(constant) => emit_constant_set(constant),
+ ArgumentEmission::Shared(shared) => emit_shared_set(shared),
ArgumentEmission::IndirectDispatch(_) => quote! {},
}
}
@@ -229,7 +284,7 @@ impl ArgumentEmission {
let buffer_type = get_generic_name_stream(buffer_arg.name.to_string().as_ref());
Some(quote! { #buffer_type })
},
- ArgumentEmission::Constant(_) => None,
+ ArgumentEmission::Constant(_) | ArgumentEmission::Shared(_) => None,
ArgumentEmission::IndirectDispatch(_) => Some(quote! { TDslIndirectDispatchBuffer }),
}
}
@@ -240,7 +295,7 @@ impl ArgumentEmission {
let buffer_type = get_generic_name_stream(buffer_arg.name.to_string().as_ref());
Some(quote! { #buffer_type: crate::backends::common::Buffer<Backend = crate::backends::metal::Metal> })
},
- ArgumentEmission::Constant(_) => None,
+ ArgumentEmission::Constant(_) | ArgumentEmission::Shared(_) => None,
ArgumentEmission::IndirectDispatch(_) => Some(
quote! { TDslIndirectDispatchBuffer: crate::backends::common::Buffer<Backend = crate::backends::metal::Metal> },
),
@@ -251,6 +306,7 @@ impl ArgumentEmission {
match self {
ArgumentEmission::Buffer(buffer) => buffer.condition.as_ref(),
ArgumentEmission::Constant(constant) => constant.condition.as_ref(),
+ ArgumentEmission::Shared(shared) => Some(&shared.condition),
ArgumentEmission::IndirectDispatch(_) => None,
}
}
@@ -356,6 +412,17 @@ fn emit_constant_set(constant: &ConstantArgument) -> TokenStream {
}
}
+fn emit_shared_set(shared: &SharedArgument) -> TokenStream {
+ let field_name = &shared.condition.field_name;
+ let threadgroup_index = shared.threadgroup_index;
+ let length_expression = &shared.length_expression;
+ quote! {
+ if self.#field_name {
+ compute_encoder.set_threadgroup_memory_length(#length_expression, #threadgroup_index);
+ }
+ }
+}
+
pub fn encode_accesses_call(arguments: &[ArgumentEmission]) -> TokenStream {
let access_expressions: Vec<TokenStream> =
arguments.iter().filter_map(|argument| argument.encode_access()).collect();
diff --git a/crates/backend-uzu/build/metal/bindgen/mod.rs b/crates/backend-uzu/build/metal/bindgen/mod.rs
index c770f0ef5..d1a9c42ad 100644
--- a/crates/backend-uzu/build/metal/bindgen/mod.rs
+++ b/crates/backend-uzu/build/metal/bindgen/mod.rs
@@ -23,12 +23,12 @@ pub fn bindgen(
let struct_name = format_ident!("{}MetalKernel", kernel_name);
let variant_binds = variants::parse(kernel)?;
- let argument_emissions = arguments::parse(kernel, enum_paths)?;
+ let mut variant_path_rewriter = VariantPathRewriter::new(&variant_binds, kernel_name);
+ let argument_emissions = arguments::parse(kernel, enum_paths, &mut variant_path_rewriter)?;
let specialize_emission =
specialize::parse(kernel, specialize_indices.get(&kernel.name).copied(), kernel_name, enum_paths)?;
let trait_wiring = trait_wiring::build(kernel, &trait_name, &struct_name);
- let mut variant_path_rewriter = VariantPathRewriter::new(&variant_binds, kernel_name);
let dispatch_emission = dispatch::parse(kernel, &mut variant_path_rewriter)?;
let referenced_parameter_names = variant_path_rewriter.finish();
@@ -37,7 +37,7 @@ pub fn bindgen(
let conditional_buffer_initializers: Vec<TokenStream> =
argument_emissions.iter().filter_map(|argument| argument.struct_initializer()).collect();
let mut encode_argument_definitions: Vec<TokenStream> =
- argument_emissions.iter().map(|argument| argument.encode_argument_definition()).collect();
+ argument_emissions.iter().filter_map(|argument| argument.encode_argument_definition()).collect();
let mut encode_lifetimes: Vec<TokenStream> =
argument_emissions.iter().filter_map(|argument| argument.encode_lifetime()).collect();
let encode_deconstructs: Vec<TokenStream> =
diff --git a/crates/backend-uzu/build/metal/wrapper.rs b/crates/backend-uzu/build/metal/wrapper.rs
index 06c465c12..61539c519 100644
--- a/crates/backend-uzu/build/metal/wrapper.rs
+++ b/crates/backend-uzu/build/metal/wrapper.rs
@@ -4,7 +4,7 @@ use anyhow::bail;
use itertools::Itertools;
use super::{
- ast::{MetalArgument, MetalArgumentType, MetalKernelInfo},
+ ast::{MetalArgument, MetalArgumentType, MetalKernelInfo, shared_element_type},
enum_path_rewrite::is_enum_c_type,
};
use crate::common::{enum_paths::EnumPaths, identifiers::KernelName, mangling::static_mangle};
@@ -205,6 +205,11 @@ fn kernel_wrappers(
})
.unzip();
+ for (threadgroup_index, argument) in optional_shared_arguments(kernel).enumerate() {
+ let element_type = shared_element_type(&argument.c_type);
+ wrapper_arguments.push(format!("{element_type}* {} [[threadgroup({threadgroup_index})]]", argument.name));
+ }
+
if kernel.has_axis() {
if kernel.has_groups() || kernel.has_threads() {
bail!("mixing groups/threads and axis is not supported");
@@ -231,8 +236,8 @@ fn kernel_wrappers(
let wrapper_arguments = wrapper_arguments.join(", ");
let shared_definitions = kernel.arguments.iter().filter_map(|a| match &a.argument_type {
- MetalArgumentType::Shared(dimensions) => {
- let element_type = a.c_type.split(['*', '&', '(']).next().unwrap_or_default().trim_end();
+ MetalArgumentType::Shared(dimensions) if a.condition.is_none() => {
+ let element_type = shared_element_type(&a.c_type);
let dimensions = dimensions.as_deref().map(|d| format!("[{d}]")).unwrap_or_default();
Some(format!("{element_type} {}{dimensions}", a.name))
},
@@ -247,6 +252,9 @@ fn kernel_wrappers(
.arguments
.iter()
.map(|a| match &a.argument_type {
+ MetalArgumentType::Shared(_) if a.condition.is_some() => {
+ format!("({}){}", a.c_type, a.name)
+ },
MetalArgumentType::Buffer(_)
| MetalArgumentType::Constant(_)
| MetalArgumentType::Shared(_)
@@ -295,3 +303,7 @@ fn kernel_wrappers(
Ok(kernel_wrappers.into())
}
+
+fn optional_shared_arguments(kernel: &MetalKernelInfo) -> impl Iterator<Item = &MetalArgument> {
+ kernel.arguments.iter().filter(|argument| argument.is_optional_shared())
+}