A high-performance inference engine for AI models
Optional threadgroup (#458)
CCYeh2746ee450393
diff --git a/crates/backend-uzu/build/metal/ast.rs b/crates/backend-uzu/build/metal/ast.rsindex d6db93130..ba24dba9d 100644--- a/crates/backend-uzu/build/metal/ast.rs+++ b/crates/backend-uzu/build/metal/ast.rs@@ -130,6 +130,38 @@ pub struct MetalArgument {pub source: Box<str>,}+pub fn shared_element_type(c_type: &str) -> &str {+ c_type.split(['*', '&', '(']).next().unwrap_or_default().trim_end()+}++pub fn shared_element_byte_size(c_type: &str) -> anyhow::Result<usize> {+ let element_type = shared_element_type(c_type).rsplit(' ').next().unwrap_or_default();++ // Split the lane digit off vectors: "float4" -> "float", 4.+ let (scalar_name, lanes) = match element_type.chars().last() {+ Some(lane_digit @ '2'..='4') => {+ (element_type.strip_suffix(lane_digit).unwrap(), lane_digit.to_digit(10).unwrap() as usize)+ },+ _ => (element_type, 1),+ };++ let scalar_size = match scalar_name {+ "bool" | "char" | "uchar" => 1,+ "short" | "ushort" | "half" | "bfloat" => 2,+ "int" | "uint" | "float" => 4,+ "long" | "ulong" => 8,+ other => bail!("unsupported OPTIONAL threadgroup element type `{other}`"),+ };++ // MSL pads 3-component vectors to 4 lanes, so `float3` is 16 bytes, not 12.+ let padded_lanes = if lanes == 3 {+ 4+ } else {+ lanes+ };+ Ok(scalar_size * padded_lanes)+}+impl MetalArgument {fn scalar_type_to_rust(c_type: &str) -> anyhow::Result<Box<str>> {let mut tokens: Vec<_> = c_type.split_whitespace().collect();@@ -203,6 +235,10 @@ impl MetalArgument {_ => None,}}++ pub fn is_optional_shared(&self) -> bool {+ matches!(self.argument_type, MetalArgumentType::Shared(_)) && self.condition.is_some()+ }}fn parse_argument_annotation(@@ -217,8 +253,11 @@ fn parse_argument_annotation(bail!("dsl.optional takes 1 argument, found {}", annotation.len() - 1);}let argument_type = parse_argument_type(c_type, source, None)?;- if !matches!(argument_type, MetalArgumentType::Buffer(_) | MetalArgumentType::Constant(_)) {- bail!("Only a buffer or a constant can be optional");+ if !matches!(+ argument_type,+ MetalArgumentType::Buffer(_) | MetalArgumentType::Constant(_) | MetalArgumentType::Shared(_)+ ) {+ bail!("Only a buffer, a constant or a threadgroup argument can be optional");}return Ok((argument_type, Some(annotation[1].clone())));}diff --git a/crates/backend-uzu/build/metal/bindgen/arguments.rs b/crates/backend-uzu/build/metal/bindgen/arguments.rsindex 6df125c15..53faf9a4c 100644--- a/crates/backend-uzu/build/metal/bindgen/arguments.rs+++ b/crates/backend-uzu/build/metal/bindgen/arguments.rs@@ -3,15 +3,22 @@ use proc_macro2::{Span, TokenStream};use quote::{format_ident, quote};use syn::{Expr, Ident, Lifetime, Type};-use super::super::{- ast::{MetalArgument, MetalArgumentType, MetalBufferAccess, MetalConstantType, MetalGroupsType, MetalKernelInfo},- enum_path_rewrite::rewrite_for_rust,+use super::{+ super::{+ ast::{+ MetalArgument, MetalArgumentType, MetalBufferAccess, MetalConstantType, MetalGroupsType, MetalKernelInfo,+ shared_element_byte_size,+ },+ enum_path_rewrite::rewrite_for_rust,+ },+ variant_path_rewriter::VariantPathRewriter,};use crate::common::{enum_paths::EnumPaths, utils::get_generic_name_stream};pub enum ArgumentEmission {Buffer(BufferArgument),Constant(ConstantArgument),+ Shared(SharedArgument),IndirectDispatch(IndirectDispatchArgument),}@@ -44,14 +51,22 @@ struct ArgumentCondition {rust_expression: TokenStream,}+pub struct SharedArgument {+ condition: ArgumentCondition,+ threadgroup_index: usize,+ length_expression: TokenStream,+}+pub struct IndirectDispatchArgument;pub fn parse(kernel: &MetalKernelInfo,enum_paths: &EnumPaths,+ variant_path_rewriter: &mut VariantPathRewriter,) -> Result<Vec<ArgumentEmission>> {let mut emissions = Vec::new();let mut next_buffer_index = 0usize;+ let mut next_threadgroup_index = 0usize;let mut indirect_dispatch_emitted = false;for argument in kernel.arguments.iter() {@@ -67,6 +82,17 @@ pub fn parse(emissions.push(ArgumentEmission::Constant(constant));next_buffer_index += 1;},+ MetalArgumentType::Shared(dimensions) if argument.condition.is_some() => {+ let shared = parse_shared_argument(+ argument,+ dimensions.as_deref(),+ next_threadgroup_index,+ enum_paths,+ variant_path_rewriter,+ )?;+ emissions.push(ArgumentEmission::Shared(shared));+ next_threadgroup_index += 1;+ },MetalArgumentType::Groups(MetalGroupsType::Indirect) if !indirect_dispatch_emitted => {emissions.push(ArgumentEmission::IndirectDispatch(IndirectDispatchArgument));indirect_dispatch_emitted = true;@@ -78,6 +104,33 @@ pub fn parse(Ok(emissions)}+fn parse_shared_argument(+ argument: &MetalArgument,+ dimensions: Option<&str>,+ threadgroup_index: usize,+ enum_paths: &EnumPaths,+ variant_path_rewriter: &mut VariantPathRewriter,+) -> Result<SharedArgument> {+ let condition = parse_argument_condition(argument, enum_paths)?+ .context("optional threadgroup argument must carry a condition")?;+ let dimensions = dimensions.context("optional threadgroup argument must have explicit array dimensions")?;+ let element_size = shared_element_byte_size(&argument.c_type)?;+ let dimension_factors = dimensions+ .split("][")+ .map(|dimension| variant_path_rewriter.rewrite(dimension))+ .collect::<Result<Vec<TokenStream>>>()?;+ let element_count = dimension_factors+ .into_iter()+ .reduce(|left, right| quote! { (#left) * (#right) })+ .expect("split always yields at least one dimension");+ let length_expression = quote! { (((#element_count) as usize) * #element_size).div_ceil(16) * 16 };+ Ok(SharedArgument {+ condition,+ threadgroup_index,+ length_expression,+ })+}+fn parse_buffer_argument(argument: &MetalArgument,access: MetalBufferAccess,@@ -160,15 +213,16 @@ impl ArgumentEmission {Some(quote! { #field_name: #rust_expression })}- pub fn encode_argument_definition(&self) -> TokenStream {+ pub fn encode_argument_definition(&self) -> Option<TokenStream> {match self {- ArgumentEmission::Buffer(buffer) => emit_buffer_argument_definition(buffer),- ArgumentEmission::Constant(constant) => emit_constant_argument_definition(constant),- ArgumentEmission::IndirectDispatch(_) => quote! {+ ArgumentEmission::Buffer(buffer) => Some(emit_buffer_argument_definition(buffer)),+ ArgumentEmission::Constant(constant) => Some(emit_constant_argument_definition(constant)),+ ArgumentEmission::Shared(_) => None,+ ArgumentEmission::IndirectDispatch(_) => Some(quote! {__dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<'__dsl_indirect_dispatch_buffer, TDslIndirectDispatchBuffer>- },+ }),}}@@ -179,7 +233,7 @@ impl ArgumentEmission {Some(quote! { #lifetime })},ArgumentEmission::IndirectDispatch(_) => Some(quote! { '__dsl_indirect_dispatch_buffer }),- ArgumentEmission::Constant(_) => None,+ ArgumentEmission::Constant(_) | ArgumentEmission::Shared(_) => None,}}@@ -196,7 +250,7 @@ impl ArgumentEmission {ArgumentEmission::IndirectDispatch(_) => Some(quote! {let __dsl_indirect_dispatch_buffer = __dsl_indirect_dispatch_buffer.into_parts();}),- ArgumentEmission::Constant(_) => None,+ ArgumentEmission::Constant(_) | ArgumentEmission::Shared(_) => None,}}@@ -211,7 +265,7 @@ impl ArgumentEmission {flags: crate::backends::common::AccessFlags::compute_read(),})}),- ArgumentEmission::Constant(_) => None,+ ArgumentEmission::Constant(_) | ArgumentEmission::Shared(_) => None,}}@@ -219,6 +273,7 @@ impl ArgumentEmission {match self {ArgumentEmission::Buffer(buffer) => emit_buffer_set(buffer),ArgumentEmission::Constant(constant) => emit_constant_set(constant),+ ArgumentEmission::Shared(shared) => emit_shared_set(shared),ArgumentEmission::IndirectDispatch(_) => quote! {},}}@@ -229,7 +284,7 @@ impl ArgumentEmission {let buffer_type = get_generic_name_stream(buffer_arg.name.to_string().as_ref());Some(quote! { #buffer_type })},- ArgumentEmission::Constant(_) => None,+ ArgumentEmission::Constant(_) | ArgumentEmission::Shared(_) => None,ArgumentEmission::IndirectDispatch(_) => Some(quote! { TDslIndirectDispatchBuffer }),}}@@ -240,7 +295,7 @@ impl ArgumentEmission {let buffer_type = get_generic_name_stream(buffer_arg.name.to_string().as_ref());Some(quote! { #buffer_type: crate::backends::common::Buffer<Backend = crate::backends::metal::Metal> })},- ArgumentEmission::Constant(_) => None,+ ArgumentEmission::Constant(_) | ArgumentEmission::Shared(_) => None,ArgumentEmission::IndirectDispatch(_) => Some(quote! { TDslIndirectDispatchBuffer: crate::backends::common::Buffer<Backend = crate::backends::metal::Metal> },),@@ -251,6 +306,7 @@ impl ArgumentEmission {match self {ArgumentEmission::Buffer(buffer) => buffer.condition.as_ref(),ArgumentEmission::Constant(constant) => constant.condition.as_ref(),+ ArgumentEmission::Shared(shared) => Some(&shared.condition),ArgumentEmission::IndirectDispatch(_) => None,}}@@ -356,6 +412,17 @@ fn emit_constant_set(constant: &ConstantArgument) -> TokenStream {}}+fn emit_shared_set(shared: &SharedArgument) -> TokenStream {+ let field_name = &shared.condition.field_name;+ let threadgroup_index = shared.threadgroup_index;+ let length_expression = &shared.length_expression;+ quote! {+ if self.#field_name {+ compute_encoder.set_threadgroup_memory_length(#length_expression, #threadgroup_index);+ }+ }+}+pub fn encode_accesses_call(arguments: &[ArgumentEmission]) -> TokenStream {let access_expressions: Vec<TokenStream> =arguments.iter().filter_map(|argument| argument.encode_access()).collect();diff --git a/crates/backend-uzu/build/metal/bindgen/mod.rs b/crates/backend-uzu/build/metal/bindgen/mod.rsindex c770f0ef5..d1a9c42ad 100644--- a/crates/backend-uzu/build/metal/bindgen/mod.rs+++ b/crates/backend-uzu/build/metal/bindgen/mod.rs@@ -23,12 +23,12 @@ pub fn bindgen(let struct_name = format_ident!("{}MetalKernel", kernel_name);let variant_binds = variants::parse(kernel)?;- let argument_emissions = arguments::parse(kernel, enum_paths)?;+ let mut variant_path_rewriter = VariantPathRewriter::new(&variant_binds, kernel_name);+ let argument_emissions = arguments::parse(kernel, enum_paths, &mut variant_path_rewriter)?;let specialize_emission =specialize::parse(kernel, specialize_indices.get(&kernel.name).copied(), kernel_name, enum_paths)?;let trait_wiring = trait_wiring::build(kernel, &trait_name, &struct_name);- let mut variant_path_rewriter = VariantPathRewriter::new(&variant_binds, kernel_name);let dispatch_emission = dispatch::parse(kernel, &mut variant_path_rewriter)?;let referenced_parameter_names = variant_path_rewriter.finish();@@ -37,7 +37,7 @@ pub fn bindgen(let conditional_buffer_initializers: Vec<TokenStream> =argument_emissions.iter().filter_map(|argument| argument.struct_initializer()).collect();let mut encode_argument_definitions: Vec<TokenStream> =- argument_emissions.iter().map(|argument| argument.encode_argument_definition()).collect();+ argument_emissions.iter().filter_map(|argument| argument.encode_argument_definition()).collect();let mut encode_lifetimes: Vec<TokenStream> =argument_emissions.iter().filter_map(|argument| argument.encode_lifetime()).collect();let encode_deconstructs: Vec<TokenStream> =diff --git a/crates/backend-uzu/build/metal/wrapper.rs b/crates/backend-uzu/build/metal/wrapper.rsindex 06c465c12..61539c519 100644--- a/crates/backend-uzu/build/metal/wrapper.rs+++ b/crates/backend-uzu/build/metal/wrapper.rs@@ -4,7 +4,7 @@ use anyhow::bail;use itertools::Itertools;use super::{- ast::{MetalArgument, MetalArgumentType, MetalKernelInfo},+ ast::{MetalArgument, MetalArgumentType, MetalKernelInfo, shared_element_type},enum_path_rewrite::is_enum_c_type,};use crate::common::{enum_paths::EnumPaths, identifiers::KernelName, mangling::static_mangle};@@ -205,6 +205,11 @@ fn kernel_wrappers(}).unzip();+ for (threadgroup_index, argument) in optional_shared_arguments(kernel).enumerate() {+ let element_type = shared_element_type(&argument.c_type);+ wrapper_arguments.push(format!("{element_type}* {} [[threadgroup({threadgroup_index})]]", argument.name));+ }+if kernel.has_axis() {if kernel.has_groups() || kernel.has_threads() {bail!("mixing groups/threads and axis is not supported");@@ -231,8 +236,8 @@ fn kernel_wrappers(let wrapper_arguments = wrapper_arguments.join(", ");let shared_definitions = kernel.arguments.iter().filter_map(|a| match &a.argument_type {- MetalArgumentType::Shared(dimensions) => {- let element_type = a.c_type.split(['*', '&', '(']).next().unwrap_or_default().trim_end();+ MetalArgumentType::Shared(dimensions) if a.condition.is_none() => {+ let element_type = shared_element_type(&a.c_type);let dimensions = dimensions.as_deref().map(|d| format!("[{d}]")).unwrap_or_default();Some(format!("{element_type} {}{dimensions}", a.name))},@@ -247,6 +252,9 @@ fn kernel_wrappers(.arguments.iter().map(|a| match &a.argument_type {+ MetalArgumentType::Shared(_) if a.condition.is_some() => {+ format!("({}){}", a.c_type, a.name)+ },MetalArgumentType::Buffer(_)| MetalArgumentType::Constant(_)| MetalArgumentType::Shared(_)@@ -295,3 +303,7 @@ fn kernel_wrappers(Ok(kernel_wrappers.into())}++fn optional_shared_arguments(kernel: &MetalKernelInfo) -> impl Iterator<Item = &MetalArgument> {+ kernel.arguments.iter().filter(|argument| argument.is_optional_shared())+}