A high-performance inference engine for AI models
Cut Metal build time and peak memory (#790)
xhjkl31d937305cce
diff --git a/crates/uzu-engine/build/common/constraints.rs b/crates/uzu-engine/build/common/constraints.rsindex c341f015f..b1b6f23ca 100644--- a/crates/uzu-engine/build/common/constraints.rs+++ b/crates/uzu-engine/build/common/constraints.rs@@ -1,21 +1,24 @@use std::collections::{BTreeMap, BTreeSet};-use rhai::{Dynamic, Engine, Module, Scope};-#[cfg(all(feature = "metal", target_os = "macos"))]-use crate::common::mangling::unqualify_variant;+use rhai::EvalAltResult;+use rhai::{AST, Dynamic, Engine, Module, Scope};-#[cfg(not(all(feature = "metal", target_os = "macos")))]-fn unqualify_variant(value: &str) -> &str {- value.rsplit("::").next().unwrap_or(value)+struct CompiledConstraint {+ source: Box<str>,+ ast: AST,}-pub struct Evaluator {+pub struct Constraints {engine: Engine,+ constraints: Box<[CompiledConstraint]>,}-impl Evaluator {- pub fn new<'a>(variant_values: impl IntoIterator<Item = &'a str>) -> Self {+impl Constraints {+ pub fn new<'a>(+ variant_values: impl IntoIterator<Item = &'a str>,+ constraints: impl IntoIterator<Item = impl AsRef<str>>,+ ) -> Self {let mut engine = Engine::new();let mut namespaces: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();for value in variant_values {@@ -30,31 +33,77 @@ impl Evaluator {}engine.register_static_module(namespace, module.into());}+ let constraints = constraints+ .into_iter()+ .map(|constraint| {+ let source: Box<str> = constraint.as_ref().into();+ let ast = engine+ .compile_expression(&source)+ .unwrap_or_else(|error| panic!("constraint `{source}` failed to compile: {error}"));+ CompiledConstraint {+ source,+ ast,+ }+ })+ .collect();Self {engine,+ constraints,}}- pub fn satisfied<N: AsRef<str>, V: AsRef<str>>(+ fn scope(&self,- bindings: &[(N, V)],- constraints: &[impl AsRef<str>],- ) -> bool {- if constraints.is_empty() {- return true;- }- let mut scope = Scope::with_capacity(bindings.len());+ bindings: impl IntoIterator<Item = (impl AsRef<str>, impl AsRef<str>)>,+ ) -> Scope<'static> {+ let bindings = bindings.into_iter();+ let mut scope = Scope::with_capacity(bindings.size_hint().0);for (name, val) in bindings {- let val = unqualify_variant(val.as_ref());+ let name = name.as_ref();+ let val = val.as_ref();+ let val = val.rsplit_once("::").map_or(val, |(_, val)| val);scope.push(- name.as_ref().to_owned(),+ name.to_owned(),self.engine.eval_expression::<Dynamic>(val).unwrap_or_else(|_| val.to_owned().into()),);}- constraints.iter().all(|c| {+ scope+ }++ /// Test partial bindings, treating a missing variable as a constraint that may still become true.+ #[cfg(all(feature = "metal", target_os = "macos"))]+ pub fn could_satisfy(+ &self,+ bindings: impl IntoIterator<Item = (impl AsRef<str>, impl AsRef<str>)>,+ ) -> bool {+ if self.constraints.is_empty() {+ return true;+ }++ let mut scope = self.scope(bindings);+ self.constraints.iter().all(|constraint| {+ match self.engine.eval_ast_with_scope::<bool>(&mut scope, &constraint.ast) {+ Ok(satisfied) => satisfied,+ Err(error) if matches!(error.as_ref(), EvalAltResult::ErrorVariableNotFound(..)) => true,+ Err(error) => panic!("constraint `{}` failed to evaluate: {error}", constraint.source),+ }+ })+ }++ /// Test complete bindings, requiring every constraint to evaluate successfully to `true`.+ pub fn satisfied(+ &self,+ bindings: impl IntoIterator<Item = (impl AsRef<str>, impl AsRef<str>)>,+ ) -> bool {+ if self.constraints.is_empty() {+ return true;+ }++ let mut scope = self.scope(bindings);+ self.constraints.iter().all(|constraint| {self.engine- .eval_expression_with_scope::<bool>(&mut scope, c.as_ref())- .unwrap_or_else(|e| panic!("constraint `{}` failed to evaluate: {e}", c.as_ref()))+ .eval_ast_with_scope::<bool>(&mut scope, &constraint.ast)+ .unwrap_or_else(|error| panic!("constraint `{}` failed to evaluate: {error}", constraint.source))})}}diff --git a/crates/uzu-engine/build/common/mangling.rs b/crates/uzu-engine/build/common/mangling.rsindex 060cfef17..af2c86465 100644--- a/crates/uzu-engine/build/common/mangling.rs+++ b/crates/uzu-engine/build/common/mangling.rs@@ -6,10 +6,6 @@ use itertools::Itertools;use proc_macro2::TokenStream;use quote::quote;-pub fn unqualify_variant(value: &str) -> &str {- value.rsplit("::").next().unwrap_or(value)-}-pub fn static_mangle(function_name: impl AsRef<str>,variant: impl IntoIterator<Item = impl AsRef<str>>,@@ -21,7 +17,8 @@ pub fn static_mangle(variant.into_iter().map(|v| {- let v = unqualify_variant(v.as_ref()).replace('-', "n");+ let v = v.as_ref();+ let v = v.rsplit_once("::").map_or(v, |(_, v)| v).replace('-', "n");format!("S{}V{}", v.len(), v)}).join("")diff --git a/crates/uzu-engine/build/cpu/compiler.rs b/crates/uzu-engine/build/cpu/compiler.rsindex 34b8e6778..baf425a0f 100644--- a/crates/uzu-engine/build/cpu/compiler.rs+++ b/crates/uzu-engine/build/cpu/compiler.rs@@ -14,6 +14,7 @@ use walkdir::WalkDir;use crate::common::{codegen::write_tokens,compiler::Compiler,+ constraints::Constraints,enum_paths::EnumPaths,gpu_types::GpuTypes,identifiers::{ArgumentName, KernelName, KernelPath},@@ -501,10 +502,12 @@ impl CpuCompiler {.iter().flat_map(|(_, variants)| variants.iter().map(|v| v.to_token_stream().to_string())).collect();- let evaluator = (!function_constraints.is_empty())- .then(|| crate::common::constraints::Evaluator::new(variant_value_strs.iter().map(|s| s.as_str())));- let constraint_strs: Vec<String> =- function_constraints.iter().map(|c| c.to_token_stream().to_string()).collect();+ let constraints = (!function_constraints.is_empty()).then(|| {+ Constraints::new(+ variant_value_strs.iter().map(|s| s.as_str()),+ function_constraints.iter().map(|c| c.to_token_stream().to_string()),+ )+ });let match_arms = function_parameters.iter()@@ -526,15 +529,15 @@ impl CpuCompiler {}).multi_cartesian_product().filter(|variants| {- let Some(evaluator) = &evaluator else {+ let Some(constraints) = &constraints else {return true;};- let bindings: Vec<(String, String)> = function_parameters- .iter()- .enumerate()- .map(|(i, p)| (p.name.to_string(), variants[i].1.to_string()))- .collect();- evaluator.satisfied(&bindings, &constraint_strs)+ constraints.satisfied(+ function_parameters+ .iter()+ .enumerate()+ .map(|(i, p)| (p.name.to_string(), variants[i].1.to_string())),+ )}).map(|variants| {let (match_variants, generic_variants): (Vec<TokenStream>, Vec<TokenStream>) =diff --git a/crates/uzu-engine/build/metal/mod.rs b/crates/uzu-engine/build/metal/mod.rsindex 29a00e23d..49e25d98d 100644--- a/crates/uzu-engine/build/metal/mod.rs+++ b/crates/uzu-engine/build/metal/mod.rs@@ -4,6 +4,7 @@ mod compiler;mod enum_path_rewrite;mod gpu_types;mod toolchain;+mod variant_combinations;mod wrapper;pub use compiler::MetalCompiler;diff --git a/crates/uzu-engine/build/metal/variant_combinations.rs b/crates/uzu-engine/build/metal/variant_combinations.rsnew file mode 100644index 000000000..64d8e907d--- /dev/null+++ b/crates/uzu-engine/build/metal/variant_combinations.rs@@ -0,0 +1,55 @@+/// Return every accepted template-variant combination as a boxed slice of indices.+///+/// `domain_lengths[i]` is the number of variants for template parameter `i`. The returned `Vec`+/// contains one `Box<[usize]>` per combination; entry `i` of that slice is the chosen index in+/// parameter `i`'s variant list. During the search, `could_satisfy` receives `&[Option<usize>]`, with+/// `None` for parameters not yet chosen, plus whether every parameter has been chosen. Returning+/// `false` discards those assignments and every complete combination containing them.+pub fn constrained_combinations(+ domain_lengths: &[usize],+ mut could_satisfy: impl FnMut(&[Option<usize>], bool) -> bool,+) -> Vec<Box<[usize]>> {+ let complete = domain_lengths.is_empty();+ let mut selection = vec![None; domain_lengths.len()];+ let viable = could_satisfy(&selection, complete);+ if !viable {+ return Vec::new();+ }+ if complete {+ return vec![Box::new([])];+ }++ // Binding small domains first exposes format and boolean constraints before+ // descending into larger tile families.+ let mut axis_order: Vec<_> = (0..domain_lengths.len()).collect();+ axis_order.sort_by_key(|&axis| domain_lengths[axis]);++ let mut combinations = Vec::new();+ let first_axis = axis_order[0];+ let mut stack = vec![(first_axis, 0..domain_lengths[first_axis])];+ while let Some((axis, values)) = stack.last_mut() {+ let axis = *axis;+ let Some(value) = values.next() else {+ selection[axis] = None;+ stack.pop();+ continue;+ };++ selection[axis] = Some(value);+ let next_axis = axis_order.get(stack.len()).copied();+ let viable = could_satisfy(&selection, next_axis.is_none());+ if !viable {+ continue;+ }+ let Some(next_axis) = next_axis else {+ combinations.push(selection.iter().map(|value| value.unwrap()).collect());+ continue;+ };++ stack.push((next_axis, 0..domain_lengths[next_axis]));+ }++ // Restoring Cartesian-product order; the caller expects the same wrapper order as exhaustive generation.+ combinations.sort_unstable();+ combinations+}diff --git a/crates/uzu-engine/build/metal/wrapper.rs b/crates/uzu-engine/build/metal/wrapper.rsindex 0f6876a93..cbd11548f 100644--- a/crates/uzu-engine/build/metal/wrapper.rs+++ b/crates/uzu-engine/build/metal/wrapper.rs@@ -6,8 +6,11 @@ use itertools::Itertools;use super::{ast::{MetalArgument, MetalArgumentType, MetalKernelInfo, shared_element_type},enum_path_rewrite::is_enum_c_type,+ variant_combinations::constrained_combinations,+};+use crate::common::{+ constraints::Constraints, enum_paths::EnumPaths, identifiers::KernelName, mangling::static_mangle,};-use crate::common::{enum_paths::EnumPaths, identifiers::KernelName, mangling::static_mangle};pub type SpecializeBaseIndices = HashMap<KernelName, usize>;@@ -144,38 +147,40 @@ fn kernel_wrappers(let bindings = specialize_bindings(kernel, enum_paths);let header = base_index.map(|&base| kernel_header(&bindings, base).into_boxed_str());-- let evaluator = crate::common::constraints::Evaluator::new(- kernel.variants.as_deref().into_iter().flatten().flat_map(|tp| tp.variants.iter().map(|v| v.as_ref())),- );- for type_variant in if let Some(variants) = &kernel.variants {- variants- .iter()- .map(|type_parameter| type_parameter.variants.iter())- .multi_cartesian_product()- .map(|values| {- Some(- variants- .iter()- .map(|tp| tp.name.to_string())- .zip(values.iter().map(|v| v.to_string()))- .collect::<Vec<_>>(),- )- })- .collect()+ let parameters = kernel.variants.as_deref();++ let selections: Vec<Box<[usize]>> = if let Some(parameters) = parameters {+ let constraints = Constraints::new(+ parameters.iter().flat_map(|tp| tp.variants.iter().map(|v| v.as_ref())),+ &kernel.constraints,+ );+ let domain_lengths = parameters.iter().map(|parameter| parameter.variants.len()).collect::<Vec<_>>();+ constrained_combinations(&domain_lengths, |selection, complete| {+ let bindings = parameters.iter().zip(selection).filter_map(|(parameter, value)| {+ value.map(|value| (parameter.name.as_ref(), parameter.variants[value].as_ref()))+ });+ if complete {+ constraints.satisfied(bindings)+ } else {+ constraints.could_satisfy(bindings)+ }+ })} else {- vec![None]- } {- if let Some(ref tv) = type_variant- && !evaluator.satisfied(tv, &kernel.constraints)- {- continue;- }+ vec![Box::new([])]+ };- let (wrapper_name, underlying_name) = if let Some(type_variant) = &type_variant {+ for selection in selections {+ let (wrapper_name, underlying_name) = if let Some(parameters) = parameters {(- static_mangle(kernel.name.as_ref(), type_variant.iter().map(|(_k, v)| v.as_str())),- format!("{}<{}>", kernel.name, type_variant.iter().map(|(_k, v)| v).join(", ")),+ static_mangle(+ kernel.name.as_ref(),+ parameters.iter().zip(&selection).map(|(parameter, &value)| ¶meter.variants[value]),+ ),+ format!(+ "{}<{}>",+ kernel.name,+ parameters.iter().zip(&selection).map(|(parameter, &value)| ¶meter.variants[value]).join(", ")+ ),)} else {(static_mangle(kernel.name.as_ref(), [] as [&str; 0]), kernel.name.to_string())@@ -287,10 +292,14 @@ fn kernel_wrappers(let wrapper_body =shared_definitions.chain(once(underlying_call)).map(|l| format!(" {l};\n")).collect::<Vec<_>>().join("");- let (defs, undefs): (Vec<_>, Vec<_>) = type_variant- .unwrap_or_default()- .iter()- .map(|(k, v)| (format!("\n#define {k} {v}"), format!("#undef {k}\n")))+ let (defs, undefs): (Vec<_>, Vec<_>) = parameters+ .into_iter()+ .flatten()+ .zip(&selection)+ .map(|(parameter, &value)| {+ let value = ¶meter.variants[value];+ (format!("\n#define {} {value}", parameter.name), format!("#undef {}\n", parameter.name))+ }).unzip();let defs = defs.join("");