Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Cut Metal build time and peak memory (#790)

xhjkl31d937305cce
diff --git a/crates/uzu-engine/build/common/constraints.rs b/crates/uzu-engine/build/common/constraints.rs
index c341f015f..b1b6f23ca 100644
--- a/crates/uzu-engine/build/common/constraints.rs
+++ b/crates/uzu-engine/build/common/constraints.rs
@@ -1,21 +1,24 @@
use std::collections::{BTreeMap, BTreeSet};
-use rhai::{Dynamic, Engine, Module, Scope};
-
#[cfg(all(feature = "metal", target_os = "macos"))]
-use crate::common::mangling::unqualify_variant;
+use rhai::EvalAltResult;
+use rhai::{AST, Dynamic, Engine, Module, Scope};
-#[cfg(not(all(feature = "metal", target_os = "macos")))]
-fn unqualify_variant(value: &str) -> &str {
- value.rsplit("::").next().unwrap_or(value)
+struct CompiledConstraint {
+ source: Box<str>,
+ ast: AST,
}
-pub struct Evaluator {
+pub struct Constraints {
engine: Engine,
+ constraints: Box<[CompiledConstraint]>,
}
-impl Evaluator {
- pub fn new<'a>(variant_values: impl IntoIterator<Item = &'a str>) -> Self {
+impl Constraints {
+ pub fn new<'a>(
+ variant_values: impl IntoIterator<Item = &'a str>,
+ constraints: impl IntoIterator<Item = impl AsRef<str>>,
+ ) -> Self {
let mut engine = Engine::new();
let mut namespaces: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();
for value in variant_values {
@@ -30,31 +33,77 @@ impl Evaluator {
}
engine.register_static_module(namespace, module.into());
}
+ let constraints = constraints
+ .into_iter()
+ .map(|constraint| {
+ let source: Box<str> = constraint.as_ref().into();
+ let ast = engine
+ .compile_expression(&source)
+ .unwrap_or_else(|error| panic!("constraint `{source}` failed to compile: {error}"));
+ CompiledConstraint {
+ source,
+ ast,
+ }
+ })
+ .collect();
Self {
engine,
+ constraints,
}
}
- pub fn satisfied<N: AsRef<str>, V: AsRef<str>>(
+ fn scope(
&self,
- bindings: &[(N, V)],
- constraints: &[impl AsRef<str>],
- ) -> bool {
- if constraints.is_empty() {
- return true;
- }
- let mut scope = Scope::with_capacity(bindings.len());
+ bindings: impl IntoIterator<Item = (impl AsRef<str>, impl AsRef<str>)>,
+ ) -> Scope<'static> {
+ let bindings = bindings.into_iter();
+ let mut scope = Scope::with_capacity(bindings.size_hint().0);
for (name, val) in bindings {
- let val = unqualify_variant(val.as_ref());
+ let name = name.as_ref();
+ let val = val.as_ref();
+ let val = val.rsplit_once("::").map_or(val, |(_, val)| val);
scope.push(
- name.as_ref().to_owned(),
+ name.to_owned(),
self.engine.eval_expression::<Dynamic>(val).unwrap_or_else(|_| val.to_owned().into()),
);
}
- constraints.iter().all(|c| {
+ scope
+ }
+
+ /// Test partial bindings, treating a missing variable as a constraint that may still become true.
+ #[cfg(all(feature = "metal", target_os = "macos"))]
+ pub fn could_satisfy(
+ &self,
+ bindings: impl IntoIterator<Item = (impl AsRef<str>, impl AsRef<str>)>,
+ ) -> bool {
+ if self.constraints.is_empty() {
+ return true;
+ }
+
+ let mut scope = self.scope(bindings);
+ self.constraints.iter().all(|constraint| {
+ match self.engine.eval_ast_with_scope::<bool>(&mut scope, &constraint.ast) {
+ Ok(satisfied) => satisfied,
+ Err(error) if matches!(error.as_ref(), EvalAltResult::ErrorVariableNotFound(..)) => true,
+ Err(error) => panic!("constraint `{}` failed to evaluate: {error}", constraint.source),
+ }
+ })
+ }
+
+ /// Test complete bindings, requiring every constraint to evaluate successfully to `true`.
+ pub fn satisfied(
+ &self,
+ bindings: impl IntoIterator<Item = (impl AsRef<str>, impl AsRef<str>)>,
+ ) -> bool {
+ if self.constraints.is_empty() {
+ return true;
+ }
+
+ let mut scope = self.scope(bindings);
+ self.constraints.iter().all(|constraint| {
self.engine
- .eval_expression_with_scope::<bool>(&mut scope, c.as_ref())
- .unwrap_or_else(|e| panic!("constraint `{}` failed to evaluate: {e}", c.as_ref()))
+ .eval_ast_with_scope::<bool>(&mut scope, &constraint.ast)
+ .unwrap_or_else(|error| panic!("constraint `{}` failed to evaluate: {error}", constraint.source))
})
}
}
diff --git a/crates/uzu-engine/build/common/mangling.rs b/crates/uzu-engine/build/common/mangling.rs
index 060cfef17..af2c86465 100644
--- a/crates/uzu-engine/build/common/mangling.rs
+++ b/crates/uzu-engine/build/common/mangling.rs
@@ -6,10 +6,6 @@ use itertools::Itertools;
use proc_macro2::TokenStream;
use quote::quote;
-pub fn unqualify_variant(value: &str) -> &str {
- value.rsplit("::").next().unwrap_or(value)
-}
-
pub fn static_mangle(
function_name: impl AsRef<str>,
variant: impl IntoIterator<Item = impl AsRef<str>>,
@@ -21,7 +17,8 @@ pub fn static_mangle(
variant
.into_iter()
.map(|v| {
- let v = unqualify_variant(v.as_ref()).replace('-', "n");
+ let v = v.as_ref();
+ let v = v.rsplit_once("::").map_or(v, |(_, v)| v).replace('-', "n");
format!("S{}V{}", v.len(), v)
})
.join("")
diff --git a/crates/uzu-engine/build/cpu/compiler.rs b/crates/uzu-engine/build/cpu/compiler.rs
index 34b8e6778..baf425a0f 100644
--- a/crates/uzu-engine/build/cpu/compiler.rs
+++ b/crates/uzu-engine/build/cpu/compiler.rs
@@ -14,6 +14,7 @@ use walkdir::WalkDir;
use crate::common::{
codegen::write_tokens,
compiler::Compiler,
+ constraints::Constraints,
enum_paths::EnumPaths,
gpu_types::GpuTypes,
identifiers::{ArgumentName, KernelName, KernelPath},
@@ -501,10 +502,12 @@ impl CpuCompiler {
.iter()
.flat_map(|(_, variants)| variants.iter().map(|v| v.to_token_stream().to_string()))
.collect();
- let evaluator = (!function_constraints.is_empty())
- .then(|| crate::common::constraints::Evaluator::new(variant_value_strs.iter().map(|s| s.as_str())));
- let constraint_strs: Vec<String> =
- function_constraints.iter().map(|c| c.to_token_stream().to_string()).collect();
+ let constraints = (!function_constraints.is_empty()).then(|| {
+ Constraints::new(
+ variant_value_strs.iter().map(|s| s.as_str()),
+ function_constraints.iter().map(|c| c.to_token_stream().to_string()),
+ )
+ });
let match_arms = function_parameters
.iter()
@@ -526,15 +529,15 @@ impl CpuCompiler {
})
.multi_cartesian_product()
.filter(|variants| {
- let Some(evaluator) = &evaluator else {
+ let Some(constraints) = &constraints else {
return true;
};
- let bindings: Vec<(String, String)> = function_parameters
- .iter()
- .enumerate()
- .map(|(i, p)| (p.name.to_string(), variants[i].1.to_string()))
- .collect();
- evaluator.satisfied(&bindings, &constraint_strs)
+ constraints.satisfied(
+ function_parameters
+ .iter()
+ .enumerate()
+ .map(|(i, p)| (p.name.to_string(), variants[i].1.to_string())),
+ )
})
.map(|variants| {
let (match_variants, generic_variants): (Vec<TokenStream>, Vec<TokenStream>) =
diff --git a/crates/uzu-engine/build/metal/mod.rs b/crates/uzu-engine/build/metal/mod.rs
index 29a00e23d..49e25d98d 100644
--- a/crates/uzu-engine/build/metal/mod.rs
+++ b/crates/uzu-engine/build/metal/mod.rs
@@ -4,6 +4,7 @@ mod compiler;
mod enum_path_rewrite;
mod gpu_types;
mod toolchain;
+mod variant_combinations;
mod wrapper;
pub use compiler::MetalCompiler;
diff --git a/crates/uzu-engine/build/metal/variant_combinations.rs b/crates/uzu-engine/build/metal/variant_combinations.rs
new file mode 100644
index 000000000..64d8e907d
--- /dev/null
+++ b/crates/uzu-engine/build/metal/variant_combinations.rs
@@ -0,0 +1,55 @@
+/// Return every accepted template-variant combination as a boxed slice of indices.
+///
+/// `domain_lengths[i]` is the number of variants for template parameter `i`. The returned `Vec`
+/// contains one `Box<[usize]>` per combination; entry `i` of that slice is the chosen index in
+/// parameter `i`'s variant list. During the search, `could_satisfy` receives `&[Option<usize>]`, with
+/// `None` for parameters not yet chosen, plus whether every parameter has been chosen. Returning
+/// `false` discards those assignments and every complete combination containing them.
+pub fn constrained_combinations(
+ domain_lengths: &[usize],
+ mut could_satisfy: impl FnMut(&[Option<usize>], bool) -> bool,
+) -> Vec<Box<[usize]>> {
+ let complete = domain_lengths.is_empty();
+ let mut selection = vec![None; domain_lengths.len()];
+ let viable = could_satisfy(&selection, complete);
+ if !viable {
+ return Vec::new();
+ }
+ if complete {
+ return vec![Box::new([])];
+ }
+
+ // Binding small domains first exposes format and boolean constraints before
+ // descending into larger tile families.
+ let mut axis_order: Vec<_> = (0..domain_lengths.len()).collect();
+ axis_order.sort_by_key(|&axis| domain_lengths[axis]);
+
+ let mut combinations = Vec::new();
+ let first_axis = axis_order[0];
+ let mut stack = vec![(first_axis, 0..domain_lengths[first_axis])];
+ while let Some((axis, values)) = stack.last_mut() {
+ let axis = *axis;
+ let Some(value) = values.next() else {
+ selection[axis] = None;
+ stack.pop();
+ continue;
+ };
+
+ selection[axis] = Some(value);
+ let next_axis = axis_order.get(stack.len()).copied();
+ let viable = could_satisfy(&selection, next_axis.is_none());
+ if !viable {
+ continue;
+ }
+ let Some(next_axis) = next_axis else {
+ combinations.push(selection.iter().map(|value| value.unwrap()).collect());
+ continue;
+ };
+
+ stack.push((next_axis, 0..domain_lengths[next_axis]));
+ }
+
+ // Restoring Cartesian-product order; the caller expects the same wrapper order as exhaustive generation.
+ combinations.sort_unstable();
+ combinations
+}
diff --git a/crates/uzu-engine/build/metal/wrapper.rs b/crates/uzu-engine/build/metal/wrapper.rs
index 0f6876a93..cbd11548f 100644
--- a/crates/uzu-engine/build/metal/wrapper.rs
+++ b/crates/uzu-engine/build/metal/wrapper.rs
@@ -6,8 +6,11 @@ use itertools::Itertools;
use super::{
ast::{MetalArgument, MetalArgumentType, MetalKernelInfo, shared_element_type},
enum_path_rewrite::is_enum_c_type,
+ variant_combinations::constrained_combinations,
+};
+use crate::common::{
+ constraints::Constraints, enum_paths::EnumPaths, identifiers::KernelName, mangling::static_mangle,
};
-use crate::common::{enum_paths::EnumPaths, identifiers::KernelName, mangling::static_mangle};
pub type SpecializeBaseIndices = HashMap<KernelName, usize>;
@@ -144,38 +147,40 @@ fn kernel_wrappers(
let bindings = specialize_bindings(kernel, enum_paths);
let header = base_index.map(|&base| kernel_header(&bindings, base).into_boxed_str());
-
- let evaluator = crate::common::constraints::Evaluator::new(
- kernel.variants.as_deref().into_iter().flatten().flat_map(|tp| tp.variants.iter().map(|v| v.as_ref())),
- );
- for type_variant in if let Some(variants) = &kernel.variants {
- variants
- .iter()
- .map(|type_parameter| type_parameter.variants.iter())
- .multi_cartesian_product()
- .map(|values| {
- Some(
- variants
- .iter()
- .map(|tp| tp.name.to_string())
- .zip(values.iter().map(|v| v.to_string()))
- .collect::<Vec<_>>(),
- )
- })
- .collect()
+ let parameters = kernel.variants.as_deref();
+
+ let selections: Vec<Box<[usize]>> = if let Some(parameters) = parameters {
+ let constraints = Constraints::new(
+ parameters.iter().flat_map(|tp| tp.variants.iter().map(|v| v.as_ref())),
+ &kernel.constraints,
+ );
+ let domain_lengths = parameters.iter().map(|parameter| parameter.variants.len()).collect::<Vec<_>>();
+ constrained_combinations(&domain_lengths, |selection, complete| {
+ let bindings = parameters.iter().zip(selection).filter_map(|(parameter, value)| {
+ value.map(|value| (parameter.name.as_ref(), parameter.variants[value].as_ref()))
+ });
+ if complete {
+ constraints.satisfied(bindings)
+ } else {
+ constraints.could_satisfy(bindings)
+ }
+ })
} else {
- vec![None]
- } {
- if let Some(ref tv) = type_variant
- && !evaluator.satisfied(tv, &kernel.constraints)
- {
- continue;
- }
+ vec![Box::new([])]
+ };
- let (wrapper_name, underlying_name) = if let Some(type_variant) = &type_variant {
+ for selection in selections {
+ let (wrapper_name, underlying_name) = if let Some(parameters) = parameters {
(
- static_mangle(kernel.name.as_ref(), type_variant.iter().map(|(_k, v)| v.as_str())),
- format!("{}<{}>", kernel.name, type_variant.iter().map(|(_k, v)| v).join(", ")),
+ static_mangle(
+ kernel.name.as_ref(),
+ parameters.iter().zip(&selection).map(|(parameter, &value)| &parameter.variants[value]),
+ ),
+ format!(
+ "{}<{}>",
+ kernel.name,
+ parameters.iter().zip(&selection).map(|(parameter, &value)| &parameter.variants[value]).join(", ")
+ ),
)
} else {
(static_mangle(kernel.name.as_ref(), [] as [&str; 0]), kernel.name.to_string())
@@ -287,10 +292,14 @@ fn kernel_wrappers(
let wrapper_body =
shared_definitions.chain(once(underlying_call)).map(|l| format!(" {l};\n")).collect::<Vec<_>>().join("");
- let (defs, undefs): (Vec<_>, Vec<_>) = type_variant
- .unwrap_or_default()
- .iter()
- .map(|(k, v)| (format!("\n#define {k} {v}"), format!("#undef {k}\n")))
+ let (defs, undefs): (Vec<_>, Vec<_>) = parameters
+ .into_iter()
+ .flatten()
+ .zip(&selection)
+ .map(|(parameter, &value)| {
+ let value = &parameter.variants[value];
+ (format!("\n#define {} {value}", parameter.name), format!("#undef {}\n", parameter.name))
+ })
.unzip();
let defs = defs.join("");