A high-performance inference engine for AI models
Replace async build tasks with Rayon parallelism (#867)
OlshaMBfd0674445f3b
diff --git a/Cargo.lock b/Cargo.lockindex 2667339c0..5e02f9c35 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -7616,7 +7616,6 @@ name = "uzu-engine"version = "0.5.30"dependencies = ["anyhow",- "async-trait","bitflags 2.13.0","blake3","bytemuck",@@ -7624,7 +7623,6 @@ dependencies = ["criterion","depfile","derive_more",- "futures","half","is_close","itertools 0.14.0",@@ -7646,6 +7644,7 @@ dependencies = ["rand 0.10.2","rand_distr","rangemap",+ "rayon","rhai","rstest","schemars 1.2.1",diff --git a/Cargo.toml b/Cargo.tomlindex 6e0b4fb0e..9eb0be1b9 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -97,6 +97,7 @@ schemars = { version = "1.2", features = ["derive", "preserve_order"] }is_close = "0.1"anyhow = "1.0.102"async-trait = "0.1.92"+rayon = "1.12.0"derive_more = { version = "2", features = ["from", "deref", "as_ref", "display"] }walkdir = "2.5"depfile = "0.1"diff --git a/crates/uzu-engine/Cargo.toml b/crates/uzu-engine/Cargo.tomlindex 1fca4904c..3d4b20c7d 100644--- a/crates/uzu-engine/Cargo.toml+++ b/crates/uzu-engine/Cargo.toml@@ -18,6 +18,7 @@ default = ["cpu", "metal", "grammar"]cpu = [] # TODO: move libm heremetal = [+ "dep:rayon","dep:clang-ast","dep:tempfile","dep:serde_json",@@ -31,13 +32,10 @@ grammar = ["dep:xgrammar", "dep:schemars"][build-dependencies]anyhow.workspace = true-async-trait.workspace = truewalkdir.workspace = truedepfile.workspace = truederive_more.workspace = trueserde.workspace = true-futures.workspace = true-tokio = { workspace = true, features = ["full"] }blake3.workspace = truequote.workspace = trueproc-macro2.workspace = true@@ -47,6 +45,7 @@ itertools.workspace = truerhai.workspace = true[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.build-dependencies]+rayon = { workspace = true, optional = true }clang-ast = { workspace = true, optional = true }tempfile = { workspace = true, optional = true }serde_json = { workspace = true, features = ["unbounded_depth"], optional = true }diff --git a/crates/uzu-engine/build/common/compiler.rs b/crates/uzu-engine/build/common/compiler.rsindex f835fbe7e..e0ff1571a 100644--- a/crates/uzu-engine/build/common/compiler.rs+++ b/crates/uzu-engine/build/common/compiler.rs@@ -1,12 +1,9 @@use std::collections::HashMap;-use async_trait::async_trait;-use super::{enum_paths::EnumPaths, gpu_types::GpuTypes, identifiers::KernelPath, kernel::Kernel};-#[async_trait]pub trait Compiler {- async fn build(+ fn build(&self,gpu_types: &GpuTypes,enum_paths: &EnumPaths,diff --git a/crates/uzu-engine/build/cpu/compiler.rs b/crates/uzu-engine/build/cpu/compiler.rsindex baf425a0f..76717d283 100644--- a/crates/uzu-engine/build/cpu/compiler.rs+++ b/crates/uzu-engine/build/cpu/compiler.rs@@ -1,7 +1,6 @@use std::{collections::HashMap, env, fs, path::PathBuf};use anyhow::{Context, bail};-use async_trait::async_trait;use itertools::Itertools;use proc_macro2::{Span, TokenStream};use quote::{ToTokens, format_ident, quote};@@ -662,9 +661,8 @@ impl CpuCompiler {}}-#[async_trait]impl Compiler for CpuCompiler {- async fn build(+ fn build(&self,_gpu_types: &GpuTypes,enum_paths: &EnumPaths,diff --git a/crates/uzu-engine/build/main.rs b/crates/uzu-engine/build/main.rsindex ae2ae724f..85c1b93ae 100644--- a/crates/uzu-engine/build/main.rs+++ b/crates/uzu-engine/build/main.rs@@ -1,7 +1,6 @@use std::{env, fs, path::PathBuf, process::ExitCode};use anyhow::Context;-use futures::future::try_join_all;mod common;use common::{compiler::Compiler, enum_paths::EnumPaths, envs, gpu_types::GpuTypes, traitgen::traitgen_all};@@ -11,8 +10,7 @@ mod cpu;#[cfg(all(feature = "metal", target_os = "macos"))]mod metal;-#[tokio::main(flavor = "current_thread")]-async fn main() -> anyhow::Result<ExitCode> {+fn main() -> anyhow::Result<ExitCode> {println!("cargo::rerun-if-changed=build");if envs::build_always() {@@ -64,7 +62,7 @@ async fn main() -> anyhow::Result<ExitCode> {#[cfg(all(feature = "metal", target_os = "macos"))]if backend_metal {- compilers.push(Box::new(metal::MetalCompiler::new().await?));+ compilers.push(Box::new(metal::MetalCompiler::new()?));}if compilers.is_empty() {@@ -72,7 +70,8 @@ async fn main() -> anyhow::Result<ExitCode> {return Ok(ExitCode::FAILURE);}- let backends_kernels = try_join_all(compilers.iter().map(|c| c.build(&gpu_types, &enum_paths))).await?;+ let backends_kernels =+ compilers.iter().map(|c| c.build(&gpu_types, &enum_paths)).collect::<anyhow::Result<Vec<_>>>()?;debug_log!("backend build end");diff --git a/crates/uzu-engine/build/metal/compiler.rs b/crates/uzu-engine/build/metal/compiler.rsindex 9f108a57f..85f17ef1d 100644--- a/crates/uzu-engine/build/metal/compiler.rs+++ b/crates/uzu-engine/build/metal/compiler.rs@@ -5,10 +5,9 @@ use std::{};use anyhow::Context;-use async_trait::async_trait;-use futures::{StreamExt, TryStreamExt, future::try_join_all, stream};use itertools::{Itertools, izip};use quote::{format_ident, quote};+use rayon::iter::{IntoParallelIterator, ParallelIterator};use serde::{Deserialize, Serialize};use walkdir::WalkDir;use xxhash_rust::xxh3::xxh3_64;@@ -89,7 +88,7 @@ pub struct MetalCompiler {}impl MetalCompiler {- pub async fn new() -> anyhow::Result<Self> {+ pub fn new() -> anyhow::Result<Self> {let source_directory = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?).join("src/backends/metal/kernel");println!("cargo::rerun-if-changed={}", source_directory.display());@@ -109,9 +108,8 @@ impl MetalCompiler {_ => true, // treat everything else (3,s,z) as release build where size matters};- let toolchain = MetalToolchain::new(modules_cache_path, gpu_types_directory.clone())- .await- .context("cannot create toolchain")?;+ let toolchain =+ MetalToolchain::new(modules_cache_path, gpu_types_directory.clone()).context("cannot create toolchain")?;let cache_key = {let build_system_hash = caching::build_system_hash().context("cannot get build system hash")?;@@ -143,7 +141,7 @@ impl MetalCompiler {}}- async fn compile(+ fn compile(&self,source_path: PathBuf,enum_paths: &EnumPaths,@@ -188,7 +186,6 @@ impl MetalCompiler {let (metal_kernel_infos, dependencies) = self.toolchain.analyze(&source_path)- .await.with_context(|| format!("cannot analyze {source_path_relative_str}"))?;let kernel_infos: Vec<MetalKernelInfo> = metal_kernel_infos.collect();@@ -232,29 +229,24 @@ impl MetalCompiler {}).collect();- let compile_outputs = try_join_all(izip!(&footers, &metallib_files, &metallib_maybe_compressed_files).map(- |(footer, metallib_file, compressed_file)| {- let source_path = &source_path;- async move {- let warnings = self.toolchain.compile(source_path, footer, metallib_file).await?;-- if self.metallib_compressed {- let metallib_file = metallib_file.clone();- let compressed_file = compressed_file.clone();- tokio::task::spawn_blocking(move || -> anyhow::Result<()> {- let metallib_source = fs::read(metallib_file)?;- fs::write(compressed_file, compression::compress(metallib_source.as_slice()))?;- Ok(())- })- .await??;- }-- anyhow::Ok(warnings)+ let compile_outputs: Vec<_> = izip!(&footers, &metallib_files, &metallib_maybe_compressed_files)+ .collect::<Vec<_>>()+ .into_par_iter()+ .map(|(footer, metallib_file, compressed_file)| {+ let footer_path = metallib_file.with_extension("metal");+ fs::write(&footer_path, footer)+ .with_context(|| format!("cannot write generated Metal source {}", footer_path.display()))?;+ let warnings = self.toolchain.compile(&source_path, &footer_path, metallib_file)?;++ if self.metallib_compressed {+ let metallib_source = fs::read(metallib_file)?;+ fs::write(compressed_file, compression::compress(&metallib_source))?;}- },- ))- .await- .with_context(|| format!("cannot compile {source_path_relative_str}"))?;++ anyhow::Ok(warnings)+ })+ .collect::<anyhow::Result<_>>()+ .with_context(|| format!("cannot compile {source_path_relative_str}"))?;for warnings in compile_outputs.into_iter().flatten() {for line in warnings.lines() {@@ -319,14 +311,13 @@ impl MetalCompiler {}}-#[async_trait]impl Compiler for MetalCompiler {- async fn build(+ fn build(&self,gpu_types: &GpuTypes,enum_paths: &EnumPaths,) -> anyhow::Result<HashMap<KernelPath, Box<[Kernel]>>> {- gpu_type_gen(&self.gpu_types_directory, gpu_types).await.context("cannot generate shared gpu types")?;+ gpu_type_gen(&self.gpu_types_directory, gpu_types).context("cannot generate shared gpu types")?;let metal_sources: Vec<PathBuf> = WalkDir::new(&self.source_directory).into_iter()@@ -335,17 +326,12 @@ impl Compiler for MetalCompiler {.map(|e| e.into_path()).collect();- let num_concurrent_compiles = std::thread::available_parallelism().map(|x| x.get()).unwrap_or(4) * 2;-- let compiled: Vec<(KernelPath, Box<[Kernel]>, bool)> = stream::iter(metal_sources)- .map(|path| async move {- self.compile(path.clone(), enum_paths)- .await- .with_context(|| format!("cannot compile {}", path.display()))+ let compiled: Vec<(KernelPath, Box<[Kernel]>, bool)> = metal_sources+ .into_par_iter()+ .map(|path| {+ self.compile(path.clone(), enum_paths).with_context(|| format!("cannot compile {}", path.display()))})- .buffer_unordered(num_concurrent_compiles)- .try_collect()- .await?;+ .collect::<anyhow::Result<_>>()?;let mut kernels_bindgen = compiled.iter()diff --git a/crates/uzu-engine/build/metal/gpu_types/mod.rs b/crates/uzu-engine/build/metal/gpu_types/mod.rsindex f36314945..2e1821686 100644--- a/crates/uzu-engine/build/metal/gpu_types/mod.rs+++ b/crates/uzu-engine/build/metal/gpu_types/mod.rs@@ -1,4 +1,4 @@-use std::path::Path;+use std::{fs, path::Path};use anyhow::Context;use itertools::Itertools;@@ -31,20 +31,19 @@ fn rust_to_metal(ty: &str) -> anyhow::Result<&'static str> {}}-pub async fn gpu_type_gen(+pub fn gpu_type_gen(gpu_types_dir: &Path,gpu_types: &GpuTypes,) -> anyhow::Result<()> {for gpu_type_file in &gpu_types.files {gpu_type_gen_file(&gpu_types_dir.join(gpu_type_file.name.as_ref()).with_extension("h"), gpu_type_file)- .await.with_context(|| format!("Cannot generate gpu types for {}", gpu_type_file.name.as_ref()))?;}Ok(())}-async fn gpu_type_gen_file(+fn gpu_type_gen_file(file_path: &Path,gpu_types_file: &GpuTypeFile,) -> anyhow::Result<()> {@@ -68,8 +67,8 @@ async fn gpu_type_gen_file(let new_contents = format!(include_str!("template.ht"), module_name = module_name, generated = generated);// Avoid advancing mtime if the contents are the same- if !tokio::fs::read(&file_path).await.is_ok_and(|old_contents| old_contents == new_contents.as_bytes()) {- tokio::fs::write(&file_path, new_contents).await.context("cannot write output")?;+ if !fs::read(file_path).is_ok_and(|old_contents| old_contents == new_contents.as_bytes()) {+ fs::write(file_path, new_contents).context("cannot write output")?;}Ok(())diff --git a/crates/uzu-engine/build/metal/toolchain.rs b/crates/uzu-engine/build/metal/toolchain.rsindex 680575077..13d608d0c 100644--- a/crates/uzu-engine/build/metal/toolchain.rs+++ b/crates/uzu-engine/build/metal/toolchain.rs@@ -3,13 +3,12 @@ use std::{ffi::OsString,fs,path::{Path, PathBuf},- process::Stdio,+ process::{Command, Stdio},};use anyhow::{Context, bail};use serde::Deserialize;use tempfile::NamedTempFile;-use tokio::{io::AsyncWriteExt, process::Command};use super::ast::{MetalAstKind, MetalAstNode, MetalKernelInfo};@@ -100,7 +99,7 @@ pub struct MetalToolchain {}impl MetalToolchain {- pub async fn new(+ pub fn new(modules_cache_path: PathBuf,include_dir: PathBuf,) -> anyhow::Result<Self> {@@ -143,7 +142,6 @@ impl MetalToolchain {.args(["-sdk", sdk.to_str()]).args(args).output()- .await.with_context(|| format!("cannot execute xcrun {}", args.join(" ")))?;if !output.status.success() {bail!("xcrun {} failed: {}", args.join(" "), String::from_utf8_lossy(&output.stderr));@@ -188,7 +186,7 @@ impl MetalToolchain {cmd.arg("-I").arg(&self.include_dir);}- pub async fn analyze(+ pub fn analyze(&self,path: impl AsRef<Path>,) -> anyhow::Result<(impl Iterator<Item = MetalKernelInfo>, impl Iterator<Item = Box<str>>)> {@@ -213,7 +211,7 @@ impl MetalToolchain {.stdout(Stdio::piped()).stderr(Stdio::piped());- let analyze_output = cmd.output().await.context("cannot execute metal analyzer")?;+ let analyze_output = cmd.output().context("cannot execute metal analyzer")?;if !analyze_output.status.success() {bail!("metal analyzer failed: {}", String::from_utf8_lossy(&analyze_output.stderr));@@ -221,26 +219,23 @@ impl MetalToolchain {let source_contents = fs::read_to_string(path).context("cannot read source file")?;- let kernel_infos = tokio::task::spawn_blocking(move || {- let mut deserializer = serde_json::Deserializer::from_slice(&analyze_output.stdout);- deserializer.disable_recursion_limit();- let ast_root = MetalAstNode::deserialize(&mut deserializer).context("cannot deserialize ast dump")?;+ let mut deserializer = serde_json::Deserializer::from_slice(&analyze_output.stdout);+ deserializer.disable_recursion_limit();+ let ast_root = MetalAstNode::deserialize(&mut deserializer).context("cannot deserialize ast dump")?;- if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {- bail!(- "unexpected kind of ast root: MetalAstKind::TranslationUnitDecl expected, but {:?} found",- ast_root.kind- );- }+ if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {+ bail!(+ "unexpected kind of ast root: MetalAstKind::TranslationUnitDecl expected, but {:?} found",+ ast_root.kind+ );+ }- ast_root- .inner- .into_iter()- .filter_map(|node| MetalKernelInfo::from_ast_node_and_source(node, &source_contents).transpose())- .collect::<anyhow::Result<Vec<_>>>()- .context("cannot parse kernel infos from AST")- })- .await??;+ let kernel_infos = ast_root+ .inner+ .into_iter()+ .filter_map(|node| MetalKernelInfo::from_ast_node_and_source(node, &source_contents).transpose())+ .collect::<anyhow::Result<Vec<_>>>()+ .context("cannot parse kernel infos from AST")?;let depfile_contents = fs::read_to_string(depfile_path.path()).context("cannot read depfile")?;@@ -255,10 +250,10 @@ impl MetalToolchain {Ok((kernel_infos.into_iter(), dependencies))}- pub async fn compile(+ pub fn compile(&self,source: impl AsRef<Path>,- footer: impl AsRef<str>,+ footer_path: impl AsRef<Path>,output: impl AsRef<Path>,) -> anyhow::Result<Option<Box<str>>> {let mut cmd = self.xcrun();@@ -272,23 +267,12 @@ impl MetalToolchain {cmd.arg("-include").arg(source.as_ref())- .arg("-")+ .arg(footer_path.as_ref()).arg("-o").arg(output.as_ref())- .stdin(Stdio::piped()).stderr(Stdio::piped());- let mut compile_child = cmd.spawn().context("cannot execute metal compiler")?;-- compile_child- .stdin- .as_mut()- .context("metal compiler stdin missing")?- .write_all(footer.as_ref().as_bytes())- .await- .context("cannot write to metal compiler stdin")?;-- let compile_output = compile_child.wait_with_output().await.context("cannot wait on metal compiler")?;+ let compile_output = cmd.output().context("cannot execute metal compiler")?;let stderr = String::from_utf8_lossy(&compile_output.stderr).into_owned().into_boxed_str();