Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Replace async build tasks with Rayon parallelism (#867)

OlshaMBfd0674445f3b
diff --git a/Cargo.lock b/Cargo.lock
index 2667339c0..5e02f9c35 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7616,7 +7616,6 @@ name = "uzu-engine"
version = "0.5.30"
dependencies = [
"anyhow",
- "async-trait",
"bitflags 2.13.0",
"blake3",
"bytemuck",
@@ -7624,7 +7623,6 @@ dependencies = [
"criterion",
"depfile",
"derive_more",
- "futures",
"half",
"is_close",
"itertools 0.14.0",
@@ -7646,6 +7644,7 @@ dependencies = [
"rand 0.10.2",
"rand_distr",
"rangemap",
+ "rayon",
"rhai",
"rstest",
"schemars 1.2.1",
diff --git a/Cargo.toml b/Cargo.toml
index 6e0b4fb0e..9eb0be1b9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -97,6 +97,7 @@ schemars = { version = "1.2", features = ["derive", "preserve_order"] }
is_close = "0.1"
anyhow = "1.0.102"
async-trait = "0.1.92"
+rayon = "1.12.0"
derive_more = { version = "2", features = ["from", "deref", "as_ref", "display"] }
walkdir = "2.5"
depfile = "0.1"
diff --git a/crates/uzu-engine/Cargo.toml b/crates/uzu-engine/Cargo.toml
index 1fca4904c..3d4b20c7d 100644
--- a/crates/uzu-engine/Cargo.toml
+++ b/crates/uzu-engine/Cargo.toml
@@ -18,6 +18,7 @@ default = ["cpu", "metal", "grammar"]
cpu = [] # TODO: move libm here
metal = [
+ "dep:rayon",
"dep:clang-ast",
"dep:tempfile",
"dep:serde_json",
@@ -31,13 +32,10 @@ grammar = ["dep:xgrammar", "dep:schemars"]
[build-dependencies]
anyhow.workspace = true
-async-trait.workspace = true
walkdir.workspace = true
depfile.workspace = true
derive_more.workspace = true
serde.workspace = true
-futures.workspace = true
-tokio = { workspace = true, features = ["full"] }
blake3.workspace = true
quote.workspace = true
proc-macro2.workspace = true
@@ -47,6 +45,7 @@ itertools.workspace = true
rhai.workspace = true
[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.build-dependencies]
+rayon = { workspace = true, optional = true }
clang-ast = { workspace = true, optional = true }
tempfile = { workspace = true, optional = true }
serde_json = { workspace = true, features = ["unbounded_depth"], optional = true }
diff --git a/crates/uzu-engine/build/common/compiler.rs b/crates/uzu-engine/build/common/compiler.rs
index f835fbe7e..e0ff1571a 100644
--- a/crates/uzu-engine/build/common/compiler.rs
+++ b/crates/uzu-engine/build/common/compiler.rs
@@ -1,12 +1,9 @@
use std::collections::HashMap;
-use async_trait::async_trait;
-
use super::{enum_paths::EnumPaths, gpu_types::GpuTypes, identifiers::KernelPath, kernel::Kernel};
-#[async_trait]
pub trait Compiler {
- async fn build(
+ fn build(
&self,
gpu_types: &GpuTypes,
enum_paths: &EnumPaths,
diff --git a/crates/uzu-engine/build/cpu/compiler.rs b/crates/uzu-engine/build/cpu/compiler.rs
index baf425a0f..76717d283 100644
--- a/crates/uzu-engine/build/cpu/compiler.rs
+++ b/crates/uzu-engine/build/cpu/compiler.rs
@@ -1,7 +1,6 @@
use std::{collections::HashMap, env, fs, path::PathBuf};
use anyhow::{Context, bail};
-use async_trait::async_trait;
use itertools::Itertools;
use proc_macro2::{Span, TokenStream};
use quote::{ToTokens, format_ident, quote};
@@ -662,9 +661,8 @@ impl CpuCompiler {
}
}
-#[async_trait]
impl Compiler for CpuCompiler {
- async fn build(
+ fn build(
&self,
_gpu_types: &GpuTypes,
enum_paths: &EnumPaths,
diff --git a/crates/uzu-engine/build/main.rs b/crates/uzu-engine/build/main.rs
index ae2ae724f..85c1b93ae 100644
--- a/crates/uzu-engine/build/main.rs
+++ b/crates/uzu-engine/build/main.rs
@@ -1,7 +1,6 @@
use std::{env, fs, path::PathBuf, process::ExitCode};
use anyhow::Context;
-use futures::future::try_join_all;
mod common;
use common::{compiler::Compiler, enum_paths::EnumPaths, envs, gpu_types::GpuTypes, traitgen::traitgen_all};
@@ -11,8 +10,7 @@ mod cpu;
#[cfg(all(feature = "metal", target_os = "macos"))]
mod metal;
-#[tokio::main(flavor = "current_thread")]
-async fn main() -> anyhow::Result<ExitCode> {
+fn main() -> anyhow::Result<ExitCode> {
println!("cargo::rerun-if-changed=build");
if envs::build_always() {
@@ -64,7 +62,7 @@ async fn main() -> anyhow::Result<ExitCode> {
#[cfg(all(feature = "metal", target_os = "macos"))]
if backend_metal {
- compilers.push(Box::new(metal::MetalCompiler::new().await?));
+ compilers.push(Box::new(metal::MetalCompiler::new()?));
}
if compilers.is_empty() {
@@ -72,7 +70,8 @@ async fn main() -> anyhow::Result<ExitCode> {
return Ok(ExitCode::FAILURE);
}
- let backends_kernels = try_join_all(compilers.iter().map(|c| c.build(&gpu_types, &enum_paths))).await?;
+ let backends_kernels =
+ compilers.iter().map(|c| c.build(&gpu_types, &enum_paths)).collect::<anyhow::Result<Vec<_>>>()?;
debug_log!("backend build end");
diff --git a/crates/uzu-engine/build/metal/compiler.rs b/crates/uzu-engine/build/metal/compiler.rs
index 9f108a57f..85f17ef1d 100644
--- a/crates/uzu-engine/build/metal/compiler.rs
+++ b/crates/uzu-engine/build/metal/compiler.rs
@@ -5,10 +5,9 @@ use std::{
};
use anyhow::Context;
-use async_trait::async_trait;
-use futures::{StreamExt, TryStreamExt, future::try_join_all, stream};
use itertools::{Itertools, izip};
use quote::{format_ident, quote};
+use rayon::iter::{IntoParallelIterator, ParallelIterator};
use serde::{Deserialize, Serialize};
use walkdir::WalkDir;
use xxhash_rust::xxh3::xxh3_64;
@@ -89,7 +88,7 @@ pub struct MetalCompiler {
}
impl MetalCompiler {
- pub async fn new() -> anyhow::Result<Self> {
+ pub fn new() -> anyhow::Result<Self> {
let source_directory = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?)
.join("src/backends/metal/kernel");
println!("cargo::rerun-if-changed={}", source_directory.display());
@@ -109,9 +108,8 @@ impl MetalCompiler {
_ => true, // treat everything else (3,s,z) as release build where size matters
};
- let toolchain = MetalToolchain::new(modules_cache_path, gpu_types_directory.clone())
- .await
- .context("cannot create toolchain")?;
+ let toolchain =
+ MetalToolchain::new(modules_cache_path, gpu_types_directory.clone()).context("cannot create toolchain")?;
let cache_key = {
let build_system_hash = caching::build_system_hash().context("cannot get build system hash")?;
@@ -143,7 +141,7 @@ impl MetalCompiler {
}
}
- async fn compile(
+ fn compile(
&self,
source_path: PathBuf,
enum_paths: &EnumPaths,
@@ -188,7 +186,6 @@ impl MetalCompiler {
let (metal_kernel_infos, dependencies) = self
.toolchain
.analyze(&source_path)
- .await
.with_context(|| format!("cannot analyze {source_path_relative_str}"))?;
let kernel_infos: Vec<MetalKernelInfo> = metal_kernel_infos.collect();
@@ -232,29 +229,24 @@ impl MetalCompiler {
})
.collect();
- let compile_outputs = try_join_all(izip!(&footers, &metallib_files, &metallib_maybe_compressed_files).map(
- |(footer, metallib_file, compressed_file)| {
- let source_path = &source_path;
- async move {
- let warnings = self.toolchain.compile(source_path, footer, metallib_file).await?;
-
- if self.metallib_compressed {
- let metallib_file = metallib_file.clone();
- let compressed_file = compressed_file.clone();
- tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
- let metallib_source = fs::read(metallib_file)?;
- fs::write(compressed_file, compression::compress(metallib_source.as_slice()))?;
- Ok(())
- })
- .await??;
- }
-
- anyhow::Ok(warnings)
+ let compile_outputs: Vec<_> = izip!(&footers, &metallib_files, &metallib_maybe_compressed_files)
+ .collect::<Vec<_>>()
+ .into_par_iter()
+ .map(|(footer, metallib_file, compressed_file)| {
+ let footer_path = metallib_file.with_extension("metal");
+ fs::write(&footer_path, footer)
+ .with_context(|| format!("cannot write generated Metal source {}", footer_path.display()))?;
+ let warnings = self.toolchain.compile(&source_path, &footer_path, metallib_file)?;
+
+ if self.metallib_compressed {
+ let metallib_source = fs::read(metallib_file)?;
+ fs::write(compressed_file, compression::compress(&metallib_source))?;
}
- },
- ))
- .await
- .with_context(|| format!("cannot compile {source_path_relative_str}"))?;
+
+ anyhow::Ok(warnings)
+ })
+ .collect::<anyhow::Result<_>>()
+ .with_context(|| format!("cannot compile {source_path_relative_str}"))?;
for warnings in compile_outputs.into_iter().flatten() {
for line in warnings.lines() {
@@ -319,14 +311,13 @@ impl MetalCompiler {
}
}
-#[async_trait]
impl Compiler for MetalCompiler {
- async fn build(
+ fn build(
&self,
gpu_types: &GpuTypes,
enum_paths: &EnumPaths,
) -> anyhow::Result<HashMap<KernelPath, Box<[Kernel]>>> {
- gpu_type_gen(&self.gpu_types_directory, gpu_types).await.context("cannot generate shared gpu types")?;
+ gpu_type_gen(&self.gpu_types_directory, gpu_types).context("cannot generate shared gpu types")?;
let metal_sources: Vec<PathBuf> = WalkDir::new(&self.source_directory)
.into_iter()
@@ -335,17 +326,12 @@ impl Compiler for MetalCompiler {
.map(|e| e.into_path())
.collect();
- let num_concurrent_compiles = std::thread::available_parallelism().map(|x| x.get()).unwrap_or(4) * 2;
-
- let compiled: Vec<(KernelPath, Box<[Kernel]>, bool)> = stream::iter(metal_sources)
- .map(|path| async move {
- self.compile(path.clone(), enum_paths)
- .await
- .with_context(|| format!("cannot compile {}", path.display()))
+ let compiled: Vec<(KernelPath, Box<[Kernel]>, bool)> = metal_sources
+ .into_par_iter()
+ .map(|path| {
+ self.compile(path.clone(), enum_paths).with_context(|| format!("cannot compile {}", path.display()))
})
- .buffer_unordered(num_concurrent_compiles)
- .try_collect()
- .await?;
+ .collect::<anyhow::Result<_>>()?;
let mut kernels_bindgen = compiled
.iter()
diff --git a/crates/uzu-engine/build/metal/gpu_types/mod.rs b/crates/uzu-engine/build/metal/gpu_types/mod.rs
index f36314945..2e1821686 100644
--- a/crates/uzu-engine/build/metal/gpu_types/mod.rs
+++ b/crates/uzu-engine/build/metal/gpu_types/mod.rs
@@ -1,4 +1,4 @@
-use std::path::Path;
+use std::{fs, path::Path};
use anyhow::Context;
use itertools::Itertools;
@@ -31,20 +31,19 @@ fn rust_to_metal(ty: &str) -> anyhow::Result<&'static str> {
}
}
-pub async fn gpu_type_gen(
+pub fn gpu_type_gen(
gpu_types_dir: &Path,
gpu_types: &GpuTypes,
) -> anyhow::Result<()> {
for gpu_type_file in &gpu_types.files {
gpu_type_gen_file(&gpu_types_dir.join(gpu_type_file.name.as_ref()).with_extension("h"), gpu_type_file)
- .await
.with_context(|| format!("Cannot generate gpu types for {}", gpu_type_file.name.as_ref()))?;
}
Ok(())
}
-async fn gpu_type_gen_file(
+fn gpu_type_gen_file(
file_path: &Path,
gpu_types_file: &GpuTypeFile,
) -> anyhow::Result<()> {
@@ -68,8 +67,8 @@ async fn gpu_type_gen_file(
let new_contents = format!(include_str!("template.ht"), module_name = module_name, generated = generated);
// Avoid advancing mtime if the contents are the same
- if !tokio::fs::read(&file_path).await.is_ok_and(|old_contents| old_contents == new_contents.as_bytes()) {
- tokio::fs::write(&file_path, new_contents).await.context("cannot write output")?;
+ if !fs::read(file_path).is_ok_and(|old_contents| old_contents == new_contents.as_bytes()) {
+ fs::write(file_path, new_contents).context("cannot write output")?;
}
Ok(())
diff --git a/crates/uzu-engine/build/metal/toolchain.rs b/crates/uzu-engine/build/metal/toolchain.rs
index 680575077..13d608d0c 100644
--- a/crates/uzu-engine/build/metal/toolchain.rs
+++ b/crates/uzu-engine/build/metal/toolchain.rs
@@ -3,13 +3,12 @@ use std::{
ffi::OsString,
fs,
path::{Path, PathBuf},
- process::Stdio,
+ process::{Command, Stdio},
};
use anyhow::{Context, bail};
use serde::Deserialize;
use tempfile::NamedTempFile;
-use tokio::{io::AsyncWriteExt, process::Command};
use super::ast::{MetalAstKind, MetalAstNode, MetalKernelInfo};
@@ -100,7 +99,7 @@ pub struct MetalToolchain {
}
impl MetalToolchain {
- pub async fn new(
+ pub fn new(
modules_cache_path: PathBuf,
include_dir: PathBuf,
) -> anyhow::Result<Self> {
@@ -143,7 +142,6 @@ impl MetalToolchain {
.args(["-sdk", sdk.to_str()])
.args(args)
.output()
- .await
.with_context(|| format!("cannot execute xcrun {}", args.join(" ")))?;
if !output.status.success() {
bail!("xcrun {} failed: {}", args.join(" "), String::from_utf8_lossy(&output.stderr));
@@ -188,7 +186,7 @@ impl MetalToolchain {
cmd.arg("-I").arg(&self.include_dir);
}
- pub async fn analyze(
+ pub fn analyze(
&self,
path: impl AsRef<Path>,
) -> anyhow::Result<(impl Iterator<Item = MetalKernelInfo>, impl Iterator<Item = Box<str>>)> {
@@ -213,7 +211,7 @@ impl MetalToolchain {
.stdout(Stdio::piped())
.stderr(Stdio::piped());
- let analyze_output = cmd.output().await.context("cannot execute metal analyzer")?;
+ let analyze_output = cmd.output().context("cannot execute metal analyzer")?;
if !analyze_output.status.success() {
bail!("metal analyzer failed: {}", String::from_utf8_lossy(&analyze_output.stderr));
@@ -221,26 +219,23 @@ impl MetalToolchain {
let source_contents = fs::read_to_string(path).context("cannot read source file")?;
- let kernel_infos = tokio::task::spawn_blocking(move || {
- let mut deserializer = serde_json::Deserializer::from_slice(&analyze_output.stdout);
- deserializer.disable_recursion_limit();
- let ast_root = MetalAstNode::deserialize(&mut deserializer).context("cannot deserialize ast dump")?;
+ let mut deserializer = serde_json::Deserializer::from_slice(&analyze_output.stdout);
+ deserializer.disable_recursion_limit();
+ let ast_root = MetalAstNode::deserialize(&mut deserializer).context("cannot deserialize ast dump")?;
- if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {
- bail!(
- "unexpected kind of ast root: MetalAstKind::TranslationUnitDecl expected, but {:?} found",
- ast_root.kind
- );
- }
+ if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {
+ bail!(
+ "unexpected kind of ast root: MetalAstKind::TranslationUnitDecl expected, but {:?} found",
+ ast_root.kind
+ );
+ }
- ast_root
- .inner
- .into_iter()
- .filter_map(|node| MetalKernelInfo::from_ast_node_and_source(node, &source_contents).transpose())
- .collect::<anyhow::Result<Vec<_>>>()
- .context("cannot parse kernel infos from AST")
- })
- .await??;
+ let kernel_infos = ast_root
+ .inner
+ .into_iter()
+ .filter_map(|node| MetalKernelInfo::from_ast_node_and_source(node, &source_contents).transpose())
+ .collect::<anyhow::Result<Vec<_>>>()
+ .context("cannot parse kernel infos from AST")?;
let depfile_contents = fs::read_to_string(depfile_path.path()).context("cannot read depfile")?;
@@ -255,10 +250,10 @@ impl MetalToolchain {
Ok((kernel_infos.into_iter(), dependencies))
}
- pub async fn compile(
+ pub fn compile(
&self,
source: impl AsRef<Path>,
- footer: impl AsRef<str>,
+ footer_path: impl AsRef<Path>,
output: impl AsRef<Path>,
) -> anyhow::Result<Option<Box<str>>> {
let mut cmd = self.xcrun();
@@ -272,23 +267,12 @@ impl MetalToolchain {
cmd.arg("-include")
.arg(source.as_ref())
- .arg("-")
+ .arg(footer_path.as_ref())
.arg("-o")
.arg(output.as_ref())
- .stdin(Stdio::piped())
.stderr(Stdio::piped());
- let mut compile_child = cmd.spawn().context("cannot execute metal compiler")?;
-
- compile_child
- .stdin
- .as_mut()
- .context("metal compiler stdin missing")?
- .write_all(footer.as_ref().as_bytes())
- .await
- .context("cannot write to metal compiler stdin")?;
-
- let compile_output = compile_child.wait_with_output().await.context("cannot wait on metal compiler")?;
+ let compile_output = cmd.output().context("cannot execute metal compiler")?;
let stderr = String::from_utf8_lossy(&compile_output.stderr).into_owned().into_boxed_str();