Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

gpu shared types autogen refactor (#336)

uuuvnf279c5f2126a
diff --git a/crates/uzu/build/common/compiler.rs b/crates/uzu/build/common/compiler.rs
index 9ffc5b0a7..564505624 100644
--- a/crates/uzu/build/common/compiler.rs
+++ b/crates/uzu/build/common/compiler.rs
@@ -2,9 +2,12 @@ use std::collections::HashMap;
use async_trait::async_trait;
-use super::kernel::Kernel;
+use super::{gpu_types::GpuTypes, kernel::Kernel};
#[async_trait]
pub trait Compiler {
- async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>;
+ async fn build(
+ &self,
+ gpu_types: &GpuTypes,
+ ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>;
}
diff --git a/crates/uzu/build/common/gpu_types/item_enum.rs b/crates/uzu/build/common/gpu_types/item_enum.rs
new file mode 100644
index 000000000..4521a2366
--- /dev/null
+++ b/crates/uzu/build/common/gpu_types/item_enum.rs
@@ -0,0 +1,63 @@
+use anyhow::{Context, bail, ensure};
+use syn::{Expr, ExprLit, ItemEnum, Lit, Variant};
+
+use crate::common::gpu_types::ensure_repr_c;
+
+#[derive(Debug)]
+pub struct GpuTypeEnumVariant {
+ pub name: Box<str>,
+ pub discriminant: u32,
+}
+
+impl GpuTypeEnumVariant {
+ fn parse(
+ variant: Variant,
+ discriminant: &mut u32,
+ ) -> anyhow::Result<Self> {
+ let name = variant.ident.to_string().into();
+
+ if let Some((_, cur_discriminant)) = variant.discriminant {
+ let Expr::Lit(ExprLit {
+ attrs: _,
+ lit: Lit::Int(cur_discriminant),
+ }) = cur_discriminant
+ else {
+ bail!("Expected integer discriminant, found: {cur_discriminant:?}");
+ };
+ *discriminant = cur_discriminant.base10_parse().context("Cannot parse discriminant")?;
+ }
+
+ let cur_discriminant = *discriminant;
+ *discriminant += 1;
+
+ ensure!(variant.fields.is_empty(), "Enum variant with fields is not supported yet");
+
+ Ok(Self {
+ name,
+ discriminant: cur_discriminant,
+ })
+ }
+}
+
+#[derive(Debug)]
+pub struct GpuTypeEnum {
+ pub name: Box<str>,
+ pub variants: Box<[GpuTypeEnumVariant]>,
+}
+
+impl GpuTypeEnum {
+ pub fn parse(item: ItemEnum) -> anyhow::Result<Self> {
+ ensure_repr_c(&item.attrs)?;
+
+ Ok(Self {
+ name: item.ident.to_string().into(),
+ variants: {
+ let mut discriminant = 0;
+ item.variants
+ .into_iter()
+ .map(|variant| GpuTypeEnumVariant::parse(variant, &mut discriminant))
+ .collect::<anyhow::Result<Box<[GpuTypeEnumVariant]>>>()?
+ },
+ })
+ }
+}
diff --git a/crates/uzu/build/common/gpu_types/item_struct.rs b/crates/uzu/build/common/gpu_types/item_struct.rs
new file mode 100644
index 000000000..fe1d1698d
--- /dev/null
+++ b/crates/uzu/build/common/gpu_types/item_struct.rs
@@ -0,0 +1,89 @@
+use anyhow::{Context, anyhow, bail};
+use quote::ToTokens;
+use syn::{Expr, ExprLit, Fields, ItemStruct, Lit, Type, TypeArray, TypePath};
+
+use crate::common::gpu_types::ensure_repr_c;
+
+#[derive(Debug)]
+pub enum GpuTypeStructFieldType {
+ Scalar(Box<str>),
+ Array {
+ element: Box<str>,
+ length: usize,
+ },
+}
+
+impl GpuTypeStructFieldType {
+ fn parse(ty: Type) -> anyhow::Result<Self> {
+ match ty {
+ Type::Path(TypePath {
+ qself: _,
+ path,
+ }) => Ok(Self::Scalar(path.into_token_stream().to_string().into())),
+ Type::Array(TypeArray {
+ bracket_token: _,
+ elem,
+ semi_token: _,
+ len,
+ }) => {
+ let Type::Path(elem) = *elem else {
+ bail!("Array element must be a Path, found {elem:?}");
+ };
+
+ let Expr::Lit(ExprLit {
+ attrs: _,
+ lit: Lit::Int(len),
+ }) = len
+ else {
+ bail!("Array length must be an integer literal, found: {len:?}");
+ };
+
+ Ok(Self::Array {
+ element: elem.into_token_stream().to_string().into(),
+ length: len.base10_parse().context("Cannot parse array length")?,
+ })
+ },
+ ty => Err(anyhow!("Expected Array or Path, found {ty:?}")),
+ }
+ }
+}
+
+#[derive(Debug)]
+pub struct GpuTypeStructField {
+ pub name: Box<str>,
+ pub ty: GpuTypeStructFieldType,
+}
+
+#[derive(Debug)]
+pub struct GpuTypeStruct {
+ pub name: Box<str>,
+ pub fields: Box<[GpuTypeStructField]>,
+}
+
+impl GpuTypeStruct {
+ pub fn parse(item: ItemStruct) -> anyhow::Result<Self> {
+ ensure_repr_c(&item.attrs)?;
+
+ let name = item.ident.to_string().into();
+
+ let Fields::Named(item_fields) = item.fields else {
+ bail!("Only structs with named fields are supported, found: {:?}", item.fields);
+ };
+
+ let fields = item_fields
+ .named
+ .into_iter()
+ .map(|field| {
+ Ok(GpuTypeStructField {
+ name: field.ident.context("Field doesn't have ident")?.to_string().into(),
+ ty: GpuTypeStructFieldType::parse(field.ty).context("Cannot parse field type")?,
+ })
+ })
+ .collect::<anyhow::Result<Box<[GpuTypeStructField]>>>()?;
+
+ Ok(Self {
+ name,
+ fields,
+ })
+ }
+}
diff --git a/crates/uzu/build/common/gpu_types/mod.rs b/crates/uzu/build/common/gpu_types/mod.rs
new file mode 100644
index 000000000..515a2a75f
--- /dev/null
+++ b/crates/uzu/build/common/gpu_types/mod.rs
@@ -0,0 +1,91 @@
+#![allow(unused)]
+use std::{
+ env, fs,
+ path::{Path, PathBuf},
+};
+
+use anyhow::Context;
+use syn::{Attribute, Ident, Item};
+use walkdir::WalkDir;
+
+mod item_enum;
+mod item_struct;
+
+pub use item_enum::GpuTypeEnum;
+pub use item_struct::{GpuTypeStruct, GpuTypeStructFieldType};
+
+fn ensure_repr_c(attrs: &[Attribute]) -> anyhow::Result<()> {
+ anyhow::ensure!(
+ attrs.iter().any(|attr| attr.path().is_ident("repr") && attr.parse_args::<Ident>().is_ok_and(|arg| arg == "C")),
+ "Does not have repr(c)"
+ );
+
+ Ok(())
+}
+
+#[derive(Debug)]
+pub enum GpuType {
+ Enum(GpuTypeEnum),
+ Struct(GpuTypeStruct),
+}
+
+#[derive(Debug)]
+pub struct GpuTypeFile {
+ pub name: Box<str>,
+ pub types: Box<[GpuType]>,
+}
+
+impl GpuTypeFile {
+ fn scan(path: &Path) -> anyhow::Result<Self> {
+ let name = path.file_stem().context("No file stem")?.to_string_lossy().into();
+
+ let source = fs::read_to_string(path).context("Cannot read source")?;
+ let ast = syn::parse_file(&source).context("Cannot parse ast")?;
+
+ let tys = ast
+ .items
+ .into_iter()
+ .filter_map(|item| match item {
+ Item::Enum(item) => Some(GpuTypeEnum::parse(item).map(GpuType::Enum)),
+ Item::Struct(item) => Some(GpuTypeStruct::parse(item).map(GpuType::Struct)),
+ _ => None,
+ })
+ .collect::<anyhow::Result<Box<[GpuType]>>>()?;
+
+ Ok(Self {
+ name,
+ types: tys,
+ })
+ }
+}
+
+#[derive(Debug)]
+pub struct GpuTypes {
+ pub files: Box<[GpuTypeFile]>,
+}
+
+impl GpuTypes {
+ pub fn scan() -> anyhow::Result<Self> {
+ let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("Missing CARGO_MANIFEST_DIR")?)
+ .join("src/backends/common/gpu_types");
+
+ let mut sources: Vec<PathBuf> = WalkDir::new(&src_dir)
+ .into_iter()
+ .filter_map(|e| e.ok())
+ .filter_map(|e| e.file_type().is_file().then(|| e.into_path()))
+ .filter(|e| e.extension().and_then(|s| s.to_str()) == Some("rs") && e.file_stem() != Some("mod".as_ref()))
+ .collect();
+
+ sources.sort();
+
+ Ok(Self {
+ files: sources
+ .into_iter()
+ .map(|source| {
+ GpuTypeFile::scan(&source)
+ .with_context(|| format!("Failed to scan {}", source.as_os_str().to_str().unwrap()))
+ })
+ .collect::<anyhow::Result<Box<[GpuTypeFile]>>>()?,
+ })
+ }
+}
diff --git a/crates/uzu/build/common/mod.rs b/crates/uzu/build/common/mod.rs
index c7b97f162..237c705e1 100644
--- a/crates/uzu/build/common/mod.rs
+++ b/crates/uzu/build/common/mod.rs
@@ -2,6 +2,7 @@ pub mod caching;
pub mod codegen;
pub mod compiler;
pub mod envs;
+pub mod gpu_types;
pub mod kernel;
pub mod logging;
pub mod mangling;
diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rs
index d825f48de..a9c5535e0 100644
--- a/crates/uzu/build/cpu/compiler.rs
+++ b/crates/uzu/build/cpu/compiler.rs
@@ -14,6 +14,7 @@ use walkdir::WalkDir;
use crate::common::{
codegen::write_tokens,
compiler::Compiler,
+ gpu_types::GpuTypes,
kernel::{Kernel, KernelArgument, KernelArgumentType, KernelBufferAccess, KernelParameter, KernelParameterType},
};
@@ -601,7 +602,10 @@ impl CpuCompiler {
#[async_trait]
impl Compiler for CpuCompiler {
- async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {
+ async fn build(
+ &self,
+ _gpu_types: &GpuTypes,
+ ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {
let objects = WalkDir::new(&self.src_dir)
.into_iter()
.filter_map(|e| e.ok())
diff --git a/crates/uzu/build/gpu_types/mod.rs b/crates/uzu/build/gpu_types/mod.rs
deleted file mode 100644
index 6ffd32d19..000000000
--- a/crates/uzu/build/gpu_types/mod.rs
+++ /dev/null
@@ -1,338 +0,0 @@
-use std::{collections::HashMap, env, fs, path::PathBuf};
-
-use anyhow::Context;
-use async_trait::async_trait;
-use syn::{Attribute, Expr, ExprLit, Fields, Item, ItemEnum, ItemStruct, Lit, Meta, Type, TypeArray, TypePath};
-use tokio::task::spawn_blocking;
-
-use crate::{
- common::{compiler::Compiler, kernel::Kernel},
- debug_log,
-};
-
-#[derive(Debug)]
-pub struct GpuTypesCompiler {
- crate_dir: PathBuf,
- generated_dir: PathBuf,
- build_dir: PathBuf,
-}
-
-impl GpuTypesCompiler {
- pub fn new() -> anyhow::Result<Self> {
- let crate_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?);
-
- let generated_dir = crate_dir.join("src/backends/metal/generated");
- fs::create_dir_all(&generated_dir).with_context(|| format!("cannot create {}", generated_dir.display()))?;
-
- let build_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?).join("gpu_types");
- fs::create_dir_all(&build_dir).with_context(|| format!("cannot create {}", build_dir.display()))?;
-
- Ok(Self {
- crate_dir,
- generated_dir,
- build_dir,
- })
- }
-
- #[cfg(all(feature = "metal", target_os = "macos"))]
- pub fn generated_header_dir(&self) -> &PathBuf {
- &self.generated_dir
- }
-
- fn generate(&self) -> anyhow::Result<()> {
- let gpu_types_dir = self.crate_dir.join("src/backends/common/gpu_types");
-
- let source_files: Vec<PathBuf> = fs::read_dir(&gpu_types_dir)
- .with_context(|| format!("cannot read {}", gpu_types_dir.display()))?
- .filter_map(|entry| entry.ok())
- .map(|entry| entry.path())
- .filter(|path| {
- path.extension().map(|ext| ext == "rs").unwrap_or(false) && path.file_stem() != Some("mod".as_ref())
- })
- .collect();
-
- let mut hasher = blake3::Hasher::new();
- for path in &source_files {
- let content = fs::read(path).with_context(|| format!("cannot read {}", path.display()))?;
- hasher.update(&content);
- }
-
- let source_hash: [u8; blake3::OUT_LEN] = hasher.finalize().into();
- let cached_hash_path = self.build_dir.join("gpu_types.hash");
-
- if let Ok(cached_hash) = fs::read(&cached_hash_path) {
- if cached_hash == source_hash {
- debug_log!("gpu_types cached");
- return Ok(());
- }
- }
-
- debug_log!("gpu_types generation started");
-
- for src_path in &source_files {
- self.generate_header_for_file(src_path)?;
- }
-
- fs::write(&cached_hash_path, &source_hash)
- .with_context(|| format!("cannot write {}", cached_hash_path.display()))?;
-
- debug_log!("gpu_types generation done");
-
- Ok(())
- }
-
- fn generate_header_for_file(
- &self,
- src_path: &PathBuf,
- ) -> anyhow::Result<()> {
- let module_name = src_path.file_stem().and_then(|s| s.to_str()).context("invalid source file name")?;
-
- let source = fs::read_to_string(src_path).with_context(|| format!("cannot read {}", src_path.display()))?;
-
- let syntax = syn::parse_file(&source).with_context(|| format!("cannot parse {}", src_path.display()))?;
-
- let mut repr_c_enums: Vec<ItemEnum> = Vec::new();
- let mut repr_c_structs: Vec<ItemStruct> = Vec::new();
- for item in syntax.items {
- if let Item::Enum(item_enum) = item {
- if have_repr_c_attr(&item_enum.attrs) {
- repr_c_enums.push(item_enum);
- }
- } else if let Item::Struct(item_struct) = item {
- if have_repr_c_attr(&item_struct.attrs) {
- repr_c_structs.push(item_struct);
- }
- }
- }
-
- let c_enum_definitions = Self::get_enum_definitions(&repr_c_enums);
- let c_struct_definitions = Self::get_struct_definitions(&repr_c_structs);
-
- let mut content = String::new();
- if !c_enum_definitions.is_empty() {
- content.push_str(&c_enum_definitions);
- }
- if !c_struct_definitions.is_empty() {
- if !content.is_empty() {
- content.push_str("\n\n");
- }
- content.push_str(&c_struct_definitions);
- }
-
- self.write_header(module_name, content.as_str())?;
-
- Ok(())
- }
-
- fn get_struct_definitions(c_structs: &Vec<ItemStruct>) -> String {
- let mut definitions = String::new();
- for (i, rust_struct) in c_structs.iter().enumerate() {
- for attribute in &rust_struct.attrs {
- if attribute.path().is_ident("doc") {
- if let Meta::NameValue(meta_name_value) = &attribute.meta {
- if let Expr::Lit(ExprLit {
- lit: Lit::Str(literal_string),
- ..
- }) = &meta_name_value.value
- {
- let doc_comment = literal_string.value();
- let doc_comment = doc_comment.trim();
- if !doc_comment.is_empty() {
- definitions.push_str("/**");
- definitions.push_str(doc_comment);
- definitions.push_str(" */\n");
- }
- }
- }
- }
- }
-
- definitions.push_str("typedef struct {\n");
-
- if let Fields::Named(named_fields) = &rust_struct.fields {
- for field in &named_fields.named {
- let field_name = field.ident.as_ref().map(|ident| ident.to_string()).unwrap_or_default();
- let c_type = rust_type_to_c(&field.ty);
-
- for attribute in &field.attrs {
- if attribute.path().is_ident("doc") {
- if let Meta::NameValue(meta_name_value) = &attribute.meta {
- if let Expr::Lit(ExprLit {
- lit: Lit::Str(literal_string),
- ..
- }) = &meta_name_value.value
- {
- let doc_comment = literal_string.value();
- let doc_comment = doc_comment.trim();
- if !doc_comment.is_empty() {
- definitions.push_str(" /**");
- definitions.push_str(doc_comment);
- definitions.push_str(" */\n");
- }
- }
- }
- }
- }
-
- if let Some((base_type, array_size)) = c_type.split_once('[') {
- definitions.push_str(&format!(" {base_type} {field_name}[{array_size};\n"));
- } else {
- definitions.push_str(&format!(" {c_type} {field_name};\n"));
- }
- }
- }
-
- let struct_name = rust_struct.ident.to_string();
- definitions.push_str(&format!("}} {struct_name};"));
- if i < c_structs.len() - 1 {
- definitions.push_str("\n\n");
- }
- }
-
- definitions
- }
-
- fn get_enum_definitions(enums: &Vec<ItemEnum>) -> String {
- let mut definitions = String::new();
- for (i, rust_enum) in enums.iter().enumerate() {
- let enum_name = rust_enum.ident.to_string();
- definitions.push_str("enum ");
- definitions.push_str(&enum_name);
- definitions.push_str(" : uint32_t {\n");
-
- for variant in &rust_enum.variants {
- definitions.push_str(" ");
- definitions.push_str(variant.ident.to_string().as_str());
- definitions.push_str(",\n");
- }
-
- definitions.push_str("};");
- if i < enums.len() - 1 {
- definitions.push_str("\n\n");
- }
- }
- definitions
- }
-
- fn write_header(
- &self,
- module_name: &str,
- content: &str,
- ) -> anyhow::Result<()> {
- let include_guard = format!("UZU_{}_H", module_name.to_uppercase());
-
- let header_content = format!(
- r#"// Auto-generated from gpu_types/{module_name}.rs - do not edit manually
-#pragma once
-
-#ifndef {include_guard}
-#define {include_guard}
-
-#ifdef __METAL_VERSION__
-#include <metal_stdlib>
-using namespace metal;
-
-namespace uzu {{
-namespace {module_name} {{
-#else
-#include <stdint.h>
-#endif
-
-{content}
-
-#ifdef __METAL_VERSION__
-}} // namespace {module_name}
-}} // namespace uzu
-#endif
-
-#endif // {include_guard}
-"#
- );
-
- let output_path = self.generated_dir.join(format!("{module_name}.h"));
- fs::write(&output_path, &header_content).with_context(|| format!("cannot write {}", output_path.display()))?;
-
- debug_log!(" -> {}", output_path.display());
-
- Ok(())
- }
-}
-
-fn rust_type_to_c(rust_type: &Type) -> String {
- match rust_type {
- Type::Path(TypePath {
- path,
- ..
- }) => {
- let segment = path.segments.last().unwrap();
- let type_name = segment.ident.to_string();
-
- match type_name.as_str() {
- "i8" => "int8_t".to_string(),
- "i16" => "int16_t".to_string(),
- "i32" => "int32_t".to_string(),
- "i64" => "int64_t".to_string(),
- "u8" => "uint8_t".to_string(),
- "u16" => "uint16_t".to_string(),
- "u32" => "uint32_t".to_string(),
- "u64" => "uint64_t".to_string(),
- "f32" => "float".to_string(),
- "f64" => "double".to_string(),
- "bool" => "bool".to_string(),
- "usize" => "size_t".to_string(),
- "isize" => "ptrdiff_t".to_string(),
- _ => type_name,
- }
- },
- Type::Array(TypeArray {
- elem,
- len,
- ..
- }) => {
- let element_type = rust_type_to_c(elem);
- if let Expr::Lit(ExprLit {
- lit: Lit::Int(literal_int),
- ..
- }) = len
- {
- let array_length = literal_int.base10_digits();
- format!("{element_type}[{array_length}]")
- } else {
- format!("{element_type}[]")
- }
- },
- _ => "/* unknown */".to_string(),
- }
-}
-
-fn have_repr_c_attr(attrs: &Vec<Attribute>) -> bool {
- attrs.iter().any(|attr| {
- if attr.path().is_ident("repr") {
- if let Ok(ident) = attr.parse_args::<syn::Ident>() {
- return ident == "C";
- }
- }
- false
- })
-}
-
-#[async_trait]
-impl Compiler for GpuTypesCompiler {
- async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {
- let crate_dir = self.crate_dir.clone();
- let generated_dir = self.generated_dir.clone();
- let build_dir = self.build_dir.clone();
-
- spawn_blocking(move || {
- GpuTypesCompiler {
- crate_dir,
- generated_dir,
- build_dir,
- }
- .generate()
- })
- .await??;
-
- Ok(HashMap::new())
- }
-}
diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rs
index 46995f3dc..7bb78cade 100644
--- a/crates/uzu/build/main.rs
+++ b/crates/uzu/build/main.rs
@@ -4,15 +4,13 @@ use anyhow::Context;
use futures::future::try_join_all;
mod common;
-mod gpu_types;
+use common::{compiler::Compiler, envs, gpu_types::GpuTypes, traitgen::traitgen_all};
mod cpu;
#[cfg(all(feature = "metal", target_os = "macos"))]
mod metal;
-use common::{compiler::Compiler, envs, traitgen::traitgen_all};
-
#[tokio::main(flavor = "current_thread")]
async fn main() -> anyhow::Result<()> {
println!("cargo::rerun-if-changed=.");
@@ -44,9 +42,8 @@ async fn main() -> anyhow::Result<()> {
debug_log!("cleaned caches");
}
- let gpu_types_compiler = gpu_types::GpuTypesCompiler::new()?;
- gpu_types_compiler.build().await?;
- debug_log!("gpu_types build done");
+ let gpu_types = GpuTypes::scan().context("Failed to scan gpu types")?;
+ debug_log!("gpu_types scan done");
let mut compilers: Vec<Box<dyn Compiler>> = Vec::new();
@@ -54,11 +51,10 @@ async fn main() -> anyhow::Result<()> {
#[cfg(all(feature = "metal", target_os = "macos"))]
if metal_backend {
- let generated_header_dir = gpu_types_compiler.generated_header_dir().clone();
- compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(generated_header_dir)?));
+ compilers.push(Box::new(metal::MetalCompiler::new()?));
}
- let backends_kernels = try_join_all(compilers.iter().map(|c| c.build())).await?;
+ let backends_kernels = try_join_all(compilers.iter().map(|c| c.build(&gpu_types))).await?;
debug_log!("backend build end");
diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rs
index ae3db74d6..03c759d02 100644
--- a/crates/uzu/build/metal/compiler.rs
+++ b/crates/uzu/build/metal/compiler.rs
@@ -15,8 +15,9 @@ use super::{
wrapper::{SpecializeBaseIndices, wrappers},
};
use crate::{
- common::{caching, codegen::write_tokens, compiler::Compiler, envs, kernel::Kernel},
+ common::{caching, codegen::write_tokens, compiler::Compiler, envs, gpu_types::GpuTypes, kernel::Kernel},
debug_log,
+ metal::gpu_types::gpu_type_gen,
};
#[derive(Serialize, Deserialize, Debug)]
@@ -83,25 +84,29 @@ fn objects_hash<'a>(objects: impl IntoIterator<Item = &'a ObjectInfo>) -> anyhow
#[derive(Debug)]
pub struct MetalCompiler {
src_dir: PathBuf,
+ gpu_types_dir: PathBuf,
build_dir: PathBuf,
out_dir: PathBuf,
toolchain: MetalToolchain,
}
impl MetalCompiler {
- pub fn new_with_include_dir(include_dir: PathBuf) -> anyhow::Result<Self> {
+ pub fn new() -> anyhow::Result<Self> {
let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?)
.join("src/backends/metal/kernel");
+ let gpu_types_dir = src_dir.join("generated");
+
let out_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
let build_dir = out_dir.join("metal");
fs::create_dir_all(&build_dir).with_context(|| format!("cannot create {}", build_dir.display()))?;
- let toolchain =
- MetalToolchain::from_env_with_include_dir(Some(include_dir)).context("cannot create toolchain")?;
+ let toolchain = MetalToolchain::from_env_with_include_dir(Some(gpu_types_dir.clone()))
+ .context("cannot create toolchain")?;
Ok(Self {
src_dir,
+ gpu_types_dir,
build_dir,
out_dir,
toolchain,
@@ -313,7 +318,12 @@ impl MetalCompiler {
#[async_trait]
impl Compiler for MetalCompiler {
- async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {
+ async fn build(
+ &self,
+ gpu_types: &GpuTypes,
+ ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {
+ gpu_type_gen(&self.gpu_types_dir, gpu_types).await.context("cannot generate shared gpu types")?;
+
let metal_sources: Vec<PathBuf> = WalkDir::new(&self.src_dir)
.into_iter()
.filter_map(|e| e.ok())
diff --git a/crates/uzu/build/metal/gpu_types/gpu_type_enum.rs b/crates/uzu/build/metal/gpu_types/gpu_type_enum.rs
new file mode 100644
index 000000000..e338a3876
--- /dev/null
+++ b/crates/uzu/build/metal/gpu_types/gpu_type_enum.rs
@@ -0,0 +1,12 @@
+use std::iter::once;
+
+use itertools::Itertools;
+
+use crate::common::gpu_types::GpuTypeEnum;
+
+pub fn gpu_type_gen_enum(gpu_type_enum: &GpuTypeEnum) -> anyhow::Result<String> {
+ Ok(once(format!("enum class {} : uint32_t {{", gpu_type_enum.name.as_ref()))
+ .chain(gpu_type_enum.variants.iter().map(|variant| format!(" {} = {},", variant.name, variant.discriminant)))
+ .chain(once("};".into()))
+ .join("\n"))
+}
diff --git a/crates/uzu/build/metal/gpu_types/gpu_type_struct.rs b/crates/uzu/build/metal/gpu_types/gpu_type_struct.rs
new file mode 100644
index 000000000..111513edb
--- /dev/null
+++ b/crates/uzu/build/metal/gpu_types/gpu_type_struct.rs
@@ -0,0 +1,37 @@
+use std::iter::once;
+
+use anyhow::anyhow;
+use itertools::Itertools;
+
+use crate::common::gpu_types::{GpuTypeStruct, GpuTypeStructFieldType};
+
+fn r2c(ty: &str) -> anyhow::Result<&'static str> {
+ match ty {
+ "i8" => Ok("int8_t"),
+ "i16" => Ok("int16_t"),
+ "i32" => Ok("int32_t"),
+ "i64" => Ok("int64_t"),
+ "u8" => Ok("uint8_t"),
+ "u16" => Ok("uint16_t"),
+ "u32" => Ok("uint32_t"),
+ "u64" => Ok("uint64_t"),
+ "f32" => Ok("float"),
+ "bool" => Ok("bool"),
+ "usize" => Ok("size_t"),
+ "isize" => Ok("ptrdiff_t"),
+ _ => Err(anyhow!("Only primitive types are supported, found {ty}")),
+ }
+}
+
+pub fn gpu_type_gen_struct(gpu_type_struct: &GpuTypeStruct) -> anyhow::Result<String> {
+ once(Ok("typedef struct {".into()))
+ .chain(gpu_type_struct.fields.iter().map(|field| match &field.ty {
+ GpuTypeStructFieldType::Scalar(ty) => Ok(format!(" {} {};", r2c(ty.as_ref())?, field.name)),
+ GpuTypeStructFieldType::Array {
+ element,
+ length,
+ } => Ok(format!(" {} {}[{}];", r2c(element.as_ref())?, field.name, *length)),
+ }))
+ .chain(once(Ok(format!("}} {};", gpu_type_struct.name.as_ref()))))
+ .process_results(|mut it| it.join("\n"))
+}
diff --git a/crates/uzu/build/metal/gpu_types/mod.rs b/crates/uzu/build/metal/gpu_types/mod.rs
new file mode 100644
index 000000000..14e3b25ab
--- /dev/null
+++ b/crates/uzu/build/metal/gpu_types/mod.rs
@@ -0,0 +1,52 @@
+use std::path::Path;
+
+use anyhow::Context;
+use itertools::Itertools;
+
+use crate::common::gpu_types::{GpuType, GpuTypeFile, GpuTypes};
+
+mod gpu_type_enum;
+mod gpu_type_struct;
+
+use gpu_type_enum::gpu_type_gen_enum;
+use gpu_type_struct::gpu_type_gen_struct;
+
+pub async fn gpu_type_gen(
+ gpu_types_dir: &Path,
+ gpu_types: &GpuTypes,
+) -> anyhow::Result<()> {
+ for gpu_type_file in &gpu_types.files {
+ gpu_type_gen_file(&gpu_types_dir.join(gpu_type_file.name.as_ref()).with_extension("h"), gpu_type_file)
+ .await
+ .with_context(|| format!("Cannot generate gpu types for {}", gpu_type_file.name.as_ref()))?;
+ }
+
+ Ok(())
+}
+
+async fn gpu_type_gen_file(
+ file_path: &Path,
+ gpu_types_file: &GpuTypeFile,
+) -> anyhow::Result<()> {
+ let module_name = gpu_types_file.name.as_ref();
+
+ let generated = gpu_types_file
+ .types
+ .iter()
+ .map(|gpu_type| match gpu_type {
+ GpuType::Enum(gpu_type_enum) => gpu_type_gen_enum(gpu_type_enum)
+ .with_context(|| format!("Failed to generate bindings for {gpu_type_enum:?}")),
+ GpuType::Struct(gpu_type_struct) => gpu_type_gen_struct(gpu_type_struct)
+ .with_context(|| format!("Failed to generate bindings for {gpu_type_struct:?}")),
+ })
+ .process_results(|mut it| it.join("\n\n"))?;
+
+ let new_contents = format!(include_str!("template.ht"), module_name = module_name, generated = generated);
+
+ // Avoid advancing mtime if the contents are the same
+ if !tokio::fs::read(&file_path).await.is_ok_and(|old_contents| &old_contents == new_contents.as_bytes()) {
+ tokio::fs::write(&file_path, new_contents).await.context("cannot write output")?;
+ }
+
+ Ok(())
+}
diff --git a/crates/uzu/build/metal/gpu_types/template.ht b/crates/uzu/build/metal/gpu_types/template.ht
new file mode 100644
index 000000000..8696754a5
--- /dev/null
+++ b/crates/uzu/build/metal/gpu_types/template.ht
@@ -0,0 +1,9 @@
+// Auto-generated from gpu_types/{module_name}.rs - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::{module_name} {{
+{generated}
+}} // namespace uzu::{module_name}
diff --git a/crates/uzu/build/metal/mod.rs b/crates/uzu/build/metal/mod.rs
index 7d2d4a094..9e5a670db 100644
--- a/crates/uzu/build/metal/mod.rs
+++ b/crates/uzu/build/metal/mod.rs
@@ -1,6 +1,7 @@
mod ast;
mod bindgen;
mod compiler;
+mod gpu_types;
mod toolchain;
mod wrapper;
diff --git a/crates/uzu/src/backends/metal/generated/activation_type.h b/crates/uzu/src/backends/metal/generated/activation_type.h
deleted file mode 100644
index 2cf6beef1..000000000
--- a/crates/uzu/src/backends/metal/generated/activation_type.h
+++ /dev/null
@@ -1,30 +0,0 @@
-// Auto-generated from gpu_types/activation_type.rs - do not edit manually
-#pragma once
-
-#ifndef UZU_ACTIVATION_TYPE_H
-#define UZU_ACTIVATION_TYPE_H
-
-#ifdef __METAL_VERSION__
-#include <metal_stdlib>
-using namespace metal;
-
-namespace uzu {
-namespace activation_type {
-#else
-#include <stdint.h>
-#endif
-
-enum ActivationType : uint32_t {
- SILU,
- GELU,
- TANH,
- IDENTITY,
- SOFTPLUS,
-};
-
-#ifdef __METAL_VERSION__
-} // namespace activation_type
-} // namespace uzu
-#endif
-
-#endif // UZU_ACTIVATION_TYPE_H
diff --git a/crates/uzu/src/backends/metal/generated/argmax.h b/crates/uzu/src/backends/metal/generated/argmax.h
deleted file mode 100644
index 8b9bcde16..000000000
--- a/crates/uzu/src/backends/metal/generated/argmax.h
+++ /dev/null
@@ -1,27 +0,0 @@
-// Auto-generated from gpu_types/argmax.rs - do not edit manually
-#pragma once
-
-#ifndef UZU_ARGMAX_H
-#define UZU_ARGMAX_H
-
-#ifdef __METAL_VERSION__
-#include <metal_stdlib>
-using namespace metal;
-
-namespace uzu {
-namespace argmax {
-#else
-#include <stdint.h>
-#endif
-
-typedef struct {
- float value;
- uint32_t index;
-} ArgmaxPair;
-
-#ifdef __METAL_VERSION__
-} // namespace argmax
-} // namespace uzu
-#endif
-
-#endif // UZU_ARGMAX_H
diff --git a/crates/uzu/src/backends/metal/generated/attention.h b/crates/uzu/src/backends/metal/generated/attention.h
deleted file mode 100644
index 7e3e3b787..000000000
--- a/crates/uzu/src/backends/metal/generated/attention.h
+++ /dev/null
@@ -1,54 +0,0 @@
-// Auto-generated from gpu_types/attention.rs - do not edit manually
-#pragma once
-
-#ifndef UZU_ATTENTION_H
-#define UZU_ATTENTION_H
-
-#ifdef __METAL_VERSION__
-#include <metal_stdlib>
-using namespace metal;
-
-namespace uzu {
-namespace attention {
-#else
-#include <stdint.h>
-#endif
-
-/**Parameters for the GEMM attention kernel. */
-/**All strides are in **elements**, not bytes. */
-/**Q/K/V/O are treated as 3D tensors with strides for: */
-/**- `[0]` batch */
-/**- `[1]` head (or kv-head for K/V) */
-/**- `[2]` sequence (row stride for [seq, head_dim]) */
-typedef struct {
- int64_t q_strides[3];
- int64_t k_strides[3];
- int64_t v_strides[3];
- int64_t o_strides[3];
- int32_t gqa_factor;
- float scale;
- /**Query length (suffix length) */
- int32_t q_len;
- /**Key length (prefix + suffix) */
- int32_t k_len;
- /**Absolute offset of the first query token in the full key sequence. */
- /**For LLM decode/prefill this is the segment prefix length. */
- int32_t q_off;
- /**Query tiling: q_len / BQ */
- int32_t nq_aligned;
- /**Query tiling: q_len % BQ */
- int32_t q_rem;
- /**Key tiling: ceil(k_len / BK) */
- int32_t nk;
- /**Key tiling: k_len / BK */
- int32_t nk_aligned;
- /**Key tiling: k_len % BK */
- int32_t k_rem;
-} AttnParams;
-
-#ifdef __METAL_VERSION__
-} // namespace attention
-} // namespace uzu
-#endif
-
-#endif // UZU_ATTENTION_H
diff --git a/crates/uzu/src/backends/metal/generated/kv_cache_update.h b/crates/uzu/src/backends/metal/generated/kv_cache_update.h
deleted file mode 100644
index fe32897f8..000000000
--- a/crates/uzu/src/backends/metal/generated/kv_cache_update.h
+++ /dev/null
@@ -1,27 +0,0 @@
-// Auto-generated from gpu_types/kv_cache_update.rs - do not edit manually
-#pragma once
-
-#ifndef UZU_KV_CACHE_UPDATE_H
-#define UZU_KV_CACHE_UPDATE_H
-
-#ifdef __METAL_VERSION__
-#include <metal_stdlib>
-using namespace metal;
-
-namespace uzu {
-namespace kv_cache_update {
-#else
-#include <stdint.h>
-#endif
-
-typedef struct {
- uint32_t source;
- uint32_t destination;
-} Swap;
-
-#ifdef __METAL_VERSION__
-} // namespace kv_cache_update
-} // namespace uzu
-#endif
-
-#endif // UZU_KV_CACHE_UPDATE_H
diff --git a/crates/uzu/src/backends/metal/generated/quantization.h b/crates/uzu/src/backends/metal/generated/quantization.h
deleted file mode 100644
index c73e771ad..000000000
--- a/crates/uzu/src/backends/metal/generated/quantization.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Auto-generated from gpu_types/quantization.rs - do not edit manually
-#pragma once
-
-#ifndef UZU_QUANTIZATION_H
-#define UZU_QUANTIZATION_H
-
-#ifdef __METAL_VERSION__
-#include <metal_stdlib>
-using namespace metal;
-
-namespace uzu {
-namespace quantization {
-#else
-#include <stdint.h>
-#endif
-
-enum QuantizationMode : uint32_t {
- UINT4,
- INT8,
- UINT8,
-};
-
-#ifdef __METAL_VERSION__
-} // namespace quantization
-} // namespace uzu
-#endif
-
-#endif // UZU_QUANTIZATION_H
diff --git a/crates/uzu/src/backends/metal/generated/ring.h b/crates/uzu/src/backends/metal/generated/ring.h
deleted file mode 100644
index 6819327e9..000000000
--- a/crates/uzu/src/backends/metal/generated/ring.h
+++ /dev/null
@@ -1,27 +0,0 @@
-// Auto-generated from gpu_types/ring.rs - do not edit manually
-#pragma once
-
-#ifndef UZU_RING_H
-#define UZU_RING_H
-
-#ifdef __METAL_VERSION__
-#include <metal_stdlib>
-using namespace metal;
-
-namespace uzu {
-namespace ring {
-#else
-#include <stdint.h>
-#endif
-
-typedef struct {
- uint32_t ring_offset;
- uint32_t ring_length;
-} RingParams;
-
-#ifdef __METAL_VERSION__
-} // namespace ring
-} // namespace uzu
-#endif
-
-#endif // UZU_RING_H
diff --git a/crates/uzu/src/backends/metal/generated/trie.h b/crates/uzu/src/backends/metal/generated/trie.h
deleted file mode 100644
index 2c2ab2497..000000000
--- a/crates/uzu/src/backends/metal/generated/trie.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Auto-generated from gpu_types/trie.rs - do not edit manually
-#pragma once
-
-#ifndef UZU_TRIE_H
-#define UZU_TRIE_H
-
-#ifdef __METAL_VERSION__
-#include <metal_stdlib>
-using namespace metal;
-
-namespace uzu {
-namespace trie {
-#else
-#include <stdint.h>
-#endif
-
-typedef struct {
- uint32_t trie_start;
- uint32_t trie_end;
- uint32_t height;
-} TrieNode;
-
-#ifdef __METAL_VERSION__
-} // namespace trie
-} // namespace uzu
-#endif
-
-#endif // UZU_TRIE_H
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activations.h b/crates/uzu/src/backends/metal/kernel/activation/activations.h
index 6df1548ae..b05b638c4 100644
--- a/crates/uzu/src/backends/metal/kernel/activation/activations.h
+++ b/crates/uzu/src/backends/metal/kernel/activation/activations.h
@@ -54,15 +54,15 @@ inline T activate_tanh(T x) {
template <typename T>
inline T activate(T x, ActivationType type) {
switch (type) {
- case SILU:
+ case ActivationType::SILU:
return activate_silu(x);
- case GELU:
+ case ActivationType::GELU:
return activate_gelu(x);
- case TANH:
+ case ActivationType::TANH:
return activate_tanh(x);
- case IDENTITY:
+ case ActivationType::IDENTITY:
return x;
- case SOFTPLUS:
+ case ActivationType::SOFTPLUS:
return activate_softplus(x);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index 67b7d6c92..f06629847 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -21,6 +21,9 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
const uint dim_idx AXIS(model_dim, 16),
const uint batch_idx AXIS(batch_size, 16)
) {
+ const QuantizationMode quantization_mode = QuantizationMode(
+ quant_mode
+ ); // TODO: should be accepted as a kernel argument
const uint thread_position_in_grid = batch_idx * model_dim + dim_idx;
const uint64_t token_id = token_ids[batch_idx];
if (token_id >= vocab_size) {
@@ -34,12 +37,13 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
const T scale = scales[token_id * num_groups + group_idx];
const T bias = biases[token_id * num_groups + group_idx];
- const uint packing_divisor = quant_mode == QuantizationMode::UINT4 ? 2 : 1;
+ const uint packing_divisor =
+ quantization_mode == QuantizationMode::UINT4 ? 2 : 1;
const uint weights_stride = model_dim / packing_divisor;
int quantized_value = 0;
- switch (quant_mode) {
- case UINT4: {
+ switch (quantization_mode) {
+ case QuantizationMode::UINT4: {
const uint byte_idx = token_id * weights_stride + (dim_idx / 2);
const uint8_t packed = weights[byte_idx];
if ((dim_idx & 1) == 0) {
@@ -49,14 +53,14 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
}
break;
};
- case INT8: {
+ case QuantizationMode::INT8: {
const uint elem_idx = token_id * weights_stride + dim_idx;
const device int8_t* weights_i8 =
reinterpret_cast<const device int8_t*>(weights);
quantized_value = int(weights_i8[elem_idx]);
break;
};
- case UINT8: {
+ case QuantizationMode::UINT8: {
const uint elem_idx = token_id * weights_stride + dim_idx;
quantized_value = int(weights[elem_idx]);
break;
diff --git a/crates/uzu/src/backends/metal/kernel/generated/activation_type.h b/crates/uzu/src/backends/metal/kernel/generated/activation_type.h
new file mode 100644
index 000000000..c5f89ed64
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/generated/activation_type.h
@@ -0,0 +1,15 @@
+// Auto-generated from gpu_types/activation_type.rs - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::activation_type {
+enum class ActivationType : uint32_t {
+ SILU = 0,
+ GELU = 1,
+ TANH = 2,
+ IDENTITY = 3,
+ SOFTPLUS = 4,
+};
+} // namespace uzu::activation_type
diff --git a/crates/uzu/src/backends/metal/kernel/generated/argmax.h b/crates/uzu/src/backends/metal/kernel/generated/argmax.h
new file mode 100644
index 000000000..3b818d559
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/generated/argmax.h
@@ -0,0 +1,12 @@
+// Auto-generated from gpu_types/argmax.rs - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::argmax {
+typedef struct {
+ float value;
+ uint32_t index;
+} ArgmaxPair;
+} // namespace uzu::argmax
diff --git a/crates/uzu/src/backends/metal/kernel/generated/attention.h b/crates/uzu/src/backends/metal/kernel/generated/attention.h
new file mode 100644
index 000000000..b46ba583f
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/generated/attention.h
@@ -0,0 +1,24 @@
+// Auto-generated from gpu_types/attention.rs - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::attention {
+typedef struct {
+ int64_t q_strides[3];
+ int64_t k_strides[3];
+ int64_t v_strides[3];
+ int64_t o_strides[3];
+ int32_t gqa_factor;
+ float scale;
+ int32_t q_len;
+ int32_t k_len;
+ int32_t q_off;
+ int32_t nq_aligned;
+ int32_t q_rem;
+ int32_t nk;
+ int32_t nk_aligned;
+ int32_t k_rem;
+} AttnParams;
+} // namespace uzu::attention
diff --git a/crates/uzu/src/backends/metal/kernel/generated/kv_cache_update.h b/crates/uzu/src/backends/metal/kernel/generated/kv_cache_update.h
new file mode 100644
index 000000000..c42228faa
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/generated/kv_cache_update.h
@@ -0,0 +1,12 @@
+// Auto-generated from gpu_types/kv_cache_update.rs - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::kv_cache_update {
+typedef struct {
+ uint32_t source;
+ uint32_t destination;
+} Swap;
+} // namespace uzu::kv_cache_update
diff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/kernel/generated/matmul.h
similarity index 65%
rename from crates/uzu/src/backends/metal/generated/matmul.h
rename to crates/uzu/src/backends/metal/kernel/generated/matmul.h
index 07525ecdb..400808477 100644
--- a/crates/uzu/src/backends/metal/generated/matmul.h
+++ b/crates/uzu/src/backends/metal/kernel/generated/matmul.h
@@ -1,19 +1,10 @@
// Auto-generated from gpu_types/matmul.rs - do not edit manually
#pragma once
-#ifndef UZU_MATMUL_H
-#define UZU_MATMUL_H
-
-#ifdef __METAL_VERSION__
#include <metal_stdlib>
using namespace metal;
-namespace uzu {
-namespace matmul {
-#else
-#include <stdint.h>
-#endif
-
+namespace uzu::matmul {
typedef struct {
uint32_t M;
uint32_t N;
@@ -27,10 +18,4 @@ typedef struct {
uint32_t aligned_inner_iterations;
bool use_morton;
} GemmParams;
-
-#ifdef __METAL_VERSION__
-} // namespace matmul
-} // namespace uzu
-#endif
-
-#endif // UZU_MATMUL_H
+} // namespace uzu::matmul
diff --git a/crates/uzu/src/backends/metal/kernel/generated/quantization.h b/crates/uzu/src/backends/metal/kernel/generated/quantization.h
new file mode 100644
index 000000000..0315c17d3
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/generated/quantization.h
@@ -0,0 +1,13 @@
+// Auto-generated from gpu_types/quantization.rs - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::quantization {
+enum class QuantizationMode : uint32_t {
+ UINT4 = 0,
+ INT8 = 1,
+ UINT8 = 2,
+};
+} // namespace uzu::quantization
diff --git a/crates/uzu/src/backends/metal/kernel/generated/ring.h b/crates/uzu/src/backends/metal/kernel/generated/ring.h
new file mode 100644
index 000000000..a10f4c220
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/generated/ring.h
@@ -0,0 +1,12 @@
+// Auto-generated from gpu_types/ring.rs - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::ring {
+typedef struct {
+ uint32_t ring_offset;
+ uint32_t ring_length;
+} RingParams;
+} // namespace uzu::ring
diff --git a/crates/uzu/src/backends/metal/kernel/generated/trie.h b/crates/uzu/src/backends/metal/kernel/generated/trie.h
new file mode 100644
index 000000000..0ccfaf5d1
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/generated/trie.h
@@ -0,0 +1,13 @@
+// Auto-generated from gpu_types/trie.rs - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::trie {
+typedef struct {
+ uint32_t trie_start;
+ uint32_t trie_end;
+ uint32_t height;
+} TrieNode;
+} // namespace uzu::trie
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
index 88130e664..2e0685009 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
@@ -2,7 +2,7 @@
#include "loader.h"
#include "threadgroup_tile.h"
-#include "../../../generated/matmul.h"
+#include "../../generated/matmul.h"
using namespace metal;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
index c4e885f9f..a275b3d00 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
@@ -2,7 +2,7 @@
#include "defines.h"
#include "loader.h"
-#include "../../../generated/matmul.h"
+#include "../../generated/matmul.h"
#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>