A high-performance inference engine for AI models
gpu shared types autogen refactor (#336)
uuuvnf279c5f2126a
diff --git a/crates/uzu/build/common/compiler.rs b/crates/uzu/build/common/compiler.rsindex 9ffc5b0a7..564505624 100644--- a/crates/uzu/build/common/compiler.rs+++ b/crates/uzu/build/common/compiler.rs@@ -2,9 +2,12 @@ use std::collections::HashMap;use async_trait::async_trait;-use super::kernel::Kernel;+use super::{gpu_types::GpuTypes, kernel::Kernel};#[async_trait]pub trait Compiler {- async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>;+ async fn build(+ &self,+ gpu_types: &GpuTypes,+ ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>>;}diff --git a/crates/uzu/build/common/gpu_types/item_enum.rs b/crates/uzu/build/common/gpu_types/item_enum.rsnew file mode 100644index 000000000..4521a2366--- /dev/null+++ b/crates/uzu/build/common/gpu_types/item_enum.rs@@ -0,0 +1,63 @@+use anyhow::{Context, bail, ensure};+use syn::{Expr, ExprLit, ItemEnum, Lit, Variant};++use crate::common::gpu_types::ensure_repr_c;++#[derive(Debug)]+pub struct GpuTypeEnumVariant {+ pub name: Box<str>,+ pub discriminant: u32,+}++impl GpuTypeEnumVariant {+ fn parse(+ variant: Variant,+ discriminant: &mut u32,+ ) -> anyhow::Result<Self> {+ let name = variant.ident.to_string().into();++ if let Some((_, cur_discriminant)) = variant.discriminant {+ let Expr::Lit(ExprLit {+ attrs: _,+ lit: Lit::Int(cur_discriminant),+ }) = cur_discriminant+ else {+ bail!("Expected integer discriminant, found: {cur_discriminant:?}");+ };+ *discriminant = cur_discriminant.base10_parse().context("Cannot parse discriminant")?;+ }++ let cur_discriminant = *discriminant;+ *discriminant += 1;++ ensure!(variant.fields.is_empty(), "Enum variant with fields is not supported yet");++ Ok(Self {+ name,+ discriminant: cur_discriminant,+ })+ }+}++#[derive(Debug)]+pub struct GpuTypeEnum {+ pub name: Box<str>,+ pub variants: Box<[GpuTypeEnumVariant]>,+}++impl GpuTypeEnum {+ pub fn parse(item: ItemEnum) -> anyhow::Result<Self> {+ ensure_repr_c(&item.attrs)?;++ Ok(Self {+ name: item.ident.to_string().into(),+ variants: {+ let mut discriminant = 0;+ item.variants+ .into_iter()+ .map(|variant| GpuTypeEnumVariant::parse(variant, &mut discriminant))+ .collect::<anyhow::Result<Box<[GpuTypeEnumVariant]>>>()?+ },+ })+ }+}diff --git a/crates/uzu/build/common/gpu_types/item_struct.rs b/crates/uzu/build/common/gpu_types/item_struct.rsnew file mode 100644index 000000000..fe1d1698d--- /dev/null+++ b/crates/uzu/build/common/gpu_types/item_struct.rs@@ -0,0 +1,89 @@+use anyhow::{Context, anyhow, bail};+use quote::ToTokens;+use syn::{Expr, ExprLit, Fields, ItemStruct, Lit, Type, TypeArray, TypePath};++use crate::common::gpu_types::ensure_repr_c;++#[derive(Debug)]+pub enum GpuTypeStructFieldType {+ Scalar(Box<str>),+ Array {+ element: Box<str>,+ length: usize,+ },+}++impl GpuTypeStructFieldType {+ fn parse(ty: Type) -> anyhow::Result<Self> {+ match ty {+ Type::Path(TypePath {+ qself: _,+ path,+ }) => Ok(Self::Scalar(path.into_token_stream().to_string().into())),+ Type::Array(TypeArray {+ bracket_token: _,+ elem,+ semi_token: _,+ len,+ }) => {+ let Type::Path(elem) = *elem else {+ bail!("Array element must be a Path, found {elem:?}");+ };++ let Expr::Lit(ExprLit {+ attrs: _,+ lit: Lit::Int(len),+ }) = len+ else {+ bail!("Array length must be an integer literal, found: {len:?}");+ };++ Ok(Self::Array {+ element: elem.into_token_stream().to_string().into(),+ length: len.base10_parse().context("Cannot parse array length")?,+ })+ },+ ty => Err(anyhow!("Expected Array or Path, found {ty:?}")),+ }+ }+}++#[derive(Debug)]+pub struct GpuTypeStructField {+ pub name: Box<str>,+ pub ty: GpuTypeStructFieldType,+}++#[derive(Debug)]+pub struct GpuTypeStruct {+ pub name: Box<str>,+ pub fields: Box<[GpuTypeStructField]>,+}++impl GpuTypeStruct {+ pub fn parse(item: ItemStruct) -> anyhow::Result<Self> {+ ensure_repr_c(&item.attrs)?;++ let name = item.ident.to_string().into();++ let Fields::Named(item_fields) = item.fields else {+ bail!("Only structs with named fields are supported, found: {:?}", item.fields);+ };++ let fields = item_fields+ .named+ .into_iter()+ .map(|field| {+ Ok(GpuTypeStructField {+ name: field.ident.context("Field doesn't have ident")?.to_string().into(),+ ty: GpuTypeStructFieldType::parse(field.ty).context("Cannot parse field type")?,+ })+ })+ .collect::<anyhow::Result<Box<[GpuTypeStructField]>>>()?;++ Ok(Self {+ name,+ fields,+ })+ }+}diff --git a/crates/uzu/build/common/gpu_types/mod.rs b/crates/uzu/build/common/gpu_types/mod.rsnew file mode 100644index 000000000..515a2a75f--- /dev/null+++ b/crates/uzu/build/common/gpu_types/mod.rs@@ -0,0 +1,91 @@+#![allow(unused)]+use std::{+ env, fs,+ path::{Path, PathBuf},+};++use anyhow::Context;+use syn::{Attribute, Ident, Item};+use walkdir::WalkDir;++mod item_enum;+mod item_struct;++pub use item_enum::GpuTypeEnum;+pub use item_struct::{GpuTypeStruct, GpuTypeStructFieldType};++fn ensure_repr_c(attrs: &[Attribute]) -> anyhow::Result<()> {+ anyhow::ensure!(+ attrs.iter().any(|attr| attr.path().is_ident("repr") && attr.parse_args::<Ident>().is_ok_and(|arg| arg == "C")),+ "Does not have repr(c)"+ );++ Ok(())+}++#[derive(Debug)]+pub enum GpuType {+ Enum(GpuTypeEnum),+ Struct(GpuTypeStruct),+}++#[derive(Debug)]+pub struct GpuTypeFile {+ pub name: Box<str>,+ pub types: Box<[GpuType]>,+}++impl GpuTypeFile {+ fn scan(path: &Path) -> anyhow::Result<Self> {+ let name = path.file_stem().context("No file stem")?.to_string_lossy().into();++ let source = fs::read_to_string(path).context("Cannot read source")?;+ let ast = syn::parse_file(&source).context("Cannot parse ast")?;++ let tys = ast+ .items+ .into_iter()+ .filter_map(|item| match item {+ Item::Enum(item) => Some(GpuTypeEnum::parse(item).map(GpuType::Enum)),+ Item::Struct(item) => Some(GpuTypeStruct::parse(item).map(GpuType::Struct)),+ _ => None,+ })+ .collect::<anyhow::Result<Box<[GpuType]>>>()?;++ Ok(Self {+ name,+ types: tys,+ })+ }+}++#[derive(Debug)]+pub struct GpuTypes {+ pub files: Box<[GpuTypeFile]>,+}++impl GpuTypes {+ pub fn scan() -> anyhow::Result<Self> {+ let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("Missing CARGO_MANIFEST_DIR")?)+ .join("src/backends/common/gpu_types");++ let mut sources: Vec<PathBuf> = WalkDir::new(&src_dir)+ .into_iter()+ .filter_map(|e| e.ok())+ .filter_map(|e| e.file_type().is_file().then(|| e.into_path()))+ .filter(|e| e.extension().and_then(|s| s.to_str()) == Some("rs") && e.file_stem() != Some("mod".as_ref()))+ .collect();++ sources.sort();++ Ok(Self {+ files: sources+ .into_iter()+ .map(|source| {+ GpuTypeFile::scan(&source)+ .with_context(|| format!("Failed to scan {}", source.as_os_str().to_str().unwrap()))+ })+ .collect::<anyhow::Result<Box<[GpuTypeFile]>>>()?,+ })+ }+}diff --git a/crates/uzu/build/common/mod.rs b/crates/uzu/build/common/mod.rsindex c7b97f162..237c705e1 100644--- a/crates/uzu/build/common/mod.rs+++ b/crates/uzu/build/common/mod.rs@@ -2,6 +2,7 @@ pub mod caching;pub mod codegen;pub mod compiler;pub mod envs;+pub mod gpu_types;pub mod kernel;pub mod logging;pub mod mangling;diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rsindex d825f48de..a9c5535e0 100644--- a/crates/uzu/build/cpu/compiler.rs+++ b/crates/uzu/build/cpu/compiler.rs@@ -14,6 +14,7 @@ use walkdir::WalkDir;use crate::common::{codegen::write_tokens,compiler::Compiler,+ gpu_types::GpuTypes,kernel::{Kernel, KernelArgument, KernelArgumentType, KernelBufferAccess, KernelParameter, KernelParameterType},};@@ -601,7 +602,10 @@ impl CpuCompiler {#[async_trait]impl Compiler for CpuCompiler {- async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {+ async fn build(+ &self,+ _gpu_types: &GpuTypes,+ ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {let objects = WalkDir::new(&self.src_dir).into_iter().filter_map(|e| e.ok())diff --git a/crates/uzu/build/gpu_types/mod.rs b/crates/uzu/build/gpu_types/mod.rsdeleted file mode 100644index 6ffd32d19..000000000--- a/crates/uzu/build/gpu_types/mod.rs+++ /dev/null@@ -1,338 +0,0 @@-use std::{collections::HashMap, env, fs, path::PathBuf};--use anyhow::Context;-use async_trait::async_trait;-use syn::{Attribute, Expr, ExprLit, Fields, Item, ItemEnum, ItemStruct, Lit, Meta, Type, TypeArray, TypePath};-use tokio::task::spawn_blocking;--use crate::{- common::{compiler::Compiler, kernel::Kernel},- debug_log,-};--#[derive(Debug)]-pub struct GpuTypesCompiler {- crate_dir: PathBuf,- generated_dir: PathBuf,- build_dir: PathBuf,-}--impl GpuTypesCompiler {- pub fn new() -> anyhow::Result<Self> {- let crate_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?);-- let generated_dir = crate_dir.join("src/backends/metal/generated");- fs::create_dir_all(&generated_dir).with_context(|| format!("cannot create {}", generated_dir.display()))?;-- let build_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?).join("gpu_types");- fs::create_dir_all(&build_dir).with_context(|| format!("cannot create {}", build_dir.display()))?;-- Ok(Self {- crate_dir,- generated_dir,- build_dir,- })- }-- #[cfg(all(feature = "metal", target_os = "macos"))]- pub fn generated_header_dir(&self) -> &PathBuf {- &self.generated_dir- }-- fn generate(&self) -> anyhow::Result<()> {- let gpu_types_dir = self.crate_dir.join("src/backends/common/gpu_types");-- let source_files: Vec<PathBuf> = fs::read_dir(&gpu_types_dir)- .with_context(|| format!("cannot read {}", gpu_types_dir.display()))?- .filter_map(|entry| entry.ok())- .map(|entry| entry.path())- .filter(|path| {- path.extension().map(|ext| ext == "rs").unwrap_or(false) && path.file_stem() != Some("mod".as_ref())- })- .collect();-- let mut hasher = blake3::Hasher::new();- for path in &source_files {- let content = fs::read(path).with_context(|| format!("cannot read {}", path.display()))?;- hasher.update(&content);- }-- let source_hash: [u8; blake3::OUT_LEN] = hasher.finalize().into();- let cached_hash_path = self.build_dir.join("gpu_types.hash");-- if let Ok(cached_hash) = fs::read(&cached_hash_path) {- if cached_hash == source_hash {- debug_log!("gpu_types cached");- return Ok(());- }- }-- debug_log!("gpu_types generation started");-- for src_path in &source_files {- self.generate_header_for_file(src_path)?;- }-- fs::write(&cached_hash_path, &source_hash)- .with_context(|| format!("cannot write {}", cached_hash_path.display()))?;-- debug_log!("gpu_types generation done");-- Ok(())- }-- fn generate_header_for_file(- &self,- src_path: &PathBuf,- ) -> anyhow::Result<()> {- let module_name = src_path.file_stem().and_then(|s| s.to_str()).context("invalid source file name")?;-- let source = fs::read_to_string(src_path).with_context(|| format!("cannot read {}", src_path.display()))?;-- let syntax = syn::parse_file(&source).with_context(|| format!("cannot parse {}", src_path.display()))?;-- let mut repr_c_enums: Vec<ItemEnum> = Vec::new();- let mut repr_c_structs: Vec<ItemStruct> = Vec::new();- for item in syntax.items {- if let Item::Enum(item_enum) = item {- if have_repr_c_attr(&item_enum.attrs) {- repr_c_enums.push(item_enum);- }- } else if let Item::Struct(item_struct) = item {- if have_repr_c_attr(&item_struct.attrs) {- repr_c_structs.push(item_struct);- }- }- }-- let c_enum_definitions = Self::get_enum_definitions(&repr_c_enums);- let c_struct_definitions = Self::get_struct_definitions(&repr_c_structs);-- let mut content = String::new();- if !c_enum_definitions.is_empty() {- content.push_str(&c_enum_definitions);- }- if !c_struct_definitions.is_empty() {- if !content.is_empty() {- content.push_str("\n\n");- }- content.push_str(&c_struct_definitions);- }-- self.write_header(module_name, content.as_str())?;-- Ok(())- }-- fn get_struct_definitions(c_structs: &Vec<ItemStruct>) -> String {- let mut definitions = String::new();- for (i, rust_struct) in c_structs.iter().enumerate() {- for attribute in &rust_struct.attrs {- if attribute.path().is_ident("doc") {- if let Meta::NameValue(meta_name_value) = &attribute.meta {- if let Expr::Lit(ExprLit {- lit: Lit::Str(literal_string),- ..- }) = &meta_name_value.value- {- let doc_comment = literal_string.value();- let doc_comment = doc_comment.trim();- if !doc_comment.is_empty() {- definitions.push_str("/**");- definitions.push_str(doc_comment);- definitions.push_str(" */\n");- }- }- }- }- }-- definitions.push_str("typedef struct {\n");-- if let Fields::Named(named_fields) = &rust_struct.fields {- for field in &named_fields.named {- let field_name = field.ident.as_ref().map(|ident| ident.to_string()).unwrap_or_default();- let c_type = rust_type_to_c(&field.ty);-- for attribute in &field.attrs {- if attribute.path().is_ident("doc") {- if let Meta::NameValue(meta_name_value) = &attribute.meta {- if let Expr::Lit(ExprLit {- lit: Lit::Str(literal_string),- ..- }) = &meta_name_value.value- {- let doc_comment = literal_string.value();- let doc_comment = doc_comment.trim();- if !doc_comment.is_empty() {- definitions.push_str(" /**");- definitions.push_str(doc_comment);- definitions.push_str(" */\n");- }- }- }- }- }-- if let Some((base_type, array_size)) = c_type.split_once('[') {- definitions.push_str(&format!(" {base_type} {field_name}[{array_size};\n"));- } else {- definitions.push_str(&format!(" {c_type} {field_name};\n"));- }- }- }-- let struct_name = rust_struct.ident.to_string();- definitions.push_str(&format!("}} {struct_name};"));- if i < c_structs.len() - 1 {- definitions.push_str("\n\n");- }- }-- definitions- }-- fn get_enum_definitions(enums: &Vec<ItemEnum>) -> String {- let mut definitions = String::new();- for (i, rust_enum) in enums.iter().enumerate() {- let enum_name = rust_enum.ident.to_string();- definitions.push_str("enum ");- definitions.push_str(&enum_name);- definitions.push_str(" : uint32_t {\n");-- for variant in &rust_enum.variants {- definitions.push_str(" ");- definitions.push_str(variant.ident.to_string().as_str());- definitions.push_str(",\n");- }-- definitions.push_str("};");- if i < enums.len() - 1 {- definitions.push_str("\n\n");- }- }- definitions- }-- fn write_header(- &self,- module_name: &str,- content: &str,- ) -> anyhow::Result<()> {- let include_guard = format!("UZU_{}_H", module_name.to_uppercase());-- let header_content = format!(- r#"// Auto-generated from gpu_types/{module_name}.rs - do not edit manually-#pragma once--#ifndef {include_guard}-#define {include_guard}--#ifdef __METAL_VERSION__-#include <metal_stdlib>-using namespace metal;--namespace uzu {{-namespace {module_name} {{-#else-#include <stdint.h>-#endif--{content}--#ifdef __METAL_VERSION__-}} // namespace {module_name}-}} // namespace uzu-#endif--#endif // {include_guard}-"#- );-- let output_path = self.generated_dir.join(format!("{module_name}.h"));- fs::write(&output_path, &header_content).with_context(|| format!("cannot write {}", output_path.display()))?;-- debug_log!(" -> {}", output_path.display());-- Ok(())- }-}--fn rust_type_to_c(rust_type: &Type) -> String {- match rust_type {- Type::Path(TypePath {- path,- ..- }) => {- let segment = path.segments.last().unwrap();- let type_name = segment.ident.to_string();-- match type_name.as_str() {- "i8" => "int8_t".to_string(),- "i16" => "int16_t".to_string(),- "i32" => "int32_t".to_string(),- "i64" => "int64_t".to_string(),- "u8" => "uint8_t".to_string(),- "u16" => "uint16_t".to_string(),- "u32" => "uint32_t".to_string(),- "u64" => "uint64_t".to_string(),- "f32" => "float".to_string(),- "f64" => "double".to_string(),- "bool" => "bool".to_string(),- "usize" => "size_t".to_string(),- "isize" => "ptrdiff_t".to_string(),- _ => type_name,- }- },- Type::Array(TypeArray {- elem,- len,- ..- }) => {- let element_type = rust_type_to_c(elem);- if let Expr::Lit(ExprLit {- lit: Lit::Int(literal_int),- ..- }) = len- {- let array_length = literal_int.base10_digits();- format!("{element_type}[{array_length}]")- } else {- format!("{element_type}[]")- }- },- _ => "/* unknown */".to_string(),- }-}--fn have_repr_c_attr(attrs: &Vec<Attribute>) -> bool {- attrs.iter().any(|attr| {- if attr.path().is_ident("repr") {- if let Ok(ident) = attr.parse_args::<syn::Ident>() {- return ident == "C";- }- }- false- })-}--#[async_trait]-impl Compiler for GpuTypesCompiler {- async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {- let crate_dir = self.crate_dir.clone();- let generated_dir = self.generated_dir.clone();- let build_dir = self.build_dir.clone();-- spawn_blocking(move || {- GpuTypesCompiler {- crate_dir,- generated_dir,- build_dir,- }- .generate()- })- .await??;-- Ok(HashMap::new())- }-}diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rsindex 46995f3dc..7bb78cade 100644--- a/crates/uzu/build/main.rs+++ b/crates/uzu/build/main.rs@@ -4,15 +4,13 @@ use anyhow::Context;use futures::future::try_join_all;mod common;-mod gpu_types;+use common::{compiler::Compiler, envs, gpu_types::GpuTypes, traitgen::traitgen_all};mod cpu;#[cfg(all(feature = "metal", target_os = "macos"))]mod metal;-use common::{compiler::Compiler, envs, traitgen::traitgen_all};-#[tokio::main(flavor = "current_thread")]async fn main() -> anyhow::Result<()> {println!("cargo::rerun-if-changed=.");@@ -44,9 +42,8 @@ async fn main() -> anyhow::Result<()> {debug_log!("cleaned caches");}- let gpu_types_compiler = gpu_types::GpuTypesCompiler::new()?;- gpu_types_compiler.build().await?;- debug_log!("gpu_types build done");+ let gpu_types = GpuTypes::scan().context("Failed to scan gpu types")?;+ debug_log!("gpu_types scan done");let mut compilers: Vec<Box<dyn Compiler>> = Vec::new();@@ -54,11 +51,10 @@ async fn main() -> anyhow::Result<()> {#[cfg(all(feature = "metal", target_os = "macos"))]if metal_backend {- let generated_header_dir = gpu_types_compiler.generated_header_dir().clone();- compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(generated_header_dir)?));+ compilers.push(Box::new(metal::MetalCompiler::new()?));}- let backends_kernels = try_join_all(compilers.iter().map(|c| c.build())).await?;+ let backends_kernels = try_join_all(compilers.iter().map(|c| c.build(&gpu_types))).await?;debug_log!("backend build end");diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rsindex ae3db74d6..03c759d02 100644--- a/crates/uzu/build/metal/compiler.rs+++ b/crates/uzu/build/metal/compiler.rs@@ -15,8 +15,9 @@ use super::{wrapper::{SpecializeBaseIndices, wrappers},};use crate::{- common::{caching, codegen::write_tokens, compiler::Compiler, envs, kernel::Kernel},+ common::{caching, codegen::write_tokens, compiler::Compiler, envs, gpu_types::GpuTypes, kernel::Kernel},debug_log,+ metal::gpu_types::gpu_type_gen,};#[derive(Serialize, Deserialize, Debug)]@@ -83,25 +84,29 @@ fn objects_hash<'a>(objects: impl IntoIterator<Item = &'a ObjectInfo>) -> anyhow#[derive(Debug)]pub struct MetalCompiler {src_dir: PathBuf,+ gpu_types_dir: PathBuf,build_dir: PathBuf,out_dir: PathBuf,toolchain: MetalToolchain,}impl MetalCompiler {- pub fn new_with_include_dir(include_dir: PathBuf) -> anyhow::Result<Self> {+ pub fn new() -> anyhow::Result<Self> {let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?).join("src/backends/metal/kernel");+ let gpu_types_dir = src_dir.join("generated");+let out_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);let build_dir = out_dir.join("metal");fs::create_dir_all(&build_dir).with_context(|| format!("cannot create {}", build_dir.display()))?;- let toolchain =- MetalToolchain::from_env_with_include_dir(Some(include_dir)).context("cannot create toolchain")?;+ let toolchain = MetalToolchain::from_env_with_include_dir(Some(gpu_types_dir.clone()))+ .context("cannot create toolchain")?;Ok(Self {src_dir,+ gpu_types_dir,build_dir,out_dir,toolchain,@@ -313,7 +318,12 @@ impl MetalCompiler {#[async_trait]impl Compiler for MetalCompiler {- async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {+ async fn build(+ &self,+ gpu_types: &GpuTypes,+ ) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {+ gpu_type_gen(&self.gpu_types_dir, gpu_types).await.context("cannot generate shared gpu types")?;+let metal_sources: Vec<PathBuf> = WalkDir::new(&self.src_dir).into_iter().filter_map(|e| e.ok())diff --git a/crates/uzu/build/metal/gpu_types/gpu_type_enum.rs b/crates/uzu/build/metal/gpu_types/gpu_type_enum.rsnew file mode 100644index 000000000..e338a3876--- /dev/null+++ b/crates/uzu/build/metal/gpu_types/gpu_type_enum.rs@@ -0,0 +1,12 @@+use std::iter::once;++use itertools::Itertools;++use crate::common::gpu_types::GpuTypeEnum;++pub fn gpu_type_gen_enum(gpu_type_enum: &GpuTypeEnum) -> anyhow::Result<String> {+ Ok(once(format!("enum class {} : uint32_t {{", gpu_type_enum.name.as_ref()))+ .chain(gpu_type_enum.variants.iter().map(|variant| format!(" {} = {},", variant.name, variant.discriminant)))+ .chain(once("};".into()))+ .join("\n"))+}diff --git a/crates/uzu/build/metal/gpu_types/gpu_type_struct.rs b/crates/uzu/build/metal/gpu_types/gpu_type_struct.rsnew file mode 100644index 000000000..111513edb--- /dev/null+++ b/crates/uzu/build/metal/gpu_types/gpu_type_struct.rs@@ -0,0 +1,37 @@+use std::iter::once;++use anyhow::anyhow;+use itertools::Itertools;++use crate::common::gpu_types::{GpuTypeStruct, GpuTypeStructFieldType};++fn r2c(ty: &str) -> anyhow::Result<&'static str> {+ match ty {+ "i8" => Ok("int8_t"),+ "i16" => Ok("int16_t"),+ "i32" => Ok("int32_t"),+ "i64" => Ok("int64_t"),+ "u8" => Ok("uint8_t"),+ "u16" => Ok("uint16_t"),+ "u32" => Ok("uint32_t"),+ "u64" => Ok("uint64_t"),+ "f32" => Ok("float"),+ "bool" => Ok("bool"),+ "usize" => Ok("size_t"),+ "isize" => Ok("ptrdiff_t"),+ _ => Err(anyhow!("Only primitive types are supported, found {ty}")),+ }+}++pub fn gpu_type_gen_struct(gpu_type_struct: &GpuTypeStruct) -> anyhow::Result<String> {+ once(Ok("typedef struct {".into()))+ .chain(gpu_type_struct.fields.iter().map(|field| match &field.ty {+ GpuTypeStructFieldType::Scalar(ty) => Ok(format!(" {} {};", r2c(ty.as_ref())?, field.name)),+ GpuTypeStructFieldType::Array {+ element,+ length,+ } => Ok(format!(" {} {}[{}];", r2c(element.as_ref())?, field.name, *length)),+ }))+ .chain(once(Ok(format!("}} {};", gpu_type_struct.name.as_ref()))))+ .process_results(|mut it| it.join("\n"))+}diff --git a/crates/uzu/build/metal/gpu_types/mod.rs b/crates/uzu/build/metal/gpu_types/mod.rsnew file mode 100644index 000000000..14e3b25ab--- /dev/null+++ b/crates/uzu/build/metal/gpu_types/mod.rs@@ -0,0 +1,52 @@+use std::path::Path;++use anyhow::Context;+use itertools::Itertools;++use crate::common::gpu_types::{GpuType, GpuTypeFile, GpuTypes};++mod gpu_type_enum;+mod gpu_type_struct;++use gpu_type_enum::gpu_type_gen_enum;+use gpu_type_struct::gpu_type_gen_struct;++pub async fn gpu_type_gen(+ gpu_types_dir: &Path,+ gpu_types: &GpuTypes,+) -> anyhow::Result<()> {+ for gpu_type_file in &gpu_types.files {+ gpu_type_gen_file(&gpu_types_dir.join(gpu_type_file.name.as_ref()).with_extension("h"), gpu_type_file)+ .await+ .with_context(|| format!("Cannot generate gpu types for {}", gpu_type_file.name.as_ref()))?;+ }++ Ok(())+}++async fn gpu_type_gen_file(+ file_path: &Path,+ gpu_types_file: &GpuTypeFile,+) -> anyhow::Result<()> {+ let module_name = gpu_types_file.name.as_ref();++ let generated = gpu_types_file+ .types+ .iter()+ .map(|gpu_type| match gpu_type {+ GpuType::Enum(gpu_type_enum) => gpu_type_gen_enum(gpu_type_enum)+ .with_context(|| format!("Failed to generate bindings for {gpu_type_enum:?}")),+ GpuType::Struct(gpu_type_struct) => gpu_type_gen_struct(gpu_type_struct)+ .with_context(|| format!("Failed to generate bindings for {gpu_type_struct:?}")),+ })+ .process_results(|mut it| it.join("\n\n"))?;++ let new_contents = format!(include_str!("template.ht"), module_name = module_name, generated = generated);++ // Avoid advancing mtime if the contents are the same+ if !tokio::fs::read(&file_path).await.is_ok_and(|old_contents| &old_contents == new_contents.as_bytes()) {+ tokio::fs::write(&file_path, new_contents).await.context("cannot write output")?;+ }++ Ok(())+}diff --git a/crates/uzu/build/metal/gpu_types/template.ht b/crates/uzu/build/metal/gpu_types/template.htnew file mode 100644index 000000000..8696754a5--- /dev/null+++ b/crates/uzu/build/metal/gpu_types/template.ht@@ -0,0 +1,9 @@+// Auto-generated from gpu_types/{module_name}.rs - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::{module_name} {{+{generated}+}} // namespace uzu::{module_name}diff --git a/crates/uzu/build/metal/mod.rs b/crates/uzu/build/metal/mod.rsindex 7d2d4a094..9e5a670db 100644--- a/crates/uzu/build/metal/mod.rs+++ b/crates/uzu/build/metal/mod.rs@@ -1,6 +1,7 @@mod ast;mod bindgen;mod compiler;+mod gpu_types;mod toolchain;mod wrapper;diff --git a/crates/uzu/src/backends/metal/generated/activation_type.h b/crates/uzu/src/backends/metal/generated/activation_type.hdeleted file mode 100644index 2cf6beef1..000000000--- a/crates/uzu/src/backends/metal/generated/activation_type.h+++ /dev/null@@ -1,30 +0,0 @@-// Auto-generated from gpu_types/activation_type.rs - do not edit manually-#pragma once--#ifndef UZU_ACTIVATION_TYPE_H-#define UZU_ACTIVATION_TYPE_H--#ifdef __METAL_VERSION__-#include <metal_stdlib>-using namespace metal;--namespace uzu {-namespace activation_type {-#else-#include <stdint.h>-#endif--enum ActivationType : uint32_t {- SILU,- GELU,- TANH,- IDENTITY,- SOFTPLUS,-};--#ifdef __METAL_VERSION__-} // namespace activation_type-} // namespace uzu-#endif--#endif // UZU_ACTIVATION_TYPE_Hdiff --git a/crates/uzu/src/backends/metal/generated/argmax.h b/crates/uzu/src/backends/metal/generated/argmax.hdeleted file mode 100644index 8b9bcde16..000000000--- a/crates/uzu/src/backends/metal/generated/argmax.h+++ /dev/null@@ -1,27 +0,0 @@-// Auto-generated from gpu_types/argmax.rs - do not edit manually-#pragma once--#ifndef UZU_ARGMAX_H-#define UZU_ARGMAX_H--#ifdef __METAL_VERSION__-#include <metal_stdlib>-using namespace metal;--namespace uzu {-namespace argmax {-#else-#include <stdint.h>-#endif--typedef struct {- float value;- uint32_t index;-} ArgmaxPair;--#ifdef __METAL_VERSION__-} // namespace argmax-} // namespace uzu-#endif--#endif // UZU_ARGMAX_Hdiff --git a/crates/uzu/src/backends/metal/generated/attention.h b/crates/uzu/src/backends/metal/generated/attention.hdeleted file mode 100644index 7e3e3b787..000000000--- a/crates/uzu/src/backends/metal/generated/attention.h+++ /dev/null@@ -1,54 +0,0 @@-// Auto-generated from gpu_types/attention.rs - do not edit manually-#pragma once--#ifndef UZU_ATTENTION_H-#define UZU_ATTENTION_H--#ifdef __METAL_VERSION__-#include <metal_stdlib>-using namespace metal;--namespace uzu {-namespace attention {-#else-#include <stdint.h>-#endif--/**Parameters for the GEMM attention kernel. */-/**All strides are in **elements**, not bytes. */-/**Q/K/V/O are treated as 3D tensors with strides for: */-/**- `[0]` batch */-/**- `[1]` head (or kv-head for K/V) */-/**- `[2]` sequence (row stride for [seq, head_dim]) */-typedef struct {- int64_t q_strides[3];- int64_t k_strides[3];- int64_t v_strides[3];- int64_t o_strides[3];- int32_t gqa_factor;- float scale;- /**Query length (suffix length) */- int32_t q_len;- /**Key length (prefix + suffix) */- int32_t k_len;- /**Absolute offset of the first query token in the full key sequence. */- /**For LLM decode/prefill this is the segment prefix length. */- int32_t q_off;- /**Query tiling: q_len / BQ */- int32_t nq_aligned;- /**Query tiling: q_len % BQ */- int32_t q_rem;- /**Key tiling: ceil(k_len / BK) */- int32_t nk;- /**Key tiling: k_len / BK */- int32_t nk_aligned;- /**Key tiling: k_len % BK */- int32_t k_rem;-} AttnParams;--#ifdef __METAL_VERSION__-} // namespace attention-} // namespace uzu-#endif--#endif // UZU_ATTENTION_Hdiff --git a/crates/uzu/src/backends/metal/generated/kv_cache_update.h b/crates/uzu/src/backends/metal/generated/kv_cache_update.hdeleted file mode 100644index fe32897f8..000000000--- a/crates/uzu/src/backends/metal/generated/kv_cache_update.h+++ /dev/null@@ -1,27 +0,0 @@-// Auto-generated from gpu_types/kv_cache_update.rs - do not edit manually-#pragma once--#ifndef UZU_KV_CACHE_UPDATE_H-#define UZU_KV_CACHE_UPDATE_H--#ifdef __METAL_VERSION__-#include <metal_stdlib>-using namespace metal;--namespace uzu {-namespace kv_cache_update {-#else-#include <stdint.h>-#endif--typedef struct {- uint32_t source;- uint32_t destination;-} Swap;--#ifdef __METAL_VERSION__-} // namespace kv_cache_update-} // namespace uzu-#endif--#endif // UZU_KV_CACHE_UPDATE_Hdiff --git a/crates/uzu/src/backends/metal/generated/quantization.h b/crates/uzu/src/backends/metal/generated/quantization.hdeleted file mode 100644index c73e771ad..000000000--- a/crates/uzu/src/backends/metal/generated/quantization.h+++ /dev/null@@ -1,28 +0,0 @@-// Auto-generated from gpu_types/quantization.rs - do not edit manually-#pragma once--#ifndef UZU_QUANTIZATION_H-#define UZU_QUANTIZATION_H--#ifdef __METAL_VERSION__-#include <metal_stdlib>-using namespace metal;--namespace uzu {-namespace quantization {-#else-#include <stdint.h>-#endif--enum QuantizationMode : uint32_t {- UINT4,- INT8,- UINT8,-};--#ifdef __METAL_VERSION__-} // namespace quantization-} // namespace uzu-#endif--#endif // UZU_QUANTIZATION_Hdiff --git a/crates/uzu/src/backends/metal/generated/ring.h b/crates/uzu/src/backends/metal/generated/ring.hdeleted file mode 100644index 6819327e9..000000000--- a/crates/uzu/src/backends/metal/generated/ring.h+++ /dev/null@@ -1,27 +0,0 @@-// Auto-generated from gpu_types/ring.rs - do not edit manually-#pragma once--#ifndef UZU_RING_H-#define UZU_RING_H--#ifdef __METAL_VERSION__-#include <metal_stdlib>-using namespace metal;--namespace uzu {-namespace ring {-#else-#include <stdint.h>-#endif--typedef struct {- uint32_t ring_offset;- uint32_t ring_length;-} RingParams;--#ifdef __METAL_VERSION__-} // namespace ring-} // namespace uzu-#endif--#endif // UZU_RING_Hdiff --git a/crates/uzu/src/backends/metal/generated/trie.h b/crates/uzu/src/backends/metal/generated/trie.hdeleted file mode 100644index 2c2ab2497..000000000--- a/crates/uzu/src/backends/metal/generated/trie.h+++ /dev/null@@ -1,28 +0,0 @@-// Auto-generated from gpu_types/trie.rs - do not edit manually-#pragma once--#ifndef UZU_TRIE_H-#define UZU_TRIE_H--#ifdef __METAL_VERSION__-#include <metal_stdlib>-using namespace metal;--namespace uzu {-namespace trie {-#else-#include <stdint.h>-#endif--typedef struct {- uint32_t trie_start;- uint32_t trie_end;- uint32_t height;-} TrieNode;--#ifdef __METAL_VERSION__-} // namespace trie-} // namespace uzu-#endif--#endif // UZU_TRIE_Hdiff --git a/crates/uzu/src/backends/metal/kernel/activation/activations.h b/crates/uzu/src/backends/metal/kernel/activation/activations.hindex 6df1548ae..b05b638c4 100644--- a/crates/uzu/src/backends/metal/kernel/activation/activations.h+++ b/crates/uzu/src/backends/metal/kernel/activation/activations.h@@ -54,15 +54,15 @@ inline T activate_tanh(T x) {template <typename T>inline T activate(T x, ActivationType type) {switch (type) {- case SILU:+ case ActivationType::SILU:return activate_silu(x);- case GELU:+ case ActivationType::GELU:return activate_gelu(x);- case TANH:+ case ActivationType::TANH:return activate_tanh(x);- case IDENTITY:+ case ActivationType::IDENTITY:return x;- case SOFTPLUS:+ case ActivationType::SOFTPLUS:return activate_softplus(x);}}diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex 67b7d6c92..f06629847 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -21,6 +21,9 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (const uint dim_idx AXIS(model_dim, 16),const uint batch_idx AXIS(batch_size, 16)) {+ const QuantizationMode quantization_mode = QuantizationMode(+ quant_mode+ ); // TODO: should be accepted as a kernel argumentconst uint thread_position_in_grid = batch_idx * model_dim + dim_idx;const uint64_t token_id = token_ids[batch_idx];if (token_id >= vocab_size) {@@ -34,12 +37,13 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (const T scale = scales[token_id * num_groups + group_idx];const T bias = biases[token_id * num_groups + group_idx];- const uint packing_divisor = quant_mode == QuantizationMode::UINT4 ? 2 : 1;+ const uint packing_divisor =+ quantization_mode == QuantizationMode::UINT4 ? 2 : 1;const uint weights_stride = model_dim / packing_divisor;int quantized_value = 0;- switch (quant_mode) {- case UINT4: {+ switch (quantization_mode) {+ case QuantizationMode::UINT4: {const uint byte_idx = token_id * weights_stride + (dim_idx / 2);const uint8_t packed = weights[byte_idx];if ((dim_idx & 1) == 0) {@@ -49,14 +53,14 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (}break;};- case INT8: {+ case QuantizationMode::INT8: {const uint elem_idx = token_id * weights_stride + dim_idx;const device int8_t* weights_i8 =reinterpret_cast<const device int8_t*>(weights);quantized_value = int(weights_i8[elem_idx]);break;};- case UINT8: {+ case QuantizationMode::UINT8: {const uint elem_idx = token_id * weights_stride + dim_idx;quantized_value = int(weights[elem_idx]);break;diff --git a/crates/uzu/src/backends/metal/kernel/generated/activation_type.h b/crates/uzu/src/backends/metal/kernel/generated/activation_type.hnew file mode 100644index 000000000..c5f89ed64--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/generated/activation_type.h@@ -0,0 +1,15 @@+// Auto-generated from gpu_types/activation_type.rs - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::activation_type {+enum class ActivationType : uint32_t {+ SILU = 0,+ GELU = 1,+ TANH = 2,+ IDENTITY = 3,+ SOFTPLUS = 4,+};+} // namespace uzu::activation_typediff --git a/crates/uzu/src/backends/metal/kernel/generated/argmax.h b/crates/uzu/src/backends/metal/kernel/generated/argmax.hnew file mode 100644index 000000000..3b818d559--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/generated/argmax.h@@ -0,0 +1,12 @@+// Auto-generated from gpu_types/argmax.rs - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::argmax {+typedef struct {+ float value;+ uint32_t index;+} ArgmaxPair;+} // namespace uzu::argmaxdiff --git a/crates/uzu/src/backends/metal/kernel/generated/attention.h b/crates/uzu/src/backends/metal/kernel/generated/attention.hnew file mode 100644index 000000000..b46ba583f--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/generated/attention.h@@ -0,0 +1,24 @@+// Auto-generated from gpu_types/attention.rs - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::attention {+typedef struct {+ int64_t q_strides[3];+ int64_t k_strides[3];+ int64_t v_strides[3];+ int64_t o_strides[3];+ int32_t gqa_factor;+ float scale;+ int32_t q_len;+ int32_t k_len;+ int32_t q_off;+ int32_t nq_aligned;+ int32_t q_rem;+ int32_t nk;+ int32_t nk_aligned;+ int32_t k_rem;+} AttnParams;+} // namespace uzu::attentiondiff --git a/crates/uzu/src/backends/metal/kernel/generated/kv_cache_update.h b/crates/uzu/src/backends/metal/kernel/generated/kv_cache_update.hnew file mode 100644index 000000000..c42228faa--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/generated/kv_cache_update.h@@ -0,0 +1,12 @@+// Auto-generated from gpu_types/kv_cache_update.rs - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::kv_cache_update {+typedef struct {+ uint32_t source;+ uint32_t destination;+} Swap;+} // namespace uzu::kv_cache_updatediff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/kernel/generated/matmul.hsimilarity index 65%rename from crates/uzu/src/backends/metal/generated/matmul.hrename to crates/uzu/src/backends/metal/kernel/generated/matmul.hindex 07525ecdb..400808477 100644--- a/crates/uzu/src/backends/metal/generated/matmul.h+++ b/crates/uzu/src/backends/metal/kernel/generated/matmul.h@@ -1,19 +1,10 @@// Auto-generated from gpu_types/matmul.rs - do not edit manually#pragma once-#ifndef UZU_MATMUL_H-#define UZU_MATMUL_H--#ifdef __METAL_VERSION__#include <metal_stdlib>using namespace metal;-namespace uzu {-namespace matmul {-#else-#include <stdint.h>-#endif-+namespace uzu::matmul {typedef struct {uint32_t M;uint32_t N;@@ -27,10 +18,4 @@ typedef struct {uint32_t aligned_inner_iterations;bool use_morton;} GemmParams;--#ifdef __METAL_VERSION__-} // namespace matmul-} // namespace uzu-#endif--#endif // UZU_MATMUL_H+} // namespace uzu::matmuldiff --git a/crates/uzu/src/backends/metal/kernel/generated/quantization.h b/crates/uzu/src/backends/metal/kernel/generated/quantization.hnew file mode 100644index 000000000..0315c17d3--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/generated/quantization.h@@ -0,0 +1,13 @@+// Auto-generated from gpu_types/quantization.rs - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::quantization {+enum class QuantizationMode : uint32_t {+ UINT4 = 0,+ INT8 = 1,+ UINT8 = 2,+};+} // namespace uzu::quantizationdiff --git a/crates/uzu/src/backends/metal/kernel/generated/ring.h b/crates/uzu/src/backends/metal/kernel/generated/ring.hnew file mode 100644index 000000000..a10f4c220--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/generated/ring.h@@ -0,0 +1,12 @@+// Auto-generated from gpu_types/ring.rs - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::ring {+typedef struct {+ uint32_t ring_offset;+ uint32_t ring_length;+} RingParams;+} // namespace uzu::ringdiff --git a/crates/uzu/src/backends/metal/kernel/generated/trie.h b/crates/uzu/src/backends/metal/kernel/generated/trie.hnew file mode 100644index 000000000..0ccfaf5d1--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/generated/trie.h@@ -0,0 +1,13 @@+// Auto-generated from gpu_types/trie.rs - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::trie {+typedef struct {+ uint32_t trie_start;+ uint32_t trie_end;+ uint32_t height;+} TrieNode;+} // namespace uzu::triediff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.hindex 88130e664..2e0685009 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h@@ -2,7 +2,7 @@#include "loader.h"#include "threadgroup_tile.h"-#include "../../../generated/matmul.h"+#include "../../generated/matmul.h"using namespace metal;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.hindex c4e885f9f..a275b3d00 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h@@ -2,7 +2,7 @@#include "defines.h"#include "loader.h"-#include "../../../generated/matmul.h"+#include "../../generated/matmul.h"#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>