A high-performance inference engine for AI models
Small cleanup part 4 (#955)
uuuvne7e8e1e0180d
diff --git a/.vscode/extensions.json b/.vscode/extensions.jsondeleted file mode 100644index 82e8a1e32..000000000--- a/.vscode/extensions.json+++ /dev/null@@ -1,9 +0,0 @@-{- "recommendations": [- "vadimcn.vscode-lldb",- "tamasfe.even-better-toml",- "rust-lang.rust-analyzer",- "eugenebokhan.metal-analyzer",- "augustocdias.tasks-shell-input"- ]-}\ No newline at end of filediff --git a/.vscode/keybindings.json b/.vscode/keybindings.jsondeleted file mode 100644index e247852b2..000000000--- a/.vscode/keybindings.json+++ /dev/null@@ -1,22 +0,0 @@-[- {- "key": "cmd+shift+b",- "command": "workbench.action.tasks.runTask",- "args": "cargo build"- },- {- "key": "cmd+shift+c",- "command": "workbench.action.tasks.runTask",- "args": "cargo check"- },- {- "key": "cmd+shift+t",- "command": "workbench.action.tasks.runTask",- "args": "cargo test"- },- {- "key": "cmd+shift+r",- "command": "workbench.action.tasks.runTask",- "args": "cargo run"- }-]\ No newline at end of filediff --git a/.vscode/settings.json b/.vscode/settings.jsondeleted file mode 100644index c9773071d..000000000--- a/.vscode/settings.json+++ /dev/null@@ -1,62 +0,0 @@-{- "editor.formatOnSave": true,- "editor.codeActionsOnSave": {- "source.organizeImports": "explicit",- },- "files.associations": {- "*.h": "metal",- },- "rust-analyzer.rustfmt.extraArgs": [- "+nightly"- ],- "[rust]": {- "editor.formatOnSave": true,- "editor.defaultFormatter": "rust-lang.rust-analyzer",- },- "rust-analyzer.testExplorer": true,- "rust-analyzer.procMacro.enable": true,- "[metal]": {- "editor.formatOnSave": true,- "editor.defaultFormatter": "eugenebokhan.metal-analyzer",- },- "metal-analyzer.diagnostics.onType": true,- "metal-analyzer.diagnostics.onSave": true,- "metal-analyzer.diagnostics.debounceMs": 300,- "metal-analyzer.indexing.enabled": true,- "metal-analyzer.indexing.concurrency": 8,- "metal-analyzer.indexing.maxFileSizeKb": 1024,- "metal-analyzer.formatting.enabled": true,- "metal-analyzer.formatting.command": "clang-format",- "metal-analyzer.formatting.args": [],- "metal-analyzer.diagnostics.scope": "workspace",- "metal-analyzer.compiler.includePaths": [],- "metal-analyzer.compiler.extraFlags": [],- "[python]": {- "editor.formatOnSave": true,- "editor.defaultFormatter": "charliermarsh.ruff",- "editor.codeActionsOnSave": {- "source.fixAll.ruff": "explicit",- "source.organizeImports.ruff": "explicit"- }- },- "python.defaultInterpreterPath": "tools/.venv/bin/python",- "python.testing.pytestArgs": [- "tests"- ],- "python.testing.unittestEnabled": false,- "python.testing.pytestEnabled": false,- "ruff.path": [- "tools/.venv/bin/ruff"- ],- "[swift]": {- "editor.formatOnSave": true,- },- "[toml]": {- "editor.formatOnSave": true,- "editor.defaultFormatter": "tamasfe.even-better-toml",- },- "[json]": {- "editor.formatOnSave": true,- "editor.defaultFormatter": "vscode.json-language-features"- },-}diff --git a/.vscode/tasks.json b/.vscode/tasks.jsondeleted file mode 100644index 08879423b..000000000--- a/.vscode/tasks.json+++ /dev/null@@ -1,11 +0,0 @@-{- "version": "2.0.0",- "tasks": [- {- "label": "Format Code",- "type": "shell",- "command": "cargo +nightly fmt && ./scripts/clang-format.sh",- "problemMatcher": []- }- ]-}\ No newline at end of filediff --git a/crates/legacy/cli/src/bench/model.rs b/crates/legacy/cli/src/bench/model.rsindex 6345fafa7..afbb05bb4 100644--- a/crates/legacy/cli/src/bench/model.rs+++ b/crates/legacy/cli/src/bench/model.rs@@ -4,7 +4,6 @@ use uzu::types::{basic::ReasoningEffort,session::chat::{ChatMessage, ChatReply, ChatReplyEnergy, ChatRole},};-use uzu_engine::data_type::DataType;use crate::{common::thinking::ThinkingSupport,@@ -105,7 +104,7 @@ pub struct BenchResult {pub device: BenchDevice,pub engine_version: String,pub timestamp: u64,- pub data_type: DataType,+ pub data_type: String,pub memory_used: Option<usize>,pub tokens_count_input: u64,pub tokens_count_output: u64,diff --git a/crates/legacy/cli/src/bench/runner.rs b/crates/legacy/cli/src/bench/runner.rsindex 3875553bb..767cff52d 100644--- a/crates/legacy/cli/src/bench/runner.rs+++ b/crates/legacy/cli/src/bench/runner.rs@@ -14,7 +14,7 @@ use uzu::{session::chat::{ChatConfig, ChatReplyConfig, ChatReplyEnergy},},};-use uzu_engine::{VERSION, data_type::DataType};+use uzu_engine::VERSION;use crate::{bench::{@@ -121,7 +121,7 @@ impl BenchRunner {device: device.clone(),engine_version: VERSION.to_string(),timestamp,- data_type: DataType::BF16,+ data_type: "bfloat16".to_string(),memory_used: session.peak_memory_usage().await,tokens_count_input,tokens_count_output,diff --git a/crates/legacy/uzu/src/engine/bridge/backend.rs b/crates/legacy/uzu/src/engine/bridge/backend.rsindex 2a79e874f..1dfb8360d 100644--- a/crates/legacy/uzu/src/engine/bridge/backend.rs+++ b/crates/legacy/uzu/src/engine/bridge/backend.rs@@ -13,7 +13,7 @@ use shoji::{};use uzu_engine::{TOOLCHAIN_VERSION,- backends::{BackendSelection, common::Backend, select_backend},+ backends::{Backend, BackendSelection, select_backend},};use crate::engine::bridge::{diff --git a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rsindex c5372c92d..56f527173 100644--- a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs+++ b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs@@ -27,7 +27,7 @@ use shoji::{use tokenizers::Tokenizer;use tokio_util::sync::CancellationToken;use uzu_engine::{- backends::common::Backend,+ backends::Backend,engine::{Engine,language_model::{LanguageModel, state::LanguageModelState, stream::LanguageModelStream},diff --git a/crates/legacy/uzu/src/engine/bridge/chat_token_state.rs b/crates/legacy/uzu/src/engine/bridge/chat_token_state.rsindex 7cc46b4b7..f14a9744c 100644--- a/crates/legacy/uzu/src/engine/bridge/chat_token_state.rs+++ b/crates/legacy/uzu/src/engine/bridge/chat_token_state.rs@@ -2,7 +2,7 @@ use std::sync::Arc;use parking_lot::Mutex;use shoji::traits::State as LlmInstanceState;-use uzu_engine::{backends::common::Backend, engine::language_model::state::LanguageModelState};+use uzu_engine::{backends::Backend, engine::language_model::state::LanguageModelState};pub struct UzuChatTokenBackendInstanceState<B: Backend> {pub value: Arc<Mutex<LanguageModelState<B>>>,diff --git a/crates/legacy/uzu/src/engine/bridge/classification_backend.rs b/crates/legacy/uzu/src/engine/bridge/classification_backend.rsindex febd04bbd..89e1226e9 100644--- a/crates/legacy/uzu/src/engine/bridge/classification_backend.rs+++ b/crates/legacy/uzu/src/engine/bridge/classification_backend.rs@@ -19,7 +19,7 @@ use shoji::{use tokenizers::Tokenizer;use tokio_util::sync::CancellationToken;use uzu_engine::{- backends::common::Backend,+ backends::Backend,engine::{Engine, classifier_model::ClassifierModel},};diff --git a/crates/legacy/uzu/src/engine/bridge/helpers.rs b/crates/legacy/uzu/src/engine/bridge/helpers.rsindex df9f03fec..94b2c336e 100644--- a/crates/legacy/uzu/src/engine/bridge/helpers.rs+++ b/crates/legacy/uzu/src/engine/bridge/helpers.rs@@ -12,7 +12,7 @@ use tokenizers::Tokenizer;#[cfg(feature = "capability-grammar")]use uzu_engine::engine::language_model::grammar::{Grammar as UzuGrammar, GrammarConfig, GrammarError};use uzu_engine::{- backends::common::Backend,+ backends::Backend,engine::language_model::{LanguageModel, stream::SamplingMethod as UzuSamplingMethod},};diff --git a/crates/uzu-engine/build/common/compiler.rs b/crates/uzu-engine/build/common/compiler.rsindex e0ff1571a..59f9f892b 100644--- a/crates/uzu-engine/build/common/compiler.rs+++ b/crates/uzu-engine/build/common/compiler.rs@@ -1,11 +1,11 @@use std::collections::HashMap;-use super::{enum_paths::EnumPaths, gpu_types::GpuTypes, identifiers::KernelPath, kernel::Kernel};+use super::{gpu_type_paths::GpuTypePaths, gpu_types::GpuTypes, identifiers::KernelPath, kernel::Kernel};pub trait Compiler {fn build(&self,gpu_types: &GpuTypes,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> anyhow::Result<HashMap<KernelPath, Box<[Kernel]>>>;}diff --git a/crates/uzu-engine/build/common/enum_paths.rs b/crates/uzu-engine/build/common/gpu_type_paths.rssimilarity index 50%rename from crates/uzu-engine/build/common/enum_paths.rsrename to crates/uzu-engine/build/common/gpu_type_paths.rsindex e641b6c06..0bd5123bc 100644--- a/crates/uzu-engine/build/common/enum_paths.rs+++ b/crates/uzu-engine/build/common/gpu_type_paths.rs@@ -1,14 +1,15 @@use std::collections::{HashMap, hash_map::Entry};-use anyhow::bail;+use anyhow::{Context, bail};use syn::{Type, TypePath, visit_mut::VisitMut};-use super::gpu_types::{GpuType, GpuTypeName, GpuTypePath, GpuTypes};+use crate::common::gpu_types::{GpuType, GpuTypeName, GpuTypePath, GpuTypes};#[derive(Copy, Clone, Debug, PartialEq, Eq)]pub enum GpuTypeKind {Enum,OptionSet,+ Struct,}#[derive(Clone)]@@ -18,24 +19,34 @@ struct GpuTypeEntry {}#[derive(Clone)]-pub struct EnumPaths {+pub struct GpuTypePaths {short_name_to_entry: HashMap<GpuTypeName, GpuTypeEntry>,}-impl EnumPaths {+impl GpuTypePaths {pub fn from_gpu_types(gpu_types: &GpuTypes) -> anyhow::Result<Self> {- let mut short_name_to_entry = HashMap::new();+ let mut paths = Self {+ short_name_to_entry: HashMap::new(),+ };+ paths.extend(gpu_types)?;+ Ok(paths)+ }++ pub fn extend(+ &mut self,+ gpu_types: &GpuTypes,+ ) -> anyhow::Result<()> {for file in gpu_types.files.iter() {for ty in file.types.iter() {let (name_str, kind) = match ty {GpuType::Enum(enum_type) => (enum_type.name.as_ref(), GpuTypeKind::Enum),GpuType::OptionSet(option_set) => (option_set.name.as_ref(), GpuTypeKind::OptionSet),- GpuType::Constant(_) | GpuType::Struct(_) => continue,+ GpuType::Struct(struct_type) => (struct_type.name.as_ref(), GpuTypeKind::Struct),+ GpuType::Constant(_) => continue,};let name = GpuTypeName::from(name_str);- let path =- GpuTypePath::from(format!("crate::backends::common::gpu_types::{}::{}", file.name, name_str));- match short_name_to_entry.entry(name) {+ let path = GpuTypePath::from(format!("{}::{}::{}", gpu_types.module_path, file.name, name_str));+ match self.short_name_to_entry.entry(name) {Entry::Occupied(occupied) => {bail!("gpu type `{}` is duplicated", occupied.key())},@@ -48,9 +59,7 @@ impl EnumPaths {}}}- Ok(Self {- short_name_to_entry,- })+ Ok(())}pub fn full_path_for(@@ -60,6 +69,11 @@ impl EnumPaths {self.short_name_to_entry.get(short_name).map(|entry| &*entry.path)}+ #[allow(dead_code)]+ pub fn full_paths(&self) -> impl Iterator<Item = &str> {+ self.short_name_to_entry.values().map(|entry| &*entry.path)+ }+#[allow(dead_code)]pub fn kind_for(&self,@@ -68,40 +82,29 @@ impl EnumPaths {self.short_name_to_entry.get(short_name).map(|entry| entry.kind)}- pub fn canonicalize_type(+ pub fn resolve_type(&self,- ty: &mut Type,- ) {- let mut canonicalizer = TypePathCanonicalizer {- enum_paths: self,- };- canonicalizer.visit_type_mut(ty);- }-}--struct TypePathCanonicalizer<'enum_paths> {- enum_paths: &'enum_paths EnumPaths,-}+ type_text: &str,+ ) -> anyhow::Result<Type> {+ struct TypePaths<'a>(&'a GpuTypePaths);-impl<'enum_paths> VisitMut for TypePathCanonicalizer<'enum_paths> {- fn visit_type_path_mut(- &mut self,- type_path: &mut TypePath,- ) {- let path = &type_path.path;- if type_path.qself.is_none()- && path.leading_colon.is_none()- && path.segments.len() == 1- && matches!(path.segments[0].arguments, syn::PathArguments::None)- {- let segment_name = path.segments[0].ident.to_string();- if let Some(full_path_text) = self.enum_paths.full_path_for(&segment_name)- && let Ok(full_path) = syn::parse_str::<syn::Path>(full_path_text)- {- type_path.path = full_path;- return;+ impl VisitMut for TypePaths<'_> {+ fn visit_type_path_mut(+ &mut self,+ ty: &mut TypePath,+ ) {+ syn::visit_mut::visit_type_path_mut(self, ty);+ if ty.qself.is_none()+ && let Some(ident) = ty.path.get_ident()+ && let Some(path) = self.0.full_path_for(&ident.to_string())+ {+ ty.path = syn::parse_str(path).unwrap();+ }}}- syn::visit_mut::visit_type_path_mut(self, type_path);++ let mut ty = syn::parse_str(type_text).with_context(|| format!("cannot parse type `{type_text}`"))?;+ TypePaths(self).visit_type_mut(&mut ty);+ Ok(ty)}}diff --git a/crates/uzu-engine/build/common/gpu_types/mod.rs b/crates/uzu-engine/build/common/gpu_types/mod.rsindex 63384d8c8..22c1a5a5e 100644--- a/crates/uzu-engine/build/common/gpu_types/mod.rs+++ b/crates/uzu-engine/build/common/gpu_types/mod.rs@@ -91,13 +91,14 @@ impl GpuTypeFile {#[derive(Debug)]pub struct GpuTypes {+ pub module_path: Box<str>,pub files: Box<[GpuTypeFile]>,}impl GpuTypes {- pub fn scan() -> anyhow::Result<Self> {+ pub fn scan(backend: &str) -> anyhow::Result<Self> {let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("Missing CARGO_MANIFEST_DIR")?)- .join("src/backends/common/gpu_types");+ .join(format!("src/backends/{backend}/gpu_types"));println!("cargo::rerun-if-changed={}", src_dir.display());let mut sources: Vec<PathBuf> = WalkDir::new(&src_dir)@@ -110,6 +111,7 @@ impl GpuTypes {sources.sort();Ok(Self {+ module_path: format!("crate::backends::{backend}::gpu_types").into(),files: sources.into_iter().map(|source| {diff --git a/crates/uzu-engine/build/common/mod.rs b/crates/uzu-engine/build/common/mod.rsindex f34304136..9ad50233d 100644--- a/crates/uzu-engine/build/common/mod.rs+++ b/crates/uzu-engine/build/common/mod.rs@@ -2,10 +2,10 @@ pub mod caching;pub mod codegen;pub mod compiler;pub mod constraints;-pub mod enum_paths;pub mod envs;#[cfg(all(feature = "metal", target_os = "macos"))]pub mod expr_rewrite;+pub mod gpu_type_paths;pub mod gpu_types;pub mod identifiers;pub mod kernel;diff --git a/crates/uzu-engine/build/common/traitgen.rs b/crates/uzu-engine/build/common/traitgen.rsindex 06d3e3a53..7881f6217 100644--- a/crates/uzu-engine/build/common/traitgen.rs+++ b/crates/uzu-engine/build/common/traitgen.rs@@ -4,30 +4,39 @@ use anyhow::{Context, bail};use itertools::Itertools;use proc_macro2::TokenStream;use quote::{format_ident, quote};-use syn::Type;use crate::common::{codegen::write_tokens,+ gpu_type_paths::GpuTypePaths,identifiers::KernelPath,kernel::{Kernel, KernelArgumentType, KernelBufferAccess, KernelParameterType},};-pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {+pub fn traitgen(+ kernel: &Kernel,+ gpu_type_paths: &GpuTypePaths,+) -> anyhow::Result<(TokenStream, TokenStream)> {let kernel_name = kernel.name.as_ref();let trait_name = format_ident!("{kernel_name}Kernel");- let params = kernel.parameters.iter().map(|p| {- let name = format_ident!("{}", p.name.as_ref());- let ty = match &p.ty {- KernelParameterType::Type => quote! { crate::data_type::DataType },- KernelParameterType::Value(ty) => {- let ty: Type = syn::parse_str(ty.as_ref()).unwrap();- quote! { #ty }- },- };+ let params = kernel+ .parameters+ .iter()+ .map(|p| {+ let name = format_ident!("{}", p.name.as_ref());+ let ty = match &p.ty {+ KernelParameterType::Type => quote! { crate::data_type::DataType },+ KernelParameterType::Value(ty) => {+ let ty = gpu_type_paths+ .resolve_type(ty)+ .with_context(|| format!("invalid type for parameter `{}`", p.name))?;+ quote! { #ty }+ },+ };- quote! { #name: #ty }- });+ Ok(quote! { #name: #ty })+ })+ .collect::<anyhow::Result<Vec<_>>>()?;let mut args = kernel.arguments@@ -45,7 +54,9 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {},},KernelArgumentType::Constant(ty) => {- let ty: Type = syn::parse_str(ty.as_ref()).unwrap();+ let ty = gpu_type_paths+ .resolve_type(ty)+ .with_context(|| format!("invalid type for argument `{}`", a.name))?;quote! { #ty }},};@@ -54,9 +65,9 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {ty = quote! { Option<#ty> };}- quote! { #name: #ty }+ Ok(quote! { #name: #ty })})- .collect::<Vec<_>>();+ .collect::<anyhow::Result<Vec<_>>>()?;args.push(quote! { command_buffer: &mut <<Self::Backend as crate::backends::common::Backend>::CommandBuffer as crate::backends::common::CommandBuffer>::Encoding });@@ -74,10 +85,13 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {let kernel_type = quote! { type #trait_name: #trait_name<Backend = Self::Backend>; };- (kernel_trait, kernel_type)+ Ok((kernel_trait, kernel_type))}-pub fn traitgen_all(backends_kernels: Vec<HashMap<KernelPath, Box<[Kernel]>>>) -> anyhow::Result<()> {+pub fn traitgen_all(+ backends_kernels: Vec<HashMap<KernelPath, Box<[Kernel]>>>,+ gpu_type_paths: &GpuTypePaths,+) -> anyhow::Result<()> {let out_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);let mut kernels: HashMap<KernelPath, Box<[Kernel]>> = HashMap::new();@@ -97,8 +111,11 @@ pub fn traitgen_all(backends_kernels: Vec<HashMap<KernelPath, Box<[Kernel]>>>) -let mut kernel_traits = Vec::new();let mut kernel_types = Vec::new();- for (_file_path, file_kernels) in kernels.into_iter().sorted_by_key(|(p, _k)| p.join("::")) {- for (tr, ty) in file_kernels.iter().map(traitgen) {+ for (file_path, file_kernels) in kernels.into_iter().sorted_by_key(|(p, _k)| p.join("::")) {+ for kernel in &file_kernels {+ let (tr, ty) = traitgen(kernel, gpu_type_paths).with_context(|| {+ format!("cannot generate shared trait for kernel `{}` in `{}`", kernel.name, file_path.join("::"))+ })?;kernel_traits.push(tr);kernel_types.push(ty);}diff --git a/crates/uzu-engine/build/cpu/compiler.rs b/crates/uzu-engine/build/cpu/compiler.rsindex 0c2c51a1c..aa11c92a4 100644--- a/crates/uzu-engine/build/cpu/compiler.rs+++ b/crates/uzu-engine/build/cpu/compiler.rs@@ -6,7 +6,7 @@ use proc_macro2::TokenStream;use quote::{ToTokens, format_ident, quote};use syn::{Expr, FnArg, GenericArgument, GenericParam, Ident, Item, ItemFn, PathArguments, Type, punctuated::Punctuated,- token::Comma,+ token::Comma, visit_mut::VisitMut,};use walkdir::WalkDir;@@ -14,7 +14,7 @@ use crate::common::{codegen::write_tokens,compiler::Compiler,constraints::Constraints,- enum_paths::EnumPaths,+ gpu_type_paths::GpuTypePaths,gpu_types::GpuTypes,identifiers::{ArgumentName, KernelName, KernelPath},kernel::{Kernel, KernelArgument, KernelArgumentType, KernelBufferAccess, KernelParameter, KernelParameterType},@@ -38,7 +38,7 @@ pub struct FunctionArgument {impl FunctionArgument {fn to_kernel_argument(&self,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> Option<KernelArgument> {Some(KernelArgument {name: ArgumentName::from(self.name.to_string()),@@ -46,14 +46,14 @@ impl FunctionArgument {ty: match &self.ty {FunctionArgumentType::Buffer(access) => KernelArgumentType::Buffer(access.clone()),FunctionArgumentType::Constant(ty, None) => KernelArgumentType::Constant(- format!("&[{}]", canonicalize_type_text(ty, enum_paths)).into_boxed_str(),+ format!("&[{}]", canonicalize_type_text(ty, gpu_type_paths)).into_boxed_str(),),FunctionArgumentType::Constant(ty, Some(size)) => KernelArgumentType::Constant(- format!("&[{}; {}]", canonicalize_type_text(ty, enum_paths), size.to_token_stream(),)+ format!("&[{}; {}]", canonicalize_type_text(ty, gpu_type_paths), size.to_token_stream(),).into_boxed_str(),),FunctionArgumentType::Scalar(ty) => {- KernelArgumentType::Constant(canonicalize_type_text(ty, enum_paths).into_boxed_str())+ KernelArgumentType::Constant(canonicalize_type_text(ty, gpu_type_paths).into_boxed_str())},FunctionArgumentType::Specialization(_) => {return None;@@ -64,13 +64,13 @@ impl FunctionArgument {fn to_kernel_parameter(&self,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> Option<KernelParameter> {Some(KernelParameter {name: self.name.to_string().into_boxed_str(),ty: match &self.ty {FunctionArgumentType::Specialization(ty) => {- KernelParameterType::Value(canonicalize_type_text(ty, enum_paths).into_boxed_str())+ KernelParameterType::Value(canonicalize_type_text(ty, gpu_type_paths).into_boxed_str())},_ => {return None;@@ -82,10 +82,29 @@ impl FunctionArgument {fn canonicalize_type_text(ty: &Type,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> String {+ struct TypeNames<'a>(&'a GpuTypePaths);++ impl VisitMut for TypeNames<'_> {+ fn visit_type_path_mut(+ &mut self,+ ty: &mut syn::TypePath,+ ) {+ if ty.qself.is_none()+ && let Some(segment) = ty.path.segments.last()+ && matches!(segment.arguments, PathArguments::None)+ && self.0.full_path_for(&segment.ident.to_string()).is_some()+ {+ let ident = &segment.ident;+ ty.path = syn::parse_quote! { #ident };+ }+ syn::visit_mut::visit_type_path_mut(self, ty);+ }+ }+let mut canonicalized = ty.clone();- enum_paths.canonicalize_type(&mut canonicalized);+ TypeNames(gpu_type_paths).visit_type_mut(&mut canonicalized);canonicalized.to_token_stream().to_string().replace(" :: ", "::")}@@ -104,14 +123,14 @@ pub struct FunctionParameter {impl FunctionParameter {fn to_kernel_parameter(&self,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> KernelParameter {KernelParameter {name: self.name.to_string().into_boxed_str(),ty: match &self.ty {FunctionParameterType::Type => KernelParameterType::Type,FunctionParameterType::Value(ty) => {- KernelParameterType::Value(canonicalize_type_text(ty, enum_paths).into_boxed_str())+ KernelParameterType::Value(canonicalize_type_text(ty, gpu_type_paths).into_boxed_str())},},}@@ -141,7 +160,7 @@ impl CpuCompiler {fn compile(&self,source_path: PathBuf,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> anyhow::Result<(KernelPath, Box<[Kernel]>)> {let src_rel_path: KernelPath = source_path.strip_prefix(&self.src_dir)@@ -164,7 +183,7 @@ impl CpuCompiler {if let Item::Fn(ifn) = item&& ifn.attrs.iter().any(|attr| attr.path().is_ident("kernel")){- Some(self.compile_kernel(ifn, enum_paths))+ Some(self.compile_kernel(ifn, gpu_type_paths))} else {None}@@ -177,7 +196,7 @@ impl CpuCompiler {fn compile_kernel(&self,ifn: ItemFn,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> anyhow::Result<Kernel> {let mut kernel_ident = None;let mut function_variants = Vec::new();@@ -294,13 +313,13 @@ impl CpuCompiler {let kernel_parameters = function_parameters.iter()- .map(|parameter| parameter.to_kernel_parameter(enum_paths))- .chain(function_arguments.iter().flat_map(|argument| argument.to_kernel_parameter(enum_paths)))+ .map(|parameter| parameter.to_kernel_parameter(gpu_type_paths))+ .chain(function_arguments.iter().flat_map(|argument| argument.to_kernel_parameter(gpu_type_paths))).collect::<Box<[KernelParameter]>>();let kernel_arguments = function_arguments.iter()- .flat_map(|argument| argument.to_kernel_argument(enum_paths))+ .flat_map(|argument| argument.to_kernel_argument(gpu_type_paths)).collect::<Box<[KernelArgument]>>();if function_parameters.len() != function_variants.len() {@@ -355,7 +374,7 @@ impl CpuCompiler {quote! { #[allow(non_snake_case)] #parameter_ident: crate::data_type::DataType }},KernelParameterType::Value(ty) => {- let ty: syn::Type = syn::parse_str(ty.as_ref()).unwrap();+ let ty = gpu_type_paths.resolve_type(ty)?;quote! { #[allow(non_snake_case)] #parameter_ident: #ty }},})@@ -377,7 +396,7 @@ impl CpuCompiler {},},KernelArgumentType::Constant(ty) => {- let ty: Type = syn::parse_str(ty.as_ref()).context("cannot parse type")?;+ let ty = gpu_type_paths.resolve_type(ty)?;quote! { #ty }},};@@ -669,7 +688,7 @@ impl Compiler for CpuCompiler {fn build(&self,_gpu_types: &GpuTypes,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> anyhow::Result<HashMap<KernelPath, Box<[Kernel]>>> {let objects = WalkDir::new(&self.src_dir).into_iter()@@ -677,7 +696,7 @@ impl Compiler for CpuCompiler {.filter(|entry| {entry.file_type().is_file() && entry.path().extension().and_then(|s| s.to_str()) == Some("rs")})- .map(|entry| self.compile(entry.into_path(), enum_paths))+ .map(|entry| self.compile(entry.into_path(), gpu_type_paths)).collect::<anyhow::Result<Vec<(KernelPath, Box<[Kernel]>)>>>().context("cannot compile cpu sources")?;diff --git a/crates/uzu-engine/build/main.rs b/crates/uzu-engine/build/main.rsindex 85c1b93ae..a4873b5c2 100644--- a/crates/uzu-engine/build/main.rs+++ b/crates/uzu-engine/build/main.rs@@ -3,7 +3,7 @@ use std::{env, fs, path::PathBuf, process::ExitCode};use anyhow::Context;mod common;-use common::{compiler::Compiler, enum_paths::EnumPaths, envs, gpu_types::GpuTypes, traitgen::traitgen_all};+use common::{compiler::Compiler, envs, gpu_type_paths::GpuTypePaths, gpu_types::GpuTypes, traitgen::traitgen_all};mod cpu;@@ -49,10 +49,10 @@ fn main() -> anyhow::Result<ExitCode> {debug_log!("cleaned caches");}- let gpu_types = GpuTypes::scan().context("Failed to scan gpu types")?;+ let gpu_types = GpuTypes::scan("common").context("Failed to scan gpu types")?;debug_log!("gpu_types scan done");- let enum_paths = EnumPaths::from_gpu_types(&gpu_types).context("Failed to build enum path map")?;+ let gpu_type_paths = GpuTypePaths::from_gpu_types(&gpu_types).context("Failed to build GPU type path map")?;let mut compilers: Vec<Box<dyn Compiler>> = Vec::new();@@ -71,11 +71,11 @@ fn main() -> anyhow::Result<ExitCode> {}let backends_kernels =- compilers.iter().map(|c| c.build(&gpu_types, &enum_paths)).collect::<anyhow::Result<Vec<_>>>()?;+ compilers.iter().map(|c| c.build(&gpu_types, &gpu_type_paths)).collect::<anyhow::Result<Vec<_>>>()?;debug_log!("backend build end");- traitgen_all(backends_kernels)?;+ traitgen_all(backends_kernels, &gpu_type_paths)?;debug_log!("build script ended");diff --git a/crates/uzu-engine/build/metal/ast.rs b/crates/uzu-engine/build/metal/ast.rsindex 94934059d..eaf8b92ed 100644--- a/crates/uzu-engine/build/metal/ast.rs+++ b/crates/uzu-engine/build/metal/ast.rs@@ -203,9 +203,7 @@ impl MetalArgument {["ulong"] | ["uint64_t"] | ["unsigned", "long"] => Ok("u64".into()),["int"] | ["int32_t"] => Ok("i32".into()),["float"] => Ok("f32".into()),- [vpath] if vpath.starts_with("uzu::") => {- Ok(vpath.replacen("uzu::", "crate::backends::common::gpu_types::", 1).into())- },+ [vpath] if vpath.starts_with("uzu::") => Ok(vpath.rsplit("::").next().unwrap().into()),_ => bail!("unknown scalar type: {c_type}"),}}diff --git a/crates/uzu-engine/build/metal/bindgen/arguments.rs b/crates/uzu-engine/build/metal/bindgen/arguments.rsindex 3a2eec622..ca3877f4e 100644--- a/crates/uzu-engine/build/metal/bindgen/arguments.rs+++ b/crates/uzu-engine/build/metal/bindgen/arguments.rs@@ -4,7 +4,7 @@ use quote::{format_ident, quote};use syn::{Expr, Ident, Type};use crate::{- common::{enum_paths::EnumPaths, expr_rewrite::rewrite_paths_with},+ common::{expr_rewrite::rewrite_paths_with, gpu_type_paths::GpuTypePaths},metal::{ast::{MetalArgument, MetalArgumentType, MetalBufferAccess, MetalConstantType, MetalGroupsType, MetalKernelInfo,@@ -60,7 +60,7 @@ pub struct IndirectDispatchArgument;pub fn parse(kernel: &MetalKernelInfo,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,host_expression_rewriter: &mut HostExpressionRewriter,) -> Result<Vec<ArgumentEmission>> {let mut emissions = Vec::new();@@ -71,13 +71,18 @@ pub fn parse(for argument in kernel.arguments.iter() {match &argument.argument_type {MetalArgumentType::Buffer(access) => {- let buffer = parse_buffer_argument(argument, *access, next_buffer_index, enum_paths)?;+ let buffer = parse_buffer_argument(argument, *access, next_buffer_index, gpu_type_paths)?;emissions.push(ArgumentEmission::Buffer(buffer));next_buffer_index += 1;},MetalArgumentType::Constant((rust_type_text, constant_type)) => {- let constant =- parse_constant_argument(argument, rust_type_text, constant_type, next_buffer_index, enum_paths)?;+ let constant = parse_constant_argument(+ argument,+ rust_type_text,+ constant_type,+ next_buffer_index,+ gpu_type_paths,+ )?;emissions.push(ArgumentEmission::Constant(constant));next_buffer_index += 1;},@@ -86,7 +91,7 @@ pub fn parse(argument,dimensions.as_deref(),next_threadgroup_index,- enum_paths,+ gpu_type_paths,host_expression_rewriter,)?;emissions.push(ArgumentEmission::Shared(shared));@@ -109,10 +114,10 @@ fn parse_shared_argument(argument: &MetalArgument,dimensions: Option<&str>,threadgroup_index: usize,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,host_expression_rewriter: &mut HostExpressionRewriter,) -> Result<SharedArgument> {- let condition = parse_argument_condition(argument, enum_paths)?+ let condition = parse_argument_condition(argument, gpu_type_paths)?.context("optional threadgroup argument must carry a condition")?;let dimensions = dimensions.context("optional threadgroup argument must have explicit array dimensions")?;let element_size = shared_element_byte_size(&argument.c_type)?;@@ -136,10 +141,10 @@ fn parse_buffer_argument(argument: &MetalArgument,access: MetalBufferAccess,buffer_index: usize,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> Result<BufferArgument> {let name = format_ident!("{}", argument.name.as_ref());- let condition = parse_argument_condition(argument, enum_paths)?;+ let condition = parse_argument_condition(argument, gpu_type_paths)?;Ok(BufferArgument {name,buffer_index,@@ -153,10 +158,11 @@ fn parse_constant_argument(rust_type_text: &str,constant_type: &MetalConstantType,buffer_index: usize,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> Result<ConstantArgument> {let name = format_ident!("{}", argument.name.as_ref());- let element_type: Type = syn::parse_str(rust_type_text)+ let element_type = gpu_type_paths+ .resolve_type(rust_type_text).with_context(|| format!("constant rust type `{}` cannot be parsed", rust_type_text))?;let shape = match constant_type {MetalConstantType::Scalar => ConstantShape::Scalar(element_type),@@ -170,7 +176,7 @@ fn parse_constant_argument(}},};- let condition = parse_argument_condition(argument, enum_paths)?;+ let condition = parse_argument_condition(argument, gpu_type_paths)?;Ok(ConstantArgument {name,buffer_index,@@ -181,12 +187,12 @@ fn parse_constant_argument(fn parse_argument_condition(argument: &MetalArgument,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> Result<Option<ArgumentCondition>> {match argument.condition.as_deref() {Some(condition_text) => {let field_name = format_ident!("has_{}", argument.name.as_ref());- let rust_expression = rewrite_for_rust(enum_paths, condition_text).with_context(|| {+ let rust_expression = rewrite_for_rust(gpu_type_paths, condition_text).with_context(|| {format!("OPTIONAL condition `{}` cannot be parsed as a rust expression", condition_text)})?;Ok(Some(ArgumentCondition {diff --git a/crates/uzu-engine/build/metal/bindgen/host_expression_rewriter.rs b/crates/uzu-engine/build/metal/bindgen/host_expression_rewriter.rsindex 582c20227..15fca0a52 100644--- a/crates/uzu-engine/build/metal/bindgen/host_expression_rewriter.rs+++ b/crates/uzu-engine/build/metal/bindgen/host_expression_rewriter.rs@@ -6,13 +6,13 @@ use quote::{format_ident, quote};use syn::Expr;use crate::{- common::{enum_paths::EnumPaths, expr_rewrite::rewrite_paths_with},+ common::{expr_rewrite::rewrite_paths_with, gpu_type_paths::GpuTypePaths},metal::{bindgen::variants::VariantBind, enum_path_rewrite::qualify_enum_path},};pub struct HostExpressionRewriter<'context> {variants: &'context [VariantBind],- enum_paths: &'context EnumPaths,+ gpu_type_paths: &'context GpuTypePaths,specialization_names: Vec<String>,referenced_parameter_names: BTreeSet<String>,kernel_name: &'context str,@@ -21,13 +21,13 @@ pub struct HostExpressionRewriter<'context> {impl<'context> HostExpressionRewriter<'context> {pub fn new(variants: &'context [VariantBind],- enum_paths: &'context EnumPaths,+ gpu_type_paths: &'context GpuTypePaths,specialization_names: Vec<String>,kernel_name: &'context str,) -> Self {Self {variants,- enum_paths,+ gpu_type_paths,specialization_names,referenced_parameter_names: BTreeSet::new(),kernel_name,@@ -43,11 +43,11 @@ impl<'context> HostExpressionRewriter<'context> {})?;let variants = self.variants;- let enum_paths = self.enum_paths;+ let gpu_type_paths = self.gpu_type_paths;let specialization_names = &self.specialization_names;let referenced_parameter_names = &mut self.referenced_parameter_names;rewrite_paths_with(&mut expression, |path| {- if let Some(qualified) = qualify_enum_path(path, enum_paths) {+ if let Some(qualified) = qualify_enum_path(path, gpu_type_paths) {return Some(qualified);}diff --git a/crates/uzu-engine/build/metal/bindgen/mod.rs b/crates/uzu-engine/build/metal/bindgen/mod.rsindex 588a61f5d..ee9cc10d1 100644--- a/crates/uzu-engine/build/metal/bindgen/mod.rs+++ b/crates/uzu-engine/build/metal/bindgen/mod.rs@@ -10,7 +10,7 @@ use proc_macro2::TokenStream;use quote::{format_ident, quote};use crate::{- common::{enum_paths::EnumPaths, kernel::Kernel, mangling::dynamic_mangle},+ common::{gpu_type_paths::GpuTypePaths, kernel::Kernel, mangling::dynamic_mangle},metal::{ast::MetalKernelInfo, bindgen::host_expression_rewriter::HostExpressionRewriter, wrapper::SpecializeBaseIndices,},@@ -19,7 +19,7 @@ use crate::{pub fn bindgen(kernel: &MetalKernelInfo,specialize_indices: &SpecializeBaseIndices,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,library_const: &proc_macro2::Ident,num_shards: usize,library_compressed: bool,@@ -28,12 +28,12 @@ pub fn bindgen(let trait_name = format_ident!("{}Kernel", kernel_name);let struct_name = format_ident!("{}MetalKernel", kernel_name);- let variant_binds = variants::parse(kernel)?;+ let variant_binds = variants::parse(kernel, gpu_type_paths)?;let specialize_emission =- specialize::parse(kernel, specialize_indices.get(&kernel.name).copied(), kernel_name, enum_paths)?;+ specialize::parse(kernel, specialize_indices.get(&kernel.name).copied(), kernel_name, gpu_type_paths)?;let mut host_expression_rewriter =- HostExpressionRewriter::new(&variant_binds, enum_paths, specialize_emission.argument_names(), kernel_name);- let argument_emissions = arguments::parse(kernel, enum_paths, &mut host_expression_rewriter)?;+ HostExpressionRewriter::new(&variant_binds, gpu_type_paths, specialize_emission.argument_names(), kernel_name);+ let argument_emissions = arguments::parse(kernel, gpu_type_paths, &mut host_expression_rewriter)?;let trait_wiring = trait_wiring::build(kernel, &trait_name, &struct_name);let dispatch_emission = dispatch::parse(kernel, &mut host_expression_rewriter)?;diff --git a/crates/uzu-engine/build/metal/bindgen/specialize.rs b/crates/uzu-engine/build/metal/bindgen/specialize.rsindex dc3d3c937..dbb0dc356 100644--- a/crates/uzu-engine/build/metal/bindgen/specialize.rs+++ b/crates/uzu-engine/build/metal/bindgen/specialize.rs@@ -9,7 +9,7 @@ use super::super::{ast::{MetalArgumentType, MetalKernelInfo},enum_path_rewrite::{gpu_type_kind_for_c_type, rewrite_for_rust},};-use crate::common::enum_paths::{EnumPaths, GpuTypeKind};+use crate::common::gpu_type_paths::{GpuTypeKind, GpuTypePaths};enum SpecializeLowering {Direct,@@ -39,7 +39,7 @@ pub fn parse(kernel: &MetalKernelInfo,base_function_constant_index: Option<usize>,kernel_name: &str,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> Result<SpecializeEmission> {let mut presence_checks = Vec::new();let arguments = kernel@@ -52,17 +52,18 @@ pub fn parse(.enumerate().map(|(offset, (argument, rust_type_text))| -> Result<SpecializeArgument> {let name = format_ident!("{}", argument.name.as_ref());- let rust_type: Type = syn::parse_str(rust_type_text).with_context(|| {+ let rust_type = gpu_type_paths.resolve_type(rust_type_text).with_context(|| {format!("specialize type `{}` in kernel `{}` cannot be parsed", rust_type_text, kernel_name)})?;let function_constant_index = base_function_constant_index.unwrap_or(0) + offset;- let lowering = match gpu_type_kind_for_c_type(enum_paths, &argument.c_type) {+ let lowering = match gpu_type_kind_for_c_type(gpu_type_paths, &argument.c_type) {Some(GpuTypeKind::Enum) => SpecializeLowering::Enum,Some(GpuTypeKind::OptionSet) => SpecializeLowering::OptionSet,+ Some(GpuTypeKind::Struct) => anyhow::bail!("cannot specialize struct argument `{name}` in kernel `{kernel_name}`"),None => SpecializeLowering::Direct,};if let Some(condition) = argument.condition.as_deref() {- let condition = rewrite_for_rust(enum_paths, condition)?;+ let condition = rewrite_for_rust(gpu_type_paths, condition)?;presence_checks.push(quote! {assert_eq!(#name.is_some(), #condition, concat!("invalid presence for specialization ", stringify!(#name)));});diff --git a/crates/uzu-engine/build/metal/bindgen/variants.rs b/crates/uzu-engine/build/metal/bindgen/variants.rsindex aac8a0af0..535b98f68 100644--- a/crates/uzu-engine/build/metal/bindgen/variants.rs+++ b/crates/uzu-engine/build/metal/bindgen/variants.rs@@ -6,6 +6,7 @@ use quote::{format_ident, quote};use syn::{Ident, Type};use super::super::ast::{MetalKernelInfo, MetalTemplateParameterType};+use crate::common::gpu_type_paths::GpuTypePaths;pub struct VariantBind {pub parameter_name: Ident,@@ -13,7 +14,10 @@ pub struct VariantBind {parsed_type: Option<Type>,}-pub fn parse(kernel: &MetalKernelInfo) -> Result<Vec<VariantBind>> {+pub fn parse(+ kernel: &MetalKernelInfo,+ gpu_type_paths: &GpuTypePaths,+) -> Result<Vec<VariantBind>> {kernel.variants.as_deref()@@ -24,7 +28,7 @@ pub fn parse(kernel: &MetalKernelInfo) -> Result<Vec<VariantBind>> {let field_name = format_ident!("{}", variant.name.as_ref().to_ascii_lowercase());let parsed_type = match &variant.ty {MetalTemplateParameterType::Type => None,- MetalTemplateParameterType::Value(type_text) => Some(syn::parse_str::<Type>(type_text.as_ref())?),+ MetalTemplateParameterType::Value(type_text) => Some(gpu_type_paths.resolve_type(type_text)?),};Ok(VariantBind {parameter_name,diff --git a/crates/uzu-engine/build/metal/compiler.rs b/crates/uzu-engine/build/metal/compiler.rsindex 85f17ef1d..002fc0744 100644--- a/crates/uzu-engine/build/metal/compiler.rs+++ b/crates/uzu-engine/build/metal/compiler.rs@@ -20,7 +20,7 @@ use super::{};use crate::{common::{- caching, codegen::write_tokens, compiler::Compiler, enum_paths::EnumPaths, gpu_types::GpuTypes,+ caching, codegen::write_tokens, compiler::Compiler, gpu_type_paths::GpuTypePaths, gpu_types::GpuTypes,identifiers::KernelPath, kernel::Kernel,},debug_log,@@ -144,7 +144,8 @@ impl MetalCompiler {fn compile(&self,source_path: PathBuf,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,+ cache_key: &[u8; blake3::OUT_LEN],) -> anyhow::Result<(KernelPath, Box<[Kernel]>, bool)> {let source_path_relative =source_path.strip_prefix(&self.source_directory).context("source is not in src_dir")?;@@ -166,7 +167,7 @@ impl MetalCompiler {if let Ok(cached_contents) = fs::read(&cached_file)&& let Ok(cached) = serde_json::from_slice::<Cached>(&cached_contents)- && cached.cache_key == self.cache_key+ && cached.cache_key == *cache_key&& cached.dependency_hashes.iter().all(|(path, hash)| {fs::read(path.as_ref()).map(|contents| blake3::hash(&contents).as_bytes() == hash).unwrap_or(false)})@@ -205,7 +206,7 @@ impl MetalCompiler {let mut num_shards = 0;if !kernel_infos.is_empty() {let (kernel_wrappers, specialize_indices) =- wrappers(&kernel_infos, enum_paths).context("cannot generate kernel wrappers")?;+ wrappers(&kernel_infos, gpu_type_paths).context("cannot generate kernel wrappers")?;num_variants = kernel_wrappers.iter().map(|kernel| kernel.variants.len()).sum();let footers = shard_footers(&kernel_wrappers);@@ -267,7 +268,7 @@ impl MetalCompiler {super::bindgen::bindgen(kernel,&specialize_indices,- enum_paths,+ gpu_type_paths,&library_const,num_shards,self.metallib_compressed,@@ -290,7 +291,7 @@ impl MetalCompiler {let has_kernels = !kernel_infos.is_empty();let cached = Cached {- cache_key: self.cache_key,+ cache_key: *cache_key,dependency_hashes,public_kernels: public_kernels.clone(),has_kernels,@@ -315,9 +316,23 @@ impl Compiler for MetalCompiler {fn build(&self,gpu_types: &GpuTypes,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> anyhow::Result<HashMap<KernelPath, Box<[Kernel]>>> {- gpu_type_gen(&self.gpu_types_directory, gpu_types).context("cannot generate shared gpu types")?;+ let private_gpu_types = GpuTypes::scan("metal").context("cannot scan Metal-private gpu types")?;+ let mut gpu_type_paths = gpu_type_paths.clone();+ gpu_type_paths.extend(&private_gpu_types).context("cannot extend Metal GPU type paths")?;+ gpu_type_gen(&self.gpu_types_directory, &[gpu_types, &private_gpu_types])+ .context("cannot generate gpu types")?;++ let cache_key = {+ let mut hasher = blake3::Hasher::new();+ hasher.update(&self.cache_key);+ for path in gpu_type_paths.full_paths().sorted_unstable() {+ hasher.update(path.as_bytes());+ hasher.update(&[0]);+ }+ hasher.finalize().into()+ };let metal_sources: Vec<PathBuf> = WalkDir::new(&self.source_directory).into_iter()@@ -329,7 +344,8 @@ impl Compiler for MetalCompiler {let compiled: Vec<(KernelPath, Box<[Kernel]>, bool)> = metal_sources.into_par_iter().map(|path| {- self.compile(path.clone(), enum_paths).with_context(|| format!("cannot compile {}", path.display()))+ self.compile(path.clone(), &gpu_type_paths, &cache_key)+ .with_context(|| format!("cannot compile {}", path.display()))}).collect::<anyhow::Result<_>>()?;diff --git a/crates/uzu-engine/build/metal/enum_path_rewrite.rs b/crates/uzu-engine/build/metal/enum_path_rewrite.rsindex c7c270bfb..2a708378a 100644--- a/crates/uzu-engine/build/metal/enum_path_rewrite.rs+++ b/crates/uzu-engine/build/metal/enum_path_rewrite.rs@@ -4,50 +4,46 @@ use quote::ToTokens;use syn::{Expr, Path, parse_quote};use crate::common::{- enum_paths::{EnumPaths, GpuTypeKind},expr_rewrite::rewrite_paths_with,+ gpu_type_paths::{GpuTypeKind, GpuTypePaths},};pub fn gpu_type_kind_for_c_type(- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,c_type: &str,) -> Option<GpuTypeKind> {let trimmed = c_type.trim_start_matches("const ").trim();- let uzu_path = trimmed.strip_prefix("uzu::")?;- let short_name = uzu_path.rsplit("::").next()?;- let expected_rust_path = format!("crate::backends::common::gpu_types::{uzu_path}");- if enum_paths.full_path_for(short_name) != Some(expected_rust_path.as_str()) {- return None;- }- enum_paths.kind_for(short_name)+ super::gpu_types::rust_path(gpu_type_paths, trimmed)?;+ let short_name = trimmed.rsplit("::").next()?;+ gpu_type_paths.kind_for(short_name)}pub fn is_enum_c_type(- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,c_type: &str,) -> bool {- gpu_type_kind_for_c_type(enum_paths, c_type).is_some()+ matches!(gpu_type_kind_for_c_type(gpu_type_paths, c_type), Some(GpuTypeKind::Enum | GpuTypeKind::OptionSet))}pub fn rewrite_for_rust(- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,condition: &str,) -> anyhow::Result<TokenStream> {let mut parsed: Expr =syn::parse_str(condition).with_context(|| format!("cannot parse rust expression `{condition}`"))?;- rewrite_paths_with(&mut parsed, |path| qualify_enum_path(path, enum_paths));+ rewrite_paths_with(&mut parsed, |path| qualify_enum_path(path, gpu_type_paths));Ok(parsed.into_token_stream())}pub(super) fn qualify_enum_path(path: &Path,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> Option<Expr> {if path.segments.len() < 2 {return None;}let head = path.segments.first()?;- let full_path_text = enum_paths.full_path_for(&head.ident.to_string())?;+ let full_path_text = gpu_type_paths.full_path_for(&head.ident.to_string())?;let prefix: Path = syn::parse_str(full_path_text).ok()?;let suffix: Vec<_> = path.segments.iter().skip(1).cloned().collect();Some(parse_quote! { #prefix #( :: #suffix )* })diff --git a/crates/uzu-engine/build/metal/gpu_types/mod.rs b/crates/uzu-engine/build/metal/gpu_types/mod.rsindex 2e1821686..bb6d5336e 100644--- a/crates/uzu-engine/build/metal/gpu_types/mod.rs+++ b/crates/uzu-engine/build/metal/gpu_types/mod.rs@@ -1,9 +1,12 @@-use std::{fs, path::Path};+use std::{collections::BTreeMap, fs, path::Path};use anyhow::Context;use itertools::Itertools;-use crate::common::gpu_types::{GpuType, GpuTypeFile, GpuTypes};+use crate::common::{+ gpu_type_paths::GpuTypePaths,+ gpu_types::{GpuType, GpuTypes},+};mod gpu_type_constant;mod gpu_type_enum;@@ -15,6 +18,17 @@ use gpu_type_enum::gpu_type_gen_enum;use gpu_type_option_set::gpu_type_gen_option_set;use gpu_type_struct::gpu_type_gen_struct;+pub fn rust_path<'a>(+ gpu_type_paths: &'a GpuTypePaths,+ c_type: &str,+) -> Option<&'a str> {+ let path = c_type.strip_prefix("uzu::")?;+ let short_name = path.rsplit("::").next()?;+ let rust_path = gpu_type_paths.full_path_for(short_name)?;+ let (_, type_path) = rust_path.split_once("::gpu_types::")?;+ (type_path == path).then_some(rust_path)+}+fn rust_to_metal(ty: &str) -> anyhow::Result<&'static str> {match ty {"i8" => Ok("int8_t"),@@ -33,11 +47,18 @@ fn rust_to_metal(ty: &str) -> anyhow::Result<&'static str> {pub fn gpu_type_gen(gpu_types_dir: &Path,- gpu_types: &GpuTypes,+ gpu_types: &[&GpuTypes],) -> anyhow::Result<()> {- for gpu_type_file in &gpu_types.files {- gpu_type_gen_file(&gpu_types_dir.join(gpu_type_file.name.as_ref()).with_extension("h"), gpu_type_file)- .with_context(|| format!("Cannot generate gpu types for {}", gpu_type_file.name.as_ref()))?;+ fs::create_dir_all(gpu_types_dir).context("cannot create gpu types directory")?;+ let mut files: BTreeMap<&str, Vec<&GpuType>> = BTreeMap::new();+ for gpu_types in gpu_types {+ for file in &gpu_types.files {+ files.entry(&file.name).or_default().extend(file.types.iter());+ }+ }+ for (module_name, types) in files {+ gpu_type_gen_file(&gpu_types_dir.join(module_name).with_extension("h"), module_name, &types)+ .with_context(|| format!("Cannot generate gpu types for {module_name}"))?;}Ok(())@@ -45,12 +66,10 @@ pub fn gpu_type_gen(fn gpu_type_gen_file(file_path: &Path,- gpu_types_file: &GpuTypeFile,+ module_name: &str,+ types: &[&GpuType],) -> anyhow::Result<()> {- let module_name = gpu_types_file.name.as_ref();-- let generated = gpu_types_file- .types+ let generated = types.iter().map(|gpu_type| match gpu_type {GpuType::Constant(gpu_type_constant) => gpu_type_gen_constant(gpu_type_constant)diff --git a/crates/uzu-engine/build/metal/wrapper.rs b/crates/uzu-engine/build/metal/wrapper.rsindex 13ba9f409..63a31b878 100644--- a/crates/uzu-engine/build/metal/wrapper.rs+++ b/crates/uzu-engine/build/metal/wrapper.rs@@ -9,7 +9,7 @@ use super::{variant_combinations::constrained_combinations,};use crate::common::{- constraints::Constraints, enum_paths::EnumPaths, identifiers::KernelName, mangling::static_mangle,+ constraints::Constraints, gpu_type_paths::GpuTypePaths, identifiers::KernelName, mangling::static_mangle,};pub type SpecializeBaseIndices = HashMap<KernelName, usize>;@@ -27,7 +27,7 @@ pub struct KernelWrappers {pub fn wrappers(kernels: &[MetalKernelInfo],- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> anyhow::Result<(Box<[KernelWrappers]>, SpecializeBaseIndices)> {let mut all_wrappers = Vec::new();let mut base_indices = SpecializeBaseIndices::new();@@ -42,7 +42,7 @@ pub fn wrappers(next_index += specialize_count;}- all_wrappers.push(kernel_wrappers(kernel, base_indices.get(&kernel.name), enum_paths)?);+ all_wrappers.push(kernel_wrappers(kernel, base_indices.get(&kernel.name), gpu_type_paths)?);}Ok((all_wrappers.into_boxed_slice(), base_indices))@@ -81,7 +81,7 @@ impl SpecializeBinding<'_> {fn specialize_bindings<'a>(kernel: &'a MetalKernelInfo,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> Vec<SpecializeBinding<'a>> {kernel.arguments@@ -89,7 +89,7 @@ fn specialize_bindings<'a>(.filter(|a| matches!(&a.argument_type, MetalArgumentType::Specialize(_))).map(|argument| {let name_suffix = format!("{}_{}", kernel.name, argument.name);- let enum_type = is_enum_c_type(enum_paths, &argument.c_type)+ let enum_type = is_enum_c_type(gpu_type_paths, &argument.c_type).then(|| argument.c_type.trim_start_matches("const ").trim().to_string());SpecializeBinding {argument,@@ -148,10 +148,10 @@ fn kernel_footer(bindings: &[SpecializeBinding<'_>]) -> String {fn kernel_wrappers(kernel: &MetalKernelInfo,base_index: Option<&usize>,- enum_paths: &EnumPaths,+ gpu_type_paths: &GpuTypePaths,) -> anyhow::Result<KernelWrappers> {let mut variants = Vec::new();- let bindings = specialize_bindings(kernel, enum_paths);+ let bindings = specialize_bindings(kernel, gpu_type_paths);let header = base_index.map(|&base| kernel_header(&bindings, base).into_boxed_str());let parameters = kernel.variants.as_deref();diff --git a/crates/uzu-engine/src/backends/common/capture_matcher.rs b/crates/uzu-engine/src/backends/common/capture_matcher.rsindex 2c3a91f74..b2d46ec3b 100644--- a/crates/uzu-engine/src/backends/common/capture_matcher.rs+++ b/crates/uzu-engine/src/backends/common/capture_matcher.rs@@ -77,6 +77,7 @@ impl CaptureMatcher {}}+ #[cfg(backend = "metal")]pub fn enabled(&self) -> bool {self.enabled}diff --git a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rsindex ddf160c8d..236b4a318 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs@@ -1,13 +1,6 @@use bitflags::bitflags;use derive_more::Display;-#[repr(C)]-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]-pub enum GemmAPrologueKind {- FullPrecision,- Int8Symmetric,-}-#[repr(C)]#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]pub enum GemmBPrologueKind {@@ -27,164 +20,3 @@ bitflags! {const SOFT_CAP = 1 << 3;}}--#[repr(C)]-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]-#[allow(non_camel_case_types)]-pub enum GemmTiling {- Tile8x32x32_Simdgroups1x1,- Tile64x32x32_Simdgroups2x2,- Tile64x64x16_Simdgroups2x2,- Tile64x64x32_Simdgroups2x2,- Tile32x32x32_Simdgroups2x2,- Tile16x32x256_Simdgroups1x1,- Tile16x32x256_Simdgroups1x1x4,- Tile16x128x256_Simdgroups1x4,- Tile32x64x256_Simdgroups2x2,- Tile64x32x256_Simdgroups4x1,- Tile64x64x256_Simdgroups2x2,- Tile128x128x256_Simdgroups4x4,-}--pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;--impl GemmTiling {- pub const fn block_m(self) -> u32 {- match self {- Self::Tile8x32x32_Simdgroups1x1 => 8,- Self::Tile64x32x32_Simdgroups2x2 => 64,- Self::Tile64x64x16_Simdgroups2x2 => 64,- Self::Tile64x64x32_Simdgroups2x2 => 64,- Self::Tile32x32x32_Simdgroups2x2 => 32,- Self::Tile16x32x256_Simdgroups1x1 => 16,- Self::Tile16x32x256_Simdgroups1x1x4 => 16,- Self::Tile16x128x256_Simdgroups1x4 => 16,- Self::Tile32x64x256_Simdgroups2x2 => 32,- Self::Tile64x32x256_Simdgroups4x1 => 64,- Self::Tile64x64x256_Simdgroups2x2 => 64,- Self::Tile128x128x256_Simdgroups4x4 => 128,- }- }-- pub const fn block_n(self) -> u32 {- match self {- Self::Tile8x32x32_Simdgroups1x1 => 32,- Self::Tile64x32x32_Simdgroups2x2 => 32,- Self::Tile64x64x16_Simdgroups2x2 => 64,- Self::Tile64x64x32_Simdgroups2x2 => 64,- Self::Tile32x32x32_Simdgroups2x2 => 32,- Self::Tile16x32x256_Simdgroups1x1 => 32,- Self::Tile16x32x256_Simdgroups1x1x4 => 32,- Self::Tile16x128x256_Simdgroups1x4 => 128,- Self::Tile32x64x256_Simdgroups2x2 => 64,- Self::Tile64x32x256_Simdgroups4x1 => 32,- Self::Tile64x64x256_Simdgroups2x2 => 64,- Self::Tile128x128x256_Simdgroups4x4 => 128,- }- }-- pub const fn block_k(self) -> u32 {- match self {- Self::Tile8x32x32_Simdgroups1x1 => 32,- Self::Tile64x32x32_Simdgroups2x2 => 32,- Self::Tile64x64x16_Simdgroups2x2 => 16,- Self::Tile64x64x32_Simdgroups2x2 => 32,- Self::Tile32x32x32_Simdgroups2x2 => 32,- Self::Tile16x32x256_Simdgroups1x1 => 256,- Self::Tile16x32x256_Simdgroups1x1x4 => 256,- Self::Tile16x128x256_Simdgroups1x4 => 256,- Self::Tile32x64x256_Simdgroups2x2 => 256,- Self::Tile64x32x256_Simdgroups4x1 => 256,- Self::Tile64x64x256_Simdgroups2x2 => 256,- Self::Tile128x128x256_Simdgroups4x4 => 256,- }- }-- pub const fn simdgroups_m(self) -> u32 {- match self {- Self::Tile8x32x32_Simdgroups1x1 => 1,- Self::Tile64x32x32_Simdgroups2x2 => 2,- Self::Tile64x64x16_Simdgroups2x2 => 2,- Self::Tile64x64x32_Simdgroups2x2 => 2,- Self::Tile32x32x32_Simdgroups2x2 => 2,- Self::Tile16x32x256_Simdgroups1x1 => 1,- Self::Tile16x32x256_Simdgroups1x1x4 => 1,- Self::Tile16x128x256_Simdgroups1x4 => 1,- Self::Tile32x64x256_Simdgroups2x2 => 2,- Self::Tile64x32x256_Simdgroups4x1 => 4,- Self::Tile64x64x256_Simdgroups2x2 => 2,- Self::Tile128x128x256_Simdgroups4x4 => 4,- }- }-- pub const fn simdgroups_n(self) -> u32 {- match self {- Self::Tile8x32x32_Simdgroups1x1 => 1,- Self::Tile64x32x32_Simdgroups2x2 => 2,- Self::Tile64x64x16_Simdgroups2x2 => 2,- Self::Tile64x64x32_Simdgroups2x2 => 2,- Self::Tile32x32x32_Simdgroups2x2 => 2,- Self::Tile16x32x256_Simdgroups1x1 => 1,- Self::Tile16x32x256_Simdgroups1x1x4 => 1,- Self::Tile16x128x256_Simdgroups1x4 => 4,- Self::Tile32x64x256_Simdgroups2x2 => 2,- Self::Tile64x32x256_Simdgroups4x1 => 1,- Self::Tile64x64x256_Simdgroups2x2 => 2,- Self::Tile128x128x256_Simdgroups4x4 => 4,- }- }-- pub const fn simdgroups_k(self) -> u32 {- match self {- Self::Tile16x32x256_Simdgroups1x1x4 => 4,- _ => 1,- }- }-- pub const fn is_mxu_variant(self) -> bool {- matches!(- self,- Self::Tile16x32x256_Simdgroups1x1- | Self::Tile16x32x256_Simdgroups1x1x4- | Self::Tile16x128x256_Simdgroups1x4- | Self::Tile32x64x256_Simdgroups2x2- | Self::Tile64x32x256_Simdgroups4x1- | Self::Tile64x64x256_Simdgroups2x2- | Self::Tile128x128x256_Simdgroups4x4- )- }-}--pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {- t.simdgroups_m()-}-pub const fn gemm_tiling_simdgroups_per_column(t: GemmTiling) -> u32 {- t.simdgroups_n()-}-pub const fn gemm_tiling_simdgroups_per_k(t: GemmTiling) -> u32 {- t.simdgroups_k()-}--bitflags! {- #[repr(transparent)]- #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]- pub struct GemmAlignment: u32 {- const M = 1 << 0;- const N = 1 << 1;- const K = 1 << 2;- }-}--impl GemmAlignment {- pub fn new(- m: bool,- n: bool,- k: bool,- ) -> Self {- let mut bits = Self::empty();- bits.set(Self::M, m);- bits.set(Self::N, n);- bits.set(Self::K, k);- bits- }-}diff --git a/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs b/crates/uzu-engine/src/backends/common/gpu_types/matmul.rsindex 77ca8331d..ce4714f04 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs@@ -1,22 +1 @@pub const QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT: u32 = 4;--#[repr(C)]-#[allow(non_snake_case)]-#[derive(Debug, Default, Copy, Clone)]-pub struct GemmParams {- pub M: u32,- pub N: u32,- pub K: u32,- pub leading_dimension_a: u32,- pub leading_dimension_b: u32,- pub scale_output_stride: u32,- pub scale_group_stride: u32,- pub zero_point_output_stride: u32,- pub zero_point_group_stride: u32,- pub leading_dimension_d: u32,- pub threadgroups_per_column: u32,- pub threadgroups_per_row: u32,- pub aligned_inner_iterations: u32,- pub use_morton: bool,- pub ab_scale: f32,-}diff --git a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs b/crates/uzu-engine/src/backends/common/gpu_types/mod.rsindex 628bd0dbe..91947e9a0 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs@@ -1,11 +1,11 @@//! GPU types shared between Rust and shader languages (Metal, HLSL, Slang).//!-//! These `#[repr(C)]` structs are the source of truth. The build system uses-//! cbindgen to generate C headers for Metal shaders.+//! These definitions are the source of truth for the headers generated by the build system.++#![allow(dead_code, reason = "Used by shaders.")]pub mod activation_transform;pub mod activation_type;-pub mod attention;pub mod d4s4;pub mod embedding;pub mod gated_act_mul;@@ -19,14 +19,3 @@ pub mod ring;pub mod trellis;pub mod trie;pub mod weaver;--pub use activation_transform::*;-pub use activation_type::*;-pub use attention::*;-pub use embedding::*;-pub use gated_act_mul::*;-pub use hadamard_order::*;-pub use kv_cache_update::*;-pub use matmul::*;-pub use quantization::*;-pub use quantization_method::*;diff --git a/crates/uzu-engine/src/backends/common/gpu_types/weaver.rs b/crates/uzu-engine/src/backends/common/gpu_types/weaver.rsindex 5e568b863..7906652ef 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/weaver.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/weaver.rs@@ -1,6 +1,4 @@pub const CANDIDATES_MAX: u32 = 512;-pub const TOP_CHILDREN_THREADS: u32 = 256;-pub const TOP_CHILDREN_SIMDGROUPS: u32 = 8;#[repr(C)]#[derive(Clone, Copy)]@@ -46,9 +44,6 @@ impl MetadataIdx {}pub const FRONTIER_NO_WINNER: u32 = !0;-pub const FRONTIER_SELECT_THREADS: u32 = 256;-pub const FRONTIER_SELECT_SIMDGROUPS: u32 = 8;-pub const FRONTIER_ENTRIES_PER_THREAD: u32 = 8;pub const FRONTIER_MAX_SLOTS: u32 = 2048;pub const FRONTIER_MAX_WIDTH: u32 = 32;pub const PADDING_DEPTH: u32 = 0;diff --git a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rsindex cf81fce78..8563d165c 100644--- a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs+++ b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs@@ -1,7 +1,7 @@use crate::{backends::common::{Backend, BufferMut, BufferRef, CommandBuffer, Kernels,- gpu_types::{ActivationTransformOp, HADAMARD_TRANSFORM_BLOCK_SIZE},+ gpu_types::{activation_transform::ActivationTransformOp, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},kernel::{ActivationTransformKernel, matmul::Int8CodeLayout},},data_type::DataType,diff --git a/crates/uzu-engine/src/backends/common/kernel/attention.rs b/crates/uzu-engine/src/backends/common/kernel/attention.rsindex 823389935..2f696060a 100644--- a/crates/uzu-engine/src/backends/common/kernel/attention.rs+++ b/crates/uzu-engine/src/backends/common/kernel/attention.rs@@ -1,3 +1,5 @@+use std::error::Error;+use crate::{backends::common::{Backend, BufferRef, CommandBuffer, Kernels},data_type::DataType,@@ -6,11 +8,12 @@ use crate::{pub trait AttentionKernel: Sized + Send + Sync {type Backend: Backend<Kernels: Kernels<AttentionKernel = Self>>;+ type Error: Error + 'static;fn new(context: &<Self::Backend as Backend>::Context,config: AttentionKernelConfig,- ) -> Result<Self, <Self::Backend as Backend>::Error>;+ ) -> Result<Self, Self::Error>;fn encode(&self,@@ -24,7 +27,7 @@ pub trait AttentionKernel: Sized + Send + Sync {impl BufferRef<Backend = Self::Backend>,>,command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<<Self::Backend as Backend>::ScratchBuffer, <Self::Backend as Backend>::Error>;+ ) -> Result<<Self::Backend as Backend>::ScratchBuffer, Self::Error>;}#[derive(Clone, Copy)]diff --git a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rsindex 5a396f962..648c1ad58 100644--- a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs+++ b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs@@ -1,5 +1,7 @@+#[cfg(backend = "cpu")]+use crate::backends::common::kernel::Unsupported;use crate::{- backends::common::{Backend, CommandBuffer, Kernels, kernel::Unsupported},+ backends::common::{Backend, CommandBuffer, Kernels},data_type::DataType,};@@ -38,6 +40,7 @@ pub trait DeltaNetChunkedPrefill: Sized + Send + Sync {) -> Result<(), <Self::Backend as Backend>::Error>;}+#[cfg(backend = "cpu")]impl<B: Backend<Kernels: Kernels<DeltaNetChunkedPrefill = Unsupported<B>>>> DeltaNetChunkedPrefill for Unsupported<B> {type Backend = B;diff --git a/crates/uzu-engine/src/backends/common/kernel/delta_net_tree_verify.rs b/crates/uzu-engine/src/backends/common/kernel/delta_net_tree_verify.rsindex c4113d1d3..25c659881 100644--- a/crates/uzu-engine/src/backends/common/kernel/delta_net_tree_verify.rs+++ b/crates/uzu-engine/src/backends/common/kernel/delta_net_tree_verify.rs@@ -1,5 +1,7 @@+#[cfg(backend = "cpu")]+use crate::backends::common::kernel::Unsupported;use crate::{- backends::common::{Backend, CommandBuffer, Kernels, kernel::Unsupported},+ backends::common::{Backend, CommandBuffer, Kernels},encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},};@@ -20,6 +22,7 @@ pub trait DeltaNetTreeVerify: Sized + Send + Sync {) -> Result<<Self::Backend as Backend>::ScratchBuffer, <Self::Backend as Backend>::Error>;}+#[cfg(backend = "cpu")]impl<B: Backend<Kernels: Kernels<DeltaNetTreeVerify = Unsupported<B>>>> DeltaNetTreeVerify for Unsupported<B> {type Backend = B;diff --git a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rsindex 170f60a48..77dc2bd4d 100644--- a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs+++ b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs@@ -3,7 +3,10 @@ use bitflags::bitflags;use crate::{backends::common::{Backend, BufferMut, BufferRef, CommandBuffer, Kernels,- gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE},+ gpu_types::{+ activation_type::ActivationType, gated_act_mul::GatedActMulOp,+ hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,+ },kernel::{ActivationQuantization, GatedActMulKernel},},config::clipping::ClippingBounds,diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rsdeleted file mode 100644index 24d59b226..000000000--- a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs+++ /dev/null@@ -1,37 +0,0 @@-use derive_more::Debug;-use thiserror::Error;--use crate::{- backends::common::{Backend, gpu_types::gemm::GemmDTransform},- data_type::DataType,-};--#[derive(Debug, Error)]-pub enum MatmulError<B: Backend> {- #[error("Unsupported data type: {0:?}")]- UnsupportedDataType(DataType),- #[error("Unsupported group size: {0}")]- UnsupportedGroupSize(u32),- #[error("Unsupported D-transform op {bit:?} on path {path}")]- UnsupportedDOp {- bit: GemmDTransform,- path: &'static str,- },- #[error("Unsupported layout on path {path}")]- UnsupportedLayout {- path: &'static str,- },- #[error("Unsupported output row stride {stride} for {columns} columns on path {path}")]- UnsupportedOutputStride {- path: &'static str,- stride: u32,- columns: u32,- },- #[error("Incompatible A operand for {path}: {reason}")]- IncompatibleA {- path: &'static str,- reason: &'static str,- },- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),-}diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rsindex 217cb7ca3..f0ba1dd1e 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs@@ -1,3 +1,5 @@+use std::error::Error;+use crate::{backends::common::{Backend, BufferMut, BufferRef, CommandBuffer, Kernels,@@ -14,13 +16,14 @@ use crate::{pub trait MatmulKernel: Sized + Send + Sync {type Backend: Backend<Kernels: Kernels<MatmulKernel = Self>>;+ type Error: Error + 'static;fn new(context: &<Self::Backend as Backend>::Context,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,- ) -> Result<Self, <Self::Backend as Backend>::Error>;+ ) -> Result<Self, Self::Error>;fn encode(&mut self,@@ -33,7 +36,7 @@ pub trait MatmulKernel: Sized + Send + Sync {impl BufferRef<Backend = Self::Backend>,>,command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), <Self::Backend as Backend>::Error>;+ ) -> Result<(), Self::Error>;fn select_activation_quantization(&self,diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rsindex 8cb287064..ca115c9b1 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs@@ -2,7 +2,7 @@ use super::{QuantParams, QuantParamsLayout};use crate::{backends::common::{BufferRef,- gpu_types::{QuantizationMode, gemm::GemmBPrologueKind},+ gpu_types::{gemm::GemmBPrologueKind, quantization::QuantizationMode},},data_type::DataType,};diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rsindex af2eef3ba..314c0ef72 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs@@ -1,6 +1,5 @@mod arguments;mod d_ops;-mod error;mod kernel;mod matmul_a;mod matmul_b;@@ -9,14 +8,15 @@ pub mod routing;pub use arguments::MatmulArguments;pub use d_ops::MatmulDOps;-pub use error::MatmulError;pub use kernel::MatmulKernel;pub use matmul_a::{Int8CodeLayout, MatmulA};-pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection, TrellisFormat};-pub use output::MatmulOutput;+#[cfg(backend = "metal")]+pub use matmul_b::TrellisFormat;+pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};+pub use output::{MatmulOutput, MatmulOutputError};pub use routing::{ActivationFormat, MatmulShape};-use crate::backends::common::gpu_types::{QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT, QuantizationMode};+use crate::backends::common::gpu_types::{matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT, quantization::QuantizationMode};#[derive(Clone, Copy, Debug, PartialEq)]pub enum QuantParamsLayout {diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/output.rsindex c7cb76391..8a73f67e1 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs@@ -1,5 +1,14 @@-use super::{MatmulDOps, MatmulError};-use crate::backends::common::{Backend, BufferMut};+use thiserror::Error;++use crate::backends::common::{Backend, BufferMut, kernel::matmul::MatmulDOps};++#[derive(Debug, Error)]+#[error("Unsupported output row stride {stride} for {columns} columns on path {path}")]+pub struct MatmulOutputError {+ path: &'static str,+ stride: u32,+ columns: u32,+}pub struct MatmulOutput<'d, B: Backend, T: BufferMut<Backend = B>> {pub values: T,@@ -24,10 +33,10 @@ impl<'d, B: Backend, T: BufferMut<Backend = B>> MatmulOutput<'d, B, T> {self,columns: u32,path: &'static str,- ) -> Result<(T, MatmulDOps<'d, B>), MatmulError<B>> {+ ) -> Result<(T, MatmulDOps<'d, B>), MatmulOutputError> {let stride = self.row_stride.unwrap_or(columns);if stride != columns {- return Err(MatmulError::UnsupportedOutputStride {+ return Err(MatmulOutputError {path,stride,columns,diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rsindex bc85140b3..0e1e89e05 100644--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs@@ -1,3 +1,4 @@+#[cfg(backend = "cpu")]use std::{convert::Infallible, marker::PhantomData};use crate::backends::common::Backend;@@ -9,12 +10,15 @@ pub mod delta_net_tree_verify;pub mod gated_act_mul;pub mod matmul;pub mod radix_top_k_small;+#[cfg(any(backend = "cpu", test))]mod trellis;pub use activation_transform::{ActivationQuantization, ActivationTransform};pub use attention::{AttentionArguments, AttentionKernel, AttentionKernelConfig};pub use gated_act_mul::{GatedActMul, GatedActMulSettings};-pub use trellis::mixing_dimension;++#[cfg(any(backend = "cpu", test))]+pub use crate::backends::common::kernel::trellis::mixing_dimension;include!(concat!(env!("OUT_DIR"), "/traits.rs"));@@ -29,6 +33,7 @@ pub trait Kernels: Sized {type RadixTopKSmall: radix_top_k_small::RadixTopKSmall<Backend = Self::Backend>;}+#[cfg(backend = "cpu")]pub struct Unsupported<B: Backend> {never: Infallible,_marker: PhantomData<fn() -> B>,diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rsindex 59a6e5e67..0b3094e9d 100644--- a/crates/uzu-engine/src/backends/common/mod.rs+++ b/crates/uzu-engine/src/backends/common/mod.rs@@ -14,7 +14,7 @@ pub use buffer::{constant::ConstantBuffer,cpu_accessible::BufferCpuAccessible,global::GlobalBuffer,- reference::{BufferMut, BufferRef, Subbuffer},+ reference::{BufferMut, BufferRef},scratch::ScratchBuffer,sparse::SparseBuffer,};diff --git a/crates/uzu-engine/src/backends/cpu/error.rs b/crates/uzu-engine/src/backends/cpu/error.rsindex 581d30aa0..f13385a4b 100644--- a/crates/uzu-engine/src/backends/cpu/error.rs+++ b/crates/uzu-engine/src/backends/cpu/error.rs@@ -1,24 +1,11 @@-use std::{error::Error as StdError, sync::mpsc::RecvError};+use std::sync::mpsc::RecvError;use thiserror::Error;-use crate::backends::{common::kernel::matmul::MatmulError, cpu::Cpu};-#[derive(Debug, Error)]pub enum CpuError {#[error("Not supported")]NotSupported,#[error("Command buffer execution failed")]CommandBufferExecutionFailed(#[from] RecvError),- #[error("Kernel dispatch failed: {0}")]- KernelDispatchFailed(#[source] Box<dyn StdError + Send + Sync + 'static>),-}--impl From<MatmulError<Cpu>> for CpuError {- fn from(value: MatmulError<Cpu>) -> Self {- match value {- MatmulError::BackendError(e) => e,- other => CpuError::KernelDispatchFailed(Box::new(other)),- }- }}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rsindex 1568d5edd..aecabd67a 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs@@ -6,7 +6,7 @@ use super::{hadamard_transform, min_max_symmetric_divisor, quantize_symmetric_i8use crate::{array::ArrayElement,backends::common::{- gpu_types::{ActivationTransformOp, HADAMARD_TRANSFORM_BLOCK_SIZE},+ gpu_types::{activation_transform::ActivationTransformOp, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},kernel::matmul::Int8CodeLayout,},};diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rsindex 0c0f28688..e357123a1 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs@@ -25,6 +25,7 @@ pub struct AttentionCpuKernel {impl AttentionKernel for AttentionCpuKernel {type Backend = Cpu;+ type Error = CpuError;fn new(_context: &CpuContext,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rsindex 73a32ca4f..91ea779f4 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs@@ -6,7 +6,8 @@ use crate::{array::ArrayElement,backends::{common::gpu_types::{- EmbeddingTableKind, HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod, QuantizationMode, d4s4,+ d4s4, embedding::EmbeddingTableKind, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,+ quantization::QuantizationMode, quantization_method::QuantizationMethod,},cpu::kernel::activation_transform::hadamard_transform,},diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rsindex a80a44e7d..72ea5555f 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs@@ -6,7 +6,11 @@ use crate::{array::ArrayElement,backends::{common::{- gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE, activation_silu_alpha},+ gpu_types::{+ activation_type::{ActivationType, activation_silu_alpha},+ gated_act_mul::GatedActMulOp,+ hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,+ },kernel::matmul::Int8CodeLayout,},cpu::kernel::activation_transform::{hadamard_transform, quantize_transformed_row},diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rsindex e45782663..20fb61d10 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs@@ -2,7 +2,7 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};// Multi-token causal conv1d with SiLU for DeltaNet.// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd valuesdiff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rsindex 8f5b56711..607b72505 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs@@ -2,7 +2,7 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};// Single-token causal conv1d with SiLU, in-place.#[kernel(DeltaNetConvUpdate)]diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rsindex 74bd19683..ad61a8a20 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs@@ -2,7 +2,7 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};// In-place RMSNorm + SiLU gate for DeltaNet prefill output.//diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rsindex 3295349b6..28f6b3df7 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs@@ -2,7 +2,7 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};#[kernel(ConvTreeScan)]#[variants(T, f32, bf16)]diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rsindex 1238972e7..6c5ac3f3e 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs@@ -2,7 +2,7 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};// Single-token delta net update: decay state, apply delta rule, RMSNorm + SiLU gate.// Steps per v-head:diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/error.rsnew file mode 100644index 000000000..fe26dc04d--- /dev/null+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/error.rs@@ -0,0 +1,25 @@+use thiserror::Error;++use crate::{+ backends::{common::kernel::matmul::MatmulOutputError, cpu::error::CpuError},+ data_type::DataType,+};++#[derive(Debug, Error)]+pub enum MatmulCpuError {+ #[error("Unsupported data type: {0:?}")]+ UnsupportedDataType(DataType),+ #[error("Unsupported layout on path {path}")]+ UnsupportedLayout {+ path: &'static str,+ },+ #[error("Incompatible A operand for {path}: {reason}")]+ IncompatibleA {+ path: &'static str,+ reason: &'static str,+ },+ #[error(transparent)]+ Output(#[from] MatmulOutputError),+ #[error("Backend error: {0}")]+ BackendError(#[from] CpuError),+}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rsindex 08d5c623d..8a91010d9 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs@@ -1,16 +1,22 @@-use super::reference::{WeightData, read_f32, write_f32};use crate::{backends::{common::{Backend, BufferCpuAccessible, BufferMut, BufferRef, Kernels,- gpu_types::QuantizationMode,+ gpu_types::quantization::QuantizationMode,kernel::{ActivationTransform, TensorAddBiasKernel,- matmul::{Int8CodeLayout, MatmulA, MatmulArguments, MatmulError, MatmulKernel},+ matmul::{Int8CodeLayout, MatmulA, MatmulArguments, MatmulKernel},},},cpu::{- Cpu, buffer::CpuBufferExt, command_buffer::CpuCommandBufferEncoding, context::CpuContext, error::CpuError,+ Cpu,+ buffer::CpuBufferExt,+ command_buffer::CpuCommandBufferEncoding,+ context::CpuContext,+ kernel::matmul::{+ MatmulCpuError,+ reference::{WeightData, read_f32, write_f32},+ },},},data_type::DataType,@@ -27,16 +33,17 @@ pub struct MatmulCpuKernel {impl MatmulKernel for MatmulCpuKernel {type Backend = Cpu;+ type Error = MatmulCpuError;fn new(context: &CpuContext,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,- ) -> Result<Self, CpuError> {+ ) -> Result<Self, Self::Error> {for data_type in [weights_data_type, input_data_type, output_data_type] {if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());+ return Err(MatmulCpuError::UnsupportedDataType(data_type));}}let output_rht = ActivationTransform::output_rht(context, output_data_type, None, true)?;@@ -66,7 +73,7 @@ impl MatmulKernel for MatmulCpuKernel {impl BufferRef<Backend = Cpu>,>,command_buffer: &mut CpuCommandBufferEncoding,- ) -> Result<(), CpuError> {+ ) -> Result<(), Self::Error> {let MatmulArguments {a,b,@@ -126,11 +133,10 @@ impl MatmulKernel for MatmulCpuKernel {&& b.quantized().is_some_and(|quantized| matches!(quantized.mode, QuantizationMode::U4 | QuantizationMode::U8));if !compatible {- return Err(MatmulError::IncompatibleA {+ return Err(MatmulCpuError::IncompatibleA {path: "CpuMatmul",reason: "symmetric int8 activations require a supported 32/64/128 activation and weight group",- }- .into());+ });}let (values, values_range) = values.parts();let (scales, scales_range) = scales.parts();@@ -148,10 +154,9 @@ impl MatmulKernel for MatmulCpuKernel {MatmulA::Trellis {..} => {- return Err(MatmulError::UnsupportedLayout {+ return Err(MatmulCpuError::UnsupportedLayout {path: "CpuMatmul",- }- .into());+ });},};let bias_ptr = bias_buffer.map(|bias| SendPtr(bias.cpu_ptr().as_ptr().cast::<u8>().cast_const()));diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/mod.rsindex aa648840e..c5323acc2 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/mod.rs@@ -1,4 +1,6 @@+mod error;mod kernel;mod reference;+pub use error::MatmulCpuError;pub use kernel::MatmulCpuKernel;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rsindex 9ca424454..0b5788925 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs@@ -4,9 +4,9 @@ use crate::{backends::{common::{BufferRef,- kernel::matmul::{MatmulB, MatmulError, QuantParamsStrides},+ kernel::matmul::{MatmulB, QuantParamsStrides},},- cpu::{Cpu, buffer::CpuBufferExt},+ cpu::{Cpu, buffer::CpuBufferExt, kernel::matmul::MatmulCpuError},},data_type::DataType,utils::pointers::SendPtr,@@ -37,7 +37,7 @@ impl WeightData {b_transpose: bool,k: usize,n: usize,- ) -> Result<Self, MatmulError<Cpu>> {+ ) -> Result<Self, MatmulCpuError> {fn buffer_ptr(view: impl BufferRef<Backend = Cpu>) -> SendPtr<u8> {let (buffer, range) = view.parts();SendPtr(buffer.cpu_address().as_ptr().cast::<u8>().cast_const().wrapping_byte_add(range.start))@@ -59,7 +59,7 @@ impl WeightData {},MatmulB::Trellis {..- } => Err(MatmulError::UnsupportedLayout {+ } => Err(MatmulCpuError::UnsupportedLayout {path: "CpuMatmul",}),MatmulB::Quantized(quantized) => Ok(WeightData::Quantized {diff --git a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rsindex 16399ee54..f87bb72ac 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs@@ -5,7 +5,8 @@ use uzu_engine_macros::kernel;use crate::{array::ArrayElement,backends::{- common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE, cpu::kernel::activation_transform::hadamard_transform,+ common::gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,+ cpu::kernel::activation_transform::hadamard_transform,},};diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rsindex 1b151e3cd..290953a07 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs@@ -2,7 +2,7 @@ use half::{bf16, f16};use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};#[kernel(SSDUpdate)]#[variants(T, f32, f16, bf16)]diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rsindex e7800cd75..d4aeedf50 100644--- a/crates/uzu-engine/src/backends/metal/error.rs+++ b/crates/uzu-engine/src/backends/metal/error.rs@@ -1,12 +1,7 @@-use std::{error::Error as StdError, sync::mpsc::RecvError};+use std::sync::mpsc::RecvError;use thiserror::Error;-use crate::backends::{- common::kernel::matmul::MatmulError,- metal::{Metal, kernel::matmul::gemm::GemmSpecializationError},-};-#[derive(Debug, Error)]pub enum MetalError {#[error("Cannot open device")]@@ -40,21 +35,4 @@ pub enum MetalError {function_name: String,error: String,},- #[error("Kernel dispatch failed: {0}")]- KernelDispatchFailed(#[source] Box<dyn StdError + Send + Sync + 'static>),-}--impl From<MatmulError<Metal>> for MetalError {- fn from(value: MatmulError<Metal>) -> Self {- match value {- MatmulError::BackendError(e) => e,- other => MetalError::KernelDispatchFailed(Box::new(other)),- }- }-}--impl From<GemmSpecializationError> for MetalError {- fn from(value: GemmSpecializationError) -> Self {- MetalError::KernelDispatchFailed(Box::new(value))- }}diff --git a/crates/uzu-engine/src/backends/common/gpu_types/attention.rs b/crates/uzu-engine/src/backends/metal/gpu_types/attention.rssimilarity index 100%rename from crates/uzu-engine/src/backends/common/gpu_types/attention.rsrename to crates/uzu-engine/src/backends/metal/gpu_types/attention.rsdiff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rsnew file mode 100644index 000000000..4a2a7e83a--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs@@ -0,0 +1,170 @@+use bitflags::bitflags;+use derive_more::Display;++#[repr(C)]+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]+pub enum GemmAPrologueKind {+ FullPrecision,+ Int8Symmetric,+}++#[repr(C)]+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]+#[allow(non_camel_case_types)]+pub enum GemmTiling {+ Tile8x32x32_Simdgroups1x1,+ Tile64x32x32_Simdgroups2x2,+ Tile64x64x16_Simdgroups2x2,+ Tile64x64x32_Simdgroups2x2,+ Tile32x32x32_Simdgroups2x2,+ Tile16x32x256_Simdgroups1x1,+ Tile16x32x256_Simdgroups1x1x4,+ Tile16x128x256_Simdgroups1x4,+ Tile32x64x256_Simdgroups2x2,+ Tile64x32x256_Simdgroups4x1,+ Tile64x64x256_Simdgroups2x2,+ Tile128x128x256_Simdgroups4x4,+}++pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;++impl GemmTiling {+ pub const fn block_m(self) -> u32 {+ match self {+ Self::Tile8x32x32_Simdgroups1x1 => 8,+ Self::Tile64x32x32_Simdgroups2x2 => 64,+ Self::Tile64x64x16_Simdgroups2x2 => 64,+ Self::Tile64x64x32_Simdgroups2x2 => 64,+ Self::Tile32x32x32_Simdgroups2x2 => 32,+ Self::Tile16x32x256_Simdgroups1x1 => 16,+ Self::Tile16x32x256_Simdgroups1x1x4 => 16,+ Self::Tile16x128x256_Simdgroups1x4 => 16,+ Self::Tile32x64x256_Simdgroups2x2 => 32,+ Self::Tile64x32x256_Simdgroups4x1 => 64,+ Self::Tile64x64x256_Simdgroups2x2 => 64,+ Self::Tile128x128x256_Simdgroups4x4 => 128,+ }+ }++ pub const fn block_n(self) -> u32 {+ match self {+ Self::Tile8x32x32_Simdgroups1x1 => 32,+ Self::Tile64x32x32_Simdgroups2x2 => 32,+ Self::Tile64x64x16_Simdgroups2x2 => 64,+ Self::Tile64x64x32_Simdgroups2x2 => 64,+ Self::Tile32x32x32_Simdgroups2x2 => 32,+ Self::Tile16x32x256_Simdgroups1x1 => 32,+ Self::Tile16x32x256_Simdgroups1x1x4 => 32,+ Self::Tile16x128x256_Simdgroups1x4 => 128,+ Self::Tile32x64x256_Simdgroups2x2 => 64,+ Self::Tile64x32x256_Simdgroups4x1 => 32,+ Self::Tile64x64x256_Simdgroups2x2 => 64,+ Self::Tile128x128x256_Simdgroups4x4 => 128,+ }+ }++ pub const fn block_k(self) -> u32 {+ match self {+ Self::Tile8x32x32_Simdgroups1x1 => 32,+ Self::Tile64x32x32_Simdgroups2x2 => 32,+ Self::Tile64x64x16_Simdgroups2x2 => 16,+ Self::Tile64x64x32_Simdgroups2x2 => 32,+ Self::Tile32x32x32_Simdgroups2x2 => 32,+ Self::Tile16x32x256_Simdgroups1x1 => 256,+ Self::Tile16x32x256_Simdgroups1x1x4 => 256,+ Self::Tile16x128x256_Simdgroups1x4 => 256,+ Self::Tile32x64x256_Simdgroups2x2 => 256,+ Self::Tile64x32x256_Simdgroups4x1 => 256,+ Self::Tile64x64x256_Simdgroups2x2 => 256,+ Self::Tile128x128x256_Simdgroups4x4 => 256,+ }+ }++ pub const fn simdgroups_m(self) -> u32 {+ match self {+ Self::Tile8x32x32_Simdgroups1x1 => 1,+ Self::Tile64x32x32_Simdgroups2x2 => 2,+ Self::Tile64x64x16_Simdgroups2x2 => 2,+ Self::Tile64x64x32_Simdgroups2x2 => 2,+ Self::Tile32x32x32_Simdgroups2x2 => 2,+ Self::Tile16x32x256_Simdgroups1x1 => 1,+ Self::Tile16x32x256_Simdgroups1x1x4 => 1,+ Self::Tile16x128x256_Simdgroups1x4 => 1,+ Self::Tile32x64x256_Simdgroups2x2 => 2,+ Self::Tile64x32x256_Simdgroups4x1 => 4,+ Self::Tile64x64x256_Simdgroups2x2 => 2,+ Self::Tile128x128x256_Simdgroups4x4 => 4,+ }+ }++ pub const fn simdgroups_n(self) -> u32 {+ match self {+ Self::Tile8x32x32_Simdgroups1x1 => 1,+ Self::Tile64x32x32_Simdgroups2x2 => 2,+ Self::Tile64x64x16_Simdgroups2x2 => 2,+ Self::Tile64x64x32_Simdgroups2x2 => 2,+ Self::Tile32x32x32_Simdgroups2x2 => 2,+ Self::Tile16x32x256_Simdgroups1x1 => 1,+ Self::Tile16x32x256_Simdgroups1x1x4 => 1,+ Self::Tile16x128x256_Simdgroups1x4 => 4,+ Self::Tile32x64x256_Simdgroups2x2 => 2,+ Self::Tile64x32x256_Simdgroups4x1 => 1,+ Self::Tile64x64x256_Simdgroups2x2 => 2,+ Self::Tile128x128x256_Simdgroups4x4 => 4,+ }+ }++ pub const fn simdgroups_k(self) -> u32 {+ match self {+ Self::Tile16x32x256_Simdgroups1x1x4 => 4,+ _ => 1,+ }+ }++ pub const fn is_mxu_variant(self) -> bool {+ matches!(+ self,+ Self::Tile16x32x256_Simdgroups1x1+ | Self::Tile16x32x256_Simdgroups1x1x4+ | Self::Tile16x128x256_Simdgroups1x4+ | Self::Tile32x64x256_Simdgroups2x2+ | Self::Tile64x32x256_Simdgroups4x1+ | Self::Tile64x64x256_Simdgroups2x2+ | Self::Tile128x128x256_Simdgroups4x4+ )+ }+}++pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {+ t.simdgroups_m()+}+pub const fn gemm_tiling_simdgroups_per_column(t: GemmTiling) -> u32 {+ t.simdgroups_n()+}+pub const fn gemm_tiling_simdgroups_per_k(t: GemmTiling) -> u32 {+ t.simdgroups_k()+}++bitflags! {+ #[repr(transparent)]+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+ pub struct GemmAlignment: u32 {+ const M = 1 << 0;+ const N = 1 << 1;+ const K = 1 << 2;+ }+}++impl GemmAlignment {+ pub fn new(+ m: bool,+ n: bool,+ k: bool,+ ) -> Self {+ let mut bits = Self::empty();+ bits.set(Self::M, m);+ bits.set(Self::N, n);+ bits.set(Self::K, k);+ bits+ }+}diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs b/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rsnew file mode 100644index 000000000..e29eea201--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs@@ -0,0 +1,20 @@+#[repr(C)]+#[allow(non_snake_case)]+#[derive(Debug, Default, Copy, Clone)]+pub struct GemmParams {+ pub M: u32,+ pub N: u32,+ pub K: u32,+ pub leading_dimension_a: u32,+ pub leading_dimension_b: u32,+ pub scale_output_stride: u32,+ pub scale_group_stride: u32,+ pub zero_point_output_stride: u32,+ pub zero_point_group_stride: u32,+ pub leading_dimension_d: u32,+ pub threadgroups_per_column: u32,+ pub threadgroups_per_row: u32,+ pub aligned_inner_iterations: u32,+ pub use_morton: bool,+ pub ab_scale: f32,+}diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/mod.rs b/crates/uzu-engine/src/backends/metal/gpu_types/mod.rsnew file mode 100644index 000000000..4f0947a4e--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/gpu_types/mod.rs@@ -0,0 +1,6 @@+#![allow(dead_code, reason = "Used by Metal shaders.")]++pub mod attention;+pub mod gemm;+pub mod matmul;+pub mod weaver;diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/weaver.rs b/crates/uzu-engine/src/backends/metal/gpu_types/weaver.rsnew file mode 100644index 000000000..ef6dcda50--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/gpu_types/weaver.rs@@ -0,0 +1,5 @@+pub const TOP_CHILDREN_THREADS: u32 = 256;+pub const TOP_CHILDREN_SIMDGROUPS: u32 = 8;+pub const FRONTIER_SELECT_THREADS: u32 = 256;+pub const FRONTIER_SELECT_SIMDGROUPS: u32 = 8;+pub const FRONTIER_ENTRIES_PER_THREAD: u32 = 8;diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/error.rsnew file mode 100644index 000000000..f31b769a4--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs@@ -0,0 +1,11 @@+use thiserror::Error;++use crate::backends::metal::{error::MetalError, kernel::matmul::MatmulMetalError};++#[derive(Debug, Error)]+pub enum AttentionMetalError {+ #[error("Backend error: {0}")]+ Backend(#[from] MetalError),+ #[error("Matmul error: {0}")]+ Matmul(#[from] MatmulMetalError),+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rsindex 114940f4a..b7bd69ab4 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs@@ -13,9 +13,9 @@ use crate::{Metal,command_buffer::MetalCommandBufferEncoding,context::MetalContext,- error::MetalError,kernel::{AttentionFallbackScatterScoresMetalKernel, AttentionFallbackScatterValuesMetalKernel, MetalKernels,+ attention::AttentionMetalError,},},},@@ -45,7 +45,7 @@ impl AttentionFallback {pub fn new(config: &AttentionKernelConfig,context: &MetalContext,- ) -> Result<Self, MetalError> {+ ) -> Result<Self, AttentionMetalError> {let scatter_scores = AttentionFallbackScatterScoresMetalKernel::new(context,config.data_type,@@ -92,7 +92,7 @@ impl AttentionFallback {impl BufferRef<Backend = Metal>,>,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {+ ) -> Result<<Metal as Backend>::ScratchBuffer, AttentionMetalError> {assert!(arguments.trie.is_none(), "fallback does not support trie");let suffix_length = arguments.suffix_length;let sequence_length = arguments.cache.prefix_len() + suffix_length;diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rsindex 6c4ea891f..78c035fb2 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs@@ -6,12 +6,11 @@ use crate::{backends::{common::{Backend, BufferRef, CommandBufferEncoding,- gpu_types::AttnParams,kernel::{AttentionArguments, AttentionKernelConfig},},metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError,- kernel::AttentionGemmMetalKernel,+ gpu_types::attention::AttnParams, kernel::AttentionGemmMetalKernel,},},data_type::DataType,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rsindex a26465005..89f3f39ed 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs@@ -4,7 +4,6 @@ use crate::{backends::{common::{Backend, BufferMut, BufferRef, CommandBufferEncoding,- gpu_types::AttnParams,kernel::{AttentionArguments, AttentionKernelConfig},},metal::{@@ -12,6 +11,7 @@ use crate::{command_buffer::MetalCommandBufferEncoding,context::MetalContext,error::MetalError,+ gpu_types::attention::AttnParams,kernel::{AttentionGemmGroupedCombineMetalKernel, AttentionGemmGroupedMetalKernel},},},diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rsindex 4452fe8a1..1530440be 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs@@ -1,20 +1,20 @@use crate::backends::common::Backend;+mod error;mod fallback;mod gemm;mod gemm_grouped;mod single_pass;mod two_pass;+pub use error::AttentionMetalError;+use crate::backends::{common::{BufferRef, CommandBufferEncoding,kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig, SigmoidGateKernel},},- metal::{- Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError,- kernel::SigmoidGateMetalKernel,- },+ metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, kernel::SigmoidGateMetalKernel},};#[derive(Clone, Copy, Debug, PartialEq, Eq)]@@ -62,11 +62,12 @@ pub struct AttentionMetalKernel {impl AttentionKernel for AttentionMetalKernel {type Backend = Metal;+ type Error = AttentionMetalError;fn new(context: &MetalContext,config: AttentionKernelConfig,- ) -> Result<Self, MetalError> {+ ) -> Result<Self, AttentionMetalError> {let grouped = gemm_grouped::AttentionGemmGrouped::is_supported(&config, context).then(|| gemm_grouped::AttentionGemmGrouped::new(&config));let gemm = gemm::AttentionGemm::is_supported(&config).then(|| gemm::AttentionGemm::new(&config));@@ -100,7 +101,7 @@ impl AttentionKernel for AttentionMetalKernel {impl BufferRef<Backend = Metal>,>,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {+ ) -> Result<<Metal as Backend>::ScratchBuffer, AttentionMetalError> {let command_buffer = &mut *command_buffer.debug_group("attention core");let (gate, suffix_length) = (arguments.gate, arguments.suffix_length);self.encode_impl(arguments, command_buffer).map(|(mut output, gate_applied)| {@@ -132,7 +133,7 @@ impl AttentionMetalKernel {impl BufferRef<Backend = Metal>,>,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<(<Metal as Backend>::ScratchBuffer, bool), MetalError> {+ ) -> Result<(<Metal as Backend>::ScratchBuffer, bool), AttentionMetalError> {let suffix_length = arguments.suffix_length;let gate_applied_by_two_pass = arguments.gate.is_some();let kv_length = arguments.cache.prefix_len() + suffix_length;@@ -145,7 +146,7 @@ impl AttentionMetalKernel {&& let Some(grouped) = &self.grouped{if grouped.should_encode(mask, suffix_length, kv_length) {- return grouped.encode(mask, arguments, command_buffer);+ return grouped.encode(mask, arguments, command_buffer).map_err(AttentionMetalError::Backend);}// These measured sibling exceptions apply only when thediff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.hindex 6a2af05cb..8f8b35954 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h@@ -5,11 +5,6 @@using namespace metal;namespace uzu::gemm {-enum class GemmAPrologueKind : uint32_t {- FullPrecision = 0,- Int8Symmetric = 1,-};-enum class GemmBPrologueKind : uint32_t {FullPrecision = 0,ScaleBiasDequant = 1,@@ -31,6 +26,11 @@ struct GemmDTransform {constexpr uint32_t bits() const constant { return raw_value; }};+enum class GemmAPrologueKind : uint32_t {+ FullPrecision = 0,+ Int8Symmetric = 1,+};+enum class GemmTiling : uint32_t {Tile8x32x32_Simdgroups1x1 = 0,Tile64x32x32_Simdgroups2x2 = 1,diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/weaver.h b/crates/uzu-engine/src/backends/metal/kernel/generated/weaver.hindex 173fdfaf4..9533a3269 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/weaver.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/weaver.h@@ -7,10 +7,6 @@ using namespace metal;namespace uzu::weaver {static constant constexpr uint32_t CANDIDATES_MAX = 512;-static constant constexpr uint32_t TOP_CHILDREN_THREADS = 256;--static constant constexpr uint32_t TOP_CHILDREN_SIMDGROUPS = 8;-enum class FrontierIdx : uint32_t {TokenId = 0,ParentSlot = 1,@@ -38,15 +34,19 @@ enum class MetadataIdx : uint32_t {static constant constexpr uint32_t FRONTIER_NO_WINNER = ~0;-static constant constexpr uint32_t FRONTIER_SELECT_THREADS = 256;--static constant constexpr uint32_t FRONTIER_SELECT_SIMDGROUPS = 8;--static constant constexpr uint32_t FRONTIER_ENTRIES_PER_THREAD = 8;-static constant constexpr uint32_t FRONTIER_MAX_SLOTS = 2048;static constant constexpr uint32_t FRONTIER_MAX_WIDTH = 32;static constant constexpr uint32_t PADDING_DEPTH = 0;++static constant constexpr uint32_t TOP_CHILDREN_THREADS = 256;++static constant constexpr uint32_t TOP_CHILDREN_SIMDGROUPS = 8;++static constant constexpr uint32_t FRONTIER_SELECT_THREADS = 256;++static constant constexpr uint32_t FRONTIER_SELECT_SIMDGROUPS = 8;++static constant constexpr uint32_t FRONTIER_ENTRIES_PER_THREAD = 8;} // namespace uzu::weaverdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/error.rsnew file mode 100644index 000000000..5d04d64be--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/error.rs@@ -0,0 +1,45 @@+use thiserror::Error;++use crate::{+ backends::{+ common::{gpu_types::gemm::GemmDTransform, kernel::matmul::MatmulOutputError},+ metal::{+ error::MetalError,+ kernel::matmul::gemm::{GemmPlanError, GemmSpecializationError},+ },+ },+ data_type::DataType,+};++#[derive(Debug, Error)]+pub enum MatmulMetalError {+ #[error("Unsupported data type: {0:?}")]+ UnsupportedDataType(DataType),+ #[error("Unsupported D-transform op {bit:?} on path {path}")]+ UnsupportedDOp {+ bit: GemmDTransform,+ path: &'static str,+ },+ #[error("Unsupported layout on path {path}")]+ UnsupportedLayout {+ path: &'static str,+ },+ #[error("Incompatible A operand for {path}: {reason}")]+ IncompatibleA {+ path: &'static str,+ reason: &'static str,+ },+ #[error("gathered readout requires the GEMV path, but shape (m={m}, n={n}) routes to GEMM")]+ UnsupportedGather {+ m: u32,+ n: u32,+ },+ #[error(transparent)]+ Output(#[from] MatmulOutputError),+ #[error(transparent)]+ Plan(#[from] GemmPlanError),+ #[error(transparent)]+ Specialization(#[from] GemmSpecializationError),+ #[error("Backend error: {0}")]+ BackendError(#[from] MetalError),+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rsindex bb635d08c..50b26d769 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs@@ -1,6 +1,6 @@use thiserror::Error;-use crate::backends::common::gpu_types::gemm::GemmTiling;+use crate::backends::metal::gpu_types::gemm::GemmTiling;#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]pub enum GemmSpecializationError {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 684ae1c35..c2eb07de3 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -1,23 +1,13 @@use std::collections::{HashMap, hash_map::Entry};-use super::{- super::{MatmulOutputWork, supports_integer_right_operand},- GemmEngine, GemmPlan,- selection::{GemmProblem, outer_block_k},- specialization::GemmSpecialization,- trellis::TrellisGemm,-};use crate::{backends::{common::{Backend, BufferMut, BufferRef, CommandBufferEncoding,- gpu_types::{- GemmParams,- gemm::{GemmAPrologueKind, GemmAlignment, GemmBPrologueKind, GemmDTransform},- },+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},kernel::matmul::{- Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulError, MatmulOutput, MatmulShape,- QuantParamsLayout, QuantParamsStrides, QuantizedB,+ Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulOutput, MatmulShape, QuantParamsLayout,+ QuantParamsStrides, QuantizedB,},},metal::{@@ -25,7 +15,23 @@ use crate::{command_buffer::MetalCommandBufferEncoding,context::MetalContext,error::MetalError,- kernel::{GemmMetalKernel, GemmSplitKReduceMetalKernel},+ gpu_types::{+ gemm::{GemmAPrologueKind, GemmAlignment},+ matmul::GemmParams,+ },+ kernel::{+ GemmMetalKernel, GemmSplitKReduceMetalKernel,+ matmul::{+ MatmulMetalError, MatmulOutputWork,+ gemm::{+ GemmEngine, GemmPlan,+ selection::{GemmProblem, outer_block_k},+ specialization::GemmSpecialization,+ trellis::TrellisGemm,+ },+ supports_integer_right_operand,+ },+ },},},data_type::DataType,@@ -131,10 +137,8 @@ impl GemmKernel {shape: &MatmulShape,engine: GemmEngine,context: &MetalContext,- ) -> Result<GemmPlan, MetalError> {- self.problem(*shape, context)- .select_plan_for_engine(engine)- .map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))+ ) -> Result<GemmPlan, MatmulMetalError> {+ self.problem(*shape, context).select_plan_for_engine(engine).map_err(MatmulMetalError::from)}pub fn encode_plan(@@ -150,21 +154,16 @@ impl GemmKernel {plan: GemmPlan,output_work: &MatmulOutputWork,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<(), MetalError> {+ ) -> Result<(), MatmulMetalError> {let shape = MatmulShape::from_arguments(&arguments);// TODO: remove after GatherGEMM is supportedif arguments.gather_indices.is_some() {- return Err(MetalError::KernelDispatchFailed(- format!(- "gathered readout requires the GEMV path, but shape (m={}, n={}) routes to GEMM",- arguments.m, arguments.n- )- .into(),- ));+ return Err(MatmulMetalError::UnsupportedGather {+ m: arguments.m,+ n: arguments.n,+ });}- self.problem(shape, command_buffer.context())- .validate_engine(plan.engine)- .map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))?;+ self.problem(shape, command_buffer.context()).validate_engine(plan.engine)?;match arguments.b {MatmulB::FullPrecision {@@ -193,7 +192,7 @@ impl GemmKernel {..} => {if self.output_data_type != DataType::BF16 {- return Err(MatmulError::UnsupportedDataType(self.output_data_type).into());+ return Err(MatmulMetalError::UnsupportedDataType(self.output_data_type));}self.trellis.encode_plan(arguments, plan, command_buffer)},@@ -210,7 +209,7 @@ impl GemmKernel {plan: GemmPlan,output_work: &MatmulOutputWork,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<(), MetalError> {+ ) -> Result<(), MatmulMetalError> {let (m, n, k) = (shape.m, shape.n, shape.k);let (mut d, d_transform) = output.into_contiguous(n, "Gemm")?;let ab_scale = d_transform.ab_scale;@@ -228,11 +227,10 @@ impl GemmKernel {let is_quant = quantized.is_some();if !is_quant && !matches!(&a, MatmulA::FullPrecision { .. }) {- return Err(MatmulError::IncompatibleA {+ return Err(MatmulMetalError::IncompatibleA {path: "Gemm",reason: "int8 activations require quantized weights",- }- .into());+ });}let (scales, biases, zero_points, scale_strides, zero_point_strides) = match quantized.as_ref() {None => (None, None, None, Default::default(), Default::default()),@@ -264,11 +262,10 @@ impl GemmKernel {activation_group_sums.is_some(),)?;if output_transform.contains(GemmDTransform::SOFT_CAP) {- return Err(MatmulError::UnsupportedDOp {+ return Err(MatmulMetalError::UnsupportedDOp {bit: GemmDTransform::SOFT_CAP,path: "Gemm int8 left operand",- }- .into());+ });}(None,@@ -282,10 +279,9 @@ impl GemmKernel {MatmulA::Trellis {..} => {- return Err(MatmulError::UnsupportedLayout {+ return Err(MatmulMetalError::UnsupportedLayout {path: "Gemm",- }- .into());+ });},};@@ -395,7 +391,7 @@ impl GemmKernel {output_transform: GemmDTransform,output_bias: Option<impl BufferRef<Backend = Metal>>,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<(), MetalError> {+ ) -> Result<(), MatmulMetalError> {let (m, n, k) = (shape.m, shape.n, shape.k);let (a_full_precision, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match a {MatmulA::FullPrecision {@@ -414,10 +410,9 @@ impl GemmKernel {MatmulA::Trellis {..} => {- return Err(MatmulError::UnsupportedLayout {+ return Err(MatmulMetalError::UnsupportedLayout {path: "Gemm",- }- .into());+ });},};let a_group_sums = a_group_sums.filter(|_| Self::requires_activation_group_sums(&shape));@@ -496,20 +491,18 @@ fn validate_int8_left_operand(a_group_size: u32,code_layout: Int8CodeLayout,has_group_sums: bool,-) -> Result<(), MetalError> {+) -> Result<(), MatmulMetalError> {if shape.b_bits.and_then(Int8CodeLayout::for_right_bits) != Some(code_layout) {- return Err(MatmulError::IncompatibleA {+ return Err(MatmulMetalError::IncompatibleA {path: "Gemm",reason: "left code layout is incompatible with the right operand",- }- .into());+ });}if GemmKernel::requires_activation_group_sums(&shape) && !has_group_sums {- return Err(MatmulError::IncompatibleA {+ return Err(MatmulMetalError::IncompatibleA {path: "Gemm",reason: "quantized correction requires left group sums",- }- .into());+ });}let compatible = use_mxu&& supports_integer_right_operand(&shape)@@ -526,11 +519,10 @@ fn validate_int8_left_operand(.b_group_size.is_some_and(|gs| matches!(gs, 32 | 64 | 128) && shape.k.is_multiple_of(gs) && a_group_size >= gs);if !compatible {- return Err(MatmulError::IncompatibleA {+ return Err(MatmulMetalError::IncompatibleA {path: "Gemm",reason: "symmetric int8 left operands require group-major metadata and supported 32/64/128 groups",- }- .into());+ });}Ok(())}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rsindex d45b0d0c0..5ce0acd5e 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs@@ -7,9 +7,10 @@ mod trellis;pub use error::GemmSpecializationError;pub use kernel::GemmKernel;+pub use selection::GemmPlanError;pub(super) use selection::GemmProblem;-use crate::backends::common::gpu_types::gemm::GemmTiling;+use crate::backends::metal::gpu_types::gemm::GemmTiling;#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum GemmEngine {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rsindex db4e537a4..c80f269a3 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs@@ -6,7 +6,7 @@use metal::MTLGPUFamily;-use crate::backends::common::gpu_types::gemm::GemmTiling;+use crate::backends::metal::gpu_types::gemm::GemmTiling;const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rsindex 86fb18c70..3b45fe82f 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -1,11 +1,16 @@use metal::MTLGPUFamily;use thiserror::Error;-use super::{GemmEngine, GemmPlan, policy};use crate::{- backends::common::{- gpu_types::gemm::{GemmBPrologueKind, GemmTiling},- kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},+ backends::{+ common::{+ gpu_types::gemm::GemmBPrologueKind,+ kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},+ },+ metal::{+ gpu_types::gemm::GemmTiling,+ kernel::matmul::gemm::{GemmEngine, GemmPlan, policy},+ },},data_type::DataType,};@@ -22,7 +27,7 @@ pub struct GemmProblem {}#[derive(Debug, Clone, Copy, PartialEq, Error)]-pub(super) enum GemmPlanError {+pub enum GemmPlanError {#[error("MXU engine is not available for this GEMM")]MxuUnavailable,#[error("GEMM input layout is unsupported: {0}")]diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rsindex d215a1d41..25ea3639f 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -1,8 +1,11 @@use super::{GemmEngine, GemmPlan, error::GemmSpecializationError};use crate::{- backends::common::{- gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},- kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},+ backends::{+ common::{+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},+ kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},+ },+ metal::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmTiling},},data_type::DataType,};diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rsindex 0929b24b3..fdfe28a11 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs@@ -1,20 +1,24 @@use std::collections::{HashMap, hash_map::Entry};-use super::{GemmEngine, GemmPlan, selection::TRELLIS_K_STEP as K_STEP};use crate::backends::{common::{BufferMut, BufferRef, CommandBufferEncoding,- gpu_types::{- GemmParams,- gemm::{GemmAlignment, GemmTiling},- },- kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError, TrellisFormat},+ kernel::matmul::{MatmulA, MatmulArguments, MatmulB, TrellisFormat},},metal::{Metal,command_buffer::MetalCommandBufferEncoding,- error::MetalError,- kernel::{GemmTrellisMetalKernel, GemmTrellisReduceMetalKernel},+ gpu_types::{+ gemm::{GemmAlignment, GemmTiling},+ matmul::GemmParams,+ },+ kernel::{+ GemmTrellisMetalKernel, GemmTrellisReduceMetalKernel,+ matmul::{+ MatmulMetalError,+ gemm::{GemmEngine, GemmPlan, selection::TRELLIS_K_STEP as K_STEP},+ },+ },},};@@ -37,11 +41,9 @@ impl TrellisGemm {>,plan: GemmPlan,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<(), MetalError> {- let unsupported = || {- MetalError::from(MatmulError::UnsupportedLayout {- path: "GemmTrellis",- })+ ) -> Result<(), MatmulMetalError> {+ let unsupported = || MatmulMetalError::UnsupportedLayout {+ path: "GemmTrellis",};let (MatmulA::Trellis {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex a69d487f2..e3ec42f2a 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -1,22 +1,26 @@use metal::MTLGPUFamily;-use super::{- super::MatmulOutputWork,- policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK},-};use crate::{backends::{common::{CommandBufferEncoding,gpu_types::{- HADAMARD_TRANSFORM_BLOCK_SIZE,gemm::{GemmBPrologueKind, GemmDTransform},+ hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,},kernel::matmul::{MatmulB, MatmulShape},},metal::{- command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError,- kernel::GemvMetalKernel,+ command_buffer::MetalCommandBufferEncoding,+ context::MetalContext,+ error::MetalError,+ kernel::{+ GemvMetalKernel,+ matmul::{+ MatmulOutputWork,+ gemv::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK},+ },+ },},},data_type::DataType,@@ -211,9 +215,9 @@ use std::collections::{HashMap, hash_map::Entry};use crate::backends::{common::{BufferMut, BufferRef,- kernel::matmul::{MatmulA, MatmulArguments, MatmulError},+ kernel::matmul::{MatmulA, MatmulArguments},},- metal::Metal,+ metal::{Metal, kernel::matmul::MatmulMetalError},};/// GEMV pipelines compiled on first use.@@ -242,13 +246,16 @@ impl GemvKernel {&mut self,context: &MetalContext,specialization: GemvSpecialization,- ) -> Result<&GemvMetalKernel, MatmulError<Metal>> {+ ) -> Result<&GemvMetalKernel, MatmulMetalError> {match self.pipelines.entry(specialization) {Entry::Occupied(entry) => Ok(entry.into_mut()),Entry::Vacant(entry) => {- let kernel = specialization- .create_pipeline(context, self.weights_data_type, self.input_data_type, self.output_data_type)- .map_err(MatmulError::BackendError)?;+ let kernel = specialization.create_pipeline(+ context,+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,+ )?;Ok(entry.insert(kernel))},}@@ -267,7 +274,7 @@ impl GemvKernel {specialization: GemvSpecialization,output_work: &MatmulOutputWork,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<(), MatmulError<Metal>> {+ ) -> Result<(), MatmulMetalError> {let MatmulArguments {a,b,@@ -294,7 +301,7 @@ impl GemvKernel {offset: a_offset,} = aelse {- return Err(MatmulError::IncompatibleA {+ return Err(MatmulMetalError::IncompatibleA {path: "Gemv",reason: "prepared int8 activations require GEMM",});@@ -364,7 +371,7 @@ impl GemvKernel {MatmulB::Trellis {..} => {- return Err(MatmulError::UnsupportedLayout {+ return Err(MatmulMetalError::UnsupportedLayout {path: "Gemv",});},diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rsindex e40b744f7..deeb74426 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs@@ -1,28 +1,31 @@use metal::MTLGPUFamily;-use super::{- GemmKernel,- gemm::{self, GemmPlan, GemmProblem},- gemv::{GemvKernel, GemvSpecialization},- matmul_dispatch::MatmulDispatch,- matmul_output_work::MatmulOutputWork,- qmv::{self, QmvRoute},-};use crate::{backends::{common::{BufferMut, BufferRef, CommandBufferEncoding,- gpu_types::gemm::GemmTiling,kernel::{ActivationQuantization,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,matmul::{- ActivationFormat, Int8CodeLayout, MatmulArguments, MatmulError, MatmulKernel, MatmulShape,- QuantParamsLayout,+ ActivationFormat, Int8CodeLayout, MatmulArguments, MatmulKernel, MatmulShape, QuantParamsLayout,},},},- metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},+ metal::{+ Metal,+ command_buffer::MetalCommandBufferEncoding,+ context::MetalContext,+ gpu_types::gemm::GemmTiling,+ kernel::matmul::{+ GemmKernel, MatmulMetalError,+ gemm::{self, GemmPlan, GemmProblem},+ gemv::{GemvKernel, GemvSpecialization},+ matmul_dispatch::MatmulDispatch,+ matmul_output_work::MatmulOutputWork,+ qmv::{self, QmvRoute},+ },+ },},data_type::DataType,};@@ -141,16 +144,17 @@ impl MatmulMetalKernel {impl MatmulKernel for MatmulMetalKernel {type Backend = Metal;+ type Error = MatmulMetalError;fn new(context: &MetalContext,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,- ) -> Result<Self, MetalError> {+ ) -> Result<Self, Self::Error> {for data_type in [weights_data_type, input_data_type, output_data_type] {if !matches!(data_type, DataType::BF16 | DataType::F32) {- return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());+ return Err(MatmulMetalError::UnsupportedDataType(data_type));}}@@ -229,25 +233,15 @@ impl MatmulKernel for MatmulMetalKernel {impl BufferRef<Backend = Metal>,>,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<(), MetalError> {+ ) -> Result<(), Self::Error> {let shape = MatmulShape::from_arguments(&arguments);let plan = match self.select_dispatch(&shape, command_buffer.context()) {MatmulDispatch::Gemv(gemv) => {- return self.gemv.encode(arguments, gemv, &self.output_work, command_buffer).map_err(MetalError::from);+ return self.gemv.encode(arguments, gemv, &self.output_work, command_buffer);},MatmulDispatch::Gemm(plan) => plan,};- // TODO: remove after GatherGEMM is supported- if arguments.gather_indices.is_some() {- return Err(MetalError::KernelDispatchFailed(- format!(- "gathered readout requires the GEMV path, but shape (m={}, n={}) routes to GEMM",- arguments.m, arguments.n- )- .into(),- ));- }self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)}}@@ -266,7 +260,7 @@ impl MatmulMetalKernel {>,engine: gemm::GemmEngine,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<(), MetalError> {+ ) -> Result<(), MatmulMetalError> {let shape = MatmulShape::from_arguments(&arguments);let plan = self.gemm.select_plan_for_engine(&shape, engine, command_buffer.context())?;self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rsindex a24948447..1a5803108 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs@@ -1,3 +1,4 @@+mod error;pub mod gemm;mod gemv;mod matmul_dispatch;@@ -5,6 +6,7 @@ mod matmul_metal_kernel;mod matmul_output_work;mod qmv;+pub use error::MatmulMetalError;pub use gemm::GemmKernel;#[cfg(test)]use matmul_dispatch::MatmulDispatch;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rsindex 2b237c776..2042c758a 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs@@ -9,9 +9,9 @@ use super::{},QmvRoute,};-use crate::backends::common::{- gpu_types::gemm::{GemmBPrologueKind, GemmTiling},- kernel::matmul::MatmulShape,+use crate::backends::{+ common::{gpu_types::gemm::GemmBPrologueKind, kernel::matmul::MatmulShape},+ metal::gpu_types::gemm::GemmTiling,};const DOWN: u8 = 1 << 0;diff --git a/crates/uzu-engine/src/backends/metal/kernel/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/mod.rsindex 30c7939fb..856bcf824 100644--- a/crates/uzu-engine/src/backends/metal/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/mod.rs@@ -1,13 +1,13 @@use crate::backends::{- common::{- Kernels,+ common::{Kernels, gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},+ metal::{+ Metal,+ buffer::MetalBufferExt,gpu_types::{- HADAMARD_TRANSFORM_BLOCK_SIZE,gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_k, gemm_tiling_simdgroups_per_row},weaver::{FRONTIER_SELECT_THREADS, TOP_CHILDREN_THREADS},},},- metal::{Metal, buffer::MetalBufferExt},};const METAL_SIMD_SIZE: u32 = 32;diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rsindex fa0b4296f..2057b7b17 100644--- a/crates/uzu-engine/src/backends/metal/mod.rs+++ b/crates/uzu-engine/src/backends/metal/mod.rs@@ -5,6 +5,7 @@ mod command_buffer;mod context;mod decompression;mod error;+mod gpu_types;mod heaps;mod kernel;mod metal_extensions;diff --git a/crates/uzu-engine/src/backends/mod.rs b/crates/uzu-engine/src/backends/mod.rsindex c9e4b8f1d..924d3133e 100644--- a/crates/uzu-engine/src/backends/mod.rs+++ b/crates/uzu-engine/src/backends/mod.rs@@ -1,11 +1,11 @@-pub mod common;+pub(crate) mod common;#[cfg(backend = "cpu")]-pub mod cpu;+pub(crate) mod cpu;#[cfg(backend = "metal")]-pub mod metal;+pub(crate) mod metal;-use crate::backends::common::Backend;+pub use common::Backend;pub trait BackendSelection {type Output;diff --git a/crates/uzu-engine/src/config/activation/mod.rs b/crates/uzu-engine/src/config/activation/mod.rsindex 8ccc592b2..e0b1a92f3 100644--- a/crates/uzu-engine/src/config/activation/mod.rs+++ b/crates/uzu-engine/src/config/activation/mod.rs@@ -1,6 +1,6 @@use uzu_engine_macros::uzu_config_abstract;-use crate::backends::common::gpu_types::ActivationType;+use crate::backends::common::gpu_types::activation_type::ActivationType;pub mod gelu;pub mod identity;diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rsindex 05fa33cad..bef35e2db 100644--- a/crates/uzu-engine/src/encodable_block/classifier.rs+++ b/crates/uzu-engine/src/encodable_block/classifier.rs@@ -13,7 +13,7 @@ use crate::{embedding::{Embedding, EmbeddingError},normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},prediction_head::{PredictionHead, PredictionHeadError},- transformer::{Transformer, TransformerNewError},+ transformer::{Transformer, TransformerEncodeError, TransformerNewError},},parameters::ParameterTree,};@@ -28,6 +28,8 @@ pub enum ClassifierError<B: Backend> {Normalization(#[from] NormalizationNewError<B>),#[error("Transformer error: {0}")]Transformer(#[from] TransformerNewError<B>),+ #[error("Transformer encoding error: {0}")]+ TransformerEncode(#[from] TransformerEncodeError<B>),#[error("Transformer error: {0}")]PredictionHead(#[from] PredictionHeadError<B>),#[error("Unsupported configuration: {0}")]@@ -144,8 +146,7 @@ impl<B: Backend> Classifier<B> {None,None,command_buffer,- )- .map_err(ClassifierError::Backend)?+ )?.output.unwrap();@@ -154,7 +155,7 @@ impl<B: Backend> Classifier<B> {.map_err(ClassifierError::Backend)?;self.pooling.encode(&hidden, &mut pooled, batch_dim, self.hidden_dim, 1, command_buffer);- let logits = self.prediction_head.encode(pooled, 1, command_buffer).map_err(ClassifierError::Backend)?;+ let logits = self.prediction_head.encode(pooled, 1, command_buffer)?;Ok(logits)}diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rsindex e3a3242aa..44b462310 100644--- a/crates/uzu-engine/src/encodable_block/decoder.rs+++ b/crates/uzu-engine/src/encodable_block/decoder.rs@@ -12,7 +12,7 @@ use crate::{embedding::{Embedding, EmbeddingError},normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},per_layer_embedding::{PerLayerEmbedding, PerLayerEmbeddingError},- transformer::{Transformer, TransformerNewError, TransformerState},+ transformer::{Transformer, TransformerEncodeError, TransformerNewError, TransformerState},},parameters::ParameterTree,};@@ -29,6 +29,8 @@ pub enum DecoderError<B: Backend> {PerLayerEmbedding(#[from] PerLayerEmbeddingError<B>),#[error("Transformer error: {0}")]Transformer(#[from] TransformerNewError<B>),+ #[error("Transformer encoding error: {0}")]+ TransformerEncode(#[from] TransformerEncodeError<B>),}pub struct Decoder<B: Backend> {@@ -155,27 +157,20 @@ impl<B: Backend> Decoder<B> {};let per_layer_inputs = if let Some(per_layer_embedding) = &self.per_layer_embedding {- Some(- per_layer_embedding- .encode(token_ids, &embedded, batch_dim.size(), command_buffer)- .map_err(DecoderError::Backend)?,- )+ Some(per_layer_embedding.encode(token_ids, &embedded, batch_dim.size(), command_buffer)?)} else {None};- let transformer_output = self- .transformer- .encode(- embedded,- per_layer_inputs.as_ref(),- batch_dim,- output_range,- hidden_feature_layer_indices,- Some(state),- command_buffer,- )- .map_err(DecoderError::Backend)?;+ let transformer_output = self.transformer.encode(+ embedded,+ per_layer_inputs.as_ref(),+ batch_dim,+ output_range,+ hidden_feature_layer_indices,+ Some(state),+ command_buffer,+ )?;let logits = if let Some(output_range) = output_range {let output = transformer_output.output.as_ref().expect("decoder output range requires transformer output");diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rsindex e2a749989..87743fc8a 100644--- a/crates/uzu-engine/src/encodable_block/dflash.rs+++ b/crates/uzu-engine/src/encodable_block/dflash.rs@@ -13,7 +13,7 @@ use crate::{encodable_block::{batch_topology::BatchTopology,embedding::{Embedding, EmbeddingError},- linear::{Linear, LinearBlockError},+ linear::{Linear, LinearBlockError, LinearEncodeError},mixer::{MixerState,attention::{@@ -83,6 +83,10 @@ pub enum DFlashNewError<B: Backend> {pub enum DFlashEncodeError<B: Backend> {#[error("Backend error: {0}")]Backend(#[source] B::Error),+ #[error("Linear error: {0}")]+ Linear(#[from] LinearEncodeError<B>),+ #[error("Transformer layer error: {0}")]+ TransformerLayer(#[from] TransformerLayerError<B>),#[error("Embedding error: {0}")]Embedding(#[from] EmbeddingError<B>),}@@ -202,7 +206,7 @@ impl<B: Backend> DFlash<B> {accepted_indices: &[u32],state: &mut DFlashState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error> {+ ) -> Result<(), DFlashEncodeError<B>> {if accepted_indices.is_empty() {return Ok(());}@@ -217,8 +221,9 @@ impl<B: Backend> DFlash<B> {assert!(context_length <= state.context_capacity, "DFlash state capacity exceeded");assert!(context_length <= self.max_context_length, "DFlash context exceeds configured RoPE capacity");- let mut packed_target_features =- command_buffer.allocate_scratch_for_shape(&[num_tokens, self.target_feature_input_dim], self.data_type)?;+ let mut packed_target_features = command_buffer+ .allocate_scratch_for_shape(&[num_tokens, self.target_feature_input_dim], self.data_type)+ .map_err(DFlashEncodeError::Backend)?;for (layer_index, features) in target_features.enumerate() {assert!(features.size() % layer_feature_bytes == 0);for (token_index, &accepted_index) in accepted_indices.iter().enumerate() {@@ -234,15 +239,13 @@ impl<B: Backend> DFlash<B> {}let projected_features =self.target_feature_projection.encode(packed_target_features, num_tokens, command_buffer)?;- let normalized_features = self.projected_feature_norm.encode(- &projected_features,- 0,- num_tokens,- None::<&mut B::ScratchBuffer>,- command_buffer,- )?;+ let normalized_features = self+ .projected_feature_norm+ .encode(&projected_features, 0, num_tokens, None::<&mut B::ScratchBuffer>, command_buffer)+ .map_err(DFlashEncodeError::Backend)?;let token_positions = (state.context_length..state.context_length + num_tokens).collect::<Box<[_]>>();- let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, command_buffer)?;+ let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, command_buffer)+ .map_err(DFlashEncodeError::Backend)?;let projected_kv = self.state_kv_projection.encode(normalized_features, num_tokens, command_buffer)?;let layer_kv_bytes = size_for_shape(&[self.layer_kv_dim], self.data_type);@@ -250,7 +253,8 @@ impl<B: Backend> DFlash<B> {|chunk_index: usize| Range::from(chunk_index * layer_kv_bytes..(chunk_index + 1) * layer_kv_bytes);let mut layer_key_values = (0..self.layers.len()).map(|_| command_buffer.allocate_scratch(num_tokens as usize * layer_kv_bytes))- .collect::<Result<Box<[_]>, _>>()?;+ .collect::<Result<Box<[_]>, _>>()+ .map_err(DFlashEncodeError::Backend)?;for (layer_index, key_value) in layer_key_values.iter_mut().enumerate() {for token_index in 0..num_tokens as usize {command_buffer.encode_copy(@@ -262,14 +266,16 @@ impl<B: Backend> DFlash<B> {for ((layer, mixer_state), mut key_value) inself.layers.iter().zip(state.layer_states.iter_mut()).zip(layer_key_values){- mixer_state.prepare(state.context_length, num_tokens)?;+ mixer_state.prepare(state.context_length, num_tokens).map_err(DFlashEncodeError::Backend)?;let attention = (layer.mixer.as_ref() as &dyn Any).downcast_ref::<Attention<B>>().expect("DFlash draft layers must use attention mixers");let attention_state = (mixer_state.as_mut() as &mut dyn Any).downcast_mut::<AttentionState<B>>().expect("DFlash draft layer states must be attention states");- attention.append_projected_kv(&mut key_value, &rope, num_tokens, attention_state, command_buffer)?;+ attention+ .append_projected_kv(&mut key_value, &rope, num_tokens, attention_state, command_buffer)+ .map_err(DFlashEncodeError::Backend)?;}state.context_length += num_tokens;@@ -315,17 +321,15 @@ impl<B: Backend> DFlash<B> {let mut residual = command_buffer.allocate_scratch(hidden.size()).map_err(DFlashEncodeError::Backend)?;for (layer, mixer_state) in self.layers.iter().zip(state.layer_states.iter_mut()) {mixer_state.prepare(state.context_length, batch_size).map_err(DFlashEncodeError::Backend)?;- hidden = layer- .encode(- hidden,- &mut residual,- None::<&B::ScratchBuffer>,- Some(&rope),- &batch_topology,- Some(MaybeMut::Mut(mixer_state.as_mut())),- command_buffer,- )- .map_err(DFlashEncodeError::Backend)?;+ hidden = layer.encode(+ hidden,+ &mut residual,+ None::<&B::ScratchBuffer>,+ Some(&rope),+ &batch_topology,+ Some(MaybeMut::Mut(mixer_state.as_mut())),+ command_buffer,+ )?;}let draft_hidden = self.output_normdiff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rsindex 88b293a90..5eb3facee 100644--- a/crates/uzu-engine/src/encodable_block/embedding.rs+++ b/crates/uzu-engine/src/encodable_block/embedding.rs@@ -14,7 +14,7 @@ use crate::{data_type::DataType,encodable_block::{embedding_table::{EmbeddingTable, EmbeddingTableError, read_output_signs},- linear::{Gather, LinearMatmulError, UntiedReadout},+ linear::{Gather, LinearEncodeError, LinearMatmulError, UntiedReadout},},parameters::{ParameterLoaderError, ParameterTree},};@@ -23,6 +23,8 @@ use crate::{pub enum EmbeddingError<B: Backend> {#[error("Backend error: {0}")]BackendError(#[source] B::Error),+ #[error("Matmul error: {0}")]+ Matmul(#[source] <<B::Kernels as Kernels>::MatmulKernel as MatmulKernel>::Error),#[error("Parameter loading error: {0}")]ParameterError(#[from] ParameterLoaderError<B>),#[error("Unsupported configuration: {0}")]@@ -31,6 +33,8 @@ pub enum EmbeddingError<B: Backend> {EmbeddingTable(#[from] EmbeddingTableError<B>),#[error(transparent)]LinearMatmul(#[from] LinearMatmulError<B>),+ #[error("Linear encoding error: {0}")]+ LinearEncode(#[from] LinearEncodeError<B>),}enum EmbeddingTying<B: Backend> {@@ -197,7 +201,7 @@ impl<B: Backend> Embedding<B> {indices,output_dim,});- output.encode(input_buffer, batch_dim, gather, command_buffer).map_err(EmbeddingError::BackendError)?+ output.encode(input_buffer, batch_dim, gather, command_buffer)?},EmbeddingTying::Tied {table,@@ -220,7 +224,7 @@ impl<B: Backend> Embedding<B> {n: output_dim,k: self.model_dim,};- readout.lock().encode(arguments, command_buffer).map_err(EmbeddingError::BackendError)?;+ readout.lock().encode(arguments, command_buffer).map_err(EmbeddingError::Matmul)?;output},};@@ -244,6 +248,6 @@ fn readout_kernel<B: Backend>(data_type: DataType,) -> Result<Mutex<<B::Kernels as Kernels>::MatmulKernel>, EmbeddingError<B>> {let kernel = <B::Kernels as Kernels>::MatmulKernel::new(context, data_type, data_type, data_type)- .map_err(EmbeddingError::BackendError)?;+ .map_err(EmbeddingError::Matmul)?;Ok(Mutex::new(kernel))}diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rsindex 5d9f67598..ab6916574 100644--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs@@ -4,7 +4,7 @@ use thiserror::Error;use crate::{backends::common::{Backend, BufferMut, BufferRef, CommandBuffer, Kernels,- gpu_types::{EmbeddingTableKind, HADAMARD_TRANSFORM_BLOCK_SIZE, d4s4},+ gpu_types::{d4s4, embedding::EmbeddingTableKind, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},kernel::InputEmbeddingLookupKernel,},config::weight_matrix::{diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rsindex 4f619f2df..6e4853a6d 100644--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs@@ -16,7 +16,7 @@ use crate::{config::weight_matrix::{AnyWeightMatrixSpec, Layout},data_type::DataType,encodable_block::{- linear::{Gather, Linear, LinearInput},+ linear::{Gather, Linear, LinearEncodeError, LinearInput},weight_matrix::{WeightMatrix, WeightMatrixError},},parameters::{ParameterLoaderError, ParameterTree},@@ -26,6 +26,8 @@ use crate::{pub enum LinearMatmulError<B: Backend> {#[error("Backend error: {0}")]BackendError(#[source] B::Error),+ #[error("Matmul error: {0}")]+ Matmul(#[source] <<B::Kernels as Kernels>::MatmulKernel as MatmulKernel>::Error),#[error("Parameter loading error: {0}")]ParameterError(#[from] ParameterLoaderError<B>),#[error("Weight matrix error: {0}")]@@ -95,7 +97,7 @@ impl<B: Backend> LinearMatmul<B> {let kernel =<B::Kernels as Kernels>::MatmulKernel::new(context, weights_data_type, input_data_type, output_data_type)- .map_err(LinearMatmulError::BackendError)?;+ .map_err(LinearMatmulError::Matmul)?;Ok(Self {kernel: Mutex::new(kernel),@@ -124,24 +126,29 @@ impl<B: Backend> LinearMatmul<B> {batch_dim: u32,gather: Option<Gather<impl BufferRef<Backend = B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {let (output_dim, gather_indices) =gather.map_or((self.output_dim, None), |gather| (gather.output_dim, Some(gather.indices)));- let mut output = command_buffer.allocate_scratch_for_shape(&[batch_dim, output_dim], self.output_data_type)?;- self.kernel.lock().encode(- MatmulArguments {- a,- b: self.matmul_b(),- b_leading_dimension: None,- b_transpose: true,- output: MatmulOutput::new(&mut output, self.d_ops()),- gather_indices,- m: batch_dim,- n: output_dim,- k: self.input_dim,- },- command_buffer,- )?;+ let mut output = command_buffer+ .allocate_scratch_for_shape(&[batch_dim, output_dim], self.output_data_type)+ .map_err(LinearEncodeError::Backend)?;+ self.kernel+ .lock()+ .encode(+ MatmulArguments {+ a,+ b: self.matmul_b(),+ b_leading_dimension: None,+ b_transpose: true,+ output: MatmulOutput::new(&mut output, self.d_ops()),+ gather_indices,+ m: batch_dim,+ n: output_dim,+ k: self.input_dim,+ },+ command_buffer,+ )+ .map_err(LinearEncodeError::Matmul)?;Ok(output)}@@ -189,7 +196,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {input: LinearInput<B>,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {self.encode_with_a(input.as_matmul_a(), batch_dim, None::<Gather<&B::ScratchBuffer>>, command_buffer)}diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rsindex 7d42396b9..c151f8d5f 100644--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs@@ -12,11 +12,11 @@ pub use untied_readout::UntiedReadout;use crate::{backends::common::{- Backend, BufferRef, CommandBuffer,- gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,+ Backend, BufferRef, CommandBuffer, Kernels,+ gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,kernel::{ActivationQuantization,- matmul::{ActivationFormat, Int8CodeLayout, MatmulA},+ matmul::{ActivationFormat, Int8CodeLayout, MatmulA, MatmulKernel},},},config::weight_matrix::{@@ -27,13 +27,21 @@ use crate::{parameters::{ParameterLoaderError, ParameterTree},};+#[derive(Debug, Error)]+pub enum LinearEncodeError<B: Backend> {+ #[error("Backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("Matmul error: {0}")]+ Matmul(#[source] <<B::Kernels as Kernels>::MatmulKernel as MatmulKernel>::Error),+}+pub trait Linear<B: Backend>: Send + Sync {fn encode(&self,input: B::ScratchBuffer,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {self.encode_input(LinearInput::FullPrecision(input), batch_dim, command_buffer)}@@ -42,7 +50,7 @@ pub trait Linear<B: Backend>: Send + Sync {input: LinearInput<B>,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error>;+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>>;fn select_activation_format(&self,diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rsindex a1e0d5a55..c97472e4c 100644--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs@@ -2,14 +2,17 @@ use derive_more::Debug;use thiserror::Error;use crate::{- backends::common::{Backend, CommandBuffer, CommandBufferEncoding, gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE},+ backends::common::{+ Backend, CommandBuffer, CommandBufferEncoding, gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,+ },config::weight_matrix::{AnyWeightMatrixSpec,hybrid_spec::{HybridSpec, IncoherenceProcessingMode},},data_type::DataType,encodable_block::linear::{- Linear, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,+ Gather, Linear, LinearEncodeError, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError,+ input_rht::InputRht,},parameters::{ParameterLoaderError, ParameterTree},};@@ -178,7 +181,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {input: LinearInput<B>,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {let input = match input {LinearInput::FullPrecision(input) => input,input => return self.inner_linear.encode_input(input, batch_dim, command_buffer),@@ -186,11 +189,14 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {let command_buffer = &mut *command_buffer.debug_group("linear (rht)");let format = self.inner_linear.select_activation_format(batch_dim, command_buffer.context());- let input = self.input_rht.prepare_in_place(input, batch_dim, format, command_buffer)?;+ let input = self+ .input_rht+ .prepare_in_place(input, batch_dim, format, command_buffer)+ .map_err(LinearEncodeError::Backend)?;let output = self.inner_linear.encode_with_a(input.as_matmul_a(),batch_dim,- None::<super::Gather<&B::ScratchBuffer>>,+ None::<Gather<&B::ScratchBuffer>>,command_buffer,)?;diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rsindex 12d6e565d..169a7a95a 100644--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs@@ -1,7 +1,7 @@use crate::{backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding,- gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,+ gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,kernel::matmul::{ActivationFormat, MatmulA},},config::weight_matrix::{@@ -10,7 +10,7 @@ use crate::{},data_type::DataType,encodable_block::linear::{- Gather, Linear, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,+ Gather, Linear, LinearEncodeError, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,},parameters::ParameterTree,};@@ -80,7 +80,7 @@ impl<B: Backend> UntiedReadout<B> {batch_dim: u32,gather: Option<Gather<impl BufferRef<Backend = B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {let Some(input_rht) = &self.input_rht else {let a = MatmulA::FullPrecision {values: input,@@ -94,7 +94,7 @@ impl<B: Backend> UntiedReadout<B> {} else {self.linear.select_activation_format(batch_dim, command_buffer.context())};- let input = input_rht.prepare(input, batch_dim, format, command_buffer)?;+ let input = input_rht.prepare(input, batch_dim, format, command_buffer).map_err(LinearEncodeError::Backend)?;self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rsindex 44f2706bb..2acb217df 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs@@ -10,9 +10,9 @@ use crate::{data_type::DataType,encodable_block::{batch_topology::BatchTopology,- linear::{Linear, LinearBlockError},+ linear::{Linear, LinearBlockError, LinearEncodeError},mixer::{- Mixer, MixerState,+ Mixer, MixerEncodeError, MixerState,attention::{mode::LinearProjection,qkv_norm::{QKVNorm, QKVNormError},@@ -50,6 +50,8 @@ pub struct Attention<B: Backend> {#[derive(Debug, Error)]pub enum AttentionNewError<B: Backend> {+ #[error("Attention kernel error: {0}")]+ Kernel(#[source] <<B::Kernels as Kernels>::AttentionKernel as AttentionKernel>::Error),#[error("Backend error: {0}")]Backend(#[source] B::Error),#[error("Parameter loader error: {0}")]@@ -60,6 +62,16 @@ pub enum AttentionNewError<B: Backend> {QKVNorm(#[from] QKVNormError<B>),}+#[derive(Debug, Error)]+pub enum AttentionEncodeError<B: Backend> {+ #[error("Backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("Linear error: {0}")]+ Linear(#[from] LinearEncodeError<B>),+ #[error("Attention kernel error: {0}")]+ Kernel(#[source] <<B::Kernels as Kernels>::AttentionKernel as AttentionKernel>::Error),+}+impl<B: Backend> Attention<B> {pub fn new(hidden_dim: u32,@@ -152,7 +164,7 @@ impl<B: Backend> Attention<B> {data_type,},)- .map_err(AttentionNewError::Backend)?;+ .map_err(AttentionNewError::Kernel)?;let out_projection = <dyn Linear<B>>::new(q_dim,@@ -211,7 +223,7 @@ impl<B: Backend> Mixer<B> for Attention<B> {batch_dim: &BatchTopology,state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {let command_buffer = &mut *command_buffer.debug_group("attention");assert_eq!(precalculated_rope.is_some(), self.max_rope_length.is_some(), "precalculated rope mismatch");diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rsindex d937a5e3c..881b08334 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs@@ -8,7 +8,10 @@ use crate::{linear::Linear,mixer::{MixerState,- attention::{Attention, KVCacheView, qkv_norm::QKVNorm, rope::PrecalculatedRoPE, state::AttentionState},+ attention::{+ Attention, AttentionEncodeError, KVCacheView, qkv_norm::QKVNorm, rope::PrecalculatedRoPE,+ state::AttentionState,+ },},},utils::maybe_mut::MaybeMut,@@ -25,7 +28,7 @@ impl<B: Backend> LinearProjection<B> {hidden: B::ScratchBuffer,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, AttentionEncodeError<B>> {let mut projected = self.lin.encode(hidden, batch_dim, command_buffer)?;self.norm.encode(&mut projected, batch_dim, command_buffer);Ok(projected)@@ -40,7 +43,7 @@ impl<B: Backend> Attention<B> {batch_dim: &BatchTopology,state: Option<MaybeMut<AttentionState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, AttentionEncodeError<B>> {let qkvg = self.projection.project(hidden, batch_dim.size(), command_buffer)?;let gate = self.has_gate.then(|| {qkvg.subrange(@@ -51,21 +54,25 @@ impl<B: Backend> Attention<B> {let attention_output = match state {Some(MaybeMut::Mut(state)) => {let prefix_len = state.view().prefix_len();- let queries = self.prepare_kv_and_queries(- &qkvg,- state.keys.as_mut(),- state.values.as_mut(),- prefix_len,- self.num_q_heads,- precalculated_rope,- batch_dim.size(),- command_buffer,- )?;+ let queries = self+ .prepare_kv_and_queries(+ &qkvg,+ state.keys.as_mut(),+ state.values.as_mut(),+ prefix_len,+ self.num_q_heads,+ precalculated_rope,+ batch_dim.size(),+ command_buffer,+ )+ .map_err(AttentionEncodeError::Backend)?;self.run_core(&queries, gate, batch_dim, state, command_buffer)?},Some(MaybeMut::Const(state)) => {// KV sharing: QKVG contains queries and an optional gate only.- let queries = self.prepare_queries(&qkvg, precalculated_rope, batch_dim.size(), command_buffer)?;+ let queries = self+ .prepare_queries(&qkvg, precalculated_rope, batch_dim.size(), command_buffer)+ .map_err(AttentionEncodeError::Backend)?;self.run_core(&queries, gate, batch_dim, state, command_buffer)?},None => {@@ -75,40 +82,48 @@ impl<B: Backend> Attention<B> {assert!(batch_dim.is_flat(), "stateless attention doesn't support trie");let mut keys = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)?;+ .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)+ .map_err(AttentionEncodeError::Backend)?;let mut values = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)?;+ .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)+ .map_err(AttentionEncodeError::Backend)?;- let queries = self.prepare_kv_and_queries(- &qkvg,- &mut keys,- &mut values,- 0,- self.num_q_heads,- precalculated_rope,- batch_dim.size(),- command_buffer,- )?;+ let queries = self+ .prepare_kv_and_queries(+ &qkvg,+ &mut keys,+ &mut values,+ 0,+ self.num_q_heads,+ precalculated_rope,+ batch_dim.size(),+ command_buffer,+ )+ .map_err(AttentionEncodeError::Backend)?;let cache = self.ring_capacity.map_or_else(|| KVCacheView::full(0), |_| KVCacheView::ring(0, 0));- self.kernel.encode(- AttentionArguments {- queries: &queries,- keys: &keys,- values: &values,- suffix_length: batch_dim.size(),- trie: None::<&B::ConstantBuffer>,- sinks: self.sinks.as_ref(),- cache,- gate,- },- command_buffer,- )?+ self.kernel+ .encode(+ AttentionArguments {+ queries: &queries,+ keys: &keys,+ values: &values,+ suffix_length: batch_dim.size(),+ trie: None::<&B::ConstantBuffer>,+ sinks: self.sinks.as_ref(),+ cache,+ gate,+ },+ command_buffer,+ )+ .map_err(AttentionEncodeError::Kernel)?},};- self.out_projection.encode(attention_output, batch_dim.size(), command_buffer)+ self.out_projection+ .encode(attention_output, batch_dim.size(), command_buffer)+ .map_err(AttentionEncodeError::Linear)}pub fn append_projected_kv(@@ -142,26 +157,32 @@ impl<B: Backend> Attention<B> {batch_dim: &BatchTopology,state: &AttentionState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, AttentionEncodeError<B>> {let trie = if batch_dim.is_flat() {None} else {- Some(command_buffer.allocate_constant_from_slice(batch_dim.nodes())?)+ Some(+ command_buffer+ .allocate_constant_from_slice(batch_dim.nodes())+ .map_err(AttentionEncodeError::Backend)?,+ )};- self.kernel.encode(- AttentionArguments {- queries,- keys: state.keys.as_ref(),- values: state.values.as_ref(),- suffix_length: batch_dim.size(),- trie: trie.as_ref(),- sinks: self.sinks.as_ref(),- cache: state.view(),- gate,- },- command_buffer,- )+ self.kernel+ .encode(+ AttentionArguments {+ queries,+ keys: state.keys.as_ref(),+ values: state.values.as_ref(),+ suffix_length: batch_dim.size(),+ trie: trie.as_ref(),+ sinks: self.sinks.as_ref(),+ cache: state.view(),+ gate,+ },+ command_buffer,+ )+ .map_err(AttentionEncodeError::Kernel)}fn prepare_kv_and_queries(diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rsindex 967fc08cd..e43044655 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs@@ -4,7 +4,7 @@ use crate::{array::size_for_shape,backends::common::{Backend, Buffer, CommandBuffer, Context, DeviceCapabilities, Kernels, SparseBuffer,- gpu_types::{Copy, ring::RingParams},+ gpu_types::{kv_cache_update::Copy, ring::RingParams},kernel::KVCacheUpdateKernel,},data_type::DataType,diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rsindex 46f4c81ec..d7b9ec6ed 100644--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs@@ -20,7 +20,7 @@ use crate::{batch_topology::BatchTopology,linear::{Linear, LinearBlockError},mixer::{- Mixer, MixerState,+ Mixer, MixerEncodeError, MixerState,attention::rope::PrecalculatedRoPE,delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},},@@ -333,22 +333,35 @@ impl<B: Backend> DeltaNet<B> {batch_dim: &BatchTopology,state: &mut DeltaNetState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {let tree_verify = self.tree_verify.as_ref().expect("DeltaNet tree verification is unsupported");let tree_size = batch_dim.size();- let parents = command_buffer.allocate_constant_from_slice(batch_dim.parents())?;- let trie = command_buffer.allocate_constant_from_slice(batch_dim.nodes())?;+ let parents =+ command_buffer.allocate_constant_from_slice(batch_dim.parents()).map_err(MixerEncodeError::Backend)?;+ let trie = command_buffer.allocate_constant_from_slice(batch_dim.nodes()).map_err(MixerEncodeError::Backend)?;let mut conv_states = command_buffer- .allocate_scratch_for_shape(&[tree_size, self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE)?;- let mut k = command_buffer.allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)?;- let mut v = command_buffer.allocate_scratch_for_shape(&[tree_size, self.value_dim], self.outer_data_type)?;- let mut beta = command_buffer.allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)?;- let mut log_decay = command_buffer.allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)?;-- let mut tree_projected =- command_buffer.allocate_scratch_for_shape(&[tree_size, self.total_proj_dim], self.outer_data_type)?;- let mut q = command_buffer.allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)?;+ .allocate_scratch_for_shape(&[tree_size, self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;+ let mut k = command_buffer+ .allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)+ .map_err(MixerEncodeError::Backend)?;+ let mut v = command_buffer+ .allocate_scratch_for_shape(&[tree_size, self.value_dim], self.outer_data_type)+ .map_err(MixerEncodeError::Backend)?;+ let mut beta = command_buffer+ .allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;+ let mut log_decay = command_buffer+ .allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;++ let mut tree_projected = command_buffer+ .allocate_scratch_for_shape(&[tree_size, self.total_proj_dim], self.outer_data_type)+ .map_err(MixerEncodeError::Backend)?;+ let mut q = command_buffer+ .allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)+ .map_err(MixerEncodeError::Backend)?;self.conv_tree_scan.encode(in_projected,@@ -381,19 +394,21 @@ impl<B: Backend> DeltaNet<B> {command_buffer,);- let mut delta_output = tree_verify.encode(- TreeVerifyEncodeArguments {- q: &q,- k: &k,- v: &v,- trie: &trie,- log_decay: &log_decay,- beta: &beta,- h0: &state.ssm_state,- tree_size,- },- command_buffer,- )?;+ let mut delta_output = tree_verify+ .encode(+ TreeVerifyEncodeArguments {+ q: &q,+ k: &k,+ v: &v,+ trie: &trie,+ log_decay: &log_decay,+ beta: &beta,+ h0: &state.ssm_state,+ tree_size,+ },+ command_buffer,+ )+ .map_err(MixerEncodeError::Backend)?;self.delta_net_norm_gate.encode(&mut delta_output,&tree_projected,@@ -469,7 +484,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {batch_dim: &BatchTopology,state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {let command_buffer = &mut *command_buffer.debug_group("delta net");assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");@@ -490,8 +505,9 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {return Ok(output);}- let mut delta_output =- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type)?;+ let mut delta_output = command_buffer+ .allocate_scratch_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type)+ .map_err(MixerEncodeError::Backend)?;if batch_dim.size() == 1 {self.conv_update.encode(&self.conv_weight,@@ -520,10 +536,12 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {command_buffer,);} else {- let mut padded = command_buffer.allocate_scratch_for_shape(- &[batch_dim.size() + self.kernel_size - 1, self.total_proj_dim],- INNER_DATA_TYPE,- )?;+ let mut padded = command_buffer+ .allocate_scratch_for_shape(+ &[batch_dim.size() + self.kernel_size - 1, self.total_proj_dim],+ INNER_DATA_TYPE,+ )+ .map_err(MixerEncodeError::Backend)?;self.conv_pack.encode(&state.conv_state,&in_projected,@@ -549,31 +567,37 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {command_buffer,);if let Some(chunked) = self.chunked.as_ref().filter(|chunked| chunked.should_use(batch_dim.size())) {- chunked.encode(- DeltaNetChunkedPrefillArgs {- in_projected: &in_projected,- a_log: &self.a_log,- dt_bias: &self.dt_bias,- ssm_state: &mut state.ssm_state,- delta_output: &mut delta_output,- num_heads: self.num_heads,- num_groups: self.num_groups,- value_head_dim: self.value_head_dim,- key_dim: self.key_dim,- value_dim: self.value_dim,- suffix_len: batch_dim.size(),- },- command_buffer,- )?;+ chunked+ .encode(+ DeltaNetChunkedPrefillArgs {+ in_projected: &in_projected,+ a_log: &self.a_log,+ dt_bias: &self.dt_bias,+ ssm_state: &mut state.ssm_state,+ delta_output: &mut delta_output,+ num_heads: self.num_heads,+ num_groups: self.num_groups,+ value_head_dim: self.value_head_dim,+ key_dim: self.key_dim,+ value_dim: self.value_dim,+ suffix_len: batch_dim.size(),+ },+ command_buffer,+ )+ .map_err(MixerEncodeError::Backend)?;} else {- let mut prep_q_norm =- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)?;- let mut prep_k_norm =- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)?;- let mut prep_beta =- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;- let mut prep_decay =- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;+ let mut prep_q_norm = command_buffer+ .allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;+ let mut prep_k_norm = command_buffer+ .allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;+ let mut prep_beta = command_buffer+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;+ let mut prep_decay = command_buffer+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;self.delta_net_prefill_prep.encode(&in_projected,&self.a_log,diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rsindex 038109caa..6bf011646 100644--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs@@ -5,7 +5,7 @@ use crate::{array::size_for_shape,backends::common::{Backend, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::ActivationType,+ gpu_types::activation_type::ActivationType,kernel::{Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDPrefill64Kernel, SSDPrefillKernel,SSDUpdateKernel, SplitInProjKernel,@@ -16,7 +16,7 @@ use crate::{encodable_block::{batch_topology::BatchTopology,linear::{Linear, LinearBlockError},- mixer::{Mixer, MixerState, attention::rope::PrecalculatedRoPE},+ mixer::{Mixer, MixerEncodeError, MixerState, attention::rope::PrecalculatedRoPE},},parameters::{ParameterLoaderError, ParameterTree},utils::maybe_mut::MaybeMut,@@ -242,7 +242,7 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {batch_dim: &BatchTopology,state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {let command_buffer = &mut *command_buffer.debug_group("mamba2");assert!(precalculated_rope.is_none(), "unexpected rope for mamba2 mixer");@@ -261,12 +261,15 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {let in_projected = self.in_projection.encode(hidden, batch_dim.size(), command_buffer)?;- let mut conv_inputs =- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.conv_dim], INNER_DATA_TYPE)?;+ let mut conv_inputs = command_buffer+ .allocate_scratch_for_shape(&[batch_dim.size(), self.conv_dim], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;let mut gate = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)?;- let mut time_step =- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;+ let mut time_step = command_buffer+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;self.split_inproj.encode(&in_projected,&mut conv_inputs,@@ -282,11 +285,14 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {);let mut conv_x = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)?;+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;let mut state_b = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)?;+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;let mut state_c = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)?;+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;let state_stride = self.kernel_size - 1;if batch_dim.size() == 1 {self.conv_decode.encode(@@ -310,7 +316,8 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {);} else {let mut padded = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size() + state_stride, self.conv_dim], INNER_DATA_TYPE)?;+ .allocate_scratch_for_shape(&[batch_dim.size() + state_stride, self.conv_dim], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;self.conv_pack.encode(&state.conv_state,&conv_inputs,@@ -341,8 +348,9 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {);}- let mut ssd_output =- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.inner_dim], INNER_DATA_TYPE)?;+ let mut ssd_output = command_buffer+ .allocate_scratch_for_shape(&[batch_dim.size(), self.inner_dim], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;let x_strides = [self.num_heads * self.head_dim, self.head_dim, 1];let dt_strides = [self.num_heads, 1];let cb_strides = [self.num_groups * self.state_dim, self.state_dim, 1];diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rsindex d4250ce14..772c4b39e 100644--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs@@ -9,8 +9,9 @@ use crate::{data_type::DataType,encodable_block::{batch_topology::BatchTopology,+ linear::LinearEncodeError,mixer::{- attention::{Attention, AttentionNewError, rope::PrecalculatedRoPE},+ attention::{Attention, AttentionEncodeError, AttentionNewError, rope::PrecalculatedRoPE},delta_net::{DeltaNet, DeltaNetNewError},mamba2::{Mamba2, Mamba2NewError},short_conv::{ShortConv, ShortConvNewError},@@ -48,6 +49,16 @@ impl<'a, B: Backend> MaybeMut<'a, dyn MixerState<B>> {}}+#[derive(Debug, Error)]+pub enum MixerEncodeError<B: Backend> {+ #[error("Backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("Linear error: {0}")]+ Linear(#[from] LinearEncodeError<B>),+ #[error("Attention error: {0}")]+ Attention(#[from] AttentionEncodeError<B>),+}+pub trait Mixer<B: Backend>: Any + Send + Sync {fn speculation_supported(&self) -> bool;@@ -66,7 +77,7 @@ pub trait Mixer<B: Backend>: Any + Send + Sync {batch_dim: &BatchTopology,state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error>;+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>>;}#[derive(Debug, Error)]diff --git a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rsindex 7bc2dd839..817d74e70 100644--- a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs@@ -13,7 +13,7 @@ use crate::{encodable_block::{batch_topology::BatchTopology,linear::{Linear, LinearBlockError},- mixer::{Mixer, MixerState, attention::rope::PrecalculatedRoPE},+ mixer::{Mixer, MixerEncodeError, MixerState, attention::rope::PrecalculatedRoPE},},parameters::{ParameterLoaderError, ParameterTree},utils::maybe_mut::MaybeMut,@@ -307,7 +307,7 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {batch_dim: &BatchTopology,state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {let command_buffer = &mut *command_buffer.debug_group("short conv");assert!(precalculated_rope.is_none(), "unexpected rope for short conv mixer");@@ -324,18 +324,21 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {let conv_output = if batch_dim.full_accept() {let conv_output = if batch_dim.size() == 1 {- self.encode_decode_conv(&in_projected, state, command_buffer)?+ self.encode_decode_conv(&in_projected, state, command_buffer).map_err(MixerEncodeError::Backend)?} else {- self.encode_prefill_conv(&in_projected, batch_dim.size(), state, command_buffer)?+ self.encode_prefill_conv(&in_projected, batch_dim.size(), state, command_buffer)+ .map_err(MixerEncodeError::Backend)?};state.suffix_state = Some(ShortConvStateSuffixStatus::Flat {suffix_length: batch_dim.size(),});conv_output} else {- let token_parents = command_buffer.allocate_constant_from_slice(batch_dim.parents())?;- let (conv_output, conv_states) =- self.encode_trie_conv(&in_projected, &token_parents, batch_dim.size(), state, command_buffer)?;+ let token_parents =+ command_buffer.allocate_constant_from_slice(batch_dim.parents()).map_err(MixerEncodeError::Backend)?;+ let (conv_output, conv_states) = self+ .encode_trie_conv(&in_projected, &token_parents, batch_dim.size(), state, command_buffer)+ .map_err(MixerEncodeError::Backend)?;state.suffix_state = Some(ShortConvStateSuffixStatus::Trie {conv_states,});diff --git a/crates/uzu-engine/src/encodable_block/mlp/dense.rs b/crates/uzu-engine/src/encodable_block/mlp/dense.rsindex 31ecba34d..2649f4a3a 100644--- a/crates/uzu-engine/src/encodable_block/mlp/dense.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/dense.rs@@ -9,7 +9,7 @@ use crate::{data_type::DataType,encodable_block::{linear::{Linear, LinearBlockError},- mlp::{Mlp, gate_act_mul::MlpGateActMulEncodable},+ mlp::{Mlp, MlpEncodeError, gate_act_mul::MlpGateActMulEncodable},},parameters::ParameterTree,};@@ -83,12 +83,15 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {input: B::ScratchBuffer,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, MlpEncodeError<B>> {let command_buffer = &mut *command_buffer.debug_group("mlp (dense)");let fused_up = self.up.encode(input, batch_dim, command_buffer)?;let act_format = self.down.select_activation_format(batch_dim, command_buffer.context());- let down_input = self.gate.encode_for_linear(&fused_up, batch_dim, act_format, command_buffer)?;+ let down_input = self+ .gate+ .encode_for_linear(&fused_up, batch_dim, act_format, command_buffer)+ .map_err(MlpEncodeError::Backend)?;let output = self.down.encode_input(down_input, batch_dim, command_buffer)?;Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rsindex 1e96ba139..25e0d78ba 100644--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs@@ -1,7 +1,7 @@use crate::{backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding,- gpu_types::ActivationType,+ gpu_types::activation_type::ActivationType,kernel::{ActivationQuantization, GatedActMul, GatedActMulSettings, matmul::ActivationFormat},},config::{activation::AnyActivation, clipping::ClippingBounds},diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rsindex 988a178e2..4dbaa9209 100644--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs@@ -8,17 +8,28 @@ use crate::{backends::common::{Backend, CommandBuffer},config::mlp::AnyMLPConfig,data_type::DataType,- encodable_block::mlp::dense::{DenseMlp, DenseMlpNewError},+ encodable_block::{+ linear::LinearEncodeError,+ mlp::dense::{DenseMlp, DenseMlpNewError},+ },parameters::ParameterTree,};+#[derive(Debug, Error)]+pub enum MlpEncodeError<B: Backend> {+ #[error("Backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("Linear error: {0}")]+ Linear(#[from] LinearEncodeError<B>),+}+pub trait Mlp<B: Backend>: Send + Sync {fn encode(&self,input: B::ScratchBuffer,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error>;+ ) -> Result<B::ScratchBuffer, MlpEncodeError<B>>;}#[derive(Debug, Error)]diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rsindex c95f24cbe..f3aa283f8 100644--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs@@ -13,7 +13,7 @@ use crate::{data_type::DataType,encodable_block::{embedding_table::{EmbeddingTable, EmbeddingTableError},- linear::{Linear, LinearBlockError},+ linear::{Linear, LinearBlockError, LinearEncodeError},normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},},parameters::ParameterTree,@@ -27,6 +27,8 @@ pub enum PerLayerEmbeddingError<B: Backend> {Normalization(#[from] NormalizationNewError<B>),#[error("Linear error: {0}")]LinearError(#[from] LinearBlockError<B>),+ #[error("Linear encoding error: {0}")]+ LinearEncode(#[from] LinearEncodeError<B>),#[error("Embedding table error: {0}")]EmbeddingTable(#[from] EmbeddingTableError<B>),}@@ -109,14 +111,16 @@ impl<B: Backend> PerLayerEmbedding<B> {inner_features: impl BufferRef<Backend = B>,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, PerLayerEmbeddingError<B>> {let command_buffer = &mut *command_buffer.debug_group("per layer embedding");let total_ple_dim = self.num_layers * self.ple_dim;let total_rows = batch_dim * self.num_layers;let total_elements = batch_dim * total_ple_dim;- let mut token_ple = command_buffer.allocate_scratch_for_shape(&[batch_dim, total_ple_dim], self.data_type)?;+ let mut token_ple = command_buffer+ .allocate_scratch_for_shape(&[batch_dim, total_ple_dim], self.data_type)+ .map_err(PerLayerEmbeddingError::BackendError)?;self.token_embedding.encode_lookup(token_ids,&mut token_ple,@@ -125,21 +129,20 @@ impl<B: Backend> PerLayerEmbedding<B> {command_buffer,);- let mut model_projection_input =- command_buffer.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;+ let mut model_projection_input = command_buffer+ .allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)+ .map_err(PerLayerEmbeddingError::BackendError)?;command_buffer.encode_copy(inner_features, &mut model_projection_input);let model_projected = self.model_projection.encode(model_projection_input, batch_dim, command_buffer)?;- let model_normed = self.projection_norm.encode(- &model_projected,- 0,- total_rows,- None::<&mut B::ScratchBuffer>,- command_buffer,- )?;+ let model_normed = self+ .projection_norm+ .encode(&model_projected, 0, total_rows, None::<&mut B::ScratchBuffer>, command_buffer)+ .map_err(PerLayerEmbeddingError::BackendError)?;- let mut per_layer_inputs =- command_buffer.allocate_scratch_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type)?;+ let mut per_layer_inputs = command_buffer+ .allocate_scratch_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type)+ .map_err(PerLayerEmbeddingError::BackendError)?;self.add_scale.encode(Some(&token_ple),&model_normed,@@ -242,7 +245,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {layer_index: u32,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error> {+ ) -> Result<(), PerLayerEmbeddingError<B>> {let command_buffer = &mut *command_buffer.debug_group("per layer embedding projection");let length = batch_dim * self.model_dim;@@ -256,11 +259,15 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {command_buffer,);- let mut gate_input = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;+ let mut gate_input = command_buffer+ .allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)+ .map_err(PerLayerEmbeddingError::BackendError)?;command_buffer.encode_copy(outputs.as_ref(), &mut gate_input);let gate_out = self.gate.encode(gate_input, batch_dim, command_buffer)?;- let mut activated = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.ple_dim], self.data_type)?;+ let mut activated = command_buffer+ .allocate_scratch_for_shape(&[batch_dim, self.ple_dim], self.data_type)+ .map_err(PerLayerEmbeddingError::BackendError)?;self.gate_act_mul.encode_fp(&gate_out,Some(per_layer_input),@@ -275,7 +282,10 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {);let projected = self.projection.encode(activated, batch_dim, command_buffer)?;- let normed = self.norm.encode(&projected, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)?;+ let normed = self+ .norm+ .encode(&projected, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)+ .map_err(PerLayerEmbeddingError::BackendError)?;self.residual_combine.encode(None::<&B::ScratchBuffer>,diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rsindex 913ac5997..e28394451 100644--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs@@ -4,13 +4,13 @@ use thiserror::Error;use crate::{backends::common::{Backend, CommandBuffer, CommandBufferEncoding,- gpu_types::ActivationType,+ gpu_types::activation_type::ActivationType,kernel::{ActivationKernel, Kernels},},config::classifier::PredictionHeadConfig,data_type::DataType,encodable_block::{- linear::{Linear, LinearBlockError},+ linear::{Linear, LinearBlockError, LinearEncodeError},normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},},parameters::ParameterTree,@@ -22,6 +22,8 @@ pub enum PredictionHeadError<B: Backend> {Backend(#[source] B::Error),#[error("Linear error: {0}")]Linear(#[from] LinearBlockError<B>),+ #[error("Linear encoding error: {0}")]+ LinearEncode(#[from] LinearEncodeError<B>),#[error("Normalization error: {0}")]Normalization(#[from] NormalizationNewError<B>),}@@ -86,7 +88,7 @@ impl<B: Backend> PredictionHead<B> {input: B::ScratchBuffer,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, PredictionHeadError<B>> {let command_buffer = &mut *command_buffer.debug_group("prediction head");let mut hidden = self.dense_projection.encode(input, batch_dim, command_buffer)?;@@ -97,8 +99,10 @@ impl<B: Backend> PredictionHead<B> {self.activation,command_buffer,);- let normalized =- self.normalization.encode(&hidden, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)?;+ let normalized = self+ .normalization+ .encode(&hidden, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)+ .map_err(PredictionHeadError::Backend)?;let logits = self.readout.encode(normalized, batch_dim, command_buffer)?;Ok(logits)diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rsindex b541d3be9..a3a9a0ed3 100644--- a/crates/uzu-engine/src/encodable_block/transformer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer.rs@@ -90,6 +90,14 @@ pub enum TransformerNewError<B: Backend> {Normalization(#[from] NormalizationNewError<B>),}+#[derive(Debug, Error)]+pub enum TransformerEncodeError<B: Backend> {+ #[error("Backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("Layer error: {0}")]+ Layer(#[from] TransformerLayerError<B>),+}+pub struct Transformer<B: Backend> {ropes: Box<[AnyRoPEConfig]>,layers: Box<[(TransformerLayer<B>, Option<usize>)]>,@@ -236,7 +244,7 @@ impl<B: Backend> Transformer<B> {hidden_feature_layer_indices: Option<&[u32]>,mut state: Option<&mut TransformerState<B>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<TransformerEncodeOutput<B>, B::Error> {+ ) -> Result<TransformerEncodeOutput<B>, TransformerEncodeError<B>> {let mut hidden = input;let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_none() {self.prefill_cache_layer_count()@@ -244,7 +252,7 @@ impl<B: Backend> Transformer<B> {self.layers.len()};- let mut shortcut = command_buffer.allocate_scratch(hidden.size())?;+ let mut shortcut = command_buffer.allocate_scratch(hidden.size()).map_err(TransformerEncodeError::Backend)?;let mut hidden_features =hidden_feature_layer_indices.map(|indices| (0..indices.len()).map(|_| None).collect::<Vec<_>>());@@ -255,7 +263,8 @@ impl<B: Backend> Transformer<B> {.ropes.iter().map(|rope_config| PrecalculatedRoPE::precalculate(rope_config, &token_positions, command_buffer))- .collect::<Result<Box<[_]>, B::Error>>()?;+ .collect::<Result<Box<[_]>, B::Error>>()+ .map_err(TransformerEncodeError::Backend)?;for (layer, layer_rope_index) in self.layers.iter().take(layer_count) {let precalculated_rope = layer_rope_index.map(|i| &precalculated_ropes[i]);@@ -289,7 +298,9 @@ impl<B: Backend> Transformer<B> {if let (Some(hidden_features), Some(indices)) = (&mut hidden_features, hidden_feature_layer_indices) {for (feature_index, &layer_index) in indices.iter().enumerate() {if layer_index == layer.layer_index {- let feature = self.capture_residual(&shortcut, &hidden, batch_dim.size(), command_buffer)?;+ let feature = self+ .capture_residual(&shortcut, &hidden, batch_dim.size(), command_buffer)+ .map_err(TransformerEncodeError::Backend)?;hidden_features[feature_index] = Some(feature);}}@@ -318,13 +329,16 @@ impl<B: Backend> Transformer<B> {});};- let output_normalized = self.output_norm.encode(- &hidden,- output_range.start,- output_range.end - output_range.start,- Some(&mut shortcut),- command_buffer,- )?;+ let output_normalized = self+ .output_norm+ .encode(+ &hidden,+ output_range.start,+ output_range.end - output_range.start,+ Some(&mut shortcut),+ command_buffer,+ )+ .map_err(TransformerEncodeError::Backend)?;Ok(TransformerEncodeOutput {output: Some(output_normalized),diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rsindex 928cf5e90..dff07cc0d 100644--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs@@ -10,11 +10,11 @@ use crate::{encodable_block::{batch_topology::BatchTopology,convolution::{ConvolutionNewError, SeparableCausalConv},- linear::{Linear, LinearBlockError},- mixer::{Mixer, MixerNewError, MixerState, attention::rope::PrecalculatedRoPE},- mlp::{Mlp, MlpBlockNewError},+ linear::{Linear, LinearBlockError, LinearEncodeError},+ mixer::{Mixer, MixerEncodeError, MixerNewError, MixerState, attention::rope::PrecalculatedRoPE},+ mlp::{Mlp, MlpBlockNewError, MlpEncodeError},normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},- per_layer_embedding::PerLayerEmbeddingProjection,+ per_layer_embedding::{PerLayerEmbeddingError, PerLayerEmbeddingProjection},},parameters::{ParameterLoaderError, ParameterTree},utils::maybe_mut::MaybeMut,@@ -28,14 +28,22 @@ pub enum TransformerLayerError<B: Backend> {ParameterLoader(#[from] ParameterLoaderError<B>),#[error("Mixer error: {0}")]Mixer(#[from] MixerNewError<B>),+ #[error("Mixer encoding error: {0}")]+ MixerEncode(#[from] MixerEncodeError<B>),#[error("MLP error: {0}")]MlpBlock(#[from] MlpBlockNewError<B>),+ #[error("MLP encoding error: {0}")]+ MlpEncode(#[from] MlpEncodeError<B>),#[error("Normalization error: {0}")]Normalization(#[from] NormalizationNewError<B>),#[error("Convolution error: {0}")]Convolution(#[from] ConvolutionNewError<B>),#[error("Linear error: {0}")]Linear(#[from] LinearBlockError<B>),+ #[error("Linear encoding error: {0}")]+ LinearEncode(#[from] LinearEncodeError<B>),+ #[error("Per-layer embedding error: {0}")]+ PerLayerEmbedding(#[from] PerLayerEmbeddingError<B>),#[error("Layer {layer_index} sets post_layer_scalar but has no post_mlp_norm")]PostLayerScalarWithoutPostMlpNorm {layer_index: u32,@@ -128,18 +136,15 @@ impl<B: Backend> TransformerLayerConv<B> {input: impl BufferRef<Backend = B>,sequence_length: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(B::ScratchBuffer, B::ScratchBuffer), B::Error> {- let mut projection_input = command_buffer.allocate_scratch(input.size())?;+ ) -> Result<(B::ScratchBuffer, B::ScratchBuffer), TransformerLayerError<B>> {+ let mut projection_input =+ command_buffer.allocate_scratch(input.size()).map_err(TransformerLayerError::Backend)?;command_buffer.encode_copy(input, &mut projection_input);let coefficients = self.kernel_projection.encode(projection_input, sequence_length, command_buffer)?;- let mut output = self.pre_conv.encode(- input,- &coefficients,- 2 * self.coefficient_count,- 0,- sequence_length,- command_buffer,- )?;+ let mut output = self+ .pre_conv+ .encode(input, &coefficients, 2 * self.coefficient_count, 0, sequence_length, command_buffer)+ .map_err(TransformerLayerError::Backend)?;if let Some((transform, factors)) = &self.input_rht {transform.encode_fp_in_place(&mut output, factors, None, sequence_length, self.model_dim, command_buffer);}@@ -349,11 +354,13 @@ impl<B: Backend> TransformerLayer<B> {batch_dim: &BatchTopology,state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, TransformerLayerError<B>> {let command_buffer = &mut *command_buffer.debug_group(&format!("transformer layer {}", self.layer_index));let mut hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {- pre_mixer_norm.encode(&input, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)?+ pre_mixer_norm+ .encode(&input, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)+ .map_err(TransformerLayerError::Backend)?} else {assert!(self.layer_index == 0);command_buffer.encode_copy(&input, shortcut.reborrow());@@ -374,15 +381,21 @@ impl<B: Backend> TransformerLayer<B> {if let Some(coefficients) = mixer_coefficients {let convolution = self.mixer_conv.as_ref().expect("mixer convolution required");- hidden = convolution.encode_post_convolution(&hidden, &coefficients, batch_dim.size(), command_buffer)?;+ hidden = convolution+ .encode_post_convolution(&hidden, &coefficients, batch_dim.size(), command_buffer)+ .map_err(TransformerLayerError::Backend)?;}if let Some(post_mixer_norm) = &self.post_mixer_norm {- hidden =- post_mixer_norm.encode(&hidden, 0, batch_dim.size(), None::<&mut B::ScratchBuffer>, command_buffer)?;+ hidden = post_mixer_norm+ .encode(&hidden, 0, batch_dim.size(), None::<&mut B::ScratchBuffer>, command_buffer)+ .map_err(TransformerLayerError::Backend)?;}- hidden = self.pre_mlp_norm.encode(&hidden, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)?;+ hidden = self+ .pre_mlp_norm+ .encode(&hidden, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)+ .map_err(TransformerLayerError::Backend)?;let mlp_coefficients = if let Some(convolution) = &self.mlp_conv {let (output, coefficients) =@@ -397,12 +410,15 @@ impl<B: Backend> TransformerLayer<B> {if let Some(coefficients) = mlp_coefficients {let convolution = self.mlp_conv.as_ref().expect("mlp convolution required");- hidden = convolution.encode_post_convolution(&hidden, &coefficients, batch_dim.size(), command_buffer)?;+ hidden = convolution+ .encode_post_convolution(&hidden, &coefficients, batch_dim.size(), command_buffer)+ .map_err(TransformerLayerError::Backend)?;}if let Some(post_mlp_norm) = &self.post_mlp_norm {- hidden =- post_mlp_norm.encode(&hidden, 0, batch_dim.size(), None::<&mut B::ScratchBuffer>, command_buffer)?;+ hidden = post_mlp_norm+ .encode(&hidden, 0, batch_dim.size(), None::<&mut B::ScratchBuffer>, command_buffer)+ .map_err(TransformerLayerError::Backend)?;}if let Some(ple_projection) = &self.ple_projection {diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex 2569a872e..d90e3474b 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -17,9 +17,9 @@ use crate::{data_type::DataType,encodable_block::{embedding::{Embedding, EmbeddingError},- linear::{Linear, LinearBlockError},+ linear::{Linear, LinearBlockError, LinearEncodeError},mixer::attention::{KVCacheView, rope::PrecalculatedRoPE},- mlp::MlpBlockNewError,+ mlp::{MlpBlockNewError, MlpEncodeError},normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},weaver_layer::{PreparedPrefixAttention, WeaverLayer},},@@ -132,6 +132,8 @@ pub struct Weaver<B: Backend> {#[derive(Debug, Error)]pub enum WeaverNewError<B: Backend> {+ #[error("Attention kernel error: {0}")]+ Attention(#[source] <<B::Kernels as Kernels>::AttentionKernel as AttentionKernel>::Error),#[error("linear error: {0}")]Linear(#[from] LinearBlockError<B>),#[error("mlp error: {0}")]@@ -162,6 +164,12 @@ pub enum WeaverNewError<B: Backend> {pub enum WeaverEncodeError<B: Backend> {#[error("backend error: {0}")]Backend(#[source] B::Error),+ #[error("Linear error: {0}")]+ Linear(#[from] LinearEncodeError<B>),+ #[error("MLP error: {0}")]+ Mlp(#[from] MlpEncodeError<B>),+ #[error("Attention kernel error: {0}")]+ Attention(#[source] <<B::Kernels as Kernels>::AttentionKernel as AttentionKernel>::Error),#[error("embedding error: {0}")]Embedding(#[from] EmbeddingError<B>),#[error("invalid Weaver tree input")]@@ -311,10 +319,7 @@ impl<B: Backend> Weaver<B> {.hidden_state_norm.encode(&prefix_hidden, 0, depth, None::<&mut B::ScratchBuffer>, command_buffer).map_err(WeaverEncodeError::Backend)?;- let mut residual_input = self- .hidden_state_projection- .encode(normalized_prefix, depth, command_buffer)- .map_err(WeaverEncodeError::Backend)?;+ let mut residual_input = self.hidden_state_projection.encode(normalized_prefix, depth, command_buffer)?;let (last_layer, preceding_layers) = self.layers.split_last().expect("Weaver must have at least one layer");let mut residual_state =command_buffer.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;@@ -323,9 +328,7 @@ impl<B: Backend> Weaver<B> {let PreparedPrefixAttention {queries,kv_cache,- } = layer- .encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, command_buffer)- .map_err(WeaverEncodeError::Backend)?;+ } = layer.encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, command_buffer)?;let cache = KVCacheView::full(0);let kv_plane_bytes = size_for_shape(&[depth, self.model_dim], DATA_TYPE);let attention_output = layer@@ -343,16 +346,14 @@ impl<B: Backend> Weaver<B> {},command_buffer,)- .map_err(WeaverEncodeError::Backend)?;- residual_input = layer- .encode_post_attention(attention_output, &mut residual_state, depth, command_buffer)- .map_err(WeaverEncodeError::Backend)?;+ .map_err(WeaverEncodeError::Attention)?;+ residual_input =+ layer.encode_post_attention(attention_output, &mut residual_state, depth, command_buffer)?;prefix_kv_layers.push(kv_cache);}prefix_kv_layers.push(last_layer- .encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, command_buffer)- .map_err(WeaverEncodeError::Backend)?+ .encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, command_buffer)?.kv_cache,);@@ -425,10 +426,8 @@ impl<B: Backend> Weaver<B> {.token_embedding_norm.encode(&token_embedding, 0, batch_node_count, None::<&mut B::ScratchBuffer>, command_buffer).map_err(WeaverEncodeError::Backend)?;- let mut residual_input = self- .token_embedding_projection- .encode(normalized_embedding, batch_node_count, command_buffer)- .map_err(WeaverEncodeError::Backend)?;+ let mut residual_input =+ self.token_embedding_projection.encode(normalized_embedding, batch_node_count, command_buffer)?;let mut residual_state =command_buffer.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;let node_metadata = node_metadata.as_ref();@@ -438,10 +437,7 @@ impl<B: Backend> Weaver<B> {.pre_attention_norm.encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), command_buffer).map_err(WeaverEncodeError::Backend)?;- let current_qkv = layer- .qkv_projection- .encode(attention_input, batch_node_count, command_buffer)- .map_err(WeaverEncodeError::Backend)?;+ let current_qkv = layer.qkv_projection.encode(attention_input, batch_node_count, command_buffer)?;let mut attention_output = command_buffer.allocate_scratch_for_shape(&[batch_node_count, self.model_dim], DATA_TYPE).map_err(WeaverEncodeError::Backend)?;@@ -464,19 +460,15 @@ impl<B: Backend> Weaver<B> {layer.attention_scale,command_buffer,);- residual_input = layer- .encode_post_attention(attention_output, &mut residual_state, batch_node_count, command_buffer)- .map_err(WeaverEncodeError::Backend)?;+ residual_input =+ layer.encode_post_attention(attention_output, &mut residual_state, batch_node_count, command_buffer)?;}let normalized_output = self.readout_norm.encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), command_buffer).map_err(WeaverEncodeError::Backend)?;- let query = self- .readout_query_projection- .encode(normalized_output, batch_node_count, command_buffer)- .map_err(WeaverEncodeError::Backend)?;+ let query = self.readout_query_projection.encode(normalized_output, batch_node_count, command_buffer)?;let logit_residuals = target_embedding.encode_readout(&query,batch_node_count,diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rsindex a71d6d5ac..6bfd7cf75 100644--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs@@ -14,7 +14,7 @@ use crate::{mixer::attention::rope::PrecalculatedRoPE,mlp::Mlp,normalization::{Normalization, PostLayerScalar, ShortcutMode},- weaver::{DATA_TYPE, ROPE_DATA_TYPE, WeaverNewError},+ weaver::{DATA_TYPE, ROPE_DATA_TYPE, WeaverEncodeError, WeaverNewError},},parameters::ParameterTree,};@@ -89,7 +89,7 @@ impl<B: Backend> WeaverLayer<B> {data_type: DATA_TYPE,},)- .map_err(WeaverNewError::Backend)?;+ .map_err(WeaverNewError::Attention)?;let attention_prepare =<B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, ROPE_DATA_TYPE, true, true).map_err(WeaverNewError::Backend)?;@@ -152,14 +152,19 @@ impl<B: Backend> WeaverLayer<B> {rope: &PrecalculatedRoPE<B>,token_count: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<PreparedPrefixAttention<B>, B::Error> {- let attention_input =- self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), command_buffer)?;+ ) -> Result<PreparedPrefixAttention<B>, WeaverEncodeError<B>> {+ let attention_input = self+ .pre_attention_norm+ .encode(residual_input, 0, token_count, Some(residual_state), command_buffer)+ .map_err(WeaverEncodeError::Backend)?;let qkv = self.qkv_projection.encode(attention_input, token_count, command_buffer)?;- let mut queries =- command_buffer.allocate_scratch_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE)?;+ let mut queries = command_buffer+ .allocate_scratch_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE)+ .map_err(WeaverEncodeError::Backend)?;let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);- let mut kv_cache = command_buffer.allocate_scratch_for_shape(&[2, token_count, self.model_dim], DATA_TYPE)?;+ let mut kv_cache = command_buffer+ .allocate_scratch_for_shape(&[2, token_count, self.model_dim], DATA_TYPE)+ .map_err(WeaverEncodeError::Backend)?;let (keys, values) = kv_cache.split_at(kv_plane_bytes);self.attention_prepare.encode(&qkv,@@ -189,10 +194,12 @@ impl<B: Backend> WeaverLayer<B> {residual_state: impl BufferMut<Backend = B>,token_count: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {+ ) -> Result<B::ScratchBuffer, WeaverEncodeError<B>> {let projected_attention = self.out_projection.encode(attention_output, token_count, command_buffer)?;- let mlp_input =- self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), command_buffer)?;- self.mlp.encode(mlp_input, token_count, command_buffer)+ let mlp_input = self+ .pre_mlp_norm+ .encode(&projected_attention, 0, token_count, Some(residual_state), command_buffer)+ .map_err(WeaverEncodeError::Backend)?;+ self.mlp.encode(mlp_input, token_count, command_buffer).map_err(WeaverEncodeError::Mlp)}}diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rsindex 1b5515e87..21731c4a4 100644--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs@@ -4,7 +4,7 @@ use thiserror::Error;use crate::{backends::common::{Backend, BufferCpuAccessible, BufferMut,- gpu_types::{QuantizationMethod, QuantizationMode},+ gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},kernel::matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection},},config::weight_matrix::{AnyWeightMatrixSpec, Layout},diff --git a/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs b/crates/uzu-engine/src/engine/language_model/grammar/grammar.rsindex 07fa38a9d..637fe4009 100644--- a/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs+++ b/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs@@ -1,7 +1,7 @@use tokenizers::Tokenizer;use xgrammar::{GrammarCompiler, GrammarMatcher, TokenizerInfo, get_bitmask_size};-use super::{GrammarConfig, GrammarError, engagement::GrammarEngagementState};+use crate::engine::language_model::grammar::{GrammarConfig, GrammarError, engagement::GrammarEngagementState};pub struct Grammar {bitmask_size: usize,diff --git a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs b/crates/uzu-engine/src/engine/language_model/grammar/mod.rsindex 9eb75ec8a..2b8bbf969 100644--- a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/grammar/mod.rs@@ -3,6 +3,4 @@ mod engagement;mod error;mod grammar;-pub use config::GrammarConfig;-pub use error::GrammarError;-pub use grammar::Grammar;+pub use crate::engine::language_model::grammar::{config::GrammarConfig, error::GrammarError, grammar::Grammar};diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rsindex 4e051ee1f..adc82fe58 100644--- a/crates/uzu-engine/src/engine/language_model/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/mod.rs@@ -6,10 +6,7 @@ use tokenizers::Tokenizer;use crate::{backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},- config::{- model::{generation::GenerationConfig, language_model::LanguageModelConfig},- token_codec::AnyTokenCodecConfig,- },+ config::{model::language_model::LanguageModelConfig, token_codec::AnyTokenCodecConfig},data_type::DataType,encodable_block::{decoder::{Decoder, DecoderError},@@ -19,6 +16,7 @@ use crate::{parameters::{ParameterLoader, ParameterLoaderError},speculators::dflash_tfm::{DFlashSpeculatorLoadError, DFlashTfmSpeculator},};+pub use crate::{config::model::generation::GenerationConfig, utils::strict_serde::Unsupported};pub mod state;pub mod stream;diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex 593db5397..706da8cd0 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -315,14 +315,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {if let Some(speculator) = model.speculator.as_ref() {let speculator_state = model_state.speculator_state.as_mut().unwrap();- speculator- .encode_accept(- speculator_state,- decoder_output.hidden_features.as_ref().unwrap().iter(),- &(0..input_chunk.len() as u32).collect::<Box<[u32]>>(),- &mut command_buffer,- )- .map_err(LanguageModelStreamError::Backend)?;+ speculator.encode_accept(+ speculator_state,+ decoder_output.hidden_features.as_ref().unwrap().iter(),+ &(0..input_chunk.len() as u32).collect::<Box<[u32]>>(),+ &mut command_buffer,+ )?;}if let Some(suffix_repetition_length) = options.sampling_method.suffix_repetition_length() {@@ -468,14 +466,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.encode_accept(&accepted_token_indicies, &mut command_buffer).map_err(LanguageModelStreamError::Backend)?;if let Some(speculator) = self.model.speculator.as_ref() {- speculator- .encode_accept(- self.model_state.speculator_state.as_mut().unwrap(),- hidden_features.as_ref().unwrap().iter(),- &accepted_token_indicies,- &mut command_buffer,- )- .map_err(LanguageModelStreamError::Backend)?;+ speculator.encode_accept(+ self.model_state.speculator_state.as_mut().unwrap(),+ hidden_features.as_ref().unwrap().iter(),+ &accepted_token_indicies,+ &mut command_buffer,+ )?;}let output_norm = if let Some((final_hidden, row_bytes)) = output_norm {let row = full.last().unwrap().0;@@ -723,14 +719,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {if let Some(speculator) = self.model.speculator.as_ref() {let speculator_state = self.model_state.speculator_state.as_mut().unwrap();- speculator- .encode_accept(- speculator_state,- decoder_output.hidden_features.as_ref().unwrap().iter(),- &(0..batch_dim.size()).collect::<Box<[u32]>>(),- &mut command_buffer,- )- .map_err(LanguageModelStreamError::Backend)?;+ speculator.encode_accept(+ speculator_state,+ decoder_output.hidden_features.as_ref().unwrap().iter(),+ &(0..batch_dim.size()).collect::<Box<[u32]>>(),+ &mut command_buffer,+ )?;}if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {diff --git a/crates/uzu-engine/src/lib.rs b/crates/uzu-engine/src/lib.rsindex f29b0b540..d073d94a5 100644--- a/crates/uzu-engine/src/lib.rs+++ b/crates/uzu-engine/src/lib.rs@@ -4,6 +4,7 @@mod array;mod config;+mod data_type;mod encodable_block;mod parameters;mod speculators;@@ -11,7 +12,6 @@ mod trie;mod utils;pub mod backends;-pub mod data_type;pub mod engine;diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex afeaf8139..b3c503b41 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -167,8 +167,10 @@ impl<B: Backend> DFlashTfmSpeculator<B> {target_features: impl ExactSizeIterator<Item = impl BufferRef<Backend = B>>,accepted_indices: &[u32],command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error> {- self.dflash.encode_accept(target_features, accepted_indices, state, command_buffer)+ ) -> Result<(), DFlashTreeError<B>> {+ self.dflash+ .encode_accept(target_features, accepted_indices, state, command_buffer)+ .map_err(DFlashTreeError::DFlash)}pub fn make_shape(diff --git a/crates/uzu-engine/src/utils/dispatch_dtype.rs b/crates/uzu-engine/src/utils/dispatch_dtype.rsdeleted file mode 100644index 423cb3472..000000000--- a/crates/uzu-engine/src/utils/dispatch_dtype.rs+++ /dev/null@@ -1,12 +0,0 @@-#[macro_export]-macro_rules! dispatch_dtype {- (|()| $body:expr) => { $body };- (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {- match $dt {- $crate::data_type::DataType::F16 => { type $T = half::f16; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }- $crate::data_type::DataType::BF16 => { type $T = half::bf16; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }- $crate::data_type::DataType::F32 => { type $T = f32; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }- other => panic!("unsupported dtype in test: {:?}", other),- }- };-}diff --git a/crates/uzu-engine/src/utils/mod.rs b/crates/uzu-engine/src/utils/mod.rsindex 11e88e2c5..e0f6473d3 100644--- a/crates/uzu-engine/src/utils/mod.rs+++ b/crates/uzu-engine/src/utils/mod.rs@@ -1,4 +1,3 @@-pub mod dispatch_dtype;pub mod downcast;pub mod maybe_mut;#[cfg(backend = "cpu")]diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_test.rsindex 9d940407d..46ab80e16 100644--- a/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs@@ -9,7 +9,7 @@ use crate::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::ActivationType, kernel::ActivationKernel,+ gpu_types::activation_type::ActivationType, kernel::ActivationKernel,},cpu::Cpu,},diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rsindex ec8d119ac..e58fd30f8 100644--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs@@ -9,7 +9,7 @@ use crate::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,- gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE as BLOCK_SIZE, kernel::ActivationTransform,+ gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE as BLOCK_SIZE, kernel::ActivationTransform,},cpu::Cpu,},diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rsindex 845c24522..4353c0c7a 100644--- a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs@@ -9,7 +9,7 @@ use crate::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::Copy, kernel::KVCacheUpdateKernel,+ gpu_types::kv_cache_update::Copy, kernel::KVCacheUpdateKernel,},cpu::Cpu,},diff --git a/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs b/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rsindex db2b3feef..e2f45a15e 100644--- a/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs@@ -11,10 +11,10 @@ use crate::{common::{Backend, Context, Kernels,gpu_types::{- EmbeddingTableKind::{self, D4S4, Dense, Quantized},- QuantizationMethod::{self, ScaleBias, ScaleSymmetric, ScaleZeroPoint},- QuantizationMode::{self, I8, U4, U8},d4s4,+ embedding::EmbeddingTableKind::{self, D4S4, Dense, Quantized},+ quantization::QuantizationMode::{self, I8, U4, U8},+ quantization_method::QuantizationMethod::{self, ScaleBias, ScaleSymmetric, ScaleZeroPoint},},kernel::InputEmbeddingLookupKernel,},diff --git a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rsindex b131a1851..2d7466022 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs@@ -9,7 +9,7 @@ use crate::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,- gpu_types::ActivationType,+ gpu_types::activation_type::ActivationType,kernel::{ActivationQuantization, GatedActMul, GatedActMulSettings, matmul::Int8CodeLayout},},cpu::Cpu,diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rsindex fed309c5b..1eb047336 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs@@ -10,7 +10,7 @@ use crate::{backends::{common::{Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,- gpu_types::QuantizationMethod,+ gpu_types::quantization_method::QuantizationMethod,kernel::{Kernels,matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rsindex ee9bf49c3..3f03a49ab 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs@@ -7,7 +7,7 @@ use crate::{array::ArrayElement,backends::common::{Backend, Context,- gpu_types::QuantizationMethod,+ gpu_types::quantization_method::QuantizationMethod,kernel::{Kernels, matmul::MatmulKernel},},tests::{diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rsindex 7f35d292e..e5a284ec2 100644--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs@@ -9,7 +9,7 @@ use crate::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::ActivationType, kernel::Conv1dDecodeKernel,+ gpu_types::activation_type::ActivationType, kernel::Conv1dDecodeKernel,},cpu::Cpu,},diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rsindex 2726ed554..5706500ba 100644--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs@@ -9,7 +9,7 @@ use crate::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::ActivationType, kernel::Conv1dScanKernel,+ gpu_types::activation_type::ActivationType, kernel::Conv1dScanKernel,},cpu::Cpu,},diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rsindex 13a77f41e..e3e39e628 100644--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs@@ -8,7 +8,7 @@ use crate::{array::ArrayElement,backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::ActivationType, kernel::SSDPrefillKernel,+ gpu_types::activation_type::ActivationType, kernel::SSDPrefillKernel,},data_type::DataType,tests::{diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rsindex f7c944dd1..abf3ca05f 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs@@ -8,7 +8,10 @@ use crate::{backends::{common::{Backend, BufferMut, CommandBuffer, CommandBufferEncoding,- gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod, QuantizationMode},+ gpu_types::{+ hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE, quantization::QuantizationMode,+ quantization_method::QuantizationMethod,+ },kernel::{ActivationQuantization, ActivationTransform, Kernels,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rsindex 4d98aaf32..abcc83a66 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs@@ -12,7 +12,7 @@ use crate::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,- gpu_types::{QuantizationMethod, QuantizationMode},+ gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},kernel::{Kernels,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rsindex a97e8234e..7d664744e 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs@@ -8,7 +8,7 @@ use crate::{backends::{common::{Backend,- gpu_types::QuantizationMethod,+ gpu_types::quantization_method::QuantizationMethod,kernel::{Kernels, matmul::MatmulKernel},},metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rsindex e4f2db786..6ade045f0 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs@@ -8,7 +8,7 @@ use crate::{backends::{common::{Backend,- gpu_types::QuantizationMethod,+ gpu_types::quantization_method::QuantizationMethod,kernel::{Kernels, matmul::MatmulKernel},},metal::{Metal, context::MetalContext},diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rsindex ab293efd1..ef5ddacfc 100644--- a/crates/uzu-engine/unit/common/matmul/quant.rs+++ b/crates/uzu-engine/unit/common/matmul/quant.rs@@ -9,7 +9,7 @@ use crate::{common::{Backend, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,Context,- gpu_types::{QuantizationMethod, QuantizationMode},+ gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},kernel::{ActivationQuantization, ActivationTransform, Kernels,matmul::{diff --git a/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs b/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rsindex e4347cfc8..3702cf678 100644--- a/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs+++ b/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs@@ -3,7 +3,7 @@ use uzu_engine_macros::uzu_test;use crate::{backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::ActivationType,+ gpu_types::activation_type::ActivationType,kernel::{Conv1dScanKernel, SSDPrefill64Kernel, SSDPrefillKernel},},data_type::DataType,diff --git a/crates/uzu-engine/unit/encodable_block/sampling_test.rs b/crates/uzu-engine/unit/encodable_block/sampling_test.rsindex c6af3b91e..fba8724f1 100644--- a/crates/uzu-engine/unit/encodable_block/sampling_test.rs+++ b/crates/uzu-engine/unit/encodable_block/sampling_test.rs@@ -13,7 +13,6 @@ use crate::{gpu_types::trie::TrieNode,},data_type::DataType,- dispatch_dtype,encodable_block::{batch_topology::BatchTopology,sampling::{Sampling, SamplingMethod},@@ -21,6 +20,18 @@ use crate::{tests::proptest::{ComparableTestResults, TestContextes, for_each_context, kernel_data_type},};+macro_rules! dispatch_dtype {+ (|()| $body:expr) => { $body };+ (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {+ match $dt {+ $crate::data_type::DataType::F16 => { type $T = half::f16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }+ $crate::data_type::DataType::BF16 => { type $T = half::bf16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }+ $crate::data_type::DataType::F32 => { type $T = f32; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }+ other => panic!("unsupported dtype in test: {:?}", other),+ }+ };+}+struct SamplingTestResults(Vec<u32>);impl ComparableTestResults for SamplingTestResults {