Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Small cleanup part 4 (#955)

uuuvne7e8e1e0180d
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
deleted file mode 100644
index 82e8a1e32..000000000
--- a/.vscode/extensions.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "recommendations": [
- "vadimcn.vscode-lldb",
- "tamasfe.even-better-toml",
- "rust-lang.rust-analyzer",
- "eugenebokhan.metal-analyzer",
- "augustocdias.tasks-shell-input"
- ]
-}
\ No newline at end of file
diff --git a/.vscode/keybindings.json b/.vscode/keybindings.json
deleted file mode 100644
index e247852b2..000000000
--- a/.vscode/keybindings.json
+++ /dev/null
@@ -1,22 +0,0 @@
-[
- {
- "key": "cmd+shift+b",
- "command": "workbench.action.tasks.runTask",
- "args": "cargo build"
- },
- {
- "key": "cmd+shift+c",
- "command": "workbench.action.tasks.runTask",
- "args": "cargo check"
- },
- {
- "key": "cmd+shift+t",
- "command": "workbench.action.tasks.runTask",
- "args": "cargo test"
- },
- {
- "key": "cmd+shift+r",
- "command": "workbench.action.tasks.runTask",
- "args": "cargo run"
- }
-]
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
deleted file mode 100644
index c9773071d..000000000
--- a/.vscode/settings.json
+++ /dev/null
@@ -1,62 +0,0 @@
-{
- "editor.formatOnSave": true,
- "editor.codeActionsOnSave": {
- "source.organizeImports": "explicit",
- },
- "files.associations": {
- "*.h": "metal",
- },
- "rust-analyzer.rustfmt.extraArgs": [
- "+nightly"
- ],
- "[rust]": {
- "editor.formatOnSave": true,
- "editor.defaultFormatter": "rust-lang.rust-analyzer",
- },
- "rust-analyzer.testExplorer": true,
- "rust-analyzer.procMacro.enable": true,
- "[metal]": {
- "editor.formatOnSave": true,
- "editor.defaultFormatter": "eugenebokhan.metal-analyzer",
- },
- "metal-analyzer.diagnostics.onType": true,
- "metal-analyzer.diagnostics.onSave": true,
- "metal-analyzer.diagnostics.debounceMs": 300,
- "metal-analyzer.indexing.enabled": true,
- "metal-analyzer.indexing.concurrency": 8,
- "metal-analyzer.indexing.maxFileSizeKb": 1024,
- "metal-analyzer.formatting.enabled": true,
- "metal-analyzer.formatting.command": "clang-format",
- "metal-analyzer.formatting.args": [],
- "metal-analyzer.diagnostics.scope": "workspace",
- "metal-analyzer.compiler.includePaths": [],
- "metal-analyzer.compiler.extraFlags": [],
- "[python]": {
- "editor.formatOnSave": true,
- "editor.defaultFormatter": "charliermarsh.ruff",
- "editor.codeActionsOnSave": {
- "source.fixAll.ruff": "explicit",
- "source.organizeImports.ruff": "explicit"
- }
- },
- "python.defaultInterpreterPath": "tools/.venv/bin/python",
- "python.testing.pytestArgs": [
- "tests"
- ],
- "python.testing.unittestEnabled": false,
- "python.testing.pytestEnabled": false,
- "ruff.path": [
- "tools/.venv/bin/ruff"
- ],
- "[swift]": {
- "editor.formatOnSave": true,
- },
- "[toml]": {
- "editor.formatOnSave": true,
- "editor.defaultFormatter": "tamasfe.even-better-toml",
- },
- "[json]": {
- "editor.formatOnSave": true,
- "editor.defaultFormatter": "vscode.json-language-features"
- },
-}
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
deleted file mode 100644
index 08879423b..000000000
--- a/.vscode/tasks.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "version": "2.0.0",
- "tasks": [
- {
- "label": "Format Code",
- "type": "shell",
- "command": "cargo +nightly fmt && ./scripts/clang-format.sh",
- "problemMatcher": []
- }
- ]
-}
\ No newline at end of file
diff --git a/crates/legacy/cli/src/bench/model.rs b/crates/legacy/cli/src/bench/model.rs
index 6345fafa7..afbb05bb4 100644
--- a/crates/legacy/cli/src/bench/model.rs
+++ b/crates/legacy/cli/src/bench/model.rs
@@ -4,7 +4,6 @@ use uzu::types::{
basic::ReasoningEffort,
session::chat::{ChatMessage, ChatReply, ChatReplyEnergy, ChatRole},
};
-use uzu_engine::data_type::DataType;
use crate::{
common::thinking::ThinkingSupport,
@@ -105,7 +104,7 @@ pub struct BenchResult {
pub device: BenchDevice,
pub engine_version: String,
pub timestamp: u64,
- pub data_type: DataType,
+ pub data_type: String,
pub memory_used: Option<usize>,
pub tokens_count_input: u64,
pub tokens_count_output: u64,
diff --git a/crates/legacy/cli/src/bench/runner.rs b/crates/legacy/cli/src/bench/runner.rs
index 3875553bb..767cff52d 100644
--- a/crates/legacy/cli/src/bench/runner.rs
+++ b/crates/legacy/cli/src/bench/runner.rs
@@ -14,7 +14,7 @@ use uzu::{
session::chat::{ChatConfig, ChatReplyConfig, ChatReplyEnergy},
},
};
-use uzu_engine::{VERSION, data_type::DataType};
+use uzu_engine::VERSION;
use crate::{
bench::{
@@ -121,7 +121,7 @@ impl BenchRunner {
device: device.clone(),
engine_version: VERSION.to_string(),
timestamp,
- data_type: DataType::BF16,
+ data_type: "bfloat16".to_string(),
memory_used: session.peak_memory_usage().await,
tokens_count_input,
tokens_count_output,
diff --git a/crates/legacy/uzu/src/engine/bridge/backend.rs b/crates/legacy/uzu/src/engine/bridge/backend.rs
index 2a79e874f..1dfb8360d 100644
--- a/crates/legacy/uzu/src/engine/bridge/backend.rs
+++ b/crates/legacy/uzu/src/engine/bridge/backend.rs
@@ -13,7 +13,7 @@ use shoji::{
};
use uzu_engine::{
TOOLCHAIN_VERSION,
- backends::{BackendSelection, common::Backend, select_backend},
+ backends::{Backend, BackendSelection, select_backend},
};
use crate::engine::bridge::{
diff --git a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
index c5372c92d..56f527173 100644
--- a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
+++ b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
@@ -27,7 +27,7 @@ use shoji::{
use tokenizers::Tokenizer;
use tokio_util::sync::CancellationToken;
use uzu_engine::{
- backends::common::Backend,
+ backends::Backend,
engine::{
Engine,
language_model::{LanguageModel, state::LanguageModelState, stream::LanguageModelStream},
diff --git a/crates/legacy/uzu/src/engine/bridge/chat_token_state.rs b/crates/legacy/uzu/src/engine/bridge/chat_token_state.rs
index 7cc46b4b7..f14a9744c 100644
--- a/crates/legacy/uzu/src/engine/bridge/chat_token_state.rs
+++ b/crates/legacy/uzu/src/engine/bridge/chat_token_state.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
use parking_lot::Mutex;
use shoji::traits::State as LlmInstanceState;
-use uzu_engine::{backends::common::Backend, engine::language_model::state::LanguageModelState};
+use uzu_engine::{backends::Backend, engine::language_model::state::LanguageModelState};
pub struct UzuChatTokenBackendInstanceState<B: Backend> {
pub value: Arc<Mutex<LanguageModelState<B>>>,
diff --git a/crates/legacy/uzu/src/engine/bridge/classification_backend.rs b/crates/legacy/uzu/src/engine/bridge/classification_backend.rs
index febd04bbd..89e1226e9 100644
--- a/crates/legacy/uzu/src/engine/bridge/classification_backend.rs
+++ b/crates/legacy/uzu/src/engine/bridge/classification_backend.rs
@@ -19,7 +19,7 @@ use shoji::{
use tokenizers::Tokenizer;
use tokio_util::sync::CancellationToken;
use uzu_engine::{
- backends::common::Backend,
+ backends::Backend,
engine::{Engine, classifier_model::ClassifierModel},
};
diff --git a/crates/legacy/uzu/src/engine/bridge/helpers.rs b/crates/legacy/uzu/src/engine/bridge/helpers.rs
index df9f03fec..94b2c336e 100644
--- a/crates/legacy/uzu/src/engine/bridge/helpers.rs
+++ b/crates/legacy/uzu/src/engine/bridge/helpers.rs
@@ -12,7 +12,7 @@ use tokenizers::Tokenizer;
#[cfg(feature = "capability-grammar")]
use uzu_engine::engine::language_model::grammar::{Grammar as UzuGrammar, GrammarConfig, GrammarError};
use uzu_engine::{
- backends::common::Backend,
+ backends::Backend,
engine::language_model::{LanguageModel, stream::SamplingMethod as UzuSamplingMethod},
};
diff --git a/crates/uzu-engine/build/common/compiler.rs b/crates/uzu-engine/build/common/compiler.rs
index e0ff1571a..59f9f892b 100644
--- a/crates/uzu-engine/build/common/compiler.rs
+++ b/crates/uzu-engine/build/common/compiler.rs
@@ -1,11 +1,11 @@
use std::collections::HashMap;
-use super::{enum_paths::EnumPaths, gpu_types::GpuTypes, identifiers::KernelPath, kernel::Kernel};
+use super::{gpu_type_paths::GpuTypePaths, gpu_types::GpuTypes, identifiers::KernelPath, kernel::Kernel};
pub trait Compiler {
fn build(
&self,
gpu_types: &GpuTypes,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> anyhow::Result<HashMap<KernelPath, Box<[Kernel]>>>;
}
diff --git a/crates/uzu-engine/build/common/enum_paths.rs b/crates/uzu-engine/build/common/gpu_type_paths.rs
similarity index 50%
rename from crates/uzu-engine/build/common/enum_paths.rs
rename to crates/uzu-engine/build/common/gpu_type_paths.rs
index e641b6c06..0bd5123bc 100644
--- a/crates/uzu-engine/build/common/enum_paths.rs
+++ b/crates/uzu-engine/build/common/gpu_type_paths.rs
@@ -1,14 +1,15 @@
use std::collections::{HashMap, hash_map::Entry};
-use anyhow::bail;
+use anyhow::{Context, bail};
use syn::{Type, TypePath, visit_mut::VisitMut};
-use super::gpu_types::{GpuType, GpuTypeName, GpuTypePath, GpuTypes};
+use crate::common::gpu_types::{GpuType, GpuTypeName, GpuTypePath, GpuTypes};
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum GpuTypeKind {
Enum,
OptionSet,
+ Struct,
}
#[derive(Clone)]
@@ -18,24 +19,34 @@ struct GpuTypeEntry {
}
#[derive(Clone)]
-pub struct EnumPaths {
+pub struct GpuTypePaths {
short_name_to_entry: HashMap<GpuTypeName, GpuTypeEntry>,
}
-impl EnumPaths {
+impl GpuTypePaths {
pub fn from_gpu_types(gpu_types: &GpuTypes) -> anyhow::Result<Self> {
- let mut short_name_to_entry = HashMap::new();
+ let mut paths = Self {
+ short_name_to_entry: HashMap::new(),
+ };
+ paths.extend(gpu_types)?;
+ Ok(paths)
+ }
+
+ pub fn extend(
+ &mut self,
+ gpu_types: &GpuTypes,
+ ) -> anyhow::Result<()> {
for file in gpu_types.files.iter() {
for ty in file.types.iter() {
let (name_str, kind) = match ty {
GpuType::Enum(enum_type) => (enum_type.name.as_ref(), GpuTypeKind::Enum),
GpuType::OptionSet(option_set) => (option_set.name.as_ref(), GpuTypeKind::OptionSet),
- GpuType::Constant(_) | GpuType::Struct(_) => continue,
+ GpuType::Struct(struct_type) => (struct_type.name.as_ref(), GpuTypeKind::Struct),
+ GpuType::Constant(_) => continue,
};
let name = GpuTypeName::from(name_str);
- let path =
- GpuTypePath::from(format!("crate::backends::common::gpu_types::{}::{}", file.name, name_str));
- match short_name_to_entry.entry(name) {
+ let path = GpuTypePath::from(format!("{}::{}::{}", gpu_types.module_path, file.name, name_str));
+ match self.short_name_to_entry.entry(name) {
Entry::Occupied(occupied) => {
bail!("gpu type `{}` is duplicated", occupied.key())
},
@@ -48,9 +59,7 @@ impl EnumPaths {
}
}
}
- Ok(Self {
- short_name_to_entry,
- })
+ Ok(())
}
pub fn full_path_for(
@@ -60,6 +69,11 @@ impl EnumPaths {
self.short_name_to_entry.get(short_name).map(|entry| &*entry.path)
}
+ #[allow(dead_code)]
+ pub fn full_paths(&self) -> impl Iterator<Item = &str> {
+ self.short_name_to_entry.values().map(|entry| &*entry.path)
+ }
+
#[allow(dead_code)]
pub fn kind_for(
&self,
@@ -68,40 +82,29 @@ impl EnumPaths {
self.short_name_to_entry.get(short_name).map(|entry| entry.kind)
}
- pub fn canonicalize_type(
+ pub fn resolve_type(
&self,
- ty: &mut Type,
- ) {
- let mut canonicalizer = TypePathCanonicalizer {
- enum_paths: self,
- };
- canonicalizer.visit_type_mut(ty);
- }
-}
-
-struct TypePathCanonicalizer<'enum_paths> {
- enum_paths: &'enum_paths EnumPaths,
-}
+ type_text: &str,
+ ) -> anyhow::Result<Type> {
+ struct TypePaths<'a>(&'a GpuTypePaths);
-impl<'enum_paths> VisitMut for TypePathCanonicalizer<'enum_paths> {
- fn visit_type_path_mut(
- &mut self,
- type_path: &mut TypePath,
- ) {
- let path = &type_path.path;
- if type_path.qself.is_none()
- && path.leading_colon.is_none()
- && path.segments.len() == 1
- && matches!(path.segments[0].arguments, syn::PathArguments::None)
- {
- let segment_name = path.segments[0].ident.to_string();
- if let Some(full_path_text) = self.enum_paths.full_path_for(&segment_name)
- && let Ok(full_path) = syn::parse_str::<syn::Path>(full_path_text)
- {
- type_path.path = full_path;
- return;
+ impl VisitMut for TypePaths<'_> {
+ fn visit_type_path_mut(
+ &mut self,
+ ty: &mut TypePath,
+ ) {
+ syn::visit_mut::visit_type_path_mut(self, ty);
+ if ty.qself.is_none()
+ && let Some(ident) = ty.path.get_ident()
+ && let Some(path) = self.0.full_path_for(&ident.to_string())
+ {
+ ty.path = syn::parse_str(path).unwrap();
+ }
}
}
- syn::visit_mut::visit_type_path_mut(self, type_path);
+
+ let mut ty = syn::parse_str(type_text).with_context(|| format!("cannot parse type `{type_text}`"))?;
+ TypePaths(self).visit_type_mut(&mut ty);
+ Ok(ty)
}
}
diff --git a/crates/uzu-engine/build/common/gpu_types/mod.rs b/crates/uzu-engine/build/common/gpu_types/mod.rs
index 63384d8c8..22c1a5a5e 100644
--- a/crates/uzu-engine/build/common/gpu_types/mod.rs
+++ b/crates/uzu-engine/build/common/gpu_types/mod.rs
@@ -91,13 +91,14 @@ impl GpuTypeFile {
#[derive(Debug)]
pub struct GpuTypes {
+ pub module_path: Box<str>,
pub files: Box<[GpuTypeFile]>,
}
impl GpuTypes {
- pub fn scan() -> anyhow::Result<Self> {
+ pub fn scan(backend: &str) -> anyhow::Result<Self> {
let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("Missing CARGO_MANIFEST_DIR")?)
- .join("src/backends/common/gpu_types");
+ .join(format!("src/backends/{backend}/gpu_types"));
println!("cargo::rerun-if-changed={}", src_dir.display());
let mut sources: Vec<PathBuf> = WalkDir::new(&src_dir)
@@ -110,6 +111,7 @@ impl GpuTypes {
sources.sort();
Ok(Self {
+ module_path: format!("crate::backends::{backend}::gpu_types").into(),
files: sources
.into_iter()
.map(|source| {
diff --git a/crates/uzu-engine/build/common/mod.rs b/crates/uzu-engine/build/common/mod.rs
index f34304136..9ad50233d 100644
--- a/crates/uzu-engine/build/common/mod.rs
+++ b/crates/uzu-engine/build/common/mod.rs
@@ -2,10 +2,10 @@ pub mod caching;
pub mod codegen;
pub mod compiler;
pub mod constraints;
-pub mod enum_paths;
pub mod envs;
#[cfg(all(feature = "metal", target_os = "macos"))]
pub mod expr_rewrite;
+pub mod gpu_type_paths;
pub mod gpu_types;
pub mod identifiers;
pub mod kernel;
diff --git a/crates/uzu-engine/build/common/traitgen.rs b/crates/uzu-engine/build/common/traitgen.rs
index 06d3e3a53..7881f6217 100644
--- a/crates/uzu-engine/build/common/traitgen.rs
+++ b/crates/uzu-engine/build/common/traitgen.rs
@@ -4,30 +4,39 @@ use anyhow::{Context, bail};
use itertools::Itertools;
use proc_macro2::TokenStream;
use quote::{format_ident, quote};
-use syn::Type;
use crate::common::{
codegen::write_tokens,
+ gpu_type_paths::GpuTypePaths,
identifiers::KernelPath,
kernel::{Kernel, KernelArgumentType, KernelBufferAccess, KernelParameterType},
};
-pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
+pub fn traitgen(
+ kernel: &Kernel,
+ gpu_type_paths: &GpuTypePaths,
+) -> anyhow::Result<(TokenStream, TokenStream)> {
let kernel_name = kernel.name.as_ref();
let trait_name = format_ident!("{kernel_name}Kernel");
- let params = kernel.parameters.iter().map(|p| {
- let name = format_ident!("{}", p.name.as_ref());
- let ty = match &p.ty {
- KernelParameterType::Type => quote! { crate::data_type::DataType },
- KernelParameterType::Value(ty) => {
- let ty: Type = syn::parse_str(ty.as_ref()).unwrap();
- quote! { #ty }
- },
- };
+ let params = kernel
+ .parameters
+ .iter()
+ .map(|p| {
+ let name = format_ident!("{}", p.name.as_ref());
+ let ty = match &p.ty {
+ KernelParameterType::Type => quote! { crate::data_type::DataType },
+ KernelParameterType::Value(ty) => {
+ let ty = gpu_type_paths
+ .resolve_type(ty)
+ .with_context(|| format!("invalid type for parameter `{}`", p.name))?;
+ quote! { #ty }
+ },
+ };
- quote! { #name: #ty }
- });
+ Ok(quote! { #name: #ty })
+ })
+ .collect::<anyhow::Result<Vec<_>>>()?;
let mut args = kernel
.arguments
@@ -45,7 +54,9 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
},
},
KernelArgumentType::Constant(ty) => {
- let ty: Type = syn::parse_str(ty.as_ref()).unwrap();
+ let ty = gpu_type_paths
+ .resolve_type(ty)
+ .with_context(|| format!("invalid type for argument `{}`", a.name))?;
quote! { #ty }
},
};
@@ -54,9 +65,9 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
ty = quote! { Option<#ty> };
}
- quote! { #name: #ty }
+ Ok(quote! { #name: #ty })
})
- .collect::<Vec<_>>();
+ .collect::<anyhow::Result<Vec<_>>>()?;
args.push(quote! { command_buffer: &mut <<Self::Backend as crate::backends::common::Backend>::CommandBuffer as crate::backends::common::CommandBuffer>::Encoding });
@@ -74,10 +85,13 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
let kernel_type = quote! { type #trait_name: #trait_name<Backend = Self::Backend>; };
- (kernel_trait, kernel_type)
+ Ok((kernel_trait, kernel_type))
}
-pub fn traitgen_all(backends_kernels: Vec<HashMap<KernelPath, Box<[Kernel]>>>) -> anyhow::Result<()> {
+pub fn traitgen_all(
+ backends_kernels: Vec<HashMap<KernelPath, Box<[Kernel]>>>,
+ gpu_type_paths: &GpuTypePaths,
+) -> anyhow::Result<()> {
let out_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
let mut kernels: HashMap<KernelPath, Box<[Kernel]>> = HashMap::new();
@@ -97,8 +111,11 @@ pub fn traitgen_all(backends_kernels: Vec<HashMap<KernelPath, Box<[Kernel]>>>) -
let mut kernel_traits = Vec::new();
let mut kernel_types = Vec::new();
- for (_file_path, file_kernels) in kernels.into_iter().sorted_by_key(|(p, _k)| p.join("::")) {
- for (tr, ty) in file_kernels.iter().map(traitgen) {
+ for (file_path, file_kernels) in kernels.into_iter().sorted_by_key(|(p, _k)| p.join("::")) {
+ for kernel in &file_kernels {
+ let (tr, ty) = traitgen(kernel, gpu_type_paths).with_context(|| {
+ format!("cannot generate shared trait for kernel `{}` in `{}`", kernel.name, file_path.join("::"))
+ })?;
kernel_traits.push(tr);
kernel_types.push(ty);
}
diff --git a/crates/uzu-engine/build/cpu/compiler.rs b/crates/uzu-engine/build/cpu/compiler.rs
index 0c2c51a1c..aa11c92a4 100644
--- a/crates/uzu-engine/build/cpu/compiler.rs
+++ b/crates/uzu-engine/build/cpu/compiler.rs
@@ -6,7 +6,7 @@ use proc_macro2::TokenStream;
use quote::{ToTokens, format_ident, quote};
use syn::{
Expr, FnArg, GenericArgument, GenericParam, Ident, Item, ItemFn, PathArguments, Type, punctuated::Punctuated,
- token::Comma,
+ token::Comma, visit_mut::VisitMut,
};
use walkdir::WalkDir;
@@ -14,7 +14,7 @@ use crate::common::{
codegen::write_tokens,
compiler::Compiler,
constraints::Constraints,
- enum_paths::EnumPaths,
+ gpu_type_paths::GpuTypePaths,
gpu_types::GpuTypes,
identifiers::{ArgumentName, KernelName, KernelPath},
kernel::{Kernel, KernelArgument, KernelArgumentType, KernelBufferAccess, KernelParameter, KernelParameterType},
@@ -38,7 +38,7 @@ pub struct FunctionArgument {
impl FunctionArgument {
fn to_kernel_argument(
&self,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> Option<KernelArgument> {
Some(KernelArgument {
name: ArgumentName::from(self.name.to_string()),
@@ -46,14 +46,14 @@ impl FunctionArgument {
ty: match &self.ty {
FunctionArgumentType::Buffer(access) => KernelArgumentType::Buffer(access.clone()),
FunctionArgumentType::Constant(ty, None) => KernelArgumentType::Constant(
- format!("&[{}]", canonicalize_type_text(ty, enum_paths)).into_boxed_str(),
+ format!("&[{}]", canonicalize_type_text(ty, gpu_type_paths)).into_boxed_str(),
),
FunctionArgumentType::Constant(ty, Some(size)) => KernelArgumentType::Constant(
- format!("&[{}; {}]", canonicalize_type_text(ty, enum_paths), size.to_token_stream(),)
+ format!("&[{}; {}]", canonicalize_type_text(ty, gpu_type_paths), size.to_token_stream(),)
.into_boxed_str(),
),
FunctionArgumentType::Scalar(ty) => {
- KernelArgumentType::Constant(canonicalize_type_text(ty, enum_paths).into_boxed_str())
+ KernelArgumentType::Constant(canonicalize_type_text(ty, gpu_type_paths).into_boxed_str())
},
FunctionArgumentType::Specialization(_) => {
return None;
@@ -64,13 +64,13 @@ impl FunctionArgument {
fn to_kernel_parameter(
&self,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> Option<KernelParameter> {
Some(KernelParameter {
name: self.name.to_string().into_boxed_str(),
ty: match &self.ty {
FunctionArgumentType::Specialization(ty) => {
- KernelParameterType::Value(canonicalize_type_text(ty, enum_paths).into_boxed_str())
+ KernelParameterType::Value(canonicalize_type_text(ty, gpu_type_paths).into_boxed_str())
},
_ => {
return None;
@@ -82,10 +82,29 @@ impl FunctionArgument {
fn canonicalize_type_text(
ty: &Type,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> String {
+ struct TypeNames<'a>(&'a GpuTypePaths);
+
+ impl VisitMut for TypeNames<'_> {
+ fn visit_type_path_mut(
+ &mut self,
+ ty: &mut syn::TypePath,
+ ) {
+ if ty.qself.is_none()
+ && let Some(segment) = ty.path.segments.last()
+ && matches!(segment.arguments, PathArguments::None)
+ && self.0.full_path_for(&segment.ident.to_string()).is_some()
+ {
+ let ident = &segment.ident;
+ ty.path = syn::parse_quote! { #ident };
+ }
+ syn::visit_mut::visit_type_path_mut(self, ty);
+ }
+ }
+
let mut canonicalized = ty.clone();
- enum_paths.canonicalize_type(&mut canonicalized);
+ TypeNames(gpu_type_paths).visit_type_mut(&mut canonicalized);
canonicalized.to_token_stream().to_string().replace(" :: ", "::")
}
@@ -104,14 +123,14 @@ pub struct FunctionParameter {
impl FunctionParameter {
fn to_kernel_parameter(
&self,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> KernelParameter {
KernelParameter {
name: self.name.to_string().into_boxed_str(),
ty: match &self.ty {
FunctionParameterType::Type => KernelParameterType::Type,
FunctionParameterType::Value(ty) => {
- KernelParameterType::Value(canonicalize_type_text(ty, enum_paths).into_boxed_str())
+ KernelParameterType::Value(canonicalize_type_text(ty, gpu_type_paths).into_boxed_str())
},
},
}
@@ -141,7 +160,7 @@ impl CpuCompiler {
fn compile(
&self,
source_path: PathBuf,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> anyhow::Result<(KernelPath, Box<[Kernel]>)> {
let src_rel_path: KernelPath = source_path
.strip_prefix(&self.src_dir)
@@ -164,7 +183,7 @@ impl CpuCompiler {
if let Item::Fn(ifn) = item
&& ifn.attrs.iter().any(|attr| attr.path().is_ident("kernel"))
{
- Some(self.compile_kernel(ifn, enum_paths))
+ Some(self.compile_kernel(ifn, gpu_type_paths))
} else {
None
}
@@ -177,7 +196,7 @@ impl CpuCompiler {
fn compile_kernel(
&self,
ifn: ItemFn,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> anyhow::Result<Kernel> {
let mut kernel_ident = None;
let mut function_variants = Vec::new();
@@ -294,13 +313,13 @@ impl CpuCompiler {
let kernel_parameters = function_parameters
.iter()
- .map(|parameter| parameter.to_kernel_parameter(enum_paths))
- .chain(function_arguments.iter().flat_map(|argument| argument.to_kernel_parameter(enum_paths)))
+ .map(|parameter| parameter.to_kernel_parameter(gpu_type_paths))
+ .chain(function_arguments.iter().flat_map(|argument| argument.to_kernel_parameter(gpu_type_paths)))
.collect::<Box<[KernelParameter]>>();
let kernel_arguments = function_arguments
.iter()
- .flat_map(|argument| argument.to_kernel_argument(enum_paths))
+ .flat_map(|argument| argument.to_kernel_argument(gpu_type_paths))
.collect::<Box<[KernelArgument]>>();
if function_parameters.len() != function_variants.len() {
@@ -355,7 +374,7 @@ impl CpuCompiler {
quote! { #[allow(non_snake_case)] #parameter_ident: crate::data_type::DataType }
},
KernelParameterType::Value(ty) => {
- let ty: syn::Type = syn::parse_str(ty.as_ref()).unwrap();
+ let ty = gpu_type_paths.resolve_type(ty)?;
quote! { #[allow(non_snake_case)] #parameter_ident: #ty }
},
})
@@ -377,7 +396,7 @@ impl CpuCompiler {
},
},
KernelArgumentType::Constant(ty) => {
- let ty: Type = syn::parse_str(ty.as_ref()).context("cannot parse type")?;
+ let ty = gpu_type_paths.resolve_type(ty)?;
quote! { #ty }
},
};
@@ -669,7 +688,7 @@ impl Compiler for CpuCompiler {
fn build(
&self,
_gpu_types: &GpuTypes,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> anyhow::Result<HashMap<KernelPath, Box<[Kernel]>>> {
let objects = WalkDir::new(&self.src_dir)
.into_iter()
@@ -677,7 +696,7 @@ impl Compiler for CpuCompiler {
.filter(|entry| {
entry.file_type().is_file() && entry.path().extension().and_then(|s| s.to_str()) == Some("rs")
})
- .map(|entry| self.compile(entry.into_path(), enum_paths))
+ .map(|entry| self.compile(entry.into_path(), gpu_type_paths))
.collect::<anyhow::Result<Vec<(KernelPath, Box<[Kernel]>)>>>()
.context("cannot compile cpu sources")?;
diff --git a/crates/uzu-engine/build/main.rs b/crates/uzu-engine/build/main.rs
index 85c1b93ae..a4873b5c2 100644
--- a/crates/uzu-engine/build/main.rs
+++ b/crates/uzu-engine/build/main.rs
@@ -3,7 +3,7 @@ use std::{env, fs, path::PathBuf, process::ExitCode};
use anyhow::Context;
mod common;
-use common::{compiler::Compiler, enum_paths::EnumPaths, envs, gpu_types::GpuTypes, traitgen::traitgen_all};
+use common::{compiler::Compiler, envs, gpu_type_paths::GpuTypePaths, gpu_types::GpuTypes, traitgen::traitgen_all};
mod cpu;
@@ -49,10 +49,10 @@ fn main() -> anyhow::Result<ExitCode> {
debug_log!("cleaned caches");
}
- let gpu_types = GpuTypes::scan().context("Failed to scan gpu types")?;
+ let gpu_types = GpuTypes::scan("common").context("Failed to scan gpu types")?;
debug_log!("gpu_types scan done");
- let enum_paths = EnumPaths::from_gpu_types(&gpu_types).context("Failed to build enum path map")?;
+ let gpu_type_paths = GpuTypePaths::from_gpu_types(&gpu_types).context("Failed to build GPU type path map")?;
let mut compilers: Vec<Box<dyn Compiler>> = Vec::new();
@@ -71,11 +71,11 @@ fn main() -> anyhow::Result<ExitCode> {
}
let backends_kernels =
- compilers.iter().map(|c| c.build(&gpu_types, &enum_paths)).collect::<anyhow::Result<Vec<_>>>()?;
+ compilers.iter().map(|c| c.build(&gpu_types, &gpu_type_paths)).collect::<anyhow::Result<Vec<_>>>()?;
debug_log!("backend build end");
- traitgen_all(backends_kernels)?;
+ traitgen_all(backends_kernels, &gpu_type_paths)?;
debug_log!("build script ended");
diff --git a/crates/uzu-engine/build/metal/ast.rs b/crates/uzu-engine/build/metal/ast.rs
index 94934059d..eaf8b92ed 100644
--- a/crates/uzu-engine/build/metal/ast.rs
+++ b/crates/uzu-engine/build/metal/ast.rs
@@ -203,9 +203,7 @@ impl MetalArgument {
["ulong"] | ["uint64_t"] | ["unsigned", "long"] => Ok("u64".into()),
["int"] | ["int32_t"] => Ok("i32".into()),
["float"] => Ok("f32".into()),
- [vpath] if vpath.starts_with("uzu::") => {
- Ok(vpath.replacen("uzu::", "crate::backends::common::gpu_types::", 1).into())
- },
+ [vpath] if vpath.starts_with("uzu::") => Ok(vpath.rsplit("::").next().unwrap().into()),
_ => bail!("unknown scalar type: {c_type}"),
}
}
diff --git a/crates/uzu-engine/build/metal/bindgen/arguments.rs b/crates/uzu-engine/build/metal/bindgen/arguments.rs
index 3a2eec622..ca3877f4e 100644
--- a/crates/uzu-engine/build/metal/bindgen/arguments.rs
+++ b/crates/uzu-engine/build/metal/bindgen/arguments.rs
@@ -4,7 +4,7 @@ use quote::{format_ident, quote};
use syn::{Expr, Ident, Type};
use crate::{
- common::{enum_paths::EnumPaths, expr_rewrite::rewrite_paths_with},
+ common::{expr_rewrite::rewrite_paths_with, gpu_type_paths::GpuTypePaths},
metal::{
ast::{
MetalArgument, MetalArgumentType, MetalBufferAccess, MetalConstantType, MetalGroupsType, MetalKernelInfo,
@@ -60,7 +60,7 @@ pub struct IndirectDispatchArgument;
pub fn parse(
kernel: &MetalKernelInfo,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
host_expression_rewriter: &mut HostExpressionRewriter,
) -> Result<Vec<ArgumentEmission>> {
let mut emissions = Vec::new();
@@ -71,13 +71,18 @@ pub fn parse(
for argument in kernel.arguments.iter() {
match &argument.argument_type {
MetalArgumentType::Buffer(access) => {
- let buffer = parse_buffer_argument(argument, *access, next_buffer_index, enum_paths)?;
+ let buffer = parse_buffer_argument(argument, *access, next_buffer_index, gpu_type_paths)?;
emissions.push(ArgumentEmission::Buffer(buffer));
next_buffer_index += 1;
},
MetalArgumentType::Constant((rust_type_text, constant_type)) => {
- let constant =
- parse_constant_argument(argument, rust_type_text, constant_type, next_buffer_index, enum_paths)?;
+ let constant = parse_constant_argument(
+ argument,
+ rust_type_text,
+ constant_type,
+ next_buffer_index,
+ gpu_type_paths,
+ )?;
emissions.push(ArgumentEmission::Constant(constant));
next_buffer_index += 1;
},
@@ -86,7 +91,7 @@ pub fn parse(
argument,
dimensions.as_deref(),
next_threadgroup_index,
- enum_paths,
+ gpu_type_paths,
host_expression_rewriter,
)?;
emissions.push(ArgumentEmission::Shared(shared));
@@ -109,10 +114,10 @@ fn parse_shared_argument(
argument: &MetalArgument,
dimensions: Option<&str>,
threadgroup_index: usize,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
host_expression_rewriter: &mut HostExpressionRewriter,
) -> Result<SharedArgument> {
- let condition = parse_argument_condition(argument, enum_paths)?
+ let condition = parse_argument_condition(argument, gpu_type_paths)?
.context("optional threadgroup argument must carry a condition")?;
let dimensions = dimensions.context("optional threadgroup argument must have explicit array dimensions")?;
let element_size = shared_element_byte_size(&argument.c_type)?;
@@ -136,10 +141,10 @@ fn parse_buffer_argument(
argument: &MetalArgument,
access: MetalBufferAccess,
buffer_index: usize,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> Result<BufferArgument> {
let name = format_ident!("{}", argument.name.as_ref());
- let condition = parse_argument_condition(argument, enum_paths)?;
+ let condition = parse_argument_condition(argument, gpu_type_paths)?;
Ok(BufferArgument {
name,
buffer_index,
@@ -153,10 +158,11 @@ fn parse_constant_argument(
rust_type_text: &str,
constant_type: &MetalConstantType,
buffer_index: usize,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> Result<ConstantArgument> {
let name = format_ident!("{}", argument.name.as_ref());
- let element_type: Type = syn::parse_str(rust_type_text)
+ let element_type = gpu_type_paths
+ .resolve_type(rust_type_text)
.with_context(|| format!("constant rust type `{}` cannot be parsed", rust_type_text))?;
let shape = match constant_type {
MetalConstantType::Scalar => ConstantShape::Scalar(element_type),
@@ -170,7 +176,7 @@ fn parse_constant_argument(
}
},
};
- let condition = parse_argument_condition(argument, enum_paths)?;
+ let condition = parse_argument_condition(argument, gpu_type_paths)?;
Ok(ConstantArgument {
name,
buffer_index,
@@ -181,12 +187,12 @@ fn parse_constant_argument(
fn parse_argument_condition(
argument: &MetalArgument,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> Result<Option<ArgumentCondition>> {
match argument.condition.as_deref() {
Some(condition_text) => {
let field_name = format_ident!("has_{}", argument.name.as_ref());
- let rust_expression = rewrite_for_rust(enum_paths, condition_text).with_context(|| {
+ let rust_expression = rewrite_for_rust(gpu_type_paths, condition_text).with_context(|| {
format!("OPTIONAL condition `{}` cannot be parsed as a rust expression", condition_text)
})?;
Ok(Some(ArgumentCondition {
diff --git a/crates/uzu-engine/build/metal/bindgen/host_expression_rewriter.rs b/crates/uzu-engine/build/metal/bindgen/host_expression_rewriter.rs
index 582c20227..15fca0a52 100644
--- a/crates/uzu-engine/build/metal/bindgen/host_expression_rewriter.rs
+++ b/crates/uzu-engine/build/metal/bindgen/host_expression_rewriter.rs
@@ -6,13 +6,13 @@ use quote::{format_ident, quote};
use syn::Expr;
use crate::{
- common::{enum_paths::EnumPaths, expr_rewrite::rewrite_paths_with},
+ common::{expr_rewrite::rewrite_paths_with, gpu_type_paths::GpuTypePaths},
metal::{bindgen::variants::VariantBind, enum_path_rewrite::qualify_enum_path},
};
pub struct HostExpressionRewriter<'context> {
variants: &'context [VariantBind],
- enum_paths: &'context EnumPaths,
+ gpu_type_paths: &'context GpuTypePaths,
specialization_names: Vec<String>,
referenced_parameter_names: BTreeSet<String>,
kernel_name: &'context str,
@@ -21,13 +21,13 @@ pub struct HostExpressionRewriter<'context> {
impl<'context> HostExpressionRewriter<'context> {
pub fn new(
variants: &'context [VariantBind],
- enum_paths: &'context EnumPaths,
+ gpu_type_paths: &'context GpuTypePaths,
specialization_names: Vec<String>,
kernel_name: &'context str,
) -> Self {
Self {
variants,
- enum_paths,
+ gpu_type_paths,
specialization_names,
referenced_parameter_names: BTreeSet::new(),
kernel_name,
@@ -43,11 +43,11 @@ impl<'context> HostExpressionRewriter<'context> {
})?;
let variants = self.variants;
- let enum_paths = self.enum_paths;
+ let gpu_type_paths = self.gpu_type_paths;
let specialization_names = &self.specialization_names;
let referenced_parameter_names = &mut self.referenced_parameter_names;
rewrite_paths_with(&mut expression, |path| {
- if let Some(qualified) = qualify_enum_path(path, enum_paths) {
+ if let Some(qualified) = qualify_enum_path(path, gpu_type_paths) {
return Some(qualified);
}
diff --git a/crates/uzu-engine/build/metal/bindgen/mod.rs b/crates/uzu-engine/build/metal/bindgen/mod.rs
index 588a61f5d..ee9cc10d1 100644
--- a/crates/uzu-engine/build/metal/bindgen/mod.rs
+++ b/crates/uzu-engine/build/metal/bindgen/mod.rs
@@ -10,7 +10,7 @@ use proc_macro2::TokenStream;
use quote::{format_ident, quote};
use crate::{
- common::{enum_paths::EnumPaths, kernel::Kernel, mangling::dynamic_mangle},
+ common::{gpu_type_paths::GpuTypePaths, kernel::Kernel, mangling::dynamic_mangle},
metal::{
ast::MetalKernelInfo, bindgen::host_expression_rewriter::HostExpressionRewriter, wrapper::SpecializeBaseIndices,
},
@@ -19,7 +19,7 @@ use crate::{
pub fn bindgen(
kernel: &MetalKernelInfo,
specialize_indices: &SpecializeBaseIndices,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
library_const: &proc_macro2::Ident,
num_shards: usize,
library_compressed: bool,
@@ -28,12 +28,12 @@ pub fn bindgen(
let trait_name = format_ident!("{}Kernel", kernel_name);
let struct_name = format_ident!("{}MetalKernel", kernel_name);
- let variant_binds = variants::parse(kernel)?;
+ let variant_binds = variants::parse(kernel, gpu_type_paths)?;
let specialize_emission =
- specialize::parse(kernel, specialize_indices.get(&kernel.name).copied(), kernel_name, enum_paths)?;
+ specialize::parse(kernel, specialize_indices.get(&kernel.name).copied(), kernel_name, gpu_type_paths)?;
let mut host_expression_rewriter =
- HostExpressionRewriter::new(&variant_binds, enum_paths, specialize_emission.argument_names(), kernel_name);
- let argument_emissions = arguments::parse(kernel, enum_paths, &mut host_expression_rewriter)?;
+ HostExpressionRewriter::new(&variant_binds, gpu_type_paths, specialize_emission.argument_names(), kernel_name);
+ let argument_emissions = arguments::parse(kernel, gpu_type_paths, &mut host_expression_rewriter)?;
let trait_wiring = trait_wiring::build(kernel, &trait_name, &struct_name);
let dispatch_emission = dispatch::parse(kernel, &mut host_expression_rewriter)?;
diff --git a/crates/uzu-engine/build/metal/bindgen/specialize.rs b/crates/uzu-engine/build/metal/bindgen/specialize.rs
index dc3d3c937..dbb0dc356 100644
--- a/crates/uzu-engine/build/metal/bindgen/specialize.rs
+++ b/crates/uzu-engine/build/metal/bindgen/specialize.rs
@@ -9,7 +9,7 @@ use super::super::{
ast::{MetalArgumentType, MetalKernelInfo},
enum_path_rewrite::{gpu_type_kind_for_c_type, rewrite_for_rust},
};
-use crate::common::enum_paths::{EnumPaths, GpuTypeKind};
+use crate::common::gpu_type_paths::{GpuTypeKind, GpuTypePaths};
enum SpecializeLowering {
Direct,
@@ -39,7 +39,7 @@ pub fn parse(
kernel: &MetalKernelInfo,
base_function_constant_index: Option<usize>,
kernel_name: &str,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> Result<SpecializeEmission> {
let mut presence_checks = Vec::new();
let arguments = kernel
@@ -52,17 +52,18 @@ pub fn parse(
.enumerate()
.map(|(offset, (argument, rust_type_text))| -> Result<SpecializeArgument> {
let name = format_ident!("{}", argument.name.as_ref());
- let rust_type: Type = syn::parse_str(rust_type_text).with_context(|| {
+ let rust_type = gpu_type_paths.resolve_type(rust_type_text).with_context(|| {
format!("specialize type `{}` in kernel `{}` cannot be parsed", rust_type_text, kernel_name)
})?;
let function_constant_index = base_function_constant_index.unwrap_or(0) + offset;
- let lowering = match gpu_type_kind_for_c_type(enum_paths, &argument.c_type) {
+ let lowering = match gpu_type_kind_for_c_type(gpu_type_paths, &argument.c_type) {
Some(GpuTypeKind::Enum) => SpecializeLowering::Enum,
Some(GpuTypeKind::OptionSet) => SpecializeLowering::OptionSet,
+ Some(GpuTypeKind::Struct) => anyhow::bail!("cannot specialize struct argument `{name}` in kernel `{kernel_name}`"),
None => SpecializeLowering::Direct,
};
if let Some(condition) = argument.condition.as_deref() {
- let condition = rewrite_for_rust(enum_paths, condition)?;
+ let condition = rewrite_for_rust(gpu_type_paths, condition)?;
presence_checks.push(quote! {
assert_eq!(#name.is_some(), #condition, concat!("invalid presence for specialization ", stringify!(#name)));
});
diff --git a/crates/uzu-engine/build/metal/bindgen/variants.rs b/crates/uzu-engine/build/metal/bindgen/variants.rs
index aac8a0af0..535b98f68 100644
--- a/crates/uzu-engine/build/metal/bindgen/variants.rs
+++ b/crates/uzu-engine/build/metal/bindgen/variants.rs
@@ -6,6 +6,7 @@ use quote::{format_ident, quote};
use syn::{Ident, Type};
use super::super::ast::{MetalKernelInfo, MetalTemplateParameterType};
+use crate::common::gpu_type_paths::GpuTypePaths;
pub struct VariantBind {
pub parameter_name: Ident,
@@ -13,7 +14,10 @@ pub struct VariantBind {
parsed_type: Option<Type>,
}
-pub fn parse(kernel: &MetalKernelInfo) -> Result<Vec<VariantBind>> {
+pub fn parse(
+ kernel: &MetalKernelInfo,
+ gpu_type_paths: &GpuTypePaths,
+) -> Result<Vec<VariantBind>> {
kernel
.variants
.as_deref()
@@ -24,7 +28,7 @@ pub fn parse(kernel: &MetalKernelInfo) -> Result<Vec<VariantBind>> {
let field_name = format_ident!("{}", variant.name.as_ref().to_ascii_lowercase());
let parsed_type = match &variant.ty {
MetalTemplateParameterType::Type => None,
- MetalTemplateParameterType::Value(type_text) => Some(syn::parse_str::<Type>(type_text.as_ref())?),
+ MetalTemplateParameterType::Value(type_text) => Some(gpu_type_paths.resolve_type(type_text)?),
};
Ok(VariantBind {
parameter_name,
diff --git a/crates/uzu-engine/build/metal/compiler.rs b/crates/uzu-engine/build/metal/compiler.rs
index 85f17ef1d..002fc0744 100644
--- a/crates/uzu-engine/build/metal/compiler.rs
+++ b/crates/uzu-engine/build/metal/compiler.rs
@@ -20,7 +20,7 @@ use super::{
};
use crate::{
common::{
- caching, codegen::write_tokens, compiler::Compiler, enum_paths::EnumPaths, gpu_types::GpuTypes,
+ caching, codegen::write_tokens, compiler::Compiler, gpu_type_paths::GpuTypePaths, gpu_types::GpuTypes,
identifiers::KernelPath, kernel::Kernel,
},
debug_log,
@@ -144,7 +144,8 @@ impl MetalCompiler {
fn compile(
&self,
source_path: PathBuf,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
+ cache_key: &[u8; blake3::OUT_LEN],
) -> anyhow::Result<(KernelPath, Box<[Kernel]>, bool)> {
let source_path_relative =
source_path.strip_prefix(&self.source_directory).context("source is not in src_dir")?;
@@ -166,7 +167,7 @@ impl MetalCompiler {
if let Ok(cached_contents) = fs::read(&cached_file)
&& let Ok(cached) = serde_json::from_slice::<Cached>(&cached_contents)
- && cached.cache_key == self.cache_key
+ && cached.cache_key == *cache_key
&& cached.dependency_hashes.iter().all(|(path, hash)| {
fs::read(path.as_ref()).map(|contents| blake3::hash(&contents).as_bytes() == hash).unwrap_or(false)
})
@@ -205,7 +206,7 @@ impl MetalCompiler {
let mut num_shards = 0;
if !kernel_infos.is_empty() {
let (kernel_wrappers, specialize_indices) =
- wrappers(&kernel_infos, enum_paths).context("cannot generate kernel wrappers")?;
+ wrappers(&kernel_infos, gpu_type_paths).context("cannot generate kernel wrappers")?;
num_variants = kernel_wrappers.iter().map(|kernel| kernel.variants.len()).sum();
let footers = shard_footers(&kernel_wrappers);
@@ -267,7 +268,7 @@ impl MetalCompiler {
super::bindgen::bindgen(
kernel,
&specialize_indices,
- enum_paths,
+ gpu_type_paths,
&library_const,
num_shards,
self.metallib_compressed,
@@ -290,7 +291,7 @@ impl MetalCompiler {
let has_kernels = !kernel_infos.is_empty();
let cached = Cached {
- cache_key: self.cache_key,
+ cache_key: *cache_key,
dependency_hashes,
public_kernels: public_kernels.clone(),
has_kernels,
@@ -315,9 +316,23 @@ impl Compiler for MetalCompiler {
fn build(
&self,
gpu_types: &GpuTypes,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> anyhow::Result<HashMap<KernelPath, Box<[Kernel]>>> {
- gpu_type_gen(&self.gpu_types_directory, gpu_types).context("cannot generate shared gpu types")?;
+ let private_gpu_types = GpuTypes::scan("metal").context("cannot scan Metal-private gpu types")?;
+ let mut gpu_type_paths = gpu_type_paths.clone();
+ gpu_type_paths.extend(&private_gpu_types).context("cannot extend Metal GPU type paths")?;
+ gpu_type_gen(&self.gpu_types_directory, &[gpu_types, &private_gpu_types])
+ .context("cannot generate gpu types")?;
+
+ let cache_key = {
+ let mut hasher = blake3::Hasher::new();
+ hasher.update(&self.cache_key);
+ for path in gpu_type_paths.full_paths().sorted_unstable() {
+ hasher.update(path.as_bytes());
+ hasher.update(&[0]);
+ }
+ hasher.finalize().into()
+ };
let metal_sources: Vec<PathBuf> = WalkDir::new(&self.source_directory)
.into_iter()
@@ -329,7 +344,8 @@ impl Compiler for MetalCompiler {
let compiled: Vec<(KernelPath, Box<[Kernel]>, bool)> = metal_sources
.into_par_iter()
.map(|path| {
- self.compile(path.clone(), enum_paths).with_context(|| format!("cannot compile {}", path.display()))
+ self.compile(path.clone(), &gpu_type_paths, &cache_key)
+ .with_context(|| format!("cannot compile {}", path.display()))
})
.collect::<anyhow::Result<_>>()?;
diff --git a/crates/uzu-engine/build/metal/enum_path_rewrite.rs b/crates/uzu-engine/build/metal/enum_path_rewrite.rs
index c7c270bfb..2a708378a 100644
--- a/crates/uzu-engine/build/metal/enum_path_rewrite.rs
+++ b/crates/uzu-engine/build/metal/enum_path_rewrite.rs
@@ -4,50 +4,46 @@ use quote::ToTokens;
use syn::{Expr, Path, parse_quote};
use crate::common::{
- enum_paths::{EnumPaths, GpuTypeKind},
expr_rewrite::rewrite_paths_with,
+ gpu_type_paths::{GpuTypeKind, GpuTypePaths},
};
pub fn gpu_type_kind_for_c_type(
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
c_type: &str,
) -> Option<GpuTypeKind> {
let trimmed = c_type.trim_start_matches("const ").trim();
- let uzu_path = trimmed.strip_prefix("uzu::")?;
- let short_name = uzu_path.rsplit("::").next()?;
- let expected_rust_path = format!("crate::backends::common::gpu_types::{uzu_path}");
- if enum_paths.full_path_for(short_name) != Some(expected_rust_path.as_str()) {
- return None;
- }
- enum_paths.kind_for(short_name)
+ super::gpu_types::rust_path(gpu_type_paths, trimmed)?;
+ let short_name = trimmed.rsplit("::").next()?;
+ gpu_type_paths.kind_for(short_name)
}
pub fn is_enum_c_type(
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
c_type: &str,
) -> bool {
- gpu_type_kind_for_c_type(enum_paths, c_type).is_some()
+ matches!(gpu_type_kind_for_c_type(gpu_type_paths, c_type), Some(GpuTypeKind::Enum | GpuTypeKind::OptionSet))
}
pub fn rewrite_for_rust(
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
condition: &str,
) -> anyhow::Result<TokenStream> {
let mut parsed: Expr =
syn::parse_str(condition).with_context(|| format!("cannot parse rust expression `{condition}`"))?;
- rewrite_paths_with(&mut parsed, |path| qualify_enum_path(path, enum_paths));
+ rewrite_paths_with(&mut parsed, |path| qualify_enum_path(path, gpu_type_paths));
Ok(parsed.into_token_stream())
}
pub(super) fn qualify_enum_path(
path: &Path,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> Option<Expr> {
if path.segments.len() < 2 {
return None;
}
let head = path.segments.first()?;
- let full_path_text = enum_paths.full_path_for(&head.ident.to_string())?;
+ let full_path_text = gpu_type_paths.full_path_for(&head.ident.to_string())?;
let prefix: Path = syn::parse_str(full_path_text).ok()?;
let suffix: Vec<_> = path.segments.iter().skip(1).cloned().collect();
Some(parse_quote! { #prefix #( :: #suffix )* })
diff --git a/crates/uzu-engine/build/metal/gpu_types/mod.rs b/crates/uzu-engine/build/metal/gpu_types/mod.rs
index 2e1821686..bb6d5336e 100644
--- a/crates/uzu-engine/build/metal/gpu_types/mod.rs
+++ b/crates/uzu-engine/build/metal/gpu_types/mod.rs
@@ -1,9 +1,12 @@
-use std::{fs, path::Path};
+use std::{collections::BTreeMap, fs, path::Path};
use anyhow::Context;
use itertools::Itertools;
-use crate::common::gpu_types::{GpuType, GpuTypeFile, GpuTypes};
+use crate::common::{
+ gpu_type_paths::GpuTypePaths,
+ gpu_types::{GpuType, GpuTypes},
+};
mod gpu_type_constant;
mod gpu_type_enum;
@@ -15,6 +18,17 @@ use gpu_type_enum::gpu_type_gen_enum;
use gpu_type_option_set::gpu_type_gen_option_set;
use gpu_type_struct::gpu_type_gen_struct;
+pub fn rust_path<'a>(
+ gpu_type_paths: &'a GpuTypePaths,
+ c_type: &str,
+) -> Option<&'a str> {
+ let path = c_type.strip_prefix("uzu::")?;
+ let short_name = path.rsplit("::").next()?;
+ let rust_path = gpu_type_paths.full_path_for(short_name)?;
+ let (_, type_path) = rust_path.split_once("::gpu_types::")?;
+ (type_path == path).then_some(rust_path)
+}
+
fn rust_to_metal(ty: &str) -> anyhow::Result<&'static str> {
match ty {
"i8" => Ok("int8_t"),
@@ -33,11 +47,18 @@ fn rust_to_metal(ty: &str) -> anyhow::Result<&'static str> {
pub fn gpu_type_gen(
gpu_types_dir: &Path,
- gpu_types: &GpuTypes,
+ gpu_types: &[&GpuTypes],
) -> anyhow::Result<()> {
- for gpu_type_file in &gpu_types.files {
- gpu_type_gen_file(&gpu_types_dir.join(gpu_type_file.name.as_ref()).with_extension("h"), gpu_type_file)
- .with_context(|| format!("Cannot generate gpu types for {}", gpu_type_file.name.as_ref()))?;
+ fs::create_dir_all(gpu_types_dir).context("cannot create gpu types directory")?;
+ let mut files: BTreeMap<&str, Vec<&GpuType>> = BTreeMap::new();
+ for gpu_types in gpu_types {
+ for file in &gpu_types.files {
+ files.entry(&file.name).or_default().extend(file.types.iter());
+ }
+ }
+ for (module_name, types) in files {
+ gpu_type_gen_file(&gpu_types_dir.join(module_name).with_extension("h"), module_name, &types)
+ .with_context(|| format!("Cannot generate gpu types for {module_name}"))?;
}
Ok(())
@@ -45,12 +66,10 @@ pub fn gpu_type_gen(
fn gpu_type_gen_file(
file_path: &Path,
- gpu_types_file: &GpuTypeFile,
+ module_name: &str,
+ types: &[&GpuType],
) -> anyhow::Result<()> {
- let module_name = gpu_types_file.name.as_ref();
-
- let generated = gpu_types_file
- .types
+ let generated = types
.iter()
.map(|gpu_type| match gpu_type {
GpuType::Constant(gpu_type_constant) => gpu_type_gen_constant(gpu_type_constant)
diff --git a/crates/uzu-engine/build/metal/wrapper.rs b/crates/uzu-engine/build/metal/wrapper.rs
index 13ba9f409..63a31b878 100644
--- a/crates/uzu-engine/build/metal/wrapper.rs
+++ b/crates/uzu-engine/build/metal/wrapper.rs
@@ -9,7 +9,7 @@ use super::{
variant_combinations::constrained_combinations,
};
use crate::common::{
- constraints::Constraints, enum_paths::EnumPaths, identifiers::KernelName, mangling::static_mangle,
+ constraints::Constraints, gpu_type_paths::GpuTypePaths, identifiers::KernelName, mangling::static_mangle,
};
pub type SpecializeBaseIndices = HashMap<KernelName, usize>;
@@ -27,7 +27,7 @@ pub struct KernelWrappers {
pub fn wrappers(
kernels: &[MetalKernelInfo],
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> anyhow::Result<(Box<[KernelWrappers]>, SpecializeBaseIndices)> {
let mut all_wrappers = Vec::new();
let mut base_indices = SpecializeBaseIndices::new();
@@ -42,7 +42,7 @@ pub fn wrappers(
next_index += specialize_count;
}
- all_wrappers.push(kernel_wrappers(kernel, base_indices.get(&kernel.name), enum_paths)?);
+ all_wrappers.push(kernel_wrappers(kernel, base_indices.get(&kernel.name), gpu_type_paths)?);
}
Ok((all_wrappers.into_boxed_slice(), base_indices))
@@ -81,7 +81,7 @@ impl SpecializeBinding<'_> {
fn specialize_bindings<'a>(
kernel: &'a MetalKernelInfo,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> Vec<SpecializeBinding<'a>> {
kernel
.arguments
@@ -89,7 +89,7 @@ fn specialize_bindings<'a>(
.filter(|a| matches!(&a.argument_type, MetalArgumentType::Specialize(_)))
.map(|argument| {
let name_suffix = format!("{}_{}", kernel.name, argument.name);
- let enum_type = is_enum_c_type(enum_paths, &argument.c_type)
+ let enum_type = is_enum_c_type(gpu_type_paths, &argument.c_type)
.then(|| argument.c_type.trim_start_matches("const ").trim().to_string());
SpecializeBinding {
argument,
@@ -148,10 +148,10 @@ fn kernel_footer(bindings: &[SpecializeBinding<'_>]) -> String {
fn kernel_wrappers(
kernel: &MetalKernelInfo,
base_index: Option<&usize>,
- enum_paths: &EnumPaths,
+ gpu_type_paths: &GpuTypePaths,
) -> anyhow::Result<KernelWrappers> {
let mut variants = Vec::new();
- let bindings = specialize_bindings(kernel, enum_paths);
+ let bindings = specialize_bindings(kernel, gpu_type_paths);
let header = base_index.map(|&base| kernel_header(&bindings, base).into_boxed_str());
let parameters = kernel.variants.as_deref();
diff --git a/crates/uzu-engine/src/backends/common/capture_matcher.rs b/crates/uzu-engine/src/backends/common/capture_matcher.rs
index 2c3a91f74..b2d46ec3b 100644
--- a/crates/uzu-engine/src/backends/common/capture_matcher.rs
+++ b/crates/uzu-engine/src/backends/common/capture_matcher.rs
@@ -77,6 +77,7 @@ impl CaptureMatcher {
}
}
+ #[cfg(backend = "metal")]
pub fn enabled(&self) -> bool {
self.enabled
}
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs
index ddf160c8d..236b4a318 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs
@@ -1,13 +1,6 @@
use bitflags::bitflags;
use derive_more::Display;
-#[repr(C)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum GemmAPrologueKind {
- FullPrecision,
- Int8Symmetric,
-}
-
#[repr(C)]
#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
pub enum GemmBPrologueKind {
@@ -27,164 +20,3 @@ bitflags! {
const SOFT_CAP = 1 << 3;
}
}
-
-#[repr(C)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
-#[allow(non_camel_case_types)]
-pub enum GemmTiling {
- Tile8x32x32_Simdgroups1x1,
- Tile64x32x32_Simdgroups2x2,
- Tile64x64x16_Simdgroups2x2,
- Tile64x64x32_Simdgroups2x2,
- Tile32x32x32_Simdgroups2x2,
- Tile16x32x256_Simdgroups1x1,
- Tile16x32x256_Simdgroups1x1x4,
- Tile16x128x256_Simdgroups1x4,
- Tile32x64x256_Simdgroups2x2,
- Tile64x32x256_Simdgroups4x1,
- Tile64x64x256_Simdgroups2x2,
- Tile128x128x256_Simdgroups4x4,
-}
-
-pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;
-
-impl GemmTiling {
- pub const fn block_m(self) -> u32 {
- match self {
- Self::Tile8x32x32_Simdgroups1x1 => 8,
- Self::Tile64x32x32_Simdgroups2x2 => 64,
- Self::Tile64x64x16_Simdgroups2x2 => 64,
- Self::Tile64x64x32_Simdgroups2x2 => 64,
- Self::Tile32x32x32_Simdgroups2x2 => 32,
- Self::Tile16x32x256_Simdgroups1x1 => 16,
- Self::Tile16x32x256_Simdgroups1x1x4 => 16,
- Self::Tile16x128x256_Simdgroups1x4 => 16,
- Self::Tile32x64x256_Simdgroups2x2 => 32,
- Self::Tile64x32x256_Simdgroups4x1 => 64,
- Self::Tile64x64x256_Simdgroups2x2 => 64,
- Self::Tile128x128x256_Simdgroups4x4 => 128,
- }
- }
-
- pub const fn block_n(self) -> u32 {
- match self {
- Self::Tile8x32x32_Simdgroups1x1 => 32,
- Self::Tile64x32x32_Simdgroups2x2 => 32,
- Self::Tile64x64x16_Simdgroups2x2 => 64,
- Self::Tile64x64x32_Simdgroups2x2 => 64,
- Self::Tile32x32x32_Simdgroups2x2 => 32,
- Self::Tile16x32x256_Simdgroups1x1 => 32,
- Self::Tile16x32x256_Simdgroups1x1x4 => 32,
- Self::Tile16x128x256_Simdgroups1x4 => 128,
- Self::Tile32x64x256_Simdgroups2x2 => 64,
- Self::Tile64x32x256_Simdgroups4x1 => 32,
- Self::Tile64x64x256_Simdgroups2x2 => 64,
- Self::Tile128x128x256_Simdgroups4x4 => 128,
- }
- }
-
- pub const fn block_k(self) -> u32 {
- match self {
- Self::Tile8x32x32_Simdgroups1x1 => 32,
- Self::Tile64x32x32_Simdgroups2x2 => 32,
- Self::Tile64x64x16_Simdgroups2x2 => 16,
- Self::Tile64x64x32_Simdgroups2x2 => 32,
- Self::Tile32x32x32_Simdgroups2x2 => 32,
- Self::Tile16x32x256_Simdgroups1x1 => 256,
- Self::Tile16x32x256_Simdgroups1x1x4 => 256,
- Self::Tile16x128x256_Simdgroups1x4 => 256,
- Self::Tile32x64x256_Simdgroups2x2 => 256,
- Self::Tile64x32x256_Simdgroups4x1 => 256,
- Self::Tile64x64x256_Simdgroups2x2 => 256,
- Self::Tile128x128x256_Simdgroups4x4 => 256,
- }
- }
-
- pub const fn simdgroups_m(self) -> u32 {
- match self {
- Self::Tile8x32x32_Simdgroups1x1 => 1,
- Self::Tile64x32x32_Simdgroups2x2 => 2,
- Self::Tile64x64x16_Simdgroups2x2 => 2,
- Self::Tile64x64x32_Simdgroups2x2 => 2,
- Self::Tile32x32x32_Simdgroups2x2 => 2,
- Self::Tile16x32x256_Simdgroups1x1 => 1,
- Self::Tile16x32x256_Simdgroups1x1x4 => 1,
- Self::Tile16x128x256_Simdgroups1x4 => 1,
- Self::Tile32x64x256_Simdgroups2x2 => 2,
- Self::Tile64x32x256_Simdgroups4x1 => 4,
- Self::Tile64x64x256_Simdgroups2x2 => 2,
- Self::Tile128x128x256_Simdgroups4x4 => 4,
- }
- }
-
- pub const fn simdgroups_n(self) -> u32 {
- match self {
- Self::Tile8x32x32_Simdgroups1x1 => 1,
- Self::Tile64x32x32_Simdgroups2x2 => 2,
- Self::Tile64x64x16_Simdgroups2x2 => 2,
- Self::Tile64x64x32_Simdgroups2x2 => 2,
- Self::Tile32x32x32_Simdgroups2x2 => 2,
- Self::Tile16x32x256_Simdgroups1x1 => 1,
- Self::Tile16x32x256_Simdgroups1x1x4 => 1,
- Self::Tile16x128x256_Simdgroups1x4 => 4,
- Self::Tile32x64x256_Simdgroups2x2 => 2,
- Self::Tile64x32x256_Simdgroups4x1 => 1,
- Self::Tile64x64x256_Simdgroups2x2 => 2,
- Self::Tile128x128x256_Simdgroups4x4 => 4,
- }
- }
-
- pub const fn simdgroups_k(self) -> u32 {
- match self {
- Self::Tile16x32x256_Simdgroups1x1x4 => 4,
- _ => 1,
- }
- }
-
- pub const fn is_mxu_variant(self) -> bool {
- matches!(
- self,
- Self::Tile16x32x256_Simdgroups1x1
- | Self::Tile16x32x256_Simdgroups1x1x4
- | Self::Tile16x128x256_Simdgroups1x4
- | Self::Tile32x64x256_Simdgroups2x2
- | Self::Tile64x32x256_Simdgroups4x1
- | Self::Tile64x64x256_Simdgroups2x2
- | Self::Tile128x128x256_Simdgroups4x4
- )
- }
-}
-
-pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {
- t.simdgroups_m()
-}
-pub const fn gemm_tiling_simdgroups_per_column(t: GemmTiling) -> u32 {
- t.simdgroups_n()
-}
-pub const fn gemm_tiling_simdgroups_per_k(t: GemmTiling) -> u32 {
- t.simdgroups_k()
-}
-
-bitflags! {
- #[repr(transparent)]
- #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
- pub struct GemmAlignment: u32 {
- const M = 1 << 0;
- const N = 1 << 1;
- const K = 1 << 2;
- }
-}
-
-impl GemmAlignment {
- pub fn new(
- m: bool,
- n: bool,
- k: bool,
- ) -> Self {
- let mut bits = Self::empty();
- bits.set(Self::M, m);
- bits.set(Self::N, n);
- bits.set(Self::K, k);
- bits
- }
-}
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs b/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs
index 77ca8331d..ce4714f04 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs
@@ -1,22 +1 @@
pub const QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT: u32 = 4;
-
-#[repr(C)]
-#[allow(non_snake_case)]
-#[derive(Debug, Default, Copy, Clone)]
-pub struct GemmParams {
- pub M: u32,
- pub N: u32,
- pub K: u32,
- pub leading_dimension_a: u32,
- pub leading_dimension_b: u32,
- pub scale_output_stride: u32,
- pub scale_group_stride: u32,
- pub zero_point_output_stride: u32,
- pub zero_point_group_stride: u32,
- pub leading_dimension_d: u32,
- pub threadgroups_per_column: u32,
- pub threadgroups_per_row: u32,
- pub aligned_inner_iterations: u32,
- pub use_morton: bool,
- pub ab_scale: f32,
-}
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
index 628bd0dbe..91947e9a0 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
@@ -1,11 +1,11 @@
//! GPU types shared between Rust and shader languages (Metal, HLSL, Slang).
//!
-//! These `#[repr(C)]` structs are the source of truth. The build system uses
-//! cbindgen to generate C headers for Metal shaders.
+//! These definitions are the source of truth for the headers generated by the build system.
+
+#![allow(dead_code, reason = "Used by shaders.")]
pub mod activation_transform;
pub mod activation_type;
-pub mod attention;
pub mod d4s4;
pub mod embedding;
pub mod gated_act_mul;
@@ -19,14 +19,3 @@ pub mod ring;
pub mod trellis;
pub mod trie;
pub mod weaver;
-
-pub use activation_transform::*;
-pub use activation_type::*;
-pub use attention::*;
-pub use embedding::*;
-pub use gated_act_mul::*;
-pub use hadamard_order::*;
-pub use kv_cache_update::*;
-pub use matmul::*;
-pub use quantization::*;
-pub use quantization_method::*;
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/weaver.rs b/crates/uzu-engine/src/backends/common/gpu_types/weaver.rs
index 5e568b863..7906652ef 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/weaver.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/weaver.rs
@@ -1,6 +1,4 @@
pub const CANDIDATES_MAX: u32 = 512;
-pub const TOP_CHILDREN_THREADS: u32 = 256;
-pub const TOP_CHILDREN_SIMDGROUPS: u32 = 8;
#[repr(C)]
#[derive(Clone, Copy)]
@@ -46,9 +44,6 @@ impl MetadataIdx {
}
pub const FRONTIER_NO_WINNER: u32 = !0;
-pub const FRONTIER_SELECT_THREADS: u32 = 256;
-pub const FRONTIER_SELECT_SIMDGROUPS: u32 = 8;
-pub const FRONTIER_ENTRIES_PER_THREAD: u32 = 8;
pub const FRONTIER_MAX_SLOTS: u32 = 2048;
pub const FRONTIER_MAX_WIDTH: u32 = 32;
pub const PADDING_DEPTH: u32 = 0;
diff --git a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
index cf81fce78..8563d165c 100644
--- a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
@@ -1,7 +1,7 @@
use crate::{
backends::common::{
Backend, BufferMut, BufferRef, CommandBuffer, Kernels,
- gpu_types::{ActivationTransformOp, HADAMARD_TRANSFORM_BLOCK_SIZE},
+ gpu_types::{activation_transform::ActivationTransformOp, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},
kernel::{ActivationTransformKernel, matmul::Int8CodeLayout},
},
data_type::DataType,
diff --git a/crates/uzu-engine/src/backends/common/kernel/attention.rs b/crates/uzu-engine/src/backends/common/kernel/attention.rs
index 823389935..2f696060a 100644
--- a/crates/uzu-engine/src/backends/common/kernel/attention.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/attention.rs
@@ -1,3 +1,5 @@
+use std::error::Error;
+
use crate::{
backends::common::{Backend, BufferRef, CommandBuffer, Kernels},
data_type::DataType,
@@ -6,11 +8,12 @@ use crate::{
pub trait AttentionKernel: Sized + Send + Sync {
type Backend: Backend<Kernels: Kernels<AttentionKernel = Self>>;
+ type Error: Error + 'static;
fn new(
context: &<Self::Backend as Backend>::Context,
config: AttentionKernelConfig,
- ) -> Result<Self, <Self::Backend as Backend>::Error>;
+ ) -> Result<Self, Self::Error>;
fn encode(
&self,
@@ -24,7 +27,7 @@ pub trait AttentionKernel: Sized + Send + Sync {
impl BufferRef<Backend = Self::Backend>,
>,
command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<<Self::Backend as Backend>::ScratchBuffer, <Self::Backend as Backend>::Error>;
+ ) -> Result<<Self::Backend as Backend>::ScratchBuffer, Self::Error>;
}
#[derive(Clone, Copy)]
diff --git a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs
index 5a396f962..648c1ad58 100644
--- a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs
@@ -1,5 +1,7 @@
+#[cfg(backend = "cpu")]
+use crate::backends::common::kernel::Unsupported;
use crate::{
- backends::common::{Backend, CommandBuffer, Kernels, kernel::Unsupported},
+ backends::common::{Backend, CommandBuffer, Kernels},
data_type::DataType,
};
@@ -38,6 +40,7 @@ pub trait DeltaNetChunkedPrefill: Sized + Send + Sync {
) -> Result<(), <Self::Backend as Backend>::Error>;
}
+#[cfg(backend = "cpu")]
impl<B: Backend<Kernels: Kernels<DeltaNetChunkedPrefill = Unsupported<B>>>> DeltaNetChunkedPrefill for Unsupported<B> {
type Backend = B;
diff --git a/crates/uzu-engine/src/backends/common/kernel/delta_net_tree_verify.rs b/crates/uzu-engine/src/backends/common/kernel/delta_net_tree_verify.rs
index c4113d1d3..25c659881 100644
--- a/crates/uzu-engine/src/backends/common/kernel/delta_net_tree_verify.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/delta_net_tree_verify.rs
@@ -1,5 +1,7 @@
+#[cfg(backend = "cpu")]
+use crate::backends::common::kernel::Unsupported;
use crate::{
- backends::common::{Backend, CommandBuffer, Kernels, kernel::Unsupported},
+ backends::common::{Backend, CommandBuffer, Kernels},
encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},
};
@@ -20,6 +22,7 @@ pub trait DeltaNetTreeVerify: Sized + Send + Sync {
) -> Result<<Self::Backend as Backend>::ScratchBuffer, <Self::Backend as Backend>::Error>;
}
+#[cfg(backend = "cpu")]
impl<B: Backend<Kernels: Kernels<DeltaNetTreeVerify = Unsupported<B>>>> DeltaNetTreeVerify for Unsupported<B> {
type Backend = B;
diff --git a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs
index 170f60a48..77dc2bd4d 100644
--- a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs
@@ -3,7 +3,10 @@ use bitflags::bitflags;
use crate::{
backends::common::{
Backend, BufferMut, BufferRef, CommandBuffer, Kernels,
- gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE},
+ gpu_types::{
+ activation_type::ActivationType, gated_act_mul::GatedActMulOp,
+ hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ },
kernel::{ActivationQuantization, GatedActMulKernel},
},
config::clipping::ClippingBounds,
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs
deleted file mode 100644
index 24d59b226..000000000
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-use derive_more::Debug;
-use thiserror::Error;
-
-use crate::{
- backends::common::{Backend, gpu_types::gemm::GemmDTransform},
- data_type::DataType,
-};
-
-#[derive(Debug, Error)]
-pub enum MatmulError<B: Backend> {
- #[error("Unsupported data type: {0:?}")]
- UnsupportedDataType(DataType),
- #[error("Unsupported group size: {0}")]
- UnsupportedGroupSize(u32),
- #[error("Unsupported D-transform op {bit:?} on path {path}")]
- UnsupportedDOp {
- bit: GemmDTransform,
- path: &'static str,
- },
- #[error("Unsupported layout on path {path}")]
- UnsupportedLayout {
- path: &'static str,
- },
- #[error("Unsupported output row stride {stride} for {columns} columns on path {path}")]
- UnsupportedOutputStride {
- path: &'static str,
- stride: u32,
- columns: u32,
- },
- #[error("Incompatible A operand for {path}: {reason}")]
- IncompatibleA {
- path: &'static str,
- reason: &'static str,
- },
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
-}
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs
index 217cb7ca3..f0ba1dd1e 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs
@@ -1,3 +1,5 @@
+use std::error::Error;
+
use crate::{
backends::common::{
Backend, BufferMut, BufferRef, CommandBuffer, Kernels,
@@ -14,13 +16,14 @@ use crate::{
pub trait MatmulKernel: Sized + Send + Sync {
type Backend: Backend<Kernels: Kernels<MatmulKernel = Self>>;
+ type Error: Error + 'static;
fn new(
context: &<Self::Backend as Backend>::Context,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
- ) -> Result<Self, <Self::Backend as Backend>::Error>;
+ ) -> Result<Self, Self::Error>;
fn encode(
&mut self,
@@ -33,7 +36,7 @@ pub trait MatmulKernel: Sized + Send + Sync {
impl BufferRef<Backend = Self::Backend>,
>,
command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), <Self::Backend as Backend>::Error>;
+ ) -> Result<(), Self::Error>;
fn select_activation_quantization(
&self,
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
index 8cb287064..ca115c9b1 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
@@ -2,7 +2,7 @@ use super::{QuantParams, QuantParamsLayout};
use crate::{
backends::common::{
BufferRef,
- gpu_types::{QuantizationMode, gemm::GemmBPrologueKind},
+ gpu_types::{gemm::GemmBPrologueKind, quantization::QuantizationMode},
},
data_type::DataType,
};
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
index af2eef3ba..314c0ef72 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
@@ -1,6 +1,5 @@
mod arguments;
mod d_ops;
-mod error;
mod kernel;
mod matmul_a;
mod matmul_b;
@@ -9,14 +8,15 @@ pub mod routing;
pub use arguments::MatmulArguments;
pub use d_ops::MatmulDOps;
-pub use error::MatmulError;
pub use kernel::MatmulKernel;
pub use matmul_a::{Int8CodeLayout, MatmulA};
-pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection, TrellisFormat};
-pub use output::MatmulOutput;
+#[cfg(backend = "metal")]
+pub use matmul_b::TrellisFormat;
+pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};
+pub use output::{MatmulOutput, MatmulOutputError};
pub use routing::{ActivationFormat, MatmulShape};
-use crate::backends::common::gpu_types::{QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT, QuantizationMode};
+use crate::backends::common::gpu_types::{matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT, quantization::QuantizationMode};
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum QuantParamsLayout {
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs
index c7cb76391..8a73f67e1 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs
@@ -1,5 +1,14 @@
-use super::{MatmulDOps, MatmulError};
-use crate::backends::common::{Backend, BufferMut};
+use thiserror::Error;
+
+use crate::backends::common::{Backend, BufferMut, kernel::matmul::MatmulDOps};
+
+#[derive(Debug, Error)]
+#[error("Unsupported output row stride {stride} for {columns} columns on path {path}")]
+pub struct MatmulOutputError {
+ path: &'static str,
+ stride: u32,
+ columns: u32,
+}
pub struct MatmulOutput<'d, B: Backend, T: BufferMut<Backend = B>> {
pub values: T,
@@ -24,10 +33,10 @@ impl<'d, B: Backend, T: BufferMut<Backend = B>> MatmulOutput<'d, B, T> {
self,
columns: u32,
path: &'static str,
- ) -> Result<(T, MatmulDOps<'d, B>), MatmulError<B>> {
+ ) -> Result<(T, MatmulDOps<'d, B>), MatmulOutputError> {
let stride = self.row_stride.unwrap_or(columns);
if stride != columns {
- return Err(MatmulError::UnsupportedOutputStride {
+ return Err(MatmulOutputError {
path,
stride,
columns,
diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rs
index bc85140b3..0e1e89e05 100644
--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs
@@ -1,3 +1,4 @@
+#[cfg(backend = "cpu")]
use std::{convert::Infallible, marker::PhantomData};
use crate::backends::common::Backend;
@@ -9,12 +10,15 @@ pub mod delta_net_tree_verify;
pub mod gated_act_mul;
pub mod matmul;
pub mod radix_top_k_small;
+#[cfg(any(backend = "cpu", test))]
mod trellis;
pub use activation_transform::{ActivationQuantization, ActivationTransform};
pub use attention::{AttentionArguments, AttentionKernel, AttentionKernelConfig};
pub use gated_act_mul::{GatedActMul, GatedActMulSettings};
-pub use trellis::mixing_dimension;
+
+#[cfg(any(backend = "cpu", test))]
+pub use crate::backends::common::kernel::trellis::mixing_dimension;
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
@@ -29,6 +33,7 @@ pub trait Kernels: Sized {
type RadixTopKSmall: radix_top_k_small::RadixTopKSmall<Backend = Self::Backend>;
}
+#[cfg(backend = "cpu")]
pub struct Unsupported<B: Backend> {
never: Infallible,
_marker: PhantomData<fn() -> B>,
diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rs
index 59a6e5e67..0b3094e9d 100644
--- a/crates/uzu-engine/src/backends/common/mod.rs
+++ b/crates/uzu-engine/src/backends/common/mod.rs
@@ -14,7 +14,7 @@ pub use buffer::{
constant::ConstantBuffer,
cpu_accessible::BufferCpuAccessible,
global::GlobalBuffer,
- reference::{BufferMut, BufferRef, Subbuffer},
+ reference::{BufferMut, BufferRef},
scratch::ScratchBuffer,
sparse::SparseBuffer,
};
diff --git a/crates/uzu-engine/src/backends/cpu/error.rs b/crates/uzu-engine/src/backends/cpu/error.rs
index 581d30aa0..f13385a4b 100644
--- a/crates/uzu-engine/src/backends/cpu/error.rs
+++ b/crates/uzu-engine/src/backends/cpu/error.rs
@@ -1,24 +1,11 @@
-use std::{error::Error as StdError, sync::mpsc::RecvError};
+use std::sync::mpsc::RecvError;
use thiserror::Error;
-use crate::backends::{common::kernel::matmul::MatmulError, cpu::Cpu};
-
#[derive(Debug, Error)]
pub enum CpuError {
#[error("Not supported")]
NotSupported,
#[error("Command buffer execution failed")]
CommandBufferExecutionFailed(#[from] RecvError),
- #[error("Kernel dispatch failed: {0}")]
- KernelDispatchFailed(#[source] Box<dyn StdError + Send + Sync + 'static>),
-}
-
-impl From<MatmulError<Cpu>> for CpuError {
- fn from(value: MatmulError<Cpu>) -> Self {
- match value {
- MatmulError::BackendError(e) => e,
- other => CpuError::KernelDispatchFailed(Box::new(other)),
- }
- }
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
index 1568d5edd..aecabd67a 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
@@ -6,7 +6,7 @@ use super::{hadamard_transform, min_max_symmetric_divisor, quantize_symmetric_i8
use crate::{
array::ArrayElement,
backends::common::{
- gpu_types::{ActivationTransformOp, HADAMARD_TRANSFORM_BLOCK_SIZE},
+ gpu_types::{activation_transform::ActivationTransformOp, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},
kernel::matmul::Int8CodeLayout,
},
};
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
index 0c0f28688..e357123a1 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
@@ -25,6 +25,7 @@ pub struct AttentionCpuKernel {
impl AttentionKernel for AttentionCpuKernel {
type Backend = Cpu;
+ type Error = CpuError;
fn new(
_context: &CpuContext,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs
index 73a32ca4f..91ea779f4 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs
@@ -6,7 +6,8 @@ use crate::{
array::ArrayElement,
backends::{
common::gpu_types::{
- EmbeddingTableKind, HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod, QuantizationMode, d4s4,
+ d4s4, embedding::EmbeddingTableKind, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ quantization::QuantizationMode, quantization_method::QuantizationMethod,
},
cpu::kernel::activation_transform::hadamard_transform,
},
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
index a80a44e7d..72ea5555f 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
@@ -6,7 +6,11 @@ use crate::{
array::ArrayElement,
backends::{
common::{
- gpu_types::{ActivationType, GatedActMulOp, HADAMARD_TRANSFORM_BLOCK_SIZE, activation_silu_alpha},
+ gpu_types::{
+ activation_type::{ActivationType, activation_silu_alpha},
+ gated_act_mul::GatedActMulOp,
+ hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ },
kernel::matmul::Int8CodeLayout,
},
cpu::kernel::activation_transform::{hadamard_transform, quantize_transformed_row},
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs
index e45782663..20fb61d10 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs
@@ -2,7 +2,7 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
// Multi-token causal conv1d with SiLU for DeltaNet.
// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd values
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs
index 8f5b56711..607b72505 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs
@@ -2,7 +2,7 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
// Single-token causal conv1d with SiLU, in-place.
#[kernel(DeltaNetConvUpdate)]
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs
index 74bd19683..ad61a8a20 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs
@@ -2,7 +2,7 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
// In-place RMSNorm + SiLU gate for DeltaNet prefill output.
//
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs
index 3295349b6..28f6b3df7 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs
@@ -2,7 +2,7 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
#[kernel(ConvTreeScan)]
#[variants(T, f32, bf16)]
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs
index 1238972e7..6c5ac3f3e 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs
@@ -2,7 +2,7 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
// Single-token delta net update: decay state, apply delta rule, RMSNorm + SiLU gate.
// Steps per v-head:
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/error.rs
new file mode 100644
index 000000000..fe26dc04d
--- /dev/null
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/error.rs
@@ -0,0 +1,25 @@
+use thiserror::Error;
+
+use crate::{
+ backends::{common::kernel::matmul::MatmulOutputError, cpu::error::CpuError},
+ data_type::DataType,
+};
+
+#[derive(Debug, Error)]
+pub enum MatmulCpuError {
+ #[error("Unsupported data type: {0:?}")]
+ UnsupportedDataType(DataType),
+ #[error("Unsupported layout on path {path}")]
+ UnsupportedLayout {
+ path: &'static str,
+ },
+ #[error("Incompatible A operand for {path}: {reason}")]
+ IncompatibleA {
+ path: &'static str,
+ reason: &'static str,
+ },
+ #[error(transparent)]
+ Output(#[from] MatmulOutputError),
+ #[error("Backend error: {0}")]
+ BackendError(#[from] CpuError),
+}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
index 08d5c623d..8a91010d9 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
@@ -1,16 +1,22 @@
-use super::reference::{WeightData, read_f32, write_f32};
use crate::{
backends::{
common::{
Backend, BufferCpuAccessible, BufferMut, BufferRef, Kernels,
- gpu_types::QuantizationMode,
+ gpu_types::quantization::QuantizationMode,
kernel::{
ActivationTransform, TensorAddBiasKernel,
- matmul::{Int8CodeLayout, MatmulA, MatmulArguments, MatmulError, MatmulKernel},
+ matmul::{Int8CodeLayout, MatmulA, MatmulArguments, MatmulKernel},
},
},
cpu::{
- Cpu, buffer::CpuBufferExt, command_buffer::CpuCommandBufferEncoding, context::CpuContext, error::CpuError,
+ Cpu,
+ buffer::CpuBufferExt,
+ command_buffer::CpuCommandBufferEncoding,
+ context::CpuContext,
+ kernel::matmul::{
+ MatmulCpuError,
+ reference::{WeightData, read_f32, write_f32},
+ },
},
},
data_type::DataType,
@@ -27,16 +33,17 @@ pub struct MatmulCpuKernel {
impl MatmulKernel for MatmulCpuKernel {
type Backend = Cpu;
+ type Error = MatmulCpuError;
fn new(
context: &CpuContext,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
- ) -> Result<Self, CpuError> {
+ ) -> Result<Self, Self::Error> {
for data_type in [weights_data_type, input_data_type, output_data_type] {
if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());
+ return Err(MatmulCpuError::UnsupportedDataType(data_type));
}
}
let output_rht = ActivationTransform::output_rht(context, output_data_type, None, true)?;
@@ -66,7 +73,7 @@ impl MatmulKernel for MatmulCpuKernel {
impl BufferRef<Backend = Cpu>,
>,
command_buffer: &mut CpuCommandBufferEncoding,
- ) -> Result<(), CpuError> {
+ ) -> Result<(), Self::Error> {
let MatmulArguments {
a,
b,
@@ -126,11 +133,10 @@ impl MatmulKernel for MatmulCpuKernel {
&& b.quantized()
.is_some_and(|quantized| matches!(quantized.mode, QuantizationMode::U4 | QuantizationMode::U8));
if !compatible {
- return Err(MatmulError::IncompatibleA {
+ return Err(MatmulCpuError::IncompatibleA {
path: "CpuMatmul",
reason: "symmetric int8 activations require a supported 32/64/128 activation and weight group",
- }
- .into());
+ });
}
let (values, values_range) = values.parts();
let (scales, scales_range) = scales.parts();
@@ -148,10 +154,9 @@ impl MatmulKernel for MatmulCpuKernel {
MatmulA::Trellis {
..
} => {
- return Err(MatmulError::UnsupportedLayout {
+ return Err(MatmulCpuError::UnsupportedLayout {
path: "CpuMatmul",
- }
- .into());
+ });
},
};
let bias_ptr = bias_buffer.map(|bias| SendPtr(bias.cpu_ptr().as_ptr().cast::<u8>().cast_const()));
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/mod.rs
index aa648840e..c5323acc2 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/mod.rs
@@ -1,4 +1,6 @@
+mod error;
mod kernel;
mod reference;
+pub use error::MatmulCpuError;
pub use kernel::MatmulCpuKernel;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
index 9ca424454..0b5788925 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
@@ -4,9 +4,9 @@ use crate::{
backends::{
common::{
BufferRef,
- kernel::matmul::{MatmulB, MatmulError, QuantParamsStrides},
+ kernel::matmul::{MatmulB, QuantParamsStrides},
},
- cpu::{Cpu, buffer::CpuBufferExt},
+ cpu::{Cpu, buffer::CpuBufferExt, kernel::matmul::MatmulCpuError},
},
data_type::DataType,
utils::pointers::SendPtr,
@@ -37,7 +37,7 @@ impl WeightData {
b_transpose: bool,
k: usize,
n: usize,
- ) -> Result<Self, MatmulError<Cpu>> {
+ ) -> Result<Self, MatmulCpuError> {
fn buffer_ptr(view: impl BufferRef<Backend = Cpu>) -> SendPtr<u8> {
let (buffer, range) = view.parts();
SendPtr(buffer.cpu_address().as_ptr().cast::<u8>().cast_const().wrapping_byte_add(range.start))
@@ -59,7 +59,7 @@ impl WeightData {
},
MatmulB::Trellis {
..
- } => Err(MatmulError::UnsupportedLayout {
+ } => Err(MatmulCpuError::UnsupportedLayout {
path: "CpuMatmul",
}),
MatmulB::Quantized(quantized) => Ok(WeightData::Quantized {
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
index 16399ee54..f87bb72ac 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
@@ -5,7 +5,8 @@ use uzu_engine_macros::kernel;
use crate::{
array::ArrayElement,
backends::{
- common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE, cpu::kernel::activation_transform::hadamard_transform,
+ common::gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ cpu::kernel::activation_transform::hadamard_transform,
},
};
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs
index 1b151e3cd..290953a07 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs
@@ -2,7 +2,7 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
+use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
#[kernel(SSDUpdate)]
#[variants(T, f32, f16, bf16)]
diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rs
index e7800cd75..d4aeedf50 100644
--- a/crates/uzu-engine/src/backends/metal/error.rs
+++ b/crates/uzu-engine/src/backends/metal/error.rs
@@ -1,12 +1,7 @@
-use std::{error::Error as StdError, sync::mpsc::RecvError};
+use std::sync::mpsc::RecvError;
use thiserror::Error;
-use crate::backends::{
- common::kernel::matmul::MatmulError,
- metal::{Metal, kernel::matmul::gemm::GemmSpecializationError},
-};
-
#[derive(Debug, Error)]
pub enum MetalError {
#[error("Cannot open device")]
@@ -40,21 +35,4 @@ pub enum MetalError {
function_name: String,
error: String,
},
- #[error("Kernel dispatch failed: {0}")]
- KernelDispatchFailed(#[source] Box<dyn StdError + Send + Sync + 'static>),
-}
-
-impl From<MatmulError<Metal>> for MetalError {
- fn from(value: MatmulError<Metal>) -> Self {
- match value {
- MatmulError::BackendError(e) => e,
- other => MetalError::KernelDispatchFailed(Box::new(other)),
- }
- }
-}
-
-impl From<GemmSpecializationError> for MetalError {
- fn from(value: GemmSpecializationError) -> Self {
- MetalError::KernelDispatchFailed(Box::new(value))
- }
}
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/attention.rs b/crates/uzu-engine/src/backends/metal/gpu_types/attention.rs
similarity index 100%
rename from crates/uzu-engine/src/backends/common/gpu_types/attention.rs
rename to crates/uzu-engine/src/backends/metal/gpu_types/attention.rs
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
new file mode 100644
index 000000000..4a2a7e83a
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
@@ -0,0 +1,170 @@
+use bitflags::bitflags;
+use derive_more::Display;
+
+#[repr(C)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum GemmAPrologueKind {
+ FullPrecision,
+ Int8Symmetric,
+}
+
+#[repr(C)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
+#[allow(non_camel_case_types)]
+pub enum GemmTiling {
+ Tile8x32x32_Simdgroups1x1,
+ Tile64x32x32_Simdgroups2x2,
+ Tile64x64x16_Simdgroups2x2,
+ Tile64x64x32_Simdgroups2x2,
+ Tile32x32x32_Simdgroups2x2,
+ Tile16x32x256_Simdgroups1x1,
+ Tile16x32x256_Simdgroups1x1x4,
+ Tile16x128x256_Simdgroups1x4,
+ Tile32x64x256_Simdgroups2x2,
+ Tile64x32x256_Simdgroups4x1,
+ Tile64x64x256_Simdgroups2x2,
+ Tile128x128x256_Simdgroups4x4,
+}
+
+pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;
+
+impl GemmTiling {
+ pub const fn block_m(self) -> u32 {
+ match self {
+ Self::Tile8x32x32_Simdgroups1x1 => 8,
+ Self::Tile64x32x32_Simdgroups2x2 => 64,
+ Self::Tile64x64x16_Simdgroups2x2 => 64,
+ Self::Tile64x64x32_Simdgroups2x2 => 64,
+ Self::Tile32x32x32_Simdgroups2x2 => 32,
+ Self::Tile16x32x256_Simdgroups1x1 => 16,
+ Self::Tile16x32x256_Simdgroups1x1x4 => 16,
+ Self::Tile16x128x256_Simdgroups1x4 => 16,
+ Self::Tile32x64x256_Simdgroups2x2 => 32,
+ Self::Tile64x32x256_Simdgroups4x1 => 64,
+ Self::Tile64x64x256_Simdgroups2x2 => 64,
+ Self::Tile128x128x256_Simdgroups4x4 => 128,
+ }
+ }
+
+ pub const fn block_n(self) -> u32 {
+ match self {
+ Self::Tile8x32x32_Simdgroups1x1 => 32,
+ Self::Tile64x32x32_Simdgroups2x2 => 32,
+ Self::Tile64x64x16_Simdgroups2x2 => 64,
+ Self::Tile64x64x32_Simdgroups2x2 => 64,
+ Self::Tile32x32x32_Simdgroups2x2 => 32,
+ Self::Tile16x32x256_Simdgroups1x1 => 32,
+ Self::Tile16x32x256_Simdgroups1x1x4 => 32,
+ Self::Tile16x128x256_Simdgroups1x4 => 128,
+ Self::Tile32x64x256_Simdgroups2x2 => 64,
+ Self::Tile64x32x256_Simdgroups4x1 => 32,
+ Self::Tile64x64x256_Simdgroups2x2 => 64,
+ Self::Tile128x128x256_Simdgroups4x4 => 128,
+ }
+ }
+
+ pub const fn block_k(self) -> u32 {
+ match self {
+ Self::Tile8x32x32_Simdgroups1x1 => 32,
+ Self::Tile64x32x32_Simdgroups2x2 => 32,
+ Self::Tile64x64x16_Simdgroups2x2 => 16,
+ Self::Tile64x64x32_Simdgroups2x2 => 32,
+ Self::Tile32x32x32_Simdgroups2x2 => 32,
+ Self::Tile16x32x256_Simdgroups1x1 => 256,
+ Self::Tile16x32x256_Simdgroups1x1x4 => 256,
+ Self::Tile16x128x256_Simdgroups1x4 => 256,
+ Self::Tile32x64x256_Simdgroups2x2 => 256,
+ Self::Tile64x32x256_Simdgroups4x1 => 256,
+ Self::Tile64x64x256_Simdgroups2x2 => 256,
+ Self::Tile128x128x256_Simdgroups4x4 => 256,
+ }
+ }
+
+ pub const fn simdgroups_m(self) -> u32 {
+ match self {
+ Self::Tile8x32x32_Simdgroups1x1 => 1,
+ Self::Tile64x32x32_Simdgroups2x2 => 2,
+ Self::Tile64x64x16_Simdgroups2x2 => 2,
+ Self::Tile64x64x32_Simdgroups2x2 => 2,
+ Self::Tile32x32x32_Simdgroups2x2 => 2,
+ Self::Tile16x32x256_Simdgroups1x1 => 1,
+ Self::Tile16x32x256_Simdgroups1x1x4 => 1,
+ Self::Tile16x128x256_Simdgroups1x4 => 1,
+ Self::Tile32x64x256_Simdgroups2x2 => 2,
+ Self::Tile64x32x256_Simdgroups4x1 => 4,
+ Self::Tile64x64x256_Simdgroups2x2 => 2,
+ Self::Tile128x128x256_Simdgroups4x4 => 4,
+ }
+ }
+
+ pub const fn simdgroups_n(self) -> u32 {
+ match self {
+ Self::Tile8x32x32_Simdgroups1x1 => 1,
+ Self::Tile64x32x32_Simdgroups2x2 => 2,
+ Self::Tile64x64x16_Simdgroups2x2 => 2,
+ Self::Tile64x64x32_Simdgroups2x2 => 2,
+ Self::Tile32x32x32_Simdgroups2x2 => 2,
+ Self::Tile16x32x256_Simdgroups1x1 => 1,
+ Self::Tile16x32x256_Simdgroups1x1x4 => 1,
+ Self::Tile16x128x256_Simdgroups1x4 => 4,
+ Self::Tile32x64x256_Simdgroups2x2 => 2,
+ Self::Tile64x32x256_Simdgroups4x1 => 1,
+ Self::Tile64x64x256_Simdgroups2x2 => 2,
+ Self::Tile128x128x256_Simdgroups4x4 => 4,
+ }
+ }
+
+ pub const fn simdgroups_k(self) -> u32 {
+ match self {
+ Self::Tile16x32x256_Simdgroups1x1x4 => 4,
+ _ => 1,
+ }
+ }
+
+ pub const fn is_mxu_variant(self) -> bool {
+ matches!(
+ self,
+ Self::Tile16x32x256_Simdgroups1x1
+ | Self::Tile16x32x256_Simdgroups1x1x4
+ | Self::Tile16x128x256_Simdgroups1x4
+ | Self::Tile32x64x256_Simdgroups2x2
+ | Self::Tile64x32x256_Simdgroups4x1
+ | Self::Tile64x64x256_Simdgroups2x2
+ | Self::Tile128x128x256_Simdgroups4x4
+ )
+ }
+}
+
+pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {
+ t.simdgroups_m()
+}
+pub const fn gemm_tiling_simdgroups_per_column(t: GemmTiling) -> u32 {
+ t.simdgroups_n()
+}
+pub const fn gemm_tiling_simdgroups_per_k(t: GemmTiling) -> u32 {
+ t.simdgroups_k()
+}
+
+bitflags! {
+ #[repr(transparent)]
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+ pub struct GemmAlignment: u32 {
+ const M = 1 << 0;
+ const N = 1 << 1;
+ const K = 1 << 2;
+ }
+}
+
+impl GemmAlignment {
+ pub fn new(
+ m: bool,
+ n: bool,
+ k: bool,
+ ) -> Self {
+ let mut bits = Self::empty();
+ bits.set(Self::M, m);
+ bits.set(Self::N, n);
+ bits.set(Self::K, k);
+ bits
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs b/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs
new file mode 100644
index 000000000..e29eea201
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs
@@ -0,0 +1,20 @@
+#[repr(C)]
+#[allow(non_snake_case)]
+#[derive(Debug, Default, Copy, Clone)]
+pub struct GemmParams {
+ pub M: u32,
+ pub N: u32,
+ pub K: u32,
+ pub leading_dimension_a: u32,
+ pub leading_dimension_b: u32,
+ pub scale_output_stride: u32,
+ pub scale_group_stride: u32,
+ pub zero_point_output_stride: u32,
+ pub zero_point_group_stride: u32,
+ pub leading_dimension_d: u32,
+ pub threadgroups_per_column: u32,
+ pub threadgroups_per_row: u32,
+ pub aligned_inner_iterations: u32,
+ pub use_morton: bool,
+ pub ab_scale: f32,
+}
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/mod.rs b/crates/uzu-engine/src/backends/metal/gpu_types/mod.rs
new file mode 100644
index 000000000..4f0947a4e
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/gpu_types/mod.rs
@@ -0,0 +1,6 @@
+#![allow(dead_code, reason = "Used by Metal shaders.")]
+
+pub mod attention;
+pub mod gemm;
+pub mod matmul;
+pub mod weaver;
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/weaver.rs b/crates/uzu-engine/src/backends/metal/gpu_types/weaver.rs
new file mode 100644
index 000000000..ef6dcda50
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/gpu_types/weaver.rs
@@ -0,0 +1,5 @@
+pub const TOP_CHILDREN_THREADS: u32 = 256;
+pub const TOP_CHILDREN_SIMDGROUPS: u32 = 8;
+pub const FRONTIER_SELECT_THREADS: u32 = 256;
+pub const FRONTIER_SELECT_SIMDGROUPS: u32 = 8;
+pub const FRONTIER_ENTRIES_PER_THREAD: u32 = 8;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs
new file mode 100644
index 000000000..f31b769a4
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs
@@ -0,0 +1,11 @@
+use thiserror::Error;
+
+use crate::backends::metal::{error::MetalError, kernel::matmul::MatmulMetalError};
+
+#[derive(Debug, Error)]
+pub enum AttentionMetalError {
+ #[error("Backend error: {0}")]
+ Backend(#[from] MetalError),
+ #[error("Matmul error: {0}")]
+ Matmul(#[from] MatmulMetalError),
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
index 114940f4a..b7bd69ab4 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
@@ -13,9 +13,9 @@ use crate::{
Metal,
command_buffer::MetalCommandBufferEncoding,
context::MetalContext,
- error::MetalError,
kernel::{
AttentionFallbackScatterScoresMetalKernel, AttentionFallbackScatterValuesMetalKernel, MetalKernels,
+ attention::AttentionMetalError,
},
},
},
@@ -45,7 +45,7 @@ impl AttentionFallback {
pub fn new(
config: &AttentionKernelConfig,
context: &MetalContext,
- ) -> Result<Self, MetalError> {
+ ) -> Result<Self, AttentionMetalError> {
let scatter_scores = AttentionFallbackScatterScoresMetalKernel::new(
context,
config.data_type,
@@ -92,7 +92,7 @@ impl AttentionFallback {
impl BufferRef<Backend = Metal>,
>,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
+ ) -> Result<<Metal as Backend>::ScratchBuffer, AttentionMetalError> {
assert!(arguments.trie.is_none(), "fallback does not support trie");
let suffix_length = arguments.suffix_length;
let sequence_length = arguments.cache.prefix_len() + suffix_length;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
index 6c4ea891f..78c035fb2 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
@@ -6,12 +6,11 @@ use crate::{
backends::{
common::{
Backend, BufferRef, CommandBufferEncoding,
- gpu_types::AttnParams,
kernel::{AttentionArguments, AttentionKernelConfig},
},
metal::{
Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError,
- kernel::AttentionGemmMetalKernel,
+ gpu_types::attention::AttnParams, kernel::AttentionGemmMetalKernel,
},
},
data_type::DataType,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
index a26465005..89f3f39ed 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
@@ -4,7 +4,6 @@ use crate::{
backends::{
common::{
Backend, BufferMut, BufferRef, CommandBufferEncoding,
- gpu_types::AttnParams,
kernel::{AttentionArguments, AttentionKernelConfig},
},
metal::{
@@ -12,6 +11,7 @@ use crate::{
command_buffer::MetalCommandBufferEncoding,
context::MetalContext,
error::MetalError,
+ gpu_types::attention::AttnParams,
kernel::{AttentionGemmGroupedCombineMetalKernel, AttentionGemmGroupedMetalKernel},
},
},
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
index 4452fe8a1..1530440be 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
@@ -1,20 +1,20 @@
use crate::backends::common::Backend;
+mod error;
mod fallback;
mod gemm;
mod gemm_grouped;
mod single_pass;
mod two_pass;
+pub use error::AttentionMetalError;
+
use crate::backends::{
common::{
BufferRef, CommandBufferEncoding,
kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig, SigmoidGateKernel},
},
- metal::{
- Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError,
- kernel::SigmoidGateMetalKernel,
- },
+ metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, kernel::SigmoidGateMetalKernel},
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -62,11 +62,12 @@ pub struct AttentionMetalKernel {
impl AttentionKernel for AttentionMetalKernel {
type Backend = Metal;
+ type Error = AttentionMetalError;
fn new(
context: &MetalContext,
config: AttentionKernelConfig,
- ) -> Result<Self, MetalError> {
+ ) -> Result<Self, AttentionMetalError> {
let grouped = gemm_grouped::AttentionGemmGrouped::is_supported(&config, context)
.then(|| gemm_grouped::AttentionGemmGrouped::new(&config));
let gemm = gemm::AttentionGemm::is_supported(&config).then(|| gemm::AttentionGemm::new(&config));
@@ -100,7 +101,7 @@ impl AttentionKernel for AttentionMetalKernel {
impl BufferRef<Backend = Metal>,
>,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
+ ) -> Result<<Metal as Backend>::ScratchBuffer, AttentionMetalError> {
let command_buffer = &mut *command_buffer.debug_group("attention core");
let (gate, suffix_length) = (arguments.gate, arguments.suffix_length);
self.encode_impl(arguments, command_buffer).map(|(mut output, gate_applied)| {
@@ -132,7 +133,7 @@ impl AttentionMetalKernel {
impl BufferRef<Backend = Metal>,
>,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<(<Metal as Backend>::ScratchBuffer, bool), MetalError> {
+ ) -> Result<(<Metal as Backend>::ScratchBuffer, bool), AttentionMetalError> {
let suffix_length = arguments.suffix_length;
let gate_applied_by_two_pass = arguments.gate.is_some();
let kv_length = arguments.cache.prefix_len() + suffix_length;
@@ -145,7 +146,7 @@ impl AttentionMetalKernel {
&& let Some(grouped) = &self.grouped
{
if grouped.should_encode(mask, suffix_length, kv_length) {
- return grouped.encode(mask, arguments, command_buffer);
+ return grouped.encode(mask, arguments, command_buffer).map_err(AttentionMetalError::Backend);
}
// These measured sibling exceptions apply only when the
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
index 6a2af05cb..8f8b35954 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
@@ -5,11 +5,6 @@
using namespace metal;
namespace uzu::gemm {
-enum class GemmAPrologueKind : uint32_t {
- FullPrecision = 0,
- Int8Symmetric = 1,
-};
-
enum class GemmBPrologueKind : uint32_t {
FullPrecision = 0,
ScaleBiasDequant = 1,
@@ -31,6 +26,11 @@ struct GemmDTransform {
constexpr uint32_t bits() const constant { return raw_value; }
};
+enum class GemmAPrologueKind : uint32_t {
+ FullPrecision = 0,
+ Int8Symmetric = 1,
+};
+
enum class GemmTiling : uint32_t {
Tile8x32x32_Simdgroups1x1 = 0,
Tile64x32x32_Simdgroups2x2 = 1,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/weaver.h b/crates/uzu-engine/src/backends/metal/kernel/generated/weaver.h
index 173fdfaf4..9533a3269 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/weaver.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/weaver.h
@@ -7,10 +7,6 @@ using namespace metal;
namespace uzu::weaver {
static constant constexpr uint32_t CANDIDATES_MAX = 512;
-static constant constexpr uint32_t TOP_CHILDREN_THREADS = 256;
-
-static constant constexpr uint32_t TOP_CHILDREN_SIMDGROUPS = 8;
-
enum class FrontierIdx : uint32_t {
TokenId = 0,
ParentSlot = 1,
@@ -38,15 +34,19 @@ enum class MetadataIdx : uint32_t {
static constant constexpr uint32_t FRONTIER_NO_WINNER = ~0;
-static constant constexpr uint32_t FRONTIER_SELECT_THREADS = 256;
-
-static constant constexpr uint32_t FRONTIER_SELECT_SIMDGROUPS = 8;
-
-static constant constexpr uint32_t FRONTIER_ENTRIES_PER_THREAD = 8;
-
static constant constexpr uint32_t FRONTIER_MAX_SLOTS = 2048;
static constant constexpr uint32_t FRONTIER_MAX_WIDTH = 32;
static constant constexpr uint32_t PADDING_DEPTH = 0;
+
+static constant constexpr uint32_t TOP_CHILDREN_THREADS = 256;
+
+static constant constexpr uint32_t TOP_CHILDREN_SIMDGROUPS = 8;
+
+static constant constexpr uint32_t FRONTIER_SELECT_THREADS = 256;
+
+static constant constexpr uint32_t FRONTIER_SELECT_SIMDGROUPS = 8;
+
+static constant constexpr uint32_t FRONTIER_ENTRIES_PER_THREAD = 8;
} // namespace uzu::weaver
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/error.rs
new file mode 100644
index 000000000..5d04d64be
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/error.rs
@@ -0,0 +1,45 @@
+use thiserror::Error;
+
+use crate::{
+ backends::{
+ common::{gpu_types::gemm::GemmDTransform, kernel::matmul::MatmulOutputError},
+ metal::{
+ error::MetalError,
+ kernel::matmul::gemm::{GemmPlanError, GemmSpecializationError},
+ },
+ },
+ data_type::DataType,
+};
+
+#[derive(Debug, Error)]
+pub enum MatmulMetalError {
+ #[error("Unsupported data type: {0:?}")]
+ UnsupportedDataType(DataType),
+ #[error("Unsupported D-transform op {bit:?} on path {path}")]
+ UnsupportedDOp {
+ bit: GemmDTransform,
+ path: &'static str,
+ },
+ #[error("Unsupported layout on path {path}")]
+ UnsupportedLayout {
+ path: &'static str,
+ },
+ #[error("Incompatible A operand for {path}: {reason}")]
+ IncompatibleA {
+ path: &'static str,
+ reason: &'static str,
+ },
+ #[error("gathered readout requires the GEMV path, but shape (m={m}, n={n}) routes to GEMM")]
+ UnsupportedGather {
+ m: u32,
+ n: u32,
+ },
+ #[error(transparent)]
+ Output(#[from] MatmulOutputError),
+ #[error(transparent)]
+ Plan(#[from] GemmPlanError),
+ #[error(transparent)]
+ Specialization(#[from] GemmSpecializationError),
+ #[error("Backend error: {0}")]
+ BackendError(#[from] MetalError),
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs
index bb635d08c..50b26d769 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs
@@ -1,6 +1,6 @@
use thiserror::Error;
-use crate::backends::common::gpu_types::gemm::GemmTiling;
+use crate::backends::metal::gpu_types::gemm::GemmTiling;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
pub enum GemmSpecializationError {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 684ae1c35..c2eb07de3 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -1,23 +1,13 @@
use std::collections::{HashMap, hash_map::Entry};
-use super::{
- super::{MatmulOutputWork, supports_integer_right_operand},
- GemmEngine, GemmPlan,
- selection::{GemmProblem, outer_block_k},
- specialization::GemmSpecialization,
- trellis::TrellisGemm,
-};
use crate::{
backends::{
common::{
Backend, BufferMut, BufferRef, CommandBufferEncoding,
- gpu_types::{
- GemmParams,
- gemm::{GemmAPrologueKind, GemmAlignment, GemmBPrologueKind, GemmDTransform},
- },
+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
kernel::matmul::{
- Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulError, MatmulOutput, MatmulShape,
- QuantParamsLayout, QuantParamsStrides, QuantizedB,
+ Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulOutput, MatmulShape, QuantParamsLayout,
+ QuantParamsStrides, QuantizedB,
},
},
metal::{
@@ -25,7 +15,23 @@ use crate::{
command_buffer::MetalCommandBufferEncoding,
context::MetalContext,
error::MetalError,
- kernel::{GemmMetalKernel, GemmSplitKReduceMetalKernel},
+ gpu_types::{
+ gemm::{GemmAPrologueKind, GemmAlignment},
+ matmul::GemmParams,
+ },
+ kernel::{
+ GemmMetalKernel, GemmSplitKReduceMetalKernel,
+ matmul::{
+ MatmulMetalError, MatmulOutputWork,
+ gemm::{
+ GemmEngine, GemmPlan,
+ selection::{GemmProblem, outer_block_k},
+ specialization::GemmSpecialization,
+ trellis::TrellisGemm,
+ },
+ supports_integer_right_operand,
+ },
+ },
},
},
data_type::DataType,
@@ -131,10 +137,8 @@ impl GemmKernel {
shape: &MatmulShape,
engine: GemmEngine,
context: &MetalContext,
- ) -> Result<GemmPlan, MetalError> {
- self.problem(*shape, context)
- .select_plan_for_engine(engine)
- .map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))
+ ) -> Result<GemmPlan, MatmulMetalError> {
+ self.problem(*shape, context).select_plan_for_engine(engine).map_err(MatmulMetalError::from)
}
pub fn encode_plan(
@@ -150,21 +154,16 @@ impl GemmKernel {
plan: GemmPlan,
output_work: &MatmulOutputWork,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<(), MetalError> {
+ ) -> Result<(), MatmulMetalError> {
let shape = MatmulShape::from_arguments(&arguments);
// TODO: remove after GatherGEMM is supported
if arguments.gather_indices.is_some() {
- return Err(MetalError::KernelDispatchFailed(
- format!(
- "gathered readout requires the GEMV path, but shape (m={}, n={}) routes to GEMM",
- arguments.m, arguments.n
- )
- .into(),
- ));
+ return Err(MatmulMetalError::UnsupportedGather {
+ m: arguments.m,
+ n: arguments.n,
+ });
}
- self.problem(shape, command_buffer.context())
- .validate_engine(plan.engine)
- .map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))?;
+ self.problem(shape, command_buffer.context()).validate_engine(plan.engine)?;
match arguments.b {
MatmulB::FullPrecision {
@@ -193,7 +192,7 @@ impl GemmKernel {
..
} => {
if self.output_data_type != DataType::BF16 {
- return Err(MatmulError::UnsupportedDataType(self.output_data_type).into());
+ return Err(MatmulMetalError::UnsupportedDataType(self.output_data_type));
}
self.trellis.encode_plan(arguments, plan, command_buffer)
},
@@ -210,7 +209,7 @@ impl GemmKernel {
plan: GemmPlan,
output_work: &MatmulOutputWork,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<(), MetalError> {
+ ) -> Result<(), MatmulMetalError> {
let (m, n, k) = (shape.m, shape.n, shape.k);
let (mut d, d_transform) = output.into_contiguous(n, "Gemm")?;
let ab_scale = d_transform.ab_scale;
@@ -228,11 +227,10 @@ impl GemmKernel {
let is_quant = quantized.is_some();
if !is_quant && !matches!(&a, MatmulA::FullPrecision { .. }) {
- return Err(MatmulError::IncompatibleA {
+ return Err(MatmulMetalError::IncompatibleA {
path: "Gemm",
reason: "int8 activations require quantized weights",
- }
- .into());
+ });
}
let (scales, biases, zero_points, scale_strides, zero_point_strides) = match quantized.as_ref() {
None => (None, None, None, Default::default(), Default::default()),
@@ -264,11 +262,10 @@ impl GemmKernel {
activation_group_sums.is_some(),
)?;
if output_transform.contains(GemmDTransform::SOFT_CAP) {
- return Err(MatmulError::UnsupportedDOp {
+ return Err(MatmulMetalError::UnsupportedDOp {
bit: GemmDTransform::SOFT_CAP,
path: "Gemm int8 left operand",
- }
- .into());
+ });
}
(
None,
@@ -282,10 +279,9 @@ impl GemmKernel {
MatmulA::Trellis {
..
} => {
- return Err(MatmulError::UnsupportedLayout {
+ return Err(MatmulMetalError::UnsupportedLayout {
path: "Gemm",
- }
- .into());
+ });
},
};
@@ -395,7 +391,7 @@ impl GemmKernel {
output_transform: GemmDTransform,
output_bias: Option<impl BufferRef<Backend = Metal>>,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<(), MetalError> {
+ ) -> Result<(), MatmulMetalError> {
let (m, n, k) = (shape.m, shape.n, shape.k);
let (a_full_precision, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match a {
MatmulA::FullPrecision {
@@ -414,10 +410,9 @@ impl GemmKernel {
MatmulA::Trellis {
..
} => {
- return Err(MatmulError::UnsupportedLayout {
+ return Err(MatmulMetalError::UnsupportedLayout {
path: "Gemm",
- }
- .into());
+ });
},
};
let a_group_sums = a_group_sums.filter(|_| Self::requires_activation_group_sums(&shape));
@@ -496,20 +491,18 @@ fn validate_int8_left_operand(
a_group_size: u32,
code_layout: Int8CodeLayout,
has_group_sums: bool,
-) -> Result<(), MetalError> {
+) -> Result<(), MatmulMetalError> {
if shape.b_bits.and_then(Int8CodeLayout::for_right_bits) != Some(code_layout) {
- return Err(MatmulError::IncompatibleA {
+ return Err(MatmulMetalError::IncompatibleA {
path: "Gemm",
reason: "left code layout is incompatible with the right operand",
- }
- .into());
+ });
}
if GemmKernel::requires_activation_group_sums(&shape) && !has_group_sums {
- return Err(MatmulError::IncompatibleA {
+ return Err(MatmulMetalError::IncompatibleA {
path: "Gemm",
reason: "quantized correction requires left group sums",
- }
- .into());
+ });
}
let compatible = use_mxu
&& supports_integer_right_operand(&shape)
@@ -526,11 +519,10 @@ fn validate_int8_left_operand(
.b_group_size
.is_some_and(|gs| matches!(gs, 32 | 64 | 128) && shape.k.is_multiple_of(gs) && a_group_size >= gs);
if !compatible {
- return Err(MatmulError::IncompatibleA {
+ return Err(MatmulMetalError::IncompatibleA {
path: "Gemm",
reason: "symmetric int8 left operands require group-major metadata and supported 32/64/128 groups",
- }
- .into());
+ });
}
Ok(())
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
index d45b0d0c0..5ce0acd5e 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
@@ -7,9 +7,10 @@ mod trellis;
pub use error::GemmSpecializationError;
pub use kernel::GemmKernel;
+pub use selection::GemmPlanError;
pub(super) use selection::GemmProblem;
-use crate::backends::common::gpu_types::gemm::GemmTiling;
+use crate::backends::metal::gpu_types::gemm::GemmTiling;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GemmEngine {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
index db4e537a4..c80f269a3 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
@@ -6,7 +6,7 @@
use metal::MTLGPUFamily;
-use crate::backends::common::gpu_types::gemm::GemmTiling;
+use crate::backends::metal::gpu_types::gemm::GemmTiling;
const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
index 86fb18c70..3b45fe82f 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -1,11 +1,16 @@
use metal::MTLGPUFamily;
use thiserror::Error;
-use super::{GemmEngine, GemmPlan, policy};
use crate::{
- backends::common::{
- gpu_types::gemm::{GemmBPrologueKind, GemmTiling},
- kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},
+ backends::{
+ common::{
+ gpu_types::gemm::GemmBPrologueKind,
+ kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},
+ },
+ metal::{
+ gpu_types::gemm::GemmTiling,
+ kernel::matmul::gemm::{GemmEngine, GemmPlan, policy},
+ },
},
data_type::DataType,
};
@@ -22,7 +27,7 @@ pub struct GemmProblem {
}
#[derive(Debug, Clone, Copy, PartialEq, Error)]
-pub(super) enum GemmPlanError {
+pub enum GemmPlanError {
#[error("MXU engine is not available for this GEMM")]
MxuUnavailable,
#[error("GEMM input layout is unsupported: {0}")]
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
index d215a1d41..25ea3639f 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -1,8 +1,11 @@
use super::{GemmEngine, GemmPlan, error::GemmSpecializationError};
use crate::{
- backends::common::{
- gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},
- kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},
+ backends::{
+ common::{
+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
+ kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},
+ },
+ metal::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmTiling},
},
data_type::DataType,
};
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs
index 0929b24b3..fdfe28a11 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs
@@ -1,20 +1,24 @@
use std::collections::{HashMap, hash_map::Entry};
-use super::{GemmEngine, GemmPlan, selection::TRELLIS_K_STEP as K_STEP};
use crate::backends::{
common::{
BufferMut, BufferRef, CommandBufferEncoding,
- gpu_types::{
- GemmParams,
- gemm::{GemmAlignment, GemmTiling},
- },
- kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError, TrellisFormat},
+ kernel::matmul::{MatmulA, MatmulArguments, MatmulB, TrellisFormat},
},
metal::{
Metal,
command_buffer::MetalCommandBufferEncoding,
- error::MetalError,
- kernel::{GemmTrellisMetalKernel, GemmTrellisReduceMetalKernel},
+ gpu_types::{
+ gemm::{GemmAlignment, GemmTiling},
+ matmul::GemmParams,
+ },
+ kernel::{
+ GemmTrellisMetalKernel, GemmTrellisReduceMetalKernel,
+ matmul::{
+ MatmulMetalError,
+ gemm::{GemmEngine, GemmPlan, selection::TRELLIS_K_STEP as K_STEP},
+ },
+ },
},
};
@@ -37,11 +41,9 @@ impl TrellisGemm {
>,
plan: GemmPlan,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<(), MetalError> {
- let unsupported = || {
- MetalError::from(MatmulError::UnsupportedLayout {
- path: "GemmTrellis",
- })
+ ) -> Result<(), MatmulMetalError> {
+ let unsupported = || MatmulMetalError::UnsupportedLayout {
+ path: "GemmTrellis",
};
let (
MatmulA::Trellis {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index a69d487f2..e3ec42f2a 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -1,22 +1,26 @@
use metal::MTLGPUFamily;
-use super::{
- super::MatmulOutputWork,
- policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK},
-};
use crate::{
backends::{
common::{
CommandBufferEncoding,
gpu_types::{
- HADAMARD_TRANSFORM_BLOCK_SIZE,
gemm::{GemmBPrologueKind, GemmDTransform},
+ hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
},
kernel::matmul::{MatmulB, MatmulShape},
},
metal::{
- command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError,
- kernel::GemvMetalKernel,
+ command_buffer::MetalCommandBufferEncoding,
+ context::MetalContext,
+ error::MetalError,
+ kernel::{
+ GemvMetalKernel,
+ matmul::{
+ MatmulOutputWork,
+ gemv::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK},
+ },
+ },
},
},
data_type::DataType,
@@ -211,9 +215,9 @@ use std::collections::{HashMap, hash_map::Entry};
use crate::backends::{
common::{
BufferMut, BufferRef,
- kernel::matmul::{MatmulA, MatmulArguments, MatmulError},
+ kernel::matmul::{MatmulA, MatmulArguments},
},
- metal::Metal,
+ metal::{Metal, kernel::matmul::MatmulMetalError},
};
/// GEMV pipelines compiled on first use.
@@ -242,13 +246,16 @@ impl GemvKernel {
&mut self,
context: &MetalContext,
specialization: GemvSpecialization,
- ) -> Result<&GemvMetalKernel, MatmulError<Metal>> {
+ ) -> Result<&GemvMetalKernel, MatmulMetalError> {
match self.pipelines.entry(specialization) {
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
- let kernel = specialization
- .create_pipeline(context, self.weights_data_type, self.input_data_type, self.output_data_type)
- .map_err(MatmulError::BackendError)?;
+ let kernel = specialization.create_pipeline(
+ context,
+ self.weights_data_type,
+ self.input_data_type,
+ self.output_data_type,
+ )?;
Ok(entry.insert(kernel))
},
}
@@ -267,7 +274,7 @@ impl GemvKernel {
specialization: GemvSpecialization,
output_work: &MatmulOutputWork,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<(), MatmulError<Metal>> {
+ ) -> Result<(), MatmulMetalError> {
let MatmulArguments {
a,
b,
@@ -294,7 +301,7 @@ impl GemvKernel {
offset: a_offset,
} = a
else {
- return Err(MatmulError::IncompatibleA {
+ return Err(MatmulMetalError::IncompatibleA {
path: "Gemv",
reason: "prepared int8 activations require GEMM",
});
@@ -364,7 +371,7 @@ impl GemvKernel {
MatmulB::Trellis {
..
} => {
- return Err(MatmulError::UnsupportedLayout {
+ return Err(MatmulMetalError::UnsupportedLayout {
path: "Gemv",
});
},
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
index e40b744f7..deeb74426 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
@@ -1,28 +1,31 @@
use metal::MTLGPUFamily;
-use super::{
- GemmKernel,
- gemm::{self, GemmPlan, GemmProblem},
- gemv::{GemvKernel, GemvSpecialization},
- matmul_dispatch::MatmulDispatch,
- matmul_output_work::MatmulOutputWork,
- qmv::{self, QmvRoute},
-};
use crate::{
backends::{
common::{
BufferMut, BufferRef, CommandBufferEncoding,
- gpu_types::gemm::GemmTiling,
kernel::{
ActivationQuantization,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
matmul::{
- ActivationFormat, Int8CodeLayout, MatmulArguments, MatmulError, MatmulKernel, MatmulShape,
- QuantParamsLayout,
+ ActivationFormat, Int8CodeLayout, MatmulArguments, MatmulKernel, MatmulShape, QuantParamsLayout,
},
},
},
- metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},
+ metal::{
+ Metal,
+ command_buffer::MetalCommandBufferEncoding,
+ context::MetalContext,
+ gpu_types::gemm::GemmTiling,
+ kernel::matmul::{
+ GemmKernel, MatmulMetalError,
+ gemm::{self, GemmPlan, GemmProblem},
+ gemv::{GemvKernel, GemvSpecialization},
+ matmul_dispatch::MatmulDispatch,
+ matmul_output_work::MatmulOutputWork,
+ qmv::{self, QmvRoute},
+ },
+ },
},
data_type::DataType,
};
@@ -141,16 +144,17 @@ impl MatmulMetalKernel {
impl MatmulKernel for MatmulMetalKernel {
type Backend = Metal;
+ type Error = MatmulMetalError;
fn new(
context: &MetalContext,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
- ) -> Result<Self, MetalError> {
+ ) -> Result<Self, Self::Error> {
for data_type in [weights_data_type, input_data_type, output_data_type] {
if !matches!(data_type, DataType::BF16 | DataType::F32) {
- return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());
+ return Err(MatmulMetalError::UnsupportedDataType(data_type));
}
}
@@ -229,25 +233,15 @@ impl MatmulKernel for MatmulMetalKernel {
impl BufferRef<Backend = Metal>,
>,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<(), MetalError> {
+ ) -> Result<(), Self::Error> {
let shape = MatmulShape::from_arguments(&arguments);
let plan = match self.select_dispatch(&shape, command_buffer.context()) {
MatmulDispatch::Gemv(gemv) => {
- return self.gemv.encode(arguments, gemv, &self.output_work, command_buffer).map_err(MetalError::from);
+ return self.gemv.encode(arguments, gemv, &self.output_work, command_buffer);
},
MatmulDispatch::Gemm(plan) => plan,
};
- // TODO: remove after GatherGEMM is supported
- if arguments.gather_indices.is_some() {
- return Err(MetalError::KernelDispatchFailed(
- format!(
- "gathered readout requires the GEMV path, but shape (m={}, n={}) routes to GEMM",
- arguments.m, arguments.n
- )
- .into(),
- ));
- }
self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)
}
}
@@ -266,7 +260,7 @@ impl MatmulMetalKernel {
>,
engine: gemm::GemmEngine,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<(), MetalError> {
+ ) -> Result<(), MatmulMetalError> {
let shape = MatmulShape::from_arguments(&arguments);
let plan = self.gemm.select_plan_for_engine(&shape, engine, command_buffer.context())?;
self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
index a24948447..1a5803108 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
@@ -1,3 +1,4 @@
+mod error;
pub mod gemm;
mod gemv;
mod matmul_dispatch;
@@ -5,6 +6,7 @@ mod matmul_metal_kernel;
mod matmul_output_work;
mod qmv;
+pub use error::MatmulMetalError;
pub use gemm::GemmKernel;
#[cfg(test)]
use matmul_dispatch::MatmulDispatch;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs
index 2b237c776..2042c758a 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs
@@ -9,9 +9,9 @@ use super::{
},
QmvRoute,
};
-use crate::backends::common::{
- gpu_types::gemm::{GemmBPrologueKind, GemmTiling},
- kernel::matmul::MatmulShape,
+use crate::backends::{
+ common::{gpu_types::gemm::GemmBPrologueKind, kernel::matmul::MatmulShape},
+ metal::gpu_types::gemm::GemmTiling,
};
const DOWN: u8 = 1 << 0;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/mod.rs
index 30c7939fb..856bcf824 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/mod.rs
@@ -1,13 +1,13 @@
use crate::backends::{
- common::{
- Kernels,
+ common::{Kernels, gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},
+ metal::{
+ Metal,
+ buffer::MetalBufferExt,
gpu_types::{
- HADAMARD_TRANSFORM_BLOCK_SIZE,
gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_k, gemm_tiling_simdgroups_per_row},
weaver::{FRONTIER_SELECT_THREADS, TOP_CHILDREN_THREADS},
},
},
- metal::{Metal, buffer::MetalBufferExt},
};
const METAL_SIMD_SIZE: u32 = 32;
diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rs
index fa0b4296f..2057b7b17 100644
--- a/crates/uzu-engine/src/backends/metal/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/mod.rs
@@ -5,6 +5,7 @@ mod command_buffer;
mod context;
mod decompression;
mod error;
+mod gpu_types;
mod heaps;
mod kernel;
mod metal_extensions;
diff --git a/crates/uzu-engine/src/backends/mod.rs b/crates/uzu-engine/src/backends/mod.rs
index c9e4b8f1d..924d3133e 100644
--- a/crates/uzu-engine/src/backends/mod.rs
+++ b/crates/uzu-engine/src/backends/mod.rs
@@ -1,11 +1,11 @@
-pub mod common;
+pub(crate) mod common;
#[cfg(backend = "cpu")]
-pub mod cpu;
+pub(crate) mod cpu;
#[cfg(backend = "metal")]
-pub mod metal;
+pub(crate) mod metal;
-use crate::backends::common::Backend;
+pub use common::Backend;
pub trait BackendSelection {
type Output;
diff --git a/crates/uzu-engine/src/config/activation/mod.rs b/crates/uzu-engine/src/config/activation/mod.rs
index 8ccc592b2..e0b1a92f3 100644
--- a/crates/uzu-engine/src/config/activation/mod.rs
+++ b/crates/uzu-engine/src/config/activation/mod.rs
@@ -1,6 +1,6 @@
use uzu_engine_macros::uzu_config_abstract;
-use crate::backends::common::gpu_types::ActivationType;
+use crate::backends::common::gpu_types::activation_type::ActivationType;
pub mod gelu;
pub mod identity;
diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rs
index 05fa33cad..bef35e2db 100644
--- a/crates/uzu-engine/src/encodable_block/classifier.rs
+++ b/crates/uzu-engine/src/encodable_block/classifier.rs
@@ -13,7 +13,7 @@ use crate::{
embedding::{Embedding, EmbeddingError},
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
prediction_head::{PredictionHead, PredictionHeadError},
- transformer::{Transformer, TransformerNewError},
+ transformer::{Transformer, TransformerEncodeError, TransformerNewError},
},
parameters::ParameterTree,
};
@@ -28,6 +28,8 @@ pub enum ClassifierError<B: Backend> {
Normalization(#[from] NormalizationNewError<B>),
#[error("Transformer error: {0}")]
Transformer(#[from] TransformerNewError<B>),
+ #[error("Transformer encoding error: {0}")]
+ TransformerEncode(#[from] TransformerEncodeError<B>),
#[error("Transformer error: {0}")]
PredictionHead(#[from] PredictionHeadError<B>),
#[error("Unsupported configuration: {0}")]
@@ -144,8 +146,7 @@ impl<B: Backend> Classifier<B> {
None,
None,
command_buffer,
- )
- .map_err(ClassifierError::Backend)?
+ )?
.output
.unwrap();
@@ -154,7 +155,7 @@ impl<B: Backend> Classifier<B> {
.map_err(ClassifierError::Backend)?;
self.pooling.encode(&hidden, &mut pooled, batch_dim, self.hidden_dim, 1, command_buffer);
- let logits = self.prediction_head.encode(pooled, 1, command_buffer).map_err(ClassifierError::Backend)?;
+ let logits = self.prediction_head.encode(pooled, 1, command_buffer)?;
Ok(logits)
}
diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rs
index e3a3242aa..44b462310 100644
--- a/crates/uzu-engine/src/encodable_block/decoder.rs
+++ b/crates/uzu-engine/src/encodable_block/decoder.rs
@@ -12,7 +12,7 @@ use crate::{
embedding::{Embedding, EmbeddingError},
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
per_layer_embedding::{PerLayerEmbedding, PerLayerEmbeddingError},
- transformer::{Transformer, TransformerNewError, TransformerState},
+ transformer::{Transformer, TransformerEncodeError, TransformerNewError, TransformerState},
},
parameters::ParameterTree,
};
@@ -29,6 +29,8 @@ pub enum DecoderError<B: Backend> {
PerLayerEmbedding(#[from] PerLayerEmbeddingError<B>),
#[error("Transformer error: {0}")]
Transformer(#[from] TransformerNewError<B>),
+ #[error("Transformer encoding error: {0}")]
+ TransformerEncode(#[from] TransformerEncodeError<B>),
}
pub struct Decoder<B: Backend> {
@@ -155,27 +157,20 @@ impl<B: Backend> Decoder<B> {
};
let per_layer_inputs = if let Some(per_layer_embedding) = &self.per_layer_embedding {
- Some(
- per_layer_embedding
- .encode(token_ids, &embedded, batch_dim.size(), command_buffer)
- .map_err(DecoderError::Backend)?,
- )
+ Some(per_layer_embedding.encode(token_ids, &embedded, batch_dim.size(), command_buffer)?)
} else {
None
};
- let transformer_output = self
- .transformer
- .encode(
- embedded,
- per_layer_inputs.as_ref(),
- batch_dim,
- output_range,
- hidden_feature_layer_indices,
- Some(state),
- command_buffer,
- )
- .map_err(DecoderError::Backend)?;
+ let transformer_output = self.transformer.encode(
+ embedded,
+ per_layer_inputs.as_ref(),
+ batch_dim,
+ output_range,
+ hidden_feature_layer_indices,
+ Some(state),
+ command_buffer,
+ )?;
let logits = if let Some(output_range) = output_range {
let output = transformer_output.output.as_ref().expect("decoder output range requires transformer output");
diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rs
index e2a749989..87743fc8a 100644
--- a/crates/uzu-engine/src/encodable_block/dflash.rs
+++ b/crates/uzu-engine/src/encodable_block/dflash.rs
@@ -13,7 +13,7 @@ use crate::{
encodable_block::{
batch_topology::BatchTopology,
embedding::{Embedding, EmbeddingError},
- linear::{Linear, LinearBlockError},
+ linear::{Linear, LinearBlockError, LinearEncodeError},
mixer::{
MixerState,
attention::{
@@ -83,6 +83,10 @@ pub enum DFlashNewError<B: Backend> {
pub enum DFlashEncodeError<B: Backend> {
#[error("Backend error: {0}")]
Backend(#[source] B::Error),
+ #[error("Linear error: {0}")]
+ Linear(#[from] LinearEncodeError<B>),
+ #[error("Transformer layer error: {0}")]
+ TransformerLayer(#[from] TransformerLayerError<B>),
#[error("Embedding error: {0}")]
Embedding(#[from] EmbeddingError<B>),
}
@@ -202,7 +206,7 @@ impl<B: Backend> DFlash<B> {
accepted_indices: &[u32],
state: &mut DFlashState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error> {
+ ) -> Result<(), DFlashEncodeError<B>> {
if accepted_indices.is_empty() {
return Ok(());
}
@@ -217,8 +221,9 @@ impl<B: Backend> DFlash<B> {
assert!(context_length <= state.context_capacity, "DFlash state capacity exceeded");
assert!(context_length <= self.max_context_length, "DFlash context exceeds configured RoPE capacity");
- let mut packed_target_features =
- command_buffer.allocate_scratch_for_shape(&[num_tokens, self.target_feature_input_dim], self.data_type)?;
+ let mut packed_target_features = command_buffer
+ .allocate_scratch_for_shape(&[num_tokens, self.target_feature_input_dim], self.data_type)
+ .map_err(DFlashEncodeError::Backend)?;
for (layer_index, features) in target_features.enumerate() {
assert!(features.size() % layer_feature_bytes == 0);
for (token_index, &accepted_index) in accepted_indices.iter().enumerate() {
@@ -234,15 +239,13 @@ impl<B: Backend> DFlash<B> {
}
let projected_features =
self.target_feature_projection.encode(packed_target_features, num_tokens, command_buffer)?;
- let normalized_features = self.projected_feature_norm.encode(
- &projected_features,
- 0,
- num_tokens,
- None::<&mut B::ScratchBuffer>,
- command_buffer,
- )?;
+ let normalized_features = self
+ .projected_feature_norm
+ .encode(&projected_features, 0, num_tokens, None::<&mut B::ScratchBuffer>, command_buffer)
+ .map_err(DFlashEncodeError::Backend)?;
let token_positions = (state.context_length..state.context_length + num_tokens).collect::<Box<[_]>>();
- let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, command_buffer)?;
+ let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, command_buffer)
+ .map_err(DFlashEncodeError::Backend)?;
let projected_kv = self.state_kv_projection.encode(normalized_features, num_tokens, command_buffer)?;
let layer_kv_bytes = size_for_shape(&[self.layer_kv_dim], self.data_type);
@@ -250,7 +253,8 @@ impl<B: Backend> DFlash<B> {
|chunk_index: usize| Range::from(chunk_index * layer_kv_bytes..(chunk_index + 1) * layer_kv_bytes);
let mut layer_key_values = (0..self.layers.len())
.map(|_| command_buffer.allocate_scratch(num_tokens as usize * layer_kv_bytes))
- .collect::<Result<Box<[_]>, _>>()?;
+ .collect::<Result<Box<[_]>, _>>()
+ .map_err(DFlashEncodeError::Backend)?;
for (layer_index, key_value) in layer_key_values.iter_mut().enumerate() {
for token_index in 0..num_tokens as usize {
command_buffer.encode_copy(
@@ -262,14 +266,16 @@ impl<B: Backend> DFlash<B> {
for ((layer, mixer_state), mut key_value) in
self.layers.iter().zip(state.layer_states.iter_mut()).zip(layer_key_values)
{
- mixer_state.prepare(state.context_length, num_tokens)?;
+ mixer_state.prepare(state.context_length, num_tokens).map_err(DFlashEncodeError::Backend)?;
let attention = (layer.mixer.as_ref() as &dyn Any)
.downcast_ref::<Attention<B>>()
.expect("DFlash draft layers must use attention mixers");
let attention_state = (mixer_state.as_mut() as &mut dyn Any)
.downcast_mut::<AttentionState<B>>()
.expect("DFlash draft layer states must be attention states");
- attention.append_projected_kv(&mut key_value, &rope, num_tokens, attention_state, command_buffer)?;
+ attention
+ .append_projected_kv(&mut key_value, &rope, num_tokens, attention_state, command_buffer)
+ .map_err(DFlashEncodeError::Backend)?;
}
state.context_length += num_tokens;
@@ -315,17 +321,15 @@ impl<B: Backend> DFlash<B> {
let mut residual = command_buffer.allocate_scratch(hidden.size()).map_err(DFlashEncodeError::Backend)?;
for (layer, mixer_state) in self.layers.iter().zip(state.layer_states.iter_mut()) {
mixer_state.prepare(state.context_length, batch_size).map_err(DFlashEncodeError::Backend)?;
- hidden = layer
- .encode(
- hidden,
- &mut residual,
- None::<&B::ScratchBuffer>,
- Some(&rope),
- &batch_topology,
- Some(MaybeMut::Mut(mixer_state.as_mut())),
- command_buffer,
- )
- .map_err(DFlashEncodeError::Backend)?;
+ hidden = layer.encode(
+ hidden,
+ &mut residual,
+ None::<&B::ScratchBuffer>,
+ Some(&rope),
+ &batch_topology,
+ Some(MaybeMut::Mut(mixer_state.as_mut())),
+ command_buffer,
+ )?;
}
let draft_hidden = self
.output_norm
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rs
index 88b293a90..5eb3facee 100644
--- a/crates/uzu-engine/src/encodable_block/embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding.rs
@@ -14,7 +14,7 @@ use crate::{
data_type::DataType,
encodable_block::{
embedding_table::{EmbeddingTable, EmbeddingTableError, read_output_signs},
- linear::{Gather, LinearMatmulError, UntiedReadout},
+ linear::{Gather, LinearEncodeError, LinearMatmulError, UntiedReadout},
},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -23,6 +23,8 @@ use crate::{
pub enum EmbeddingError<B: Backend> {
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
+ #[error("Matmul error: {0}")]
+ Matmul(#[source] <<B::Kernels as Kernels>::MatmulKernel as MatmulKernel>::Error),
#[error("Parameter loading error: {0}")]
ParameterError(#[from] ParameterLoaderError<B>),
#[error("Unsupported configuration: {0}")]
@@ -31,6 +33,8 @@ pub enum EmbeddingError<B: Backend> {
EmbeddingTable(#[from] EmbeddingTableError<B>),
#[error(transparent)]
LinearMatmul(#[from] LinearMatmulError<B>),
+ #[error("Linear encoding error: {0}")]
+ LinearEncode(#[from] LinearEncodeError<B>),
}
enum EmbeddingTying<B: Backend> {
@@ -197,7 +201,7 @@ impl<B: Backend> Embedding<B> {
indices,
output_dim,
});
- output.encode(input_buffer, batch_dim, gather, command_buffer).map_err(EmbeddingError::BackendError)?
+ output.encode(input_buffer, batch_dim, gather, command_buffer)?
},
EmbeddingTying::Tied {
table,
@@ -220,7 +224,7 @@ impl<B: Backend> Embedding<B> {
n: output_dim,
k: self.model_dim,
};
- readout.lock().encode(arguments, command_buffer).map_err(EmbeddingError::BackendError)?;
+ readout.lock().encode(arguments, command_buffer).map_err(EmbeddingError::Matmul)?;
output
},
};
@@ -244,6 +248,6 @@ fn readout_kernel<B: Backend>(
data_type: DataType,
) -> Result<Mutex<<B::Kernels as Kernels>::MatmulKernel>, EmbeddingError<B>> {
let kernel = <B::Kernels as Kernels>::MatmulKernel::new(context, data_type, data_type, data_type)
- .map_err(EmbeddingError::BackendError)?;
+ .map_err(EmbeddingError::Matmul)?;
Ok(Mutex::new(kernel))
}
diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rs
index 5d9f67598..ab6916574 100644
--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs
@@ -4,7 +4,7 @@ use thiserror::Error;
use crate::{
backends::common::{
Backend, BufferMut, BufferRef, CommandBuffer, Kernels,
- gpu_types::{EmbeddingTableKind, HADAMARD_TRANSFORM_BLOCK_SIZE, d4s4},
+ gpu_types::{d4s4, embedding::EmbeddingTableKind, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},
kernel::InputEmbeddingLookupKernel,
},
config::weight_matrix::{
diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
index 4f619f2df..6e4853a6d 100644
--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
@@ -16,7 +16,7 @@ use crate::{
config::weight_matrix::{AnyWeightMatrixSpec, Layout},
data_type::DataType,
encodable_block::{
- linear::{Gather, Linear, LinearInput},
+ linear::{Gather, Linear, LinearEncodeError, LinearInput},
weight_matrix::{WeightMatrix, WeightMatrixError},
},
parameters::{ParameterLoaderError, ParameterTree},
@@ -26,6 +26,8 @@ use crate::{
pub enum LinearMatmulError<B: Backend> {
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
+ #[error("Matmul error: {0}")]
+ Matmul(#[source] <<B::Kernels as Kernels>::MatmulKernel as MatmulKernel>::Error),
#[error("Parameter loading error: {0}")]
ParameterError(#[from] ParameterLoaderError<B>),
#[error("Weight matrix error: {0}")]
@@ -95,7 +97,7 @@ impl<B: Backend> LinearMatmul<B> {
let kernel =
<B::Kernels as Kernels>::MatmulKernel::new(context, weights_data_type, input_data_type, output_data_type)
- .map_err(LinearMatmulError::BackendError)?;
+ .map_err(LinearMatmulError::Matmul)?;
Ok(Self {
kernel: Mutex::new(kernel),
@@ -124,24 +126,29 @@ impl<B: Backend> LinearMatmul<B> {
batch_dim: u32,
gather: Option<Gather<impl BufferRef<Backend = B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {
let (output_dim, gather_indices) =
gather.map_or((self.output_dim, None), |gather| (gather.output_dim, Some(gather.indices)));
- let mut output = command_buffer.allocate_scratch_for_shape(&[batch_dim, output_dim], self.output_data_type)?;
- self.kernel.lock().encode(
- MatmulArguments {
- a,
- b: self.matmul_b(),
- b_leading_dimension: None,
- b_transpose: true,
- output: MatmulOutput::new(&mut output, self.d_ops()),
- gather_indices,
- m: batch_dim,
- n: output_dim,
- k: self.input_dim,
- },
- command_buffer,
- )?;
+ let mut output = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim, output_dim], self.output_data_type)
+ .map_err(LinearEncodeError::Backend)?;
+ self.kernel
+ .lock()
+ .encode(
+ MatmulArguments {
+ a,
+ b: self.matmul_b(),
+ b_leading_dimension: None,
+ b_transpose: true,
+ output: MatmulOutput::new(&mut output, self.d_ops()),
+ gather_indices,
+ m: batch_dim,
+ n: output_dim,
+ k: self.input_dim,
+ },
+ command_buffer,
+ )
+ .map_err(LinearEncodeError::Matmul)?;
Ok(output)
}
@@ -189,7 +196,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
input: LinearInput<B>,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {
self.encode_with_a(input.as_matmul_a(), batch_dim, None::<Gather<&B::ScratchBuffer>>, command_buffer)
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rs
index 7d42396b9..c151f8d5f 100644
--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs
@@ -12,11 +12,11 @@ pub use untied_readout::UntiedReadout;
use crate::{
backends::common::{
- Backend, BufferRef, CommandBuffer,
- gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ Backend, BufferRef, CommandBuffer, Kernels,
+ gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
kernel::{
ActivationQuantization,
- matmul::{ActivationFormat, Int8CodeLayout, MatmulA},
+ matmul::{ActivationFormat, Int8CodeLayout, MatmulA, MatmulKernel},
},
},
config::weight_matrix::{
@@ -27,13 +27,21 @@ use crate::{
parameters::{ParameterLoaderError, ParameterTree},
};
+#[derive(Debug, Error)]
+pub enum LinearEncodeError<B: Backend> {
+ #[error("Backend error: {0}")]
+ Backend(#[source] B::Error),
+ #[error("Matmul error: {0}")]
+ Matmul(#[source] <<B::Kernels as Kernels>::MatmulKernel as MatmulKernel>::Error),
+}
+
pub trait Linear<B: Backend>: Send + Sync {
fn encode(
&self,
input: B::ScratchBuffer,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {
self.encode_input(LinearInput::FullPrecision(input), batch_dim, command_buffer)
}
@@ -42,7 +50,7 @@ pub trait Linear<B: Backend>: Send + Sync {
input: LinearInput<B>,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error>;
+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>>;
fn select_activation_format(
&self,
diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
index a1e0d5a55..c97472e4c 100644
--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
@@ -2,14 +2,17 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- backends::common::{Backend, CommandBuffer, CommandBufferEncoding, gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE},
+ backends::common::{
+ Backend, CommandBuffer, CommandBufferEncoding, gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ },
config::weight_matrix::{
AnyWeightMatrixSpec,
hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
},
data_type::DataType,
encodable_block::linear::{
- Linear, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,
+ Gather, Linear, LinearEncodeError, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError,
+ input_rht::InputRht,
},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -178,7 +181,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
input: LinearInput<B>,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {
let input = match input {
LinearInput::FullPrecision(input) => input,
input => return self.inner_linear.encode_input(input, batch_dim, command_buffer),
@@ -186,11 +189,14 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
let command_buffer = &mut *command_buffer.debug_group("linear (rht)");
let format = self.inner_linear.select_activation_format(batch_dim, command_buffer.context());
- let input = self.input_rht.prepare_in_place(input, batch_dim, format, command_buffer)?;
+ let input = self
+ .input_rht
+ .prepare_in_place(input, batch_dim, format, command_buffer)
+ .map_err(LinearEncodeError::Backend)?;
let output = self.inner_linear.encode_with_a(
input.as_matmul_a(),
batch_dim,
- None::<super::Gather<&B::ScratchBuffer>>,
+ None::<Gather<&B::ScratchBuffer>>,
command_buffer,
)?;
diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
index 12d6e565d..169a7a95a 100644
--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
@@ -1,7 +1,7 @@
use crate::{
backends::common::{
Backend, BufferRef, CommandBuffer, CommandBufferEncoding,
- gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
kernel::matmul::{ActivationFormat, MatmulA},
},
config::weight_matrix::{
@@ -10,7 +10,7 @@ use crate::{
},
data_type::DataType,
encodable_block::linear::{
- Gather, Linear, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,
+ Gather, Linear, LinearEncodeError, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,
},
parameters::ParameterTree,
};
@@ -80,7 +80,7 @@ impl<B: Backend> UntiedReadout<B> {
batch_dim: u32,
gather: Option<Gather<impl BufferRef<Backend = B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {
let Some(input_rht) = &self.input_rht else {
let a = MatmulA::FullPrecision {
values: input,
@@ -94,7 +94,7 @@ impl<B: Backend> UntiedReadout<B> {
} else {
self.linear.select_activation_format(batch_dim, command_buffer.context())
};
- let input = input_rht.prepare(input, batch_dim, format, command_buffer)?;
+ let input = input_rht.prepare(input, batch_dim, format, command_buffer).map_err(LinearEncodeError::Backend)?;
self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
index 44f2706bb..2acb217df 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
@@ -10,9 +10,9 @@ use crate::{
data_type::DataType,
encodable_block::{
batch_topology::BatchTopology,
- linear::{Linear, LinearBlockError},
+ linear::{Linear, LinearBlockError, LinearEncodeError},
mixer::{
- Mixer, MixerState,
+ Mixer, MixerEncodeError, MixerState,
attention::{
mode::LinearProjection,
qkv_norm::{QKVNorm, QKVNormError},
@@ -50,6 +50,8 @@ pub struct Attention<B: Backend> {
#[derive(Debug, Error)]
pub enum AttentionNewError<B: Backend> {
+ #[error("Attention kernel error: {0}")]
+ Kernel(#[source] <<B::Kernels as Kernels>::AttentionKernel as AttentionKernel>::Error),
#[error("Backend error: {0}")]
Backend(#[source] B::Error),
#[error("Parameter loader error: {0}")]
@@ -60,6 +62,16 @@ pub enum AttentionNewError<B: Backend> {
QKVNorm(#[from] QKVNormError<B>),
}
+#[derive(Debug, Error)]
+pub enum AttentionEncodeError<B: Backend> {
+ #[error("Backend error: {0}")]
+ Backend(#[source] B::Error),
+ #[error("Linear error: {0}")]
+ Linear(#[from] LinearEncodeError<B>),
+ #[error("Attention kernel error: {0}")]
+ Kernel(#[source] <<B::Kernels as Kernels>::AttentionKernel as AttentionKernel>::Error),
+}
+
impl<B: Backend> Attention<B> {
pub fn new(
hidden_dim: u32,
@@ -152,7 +164,7 @@ impl<B: Backend> Attention<B> {
data_type,
},
)
- .map_err(AttentionNewError::Backend)?;
+ .map_err(AttentionNewError::Kernel)?;
let out_projection = <dyn Linear<B>>::new(
q_dim,
@@ -211,7 +223,7 @@ impl<B: Backend> Mixer<B> for Attention<B> {
batch_dim: &BatchTopology,
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {
let command_buffer = &mut *command_buffer.debug_group("attention");
assert_eq!(precalculated_rope.is_some(), self.max_rope_length.is_some(), "precalculated rope mismatch");
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
index d937a5e3c..881b08334 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
@@ -8,7 +8,10 @@ use crate::{
linear::Linear,
mixer::{
MixerState,
- attention::{Attention, KVCacheView, qkv_norm::QKVNorm, rope::PrecalculatedRoPE, state::AttentionState},
+ attention::{
+ Attention, AttentionEncodeError, KVCacheView, qkv_norm::QKVNorm, rope::PrecalculatedRoPE,
+ state::AttentionState,
+ },
},
},
utils::maybe_mut::MaybeMut,
@@ -25,7 +28,7 @@ impl<B: Backend> LinearProjection<B> {
hidden: B::ScratchBuffer,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, AttentionEncodeError<B>> {
let mut projected = self.lin.encode(hidden, batch_dim, command_buffer)?;
self.norm.encode(&mut projected, batch_dim, command_buffer);
Ok(projected)
@@ -40,7 +43,7 @@ impl<B: Backend> Attention<B> {
batch_dim: &BatchTopology,
state: Option<MaybeMut<AttentionState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, AttentionEncodeError<B>> {
let qkvg = self.projection.project(hidden, batch_dim.size(), command_buffer)?;
let gate = self.has_gate.then(|| {
qkvg.subrange(
@@ -51,21 +54,25 @@ impl<B: Backend> Attention<B> {
let attention_output = match state {
Some(MaybeMut::Mut(state)) => {
let prefix_len = state.view().prefix_len();
- let queries = self.prepare_kv_and_queries(
- &qkvg,
- state.keys.as_mut(),
- state.values.as_mut(),
- prefix_len,
- self.num_q_heads,
- precalculated_rope,
- batch_dim.size(),
- command_buffer,
- )?;
+ let queries = self
+ .prepare_kv_and_queries(
+ &qkvg,
+ state.keys.as_mut(),
+ state.values.as_mut(),
+ prefix_len,
+ self.num_q_heads,
+ precalculated_rope,
+ batch_dim.size(),
+ command_buffer,
+ )
+ .map_err(AttentionEncodeError::Backend)?;
self.run_core(&queries, gate, batch_dim, state, command_buffer)?
},
Some(MaybeMut::Const(state)) => {
// KV sharing: QKVG contains queries and an optional gate only.
- let queries = self.prepare_queries(&qkvg, precalculated_rope, batch_dim.size(), command_buffer)?;
+ let queries = self
+ .prepare_queries(&qkvg, precalculated_rope, batch_dim.size(), command_buffer)
+ .map_err(AttentionEncodeError::Backend)?;
self.run_core(&queries, gate, batch_dim, state, command_buffer)?
},
None => {
@@ -75,40 +82,48 @@ impl<B: Backend> Attention<B> {
assert!(batch_dim.is_flat(), "stateless attention doesn't support trie");
let mut keys = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)?;
+ .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)
+ .map_err(AttentionEncodeError::Backend)?;
let mut values = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)?;
+ .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)
+ .map_err(AttentionEncodeError::Backend)?;
- let queries = self.prepare_kv_and_queries(
- &qkvg,
- &mut keys,
- &mut values,
- 0,
- self.num_q_heads,
- precalculated_rope,
- batch_dim.size(),
- command_buffer,
- )?;
+ let queries = self
+ .prepare_kv_and_queries(
+ &qkvg,
+ &mut keys,
+ &mut values,
+ 0,
+ self.num_q_heads,
+ precalculated_rope,
+ batch_dim.size(),
+ command_buffer,
+ )
+ .map_err(AttentionEncodeError::Backend)?;
let cache = self.ring_capacity.map_or_else(|| KVCacheView::full(0), |_| KVCacheView::ring(0, 0));
- self.kernel.encode(
- AttentionArguments {
- queries: &queries,
- keys: &keys,
- values: &values,
- suffix_length: batch_dim.size(),
- trie: None::<&B::ConstantBuffer>,
- sinks: self.sinks.as_ref(),
- cache,
- gate,
- },
- command_buffer,
- )?
+ self.kernel
+ .encode(
+ AttentionArguments {
+ queries: &queries,
+ keys: &keys,
+ values: &values,
+ suffix_length: batch_dim.size(),
+ trie: None::<&B::ConstantBuffer>,
+ sinks: self.sinks.as_ref(),
+ cache,
+ gate,
+ },
+ command_buffer,
+ )
+ .map_err(AttentionEncodeError::Kernel)?
},
};
- self.out_projection.encode(attention_output, batch_dim.size(), command_buffer)
+ self.out_projection
+ .encode(attention_output, batch_dim.size(), command_buffer)
+ .map_err(AttentionEncodeError::Linear)
}
pub fn append_projected_kv(
@@ -142,26 +157,32 @@ impl<B: Backend> Attention<B> {
batch_dim: &BatchTopology,
state: &AttentionState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, AttentionEncodeError<B>> {
let trie = if batch_dim.is_flat() {
None
} else {
- Some(command_buffer.allocate_constant_from_slice(batch_dim.nodes())?)
+ Some(
+ command_buffer
+ .allocate_constant_from_slice(batch_dim.nodes())
+ .map_err(AttentionEncodeError::Backend)?,
+ )
};
- self.kernel.encode(
- AttentionArguments {
- queries,
- keys: state.keys.as_ref(),
- values: state.values.as_ref(),
- suffix_length: batch_dim.size(),
- trie: trie.as_ref(),
- sinks: self.sinks.as_ref(),
- cache: state.view(),
- gate,
- },
- command_buffer,
- )
+ self.kernel
+ .encode(
+ AttentionArguments {
+ queries,
+ keys: state.keys.as_ref(),
+ values: state.values.as_ref(),
+ suffix_length: batch_dim.size(),
+ trie: trie.as_ref(),
+ sinks: self.sinks.as_ref(),
+ cache: state.view(),
+ gate,
+ },
+ command_buffer,
+ )
+ .map_err(AttentionEncodeError::Kernel)
}
fn prepare_kv_and_queries(
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
index 967fc08cd..e43044655 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
@@ -4,7 +4,7 @@ use crate::{
array::size_for_shape,
backends::common::{
Backend, Buffer, CommandBuffer, Context, DeviceCapabilities, Kernels, SparseBuffer,
- gpu_types::{Copy, ring::RingParams},
+ gpu_types::{kv_cache_update::Copy, ring::RingParams},
kernel::KVCacheUpdateKernel,
},
data_type::DataType,
diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
index 46f4c81ec..d7b9ec6ed 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
@@ -20,7 +20,7 @@ use crate::{
batch_topology::BatchTopology,
linear::{Linear, LinearBlockError},
mixer::{
- Mixer, MixerState,
+ Mixer, MixerEncodeError, MixerState,
attention::rope::PrecalculatedRoPE,
delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},
},
@@ -333,22 +333,35 @@ impl<B: Backend> DeltaNet<B> {
batch_dim: &BatchTopology,
state: &mut DeltaNetState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {
let tree_verify = self.tree_verify.as_ref().expect("DeltaNet tree verification is unsupported");
let tree_size = batch_dim.size();
- let parents = command_buffer.allocate_constant_from_slice(batch_dim.parents())?;
- let trie = command_buffer.allocate_constant_from_slice(batch_dim.nodes())?;
+ let parents =
+ command_buffer.allocate_constant_from_slice(batch_dim.parents()).map_err(MixerEncodeError::Backend)?;
+ let trie = command_buffer.allocate_constant_from_slice(batch_dim.nodes()).map_err(MixerEncodeError::Backend)?;
let mut conv_states = command_buffer
- .allocate_scratch_for_shape(&[tree_size, self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE)?;
- let mut k = command_buffer.allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)?;
- let mut v = command_buffer.allocate_scratch_for_shape(&[tree_size, self.value_dim], self.outer_data_type)?;
- let mut beta = command_buffer.allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)?;
- let mut log_decay = command_buffer.allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)?;
-
- let mut tree_projected =
- command_buffer.allocate_scratch_for_shape(&[tree_size, self.total_proj_dim], self.outer_data_type)?;
- let mut q = command_buffer.allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)?;
+ .allocate_scratch_for_shape(&[tree_size, self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
+ let mut k = command_buffer
+ .allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)
+ .map_err(MixerEncodeError::Backend)?;
+ let mut v = command_buffer
+ .allocate_scratch_for_shape(&[tree_size, self.value_dim], self.outer_data_type)
+ .map_err(MixerEncodeError::Backend)?;
+ let mut beta = command_buffer
+ .allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
+ let mut log_decay = command_buffer
+ .allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
+
+ let mut tree_projected = command_buffer
+ .allocate_scratch_for_shape(&[tree_size, self.total_proj_dim], self.outer_data_type)
+ .map_err(MixerEncodeError::Backend)?;
+ let mut q = command_buffer
+ .allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)
+ .map_err(MixerEncodeError::Backend)?;
self.conv_tree_scan.encode(
in_projected,
@@ -381,19 +394,21 @@ impl<B: Backend> DeltaNet<B> {
command_buffer,
);
- let mut delta_output = tree_verify.encode(
- TreeVerifyEncodeArguments {
- q: &q,
- k: &k,
- v: &v,
- trie: &trie,
- log_decay: &log_decay,
- beta: &beta,
- h0: &state.ssm_state,
- tree_size,
- },
- command_buffer,
- )?;
+ let mut delta_output = tree_verify
+ .encode(
+ TreeVerifyEncodeArguments {
+ q: &q,
+ k: &k,
+ v: &v,
+ trie: &trie,
+ log_decay: &log_decay,
+ beta: &beta,
+ h0: &state.ssm_state,
+ tree_size,
+ },
+ command_buffer,
+ )
+ .map_err(MixerEncodeError::Backend)?;
self.delta_net_norm_gate.encode(
&mut delta_output,
&tree_projected,
@@ -469,7 +484,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
batch_dim: &BatchTopology,
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {
let command_buffer = &mut *command_buffer.debug_group("delta net");
assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");
@@ -490,8 +505,9 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
return Ok(output);
}
- let mut delta_output =
- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type)?;
+ let mut delta_output = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type)
+ .map_err(MixerEncodeError::Backend)?;
if batch_dim.size() == 1 {
self.conv_update.encode(
&self.conv_weight,
@@ -520,10 +536,12 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
command_buffer,
);
} else {
- let mut padded = command_buffer.allocate_scratch_for_shape(
- &[batch_dim.size() + self.kernel_size - 1, self.total_proj_dim],
- INNER_DATA_TYPE,
- )?;
+ let mut padded = command_buffer
+ .allocate_scratch_for_shape(
+ &[batch_dim.size() + self.kernel_size - 1, self.total_proj_dim],
+ INNER_DATA_TYPE,
+ )
+ .map_err(MixerEncodeError::Backend)?;
self.conv_pack.encode(
&state.conv_state,
&in_projected,
@@ -549,31 +567,37 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
command_buffer,
);
if let Some(chunked) = self.chunked.as_ref().filter(|chunked| chunked.should_use(batch_dim.size())) {
- chunked.encode(
- DeltaNetChunkedPrefillArgs {
- in_projected: &in_projected,
- a_log: &self.a_log,
- dt_bias: &self.dt_bias,
- ssm_state: &mut state.ssm_state,
- delta_output: &mut delta_output,
- num_heads: self.num_heads,
- num_groups: self.num_groups,
- value_head_dim: self.value_head_dim,
- key_dim: self.key_dim,
- value_dim: self.value_dim,
- suffix_len: batch_dim.size(),
- },
- command_buffer,
- )?;
+ chunked
+ .encode(
+ DeltaNetChunkedPrefillArgs {
+ in_projected: &in_projected,
+ a_log: &self.a_log,
+ dt_bias: &self.dt_bias,
+ ssm_state: &mut state.ssm_state,
+ delta_output: &mut delta_output,
+ num_heads: self.num_heads,
+ num_groups: self.num_groups,
+ value_head_dim: self.value_head_dim,
+ key_dim: self.key_dim,
+ value_dim: self.value_dim,
+ suffix_len: batch_dim.size(),
+ },
+ command_buffer,
+ )
+ .map_err(MixerEncodeError::Backend)?;
} else {
- let mut prep_q_norm =
- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)?;
- let mut prep_k_norm =
- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)?;
- let mut prep_beta =
- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;
- let mut prep_decay =
- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;
+ let mut prep_q_norm = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
+ let mut prep_k_norm = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
+ let mut prep_beta = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
+ let mut prep_decay = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
self.delta_net_prefill_prep.encode(
&in_projected,
&self.a_log,
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
index 038109caa..6bf011646 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
@@ -5,7 +5,7 @@ use crate::{
array::size_for_shape,
backends::common::{
Backend, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::ActivationType,
+ gpu_types::activation_type::ActivationType,
kernel::{
Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDPrefill64Kernel, SSDPrefillKernel,
SSDUpdateKernel, SplitInProjKernel,
@@ -16,7 +16,7 @@ use crate::{
encodable_block::{
batch_topology::BatchTopology,
linear::{Linear, LinearBlockError},
- mixer::{Mixer, MixerState, attention::rope::PrecalculatedRoPE},
+ mixer::{Mixer, MixerEncodeError, MixerState, attention::rope::PrecalculatedRoPE},
},
parameters::{ParameterLoaderError, ParameterTree},
utils::maybe_mut::MaybeMut,
@@ -242,7 +242,7 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
batch_dim: &BatchTopology,
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {
let command_buffer = &mut *command_buffer.debug_group("mamba2");
assert!(precalculated_rope.is_none(), "unexpected rope for mamba2 mixer");
@@ -261,12 +261,15 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
let in_projected = self.in_projection.encode(hidden, batch_dim.size(), command_buffer)?;
- let mut conv_inputs =
- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.conv_dim], INNER_DATA_TYPE)?;
+ let mut conv_inputs = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.conv_dim], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
let mut gate = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)?;
- let mut time_step =
- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
+ let mut time_step = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
self.split_inproj.encode(
&in_projected,
&mut conv_inputs,
@@ -282,11 +285,14 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
);
let mut conv_x = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)?;
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
let mut state_b = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)?;
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
let mut state_c = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)?;
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
let state_stride = self.kernel_size - 1;
if batch_dim.size() == 1 {
self.conv_decode.encode(
@@ -310,7 +316,8 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
);
} else {
let mut padded = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size() + state_stride, self.conv_dim], INNER_DATA_TYPE)?;
+ .allocate_scratch_for_shape(&[batch_dim.size() + state_stride, self.conv_dim], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
self.conv_pack.encode(
&state.conv_state,
&conv_inputs,
@@ -341,8 +348,9 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
);
}
- let mut ssd_output =
- command_buffer.allocate_scratch_for_shape(&[batch_dim.size(), self.inner_dim], INNER_DATA_TYPE)?;
+ let mut ssd_output = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.inner_dim], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
let x_strides = [self.num_heads * self.head_dim, self.head_dim, 1];
let dt_strides = [self.num_heads, 1];
let cb_strides = [self.num_groups * self.state_dim, self.state_dim, 1];
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
index d4250ce14..772c4b39e 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
@@ -9,8 +9,9 @@ use crate::{
data_type::DataType,
encodable_block::{
batch_topology::BatchTopology,
+ linear::LinearEncodeError,
mixer::{
- attention::{Attention, AttentionNewError, rope::PrecalculatedRoPE},
+ attention::{Attention, AttentionEncodeError, AttentionNewError, rope::PrecalculatedRoPE},
delta_net::{DeltaNet, DeltaNetNewError},
mamba2::{Mamba2, Mamba2NewError},
short_conv::{ShortConv, ShortConvNewError},
@@ -48,6 +49,16 @@ impl<'a, B: Backend> MaybeMut<'a, dyn MixerState<B>> {
}
}
+#[derive(Debug, Error)]
+pub enum MixerEncodeError<B: Backend> {
+ #[error("Backend error: {0}")]
+ Backend(#[source] B::Error),
+ #[error("Linear error: {0}")]
+ Linear(#[from] LinearEncodeError<B>),
+ #[error("Attention error: {0}")]
+ Attention(#[from] AttentionEncodeError<B>),
+}
+
pub trait Mixer<B: Backend>: Any + Send + Sync {
fn speculation_supported(&self) -> bool;
@@ -66,7 +77,7 @@ pub trait Mixer<B: Backend>: Any + Send + Sync {
batch_dim: &BatchTopology,
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error>;
+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>>;
}
#[derive(Debug, Error)]
diff --git a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
index 7bc2dd839..817d74e70 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
@@ -13,7 +13,7 @@ use crate::{
encodable_block::{
batch_topology::BatchTopology,
linear::{Linear, LinearBlockError},
- mixer::{Mixer, MixerState, attention::rope::PrecalculatedRoPE},
+ mixer::{Mixer, MixerEncodeError, MixerState, attention::rope::PrecalculatedRoPE},
},
parameters::{ParameterLoaderError, ParameterTree},
utils::maybe_mut::MaybeMut,
@@ -307,7 +307,7 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {
batch_dim: &BatchTopology,
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {
let command_buffer = &mut *command_buffer.debug_group("short conv");
assert!(precalculated_rope.is_none(), "unexpected rope for short conv mixer");
@@ -324,18 +324,21 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {
let conv_output = if batch_dim.full_accept() {
let conv_output = if batch_dim.size() == 1 {
- self.encode_decode_conv(&in_projected, state, command_buffer)?
+ self.encode_decode_conv(&in_projected, state, command_buffer).map_err(MixerEncodeError::Backend)?
} else {
- self.encode_prefill_conv(&in_projected, batch_dim.size(), state, command_buffer)?
+ self.encode_prefill_conv(&in_projected, batch_dim.size(), state, command_buffer)
+ .map_err(MixerEncodeError::Backend)?
};
state.suffix_state = Some(ShortConvStateSuffixStatus::Flat {
suffix_length: batch_dim.size(),
});
conv_output
} else {
- let token_parents = command_buffer.allocate_constant_from_slice(batch_dim.parents())?;
- let (conv_output, conv_states) =
- self.encode_trie_conv(&in_projected, &token_parents, batch_dim.size(), state, command_buffer)?;
+ let token_parents =
+ command_buffer.allocate_constant_from_slice(batch_dim.parents()).map_err(MixerEncodeError::Backend)?;
+ let (conv_output, conv_states) = self
+ .encode_trie_conv(&in_projected, &token_parents, batch_dim.size(), state, command_buffer)
+ .map_err(MixerEncodeError::Backend)?;
state.suffix_state = Some(ShortConvStateSuffixStatus::Trie {
conv_states,
});
diff --git a/crates/uzu-engine/src/encodable_block/mlp/dense.rs b/crates/uzu-engine/src/encodable_block/mlp/dense.rs
index 31ecba34d..2649f4a3a 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/dense.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/dense.rs
@@ -9,7 +9,7 @@ use crate::{
data_type::DataType,
encodable_block::{
linear::{Linear, LinearBlockError},
- mlp::{Mlp, gate_act_mul::MlpGateActMulEncodable},
+ mlp::{Mlp, MlpEncodeError, gate_act_mul::MlpGateActMulEncodable},
},
parameters::ParameterTree,
};
@@ -83,12 +83,15 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {
input: B::ScratchBuffer,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, MlpEncodeError<B>> {
let command_buffer = &mut *command_buffer.debug_group("mlp (dense)");
let fused_up = self.up.encode(input, batch_dim, command_buffer)?;
let act_format = self.down.select_activation_format(batch_dim, command_buffer.context());
- let down_input = self.gate.encode_for_linear(&fused_up, batch_dim, act_format, command_buffer)?;
+ let down_input = self
+ .gate
+ .encode_for_linear(&fused_up, batch_dim, act_format, command_buffer)
+ .map_err(MlpEncodeError::Backend)?;
let output = self.down.encode_input(down_input, batch_dim, command_buffer)?;
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
index 1e96ba139..25e0d78ba 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
@@ -1,7 +1,7 @@
use crate::{
backends::common::{
Backend, BufferRef, CommandBuffer, CommandBufferEncoding,
- gpu_types::ActivationType,
+ gpu_types::activation_type::ActivationType,
kernel::{ActivationQuantization, GatedActMul, GatedActMulSettings, matmul::ActivationFormat},
},
config::{activation::AnyActivation, clipping::ClippingBounds},
diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
index 988a178e2..4dbaa9209 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
@@ -8,17 +8,28 @@ use crate::{
backends::common::{Backend, CommandBuffer},
config::mlp::AnyMLPConfig,
data_type::DataType,
- encodable_block::mlp::dense::{DenseMlp, DenseMlpNewError},
+ encodable_block::{
+ linear::LinearEncodeError,
+ mlp::dense::{DenseMlp, DenseMlpNewError},
+ },
parameters::ParameterTree,
};
+#[derive(Debug, Error)]
+pub enum MlpEncodeError<B: Backend> {
+ #[error("Backend error: {0}")]
+ Backend(#[source] B::Error),
+ #[error("Linear error: {0}")]
+ Linear(#[from] LinearEncodeError<B>),
+}
+
pub trait Mlp<B: Backend>: Send + Sync {
fn encode(
&self,
input: B::ScratchBuffer,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error>;
+ ) -> Result<B::ScratchBuffer, MlpEncodeError<B>>;
}
#[derive(Debug, Error)]
diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
index c95f24cbe..f3aa283f8 100644
--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
@@ -13,7 +13,7 @@ use crate::{
data_type::DataType,
encodable_block::{
embedding_table::{EmbeddingTable, EmbeddingTableError},
- linear::{Linear, LinearBlockError},
+ linear::{Linear, LinearBlockError, LinearEncodeError},
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
},
parameters::ParameterTree,
@@ -27,6 +27,8 @@ pub enum PerLayerEmbeddingError<B: Backend> {
Normalization(#[from] NormalizationNewError<B>),
#[error("Linear error: {0}")]
LinearError(#[from] LinearBlockError<B>),
+ #[error("Linear encoding error: {0}")]
+ LinearEncode(#[from] LinearEncodeError<B>),
#[error("Embedding table error: {0}")]
EmbeddingTable(#[from] EmbeddingTableError<B>),
}
@@ -109,14 +111,16 @@ impl<B: Backend> PerLayerEmbedding<B> {
inner_features: impl BufferRef<Backend = B>,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, PerLayerEmbeddingError<B>> {
let command_buffer = &mut *command_buffer.debug_group("per layer embedding");
let total_ple_dim = self.num_layers * self.ple_dim;
let total_rows = batch_dim * self.num_layers;
let total_elements = batch_dim * total_ple_dim;
- let mut token_ple = command_buffer.allocate_scratch_for_shape(&[batch_dim, total_ple_dim], self.data_type)?;
+ let mut token_ple = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim, total_ple_dim], self.data_type)
+ .map_err(PerLayerEmbeddingError::BackendError)?;
self.token_embedding.encode_lookup(
token_ids,
&mut token_ple,
@@ -125,21 +129,20 @@ impl<B: Backend> PerLayerEmbedding<B> {
command_buffer,
);
- let mut model_projection_input =
- command_buffer.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;
+ let mut model_projection_input = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)
+ .map_err(PerLayerEmbeddingError::BackendError)?;
command_buffer.encode_copy(inner_features, &mut model_projection_input);
let model_projected = self.model_projection.encode(model_projection_input, batch_dim, command_buffer)?;
- let model_normed = self.projection_norm.encode(
- &model_projected,
- 0,
- total_rows,
- None::<&mut B::ScratchBuffer>,
- command_buffer,
- )?;
+ let model_normed = self
+ .projection_norm
+ .encode(&model_projected, 0, total_rows, None::<&mut B::ScratchBuffer>, command_buffer)
+ .map_err(PerLayerEmbeddingError::BackendError)?;
- let mut per_layer_inputs =
- command_buffer.allocate_scratch_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type)?;
+ let mut per_layer_inputs = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type)
+ .map_err(PerLayerEmbeddingError::BackendError)?;
self.add_scale.encode(
Some(&token_ple),
&model_normed,
@@ -242,7 +245,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
layer_index: u32,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error> {
+ ) -> Result<(), PerLayerEmbeddingError<B>> {
let command_buffer = &mut *command_buffer.debug_group("per layer embedding projection");
let length = batch_dim * self.model_dim;
@@ -256,11 +259,15 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
command_buffer,
);
- let mut gate_input = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;
+ let mut gate_input = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)
+ .map_err(PerLayerEmbeddingError::BackendError)?;
command_buffer.encode_copy(outputs.as_ref(), &mut gate_input);
let gate_out = self.gate.encode(gate_input, batch_dim, command_buffer)?;
- let mut activated = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.ple_dim], self.data_type)?;
+ let mut activated = command_buffer
+ .allocate_scratch_for_shape(&[batch_dim, self.ple_dim], self.data_type)
+ .map_err(PerLayerEmbeddingError::BackendError)?;
self.gate_act_mul.encode_fp(
&gate_out,
Some(per_layer_input),
@@ -275,7 +282,10 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
);
let projected = self.projection.encode(activated, batch_dim, command_buffer)?;
- let normed = self.norm.encode(&projected, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)?;
+ let normed = self
+ .norm
+ .encode(&projected, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)
+ .map_err(PerLayerEmbeddingError::BackendError)?;
self.residual_combine.encode(
None::<&B::ScratchBuffer>,
diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rs
index 913ac5997..e28394451 100644
--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs
+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs
@@ -4,13 +4,13 @@ use thiserror::Error;
use crate::{
backends::common::{
Backend, CommandBuffer, CommandBufferEncoding,
- gpu_types::ActivationType,
+ gpu_types::activation_type::ActivationType,
kernel::{ActivationKernel, Kernels},
},
config::classifier::PredictionHeadConfig,
data_type::DataType,
encodable_block::{
- linear::{Linear, LinearBlockError},
+ linear::{Linear, LinearBlockError, LinearEncodeError},
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
},
parameters::ParameterTree,
@@ -22,6 +22,8 @@ pub enum PredictionHeadError<B: Backend> {
Backend(#[source] B::Error),
#[error("Linear error: {0}")]
Linear(#[from] LinearBlockError<B>),
+ #[error("Linear encoding error: {0}")]
+ LinearEncode(#[from] LinearEncodeError<B>),
#[error("Normalization error: {0}")]
Normalization(#[from] NormalizationNewError<B>),
}
@@ -86,7 +88,7 @@ impl<B: Backend> PredictionHead<B> {
input: B::ScratchBuffer,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, PredictionHeadError<B>> {
let command_buffer = &mut *command_buffer.debug_group("prediction head");
let mut hidden = self.dense_projection.encode(input, batch_dim, command_buffer)?;
@@ -97,8 +99,10 @@ impl<B: Backend> PredictionHead<B> {
self.activation,
command_buffer,
);
- let normalized =
- self.normalization.encode(&hidden, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)?;
+ let normalized = self
+ .normalization
+ .encode(&hidden, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)
+ .map_err(PredictionHeadError::Backend)?;
let logits = self.readout.encode(normalized, batch_dim, command_buffer)?;
Ok(logits)
diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rs
index b541d3be9..a3a9a0ed3 100644
--- a/crates/uzu-engine/src/encodable_block/transformer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer.rs
@@ -90,6 +90,14 @@ pub enum TransformerNewError<B: Backend> {
Normalization(#[from] NormalizationNewError<B>),
}
+#[derive(Debug, Error)]
+pub enum TransformerEncodeError<B: Backend> {
+ #[error("Backend error: {0}")]
+ Backend(#[source] B::Error),
+ #[error("Layer error: {0}")]
+ Layer(#[from] TransformerLayerError<B>),
+}
+
pub struct Transformer<B: Backend> {
ropes: Box<[AnyRoPEConfig]>,
layers: Box<[(TransformerLayer<B>, Option<usize>)]>,
@@ -236,7 +244,7 @@ impl<B: Backend> Transformer<B> {
hidden_feature_layer_indices: Option<&[u32]>,
mut state: Option<&mut TransformerState<B>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<TransformerEncodeOutput<B>, B::Error> {
+ ) -> Result<TransformerEncodeOutput<B>, TransformerEncodeError<B>> {
let mut hidden = input;
let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_none() {
self.prefill_cache_layer_count()
@@ -244,7 +252,7 @@ impl<B: Backend> Transformer<B> {
self.layers.len()
};
- let mut shortcut = command_buffer.allocate_scratch(hidden.size())?;
+ let mut shortcut = command_buffer.allocate_scratch(hidden.size()).map_err(TransformerEncodeError::Backend)?;
let mut hidden_features =
hidden_feature_layer_indices.map(|indices| (0..indices.len()).map(|_| None).collect::<Vec<_>>());
@@ -255,7 +263,8 @@ impl<B: Backend> Transformer<B> {
.ropes
.iter()
.map(|rope_config| PrecalculatedRoPE::precalculate(rope_config, &token_positions, command_buffer))
- .collect::<Result<Box<[_]>, B::Error>>()?;
+ .collect::<Result<Box<[_]>, B::Error>>()
+ .map_err(TransformerEncodeError::Backend)?;
for (layer, layer_rope_index) in self.layers.iter().take(layer_count) {
let precalculated_rope = layer_rope_index.map(|i| &precalculated_ropes[i]);
@@ -289,7 +298,9 @@ impl<B: Backend> Transformer<B> {
if let (Some(hidden_features), Some(indices)) = (&mut hidden_features, hidden_feature_layer_indices) {
for (feature_index, &layer_index) in indices.iter().enumerate() {
if layer_index == layer.layer_index {
- let feature = self.capture_residual(&shortcut, &hidden, batch_dim.size(), command_buffer)?;
+ let feature = self
+ .capture_residual(&shortcut, &hidden, batch_dim.size(), command_buffer)
+ .map_err(TransformerEncodeError::Backend)?;
hidden_features[feature_index] = Some(feature);
}
}
@@ -318,13 +329,16 @@ impl<B: Backend> Transformer<B> {
});
};
- let output_normalized = self.output_norm.encode(
- &hidden,
- output_range.start,
- output_range.end - output_range.start,
- Some(&mut shortcut),
- command_buffer,
- )?;
+ let output_normalized = self
+ .output_norm
+ .encode(
+ &hidden,
+ output_range.start,
+ output_range.end - output_range.start,
+ Some(&mut shortcut),
+ command_buffer,
+ )
+ .map_err(TransformerEncodeError::Backend)?;
Ok(TransformerEncodeOutput {
output: Some(output_normalized),
diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
index 928cf5e90..dff07cc0d 100644
--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
@@ -10,11 +10,11 @@ use crate::{
encodable_block::{
batch_topology::BatchTopology,
convolution::{ConvolutionNewError, SeparableCausalConv},
- linear::{Linear, LinearBlockError},
- mixer::{Mixer, MixerNewError, MixerState, attention::rope::PrecalculatedRoPE},
- mlp::{Mlp, MlpBlockNewError},
+ linear::{Linear, LinearBlockError, LinearEncodeError},
+ mixer::{Mixer, MixerEncodeError, MixerNewError, MixerState, attention::rope::PrecalculatedRoPE},
+ mlp::{Mlp, MlpBlockNewError, MlpEncodeError},
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
- per_layer_embedding::PerLayerEmbeddingProjection,
+ per_layer_embedding::{PerLayerEmbeddingError, PerLayerEmbeddingProjection},
},
parameters::{ParameterLoaderError, ParameterTree},
utils::maybe_mut::MaybeMut,
@@ -28,14 +28,22 @@ pub enum TransformerLayerError<B: Backend> {
ParameterLoader(#[from] ParameterLoaderError<B>),
#[error("Mixer error: {0}")]
Mixer(#[from] MixerNewError<B>),
+ #[error("Mixer encoding error: {0}")]
+ MixerEncode(#[from] MixerEncodeError<B>),
#[error("MLP error: {0}")]
MlpBlock(#[from] MlpBlockNewError<B>),
+ #[error("MLP encoding error: {0}")]
+ MlpEncode(#[from] MlpEncodeError<B>),
#[error("Normalization error: {0}")]
Normalization(#[from] NormalizationNewError<B>),
#[error("Convolution error: {0}")]
Convolution(#[from] ConvolutionNewError<B>),
#[error("Linear error: {0}")]
Linear(#[from] LinearBlockError<B>),
+ #[error("Linear encoding error: {0}")]
+ LinearEncode(#[from] LinearEncodeError<B>),
+ #[error("Per-layer embedding error: {0}")]
+ PerLayerEmbedding(#[from] PerLayerEmbeddingError<B>),
#[error("Layer {layer_index} sets post_layer_scalar but has no post_mlp_norm")]
PostLayerScalarWithoutPostMlpNorm {
layer_index: u32,
@@ -128,18 +136,15 @@ impl<B: Backend> TransformerLayerConv<B> {
input: impl BufferRef<Backend = B>,
sequence_length: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(B::ScratchBuffer, B::ScratchBuffer), B::Error> {
- let mut projection_input = command_buffer.allocate_scratch(input.size())?;
+ ) -> Result<(B::ScratchBuffer, B::ScratchBuffer), TransformerLayerError<B>> {
+ let mut projection_input =
+ command_buffer.allocate_scratch(input.size()).map_err(TransformerLayerError::Backend)?;
command_buffer.encode_copy(input, &mut projection_input);
let coefficients = self.kernel_projection.encode(projection_input, sequence_length, command_buffer)?;
- let mut output = self.pre_conv.encode(
- input,
- &coefficients,
- 2 * self.coefficient_count,
- 0,
- sequence_length,
- command_buffer,
- )?;
+ let mut output = self
+ .pre_conv
+ .encode(input, &coefficients, 2 * self.coefficient_count, 0, sequence_length, command_buffer)
+ .map_err(TransformerLayerError::Backend)?;
if let Some((transform, factors)) = &self.input_rht {
transform.encode_fp_in_place(&mut output, factors, None, sequence_length, self.model_dim, command_buffer);
}
@@ -349,11 +354,13 @@ impl<B: Backend> TransformerLayer<B> {
batch_dim: &BatchTopology,
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, TransformerLayerError<B>> {
let command_buffer = &mut *command_buffer.debug_group(&format!("transformer layer {}", self.layer_index));
let mut hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {
- pre_mixer_norm.encode(&input, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)?
+ pre_mixer_norm
+ .encode(&input, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)
+ .map_err(TransformerLayerError::Backend)?
} else {
assert!(self.layer_index == 0);
command_buffer.encode_copy(&input, shortcut.reborrow());
@@ -374,15 +381,21 @@ impl<B: Backend> TransformerLayer<B> {
if let Some(coefficients) = mixer_coefficients {
let convolution = self.mixer_conv.as_ref().expect("mixer convolution required");
- hidden = convolution.encode_post_convolution(&hidden, &coefficients, batch_dim.size(), command_buffer)?;
+ hidden = convolution
+ .encode_post_convolution(&hidden, &coefficients, batch_dim.size(), command_buffer)
+ .map_err(TransformerLayerError::Backend)?;
}
if let Some(post_mixer_norm) = &self.post_mixer_norm {
- hidden =
- post_mixer_norm.encode(&hidden, 0, batch_dim.size(), None::<&mut B::ScratchBuffer>, command_buffer)?;
+ hidden = post_mixer_norm
+ .encode(&hidden, 0, batch_dim.size(), None::<&mut B::ScratchBuffer>, command_buffer)
+ .map_err(TransformerLayerError::Backend)?;
}
- hidden = self.pre_mlp_norm.encode(&hidden, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)?;
+ hidden = self
+ .pre_mlp_norm
+ .encode(&hidden, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)
+ .map_err(TransformerLayerError::Backend)?;
let mlp_coefficients = if let Some(convolution) = &self.mlp_conv {
let (output, coefficients) =
@@ -397,12 +410,15 @@ impl<B: Backend> TransformerLayer<B> {
if let Some(coefficients) = mlp_coefficients {
let convolution = self.mlp_conv.as_ref().expect("mlp convolution required");
- hidden = convolution.encode_post_convolution(&hidden, &coefficients, batch_dim.size(), command_buffer)?;
+ hidden = convolution
+ .encode_post_convolution(&hidden, &coefficients, batch_dim.size(), command_buffer)
+ .map_err(TransformerLayerError::Backend)?;
}
if let Some(post_mlp_norm) = &self.post_mlp_norm {
- hidden =
- post_mlp_norm.encode(&hidden, 0, batch_dim.size(), None::<&mut B::ScratchBuffer>, command_buffer)?;
+ hidden = post_mlp_norm
+ .encode(&hidden, 0, batch_dim.size(), None::<&mut B::ScratchBuffer>, command_buffer)
+ .map_err(TransformerLayerError::Backend)?;
}
if let Some(ple_projection) = &self.ple_projection {
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index 2569a872e..d90e3474b 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -17,9 +17,9 @@ use crate::{
data_type::DataType,
encodable_block::{
embedding::{Embedding, EmbeddingError},
- linear::{Linear, LinearBlockError},
+ linear::{Linear, LinearBlockError, LinearEncodeError},
mixer::attention::{KVCacheView, rope::PrecalculatedRoPE},
- mlp::MlpBlockNewError,
+ mlp::{MlpBlockNewError, MlpEncodeError},
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
weaver_layer::{PreparedPrefixAttention, WeaverLayer},
},
@@ -132,6 +132,8 @@ pub struct Weaver<B: Backend> {
#[derive(Debug, Error)]
pub enum WeaverNewError<B: Backend> {
+ #[error("Attention kernel error: {0}")]
+ Attention(#[source] <<B::Kernels as Kernels>::AttentionKernel as AttentionKernel>::Error),
#[error("linear error: {0}")]
Linear(#[from] LinearBlockError<B>),
#[error("mlp error: {0}")]
@@ -162,6 +164,12 @@ pub enum WeaverNewError<B: Backend> {
pub enum WeaverEncodeError<B: Backend> {
#[error("backend error: {0}")]
Backend(#[source] B::Error),
+ #[error("Linear error: {0}")]
+ Linear(#[from] LinearEncodeError<B>),
+ #[error("MLP error: {0}")]
+ Mlp(#[from] MlpEncodeError<B>),
+ #[error("Attention kernel error: {0}")]
+ Attention(#[source] <<B::Kernels as Kernels>::AttentionKernel as AttentionKernel>::Error),
#[error("embedding error: {0}")]
Embedding(#[from] EmbeddingError<B>),
#[error("invalid Weaver tree input")]
@@ -311,10 +319,7 @@ impl<B: Backend> Weaver<B> {
.hidden_state_norm
.encode(&prefix_hidden, 0, depth, None::<&mut B::ScratchBuffer>, command_buffer)
.map_err(WeaverEncodeError::Backend)?;
- let mut residual_input = self
- .hidden_state_projection
- .encode(normalized_prefix, depth, command_buffer)
- .map_err(WeaverEncodeError::Backend)?;
+ let mut residual_input = self.hidden_state_projection.encode(normalized_prefix, depth, command_buffer)?;
let (last_layer, preceding_layers) = self.layers.split_last().expect("Weaver must have at least one layer");
let mut residual_state =
command_buffer.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;
@@ -323,9 +328,7 @@ impl<B: Backend> Weaver<B> {
let PreparedPrefixAttention {
queries,
kv_cache,
- } = layer
- .encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, command_buffer)
- .map_err(WeaverEncodeError::Backend)?;
+ } = layer.encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, command_buffer)?;
let cache = KVCacheView::full(0);
let kv_plane_bytes = size_for_shape(&[depth, self.model_dim], DATA_TYPE);
let attention_output = layer
@@ -343,16 +346,14 @@ impl<B: Backend> Weaver<B> {
},
command_buffer,
)
- .map_err(WeaverEncodeError::Backend)?;
- residual_input = layer
- .encode_post_attention(attention_output, &mut residual_state, depth, command_buffer)
- .map_err(WeaverEncodeError::Backend)?;
+ .map_err(WeaverEncodeError::Attention)?;
+ residual_input =
+ layer.encode_post_attention(attention_output, &mut residual_state, depth, command_buffer)?;
prefix_kv_layers.push(kv_cache);
}
prefix_kv_layers.push(
last_layer
- .encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, command_buffer)
- .map_err(WeaverEncodeError::Backend)?
+ .encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, command_buffer)?
.kv_cache,
);
@@ -425,10 +426,8 @@ impl<B: Backend> Weaver<B> {
.token_embedding_norm
.encode(&token_embedding, 0, batch_node_count, None::<&mut B::ScratchBuffer>, command_buffer)
.map_err(WeaverEncodeError::Backend)?;
- let mut residual_input = self
- .token_embedding_projection
- .encode(normalized_embedding, batch_node_count, command_buffer)
- .map_err(WeaverEncodeError::Backend)?;
+ let mut residual_input =
+ self.token_embedding_projection.encode(normalized_embedding, batch_node_count, command_buffer)?;
let mut residual_state =
command_buffer.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;
let node_metadata = node_metadata.as_ref();
@@ -438,10 +437,7 @@ impl<B: Backend> Weaver<B> {
.pre_attention_norm
.encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), command_buffer)
.map_err(WeaverEncodeError::Backend)?;
- let current_qkv = layer
- .qkv_projection
- .encode(attention_input, batch_node_count, command_buffer)
- .map_err(WeaverEncodeError::Backend)?;
+ let current_qkv = layer.qkv_projection.encode(attention_input, batch_node_count, command_buffer)?;
let mut attention_output = command_buffer
.allocate_scratch_for_shape(&[batch_node_count, self.model_dim], DATA_TYPE)
.map_err(WeaverEncodeError::Backend)?;
@@ -464,19 +460,15 @@ impl<B: Backend> Weaver<B> {
layer.attention_scale,
command_buffer,
);
- residual_input = layer
- .encode_post_attention(attention_output, &mut residual_state, batch_node_count, command_buffer)
- .map_err(WeaverEncodeError::Backend)?;
+ residual_input =
+ layer.encode_post_attention(attention_output, &mut residual_state, batch_node_count, command_buffer)?;
}
let normalized_output = self
.readout_norm
.encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), command_buffer)
.map_err(WeaverEncodeError::Backend)?;
- let query = self
- .readout_query_projection
- .encode(normalized_output, batch_node_count, command_buffer)
- .map_err(WeaverEncodeError::Backend)?;
+ let query = self.readout_query_projection.encode(normalized_output, batch_node_count, command_buffer)?;
let logit_residuals = target_embedding.encode_readout(
&query,
batch_node_count,
diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
index a71d6d5ac..6bfd7cf75 100644
--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
@@ -14,7 +14,7 @@ use crate::{
mixer::attention::rope::PrecalculatedRoPE,
mlp::Mlp,
normalization::{Normalization, PostLayerScalar, ShortcutMode},
- weaver::{DATA_TYPE, ROPE_DATA_TYPE, WeaverNewError},
+ weaver::{DATA_TYPE, ROPE_DATA_TYPE, WeaverEncodeError, WeaverNewError},
},
parameters::ParameterTree,
};
@@ -89,7 +89,7 @@ impl<B: Backend> WeaverLayer<B> {
data_type: DATA_TYPE,
},
)
- .map_err(WeaverNewError::Backend)?;
+ .map_err(WeaverNewError::Attention)?;
let attention_prepare =
<B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, ROPE_DATA_TYPE, true, true)
.map_err(WeaverNewError::Backend)?;
@@ -152,14 +152,19 @@ impl<B: Backend> WeaverLayer<B> {
rope: &PrecalculatedRoPE<B>,
token_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<PreparedPrefixAttention<B>, B::Error> {
- let attention_input =
- self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), command_buffer)?;
+ ) -> Result<PreparedPrefixAttention<B>, WeaverEncodeError<B>> {
+ let attention_input = self
+ .pre_attention_norm
+ .encode(residual_input, 0, token_count, Some(residual_state), command_buffer)
+ .map_err(WeaverEncodeError::Backend)?;
let qkv = self.qkv_projection.encode(attention_input, token_count, command_buffer)?;
- let mut queries =
- command_buffer.allocate_scratch_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE)?;
+ let mut queries = command_buffer
+ .allocate_scratch_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE)
+ .map_err(WeaverEncodeError::Backend)?;
let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);
- let mut kv_cache = command_buffer.allocate_scratch_for_shape(&[2, token_count, self.model_dim], DATA_TYPE)?;
+ let mut kv_cache = command_buffer
+ .allocate_scratch_for_shape(&[2, token_count, self.model_dim], DATA_TYPE)
+ .map_err(WeaverEncodeError::Backend)?;
let (keys, values) = kv_cache.split_at(kv_plane_bytes);
self.attention_prepare.encode(
&qkv,
@@ -189,10 +194,12 @@ impl<B: Backend> WeaverLayer<B> {
residual_state: impl BufferMut<Backend = B>,
token_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
+ ) -> Result<B::ScratchBuffer, WeaverEncodeError<B>> {
let projected_attention = self.out_projection.encode(attention_output, token_count, command_buffer)?;
- let mlp_input =
- self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), command_buffer)?;
- self.mlp.encode(mlp_input, token_count, command_buffer)
+ let mlp_input = self
+ .pre_mlp_norm
+ .encode(&projected_attention, 0, token_count, Some(residual_state), command_buffer)
+ .map_err(WeaverEncodeError::Backend)?;
+ self.mlp.encode(mlp_input, token_count, command_buffer).map_err(WeaverEncodeError::Mlp)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
index 1b5515e87..21731c4a4 100644
--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs
+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
@@ -4,7 +4,7 @@ use thiserror::Error;
use crate::{
backends::common::{
Backend, BufferCpuAccessible, BufferMut,
- gpu_types::{QuantizationMethod, QuantizationMode},
+ gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},
kernel::matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection},
},
config::weight_matrix::{AnyWeightMatrixSpec, Layout},
diff --git a/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs b/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs
index 07fa38a9d..637fe4009 100644
--- a/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs
+++ b/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs
@@ -1,7 +1,7 @@
use tokenizers::Tokenizer;
use xgrammar::{GrammarCompiler, GrammarMatcher, TokenizerInfo, get_bitmask_size};
-use super::{GrammarConfig, GrammarError, engagement::GrammarEngagementState};
+use crate::engine::language_model::grammar::{GrammarConfig, GrammarError, engagement::GrammarEngagementState};
pub struct Grammar {
bitmask_size: usize,
diff --git a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs b/crates/uzu-engine/src/engine/language_model/grammar/mod.rs
index 9eb75ec8a..2b8bbf969 100644
--- a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/grammar/mod.rs
@@ -3,6 +3,4 @@ mod engagement;
mod error;
mod grammar;
-pub use config::GrammarConfig;
-pub use error::GrammarError;
-pub use grammar::Grammar;
+pub use crate::engine::language_model::grammar::{config::GrammarConfig, error::GrammarError, grammar::Grammar};
diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rs
index 4e051ee1f..adc82fe58 100644
--- a/crates/uzu-engine/src/engine/language_model/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/mod.rs
@@ -6,10 +6,7 @@ use tokenizers::Tokenizer;
use crate::{
backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},
- config::{
- model::{generation::GenerationConfig, language_model::LanguageModelConfig},
- token_codec::AnyTokenCodecConfig,
- },
+ config::{model::language_model::LanguageModelConfig, token_codec::AnyTokenCodecConfig},
data_type::DataType,
encodable_block::{
decoder::{Decoder, DecoderError},
@@ -19,6 +16,7 @@ use crate::{
parameters::{ParameterLoader, ParameterLoaderError},
speculators::dflash_tfm::{DFlashSpeculatorLoadError, DFlashTfmSpeculator},
};
+pub use crate::{config::model::generation::GenerationConfig, utils::strict_serde::Unsupported};
pub mod state;
pub mod stream;
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index 593db5397..706da8cd0 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -315,14 +315,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if let Some(speculator) = model.speculator.as_ref() {
let speculator_state = model_state.speculator_state.as_mut().unwrap();
- speculator
- .encode_accept(
- speculator_state,
- decoder_output.hidden_features.as_ref().unwrap().iter(),
- &(0..input_chunk.len() as u32).collect::<Box<[u32]>>(),
- &mut command_buffer,
- )
- .map_err(LanguageModelStreamError::Backend)?;
+ speculator.encode_accept(
+ speculator_state,
+ decoder_output.hidden_features.as_ref().unwrap().iter(),
+ &(0..input_chunk.len() as u32).collect::<Box<[u32]>>(),
+ &mut command_buffer,
+ )?;
}
if let Some(suffix_repetition_length) = options.sampling_method.suffix_repetition_length() {
@@ -468,14 +466,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.encode_accept(&accepted_token_indicies, &mut command_buffer)
.map_err(LanguageModelStreamError::Backend)?;
if let Some(speculator) = self.model.speculator.as_ref() {
- speculator
- .encode_accept(
- self.model_state.speculator_state.as_mut().unwrap(),
- hidden_features.as_ref().unwrap().iter(),
- &accepted_token_indicies,
- &mut command_buffer,
- )
- .map_err(LanguageModelStreamError::Backend)?;
+ speculator.encode_accept(
+ self.model_state.speculator_state.as_mut().unwrap(),
+ hidden_features.as_ref().unwrap().iter(),
+ &accepted_token_indicies,
+ &mut command_buffer,
+ )?;
}
let output_norm = if let Some((final_hidden, row_bytes)) = output_norm {
let row = full.last().unwrap().0;
@@ -723,14 +719,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if let Some(speculator) = self.model.speculator.as_ref() {
let speculator_state = self.model_state.speculator_state.as_mut().unwrap();
- speculator
- .encode_accept(
- speculator_state,
- decoder_output.hidden_features.as_ref().unwrap().iter(),
- &(0..batch_dim.size()).collect::<Box<[u32]>>(),
- &mut command_buffer,
- )
- .map_err(LanguageModelStreamError::Backend)?;
+ speculator.encode_accept(
+ speculator_state,
+ decoder_output.hidden_features.as_ref().unwrap().iter(),
+ &(0..batch_dim.size()).collect::<Box<[u32]>>(),
+ &mut command_buffer,
+ )?;
}
if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {
diff --git a/crates/uzu-engine/src/lib.rs b/crates/uzu-engine/src/lib.rs
index f29b0b540..d073d94a5 100644
--- a/crates/uzu-engine/src/lib.rs
+++ b/crates/uzu-engine/src/lib.rs
@@ -4,6 +4,7 @@
mod array;
mod config;
+mod data_type;
mod encodable_block;
mod parameters;
mod speculators;
@@ -11,7 +12,6 @@ mod trie;
mod utils;
pub mod backends;
-pub mod data_type;
pub mod engine;
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index afeaf8139..b3c503b41 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -167,8 +167,10 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
target_features: impl ExactSizeIterator<Item = impl BufferRef<Backend = B>>,
accepted_indices: &[u32],
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error> {
- self.dflash.encode_accept(target_features, accepted_indices, state, command_buffer)
+ ) -> Result<(), DFlashTreeError<B>> {
+ self.dflash
+ .encode_accept(target_features, accepted_indices, state, command_buffer)
+ .map_err(DFlashTreeError::DFlash)
}
pub fn make_shape(
diff --git a/crates/uzu-engine/src/utils/dispatch_dtype.rs b/crates/uzu-engine/src/utils/dispatch_dtype.rs
deleted file mode 100644
index 423cb3472..000000000
--- a/crates/uzu-engine/src/utils/dispatch_dtype.rs
+++ /dev/null
@@ -1,12 +0,0 @@
-#[macro_export]
-macro_rules! dispatch_dtype {
- (|()| $body:expr) => { $body };
- (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {
- match $dt {
- $crate::data_type::DataType::F16 => { type $T = half::f16; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
- $crate::data_type::DataType::BF16 => { type $T = half::bf16; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
- $crate::data_type::DataType::F32 => { type $T = f32; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
- other => panic!("unsupported dtype in test: {:?}", other),
- }
- };
-}
diff --git a/crates/uzu-engine/src/utils/mod.rs b/crates/uzu-engine/src/utils/mod.rs
index 11e88e2c5..e0f6473d3 100644
--- a/crates/uzu-engine/src/utils/mod.rs
+++ b/crates/uzu-engine/src/utils/mod.rs
@@ -1,4 +1,3 @@
-pub mod dispatch_dtype;
pub mod downcast;
pub mod maybe_mut;
#[cfg(backend = "cpu")]
diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs
index 9d940407d..46ab80e16 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs
@@ -9,7 +9,7 @@ use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::ActivationType, kernel::ActivationKernel,
+ gpu_types::activation_type::ActivationType, kernel::ActivationKernel,
},
cpu::Cpu,
},
diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
index ec8d119ac..e58fd30f8 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
@@ -9,7 +9,7 @@ use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
- gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE as BLOCK_SIZE, kernel::ActivationTransform,
+ gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE as BLOCK_SIZE, kernel::ActivationTransform,
},
cpu::Cpu,
},
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
index 845c24522..4353c0c7a 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
@@ -9,7 +9,7 @@ use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::Copy, kernel::KVCacheUpdateKernel,
+ gpu_types::kv_cache_update::Copy, kernel::KVCacheUpdateKernel,
},
cpu::Cpu,
},
diff --git a/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs b/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs
index db2b3feef..e2f45a15e 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs
@@ -11,10 +11,10 @@ use crate::{
common::{
Backend, Context, Kernels,
gpu_types::{
- EmbeddingTableKind::{self, D4S4, Dense, Quantized},
- QuantizationMethod::{self, ScaleBias, ScaleSymmetric, ScaleZeroPoint},
- QuantizationMode::{self, I8, U4, U8},
d4s4,
+ embedding::EmbeddingTableKind::{self, D4S4, Dense, Quantized},
+ quantization::QuantizationMode::{self, I8, U4, U8},
+ quantization_method::QuantizationMethod::{self, ScaleBias, ScaleSymmetric, ScaleZeroPoint},
},
kernel::InputEmbeddingLookupKernel,
},
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
index b131a1851..2d7466022 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
@@ -9,7 +9,7 @@ use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
- gpu_types::ActivationType,
+ gpu_types::activation_type::ActivationType,
kernel::{ActivationQuantization, GatedActMul, GatedActMulSettings, matmul::Int8CodeLayout},
},
cpu::Cpu,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
index fed309c5b..1eb047336 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
@@ -10,7 +10,7 @@ use crate::{
backends::{
common::{
Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
- gpu_types::QuantizationMethod,
+ gpu_types::quantization_method::QuantizationMethod,
kernel::{
Kernels,
matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
index ee9bf49c3..3f03a49ab 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
@@ -7,7 +7,7 @@ use crate::{
array::ArrayElement,
backends::common::{
Backend, Context,
- gpu_types::QuantizationMethod,
+ gpu_types::quantization_method::QuantizationMethod,
kernel::{Kernels, matmul::MatmulKernel},
},
tests::{
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs
index 7f35d292e..e5a284ec2 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs
@@ -9,7 +9,7 @@ use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::ActivationType, kernel::Conv1dDecodeKernel,
+ gpu_types::activation_type::ActivationType, kernel::Conv1dDecodeKernel,
},
cpu::Cpu,
},
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs
index 2726ed554..5706500ba 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs
@@ -9,7 +9,7 @@ use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::ActivationType, kernel::Conv1dScanKernel,
+ gpu_types::activation_type::ActivationType, kernel::Conv1dScanKernel,
},
cpu::Cpu,
},
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs
index 13a77f41e..e3e39e628 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs
@@ -8,7 +8,7 @@ use crate::{
array::ArrayElement,
backends::common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::ActivationType, kernel::SSDPrefillKernel,
+ gpu_types::activation_type::ActivationType, kernel::SSDPrefillKernel,
},
data_type::DataType,
tests::{
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs
index f7c944dd1..abf3ca05f 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs
@@ -8,7 +8,10 @@ use crate::{
backends::{
common::{
Backend, BufferMut, CommandBuffer, CommandBufferEncoding,
- gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod, QuantizationMode},
+ gpu_types::{
+ hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE, quantization::QuantizationMode,
+ quantization_method::QuantizationMethod,
+ },
kernel::{
ActivationQuantization, ActivationTransform, Kernels,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
index 4d98aaf32..abcc83a66 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
@@ -12,7 +12,7 @@ use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
- gpu_types::{QuantizationMethod, QuantizationMode},
+ gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},
kernel::{
Kernels,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs
index a97e8234e..7d664744e 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs
@@ -8,7 +8,7 @@ use crate::{
backends::{
common::{
Backend,
- gpu_types::QuantizationMethod,
+ gpu_types::quantization_method::QuantizationMethod,
kernel::{Kernels, matmul::MatmulKernel},
},
metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs
index e4f2db786..6ade045f0 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs
@@ -8,7 +8,7 @@ use crate::{
backends::{
common::{
Backend,
- gpu_types::QuantizationMethod,
+ gpu_types::quantization_method::QuantizationMethod,
kernel::{Kernels, matmul::MatmulKernel},
},
metal::{Metal, context::MetalContext},
diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rs
index ab293efd1..ef5ddacfc 100644
--- a/crates/uzu-engine/unit/common/matmul/quant.rs
+++ b/crates/uzu-engine/unit/common/matmul/quant.rs
@@ -9,7 +9,7 @@ use crate::{
common::{
Backend, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,
Context,
- gpu_types::{QuantizationMethod, QuantizationMode},
+ gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},
kernel::{
ActivationQuantization, ActivationTransform, Kernels,
matmul::{
diff --git a/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs b/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs
index e4347cfc8..3702cf678 100644
--- a/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs
+++ b/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs
@@ -3,7 +3,7 @@ use uzu_engine_macros::uzu_test;
use crate::{
backends::common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::ActivationType,
+ gpu_types::activation_type::ActivationType,
kernel::{Conv1dScanKernel, SSDPrefill64Kernel, SSDPrefillKernel},
},
data_type::DataType,
diff --git a/crates/uzu-engine/unit/encodable_block/sampling_test.rs b/crates/uzu-engine/unit/encodable_block/sampling_test.rs
index c6af3b91e..fba8724f1 100644
--- a/crates/uzu-engine/unit/encodable_block/sampling_test.rs
+++ b/crates/uzu-engine/unit/encodable_block/sampling_test.rs
@@ -13,7 +13,6 @@ use crate::{
gpu_types::trie::TrieNode,
},
data_type::DataType,
- dispatch_dtype,
encodable_block::{
batch_topology::BatchTopology,
sampling::{Sampling, SamplingMethod},
@@ -21,6 +20,18 @@ use crate::{
tests::proptest::{ComparableTestResults, TestContextes, for_each_context, kernel_data_type},
};
+macro_rules! dispatch_dtype {
+ (|()| $body:expr) => { $body };
+ (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {
+ match $dt {
+ $crate::data_type::DataType::F16 => { type $T = half::f16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
+ $crate::data_type::DataType::BF16 => { type $T = half::bf16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
+ $crate::data_type::DataType::F32 => { type $T = f32; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
+ other => panic!("unsupported dtype in test: {:?}", other),
+ }
+ };
+}
+
struct SamplingTestResults(Vec<u32>);
impl ComparableTestResults for SamplingTestResults {