Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

beginnings of the cpu backend (#226)

uuuvncd183c2e4ad9
diff --git a/Cargo.lock b/Cargo.lock
index e66a95c0d..5eca14321 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1125,6 +1125,15 @@ dependencies = [
"litrs",
]
+[[package]]
+name = "dsl"
+version = "0.1.8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.114",
+]
+
[[package]]
name = "dyn-clone"
version = "1.0.20"
@@ -4340,6 +4349,7 @@ dependencies = [
"clang-ast",
"clap",
"depfile",
+ "dsl",
"futures",
"half",
"is_close",
diff --git a/Cargo.toml b/Cargo.toml
index ebf12810a..da4244059 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,5 @@
[workspace]
-members = ["crates/cli", "crates/uzu", "crates/benchmarks"]
+members = ["crates/cli", "crates/uzu", "crates/benchmarks", "crates/dsl"]
resolver = "3"
[workspace.package]
@@ -12,6 +12,7 @@ repository = "https://github.com/trymirai/uzu"
[workspace.dependencies]
benchmarks = { path = "crates/benchmarks" }
+dsl = { path = "crates/dsl" }
uzu = { path = "crates/uzu" }
xgrammar = { package = "xgrammar-rs", version = "0.1", features = ["hf"] }
half = { version = "2.7", features = ["bytemuck", "num-traits"] }
diff --git a/crates/dsl/Cargo.toml b/crates/dsl/Cargo.toml
new file mode 100644
index 000000000..7255c8b01
--- /dev/null
+++ b/crates/dsl/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "dsl"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+authors.workspace = true
+repository.workspace = true
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2.workspace = true
+quote.workspace = true
+syn.workspace = true
diff --git a/crates/dsl/src/lib.rs b/crates/dsl/src/lib.rs
new file mode 100644
index 000000000..1d84d30b8
--- /dev/null
+++ b/crates/dsl/src/lib.rs
@@ -0,0 +1,30 @@
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::{FnArg, ItemFn, parse_macro_input};
+
+#[proc_macro_attribute]
+pub fn kernel(
+ args: TokenStream,
+ input: TokenStream,
+) -> TokenStream {
+ let file = format!("/cpu/{}.rs", args.to_string());
+
+ let mut func = parse_macro_input!(input as ItemFn);
+
+ func.attrs.retain(|attr| !attr.path().is_ident("variants"));
+
+ for arg in &mut func.sig.inputs {
+ if let FnArg::Typed(pat) = arg {
+ pat.attrs.retain(|attr| {
+ let attr_path = attr.path();
+
+ !(attr_path.is_ident("specialize") || attr_path.is_ident("optional"))
+ });
+ }
+ }
+ quote! {
+ #func
+ include!(concat!(env!("OUT_DIR"), #file));
+ }
+ .into()
+}
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 0fc8e792e..951b0a513 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -5,6 +5,7 @@ edition.workspace = true
build = "build/main.rs"
[dependencies]
+dsl.workspace = true
xgrammar.workspace = true
half.workspace = true
ndarray.workspace = true
diff --git a/crates/uzu/build/common/codegen.rs b/crates/uzu/build/common/codegen.rs
index 5b8a97cd2..ce2f29194 100644
--- a/crates/uzu/build/common/codegen.rs
+++ b/crates/uzu/build/common/codegen.rs
@@ -10,7 +10,8 @@ pub fn write_tokens(
let tokens = tokens.into();
let file = file.as_ref();
- let parsed = syn::parse2(tokens.into()).context("cannot parse generated bindings")?;
+ let parsed = syn::parse2(tokens.clone().into())
+ .with_context(|| format!("cannot parse generated bindings: {}", tokens.to_string()))?;
fs::write(&file, prettyplease::unparse(&parsed))
.with_context(|| format!("cannot write file {}", file.display()))?;
diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rs
new file mode 100644
index 000000000..feee0dd60
--- /dev/null
+++ b/crates/uzu/build/cpu/compiler.rs
@@ -0,0 +1,606 @@
+use std::{collections::HashMap, env, fs, path::PathBuf};
+
+use anyhow::{Context, bail};
+use async_trait::async_trait;
+use itertools::Itertools;
+use proc_macro2::{Span, TokenStream};
+use quote::{ToTokens, format_ident, quote};
+use syn::{
+ Expr, FnArg, GenericArgument, GenericParam, Ident, Item, ItemFn, Lifetime, PathArguments, Type,
+ punctuated::Punctuated, token::Comma,
+};
+use walkdir::WalkDir;
+
+use crate::common::{
+ codegen::write_tokens,
+ compiler::Compiler,
+ kernel::{Kernel, KernelArgument, KernelArgumentType, KernelParameter, KernelParameterType},
+};
+
+#[derive(PartialEq, Debug)]
+pub enum FunctionArgumentType {
+ Buffer,
+ Constant(Type),
+ Scalar(Type),
+ Specialization(Type),
+}
+
+#[derive(PartialEq, Debug)]
+pub struct FunctionArgument {
+ pub name: Ident,
+ pub conditional: Option<Ident>,
+ pub ty: FunctionArgumentType,
+}
+
+impl FunctionArgument {
+ fn to_kernel_argument(&self) -> Option<KernelArgument> {
+ Some(KernelArgument {
+ name: self.name.to_string().into_boxed_str(),
+ conditional: self.conditional.is_some(),
+ ty: match &self.ty {
+ FunctionArgumentType::Buffer => KernelArgumentType::Buffer,
+ FunctionArgumentType::Constant(ty) => KernelArgumentType::Constant(
+ ty.to_token_stream().to_string().replace(" :: ", "::").into_boxed_str(),
+ ),
+ FunctionArgumentType::Scalar(ty) => {
+ KernelArgumentType::Scalar(ty.to_token_stream().to_string().replace(" :: ", "::").into_boxed_str())
+ },
+ FunctionArgumentType::Specialization(_) => {
+ return None;
+ },
+ },
+ })
+ }
+
+ fn to_kernel_parameter(&self) -> Option<KernelParameter> {
+ Some(KernelParameter {
+ name: self.name.to_string().into_boxed_str(),
+ ty: match &self.ty {
+ FunctionArgumentType::Specialization(ty) => {
+ KernelParameterType::Value(ty.to_token_stream().to_string().into_boxed_str())
+ },
+ _ => {
+ return None;
+ },
+ },
+ })
+ }
+}
+
+#[derive(PartialEq, Debug)]
+pub enum FunctionParameterType {
+ Type,
+ Value(Type),
+}
+
+#[derive(PartialEq, Debug)]
+pub struct FunctionParameter {
+ pub name: Ident,
+ pub ty: FunctionParameterType,
+}
+
+impl FunctionParameter {
+ fn to_kernel_parameter(&self) -> KernelParameter {
+ KernelParameter {
+ name: self.name.to_string().into_boxed_str(),
+ ty: match &self.ty {
+ FunctionParameterType::Type => KernelParameterType::Type,
+ FunctionParameterType::Value(ty) => {
+ KernelParameterType::Value(ty.to_token_stream().to_string().into_boxed_str())
+ },
+ },
+ }
+ }
+}
+
+pub struct CpuCompiler {
+ src_dir: PathBuf,
+ build_dir: PathBuf,
+}
+
+impl CpuCompiler {
+ pub fn new() -> anyhow::Result<Self> {
+ let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?)
+ .join("src/backends/cpu/kernel");
+
+ let build_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?).join("cpu");
+ fs::create_dir_all(&build_dir).with_context(|| format!("cannot create {}", build_dir.display()))?;
+
+ Ok(Self {
+ src_dir,
+ build_dir,
+ })
+ }
+
+ fn compile(
+ &self,
+ source_path: PathBuf,
+ ) -> anyhow::Result<(Box<[Box<str>]>, Box<[Kernel]>)> {
+ let src_rel_path: Box<[Box<str>]> = source_path
+ .strip_prefix(&self.src_dir)
+ .context("source is not in src_dir")?
+ .with_extension("")
+ .as_os_str()
+ .to_str()
+ .unwrap()
+ .split("/")
+ .map(|s| s.to_string().into_boxed_str())
+ .collect();
+
+ let source_contents = fs::read_to_string(&source_path).context("cannot read the source file")?;
+ let source_ast = syn::parse_file(&source_contents).context("cannot parse ast")?;
+
+ let kernels = source_ast
+ .items
+ .into_iter()
+ .filter_map(|item| {
+ if let Item::Fn(ifn) = item
+ && ifn.attrs.iter().any(|attr| attr.path().is_ident("kernel"))
+ {
+ Some(self.compile_kernel(ifn))
+ } else {
+ None
+ }
+ })
+ .collect::<anyhow::Result<_>>()?;
+
+ Ok((src_rel_path, kernels))
+ }
+
+ fn compile_kernel(
+ &self,
+ ifn: ItemFn,
+ ) -> anyhow::Result<Kernel> {
+ let mut kernel_ident = None;
+ let mut function_variants = Vec::new();
+
+ for attr in ifn.attrs {
+ match attr.path().get_ident().map(|i| i.to_string()).as_ref().map(|s| s.as_ref()) {
+ Some("kernel") => {
+ if kernel_ident.is_some() {
+ bail!("Multiple kernel attributes!");
+ }
+
+ kernel_ident = Some(attr.parse_args::<Ident>().context("cannot parse kernel attribute arg")?);
+ },
+ Some("variants") => {
+ let mut args = attr
+ .parse_args_with(Punctuated::<Expr, Comma>::parse_terminated)
+ .context("cannot parse variants attribute args")?
+ .into_iter();
+
+ function_variants.push((
+ syn::parse2::<Ident>(args.next().context("variant must have a name")?.into_token_stream())
+ .unwrap(),
+ args.collect::<Box<[_]>>(),
+ ));
+ },
+ _ => bail!("Unexpected attr {attr:?}"),
+ }
+ }
+
+ let Some(kernel_ident) = kernel_ident else {
+ bail!("Not a kernel")
+ };
+
+ let function_ident = ifn.sig.ident;
+ let function_parameters = ifn
+ .sig
+ .generics
+ .params
+ .into_iter()
+ .map(|parameter| {
+ Ok(match parameter {
+ GenericParam::Type(parameter) => FunctionParameter {
+ name: parameter.ident,
+ ty: FunctionParameterType::Type,
+ },
+ GenericParam::Const(parameter) => FunctionParameter {
+ name: parameter.ident,
+ ty: FunctionParameterType::Value(parameter.ty),
+ },
+ parameter => {
+ bail!("unsupported kernel parameter type: {parameter:?}")
+ },
+ })
+ })
+ .collect::<anyhow::Result<Box<[FunctionParameter]>>>()?;
+ let function_arguments = ifn
+ .sig
+ .inputs
+ .into_iter()
+ .map(|argument| {
+ let FnArg::Typed(argument) = argument else {
+ bail!("self argument in a kernel is not supported");
+ };
+
+ let name = syn::parse2(argument.pat.into_token_stream()).context("cannot parse the argument name")?;
+
+ let specialize = argument.attrs.iter().any(|attr| attr.path().is_ident("specialize"));
+
+ let conditional = argument
+ .attrs
+ .iter()
+ .find_map(|attr| attr.path().is_ident("optional").then(|| attr.parse_args::<Ident>().unwrap()));
+
+ let ty = if specialize {
+ FunctionArgumentType::Specialization(*argument.ty)
+ } else if conditional.is_some() {
+ let Type::Path(ty) = *argument.ty else {
+ bail!("conditional argument must be a type path");
+ };
+ if ty.path.segments.len() != 1 {
+ bail!("conditional argument type path must have one segment");
+ }
+ let seg = &ty.path.segments[0];
+ if seg.ident.to_string() != "Option" {
+ bail!("conditional argument type must be Option<...>");
+ }
+ let PathArguments::AngleBracketed(option_arguments) = &seg.arguments else {
+ bail!("conditional argument type must be angle bracketed");
+ };
+ if option_arguments.args.len() != 1 {
+ bail!("conditional argument type Option must have one generic argument");
+ }
+ let generic_argument = &option_arguments.args[0];
+ let GenericArgument::Type(inner_ty) = generic_argument else {
+ bail!("conditional argument type Option must have a type argument")
+ };
+ Self::parse_type(inner_ty.clone()).context("failed to parse conditional argument type")?
+ } else {
+ Self::parse_type(*argument.ty).context("failed to parse argument type")?
+ };
+
+ Ok(FunctionArgument {
+ name,
+ conditional,
+ ty,
+ })
+ })
+ .collect::<anyhow::Result<Box<[FunctionArgument]>>>()?;
+
+ let kernel_parameters = function_parameters
+ .iter()
+ .map(|p| p.to_kernel_parameter())
+ .chain(function_arguments.iter().flat_map(|p| p.to_kernel_parameter()))
+ .collect::<Box<[KernelParameter]>>();
+
+ let kernel_arguments =
+ function_arguments.iter().flat_map(|p| p.to_kernel_argument()).collect::<Box<[KernelArgument]>>();
+
+ if function_parameters.len() != function_variants.len() {
+ bail!(
+ "Kernel function has {} generics != {} #[variants(...)]!",
+ function_parameters.len(),
+ function_variants.len()
+ );
+ }
+
+ for (parameter, (variant_name, _)) in std::iter::zip(function_parameters.iter(), function_variants.iter()) {
+ if &parameter.name != variant_name {
+ bail!(
+ "Parameter name doesn't match variant name: {} | {}",
+ parameter.name.to_string(),
+ variant_name.to_string(),
+ );
+ }
+ }
+
+ // === Bindgen ===
+
+ let trait_ident = format_ident!("{kernel_ident}Kernel");
+ let struct_ident = format_ident!("{kernel_ident}CpuKernel");
+
+ let (struct_fields_defs, struct_fields_sets): (Vec<TokenStream>, Vec<TokenStream>) = function_parameters
+ .iter()
+ .map(|parameter| {
+ let ident = &parameter.name;
+
+ let ty = match &parameter.ty {
+ FunctionParameterType::Type => quote! { crate::DataType },
+ FunctionParameterType::Value(ty) => quote! { #ty },
+ };
+
+ (quote! { #ident: #ty }, quote! { #ident })
+ })
+ .chain(function_arguments.iter().flat_map(|argument| {
+ let FunctionArgumentType::Specialization(ty) = &argument.ty else {
+ return None;
+ };
+
+ let ident = &argument.name;
+
+ Some((quote! { #ident: #ty }, quote! { #ident }))
+ }))
+ .collect();
+
+ let parameter_args: Vec<TokenStream> = kernel_parameters
+ .iter()
+ .map(|parameter| {
+ let parameter_ident: Ident = syn::parse_str(parameter.name.as_ref())?;
+
+ Ok(match &parameter.ty {
+ KernelParameterType::Type => quote! { #[allow(non_snake_case)] #parameter_ident: crate::DataType },
+ KernelParameterType::Value(ty) => {
+ let ty: syn::Type = syn::parse_str(ty.as_ref()).unwrap();
+ quote! { #[allow(non_snake_case)] #parameter_ident: #ty }
+ },
+ })
+ })
+ .collect::<anyhow::Result<_>>()?;
+
+ let (encode_generics, encode_args_defs, encode_args_passthrough): (Vec<_>, Vec<_>, Vec<_>) = kernel_arguments
+ .iter()
+ .map(|argument| {
+ let argument_ident: Ident = syn::parse_str(argument.name.as_ref()).context("cannot parse ident")?;
+
+ let (generic, mut ty) = match &argument.ty {
+ KernelArgumentType::Buffer => {
+ let buffer_lifetime = Lifetime::new(&format!("'{}", argument.name.as_ref()), Span::call_site());
+ (
+ Some(quote! { #buffer_lifetime }),
+ quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, Box<[u8]>> },
+ )
+ },
+ KernelArgumentType::Constant(ty) => {
+ let ty: Type = syn::parse_str(ty.as_ref()).context("cannot parse type")?;
+ (None, quote! { &[#ty] })
+ },
+ KernelArgumentType::Scalar(ty) => {
+ let ty: Type = syn::parse_str(ty.as_ref()).context("cannot parse type")?;
+ (None, quote! { #ty })
+ },
+ };
+
+ if argument.conditional {
+ ty = quote! { Option<#ty> };
+ }
+
+ Ok((generic, quote! { #argument_ident: #ty }, quote! { #argument_ident }))
+ })
+ .collect::<anyhow::Result<_>>()?;
+
+ let encode_generics = encode_generics.into_iter().flatten().collect::<Vec<_>>();
+
+ let argument_copies = function_arguments
+ .iter()
+ .flat_map(|argument| {
+ let argument_ident = &argument.name;
+ match &argument.ty {
+ FunctionArgumentType::Buffer => {
+ if argument.conditional.is_some() {
+ Some(quote! {
+ let #argument_ident = #argument_ident.map(|__dsl_buffer_impl| unsafe {
+ let (__dsl_buffer, __dsl_offset) = __dsl_buffer_impl.into_parts();
+
+ __dsl_buffer.as_ptr().byte_add(__dsl_offset)
+ });
+ })
+ } else {
+ Some(quote! {
+ let #argument_ident = unsafe {
+ let (__dsl_buffer, __dsl_offset) = #argument_ident.into_parts();
+
+ __dsl_buffer.as_ptr().byte_add(__dsl_offset)
+ };
+ })
+ }
+ },
+ FunctionArgumentType::Constant(_) => {
+ Some(quote! { let #argument_ident = #argument_ident.to_vec().into_boxed_slice(); })
+ },
+ FunctionArgumentType::Scalar(_) => None,
+ FunctionArgumentType::Specialization(_) => {
+ Some(quote! { let #argument_ident = self.#argument_ident; })
+ },
+ }
+ })
+ .collect::<Vec<_>>();
+
+ let make_encode = |generics: TokenStream| -> TokenStream {
+ let monomorphized_function = if !generics.is_empty() {
+ quote! { self::#function_ident::<#generics> }
+ } else {
+ quote! { self::#function_ident }
+ };
+
+ let function_call_args = function_arguments.iter().map(|argument| {
+ let argument_ident = &argument.name;
+
+ match &argument.ty {
+ FunctionArgumentType::Buffer => {
+ if argument.conditional.is_some() {
+ quote! { #argument_ident.map(|p| p as _) }
+ } else {
+ quote! { #argument_ident as _ }
+ }
+ },
+ FunctionArgumentType::Constant(_) => quote! { #argument_ident.as_ref() },
+ FunctionArgumentType::Scalar(_) | FunctionArgumentType::Specialization(_) => {
+ quote! { #argument_ident }
+ },
+ }
+ });
+
+ let function_call_args_joined = function_call_args.fold(quote! {}, |a, b| {
+ if !a.is_empty() && !b.is_empty() {
+ quote! {#a , #b}
+ } else {
+ quote! {#a #b}
+ }
+ });
+
+ quote! {
+ encoder.borrow_mut().push_command(move || {
+ #monomorphized_function(#function_call_args_joined)
+ });
+ }
+ };
+
+ let encode_body = if function_parameters.len() > 0 {
+ let mut parameter_idents = function_parameters.iter().map(|p| p.name.clone()).fold(quote! {}, |a, b| {
+ if !a.is_empty() {
+ quote! {#a , self.#b}
+ } else {
+ quote! {self.#b}
+ }
+ });
+
+ if function_parameters.len() > 1 {
+ parameter_idents = quote! { (#parameter_idents) };
+ }
+
+ let match_arms = function_parameters
+ .iter()
+ .zip(function_variants.iter())
+ .map(|(parameter, (_, variants))| {
+ variants.iter().map(|variant| {
+ (
+ match parameter.ty {
+ FunctionParameterType::Type => {
+ let dtype =
+ format_ident!("{}", variant.to_token_stream().to_string().to_uppercase());
+ quote! { crate::DataType::#dtype }
+ },
+ FunctionParameterType::Value(_) => quote! { #variant },
+ },
+ quote! { #variant },
+ )
+ })
+ })
+ .multi_cartesian_product()
+ .map(|variants| {
+ let (match_variants, generic_variants): (Vec<TokenStream>, Vec<TokenStream>) =
+ variants.into_iter().unzip();
+
+ let mut match_variant = match_variants.iter().fold(quote! {}, |a, b| {
+ if !a.is_empty() && !b.is_empty() {
+ quote! {#a , #b}
+ } else {
+ quote! {#a #b}
+ }
+ });
+
+ if match_variants.len() > 1 {
+ match_variant = quote! { (#match_variant) };
+ }
+
+ let generic_variant = generic_variants.iter().fold(quote! {}, |a, b| {
+ if !a.is_empty() && !b.is_empty() {
+ quote! {#a , #b}
+ } else {
+ quote! {#a #b}
+ }
+ });
+
+ let encode = make_encode(generic_variant);
+
+ quote! { #match_variant => { #encode } }
+ })
+ .collect::<Vec<_>>();
+
+ quote! {
+ match #parameter_idents {
+ #(#match_arms ,)*
+ __dsl_variant => unimplemented!("variant doesn't exist: {__dsl_variant:?}"),
+ }
+ }
+ } else {
+ make_encode(quote! {})
+ };
+
+ let tokens = quote! {
+ #[allow(non_snake_case)]
+ pub struct #struct_ident {
+ #(#struct_fields_defs ,)*
+ }
+
+ impl crate::backends::common::kernel::#trait_ident for #struct_ident {
+ type Backend = crate::backends::cpu::Cpu;
+
+ fn new(context: &crate::backends::cpu::context::CpuContext #(, #parameter_args)*) -> Result<Self, crate::backends::cpu::error::CpuError> {
+ Ok(Self {
+ #(#struct_fields_sets ,)*
+ })
+ }
+
+ fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder std::cell::RefCell<crate::backends::cpu::command_buffer::CpuCommandBuffer>) {
+ self.encode_if(#(#encode_args_passthrough ,)* encoder, None::<&Box<[u8]>>)
+ }
+
+ fn encode_if<#(#encode_generics ,)* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* encoder: &'encoder std::cell::RefCell<crate::backends::cpu::command_buffer::CpuCommandBuffer>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Box<[u8]>>>) {
+ #(#argument_copies)*
+ #encode_body
+ }
+ }
+ };
+
+ let out_path = self.build_dir.join(kernel_ident.to_string()).with_extension("rs");
+ write_tokens(tokens, &out_path).context("cannot write bindings")?;
+
+ Ok(Kernel {
+ name: kernel_ident.to_string().into_boxed_str(),
+ parameters: kernel_parameters,
+ arguments: kernel_arguments,
+ })
+ }
+
+ fn parse_type(ty: Type) -> anyhow::Result<FunctionArgumentType> {
+ Ok(match ty {
+ Type::Ptr(_) => FunctionArgumentType::Buffer,
+ Type::Reference(ty) => match *ty.elem {
+ Type::Slice(ty) => FunctionArgumentType::Constant(*ty.elem),
+ ty => bail!("unsupported reference type: {} ({:?})", ty.to_token_stream().to_string(), ty),
+ },
+ Type::Path(ty) => FunctionArgumentType::Scalar(Type::Path(ty)),
+ ty => bail!("unsupported type: {} ({:?})", ty.to_token_stream().to_string(), ty),
+ })
+ }
+
+ fn bindgen<'a>(
+ &self,
+ objects: impl IntoIterator<Item = &'a (Box<[Box<str>]>, Box<[Kernel]>)> + Clone,
+ ) -> anyhow::Result<()> {
+ let out_path = self.build_dir.join("dsl.rs");
+
+ let associated_types = objects.into_iter().flat_map(|(file_path, kernels)| {
+ kernels.iter().map(|kernel| {
+ let file_path: TokenStream = file_path.iter().join("::").parse().unwrap();
+ let kernel_trait_name = format_ident!("{}Kernel", kernel.name.as_ref());
+ let kernel_struct_name = format_ident!("{}CpuKernel", kernel.name.as_ref());
+ quote! { type #kernel_trait_name = #file_path::#kernel_struct_name; }
+ })
+ });
+
+ let tokens = quote! {
+ pub struct CpuKernels;
+
+ impl crate::backends::common::kernel::Kernels for CpuKernels {
+ type Backend = crate::backends::cpu::Cpu;
+
+ #(#associated_types)*
+ }
+ };
+
+ write_tokens(tokens, &out_path).context("cannot write dsl bindings")?;
+
+ Ok(())
+ }
+}
+
+#[async_trait]
+impl Compiler for CpuCompiler {
+ async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {
+ let objects = WalkDir::new(&self.src_dir)
+ .into_iter()
+ .filter_map(|e| e.ok())
+ .filter(|e| e.file_type().is_file() && e.path().extension().and_then(|s| s.to_str()) == Some("rs"))
+ .map(|e| self.compile(e.into_path()))
+ .collect::<anyhow::Result<Vec<(Box<[Box<str>]>, Box<[Kernel]>)>>>()
+ .context("cannot compile cpu sources")?;
+
+ self.bindgen(&objects).context("cannot generate bindings")?;
+
+ Ok(objects.into_iter().collect())
+ }
+}
diff --git a/crates/uzu/build/cpu/mod.rs b/crates/uzu/build/cpu/mod.rs
new file mode 100644
index 000000000..23a57fd58
--- /dev/null
+++ b/crates/uzu/build/cpu/mod.rs
@@ -0,0 +1,3 @@
+mod compiler;
+
+pub use compiler::CpuCompiler;
diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rs
index bc25ca1bd..ae9dd59f2 100644
--- a/crates/uzu/build/main.rs
+++ b/crates/uzu/build/main.rs
@@ -6,6 +6,8 @@ use futures::future::try_join_all;
mod common;
mod gpu_types;
+mod cpu;
+
#[cfg(feature = "metal")]
mod metal;
@@ -37,6 +39,8 @@ async fn main() -> anyhow::Result<()> {
let mut compilers: Vec<Box<dyn Compiler>> = Vec::new();
+ compilers.push(Box::new(cpu::CpuCompiler::new()?));
+
#[cfg(feature = "metal")]
compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(generated_header_dir)?));
diff --git a/crates/uzu/src/backends/cpu/backend.rs b/crates/uzu/src/backends/cpu/backend.rs
new file mode 100644
index 000000000..ac7bd8408
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/backend.rs
@@ -0,0 +1,20 @@
+use std::cell::RefCell;
+
+use super::{command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError, kernel::CpuKernels};
+use crate::backends::common::Backend;
+
+#[derive(Debug, Clone)]
+pub struct Cpu;
+
+impl Backend for Cpu {
+ type Context = CpuContext;
+ type NativeBuffer = Box<[u8]>;
+ type CommandBuffer = RefCell<CpuCommandBuffer>;
+ type ComputeEncoder = RefCell<CpuCommandBuffer>;
+ type CopyEncoder = RefCell<CpuCommandBuffer>;
+ type Event = RefCell<u64>;
+ type Kernels = CpuKernels;
+ type Error = CpuError;
+
+ const MAX_INLINE_BYTES: usize = usize::MAX;
+}
diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rs
new file mode 100644
index 000000000..fa780d1e2
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/command_buffer.rs
@@ -0,0 +1,117 @@
+use std::{cell::RefCell, time::Instant};
+
+use super::Cpu;
+use crate::backends::common::{CommandBuffer, CopyEncoder};
+
+pub struct CpuCommandBuffer {
+ commands: Vec<Box<dyn Fn()>>,
+ completion_handlers: Vec<Box<dyn Fn()>>,
+ gpu_execution_time_ms: Option<f64>,
+}
+
+impl CpuCommandBuffer {
+ pub fn new() -> RefCell<CpuCommandBuffer> {
+ RefCell::new(CpuCommandBuffer {
+ commands: Vec::new(),
+ completion_handlers: Vec::new(),
+ gpu_execution_time_ms: None,
+ })
+ }
+
+ pub fn push_command(
+ &mut self,
+ command: impl Fn() + 'static,
+ ) {
+ self.commands.push(Box::new(command))
+ }
+}
+
+impl CommandBuffer for RefCell<CpuCommandBuffer> {
+ type Backend = Cpu;
+
+ fn with_compute_encoder<T>(
+ &mut self,
+ callback: impl FnOnce(&mut RefCell<CpuCommandBuffer>) -> T,
+ ) -> T {
+ callback(self)
+ }
+
+ fn with_copy_encoder<T>(
+ &mut self,
+ callback: impl FnOnce(&mut RefCell<CpuCommandBuffer>) -> T,
+ ) -> T {
+ callback(self)
+ }
+
+ fn encode_wait_for_event(
+ &mut self,
+ event: &RefCell<u64>,
+ value: u64,
+ ) {
+ let event = event.as_ptr();
+ self.borrow_mut().push_command(Box::new(move || unsafe { assert!(*event >= value, "deadlock") }))
+ }
+
+ fn encode_signal_event(
+ &mut self,
+ event: &RefCell<u64>,
+ value: u64,
+ ) {
+ let event = event.as_ptr();
+ self.borrow_mut().push_command(Box::new(move || unsafe { *event = value }))
+ }
+
+ fn add_completion_handler(
+ &mut self,
+ handler: impl Fn() + 'static,
+ ) {
+ self.borrow_mut().completion_handlers.push(Box::new(move || handler()))
+ }
+
+ fn submit(&self) {
+ let start = Instant::now();
+ for command in &self.borrow().commands {
+ command()
+ }
+ self.borrow_mut().gpu_execution_time_ms = Some((Instant::now() - start).as_secs_f64() * 1000.0);
+ for completion_handler in self.borrow().completion_handlers.iter() {
+ completion_handler();
+ }
+ }
+
+ fn wait_until_completed(&self) {}
+
+ fn gpu_execution_time_ms(&self) -> Option<f64> {
+ self.borrow().gpu_execution_time_ms
+ }
+}
+
+impl CopyEncoder for RefCell<CpuCommandBuffer> {
+ type Backend = Cpu;
+
+ fn encode_copy(
+ &mut self,
+ src: &Box<[u8]>,
+ dst: &mut Box<[u8]>,
+ size: usize,
+ ) {
+ let src = src.as_ptr();
+ let dst = dst.as_ptr() as *mut u8;
+ self.borrow_mut().push_command(Box::new(move || unsafe {
+ std::ptr::copy(src, dst, size);
+ }))
+ }
+
+ fn encode_fill(
+ &mut self,
+ dst: &mut Box<[u8]>,
+ range: std::ops::Range<usize>,
+ value: u8,
+ ) {
+ let size = range.end - range.start;
+ let dst = dst[range].as_ptr() as *mut u8;
+ self.borrow_mut().push_command(Box::new(move || unsafe {
+ dst.write_bytes(value, size);
+ }))
+ }
+}
diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rs
new file mode 100644
index 000000000..cbf2d0aa2
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/context.rs
@@ -0,0 +1,54 @@
+use std::{cell::RefCell, rc::Rc};
+
+use super::{Cpu, command_buffer::CpuCommandBuffer, error::CpuError};
+use crate::backends::common::{Allocator, Context, DeviceClass};
+
+pub struct CpuContext;
+
+impl Context for CpuContext {
+ type Backend = Cpu;
+
+ fn new() -> Result<Rc<Self>, CpuError> {
+ Ok(Rc::new(CpuContext))
+ }
+
+ fn device_class(&self) -> DeviceClass {
+ DeviceClass::Base
+ }
+
+ fn debug_active(&self) -> bool {
+ false
+ }
+
+ fn allocator(&self) -> &Allocator<Self::Backend> {
+ unimplemented!()
+ }
+
+ fn create_buffer(
+ &self,
+ size: usize,
+ ) -> Result<Box<[u8]>, CpuError> {
+ Ok(vec![0; size].into_boxed_slice())
+ }
+
+ fn create_command_buffer(&self) -> Result<RefCell<CpuCommandBuffer>, CpuError> {
+ Ok(CpuCommandBuffer::new())
+ }
+
+ fn create_event(&self) -> Result<RefCell<u64>, CpuError> {
+ Ok(RefCell::new(0))
+ }
+
+ fn enable_capture() {}
+
+ fn start_capture(
+ &self,
+ _trace_path: &std::path::Path,
+ ) -> Result<(), CpuError> {
+ Err(CpuError::NotSupported)
+ }
+
+ fn stop_capture(&self) -> Result<(), CpuError> {
+ Err(CpuError::NotSupported)
+ }
+}
diff --git a/crates/uzu/src/backends/cpu/error.rs b/crates/uzu/src/backends/cpu/error.rs
new file mode 100644
index 000000000..e221f160a
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/error.rs
@@ -0,0 +1,7 @@
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum CpuError {
+ #[error("not supported")]
+ NotSupported,
+}
diff --git a/crates/uzu/src/backends/cpu/event.rs b/crates/uzu/src/backends/cpu/event.rs
new file mode 100644
index 000000000..aa3309c3e
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/event.rs
@@ -0,0 +1,8 @@
+use std::cell::RefCell;
+
+use super::Cpu;
+use crate::backends::common::Event;
+
+impl Event for RefCell<u64> {
+ type Backend = Cpu;
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/activation/activation.rs b/crates/uzu/src/backends/cpu/kernel/activation/activation.rs
new file mode 100644
index 000000000..5f1c9c763
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/activation/activation.rs
@@ -0,0 +1,16 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(Activation)]
+#[variants(T, f16, f32, bf16)]
+pub fn activation<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] n: u32,
+ #[allow(unused)] act_type: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/activation/mod.rs b/crates/uzu/src/backends/cpu/kernel/activation/mod.rs
new file mode 100644
index 000000000..3e9730314
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/activation/mod.rs
@@ -0,0 +1 @@
+pub mod activation;
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs
new file mode 100644
index 000000000..4c2471852
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs
@@ -0,0 +1,45 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AttentionGemm)]
+#[variants(T, f32, f16, bf16)]
+#[variants(BK, 16, 32)]
+#[variants(BD, 64, 128, 256)]
+pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(
+ #[allow(unused)] q: *const T,
+ #[allow(unused)] k: *const T,
+ #[allow(unused)] v: *const T,
+ #[allow(unused)] o: *mut T,
+ #[allow(unused)] params: crate::backends::common::gpu_types::attention::AttnParams,
+ #[allow(unused)]
+ #[optional(has_mask)]
+ mask_params: Option<crate::backends::common::gpu_types::attention::AttnMaskParams>,
+ #[allow(unused)]
+ #[optional(has_mask)]
+ mask: Option<*const T>,
+ #[allow(unused)]
+ #[optional(has_sinks)]
+ sinks: Option<*const f32>,
+ #[allow(unused)] num_heads: u32,
+ #[allow(unused)] suffix_length: u32,
+ #[allow(unused)]
+ #[specialize]
+ align_q: bool,
+ #[allow(unused)]
+ #[specialize]
+ align_k: bool,
+ #[allow(unused)]
+ #[specialize]
+ do_causal: bool,
+ #[allow(unused)]
+ #[specialize]
+ has_mask: bool,
+ #[allow(unused)]
+ #[specialize]
+ has_sinks: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs
new file mode 100644
index 000000000..af87fb206
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs
@@ -0,0 +1,53 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AttentionSinglePass)]
+#[variants(T, f32, f16, bf16)]
+#[variants(HEAD_DIM, 64, 128, 256)]
+pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(
+ #[allow(unused)] queries: *const T,
+ #[allow(unused)] keys: *const T,
+ #[allow(unused)] values: *const T,
+ #[allow(unused)] out: *mut T,
+ #[allow(unused)] gqa_factor: u32,
+ #[allow(unused)] sequence_length: u32,
+ #[allow(unused)] k_head_stride: u32,
+ #[allow(unused)] k_seq_stride: u32,
+ #[allow(unused)] v_head_stride: u32,
+ #[allow(unused)] v_seq_stride: u32,
+ #[allow(unused)] scale: f32,
+ #[allow(unused)]
+ #[optional(float_mask)]
+ fmask: Option<*const T>,
+ #[allow(unused)]
+ #[optional(has_mask)]
+ mask_kv_seq_stride: Option<u32>,
+ #[allow(unused)]
+ #[optional(has_mask)]
+ mask_q_seq_stride: Option<u32>,
+ #[allow(unused)]
+ #[optional(has_mask)]
+ mask_head_stride: Option<u32>,
+ #[allow(unused)]
+ #[optional(has_sinks)]
+ sinks: Option<*const f32>,
+ #[allow(unused)] num_heads: u32,
+ #[allow(unused)] suffix_length: u32,
+ #[allow(unused)]
+ #[specialize]
+ float_mask: bool,
+ #[allow(unused)]
+ #[specialize]
+ has_mask: bool,
+ #[allow(unused)]
+ #[specialize]
+ has_sinks: bool,
+ #[allow(unused)]
+ #[specialize]
+ do_causal: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
new file mode 100644
index 000000000..7afa88cad
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
@@ -0,0 +1,69 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AttentionTwoPass1)]
+#[variants(T, f32, f16, bf16)]
+#[variants(HEAD_DIM, 64, 128, 256)]
+pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
+ #[allow(unused)] queries: *const T,
+ #[allow(unused)] keys: *const T,
+ #[allow(unused)] values: *const T,
+ #[allow(unused)] out: *mut f32,
+ #[allow(unused)] sums: *mut f32,
+ #[allow(unused)] maxs: *mut f32,
+ #[allow(unused)] gqa_factor: u32,
+ #[allow(unused)] sequence_length: u32,
+ #[allow(unused)] k_head_stride: u32,
+ #[allow(unused)] k_seq_stride: u32,
+ #[allow(unused)] v_head_stride: u32,
+ #[allow(unused)] v_seq_stride: u32,
+ #[allow(unused)] scale: f32,
+ #[allow(unused)] num_heads: u32,
+ #[allow(unused)] suffix_length: u32,
+ #[allow(unused)]
+ #[optional(float_mask)]
+ fmask: Option<*const T>,
+ #[allow(unused)]
+ #[optional(has_mask)]
+ mask_kv_seq_stride: Option<u32>,
+ #[allow(unused)]
+ #[optional(has_mask)]
+ mask_q_seq_stride: Option<u32>,
+ #[allow(unused)]
+ #[optional(has_mask)]
+ mask_head_stride: Option<u32>,
+ #[allow(unused)]
+ #[optional(has_sinks)]
+ sinks: Option<*const f32>,
+ #[allow(unused)]
+ #[specialize]
+ float_mask: bool,
+ #[allow(unused)]
+ #[specialize]
+ has_mask: bool,
+ #[allow(unused)]
+ #[specialize]
+ has_sinks: bool,
+ #[allow(unused)]
+ #[specialize]
+ do_causal: bool,
+) {
+ todo!()
+}
+
+#[kernel(AttentionTwoPass2)]
+#[variants(T, f32, f16, bf16)]
+#[variants(HEAD_DIM, 64, 128, 256)]
+pub fn attention_two_pass2<T: ArrayElement + Float, const HEAD_DIM: u32>(
+ #[allow(unused)] partials: *const f32,
+ #[allow(unused)] sums: *const f32,
+ #[allow(unused)] maxs: *const f32,
+ #[allow(unused)] out: *mut T,
+ #[allow(unused)] num_heads: u32,
+ #[allow(unused)] suffix_length: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_update_kv_cache.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_update_kv_cache.rs
new file mode 100644
index 000000000..a5147b8a3
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_update_kv_cache.rs
@@ -0,0 +1,22 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AttentionUpdateKVCache)]
+#[variants(T, f32, f16, bf16)]
+pub fn attention_update_kv_cache<T: ArrayElement + Float>(
+ #[allow(unused)] rotated_keys: *const T,
+ #[allow(unused)] qkv: *const T,
+ #[allow(unused)] key_cache: *mut T,
+ #[allow(unused)] value_cache: *mut T,
+ #[allow(unused)] num_groups: u32,
+ #[allow(unused)] num_heads: u32,
+ #[allow(unused)] head_dim: u32,
+ #[allow(unused)] suffix_length: u32,
+ #[allow(unused)] prefix_segment_length: u32,
+ #[allow(unused)] max_sequence_length: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu/src/backends/cpu/kernel/attention/mod.rs
new file mode 100644
index 000000000..24f00ec54
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/attention/mod.rs
@@ -0,0 +1,4 @@
+pub mod attention_gemm;
+pub mod attention_single_pass;
+pub mod attention_two_pass;
+pub mod attention_update_kv_cache;
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/audio.rs b/crates/uzu/src/backends/cpu/kernel/audio/audio.rs
new file mode 100644
index 000000000..81a2fd57a
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/audio.rs
@@ -0,0 +1,167 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioFsqDecode)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_fsq_decode<T: ArrayElement + Float>(
+ #[allow(unused)] tokens: *const i32,
+ #[allow(unused)] out: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] num_groups: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] codebook_dim: i32,
+ #[allow(unused)] num_levels: &[i32],
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
+
+#[kernel(AudioLeakyRelu)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_leaky_relu<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] n: i32,
+ #[allow(unused)] negative_slope: f32,
+) {
+ todo!()
+}
+
+#[kernel(AudioTanh)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_tanh<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] n: i32,
+) {
+ todo!()
+}
+
+#[kernel(AudioAdd)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_add<T: ArrayElement + Float>(
+ #[allow(unused)] a: *const T,
+ #[allow(unused)] b: *const T,
+ #[allow(unused)] out: *mut T,
+ #[allow(unused)] n: i32,
+) {
+ todo!()
+}
+
+#[kernel(AudioScale)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_scale<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] n: i32,
+ #[allow(unused)] scale: f32,
+) {
+ todo!()
+}
+
+#[kernel(AudioCausalConv1d)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_causal_conv1d<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] weight: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] cin: i32,
+ #[allow(unused)] cout: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] kernel_size: i32,
+ #[allow(unused)] dilation: i32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
+
+#[kernel(AudioCausalConvTranspose1d)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_causal_conv_transpose1d<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] weight: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] cin: i32,
+ #[allow(unused)] cout: i32,
+ #[allow(unused)] seq_len_in: i32,
+ #[allow(unused)] seq_len_out: i32,
+ #[allow(unused)] stride: i32,
+ #[allow(unused)] groups: i32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
+
+#[kernel(AudioHalfSnake)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_half_snake<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] alpha: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] channels: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] snake_channels: i32,
+ #[allow(unused)] negative_slope: f32,
+ #[allow(unused)] eps: f32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
+
+#[kernel(AudioClamp)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_clamp<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] n: i32,
+ #[allow(unused)] min_value: f32,
+ #[allow(unused)] max_value: f32,
+) {
+ todo!()
+}
+
+#[kernel(AudioConv1d)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_conv1d<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] weight: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] cin: i32,
+ #[allow(unused)] cout: i32,
+ #[allow(unused)] seq_len_in: i32,
+ #[allow(unused)] seq_len_out: i32,
+ #[allow(unused)] kernel_size: i32,
+ #[allow(unused)] stride: i32,
+ #[allow(unused)] dilation: i32,
+ #[allow(unused)] padding: i32,
+ #[allow(unused)] pad_mode: i32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
+
+#[kernel(AudioFsqEncode)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_fsq_encode<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] tokens: *mut i32,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] num_groups: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] codebook_dim: i32,
+ #[allow(unused)] num_levels: &[i32],
+ #[allow(unused)] dim_base_index: &[i32],
+ #[allow(unused)] eps: f32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/mod.rs b/crates/uzu/src/backends/cpu/kernel/audio/mod.rs
new file mode 100644
index 000000000..fa3e4e468
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/mod.rs
@@ -0,0 +1 @@
+pub mod audio;
diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs
new file mode 100644
index 000000000..3d8e27671
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs
@@ -0,0 +1,19 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(FullPrecisionEmbeddingLookup)]
+#[variants(T, f32, f16, bf16)]
+pub fn full_precision_embedding_lookup<T: ArrayElement + Float>(
+ #[allow(unused)] token_ids: *const u64,
+ #[allow(unused)] weights: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] vocab_size: u32,
+ #[allow(unused)] model_dim: u32,
+ #[allow(unused)] input_scale: f32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs b/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs
new file mode 100644
index 000000000..93f7cf1f0
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs
@@ -0,0 +1,2 @@
+pub mod full_precision_embedding;
+pub mod quant_embedding;
diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
new file mode 100644
index 000000000..f19235562
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
@@ -0,0 +1,23 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(QuantizedEmbeddingLookup)]
+#[variants(T, f32, f16, bf16)]
+pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
+ #[allow(unused)] token_ids: *const u64,
+ #[allow(unused)] weights: *const u8,
+ #[allow(unused)] scales: *const T,
+ #[allow(unused)] biases: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] vocab_size: u32,
+ #[allow(unused)] model_dim: u32,
+ #[allow(unused)] group_size: u32,
+ #[allow(unused)] input_scale: f32,
+ #[allow(unused)] quant_mode: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs
new file mode 100644
index 000000000..58e758c58
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs
@@ -0,0 +1,19 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(KVCacheUpdate)]
+#[variants(T, f32, bf16, f16)]
+pub fn kv_cache_update<T: ArrayElement + Float>(
+ #[allow(unused)] in_place_keys: *mut T,
+ #[allow(unused)] in_place_values: *mut T,
+ #[allow(unused)] swaps: &[crate::backends::common::gpu_types::kv_cache_update::Swap],
+ #[allow(unused)] swap_count: u32,
+ #[allow(unused)] num_heads: u32,
+ #[allow(unused)] max_sequence_length: u32,
+ #[allow(unused)] head_dim: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/kv_cache_update/mod.rs b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/mod.rs
new file mode 100644
index 000000000..128b5561e
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/mod.rs
@@ -0,0 +1 @@
+pub mod kv_cache_update;
diff --git a/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs b/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs
new file mode 100644
index 000000000..c9be4f254
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs
@@ -0,0 +1,28 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(LayerNorm)]
+#[variants(IN, f32, f16, bf16)]
+#[variants(SC, f32, f16, bf16)]
+#[variants(OUT, f32, f16, bf16)]
+#[variants(ACC, f32)]
+pub fn layer_norm<
+ IN: ArrayElement + Float,
+ SC: ArrayElement + Float,
+ OUT: ArrayElement + Float,
+ ACC: ArrayElement + Float,
+>(
+ #[allow(unused)] input: *const IN,
+ #[allow(unused)] scales: *const SC,
+ #[allow(unused)] output: *mut OUT,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] model_dim: u32,
+ #[allow(unused)] epsilon: f32,
+ #[allow(unused)] scale_offset: f32,
+ #[allow(unused)] full_layer: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/layer_norm/mod.rs b/crates/uzu/src/backends/cpu/kernel/layer_norm/mod.rs
new file mode 100644
index 000000000..89b2f7b11
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/layer_norm/mod.rs
@@ -0,0 +1 @@
+pub mod layer_norm;
diff --git a/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs b/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs
new file mode 100644
index 000000000..97498c6e4
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs
@@ -0,0 +1,15 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MaskUpdate)]
+#[variants(T, f32, f16, bf16)]
+pub fn mask_update<T: ArrayElement + Float>(
+ #[allow(unused)] mask: *mut T,
+ #[allow(unused)] unmask_col: i32,
+ #[allow(unused)] mask_col: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rs b/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rs
new file mode 100644
index 000000000..75a6d52a2
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rs
@@ -0,0 +1 @@
+pub mod mask_update;
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs
new file mode 100644
index 000000000..b68fab06e
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs
@@ -0,0 +1 @@
+pub mod shaders;
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rs
new file mode 100644
index 000000000..c55461396
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rs
@@ -0,0 +1 @@
+pub mod steel_gemm;
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
new file mode 100644
index 000000000..91730cabb
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
@@ -0,0 +1,43 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MatmulGemm)]
+#[variants(T, f32, f16, bf16)]
+pub fn matmul_gemm<T: ArrayElement + Float>(
+ #[allow(unused)] a: *const T,
+ #[allow(unused)] b: *const T,
+ #[allow(unused)] d: *mut T,
+ #[allow(unused)] params: &[crate::backends::common::gpu_types::matmul::GEMMParams],
+ #[allow(unused)] group_count_x: u32,
+ #[allow(unused)] group_count_y: u32,
+ #[allow(unused)] group_count_z: u32,
+ #[allow(unused)]
+ #[specialize]
+ block_rows: u32,
+ #[allow(unused)]
+ #[specialize]
+ block_cols: u32,
+ #[allow(unused)]
+ #[specialize]
+ block_depth: u32,
+ #[allow(unused)]
+ #[specialize]
+ warps_per_row: u32,
+ #[allow(unused)]
+ #[specialize]
+ warps_per_col: u32,
+ #[allow(unused)]
+ #[specialize]
+ align_m: bool,
+ #[allow(unused)]
+ #[specialize]
+ align_n: bool,
+ #[allow(unused)]
+ #[specialize]
+ align_k: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs
new file mode 100644
index 000000000..b68fab06e
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs
@@ -0,0 +1 @@
+pub mod shaders;
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs
new file mode 100644
index 000000000..286db8347
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs
@@ -0,0 +1,51 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MatmulGemv)]
+#[variants(T, f32, f16, bf16)]
+pub fn matmul_gemv<T: ArrayElement + Float>(
+ #[allow(unused)] matrix: *const T,
+ #[allow(unused)] input_vector: *const T,
+ #[allow(unused)]
+ #[optional(apply_output_scale_and_accumulate)]
+ output_source: Option<*const T>,
+ #[allow(unused)] output_vector: *mut T,
+ #[allow(unused)] input_dimension: i32,
+ #[allow(unused)] output_dimension: i32,
+ #[allow(unused)] matrix_leading_dimension: i32,
+ #[allow(unused)] output_scale: f32,
+ #[allow(unused)] output_accumulate_scale: f32,
+ #[allow(unused)] batch_shape: &[i32],
+ #[allow(unused)] vector_batch_stride: &[i32],
+ #[allow(unused)] matrix_batch_stride: &[i32],
+ #[allow(unused)] output_source_batch_stride: &[i32],
+ #[allow(unused)] output_source_stride: i32,
+ #[allow(unused)] batch_rows: i32,
+ #[allow(unused)] output_rows_per_threadgroup: i32,
+ #[allow(unused)]
+ #[specialize]
+ tg_simd_rows: u32,
+ #[allow(unused)]
+ #[specialize]
+ tg_simd_cols: u32,
+ #[allow(unused)]
+ #[specialize]
+ sg_thread_rows: u32,
+ #[allow(unused)]
+ #[specialize]
+ sg_thread_cols: u32,
+ #[allow(unused)]
+ #[specialize]
+ thread_out_rows: u32,
+ #[allow(unused)]
+ #[specialize]
+ thread_out_cols: u32,
+ #[allow(unused)]
+ #[specialize]
+ apply_output_scale_and_accumulate: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/mod.rs
new file mode 100644
index 000000000..fc35204af
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/mod.rs
@@ -0,0 +1 @@
+pub mod gemv;
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs
new file mode 100644
index 000000000..26abe007d
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs
@@ -0,0 +1,3 @@
+pub mod gemm;
+pub mod gemv;
+pub mod split_k;
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/mod.rs
new file mode 100644
index 000000000..b68fab06e
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/mod.rs
@@ -0,0 +1 @@
+pub mod shaders;
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/mod.rs
new file mode 100644
index 000000000..8295f2b83
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/mod.rs
@@ -0,0 +1 @@
+pub mod steel_gemm_splitk;
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/steel_gemm_splitk.rs b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/steel_gemm_splitk.rs
new file mode 100644
index 000000000..4a3efbd68
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/steel_gemm_splitk.rs
@@ -0,0 +1,31 @@
+use dsl::kernel;
+use half::bf16;
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MatmulSplitKPartialBfloat16)]
+pub fn matmul_split_k_partial_bfloat16(
+ #[allow(unused)] a: *const bf16,
+ #[allow(unused)] b: *const bf16,
+ #[allow(unused)] c: *mut f32,
+ #[allow(unused)] params: &[crate::backends::common::gpu_types::matmul::GEMMSpiltKParams],
+ #[allow(unused)] partial_group_count_x: u32,
+ #[allow(unused)] partial_group_count_y: u32,
+ #[allow(unused)] partial_group_count_z: u32,
+) {
+ todo!()
+}
+
+#[kernel(MatmulSplitKAccumBfloat16)]
+pub fn matmul_split_k_accum_bfloat16(
+ #[allow(unused)] c_split: *const f32,
+ #[allow(unused)] d: *mut bf16,
+ #[allow(unused)] k_partitions: i32,
+ #[allow(unused)] partition_stride: i32,
+ #[allow(unused)] ldd: i32,
+ #[allow(unused)] accum_total_threads_x: u32,
+ #[allow(unused)] accum_total_threads_y: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs
new file mode 100644
index 000000000..b831f6dbc
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs
@@ -0,0 +1,17 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MlpGateActMul)]
+#[variants(T, f32, f16, bf16)]
+pub fn mlp_gate_act_mul<T: ArrayElement + Float>(
+ #[allow(unused)] fused_up: *const T,
+ #[allow(unused)] hidden: *mut T,
+ #[allow(unused)] h: i32,
+ #[allow(unused)] m: i32,
+ #[allow(unused)] act_type: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/mlp/mod.rs b/crates/uzu/src/backends/cpu/kernel/mlp/mod.rs
new file mode 100644
index 000000000..c6ca0a7ce
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/mlp/mod.rs
@@ -0,0 +1 @@
+pub mod activation_mul;
diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rs
new file mode 100644
index 000000000..ec3bb5c04
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs
@@ -0,0 +1,65 @@
+#![allow(unused)]
+mod activation;
+mod attention;
+mod audio;
+mod embedding;
+mod kv_cache_update;
+mod layer_norm;
+mod mask_update;
+mod matmul;
+mod mlp;
+mod moe;
+mod pooling;
+mod quant_matmul;
+mod rms_norm;
+mod rope;
+mod sampling;
+mod short_conv;
+mod sigmoid;
+mod ssm;
+mod tensor_add_bias;
+mod tensor_add_swap;
+mod tensor_copy;
+mod token_copy;
+
+include!(concat!(env!("OUT_DIR"), "/cpu/dsl.rs"));
+
+use crate::{
+ DataType,
+ backends::{
+ common::{
+ Backend,
+ kernel::matmul::{
+ FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,
+ MatmulKernels,
+ },
+ },
+ cpu::{Cpu, context::CpuContext, error::CpuError},
+ },
+};
+
+pub struct FullPrecisionMatmulCpuKernel;
+
+impl FullPrecisionMatmulKernelTrait for FullPrecisionMatmulCpuKernel {
+ type Backend = Cpu;
+
+ fn new(
+ #[allow(unused)] context: &CpuContext,
+ #[allow(unused)] data_type: DataType,
+ ) -> Result<Self, CpuError> {
+ Ok(Self)
+ }
+
+ fn encode(
+ &mut self,
+ #[allow(unused)] context: &CpuContext,
+ #[allow(unused)] encoder: &<Cpu as Backend>::ComputeEncoder,
+ #[allow(unused)] arguments: FullPrecisionMatmulArguments<Cpu>,
+ ) {
+ encoder.borrow_mut().push_command(move || todo!());
+ }
+}
+
+impl MatmulKernels for CpuKernels {
+ type FullPrecisionMatmulKernel = FullPrecisionMatmulCpuKernel;
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/moe/counts_offsets_fused.rs b/crates/uzu/src/backends/cpu/kernel/moe/counts_offsets_fused.rs
new file mode 100644
index 000000000..f4f39e19d
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/moe/counts_offsets_fused.rs
@@ -0,0 +1,17 @@
+use dsl::kernel;
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MoeCountsOffsetsFused)]
+pub fn moe_counts_offsets_fused(
+ #[allow(unused)] topk_ids: *const i32,
+ #[allow(unused)] offsets: *mut u32,
+ #[allow(unused)] sum_k_out: *mut u32,
+ #[allow(unused)] partials: *mut u32,
+ #[allow(unused)] t_input: u32,
+ #[allow(unused)] e_input: u32,
+ #[allow(unused)] k_input: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/moe/experts_single.rs b/crates/uzu/src/backends/cpu/kernel/moe/experts_single.rs
new file mode 100644
index 000000000..064e54d8b
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/moe/experts_single.rs
@@ -0,0 +1,44 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MoeExpertsDecodeSinglePassA)]
+#[variants(T, f32, f16, bf16)]
+pub fn moe_experts_decode_single_pass_a<T: ArrayElement + Float>(
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] topk_ids: *const i32,
+ #[allow(unused)] w13_all: *const T,
+ #[allow(unused)] biases: *const T,
+ #[allow(unused)] hidden_out: *mut f32,
+ #[allow(unused)] d_model: u32,
+ #[allow(unused)] d_ff: u32,
+ #[allow(unused)] k: u32,
+ #[allow(unused)] silu_alpha: f32,
+ #[allow(unused)] gate_clip_min: f32,
+ #[allow(unused)] gate_clip_max: f32,
+ #[allow(unused)] up_clip_min: f32,
+ #[allow(unused)] up_clip_max: f32,
+ #[allow(unused)]
+ #[specialize]
+ gating_sel: u32,
+) {
+ todo!()
+}
+
+#[kernel(MoeExpertsDecodeSinglePassB)]
+#[variants(T, f32, f16, bf16)]
+pub fn moe_experts_decode_single_pass_b<T: ArrayElement + Float>(
+ #[allow(unused)] hidden: *const f32,
+ #[allow(unused)] topk_ids: *const i32,
+ #[allow(unused)] topk_probs: *const T,
+ #[allow(unused)] w2_all: *const T,
+ #[allow(unused)] biases: *const T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] d_model: u32,
+ #[allow(unused)] d_ff: u32,
+ #[allow(unused)] k_input: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_decode.rs
new file mode 100644
index 000000000..c61db4f40
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_decode.rs
@@ -0,0 +1,47 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MoeExpertsDecodePassA)]
+#[variants(T, f32, f16, bf16)]
+pub fn moe_experts_decode_pass_a<T: ArrayElement + Float>(
+ #[allow(unused)] x_perm: *const T,
+ #[allow(unused)] expert_offsets: *const u32,
+ #[allow(unused)] w13_all: *const T,
+ #[allow(unused)] hidden_out: *mut f32,
+ #[allow(unused)] up_biases: *const T,
+ #[allow(unused)] d_model: u32,
+ #[allow(unused)] d_ff: u32,
+ #[allow(unused)] e: u32,
+ #[allow(unused)] gate_clip_min: f32,
+ #[allow(unused)] gate_clip_max: f32,
+ #[allow(unused)] up_clip_min: f32,
+ #[allow(unused)] up_clip_max: f32,
+ #[allow(unused)] silu_alpha: f32,
+ #[allow(unused)] tile_map: *const u32,
+ #[allow(unused)]
+ #[specialize]
+ gating_sel: u32,
+ #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,
+) {
+ todo!()
+}
+
+#[kernel(MoeExpertsDecodeDownFused2D)]
+#[variants(T, f32, f16, bf16)]
+#[variants(AccumT, f32)]
+pub fn moe_experts_decode_down_fused2_d<T: ArrayElement + Float, AccumT: ArrayElement + Float>(
+ #[allow(unused)] hidden: *const f32,
+ #[allow(unused)] row_expert_map: *const u32,
+ #[allow(unused)] w2_all: *const T,
+ #[allow(unused)] down_biases: *const T,
+ #[allow(unused)] y_out: *mut T,
+ #[allow(unused)] total_rows: u32,
+ #[allow(unused)] d_model: u32,
+ #[allow(unused)] d_ff: u32,
+ #[allow(unused)] e: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rs
new file mode 100644
index 000000000..119f4d585
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rs
@@ -0,0 +1,47 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MoeExpertsPrefillPassA)]
+#[variants(T, f32, f16, bf16)]
+pub fn moe_experts_prefill_pass_a<T: ArrayElement + Float>(
+ #[allow(unused)] x_perm: *const T,
+ #[allow(unused)] expert_offsets: *const u32,
+ #[allow(unused)] w13_all: *const T,
+ #[allow(unused)] up_biases: *const T,
+ #[allow(unused)] hidden_out: *mut f32,
+ #[allow(unused)] d_model: u32,
+ #[allow(unused)] d_ff: u32,
+ #[allow(unused)] e: u32,
+ #[allow(unused)] gate_clip_min: f32,
+ #[allow(unused)] gate_clip_max: f32,
+ #[allow(unused)] up_clip_min: f32,
+ #[allow(unused)] up_clip_max: f32,
+ #[allow(unused)] silu_alpha: f32,
+ #[allow(unused)] tile_map: *const u32,
+ #[allow(unused)]
+ #[specialize]
+ gating_sel: u32,
+ #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,
+) {
+ todo!()
+}
+
+#[kernel(MoeExpertsPrefillPassB)]
+#[variants(T, f32, f16, bf16)]
+pub fn moe_experts_prefill_pass_b<T: ArrayElement + Float>(
+ #[allow(unused)] hidden: *const f32,
+ #[allow(unused)] expert_offsets: *const u32,
+ #[allow(unused)] w2_all: *const T,
+ #[allow(unused)] down_biases: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] d_model: u32,
+ #[allow(unused)] d_ff: u32,
+ #[allow(unused)] e: u32,
+ #[allow(unused)] tile_map: *const u32,
+ #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/moe/finalize.rs b/crates/uzu/src/backends/cpu/kernel/moe/finalize.rs
new file mode 100644
index 000000000..b1ecf696d
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/moe/finalize.rs
@@ -0,0 +1,19 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MoeFinalize)]
+#[variants(T, f32, f16, bf16)]
+pub fn moe_finalize<T: ArrayElement + Float>(
+ #[allow(unused)] tok2row: *const i32,
+ #[allow(unused)] probs: *const T,
+ #[allow(unused)] y_partial: *const T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] t_count: u32,
+ #[allow(unused)] d_model: u32,
+ #[allow(unused)] k_input: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/moe/gather.rs b/crates/uzu/src/backends/cpu/kernel/moe/gather.rs
new file mode 100644
index 000000000..c6d9f6329
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/moe/gather.rs
@@ -0,0 +1,33 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MoeGatherXPerm2D)]
+#[variants(T, f32, f16, bf16)]
+pub fn moe_gather_x_perm2_d<T: ArrayElement + Float>(
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] bucketed_ids: *const i32,
+ #[allow(unused)] x_perm: *mut T,
+ #[allow(unused)] sumk_buf: *const u32,
+ #[allow(unused)] d_model: u32,
+ #[allow(unused)] t: u32,
+ #[allow(unused)] k: u32,
+) {
+ todo!()
+}
+
+#[kernel(MoeGatherXPerm1D)]
+#[variants(T, f32, f16, bf16)]
+pub fn moe_gather_x_perm1_d<T: ArrayElement + Float>(
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] bucketed_ids: *const i32,
+ #[allow(unused)] x_perm: *mut T,
+ #[allow(unused)] sumk_buf: *const u32,
+ #[allow(unused)] d_model: u32,
+ #[allow(unused)] t: u32,
+ #[allow(unused)] k: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/moe/mod.rs b/crates/uzu/src/backends/cpu/kernel/moe/mod.rs
new file mode 100644
index 000000000..5f029d341
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/moe/mod.rs
@@ -0,0 +1,10 @@
+pub mod counts_offsets_fused;
+pub mod experts_single;
+pub mod experts_two_pass_decode;
+pub mod experts_two_pass_prefill;
+pub mod finalize;
+pub mod gather;
+pub mod router_topk;
+pub mod scatter_buckets;
+pub mod tiles_map;
+pub mod tiles_pass_a;
diff --git a/crates/uzu/src/backends/cpu/kernel/moe/router_topk.rs b/crates/uzu/src/backends/cpu/kernel/moe/router_topk.rs
new file mode 100644
index 000000000..69eb270bf
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/moe/router_topk.rs
@@ -0,0 +1,22 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MoeRouterTopK)]
+#[variants(ScalarT, f16, bf16, f32)]
+pub fn moe_router_top_k<ScalarT: ArrayElement + Float>(
+ #[allow(unused)] input: *const ScalarT,
+ #[allow(unused)] weight: *const ScalarT,
+ #[allow(unused)] bias: *const ScalarT,
+ #[allow(unused)] topk_ids: *mut i32,
+ #[allow(unused)] topk_probs: *mut ScalarT,
+ #[allow(unused)] t: u32,
+ #[allow(unused)] d_model: u32,
+ #[allow(unused)] e: u32,
+ #[allow(unused)] k: u32,
+ #[allow(unused)] renorm: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/moe/scatter_buckets.rs b/crates/uzu/src/backends/cpu/kernel/moe/scatter_buckets.rs
new file mode 100644
index 000000000..e0b521345
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/moe/scatter_buckets.rs
@@ -0,0 +1,57 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MoeBlockBasesFromPartials)]
+pub fn moe_block_bases_from_partials(
+ #[allow(unused)] partials: *const u32,
+ #[allow(unused)] block_bases: *mut u32,
+ #[allow(unused)] block_alloc: *mut u32,
+ #[allow(unused)] e_input: u32,
+ #[allow(unused)] num_blocks: u32,
+ #[allow(unused)] num_tiles: u32,
+ #[allow(unused)] capacity_per_expert: u32,
+) {
+ todo!()
+}
+
+#[kernel(MoeScatterBuckets)]
+#[variants(T, f32, f16, bf16)]
+pub fn moe_scatter_buckets<T: ArrayElement + Float>(
+ #[allow(unused)] topk_ids: *const i32,
+ #[allow(unused)] topk_probs: *const T,
+ #[allow(unused)] offsets: *const u32,
+ #[allow(unused)] block_bases: *const u32,
+ #[allow(unused)] block_alloc: *const u32,
+ #[allow(unused)] out_ids: *mut i32,
+ #[allow(unused)] out_probs: *mut T,
+ #[allow(unused)] t: u32,
+ #[allow(unused)] e: u32,
+ #[allow(unused)] k: u32,
+ #[allow(unused)] num_blocks: u32,
+ #[allow(unused)] num_tiles: u32,
+) {
+ todo!()
+}
+
+#[kernel(MoeScatterBucketsMap)]
+#[variants(T, f32, f16, bf16)]
+pub fn moe_scatter_buckets_map<T: ArrayElement + Float>(
+ #[allow(unused)] topk_ids: *const i32,
+ #[allow(unused)] topk_probs: *const T,
+ #[allow(unused)] offsets: *const u32,
+ #[allow(unused)] block_bases: *const u32,
+ #[allow(unused)] block_alloc: *const u32,
+ #[allow(unused)] out_ids: *mut i32,
+ #[allow(unused)] out_probs: *mut T,
+ #[allow(unused)] t: u32,
+ #[allow(unused)] e: u32,
+ #[allow(unused)] k: u32,
+ #[allow(unused)] num_blocks: u32,
+ #[allow(unused)] num_tiles: u32,
+ #[allow(unused)] tok2row: *mut i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/cpu/kernel/moe/tiles_map.rs
new file mode 100644
index 000000000..2314c6cbb
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/moe/tiles_map.rs
@@ -0,0 +1,43 @@
+use dsl::kernel;
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MoeTileCounts)]
+pub fn moe_tile_counts(
+ #[allow(unused)] offsets: *const u32,
+ #[allow(unused)] tile_counts: *mut u32,
+ #[allow(unused)] e: u32,
+) {
+ todo!()
+}
+
+#[kernel(MoeTileScan)]
+pub fn moe_tile_scan(
+ #[allow(unused)] tile_counts: *const u32,
+ #[allow(unused)] tile_row_offsets: *mut u32,
+ #[allow(unused)] total_tiles_buf: *mut u32,
+ #[allow(unused)] e: u32,
+) {
+ todo!()
+}
+
+#[kernel(MoeBuildTileMap)]
+pub fn moe_build_tile_map(
+ #[allow(unused)] offsets: *const u32,
+ #[allow(unused)] tile_row_offsets: *const u32,
+ #[allow(unused)] tile_counts: *const u32,
+ #[allow(unused)] tile_map: *mut u32,
+ #[allow(unused)] e: u32,
+) {
+ todo!()
+}
+
+#[kernel(MoeWriteDispatchArgs)]
+pub fn moe_write_dispatch_args(
+ #[allow(unused)] total_tiles_buf: *const u32,
+ #[allow(unused)] dispatch_args: *mut u32,
+ #[allow(unused)] num_tiles_n: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/cpu/kernel/moe/tiles_pass_a.rs
new file mode 100644
index 000000000..7a165441c
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/moe/tiles_pass_a.rs
@@ -0,0 +1,55 @@
+use dsl::kernel;
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MoePassATileCounts)]
+pub fn moe_pass_a_tile_counts(
+ #[allow(unused)] expert_offsets: *const u32,
+ #[allow(unused)] tile_counts: *mut u32,
+ #[allow(unused)] e: u32,
+ #[allow(unused)] h_blocks: u32,
+) {
+ todo!()
+}
+
+#[kernel(MoePassATileScan)]
+pub fn moe_pass_a_tile_scan(
+ #[allow(unused)] tile_counts: *const u32,
+ #[allow(unused)] tile_offsets: *mut u32,
+ #[allow(unused)] total_tiles: *mut u32,
+ #[allow(unused)] e: u32,
+) {
+ todo!()
+}
+
+#[kernel(MoePassABuildRowMap)]
+pub fn moe_pass_a_build_row_map(
+ #[allow(unused)] expert_offsets: *const u32,
+ #[allow(unused)] row_expert_map: *mut u32,
+ #[allow(unused)] total_rows: u32,
+ #[allow(unused)] e: u32,
+) {
+ todo!()
+}
+
+#[kernel(MoePassABuildTileMap)]
+pub fn moe_pass_a_build_tile_map(
+ #[allow(unused)] expert_offsets: *const u32,
+ #[allow(unused)] tile_offsets: *const u32,
+ #[allow(unused)] row_expert_map: *const u32,
+ #[allow(unused)] tile_map: *mut u32,
+ #[allow(unused)] total_rows: u32,
+ #[allow(unused)] h_blocks: u32,
+) {
+ todo!()
+}
+
+#[kernel(MoePassAWriteDispatchArgs)]
+pub fn moe_pass_a_write_dispatch_args(
+ #[allow(unused)] total_tiles: *const u32,
+ #[allow(unused)] dispatch_args: *mut u32,
+ #[allow(unused)] num_tiles_y: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/pooling/mod.rs b/crates/uzu/src/backends/cpu/kernel/pooling/mod.rs
new file mode 100644
index 000000000..f30070eb3
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/pooling/mod.rs
@@ -0,0 +1 @@
+pub mod pooling;
diff --git a/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs b/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs
new file mode 100644
index 000000000..9daf2804b
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs
@@ -0,0 +1,29 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(PoolingCls)]
+#[variants(T, f32, f16, bf16)]
+pub fn pooling_cls<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] seq_len: u32,
+ #[allow(unused)] hidden_dim: u32,
+ #[allow(unused)] batch_size: u32,
+) {
+ todo!()
+}
+
+#[kernel(PoolingMean)]
+#[variants(T, f32, f16, bf16)]
+pub fn pooling_mean<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] seq_len: u32,
+ #[allow(unused)] hidden_dim: u32,
+ #[allow(unused)] batch_size: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
new file mode 100644
index 000000000..dadd0367b
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
@@ -0,0 +1,6 @@
+pub mod qmm;
+pub mod qmm_transposed;
+pub mod qmm_transposed_64x64;
+pub mod qmv;
+pub mod qmv_fast;
+pub mod qvm;
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rs
new file mode 100644
index 000000000..823301f3d
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rs
@@ -0,0 +1,36 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(QuantizedMatmulQmm)]
+#[variants(T, f32, f16, bf16)]
+#[variants(GROUP_SIZE, 32, 64, 128)]
+#[variants(BITS, 4, 8)]
+pub fn quantized_matmul_qmm<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
+ #[allow(unused)] w: *const u32,
+ #[allow(unused)] scales: *const T,
+ #[allow(unused)]
+ #[optional(use_zero_points)]
+ zero_points: Option<*const u8>,
+ #[allow(unused)]
+ #[optional(use_mlx_quant)]
+ biases: Option<*const T>,
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] k: i32,
+ #[allow(unused)] n: i32,
+ #[allow(unused)] m: i32,
+ #[allow(unused)]
+ #[specialize]
+ use_zero_points: bool,
+ #[allow(unused)]
+ #[specialize]
+ use_mlx_quant: bool,
+ #[allow(unused)]
+ #[specialize]
+ aligned_k: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
new file mode 100644
index 000000000..62f3f9456
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
@@ -0,0 +1,36 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(QuantizedMatmulQmmTransposed)]
+#[variants(T, f32, f16, bf16)]
+#[variants(GROUP_SIZE, 32, 64, 128)]
+#[variants(BITS, 4, 8)]
+pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
+ #[allow(unused)] w: *const u32,
+ #[allow(unused)] scales: *const T,
+ #[allow(unused)]
+ #[optional(use_zero_points)]
+ zero_points: Option<*const u8>,
+ #[allow(unused)]
+ #[optional(use_mlx_quant)]
+ biases: Option<*const T>,
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] k: i32,
+ #[allow(unused)] n: i32,
+ #[allow(unused)] m: i32,
+ #[allow(unused)]
+ #[specialize]
+ use_zero_points: bool,
+ #[allow(unused)]
+ #[specialize]
+ use_mlx_quant: bool,
+ #[allow(unused)]
+ #[specialize]
+ aligned_n: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs
new file mode 100644
index 000000000..e252f69bc
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs
@@ -0,0 +1,33 @@
+use dsl::kernel;
+use half::bf16;
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(QuantizedMatmulQmmTransposed64x64)]
+#[variants(T, bf16)]
+#[variants(GROUP_SIZE, 64, 128)]
+#[variants(BITS, 4, 8)]
+pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
+ #[allow(unused)] w: *const u32,
+ #[allow(unused)] scales: *const T,
+ #[allow(unused)]
+ #[optional(use_zero_points)]
+ zero_points: Option<*const u8>,
+ #[allow(unused)]
+ #[optional(use_mlx_quant)]
+ biases: Option<*const T>,
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] k: i32,
+ #[allow(unused)] n: i32,
+ #[allow(unused)] m: i32,
+ #[allow(unused)]
+ #[specialize]
+ use_zero_points: bool,
+ #[allow(unused)]
+ #[specialize]
+ use_mlx_quant: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs
new file mode 100644
index 000000000..5ae376ea9
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs
@@ -0,0 +1,33 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(QuantizedMatmulQmv)]
+#[variants(T, f32, f16, bf16)]
+#[variants(GROUP_SIZE, 32, 64, 128)]
+#[variants(BITS, 4, 8)]
+pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
+ #[allow(unused)] w: *const u32,
+ #[allow(unused)] scales: *const T,
+ #[allow(unused)]
+ #[optional(use_zero_points)]
+ zero_points: Option<*const u8>,
+ #[allow(unused)]
+ #[optional(use_mlx_quant)]
+ biases: Option<*const T>,
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] k: i32,
+ #[allow(unused)] n: i32,
+ #[allow(unused)] m: i32,
+ #[allow(unused)]
+ #[specialize]
+ use_zero_points: bool,
+ #[allow(unused)]
+ #[specialize]
+ use_mlx_quant: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
new file mode 100644
index 000000000..33f16dbcc
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
@@ -0,0 +1,33 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(QuantizedMatmulQmvFast)]
+#[variants(T, f32, f16, bf16)]
+#[variants(GROUP_SIZE, 32, 64, 128)]
+#[variants(BITS, 4, 8)]
+pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
+ #[allow(unused)] w: *const u32,
+ #[allow(unused)] scales: *const T,
+ #[allow(unused)]
+ #[optional(use_zero_points)]
+ zero_points: Option<*const u8>,
+ #[allow(unused)]
+ #[optional(use_mlx_quant)]
+ biases: Option<*const T>,
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] k: i32,
+ #[allow(unused)] n: i32,
+ #[allow(unused)] m: i32,
+ #[allow(unused)]
+ #[specialize]
+ use_zero_points: bool,
+ #[allow(unused)]
+ #[specialize]
+ use_mlx_quant: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rs
new file mode 100644
index 000000000..6eeba6605
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rs
@@ -0,0 +1,33 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(QuantizedMatmulQvm)]
+#[variants(T, f32, f16, bf16)]
+#[variants(GROUP_SIZE, 32, 64, 128)]
+#[variants(BITS, 4, 8)]
+pub fn quantized_matmul_qvm<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
+ #[allow(unused)] w: *const u32,
+ #[allow(unused)] scales: *const T,
+ #[allow(unused)]
+ #[optional(use_zero_points)]
+ zero_points: Option<*const u8>,
+ #[allow(unused)]
+ #[optional(use_mlx_quant)]
+ biases: Option<*const T>,
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] k: i32,
+ #[allow(unused)] n: i32,
+ #[allow(unused)] m: i32,
+ #[allow(unused)]
+ #[specialize]
+ use_zero_points: bool,
+ #[allow(unused)]
+ #[specialize]
+ use_mlx_quant: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/mod.rs
new file mode 100644
index 000000000..c26f009f4
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/mod.rs
@@ -0,0 +1,2 @@
+pub mod qk_norm;
+pub mod rms_norm;
diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/qk_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/qk_norm.rs
new file mode 100644
index 000000000..935948d4d
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/qk_norm.rs
@@ -0,0 +1,32 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(QKNorm)]
+#[variants(InputT, f32, f16, bf16)]
+#[variants(ScaleT, f32, f16, bf16)]
+#[variants(OutputT, f32, f16, bf16)]
+#[variants(AccumT, f32, f16)]
+pub fn qk_norm<
+ InputT: ArrayElement + Float,
+ ScaleT: ArrayElement + Float,
+ OutputT: ArrayElement + Float,
+ AccumT: ArrayElement + Float,
+>(
+ #[allow(unused)] qkv_input: *const InputT,
+ #[allow(unused)] scales: *const ScaleT,
+ #[allow(unused)] qkv_output: *mut OutputT,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] num_q_heads: u32,
+ #[allow(unused)] num_kv_heads: u32,
+ #[allow(unused)] head_dim: u32,
+ #[allow(unused)] epsilon: f32,
+ #[allow(unused)] scale_offset: f32,
+ #[allow(unused)] head_offset: u32,
+ #[allow(unused)] head_count: u32,
+ #[allow(unused)] full_layer: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
new file mode 100644
index 000000000..a778079fd
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
@@ -0,0 +1,28 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(RMSNorm)]
+#[variants(InputT, f32, f16, bf16)]
+#[variants(ScaleT, f32, f16, bf16)]
+#[variants(OutputT, f32, f16, bf16)]
+#[variants(AccumT, f32, f16)]
+pub fn rms_norm<
+ InputT: ArrayElement + Float,
+ ScaleT: ArrayElement + Float,
+ OutputT: ArrayElement + Float,
+ AccumT: ArrayElement + Float,
+>(
+ #[allow(unused)] input: *const InputT,
+ #[allow(unused)] scales: *const ScaleT,
+ #[allow(unused)] output: *mut OutputT,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] element_count: u32,
+ #[allow(unused)] epsilon: f32,
+ #[allow(unused)] scale_offset: f32,
+ #[allow(unused)] full_layer: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/rope/mod.rs b/crates/uzu/src/backends/cpu/kernel/rope/mod.rs
new file mode 100644
index 000000000..f67f4f08b
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/rope/mod.rs
@@ -0,0 +1 @@
+pub mod rope;
diff --git a/crates/uzu/src/backends/cpu/kernel/rope/rope.rs b/crates/uzu/src/backends/cpu/kernel/rope/rope.rs
new file mode 100644
index 000000000..ea5da869f
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/rope/rope.rs
@@ -0,0 +1,23 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(Rope)]
+#[variants(T, f32, f16, bf16)]
+pub fn rope<T: ArrayElement + Float>(
+ #[allow(unused)] qkv: *const T,
+ #[allow(unused)] cosines: *const T,
+ #[allow(unused)] sines: *const T,
+ #[allow(unused)] token_positions: *const i32,
+ #[allow(unused)] rotated_queries: *mut T,
+ #[allow(unused)] rotated_keys: *mut T,
+ #[allow(unused)] head_dim: u32,
+ #[allow(unused)] num_heads: u32,
+ #[allow(unused)] num_groups: u32,
+ #[allow(unused)] suffix_length: u32,
+ #[allow(unused)] max_sequence_length: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/argmax.rs b/crates/uzu/src/backends/cpu/kernel/sampling/argmax.rs
new file mode 100644
index 000000000..61aea8a75
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/argmax.rs
@@ -0,0 +1,37 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(ArgmaxSingle)]
+#[variants(T, f32, f16, bf16)]
+pub fn argmax_single<T: ArrayElement + Float>(
+ #[allow(unused)] logits_data: *const T,
+ #[allow(unused)] final_tokens: *mut u32,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] vocab_size: u32,
+) {
+ todo!()
+}
+
+#[kernel(ArgmaxMain)]
+#[variants(T, f32, f16, bf16)]
+pub fn argmax_main<T: ArrayElement + Float>(
+ #[allow(unused)] logits_data: *const T,
+ #[allow(unused)] partial_results: *mut crate::backends::common::gpu_types::argmax::ArgmaxPair,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] vocab_size: u32,
+) {
+ todo!()
+}
+
+#[kernel(ArgmaxFinal)]
+pub fn argmax_final(
+ #[allow(unused)] partial_results: *const crate::backends::common::gpu_types::argmax::ArgmaxPair,
+ #[allow(unused)] final_tokens: *mut u32,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] vocab_size: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs b/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs
new file mode 100644
index 000000000..61702f72a
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs
@@ -0,0 +1,17 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(Bitmask)]
+#[variants(T, f32, f16, bf16)]
+pub fn bitmask<T: ArrayElement + Float>(
+ #[allow(unused)] logits: *const T,
+ #[allow(unused)] bitmask: *const u32,
+ #[allow(unused)] processed_logits: *mut T,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] vocab_size: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs b/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs
new file mode 100644
index 000000000..943b956e6
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs
@@ -0,0 +1,17 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(Gumbel)]
+#[variants(T, f32, f16, bf16)]
+pub fn gumbel<T: ArrayElement + Float>(
+ #[allow(unused)] logits: *const T,
+ #[allow(unused)] batch_seeds: *const u64,
+ #[allow(unused)] processed_logits: *mut T,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] vocab_size: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs b/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs
new file mode 100644
index 000000000..58e9deb06
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs
@@ -0,0 +1,17 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(MinP)]
+#[variants(T, f32, f16, bf16)]
+pub fn min_p<T: ArrayElement + Float>(
+ #[allow(unused)] logits: *const T,
+ #[allow(unused)] processed_logits: *mut T,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] vocab_size: u32,
+ #[allow(unused)] min_p: f32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/mod.rs b/crates/uzu/src/backends/cpu/kernel/sampling/mod.rs
new file mode 100644
index 000000000..db49e2e7a
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/mod.rs
@@ -0,0 +1,7 @@
+pub mod argmax;
+pub mod bitmask;
+pub mod gumbel;
+pub mod min_p;
+pub mod temperature;
+pub mod top_k;
+pub mod top_p;
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs b/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs
new file mode 100644
index 000000000..b40ee1cea
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs
@@ -0,0 +1,17 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(Temperature)]
+#[variants(T, f32, f16, bf16)]
+pub fn temperature<T: ArrayElement + Float>(
+ #[allow(unused)] logits: *const T,
+ #[allow(unused)] processed_logits: *mut T,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] vocab_size: u32,
+ #[allow(unused)] temperature: f32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs b/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs
new file mode 100644
index 000000000..909cebd82
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs
@@ -0,0 +1,17 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(TopK)]
+#[variants(T, f32, f16, bf16)]
+pub fn top_k<T: ArrayElement + Float>(
+ #[allow(unused)] logits: *const T,
+ #[allow(unused)] processed_logits: *mut T,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] vocab_size: u32,
+ #[allow(unused)] top_k: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs b/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs
new file mode 100644
index 000000000..f6879e479
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs
@@ -0,0 +1,17 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(TopP)]
+#[variants(T, f32, f16, bf16)]
+pub fn top_p<T: ArrayElement + Float>(
+ #[allow(unused)] logits: *const T,
+ #[allow(unused)] processed_logits: *mut T,
+ #[allow(unused)] batch_size: u32,
+ #[allow(unused)] vocab_size: u32,
+ #[allow(unused)] top_p: f32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/short_conv/mod.rs b/crates/uzu/src/backends/cpu/kernel/short_conv/mod.rs
new file mode 100644
index 000000000..2834d2c1d
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/short_conv/mod.rs
@@ -0,0 +1 @@
+pub mod short_conv;
diff --git a/crates/uzu/src/backends/cpu/kernel/short_conv/short_conv.rs b/crates/uzu/src/backends/cpu/kernel/short_conv/short_conv.rs
new file mode 100644
index 000000000..fe031529a
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/short_conv/short_conv.rs
@@ -0,0 +1,89 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(ShortConvPack)]
+#[variants(T, f32, f16, bf16)]
+pub fn short_conv_pack<T: ArrayElement + Float>(
+ #[allow(unused)] state_in: *const T,
+ #[allow(unused)] in_proj: *const T,
+ #[allow(unused)] padded: *mut T,
+ #[allow(unused)] state_stride: u32,
+ #[allow(unused)] suffix_len: u32,
+ #[allow(unused)] in_proj_stride: u32,
+ #[allow(unused)] model_dim: u32,
+) {
+ todo!()
+}
+
+#[kernel(ShortConvPrefill)]
+#[variants(T, f32, f16, bf16)]
+pub fn short_conv_prefill<T: ArrayElement + Float>(
+ #[allow(unused)] padded: *const T,
+ #[allow(unused)] in_proj: *const T,
+ #[allow(unused)] w: *const T,
+ #[allow(unused)]
+ #[optional(has_bias)]
+ b: Option<*const T>,
+ #[allow(unused)] out: *mut T,
+ #[allow(unused)] state_out: *mut T,
+ #[allow(unused)] suffix_len: u32,
+ #[allow(unused)] kernel_size: u32,
+ #[allow(unused)] in_proj_stride: u32,
+ #[allow(unused)] state_stride: u32,
+ #[allow(unused)] model_dim: u32,
+ #[allow(unused)]
+ #[specialize]
+ has_bias: bool,
+) {
+ todo!()
+}
+
+#[kernel(ShortConvDecode)]
+#[variants(T, f32, f16, bf16)]
+pub fn short_conv_decode<T: ArrayElement + Float>(
+ #[allow(unused)] in_proj: *const T,
+ #[allow(unused)] w: *const T,
+ #[allow(unused)]
+ #[optional(has_bias)]
+ b: Option<*const T>,
+ #[allow(unused)] state: *const T,
+ #[allow(unused)] out: *mut T,
+ #[allow(unused)] next_state: *mut T,
+ #[allow(unused)] suffix_len: u32,
+ #[allow(unused)] kernel_size: u32,
+ #[allow(unused)] in_proj_stride: u32,
+ #[allow(unused)] state_stride: u32,
+ #[allow(unused)] model_dim: u32,
+ #[allow(unused)]
+ #[specialize]
+ has_bias: bool,
+) {
+ todo!()
+}
+
+#[kernel(ShortConvTrie)]
+#[variants(T, f32, f16, bf16)]
+pub fn short_conv_trie<T: ArrayElement + Float>(
+ #[allow(unused)] in_proj: *const T,
+ #[allow(unused)] w: *const T,
+ #[allow(unused)]
+ #[optional(has_bias)]
+ b: Option<*const T>,
+ #[allow(unused)] base_state: *const T,
+ #[allow(unused)] parents: *const i32,
+ #[allow(unused)] out: *mut T,
+ #[allow(unused)] suffix_state: *mut T,
+ #[allow(unused)] suffix_len: u32,
+ #[allow(unused)] kernel_size: u32,
+ #[allow(unused)] in_proj_stride: u32,
+ #[allow(unused)] state_stride: u32,
+ #[allow(unused)] model_dim: u32,
+ #[allow(unused)]
+ #[specialize]
+ has_bias: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rs b/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rs
new file mode 100644
index 000000000..33caeb2aa
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rs
@@ -0,0 +1 @@
+pub mod sigmoid;
diff --git a/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs b/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs
new file mode 100644
index 000000000..dc3aacc56
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs
@@ -0,0 +1,15 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(Sigmoid)]
+#[variants(T, f32, f16, bf16)]
+pub fn sigmoid<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] total_elements: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs
new file mode 100644
index 000000000..4b8098ac0
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs
@@ -0,0 +1,78 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(Conv1dPack)]
+#[variants(T, f32, f16, bf16)]
+pub fn conv1d_pack<T: ArrayElement + Float>(
+ #[allow(unused)] state_in: *const T,
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] padded: *mut T,
+ #[allow(unused)] state_stride: u32,
+ #[allow(unused)] row_stride: u32,
+ #[allow(unused)] suffix_len: u32,
+ #[allow(unused)] num_channels: u32,
+) {
+ todo!()
+}
+
+#[kernel(Conv1dDecode)]
+#[variants(T, f32, f16, bf16)]
+pub fn conv1d_decode<T: ArrayElement + Float>(
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] w: *const T,
+ #[allow(unused)]
+ #[optional(has_bias)]
+ b: Option<*const T>,
+ #[allow(unused)] state: *const T,
+ #[allow(unused)] x_out: *mut T,
+ #[allow(unused)] b_out: *mut T,
+ #[allow(unused)] c_out: *mut T,
+ #[allow(unused)] next_state: *mut T,
+ #[allow(unused)] kernel_size: u32,
+ #[allow(unused)] row_stride: u32,
+ #[allow(unused)] state_stride: u32,
+ #[allow(unused)] num_channels: u32,
+ #[allow(unused)] suffix_len: u32,
+ #[allow(unused)] inner_dim: u32,
+ #[allow(unused)] proj_dim: u32,
+ #[allow(unused)]
+ #[specialize]
+ activation_type: u32,
+ #[allow(unused)]
+ #[specialize]
+ has_bias: bool,
+) {
+ todo!()
+}
+
+#[kernel(Conv1dScan)]
+#[variants(T, f32, f16, bf16)]
+pub fn conv1d_scan<T: ArrayElement + Float>(
+ #[allow(unused)] padded: *const T,
+ #[allow(unused)] w: *const T,
+ #[allow(unused)]
+ #[optional(has_bias)]
+ b: Option<*const T>,
+ #[allow(unused)] x_out: *mut T,
+ #[allow(unused)] b_out: *mut T,
+ #[allow(unused)] c_out: *mut T,
+ #[allow(unused)] state_out: *mut T,
+ #[allow(unused)] suffix_len: u32,
+ #[allow(unused)] kernel_size: u32,
+ #[allow(unused)] row_stride: u32,
+ #[allow(unused)] state_stride: u32,
+ #[allow(unused)] num_channels: u32,
+ #[allow(unused)] inner_dim: u32,
+ #[allow(unused)] proj_dim: u32,
+ #[allow(unused)]
+ #[specialize]
+ activation_type: u32,
+ #[allow(unused)]
+ #[specialize]
+ has_bias: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/mod.rs b/crates/uzu/src/backends/cpu/kernel/ssm/mod.rs
new file mode 100644
index 000000000..fb01e93a7
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/ssm/mod.rs
@@ -0,0 +1,4 @@
+pub mod conv1d;
+pub mod split_inproj;
+pub mod ssd_prefill;
+pub mod ssd_update;
diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs b/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs
new file mode 100644
index 000000000..873e4908f
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs
@@ -0,0 +1,22 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(SplitInProj)]
+#[variants(T, f32, f16, bf16)]
+pub fn split_in_proj<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] conv_out: *mut T,
+ #[allow(unused)] z_out: *mut T,
+ #[allow(unused)] dt_out: *mut T,
+ #[allow(unused)] z_bias: *const T,
+ #[allow(unused)] suffix_length: u32,
+ #[allow(unused)] total_dim: u32,
+ #[allow(unused)] conv_dim: u32,
+ #[allow(unused)] inner_dim: u32,
+ #[allow(unused)] num_heads: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs
new file mode 100644
index 000000000..0ca159e13
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs
@@ -0,0 +1,77 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(SSDPrefill64)]
+#[variants(T, f32, f16, bf16)]
+pub fn ssd_prefill64<T: ArrayElement + Float>(
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] dt_raw: *const T,
+ #[allow(unused)] b: *const T,
+ #[allow(unused)] c: *const T,
+ #[allow(unused)] d: *const T,
+ #[allow(unused)] z: *const T,
+ #[allow(unused)] state: *mut T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] suffix_len: u32,
+ #[allow(unused)] group_size: i32,
+ #[allow(unused)] state_size: i32,
+ #[allow(unused)] x_strides: &[u32],
+ #[allow(unused)] dt_strides: &[u32],
+ #[allow(unused)] cb_strides: &[u32],
+ #[allow(unused)] state_strides: &[u32],
+ #[allow(unused)] num_heads: u32,
+ #[allow(unused)] head_dim: u32,
+) {
+ todo!()
+}
+
+#[kernel(SSDPrefill)]
+#[variants(T, f32, f16, bf16)]
+pub fn ssd_prefill<T: ArrayElement + Float>(
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] dt_raw: *const T,
+ #[allow(unused)] b: *const T,
+ #[allow(unused)] c: *const T,
+ #[allow(unused)] d: *const T,
+ #[allow(unused)] z: *const T,
+ #[allow(unused)] state: *mut T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] suffix_len: u32,
+ #[allow(unused)] group_size: i32,
+ #[allow(unused)] state_size: i32,
+ #[allow(unused)] x_strides: &[u32],
+ #[allow(unused)] dt_strides: &[u32],
+ #[allow(unused)] cb_strides: &[u32],
+ #[allow(unused)] state_strides: &[u32],
+ #[allow(unused)] num_heads: u32,
+ #[allow(unused)] head_dim: u32,
+) {
+ todo!()
+}
+
+#[kernel(SSDPrefillSequential)]
+#[variants(T, f32, f16, bf16)]
+pub fn ssd_prefill_sequential<T: ArrayElement + Float>(
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] dt_raw: *const T,
+ #[allow(unused)] b: *const T,
+ #[allow(unused)] c: *const T,
+ #[allow(unused)] d: *const T,
+ #[allow(unused)] z: *const T,
+ #[allow(unused)] state: *mut T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] suffix_len: u32,
+ #[allow(unused)] group_size: i32,
+ #[allow(unused)] state_size: i32,
+ #[allow(unused)] x_strides: &[u32],
+ #[allow(unused)] dt_strides: &[u32],
+ #[allow(unused)] cb_strides: &[u32],
+ #[allow(unused)] state_strides: &[u32],
+ #[allow(unused)] channels: u32,
+ #[allow(unused)] head_dim: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs
new file mode 100644
index 000000000..b888db629
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs
@@ -0,0 +1,30 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(SSDUpdate)]
+#[variants(T, f32, f16, bf16)]
+pub fn ssd_update<T: ArrayElement + Float>(
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] dt_raw: *const T,
+ #[allow(unused)] b: *const T,
+ #[allow(unused)] c: *const T,
+ #[allow(unused)] d: *const T,
+ #[allow(unused)] z: *const T,
+ #[allow(unused)] state: *const T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] next_state: *mut T,
+ #[allow(unused)] group_size: u32,
+ #[allow(unused)] state_size: u32,
+ #[allow(unused)] x_strides: &[u32],
+ #[allow(unused)] dt_strides: &[u32],
+ #[allow(unused)] cb_strides: &[u32],
+ #[allow(unused)] state_strides: &[u32],
+ #[allow(unused)] b_size: u32,
+ #[allow(unused)] h_size: u32,
+ #[allow(unused)] dh_size: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/mod.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/mod.rs
new file mode 100644
index 000000000..f8e961796
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/mod.rs
@@ -0,0 +1 @@
+pub mod tensor_add_bias;
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
new file mode 100644
index 000000000..c44c9c813
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
@@ -0,0 +1,17 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(TensorAddBias)]
+#[variants(T, f32, f16, bf16)]
+pub fn tensor_add_bias<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] num_cols: u32,
+ #[allow(unused)] length: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/mod.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/mod.rs
new file mode 100644
index 000000000..a429b0db3
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/mod.rs
@@ -0,0 +1 @@
+pub mod tensor_add_swap;
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs
new file mode 100644
index 000000000..c4f234600
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs
@@ -0,0 +1,15 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(TensorAddSwap)]
+#[variants(T, f32, f16, bf16)]
+pub fn tensor_add_swap<T: ArrayElement + Float>(
+ #[allow(unused)] skip_buffer: *mut T,
+ #[allow(unused)] main_buffer: *mut T,
+ #[allow(unused)] length: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_copy/mod.rs b/crates/uzu/src/backends/cpu/kernel/tensor_copy/mod.rs
new file mode 100644
index 000000000..a61a2ebd2
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_copy/mod.rs
@@ -0,0 +1 @@
+pub mod tensor_copy;
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs b/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs
new file mode 100644
index 000000000..c2e19054a
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs
@@ -0,0 +1,15 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(TensorCopy)]
+#[variants(T, f32, f16, bf16)]
+pub fn tensor_copy<T: ArrayElement + Float>(
+ #[allow(unused)] src_buffer: *const T,
+ #[allow(unused)] dst_buffer: *mut T,
+ #[allow(unused)] length: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/token_copy/mod.rs b/crates/uzu/src/backends/cpu/kernel/token_copy/mod.rs
new file mode 100644
index 000000000..90375d281
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/token_copy/mod.rs
@@ -0,0 +1 @@
+pub mod token_copy;
diff --git a/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs b/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs
new file mode 100644
index 000000000..9da03daaa
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs
@@ -0,0 +1,20 @@
+use dsl::kernel;
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(TokenCopySampled)]
+pub fn token_copy_sampled(
+ #[allow(unused)] src: *const u32,
+ #[allow(unused)] dst: *mut u64,
+) {
+ todo!()
+}
+
+#[kernel(TokenCopyToResults)]
+pub fn token_copy_to_results(
+ #[allow(unused)] src: *const u32,
+ #[allow(unused)] dst: *mut u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/mod.rs b/crates/uzu/src/backends/cpu/mod.rs
index 5d259bb5f..bd15afca5 100644
--- a/crates/uzu/src/backends/cpu/mod.rs
+++ b/crates/uzu/src/backends/cpu/mod.rs
@@ -1 +1,11 @@
+mod backend;
+mod command_buffer;
+mod context;
+mod error;
+mod event;
+mod kernel;
+mod native_buffer;
+
+pub use backend::Cpu;
+
pub mod argmax;
diff --git a/crates/uzu/src/backends/cpu/native_buffer.rs b/crates/uzu/src/backends/cpu/native_buffer.rs
new file mode 100644
index 000000000..a885455be
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/native_buffer.rs
@@ -0,0 +1,24 @@
+use std::{os::raw::c_void, ptr::NonNull};
+
+use super::Cpu;
+use crate::backends::common::NativeBuffer;
+
+impl NativeBuffer for Box<[u8]> {
+ type Backend = Cpu;
+
+ fn set_label(
+ &mut self,
+ _label: Option<&str>,
+ ) {
+ }
+
+ fn cpu_ptr(&self) -> NonNull<c_void> {
+ unsafe { NonNull::new_unchecked(self.as_ptr() as *mut c_void) }
+ }
+ fn length(&self) -> usize {
+ self.len()
+ }
+ fn id(&self) -> usize {
+ unimplemented!()
+ }
+}
diff --git a/crates/uzu/src/backends/mod.rs b/crates/uzu/src/backends/mod.rs
index 5560db88f..d123fda17 100644
--- a/crates/uzu/src/backends/mod.rs
+++ b/crates/uzu/src/backends/mod.rs
@@ -7,6 +7,10 @@ pub mod metal;
macro_rules! select_backend {
($expr:expr, $unk:expr) => {{
match std::env::var("UZU_BACKEND").map(|s| s.to_lowercase()).as_deref().unwrap_or("metal") {
+ "cpu" => {
+ type B = crate::backends::cpu::Cpu;
+ $expr
+ },
#[cfg(feature = "metal")]
"metal" => {
type B = crate::backends::metal::Metal;