A high-performance inference engine for AI models
beginnings of the cpu backend (#226)
uuuvncd183c2e4ad9
diff --git a/Cargo.lock b/Cargo.lockindex e66a95c0d..5eca14321 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1125,6 +1125,15 @@ dependencies = ["litrs",]+[[package]]+name = "dsl"+version = "0.1.8"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.114",+]+[[package]]name = "dyn-clone"version = "1.0.20"@@ -4340,6 +4349,7 @@ dependencies = ["clang-ast","clap","depfile",+ "dsl","futures","half","is_close",diff --git a/Cargo.toml b/Cargo.tomlindex ebf12810a..da4244059 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,5 +1,5 @@[workspace]-members = ["crates/cli", "crates/uzu", "crates/benchmarks"]+members = ["crates/cli", "crates/uzu", "crates/benchmarks", "crates/dsl"]resolver = "3"[workspace.package]@@ -12,6 +12,7 @@ repository = "https://github.com/trymirai/uzu"[workspace.dependencies]benchmarks = { path = "crates/benchmarks" }+dsl = { path = "crates/dsl" }uzu = { path = "crates/uzu" }xgrammar = { package = "xgrammar-rs", version = "0.1", features = ["hf"] }half = { version = "2.7", features = ["bytemuck", "num-traits"] }diff --git a/crates/dsl/Cargo.toml b/crates/dsl/Cargo.tomlnew file mode 100644index 000000000..7255c8b01--- /dev/null+++ b/crates/dsl/Cargo.toml@@ -0,0 +1,16 @@+[package]+name = "dsl"+version.workspace = true+edition.workspace = true+rust-version.workspace = true+license.workspace = true+authors.workspace = true+repository.workspace = true++[lib]+proc-macro = true++[dependencies]+proc-macro2.workspace = true+quote.workspace = true+syn.workspace = truediff --git a/crates/dsl/src/lib.rs b/crates/dsl/src/lib.rsnew file mode 100644index 000000000..1d84d30b8--- /dev/null+++ b/crates/dsl/src/lib.rs@@ -0,0 +1,30 @@+use proc_macro::TokenStream;+use quote::quote;+use syn::{FnArg, ItemFn, parse_macro_input};++#[proc_macro_attribute]+pub fn kernel(+ args: TokenStream,+ input: TokenStream,+) -> TokenStream {+ let file = format!("/cpu/{}.rs", args.to_string());++ let mut func = parse_macro_input!(input as ItemFn);++ func.attrs.retain(|attr| !attr.path().is_ident("variants"));++ for arg in &mut func.sig.inputs {+ if let FnArg::Typed(pat) = arg {+ pat.attrs.retain(|attr| {+ let attr_path = attr.path();++ !(attr_path.is_ident("specialize") || attr_path.is_ident("optional"))+ });+ }+ }+ quote! {+ #func+ include!(concat!(env!("OUT_DIR"), #file));+ }+ .into()+}diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 0fc8e792e..951b0a513 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -5,6 +5,7 @@ edition.workspace = truebuild = "build/main.rs"[dependencies]+dsl.workspace = truexgrammar.workspace = truehalf.workspace = truendarray.workspace = truediff --git a/crates/uzu/build/common/codegen.rs b/crates/uzu/build/common/codegen.rsindex 5b8a97cd2..ce2f29194 100644--- a/crates/uzu/build/common/codegen.rs+++ b/crates/uzu/build/common/codegen.rs@@ -10,7 +10,8 @@ pub fn write_tokens(let tokens = tokens.into();let file = file.as_ref();- let parsed = syn::parse2(tokens.into()).context("cannot parse generated bindings")?;+ let parsed = syn::parse2(tokens.clone().into())+ .with_context(|| format!("cannot parse generated bindings: {}", tokens.to_string()))?;fs::write(&file, prettyplease::unparse(&parsed)).with_context(|| format!("cannot write file {}", file.display()))?;diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rsnew file mode 100644index 000000000..feee0dd60--- /dev/null+++ b/crates/uzu/build/cpu/compiler.rs@@ -0,0 +1,606 @@+use std::{collections::HashMap, env, fs, path::PathBuf};++use anyhow::{Context, bail};+use async_trait::async_trait;+use itertools::Itertools;+use proc_macro2::{Span, TokenStream};+use quote::{ToTokens, format_ident, quote};+use syn::{+ Expr, FnArg, GenericArgument, GenericParam, Ident, Item, ItemFn, Lifetime, PathArguments, Type,+ punctuated::Punctuated, token::Comma,+};+use walkdir::WalkDir;++use crate::common::{+ codegen::write_tokens,+ compiler::Compiler,+ kernel::{Kernel, KernelArgument, KernelArgumentType, KernelParameter, KernelParameterType},+};++#[derive(PartialEq, Debug)]+pub enum FunctionArgumentType {+ Buffer,+ Constant(Type),+ Scalar(Type),+ Specialization(Type),+}++#[derive(PartialEq, Debug)]+pub struct FunctionArgument {+ pub name: Ident,+ pub conditional: Option<Ident>,+ pub ty: FunctionArgumentType,+}++impl FunctionArgument {+ fn to_kernel_argument(&self) -> Option<KernelArgument> {+ Some(KernelArgument {+ name: self.name.to_string().into_boxed_str(),+ conditional: self.conditional.is_some(),+ ty: match &self.ty {+ FunctionArgumentType::Buffer => KernelArgumentType::Buffer,+ FunctionArgumentType::Constant(ty) => KernelArgumentType::Constant(+ ty.to_token_stream().to_string().replace(" :: ", "::").into_boxed_str(),+ ),+ FunctionArgumentType::Scalar(ty) => {+ KernelArgumentType::Scalar(ty.to_token_stream().to_string().replace(" :: ", "::").into_boxed_str())+ },+ FunctionArgumentType::Specialization(_) => {+ return None;+ },+ },+ })+ }++ fn to_kernel_parameter(&self) -> Option<KernelParameter> {+ Some(KernelParameter {+ name: self.name.to_string().into_boxed_str(),+ ty: match &self.ty {+ FunctionArgumentType::Specialization(ty) => {+ KernelParameterType::Value(ty.to_token_stream().to_string().into_boxed_str())+ },+ _ => {+ return None;+ },+ },+ })+ }+}++#[derive(PartialEq, Debug)]+pub enum FunctionParameterType {+ Type,+ Value(Type),+}++#[derive(PartialEq, Debug)]+pub struct FunctionParameter {+ pub name: Ident,+ pub ty: FunctionParameterType,+}++impl FunctionParameter {+ fn to_kernel_parameter(&self) -> KernelParameter {+ KernelParameter {+ name: self.name.to_string().into_boxed_str(),+ ty: match &self.ty {+ FunctionParameterType::Type => KernelParameterType::Type,+ FunctionParameterType::Value(ty) => {+ KernelParameterType::Value(ty.to_token_stream().to_string().into_boxed_str())+ },+ },+ }+ }+}++pub struct CpuCompiler {+ src_dir: PathBuf,+ build_dir: PathBuf,+}++impl CpuCompiler {+ pub fn new() -> anyhow::Result<Self> {+ let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?)+ .join("src/backends/cpu/kernel");++ let build_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?).join("cpu");+ fs::create_dir_all(&build_dir).with_context(|| format!("cannot create {}", build_dir.display()))?;++ Ok(Self {+ src_dir,+ build_dir,+ })+ }++ fn compile(+ &self,+ source_path: PathBuf,+ ) -> anyhow::Result<(Box<[Box<str>]>, Box<[Kernel]>)> {+ let src_rel_path: Box<[Box<str>]> = source_path+ .strip_prefix(&self.src_dir)+ .context("source is not in src_dir")?+ .with_extension("")+ .as_os_str()+ .to_str()+ .unwrap()+ .split("/")+ .map(|s| s.to_string().into_boxed_str())+ .collect();++ let source_contents = fs::read_to_string(&source_path).context("cannot read the source file")?;+ let source_ast = syn::parse_file(&source_contents).context("cannot parse ast")?;++ let kernels = source_ast+ .items+ .into_iter()+ .filter_map(|item| {+ if let Item::Fn(ifn) = item+ && ifn.attrs.iter().any(|attr| attr.path().is_ident("kernel"))+ {+ Some(self.compile_kernel(ifn))+ } else {+ None+ }+ })+ .collect::<anyhow::Result<_>>()?;++ Ok((src_rel_path, kernels))+ }++ fn compile_kernel(+ &self,+ ifn: ItemFn,+ ) -> anyhow::Result<Kernel> {+ let mut kernel_ident = None;+ let mut function_variants = Vec::new();++ for attr in ifn.attrs {+ match attr.path().get_ident().map(|i| i.to_string()).as_ref().map(|s| s.as_ref()) {+ Some("kernel") => {+ if kernel_ident.is_some() {+ bail!("Multiple kernel attributes!");+ }++ kernel_ident = Some(attr.parse_args::<Ident>().context("cannot parse kernel attribute arg")?);+ },+ Some("variants") => {+ let mut args = attr+ .parse_args_with(Punctuated::<Expr, Comma>::parse_terminated)+ .context("cannot parse variants attribute args")?+ .into_iter();++ function_variants.push((+ syn::parse2::<Ident>(args.next().context("variant must have a name")?.into_token_stream())+ .unwrap(),+ args.collect::<Box<[_]>>(),+ ));+ },+ _ => bail!("Unexpected attr {attr:?}"),+ }+ }++ let Some(kernel_ident) = kernel_ident else {+ bail!("Not a kernel")+ };++ let function_ident = ifn.sig.ident;+ let function_parameters = ifn+ .sig+ .generics+ .params+ .into_iter()+ .map(|parameter| {+ Ok(match parameter {+ GenericParam::Type(parameter) => FunctionParameter {+ name: parameter.ident,+ ty: FunctionParameterType::Type,+ },+ GenericParam::Const(parameter) => FunctionParameter {+ name: parameter.ident,+ ty: FunctionParameterType::Value(parameter.ty),+ },+ parameter => {+ bail!("unsupported kernel parameter type: {parameter:?}")+ },+ })+ })+ .collect::<anyhow::Result<Box<[FunctionParameter]>>>()?;+ let function_arguments = ifn+ .sig+ .inputs+ .into_iter()+ .map(|argument| {+ let FnArg::Typed(argument) = argument else {+ bail!("self argument in a kernel is not supported");+ };++ let name = syn::parse2(argument.pat.into_token_stream()).context("cannot parse the argument name")?;++ let specialize = argument.attrs.iter().any(|attr| attr.path().is_ident("specialize"));++ let conditional = argument+ .attrs+ .iter()+ .find_map(|attr| attr.path().is_ident("optional").then(|| attr.parse_args::<Ident>().unwrap()));++ let ty = if specialize {+ FunctionArgumentType::Specialization(*argument.ty)+ } else if conditional.is_some() {+ let Type::Path(ty) = *argument.ty else {+ bail!("conditional argument must be a type path");+ };+ if ty.path.segments.len() != 1 {+ bail!("conditional argument type path must have one segment");+ }+ let seg = &ty.path.segments[0];+ if seg.ident.to_string() != "Option" {+ bail!("conditional argument type must be Option<...>");+ }+ let PathArguments::AngleBracketed(option_arguments) = &seg.arguments else {+ bail!("conditional argument type must be angle bracketed");+ };+ if option_arguments.args.len() != 1 {+ bail!("conditional argument type Option must have one generic argument");+ }+ let generic_argument = &option_arguments.args[0];+ let GenericArgument::Type(inner_ty) = generic_argument else {+ bail!("conditional argument type Option must have a type argument")+ };+ Self::parse_type(inner_ty.clone()).context("failed to parse conditional argument type")?+ } else {+ Self::parse_type(*argument.ty).context("failed to parse argument type")?+ };++ Ok(FunctionArgument {+ name,+ conditional,+ ty,+ })+ })+ .collect::<anyhow::Result<Box<[FunctionArgument]>>>()?;++ let kernel_parameters = function_parameters+ .iter()+ .map(|p| p.to_kernel_parameter())+ .chain(function_arguments.iter().flat_map(|p| p.to_kernel_parameter()))+ .collect::<Box<[KernelParameter]>>();++ let kernel_arguments =+ function_arguments.iter().flat_map(|p| p.to_kernel_argument()).collect::<Box<[KernelArgument]>>();++ if function_parameters.len() != function_variants.len() {+ bail!(+ "Kernel function has {} generics != {} #[variants(...)]!",+ function_parameters.len(),+ function_variants.len()+ );+ }++ for (parameter, (variant_name, _)) in std::iter::zip(function_parameters.iter(), function_variants.iter()) {+ if ¶meter.name != variant_name {+ bail!(+ "Parameter name doesn't match variant name: {} | {}",+ parameter.name.to_string(),+ variant_name.to_string(),+ );+ }+ }++ // === Bindgen ===++ let trait_ident = format_ident!("{kernel_ident}Kernel");+ let struct_ident = format_ident!("{kernel_ident}CpuKernel");++ let (struct_fields_defs, struct_fields_sets): (Vec<TokenStream>, Vec<TokenStream>) = function_parameters+ .iter()+ .map(|parameter| {+ let ident = ¶meter.name;++ let ty = match ¶meter.ty {+ FunctionParameterType::Type => quote! { crate::DataType },+ FunctionParameterType::Value(ty) => quote! { #ty },+ };++ (quote! { #ident: #ty }, quote! { #ident })+ })+ .chain(function_arguments.iter().flat_map(|argument| {+ let FunctionArgumentType::Specialization(ty) = &argument.ty else {+ return None;+ };++ let ident = &argument.name;++ Some((quote! { #ident: #ty }, quote! { #ident }))+ }))+ .collect();++ let parameter_args: Vec<TokenStream> = kernel_parameters+ .iter()+ .map(|parameter| {+ let parameter_ident: Ident = syn::parse_str(parameter.name.as_ref())?;++ Ok(match ¶meter.ty {+ KernelParameterType::Type => quote! { #[allow(non_snake_case)] #parameter_ident: crate::DataType },+ KernelParameterType::Value(ty) => {+ let ty: syn::Type = syn::parse_str(ty.as_ref()).unwrap();+ quote! { #[allow(non_snake_case)] #parameter_ident: #ty }+ },+ })+ })+ .collect::<anyhow::Result<_>>()?;++ let (encode_generics, encode_args_defs, encode_args_passthrough): (Vec<_>, Vec<_>, Vec<_>) = kernel_arguments+ .iter()+ .map(|argument| {+ let argument_ident: Ident = syn::parse_str(argument.name.as_ref()).context("cannot parse ident")?;++ let (generic, mut ty) = match &argument.ty {+ KernelArgumentType::Buffer => {+ let buffer_lifetime = Lifetime::new(&format!("'{}", argument.name.as_ref()), Span::call_site());+ (+ Some(quote! { #buffer_lifetime }),+ quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, Box<[u8]>> },+ )+ },+ KernelArgumentType::Constant(ty) => {+ let ty: Type = syn::parse_str(ty.as_ref()).context("cannot parse type")?;+ (None, quote! { &[#ty] })+ },+ KernelArgumentType::Scalar(ty) => {+ let ty: Type = syn::parse_str(ty.as_ref()).context("cannot parse type")?;+ (None, quote! { #ty })+ },+ };++ if argument.conditional {+ ty = quote! { Option<#ty> };+ }++ Ok((generic, quote! { #argument_ident: #ty }, quote! { #argument_ident }))+ })+ .collect::<anyhow::Result<_>>()?;++ let encode_generics = encode_generics.into_iter().flatten().collect::<Vec<_>>();++ let argument_copies = function_arguments+ .iter()+ .flat_map(|argument| {+ let argument_ident = &argument.name;+ match &argument.ty {+ FunctionArgumentType::Buffer => {+ if argument.conditional.is_some() {+ Some(quote! {+ let #argument_ident = #argument_ident.map(|__dsl_buffer_impl| unsafe {+ let (__dsl_buffer, __dsl_offset) = __dsl_buffer_impl.into_parts();++ __dsl_buffer.as_ptr().byte_add(__dsl_offset)+ });+ })+ } else {+ Some(quote! {+ let #argument_ident = unsafe {+ let (__dsl_buffer, __dsl_offset) = #argument_ident.into_parts();++ __dsl_buffer.as_ptr().byte_add(__dsl_offset)+ };+ })+ }+ },+ FunctionArgumentType::Constant(_) => {+ Some(quote! { let #argument_ident = #argument_ident.to_vec().into_boxed_slice(); })+ },+ FunctionArgumentType::Scalar(_) => None,+ FunctionArgumentType::Specialization(_) => {+ Some(quote! { let #argument_ident = self.#argument_ident; })+ },+ }+ })+ .collect::<Vec<_>>();++ let make_encode = |generics: TokenStream| -> TokenStream {+ let monomorphized_function = if !generics.is_empty() {+ quote! { self::#function_ident::<#generics> }+ } else {+ quote! { self::#function_ident }+ };++ let function_call_args = function_arguments.iter().map(|argument| {+ let argument_ident = &argument.name;++ match &argument.ty {+ FunctionArgumentType::Buffer => {+ if argument.conditional.is_some() {+ quote! { #argument_ident.map(|p| p as _) }+ } else {+ quote! { #argument_ident as _ }+ }+ },+ FunctionArgumentType::Constant(_) => quote! { #argument_ident.as_ref() },+ FunctionArgumentType::Scalar(_) | FunctionArgumentType::Specialization(_) => {+ quote! { #argument_ident }+ },+ }+ });++ let function_call_args_joined = function_call_args.fold(quote! {}, |a, b| {+ if !a.is_empty() && !b.is_empty() {+ quote! {#a , #b}+ } else {+ quote! {#a #b}+ }+ });++ quote! {+ encoder.borrow_mut().push_command(move || {+ #monomorphized_function(#function_call_args_joined)+ });+ }+ };++ let encode_body = if function_parameters.len() > 0 {+ let mut parameter_idents = function_parameters.iter().map(|p| p.name.clone()).fold(quote! {}, |a, b| {+ if !a.is_empty() {+ quote! {#a , self.#b}+ } else {+ quote! {self.#b}+ }+ });++ if function_parameters.len() > 1 {+ parameter_idents = quote! { (#parameter_idents) };+ }++ let match_arms = function_parameters+ .iter()+ .zip(function_variants.iter())+ .map(|(parameter, (_, variants))| {+ variants.iter().map(|variant| {+ (+ match parameter.ty {+ FunctionParameterType::Type => {+ let dtype =+ format_ident!("{}", variant.to_token_stream().to_string().to_uppercase());+ quote! { crate::DataType::#dtype }+ },+ FunctionParameterType::Value(_) => quote! { #variant },+ },+ quote! { #variant },+ )+ })+ })+ .multi_cartesian_product()+ .map(|variants| {+ let (match_variants, generic_variants): (Vec<TokenStream>, Vec<TokenStream>) =+ variants.into_iter().unzip();++ let mut match_variant = match_variants.iter().fold(quote! {}, |a, b| {+ if !a.is_empty() && !b.is_empty() {+ quote! {#a , #b}+ } else {+ quote! {#a #b}+ }+ });++ if match_variants.len() > 1 {+ match_variant = quote! { (#match_variant) };+ }++ let generic_variant = generic_variants.iter().fold(quote! {}, |a, b| {+ if !a.is_empty() && !b.is_empty() {+ quote! {#a , #b}+ } else {+ quote! {#a #b}+ }+ });++ let encode = make_encode(generic_variant);++ quote! { #match_variant => { #encode } }+ })+ .collect::<Vec<_>>();++ quote! {+ match #parameter_idents {+ #(#match_arms ,)*+ __dsl_variant => unimplemented!("variant doesn't exist: {__dsl_variant:?}"),+ }+ }+ } else {+ make_encode(quote! {})+ };++ let tokens = quote! {+ #[allow(non_snake_case)]+ pub struct #struct_ident {+ #(#struct_fields_defs ,)*+ }++ impl crate::backends::common::kernel::#trait_ident for #struct_ident {+ type Backend = crate::backends::cpu::Cpu;++ fn new(context: &crate::backends::cpu::context::CpuContext #(, #parameter_args)*) -> Result<Self, crate::backends::cpu::error::CpuError> {+ Ok(Self {+ #(#struct_fields_sets ,)*+ })+ }++ fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder std::cell::RefCell<crate::backends::cpu::command_buffer::CpuCommandBuffer>) {+ self.encode_if(#(#encode_args_passthrough ,)* encoder, None::<&Box<[u8]>>)+ }++ fn encode_if<#(#encode_generics ,)* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* encoder: &'encoder std::cell::RefCell<crate::backends::cpu::command_buffer::CpuCommandBuffer>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Box<[u8]>>>) {+ #(#argument_copies)*+ #encode_body+ }+ }+ };++ let out_path = self.build_dir.join(kernel_ident.to_string()).with_extension("rs");+ write_tokens(tokens, &out_path).context("cannot write bindings")?;++ Ok(Kernel {+ name: kernel_ident.to_string().into_boxed_str(),+ parameters: kernel_parameters,+ arguments: kernel_arguments,+ })+ }++ fn parse_type(ty: Type) -> anyhow::Result<FunctionArgumentType> {+ Ok(match ty {+ Type::Ptr(_) => FunctionArgumentType::Buffer,+ Type::Reference(ty) => match *ty.elem {+ Type::Slice(ty) => FunctionArgumentType::Constant(*ty.elem),+ ty => bail!("unsupported reference type: {} ({:?})", ty.to_token_stream().to_string(), ty),+ },+ Type::Path(ty) => FunctionArgumentType::Scalar(Type::Path(ty)),+ ty => bail!("unsupported type: {} ({:?})", ty.to_token_stream().to_string(), ty),+ })+ }++ fn bindgen<'a>(+ &self,+ objects: impl IntoIterator<Item = &'a (Box<[Box<str>]>, Box<[Kernel]>)> + Clone,+ ) -> anyhow::Result<()> {+ let out_path = self.build_dir.join("dsl.rs");++ let associated_types = objects.into_iter().flat_map(|(file_path, kernels)| {+ kernels.iter().map(|kernel| {+ let file_path: TokenStream = file_path.iter().join("::").parse().unwrap();+ let kernel_trait_name = format_ident!("{}Kernel", kernel.name.as_ref());+ let kernel_struct_name = format_ident!("{}CpuKernel", kernel.name.as_ref());+ quote! { type #kernel_trait_name = #file_path::#kernel_struct_name; }+ })+ });++ let tokens = quote! {+ pub struct CpuKernels;++ impl crate::backends::common::kernel::Kernels for CpuKernels {+ type Backend = crate::backends::cpu::Cpu;++ #(#associated_types)*+ }+ };++ write_tokens(tokens, &out_path).context("cannot write dsl bindings")?;++ Ok(())+ }+}++#[async_trait]+impl Compiler for CpuCompiler {+ async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {+ let objects = WalkDir::new(&self.src_dir)+ .into_iter()+ .filter_map(|e| e.ok())+ .filter(|e| e.file_type().is_file() && e.path().extension().and_then(|s| s.to_str()) == Some("rs"))+ .map(|e| self.compile(e.into_path()))+ .collect::<anyhow::Result<Vec<(Box<[Box<str>]>, Box<[Kernel]>)>>>()+ .context("cannot compile cpu sources")?;++ self.bindgen(&objects).context("cannot generate bindings")?;++ Ok(objects.into_iter().collect())+ }+}diff --git a/crates/uzu/build/cpu/mod.rs b/crates/uzu/build/cpu/mod.rsnew file mode 100644index 000000000..23a57fd58--- /dev/null+++ b/crates/uzu/build/cpu/mod.rs@@ -0,0 +1,3 @@+mod compiler;++pub use compiler::CpuCompiler;diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rsindex bc25ca1bd..ae9dd59f2 100644--- a/crates/uzu/build/main.rs+++ b/crates/uzu/build/main.rs@@ -6,6 +6,8 @@ use futures::future::try_join_all;mod common;mod gpu_types;+mod cpu;+#[cfg(feature = "metal")]mod metal;@@ -37,6 +39,8 @@ async fn main() -> anyhow::Result<()> {let mut compilers: Vec<Box<dyn Compiler>> = Vec::new();+ compilers.push(Box::new(cpu::CpuCompiler::new()?));+#[cfg(feature = "metal")]compilers.push(Box::new(metal::MetalCompiler::new_with_include_dir(generated_header_dir)?));diff --git a/crates/uzu/src/backends/cpu/backend.rs b/crates/uzu/src/backends/cpu/backend.rsnew file mode 100644index 000000000..ac7bd8408--- /dev/null+++ b/crates/uzu/src/backends/cpu/backend.rs@@ -0,0 +1,20 @@+use std::cell::RefCell;++use super::{command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError, kernel::CpuKernels};+use crate::backends::common::Backend;++#[derive(Debug, Clone)]+pub struct Cpu;++impl Backend for Cpu {+ type Context = CpuContext;+ type NativeBuffer = Box<[u8]>;+ type CommandBuffer = RefCell<CpuCommandBuffer>;+ type ComputeEncoder = RefCell<CpuCommandBuffer>;+ type CopyEncoder = RefCell<CpuCommandBuffer>;+ type Event = RefCell<u64>;+ type Kernels = CpuKernels;+ type Error = CpuError;++ const MAX_INLINE_BYTES: usize = usize::MAX;+}diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rsnew file mode 100644index 000000000..fa780d1e2--- /dev/null+++ b/crates/uzu/src/backends/cpu/command_buffer.rs@@ -0,0 +1,117 @@+use std::{cell::RefCell, time::Instant};++use super::Cpu;+use crate::backends::common::{CommandBuffer, CopyEncoder};++pub struct CpuCommandBuffer {+ commands: Vec<Box<dyn Fn()>>,+ completion_handlers: Vec<Box<dyn Fn()>>,+ gpu_execution_time_ms: Option<f64>,+}++impl CpuCommandBuffer {+ pub fn new() -> RefCell<CpuCommandBuffer> {+ RefCell::new(CpuCommandBuffer {+ commands: Vec::new(),+ completion_handlers: Vec::new(),+ gpu_execution_time_ms: None,+ })+ }++ pub fn push_command(+ &mut self,+ command: impl Fn() + 'static,+ ) {+ self.commands.push(Box::new(command))+ }+}++impl CommandBuffer for RefCell<CpuCommandBuffer> {+ type Backend = Cpu;++ fn with_compute_encoder<T>(+ &mut self,+ callback: impl FnOnce(&mut RefCell<CpuCommandBuffer>) -> T,+ ) -> T {+ callback(self)+ }++ fn with_copy_encoder<T>(+ &mut self,+ callback: impl FnOnce(&mut RefCell<CpuCommandBuffer>) -> T,+ ) -> T {+ callback(self)+ }++ fn encode_wait_for_event(+ &mut self,+ event: &RefCell<u64>,+ value: u64,+ ) {+ let event = event.as_ptr();+ self.borrow_mut().push_command(Box::new(move || unsafe { assert!(*event >= value, "deadlock") }))+ }++ fn encode_signal_event(+ &mut self,+ event: &RefCell<u64>,+ value: u64,+ ) {+ let event = event.as_ptr();+ self.borrow_mut().push_command(Box::new(move || unsafe { *event = value }))+ }++ fn add_completion_handler(+ &mut self,+ handler: impl Fn() + 'static,+ ) {+ self.borrow_mut().completion_handlers.push(Box::new(move || handler()))+ }++ fn submit(&self) {+ let start = Instant::now();+ for command in &self.borrow().commands {+ command()+ }+ self.borrow_mut().gpu_execution_time_ms = Some((Instant::now() - start).as_secs_f64() * 1000.0);+ for completion_handler in self.borrow().completion_handlers.iter() {+ completion_handler();+ }+ }++ fn wait_until_completed(&self) {}++ fn gpu_execution_time_ms(&self) -> Option<f64> {+ self.borrow().gpu_execution_time_ms+ }+}++impl CopyEncoder for RefCell<CpuCommandBuffer> {+ type Backend = Cpu;++ fn encode_copy(+ &mut self,+ src: &Box<[u8]>,+ dst: &mut Box<[u8]>,+ size: usize,+ ) {+ let src = src.as_ptr();+ let dst = dst.as_ptr() as *mut u8;+ self.borrow_mut().push_command(Box::new(move || unsafe {+ std::ptr::copy(src, dst, size);+ }))+ }++ fn encode_fill(+ &mut self,+ dst: &mut Box<[u8]>,+ range: std::ops::Range<usize>,+ value: u8,+ ) {+ let size = range.end - range.start;+ let dst = dst[range].as_ptr() as *mut u8;+ self.borrow_mut().push_command(Box::new(move || unsafe {+ dst.write_bytes(value, size);+ }))+ }+}diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rsnew file mode 100644index 000000000..cbf2d0aa2--- /dev/null+++ b/crates/uzu/src/backends/cpu/context.rs@@ -0,0 +1,54 @@+use std::{cell::RefCell, rc::Rc};++use super::{Cpu, command_buffer::CpuCommandBuffer, error::CpuError};+use crate::backends::common::{Allocator, Context, DeviceClass};++pub struct CpuContext;++impl Context for CpuContext {+ type Backend = Cpu;++ fn new() -> Result<Rc<Self>, CpuError> {+ Ok(Rc::new(CpuContext))+ }++ fn device_class(&self) -> DeviceClass {+ DeviceClass::Base+ }++ fn debug_active(&self) -> bool {+ false+ }++ fn allocator(&self) -> &Allocator<Self::Backend> {+ unimplemented!()+ }++ fn create_buffer(+ &self,+ size: usize,+ ) -> Result<Box<[u8]>, CpuError> {+ Ok(vec![0; size].into_boxed_slice())+ }++ fn create_command_buffer(&self) -> Result<RefCell<CpuCommandBuffer>, CpuError> {+ Ok(CpuCommandBuffer::new())+ }++ fn create_event(&self) -> Result<RefCell<u64>, CpuError> {+ Ok(RefCell::new(0))+ }++ fn enable_capture() {}++ fn start_capture(+ &self,+ _trace_path: &std::path::Path,+ ) -> Result<(), CpuError> {+ Err(CpuError::NotSupported)+ }++ fn stop_capture(&self) -> Result<(), CpuError> {+ Err(CpuError::NotSupported)+ }+}diff --git a/crates/uzu/src/backends/cpu/error.rs b/crates/uzu/src/backends/cpu/error.rsnew file mode 100644index 000000000..e221f160a--- /dev/null+++ b/crates/uzu/src/backends/cpu/error.rs@@ -0,0 +1,7 @@+use thiserror::Error;++#[derive(Debug, Error)]+pub enum CpuError {+ #[error("not supported")]+ NotSupported,+}diff --git a/crates/uzu/src/backends/cpu/event.rs b/crates/uzu/src/backends/cpu/event.rsnew file mode 100644index 000000000..aa3309c3e--- /dev/null+++ b/crates/uzu/src/backends/cpu/event.rs@@ -0,0 +1,8 @@+use std::cell::RefCell;++use super::Cpu;+use crate::backends::common::Event;++impl Event for RefCell<u64> {+ type Backend = Cpu;+}diff --git a/crates/uzu/src/backends/cpu/kernel/activation/activation.rs b/crates/uzu/src/backends/cpu/kernel/activation/activation.rsnew file mode 100644index 000000000..5f1c9c763--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/activation/activation.rs@@ -0,0 +1,16 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(Activation)]+#[variants(T, f16, f32, bf16)]+pub fn activation<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] n: u32,+ #[allow(unused)] act_type: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/activation/mod.rs b/crates/uzu/src/backends/cpu/kernel/activation/mod.rsnew file mode 100644index 000000000..3e9730314--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/activation/mod.rs@@ -0,0 +1 @@+pub mod activation;diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rsnew file mode 100644index 000000000..4c2471852--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs@@ -0,0 +1,45 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AttentionGemm)]+#[variants(T, f32, f16, bf16)]+#[variants(BK, 16, 32)]+#[variants(BD, 64, 128, 256)]+pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(+ #[allow(unused)] q: *const T,+ #[allow(unused)] k: *const T,+ #[allow(unused)] v: *const T,+ #[allow(unused)] o: *mut T,+ #[allow(unused)] params: crate::backends::common::gpu_types::attention::AttnParams,+ #[allow(unused)]+ #[optional(has_mask)]+ mask_params: Option<crate::backends::common::gpu_types::attention::AttnMaskParams>,+ #[allow(unused)]+ #[optional(has_mask)]+ mask: Option<*const T>,+ #[allow(unused)]+ #[optional(has_sinks)]+ sinks: Option<*const f32>,+ #[allow(unused)] num_heads: u32,+ #[allow(unused)] suffix_length: u32,+ #[allow(unused)]+ #[specialize]+ align_q: bool,+ #[allow(unused)]+ #[specialize]+ align_k: bool,+ #[allow(unused)]+ #[specialize]+ do_causal: bool,+ #[allow(unused)]+ #[specialize]+ has_mask: bool,+ #[allow(unused)]+ #[specialize]+ has_sinks: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rsnew file mode 100644index 000000000..af87fb206--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs@@ -0,0 +1,53 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AttentionSinglePass)]+#[variants(T, f32, f16, bf16)]+#[variants(HEAD_DIM, 64, 128, 256)]+pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(+ #[allow(unused)] queries: *const T,+ #[allow(unused)] keys: *const T,+ #[allow(unused)] values: *const T,+ #[allow(unused)] out: *mut T,+ #[allow(unused)] gqa_factor: u32,+ #[allow(unused)] sequence_length: u32,+ #[allow(unused)] k_head_stride: u32,+ #[allow(unused)] k_seq_stride: u32,+ #[allow(unused)] v_head_stride: u32,+ #[allow(unused)] v_seq_stride: u32,+ #[allow(unused)] scale: f32,+ #[allow(unused)]+ #[optional(float_mask)]+ fmask: Option<*const T>,+ #[allow(unused)]+ #[optional(has_mask)]+ mask_kv_seq_stride: Option<u32>,+ #[allow(unused)]+ #[optional(has_mask)]+ mask_q_seq_stride: Option<u32>,+ #[allow(unused)]+ #[optional(has_mask)]+ mask_head_stride: Option<u32>,+ #[allow(unused)]+ #[optional(has_sinks)]+ sinks: Option<*const f32>,+ #[allow(unused)] num_heads: u32,+ #[allow(unused)] suffix_length: u32,+ #[allow(unused)]+ #[specialize]+ float_mask: bool,+ #[allow(unused)]+ #[specialize]+ has_mask: bool,+ #[allow(unused)]+ #[specialize]+ has_sinks: bool,+ #[allow(unused)]+ #[specialize]+ do_causal: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rsnew file mode 100644index 000000000..7afa88cad--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs@@ -0,0 +1,69 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AttentionTwoPass1)]+#[variants(T, f32, f16, bf16)]+#[variants(HEAD_DIM, 64, 128, 256)]+pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(+ #[allow(unused)] queries: *const T,+ #[allow(unused)] keys: *const T,+ #[allow(unused)] values: *const T,+ #[allow(unused)] out: *mut f32,+ #[allow(unused)] sums: *mut f32,+ #[allow(unused)] maxs: *mut f32,+ #[allow(unused)] gqa_factor: u32,+ #[allow(unused)] sequence_length: u32,+ #[allow(unused)] k_head_stride: u32,+ #[allow(unused)] k_seq_stride: u32,+ #[allow(unused)] v_head_stride: u32,+ #[allow(unused)] v_seq_stride: u32,+ #[allow(unused)] scale: f32,+ #[allow(unused)] num_heads: u32,+ #[allow(unused)] suffix_length: u32,+ #[allow(unused)]+ #[optional(float_mask)]+ fmask: Option<*const T>,+ #[allow(unused)]+ #[optional(has_mask)]+ mask_kv_seq_stride: Option<u32>,+ #[allow(unused)]+ #[optional(has_mask)]+ mask_q_seq_stride: Option<u32>,+ #[allow(unused)]+ #[optional(has_mask)]+ mask_head_stride: Option<u32>,+ #[allow(unused)]+ #[optional(has_sinks)]+ sinks: Option<*const f32>,+ #[allow(unused)]+ #[specialize]+ float_mask: bool,+ #[allow(unused)]+ #[specialize]+ has_mask: bool,+ #[allow(unused)]+ #[specialize]+ has_sinks: bool,+ #[allow(unused)]+ #[specialize]+ do_causal: bool,+) {+ todo!()+}++#[kernel(AttentionTwoPass2)]+#[variants(T, f32, f16, bf16)]+#[variants(HEAD_DIM, 64, 128, 256)]+pub fn attention_two_pass2<T: ArrayElement + Float, const HEAD_DIM: u32>(+ #[allow(unused)] partials: *const f32,+ #[allow(unused)] sums: *const f32,+ #[allow(unused)] maxs: *const f32,+ #[allow(unused)] out: *mut T,+ #[allow(unused)] num_heads: u32,+ #[allow(unused)] suffix_length: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_update_kv_cache.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_update_kv_cache.rsnew file mode 100644index 000000000..a5147b8a3--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_update_kv_cache.rs@@ -0,0 +1,22 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AttentionUpdateKVCache)]+#[variants(T, f32, f16, bf16)]+pub fn attention_update_kv_cache<T: ArrayElement + Float>(+ #[allow(unused)] rotated_keys: *const T,+ #[allow(unused)] qkv: *const T,+ #[allow(unused)] key_cache: *mut T,+ #[allow(unused)] value_cache: *mut T,+ #[allow(unused)] num_groups: u32,+ #[allow(unused)] num_heads: u32,+ #[allow(unused)] head_dim: u32,+ #[allow(unused)] suffix_length: u32,+ #[allow(unused)] prefix_segment_length: u32,+ #[allow(unused)] max_sequence_length: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu/src/backends/cpu/kernel/attention/mod.rsnew file mode 100644index 000000000..24f00ec54--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/attention/mod.rs@@ -0,0 +1,4 @@+pub mod attention_gemm;+pub mod attention_single_pass;+pub mod attention_two_pass;+pub mod attention_update_kv_cache;diff --git a/crates/uzu/src/backends/cpu/kernel/audio/audio.rs b/crates/uzu/src/backends/cpu/kernel/audio/audio.rsnew file mode 100644index 000000000..81a2fd57a--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/audio.rs@@ -0,0 +1,167 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioFsqDecode)]+#[variants(T, f32, f16, bf16)]+pub fn audio_fsq_decode<T: ArrayElement + Float>(+ #[allow(unused)] tokens: *const i32,+ #[allow(unused)] out: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] num_groups: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] codebook_dim: i32,+ #[allow(unused)] num_levels: &[i32],+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}++#[kernel(AudioLeakyRelu)]+#[variants(T, f32, f16, bf16)]+pub fn audio_leaky_relu<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] n: i32,+ #[allow(unused)] negative_slope: f32,+) {+ todo!()+}++#[kernel(AudioTanh)]+#[variants(T, f32, f16, bf16)]+pub fn audio_tanh<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] n: i32,+) {+ todo!()+}++#[kernel(AudioAdd)]+#[variants(T, f32, f16, bf16)]+pub fn audio_add<T: ArrayElement + Float>(+ #[allow(unused)] a: *const T,+ #[allow(unused)] b: *const T,+ #[allow(unused)] out: *mut T,+ #[allow(unused)] n: i32,+) {+ todo!()+}++#[kernel(AudioScale)]+#[variants(T, f32, f16, bf16)]+pub fn audio_scale<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] n: i32,+ #[allow(unused)] scale: f32,+) {+ todo!()+}++#[kernel(AudioCausalConv1d)]+#[variants(T, f32, f16, bf16)]+pub fn audio_causal_conv1d<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] weight: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] cin: i32,+ #[allow(unused)] cout: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] kernel_size: i32,+ #[allow(unused)] dilation: i32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}++#[kernel(AudioCausalConvTranspose1d)]+#[variants(T, f32, f16, bf16)]+pub fn audio_causal_conv_transpose1d<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] weight: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] cin: i32,+ #[allow(unused)] cout: i32,+ #[allow(unused)] seq_len_in: i32,+ #[allow(unused)] seq_len_out: i32,+ #[allow(unused)] stride: i32,+ #[allow(unused)] groups: i32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}++#[kernel(AudioHalfSnake)]+#[variants(T, f32, f16, bf16)]+pub fn audio_half_snake<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] alpha: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] channels: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] snake_channels: i32,+ #[allow(unused)] negative_slope: f32,+ #[allow(unused)] eps: f32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}++#[kernel(AudioClamp)]+#[variants(T, f32, f16, bf16)]+pub fn audio_clamp<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] n: i32,+ #[allow(unused)] min_value: f32,+ #[allow(unused)] max_value: f32,+) {+ todo!()+}++#[kernel(AudioConv1d)]+#[variants(T, f32, f16, bf16)]+pub fn audio_conv1d<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] weight: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] cin: i32,+ #[allow(unused)] cout: i32,+ #[allow(unused)] seq_len_in: i32,+ #[allow(unused)] seq_len_out: i32,+ #[allow(unused)] kernel_size: i32,+ #[allow(unused)] stride: i32,+ #[allow(unused)] dilation: i32,+ #[allow(unused)] padding: i32,+ #[allow(unused)] pad_mode: i32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}++#[kernel(AudioFsqEncode)]+#[variants(T, f32, f16, bf16)]+pub fn audio_fsq_encode<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] tokens: *mut i32,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] num_groups: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] codebook_dim: i32,+ #[allow(unused)] num_levels: &[i32],+ #[allow(unused)] dim_base_index: &[i32],+ #[allow(unused)] eps: f32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/mod.rs b/crates/uzu/src/backends/cpu/kernel/audio/mod.rsnew file mode 100644index 000000000..fa3e4e468--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/mod.rs@@ -0,0 +1 @@+pub mod audio;diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rsnew file mode 100644index 000000000..3d8e27671--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs@@ -0,0 +1,19 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(FullPrecisionEmbeddingLookup)]+#[variants(T, f32, f16, bf16)]+pub fn full_precision_embedding_lookup<T: ArrayElement + Float>(+ #[allow(unused)] token_ids: *const u64,+ #[allow(unused)] weights: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] vocab_size: u32,+ #[allow(unused)] model_dim: u32,+ #[allow(unused)] input_scale: f32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs b/crates/uzu/src/backends/cpu/kernel/embedding/mod.rsnew file mode 100644index 000000000..93f7cf1f0--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs@@ -0,0 +1,2 @@+pub mod full_precision_embedding;+pub mod quant_embedding;diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rsnew file mode 100644index 000000000..f19235562--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs@@ -0,0 +1,23 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(QuantizedEmbeddingLookup)]+#[variants(T, f32, f16, bf16)]+pub fn quantized_embedding_lookup<T: ArrayElement + Float>(+ #[allow(unused)] token_ids: *const u64,+ #[allow(unused)] weights: *const u8,+ #[allow(unused)] scales: *const T,+ #[allow(unused)] biases: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] vocab_size: u32,+ #[allow(unused)] model_dim: u32,+ #[allow(unused)] group_size: u32,+ #[allow(unused)] input_scale: f32,+ #[allow(unused)] quant_mode: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rsnew file mode 100644index 000000000..58e758c58--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs@@ -0,0 +1,19 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(KVCacheUpdate)]+#[variants(T, f32, bf16, f16)]+pub fn kv_cache_update<T: ArrayElement + Float>(+ #[allow(unused)] in_place_keys: *mut T,+ #[allow(unused)] in_place_values: *mut T,+ #[allow(unused)] swaps: &[crate::backends::common::gpu_types::kv_cache_update::Swap],+ #[allow(unused)] swap_count: u32,+ #[allow(unused)] num_heads: u32,+ #[allow(unused)] max_sequence_length: u32,+ #[allow(unused)] head_dim: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/kv_cache_update/mod.rs b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/mod.rsnew file mode 100644index 000000000..128b5561e--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/mod.rs@@ -0,0 +1 @@+pub mod kv_cache_update;diff --git a/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs b/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rsnew file mode 100644index 000000000..c9be4f254--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs@@ -0,0 +1,28 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(LayerNorm)]+#[variants(IN, f32, f16, bf16)]+#[variants(SC, f32, f16, bf16)]+#[variants(OUT, f32, f16, bf16)]+#[variants(ACC, f32)]+pub fn layer_norm<+ IN: ArrayElement + Float,+ SC: ArrayElement + Float,+ OUT: ArrayElement + Float,+ ACC: ArrayElement + Float,+>(+ #[allow(unused)] input: *const IN,+ #[allow(unused)] scales: *const SC,+ #[allow(unused)] output: *mut OUT,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] model_dim: u32,+ #[allow(unused)] epsilon: f32,+ #[allow(unused)] scale_offset: f32,+ #[allow(unused)] full_layer: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/layer_norm/mod.rs b/crates/uzu/src/backends/cpu/kernel/layer_norm/mod.rsnew file mode 100644index 000000000..89b2f7b11--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/layer_norm/mod.rs@@ -0,0 +1 @@+pub mod layer_norm;diff --git a/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs b/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rsnew file mode 100644index 000000000..97498c6e4--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs@@ -0,0 +1,15 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MaskUpdate)]+#[variants(T, f32, f16, bf16)]+pub fn mask_update<T: ArrayElement + Float>(+ #[allow(unused)] mask: *mut T,+ #[allow(unused)] unmask_col: i32,+ #[allow(unused)] mask_col: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rs b/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rsnew file mode 100644index 000000000..75a6d52a2--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rs@@ -0,0 +1 @@+pub mod mask_update;diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/mod.rsnew file mode 100644index 000000000..b68fab06e--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs@@ -0,0 +1 @@+pub mod shaders;diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rsnew file mode 100644index 000000000..c55461396--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rs@@ -0,0 +1 @@+pub mod steel_gemm;diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rsnew file mode 100644index 000000000..91730cabb--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs@@ -0,0 +1,43 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MatmulGemm)]+#[variants(T, f32, f16, bf16)]+pub fn matmul_gemm<T: ArrayElement + Float>(+ #[allow(unused)] a: *const T,+ #[allow(unused)] b: *const T,+ #[allow(unused)] d: *mut T,+ #[allow(unused)] params: &[crate::backends::common::gpu_types::matmul::GEMMParams],+ #[allow(unused)] group_count_x: u32,+ #[allow(unused)] group_count_y: u32,+ #[allow(unused)] group_count_z: u32,+ #[allow(unused)]+ #[specialize]+ block_rows: u32,+ #[allow(unused)]+ #[specialize]+ block_cols: u32,+ #[allow(unused)]+ #[specialize]+ block_depth: u32,+ #[allow(unused)]+ #[specialize]+ warps_per_row: u32,+ #[allow(unused)]+ #[specialize]+ warps_per_col: u32,+ #[allow(unused)]+ #[specialize]+ align_m: bool,+ #[allow(unused)]+ #[specialize]+ align_n: bool,+ #[allow(unused)]+ #[specialize]+ align_k: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/mod.rsnew file mode 100644index 000000000..b68fab06e--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs@@ -0,0 +1 @@+pub mod shaders;diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rsnew file mode 100644index 000000000..286db8347--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/gemv.rs@@ -0,0 +1,51 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MatmulGemv)]+#[variants(T, f32, f16, bf16)]+pub fn matmul_gemv<T: ArrayElement + Float>(+ #[allow(unused)] matrix: *const T,+ #[allow(unused)] input_vector: *const T,+ #[allow(unused)]+ #[optional(apply_output_scale_and_accumulate)]+ output_source: Option<*const T>,+ #[allow(unused)] output_vector: *mut T,+ #[allow(unused)] input_dimension: i32,+ #[allow(unused)] output_dimension: i32,+ #[allow(unused)] matrix_leading_dimension: i32,+ #[allow(unused)] output_scale: f32,+ #[allow(unused)] output_accumulate_scale: f32,+ #[allow(unused)] batch_shape: &[i32],+ #[allow(unused)] vector_batch_stride: &[i32],+ #[allow(unused)] matrix_batch_stride: &[i32],+ #[allow(unused)] output_source_batch_stride: &[i32],+ #[allow(unused)] output_source_stride: i32,+ #[allow(unused)] batch_rows: i32,+ #[allow(unused)] output_rows_per_threadgroup: i32,+ #[allow(unused)]+ #[specialize]+ tg_simd_rows: u32,+ #[allow(unused)]+ #[specialize]+ tg_simd_cols: u32,+ #[allow(unused)]+ #[specialize]+ sg_thread_rows: u32,+ #[allow(unused)]+ #[specialize]+ sg_thread_cols: u32,+ #[allow(unused)]+ #[specialize]+ thread_out_rows: u32,+ #[allow(unused)]+ #[specialize]+ thread_out_cols: u32,+ #[allow(unused)]+ #[specialize]+ apply_output_scale_and_accumulate: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/mod.rsnew file mode 100644index 000000000..fc35204af--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemv/shaders/mod.rs@@ -0,0 +1 @@+pub mod gemv;diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rsnew file mode 100644index 000000000..26abe007d--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs@@ -0,0 +1,3 @@+pub mod gemm;+pub mod gemv;+pub mod split_k;diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/mod.rsnew file mode 100644index 000000000..b68fab06e--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/mod.rs@@ -0,0 +1 @@+pub mod shaders;diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/mod.rsnew file mode 100644index 000000000..8295f2b83--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/mod.rs@@ -0,0 +1 @@+pub mod steel_gemm_splitk;diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/steel_gemm_splitk.rs b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/steel_gemm_splitk.rsnew file mode 100644index 000000000..4a3efbd68--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/matmul/split_k/shaders/steel_gemm_splitk.rs@@ -0,0 +1,31 @@+use dsl::kernel;+use half::bf16;+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MatmulSplitKPartialBfloat16)]+pub fn matmul_split_k_partial_bfloat16(+ #[allow(unused)] a: *const bf16,+ #[allow(unused)] b: *const bf16,+ #[allow(unused)] c: *mut f32,+ #[allow(unused)] params: &[crate::backends::common::gpu_types::matmul::GEMMSpiltKParams],+ #[allow(unused)] partial_group_count_x: u32,+ #[allow(unused)] partial_group_count_y: u32,+ #[allow(unused)] partial_group_count_z: u32,+) {+ todo!()+}++#[kernel(MatmulSplitKAccumBfloat16)]+pub fn matmul_split_k_accum_bfloat16(+ #[allow(unused)] c_split: *const f32,+ #[allow(unused)] d: *mut bf16,+ #[allow(unused)] k_partitions: i32,+ #[allow(unused)] partition_stride: i32,+ #[allow(unused)] ldd: i32,+ #[allow(unused)] accum_total_threads_x: u32,+ #[allow(unused)] accum_total_threads_y: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rsnew file mode 100644index 000000000..b831f6dbc--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs@@ -0,0 +1,17 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MlpGateActMul)]+#[variants(T, f32, f16, bf16)]+pub fn mlp_gate_act_mul<T: ArrayElement + Float>(+ #[allow(unused)] fused_up: *const T,+ #[allow(unused)] hidden: *mut T,+ #[allow(unused)] h: i32,+ #[allow(unused)] m: i32,+ #[allow(unused)] act_type: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/mlp/mod.rs b/crates/uzu/src/backends/cpu/kernel/mlp/mod.rsnew file mode 100644index 000000000..c6ca0a7ce--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/mlp/mod.rs@@ -0,0 +1 @@+pub mod activation_mul;diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rsnew file mode 100644index 000000000..ec3bb5c04--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs@@ -0,0 +1,65 @@+#![allow(unused)]+mod activation;+mod attention;+mod audio;+mod embedding;+mod kv_cache_update;+mod layer_norm;+mod mask_update;+mod matmul;+mod mlp;+mod moe;+mod pooling;+mod quant_matmul;+mod rms_norm;+mod rope;+mod sampling;+mod short_conv;+mod sigmoid;+mod ssm;+mod tensor_add_bias;+mod tensor_add_swap;+mod tensor_copy;+mod token_copy;++include!(concat!(env!("OUT_DIR"), "/cpu/dsl.rs"));++use crate::{+ DataType,+ backends::{+ common::{+ Backend,+ kernel::matmul::{+ FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,+ MatmulKernels,+ },+ },+ cpu::{Cpu, context::CpuContext, error::CpuError},+ },+};++pub struct FullPrecisionMatmulCpuKernel;++impl FullPrecisionMatmulKernelTrait for FullPrecisionMatmulCpuKernel {+ type Backend = Cpu;++ fn new(+ #[allow(unused)] context: &CpuContext,+ #[allow(unused)] data_type: DataType,+ ) -> Result<Self, CpuError> {+ Ok(Self)+ }++ fn encode(+ &mut self,+ #[allow(unused)] context: &CpuContext,+ #[allow(unused)] encoder: &<Cpu as Backend>::ComputeEncoder,+ #[allow(unused)] arguments: FullPrecisionMatmulArguments<Cpu>,+ ) {+ encoder.borrow_mut().push_command(move || todo!());+ }+}++impl MatmulKernels for CpuKernels {+ type FullPrecisionMatmulKernel = FullPrecisionMatmulCpuKernel;+}diff --git a/crates/uzu/src/backends/cpu/kernel/moe/counts_offsets_fused.rs b/crates/uzu/src/backends/cpu/kernel/moe/counts_offsets_fused.rsnew file mode 100644index 000000000..f4f39e19d--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/moe/counts_offsets_fused.rs@@ -0,0 +1,17 @@+use dsl::kernel;+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MoeCountsOffsetsFused)]+pub fn moe_counts_offsets_fused(+ #[allow(unused)] topk_ids: *const i32,+ #[allow(unused)] offsets: *mut u32,+ #[allow(unused)] sum_k_out: *mut u32,+ #[allow(unused)] partials: *mut u32,+ #[allow(unused)] t_input: u32,+ #[allow(unused)] e_input: u32,+ #[allow(unused)] k_input: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/moe/experts_single.rs b/crates/uzu/src/backends/cpu/kernel/moe/experts_single.rsnew file mode 100644index 000000000..064e54d8b--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/moe/experts_single.rs@@ -0,0 +1,44 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MoeExpertsDecodeSinglePassA)]+#[variants(T, f32, f16, bf16)]+pub fn moe_experts_decode_single_pass_a<T: ArrayElement + Float>(+ #[allow(unused)] x: *const T,+ #[allow(unused)] topk_ids: *const i32,+ #[allow(unused)] w13_all: *const T,+ #[allow(unused)] biases: *const T,+ #[allow(unused)] hidden_out: *mut f32,+ #[allow(unused)] d_model: u32,+ #[allow(unused)] d_ff: u32,+ #[allow(unused)] k: u32,+ #[allow(unused)] silu_alpha: f32,+ #[allow(unused)] gate_clip_min: f32,+ #[allow(unused)] gate_clip_max: f32,+ #[allow(unused)] up_clip_min: f32,+ #[allow(unused)] up_clip_max: f32,+ #[allow(unused)]+ #[specialize]+ gating_sel: u32,+) {+ todo!()+}++#[kernel(MoeExpertsDecodeSinglePassB)]+#[variants(T, f32, f16, bf16)]+pub fn moe_experts_decode_single_pass_b<T: ArrayElement + Float>(+ #[allow(unused)] hidden: *const f32,+ #[allow(unused)] topk_ids: *const i32,+ #[allow(unused)] topk_probs: *const T,+ #[allow(unused)] w2_all: *const T,+ #[allow(unused)] biases: *const T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] d_model: u32,+ #[allow(unused)] d_ff: u32,+ #[allow(unused)] k_input: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_decode.rsnew file mode 100644index 000000000..c61db4f40--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_decode.rs@@ -0,0 +1,47 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MoeExpertsDecodePassA)]+#[variants(T, f32, f16, bf16)]+pub fn moe_experts_decode_pass_a<T: ArrayElement + Float>(+ #[allow(unused)] x_perm: *const T,+ #[allow(unused)] expert_offsets: *const u32,+ #[allow(unused)] w13_all: *const T,+ #[allow(unused)] hidden_out: *mut f32,+ #[allow(unused)] up_biases: *const T,+ #[allow(unused)] d_model: u32,+ #[allow(unused)] d_ff: u32,+ #[allow(unused)] e: u32,+ #[allow(unused)] gate_clip_min: f32,+ #[allow(unused)] gate_clip_max: f32,+ #[allow(unused)] up_clip_min: f32,+ #[allow(unused)] up_clip_max: f32,+ #[allow(unused)] silu_alpha: f32,+ #[allow(unused)] tile_map: *const u32,+ #[allow(unused)]+ #[specialize]+ gating_sel: u32,+ #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,+) {+ todo!()+}++#[kernel(MoeExpertsDecodeDownFused2D)]+#[variants(T, f32, f16, bf16)]+#[variants(AccumT, f32)]+pub fn moe_experts_decode_down_fused2_d<T: ArrayElement + Float, AccumT: ArrayElement + Float>(+ #[allow(unused)] hidden: *const f32,+ #[allow(unused)] row_expert_map: *const u32,+ #[allow(unused)] w2_all: *const T,+ #[allow(unused)] down_biases: *const T,+ #[allow(unused)] y_out: *mut T,+ #[allow(unused)] total_rows: u32,+ #[allow(unused)] d_model: u32,+ #[allow(unused)] d_ff: u32,+ #[allow(unused)] e: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rsnew file mode 100644index 000000000..119f4d585--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rs@@ -0,0 +1,47 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MoeExpertsPrefillPassA)]+#[variants(T, f32, f16, bf16)]+pub fn moe_experts_prefill_pass_a<T: ArrayElement + Float>(+ #[allow(unused)] x_perm: *const T,+ #[allow(unused)] expert_offsets: *const u32,+ #[allow(unused)] w13_all: *const T,+ #[allow(unused)] up_biases: *const T,+ #[allow(unused)] hidden_out: *mut f32,+ #[allow(unused)] d_model: u32,+ #[allow(unused)] d_ff: u32,+ #[allow(unused)] e: u32,+ #[allow(unused)] gate_clip_min: f32,+ #[allow(unused)] gate_clip_max: f32,+ #[allow(unused)] up_clip_min: f32,+ #[allow(unused)] up_clip_max: f32,+ #[allow(unused)] silu_alpha: f32,+ #[allow(unused)] tile_map: *const u32,+ #[allow(unused)]+ #[specialize]+ gating_sel: u32,+ #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,+) {+ todo!()+}++#[kernel(MoeExpertsPrefillPassB)]+#[variants(T, f32, f16, bf16)]+pub fn moe_experts_prefill_pass_b<T: ArrayElement + Float>(+ #[allow(unused)] hidden: *const f32,+ #[allow(unused)] expert_offsets: *const u32,+ #[allow(unused)] w2_all: *const T,+ #[allow(unused)] down_biases: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] d_model: u32,+ #[allow(unused)] d_ff: u32,+ #[allow(unused)] e: u32,+ #[allow(unused)] tile_map: *const u32,+ #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/moe/finalize.rs b/crates/uzu/src/backends/cpu/kernel/moe/finalize.rsnew file mode 100644index 000000000..b1ecf696d--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/moe/finalize.rs@@ -0,0 +1,19 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MoeFinalize)]+#[variants(T, f32, f16, bf16)]+pub fn moe_finalize<T: ArrayElement + Float>(+ #[allow(unused)] tok2row: *const i32,+ #[allow(unused)] probs: *const T,+ #[allow(unused)] y_partial: *const T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] t_count: u32,+ #[allow(unused)] d_model: u32,+ #[allow(unused)] k_input: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/moe/gather.rs b/crates/uzu/src/backends/cpu/kernel/moe/gather.rsnew file mode 100644index 000000000..c6d9f6329--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/moe/gather.rs@@ -0,0 +1,33 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MoeGatherXPerm2D)]+#[variants(T, f32, f16, bf16)]+pub fn moe_gather_x_perm2_d<T: ArrayElement + Float>(+ #[allow(unused)] x: *const T,+ #[allow(unused)] bucketed_ids: *const i32,+ #[allow(unused)] x_perm: *mut T,+ #[allow(unused)] sumk_buf: *const u32,+ #[allow(unused)] d_model: u32,+ #[allow(unused)] t: u32,+ #[allow(unused)] k: u32,+) {+ todo!()+}++#[kernel(MoeGatherXPerm1D)]+#[variants(T, f32, f16, bf16)]+pub fn moe_gather_x_perm1_d<T: ArrayElement + Float>(+ #[allow(unused)] x: *const T,+ #[allow(unused)] bucketed_ids: *const i32,+ #[allow(unused)] x_perm: *mut T,+ #[allow(unused)] sumk_buf: *const u32,+ #[allow(unused)] d_model: u32,+ #[allow(unused)] t: u32,+ #[allow(unused)] k: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/moe/mod.rs b/crates/uzu/src/backends/cpu/kernel/moe/mod.rsnew file mode 100644index 000000000..5f029d341--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/moe/mod.rs@@ -0,0 +1,10 @@+pub mod counts_offsets_fused;+pub mod experts_single;+pub mod experts_two_pass_decode;+pub mod experts_two_pass_prefill;+pub mod finalize;+pub mod gather;+pub mod router_topk;+pub mod scatter_buckets;+pub mod tiles_map;+pub mod tiles_pass_a;diff --git a/crates/uzu/src/backends/cpu/kernel/moe/router_topk.rs b/crates/uzu/src/backends/cpu/kernel/moe/router_topk.rsnew file mode 100644index 000000000..69eb270bf--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/moe/router_topk.rs@@ -0,0 +1,22 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MoeRouterTopK)]+#[variants(ScalarT, f16, bf16, f32)]+pub fn moe_router_top_k<ScalarT: ArrayElement + Float>(+ #[allow(unused)] input: *const ScalarT,+ #[allow(unused)] weight: *const ScalarT,+ #[allow(unused)] bias: *const ScalarT,+ #[allow(unused)] topk_ids: *mut i32,+ #[allow(unused)] topk_probs: *mut ScalarT,+ #[allow(unused)] t: u32,+ #[allow(unused)] d_model: u32,+ #[allow(unused)] e: u32,+ #[allow(unused)] k: u32,+ #[allow(unused)] renorm: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/moe/scatter_buckets.rs b/crates/uzu/src/backends/cpu/kernel/moe/scatter_buckets.rsnew file mode 100644index 000000000..e0b521345--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/moe/scatter_buckets.rs@@ -0,0 +1,57 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MoeBlockBasesFromPartials)]+pub fn moe_block_bases_from_partials(+ #[allow(unused)] partials: *const u32,+ #[allow(unused)] block_bases: *mut u32,+ #[allow(unused)] block_alloc: *mut u32,+ #[allow(unused)] e_input: u32,+ #[allow(unused)] num_blocks: u32,+ #[allow(unused)] num_tiles: u32,+ #[allow(unused)] capacity_per_expert: u32,+) {+ todo!()+}++#[kernel(MoeScatterBuckets)]+#[variants(T, f32, f16, bf16)]+pub fn moe_scatter_buckets<T: ArrayElement + Float>(+ #[allow(unused)] topk_ids: *const i32,+ #[allow(unused)] topk_probs: *const T,+ #[allow(unused)] offsets: *const u32,+ #[allow(unused)] block_bases: *const u32,+ #[allow(unused)] block_alloc: *const u32,+ #[allow(unused)] out_ids: *mut i32,+ #[allow(unused)] out_probs: *mut T,+ #[allow(unused)] t: u32,+ #[allow(unused)] e: u32,+ #[allow(unused)] k: u32,+ #[allow(unused)] num_blocks: u32,+ #[allow(unused)] num_tiles: u32,+) {+ todo!()+}++#[kernel(MoeScatterBucketsMap)]+#[variants(T, f32, f16, bf16)]+pub fn moe_scatter_buckets_map<T: ArrayElement + Float>(+ #[allow(unused)] topk_ids: *const i32,+ #[allow(unused)] topk_probs: *const T,+ #[allow(unused)] offsets: *const u32,+ #[allow(unused)] block_bases: *const u32,+ #[allow(unused)] block_alloc: *const u32,+ #[allow(unused)] out_ids: *mut i32,+ #[allow(unused)] out_probs: *mut T,+ #[allow(unused)] t: u32,+ #[allow(unused)] e: u32,+ #[allow(unused)] k: u32,+ #[allow(unused)] num_blocks: u32,+ #[allow(unused)] num_tiles: u32,+ #[allow(unused)] tok2row: *mut i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/cpu/kernel/moe/tiles_map.rsnew file mode 100644index 000000000..2314c6cbb--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/moe/tiles_map.rs@@ -0,0 +1,43 @@+use dsl::kernel;+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MoeTileCounts)]+pub fn moe_tile_counts(+ #[allow(unused)] offsets: *const u32,+ #[allow(unused)] tile_counts: *mut u32,+ #[allow(unused)] e: u32,+) {+ todo!()+}++#[kernel(MoeTileScan)]+pub fn moe_tile_scan(+ #[allow(unused)] tile_counts: *const u32,+ #[allow(unused)] tile_row_offsets: *mut u32,+ #[allow(unused)] total_tiles_buf: *mut u32,+ #[allow(unused)] e: u32,+) {+ todo!()+}++#[kernel(MoeBuildTileMap)]+pub fn moe_build_tile_map(+ #[allow(unused)] offsets: *const u32,+ #[allow(unused)] tile_row_offsets: *const u32,+ #[allow(unused)] tile_counts: *const u32,+ #[allow(unused)] tile_map: *mut u32,+ #[allow(unused)] e: u32,+) {+ todo!()+}++#[kernel(MoeWriteDispatchArgs)]+pub fn moe_write_dispatch_args(+ #[allow(unused)] total_tiles_buf: *const u32,+ #[allow(unused)] dispatch_args: *mut u32,+ #[allow(unused)] num_tiles_n: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/cpu/kernel/moe/tiles_pass_a.rsnew file mode 100644index 000000000..7a165441c--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/moe/tiles_pass_a.rs@@ -0,0 +1,55 @@+use dsl::kernel;+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MoePassATileCounts)]+pub fn moe_pass_a_tile_counts(+ #[allow(unused)] expert_offsets: *const u32,+ #[allow(unused)] tile_counts: *mut u32,+ #[allow(unused)] e: u32,+ #[allow(unused)] h_blocks: u32,+) {+ todo!()+}++#[kernel(MoePassATileScan)]+pub fn moe_pass_a_tile_scan(+ #[allow(unused)] tile_counts: *const u32,+ #[allow(unused)] tile_offsets: *mut u32,+ #[allow(unused)] total_tiles: *mut u32,+ #[allow(unused)] e: u32,+) {+ todo!()+}++#[kernel(MoePassABuildRowMap)]+pub fn moe_pass_a_build_row_map(+ #[allow(unused)] expert_offsets: *const u32,+ #[allow(unused)] row_expert_map: *mut u32,+ #[allow(unused)] total_rows: u32,+ #[allow(unused)] e: u32,+) {+ todo!()+}++#[kernel(MoePassABuildTileMap)]+pub fn moe_pass_a_build_tile_map(+ #[allow(unused)] expert_offsets: *const u32,+ #[allow(unused)] tile_offsets: *const u32,+ #[allow(unused)] row_expert_map: *const u32,+ #[allow(unused)] tile_map: *mut u32,+ #[allow(unused)] total_rows: u32,+ #[allow(unused)] h_blocks: u32,+) {+ todo!()+}++#[kernel(MoePassAWriteDispatchArgs)]+pub fn moe_pass_a_write_dispatch_args(+ #[allow(unused)] total_tiles: *const u32,+ #[allow(unused)] dispatch_args: *mut u32,+ #[allow(unused)] num_tiles_y: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/pooling/mod.rs b/crates/uzu/src/backends/cpu/kernel/pooling/mod.rsnew file mode 100644index 000000000..f30070eb3--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/pooling/mod.rs@@ -0,0 +1 @@+pub mod pooling;diff --git a/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs b/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rsnew file mode 100644index 000000000..9daf2804b--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs@@ -0,0 +1,29 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(PoolingCls)]+#[variants(T, f32, f16, bf16)]+pub fn pooling_cls<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] seq_len: u32,+ #[allow(unused)] hidden_dim: u32,+ #[allow(unused)] batch_size: u32,+) {+ todo!()+}++#[kernel(PoolingMean)]+#[variants(T, f32, f16, bf16)]+pub fn pooling_mean<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] seq_len: u32,+ #[allow(unused)] hidden_dim: u32,+ #[allow(unused)] batch_size: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rsnew file mode 100644index 000000000..dadd0367b--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs@@ -0,0 +1,6 @@+pub mod qmm;+pub mod qmm_transposed;+pub mod qmm_transposed_64x64;+pub mod qmv;+pub mod qmv_fast;+pub mod qvm;diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rsnew file mode 100644index 000000000..823301f3d--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rs@@ -0,0 +1,36 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(QuantizedMatmulQmm)]+#[variants(T, f32, f16, bf16)]+#[variants(GROUP_SIZE, 32, 64, 128)]+#[variants(BITS, 4, 8)]+pub fn quantized_matmul_qmm<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(+ #[allow(unused)] w: *const u32,+ #[allow(unused)] scales: *const T,+ #[allow(unused)]+ #[optional(use_zero_points)]+ zero_points: Option<*const u8>,+ #[allow(unused)]+ #[optional(use_mlx_quant)]+ biases: Option<*const T>,+ #[allow(unused)] x: *const T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] k: i32,+ #[allow(unused)] n: i32,+ #[allow(unused)] m: i32,+ #[allow(unused)]+ #[specialize]+ use_zero_points: bool,+ #[allow(unused)]+ #[specialize]+ use_mlx_quant: bool,+ #[allow(unused)]+ #[specialize]+ aligned_k: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rsnew file mode 100644index 000000000..62f3f9456--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs@@ -0,0 +1,36 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(QuantizedMatmulQmmTransposed)]+#[variants(T, f32, f16, bf16)]+#[variants(GROUP_SIZE, 32, 64, 128)]+#[variants(BITS, 4, 8)]+pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(+ #[allow(unused)] w: *const u32,+ #[allow(unused)] scales: *const T,+ #[allow(unused)]+ #[optional(use_zero_points)]+ zero_points: Option<*const u8>,+ #[allow(unused)]+ #[optional(use_mlx_quant)]+ biases: Option<*const T>,+ #[allow(unused)] x: *const T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] k: i32,+ #[allow(unused)] n: i32,+ #[allow(unused)] m: i32,+ #[allow(unused)]+ #[specialize]+ use_zero_points: bool,+ #[allow(unused)]+ #[specialize]+ use_mlx_quant: bool,+ #[allow(unused)]+ #[specialize]+ aligned_n: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rsnew file mode 100644index 000000000..e252f69bc--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs@@ -0,0 +1,33 @@+use dsl::kernel;+use half::bf16;+use num_traits::Float;++use crate::ArrayElement;++#[kernel(QuantizedMatmulQmmTransposed64x64)]+#[variants(T, bf16)]+#[variants(GROUP_SIZE, 64, 128)]+#[variants(BITS, 4, 8)]+pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(+ #[allow(unused)] w: *const u32,+ #[allow(unused)] scales: *const T,+ #[allow(unused)]+ #[optional(use_zero_points)]+ zero_points: Option<*const u8>,+ #[allow(unused)]+ #[optional(use_mlx_quant)]+ biases: Option<*const T>,+ #[allow(unused)] x: *const T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] k: i32,+ #[allow(unused)] n: i32,+ #[allow(unused)] m: i32,+ #[allow(unused)]+ #[specialize]+ use_zero_points: bool,+ #[allow(unused)]+ #[specialize]+ use_mlx_quant: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rsnew file mode 100644index 000000000..5ae376ea9--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs@@ -0,0 +1,33 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(QuantizedMatmulQmv)]+#[variants(T, f32, f16, bf16)]+#[variants(GROUP_SIZE, 32, 64, 128)]+#[variants(BITS, 4, 8)]+pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(+ #[allow(unused)] w: *const u32,+ #[allow(unused)] scales: *const T,+ #[allow(unused)]+ #[optional(use_zero_points)]+ zero_points: Option<*const u8>,+ #[allow(unused)]+ #[optional(use_mlx_quant)]+ biases: Option<*const T>,+ #[allow(unused)] x: *const T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] k: i32,+ #[allow(unused)] n: i32,+ #[allow(unused)] m: i32,+ #[allow(unused)]+ #[specialize]+ use_zero_points: bool,+ #[allow(unused)]+ #[specialize]+ use_mlx_quant: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rsnew file mode 100644index 000000000..33f16dbcc--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs@@ -0,0 +1,33 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(QuantizedMatmulQmvFast)]+#[variants(T, f32, f16, bf16)]+#[variants(GROUP_SIZE, 32, 64, 128)]+#[variants(BITS, 4, 8)]+pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(+ #[allow(unused)] w: *const u32,+ #[allow(unused)] scales: *const T,+ #[allow(unused)]+ #[optional(use_zero_points)]+ zero_points: Option<*const u8>,+ #[allow(unused)]+ #[optional(use_mlx_quant)]+ biases: Option<*const T>,+ #[allow(unused)] x: *const T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] k: i32,+ #[allow(unused)] n: i32,+ #[allow(unused)] m: i32,+ #[allow(unused)]+ #[specialize]+ use_zero_points: bool,+ #[allow(unused)]+ #[specialize]+ use_mlx_quant: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rsnew file mode 100644index 000000000..6eeba6605--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rs@@ -0,0 +1,33 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(QuantizedMatmulQvm)]+#[variants(T, f32, f16, bf16)]+#[variants(GROUP_SIZE, 32, 64, 128)]+#[variants(BITS, 4, 8)]+pub fn quantized_matmul_qvm<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(+ #[allow(unused)] w: *const u32,+ #[allow(unused)] scales: *const T,+ #[allow(unused)]+ #[optional(use_zero_points)]+ zero_points: Option<*const u8>,+ #[allow(unused)]+ #[optional(use_mlx_quant)]+ biases: Option<*const T>,+ #[allow(unused)] x: *const T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] k: i32,+ #[allow(unused)] n: i32,+ #[allow(unused)] m: i32,+ #[allow(unused)]+ #[specialize]+ use_zero_points: bool,+ #[allow(unused)]+ #[specialize]+ use_mlx_quant: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/mod.rsnew file mode 100644index 000000000..c26f009f4--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/mod.rs@@ -0,0 +1,2 @@+pub mod qk_norm;+pub mod rms_norm;diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/qk_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/qk_norm.rsnew file mode 100644index 000000000..935948d4d--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/qk_norm.rs@@ -0,0 +1,32 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(QKNorm)]+#[variants(InputT, f32, f16, bf16)]+#[variants(ScaleT, f32, f16, bf16)]+#[variants(OutputT, f32, f16, bf16)]+#[variants(AccumT, f32, f16)]+pub fn qk_norm<+ InputT: ArrayElement + Float,+ ScaleT: ArrayElement + Float,+ OutputT: ArrayElement + Float,+ AccumT: ArrayElement + Float,+>(+ #[allow(unused)] qkv_input: *const InputT,+ #[allow(unused)] scales: *const ScaleT,+ #[allow(unused)] qkv_output: *mut OutputT,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] num_q_heads: u32,+ #[allow(unused)] num_kv_heads: u32,+ #[allow(unused)] head_dim: u32,+ #[allow(unused)] epsilon: f32,+ #[allow(unused)] scale_offset: f32,+ #[allow(unused)] head_offset: u32,+ #[allow(unused)] head_count: u32,+ #[allow(unused)] full_layer: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rsnew file mode 100644index 000000000..a778079fd--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs@@ -0,0 +1,28 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(RMSNorm)]+#[variants(InputT, f32, f16, bf16)]+#[variants(ScaleT, f32, f16, bf16)]+#[variants(OutputT, f32, f16, bf16)]+#[variants(AccumT, f32, f16)]+pub fn rms_norm<+ InputT: ArrayElement + Float,+ ScaleT: ArrayElement + Float,+ OutputT: ArrayElement + Float,+ AccumT: ArrayElement + Float,+>(+ #[allow(unused)] input: *const InputT,+ #[allow(unused)] scales: *const ScaleT,+ #[allow(unused)] output: *mut OutputT,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] element_count: u32,+ #[allow(unused)] epsilon: f32,+ #[allow(unused)] scale_offset: f32,+ #[allow(unused)] full_layer: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/rope/mod.rs b/crates/uzu/src/backends/cpu/kernel/rope/mod.rsnew file mode 100644index 000000000..f67f4f08b--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/rope/mod.rs@@ -0,0 +1 @@+pub mod rope;diff --git a/crates/uzu/src/backends/cpu/kernel/rope/rope.rs b/crates/uzu/src/backends/cpu/kernel/rope/rope.rsnew file mode 100644index 000000000..ea5da869f--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/rope/rope.rs@@ -0,0 +1,23 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(Rope)]+#[variants(T, f32, f16, bf16)]+pub fn rope<T: ArrayElement + Float>(+ #[allow(unused)] qkv: *const T,+ #[allow(unused)] cosines: *const T,+ #[allow(unused)] sines: *const T,+ #[allow(unused)] token_positions: *const i32,+ #[allow(unused)] rotated_queries: *mut T,+ #[allow(unused)] rotated_keys: *mut T,+ #[allow(unused)] head_dim: u32,+ #[allow(unused)] num_heads: u32,+ #[allow(unused)] num_groups: u32,+ #[allow(unused)] suffix_length: u32,+ #[allow(unused)] max_sequence_length: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/argmax.rs b/crates/uzu/src/backends/cpu/kernel/sampling/argmax.rsnew file mode 100644index 000000000..61aea8a75--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/sampling/argmax.rs@@ -0,0 +1,37 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(ArgmaxSingle)]+#[variants(T, f32, f16, bf16)]+pub fn argmax_single<T: ArrayElement + Float>(+ #[allow(unused)] logits_data: *const T,+ #[allow(unused)] final_tokens: *mut u32,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] vocab_size: u32,+) {+ todo!()+}++#[kernel(ArgmaxMain)]+#[variants(T, f32, f16, bf16)]+pub fn argmax_main<T: ArrayElement + Float>(+ #[allow(unused)] logits_data: *const T,+ #[allow(unused)] partial_results: *mut crate::backends::common::gpu_types::argmax::ArgmaxPair,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] vocab_size: u32,+) {+ todo!()+}++#[kernel(ArgmaxFinal)]+pub fn argmax_final(+ #[allow(unused)] partial_results: *const crate::backends::common::gpu_types::argmax::ArgmaxPair,+ #[allow(unused)] final_tokens: *mut u32,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] vocab_size: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs b/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rsnew file mode 100644index 000000000..61702f72a--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs@@ -0,0 +1,17 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(Bitmask)]+#[variants(T, f32, f16, bf16)]+pub fn bitmask<T: ArrayElement + Float>(+ #[allow(unused)] logits: *const T,+ #[allow(unused)] bitmask: *const u32,+ #[allow(unused)] processed_logits: *mut T,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] vocab_size: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs b/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rsnew file mode 100644index 000000000..943b956e6--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs@@ -0,0 +1,17 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(Gumbel)]+#[variants(T, f32, f16, bf16)]+pub fn gumbel<T: ArrayElement + Float>(+ #[allow(unused)] logits: *const T,+ #[allow(unused)] batch_seeds: *const u64,+ #[allow(unused)] processed_logits: *mut T,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] vocab_size: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs b/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rsnew file mode 100644index 000000000..58e9deb06--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs@@ -0,0 +1,17 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(MinP)]+#[variants(T, f32, f16, bf16)]+pub fn min_p<T: ArrayElement + Float>(+ #[allow(unused)] logits: *const T,+ #[allow(unused)] processed_logits: *mut T,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] vocab_size: u32,+ #[allow(unused)] min_p: f32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/mod.rs b/crates/uzu/src/backends/cpu/kernel/sampling/mod.rsnew file mode 100644index 000000000..db49e2e7a--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/sampling/mod.rs@@ -0,0 +1,7 @@+pub mod argmax;+pub mod bitmask;+pub mod gumbel;+pub mod min_p;+pub mod temperature;+pub mod top_k;+pub mod top_p;diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs b/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rsnew file mode 100644index 000000000..b40ee1cea--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs@@ -0,0 +1,17 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(Temperature)]+#[variants(T, f32, f16, bf16)]+pub fn temperature<T: ArrayElement + Float>(+ #[allow(unused)] logits: *const T,+ #[allow(unused)] processed_logits: *mut T,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] vocab_size: u32,+ #[allow(unused)] temperature: f32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs b/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rsnew file mode 100644index 000000000..909cebd82--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs@@ -0,0 +1,17 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(TopK)]+#[variants(T, f32, f16, bf16)]+pub fn top_k<T: ArrayElement + Float>(+ #[allow(unused)] logits: *const T,+ #[allow(unused)] processed_logits: *mut T,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] vocab_size: u32,+ #[allow(unused)] top_k: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs b/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rsnew file mode 100644index 000000000..f6879e479--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs@@ -0,0 +1,17 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(TopP)]+#[variants(T, f32, f16, bf16)]+pub fn top_p<T: ArrayElement + Float>(+ #[allow(unused)] logits: *const T,+ #[allow(unused)] processed_logits: *mut T,+ #[allow(unused)] batch_size: u32,+ #[allow(unused)] vocab_size: u32,+ #[allow(unused)] top_p: f32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/short_conv/mod.rs b/crates/uzu/src/backends/cpu/kernel/short_conv/mod.rsnew file mode 100644index 000000000..2834d2c1d--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/short_conv/mod.rs@@ -0,0 +1 @@+pub mod short_conv;diff --git a/crates/uzu/src/backends/cpu/kernel/short_conv/short_conv.rs b/crates/uzu/src/backends/cpu/kernel/short_conv/short_conv.rsnew file mode 100644index 000000000..fe031529a--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/short_conv/short_conv.rs@@ -0,0 +1,89 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(ShortConvPack)]+#[variants(T, f32, f16, bf16)]+pub fn short_conv_pack<T: ArrayElement + Float>(+ #[allow(unused)] state_in: *const T,+ #[allow(unused)] in_proj: *const T,+ #[allow(unused)] padded: *mut T,+ #[allow(unused)] state_stride: u32,+ #[allow(unused)] suffix_len: u32,+ #[allow(unused)] in_proj_stride: u32,+ #[allow(unused)] model_dim: u32,+) {+ todo!()+}++#[kernel(ShortConvPrefill)]+#[variants(T, f32, f16, bf16)]+pub fn short_conv_prefill<T: ArrayElement + Float>(+ #[allow(unused)] padded: *const T,+ #[allow(unused)] in_proj: *const T,+ #[allow(unused)] w: *const T,+ #[allow(unused)]+ #[optional(has_bias)]+ b: Option<*const T>,+ #[allow(unused)] out: *mut T,+ #[allow(unused)] state_out: *mut T,+ #[allow(unused)] suffix_len: u32,+ #[allow(unused)] kernel_size: u32,+ #[allow(unused)] in_proj_stride: u32,+ #[allow(unused)] state_stride: u32,+ #[allow(unused)] model_dim: u32,+ #[allow(unused)]+ #[specialize]+ has_bias: bool,+) {+ todo!()+}++#[kernel(ShortConvDecode)]+#[variants(T, f32, f16, bf16)]+pub fn short_conv_decode<T: ArrayElement + Float>(+ #[allow(unused)] in_proj: *const T,+ #[allow(unused)] w: *const T,+ #[allow(unused)]+ #[optional(has_bias)]+ b: Option<*const T>,+ #[allow(unused)] state: *const T,+ #[allow(unused)] out: *mut T,+ #[allow(unused)] next_state: *mut T,+ #[allow(unused)] suffix_len: u32,+ #[allow(unused)] kernel_size: u32,+ #[allow(unused)] in_proj_stride: u32,+ #[allow(unused)] state_stride: u32,+ #[allow(unused)] model_dim: u32,+ #[allow(unused)]+ #[specialize]+ has_bias: bool,+) {+ todo!()+}++#[kernel(ShortConvTrie)]+#[variants(T, f32, f16, bf16)]+pub fn short_conv_trie<T: ArrayElement + Float>(+ #[allow(unused)] in_proj: *const T,+ #[allow(unused)] w: *const T,+ #[allow(unused)]+ #[optional(has_bias)]+ b: Option<*const T>,+ #[allow(unused)] base_state: *const T,+ #[allow(unused)] parents: *const i32,+ #[allow(unused)] out: *mut T,+ #[allow(unused)] suffix_state: *mut T,+ #[allow(unused)] suffix_len: u32,+ #[allow(unused)] kernel_size: u32,+ #[allow(unused)] in_proj_stride: u32,+ #[allow(unused)] state_stride: u32,+ #[allow(unused)] model_dim: u32,+ #[allow(unused)]+ #[specialize]+ has_bias: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rs b/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rsnew file mode 100644index 000000000..33caeb2aa--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rs@@ -0,0 +1 @@+pub mod sigmoid;diff --git a/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs b/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rsnew file mode 100644index 000000000..dc3aacc56--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs@@ -0,0 +1,15 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(Sigmoid)]+#[variants(T, f32, f16, bf16)]+pub fn sigmoid<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] total_elements: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rsnew file mode 100644index 000000000..4b8098ac0--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs@@ -0,0 +1,78 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(Conv1dPack)]+#[variants(T, f32, f16, bf16)]+pub fn conv1d_pack<T: ArrayElement + Float>(+ #[allow(unused)] state_in: *const T,+ #[allow(unused)] x: *const T,+ #[allow(unused)] padded: *mut T,+ #[allow(unused)] state_stride: u32,+ #[allow(unused)] row_stride: u32,+ #[allow(unused)] suffix_len: u32,+ #[allow(unused)] num_channels: u32,+) {+ todo!()+}++#[kernel(Conv1dDecode)]+#[variants(T, f32, f16, bf16)]+pub fn conv1d_decode<T: ArrayElement + Float>(+ #[allow(unused)] x: *const T,+ #[allow(unused)] w: *const T,+ #[allow(unused)]+ #[optional(has_bias)]+ b: Option<*const T>,+ #[allow(unused)] state: *const T,+ #[allow(unused)] x_out: *mut T,+ #[allow(unused)] b_out: *mut T,+ #[allow(unused)] c_out: *mut T,+ #[allow(unused)] next_state: *mut T,+ #[allow(unused)] kernel_size: u32,+ #[allow(unused)] row_stride: u32,+ #[allow(unused)] state_stride: u32,+ #[allow(unused)] num_channels: u32,+ #[allow(unused)] suffix_len: u32,+ #[allow(unused)] inner_dim: u32,+ #[allow(unused)] proj_dim: u32,+ #[allow(unused)]+ #[specialize]+ activation_type: u32,+ #[allow(unused)]+ #[specialize]+ has_bias: bool,+) {+ todo!()+}++#[kernel(Conv1dScan)]+#[variants(T, f32, f16, bf16)]+pub fn conv1d_scan<T: ArrayElement + Float>(+ #[allow(unused)] padded: *const T,+ #[allow(unused)] w: *const T,+ #[allow(unused)]+ #[optional(has_bias)]+ b: Option<*const T>,+ #[allow(unused)] x_out: *mut T,+ #[allow(unused)] b_out: *mut T,+ #[allow(unused)] c_out: *mut T,+ #[allow(unused)] state_out: *mut T,+ #[allow(unused)] suffix_len: u32,+ #[allow(unused)] kernel_size: u32,+ #[allow(unused)] row_stride: u32,+ #[allow(unused)] state_stride: u32,+ #[allow(unused)] num_channels: u32,+ #[allow(unused)] inner_dim: u32,+ #[allow(unused)] proj_dim: u32,+ #[allow(unused)]+ #[specialize]+ activation_type: u32,+ #[allow(unused)]+ #[specialize]+ has_bias: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/mod.rs b/crates/uzu/src/backends/cpu/kernel/ssm/mod.rsnew file mode 100644index 000000000..fb01e93a7--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/ssm/mod.rs@@ -0,0 +1,4 @@+pub mod conv1d;+pub mod split_inproj;+pub mod ssd_prefill;+pub mod ssd_update;diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs b/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rsnew file mode 100644index 000000000..873e4908f--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs@@ -0,0 +1,22 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(SplitInProj)]+#[variants(T, f32, f16, bf16)]+pub fn split_in_proj<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] conv_out: *mut T,+ #[allow(unused)] z_out: *mut T,+ #[allow(unused)] dt_out: *mut T,+ #[allow(unused)] z_bias: *const T,+ #[allow(unused)] suffix_length: u32,+ #[allow(unused)] total_dim: u32,+ #[allow(unused)] conv_dim: u32,+ #[allow(unused)] inner_dim: u32,+ #[allow(unused)] num_heads: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rsnew file mode 100644index 000000000..0ca159e13--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs@@ -0,0 +1,77 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(SSDPrefill64)]+#[variants(T, f32, f16, bf16)]+pub fn ssd_prefill64<T: ArrayElement + Float>(+ #[allow(unused)] x: *const T,+ #[allow(unused)] dt_raw: *const T,+ #[allow(unused)] b: *const T,+ #[allow(unused)] c: *const T,+ #[allow(unused)] d: *const T,+ #[allow(unused)] z: *const T,+ #[allow(unused)] state: *mut T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] suffix_len: u32,+ #[allow(unused)] group_size: i32,+ #[allow(unused)] state_size: i32,+ #[allow(unused)] x_strides: &[u32],+ #[allow(unused)] dt_strides: &[u32],+ #[allow(unused)] cb_strides: &[u32],+ #[allow(unused)] state_strides: &[u32],+ #[allow(unused)] num_heads: u32,+ #[allow(unused)] head_dim: u32,+) {+ todo!()+}++#[kernel(SSDPrefill)]+#[variants(T, f32, f16, bf16)]+pub fn ssd_prefill<T: ArrayElement + Float>(+ #[allow(unused)] x: *const T,+ #[allow(unused)] dt_raw: *const T,+ #[allow(unused)] b: *const T,+ #[allow(unused)] c: *const T,+ #[allow(unused)] d: *const T,+ #[allow(unused)] z: *const T,+ #[allow(unused)] state: *mut T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] suffix_len: u32,+ #[allow(unused)] group_size: i32,+ #[allow(unused)] state_size: i32,+ #[allow(unused)] x_strides: &[u32],+ #[allow(unused)] dt_strides: &[u32],+ #[allow(unused)] cb_strides: &[u32],+ #[allow(unused)] state_strides: &[u32],+ #[allow(unused)] num_heads: u32,+ #[allow(unused)] head_dim: u32,+) {+ todo!()+}++#[kernel(SSDPrefillSequential)]+#[variants(T, f32, f16, bf16)]+pub fn ssd_prefill_sequential<T: ArrayElement + Float>(+ #[allow(unused)] x: *const T,+ #[allow(unused)] dt_raw: *const T,+ #[allow(unused)] b: *const T,+ #[allow(unused)] c: *const T,+ #[allow(unused)] d: *const T,+ #[allow(unused)] z: *const T,+ #[allow(unused)] state: *mut T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] suffix_len: u32,+ #[allow(unused)] group_size: i32,+ #[allow(unused)] state_size: i32,+ #[allow(unused)] x_strides: &[u32],+ #[allow(unused)] dt_strides: &[u32],+ #[allow(unused)] cb_strides: &[u32],+ #[allow(unused)] state_strides: &[u32],+ #[allow(unused)] channels: u32,+ #[allow(unused)] head_dim: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rsnew file mode 100644index 000000000..b888db629--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs@@ -0,0 +1,30 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(SSDUpdate)]+#[variants(T, f32, f16, bf16)]+pub fn ssd_update<T: ArrayElement + Float>(+ #[allow(unused)] x: *const T,+ #[allow(unused)] dt_raw: *const T,+ #[allow(unused)] b: *const T,+ #[allow(unused)] c: *const T,+ #[allow(unused)] d: *const T,+ #[allow(unused)] z: *const T,+ #[allow(unused)] state: *const T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] next_state: *mut T,+ #[allow(unused)] group_size: u32,+ #[allow(unused)] state_size: u32,+ #[allow(unused)] x_strides: &[u32],+ #[allow(unused)] dt_strides: &[u32],+ #[allow(unused)] cb_strides: &[u32],+ #[allow(unused)] state_strides: &[u32],+ #[allow(unused)] b_size: u32,+ #[allow(unused)] h_size: u32,+ #[allow(unused)] dh_size: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/mod.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/mod.rsnew file mode 100644index 000000000..f8e961796--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/mod.rs@@ -0,0 +1 @@+pub mod tensor_add_bias;diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rsnew file mode 100644index 000000000..c44c9c813--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs@@ -0,0 +1,17 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(TensorAddBias)]+#[variants(T, f32, f16, bf16)]+pub fn tensor_add_bias<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] num_cols: u32,+ #[allow(unused)] length: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/mod.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/mod.rsnew file mode 100644index 000000000..a429b0db3--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/mod.rs@@ -0,0 +1 @@+pub mod tensor_add_swap;diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rsnew file mode 100644index 000000000..c4f234600--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs@@ -0,0 +1,15 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(TensorAddSwap)]+#[variants(T, f32, f16, bf16)]+pub fn tensor_add_swap<T: ArrayElement + Float>(+ #[allow(unused)] skip_buffer: *mut T,+ #[allow(unused)] main_buffer: *mut T,+ #[allow(unused)] length: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_copy/mod.rs b/crates/uzu/src/backends/cpu/kernel/tensor_copy/mod.rsnew file mode 100644index 000000000..a61a2ebd2--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/tensor_copy/mod.rs@@ -0,0 +1 @@+pub mod tensor_copy;diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs b/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rsnew file mode 100644index 000000000..c2e19054a--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs@@ -0,0 +1,15 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(TensorCopy)]+#[variants(T, f32, f16, bf16)]+pub fn tensor_copy<T: ArrayElement + Float>(+ #[allow(unused)] src_buffer: *const T,+ #[allow(unused)] dst_buffer: *mut T,+ #[allow(unused)] length: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/token_copy/mod.rs b/crates/uzu/src/backends/cpu/kernel/token_copy/mod.rsnew file mode 100644index 000000000..90375d281--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/token_copy/mod.rs@@ -0,0 +1 @@+pub mod token_copy;diff --git a/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs b/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rsnew file mode 100644index 000000000..9da03daaa--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs@@ -0,0 +1,20 @@+use dsl::kernel;+use num_traits::Float;++use crate::ArrayElement;++#[kernel(TokenCopySampled)]+pub fn token_copy_sampled(+ #[allow(unused)] src: *const u32,+ #[allow(unused)] dst: *mut u64,+) {+ todo!()+}++#[kernel(TokenCopyToResults)]+pub fn token_copy_to_results(+ #[allow(unused)] src: *const u32,+ #[allow(unused)] dst: *mut u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/mod.rs b/crates/uzu/src/backends/cpu/mod.rsindex 5d259bb5f..bd15afca5 100644--- a/crates/uzu/src/backends/cpu/mod.rs+++ b/crates/uzu/src/backends/cpu/mod.rs@@ -1 +1,11 @@+mod backend;+mod command_buffer;+mod context;+mod error;+mod event;+mod kernel;+mod native_buffer;++pub use backend::Cpu;+pub mod argmax;diff --git a/crates/uzu/src/backends/cpu/native_buffer.rs b/crates/uzu/src/backends/cpu/native_buffer.rsnew file mode 100644index 000000000..a885455be--- /dev/null+++ b/crates/uzu/src/backends/cpu/native_buffer.rs@@ -0,0 +1,24 @@+use std::{os::raw::c_void, ptr::NonNull};++use super::Cpu;+use crate::backends::common::NativeBuffer;++impl NativeBuffer for Box<[u8]> {+ type Backend = Cpu;++ fn set_label(+ &mut self,+ _label: Option<&str>,+ ) {+ }++ fn cpu_ptr(&self) -> NonNull<c_void> {+ unsafe { NonNull::new_unchecked(self.as_ptr() as *mut c_void) }+ }+ fn length(&self) -> usize {+ self.len()+ }+ fn id(&self) -> usize {+ unimplemented!()+ }+}diff --git a/crates/uzu/src/backends/mod.rs b/crates/uzu/src/backends/mod.rsindex 5560db88f..d123fda17 100644--- a/crates/uzu/src/backends/mod.rs+++ b/crates/uzu/src/backends/mod.rs@@ -7,6 +7,10 @@ pub mod metal;macro_rules! select_backend {($expr:expr, $unk:expr) => {{match std::env::var("UZU_BACKEND").map(|s| s.to_lowercase()).as_deref().unwrap_or("metal") {+ "cpu" => {+ type B = crate::backends::cpu::Cpu;+ $expr+ },#[cfg(feature = "metal")]"metal" => {type B = crate::backends::metal::Metal;