A high-performance inference engine for AI models
Move ugly BufferArg into it's own mod (#584)
uuuvndbaa32616b45
diff --git a/crates/backend-uzu/build/common/traitgen.rs b/crates/backend-uzu/build/common/traitgen.rsindex a27b4c753..9acdab424 100644--- a/crates/backend-uzu/build/common/traitgen.rs+++ b/crates/backend-uzu/build/common/traitgen.rs@@ -41,9 +41,11 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {(Some(quote! { #buffer_lifetime }),match access {- KernelBufferAccess::Read => quote! { impl BufferArg<#buffer_lifetime, Self::Backend> },+ KernelBufferAccess::Read => {+ quote! { impl crate::backends::common::BufferArg<#buffer_lifetime, Self::Backend> }+ },KernelBufferAccess::ReadWrite => {- quote! { impl BufferArgMut<#buffer_lifetime, Self::Backend> }+ quote! { impl crate::backends::common::BufferArgMut<#buffer_lifetime, Self::Backend> }},},)diff --git a/crates/backend-uzu/build/cpu/compiler.rs b/crates/backend-uzu/build/cpu/compiler.rsindex 5fa114203..bb76257b7 100644--- a/crates/backend-uzu/build/cpu/compiler.rs+++ b/crates/backend-uzu/build/cpu/compiler.rs@@ -373,10 +373,10 @@ impl CpuCompiler {Some(quote! { #buffer_lifetime }),match access {KernelBufferAccess::Read => {- quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, crate::backends::cpu::Cpu> }+ quote! { impl crate::backends::common::BufferArg<#buffer_lifetime, crate::backends::cpu::Cpu> }},KernelBufferAccess::ReadWrite => {- quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, crate::backends::cpu::Cpu> }+ quote! { impl crate::backends::common::BufferArgMut<#buffer_lifetime, crate::backends::cpu::Cpu> }},},)diff --git a/crates/backend-uzu/build/metal/bindgen/arguments.rs b/crates/backend-uzu/build/metal/bindgen/arguments.rsindex 985f2ab01..cd961c42f 100644--- a/crates/backend-uzu/build/metal/bindgen/arguments.rs+++ b/crates/backend-uzu/build/metal/bindgen/arguments.rs@@ -219,7 +219,7 @@ impl ArgumentEmission {ArgumentEmission::Constant(constant) => Some(emit_constant_argument_definition(constant)),ArgumentEmission::Shared(_) => None,ArgumentEmission::IndirectDispatch(_) => Some(quote! {- __dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<+ __dsl_indirect_dispatch_buffer: impl crate::backends::common::BufferArg<'__dsl_indirect_dispatch_buffer, crate::backends::metal::Metal>}),@@ -292,8 +292,8 @@ fn emit_buffer_argument_definition(buffer: &BufferArgument) -> TokenStream {let name = &buffer.name;let lifetime = &buffer.lifetime;let trait_path = match buffer.access {- MetalBufferAccess::Read => quote! { crate::backends::common::kernel::BufferArg },- MetalBufferAccess::ReadWrite => quote! { crate::backends::common::kernel::BufferArgMut },+ MetalBufferAccess::Read => quote! { crate::backends::common::BufferArg },+ MetalBufferAccess::ReadWrite => quote! { crate::backends::common::BufferArgMut },};let buffer_argument_type = quote! { impl #trait_path<#lifetime, crate::backends::metal::Metal> };if buffer.condition.is_some() {diff --git a/crates/backend-uzu/src/backends/common/buffer/arg.rs b/crates/backend-uzu/src/backends/common/buffer/arg.rsnew file mode 100644index 000000000..ad56f62b0--- /dev/null+++ b/crates/backend-uzu/src/backends/common/buffer/arg.rs@@ -0,0 +1,52 @@+use crate::backends::common::{AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer};++// TODO: This whole thing is horrible and should be unified with AsBufferRangeRef+pub trait BufferArg<'a, B: Backend>: Copy {+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize);+}++impl<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = B>>> BufferArg<'a, B> for &'a T {+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {+ let buffer_range = self.as_buffer_range_ref();+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());+ (buffer, range.start, range.end - range.start)+ }+}++impl<'a, B: Backend> BufferArg<'a, B> for &'a dyn Buffer<Backend = B> {+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {+ (self, 0, self.size())+ }+}++impl<'a, B: Backend, T: BufferArg<'a, B>> BufferArg<'a, B> for (T, usize) {+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {+ let (buffer, offset, length) = self.0.into_parts();+ (buffer, offset + self.1, length - self.1)+ }+}++pub trait BufferArgMut<'a, B: Backend> {+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize);+}++impl<'a, B: Backend, T: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> BufferArgMut<'a, B> for &'a mut T {+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {+ let buffer_range = self.as_buffer_range_mut();+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());+ (buffer, range.start, range.end - range.start)+ }+}++impl<'a, B: Backend> BufferArgMut<'a, B> for &'a mut dyn Buffer<Backend = B> {+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {+ (self, 0, self.size())+ }+}++impl<'a, B: Backend, T: BufferArgMut<'a, B>> BufferArgMut<'a, B> for (T, usize) {+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {+ let (buffer, offset, length) = self.0.into_parts();+ (buffer, offset + self.1, length - self.1)+ }+}diff --git a/crates/backend-uzu/src/backends/common/buffer/mod.rs b/crates/backend-uzu/src/backends/common/buffer/mod.rsindex 99d0c21fa..b9130c339 100644--- a/crates/backend-uzu/src/backends/common/buffer/mod.rs+++ b/crates/backend-uzu/src/backends/common/buffer/mod.rs@@ -5,6 +5,7 @@ use crate::backends::common::{AsBufferRangeMut, AsBufferRangeRef, Backend, Buffepub mod dense;pub mod sparse;+pub mod arg;pub mod range;pub trait Buffer: Any + Debug {diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rsindex 594d54c2c..30ce849fc 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs@@ -1,5 +1,5 @@use super::{d_ops::MatmulDOps, matmul_b::MatmulB};-use crate::backends::common::{Allocation, Backend, kernel::BufferArg};+use crate::backends::common::{Allocation, Backend, BufferArg};pub struct MatmulArguments<'a, 'b, 'd, B: Backend, TB: BufferArg<'b, B> = &'b Allocation<B>> {pub a: &'a Allocation<B>,diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rsindex 844d845c6..7e02d73d4 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs@@ -1,8 +1,5 @@use crate::{- backends::common::{- Backend, Encoder, Kernels,- kernel::{BufferArg, matmul::arguments::MatmulArguments},- },+ backends::common::{Backend, BufferArg, Encoder, Kernels, kernel::matmul::arguments::MatmulArguments},data_type::DataType,};diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rsindex a795d09d8..6efb603ba 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs@@ -1,8 +1,7 @@use crate::{backends::common::{- Allocation, Backend,+ Allocation, Backend, BufferArg,gpu_types::{QuantizationMode, gemm::GemmBPrologueKind},- kernel::BufferArg,},data_type::DataType,};diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rsindex b4bf315b2..0bd082fc3 100644--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs@@ -1,58 +1,7 @@-use crate::backends::common::{AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer};+use crate::backends::common::Backend;pub mod matmul;-// TODO: This whole thing is horrible and should be unified with AsBufferRangeRef-pub trait BufferArg<'a, B: Backend>: Copy {- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize);-}--impl<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = B>>> BufferArg<'a, B> for &'a T {- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {- let buffer_range = self.as_buffer_range_ref();- let (buffer, range) = (buffer_range.buffer(), buffer_range.range());- (buffer, range.start, range.end - range.start)- }-}--impl<'a, B: Backend> BufferArg<'a, B> for &'a dyn Buffer<Backend = B> {- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {- (self, 0, self.size())- }-}--impl<'a, B: Backend, T: BufferArg<'a, B>> BufferArg<'a, B> for (T, usize) {- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {- let (buffer, offset, length) = self.0.into_parts();- (buffer, offset + self.1, length - self.1)- }-}--pub trait BufferArgMut<'a, B: Backend> {- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize);-}--impl<'a, B: Backend, T: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> BufferArgMut<'a, B> for &'a mut T {- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {- let buffer_range = self.as_buffer_range_mut();- let (buffer, range) = (buffer_range.buffer(), buffer_range.range());- (buffer, range.start, range.end - range.start)- }-}--impl<'a, B: Backend> BufferArgMut<'a, B> for &'a mut dyn Buffer<Backend = B> {- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {- (self, 0, self.size())- }-}--impl<'a, B: Backend, T: BufferArgMut<'a, B>> BufferArgMut<'a, B> for (T, usize) {- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {- let (buffer, offset, length) = self.0.into_parts();- (buffer, offset + self.1, length - self.1)- }-}-include!(concat!(env!("OUT_DIR"), "/traits.rs"));pub trait Kernels: Sized {diff --git a/crates/backend-uzu/src/backends/common/mod.rs b/crates/backend-uzu/src/backends/common/mod.rsindex 6345139b2..541870438 100644--- a/crates/backend-uzu/src/backends/common/mod.rs+++ b/crates/backend-uzu/src/backends/common/mod.rs@@ -12,6 +12,7 @@ pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};pub use backend::Backend;pub use buffer::{Buffer, BufferGpuAddressRangeExt,+ arg::{BufferArg, BufferArgMut},dense::DenseBuffer,range::{AsBufferRangeMut, AsBufferRangeRef, BufferRangeMut, BufferRangeRef},sparse::{SparseBuffer, SparseBufferExt},diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rsindex 734dca683..bfd00c048 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs@@ -2,10 +2,10 @@ use super::reference::{WeightData, read_f32, write_f32};use crate::{backends::{common::{- AsBufferRangeMut, AsBufferRangeRef, Backend, Encoder, Kernels,+ AsBufferRangeMut, AsBufferRangeRef, Backend, BufferArg, Encoder, Kernels,gpu_types::HadamardTransformOrder,kernel::{- BufferArg, HadamardTransformKernel,+ HadamardTransformKernel,matmul::{MatmulArguments, MatmulError, MatmulKernel},},},diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rsindex b570d4c82..686ce50db 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs@@ -2,11 +2,7 @@ use half::{bf16, f16};use crate::{backends::{- common::{- AsBufferRangeRef,- gpu_types::QuantizationMode,- kernel::{BufferArg, matmul::MatmulB},- },+ common::{AsBufferRangeRef, BufferArg, gpu_types::QuantizationMode, kernel::matmul::MatmulB},cpu::Cpu,},data_type::DataType,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex a14a5e1b4..92a479edc 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -4,13 +4,13 @@ use super::specialization::GemmSpecialization;use crate::{backends::{common::{- Allocation, Backend, Encoder,+ Allocation, Backend, BufferArg, Encoder,gpu_types::{GemmParams, HadamardTransformOrder,gemm::{GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},},kernel::{- BufferArg, HadamardTransformKernel, Kernels, TensorAddBiasKernel,+ HadamardTransformKernel, Kernels, TensorAddBiasKernel,matmul::{MatmulArguments, MatmulB, MatmulError},},},diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 0e847cfaa..fdaf77b73 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -7,12 +7,9 @@ use super::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK};use crate::{backends::{common::{- Allocation, Encoder,+ Allocation, BufferArg, Encoder,gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},- kernel::{- BufferArg,- matmul::{MatmulArguments, MatmulB, MatmulError},- },+ kernel::matmul::{MatmulArguments, MatmulB, MatmulError},},metal::{Metal, context::MetalContext, device_tier::DeviceTier, kernel::GemvMetalKernel},},diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex 27afb618f..574e1afaa 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -6,11 +6,8 @@ use self::gemv::{GemvDispatch, GemvSpecialization};use crate::{backends::{common::{- Encoder,- kernel::{- BufferArg,- matmul::{MatmulArguments, MatmulError, MatmulKernel},- },+ BufferArg, Encoder,+ kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},},metal::{Metal, context::MetalContext, error::MetalError, metal_extensions::DeviceExt},},diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rsindex a6f7691c4..ef05bcc7a 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs@@ -3,9 +3,9 @@ use std::cell::RefCell;use crate::{array::size_for_shape,backends::common::{- Allocation, Backend, Encoder, Kernels,+ Allocation, Backend, BufferArg, Encoder, Kernels,kernel::{- AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, BufferArg, SoftmaxKernel,+ AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, SoftmaxKernel,matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/gemm.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/gemm.rsindex bbf6dfc33..6646415ca 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/gemm.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/gemm.rs@@ -5,9 +5,7 @@ use bitflags::bitflags;use crate::{array::size_for_shape,backends::common::{- Allocation, Backend, Encoder, Kernels,- gpu_types::AttnParams,- kernel::{AttentionGemmKernel, BufferArg},+ Allocation, Backend, BufferArg, Encoder, Kernels, gpu_types::AttnParams, kernel::AttentionGemmKernel,},data_type::DataType,encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rsindex b741ebadc..4e4c8db8a 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs@@ -1,5 +1,5 @@use crate::{- backends::common::{Allocation, Backend, Context, Encoder, kernel::BufferArg},+ backends::common::{Allocation, Backend, BufferArg, Context, Encoder},data_type::DataType,encodable_block::mixer::attention::{core::{diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rsindex f35ee4d97..710fbaf0d 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs@@ -1,9 +1,6 @@use crate::{array::size_for_shape,- backends::common::{- Allocation, Backend, Encoder, Kernels,- kernel::{AttentionSinglePassKernel, BufferArg},- },+ backends::common::{Allocation, Backend, BufferArg, Encoder, Kernels, kernel::AttentionSinglePassKernel},data_type::DataType,encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},};diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rsindex 0141ed0e4..bc905558b 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs@@ -1,8 +1,8 @@use crate::{array::size_for_shape,backends::common::{- Allocation, Backend, Encoder, Kernels,- kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, BufferArg},+ Allocation, Backend, BufferArg, Encoder, Kernels,+ kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},},data_type::DataType,encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rsindex 759c43cdb..c5ca02ae3 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs@@ -1,9 +1,9 @@use crate::{array::size_for_shape,backends::common::{- Allocation, Backend, Encoder, Kernels,+ Allocation, Backend, BufferArgMut, Encoder, Kernels,gpu_types::trie::TrieNode,- kernel::{AttentionPrepareKernel, BufferArgMut, SigmoidGateKernel},+ kernel::{AttentionPrepareKernel, SigmoidGateKernel},},encodable_block::{batch_topology::BatchTopology,