Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Move ugly BufferArg into it's own mod (#584)

uuuvndbaa32616b45
diff --git a/crates/backend-uzu/build/common/traitgen.rs b/crates/backend-uzu/build/common/traitgen.rs
index a27b4c753..9acdab424 100644
--- a/crates/backend-uzu/build/common/traitgen.rs
+++ b/crates/backend-uzu/build/common/traitgen.rs
@@ -41,9 +41,11 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
(
Some(quote! { #buffer_lifetime }),
match access {
- KernelBufferAccess::Read => quote! { impl BufferArg<#buffer_lifetime, Self::Backend> },
+ KernelBufferAccess::Read => {
+ quote! { impl crate::backends::common::BufferArg<#buffer_lifetime, Self::Backend> }
+ },
KernelBufferAccess::ReadWrite => {
- quote! { impl BufferArgMut<#buffer_lifetime, Self::Backend> }
+ quote! { impl crate::backends::common::BufferArgMut<#buffer_lifetime, Self::Backend> }
},
},
)
diff --git a/crates/backend-uzu/build/cpu/compiler.rs b/crates/backend-uzu/build/cpu/compiler.rs
index 5fa114203..bb76257b7 100644
--- a/crates/backend-uzu/build/cpu/compiler.rs
+++ b/crates/backend-uzu/build/cpu/compiler.rs
@@ -373,10 +373,10 @@ impl CpuCompiler {
Some(quote! { #buffer_lifetime }),
match access {
KernelBufferAccess::Read => {
- quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, crate::backends::cpu::Cpu> }
+ quote! { impl crate::backends::common::BufferArg<#buffer_lifetime, crate::backends::cpu::Cpu> }
},
KernelBufferAccess::ReadWrite => {
- quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, crate::backends::cpu::Cpu> }
+ quote! { impl crate::backends::common::BufferArgMut<#buffer_lifetime, crate::backends::cpu::Cpu> }
},
},
)
diff --git a/crates/backend-uzu/build/metal/bindgen/arguments.rs b/crates/backend-uzu/build/metal/bindgen/arguments.rs
index 985f2ab01..cd961c42f 100644
--- a/crates/backend-uzu/build/metal/bindgen/arguments.rs
+++ b/crates/backend-uzu/build/metal/bindgen/arguments.rs
@@ -219,7 +219,7 @@ impl ArgumentEmission {
ArgumentEmission::Constant(constant) => Some(emit_constant_argument_definition(constant)),
ArgumentEmission::Shared(_) => None,
ArgumentEmission::IndirectDispatch(_) => Some(quote! {
- __dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<
+ __dsl_indirect_dispatch_buffer: impl crate::backends::common::BufferArg<
'__dsl_indirect_dispatch_buffer, crate::backends::metal::Metal
>
}),
@@ -292,8 +292,8 @@ fn emit_buffer_argument_definition(buffer: &BufferArgument) -> TokenStream {
let name = &buffer.name;
let lifetime = &buffer.lifetime;
let trait_path = match buffer.access {
- MetalBufferAccess::Read => quote! { crate::backends::common::kernel::BufferArg },
- MetalBufferAccess::ReadWrite => quote! { crate::backends::common::kernel::BufferArgMut },
+ MetalBufferAccess::Read => quote! { crate::backends::common::BufferArg },
+ MetalBufferAccess::ReadWrite => quote! { crate::backends::common::BufferArgMut },
};
let buffer_argument_type = quote! { impl #trait_path<#lifetime, crate::backends::metal::Metal> };
if buffer.condition.is_some() {
diff --git a/crates/backend-uzu/src/backends/common/buffer/arg.rs b/crates/backend-uzu/src/backends/common/buffer/arg.rs
new file mode 100644
index 000000000..ad56f62b0
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/buffer/arg.rs
@@ -0,0 +1,52 @@
+use crate::backends::common::{AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer};
+
+// TODO: This whole thing is horrible and should be unified with AsBufferRangeRef
+pub trait BufferArg<'a, B: Backend>: Copy {
+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize);
+}
+
+impl<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = B>>> BufferArg<'a, B> for &'a T {
+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
+ let buffer_range = self.as_buffer_range_ref();
+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
+ (buffer, range.start, range.end - range.start)
+ }
+}
+
+impl<'a, B: Backend> BufferArg<'a, B> for &'a dyn Buffer<Backend = B> {
+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
+ (self, 0, self.size())
+ }
+}
+
+impl<'a, B: Backend, T: BufferArg<'a, B>> BufferArg<'a, B> for (T, usize) {
+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
+ let (buffer, offset, length) = self.0.into_parts();
+ (buffer, offset + self.1, length - self.1)
+ }
+}
+
+pub trait BufferArgMut<'a, B: Backend> {
+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize);
+}
+
+impl<'a, B: Backend, T: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> BufferArgMut<'a, B> for &'a mut T {
+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
+ let buffer_range = self.as_buffer_range_mut();
+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
+ (buffer, range.start, range.end - range.start)
+ }
+}
+
+impl<'a, B: Backend> BufferArgMut<'a, B> for &'a mut dyn Buffer<Backend = B> {
+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
+ (self, 0, self.size())
+ }
+}
+
+impl<'a, B: Backend, T: BufferArgMut<'a, B>> BufferArgMut<'a, B> for (T, usize) {
+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
+ let (buffer, offset, length) = self.0.into_parts();
+ (buffer, offset + self.1, length - self.1)
+ }
+}
diff --git a/crates/backend-uzu/src/backends/common/buffer/mod.rs b/crates/backend-uzu/src/backends/common/buffer/mod.rs
index 99d0c21fa..b9130c339 100644
--- a/crates/backend-uzu/src/backends/common/buffer/mod.rs
+++ b/crates/backend-uzu/src/backends/common/buffer/mod.rs
@@ -5,6 +5,7 @@ use crate::backends::common::{AsBufferRangeMut, AsBufferRangeRef, Backend, Buffe
pub mod dense;
pub mod sparse;
+pub mod arg;
pub mod range;
pub trait Buffer: Any + Debug {
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs
index 594d54c2c..30ce849fc 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/arguments.rs
@@ -1,5 +1,5 @@
use super::{d_ops::MatmulDOps, matmul_b::MatmulB};
-use crate::backends::common::{Allocation, Backend, kernel::BufferArg};
+use crate::backends::common::{Allocation, Backend, BufferArg};
pub struct MatmulArguments<'a, 'b, 'd, B: Backend, TB: BufferArg<'b, B> = &'b Allocation<B>> {
pub a: &'a Allocation<B>,
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
index 844d845c6..7e02d73d4 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
@@ -1,8 +1,5 @@
use crate::{
- backends::common::{
- Backend, Encoder, Kernels,
- kernel::{BufferArg, matmul::arguments::MatmulArguments},
- },
+ backends::common::{Backend, BufferArg, Encoder, Kernels, kernel::matmul::arguments::MatmulArguments},
data_type::DataType,
};
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs
index a795d09d8..6efb603ba 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs
@@ -1,8 +1,7 @@
use crate::{
backends::common::{
- Allocation, Backend,
+ Allocation, Backend, BufferArg,
gpu_types::{QuantizationMode, gemm::GemmBPrologueKind},
- kernel::BufferArg,
},
data_type::DataType,
};
diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rs
index b4bf315b2..0bd082fc3 100644
--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs
@@ -1,58 +1,7 @@
-use crate::backends::common::{AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer};
+use crate::backends::common::Backend;
pub mod matmul;
-// TODO: This whole thing is horrible and should be unified with AsBufferRangeRef
-pub trait BufferArg<'a, B: Backend>: Copy {
- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize);
-}
-
-impl<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = B>>> BufferArg<'a, B> for &'a T {
- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
- let buffer_range = self.as_buffer_range_ref();
- let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
- (buffer, range.start, range.end - range.start)
- }
-}
-
-impl<'a, B: Backend> BufferArg<'a, B> for &'a dyn Buffer<Backend = B> {
- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
- (self, 0, self.size())
- }
-}
-
-impl<'a, B: Backend, T: BufferArg<'a, B>> BufferArg<'a, B> for (T, usize) {
- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
- let (buffer, offset, length) = self.0.into_parts();
- (buffer, offset + self.1, length - self.1)
- }
-}
-
-pub trait BufferArgMut<'a, B: Backend> {
- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize);
-}
-
-impl<'a, B: Backend, T: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> BufferArgMut<'a, B> for &'a mut T {
- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
- let buffer_range = self.as_buffer_range_mut();
- let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
- (buffer, range.start, range.end - range.start)
- }
-}
-
-impl<'a, B: Backend> BufferArgMut<'a, B> for &'a mut dyn Buffer<Backend = B> {
- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
- (self, 0, self.size())
- }
-}
-
-impl<'a, B: Backend, T: BufferArgMut<'a, B>> BufferArgMut<'a, B> for (T, usize) {
- fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
- let (buffer, offset, length) = self.0.into_parts();
- (buffer, offset + self.1, length - self.1)
- }
-}
-
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
pub trait Kernels: Sized {
diff --git a/crates/backend-uzu/src/backends/common/mod.rs b/crates/backend-uzu/src/backends/common/mod.rs
index 6345139b2..541870438 100644
--- a/crates/backend-uzu/src/backends/common/mod.rs
+++ b/crates/backend-uzu/src/backends/common/mod.rs
@@ -12,6 +12,7 @@ pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};
pub use backend::Backend;
pub use buffer::{
Buffer, BufferGpuAddressRangeExt,
+ arg::{BufferArg, BufferArgMut},
dense::DenseBuffer,
range::{AsBufferRangeMut, AsBufferRangeRef, BufferRangeMut, BufferRangeRef},
sparse::{SparseBuffer, SparseBufferExt},
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs
index 734dca683..bfd00c048 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs
@@ -2,10 +2,10 @@ use super::reference::{WeightData, read_f32, write_f32};
use crate::{
backends::{
common::{
- AsBufferRangeMut, AsBufferRangeRef, Backend, Encoder, Kernels,
+ AsBufferRangeMut, AsBufferRangeRef, Backend, BufferArg, Encoder, Kernels,
gpu_types::HadamardTransformOrder,
kernel::{
- BufferArg, HadamardTransformKernel,
+ HadamardTransformKernel,
matmul::{MatmulArguments, MatmulError, MatmulKernel},
},
},
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs
index b570d4c82..686ce50db 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs
@@ -2,11 +2,7 @@ use half::{bf16, f16};
use crate::{
backends::{
- common::{
- AsBufferRangeRef,
- gpu_types::QuantizationMode,
- kernel::{BufferArg, matmul::MatmulB},
- },
+ common::{AsBufferRangeRef, BufferArg, gpu_types::QuantizationMode, kernel::matmul::MatmulB},
cpu::Cpu,
},
data_type::DataType,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index a14a5e1b4..92a479edc 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -4,13 +4,13 @@ use super::specialization::GemmSpecialization;
use crate::{
backends::{
common::{
- Allocation, Backend, Encoder,
+ Allocation, Backend, BufferArg, Encoder,
gpu_types::{
GemmParams, HadamardTransformOrder,
gemm::{GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},
},
kernel::{
- BufferArg, HadamardTransformKernel, Kernels, TensorAddBiasKernel,
+ HadamardTransformKernel, Kernels, TensorAddBiasKernel,
matmul::{MatmulArguments, MatmulB, MatmulError},
},
},
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 0e847cfaa..fdaf77b73 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -7,12 +7,9 @@ use super::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK};
use crate::{
backends::{
common::{
- Allocation, Encoder,
+ Allocation, BufferArg, Encoder,
gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
- kernel::{
- BufferArg,
- matmul::{MatmulArguments, MatmulB, MatmulError},
- },
+ kernel::matmul::{MatmulArguments, MatmulB, MatmulError},
},
metal::{Metal, context::MetalContext, device_tier::DeviceTier, kernel::GemvMetalKernel},
},
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index 27afb618f..574e1afaa 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -6,11 +6,8 @@ use self::gemv::{GemvDispatch, GemvSpecialization};
use crate::{
backends::{
common::{
- Encoder,
- kernel::{
- BufferArg,
- matmul::{MatmulArguments, MatmulError, MatmulKernel},
- },
+ BufferArg, Encoder,
+ kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},
},
metal::{Metal, context::MetalContext, error::MetalError, metal_extensions::DeviceExt},
},
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs
index a6f7691c4..ef05bcc7a 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs
@@ -3,9 +3,9 @@ use std::cell::RefCell;
use crate::{
array::size_for_shape,
backends::common::{
- Allocation, Backend, Encoder, Kernels,
+ Allocation, Backend, BufferArg, Encoder, Kernels,
kernel::{
- AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, BufferArg, SoftmaxKernel,
+ AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, SoftmaxKernel,
matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/gemm.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/gemm.rs
index bbf6dfc33..6646415ca 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/gemm.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/gemm.rs
@@ -5,9 +5,7 @@ use bitflags::bitflags;
use crate::{
array::size_for_shape,
backends::common::{
- Allocation, Backend, Encoder, Kernels,
- gpu_types::AttnParams,
- kernel::{AttentionGemmKernel, BufferArg},
+ Allocation, Backend, BufferArg, Encoder, Kernels, gpu_types::AttnParams, kernel::AttentionGemmKernel,
},
data_type::DataType,
encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs
index b741ebadc..4e4c8db8a 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs
@@ -1,5 +1,5 @@
use crate::{
- backends::common::{Allocation, Backend, Context, Encoder, kernel::BufferArg},
+ backends::common::{Allocation, Backend, BufferArg, Context, Encoder},
data_type::DataType,
encodable_block::mixer::attention::{
core::{
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs
index f35ee4d97..710fbaf0d 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs
@@ -1,9 +1,6 @@
use crate::{
array::size_for_shape,
- backends::common::{
- Allocation, Backend, Encoder, Kernels,
- kernel::{AttentionSinglePassKernel, BufferArg},
- },
+ backends::common::{Allocation, Backend, BufferArg, Encoder, Kernels, kernel::AttentionSinglePassKernel},
data_type::DataType,
encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
};
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs
index 0141ed0e4..bc905558b 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs
@@ -1,8 +1,8 @@
use crate::{
array::size_for_shape,
backends::common::{
- Allocation, Backend, Encoder, Kernels,
- kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, BufferArg},
+ Allocation, Backend, BufferArg, Encoder, Kernels,
+ kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},
},
data_type::DataType,
encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs
index 759c43cdb..c5ca02ae3 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs
@@ -1,9 +1,9 @@
use crate::{
array::size_for_shape,
backends::common::{
- Allocation, Backend, Encoder, Kernels,
+ Allocation, Backend, BufferArgMut, Encoder, Kernels,
gpu_types::trie::TrieNode,
- kernel::{AttentionPrepareKernel, BufferArgMut, SigmoidGateKernel},
+ kernel::{AttentionPrepareKernel, SigmoidGateKernel},
},
encodable_block::{
batch_topology::BatchTopology,