A high-performance inference engine for AI models
Tighten allocator alignment and granularity (#488)
uuuvn18ccba93e6f7
diff --git a/crates/backend-uzu/src/backends/common/allocator/allocator.rs b/crates/backend-uzu/src/backends/common/allocator/allocator.rsindex 75836fc0a..d317b6cdf 100644--- a/crates/backend-uzu/src/backends/common/allocator/allocator.rs+++ b/crates/backend-uzu/src/backends/common/allocator/allocator.rs@@ -105,7 +105,8 @@ impl<B: Backend> Allocator<B> {allocation_type: AllocationType<B>,) -> Result<Allocation<B>, B::Error> {assert!(size > 0, "allocation size must be greater than 0");- let alignment = usize::clamp(size.next_power_of_two(), B::MIN_ALLOCATION_ALIGNMENT, 16_384);+ let alignment =+ usize::clamp(size.next_power_of_two(), B::MIN_ALLOCATION_ALIGNMENT, B::MAX_ALLOCATION_ALIGNMENT);let allocation_type = match allocation_type {AllocationType::Global => RangeAllocationType::Global,AllocationType::Pooled {@@ -132,7 +133,7 @@ impl<B: Backend> Allocator<B> {(buffer, range)} else {- let new_allocator_buffer_size = usize::max(size, 268_435_456);+ let new_allocator_buffer_size = usize::max(size, B::ALLOCATION_GRANULARITY);let mut allocator_buffer = AllocatorBuffer::<B> {buffer: Box::pin(self.context.upgrade().unwrap().create_buffer(new_allocator_buffer_size)?), // Upgrade can never faildiff --git a/crates/backend-uzu/src/backends/common/backend.rs b/crates/backend-uzu/src/backends/common/backend.rsindex bd6a4c1fd..70ea34230 100644--- a/crates/backend-uzu/src/backends/common/backend.rs+++ b/crates/backend-uzu/src/backends/common/backend.rs@@ -11,5 +11,7 @@ pub trait Backend: Debug + Clone + 'static {type Error: Error + Debug;const MIN_ALLOCATION_ALIGNMENT: usize;+ const MAX_ALLOCATION_ALIGNMENT: usize;+ const ALLOCATION_GRANULARITY: usize;const MAX_INLINE_BYTES: usize;}diff --git a/crates/backend-uzu/src/backends/cpu/backend.rs b/crates/backend-uzu/src/backends/cpu/backend.rsindex d8f96544a..71e654465 100644--- a/crates/backend-uzu/src/backends/cpu/backend.rs+++ b/crates/backend-uzu/src/backends/cpu/backend.rs@@ -16,6 +16,8 @@ impl Backend for Cpu {type Kernels = CpuKernels;type Error = CpuError;- const MIN_ALLOCATION_ALIGNMENT: usize = 64;+ const MIN_ALLOCATION_ALIGNMENT: usize = 4;+ const MAX_ALLOCATION_ALIGNMENT: usize = 64;+ const ALLOCATION_GRANULARITY: usize = 8 * 1024 * 1024;const MAX_INLINE_BYTES: usize = usize::MAX;}diff --git a/crates/backend-uzu/src/backends/metal/backend.rs b/crates/backend-uzu/src/backends/metal/backend.rsindex 5dd2a1da9..fe777ae3f 100644--- a/crates/backend-uzu/src/backends/metal/backend.rs+++ b/crates/backend-uzu/src/backends/metal/backend.rs@@ -15,7 +15,9 @@ impl Backend for Metal {type Kernels = MetalKernels;type Error = MetalError;- const MIN_ALLOCATION_ALIGNMENT: usize = 64;+ const MIN_ALLOCATION_ALIGNMENT: usize = 4;+ const MAX_ALLOCATION_ALIGNMENT: usize = 64;+ const ALLOCATION_GRANULARITY: usize = 8 * 1024 * 1024;// Metal's set_bytes supports up to 4KB per bound value.// https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setbytes(_:length:index:)?language=objcconst MAX_INLINE_BYTES: usize = 4096;diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rsindex 4afabff72..26651e852 100644--- a/crates/backend-uzu/src/backends/metal/context.rs+++ b/crates/backend-uzu/src/backends/metal/context.rs@@ -26,7 +26,6 @@ use crate::{common::{Allocation, AllocationPool, AllocationType, Allocator, Backend, Context},metal::{command_buffer::MetalCommandBufferInitial,- metal_extensions::SparsePageSizeExt,sparse::{MetalSparseBuffer, MetalSparseHeapPool, MetalSparseMappingOpsBatch},},},@@ -133,7 +132,7 @@ impl Context for MetalContext {let gpu_core_count = device.gpu_core_count();let page_size = MTLSparsePageSize::KB256;- let heap_capacity = 64 * 4 * page_size.in_bytes();+ let heap_capacity = Metal::ALLOCATION_GRANULARITY;let sparse_pool = MetalSparseHeapPool::new(page_size, heap_capacity);let timeline_event = device.new_event().ok_or(MetalError::CannotCreateEvent)?;#[cfg(test)]