Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Tighten allocator alignment and granularity (#488)

uuuvn18ccba93e6f7
diff --git a/crates/backend-uzu/src/backends/common/allocator/allocator.rs b/crates/backend-uzu/src/backends/common/allocator/allocator.rs
index 75836fc0a..d317b6cdf 100644
--- a/crates/backend-uzu/src/backends/common/allocator/allocator.rs
+++ b/crates/backend-uzu/src/backends/common/allocator/allocator.rs
@@ -105,7 +105,8 @@ impl<B: Backend> Allocator<B> {
allocation_type: AllocationType<B>,
) -> Result<Allocation<B>, B::Error> {
assert!(size > 0, "allocation size must be greater than 0");
- let alignment = usize::clamp(size.next_power_of_two(), B::MIN_ALLOCATION_ALIGNMENT, 16_384);
+ let alignment =
+ usize::clamp(size.next_power_of_two(), B::MIN_ALLOCATION_ALIGNMENT, B::MAX_ALLOCATION_ALIGNMENT);
let allocation_type = match allocation_type {
AllocationType::Global => RangeAllocationType::Global,
AllocationType::Pooled {
@@ -132,7 +133,7 @@ impl<B: Backend> Allocator<B> {
(buffer, range)
} else {
- let new_allocator_buffer_size = usize::max(size, 268_435_456);
+ let new_allocator_buffer_size = usize::max(size, B::ALLOCATION_GRANULARITY);
let mut allocator_buffer = AllocatorBuffer::<B> {
buffer: Box::pin(self.context.upgrade().unwrap().create_buffer(new_allocator_buffer_size)?), // Upgrade can never fail
diff --git a/crates/backend-uzu/src/backends/common/backend.rs b/crates/backend-uzu/src/backends/common/backend.rs
index bd6a4c1fd..70ea34230 100644
--- a/crates/backend-uzu/src/backends/common/backend.rs
+++ b/crates/backend-uzu/src/backends/common/backend.rs
@@ -11,5 +11,7 @@ pub trait Backend: Debug + Clone + 'static {
type Error: Error + Debug;
const MIN_ALLOCATION_ALIGNMENT: usize;
+ const MAX_ALLOCATION_ALIGNMENT: usize;
+ const ALLOCATION_GRANULARITY: usize;
const MAX_INLINE_BYTES: usize;
}
diff --git a/crates/backend-uzu/src/backends/cpu/backend.rs b/crates/backend-uzu/src/backends/cpu/backend.rs
index d8f96544a..71e654465 100644
--- a/crates/backend-uzu/src/backends/cpu/backend.rs
+++ b/crates/backend-uzu/src/backends/cpu/backend.rs
@@ -16,6 +16,8 @@ impl Backend for Cpu {
type Kernels = CpuKernels;
type Error = CpuError;
- const MIN_ALLOCATION_ALIGNMENT: usize = 64;
+ const MIN_ALLOCATION_ALIGNMENT: usize = 4;
+ const MAX_ALLOCATION_ALIGNMENT: usize = 64;
+ const ALLOCATION_GRANULARITY: usize = 8 * 1024 * 1024;
const MAX_INLINE_BYTES: usize = usize::MAX;
}
diff --git a/crates/backend-uzu/src/backends/metal/backend.rs b/crates/backend-uzu/src/backends/metal/backend.rs
index 5dd2a1da9..fe777ae3f 100644
--- a/crates/backend-uzu/src/backends/metal/backend.rs
+++ b/crates/backend-uzu/src/backends/metal/backend.rs
@@ -15,7 +15,9 @@ impl Backend for Metal {
type Kernels = MetalKernels;
type Error = MetalError;
- const MIN_ALLOCATION_ALIGNMENT: usize = 64;
+ const MIN_ALLOCATION_ALIGNMENT: usize = 4;
+ const MAX_ALLOCATION_ALIGNMENT: usize = 64;
+ const ALLOCATION_GRANULARITY: usize = 8 * 1024 * 1024;
// Metal's set_bytes supports up to 4KB per bound value.
// https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setbytes(_:length:index:)?language=objc
const MAX_INLINE_BYTES: usize = 4096;
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index 4afabff72..26651e852 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -26,7 +26,6 @@ use crate::{
common::{Allocation, AllocationPool, AllocationType, Allocator, Backend, Context},
metal::{
command_buffer::MetalCommandBufferInitial,
- metal_extensions::SparsePageSizeExt,
sparse::{MetalSparseBuffer, MetalSparseHeapPool, MetalSparseMappingOpsBatch},
},
},
@@ -133,7 +132,7 @@ impl Context for MetalContext {
let gpu_core_count = device.gpu_core_count();
let page_size = MTLSparsePageSize::KB256;
- let heap_capacity = 64 * 4 * page_size.in_bytes();
+ let heap_capacity = Metal::ALLOCATION_GRANULARITY;
let sparse_pool = MetalSparseHeapPool::new(page_size, heap_capacity);
let timeline_event = device.new_event().ok_or(MetalError::CannotCreateEvent)?;
#[cfg(test)]