Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Temporary allocator optimization (#417)

uuuvneb93dfcb9484
diff --git a/crates/backend-uzu/src/backends/common/allocator/allocator.rs b/crates/backend-uzu/src/backends/common/allocator/allocator.rs
index 62683302e..e07e111b7 100644
--- a/crates/backend-uzu/src/backends/common/allocator/allocator.rs
+++ b/crates/backend-uzu/src/backends/common/allocator/allocator.rs
@@ -15,6 +15,7 @@ pub struct Allocation<B: Backend> {
allocator: Rc<Allocator<B>>,
buffer: *const B::DenseBuffer,
range: Range<usize>,
+ allocation_type: RangeAllocationType,
}
impl<B: Backend> AsBufferRangeRef for Allocation<B> {
@@ -65,7 +66,7 @@ pub enum AllocationType<'a, B: Backend> {
struct AllocatorBuffer<B: Backend> {
buffer: Pin<Box<B::DenseBuffer>>,
- range_allocator: RangeAllocator<usize>,
+ range_allocator: RangeAllocator,
}
pub struct Allocator<B: Backend> {
@@ -106,18 +107,16 @@ impl<B: Backend> Allocator<B> {
let mut allocator_buffers = self.allocator_buffers.borrow_mut();
- let mut found: Option<(*const B::DenseBuffer, Range<usize>)> = None;
+ let found = allocator_buffers.iter_mut().enumerate().find_map(|(allocator_buffer_index, allocator_buffer)| {
+ let range = allocator_buffer.range_allocator.allocate_range_aligned(size, alignment, allocation_type)?;
+ let buffer = allocator_buffer.buffer.as_ref().get_ref() as *const B::DenseBuffer;
- for allocator_buffer in allocator_buffers.iter_mut() {
- if let Some(range) =
- allocator_buffer.range_allocator.allocate_range_aligned(size, alignment, allocation_type.clone())
- {
- found = Some((allocator_buffer.buffer.as_ref().get_ref() as *const B::DenseBuffer, range));
- break;
- }
- }
+ Some((allocator_buffer_index, buffer, range))
+ });
+
+ let (buffer, range) = if let Some((allocator_buffer_index, buffer, range)) = found {
+ Self::restore_buffer_order(&mut allocator_buffers, allocator_buffer_index);
- let (buffer, range) = if let Some((buffer, range)) = found {
(buffer, range)
} else {
let new_allocator_buffer_size = usize::max(size, 268_435_456);
@@ -132,6 +131,8 @@ impl<B: Backend> Allocator<B> {
allocator_buffer.range_allocator.allocate_range_aligned(size, alignment, allocation_type).unwrap(); // Can never fail
allocator_buffers.push(allocator_buffer);
+ let allocator_buffer_index = allocator_buffers.len() - 1;
+ Self::restore_buffer_order(&mut allocator_buffers, allocator_buffer_index);
*self.peak_memory_usage.borrow_mut() =
allocator_buffers.iter().map(|allocator_buffer| allocator_buffer.buffer.size()).sum();
@@ -139,12 +140,11 @@ impl<B: Backend> Allocator<B> {
(buffer, range)
};
- allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());
-
Ok(Allocation {
allocator: self.clone(),
buffer,
range,
+ allocation_type,
})
}
@@ -175,21 +175,22 @@ impl<B: Backend> Allocator<B> {
) {
let mut allocator_buffers = self.allocator_buffers.borrow_mut();
- let (allocator_buffer_index, allocator_buffer) = allocator_buffers
- .iter_mut()
- .enumerate()
- .find(|(_allocator_buffer_index, allocator_buffer)| {
+ let allocator_buffer_index = allocator_buffers
+ .iter()
+ .position(|allocator_buffer| {
(allocator_buffer.buffer.as_ref().get_ref() as *const B::DenseBuffer) == allocation.buffer
})
.unwrap(); // Can never fail
- allocator_buffer.range_allocator.free_range(allocation.range.clone());
+ allocator_buffers[allocator_buffer_index]
+ .range_allocator
+ .free_range(allocation.range.clone(), allocation.allocation_type);
- if allocator_buffer.range_allocator.is_empty() {
+ if allocator_buffers[allocator_buffer_index].range_allocator.is_empty() {
allocator_buffers.remove(allocator_buffer_index);
+ } else {
+ Self::restore_buffer_order(&mut allocator_buffers, allocator_buffer_index);
}
-
- allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());
}
fn free_pool(
@@ -197,19 +198,39 @@ impl<B: Backend> Allocator<B> {
pool: &AllocationPool<B>,
) {
let mut allocator_buffers = self.allocator_buffers.borrow_mut();
- let mut free_idxs = Vec::new();
- for (allocation_buffer_idx, allocation_buffer) in allocator_buffers.iter_mut().enumerate() {
+ allocator_buffers.retain_mut(|allocation_buffer| {
allocation_buffer.range_allocator.free_pool(pool.pool_number);
- if allocation_buffer.range_allocator.is_empty() {
- free_idxs.push(allocation_buffer_idx);
- }
+ !allocation_buffer.range_allocator.is_empty()
+ });
+
+ if allocator_buffers.len() > 1 {
+ allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());
}
+ }
- for (i, free_idx) in free_idxs.into_iter().enumerate() {
- allocator_buffers.remove(free_idx - i);
+ fn restore_buffer_order(
+ allocator_buffers: &mut [AllocatorBuffer<B>],
+ mut index: usize,
+ ) {
+ while index > 0
+ && allocator_buffers[index].range_allocator.total_available()
+ < allocator_buffers[index - 1].range_allocator.total_available()
+ {
+ allocator_buffers.swap(index, index - 1);
+ index -= 1;
}
- allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());
+ while index + 1 < allocator_buffers.len()
+ && allocator_buffers[index].range_allocator.total_available()
+ > allocator_buffers[index + 1].range_allocator.total_available()
+ {
+ allocator_buffers.swap(index, index + 1);
+ index += 1;
+ }
}
}
+
+#[cfg(all(test, metal_backend))]
+#[path = "../../../../tests/unit/backends/common/allocator/allocator.rs"]
+mod tests;
diff --git a/crates/backend-uzu/src/backends/common/allocator/range_allocator.rs b/crates/backend-uzu/src/backends/common/allocator/range_allocator.rs
index 635729787..76a982bb3 100644
--- a/crates/backend-uzu/src/backends/common/allocator/range_allocator.rs
+++ b/crates/backend-uzu/src/backends/common/allocator/range_allocator.rs
@@ -1,45 +1,178 @@
-use std::ops::Range;
+use std::{
+ collections::{BTreeMap, BTreeSet},
+ ops::Range,
+};
-use rangemap::{RangeMap, RangeSet};
-
-#[derive(PartialEq, Eq, Clone)]
-pub enum AllocationType<P> {
+#[derive(PartialEq, Eq, Clone, Copy)]
+pub enum AllocationType {
Global,
Pooled {
- pool: P,
+ pool: usize,
can_alias_before: bool,
can_alias_after: bool,
},
}
-#[derive(PartialEq, Eq, Clone)]
-enum PooledAllocationRangeStatus {
- Allocated {
- can_alias_after: bool,
- },
- Aliasable,
- Frozen,
+#[derive(Default)]
+struct AvailableRanges {
+ by_start: BTreeMap<usize, usize>,
+ by_len: BTreeSet<(usize, usize, usize)>,
+ total_len: usize,
}
-#[derive(PartialEq, Eq, Clone)]
-enum AllocationRangeType<P: Eq + Clone> {
- Global,
- Pooled {
- pool: P,
- status: PooledAllocationRangeStatus,
- },
+#[derive(Clone, Copy)]
+struct AvailableRangeFit {
+ available_start: usize,
+ available_end: usize,
+ allocated_start: usize,
+ allocated_end: usize,
+}
+
+impl AvailableRangeFit {
+ fn available_key(self) -> (usize, usize) {
+ (self.available_end - self.available_start, self.available_start)
+ }
+
+ fn allocated_range(self) -> Range<usize> {
+ self.allocated_start..self.allocated_end
+ }
+
+ fn allocated_len(self) -> usize {
+ self.allocated_end - self.allocated_start
+ }
+}
+
+impl AvailableRanges {
+ fn new(range: Range<usize>) -> Self {
+ let mut available_ranges = Self::default();
+ available_ranges.insert(range);
+ available_ranges
+ }
+
+ fn total_len(&self) -> usize {
+ self.total_len
+ }
+
+ fn insert(
+ &mut self,
+ mut range: Range<usize>,
+ ) {
+ if range.is_empty() {
+ return;
+ }
+
+ if let Some((&start, &end)) = self.by_start.range(..=range.start).next_back()
+ && end >= range.start
+ {
+ self.remove_entry(start, end);
+ range.start = usize::min(range.start, start);
+ range.end = usize::max(range.end, end);
+ }
+
+ while let Some((&start, &end)) = self.by_start.range(range.start..).next() {
+ if start > range.end {
+ break;
+ }
+
+ self.remove_entry(start, end);
+ range.end = usize::max(range.end, end);
+ }
+
+ self.insert_entry(range.start, range.end);
+ }
+
+ fn remove_fit(
+ &mut self,
+ fit: AvailableRangeFit,
+ ) -> Range<usize> {
+ self.remove_entry(fit.available_start, fit.available_end);
+
+ if fit.available_start < fit.allocated_start {
+ self.insert_entry(fit.available_start, fit.allocated_start);
+ }
+
+ if fit.allocated_end < fit.available_end {
+ self.insert_entry(fit.allocated_end, fit.available_end);
+ }
+
+ fit.allocated_range()
+ }
+
+ fn best_fit(
+ &self,
+ size: usize,
+ alignment: usize,
+ ) -> Option<AvailableRangeFit> {
+ for &(range_len, range_start, range_end) in self.by_len.range((size, 0, 0)..) {
+ let allocated_start = range_start.next_multiple_of(alignment);
+
+ if range_end.saturating_sub(allocated_start) >= size {
+ debug_assert_eq!(range_end - range_start, range_len);
+
+ return Some(AvailableRangeFit {
+ available_start: range_start,
+ available_end: range_end,
+ allocated_start,
+ allocated_end: allocated_start + size,
+ });
+ }
+ }
+
+ None
+ }
+
+ fn into_ranges(self) -> impl Iterator<Item = Range<usize>> {
+ self.by_start.into_iter().map(|(start, end)| start..end)
+ }
+
+ fn insert_entry(
+ &mut self,
+ start: usize,
+ end: usize,
+ ) {
+ debug_assert!(start < end);
+
+ let len = end - start;
+ let old_end = self.by_start.insert(start, end);
+ debug_assert!(old_end.is_none());
+
+ self.by_len.insert((len, start, end));
+ self.total_len += len;
+ }
+
+ fn remove_entry(
+ &mut self,
+ start: usize,
+ end: usize,
+ ) {
+ let len = end - start;
+
+ assert_eq!(self.by_start.remove(&start), Some(end));
+ assert!(self.by_len.remove(&(len, start, end)));
+ self.total_len -= len;
+ }
}
-pub struct RangeAllocator<P: Eq + Clone> {
- full_range: Range<usize>,
- ranges: RangeMap<usize, AllocationRangeType<P>>,
+pub struct RangeAllocator {
+ full_len: usize,
+ free_ranges: AvailableRanges,
+ aliasable_ranges_by_pool: Vec<AvailableRanges>,
+ pool_ranges_by_pool: Vec<AvailableRanges>,
+ pool_live_allocations: Vec<usize>,
+ total_available: usize,
}
-impl<P: Eq + Clone> RangeAllocator<P> {
+impl RangeAllocator {
pub fn new(full_range: Range<usize>) -> Self {
+ let full_len = full_range.len();
+
Self {
- full_range,
- ranges: RangeMap::new(),
+ full_len,
+ free_ranges: AvailableRanges::new(full_range),
+ aliasable_ranges_by_pool: Vec::new(),
+ pool_ranges_by_pool: Vec::new(),
+ pool_live_allocations: Vec::new(),
+ total_available: full_len,
}
}
@@ -47,151 +180,123 @@ impl<P: Eq + Clone> RangeAllocator<P> {
&mut self,
size: usize,
alignment: usize,
- allocation_type: AllocationType<P>,
+ allocation_type: AllocationType,
) -> Option<Range<usize>> {
- let mut available_range = RangeSet::new();
- available_range.extend(self.ranges.gaps(&self.full_range));
-
- if let AllocationType::Pooled {
- pool,
- can_alias_before: true,
- can_alias_after: _,
- } = &allocation_type
- {
- available_range.extend(self.ranges.iter().flat_map(|(range, range_type)| {
- if let AllocationRangeType::Pooled {
- pool: range_pool,
- status: PooledAllocationRangeStatus::Aliasable,
- } = range_type
- && range_pool == pool
- {
- Some(range.clone())
- } else {
- None
- }
- }));
- }
+ let aliasable_pool = match allocation_type {
+ AllocationType::Pooled {
+ pool,
+ can_alias_before: true,
+ can_alias_after: _,
+ } => Some(pool),
+ _ => None,
+ };
+
+ let free_candidate = self.free_ranges.best_fit(size, alignment);
+ let aliasable_candidate = aliasable_pool.and_then(|pool| {
+ self.aliasable_ranges_by_pool.get(pool).and_then(|ranges| ranges.best_fit(size, alignment))
+ });
+
+ let (selected_aliasable_pool, fit) = match (free_candidate, aliasable_candidate) {
+ (None, None) => return None,
+ (Some(fit), None) => (None, fit),
+ (None, Some(fit)) => (aliasable_pool, fit),
+ (Some(free_fit), Some(aliasable_fit)) if aliasable_fit.available_key() < free_fit.available_key() => {
+ (aliasable_pool, aliasable_fit)
+ },
+ (Some(fit), Some(_)) => (None, fit),
+ };
- let Some(smallest_fitting_free_range) = available_range
- .iter()
- .filter(|range| range.end.saturating_sub(range.start.next_multiple_of(alignment)) >= size)
- .min_by_key(|range| range.len())
- else {
- return None;
+ let allocated_range = if let Some(pool) = selected_aliasable_pool {
+ self.aliasable_ranges_by_pool[pool].remove_fit(fit)
+ } else {
+ self.free_ranges.remove_fit(fit)
};
+ self.total_available -= fit.allocated_len();
- let allocated_range_start = smallest_fitting_free_range.start.next_multiple_of(alignment);
- let allocated_range = allocated_range_start..(allocated_range_start + size);
- let allocated_range_status = match allocation_type {
- AllocationType::Global => AllocationRangeType::Global,
+ match allocation_type {
+ AllocationType::Global => {},
AllocationType::Pooled {
pool,
can_alias_before: _,
- can_alias_after,
- } => AllocationRangeType::Pooled {
- pool,
- status: PooledAllocationRangeStatus::Allocated {
- can_alias_after,
- },
+ can_alias_after: _,
+ } => {
+ self.ensure_pool(pool);
+ self.pool_live_allocations[pool] += 1;
+
+ if selected_aliasable_pool.is_none() {
+ self.pool_ranges_by_pool[pool].insert(allocated_range.clone());
+ }
},
};
- self.ranges.insert(allocated_range.clone(), allocated_range_status);
-
Some(allocated_range)
}
pub fn free_range(
&mut self,
range: Range<usize>,
+ allocation_type: AllocationType,
) {
- let overlapping_ranges = self.ranges.overlapping(range.clone()).collect::<Vec<_>>();
- assert!(overlapping_ranges.len() == 1, "expected only 1 overlapping range");
-
- let (overlapping_range, range_type) = overlapping_ranges[0];
- assert!(
- overlapping_range.start <= range.start && overlapping_range.end >= range.end,
- "overlapping range doesn't fully cover the range"
- );
-
- match range_type {
- AllocationRangeType::Global => self.ranges.remove(range),
- AllocationRangeType::Pooled {
+ match allocation_type {
+ AllocationType::Global => {
+ self.free_ranges.insert(range.clone());
+ self.total_available += range.len();
+ },
+ AllocationType::Pooled {
pool,
- status,
+ can_alias_before: _,
+ can_alias_after,
} => {
- let status = match status {
- PooledAllocationRangeStatus::Allocated {
- can_alias_after: true,
- } => PooledAllocationRangeStatus::Aliasable,
- PooledAllocationRangeStatus::Allocated {
- can_alias_after: false,
- } => PooledAllocationRangeStatus::Frozen,
- _ => panic!("attempted to free a range that isn't allocated"),
- };
-
- self.ranges.insert(
- range,
- AllocationRangeType::Pooled {
- pool: pool.clone(),
- status,
- },
- )
+ self.pool_live_allocations[pool] -= 1;
+
+ if can_alias_after {
+ self.aliasable_ranges_by_pool[pool].insert(range.clone());
+ self.total_available += range.len();
+ }
},
};
}
pub fn free_pool(
&mut self,
- pool: P,
+ pool: usize,
) {
- let pool_ranges = self
- .ranges
- .iter()
- .filter_map(|(range, range_type)| {
- if let AllocationRangeType::Pooled {
- pool: range_pool,
- status,
- } = range_type
- && range_pool == &pool
- {
- assert!(
- !matches!(status, PooledAllocationRangeStatus::Allocated { .. }),
- "attempted to free a pool that has live allocations"
- );
- Some(range.clone())
- } else {
- None
- }
- })
- .collect::<Vec<_>>();
+ if let Some(live_allocations) = self.pool_live_allocations.get(pool) {
+ assert_eq!(*live_allocations, 0, "attempted to free a pool that has live allocations");
+ }
+
+ if let Some(aliasable_ranges) = self.aliasable_ranges_by_pool.get_mut(pool) {
+ self.total_available -= aliasable_ranges.total_len();
+ *aliasable_ranges = AvailableRanges::default();
+ }
- for pool_range in pool_ranges {
- self.ranges.remove(pool_range);
+ let Some(pool_ranges) = self.pool_ranges_by_pool.get_mut(pool) else {
+ return;
+ };
+ let pool_ranges = std::mem::take(pool_ranges);
+
+ for pool_range in pool_ranges.into_ranges() {
+ self.free_ranges.insert(pool_range.clone());
+ self.total_available += pool_range.len();
}
}
pub fn total_available(&self) -> usize {
- self.ranges
- .gaps(&self.full_range)
- .map(|gap| gap.len())
- .chain(self.ranges.iter().filter_map(|(range, range_type)| {
- if matches!(
- range_type,
- AllocationRangeType::Pooled {
- pool: _,
- status: PooledAllocationRangeStatus::Aliasable
- }
- ) {
- Some(range.len())
- } else {
- None
- }
- }))
- .sum()
+ self.total_available
}
pub fn is_empty(&self) -> bool {
- self.ranges.is_empty()
+ self.free_ranges.total_len() == self.full_len
+ }
+
+ fn ensure_pool(
+ &mut self,
+ pool: usize,
+ ) {
+ let pool_len = pool + 1;
+
+ self.aliasable_ranges_by_pool.resize_with(pool_len, AvailableRanges::default);
+ self.pool_ranges_by_pool.resize_with(pool_len, AvailableRanges::default);
+ self.pool_live_allocations.resize(pool_len, 0);
}
}
diff --git a/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs b/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs
new file mode 100644
index 000000000..bc4351f06
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs
@@ -0,0 +1,128 @@
+use std::{
+ collections::{HashMap, hash_map::Entry},
+ fs::File,
+ io::BufReader,
+ rc::Rc,
+ time::Instant,
+};
+
+use serde::{Deserialize, Serialize};
+
+use crate::backends::{
+ common::{Allocation, AllocationPool, AllocationType, Allocator, Backend, Context},
+ metal::Metal,
+};
+
+#[derive(Serialize, Deserialize)]
+struct TraceEvent {
+ allocator_id: usize,
+ event_type: TraceEventType,
+}
+
+#[derive(Serialize, Deserialize)]
+pub enum TraceEventAllocationType {
+ Global,
+ Pooled {
+ pool_id: usize,
+ cpu_available: bool,
+ },
+}
+
+#[derive(Serialize, Deserialize)]
+enum TraceEventType {
+ Create,
+ CreatePool {
+ reusable: bool,
+ pool_id: usize,
+ },
+ Allocate {
+ size: usize,
+ allocation_type: TraceEventAllocationType,
+ allocation_id: usize,
+ },
+ Free {
+ allocation_id: usize,
+ },
+ FreePool {
+ pool_id: usize,
+ },
+}
+
+#[test]
+#[ignore]
+fn bench_allocator_generation_trace() {
+ let file = BufReader::with_capacity(1 << 20, File::open("/tmp/allocator_trace.jsonl").unwrap());
+ let events: Vec<TraceEvent> = serde_json::Deserializer::from_reader(file)
+ .into_iter::<TraceEvent>()
+ .map(|r| r.expect("malformed trace event"))
+ .collect();
+
+ let context = <<Metal as Backend>::Context as Context>::new().unwrap();
+
+ let mut allocators: HashMap<usize, Rc<Allocator<Metal>>> = HashMap::new();
+ let mut pools: HashMap<(usize, usize), AllocationPool<Metal>> = HashMap::new();
+ let mut allocations: HashMap<(usize, usize), Allocation<Metal>> = HashMap::new();
+
+ let start = Instant::now();
+
+ for event in events {
+ match event.event_type {
+ TraceEventType::Create => {
+ let Entry::Vacant(vacant) = allocators.entry(event.allocator_id) else {
+ panic!("Duplicate allocator id: {}", event.allocator_id);
+ };
+
+ vacant.insert(Allocator::new(Rc::downgrade(&context)));
+ },
+ TraceEventType::CreatePool {
+ reusable,
+ pool_id,
+ } => {
+ let Entry::Vacant(vacant) = pools.entry((event.allocator_id, pool_id)) else {
+ panic!("Duplicate pool id {} in allocator {}", pool_id, event.allocator_id);
+ };
+
+ vacant.insert(allocators[&event.allocator_id].create_pool(reusable));
+ },
+ TraceEventType::Allocate {
+ size,
+ allocation_type,
+ allocation_id,
+ } => {
+ let Entry::Vacant(vacant) = allocations.entry((event.allocator_id, allocation_id)) else {
+ panic!("Duplicate allocation id {} in allocator {}", allocation_id, event.allocator_id);
+ };
+
+ let allocation_type = match allocation_type {
+ TraceEventAllocationType::Global => AllocationType::Global,
+ TraceEventAllocationType::Pooled {
+ pool_id,
+ cpu_available,
+ } => AllocationType::Pooled {
+ pool: &pools[&(event.allocator_id, pool_id)],
+ cpu_available,
+ },
+ };
+
+ vacant.insert(allocators[&event.allocator_id].allocate(size, allocation_type).unwrap());
+ },
+ TraceEventType::Free {
+ allocation_id,
+ } => {
+ allocations.remove(&(event.allocator_id, allocation_id)).unwrap();
+ },
+ TraceEventType::FreePool {
+ pool_id,
+ } => {
+ pools.remove(&(event.allocator_id, pool_id)).unwrap();
+ },
+ }
+ }
+
+ let elapsed = start.elapsed();
+
+ eprintln!();
+ eprintln!("--------------------");
+ eprintln!("Cpu time: {} ms", elapsed.as_millis());
+ eprintln!("--------------------");
+}