A high-performance inference engine for AI models
Temporary allocator optimization (#417)
uuuvneb93dfcb9484
diff --git a/crates/backend-uzu/src/backends/common/allocator/allocator.rs b/crates/backend-uzu/src/backends/common/allocator/allocator.rsindex 62683302e..e07e111b7 100644--- a/crates/backend-uzu/src/backends/common/allocator/allocator.rs+++ b/crates/backend-uzu/src/backends/common/allocator/allocator.rs@@ -15,6 +15,7 @@ pub struct Allocation<B: Backend> {allocator: Rc<Allocator<B>>,buffer: *const B::DenseBuffer,range: Range<usize>,+ allocation_type: RangeAllocationType,}impl<B: Backend> AsBufferRangeRef for Allocation<B> {@@ -65,7 +66,7 @@ pub enum AllocationType<'a, B: Backend> {struct AllocatorBuffer<B: Backend> {buffer: Pin<Box<B::DenseBuffer>>,- range_allocator: RangeAllocator<usize>,+ range_allocator: RangeAllocator,}pub struct Allocator<B: Backend> {@@ -106,18 +107,16 @@ impl<B: Backend> Allocator<B> {let mut allocator_buffers = self.allocator_buffers.borrow_mut();- let mut found: Option<(*const B::DenseBuffer, Range<usize>)> = None;+ let found = allocator_buffers.iter_mut().enumerate().find_map(|(allocator_buffer_index, allocator_buffer)| {+ let range = allocator_buffer.range_allocator.allocate_range_aligned(size, alignment, allocation_type)?;+ let buffer = allocator_buffer.buffer.as_ref().get_ref() as *const B::DenseBuffer;- for allocator_buffer in allocator_buffers.iter_mut() {- if let Some(range) =- allocator_buffer.range_allocator.allocate_range_aligned(size, alignment, allocation_type.clone())- {- found = Some((allocator_buffer.buffer.as_ref().get_ref() as *const B::DenseBuffer, range));- break;- }- }+ Some((allocator_buffer_index, buffer, range))+ });++ let (buffer, range) = if let Some((allocator_buffer_index, buffer, range)) = found {+ Self::restore_buffer_order(&mut allocator_buffers, allocator_buffer_index);- let (buffer, range) = if let Some((buffer, range)) = found {(buffer, range)} else {let new_allocator_buffer_size = usize::max(size, 268_435_456);@@ -132,6 +131,8 @@ impl<B: Backend> Allocator<B> {allocator_buffer.range_allocator.allocate_range_aligned(size, alignment, allocation_type).unwrap(); // Can never failallocator_buffers.push(allocator_buffer);+ let allocator_buffer_index = allocator_buffers.len() - 1;+ Self::restore_buffer_order(&mut allocator_buffers, allocator_buffer_index);*self.peak_memory_usage.borrow_mut() =allocator_buffers.iter().map(|allocator_buffer| allocator_buffer.buffer.size()).sum();@@ -139,12 +140,11 @@ impl<B: Backend> Allocator<B> {(buffer, range)};- allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());-Ok(Allocation {allocator: self.clone(),buffer,range,+ allocation_type,})}@@ -175,21 +175,22 @@ impl<B: Backend> Allocator<B> {) {let mut allocator_buffers = self.allocator_buffers.borrow_mut();- let (allocator_buffer_index, allocator_buffer) = allocator_buffers- .iter_mut()- .enumerate()- .find(|(_allocator_buffer_index, allocator_buffer)| {+ let allocator_buffer_index = allocator_buffers+ .iter()+ .position(|allocator_buffer| {(allocator_buffer.buffer.as_ref().get_ref() as *const B::DenseBuffer) == allocation.buffer}).unwrap(); // Can never fail- allocator_buffer.range_allocator.free_range(allocation.range.clone());+ allocator_buffers[allocator_buffer_index]+ .range_allocator+ .free_range(allocation.range.clone(), allocation.allocation_type);- if allocator_buffer.range_allocator.is_empty() {+ if allocator_buffers[allocator_buffer_index].range_allocator.is_empty() {allocator_buffers.remove(allocator_buffer_index);+ } else {+ Self::restore_buffer_order(&mut allocator_buffers, allocator_buffer_index);}-- allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());}fn free_pool(@@ -197,19 +198,39 @@ impl<B: Backend> Allocator<B> {pool: &AllocationPool<B>,) {let mut allocator_buffers = self.allocator_buffers.borrow_mut();- let mut free_idxs = Vec::new();- for (allocation_buffer_idx, allocation_buffer) in allocator_buffers.iter_mut().enumerate() {+ allocator_buffers.retain_mut(|allocation_buffer| {allocation_buffer.range_allocator.free_pool(pool.pool_number);- if allocation_buffer.range_allocator.is_empty() {- free_idxs.push(allocation_buffer_idx);- }+ !allocation_buffer.range_allocator.is_empty()+ });++ if allocator_buffers.len() > 1 {+ allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());}+ }- for (i, free_idx) in free_idxs.into_iter().enumerate() {- allocator_buffers.remove(free_idx - i);+ fn restore_buffer_order(+ allocator_buffers: &mut [AllocatorBuffer<B>],+ mut index: usize,+ ) {+ while index > 0+ && allocator_buffers[index].range_allocator.total_available()+ < allocator_buffers[index - 1].range_allocator.total_available()+ {+ allocator_buffers.swap(index, index - 1);+ index -= 1;}- allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());+ while index + 1 < allocator_buffers.len()+ && allocator_buffers[index].range_allocator.total_available()+ > allocator_buffers[index + 1].range_allocator.total_available()+ {+ allocator_buffers.swap(index, index + 1);+ index += 1;+ }}}++#[cfg(all(test, metal_backend))]+#[path = "../../../../tests/unit/backends/common/allocator/allocator.rs"]+mod tests;diff --git a/crates/backend-uzu/src/backends/common/allocator/range_allocator.rs b/crates/backend-uzu/src/backends/common/allocator/range_allocator.rsindex 635729787..76a982bb3 100644--- a/crates/backend-uzu/src/backends/common/allocator/range_allocator.rs+++ b/crates/backend-uzu/src/backends/common/allocator/range_allocator.rs@@ -1,45 +1,178 @@-use std::ops::Range;+use std::{+ collections::{BTreeMap, BTreeSet},+ ops::Range,+};-use rangemap::{RangeMap, RangeSet};--#[derive(PartialEq, Eq, Clone)]-pub enum AllocationType<P> {+#[derive(PartialEq, Eq, Clone, Copy)]+pub enum AllocationType {Global,Pooled {- pool: P,+ pool: usize,can_alias_before: bool,can_alias_after: bool,},}-#[derive(PartialEq, Eq, Clone)]-enum PooledAllocationRangeStatus {- Allocated {- can_alias_after: bool,- },- Aliasable,- Frozen,+#[derive(Default)]+struct AvailableRanges {+ by_start: BTreeMap<usize, usize>,+ by_len: BTreeSet<(usize, usize, usize)>,+ total_len: usize,}-#[derive(PartialEq, Eq, Clone)]-enum AllocationRangeType<P: Eq + Clone> {- Global,- Pooled {- pool: P,- status: PooledAllocationRangeStatus,- },+#[derive(Clone, Copy)]+struct AvailableRangeFit {+ available_start: usize,+ available_end: usize,+ allocated_start: usize,+ allocated_end: usize,+}++impl AvailableRangeFit {+ fn available_key(self) -> (usize, usize) {+ (self.available_end - self.available_start, self.available_start)+ }++ fn allocated_range(self) -> Range<usize> {+ self.allocated_start..self.allocated_end+ }++ fn allocated_len(self) -> usize {+ self.allocated_end - self.allocated_start+ }+}++impl AvailableRanges {+ fn new(range: Range<usize>) -> Self {+ let mut available_ranges = Self::default();+ available_ranges.insert(range);+ available_ranges+ }++ fn total_len(&self) -> usize {+ self.total_len+ }++ fn insert(+ &mut self,+ mut range: Range<usize>,+ ) {+ if range.is_empty() {+ return;+ }++ if let Some((&start, &end)) = self.by_start.range(..=range.start).next_back()+ && end >= range.start+ {+ self.remove_entry(start, end);+ range.start = usize::min(range.start, start);+ range.end = usize::max(range.end, end);+ }++ while let Some((&start, &end)) = self.by_start.range(range.start..).next() {+ if start > range.end {+ break;+ }++ self.remove_entry(start, end);+ range.end = usize::max(range.end, end);+ }++ self.insert_entry(range.start, range.end);+ }++ fn remove_fit(+ &mut self,+ fit: AvailableRangeFit,+ ) -> Range<usize> {+ self.remove_entry(fit.available_start, fit.available_end);++ if fit.available_start < fit.allocated_start {+ self.insert_entry(fit.available_start, fit.allocated_start);+ }++ if fit.allocated_end < fit.available_end {+ self.insert_entry(fit.allocated_end, fit.available_end);+ }++ fit.allocated_range()+ }++ fn best_fit(+ &self,+ size: usize,+ alignment: usize,+ ) -> Option<AvailableRangeFit> {+ for &(range_len, range_start, range_end) in self.by_len.range((size, 0, 0)..) {+ let allocated_start = range_start.next_multiple_of(alignment);++ if range_end.saturating_sub(allocated_start) >= size {+ debug_assert_eq!(range_end - range_start, range_len);++ return Some(AvailableRangeFit {+ available_start: range_start,+ available_end: range_end,+ allocated_start,+ allocated_end: allocated_start + size,+ });+ }+ }++ None+ }++ fn into_ranges(self) -> impl Iterator<Item = Range<usize>> {+ self.by_start.into_iter().map(|(start, end)| start..end)+ }++ fn insert_entry(+ &mut self,+ start: usize,+ end: usize,+ ) {+ debug_assert!(start < end);++ let len = end - start;+ let old_end = self.by_start.insert(start, end);+ debug_assert!(old_end.is_none());++ self.by_len.insert((len, start, end));+ self.total_len += len;+ }++ fn remove_entry(+ &mut self,+ start: usize,+ end: usize,+ ) {+ let len = end - start;++ assert_eq!(self.by_start.remove(&start), Some(end));+ assert!(self.by_len.remove(&(len, start, end)));+ self.total_len -= len;+ }}-pub struct RangeAllocator<P: Eq + Clone> {- full_range: Range<usize>,- ranges: RangeMap<usize, AllocationRangeType<P>>,+pub struct RangeAllocator {+ full_len: usize,+ free_ranges: AvailableRanges,+ aliasable_ranges_by_pool: Vec<AvailableRanges>,+ pool_ranges_by_pool: Vec<AvailableRanges>,+ pool_live_allocations: Vec<usize>,+ total_available: usize,}-impl<P: Eq + Clone> RangeAllocator<P> {+impl RangeAllocator {pub fn new(full_range: Range<usize>) -> Self {+ let full_len = full_range.len();+Self {- full_range,- ranges: RangeMap::new(),+ full_len,+ free_ranges: AvailableRanges::new(full_range),+ aliasable_ranges_by_pool: Vec::new(),+ pool_ranges_by_pool: Vec::new(),+ pool_live_allocations: Vec::new(),+ total_available: full_len,}}@@ -47,151 +180,123 @@ impl<P: Eq + Clone> RangeAllocator<P> {&mut self,size: usize,alignment: usize,- allocation_type: AllocationType<P>,+ allocation_type: AllocationType,) -> Option<Range<usize>> {- let mut available_range = RangeSet::new();- available_range.extend(self.ranges.gaps(&self.full_range));-- if let AllocationType::Pooled {- pool,- can_alias_before: true,- can_alias_after: _,- } = &allocation_type- {- available_range.extend(self.ranges.iter().flat_map(|(range, range_type)| {- if let AllocationRangeType::Pooled {- pool: range_pool,- status: PooledAllocationRangeStatus::Aliasable,- } = range_type- && range_pool == pool- {- Some(range.clone())- } else {- None- }- }));- }+ let aliasable_pool = match allocation_type {+ AllocationType::Pooled {+ pool,+ can_alias_before: true,+ can_alias_after: _,+ } => Some(pool),+ _ => None,+ };++ let free_candidate = self.free_ranges.best_fit(size, alignment);+ let aliasable_candidate = aliasable_pool.and_then(|pool| {+ self.aliasable_ranges_by_pool.get(pool).and_then(|ranges| ranges.best_fit(size, alignment))+ });++ let (selected_aliasable_pool, fit) = match (free_candidate, aliasable_candidate) {+ (None, None) => return None,+ (Some(fit), None) => (None, fit),+ (None, Some(fit)) => (aliasable_pool, fit),+ (Some(free_fit), Some(aliasable_fit)) if aliasable_fit.available_key() < free_fit.available_key() => {+ (aliasable_pool, aliasable_fit)+ },+ (Some(fit), Some(_)) => (None, fit),+ };- let Some(smallest_fitting_free_range) = available_range- .iter()- .filter(|range| range.end.saturating_sub(range.start.next_multiple_of(alignment)) >= size)- .min_by_key(|range| range.len())- else {- return None;+ let allocated_range = if let Some(pool) = selected_aliasable_pool {+ self.aliasable_ranges_by_pool[pool].remove_fit(fit)+ } else {+ self.free_ranges.remove_fit(fit)};+ self.total_available -= fit.allocated_len();- let allocated_range_start = smallest_fitting_free_range.start.next_multiple_of(alignment);- let allocated_range = allocated_range_start..(allocated_range_start + size);- let allocated_range_status = match allocation_type {- AllocationType::Global => AllocationRangeType::Global,+ match allocation_type {+ AllocationType::Global => {},AllocationType::Pooled {pool,can_alias_before: _,- can_alias_after,- } => AllocationRangeType::Pooled {- pool,- status: PooledAllocationRangeStatus::Allocated {- can_alias_after,- },+ can_alias_after: _,+ } => {+ self.ensure_pool(pool);+ self.pool_live_allocations[pool] += 1;++ if selected_aliasable_pool.is_none() {+ self.pool_ranges_by_pool[pool].insert(allocated_range.clone());+ }},};- self.ranges.insert(allocated_range.clone(), allocated_range_status);-Some(allocated_range)}pub fn free_range(&mut self,range: Range<usize>,+ allocation_type: AllocationType,) {- let overlapping_ranges = self.ranges.overlapping(range.clone()).collect::<Vec<_>>();- assert!(overlapping_ranges.len() == 1, "expected only 1 overlapping range");-- let (overlapping_range, range_type) = overlapping_ranges[0];- assert!(- overlapping_range.start <= range.start && overlapping_range.end >= range.end,- "overlapping range doesn't fully cover the range"- );-- match range_type {- AllocationRangeType::Global => self.ranges.remove(range),- AllocationRangeType::Pooled {+ match allocation_type {+ AllocationType::Global => {+ self.free_ranges.insert(range.clone());+ self.total_available += range.len();+ },+ AllocationType::Pooled {pool,- status,+ can_alias_before: _,+ can_alias_after,} => {- let status = match status {- PooledAllocationRangeStatus::Allocated {- can_alias_after: true,- } => PooledAllocationRangeStatus::Aliasable,- PooledAllocationRangeStatus::Allocated {- can_alias_after: false,- } => PooledAllocationRangeStatus::Frozen,- _ => panic!("attempted to free a range that isn't allocated"),- };-- self.ranges.insert(- range,- AllocationRangeType::Pooled {- pool: pool.clone(),- status,- },- )+ self.pool_live_allocations[pool] -= 1;++ if can_alias_after {+ self.aliasable_ranges_by_pool[pool].insert(range.clone());+ self.total_available += range.len();+ }},};}pub fn free_pool(&mut self,- pool: P,+ pool: usize,) {- let pool_ranges = self- .ranges- .iter()- .filter_map(|(range, range_type)| {- if let AllocationRangeType::Pooled {- pool: range_pool,- status,- } = range_type- && range_pool == &pool- {- assert!(- !matches!(status, PooledAllocationRangeStatus::Allocated { .. }),- "attempted to free a pool that has live allocations"- );- Some(range.clone())- } else {- None- }- })- .collect::<Vec<_>>();+ if let Some(live_allocations) = self.pool_live_allocations.get(pool) {+ assert_eq!(*live_allocations, 0, "attempted to free a pool that has live allocations");+ }++ if let Some(aliasable_ranges) = self.aliasable_ranges_by_pool.get_mut(pool) {+ self.total_available -= aliasable_ranges.total_len();+ *aliasable_ranges = AvailableRanges::default();+ }- for pool_range in pool_ranges {- self.ranges.remove(pool_range);+ let Some(pool_ranges) = self.pool_ranges_by_pool.get_mut(pool) else {+ return;+ };+ let pool_ranges = std::mem::take(pool_ranges);++ for pool_range in pool_ranges.into_ranges() {+ self.free_ranges.insert(pool_range.clone());+ self.total_available += pool_range.len();}}pub fn total_available(&self) -> usize {- self.ranges- .gaps(&self.full_range)- .map(|gap| gap.len())- .chain(self.ranges.iter().filter_map(|(range, range_type)| {- if matches!(- range_type,- AllocationRangeType::Pooled {- pool: _,- status: PooledAllocationRangeStatus::Aliasable- }- ) {- Some(range.len())- } else {- None- }- }))- .sum()+ self.total_available}pub fn is_empty(&self) -> bool {- self.ranges.is_empty()+ self.free_ranges.total_len() == self.full_len+ }++ fn ensure_pool(+ &mut self,+ pool: usize,+ ) {+ let pool_len = pool + 1;++ self.aliasable_ranges_by_pool.resize_with(pool_len, AvailableRanges::default);+ self.pool_ranges_by_pool.resize_with(pool_len, AvailableRanges::default);+ self.pool_live_allocations.resize(pool_len, 0);}}diff --git a/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs b/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rsnew file mode 100644index 000000000..bc4351f06--- /dev/null+++ b/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs@@ -0,0 +1,128 @@+use std::{+ collections::{HashMap, hash_map::Entry},+ fs::File,+ io::BufReader,+ rc::Rc,+ time::Instant,+};++use serde::{Deserialize, Serialize};++use crate::backends::{+ common::{Allocation, AllocationPool, AllocationType, Allocator, Backend, Context},+ metal::Metal,+};++#[derive(Serialize, Deserialize)]+struct TraceEvent {+ allocator_id: usize,+ event_type: TraceEventType,+}++#[derive(Serialize, Deserialize)]+pub enum TraceEventAllocationType {+ Global,+ Pooled {+ pool_id: usize,+ cpu_available: bool,+ },+}++#[derive(Serialize, Deserialize)]+enum TraceEventType {+ Create,+ CreatePool {+ reusable: bool,+ pool_id: usize,+ },+ Allocate {+ size: usize,+ allocation_type: TraceEventAllocationType,+ allocation_id: usize,+ },+ Free {+ allocation_id: usize,+ },+ FreePool {+ pool_id: usize,+ },+}++#[test]+#[ignore]+fn bench_allocator_generation_trace() {+ let file = BufReader::with_capacity(1 << 20, File::open("/tmp/allocator_trace.jsonl").unwrap());+ let events: Vec<TraceEvent> = serde_json::Deserializer::from_reader(file)+ .into_iter::<TraceEvent>()+ .map(|r| r.expect("malformed trace event"))+ .collect();++ let context = <<Metal as Backend>::Context as Context>::new().unwrap();++ let mut allocators: HashMap<usize, Rc<Allocator<Metal>>> = HashMap::new();+ let mut pools: HashMap<(usize, usize), AllocationPool<Metal>> = HashMap::new();+ let mut allocations: HashMap<(usize, usize), Allocation<Metal>> = HashMap::new();++ let start = Instant::now();++ for event in events {+ match event.event_type {+ TraceEventType::Create => {+ let Entry::Vacant(vacant) = allocators.entry(event.allocator_id) else {+ panic!("Duplicate allocator id: {}", event.allocator_id);+ };++ vacant.insert(Allocator::new(Rc::downgrade(&context)));+ },+ TraceEventType::CreatePool {+ reusable,+ pool_id,+ } => {+ let Entry::Vacant(vacant) = pools.entry((event.allocator_id, pool_id)) else {+ panic!("Duplicate pool id {} in allocator {}", pool_id, event.allocator_id);+ };++ vacant.insert(allocators[&event.allocator_id].create_pool(reusable));+ },+ TraceEventType::Allocate {+ size,+ allocation_type,+ allocation_id,+ } => {+ let Entry::Vacant(vacant) = allocations.entry((event.allocator_id, allocation_id)) else {+ panic!("Duplicate allocation id {} in allocator {}", allocation_id, event.allocator_id);+ };++ let allocation_type = match allocation_type {+ TraceEventAllocationType::Global => AllocationType::Global,+ TraceEventAllocationType::Pooled {+ pool_id,+ cpu_available,+ } => AllocationType::Pooled {+ pool: &pools[&(event.allocator_id, pool_id)],+ cpu_available,+ },+ };++ vacant.insert(allocators[&event.allocator_id].allocate(size, allocation_type).unwrap());+ },+ TraceEventType::Free {+ allocation_id,+ } => {+ allocations.remove(&(event.allocator_id, allocation_id)).unwrap();+ },+ TraceEventType::FreePool {+ pool_id,+ } => {+ pools.remove(&(event.allocator_id, pool_id)).unwrap();+ },+ }+ }++ let elapsed = start.elapsed();++ eprintln!();+ eprintln!("--------------------");+ eprintln!("Cpu time: {} ms", elapsed.as_millis());+ eprintln!("--------------------");+}