Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Manual blit to workaround profiling segfault (#934)

uuuvn8b5aed4bf99b
diff --git a/crates/uzu-engine/build/metal/ast.rs b/crates/uzu-engine/build/metal/ast.rs
index 01ce86c74..94934059d 100644
--- a/crates/uzu-engine/build/metal/ast.rs
+++ b/crates/uzu-engine/build/metal/ast.rs
@@ -200,6 +200,7 @@ impl MetalArgument {
match tokens.as_slice() {
["bool"] => Ok("bool".into()),
["uint"] | ["uint32_t"] | ["unsigned", "int"] => Ok("u32".into()),
+ ["ulong"] | ["uint64_t"] | ["unsigned", "long"] => Ok("u64".into()),
["int"] | ["int32_t"] => Ok("i32".into()),
["float"] => Ok("f32".into()),
[vpath] if vpath.starts_with("uzu::") => {
diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rs
index 727de6a8d..02565f5e8 100644
--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs
@@ -8,7 +8,7 @@ use metal::{
MTL4ArgumentTable, MTL4ArgumentTableDescriptor, MTL4CommandAllocator, MTL4CommandBuffer, MTL4CommandBufferExt,
MTL4CommandEncoder, MTL4CommandEncoderExt, MTL4CommandQueue, MTL4CommandQueueExt, MTL4CommitFeedback,
MTL4CommitFeedbackExt, MTL4CommitFeedbackHandler, MTL4CommitOptions, MTL4ComputeCommandEncoder,
- MTL4ComputeCommandEncoderExt, MTL4VisibilityOptions, MTLDeviceExt, MTLStages,
+ MTL4VisibilityOptions, MTLDeviceExt, MTLStages,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
use rangemap::RangeSet;
@@ -18,7 +18,12 @@ use crate::backends::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,
CommandBufferExecutable, CommandBufferPending, Context, allocator::bump::BumpAllocator,
},
- metal::{Metal, MetalContext, buffer::MetalBufferExt, error::MetalError},
+ metal::{
+ Metal, MetalContext,
+ buffer::MetalBufferExt,
+ error::MetalError,
+ kernel::{BlitCopyMetalKernel, BlitFillMetalKernel},
+ },
};
pub struct MetalCommandBuffer;
@@ -70,8 +75,8 @@ impl MetalCommandBufferEncoding {
ProtocolObject::<dyn MTL4CommandEncoder>::set_label(compute_encoder.as_ref(), name);
compute_encoder.barrier_after_queue_stages_before_stages_visibility_options(
- MTLStages::Dispatch | MTLStages::Blit | MTLStages::ResourceState,
- MTLStages::Dispatch | MTLStages::Blit,
+ MTLStages::Dispatch | MTLStages::ResourceState,
+ MTLStages::Dispatch,
MTL4VisibilityOptions::Device,
);
@@ -110,8 +115,8 @@ impl MetalCommandBufferEncoding {
self.writes.overlaps(&access.range.into()) || (access.write && self.reads.overlaps(&access.range.into()))
}) {
self.compute_encoder.barrier_after_encoder_stages_before_encoder_stages_visibility_options(
- MTLStages::Dispatch | MTLStages::Blit,
- MTLStages::Dispatch | MTLStages::Blit,
+ MTLStages::Dispatch,
+ MTLStages::Dispatch,
MTL4VisibilityOptions::Device,
);
self.reads.clear();
@@ -154,30 +159,25 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
src: impl BufferRef<Backend = Metal>,
dst: impl BufferMut<Backend = Metal>,
) {
- let (src, src_range) = src.parts();
- let (dst, dst_range) = dst.parts();
- assert_eq!(src_range.iter().len(), dst_range.iter().len());
-
- self.access(&[
- Access {
- range: src.gpu_address_subrange(src_range),
- write: false,
- },
- Access {
- range: dst.gpu_address_subrange(dst_range),
- write: true,
- },
- ]);
-
- let (src_buffer, src_offset) = src.downcast();
- let (dst_buffer, dst_offset) = dst.downcast();
- self.compute_encoder.copy_from_buffer_source_offset_to_buffer_destination_offset_size(
- src_buffer,
- src_offset + src_range.start,
- dst_buffer,
- dst_offset + dst_range.start,
- src_range.iter().len(),
- );
+ let size = src.size();
+ assert_eq!(size, dst.size());
+ if size == 0 {
+ return;
+ }
+ let (src_buffer, src_range) = src.parts();
+ let (dst_buffer, dst_range) = dst.as_ref().parts();
+ let alignment = ((src_buffer.gpu_address() + src_range.start as u64)
+ | (dst_buffer.gpu_address() + dst_range.start as u64))
+ & 15;
+ let (variant, alignment) = match alignment {
+ 0 => (0, 0),
+ 4 | 8 | 12 => (1, 4),
+ _ => (2, 1),
+ };
+ let context = self.context.clone();
+ let kernel = context.blit_copy[variant]
+ .get_or_init(|| BlitCopyMetalKernel::new(&context, alignment).expect("Failed to create blit copy kernel"));
+ kernel.encode(src, dst, size as u64, self);
}
fn encode_fill(
@@ -185,17 +185,14 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
dst: impl BufferMut<Backend = Metal>,
value: u8,
) {
- let (dst, range) = dst.parts();
+ let (buffer, range) = dst.as_ref().parts();
assert!(range.end > range.start);
assert!(range.start.is_multiple_of(4) && range.end.is_multiple_of(4));
-
- self.access(&[Access {
- range: dst.gpu_address_subrange(range),
- write: true,
- }]);
-
- let (buffer, offset) = dst.downcast();
- self.compute_encoder.fill_buffer_range_value(buffer, offset + range.start..offset + range.end, value);
+ let aligned = (buffer.gpu_address() + range.start as u64).is_multiple_of(16);
+ let context = self.context.clone();
+ let kernel = context.blit_fill[usize::from(aligned)]
+ .get_or_init(|| BlitFillMetalKernel::new(&context, aligned).expect("Failed to create blit fill kernel"));
+ kernel.encode(dst, (range.end - range.start) as u64, u32::from(value) * 0x0101_0101, self);
}
// TODO: maybe port previous debug command_buffer labels
@@ -234,8 +231,8 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
impl Drop for MetalCommandBufferEncoding {
fn drop(&mut self) {
self.compute_encoder.barrier_after_stages_before_queue_stages_visibility_options(
- MTLStages::Dispatch | MTLStages::Blit,
- MTLStages::Dispatch | MTLStages::Blit | MTLStages::ResourceState,
+ MTLStages::Dispatch,
+ MTLStages::Dispatch | MTLStages::ResourceState,
MTL4VisibilityOptions::Device,
);
self.compute_encoder.end_encoding();
diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rs
index dd77f6ed3..bf64fd131 100644
--- a/crates/uzu-engine/src/backends/metal/context.rs
+++ b/crates/uzu-engine/src/backends/metal/context.rs
@@ -2,7 +2,7 @@ use std::{
collections::HashMap,
path::Path,
sync::{
- Arc, Weak,
+ Arc, OnceLock, Weak,
atomic::{AtomicUsize, Ordering},
},
};
@@ -29,6 +29,7 @@ use crate::backends::{
decompression,
error::MetalError,
heaps::MetalHeaps,
+ kernel::{BlitCopyMetalKernel, BlitFillMetalKernel},
metal_extensions::{DeviceExt, LibraryPipelineExtensions},
},
};
@@ -57,6 +58,8 @@ pub struct MetalContext {
pub(super) command_allocator_cache: Mutex<Vec<Retained<ProtocolObject<dyn MTL4CommandAllocator>>>>,
library_cache: Mutex<HashMap<usize, Retained<ProtocolObject<dyn MTLLibrary>>>>,
pipeline_cache: Mutex<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,
+ pub(super) blit_copy: [OnceLock<BlitCopyMetalKernel>; 3],
+ pub(super) blit_fill: [OnceLock<BlitFillMetalKernel>; 2],
weak_self: Weak<MetalContext>,
}
@@ -176,6 +179,8 @@ impl Context for MetalContext {
command_allocator_cache: Mutex::new(Vec::with_capacity(32)),
library_cache: Mutex::new(HashMap::new()),
pipeline_cache: Mutex::new(HashMap::new()),
+ blit_copy: std::array::from_fn(|_| OnceLock::new()),
+ blit_fill: std::array::from_fn(|_| OnceLock::new()),
weak_self: weak_self.clone(),
}))
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/blit.metal b/crates/uzu-engine/src/backends/metal/kernel/blit.metal
new file mode 100644
index 000000000..98730163c
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/blit.metal
@@ -0,0 +1,55 @@
+#include <metal_stdlib>
+#include "common/dsl.h"
+
+#define BLIT_MAX_THREADS 16777216
+
+KERNEL(BlitCopy)(
+ const device uchar* src,
+ device uchar* dst,
+ constant uint64_t& size,
+ const uint alignment SPECIALIZE,
+ uint position AXIS(size.div_ceil(16).min(BLIT_MAX_THREADS), 256)
+) {
+ for (ulong index = position; index < (size + 15) / 16; index += BLIT_MAX_THREADS) {
+ const ulong offset = index * 16;
+ if (size - offset >= 16 && alignment == 0) {
+ reinterpret_cast<device uint4*>(dst)[index] = reinterpret_cast<const device uint4*>(src)[index];
+ } else if (size - offset >= 16 && (alignment & 3) == 0) {
+ reinterpret_cast<device packed_uint4*>(dst)[index] = reinterpret_cast<const device packed_uint4*>(src)[index];
+ } else if ((alignment & 3) == 0) {
+ for (ulong word = offset / 4; word < size / 4; ++word) {
+ reinterpret_cast<device uint*>(dst)[word] = reinterpret_cast<const device uint*>(src)[word];
+ }
+ for (ulong byte = size / 4 * 4; byte < size; ++byte) {
+ dst[byte] = src[byte];
+ }
+ } else {
+ for (ulong i = offset; i < min(offset + 16, size); ++i) {
+ dst[i] = src[i];
+ }
+ }
+ }
+}
+
+KERNEL(BlitFill)(
+ device uint* dst,
+ constant uint64_t& size,
+ constant uint& value,
+ const bool aligned SPECIALIZE,
+ uint position AXIS(size.div_ceil(16).min(BLIT_MAX_THREADS), 256)
+) {
+ for (ulong index = position; index < (size + 15) / 16; index += BLIT_MAX_THREADS) {
+ const ulong offset = index * 16;
+ if (size - offset >= 16) {
+ if (aligned) {
+ reinterpret_cast<device uint4*>(dst)[index] = uint4(value);
+ } else {
+ reinterpret_cast<device packed_uint4*>(dst)[index] = uint4(value);
+ }
+ } else {
+ for (ulong i = offset / 4; i < size / 4; ++i) {
+ dst[i] = value;
+ }
+ }
+ }
+}