Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add vulkan buffers

eugene1429c21f0520
diff --git a/crates/uzu-engine/src/backends/common/buffer/global.rs b/crates/uzu-engine/src/backends/common/buffer/global.rs
index a117557d8..d119c6010 100644
--- a/crates/uzu-engine/src/backends/common/buffer/global.rs
+++ b/crates/uzu-engine/src/backends/common/buffer/global.rs
@@ -3,5 +3,5 @@ use crate::backends::common::{Backend, CpuBuffer, GpuBuffer};
pub trait GlobalGpuBuffer: GpuBuffer<Backend: Backend<GlobalGpuBuffer = Self>> {}
pub trait GlobalBufferCpuGpu: CpuBuffer<Backend: Backend<GlobalBufferCpuGpu = Self>> {
- fn handoff(self) -> <Self::Backend as Backend>::GlobalGpuBuffer;
+ fn handoff(self) -> Result<<Self::Backend as Backend>::GlobalGpuBuffer, <Self::Backend as Backend>::Error>;
}
diff --git a/crates/uzu-engine/src/backends/cpu/buffer/dense.rs b/crates/uzu-engine/src/backends/cpu/buffer/dense.rs
index d354743e5..93182f2a2 100644
--- a/crates/uzu-engine/src/backends/cpu/buffer/dense.rs
+++ b/crates/uzu-engine/src/backends/cpu/buffer/dense.rs
@@ -6,7 +6,7 @@ use std::{
use crate::backends::{
common::{Buffer, CpuBuffer, GlobalBufferCpuGpu, GlobalGpuBuffer, GpuBuffer},
- cpu::Cpu,
+ cpu::{Cpu, error::CpuError},
};
#[derive(Debug)]
@@ -56,7 +56,7 @@ impl CpuBuffer for CpuDenseBuffer {
impl GlobalGpuBuffer for CpuDenseBuffer {}
impl GlobalBufferCpuGpu for CpuDenseBuffer {
- fn handoff(self) -> Self {
- self
+ fn handoff(self) -> Result<Self, CpuError> {
+ Ok(self)
}
}
diff --git a/crates/uzu-engine/src/backends/metal/buffer/dense.rs b/crates/uzu-engine/src/backends/metal/buffer/dense.rs
index 9a21df27c..657f86275 100644
--- a/crates/uzu-engine/src/backends/metal/buffer/dense.rs
+++ b/crates/uzu-engine/src/backends/metal/buffer/dense.rs
@@ -74,7 +74,7 @@ impl CpuBuffer for MetalDenseBuffer {
impl GlobalGpuBuffer for BlockAllocation<MetalDenseBuffer> {}
impl GlobalBufferCpuGpu for BlockAllocation<MetalDenseBuffer> {
- fn handoff(self) -> Self {
- self
+ fn handoff(self) -> Result<Self, MetalError> {
+ Ok(self)
}
}
diff --git a/crates/uzu-engine/src/backends/vulkan/buffer/mod.rs b/crates/uzu-engine/src/backends/vulkan/buffer/mod.rs
new file mode 100644
index 000000000..869162333
--- /dev/null
+++ b/crates/uzu-engine/src/backends/vulkan/buffer/mod.rs
@@ -0,0 +1,5 @@
+mod vk_buffer;
+mod vk_buffer_initializer;
+
+pub use vk_buffer::{VkBuffer, VkBufferMemory, VkSparseBuffer};
+pub use vk_buffer_initializer::VkBufferInitializer;
diff --git a/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer.rs b/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer.rs
new file mode 100644
index 000000000..e324c7209
--- /dev/null
+++ b/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer.rs
@@ -0,0 +1,173 @@
+use std::{convert::Infallible, ops::Range, os::raw::c_void, ptr::NonNull, sync::Arc};
+
+use ash::vk;
+
+use crate::{
+ backends::{
+ common::{
+ Buffer, BufferRef, CpuBuffer, GlobalGpuBuffer, GpuBuffer, SparseBuffer,
+ allocator::{bump::BumpAllocation, pool::PoolAllocation},
+ },
+ vulkan::{Error, VkContext, Vulkan},
+ },
+ utils::downcast::downcast_ref,
+};
+
+pub struct VkBuffer {
+ context: Arc<VkContext>,
+ allocation: vk_mem::Allocation,
+ buffer: vk::Buffer,
+ device_address: vk::DeviceAddress,
+ cpu_ptr: Option<NonNull<c_void>>,
+ size: vk::DeviceSize,
+}
+
+unsafe impl Send for VkBuffer {}
+unsafe impl Sync for VkBuffer {}
+
+impl VkBuffer {
+ pub fn new(
+ context: Arc<VkContext>,
+ size: usize,
+ memory: VkBufferMemory,
+ ) -> Result<Arc<Self>, Error> {
+ let size = size as u64;
+ let (buffer, allocation, info) = context.allocate_buffer(size, memory)?;
+ let cpu_ptr = (memory != VkBufferMemory::Device)
+ .then(|| NonNull::new(info.mapped_data).expect("VMA maps host-visible MAPPED allocations"));
+ let device_address = unsafe {
+ context.device().get_buffer_device_address(&vk::BufferDeviceAddressInfo::default().buffer(buffer))
+ };
+ Ok(Arc::new(Self {
+ context,
+ allocation,
+ buffer,
+ device_address,
+ cpu_ptr,
+ size,
+ }))
+ }
+
+ pub fn context(&self) -> &Arc<VkContext> {
+ &self.context
+ }
+
+ pub fn buffer(&self) -> vk::Buffer {
+ self.buffer
+ }
+
+ pub fn device_address(&self) -> vk::DeviceAddress {
+ self.device_address
+ }
+
+ pub fn size(&self) -> vk::DeviceSize {
+ self.size
+ }
+
+ pub fn resolve(
+ buffer: &(impl Buffer<Backend = Vulkan> + ?Sized),
+ range: std::range::Range<usize>,
+ ) -> (&Arc<VkBuffer>, Range<u64>) {
+ let (buffer, offset) = if let Some(buffer) = downcast_ref::<Arc<VkBuffer>>(buffer) {
+ (buffer, 0)
+ } else if let Some(allocation) = downcast_ref::<BumpAllocation<Arc<VkBuffer>>>(buffer) {
+ (allocation.page(), allocation.range().start)
+ } else if let Some(allocation) = downcast_ref::<PoolAllocation<Arc<VkBuffer>>>(buffer) {
+ (allocation.page(), allocation.range().start)
+ } else {
+ unreachable!("unsupported Vulkan buffer type")
+ };
+ (buffer, (offset + range.start) as u64..(offset + range.end) as u64)
+ }
+
+ pub fn resolve_ref<'a>(buffer: impl BufferRef<Backend = Vulkan> + 'a) -> (&'a Arc<Self>, Range<u64>) {
+ let (buffer, range) = buffer.parts();
+ Self::resolve(buffer, range)
+ }
+}
+
+impl Drop for VkBuffer {
+ fn drop(&mut self) {
+ unsafe { self.context.free_buffer(self.buffer, &mut self.allocation) };
+ }
+}
+
+impl Buffer for Arc<VkBuffer> {
+ type Backend = Vulkan;
+
+ fn size(&self) -> usize {
+ VkBuffer::size(self) as usize
+ }
+}
+
+impl GpuBuffer for Arc<VkBuffer> {}
+
+impl CpuBuffer for Arc<VkBuffer> {
+ fn cpu_ptr(&self) -> NonNull<c_void> {
+ self.cpu_ptr.expect("device-local Vulkan buffers are not CPU-accessible")
+ }
+}
+
+impl GlobalGpuBuffer for Arc<VkBuffer> {}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum VkBufferMemory {
+ Device,
+ Staging,
+ Readback,
+}
+
+impl VkBufferMemory {
+ pub fn allocation_create_info(
+ self,
+ discrete: bool,
+ ) -> vk_mem::AllocationCreateInfo {
+ let mapped = vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT;
+ let write = vk_mem::AllocationCreateFlags::MAPPED | vk_mem::AllocationCreateFlags::HOST_ACCESS_SEQUENTIAL_WRITE;
+ let (usage, flags, required_flags) = match self {
+ Self::Device if discrete => (
+ vk_mem::MemoryUsage::AutoPreferDevice,
+ vk_mem::AllocationCreateFlags::WITHIN_BUDGET,
+ vk::MemoryPropertyFlags::DEVICE_LOCAL,
+ ),
+ Self::Device => {
+ (vk_mem::MemoryUsage::AutoPreferDevice, vk_mem::AllocationCreateFlags::empty(), Default::default())
+ },
+ Self::Staging => (vk_mem::MemoryUsage::AutoPreferHost, write, mapped),
+ Self::Readback => (
+ vk_mem::MemoryUsage::AutoPreferHost,
+ vk_mem::AllocationCreateFlags::MAPPED | vk_mem::AllocationCreateFlags::HOST_ACCESS_RANDOM,
+ mapped,
+ ),
+ };
+ vk_mem::AllocationCreateInfo {
+ usage,
+ flags,
+ required_flags,
+ ..Default::default()
+ }
+ }
+}
+
+pub struct VkSparseBuffer {
+ never: Infallible,
+}
+
+impl Buffer for VkSparseBuffer {
+ type Backend = Vulkan;
+
+ fn size(&self) -> usize {
+ match self.never {}
+ }
+}
+
+impl GpuBuffer for VkSparseBuffer {}
+
+impl SparseBuffer for VkSparseBuffer {
+ fn map(
+ &mut self,
+ _until: usize,
+ ) -> Result<(), Error> {
+ match self.never {}
+ }
+}
diff --git a/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer_initializer.rs b/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer_initializer.rs
new file mode 100644
index 000000000..38047533c
--- /dev/null
+++ b/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer_initializer.rs
@@ -0,0 +1,52 @@
+use std::{os::raw::c_void, ptr::NonNull, sync::Arc};
+
+use crate::backends::{
+ common::{Buffer, CommandBufferEncoding, Context, CpuBuffer, GlobalBufferCpuGpu},
+ vulkan::{Error, VkBuffer, VkBufferMemory, VkContext, VkDeviceCapabilities, Vulkan},
+};
+
+pub struct VkBufferInitializer {
+ host: Arc<VkBuffer>,
+ device: Option<Arc<VkBuffer>>,
+}
+
+impl VkBufferInitializer {
+ pub fn new(
+ context: Arc<VkContext>,
+ size: usize,
+ ) -> Result<Self, Error> {
+ let device = (!context.physical_device().capabilities.contains(VkDeviceCapabilities::UNIFIED_MEMORY))
+ .then(|| VkBuffer::new(context.clone(), size, VkBufferMemory::Device))
+ .transpose()?;
+ Ok(Self {
+ host: VkBuffer::new(context, size, VkBufferMemory::Staging)?,
+ device,
+ })
+ }
+}
+
+impl Buffer for VkBufferInitializer {
+ type Backend = Vulkan;
+
+ fn size(&self) -> usize {
+ Buffer::size(&self.host)
+ }
+}
+
+impl CpuBuffer for VkBufferInitializer {
+ fn cpu_ptr(&self) -> NonNull<c_void> {
+ self.host.cpu_ptr()
+ }
+}
+
+impl GlobalBufferCpuGpu for VkBufferInitializer {
+ fn handoff(self) -> Result<Arc<VkBuffer>, Error> {
+ let Some(mut device) = self.device else {
+ return Ok(self.host);
+ };
+ let mut encoding = self.host.context().create_command_buffer(None, None)?;
+ encoding.encode_copy(&self.host, &mut device);
+ encoding.end_encoding().try_submit()?;
+ Ok(device)
+ }
+}
diff --git a/crates/uzu-engine/src/backends/vulkan/device/vk_physical_device.rs b/crates/uzu-engine/src/backends/vulkan/device/vk_physical_device.rs
index 35752cf0e..4381e9dab 100644
--- a/crates/uzu-engine/src/backends/vulkan/device/vk_physical_device.rs
+++ b/crates/uzu-engine/src/backends/vulkan/device/vk_physical_device.rs
@@ -215,6 +215,80 @@ impl VkPhysicalDevice {
}
}
+impl VkPhysicalDevice {
+ pub fn create_device(
+ &self,
+ instance: &ash::Instance,
+ queue_family_index: u32,
+ ) -> Result<ash::Device, VkContextError> {
+ let queue_priorities = [1.0_f32];
+ let queue_create_info = vk::DeviceQueueCreateInfo::default()
+ .queue_family_index(queue_family_index)
+ .queue_priorities(&queue_priorities);
+
+ let bf16_type = self.capabilities.contains(VkDeviceCapabilities::BF16_TYPE);
+ let matrices = self
+ .capabilities
+ .intersects(VkDeviceCapabilities::BF16_TENSOR_CORES | VkDeviceCapabilities::INT8_TENSOR_CORES);
+ let element_fold = self.capabilities.contains(VkDeviceCapabilities::INT8_ELEMENT_FOLD);
+ let mut device_extensions = Vec::new();
+ if matrices {
+ device_extensions.push(khr::cooperative_matrix::NAME.as_ptr());
+ }
+ if bf16_type {
+ device_extensions.push(VkShaderBfloat16Features::EXTENSION_NAME.as_ptr());
+ }
+ if element_fold {
+ device_extensions.push(VkCooperativeMatrix2Features::EXTENSION_NAME.as_ptr());
+ }
+
+ let vk10_features = vk::PhysicalDeviceFeatures::default().shader_int16(true);
+ let mut vk11_features = vk::PhysicalDeviceVulkan11Features::default()
+ .storage_buffer16_bit_access(true)
+ .storage_push_constant16(true);
+ let mut vk12_features = vk::PhysicalDeviceVulkan12Features::default()
+ .shader_int8(true)
+ .storage_buffer8_bit_access(true)
+ .storage_push_constant8(true)
+ .shader_float16(true)
+ .shader_subgroup_extended_types(true)
+ .buffer_device_address(true)
+ .vulkan_memory_model(true)
+ .vulkan_memory_model_device_scope(true);
+ let mut vk13_features = vk::PhysicalDeviceVulkan13Features::default()
+ .maintenance4(true)
+ .synchronization2(true)
+ .subgroup_size_control(true)
+ .compute_full_subgroups(true)
+ .shader_integer_dot_product(true);
+
+ let mut matrix_features = vk::PhysicalDeviceCooperativeMatrixFeaturesKHR::default().cooperative_matrix(true);
+ let mut bfloat16_features = VkShaderBfloat16Features::default();
+ bfloat16_features.shader_bfloat16_type = vk::TRUE;
+ bfloat16_features.shader_bfloat16_cooperative_matrix =
+ self.capabilities.contains(VkDeviceCapabilities::BF16_TENSOR_CORES).into();
+ let mut matrix2_features = VkCooperativeMatrix2Features::default();
+ matrix2_features.cooperative_matrix_per_element_operations = vk::TRUE;
+ let mut device_create_info = vk::DeviceCreateInfo::default()
+ .queue_create_infos(std::slice::from_ref(&queue_create_info))
+ .enabled_extension_names(&device_extensions)
+ .enabled_features(&vk10_features)
+ .push_next(&mut vk11_features)
+ .push_next(&mut vk12_features)
+ .push_next(&mut vk13_features);
+ if matrices {
+ device_create_info = device_create_info.push_next(&mut matrix_features);
+ }
+ if bf16_type {
+ device_create_info = device_create_info.push_next(&mut bfloat16_features);
+ }
+ if element_fold {
+ device_create_info = device_create_info.push_next(&mut matrix2_features);
+ }
+ unsafe { instance.create_device(self.handle, &device_create_info, None) }.map_err(VkContextError::DeviceCreate)
+ }
+}
+
pub struct VkDeviceLimits {
pub max_work_group_count: [u32; 3],
pub max_work_group_size: [u32; 3],
diff --git a/crates/uzu-engine/src/backends/vulkan/error.rs b/crates/uzu-engine/src/backends/vulkan/error.rs
index 9e9928dc6..245cf353f 100644
--- a/crates/uzu-engine/src/backends/vulkan/error.rs
+++ b/crates/uzu-engine/src/backends/vulkan/error.rs
@@ -1,5 +1,3 @@
-use std::sync::Arc;
-
use ash::vk;
use crate::data_type::DataType;
@@ -91,9 +89,6 @@ pub enum VkContextError {
#[error("the Vulkan validation layer is not installed")]
ValidationNotSupported,
-
- #[error("GPU work that nobody awaited failed, so buffers may hold wrong data: {0}")]
- Failed(#[source] Arc<Error>),
}
#[derive(Debug, thiserror::Error)]
diff --git a/crates/uzu-engine/src/backends/vulkan/mod.rs b/crates/uzu-engine/src/backends/vulkan/mod.rs
index d225d1a3d..8feb3b4ea 100644
--- a/crates/uzu-engine/src/backends/vulkan/mod.rs
+++ b/crates/uzu-engine/src/backends/vulkan/mod.rs
@@ -1,5 +1,7 @@
+mod buffer;
mod device;
mod error;
+pub use buffer::*;
pub use device::*;
pub use error::{Error, VkBufferError, VkCommandBufferError, VkContextError, VkKernelError, VkPipelineError};
diff --git a/crates/uzu-engine/src/backends/webgpu/buffer/dense.rs b/crates/uzu-engine/src/backends/webgpu/buffer/dense.rs
index 2caf69f0f..09779360c 100644
--- a/crates/uzu-engine/src/backends/webgpu/buffer/dense.rs
+++ b/crates/uzu-engine/src/backends/webgpu/buffer/dense.rs
@@ -117,12 +117,12 @@ impl CpuBuffer for WebGPUDenseBufferCpuGpu {
}
impl GlobalBufferCpuGpu for WebGPUDenseBufferCpuGpu {
- fn handoff(self) -> WebGPUDenseBufferGpu {
+ fn handoff(self) -> Result<WebGPUDenseBufferGpu, WebGPUError> {
let gpu = WebGPUDenseBufferGpu {
buffer: self.buffer.clone(),
size: self.size,
};
drop(self);
- gpu
+ Ok(gpu)
}
}
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index ed6af073a..fb6cd9fb9 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -134,7 +134,7 @@ impl<'a, B: Backend> StreamEnqueuer<'a, B> {
.collect::<Box<[_]>>(),
);
- Some(context_ring.handoff())
+ Some(context_ring.handoff().map_err(LanguageModelStreamError::Backend)?)
} else {
None
};
diff --git a/crates/uzu-engine/src/safetensors.rs b/crates/uzu-engine/src/safetensors.rs
index a5082829e..2e1f9139a 100644
--- a/crates/uzu-engine/src/safetensors.rs
+++ b/crates/uzu-engine/src/safetensors.rs
@@ -213,7 +213,7 @@ impl<B: Backend> ParameterTree<'_, B> {
.map_err(ParameterLoaderError::BackendError)?;
self.loader.file.read_exact_at(buffer.as_slice_mut::<u8>(), self.loader.data_offset + start).await?;
tensor.read.store(true, Ordering::Relaxed);
- Ok(buffer.handoff())
+ buffer.handoff().map_err(ParameterLoaderError::BackendError)
}
pub async fn read_slice<T: DataTypeRepr>(
diff --git a/crates/uzu-engine/unit/common/helpers.rs b/crates/uzu-engine/unit/common/helpers.rs
index 53acfa772..ee4c5b9c1 100644
--- a/crates/uzu-engine/unit/common/helpers.rs
+++ b/crates/uzu-engine/unit/common/helpers.rs
@@ -61,7 +61,7 @@ pub fn create_buffer_with_data<B: Backend, T: DataTypeRepr>(
let mut buffer =
context.create_buffer_cpu_gpu(buffer_size_bytes::<T>(data.len())).expect("Failed to create buffer");
buffer.copyin(data);
- buffer.handoff()
+ buffer.handoff().expect("Failed to hand off buffer")
}
pub fn buffer_to_vec<B: Backend, T: DataTypeRepr>(