A high-performance inference engine for AI models
add vulkan buffers
eugene1429c21f0520
diff --git a/crates/uzu-engine/src/backends/common/buffer/global.rs b/crates/uzu-engine/src/backends/common/buffer/global.rsindex a117557d8..d119c6010 100644--- a/crates/uzu-engine/src/backends/common/buffer/global.rs+++ b/crates/uzu-engine/src/backends/common/buffer/global.rs@@ -3,5 +3,5 @@ use crate::backends::common::{Backend, CpuBuffer, GpuBuffer};pub trait GlobalGpuBuffer: GpuBuffer<Backend: Backend<GlobalGpuBuffer = Self>> {}pub trait GlobalBufferCpuGpu: CpuBuffer<Backend: Backend<GlobalBufferCpuGpu = Self>> {- fn handoff(self) -> <Self::Backend as Backend>::GlobalGpuBuffer;+ fn handoff(self) -> Result<<Self::Backend as Backend>::GlobalGpuBuffer, <Self::Backend as Backend>::Error>;}diff --git a/crates/uzu-engine/src/backends/cpu/buffer/dense.rs b/crates/uzu-engine/src/backends/cpu/buffer/dense.rsindex d354743e5..93182f2a2 100644--- a/crates/uzu-engine/src/backends/cpu/buffer/dense.rs+++ b/crates/uzu-engine/src/backends/cpu/buffer/dense.rs@@ -6,7 +6,7 @@ use std::{use crate::backends::{common::{Buffer, CpuBuffer, GlobalBufferCpuGpu, GlobalGpuBuffer, GpuBuffer},- cpu::Cpu,+ cpu::{Cpu, error::CpuError},};#[derive(Debug)]@@ -56,7 +56,7 @@ impl CpuBuffer for CpuDenseBuffer {impl GlobalGpuBuffer for CpuDenseBuffer {}impl GlobalBufferCpuGpu for CpuDenseBuffer {- fn handoff(self) -> Self {- self+ fn handoff(self) -> Result<Self, CpuError> {+ Ok(self)}}diff --git a/crates/uzu-engine/src/backends/metal/buffer/dense.rs b/crates/uzu-engine/src/backends/metal/buffer/dense.rsindex 9a21df27c..657f86275 100644--- a/crates/uzu-engine/src/backends/metal/buffer/dense.rs+++ b/crates/uzu-engine/src/backends/metal/buffer/dense.rs@@ -74,7 +74,7 @@ impl CpuBuffer for MetalDenseBuffer {impl GlobalGpuBuffer for BlockAllocation<MetalDenseBuffer> {}impl GlobalBufferCpuGpu for BlockAllocation<MetalDenseBuffer> {- fn handoff(self) -> Self {- self+ fn handoff(self) -> Result<Self, MetalError> {+ Ok(self)}}diff --git a/crates/uzu-engine/src/backends/vulkan/buffer/mod.rs b/crates/uzu-engine/src/backends/vulkan/buffer/mod.rsnew file mode 100644index 000000000..869162333--- /dev/null+++ b/crates/uzu-engine/src/backends/vulkan/buffer/mod.rs@@ -0,0 +1,5 @@+mod vk_buffer;+mod vk_buffer_initializer;++pub use vk_buffer::{VkBuffer, VkBufferMemory, VkSparseBuffer};+pub use vk_buffer_initializer::VkBufferInitializer;diff --git a/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer.rs b/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer.rsnew file mode 100644index 000000000..e324c7209--- /dev/null+++ b/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer.rs@@ -0,0 +1,173 @@+use std::{convert::Infallible, ops::Range, os::raw::c_void, ptr::NonNull, sync::Arc};++use ash::vk;++use crate::{+ backends::{+ common::{+ Buffer, BufferRef, CpuBuffer, GlobalGpuBuffer, GpuBuffer, SparseBuffer,+ allocator::{bump::BumpAllocation, pool::PoolAllocation},+ },+ vulkan::{Error, VkContext, Vulkan},+ },+ utils::downcast::downcast_ref,+};++pub struct VkBuffer {+ context: Arc<VkContext>,+ allocation: vk_mem::Allocation,+ buffer: vk::Buffer,+ device_address: vk::DeviceAddress,+ cpu_ptr: Option<NonNull<c_void>>,+ size: vk::DeviceSize,+}++unsafe impl Send for VkBuffer {}+unsafe impl Sync for VkBuffer {}++impl VkBuffer {+ pub fn new(+ context: Arc<VkContext>,+ size: usize,+ memory: VkBufferMemory,+ ) -> Result<Arc<Self>, Error> {+ let size = size as u64;+ let (buffer, allocation, info) = context.allocate_buffer(size, memory)?;+ let cpu_ptr = (memory != VkBufferMemory::Device)+ .then(|| NonNull::new(info.mapped_data).expect("VMA maps host-visible MAPPED allocations"));+ let device_address = unsafe {+ context.device().get_buffer_device_address(&vk::BufferDeviceAddressInfo::default().buffer(buffer))+ };+ Ok(Arc::new(Self {+ context,+ allocation,+ buffer,+ device_address,+ cpu_ptr,+ size,+ }))+ }++ pub fn context(&self) -> &Arc<VkContext> {+ &self.context+ }++ pub fn buffer(&self) -> vk::Buffer {+ self.buffer+ }++ pub fn device_address(&self) -> vk::DeviceAddress {+ self.device_address+ }++ pub fn size(&self) -> vk::DeviceSize {+ self.size+ }++ pub fn resolve(+ buffer: &(impl Buffer<Backend = Vulkan> + ?Sized),+ range: std::range::Range<usize>,+ ) -> (&Arc<VkBuffer>, Range<u64>) {+ let (buffer, offset) = if let Some(buffer) = downcast_ref::<Arc<VkBuffer>>(buffer) {+ (buffer, 0)+ } else if let Some(allocation) = downcast_ref::<BumpAllocation<Arc<VkBuffer>>>(buffer) {+ (allocation.page(), allocation.range().start)+ } else if let Some(allocation) = downcast_ref::<PoolAllocation<Arc<VkBuffer>>>(buffer) {+ (allocation.page(), allocation.range().start)+ } else {+ unreachable!("unsupported Vulkan buffer type")+ };+ (buffer, (offset + range.start) as u64..(offset + range.end) as u64)+ }++ pub fn resolve_ref<'a>(buffer: impl BufferRef<Backend = Vulkan> + 'a) -> (&'a Arc<Self>, Range<u64>) {+ let (buffer, range) = buffer.parts();+ Self::resolve(buffer, range)+ }+}++impl Drop for VkBuffer {+ fn drop(&mut self) {+ unsafe { self.context.free_buffer(self.buffer, &mut self.allocation) };+ }+}++impl Buffer for Arc<VkBuffer> {+ type Backend = Vulkan;++ fn size(&self) -> usize {+ VkBuffer::size(self) as usize+ }+}++impl GpuBuffer for Arc<VkBuffer> {}++impl CpuBuffer for Arc<VkBuffer> {+ fn cpu_ptr(&self) -> NonNull<c_void> {+ self.cpu_ptr.expect("device-local Vulkan buffers are not CPU-accessible")+ }+}++impl GlobalGpuBuffer for Arc<VkBuffer> {}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum VkBufferMemory {+ Device,+ Staging,+ Readback,+}++impl VkBufferMemory {+ pub fn allocation_create_info(+ self,+ discrete: bool,+ ) -> vk_mem::AllocationCreateInfo {+ let mapped = vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT;+ let write = vk_mem::AllocationCreateFlags::MAPPED | vk_mem::AllocationCreateFlags::HOST_ACCESS_SEQUENTIAL_WRITE;+ let (usage, flags, required_flags) = match self {+ Self::Device if discrete => (+ vk_mem::MemoryUsage::AutoPreferDevice,+ vk_mem::AllocationCreateFlags::WITHIN_BUDGET,+ vk::MemoryPropertyFlags::DEVICE_LOCAL,+ ),+ Self::Device => {+ (vk_mem::MemoryUsage::AutoPreferDevice, vk_mem::AllocationCreateFlags::empty(), Default::default())+ },+ Self::Staging => (vk_mem::MemoryUsage::AutoPreferHost, write, mapped),+ Self::Readback => (+ vk_mem::MemoryUsage::AutoPreferHost,+ vk_mem::AllocationCreateFlags::MAPPED | vk_mem::AllocationCreateFlags::HOST_ACCESS_RANDOM,+ mapped,+ ),+ };+ vk_mem::AllocationCreateInfo {+ usage,+ flags,+ required_flags,+ ..Default::default()+ }+ }+}++pub struct VkSparseBuffer {+ never: Infallible,+}++impl Buffer for VkSparseBuffer {+ type Backend = Vulkan;++ fn size(&self) -> usize {+ match self.never {}+ }+}++impl GpuBuffer for VkSparseBuffer {}++impl SparseBuffer for VkSparseBuffer {+ fn map(+ &mut self,+ _until: usize,+ ) -> Result<(), Error> {+ match self.never {}+ }+}diff --git a/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer_initializer.rs b/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer_initializer.rsnew file mode 100644index 000000000..38047533c--- /dev/null+++ b/crates/uzu-engine/src/backends/vulkan/buffer/vk_buffer_initializer.rs@@ -0,0 +1,52 @@+use std::{os::raw::c_void, ptr::NonNull, sync::Arc};++use crate::backends::{+ common::{Buffer, CommandBufferEncoding, Context, CpuBuffer, GlobalBufferCpuGpu},+ vulkan::{Error, VkBuffer, VkBufferMemory, VkContext, VkDeviceCapabilities, Vulkan},+};++pub struct VkBufferInitializer {+ host: Arc<VkBuffer>,+ device: Option<Arc<VkBuffer>>,+}++impl VkBufferInitializer {+ pub fn new(+ context: Arc<VkContext>,+ size: usize,+ ) -> Result<Self, Error> {+ let device = (!context.physical_device().capabilities.contains(VkDeviceCapabilities::UNIFIED_MEMORY))+ .then(|| VkBuffer::new(context.clone(), size, VkBufferMemory::Device))+ .transpose()?;+ Ok(Self {+ host: VkBuffer::new(context, size, VkBufferMemory::Staging)?,+ device,+ })+ }+}++impl Buffer for VkBufferInitializer {+ type Backend = Vulkan;++ fn size(&self) -> usize {+ Buffer::size(&self.host)+ }+}++impl CpuBuffer for VkBufferInitializer {+ fn cpu_ptr(&self) -> NonNull<c_void> {+ self.host.cpu_ptr()+ }+}++impl GlobalBufferCpuGpu for VkBufferInitializer {+ fn handoff(self) -> Result<Arc<VkBuffer>, Error> {+ let Some(mut device) = self.device else {+ return Ok(self.host);+ };+ let mut encoding = self.host.context().create_command_buffer(None, None)?;+ encoding.encode_copy(&self.host, &mut device);+ encoding.end_encoding().try_submit()?;+ Ok(device)+ }+}diff --git a/crates/uzu-engine/src/backends/vulkan/device/vk_physical_device.rs b/crates/uzu-engine/src/backends/vulkan/device/vk_physical_device.rsindex 35752cf0e..4381e9dab 100644--- a/crates/uzu-engine/src/backends/vulkan/device/vk_physical_device.rs+++ b/crates/uzu-engine/src/backends/vulkan/device/vk_physical_device.rs@@ -215,6 +215,80 @@ impl VkPhysicalDevice {}}+impl VkPhysicalDevice {+ pub fn create_device(+ &self,+ instance: &ash::Instance,+ queue_family_index: u32,+ ) -> Result<ash::Device, VkContextError> {+ let queue_priorities = [1.0_f32];+ let queue_create_info = vk::DeviceQueueCreateInfo::default()+ .queue_family_index(queue_family_index)+ .queue_priorities(&queue_priorities);++ let bf16_type = self.capabilities.contains(VkDeviceCapabilities::BF16_TYPE);+ let matrices = self+ .capabilities+ .intersects(VkDeviceCapabilities::BF16_TENSOR_CORES | VkDeviceCapabilities::INT8_TENSOR_CORES);+ let element_fold = self.capabilities.contains(VkDeviceCapabilities::INT8_ELEMENT_FOLD);+ let mut device_extensions = Vec::new();+ if matrices {+ device_extensions.push(khr::cooperative_matrix::NAME.as_ptr());+ }+ if bf16_type {+ device_extensions.push(VkShaderBfloat16Features::EXTENSION_NAME.as_ptr());+ }+ if element_fold {+ device_extensions.push(VkCooperativeMatrix2Features::EXTENSION_NAME.as_ptr());+ }++ let vk10_features = vk::PhysicalDeviceFeatures::default().shader_int16(true);+ let mut vk11_features = vk::PhysicalDeviceVulkan11Features::default()+ .storage_buffer16_bit_access(true)+ .storage_push_constant16(true);+ let mut vk12_features = vk::PhysicalDeviceVulkan12Features::default()+ .shader_int8(true)+ .storage_buffer8_bit_access(true)+ .storage_push_constant8(true)+ .shader_float16(true)+ .shader_subgroup_extended_types(true)+ .buffer_device_address(true)+ .vulkan_memory_model(true)+ .vulkan_memory_model_device_scope(true);+ let mut vk13_features = vk::PhysicalDeviceVulkan13Features::default()+ .maintenance4(true)+ .synchronization2(true)+ .subgroup_size_control(true)+ .compute_full_subgroups(true)+ .shader_integer_dot_product(true);++ let mut matrix_features = vk::PhysicalDeviceCooperativeMatrixFeaturesKHR::default().cooperative_matrix(true);+ let mut bfloat16_features = VkShaderBfloat16Features::default();+ bfloat16_features.shader_bfloat16_type = vk::TRUE;+ bfloat16_features.shader_bfloat16_cooperative_matrix =+ self.capabilities.contains(VkDeviceCapabilities::BF16_TENSOR_CORES).into();+ let mut matrix2_features = VkCooperativeMatrix2Features::default();+ matrix2_features.cooperative_matrix_per_element_operations = vk::TRUE;+ let mut device_create_info = vk::DeviceCreateInfo::default()+ .queue_create_infos(std::slice::from_ref(&queue_create_info))+ .enabled_extension_names(&device_extensions)+ .enabled_features(&vk10_features)+ .push_next(&mut vk11_features)+ .push_next(&mut vk12_features)+ .push_next(&mut vk13_features);+ if matrices {+ device_create_info = device_create_info.push_next(&mut matrix_features);+ }+ if bf16_type {+ device_create_info = device_create_info.push_next(&mut bfloat16_features);+ }+ if element_fold {+ device_create_info = device_create_info.push_next(&mut matrix2_features);+ }+ unsafe { instance.create_device(self.handle, &device_create_info, None) }.map_err(VkContextError::DeviceCreate)+ }+}+pub struct VkDeviceLimits {pub max_work_group_count: [u32; 3],pub max_work_group_size: [u32; 3],diff --git a/crates/uzu-engine/src/backends/vulkan/error.rs b/crates/uzu-engine/src/backends/vulkan/error.rsindex 9e9928dc6..245cf353f 100644--- a/crates/uzu-engine/src/backends/vulkan/error.rs+++ b/crates/uzu-engine/src/backends/vulkan/error.rs@@ -1,5 +1,3 @@-use std::sync::Arc;-use ash::vk;use crate::data_type::DataType;@@ -91,9 +89,6 @@ pub enum VkContextError {#[error("the Vulkan validation layer is not installed")]ValidationNotSupported,-- #[error("GPU work that nobody awaited failed, so buffers may hold wrong data: {0}")]- Failed(#[source] Arc<Error>),}#[derive(Debug, thiserror::Error)]diff --git a/crates/uzu-engine/src/backends/vulkan/mod.rs b/crates/uzu-engine/src/backends/vulkan/mod.rsindex d225d1a3d..8feb3b4ea 100644--- a/crates/uzu-engine/src/backends/vulkan/mod.rs+++ b/crates/uzu-engine/src/backends/vulkan/mod.rs@@ -1,5 +1,7 @@+mod buffer;mod device;mod error;+pub use buffer::*;pub use device::*;pub use error::{Error, VkBufferError, VkCommandBufferError, VkContextError, VkKernelError, VkPipelineError};diff --git a/crates/uzu-engine/src/backends/webgpu/buffer/dense.rs b/crates/uzu-engine/src/backends/webgpu/buffer/dense.rsindex 2caf69f0f..09779360c 100644--- a/crates/uzu-engine/src/backends/webgpu/buffer/dense.rs+++ b/crates/uzu-engine/src/backends/webgpu/buffer/dense.rs@@ -117,12 +117,12 @@ impl CpuBuffer for WebGPUDenseBufferCpuGpu {}impl GlobalBufferCpuGpu for WebGPUDenseBufferCpuGpu {- fn handoff(self) -> WebGPUDenseBufferGpu {+ fn handoff(self) -> Result<WebGPUDenseBufferGpu, WebGPUError> {let gpu = WebGPUDenseBufferGpu {buffer: self.buffer.clone(),size: self.size,};drop(self);- gpu+ Ok(gpu)}}diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex ed6af073a..fb6cd9fb9 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -134,7 +134,7 @@ impl<'a, B: Backend> StreamEnqueuer<'a, B> {.collect::<Box<[_]>>(),);- Some(context_ring.handoff())+ Some(context_ring.handoff().map_err(LanguageModelStreamError::Backend)?)} else {None};diff --git a/crates/uzu-engine/src/safetensors.rs b/crates/uzu-engine/src/safetensors.rsindex a5082829e..2e1f9139a 100644--- a/crates/uzu-engine/src/safetensors.rs+++ b/crates/uzu-engine/src/safetensors.rs@@ -213,7 +213,7 @@ impl<B: Backend> ParameterTree<'_, B> {.map_err(ParameterLoaderError::BackendError)?;self.loader.file.read_exact_at(buffer.as_slice_mut::<u8>(), self.loader.data_offset + start).await?;tensor.read.store(true, Ordering::Relaxed);- Ok(buffer.handoff())+ buffer.handoff().map_err(ParameterLoaderError::BackendError)}pub async fn read_slice<T: DataTypeRepr>(diff --git a/crates/uzu-engine/unit/common/helpers.rs b/crates/uzu-engine/unit/common/helpers.rsindex 53acfa772..ee4c5b9c1 100644--- a/crates/uzu-engine/unit/common/helpers.rs+++ b/crates/uzu-engine/unit/common/helpers.rs@@ -61,7 +61,7 @@ pub fn create_buffer_with_data<B: Backend, T: DataTypeRepr>(let mut buffer =context.create_buffer_cpu_gpu(buffer_size_bytes::<T>(data.len())).expect("Failed to create buffer");buffer.copyin(data);- buffer.handoff()+ buffer.handoff().expect("Failed to hand off buffer")}pub fn buffer_to_vec<B: Backend, T: DataTypeRepr>(