A high-performance inference engine for AI models
use std::{mem::size_of_val, time::Duration};
use bytemuck::{AnyBitPattern, NoUninit};
use crate::{
backends::common::{Backend, BufferMut, BufferRef, CpuBuffer, TimestampSpan},
data_type::DataType,
utils::size_for_shape::size_for_shape,
};
pub trait CommandBuffer {
type Backend: Backend<CommandBuffer = Self>;
type Encoding: CommandBufferEncoding<CommandBuffer = Self>;
type Executable: CommandBufferExecutable<CommandBuffer = Self>;
type Pending: CommandBufferPending<CommandBuffer = Self>;
type Completed: CommandBufferCompleted<CommandBuffer = Self>;
}
pub trait CommandBufferEncoding: Send {
type CommandBuffer: CommandBuffer<Encoding = Self>;
fn context(&self) -> &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Context;
fn allocate_constant(
&mut self,
size: usize,
) -> Result<
<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::ConstantBuffer,
<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,
>;
fn allocate_constant_from_slice(
&mut self,
data: &[impl NoUninit + AnyBitPattern],
) -> Result<
<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::ConstantBuffer,
<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,
> {
let mut buffer = self.allocate_constant(size_of_val(data))?;
buffer.copyin(data);
Ok(buffer)
}
fn allocate_scratch(
&mut self,
size: usize,
) -> Result<
<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::ScratchBuffer,
<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,
>;
fn allocate_scratch_for_shape(
&mut self,
shape: &[u32],
data_type: DataType,
) -> Result<
<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::ScratchBuffer,
<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,
> {
self.allocate_scratch(size_for_shape(shape, data_type))
}
fn encode_copy(
&mut self,
src: impl BufferRef<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>,
dst: impl BufferMut<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>,
);
fn encode_fill(
&mut self,
dst: impl BufferMut<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>,
value: u8,
);
#[must_use]
fn debug_group(
&mut self,
name: &str,
) -> <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DebugGroupSpan<'_>;
fn debug_point(
&mut self,
name: &str,
);
fn with_timestamps(self) -> Self;
#[track_caller]
fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;
}
pub trait CommandBufferExecutable: Send {
type CommandBuffer: CommandBuffer<Executable = Self>;
fn submit(self) -> <Self::CommandBuffer as CommandBuffer>::Pending;
}
pub trait CommandBufferPending: Send {
type CommandBuffer: CommandBuffer<Pending = Self>;
fn wait_until_completed(
self
) -> impl Future<
Output = Result<
<Self::CommandBuffer as CommandBuffer>::Completed,
<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,
>,
> + Send;
}
pub trait CommandBufferCompleted: Send {
type CommandBuffer: CommandBuffer<Completed = Self>;
fn gpu_execution_time(&self) -> Option<Duration>;
fn timestamps(&self) -> &[TimestampSpan];
}