Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

migrate from size to u32

eugene1bbb59ebe273
diff --git a/Cargo.toml b/Cargo.toml
index 596283a95..13e51d421 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -231,6 +231,7 @@ uzu_bench = "dinghy_apple_runner_macros::bench_case"
[profile.dev]
opt-level = 1
+overflow-checks = true
[profile.dev.package."*"]
opt-level = 2
@@ -243,3 +244,4 @@ lto = "thin"
[profile.test]
opt-level = 2
+overflow-checks = true
diff --git a/crates/backend-uzu/build/metal/gpu_types/mod.rs b/crates/backend-uzu/build/metal/gpu_types/mod.rs
index b5968d062..f36314945 100644
--- a/crates/backend-uzu/build/metal/gpu_types/mod.rs
+++ b/crates/backend-uzu/build/metal/gpu_types/mod.rs
@@ -27,8 +27,6 @@ fn rust_to_metal(ty: &str) -> anyhow::Result<&'static str> {
"u64" => Ok("uint64_t"),
"f32" => Ok("float"),
"bool" => Ok("bool"),
- "usize" => Ok("size_t"),
- "isize" => Ok("ptrdiff_t"),
unknown => anyhow::bail!("Unsupported GPU type: {unknown}"),
}
}
diff --git a/crates/backend-uzu/src/array/util.rs b/crates/backend-uzu/src/array/util.rs
index cfcc03d09..b0c6a5b3b 100644
--- a/crates/backend-uzu/src/array/util.rs
+++ b/crates/backend-uzu/src/array/util.rs
@@ -1,17 +1,17 @@
use crate::data_type::DataType;
pub fn size_for_shape(
- shape: &[usize],
+ shape: &[u32],
data_type: DataType,
) -> usize {
let Some(last_dim) = shape.last() else {
return data_type.size_in_bytes();
};
- let bits_per_row = last_dim * data_type.size_in_bits();
+ let bits_per_row = *last_dim as usize * data_type.size_in_bits();
let padded_bytes_per_row = bits_per_row.div_ceil(8);
- let num_rows: usize = shape.iter().rev().skip(1).product();
+ let num_rows: usize = shape.iter().rev().skip(1).map(|&dim| dim as usize).product();
num_rows * padded_bytes_per_row
}
diff --git a/crates/backend-uzu/src/backends/common/encoder.rs b/crates/backend-uzu/src/backends/common/encoder.rs
index 995608391..987a37c9c 100644
--- a/crates/backend-uzu/src/backends/common/encoder.rs
+++ b/crates/backend-uzu/src/backends/common/encoder.rs
@@ -7,11 +7,15 @@ use std::{
use bytemuck::{AnyBitPattern, NoUninit};
-use crate::backends::common::{
- AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer,
- BufferGpuAddressRangeExt, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
- CommandBufferInitial, CommandBufferPending, Context,
- hazard_tracker::{Access, HazardTracker},
+use crate::{
+ array::size_for_shape,
+ backends::common::{
+ AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer,
+ BufferGpuAddressRangeExt, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,
+ CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ hazard_tracker::{Access, HazardTracker},
+ },
+ data_type::DataType,
};
fn resolve_copy_range(
@@ -83,6 +87,14 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {
)
}
+ pub fn allocate_constant_for_shape(
+ &mut self,
+ shape: &[u32],
+ data_type: DataType,
+ ) -> Result<Allocation<B>, B::Error> {
+ self.allocate_constant(size_for_shape(shape, data_type))
+ }
+
pub fn allocate_constant_from_slice<T: NoUninit + AnyBitPattern>(
&mut self,
data: &[T],
@@ -106,6 +118,14 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {
)
}
+ pub fn allocate_scratch_for_shape(
+ &mut self,
+ shape: &[u32],
+ data_type: DataType,
+ ) -> Result<Allocation<B>, B::Error> {
+ self.allocate_scratch(size_for_shape(shape, data_type))
+ }
+
pub fn encode_copy<
Src: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,
Dst: AsBufferRangeMut<Buffer: Buffer<Backend = B>>,
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/attention.rs b/crates/backend-uzu/src/backends/common/gpu_types/attention.rs
index 7df3d5ee6..e0b648ead 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/attention.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/attention.rs
@@ -10,10 +10,10 @@
#[repr(C)]
#[derive(Debug, Default, Copy, Clone)]
pub struct AttnParams {
- pub q_strides: [u64; 3],
- pub k_strides: [u64; 3],
- pub v_strides: [u64; 3],
- pub o_strides: [u64; 3],
+ pub q_strides: [u32; 3],
+ pub k_strides: [u32; 3],
+ pub v_strides: [u32; 3],
+ pub o_strides: [u32; 3],
pub gqa_factor: u32,
pub scale: f32,
/// Query length (suffix length)
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs
index 429450dac..a727502c1 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs
@@ -1 +1 @@
-pub const HADAMARD_TRANSFORM_BLOCK_SIZE: usize = 32;
+pub const HADAMARD_TRANSFORM_BLOCK_SIZE: u32 = 32;
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs b/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs
index fce644746..9896c7372 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs
@@ -16,8 +16,8 @@ pub enum QuantizationMode {
}
impl QuantizationMode {
- pub fn packing_divisor(&self) -> usize {
- let bits = DataType::from(*self).size_in_bits();
+ pub fn packing_divisor(&self) -> u32 {
+ let bits = DataType::from(*self).size_in_bits() as u32;
assert_eq!(8 % bits, 0, "QuantizationMode bit width ({bits}) must divide 8 evenly");
8 / bits
}
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs
index 09d0e86cd..bada0a1ec 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs
@@ -1,6 +1,6 @@
-pub const CANDIDATES_MAX: usize = 512;
-pub const TOP_CHILDREN_THREADS: usize = 256;
-pub const TOP_CHILDREN_SIMDGROUPS: usize = 8;
+pub const CANDIDATES_MAX: u32 = 512;
+pub const TOP_CHILDREN_THREADS: u32 = 256;
+pub const TOP_CHILDREN_SIMDGROUPS: u32 = 8;
#[repr(C)]
#[derive(Clone, Copy)]
@@ -46,8 +46,8 @@ impl MetadataIdx {
}
pub const FRONTIER_NO_WINNER: u32 = !0;
-pub const FRONTIER_SELECT_THREADS: usize = 256;
-pub const FRONTIER_SELECT_SIMDGROUPS: usize = 8;
-pub const FRONTIER_ENTRIES_PER_THREAD: usize = 8;
-pub const FRONTIER_MAX_SLOTS: usize = 2048;
-pub const FRONTIER_MAX_WIDTH: usize = 32;
+pub const FRONTIER_SELECT_THREADS: u32 = 256;
+pub const FRONTIER_SELECT_SIMDGROUPS: u32 = 8;
+pub const FRONTIER_ENTRIES_PER_THREAD: u32 = 8;
+pub const FRONTIER_MAX_SLOTS: u32 = 2048;
+pub const FRONTIER_MAX_WIDTH: u32 = 32;
diff --git a/crates/backend-uzu/src/backends/common/kernel/activation_transform.rs b/crates/backend-uzu/src/backends/common/kernel/activation_transform.rs
index 87b732c43..17cb688c8 100644
--- a/crates/backend-uzu/src/backends/common/kernel/activation_transform.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/activation_transform.rs
@@ -9,7 +9,7 @@ use crate::{
fn assert_row_width(element_count: u32) {
assert!(
- (element_count as usize).is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE),
+ element_count.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE),
"activation transform requires element_count ({element_count}) to be a multiple of {HADAMARD_TRANSFORM_BLOCK_SIZE}"
);
}
@@ -20,7 +20,7 @@ pub struct ActivationTransform<B: Backend> {
kernel: <B::Kernels as Kernels>::ActivationTransformKernel,
ops: ActivationTransformOp,
in_place: bool,
- activation_group_size: usize,
+ activation_group_size: u32,
sum_group_size: Option<u32>,
}
@@ -30,7 +30,7 @@ impl<B: Backend> ActivationTransform<B> {
data_type: DataType,
ops: ActivationTransformOp,
in_place: bool,
- activation_group_size: usize,
+ activation_group_size: u32,
sum_group_size: Option<u32>,
) -> Result<Self, B::Error> {
let kernel = <B::Kernels as Kernels>::ActivationTransformKernel::new(
@@ -38,8 +38,8 @@ impl<B: Backend> ActivationTransform<B> {
data_type,
ops,
in_place,
- activation_group_size as u32,
- sum_group_size.unwrap_or(HADAMARD_TRANSFORM_BLOCK_SIZE as u32),
+ activation_group_size,
+ sum_group_size.unwrap_or(HADAMARD_TRANSFORM_BLOCK_SIZE),
)?;
Ok(Self {
kernel,
@@ -69,7 +69,7 @@ impl<B: Backend> ActivationTransform<B> {
pub fn quantize(
context: &B::Context,
data_type: DataType,
- activation_group_size: usize,
+ activation_group_size: u32,
sum_group_size: Option<u32>,
) -> Result<Self, B::Error> {
let ops = if sum_group_size.is_some() {
@@ -149,7 +149,7 @@ impl<B: Backend> ActivationTransform<B> {
assert!(self.quantizes());
assert_row_width(element_count);
assert!(
- element_count.is_multiple_of(self.activation_group_size as u32),
+ element_count.is_multiple_of(self.activation_group_size),
"quantized activation row ({element_count}) must be a multiple of scale group ({})",
self.activation_group_size
);
@@ -177,7 +177,7 @@ impl<B: Backend> ActivationTransform<B> {
self.sum_group_size.is_some()
}
- pub fn activation_group_size(&self) -> usize {
+ pub fn activation_group_size(&self) -> u32 {
self.activation_group_size
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs
index 28ec0a2df..9d1e3ec4c 100644
--- a/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs
@@ -16,7 +16,7 @@ pub struct DeltaNetChunkedPrefillArgs<'a, B: Backend> {
pub value_head_dim: u32,
pub key_dim: u32,
pub value_dim: u32,
- pub suffix_len: usize,
+ pub suffix_len: u32,
}
pub trait DeltaNetChunkedPrefill<B: Backend>: Sized + Send + Sync {
@@ -28,7 +28,7 @@ pub trait DeltaNetChunkedPrefill<B: Backend>: Sized + Send + Sync {
fn should_use(
&self,
- suffix_len: usize,
+ suffix_len: u32,
) -> bool;
fn encode(
@@ -49,7 +49,7 @@ impl<B: Backend> DeltaNetChunkedPrefill<B> for Infallible {
fn should_use(
&self,
- _suffix_len: usize,
+ _suffix_len: u32,
) -> bool {
match *self {}
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs
index c98795a5a..96be2b02a 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs
@@ -10,7 +10,7 @@ pub enum MatmulError<B: Backend> {
#[error("Unsupported data type: {0:?}")]
UnsupportedDataType(DataType),
#[error("Unsupported group size: {0}")]
- UnsupportedGroupSize(usize),
+ UnsupportedGroupSize(u32),
#[error("Unsupported D-transform op {bit:?} on path {path}")]
UnsupportedDOp {
bit: GemmDTransform,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/activation_transform.rs b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/activation_transform.rs
index b91b027d2..df6b0dc5d 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/activation_transform.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/activation_transform.rs
@@ -72,9 +72,9 @@ pub fn activation_transform<T: ArrayElement + Float>(
let mut transformed = vec![0.0f32; columns];
for row in 0..rows {
let row_offset = row * columns;
- for stripe_start in (0..columns).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE) {
- let mut stripe = [0.0f32; HADAMARD_TRANSFORM_BLOCK_SIZE];
- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {
+ for stripe_start in (0..columns).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {
+ let mut stripe = [0.0f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize];
+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE as usize {
let index = stripe_start + lane;
let value: f32 = NumCast::from(unsafe { *input.add(row_offset + index) }).unwrap();
let factor = unsafe { *rht_factors.add(index) } as f32;
@@ -87,7 +87,7 @@ pub fn activation_transform<T: ArrayElement + Float>(
hadamard_transform(&mut stripe);
- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {
+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE as usize {
let index = stripe_start + lane;
let factor = unsafe { *rht_factors.add(index) } as f32;
transformed[index] = if input_rht {
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs
index 3c3defd42..b33ecaff9 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs
@@ -22,10 +22,10 @@ pub fn quantize_symmetric_i8(
(value / divisor).round().clamp(-INT8_SYMMETRIC_QUANTIZATION_MAXIMUM, INT8_SYMMETRIC_QUANTIZATION_MAXIMUM) as i8
}
-pub(crate) fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZE]) {
+pub(crate) fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize]) {
let mut stride = 1;
- while stride < HADAMARD_TRANSFORM_BLOCK_SIZE {
- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {
+ while stride < HADAMARD_TRANSFORM_BLOCK_SIZE as usize {
+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE as usize {
if lane & stride == 0 {
let a = values[lane];
let b = values[lane | stride];
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
index 8a071e503..45cffeb8d 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
@@ -33,7 +33,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
unimplemented!("not supported yet");
}
- let packing_divisor = quantization_mode.packing_divisor() as u32;
+ let packing_divisor = quantization_mode.packing_divisor();
let weights_stride = model_dim / packing_divisor;
let num_groups = model_dim.div_ceil(group_size);
let zero_points_stride = match quantization_mode {
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
index 0ea58c3c3..365ce8620 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
@@ -28,9 +28,9 @@ pub fn weaver_frontier_select(
candidates_per_depth: u32,
) {
if frontier_capacity == 0
- || frontier_capacity as usize > FRONTIER_MAX_SLOTS
+ || frontier_capacity > FRONTIER_MAX_SLOTS
|| node_count == 0
- || node_count as usize > FRONTIER_MAX_WIDTH
+ || node_count > FRONTIER_MAX_WIDTH
|| ancestor_stride == 0
|| max_depth == 0
|| tree_slot_count == 0
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs
index f4a5bd65c..87ea5135f 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs
@@ -23,7 +23,7 @@ pub fn weaver_top_children(
let rows = rows as usize;
let candidates = candidates as usize;
let expand_width = expand_width as usize;
- if candidates == 0 || candidates > CANDIDATES_MAX || expand_width == 0 || expand_width > candidates {
+ if candidates == 0 || candidates > CANDIDATES_MAX as usize || expand_width == 0 || expand_width > candidates {
return;
}
for row in 0..rows {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention.rs b/crates/backend-uzu/src/backends/metal/kernel/attention.rs
index cb48dcffb..bd53cdca9 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention.rs
@@ -3,7 +3,6 @@ use std::collections::{HashMap, hash_map::Entry};
use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};
use crate::{
- array::size_for_shape,
backends::{
common::{Allocation, BufferArg, Encoder, gpu_types::AttnParams, kernel::attention_gemm::AttentionGemmCore},
metal::{Metal, context::MetalContext, error::MetalError, kernel::AttentionGemmMetalKernel},
@@ -14,10 +13,10 @@ use crate::{
pub struct AttentionGemmMetalCore {
kernels: Mutex<HashMap<AttentionGemmKey, AttentionGemmMetalKernel>>,
- head_dim: usize,
- num_groups: usize,
- num_q_heads: usize,
- sliding_window_size: Option<usize>,
+ head_dim: u32,
+ num_groups: u32,
+ num_q_heads: u32,
+ sliding_window_size: Option<u32>,
scale: Option<f32>,
data_type: DataType,
simd_bk: u32,
@@ -65,7 +64,7 @@ impl AttentionGemmMetalCore {
context,
self.data_type,
bk,
- self.head_dim as u32,
+ self.head_dim,
key.use_mxu,
key.align_q,
key.align_k,
@@ -121,10 +120,8 @@ impl AttentionGemmCore<Metal> for AttentionGemmMetalCore {
arguments: AttentionCoreEncodeArguments<'a, Metal, KT, VT>,
encoder: &mut Encoder<Metal>,
) -> Result<Allocation<Metal>, MetalError> {
- let mut output = encoder.allocate_constant(size_for_shape(
- &[arguments.suffix_length, self.num_q_heads, self.head_dim],
- self.data_type,
- ))?;
+ let mut output = encoder
+ .allocate_constant_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;
let use_mxu = arguments.suffix_length >= 64
&& encoder.context().supports_mxu()
@@ -137,15 +134,15 @@ impl AttentionGemmCore<Metal> for AttentionGemmMetalCore {
};
let params = retile_params(
AttnParams {
- q_strides: [0, (arguments.suffix_length * self.head_dim) as u64, self.head_dim as u64],
- k_strides: [0, self.head_dim as u64, (self.num_groups * self.head_dim) as u64],
- v_strides: [0, self.head_dim as u64, (self.num_groups * self.head_dim) as u64],
- o_strides: [0, self.head_dim as u64, (self.num_q_heads * self.head_dim) as u64],
- gqa_factor: (self.num_q_heads / self.num_groups) as u32,
+ q_strides: [0, arguments.suffix_length * self.head_dim, self.head_dim],
+ k_strides: [0, self.head_dim, self.num_groups * self.head_dim],
+ v_strides: [0, self.head_dim, self.num_groups * self.head_dim],
+ o_strides: [0, self.head_dim, self.num_q_heads * self.head_dim],
+ gqa_factor: self.num_q_heads / self.num_groups,
scale: self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),
- q_len: arguments.suffix_length as u32,
- k_len: (arguments.state_type.physical_prefix_length() + arguments.suffix_length) as u32,
- q_off: arguments.state_type.physical_prefix_length() as u32,
+ q_len: arguments.suffix_length,
+ k_len: arguments.state_type.physical_prefix_length() + arguments.suffix_length,
+ q_off: arguments.state_type.physical_prefix_length(),
nq_aligned: 0,
q_rem: 0,
nk: 0,
@@ -170,10 +167,10 @@ impl AttentionGemmCore<Metal> for AttentionGemmMetalCore {
params,
arguments.state_type.ring_params(),
arguments.trie,
- self.sliding_window_size.map(|sliding_window_size| sliding_window_size as u32),
+ self.sliding_window_size,
arguments.sinks,
- self.num_q_heads as u32,
- arguments.suffix_length as u32,
+ self.num_q_heads,
+ arguments.suffix_length,
encoder,
);
Ok(output)
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index 728c090e4..b19f733f1 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -82,14 +82,14 @@ KERNEL(AttentionGemm)(
using InputType = metal::conditional_t<USE_MXU, T, AccumType>;
- q += batch_idx * params.q_strides[0] + head_idx * params.q_strides[1] +
+ q += batch_idx * uint64_t(params.q_strides[0]) + head_idx * uint64_t(params.q_strides[1]) +
q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.q_strides[2];
const int kv_head_idx = int(head_idx) / params.gqa_factor;
- k += batch_idx * params.k_strides[0] + int64_t(kv_head_idx) * params.k_strides[1];
- v += batch_idx * params.v_strides[0] + int64_t(kv_head_idx) * params.v_strides[1];
+ k += batch_idx * uint64_t(params.k_strides[0]) + int64_t(kv_head_idx) * params.k_strides[1];
+ v += batch_idx * uint64_t(params.v_strides[0]) + int64_t(kv_head_idx) * params.v_strides[1];
- o += batch_idx * params.o_strides[0] + head_idx * params.o_strides[1] +
+ o += batch_idx * uint64_t(params.o_strides[0]) + head_idx * uint64_t(params.o_strides[1]) +
q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.o_strides[2];
if (is_trie) {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs
index 716f6086c..b23317a9f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs
@@ -3,7 +3,6 @@ use super::super::{
DeltaNetChunkedGramMetalKernel, DeltaNetChunkedOutputAndStateMetalKernel, DeltaNetPrefillPrepMetalKernel,
};
use crate::{
- array::size_for_shape,
backends::{
common::{
Backend, Encoder,
@@ -17,13 +16,14 @@ use crate::{
data_type::DataType,
};
-const MXU_MIN_T: usize = 256;
-const CHUNK_SIZE: usize = 64;
-const BLOCK_SIZE: usize = 16;
-const VT: usize = 32;
+const MXU_MIN_T: u32 = 256;
+const CHUNK_SIZE: u32 = 64;
+const BLOCK_SIZE: u32 = 16;
+const VT: u32 = 32;
+const INNER_DATA_TYPE: DataType = DataType::F32;
pub struct MetalDeltaNetChunkedPrefill {
- min_t: usize,
+ min_t: u32,
prep: DeltaNetPrefillPrepMetalKernel,
cumsum: DeltaNetChunkedCumsumMetalKernel,
gram: DeltaNetChunkedGramMetalKernel,
@@ -49,23 +49,30 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {
let scratch_data_type = if outer_data_type == DataType::BF16 {
DataType::BF16
} else {
- DataType::F32
+ INNER_DATA_TYPE
};
Ok(Some(Self {
min_t: MXU_MIN_T,
- prep: DeltaNetPrefillPrepMetalKernel::new(context, outer_data_type, DataType::F32, head_dim, true, false)?,
- cumsum: DeltaNetChunkedCumsumMetalKernel::new(context, CHUNK_SIZE as u32)?,
- gram: DeltaNetChunkedGramMetalKernel::new(context, head_dim, CHUNK_SIZE as u32)?,
- a_diag_inv: DeltaNetChunkedADiagInvMetalKernel::new(context, CHUNK_SIZE as u32)?,
- causal_inv: DeltaNetChunkedCausalInvMetalKernel::new(context, CHUNK_SIZE as u32, VT as u32)?,
+ prep: DeltaNetPrefillPrepMetalKernel::new(
+ context,
+ outer_data_type,
+ INNER_DATA_TYPE,
+ head_dim,
+ true,
+ false,
+ )?,
+ cumsum: DeltaNetChunkedCumsumMetalKernel::new(context, CHUNK_SIZE)?,
+ gram: DeltaNetChunkedGramMetalKernel::new(context, head_dim, CHUNK_SIZE)?,
+ a_diag_inv: DeltaNetChunkedADiagInvMetalKernel::new(context, CHUNK_SIZE)?,
+ causal_inv: DeltaNetChunkedCausalInvMetalKernel::new(context, CHUNK_SIZE, VT)?,
output_and_state: DeltaNetChunkedOutputAndStateMetalKernel::new(
context,
outer_data_type,
outer_data_type,
scratch_data_type,
head_dim,
- VT as u32,
+ VT,
true,
)?,
}))
@@ -73,7 +80,7 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {
fn should_use(
&self,
- suffix_len: usize,
+ suffix_len: u32,
) -> bool {
suffix_len >= self.min_t
}
@@ -88,35 +95,25 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {
let num_blocks = CHUNK_SIZE.div_ceil(BLOCK_SIZE);
let num_col_pairs = num_blocks.div_ceil(2);
- let mut q_norm =
- encoder.allocate_scratch(size_for_shape(&[suffix_len * args.key_dim as usize], DataType::F32))?;
- let mut k_norm =
- encoder.allocate_scratch(size_for_shape(&[suffix_len * args.key_dim as usize], DataType::F32))?;
- let mut beta =
- encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;
- let mut log_decay =
- encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;
- let mut g = encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;
- let mut kk = encoder.allocate_scratch(size_for_shape(
- &[num_chunks * args.num_groups as usize * CHUNK_SIZE * CHUNK_SIZE],
- DataType::F32,
- ))?;
- let mut qk = encoder.allocate_scratch(size_for_shape(
- &[num_chunks * args.num_heads as usize * CHUNK_SIZE * CHUNK_SIZE],
- DataType::F32,
- ))?;
- let mut a_packed = encoder.allocate_scratch(size_for_shape(
- &[num_chunks * args.num_heads as usize * num_blocks * num_col_pairs * BLOCK_SIZE * 2 * BLOCK_SIZE],
- DataType::F32,
- ))?;
- let mut a_inv = encoder.allocate_scratch(size_for_shape(
- &[num_chunks * args.num_heads as usize * num_blocks * BLOCK_SIZE * BLOCK_SIZE],
- DataType::F32,
- ))?;
- let mut t_mat = encoder.allocate_scratch(size_for_shape(
- &[num_chunks * args.num_heads as usize * CHUNK_SIZE * CHUNK_SIZE],
- DataType::BF16,
- ))?;
+ let mut q_norm = encoder.allocate_scratch_for_shape(&[suffix_len, args.key_dim], INNER_DATA_TYPE)?;
+ let mut k_norm = encoder.allocate_scratch_for_shape(&[suffix_len, args.key_dim], INNER_DATA_TYPE)?;
+ let mut beta = encoder.allocate_scratch_for_shape(&[suffix_len, args.num_heads], INNER_DATA_TYPE)?;
+ let mut log_decay = encoder.allocate_scratch_for_shape(&[suffix_len, args.num_heads], INNER_DATA_TYPE)?;
+ let mut g = encoder.allocate_scratch_for_shape(&[suffix_len, args.num_heads], INNER_DATA_TYPE)?;
+ let mut kk = encoder
+ .allocate_scratch_for_shape(&[num_chunks, args.num_groups, CHUNK_SIZE, CHUNK_SIZE], INNER_DATA_TYPE)?;
+ let mut qk = encoder
+ .allocate_scratch_for_shape(&[num_chunks, args.num_heads, CHUNK_SIZE, CHUNK_SIZE], INNER_DATA_TYPE)?;
+ let mut a_packed = encoder.allocate_scratch_for_shape(
+ &[num_chunks, args.num_heads, num_blocks, num_col_pairs, BLOCK_SIZE, 2 * BLOCK_SIZE],
+ INNER_DATA_TYPE,
+ )?;
+ let mut a_inv = encoder.allocate_scratch_for_shape(
+ &[num_chunks, args.num_heads, num_blocks, BLOCK_SIZE, BLOCK_SIZE],
+ INNER_DATA_TYPE,
+ )?;
+ let mut t_mat = encoder
+ .allocate_scratch_for_shape(&[num_chunks, args.num_heads, CHUNK_SIZE, CHUNK_SIZE], DataType::BF16)?;
self.prep.encode(
args.in_projected,
@@ -131,10 +128,10 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {
args.num_groups,
args.key_dim,
args.value_dim,
- suffix_len as u32,
+ suffix_len,
encoder,
);
- self.cumsum.encode(&log_decay, &mut g, args.num_heads, suffix_len as u32, encoder);
+ self.cumsum.encode(&log_decay, &mut g, args.num_heads, suffix_len, encoder);
self.gram.encode(
&q_norm,
&k_norm,
@@ -144,7 +141,7 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {
args.num_heads,
args.num_groups,
args.key_dim,
- suffix_len as u32,
+ suffix_len,
encoder,
);
self.a_diag_inv.encode(
@@ -155,10 +152,10 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {
&mut a_inv,
args.num_heads,
args.num_groups,
- suffix_len as u32,
+ suffix_len,
encoder,
);
- self.causal_inv.encode(&a_packed, &a_inv, &mut t_mat, args.num_heads, suffix_len as u32, encoder);
+ self.causal_inv.encode(&a_packed, &a_inv, &mut t_mat, args.num_heads, suffix_len, encoder);
self.output_and_state.encode(
&q_norm,
&k_norm,
@@ -174,7 +171,7 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {
args.value_head_dim,
args.key_dim,
args.value_dim,
- suffix_len as u32,
+ suffix_len,
encoder,
);
Ok(())
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs
index c786faad6..095d6c161 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs
@@ -1,5 +1,4 @@
use crate::{
- array::size_for_shape,
backends::{
common::{
Allocation, Encoder, Kernels,
@@ -14,20 +13,21 @@ use crate::{
encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},
};
-const TOKEN_BLOCK: usize = 16;
-const BLOCK_PAIR_WIDTH: usize = 2 * TOKEN_BLOCK;
+const TOKEN_BLOCK: u32 = 16;
+const BLOCK_PAIR_WIDTH: u32 = 2 * TOKEN_BLOCK;
+const INNER_DATA_TYPE: DataType = DataType::F32;
struct Layout {
- tree_size: usize,
- num_blocks: usize,
- num_block_pairs: usize,
- num_v_heads: usize,
- head_v_dim: usize,
+ tree_size: u32,
+ num_blocks: u32,
+ num_block_pairs: u32,
+ num_v_heads: u32,
+ head_v_dim: u32,
}
impl Layout {
const fn new(
- tree_size: usize,
+ tree_size: u32,
arguments: &TreeVerifyNewArguments,
) -> Self {
let num_blocks = tree_size.div_ceil(TOKEN_BLOCK);
@@ -40,11 +40,11 @@ impl Layout {
}
}
- const fn a_packed_shape(&self) -> [usize; 5] {
+ const fn a_packed_shape(&self) -> [u32; 5] {
[self.num_v_heads, self.num_blocks, self.num_block_pairs, TOKEN_BLOCK, BLOCK_PAIR_WIDTH]
}
- const fn a_inverse_shape(&self) -> [usize; 4] {
+ const fn a_inverse_shape(&self) -> [u32; 4] {
[self.num_v_heads, self.num_blocks, TOKEN_BLOCK, TOKEN_BLOCK]
}
}
@@ -93,21 +93,31 @@ impl DeltaNetTreeVerify<Metal> for MetalDeltaNetTreeVerify {
let layout = Layout::new(arguments.tree_size, &self.arguments);
let mut h0_indices = encoder.allocate_constant(DataType::I32.size_in_bytes())?;
h0_indices.copyin(&[0i32]);
- let mut scratch = |shape: &[usize], data_type| encoder.allocate_scratch(size_for_shape(shape, data_type));
- let mut prefix = scratch(&[layout.tree_size, layout.num_v_heads], DataType::F32)?;
- let mut a_packed = scratch(&layout.a_packed_shape(), DataType::F32)?;
- let mut qkd = scratch(&[layout.num_v_heads, layout.tree_size, layout.tree_size], DataType::F32)?;
- let mut a_inverse = scratch(&layout.a_inverse_shape(), DataType::F32)?;
- let mut kh0 = scratch(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], DataType::F32)?;
- let mut u = scratch(&[layout.num_v_heads, layout.tree_size, layout.head_v_dim], DataType::F32)?;
- let mut output = scratch(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], self.arguments.data_type)?;
- let tree_size = arguments.tree_size.try_into().expect("tree size exceeds u32");
- let num_k_heads = self.arguments.num_k_heads.try_into().expect("K head count exceeds u32");
- let num_v_heads = self.arguments.num_v_heads.try_into().expect("V head count exceeds u32");
- let head_k_dim = self.arguments.head_k_dim.try_into().expect("K head dimension exceeds u32");
- let head_v_dim = self.arguments.head_v_dim.try_into().expect("V head dimension exceeds u32");
- self.prefix.encode(arguments.trie, arguments.log_decay, &mut prefix, 1, tree_size, num_v_heads, encoder);
+ let mut prefix =
+ encoder.allocate_scratch_for_shape(&[layout.tree_size, layout.num_v_heads], INNER_DATA_TYPE)?;
+ let mut a_packed = encoder.allocate_scratch_for_shape(&layout.a_packed_shape(), INNER_DATA_TYPE)?;
+ let mut qkd = encoder
+ .allocate_scratch_for_shape(&[layout.num_v_heads, layout.tree_size, layout.tree_size], INNER_DATA_TYPE)?;
+ let mut a_inverse = encoder.allocate_scratch_for_shape(&layout.a_inverse_shape(), INNER_DATA_TYPE)?;
+ let mut kh0 = encoder
+ .allocate_scratch_for_shape(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], INNER_DATA_TYPE)?;
+ let mut u = encoder
+ .allocate_scratch_for_shape(&[layout.num_v_heads, layout.tree_size, layout.head_v_dim], INNER_DATA_TYPE)?;
+ let mut output = encoder.allocate_scratch_for_shape(
+ &[layout.tree_size, layout.num_v_heads, layout.head_v_dim],
+ self.arguments.data_type,
+ )?;
+
+ self.prefix.encode(
+ arguments.trie,
+ arguments.log_decay,
+ &mut prefix,
+ 1,
+ arguments.tree_size,
+ self.arguments.num_v_heads,
+ encoder,
+ );
self.gram.encode(
arguments.q,
arguments.k,
@@ -122,11 +132,11 @@ impl DeltaNetTreeVerify<Metal> for MetalDeltaNetTreeVerify {
Some(&mut kh0),
1.0,
1,
- tree_size,
- num_k_heads,
- num_v_heads,
- head_k_dim,
- head_v_dim,
+ arguments.tree_size,
+ self.arguments.num_k_heads,
+ self.arguments.num_v_heads,
+ self.arguments.head_k_dim,
+ self.arguments.head_v_dim,
encoder,
);
self.solve.encode(
@@ -139,9 +149,9 @@ impl DeltaNetTreeVerify<Metal> for MetalDeltaNetTreeVerify {
Some(&h0_indices),
&mut u,
1,
- tree_size,
- num_v_heads,
- head_v_dim,
+ arguments.tree_size,
+ self.arguments.num_v_heads,
+ self.arguments.head_v_dim,
encoder,
);
self.out.encode(
@@ -154,11 +164,11 @@ impl DeltaNetTreeVerify<Metal> for MetalDeltaNetTreeVerify {
&mut output,
1.0,
1,
- tree_size,
- num_k_heads,
- num_v_heads,
- head_k_dim,
- head_v_dim,
+ arguments.tree_size,
+ self.arguments.num_k_heads,
+ self.arguments.num_v_heads,
+ self.arguments.head_k_dim,
+ self.arguments.head_v_dim,
encoder,
);
Ok(output)
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h b/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h
index 55951bc81..55f58b29f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h
@@ -6,10 +6,10 @@ using namespace metal;
namespace uzu::attention {
typedef struct {
- uint64_t q_strides[3];
- uint64_t k_strides[3];
- uint64_t v_strides[3];
- uint64_t o_strides[3];
+ uint32_t q_strides[3];
+ uint32_t k_strides[3];
+ uint32_t v_strides[3];
+ uint32_t o_strides[3];
uint32_t gqa_factor;
float scale;
uint32_t q_len;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h
index d36ab48c9..aed63aa23 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h
@@ -5,5 +5,5 @@
using namespace metal;
namespace uzu::hadamard_order {
-static constant constexpr size_t HADAMARD_TRANSFORM_BLOCK_SIZE = 32;
+static constant constexpr uint32_t HADAMARD_TRANSFORM_BLOCK_SIZE = 32;
} // namespace uzu::hadamard_order
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h
index f453dcc9e..fd799486f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h
@@ -5,11 +5,11 @@
using namespace metal;
namespace uzu::weaver {
-static constant constexpr size_t CANDIDATES_MAX = 512;
+static constant constexpr uint32_t CANDIDATES_MAX = 512;
-static constant constexpr size_t TOP_CHILDREN_THREADS = 256;
+static constant constexpr uint32_t TOP_CHILDREN_THREADS = 256;
-static constant constexpr size_t TOP_CHILDREN_SIMDGROUPS = 8;
+static constant constexpr uint32_t TOP_CHILDREN_SIMDGROUPS = 8;
enum class FrontierIdx : uint32_t {
TokenId = 0,
@@ -38,13 +38,13 @@ enum class MetadataIdx : uint32_t {
static constant constexpr uint32_t FRONTIER_NO_WINNER = ~0;
-static constant constexpr size_t FRONTIER_SELECT_THREADS = 256;
+static constant constexpr uint32_t FRONTIER_SELECT_THREADS = 256;
-static constant constexpr size_t FRONTIER_SELECT_SIMDGROUPS = 8;
+static constant constexpr uint32_t FRONTIER_SELECT_SIMDGROUPS = 8;
-static constant constexpr size_t FRONTIER_ENTRIES_PER_THREAD = 8;
+static constant constexpr uint32_t FRONTIER_ENTRIES_PER_THREAD = 8;
-static constant constexpr size_t FRONTIER_MAX_SLOTS = 2048;
+static constant constexpr uint32_t FRONTIER_MAX_SLOTS = 2048;
-static constant constexpr size_t FRONTIER_MAX_WIDTH = 32;
+static constant constexpr uint32_t FRONTIER_MAX_WIDTH = 32;
} // namespace uzu::weaver
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 2c0a0f998..77ff41cd0 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -66,9 +66,7 @@ impl GemvSpecialization {
if shape.d_transform.contains(GemmDTransform::ACCUMULATE) && !shape.n.is_multiple_of(32) {
return None;
}
- if shape.d_transform.contains(GemmDTransform::RHT)
- && !shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE as u32)
- {
+ if shape.d_transform.contains(GemmDTransform::RHT) && !shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE) {
return None;
}
if shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > max_gemv_batch_threshold() {
diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rs
index a8917c48a..1d182db33 100644
--- a/crates/backend-uzu/src/backends/metal/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/mod.rs
@@ -14,7 +14,7 @@ use crate::backends::common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE;
pub(crate) const METAL_SIMD_SIZE: u32 = 32;
const _: () = {
- assert!(HADAMARD_TRANSFORM_BLOCK_SIZE == METAL_SIMD_SIZE as usize);
+ assert!(HADAMARD_TRANSFORM_BLOCK_SIZE == METAL_SIMD_SIZE);
};
pub use backend::Metal;
diff --git a/crates/backend-uzu/src/bridge/chat_token_backend.rs b/crates/backend-uzu/src/bridge/chat_token_backend.rs
index 5c60f58ae..01fa5dc95 100644
--- a/crates/backend-uzu/src/bridge/chat_token_backend.rs
+++ b/crates/backend-uzu/src/bridge/chat_token_backend.rs
@@ -50,7 +50,7 @@ pub struct UzuChatTokenBackendInstance<B: Backend> {
#[cfg(grammar)]
tokenizer: Arc<Tokenizer>,
stop_token_ids: Vec<i32>,
- max_context_length: Option<usize>,
+ max_context_length: Option<u32>,
}
impl<B: Backend> UzuChatTokenBackendInstance<B> {
@@ -167,7 +167,7 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
impl<B: Backend> ChatTokenBackendInstance for UzuChatTokenBackendInstance<B> {
fn max_context_length(&self) -> Option<usize> {
- self.max_context_length
+ self.max_context_length.map(|max_context_length| max_context_length as usize)
}
fn stop_token_ids(&self) -> Option<Box<[u64]>> {
diff --git a/crates/backend-uzu/src/bridge/helpers.rs b/crates/backend-uzu/src/bridge/helpers.rs
index c52d2375e..36eb42de9 100644
--- a/crates/backend-uzu/src/bridge/helpers.rs
+++ b/crates/backend-uzu/src/bridge/helpers.rs
@@ -48,7 +48,7 @@ pub fn get_grammar(
pub fn get_max_context_length<B: Backend>(
model: &LanguageModel<B>,
context_length: ContextLength,
-) -> Option<usize> {
+) -> Option<u32> {
match context_length {
ContextLength::Default {
..
@@ -58,7 +58,7 @@ pub fn get_max_context_length<B: Backend>(
} => model.max_context_length(),
ContextLength::Custom {
length,
- } => Some(length.max(0) as usize),
+ } => Some(length.max(0) as u32),
}
}
@@ -89,7 +89,7 @@ pub fn get_sampling_method<B: Backend>(
top_p: top_p.map(|value| value as f32),
min_p: min_p.map(|value| value as f32),
repetition_penalty: repetition_penalty.map(|value| value as f32),
- suffix_repetition_length: suffix_repetition_length.map(|value| value as usize),
+ suffix_repetition_length: suffix_repetition_length.map(|value| value as u32),
},
},
}
diff --git a/crates/backend-uzu/src/config/classifier.rs b/crates/backend-uzu/src/config/classifier.rs
index 827eab624..5934bf35a 100644
--- a/crates/backend-uzu/src/config/classifier.rs
+++ b/crates/backend-uzu/src/config/classifier.rs
@@ -29,10 +29,10 @@ pub struct ClassifierConfig {
pub transformer_config: TransformerConfig,
pub prediction_head_config: PredictionHeadConfig,
- pub vocab_size: usize,
- pub model_dim: usize,
- pub hidden_dim: usize,
- pub num_labels: usize,
+ pub vocab_size: u32,
+ pub model_dim: u32,
+ pub hidden_dim: u32,
+ pub num_labels: u32,
pub classifier_pooling: PoolingType,
pub output_labels: Option<Box<[String]>>,
}
diff --git a/crates/backend-uzu/src/config/decoder.rs b/crates/backend-uzu/src/config/decoder.rs
index 1c6d4af35..23ccab305 100644
--- a/crates/backend-uzu/src/config/decoder.rs
+++ b/crates/backend-uzu/src/config/decoder.rs
@@ -10,7 +10,7 @@ pub struct DecoderConfig {
pub embedding_config: AnyEmbeddingConfig,
pub transformer_config: TransformerConfig,
- pub vocab_size: usize,
+ pub vocab_size: u32,
pub ple_model_config: Option<PLEModelConfig>,
pub embedding_norm_config: Option<NormalizationConfig>,
}
diff --git a/crates/backend-uzu/src/config/dflash.rs b/crates/backend-uzu/src/config/dflash.rs
index 71f06dbd8..c8c4391d6 100644
--- a/crates/backend-uzu/src/config/dflash.rs
+++ b/crates/backend-uzu/src/config/dflash.rs
@@ -7,13 +7,13 @@ use crate::config::{
#[uzu_config]
pub struct DFlashDraftConfig {
- pub model_dim: usize,
- pub hidden_dim: usize,
- pub block_size: usize,
+ pub model_dim: u32,
+ pub hidden_dim: u32,
+ pub block_size: u32,
pub mask_token_id: u64,
- pub target_layer_ids: Box<[usize]>,
- pub num_target_layers: usize,
- pub vocab_size: usize,
+ pub target_layer_ids: Box<[u32]>,
+ pub num_target_layers: u32,
+ pub vocab_size: u32,
pub context_projection_config: LinearConfig,
pub context_norm_config: NormalizationConfig,
pub rope_config: AnyRoPEConfig,
diff --git a/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs b/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs
index 8678c539b..624ce4fbf 100644
--- a/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs
+++ b/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs
@@ -11,11 +11,11 @@ pub struct MixtureOfExpertsConfig {
pub router_config: LinearConfig,
pub routing_function: AnyRoutingFunction,
- pub num_routed_experts: usize,
- pub num_active_routed_experts: usize,
+ pub num_routed_experts: u32,
+ pub num_active_routed_experts: u32,
pub router_has_biases: bool,
- pub num_shared_experts: usize,
- pub expert_hidden_dim: usize,
+ pub num_shared_experts: u32,
+ pub expert_hidden_dim: u32,
pub gate_config: Option<LinearConfig>,
}
diff --git a/crates/backend-uzu/src/config/model/generation.rs b/crates/backend-uzu/src/config/model/generation.rs
index 2be8ff950..e85b2deef 100644
--- a/crates/backend-uzu/src/config/model/generation.rs
+++ b/crates/backend-uzu/src/config/model/generation.rs
@@ -13,5 +13,5 @@ pub struct GenerationConfig {
pub repetition_penalty: Option<f32>,
pub presence_penalty: Option<Unsupported>,
pub frequency_penalty: Option<Unsupported>,
- pub suffix_repetition_length: Option<usize>,
+ pub suffix_repetition_length: Option<u32>,
}
diff --git a/crates/backend-uzu/src/config/per_layer_embedding.rs b/crates/backend-uzu/src/config/per_layer_embedding.rs
index dc472f437..adf5034a1 100644
--- a/crates/backend-uzu/src/config/per_layer_embedding.rs
+++ b/crates/backend-uzu/src/config/per_layer_embedding.rs
@@ -4,9 +4,9 @@ use super::{activation::AnyActivation, linear::LinearConfig, normalization::Norm
#[uzu_config]
pub struct PLEModelConfig {
- pub ple_dim: usize,
- pub num_layers: usize,
- pub ple_vocab_size: usize,
+ pub ple_dim: u32,
+ pub num_layers: u32,
+ pub ple_vocab_size: u32,
pub ple_embed_scale: f32,
pub model_projection_scale: f32,
pub input_scale: f32,
@@ -18,6 +18,6 @@ pub struct PLEModelConfig {
pub struct PLELayerConfig {
pub linear_config: LinearConfig,
pub norm_config: NormalizationConfig,
- pub ple_dim: usize,
+ pub ple_dim: u32,
pub activation: AnyActivation,
}
diff --git a/crates/backend-uzu/src/config/rope/llama_rope.rs b/crates/backend-uzu/src/config/rope/llama_rope.rs
index 8e6ac47ba..9b304b06a 100644
--- a/crates/backend-uzu/src/config/rope/llama_rope.rs
+++ b/crates/backend-uzu/src/config/rope/llama_rope.rs
@@ -3,7 +3,7 @@ use proc_macros::uzu_config;
#[uzu_config(super::RoPEConfig)]
pub struct LlamaRoPEConfig {
pub scaling_factor: f32,
- pub original_context_length: usize,
+ pub original_context_length: u32,
pub low_frequency_factor: f32,
pub high_frequency_factor: f32,
}
diff --git a/crates/backend-uzu/src/config/rope/longrope.rs b/crates/backend-uzu/src/config/rope/longrope.rs
index 403ef4581..f0a65cfd4 100644
--- a/crates/backend-uzu/src/config/rope/longrope.rs
+++ b/crates/backend-uzu/src/config/rope/longrope.rs
@@ -4,6 +4,6 @@ use proc_macros::uzu_config;
pub struct LongRoPEConfig {
pub short_factor: Box<[f32]>,
pub long_factor: Box<[f32]>,
- pub original_context_length: usize,
+ pub original_context_length: u32,
pub scaling_factor: f32,
}
diff --git a/crates/backend-uzu/src/config/rope/mod.rs b/crates/backend-uzu/src/config/rope/mod.rs
index e493f5ab7..7bf793f99 100644
--- a/crates/backend-uzu/src/config/rope/mod.rs
+++ b/crates/backend-uzu/src/config/rope/mod.rs
@@ -15,6 +15,6 @@ pub mod yarn_rope;
)]
pub struct RoPEConfig {
pub base: f32,
- pub max_sequence_length: usize,
- pub head_dim: usize,
+ pub max_sequence_length: u32,
+ pub head_dim: u32,
}
diff --git a/crates/backend-uzu/src/config/rope/yarn_rope.rs b/crates/backend-uzu/src/config/rope/yarn_rope.rs
index 491b10707..6cb977f3b 100644
--- a/crates/backend-uzu/src/config/rope/yarn_rope.rs
+++ b/crates/backend-uzu/src/config/rope/yarn_rope.rs
@@ -3,7 +3,7 @@ use proc_macros::uzu_config;
#[uzu_config(super::RoPEConfig)]
pub struct YARNRoPEConfig {
pub scaling_factor: f32,
- pub original_context_length: usize,
+ pub original_context_length: u32,
pub beta_fast: f32,
pub beta_slow: f32,
pub truncate: bool,
diff --git a/crates/backend-uzu/src/config/token_mixer/attention.rs b/crates/backend-uzu/src/config/token_mixer/attention.rs
index 9f5f3f09d..96c2b48a1 100644
--- a/crates/backend-uzu/src/config/token_mixer/attention.rs
+++ b/crates/backend-uzu/src/config/token_mixer/attention.rs
@@ -13,13 +13,12 @@ pub struct AttentionConfig {
pub query_norm_config: Option<NormalizationConfig>,
pub key_norm_config: Option<NormalizationConfig>,
- pub num_heads: usize,
- pub num_groups: usize,
- pub head_dim: usize,
+ pub num_heads: u32,
+ pub num_groups: u32,
+ pub head_dim: u32,
pub is_causal: bool,
pub scale: Option<f32>,
- pub sliding_window_size: Option<usize>,
-
+ pub sliding_window_size: Option<u32>,
pub logit_soft_cap: Option<f32>,
pub has_sinks: bool,
pub has_qkv_biases: bool,
diff --git a/crates/backend-uzu/src/config/token_mixer/delta_net.rs b/crates/backend-uzu/src/config/token_mixer/delta_net.rs
index 960559fc7..d0ae33ab6 100644
--- a/crates/backend-uzu/src/config/token_mixer/delta_net.rs
+++ b/crates/backend-uzu/src/config/token_mixer/delta_net.rs
@@ -11,9 +11,9 @@ pub struct DeltaNetConfig {
pub out_proj_config: LinearConfig,
pub norm_config: NormalizationConfig,
- pub num_heads: usize,
- pub num_groups: usize,
- pub head_dim: usize,
- pub value_head_dim: usize,
- pub kernel_size: usize,
+ pub num_heads: u32,
+ pub num_groups: u32,
+ pub head_dim: u32,
+ pub value_head_dim: u32,
+ pub kernel_size: u32,
}
diff --git a/crates/backend-uzu/src/config/token_mixer/mamba2.rs b/crates/backend-uzu/src/config/token_mixer/mamba2.rs
index d3139073a..0c17eddf6 100644
--- a/crates/backend-uzu/src/config/token_mixer/mamba2.rs
+++ b/crates/backend-uzu/src/config/token_mixer/mamba2.rs
@@ -11,12 +11,11 @@ pub struct Mamba2Config {
pub conv_config: SeparableCausalConvConfig,
pub activation: AnyActivation,
- pub kernel_size: usize,
- pub num_heads: usize,
- pub num_groups: usize,
- pub head_dim: usize,
- pub state_dim: usize,
-
+ pub kernel_size: u32,
+ pub num_heads: u32,
+ pub num_groups: u32,
+ pub head_dim: u32,
+ pub state_dim: u32,
pub has_in_biases: bool,
pub has_out_biases: bool,
}
diff --git a/crates/backend-uzu/src/config/token_mixer/short_conv.rs b/crates/backend-uzu/src/config/token_mixer/short_conv.rs
index aa0c11f9c..30c05933d 100644
--- a/crates/backend-uzu/src/config/token_mixer/short_conv.rs
+++ b/crates/backend-uzu/src/config/token_mixer/short_conv.rs
@@ -8,5 +8,5 @@ pub struct ShortConvConfig {
pub conv_config: SeparableCausalConvConfig,
pub out_projection_config: LinearConfig,
- pub kernel_size: usize,
+ pub kernel_size: u32,
}
diff --git a/crates/backend-uzu/src/config/transformer.rs b/crates/backend-uzu/src/config/transformer.rs
index 9bf25d872..d2041d4ec 100644
--- a/crates/backend-uzu/src/config/transformer.rs
+++ b/crates/backend-uzu/src/config/transformer.rs
@@ -6,6 +6,6 @@ use crate::config::{normalization::NormalizationConfig, transformer_layer::Trans
pub struct TransformerConfig {
pub layer_configs: Box<[TransformerLayerConfig]>,
pub output_norm_config: NormalizationConfig,
- pub model_dim: usize,
- pub hidden_dim: usize,
+ pub model_dim: u32,
+ pub hidden_dim: u32,
}
diff --git a/crates/backend-uzu/src/config/transformer_layer.rs b/crates/backend-uzu/src/config/transformer_layer.rs
index 7bd395cb6..0fce99a05 100644
--- a/crates/backend-uzu/src/config/transformer_layer.rs
+++ b/crates/backend-uzu/src/config/transformer_layer.rs
@@ -13,9 +13,9 @@ pub struct TransformerLayerConfig {
pub pre_mlp_norm_config: NormalizationConfig,
pub mlp_config: AnyMLPConfig,
pub post_mlp_norm_config: Option<NormalizationConfig>,
- pub hidden_dim: Option<usize>,
+ pub hidden_dim: Option<u32>,
pub ple_config: Option<PLELayerConfig>,
pub has_post_layer_scalar: bool,
- pub kv_source_layer_index: Option<usize>,
+ pub kv_source_layer_index: Option<u32>,
pub rope_config: Option<AnyRoPEConfig>,
}
diff --git a/crates/backend-uzu/src/config/weaver.rs b/crates/backend-uzu/src/config/weaver.rs
index c106830a1..fe2877513 100644
--- a/crates/backend-uzu/src/config/weaver.rs
+++ b/crates/backend-uzu/src/config/weaver.rs
@@ -4,14 +4,14 @@ use crate::config::{linear::LinearConfig, normalization::NormalizationConfig, ro
#[uzu_config]
pub struct WeaverConfig {
- pub model_dim: usize,
- pub target_model_dim: usize,
- pub target_embedding_dim: usize,
- pub num_layers: usize,
- pub num_heads: usize,
- pub hidden_dim: usize,
- pub max_depth: usize,
- pub candidate_pool_size: usize,
+ pub model_dim: u32,
+ pub target_model_dim: u32,
+ pub target_embedding_dim: u32,
+ pub num_layers: u32,
+ pub num_heads: u32,
+ pub hidden_dim: u32,
+ pub max_depth: u32,
+ pub candidate_pool_size: u32,
pub linear_config: LinearConfig,
pub norm_config: NormalizationConfig,
pub rope_config: AnyRoPEConfig,
diff --git a/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs b/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs
index 7c9e08009..cfc1b6a2a 100644
--- a/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs
+++ b/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs
@@ -14,6 +14,6 @@ pub enum IncoherenceProcessingMode {
pub struct HybridSpec {
pub quantization_spec: Box<AnyWeightMatrixSpec>,
pub adapter_spec: Option<Box<AnyWeightMatrixSpec>>,
- pub incoherence_block_size: Option<usize>,
+ pub incoherence_block_size: Option<u32>,
pub incoherence_processing_mode: IncoherenceProcessingMode,
}
diff --git a/crates/backend-uzu/src/config/weight_matrix/int_spec.rs b/crates/backend-uzu/src/config/weight_matrix/int_spec.rs
index 500478697..ee46d63d2 100644
--- a/crates/backend-uzu/src/config/weight_matrix/int_spec.rs
+++ b/crates/backend-uzu/src/config/weight_matrix/int_spec.rs
@@ -5,7 +5,7 @@ use crate::config::weight_matrix::Layout;
#[uzu_config(super::WeightMatrixSpec)]
pub struct IntSpec {
pub bits: u32,
- pub group_size: usize,
+ pub group_size: u32,
pub is_symmetric: bool,
pub layout: Layout,
}
diff --git a/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs b/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs
index d056781e4..a4d9d90a3 100644
--- a/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs
+++ b/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs
@@ -2,5 +2,5 @@ use proc_macros::uzu_config;
#[uzu_config(super::WeightMatrixSpec)]
pub struct LowRankSpec {
- pub rank: usize,
+ pub rank: u32,
}
diff --git a/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs b/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs
index eda4bf6c5..ba60186e0 100644
--- a/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs
+++ b/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs
@@ -5,6 +5,6 @@ use crate::config::weight_matrix::Layout;
#[uzu_config(super::WeightMatrixSpec)]
pub struct MLXSpec {
pub bits: u32,
- pub group_size: usize,
+ pub group_size: u32,
pub layout: Layout,
}
diff --git a/crates/backend-uzu/src/encodable_block/batch_topology.rs b/crates/backend-uzu/src/encodable_block/batch_topology.rs
index 2c8124247..4606f8562 100644
--- a/crates/backend-uzu/src/encodable_block/batch_topology.rs
+++ b/crates/backend-uzu/src/encodable_block/batch_topology.rs
@@ -17,12 +17,12 @@ impl<'a> BatchTopology<'a> {
let mut parents = Box::new_uninit_slice(nodes.len());
let mut is_flat = true;
- for (index, node) in nodes.iter().enumerate() {
+ for (node, index) in nodes.iter().zip(0u32..) {
stack.truncate(node.height as usize);
- parents[index].write(stack.last().map(|i| *i as i32).unwrap_or(-1));
- stack.push(index);
+ parents[index as usize].write(stack.last().map(|i| *i as i32).unwrap_or(-1));
+ stack.push(index as usize);
- is_flat &= node.height == index as u32;
+ is_flat &= node.height == index;
}
let parents = unsafe { parents.assume_init() };
@@ -40,8 +40,8 @@ impl<'a> BatchTopology<'a> {
self.nodes
}
- pub fn size(&self) -> usize {
- self.nodes.len()
+ pub fn size(&self) -> u32 {
+ self.nodes.len() as u32
}
pub fn is_flat(&self) -> bool {
diff --git a/crates/backend-uzu/src/encodable_block/classifier.rs b/crates/backend-uzu/src/encodable_block/classifier.rs
index f2af4a3b3..2f5ee208d 100644
--- a/crates/backend-uzu/src/encodable_block/classifier.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier.rs
@@ -1,7 +1,6 @@
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{Allocation, Backend, Encoder, Kernels, gpu_types::trie::TrieNode, kernel::PoolingMeanKernel},
config::classifier::{ClassifierConfig, PoolingType},
data_type::DataType,
@@ -32,7 +31,7 @@ pub enum ClassifierError<B: Backend> {
}
pub struct Classifier<B: Backend> {
- hidden_dim: usize,
+ hidden_dim: u32,
data_type: DataType,
embedding: Embedding<B>,
embedding_norm: Normalization<B>,
@@ -50,8 +49,8 @@ impl<B: Backend> Classifier<B> {
) -> Result<Self, ClassifierError<B>> {
let (embedding, _) = Embedding::new(
context,
- config.vocab_size as u32,
- config.transformer_config.model_dim as u32,
+ config.vocab_size,
+ config.transformer_config.model_dim,
&config.embedding_config,
&parameter_tree.subtree("embedding"),
data_type,
@@ -105,14 +104,14 @@ impl<B: Backend> Classifier<B> {
})
}
- pub fn max_context_length(&self) -> Option<usize> {
+ pub fn max_context_length(&self) -> Option<u32> {
self.transformer.max_context_length()
}
pub fn encode(
&self,
token_ids: &Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, ClassifierError<B>> {
encoder.push_debug_group("classifier");
@@ -124,9 +123,9 @@ impl<B: Backend> Classifier<B> {
let nodes = (0..batch_dim)
.map(|index| TrieNode {
- trie_start: index as u32,
- trie_end: (batch_dim - 1) as u32,
- height: index as u32,
+ trie_start: index,
+ trie_end: batch_dim - 1,
+ height: index,
})
.collect::<Box<[TrieNode]>>();
let hidden = self
@@ -136,10 +135,9 @@ impl<B: Backend> Classifier<B> {
.output
.unwrap();
- let mut pooled = encoder
- .allocate_scratch(size_for_shape(&[self.hidden_dim], self.data_type))
- .map_err(ClassifierError::Backend)?;
- self.pooling.encode(&hidden, &mut pooled, batch_dim as u32, self.hidden_dim as u32, 1, encoder);
+ let mut pooled =
+ encoder.allocate_scratch_for_shape(&[self.hidden_dim], self.data_type).map_err(ClassifierError::Backend)?;
+ self.pooling.encode(&hidden, &mut pooled, batch_dim, self.hidden_dim, 1, encoder);
let logits = self.prediction_head.encode(pooled, 1, encoder).map_err(ClassifierError::Backend)?;
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index 957ca02d2..6a145194f 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -58,8 +58,8 @@ impl<B: Backend> Decoder<B> {
) -> Result<Self, DecoderError<B>> {
let (embedding, readout_input_hadamard_factors) = Embedding::new(
context,
- config.vocab_size as u32,
- config.transformer_config.model_dim as u32,
+ config.vocab_size,
+ config.transformer_config.model_dim,
&config.embedding_config,
&parameter_tree.subtree("embedding"),
data_type,
@@ -85,7 +85,7 @@ impl<B: Backend> Decoder<B> {
let per_layer_embedding = if let Some(ple_config) = &config.ple_model_config {
assert_eq!(
ple_config.num_layers,
- config.transformer_config.layer_configs.len(),
+ config.transformer_config.layer_configs.len() as u32,
"per-layer embedding num_layers must match transformer layer count"
);
Some(PerLayerEmbedding::new(
@@ -119,7 +119,7 @@ impl<B: Backend> Decoder<B> {
self.transformer.speculation_supported()
}
- pub fn max_context_length(&self) -> Option<usize> {
+ pub fn max_context_length(&self) -> Option<u32> {
self.transformer.max_context_length()
}
@@ -129,7 +129,7 @@ impl<B: Backend> Decoder<B> {
pub fn create_empty_state(
&self,
- max_context_length: Option<usize>,
+ max_context_length: Option<u32>,
context: &B::Context,
) -> Result<TransformerState<B>, B::Error> {
self.transformer.create_empty_state(max_context_length, context)
@@ -139,8 +139,8 @@ impl<B: Backend> Decoder<B> {
&self,
token_ids: &Allocation<B>,
batch_dim: &BatchTopology,
- output_range: Option<Range<usize>>,
- hidden_feature_layer_indices: Option<&[usize]>,
+ output_range: Option<Range<u32>>,
+ hidden_feature_layer_indices: Option<&[u32]>,
state: &mut TransformerState<B>,
encoder: &mut Encoder<B>,
) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {
@@ -178,7 +178,12 @@ impl<B: Backend> Decoder<B> {
let logits = if let Some(output_range) = output_range {
let output = transformer_output.output.as_ref().expect("decoder output range requires transformer output");
- Some(self.embedding.encode_readout(output_range.len(), output, self.embedding.data_type(), encoder)?)
+ Some(self.embedding.encode_readout(
+ output_range.end - output_range.start,
+ output,
+ self.embedding.data_type(),
+ encoder,
+ )?)
} else {
None
};
diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rs
index 475a0c077..1eba2a76e 100644
--- a/crates/backend-uzu/src/encodable_block/dflash.rs
+++ b/crates/backend-uzu/src/encodable_block/dflash.rs
@@ -3,6 +3,7 @@ use std::any::Any;
use thiserror::Error;
use crate::{
+ array::size_for_shape,
backends::common::{Allocation, Backend, Encoder, gpu_types::trie::TrieNode},
config::{dflash::DFlashDraftConfig, rope::AnyRoPEConfig, token_mixer::AnyTokenMixerConfig},
data_type::DataType,
@@ -26,12 +27,12 @@ use crate::{
pub struct DFlashState<B: Backend> {
layer_states: Box<[Box<dyn MixerState<B>>]>,
- context_length: usize,
- context_capacity: usize,
+ context_length: u32,
+ context_capacity: u32,
}
impl<B: Backend> DFlashState<B> {
- pub fn context_length(&self) -> usize {
+ pub fn context_length(&self) -> u32 {
self.context_length
}
}
@@ -40,15 +41,15 @@ pub struct DFlash<B: Backend> {
target_feature_projection: Box<dyn Linear<B>>,
projected_feature_norm: Normalization<B>,
state_kv_projection: Box<dyn Linear<B>>,
- layer_kv_dim: usize,
+ layer_kv_dim: u32,
layers: Box<[TransformerLayer<B>]>,
output_norm: Normalization<B>,
rope_config: AnyRoPEConfig,
- model_dim: usize,
- max_context_length: usize,
- block_size: usize,
+ model_dim: u32,
+ max_context_length: u32,
+ block_size: u32,
mask_token_id: u32,
- target_feature_input_dim: usize,
+ target_feature_input_dim: u32,
data_type: DataType,
}
@@ -92,8 +93,13 @@ impl<B: Backend> DFlash<B> {
) -> Result<Self, DFlashNewError<B>> {
let mask_token_id = config.mask_token_id as u32;
assert!(config.block_size <= ATTENTION_SUFFIX_CAPACITY, "DFlash block_size exceeds attention suffix capacity");
+ assert_eq!(
+ config.num_target_layers,
+ config.target_layer_ids.len() as u32,
+ "num_target_layers must match target_layer_ids length"
+ );
let target_feature_projection = <dyn Linear<B>>::new(
- config.model_dim * config.target_layer_ids.len(),
+ config.model_dim * config.num_target_layers,
[config.model_dim],
false,
context,
@@ -116,9 +122,10 @@ impl<B: Backend> DFlash<B> {
return Err(DFlashNewError::InvalidAttentionConfig("DFlash layers must use attention mixers"));
};
let layer_kv_dim = 2 * attention_config.num_groups * attention_config.head_dim;
+ let num_layers = config.layer_configs.len() as u32;
let state_kv_projection = <dyn Linear<B>>::new(
config.model_dim,
- [config.layer_configs.len() * layer_kv_dim],
+ [num_layers * layer_kv_dim],
false,
context,
data_type,
@@ -127,13 +134,13 @@ impl<B: Backend> DFlash<B> {
let layers = config
.layer_configs
.iter()
- .enumerate()
- .map(|(index, layer_config)| {
+ .zip(0u32..)
+ .map(|(layer_config, index)| {
TransformerLayer::new(
context,
config.model_dim,
config.hidden_dim,
- config.layer_configs.len(),
+ num_layers,
layer_config,
index,
&layers_tree.subtree(&index.to_string()),
@@ -164,18 +171,18 @@ impl<B: Backend> DFlash<B> {
max_context_length: *config.rope_config.max_sequence_length(),
block_size: config.block_size,
mask_token_id,
- target_feature_input_dim: config.model_dim * config.target_layer_ids.len(),
+ target_feature_input_dim: config.model_dim * config.num_target_layers,
data_type,
})
}
- pub fn block_size(&self) -> usize {
+ pub fn block_size(&self) -> u32 {
self.block_size
}
pub fn empty_state(
&self,
- context_capacity: usize,
+ context_capacity: u32,
context: &B::Context,
) -> Result<DFlashState<B>, B::Error> {
assert!(context_capacity <= self.max_context_length, "DFlash state capacity exceeds configured RoPE capacity");
@@ -195,7 +202,7 @@ impl<B: Backend> DFlash<B> {
&self,
state: &mut DFlashState<B>,
target_features: &[Allocation<B>],
- accepted_indices: &[usize],
+ accepted_indices: &[u32],
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
if accepted_indices.is_empty() {
@@ -204,20 +211,20 @@ impl<B: Backend> DFlash<B> {
encoder.push_debug_group("dflash accept");
- let num_tokens = accepted_indices.len();
+ let num_tokens = accepted_indices.len() as u32;
let captured_layer_count = self.target_feature_input_dim / self.model_dim;
- assert_eq!(target_features.len(), captured_layer_count);
- let layer_feature_bytes = self.model_dim * self.data_type.size_in_bytes();
+ assert_eq!(target_features.len() as u32, captured_layer_count);
+ let layer_feature_bytes = size_for_shape(&[self.model_dim], self.data_type);
assert!(target_features.iter().all(|features| features.size() % layer_feature_bytes == 0));
let context_length = state.context_length + num_tokens;
assert!(context_length <= state.context_capacity, "DFlash state capacity exceeded");
assert!(context_length <= self.max_context_length, "DFlash context exceeds configured RoPE capacity");
let mut packed_target_features =
- encoder.allocate_scratch(num_tokens * self.target_feature_input_dim * self.data_type.size_in_bytes())?;
+ encoder.allocate_scratch_for_shape(&[num_tokens, self.target_feature_input_dim], self.data_type)?;
for (layer_index, features) in target_features.iter().enumerate() {
for (token_index, &accepted_index) in accepted_indices.iter().enumerate() {
- let source_start = accepted_index * layer_feature_bytes;
+ let source_start = accepted_index as usize * layer_feature_bytes;
assert!(source_start + layer_feature_bytes <= features.size(), "accepted index out of feature bounds");
let destination_start = (token_index * target_features.len() + layer_index) * layer_feature_bytes;
encoder.encode_copy(
@@ -235,13 +242,13 @@ impl<B: Backend> DFlash<B> {
let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, encoder)?;
let projected_kv = self.state_kv_projection.encode(normalized_features, num_tokens, encoder)?;
- let layer_kv_bytes = self.layer_kv_dim * self.data_type.size_in_bytes();
+ let layer_kv_bytes = size_for_shape(&[self.layer_kv_dim], self.data_type);
let kv_chunk = |chunk_index: usize| chunk_index * layer_kv_bytes..(chunk_index + 1) * layer_kv_bytes;
let mut layer_key_values = (0..self.layers.len())
- .map(|_| encoder.allocate_scratch(num_tokens * layer_kv_bytes))
+ .map(|_| encoder.allocate_scratch(num_tokens as usize * layer_kv_bytes))
.collect::<Result<Box<[_]>, _>>()?;
for (layer_index, key_value) in layer_key_values.iter_mut().enumerate() {
- for token_index in 0..num_tokens {
+ for token_index in 0..num_tokens as usize {
encoder.encode_copy(
&projected_kv,
kv_chunk(token_index * self.layers.len() + layer_index),
@@ -275,7 +282,7 @@ impl<B: Backend> DFlash<B> {
state: &mut DFlashState<B>,
target_output_token: u32,
target_embedding: &Embedding<B>,
- batch_size: usize,
+ batch_size: u32,
encoder: &mut Encoder<B>,
) -> Result<DFlashOutput<B>, DFlashEncodeError<B>> {
encoder.push_debug_group("dflash draft");
@@ -286,7 +293,7 @@ impl<B: Backend> DFlash<B> {
"DFlash block positions exceed configured RoPE capacity"
);
- let mut tokens = vec![self.mask_token_id; batch_size];
+ let mut tokens = vec![self.mask_token_id; batch_size as usize];
tokens[0] = target_output_token;
let token_ids = encoder.allocate_constant_from_slice(&tokens).map_err(DFlashEncodeError::Backend)?;
@@ -294,9 +301,9 @@ impl<B: Backend> DFlash<B> {
let nodes = (0..batch_size)
.map(|index| TrieNode {
- trie_start: index as u32,
- trie_end: (batch_size - 1) as u32,
- height: index as u32,
+ trie_start: index,
+ trie_end: batch_size - 1,
+ height: index,
})
.collect::<Box<[_]>>();
let batch_topology = BatchTopology::new(&nodes, true);
@@ -327,10 +334,11 @@ impl<B: Backend> DFlash<B> {
.encode(&hidden, 0, batch_size, Some(&mut residual), encoder)
.map_err(DFlashEncodeError::Backend)?;
- let row_bytes = target_embedding.model_dim() * DataType::BF16.size_in_bytes();
+ let row_bytes = size_for_shape(&[target_embedding.model_dim()], DataType::BF16);
+ let lookahead_rows = row_bytes..batch_size as usize * row_bytes;
let mut lookahead_hidden =
- encoder.allocate_scratch((batch_size - 1) * row_bytes).map_err(DFlashEncodeError::Backend)?;
- encoder.encode_copy(&draft_hidden, row_bytes..batch_size * row_bytes, &mut lookahead_hidden, ..);
+ encoder.allocate_scratch(lookahead_rows.len()).map_err(DFlashEncodeError::Backend)?;
+ encoder.encode_copy(&draft_hidden, lookahead_rows, &mut lookahead_hidden, ..);
let logits = target_embedding.encode_readout(batch_size - 1, &lookahead_hidden, DataType::F32, encoder)?;
encoder.pop_debug_group();
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 13ba866eb..e04b78329 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -2,7 +2,6 @@ use parking_lot::Mutex;
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder, Kernels,
gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
@@ -83,12 +82,12 @@ impl<B: Backend> Embedding<B> {
self.data_type
}
- pub(crate) fn vocab_size(&self) -> usize {
- self.vocab_size as usize
+ pub(crate) fn vocab_size(&self) -> u32 {
+ self.vocab_size
}
- pub(crate) fn model_dim(&self) -> usize {
- self.model_dim as usize
+ pub(crate) fn model_dim(&self) -> u32 {
+ self.model_dim
}
fn readout_input_hadamard(&self) -> Option<&InputHadamard<B>> {
@@ -136,8 +135,8 @@ impl<B: Backend> Embedding<B> {
let table = EmbeddingTable::load_with_spec(
context,
&embedding_tree,
- vocab_size as usize,
- model_dim as usize,
+ vocab_size,
+ model_dim,
data_type,
spec,
None,
@@ -162,21 +161,21 @@ impl<B: Backend> Embedding<B> {
let output_hadamard_factors = Some(
incoherence_signs_tree
.leaf("output_signs")?
- .validate(&[model_dim as usize], DataType::I32)?
+ .validate(&[model_dim], DataType::I32)?
.read_allocation()?,
);
let readout_input_hadamard_factors = Some(
incoherence_signs_tree
.leaf("output_signs")?
- .validate(&[model_dim as usize], DataType::I32)?
+ .validate(&[model_dim], DataType::I32)?
.read_allocation()?,
);
let table = EmbeddingTable::load_with_spec(
context,
&embedding_tree.subtree("quantized"),
- vocab_size as usize,
- model_dim as usize,
+ vocab_size,
+ model_dim,
data_type,
*quantization_spec,
output_hadamard_factors,
@@ -210,14 +209,14 @@ impl<B: Backend> Embedding<B> {
input_embedding_tree
.subtree("incoherence_signs")
.leaf("output_signs")?
- .validate(&[model_dim as usize], DataType::I32)?
+ .validate(&[model_dim], DataType::I32)?
.read_allocation()?,
);
EmbeddingTable::load_with_spec(
context,
&input_embedding_tree.subtree("quantized"),
- vocab_size as usize,
- model_dim as usize,
+ vocab_size,
+ model_dim,
data_type,
*quantization_spec,
output_hadamard_factors,
@@ -226,8 +225,8 @@ impl<B: Backend> Embedding<B> {
spec => EmbeddingTable::load_with_spec(
context,
&input_embedding_tree,
- vocab_size as usize,
- model_dim as usize,
+ vocab_size,
+ model_dim,
data_type,
spec,
None,
@@ -249,8 +248,8 @@ impl<B: Backend> Embedding<B> {
&output_embedding_tree.subtree("quantized"),
*quantization_spec,
Layout::OutputInput,
- vocab_size as usize,
- model_dim as usize,
+ vocab_size,
+ model_dim,
data_type,
)?;
@@ -260,7 +259,7 @@ impl<B: Backend> Embedding<B> {
let factors = output_embedding_tree
.subtree("incoherence_signs")
.leaf("input_signs")?
- .validate(&[model_dim as usize], DataType::I32)?
+ .validate(&[model_dim], DataType::I32)?
.read_allocation()?;
let kernel = ActivationTransform::input_rht(context, data_type, false)
.map_err(EmbeddingError::BackendError)?;
@@ -279,8 +278,8 @@ impl<B: Backend> Embedding<B> {
&output_embedding_tree,
spec,
Layout::OutputInput,
- vocab_size as usize,
- model_dim as usize,
+ vocab_size,
+ model_dim,
data_type,
)?;
UntiedReadout {
@@ -346,15 +345,14 @@ impl<B: Backend> Embedding<B> {
pub fn encode_lookup(
&self,
token_ids: &Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, EmbeddingError<B>> {
encoder.push_debug_group("embedding lookup");
let mut output = encoder
- .allocate_scratch(size_for_shape(&[batch_dim, self.model_dim as usize], self.data_type))
+ .allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)
.map_err(EmbeddingError::BackendError)?;
- let batch_dim = batch_dim as u32;
let table = match &self.tying {
EmbeddingTying::Tied {
@@ -375,7 +373,7 @@ impl<B: Backend> Embedding<B> {
pub fn encode_readout(
&self,
- batch_dim: usize,
+ batch_dim: u32,
input_allocation: &Allocation<B>,
output_data_type: DataType,
encoder: &mut Encoder<B>,
@@ -386,7 +384,7 @@ impl<B: Backend> Embedding<B> {
let native_output = output_data_type == self.data_type;
let input_hadamard = self.readout_input_hadamard();
let mut output_allocation = encoder
- .allocate_scratch(size_for_shape(&[batch_dim, self.vocab_size as usize], output_data_type))
+ .allocate_scratch_for_shape(&[batch_dim, self.vocab_size], output_data_type)
.map_err(EmbeddingError::BackendError)?;
let (matrix, readout) = self.readout_operands();
@@ -399,7 +397,7 @@ impl<B: Backend> Embedding<B> {
input_allocation,
&mut transformed,
&input_hadamard.factors,
- batch_dim as u32,
+ batch_dim,
self.model_dim,
encoder,
);
@@ -418,7 +416,7 @@ impl<B: Backend> Embedding<B> {
d: &mut output_allocation,
d_transform: MatmulDOps::none(),
gather_indices: None,
- m: batch_dim as u32,
+ m: batch_dim,
n: self.vocab_size,
k: self.model_dim,
};
@@ -436,7 +434,7 @@ impl<B: Backend> Embedding<B> {
}
if let Some(logit_transform) = &self.logit_transform {
- let length = (batch_dim * self.vocab_size as usize) as u32;
+ let length = batch_dim * self.vocab_size;
let kernel = if native_output {
&logit_transform.kernel
} else {
@@ -463,8 +461,8 @@ impl<B: Backend> Embedding<B> {
&self,
input: &Allocation<B>,
token_ids: &Allocation<B>,
- rows: usize,
- ids_per_row: usize,
+ rows: u32,
+ ids_per_row: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, EmbeddingError<B>> {
encoder.push_debug_group("embedding readout (sparse)");
@@ -475,7 +473,7 @@ impl<B: Backend> Embedding<B> {
let b = matrix.matmul_b();
let mut output = encoder
- .allocate_scratch(size_for_shape(&[rows, ids_per_row], self.data_type))
+ .allocate_scratch_for_shape(&[rows, ids_per_row], self.data_type)
.map_err(EmbeddingError::BackendError)?;
let mut rht_input: Option<Allocation<B>> = None;
@@ -486,7 +484,7 @@ impl<B: Backend> Embedding<B> {
input,
&mut transformed,
&input_hadamard.factors,
- rows as u32,
+ rows,
self.model_dim,
encoder,
);
@@ -517,8 +515,8 @@ impl<B: Backend> Embedding<B> {
..MatmulDOps::none()
},
gather_indices: Some(token_ids),
- m: rows as u32,
- n: ids_per_row as u32,
+ m: rows,
+ n: ids_per_row,
k: self.model_dim,
},
encoder,
@@ -528,7 +526,7 @@ impl<B: Backend> Embedding<B> {
if let Some(logit_transform) = &self.logit_transform
&& logit_transform.scale != 1.0
{
- let length = (rows * ids_per_row) as u32;
+ let length = rows * ids_per_row;
logit_transform.kernel.encode(
&mut output,
length,
diff --git a/crates/backend-uzu/src/encodable_block/embedding_table.rs b/crates/backend-uzu/src/encodable_block/embedding_table.rs
index 3df7e8e73..250a1e70c 100644
--- a/crates/backend-uzu/src/encodable_block/embedding_table.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding_table.rs
@@ -40,8 +40,8 @@ impl<B: Backend> EmbeddingTable<B> {
pub fn load(
context: &B::Context,
tree: &ParameterTree<B>,
- vocab_size: usize,
- embedding_dim: usize,
+ vocab_size: u32,
+ embedding_dim: u32,
data_type: DataType,
) -> Result<Self, EmbeddingTableError<B>> {
let spec = tree.metadata::<AnyWeightMatrixSpec>("spec")?;
@@ -51,8 +51,8 @@ impl<B: Backend> EmbeddingTable<B> {
pub fn load_with_spec(
context: &B::Context,
tree: &ParameterTree<B>,
- vocab_size: usize,
- embedding_dim: usize,
+ vocab_size: u32,
+ embedding_dim: u32,
data_type: DataType,
spec: AnyWeightMatrixSpec,
output_hadamard_factors: Option<Allocation<B>>,
@@ -86,8 +86,8 @@ impl<B: Backend> EmbeddingTable<B> {
matrix,
lookup,
output_hadamard_factors,
- vocab_size: vocab_size as u32,
- embedding_dim: embedding_dim as u32,
+ vocab_size,
+ embedding_dim,
})
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
index e64f5a23c..213a857ad 100644
--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
@@ -2,7 +2,6 @@ use parking_lot::Mutex;
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder,
kernel::{
@@ -38,15 +37,15 @@ pub struct LinearMatmul<B: Backend> {
matrix: WeightMatrix<B>,
biases: Option<Allocation<B>>,
output_hadamard_factors: Option<Allocation<B>>,
- input_dim: usize,
- output_dim: usize,
+ input_dim: u32,
+ output_dim: u32,
output_data_type: DataType,
}
fn load_biases<B: Backend>(
weights_data_type: DataType,
output_data_type: DataType,
- output_dim: usize,
+ output_dim: u32,
parameter_tree: Option<&ParameterTree<B>>,
) -> Result<Option<Allocation<B>>, LinearMatmulError<B>> {
if parameter_tree.is_some() && weights_data_type != output_data_type {
@@ -65,8 +64,8 @@ impl<B: Backend> LinearMatmul<B> {
pub fn load(
context: &B::Context,
spec: AnyWeightMatrixSpec,
- input_dim: usize,
- output_dim: usize,
+ input_dim: u32,
+ output_dim: u32,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
@@ -112,11 +111,10 @@ impl<B: Backend> LinearMatmul<B> {
pub(super) fn encode_with_a(
&self,
a: MatmulA<'_, B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut output =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;
+ let mut output = encoder.allocate_scratch_for_shape(&[batch_dim, self.output_dim], self.output_data_type)?;
self.kernel.lock().encode(
MatmulArguments {
@@ -127,9 +125,9 @@ impl<B: Backend> LinearMatmul<B> {
d: &mut output,
d_transform: self.d_ops(),
gather_indices: None,
- m: batch_dim as u32,
- n: self.output_dim as u32,
- k: self.input_dim as u32,
+ m: batch_dim,
+ n: self.output_dim,
+ k: self.input_dim,
},
encoder,
)?;
@@ -151,14 +149,14 @@ impl<B: Backend> LinearMatmul<B> {
pub(super) fn select_path(
&self,
- batch_dim: usize,
+ batch_dim: u32,
context: &B::Context,
) -> MatmulPath {
let b = self.matmul_b();
let shape = MatmulShape {
- m: batch_dim as u32,
- n: self.output_dim as u32,
- k: self.input_dim as u32,
+ m: batch_dim,
+ n: self.output_dim,
+ k: self.input_dim,
b_transpose: true,
b_leading_dimension: None,
b_prologue: b.b_prologue(),
@@ -177,7 +175,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
fn encode(
&self,
input: Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group("matmul");
diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rs
index cce6c7ba2..42594c9cf 100644
--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs
@@ -26,7 +26,7 @@ pub trait Linear<B: Backend>: Send + Sync {
fn encode(
&self,
input: Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error>;
}
@@ -59,8 +59,8 @@ pub enum OutputHadamardLinearError<B: Backend> {
impl<B: Backend> dyn Linear<B> {
pub fn new_mixed_precision<const N: usize>(
- input_dimension: usize,
- output_dimensions: [usize; N],
+ input_dimension: u32,
+ output_dimensions: [u32; N],
has_biases: bool,
context: &B::Context,
weights_data_type: DataType,
@@ -68,7 +68,7 @@ impl<B: Backend> dyn Linear<B> {
output_data_type: DataType,
parameter_tree: &ParameterTree<B>,
) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
- let output_dimension_sum: usize = output_dimensions.iter().sum();
+ let output_dimension_sum: u32 = output_dimensions.iter().sum();
let weights_tree = parameter_tree.subtree("weights");
let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
match spec {
@@ -135,8 +135,8 @@ impl<B: Backend> dyn Linear<B> {
}
pub fn new<const N: usize>(
- input_dimension: usize,
- output_dimensions: [usize; N],
+ input_dimension: u32,
+ output_dimensions: [u32; N],
has_biases: bool,
context: &B::Context,
data_type: DataType,
@@ -158,8 +158,8 @@ impl<B: Backend> dyn Linear<B> {
context: &B::Context,
parameter_tree: &ParameterTree<B>,
output_factors: Allocation<B>,
- input_dim: usize,
- output_dim: usize,
+ input_dim: u32,
+ output_dim: u32,
has_biases: bool,
weights_data_type: DataType,
input_data_type: DataType,
@@ -189,8 +189,8 @@ impl<B: Backend> dyn Linear<B> {
}
pub fn new_extracting_input_hadamard_mixed_precision<const N: usize>(
- input_dimension: usize,
- output_dimensions: [usize; N],
+ input_dimension: u32,
+ output_dimensions: [u32; N],
has_biases: bool,
context: &B::Context,
weights_data_type: DataType,
@@ -198,7 +198,7 @@ impl<B: Backend> dyn Linear<B> {
output_data_type: DataType,
parameter_tree: &ParameterTree<B>,
) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {
- let output_dimension_sum: usize = output_dimensions.iter().sum();
+ let output_dimension_sum: u32 = output_dimensions.iter().sum();
let weights_tree = parameter_tree.subtree("weights");
let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
if let AnyWeightMatrixSpec::HybridSpec(HybridSpec {
@@ -217,7 +217,7 @@ impl<B: Backend> dyn Linear<B> {
input_dimension,
input_data_type,
output_data_type,
- ACTIVATION_SCALE_GROUP_SIZE as usize,
+ ACTIVATION_SCALE_GROUP_SIZE,
)
}) {
let linear = RHTLinearWrapper::new(
@@ -269,8 +269,8 @@ impl<B: Backend> dyn Linear<B> {
}
pub fn new_extracting_input_hadamard<const N: usize>(
- input_dimension: usize,
- output_dimensions: [usize; N],
+ input_dimension: u32,
+ output_dimensions: [u32; N],
has_biases: bool,
context: &B::Context,
data_type: DataType,
diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
index 93d771b76..f5866e5b2 100644
--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
@@ -2,7 +2,6 @@ use parking_lot::Mutex;
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder,
gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
@@ -42,9 +41,9 @@ pub struct QLoRALinearWrapper<B: Backend> {
adapter_up_kernel: Mutex<<B::Kernels as Kernels>::MatmulKernel>,
adapter_down: Allocation<B>,
adapter_up: Allocation<B>,
- input_dim: usize,
- output_dim: usize,
- lora_rank: usize,
+ input_dim: u32,
+ output_dim: u32,
+ lora_rank: u32,
weights_data_type: DataType,
input_data_type: DataType,
}
@@ -54,10 +53,10 @@ impl<B: Backend> QLoRALinearWrapper<B> {
context: &B::Context,
quantization_spec: AnyWeightMatrixSpec,
adapter_spec: LowRankSpec,
- incoherence_block_size: Option<usize>,
+ incoherence_block_size: Option<u32>,
incoherence_processing_mode: IncoherenceProcessingMode,
- input_dim: usize,
- output_dim: usize,
+ input_dim: u32,
+ output_dim: u32,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
@@ -168,13 +167,13 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
fn encode(
&self,
input: Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group("linear (qlora)");
let mut intermediate =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type))?;
+ encoder.allocate_scratch_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type)?;
{
let mut adapter_kernel = self.adapter_down_kernel.lock();
@@ -192,9 +191,9 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
d: &mut intermediate,
d_transform: MatmulDOps::none(),
gather_indices: None,
- m: batch_dim as u32,
- n: self.lora_rank as u32,
- k: self.input_dim as u32,
+ m: batch_dim,
+ n: self.lora_rank,
+ k: self.input_dim,
},
encoder,
)?;
@@ -202,15 +201,8 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
let base_input = if let Some((input_hadamard_kernel, input_factors)) = &self.input_hadamard {
let mut base_input =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dim], self.input_data_type))?;
- input_hadamard_kernel.encode_fp(
- &input,
- &mut base_input,
- input_factors,
- batch_dim as u32,
- self.input_dim as u32,
- encoder,
- );
+ encoder.allocate_scratch_for_shape(&[batch_dim, self.input_dim], self.input_data_type)?;
+ input_hadamard_kernel.encode_fp(&input, &mut base_input, input_factors, batch_dim, self.input_dim, encoder);
base_input
} else {
input
@@ -237,22 +229,16 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
..MatmulDOps::none()
},
gather_indices: None,
- m: batch_dim as u32,
- n: self.output_dim as u32,
- k: self.lora_rank as u32,
+ m: batch_dim,
+ n: self.output_dim,
+ k: self.lora_rank,
},
encoder,
)?;
}
if let Some((output_hadamard_kernel, output_factors)) = &self.output_hadamard {
- output_hadamard_kernel.encode_fp_in_place(
- &mut output,
- output_factors,
- batch_dim as u32,
- self.output_dim as u32,
- encoder,
- );
+ output_hadamard_kernel.encode_fp_in_place(&mut output, output_factors, batch_dim, self.output_dim, encoder);
}
encoder.pop_debug_group();
diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
index 78bc41d01..08488ef70 100644
--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
@@ -1,7 +1,6 @@
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, Context, DeviceCapabilities, Encoder,
gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod},
@@ -30,10 +29,10 @@ const INT8_ACTIVATIONS_ENABLED: bool = true;
pub(super) fn int8_activations_eligible<B: Backend>(
context: &B::Context,
spec: &ParsedWeightSpec,
- input_dimension: usize,
+ input_dimension: u32,
input_data_type: DataType,
output_data_type: DataType,
- activation_group_size: usize,
+ activation_group_size: u32,
) -> bool {
if !INT8_ACTIVATIONS_ENABLED {
return false;
@@ -47,7 +46,7 @@ pub(super) fn int8_activations_eligible<B: Backend>(
let (Some(info), Layout::OutputInput) = (spec.quantization, &spec.layout) else {
return false;
};
- let group_size = info.group_size as usize;
+ let group_size = info.group_size;
matches!(group_size, 32 | 64 | 128)
&& input_dimension.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE)
&& input_dimension.is_multiple_of(group_size)
@@ -73,14 +72,14 @@ pub struct RHTLinearWrapper<B: Backend> {
quantize_transform: Option<ActivationTransform<B>>,
input_factors: Allocation<B>,
inner_linear: LinearMatmul<B>,
- input_dimension: usize,
+ input_dimension: u32,
}
impl<B: Backend> RHTLinearWrapper<B> {
pub fn new(
context: &B::Context,
- input_dimension: usize,
- output_dimension: usize,
+ input_dimension: u32,
+ output_dimension: u32,
has_biases: bool,
weights_data_type: DataType,
input_data_type: DataType,
@@ -113,7 +112,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
let Some(quantization) = parsed.quantization else {
return Err(RHTLinearWrapperError::UnsupportedConfiguration("RHT requires a quantized inner spec".into()));
};
- let activation_group_size = ACTIVATION_SCALE_GROUP_SIZE as usize;
+ let activation_group_size = ACTIVATION_SCALE_GROUP_SIZE;
let input_transform = ActivationTransform::input_rht(context, input_data_type, true)
.map_err(RHTLinearWrapperError::BackendError)?;
@@ -144,7 +143,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
context,
input_data_type,
activation_group_size,
- emit_group_sums.then_some(quantization.group_size.min(activation_group_size as u32)),
+ emit_group_sums.then_some(quantization.group_size.min(activation_group_size)),
)
.map_err(RHTLinearWrapperError::BackendError)?;
inner_linear.make_codes_signed();
@@ -167,7 +166,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
fn encode(
&self,
input: Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group("linear (rht)");
@@ -176,15 +175,12 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
&& self.inner_linear.select_path(batch_dim, encoder.context()) == MatmulPath::Gemm
{
let scale_groups_per_row = self.input_dimension.div_ceil(quantize_transform.activation_group_size());
- let sum_groups_per_row = quantize_transform
- .sum_group_size()
- .map(|group_size| self.input_dimension.div_ceil(group_size as usize));
- let mut values =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dimension], DataType::I8))?;
- let mut scales =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, scale_groups_per_row], DataType::F32))?;
+ let sum_groups_per_row =
+ quantize_transform.sum_group_size().map(|group_size| self.input_dimension.div_ceil(group_size));
+ let mut values = encoder.allocate_scratch_for_shape(&[batch_dim, self.input_dimension], DataType::I8)?;
+ let mut scales = encoder.allocate_scratch_for_shape(&[batch_dim, scale_groups_per_row], DataType::F32)?;
let mut group_sums = sum_groups_per_row
- .map(|groups| encoder.allocate_scratch(size_for_shape(&[batch_dim, groups], DataType::I32)))
+ .map(|groups| encoder.allocate_scratch_for_shape(&[batch_dim, groups], DataType::I32))
.transpose()?;
quantize_transform.encode_quantize(
@@ -193,8 +189,8 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
&mut scales,
group_sums.as_mut(),
&self.input_factors,
- batch_dim as u32,
- self.input_dimension as u32,
+ batch_dim,
+ self.input_dimension,
encoder,
);
let output = self.inner_linear.encode_with_a(
@@ -202,7 +198,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
values: &values,
scales: &scales,
group_sums: group_sums.as_ref(),
- group_size: quantize_transform.activation_group_size() as u32,
+ group_size: quantize_transform.activation_group_size(),
},
batch_dim,
encoder,
@@ -217,8 +213,8 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
self.input_transform.encode_fp_in_place(
&mut input,
&self.input_factors,
- batch_dim as u32,
- self.input_dimension as u32,
+ batch_dim,
+ self.input_dimension,
encoder,
);
let output = self.inner_linear.encode(input, batch_dim, encoder)?;
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs
index c5d5e421b..37fb77d5d 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs
@@ -1,7 +1,6 @@
use parking_lot::Mutex;
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, BufferArg, Encoder, Kernels,
kernel::{
@@ -14,10 +13,10 @@ use crate::{
};
pub struct AttentionFallbackCore<B: Backend> {
- head_dim: usize,
- num_groups: usize,
- num_q_heads: usize,
- sliding_window_size: Option<usize>,
+ head_dim: u32,
+ num_groups: u32,
+ num_q_heads: u32,
+ sliding_window_size: Option<u32>,
scale: Option<f32>,
data_type: DataType,
scatter_scores: <B::Kernels as Kernels>::AttentionFallbackScatterScoresKernel,
@@ -70,35 +69,33 @@ impl<B: Backend> AttentionFallbackCore<B> {
arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let sequence_length = arguments.state_type.physical_prefix_length() + arguments.suffix_length;
+ let suffix_length = arguments.suffix_length;
+ let sequence_length = arguments.state_type.physical_prefix_length() + suffix_length;
let gqa_factor = self.num_q_heads / self.num_groups;
let scale = self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt());
+
let dt_bytes = self.data_type.size_in_bytes();
+ let head_dim_bytes = self.head_dim as usize * dt_bytes;
+ let group_rows = (gqa_factor * suffix_length) as usize;
- let mut output = encoder.allocate_constant(size_for_shape(
- &[arguments.suffix_length, self.num_q_heads, self.head_dim],
- self.data_type,
- ))?;
- let mut scores = encoder.allocate_scratch(size_for_shape(
- &[self.num_q_heads * arguments.suffix_length * sequence_length],
- self.data_type,
- ))?;
- let mut group_scores = encoder.allocate_scratch(size_for_shape(
- &[gqa_factor * arguments.suffix_length, sequence_length],
- self.data_type,
- ))?;
+ let mut output =
+ encoder.allocate_constant_for_shape(&[suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;
+ let mut scores =
+ encoder.allocate_scratch_for_shape(&[self.num_q_heads, suffix_length, sequence_length], self.data_type)?;
+ let mut group_scores =
+ encoder.allocate_scratch_for_shape(&[gqa_factor * suffix_length, sequence_length], self.data_type)?;
for group_index in 0..self.num_groups {
self.matmul.lock().encode(
MatmulArguments {
a: MatmulA::FullPrecision {
values: arguments.queries,
- offset: group_index * gqa_factor * arguments.suffix_length * self.head_dim * dt_bytes,
+ offset: group_index as usize * group_rows * head_dim_bytes,
},
b: MatmulB::FullPrecision {
- b: (arguments.keys, group_index * self.head_dim * dt_bytes),
+ b: (arguments.keys, group_index as usize * head_dim_bytes),
},
- b_leading_dimension: Some((self.num_groups * self.head_dim) as u32),
+ b_leading_dimension: Some(self.num_groups * self.head_dim),
b_transpose: true,
d: &mut group_scores,
d_transform: MatmulDOps {
@@ -106,9 +103,9 @@ impl<B: Backend> AttentionFallbackCore<B> {
..MatmulDOps::none()
},
gather_indices: None,
- m: (gqa_factor * arguments.suffix_length) as u32,
- n: sequence_length as u32,
- k: self.head_dim as u32,
+ m: gqa_factor * suffix_length,
+ n: sequence_length,
+ k: self.head_dim,
},
encoder,
)?;
@@ -117,58 +114,51 @@ impl<B: Backend> AttentionFallbackCore<B> {
&mut scores,
arguments.state_type.ring_params(),
None::<&Allocation<B>>,
- self.sliding_window_size.map(|sliding_window_size| sliding_window_size as u32),
- group_index as u32,
- gqa_factor as u32,
- sequence_length as u32,
- arguments.suffix_length as u32,
- (gqa_factor * arguments.suffix_length * sequence_length) as u32,
+ self.sliding_window_size,
+ group_index,
+ gqa_factor,
+ sequence_length,
+ suffix_length,
+ gqa_factor * suffix_length * sequence_length,
encoder,
);
}
- self.softmax.encode(
- &mut scores,
- arguments.sinks,
- sequence_length as u32,
- self.num_q_heads as u32,
- arguments.suffix_length as u32,
- encoder,
- );
+ self.softmax.encode(&mut scores, arguments.sinks, sequence_length, self.num_q_heads, suffix_length, encoder);
- let mut group_output = encoder
- .allocate_scratch(size_for_shape(&[gqa_factor * arguments.suffix_length, self.head_dim], self.data_type))?;
+ let mut group_output =
+ encoder.allocate_scratch_for_shape(&[gqa_factor * suffix_length, self.head_dim], self.data_type)?;
for group_index in 0..self.num_groups {
self.matmul.lock().encode(
MatmulArguments {
a: MatmulA::FullPrecision {
values: &scores,
- offset: group_index * gqa_factor * arguments.suffix_length * sequence_length * dt_bytes,
+ offset: group_index as usize * group_rows * sequence_length as usize * dt_bytes,
},
b: MatmulB::FullPrecision {
- b: (arguments.values, group_index * self.head_dim * dt_bytes),
+ b: (arguments.values, group_index as usize * head_dim_bytes),
},
- b_leading_dimension: Some((self.num_groups * self.head_dim) as u32),
+ b_leading_dimension: Some(self.num_groups * self.head_dim),
b_transpose: false,
d: &mut group_output,
d_transform: MatmulDOps::none(),
gather_indices: None,
- m: (gqa_factor * arguments.suffix_length) as u32,
- n: self.head_dim as u32,
- k: sequence_length as u32,
+ m: gqa_factor * suffix_length,
+ n: self.head_dim,
+ k: sequence_length,
},
encoder,
)?;
self.scatter_values.encode(
&group_output,
&mut output,
- group_index as u32,
- gqa_factor as u32,
- arguments.suffix_length as u32,
- self.num_q_heads as u32,
- self.head_dim as u32,
- (gqa_factor * arguments.suffix_length * self.head_dim) as u32,
+ group_index,
+ gqa_factor,
+ suffix_length,
+ self.num_q_heads,
+ self.head_dim,
+ gqa_factor * suffix_length * self.head_dim,
encoder,
);
}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs
index 456d02d30..419549143 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs
@@ -15,14 +15,14 @@ mod single_pass;
mod two_pass;
pub struct AttentionCoreNewArguments {
- pub head_dim: usize,
- pub num_groups: usize,
- pub num_q_heads: usize,
+ pub head_dim: u32,
+ pub num_groups: u32,
+ pub num_q_heads: u32,
pub has_sinks: bool,
pub is_kv_cache_ring: bool,
pub is_causal: bool,
pub is_trie: bool,
- pub sliding_window_size: Option<usize>,
+ pub sliding_window_size: Option<u32>,
pub scale: Option<f32>,
pub data_type: DataType,
}
@@ -31,7 +31,7 @@ pub struct AttentionCoreEncodeArguments<'a, B: Backend, KT: BufferArg<'a, B>, VT
pub queries: &'a Allocation<B>,
pub keys: KT,
pub values: VT,
- pub suffix_length: usize,
+ pub suffix_length: u32,
pub trie: Option<&'a Allocation<B>>,
pub sinks: Option<&'a Allocation<B>>,
pub state_type: &'a AttentionStateType,
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs
index 710fbaf0d..a2e89b301 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs
@@ -1,15 +1,14 @@
use crate::{
- array::size_for_shape,
backends::common::{Allocation, Backend, BufferArg, Encoder, Kernels, kernel::AttentionSinglePassKernel},
data_type::DataType,
encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
};
pub struct AttentionSinglePassCore<B: Backend> {
- head_dim: usize,
- num_groups: usize,
- num_q_heads: usize,
- sliding_window_size: Option<usize>,
+ head_dim: u32,
+ num_groups: u32,
+ num_q_heads: u32,
+ sliding_window_size: Option<u32>,
scale: Option<f32>,
data_type: DataType,
kernel: <B::Kernels as Kernels>::AttentionSinglePassKernel,
@@ -23,7 +22,7 @@ impl<B: Backend> AttentionSinglePassCore<B> {
let kernel = <B::Kernels as Kernels>::AttentionSinglePassKernel::new(
context,
arguments.data_type,
- arguments.head_dim as u32,
+ arguments.head_dim,
arguments.has_sinks,
arguments.is_kv_cache_ring,
arguments.is_causal,
@@ -47,28 +46,26 @@ impl<B: Backend> AttentionSinglePassCore<B> {
arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut output = encoder.allocate_constant(size_for_shape(
- &[arguments.suffix_length, self.num_q_heads, self.head_dim],
- self.data_type,
- ))?;
+ let mut output = encoder
+ .allocate_constant_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;
self.kernel.encode(
arguments.queries,
arguments.keys,
arguments.values,
&mut output,
- (self.num_q_heads / self.num_groups) as u32,
- (arguments.state_type.physical_prefix_length() + arguments.suffix_length) as u32,
- self.head_dim as u32,
- (self.num_groups * self.head_dim) as u32,
- self.head_dim as u32,
- (self.num_groups * self.head_dim) as u32,
+ self.num_q_heads / self.num_groups,
+ arguments.state_type.physical_prefix_length() + arguments.suffix_length,
+ self.head_dim,
+ self.num_groups * self.head_dim,
+ self.head_dim,
+ self.num_groups * self.head_dim,
arguments.state_type.ring_params(),
self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),
arguments.trie,
- self.sliding_window_size.map(|sliding_window_size| sliding_window_size as u32),
+ self.sliding_window_size,
arguments.sinks,
- self.num_q_heads as u32,
- arguments.suffix_length as u32,
+ self.num_q_heads,
+ arguments.suffix_length,
encoder,
);
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs
index bc905558b..305c184dd 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs
@@ -1,5 +1,4 @@
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, BufferArg, Encoder, Kernels,
kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},
@@ -8,11 +7,14 @@ use crate::{
encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
};
+const INNER_DATA_TYPE: DataType = DataType::F32;
+const TWO_PASS_BLOCKS: u32 = 32;
+
pub struct AttentionTwoPassCore<B: Backend> {
- head_dim: usize,
- num_groups: usize,
- num_q_heads: usize,
- sliding_window_size: Option<usize>,
+ head_dim: u32,
+ num_groups: u32,
+ num_q_heads: u32,
+ sliding_window_size: Option<u32>,
scale: Option<f32>,
data_type: DataType,
pass_1: <B::Kernels as Kernels>::AttentionTwoPass1Kernel,
@@ -27,7 +29,7 @@ impl<B: Backend> AttentionTwoPassCore<B> {
let pass_1 = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
context,
arguments.data_type,
- arguments.head_dim as u32,
+ arguments.head_dim,
arguments.has_sinks,
arguments.is_kv_cache_ring,
arguments.is_causal,
@@ -35,11 +37,8 @@ impl<B: Backend> AttentionTwoPassCore<B> {
arguments.sliding_window_size.is_some(),
)?;
- let pass_2 = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(
- context,
- arguments.data_type,
- arguments.head_dim as u32,
- )?;
+ let pass_2 =
+ <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, arguments.data_type, arguments.head_dim)?;
Ok(Self {
head_dim: arguments.head_dim,
@@ -58,14 +57,18 @@ impl<B: Backend> AttentionTwoPassCore<B> {
arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut partials = encoder.allocate_scratch(size_for_shape(
- &[self.num_q_heads * arguments.suffix_length * 32 * self.head_dim],
- DataType::F32,
- ))?;
- let mut sums = encoder
- .allocate_scratch(size_for_shape(&[self.num_q_heads * arguments.suffix_length * 32], DataType::F32))?;
- let mut maxs = encoder
- .allocate_scratch(size_for_shape(&[self.num_q_heads * arguments.suffix_length * 32], DataType::F32))?;
+ let mut partials = encoder.allocate_scratch_for_shape(
+ &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS, self.head_dim],
+ INNER_DATA_TYPE,
+ )?;
+ let mut sums = encoder.allocate_scratch_for_shape(
+ &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS],
+ INNER_DATA_TYPE,
+ )?;
+ let mut maxs = encoder.allocate_scratch_for_shape(
+ &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS],
+ INNER_DATA_TYPE,
+ )?;
self.pass_1.encode(
arguments.queries,
@@ -74,35 +77,25 @@ impl<B: Backend> AttentionTwoPassCore<B> {
&mut partials,
&mut sums,
&mut maxs,
- (self.num_q_heads / self.num_groups) as u32,
- (arguments.state_type.physical_prefix_length() + arguments.suffix_length) as u32,
- self.head_dim as u32,
- (self.num_groups * self.head_dim) as u32,
- self.head_dim as u32,
- (self.num_groups * self.head_dim) as u32,
+ self.num_q_heads / self.num_groups,
+ arguments.state_type.physical_prefix_length() + arguments.suffix_length,
+ self.head_dim,
+ self.num_groups * self.head_dim,
+ self.head_dim,
+ self.num_groups * self.head_dim,
arguments.state_type.ring_params(),
self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),
- self.num_q_heads as u32,
- arguments.suffix_length as u32,
+ self.num_q_heads,
+ arguments.suffix_length,
arguments.trie,
- self.sliding_window_size.map(|sliding_window_size| sliding_window_size as u32),
+ self.sliding_window_size,
arguments.sinks,
encoder,
);
- let mut output = encoder.allocate_constant(size_for_shape(
- &[arguments.suffix_length, self.num_q_heads, self.head_dim],
- self.data_type,
- ))?;
- self.pass_2.encode(
- &partials,
- &sums,
- &maxs,
- &mut output,
- self.num_q_heads as u32,
- arguments.suffix_length as u32,
- encoder,
- );
+ let mut output = encoder
+ .allocate_constant_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;
+ self.pass_2.encode(&partials, &sums, &maxs, &mut output, self.num_q_heads, arguments.suffix_length, encoder);
Ok(output)
}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs
index 276c96338..a95e84210 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs
@@ -34,12 +34,12 @@ pub(crate) use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, AttentionState
pub mod rope;
pub struct Attention<B: Backend> {
- head_dim: usize,
- num_q_heads: usize,
- num_kv_heads: Option<usize>,
+ head_dim: u32,
+ num_q_heads: u32,
+ num_kv_heads: Option<u32>,
is_causal: bool,
- sliding_window_size: Option<usize>,
- max_rope_length: Option<usize>,
+ sliding_window_size: Option<u32>,
+ max_rope_length: Option<u32>,
data_type: DataType,
qkv: LinearProjection<B>,
prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
@@ -65,7 +65,7 @@ pub enum AttentionNewError<B: Backend> {
impl<B: Backend> Attention<B> {
pub fn new(
- hidden_dim: usize,
+ hidden_dim: u32,
data_type: DataType,
rope_config: Option<&AnyRoPEConfig>,
config: &AttentionConfig,
@@ -245,13 +245,13 @@ impl<B: Backend> Mixer<B> for Attention<B> {
true
}
- fn max_context_length(&self) -> Option<usize> {
+ fn max_context_length(&self) -> Option<u32> {
self.max_rope_length
}
fn create_empty_state(
&self,
- max_context_length: Option<usize>,
+ max_context_length: Option<u32>,
context: &B::Context,
) -> Result<Box<dyn MixerState<B>>, B::Error> {
Ok(Box::new(AttentionState::create_empty(self, max_context_length, context)?))
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs
index c0f3a4b51..1fc1284ea 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs
@@ -1,5 +1,4 @@
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, BufferArgMut, Encoder,
gpu_types::trie::TrieNode,
@@ -31,7 +30,7 @@ impl<B: Backend> LinearProjection<B> {
fn project(
&self,
hidden: Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
let mut projected = self.lin.encode(hidden, batch_dim, encoder)?;
@@ -69,7 +68,7 @@ impl<B: Backend> Attention<B> {
state.keys.as_mut(),
state.values.as_mut(),
state.state_type.physical_prefix_length(),
- self.num_q_heads as u32,
+ self.num_q_heads,
precalculated_rope,
batch_dim.size(),
encoder,
@@ -89,21 +88,17 @@ impl<B: Backend> Attention<B> {
assert!(batch_dim.is_flat(), "stateless attention doesn't support trie");
let qkv = self.qkv.project(hidden, batch_dim.size(), encoder)?;
- let mut keys = encoder.allocate_scratch(size_for_shape(
- &[batch_dim.size(), num_kv_heads, self.head_dim],
- self.data_type,
- ))?;
- let mut values = encoder.allocate_scratch(size_for_shape(
- &[batch_dim.size(), num_kv_heads, self.head_dim],
- self.data_type,
- ))?;
+ let mut keys = encoder
+ .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)?;
+ let mut values = encoder
+ .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)?;
let queries = self.prepare_kv_and_queries(
&qkv,
&mut keys,
&mut values,
0,
- self.num_q_heads as u32,
+ self.num_q_heads,
precalculated_rope,
batch_dim.size(),
encoder,
@@ -141,7 +136,7 @@ impl<B: Backend> Attention<B> {
gate_kernel.encode(
&gate.unwrap(),
&mut attention_output,
- (batch_dim.size() * self.num_q_heads * self.head_dim) as u32,
+ batch_dim.size() * (self.num_q_heads * self.head_dim),
encoder,
);
}
@@ -152,7 +147,7 @@ impl<B: Backend> Attention<B> {
&self,
mut key_value: Allocation<B>,
precalculated_rope: &PrecalculatedRoPE<B>,
- batch_dim: usize,
+ batch_dim: u32,
state: &mut AttentionState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
@@ -169,7 +164,7 @@ impl<B: Backend> Attention<B> {
batch_dim,
encoder,
)?;
- state.encode_accept(&(0..batch_dim).collect::<Box<[usize]>>(), encoder)?;
+ state.encode_accept(&(0..batch_dim).collect::<Box<[u32]>>(), encoder)?;
Ok(())
}
@@ -183,7 +178,7 @@ impl<B: Backend> Attention<B> {
let (core, trie) = if batch_dim.is_flat() {
(&self.flat_core, None)
} else {
- let mut trie = encoder.allocate_constant(batch_dim.size() * size_of::<TrieNode>())?;
+ let mut trie = encoder.allocate_constant(batch_dim.size() as usize * size_of::<TrieNode>())?;
trie.copyin(batch_dim.nodes());
(&self.trie_core, Some(trie))
};
@@ -207,16 +202,16 @@ impl<B: Backend> Attention<B> {
input: &Allocation<B>,
keys: impl BufferArgMut<'keys, B>,
values: impl BufferArgMut<'values, B>,
- kv_token_offset: usize,
+ kv_token_offset: u32,
num_q_heads: u32,
precalculated_rope: Option<&PrecalculatedRoPE<B>>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
let mut queries = if num_q_heads == 0 {
encoder.allocate_scratch(self.data_type.size_in_bytes())?
} else {
- encoder.allocate_scratch(size_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type))?
+ encoder.allocate_scratch_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type)?
};
self.prepare.encode(
input,
@@ -226,11 +221,11 @@ impl<B: Backend> Attention<B> {
precalculated_rope.map(|precalculated_rope| &precalculated_rope.cosines),
precalculated_rope.map(|precalculated_rope| &precalculated_rope.sines),
num_q_heads,
- Some(self.num_kv_heads.expect("KV prepare requires KV heads") as u32),
- self.head_dim as u32,
- precalculated_rope.map(|precalculated_rope| precalculated_rope.dim as u32),
- Some(kv_token_offset as u32),
- batch_dim as u32,
+ Some(self.num_kv_heads.expect("KV prepare requires KV heads")),
+ self.head_dim,
+ precalculated_rope.map(|precalculated_rope| precalculated_rope.dim),
+ Some(kv_token_offset),
+ batch_dim,
encoder,
);
Ok(queries)
@@ -240,11 +235,11 @@ impl<B: Backend> Attention<B> {
&self,
query: &Allocation<B>,
precalculated_rope: Option<&PrecalculatedRoPE<B>>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
let mut queries =
- encoder.allocate_scratch(size_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type))?;
+ encoder.allocate_scratch_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type)?;
self.prepare.encode(
query,
&mut queries,
@@ -252,12 +247,12 @@ impl<B: Backend> Attention<B> {
None::<&mut Allocation<B>>,
precalculated_rope.map(|rope| &rope.cosines),
precalculated_rope.map(|rope| &rope.sines),
- self.num_q_heads as u32,
+ self.num_q_heads,
None,
- self.head_dim as u32,
- precalculated_rope.map(|rope| rope.dim as u32),
+ self.head_dim,
+ precalculated_rope.map(|rope| rope.dim),
None,
- batch_dim as u32,
+ batch_dim,
encoder,
);
Ok(queries)
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rs
index 27504b525..45eb93a93 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rs
@@ -28,9 +28,9 @@ pub struct QKVNorm<B: Backend> {
query: Option<Head<B>>,
key: Option<Head<B>>,
value: Option<Head<B>>,
- num_q_heads: usize,
- num_kv_heads: usize,
- head_dim: usize,
+ num_q_heads: u32,
+ num_kv_heads: u32,
+ head_dim: u32,
}
impl<B: Backend> QKVNorm<B> {
@@ -41,9 +41,9 @@ impl<B: Backend> QKVNorm<B> {
key_config: Option<NormalizationConfig>,
value_config: Option<NormalizationConfig>,
parameter_tree: &ParameterTree<B>,
- num_q_heads: usize,
- num_kv_heads: usize,
- head_dim: usize,
+ num_q_heads: u32,
+ num_kv_heads: u32,
+ head_dim: u32,
) -> Result<Self, QKVNormError<B>> {
let query = query_config
.map(|cfg| {
@@ -86,7 +86,7 @@ impl<B: Backend> QKVNorm<B> {
intermediate_data_type: DataType,
config: NormalizationConfig,
parameter_tree: Option<&ParameterTree<B>>,
- head_dim: usize,
+ head_dim: u32,
) -> Result<Head<B>, QKVNormError<B>> {
let scales = if config.has_scale {
Some(
@@ -119,7 +119,7 @@ impl<B: Backend> QKVNorm<B> {
pub fn encode(
&self,
qkv: &mut Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
self.encode_packed(qkv, batch_dim, self.num_q_heads, encoder)
@@ -128,7 +128,7 @@ impl<B: Backend> QKVNorm<B> {
pub fn encode_key_value(
&self,
key_value: &mut Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
self.encode_packed(key_value, batch_dim, 0, encoder)
@@ -137,8 +137,8 @@ impl<B: Backend> QKVNorm<B> {
fn encode_packed(
&self,
buffer: &mut Allocation<B>,
- batch_dim: usize,
- q_heads: usize,
+ batch_dim: u32,
+ q_heads: u32,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
encoder.push_debug_group("qkv norm");
@@ -157,13 +157,13 @@ impl<B: Backend> QKVNorm<B> {
None::<&Allocation<B>>,
head.scales.as_ref(),
&mut *buffer,
- batch_dim as u32,
- total_heads as u32,
- self.head_dim as u32,
+ batch_dim,
+ total_heads,
+ self.head_dim,
head.config.epsilon,
head.config.scale_offset.unwrap_or(0.0),
- head_offset as u32,
- head_count as u32,
+ head_offset,
+ head_count,
head.config.upcast_mode == UpcastMode::FullLayer,
encoder,
);
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs
index b873b2d89..97009a5b1 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs
@@ -6,13 +6,13 @@ use crate::{
pub struct PrecalculatedRoPE<B: Backend> {
pub cosines: Allocation<B>,
pub sines: Allocation<B>,
- pub dim: usize,
+ pub dim: u32,
}
impl<B: Backend> PrecalculatedRoPE<B> {
pub fn precalculate(
rope_config: &AnyRoPEConfig,
- token_positions: &[usize],
+ token_positions: &[u32],
encoder: &mut Encoder<B>,
) -> Result<Self, B::Error> {
let head_dim = *rope_config.head_dim();
@@ -26,7 +26,8 @@ impl<B: Backend> PrecalculatedRoPE<B> {
_ => 1.0,
};
- let element_count = token_positions.len() * head_dim;
+ let element_count = token_positions.len() * head_dim as usize;
+ let half_offset = half_dim as usize;
let mut sines = vec![0.0; element_count];
let mut cosines = vec![0.0; element_count];
for pair_index in 0..half_dim {
@@ -84,7 +85,7 @@ impl<B: Backend> PrecalculatedRoPE<B> {
} else {
&config.short_factor
};
- inverse_frequency / factors[pair_index]
+ inverse_frequency / factors[pair_index as usize]
},
};
@@ -92,11 +93,11 @@ impl<B: Backend> PrecalculatedRoPE<B> {
let embedding = *token_position as f32 * inverse_frequency;
let sine = embedding.sin() * attention_scaling_factor;
let cosine = embedding.cos() * attention_scaling_factor;
- let row_offset = token_index * head_dim;
- sines[row_offset + pair_index] = sine;
- sines[row_offset + half_dim + pair_index] = sine;
- cosines[row_offset + pair_index] = cosine;
- cosines[row_offset + half_dim + pair_index] = cosine;
+ let pair_offset = token_index * head_dim as usize + pair_index as usize;
+ sines[pair_offset] = sine;
+ sines[pair_offset + half_offset] = sine;
+ cosines[pair_offset] = cosine;
+ cosines[pair_offset + half_offset] = cosine;
}
}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/state.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/state.rs
index 6fb9fc64e..ea48bc691 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/state.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/state.rs
@@ -1,6 +1,7 @@
use std::any::Any;
use crate::{
+ array::size_for_shape,
backends::common::{
Backend, Buffer, Context, DeviceCapabilities, Encoder, Kernels, SparseBuffer,
gpu_types::{Copy, ring::RingParams},
@@ -10,21 +11,21 @@ use crate::{
encodable_block::mixer::{MixerState, attention::Attention},
};
-pub(crate) const ATTENTION_SUFFIX_CAPACITY: usize = 1024; // TODO: remove hardcoded suffix capacity
+pub(crate) const ATTENTION_SUFFIX_CAPACITY: u32 = 1024; // TODO: remove hardcoded suffix capacity
pub enum AttentionStateType {
Full {
- length: usize,
+ length: u32,
},
Ring {
- offset: usize,
- length: usize,
- max_length: usize,
+ offset: u32,
+ length: u32,
+ max_length: u32,
},
}
impl AttentionStateType {
- pub fn physical_prefix_length(&self) -> usize {
+ pub fn physical_prefix_length(&self) -> u32 {
match self {
Self::Full {
length,
@@ -47,16 +48,16 @@ impl AttentionStateType {
};
Some(RingParams {
- ring_offset: *offset as u32,
- ring_length: *length as u32,
+ ring_offset: *offset,
+ ring_length: *length,
})
}
}
pub struct AttentionState<B: Backend> {
- pub cur_context_length: usize,
- pub elements_prepared: usize,
- pub element_dim: usize,
+ pub cur_context_length: u32,
+ pub elements_prepared: u32,
+ pub element_dim: u32,
pub data_type: DataType,
pub state_type: AttentionStateType,
pub is_sparse: bool,
@@ -68,7 +69,7 @@ pub struct AttentionState<B: Backend> {
impl<B: Backend> AttentionState<B> {
pub fn create_empty(
attention: &Attention<B>,
- max_context_length: Option<usize>,
+ max_context_length: Option<u32>,
context: &B::Context,
) -> Result<Self, B::Error> {
if let Some(max_context_length) = max_context_length {
@@ -106,7 +107,7 @@ impl<B: Backend> AttentionState<B> {
let max_elements = max_prefix_elements + ATTENTION_SUFFIX_CAPACITY;
let element_size = attention.num_kv_heads.unwrap() * attention.head_dim;
- let kv_buffer_bytes = max_elements * element_size * data_type.size_in_bytes();
+ let kv_buffer_bytes = size_for_shape(&[max_elements, element_size], data_type);
let is_ring = matches!(state_type, AttentionStateType::Ring { .. });
let is_sparse = !is_ring && context.device_capabilities().contains(DeviceCapabilities::SPARSE_BUFFERS);
@@ -139,8 +140,8 @@ impl<B: Backend> AttentionState<B> {
impl<B: Backend> MixerState<B> for AttentionState<B> {
fn prepare(
&mut self,
- context_length: usize,
- suffix_length: usize,
+ context_length: u32,
+ suffix_length: u32,
context: &B::Context,
) -> Result<(), B::Error> {
if !self.is_sparse {
@@ -149,8 +150,8 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
assert!(suffix_length <= ATTENTION_SUFFIX_CAPACITY, "attention suffix length exceeds hardcoded capacity");
let elements_required = context_length + suffix_length;
- let bytes_required = elements_required * self.element_dim * self.data_type.size_in_bytes();
- let bytes_prepared = self.elements_prepared * self.element_dim * self.data_type.size_in_bytes();
+ let bytes_required = size_for_shape(&[elements_required, self.element_dim], self.data_type);
+ let bytes_prepared = size_for_shape(&[self.elements_prepared, self.element_dim], self.data_type);
let keys = (self.keys.as_mut() as &mut dyn Any).downcast_mut::<B::SparseBuffer>().unwrap();
let values = (self.values.as_mut() as &mut dyn Any).downcast_mut::<B::SparseBuffer>().unwrap();
@@ -172,7 +173,7 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
fn encode_accept(
&mut self,
- accepted_indices: &[usize],
+ accepted_indices: &[u32],
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
assert!(accepted_indices.is_sorted_by(|a, b| a < b), "invalid accepted indicies");
@@ -184,15 +185,15 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
let copies = accepted_indices
.iter()
.copied()
- .enumerate()
- .filter(|(index, accepted_index)| index != accepted_index)
- .map(|(index, accepted_index)| Copy {
- source: (*length + accepted_index) as u32,
- destination: (*length + index) as u32,
+ .zip(0u32..)
+ .filter(|&(accepted_index, index)| index != accepted_index)
+ .map(|(accepted_index, index)| Copy {
+ source: *length + accepted_index,
+ destination: *length + index,
})
.collect::<Vec<Copy>>();
- *length += accepted_indices.len();
+ *length += accepted_indices.len() as u32;
copies
},
@@ -204,8 +205,8 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
let mut copies = Vec::new();
for accepted_index in accepted_indices {
copies.push(Copy {
- source: (*max_length + accepted_index) as u32,
- destination: ((*offset + *length) % *max_length) as u32,
+ source: *max_length + *accepted_index,
+ destination: (*offset + *length) % *max_length,
});
if length < max_length {
@@ -224,12 +225,12 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
self.values.as_mut(),
copies_chunk,
copies_chunk.len() as u32,
- self.element_dim as u32,
+ self.element_dim,
encoder,
);
}
- self.cur_context_length += accepted_indices.len();
+ self.cur_context_length += accepted_indices.len() as u32;
Ok(())
}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
index ffaf0e3c0..8e8cfa965 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
@@ -1,12 +1,9 @@
-use std::mem::size_of;
-
use thiserror::Error;
use crate::{
array::size_for_shape,
backends::common::{
Allocation, AllocationType, Backend, Context, Encoder, Kernels,
- gpu_types::trie::TrieNode,
kernel::{
Conv1dPackKernel, ConvTreeScanKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel,
DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,
@@ -32,9 +29,11 @@ use crate::{
pub(crate) mod tree_verify;
+const INNER_DATA_TYPE: DataType = DataType::F32;
+
enum DeltaNetSuffixStatus<B: Backend> {
Flat {
- suffix_length: usize,
+ suffix_length: u32,
},
Tree {
conv_states: Allocation<B>,
@@ -56,8 +55,8 @@ pub struct DeltaNetState<B: Backend> {
impl<B: Backend> MixerState<B> for DeltaNetState<B> {
fn prepare(
&mut self,
- _context_length: usize,
- _suffix_length: usize,
+ _context_length: u32,
+ _suffix_length: u32,
_context: &B::Context,
) -> Result<(), B::Error> {
Ok(())
@@ -65,7 +64,7 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {
fn encode_accept(
&mut self,
- accepted_indices: &[usize],
+ accepted_indices: &[u32],
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let suffix_status = self.suffix_status.take().expect("delta net state has no suffix to accept");
@@ -85,22 +84,20 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {
beta,
parents,
} => {
- assert!(accepted_indices.iter().all(|&index| index < parents.len()));
- assert_eq!(parents[accepted_indices[0]], -1);
- assert!(accepted_indices.windows(2).all(|edge| parents[edge[1]] == edge[0] as i32));
+ assert!(accepted_indices.iter().all(|&index| (index as usize) < parents.len()));
+ assert_eq!(parents[accepted_indices[0] as usize], -1);
+ assert!(accepted_indices.windows(2).all(|edge| parents[edge[1] as usize] == edge[0] as i32));
let conv_state_size = self.conv_state.size();
+ let accepted_offset = accepted_index as usize * conv_state_size;
encoder.encode_copy(
&conv_states,
- accepted_index * conv_state_size..(accepted_index + 1) * conv_state_size,
+ accepted_offset..accepted_offset + conv_state_size,
&mut self.conv_state,
..,
);
- let accepted_indices = accepted_indices.iter().map(|&index| index as u32).collect::<Box<[_]>>();
- let mut accepted_indices_buffer =
- encoder.allocate_constant(accepted_indices.len() * DataType::U32.size_in_bytes())?;
- accepted_indices_buffer.copyin(&accepted_indices);
+ let accepted_indices_buffer = encoder.allocate_constant_from_slice(accepted_indices)?;
self.state_advance.encode(
&k,
&v,
@@ -118,15 +115,15 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {
}
pub struct DeltaNet<B: Backend> {
- num_heads: usize,
- head_dim: usize,
- num_groups: usize,
- value_head_dim: usize,
- key_dim: usize,
- value_dim: usize,
- conv_dim: usize,
- total_proj_dim: usize,
- kernel_size: usize,
+ num_heads: u32,
+ head_dim: u32,
+ num_groups: u32,
+ value_head_dim: u32,
+ key_dim: u32,
+ value_dim: u32,
+ conv_dim: u32,
+ total_proj_dim: u32,
+ kernel_size: u32,
outer_data_type: DataType,
in_projection: Box<dyn Linear<B>>,
conv_weight: Allocation<B>,
@@ -163,7 +160,7 @@ pub enum DeltaNetNewError<B: Backend> {
impl<B: Backend> DeltaNet<B> {
pub fn new(
- hidden_dim: usize,
+ hidden_dim: u32,
outer_data_type: DataType,
config: &DeltaNetConfig,
parameter_tree: &ParameterTree<B>,
@@ -188,8 +185,6 @@ impl<B: Backend> DeltaNet<B> {
)));
}
- let inner_data_type = DataType::F32;
-
let key_dim = config.num_groups * config.head_dim;
let value_dim = config.num_heads * config.value_head_dim;
let conv_dim = key_dim * 2 + value_dim;
@@ -211,16 +206,16 @@ impl<B: Backend> DeltaNet<B> {
let conv_tree = parameter_tree.subtree("conv");
let conv_weight =
- conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], inner_data_type)?.read_allocation()?;
+ conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], INNER_DATA_TYPE)?.read_allocation()?;
let conv_bias = if conv_config.has_biases {
- Some(conv_tree.leaf("biases")?.validate(&[conv_dim], inner_data_type)?.read_allocation()?)
+ Some(conv_tree.leaf("biases")?.validate(&[conv_dim], INNER_DATA_TYPE)?.read_allocation()?)
} else {
None
};
let conv_update =
<B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, outer_data_type, conv_config.has_biases)
.map_err(DeltaNetNewError::Backend)?;
- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type, outer_data_type)
+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, INNER_DATA_TYPE, outer_data_type)
.map_err(DeltaNetNewError::Backend)?;
let conv_scan =
<B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, outer_data_type, conv_config.has_biases)
@@ -228,27 +223,27 @@ impl<B: Backend> DeltaNet<B> {
let conv_tree_scan = <B::Kernels as Kernels>::ConvTreeScanKernel::new(
context,
outer_data_type,
- config.kernel_size as u32,
+ config.kernel_size,
conv_config.has_biases,
)
.map_err(DeltaNetNewError::Backend)?;
- let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], inner_data_type)?.read_allocation()?;
+ let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], INNER_DATA_TYPE)?.read_allocation()?;
let dt_bias =
- parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], inner_data_type)?.read_allocation()?;
+ parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], INNER_DATA_TYPE)?.read_allocation()?;
let norm_weight = parameter_tree
.leaf("norm.scales")?
- .validate(&[config.value_head_dim], inner_data_type)?
+ .validate(&[config.value_head_dim], INNER_DATA_TYPE)?
.read_allocation()?;
let norm_epsilon = config.norm_config.epsilon;
let delta_net_update =
- <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, outer_data_type, config.head_dim as u32)
+ <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, outer_data_type, config.head_dim)
.map_err(DeltaNetNewError::Backend)?;
let delta_net_prefill_prep = <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
context,
outer_data_type,
- DataType::F32,
- config.head_dim as u32,
+ INNER_DATA_TYPE,
+ config.head_dim,
false,
false,
)
@@ -257,13 +252,13 @@ impl<B: Backend> DeltaNet<B> {
context,
outer_data_type,
outer_data_type,
- config.head_dim as u32,
+ config.head_dim,
true,
true,
)
.map_err(DeltaNetNewError::Backend)?;
let delta_net_prefill =
- <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, outer_data_type, config.head_dim as u32)
+ <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, outer_data_type, config.head_dim)
.map_err(DeltaNetNewError::Backend)?;
let delta_net_norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, outer_data_type)
.map_err(DeltaNetNewError::Backend)?;
@@ -286,9 +281,8 @@ impl<B: Backend> DeltaNet<B> {
None
};
- let chunked =
- <B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(context, outer_data_type, config.head_dim as u32)
- .map_err(DeltaNetNewError::Backend)?;
+ let chunked = <B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(context, outer_data_type, config.head_dim)
+ .map_err(DeltaNetNewError::Backend)?;
let out_projection = <dyn Linear<B>>::new_mixed_precision(
value_dim,
@@ -346,21 +340,19 @@ impl<B: Backend> DeltaNet<B> {
) -> Result<Allocation<B>, B::Error> {
let tree_verify = self.tree_verify.as_ref().expect("DeltaNet tree verification is unsupported");
let tree_size = batch_dim.size();
- let mut parents = encoder.allocate_constant(tree_size * DataType::I32.size_in_bytes())?;
- parents.copyin(batch_dim.parents());
- let mut trie = encoder.allocate_constant(tree_size * size_of::<TrieNode>())?;
- trie.copyin(batch_dim.nodes());
-
- let mut conv_states = encoder
- .allocate_scratch(size_for_shape(&[tree_size, self.conv_dim, self.kernel_size - 1], DataType::F32))?;
- let mut k = encoder.allocate_scratch(size_for_shape(&[tree_size, self.key_dim], self.outer_data_type))?;
- let mut v = encoder.allocate_scratch(size_for_shape(&[tree_size, self.value_dim], self.outer_data_type))?;
- let mut beta = encoder.allocate_scratch(size_for_shape(&[tree_size, self.num_heads], DataType::F32))?;
- let mut log_decay = encoder.allocate_scratch(size_for_shape(&[tree_size, self.num_heads], DataType::F32))?;
-
- let mut scratch = |shape: &[usize], data_type| encoder.allocate_scratch(size_for_shape(shape, data_type));
- let mut tree_projected = scratch(&[tree_size, self.total_proj_dim], self.outer_data_type)?;
- let mut q = scratch(&[tree_size, self.key_dim], self.outer_data_type)?;
+ let parents = encoder.allocate_constant_from_slice(batch_dim.parents())?;
+ let trie = encoder.allocate_constant_from_slice(batch_dim.nodes())?;
+
+ let mut conv_states =
+ encoder.allocate_scratch_for_shape(&[tree_size, self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE)?;
+ let mut k = encoder.allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)?;
+ let mut v = encoder.allocate_scratch_for_shape(&[tree_size, self.value_dim], self.outer_data_type)?;
+ let mut beta = encoder.allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)?;
+ let mut log_decay = encoder.allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)?;
+
+ let mut tree_projected =
+ encoder.allocate_scratch_for_shape(&[tree_size, self.total_proj_dim], self.outer_data_type)?;
+ let mut q = encoder.allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)?;
self.conv_tree_scan.encode(
&in_projected,
@@ -370,9 +362,9 @@ impl<B: Backend> DeltaNet<B> {
&parents,
&mut tree_projected,
&mut conv_states,
- tree_size as u32,
- self.total_proj_dim as u32,
- self.conv_dim as u32,
+ tree_size,
+ self.total_proj_dim,
+ self.conv_dim,
encoder,
);
@@ -385,11 +377,11 @@ impl<B: Backend> DeltaNet<B> {
Some(&mut v),
&mut beta,
&mut log_decay,
- self.num_heads as u32,
- self.num_groups as u32,
- self.key_dim as u32,
- self.value_dim as u32,
- tree_size as u32,
+ self.num_heads,
+ self.num_groups,
+ self.key_dim,
+ self.value_dim,
+ tree_size,
encoder,
);
@@ -410,13 +402,13 @@ impl<B: Backend> DeltaNet<B> {
&mut delta_output,
&tree_projected,
&self.norm_weight,
- self.num_heads as u32,
- self.value_head_dim as u32,
- self.value_dim as u32,
- self.conv_dim as u32,
- self.total_proj_dim as u32,
+ self.num_heads,
+ self.value_head_dim,
+ self.value_dim,
+ self.conv_dim,
+ self.total_proj_dim,
self.norm_epsilon,
- tree_size as u32,
+ tree_size,
encoder,
);
@@ -438,22 +430,22 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
self.tree_verify.is_some()
}
- fn max_context_length(&self) -> Option<usize> {
+ fn max_context_length(&self) -> Option<u32> {
None
}
fn create_empty_state(
&self,
- _max_context_length: Option<usize>,
+ _max_context_length: Option<u32>,
context: &B::Context,
) -> Result<Box<dyn MixerState<B>>, B::Error> {
let mut conv_state = context.create_allocation(
- size_for_shape(&[self.conv_dim, self.kernel_size - 1], DataType::F32),
+ size_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE),
AllocationType::Global,
)?;
let mut ssm_state = context.create_allocation(
- size_for_shape(&[self.num_heads, self.value_head_dim, self.head_dim], DataType::F32),
+ size_for_shape(&[self.num_heads, self.value_head_dim, self.head_dim], INNER_DATA_TYPE),
AllocationType::Global,
)?;
@@ -465,9 +457,9 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
let state_advance = <B::Kernels as Kernels>::StateAdvanceKernel::new(
context,
self.outer_data_type,
- self.head_dim as u32,
- self.num_heads as u32,
- self.num_groups as u32,
+ self.head_dim,
+ self.num_heads,
+ self.num_groups,
)?;
Ok(Box::new(DeltaNetState {
@@ -509,16 +501,16 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
}
let mut delta_output =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type))?;
+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type)?;
if batch_dim.size() == 1 {
self.conv_update.encode(
&self.conv_weight,
self.conv_bias.as_ref(),
&mut in_projected,
&mut state.conv_state,
- self.kernel_size as u32,
- self.conv_dim as u32,
- (self.kernel_size - 1) as u32,
+ self.kernel_size,
+ self.conv_dim,
+ self.kernel_size - 1,
encoder,
);
@@ -529,27 +521,27 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
&self.norm_weight,
&mut state.ssm_state,
&mut delta_output,
- self.num_heads as u32,
- self.num_groups as u32,
- self.value_head_dim as u32,
- self.key_dim as u32,
- self.value_dim as u32,
+ self.num_heads,
+ self.num_groups,
+ self.value_head_dim,
+ self.key_dim,
+ self.value_dim,
self.norm_epsilon,
encoder,
);
} else {
- let mut padded = encoder.allocate_scratch(size_for_shape(
+ let mut padded = encoder.allocate_scratch_for_shape(
&[batch_dim.size() + self.kernel_size - 1, self.total_proj_dim],
- DataType::F32,
- ))?;
+ INNER_DATA_TYPE,
+ )?;
self.conv_pack.encode(
&state.conv_state,
&in_projected,
&mut padded,
- (self.kernel_size - 1) as u32,
- self.total_proj_dim as u32,
- batch_dim.size() as u32,
- self.conv_dim as u32,
+ self.kernel_size - 1,
+ self.total_proj_dim,
+ batch_dim.size(),
+ self.conv_dim,
encoder,
);
self.conv_scan.encode(
@@ -558,12 +550,12 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
self.conv_bias.as_ref(),
&mut in_projected,
&mut state.conv_state,
- batch_dim.size() as u32,
- self.kernel_size as u32,
- self.total_proj_dim as u32,
- (self.kernel_size - 1) as u32,
- self.conv_dim as u32,
- self.total_proj_dim as u32,
+ batch_dim.size(),
+ self.kernel_size,
+ self.total_proj_dim,
+ self.kernel_size - 1,
+ self.conv_dim,
+ self.total_proj_dim,
encoder,
);
if let Some(chunked) = self.chunked.as_ref().filter(|chunked| chunked.should_use(batch_dim.size())) {
@@ -574,24 +566,24 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
dt_bias: &self.dt_bias,
ssm_state: &mut state.ssm_state,
delta_output: &mut delta_output,
- num_heads: self.num_heads as u32,
- num_groups: self.num_groups as u32,
- value_head_dim: self.value_head_dim as u32,
- key_dim: self.key_dim as u32,
- value_dim: self.value_dim as u32,
+ num_heads: self.num_heads,
+ num_groups: self.num_groups,
+ value_head_dim: self.value_head_dim,
+ key_dim: self.key_dim,
+ value_dim: self.value_dim,
suffix_len: batch_dim.size(),
},
encoder,
)?;
} else {
let mut prep_q_norm =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;
+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)?;
let mut prep_k_norm =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;
+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)?;
let mut prep_beta =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;
+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;
let mut prep_decay =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;
+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;
self.delta_net_prefill_prep.encode(
&in_projected,
&self.a_log,
@@ -601,11 +593,11 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
None::<&mut Allocation<B>>,
&mut prep_beta,
&mut prep_decay,
- self.num_heads as u32,
- self.num_groups as u32,
- self.key_dim as u32,
- self.value_dim as u32,
- batch_dim.size() as u32,
+ self.num_heads,
+ self.num_groups,
+ self.key_dim,
+ self.value_dim,
+ batch_dim.size(),
encoder,
);
self.delta_net_prefill.encode(
@@ -616,13 +608,13 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
&in_projected,
&mut state.ssm_state,
&mut delta_output,
- self.num_heads as u32,
- self.num_groups as u32,
- self.value_head_dim as u32,
- self.key_dim as u32,
- self.value_dim as u32,
- batch_dim.size() as u32,
- self.value_head_dim.div_ceil(16) as u32,
+ self.num_heads,
+ self.num_groups,
+ self.value_head_dim,
+ self.key_dim,
+ self.value_dim,
+ batch_dim.size(),
+ self.value_head_dim.div_ceil(16),
encoder,
);
}
@@ -630,13 +622,13 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
&mut delta_output,
&in_projected,
&self.norm_weight,
- self.num_heads as u32,
- self.value_head_dim as u32,
- self.value_dim as u32,
- self.conv_dim as u32,
- self.total_proj_dim as u32,
+ self.num_heads,
+ self.value_head_dim,
+ self.value_dim,
+ self.conv_dim,
+ self.total_proj_dim,
self.norm_epsilon,
- batch_dim.size() as u32,
+ batch_dim.size(),
encoder,
);
}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs
index 233619b08..92c97af2c 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs
@@ -6,10 +6,10 @@ use crate::{
#[derive(Clone, Copy)]
pub struct TreeVerifyNewArguments {
pub data_type: DataType,
- pub num_k_heads: usize,
- pub num_v_heads: usize,
- pub head_k_dim: usize,
- pub head_v_dim: usize,
+ pub num_k_heads: u32,
+ pub num_v_heads: u32,
+ pub head_k_dim: u32,
+ pub head_v_dim: u32,
}
#[derive(Clone, Copy)]
@@ -21,7 +21,7 @@ pub struct TreeVerifyEncodeArguments<'a, B: Backend> {
pub log_decay: &'a Allocation<B>,
pub beta: &'a Allocation<B>,
pub h0: &'a Allocation<B>,
- pub tree_size: usize,
+ pub tree_size: u32,
}
#[cfg(test)]
diff --git a/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs b/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs
index d84b58016..01e7e0bf0 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs
@@ -21,17 +21,19 @@ use crate::{
utils::maybe_mut::MaybeMut,
};
+const INNER_DATA_TYPE: DataType = DataType::F32;
+
pub struct Mamba2State<B: Backend> {
conv_state: Allocation<B>,
ssm_state: Allocation<B>,
- suffix_length: Option<usize>,
+ suffix_length: Option<u32>,
}
impl<B: Backend> MixerState<B> for Mamba2State<B> {
fn prepare(
&mut self,
- _context_length: usize,
- _suffix_length: usize,
+ _context_length: u32,
+ _suffix_length: u32,
_context: &B::Context,
) -> Result<(), B::Error> {
Ok(())
@@ -39,7 +41,7 @@ impl<B: Backend> MixerState<B> for Mamba2State<B> {
fn encode_accept(
&mut self,
- accepted_indices: &[usize],
+ accepted_indices: &[u32],
_encoder: &mut Encoder<B>,
) -> Result<(), <B as Backend>::Error> {
assert!(self.suffix_length.take() == Some(*accepted_indices.last().unwrap() + 1));
@@ -53,14 +55,13 @@ enum Mamba2SSDPrefillVariant<B: Backend> {
}
pub struct Mamba2<B: Backend> {
- kernel_size: usize,
- num_heads: usize,
- num_groups: usize,
- head_dim: usize,
- state_dim: usize,
- inner_dim: usize,
- conv_dim: usize,
- inner_data_type: DataType,
+ kernel_size: u32,
+ num_heads: u32,
+ num_groups: u32,
+ head_dim: u32,
+ state_dim: u32,
+ inner_dim: u32,
+ conv_dim: u32,
activation_type: ActivationType,
in_projection: Box<dyn Linear<B>>,
gate_bias: Allocation<B>,
@@ -90,14 +91,12 @@ pub enum Mamba2NewError<B: Backend> {
impl<B: Backend> Mamba2<B> {
pub fn new(
- hidden_dim: usize,
+ hidden_dim: u32,
outer_data_type: DataType,
config: &Mamba2Config,
parameter_tree: &ParameterTree<B>,
context: &B::Context,
) -> Result<(Self, Option<Allocation<B>>), Mamba2NewError<B>> {
- let inner_data_type = DataType::F32;
-
let kernel_size = config.kernel_size;
let num_heads = config.num_heads;
let num_groups = config.num_groups;
@@ -121,48 +120,48 @@ impl<B: Backend> Mamba2<B> {
context,
outer_data_type,
outer_data_type,
- inner_data_type,
+ INNER_DATA_TYPE,
&parameter_tree.subtree("in_projection"),
)?;
- let gate_bias = parameter_tree.leaf("gate_bias")?.validate(&[inner_dim], inner_data_type)?.read_allocation()?;
- let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, inner_data_type)
+ let gate_bias = parameter_tree.leaf("gate_bias")?.validate(&[inner_dim], INNER_DATA_TYPE)?.read_allocation()?;
+ let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, INNER_DATA_TYPE)
.map_err(Mamba2NewError::Backend)?;
let conv_config = &config.conv_config;
let conv_tree = parameter_tree.subtree("conv");
let conv_weight =
- conv_tree.leaf("weights")?.validate(&[conv_dim, kernel_size], inner_data_type)?.read_allocation()?;
+ conv_tree.leaf("weights")?.validate(&[conv_dim, kernel_size], INNER_DATA_TYPE)?.read_allocation()?;
let conv_bias = if conv_config.has_biases {
- Some(conv_tree.leaf("biases")?.validate(&[conv_dim], inner_data_type)?.read_allocation()?)
+ Some(conv_tree.leaf("biases")?.validate(&[conv_dim], INNER_DATA_TYPE)?.read_allocation()?)
} else {
None
};
let conv_decode =
- <B::Kernels as Kernels>::Conv1dDecodeKernel::new(context, inner_data_type, conv_config.has_biases, true)
+ <B::Kernels as Kernels>::Conv1dDecodeKernel::new(context, INNER_DATA_TYPE, conv_config.has_biases, true)
.map_err(Mamba2NewError::Backend)?;
- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type, inner_data_type)
+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, INNER_DATA_TYPE, INNER_DATA_TYPE)
.map_err(Mamba2NewError::Backend)?;
let conv_scan =
- <B::Kernels as Kernels>::Conv1dScanKernel::new(context, inner_data_type, conv_config.has_biases)
+ <B::Kernels as Kernels>::Conv1dScanKernel::new(context, INNER_DATA_TYPE, conv_config.has_biases)
.map_err(Mamba2NewError::Backend)?;
let skip_connection_weight = parameter_tree
.leaf("skip_connection_weight")?
- .validate(&[num_heads], inner_data_type)?
+ .validate(&[num_heads], INNER_DATA_TYPE)?
.read_allocation()?;
- let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, inner_data_type, true)
+ let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, INNER_DATA_TYPE, true)
.map_err(Mamba2NewError::Backend)?;
let ssd_prefill = if state_dim == 64 {
Mamba2SSDPrefillVariant::Special64(
- <B::Kernels as Kernels>::SSDPrefill64Kernel::new(context, inner_data_type)
+ <B::Kernels as Kernels>::SSDPrefill64Kernel::new(context, INNER_DATA_TYPE)
.map_err(Mamba2NewError::Backend)?,
)
} else {
Mamba2SSDPrefillVariant::Universal(
- <B::Kernels as Kernels>::SSDPrefillKernel::new(context, inner_data_type)
+ <B::Kernels as Kernels>::SSDPrefillKernel::new(context, INNER_DATA_TYPE)
.map_err(Mamba2NewError::Backend)?,
)
};
@@ -173,7 +172,7 @@ impl<B: Backend> Mamba2<B> {
config.has_out_biases,
context,
outer_data_type,
- inner_data_type,
+ INNER_DATA_TYPE,
outer_data_type,
&parameter_tree.subtree("out_projection"),
)?;
@@ -187,7 +186,6 @@ impl<B: Backend> Mamba2<B> {
state_dim,
inner_dim,
conv_dim,
- inner_data_type,
activation_type,
in_projection,
gate_bias,
@@ -212,22 +210,22 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
false
}
- fn max_context_length(&self) -> Option<usize> {
+ fn max_context_length(&self) -> Option<u32> {
None
}
fn create_empty_state(
&self,
- _max_context_length: Option<usize>,
+ _max_context_length: Option<u32>,
context: &B::Context,
) -> Result<Box<dyn MixerState<B>>, B::Error> {
let mut conv_state = context.create_allocation(
- size_for_shape(&[self.conv_dim, self.kernel_size - 1], DataType::F32),
+ size_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE),
AllocationType::Global,
)?;
let mut ssm_state = context.create_allocation(
- size_for_shape(&[self.num_heads, self.head_dim, self.state_dim], DataType::F32),
+ size_for_shape(&[self.num_heads, self.head_dim, self.state_dim], INNER_DATA_TYPE),
AllocationType::Global,
)?;
@@ -270,39 +268,30 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
let in_projected = self.in_projection.encode(hidden, batch_dim.size(), encoder)?;
let mut conv_inputs =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.conv_dim], self.inner_data_type))?;
- let mut gate = encoder.allocate_scratch(size_for_shape(
- &[batch_dim.size(), self.num_heads, self.head_dim],
- self.inner_data_type,
- ))?;
- let mut time_step =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], self.inner_data_type))?;
+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.conv_dim], INNER_DATA_TYPE)?;
+ let mut gate =
+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)?;
+ let mut time_step = encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;
self.split_inproj.encode(
&in_projected,
&mut conv_inputs,
&mut gate,
&mut time_step,
&self.gate_bias,
- batch_dim.size() as u32,
- (self.conv_dim + self.inner_dim + self.num_heads) as u32,
- self.conv_dim as u32,
- self.inner_dim as u32,
- self.num_heads as u32,
+ batch_dim.size(),
+ self.conv_dim + self.inner_dim + self.num_heads,
+ self.conv_dim,
+ self.inner_dim,
+ self.num_heads,
encoder,
);
- let mut conv_x = encoder.allocate_scratch(size_for_shape(
- &[batch_dim.size(), self.num_heads, self.head_dim],
- self.inner_data_type,
- ))?;
- let mut state_b = encoder.allocate_scratch(size_for_shape(
- &[batch_dim.size(), self.num_groups, self.state_dim],
- self.inner_data_type,
- ))?;
- let mut state_c = encoder.allocate_scratch(size_for_shape(
- &[batch_dim.size(), self.num_groups, self.state_dim],
- self.inner_data_type,
- ))?;
+ let mut conv_x =
+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)?;
+ let mut state_b = encoder
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)?;
+ let mut state_c = encoder
+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)?;
let state_stride = self.kernel_size - 1;
if batch_dim.size() == 1 {
self.conv_decode.encode(
@@ -314,29 +303,27 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
&mut state_b,
&mut state_c,
&mut state.conv_state,
- self.kernel_size as u32,
- self.conv_dim as u32,
- state_stride as u32,
- self.conv_dim as u32,
- batch_dim.size() as u32,
- self.inner_dim as u32,
- (self.num_groups * self.state_dim) as u32,
+ self.kernel_size,
+ self.conv_dim,
+ state_stride,
+ self.conv_dim,
+ batch_dim.size(),
+ self.inner_dim,
+ self.num_groups * self.state_dim,
self.activation_type,
encoder,
);
} else {
- let mut padded = encoder.allocate_scratch(size_for_shape(
- &[batch_dim.size() + state_stride, self.conv_dim],
- self.inner_data_type,
- ))?;
+ let mut padded = encoder
+ .allocate_scratch_for_shape(&[batch_dim.size() + state_stride, self.conv_dim], INNER_DATA_TYPE)?;
self.conv_pack.encode(
&state.conv_state,
&conv_inputs,
&mut padded,
- state_stride as u32,
- self.conv_dim as u32,
- batch_dim.size() as u32,
- self.conv_dim as u32,
+ state_stride,
+ self.conv_dim,
+ batch_dim.size(),
+ self.conv_dim,
encoder,
);
self.conv_scan.encode(
@@ -347,31 +334,27 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
&mut state_b,
&mut state_c,
&mut state.conv_state,
- batch_dim.size() as u32,
- self.kernel_size as u32,
- self.conv_dim as u32,
- state_stride as u32,
- self.conv_dim as u32,
- self.inner_dim as u32,
- (self.num_groups * self.state_dim) as u32,
+ batch_dim.size(),
+ self.kernel_size,
+ self.conv_dim,
+ state_stride,
+ self.conv_dim,
+ self.inner_dim,
+ self.num_groups * self.state_dim,
self.activation_type,
encoder,
);
}
let mut ssd_output =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.inner_dim], self.inner_data_type))?;
- let x_strides = [(self.num_heads * self.head_dim) as u32, self.head_dim as u32, 1];
- let dt_strides = [self.num_heads as u32, 1];
- let cb_strides = [(self.num_groups * self.state_dim) as u32, self.state_dim as u32, 1];
+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.inner_dim], INNER_DATA_TYPE)?;
+ let x_strides = [self.num_heads * self.head_dim, self.head_dim, 1];
+ let dt_strides = [self.num_heads, 1];
+ let cb_strides = [self.num_groups * self.state_dim, self.state_dim, 1];
let group_size = self.num_heads / self.num_groups;
if batch_dim.size() == 1 {
- let state_strides = [
- (self.num_heads * self.head_dim * self.state_dim) as u32,
- (self.head_dim * self.state_dim) as u32,
- self.state_dim as u32,
- 1,
- ];
+ let state_strides =
+ [self.num_heads * self.head_dim * self.state_dim, self.head_dim * self.state_dim, self.state_dim, 1];
self.ssd_update.encode(
&conv_x,
&time_step,
@@ -382,19 +365,19 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
None::<&Allocation<B>>,
&mut ssd_output,
&mut state.ssm_state,
- group_size as u32,
- self.state_dim as u32,
+ group_size,
+ self.state_dim,
&x_strides,
&dt_strides,
&cb_strides,
&state_strides,
- batch_dim.size() as u32,
- self.num_heads as u32,
- self.head_dim as u32,
+ batch_dim.size(),
+ self.num_heads,
+ self.head_dim,
encoder,
);
} else {
- let state_strides = [(self.head_dim * self.state_dim) as u32, self.state_dim as u32, 1];
+ let state_strides = [self.head_dim * self.state_dim, self.state_dim, 1];
match &self.ssd_prefill {
Mamba2SSDPrefillVariant::Universal(ssd_prefill) => ssd_prefill.encode(
&conv_x,
@@ -405,15 +388,15 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
&gate,
&mut state.ssm_state,
&mut ssd_output,
- batch_dim.size() as u32,
- group_size as u32,
- self.state_dim as u32,
+ batch_dim.size(),
+ group_size,
+ self.state_dim,
&x_strides,
&dt_strides,
&cb_strides,
&state_strides,
- self.num_heads as u32,
- self.head_dim as u32,
+ self.num_heads,
+ self.head_dim,
encoder,
),
Mamba2SSDPrefillVariant::Special64(ssd_prefill) => ssd_prefill.encode(
@@ -425,15 +408,15 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
&gate,
&mut state.ssm_state,
&mut ssd_output,
- batch_dim.size() as u32,
- group_size as u32,
- self.state_dim as u32,
+ batch_dim.size(),
+ group_size,
+ self.state_dim,
&x_strides,
&dt_strides,
&cb_strides,
&state_strides,
- self.num_heads as u32,
- self.head_dim as u32,
+ self.num_heads,
+ self.head_dim,
encoder,
),
}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/mod.rs
index c3ae9178c..e38911c01 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/mod.rs
@@ -27,14 +27,14 @@ pub mod short_conv;
pub trait MixerState<B: Backend>: Any + Send {
fn prepare(
&mut self,
- context_length: usize,
- suffix_length: usize,
+ context_length: u32,
+ suffix_length: u32,
context: &B::Context,
) -> Result<(), B::Error>;
fn encode_accept(
&mut self,
- accepted_indices: &[usize],
+ accepted_indices: &[u32],
encoder: &mut Encoder<B>,
) -> Result<(), B::Error>;
}
@@ -51,11 +51,11 @@ impl<'a, B: Backend> MaybeMut<'a, dyn MixerState<B>> {
pub trait Mixer<B: Backend>: Any + Send + Sync {
fn speculation_supported(&self) -> bool;
- fn max_context_length(&self) -> Option<usize>;
+ fn max_context_length(&self) -> Option<u32>;
fn create_empty_state(
&self,
- max_context_length: Option<usize>,
+ max_context_length: Option<u32>,
context: &B::Context,
) -> Result<Box<dyn MixerState<B>>, B::Error>;
@@ -83,7 +83,7 @@ pub enum MixerNewError<B: Backend> {
impl<B: Backend> dyn Mixer<B> {
pub fn new(
- hidden_dim: usize,
+ hidden_dim: u32,
data_type: DataType,
rope_config: Option<&AnyRoPEConfig>,
config: &AnyTokenMixerConfig,
diff --git a/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs b/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs
index 3d285ca7d..bd544b76a 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs
@@ -19,7 +19,7 @@ use crate::{
enum ShortConvStateSuffixStatus<B: Backend> {
Flat {
- suffix_length: usize,
+ suffix_length: u32,
},
Trie {
conv_states: Allocation<B>,
@@ -34,8 +34,8 @@ pub struct ShortConvState<B: Backend> {
impl<B: Backend> MixerState<B> for ShortConvState<B> {
fn prepare(
&mut self,
- _context_length: usize,
- _suffix_length: usize,
+ _context_length: u32,
+ _suffix_length: u32,
_context: &B::Context,
) -> Result<(), B::Error> {
Ok(())
@@ -43,7 +43,7 @@ impl<B: Backend> MixerState<B> for ShortConvState<B> {
fn encode_accept(
&mut self,
- accepted_indices: &[usize],
+ accepted_indices: &[u32],
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let suffix_state_type =
@@ -62,9 +62,10 @@ impl<B: Backend> MixerState<B> for ShortConvState<B> {
conv_states,
} => {
let conv_state_size = self.conv_state.size();
+ let accepted_offset = accepted_index as usize * conv_state_size;
encoder.encode_copy(
&conv_states,
- (accepted_index * conv_state_size)..((accepted_index + 1) * conv_state_size),
+ accepted_offset..accepted_offset + conv_state_size,
&mut self.conv_state,
..,
);
@@ -75,9 +76,9 @@ impl<B: Backend> MixerState<B> for ShortConvState<B> {
}
pub struct ShortConv<B: Backend> {
- hidden_dim: usize,
+ hidden_dim: u32,
data_type: DataType,
- kernel_size: usize,
+ kernel_size: u32,
in_projection: Box<dyn Linear<B>>,
out_projection: Box<dyn Linear<B>>,
short_conv_pack: <B::Kernels as Kernels>::ShortConvPackKernel,
@@ -102,7 +103,7 @@ pub enum ShortConvNewError<B: Backend> {
impl<B: Backend> ShortConv<B> {
pub fn new(
- hidden_dim: usize,
+ hidden_dim: u32,
data_type: DataType,
config: &ShortConvConfig,
parameter_tree: &ParameterTree<B>,
@@ -183,7 +184,7 @@ impl<B: Backend> ShortConv<B> {
state: &mut ShortConvState<B>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut conv_output = encoder.allocate_scratch(size_for_shape(&[self.hidden_dim], self.data_type))?;
+ let mut conv_output = encoder.allocate_scratch_for_shape(&[self.hidden_dim], self.data_type)?;
self.short_conv_decode.encode(
in_projected,
&self.conv_weight,
@@ -192,10 +193,10 @@ impl<B: Backend> ShortConv<B> {
&mut conv_output,
&mut state.conv_state,
1,
- self.kernel_size as u32,
- self.hidden_dim as u32 * 3,
- (self.kernel_size - 1) as u32,
- self.hidden_dim as u32,
+ self.kernel_size,
+ self.hidden_dim * 3,
+ self.kernel_size - 1,
+ self.hidden_dim,
encoder,
);
Ok(conv_output)
@@ -204,27 +205,26 @@ impl<B: Backend> ShortConv<B> {
fn encode_prefill_conv(
&self,
in_projected: &Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
state: &mut ShortConvState<B>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
let state_stride = self.kernel_size - 1;
let padded_rows = state_stride + batch_dim;
- let mut padded = encoder.allocate_scratch(size_for_shape(&[padded_rows, self.hidden_dim], self.data_type))?;
+ let mut padded = encoder.allocate_scratch_for_shape(&[padded_rows, self.hidden_dim], self.data_type)?;
self.short_conv_pack.encode(
&state.conv_state,
in_projected,
&mut padded,
- state_stride as u32,
- batch_dim as u32,
- self.hidden_dim as u32 * 3,
- self.hidden_dim as u32,
+ state_stride,
+ batch_dim,
+ self.hidden_dim * 3,
+ self.hidden_dim,
encoder,
);
- let mut conv_output =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;
+ let mut conv_output = encoder.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;
self.short_conv_prefill.encode(
&padded,
in_projected,
@@ -232,11 +232,11 @@ impl<B: Backend> ShortConv<B> {
self.conv_bias.as_ref(),
&mut conv_output,
&mut state.conv_state,
- batch_dim as u32,
- self.kernel_size as u32,
- self.hidden_dim as u32 * 3,
- state_stride as u32,
- self.hidden_dim as u32,
+ batch_dim,
+ self.kernel_size,
+ self.hidden_dim * 3,
+ state_stride,
+ self.hidden_dim,
encoder,
);
Ok(conv_output)
@@ -245,15 +245,14 @@ impl<B: Backend> ShortConv<B> {
fn encode_trie_conv(
&self,
in_projected: &Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
token_parents: &Allocation<B>,
state: &mut ShortConvState<B>,
encoder: &mut Encoder<B>,
) -> Result<(Allocation<B>, Allocation<B>), B::Error> {
- let mut conv_output =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;
- let mut conv_states = encoder
- .allocate_scratch(size_for_shape(&[batch_dim, self.kernel_size - 1, self.hidden_dim], self.data_type))?;
+ let mut conv_output = encoder.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;
+ let mut conv_states =
+ encoder.allocate_scratch_for_shape(&[batch_dim, self.kernel_size - 1, self.hidden_dim], self.data_type)?;
self.short_conv_trie.encode(
in_projected,
&self.conv_weight,
@@ -262,11 +261,11 @@ impl<B: Backend> ShortConv<B> {
token_parents,
&mut conv_output,
&mut conv_states,
- batch_dim as u32,
- self.kernel_size as u32,
- (self.hidden_dim * 3) as u32,
- (self.kernel_size - 1) as u32,
- self.hidden_dim as u32,
+ batch_dim,
+ self.kernel_size,
+ self.hidden_dim * 3,
+ self.kernel_size - 1,
+ self.hidden_dim,
encoder,
);
Ok((conv_output, conv_states))
@@ -278,13 +277,13 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {
true
}
- fn max_context_length(&self) -> Option<usize> {
+ fn max_context_length(&self) -> Option<u32> {
None
}
fn create_empty_state(
&self,
- _max_context_length: Option<usize>,
+ _max_context_length: Option<u32>,
context: &B::Context,
) -> Result<Box<dyn MixerState<B>>, B::Error> {
let mut conv_state = context.create_allocation(
@@ -342,8 +341,7 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {
});
conv_output
} else {
- let mut token_parents = encoder.allocate_constant(batch_dim.size() * DataType::I32.size_in_bytes())?;
- token_parents.copyin(batch_dim.parents());
+ let token_parents = encoder.allocate_constant_from_slice(batch_dim.parents())?;
let (conv_output, conv_states) =
self.encode_trie_conv(&in_projected, batch_dim.size(), &token_parents, state, encoder)?;
state.suffix_state = Some(ShortConvStateSuffixStatus::Trie {
diff --git a/crates/backend-uzu/src/encodable_block/mlp/dense.rs b/crates/backend-uzu/src/encodable_block/mlp/dense.rs
index eafd4ac4b..a646339f3 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/dense.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/dense.rs
@@ -32,7 +32,7 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {
fn encode(
&self,
input: Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group("mlp (dense)");
diff --git a/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs b/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs
index 231dcb145..73799c439 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs
@@ -1,5 +1,4 @@
use crate::{
- array::size_for_shape,
backends::common::{Allocation, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::GatedActMulKernel},
config::activation::AnyActivation,
data_type::DataType,
@@ -8,7 +7,7 @@ use crate::{
pub struct MlpGateActMulEncodable<B: Backend> {
kernel: <B::Kernels as Kernels>::GatedActMulKernel,
activation: AnyActivation,
- hidden_dim: usize,
+ hidden_dim: u32,
data_type: DataType,
hadamard_factors: Option<Allocation<B>>,
}
@@ -18,7 +17,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
context: &B::Context,
data_type: DataType,
activation: AnyActivation,
- hidden_dim: usize,
+ hidden_dim: u32,
hadamard_factors: Option<Allocation<B>>,
) -> Result<Self, B::Error> {
let kernel =
@@ -36,21 +35,21 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
&self,
encoder: &mut Encoder<B>,
fused_up: &Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group("gate act mul");
if self.activation.act_type() == ActivationType::IDENTITY {
panic!("Identity activation is not supported for kernel")
}
- let mut hidden = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;
+ let mut hidden = encoder.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;
self.kernel.encode(
fused_up,
None::<&Allocation<B>>,
&mut hidden,
self.hadamard_factors.as_ref(),
- self.hidden_dim as u32,
- batch_dim as u32,
+ self.hidden_dim,
+ batch_dim,
0,
0,
self.activation.act_type(),
diff --git a/crates/backend-uzu/src/encodable_block/mlp/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/mod.rs
index 73bff433c..57230dd8c 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/mod.rs
@@ -19,7 +19,7 @@ pub trait Mlp<B: Backend>: Send + Sync {
fn encode(
&self,
input: Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error>;
}
@@ -37,8 +37,8 @@ pub enum MlpBlockError<B: Backend> {
impl<B: Backend> dyn Mlp<B> {
pub fn new(
config: &AnyMLPConfig,
- model_dimension: usize,
- hidden_dimension: usize,
+ model_dimension: u32,
+ hidden_dimension: u32,
context: &B::Context,
parameter_tree: &ParameterTree<B>,
data_type: DataType,
diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_decode.rs b/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_decode.rs
index 7495f995c..e1ff9e4f7 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_decode.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_decode.rs
@@ -1,5 +1,4 @@
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder,
kernel::{
@@ -46,42 +45,40 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
args: MoeExpertsTwoPassArguments<B>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- const BLOCK_M: usize = 4;
- let h_blocks = args.d_ff.div_ceil(BLOCK_M) as u32;
+ const BLOCK_M: u32 = 4;
+ let h_blocks = args.d_ff.div_ceil(BLOCK_M);
- let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[args.num_routed_experts], DataType::U32))?;
- self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts as u32, h_blocks, encoder);
+ let mut tile_counts = encoder.allocate_scratch_for_shape(&[args.num_routed_experts], DataType::U32)?;
+ self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts, h_blocks, encoder);
- let mut tile_offsets =
- encoder.allocate_scratch(size_for_shape(&[args.num_routed_experts + 1], DataType::U32))?;
- let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[1], DataType::U32))?;
- self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts as u32, encoder);
+ let mut tile_offsets = encoder.allocate_scratch_for_shape(&[args.num_routed_experts + 1], DataType::U32)?;
+ let mut total_tiles = encoder.allocate_scratch_for_shape(&[1], DataType::U32)?;
+ self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts, encoder);
- let mut row_expert_map = encoder.allocate_scratch(size_for_shape(&[args.total_rows], DataType::U32))?;
+ let mut row_expert_map = encoder.allocate_scratch_for_shape(&[args.total_rows], DataType::U32)?;
self.row_map.encode(
args.expert_offsets,
&mut row_expert_map,
- args.total_rows as u32,
- args.num_routed_experts as u32,
+ args.total_rows,
+ args.num_routed_experts,
encoder,
);
- let mut tile_map =
- encoder.allocate_scratch(size_for_shape(&[args.total_rows * h_blocks as usize * 3], DataType::U32))?;
+ let mut tile_map = encoder.allocate_scratch_for_shape(&[args.total_rows, h_blocks, 3], DataType::U32)?;
self.build_map.encode(
args.expert_offsets,
&tile_offsets,
&row_expert_map,
&mut tile_map,
- args.total_rows as u32,
+ args.total_rows,
h_blocks,
encoder,
);
- let mut dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;
+ let mut dispatch_args = encoder.allocate_scratch_for_shape(&[3], DataType::U32)?;
self.dispatch.encode(&total_tiles, &mut dispatch_args, 1, encoder);
- let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;
+ let mut hidden = encoder.allocate_scratch_for_shape(&[args.total_rows, args.d_ff], DataType::F32)?;
self.pass_a_indirect.encode(
args.x_perm,
@@ -89,9 +86,9 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
args.w13_all,
&mut hidden,
args.up_biases,
- args.d_model as u32,
- args.d_ff as u32,
- args.num_routed_experts as u32,
+ args.d_model,
+ args.d_ff,
+ args.num_routed_experts,
args.gate_clip_min,
args.gate_clip_max,
args.up_clip_min,
@@ -102,17 +99,17 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
encoder,
);
- let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], self.data_type))?;
+ let mut output = encoder.allocate_scratch_for_shape(&[args.total_rows, args.d_model], self.data_type)?;
self.fused_down.encode(
&hidden,
&row_expert_map,
args.w2_all,
args.down_biases,
&mut output,
- args.total_rows as u32,
- args.d_model as u32,
- args.d_ff as u32,
- args.num_routed_experts as u32,
+ args.total_rows,
+ args.d_model,
+ args.d_ff,
+ args.num_routed_experts,
encoder,
);
diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs b/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs
index 08d572785..f53249825 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs
@@ -1,5 +1,4 @@
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder, Kernels,
kernel::{
@@ -42,31 +41,30 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
args: MoeExpertsTwoPassArguments<B>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[args.num_routed_experts], DataType::U32))?;
- self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts as u32, encoder);
+ let mut tile_counts = encoder.allocate_scratch_for_shape(&[args.num_routed_experts], DataType::U32)?;
+ self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts, encoder);
- let mut tile_offsets =
- encoder.allocate_scratch(size_for_shape(&[args.num_routed_experts + 1], DataType::U32))?;
- let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[8], DataType::U32))?;
- self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts as u32, encoder);
+ let mut tile_offsets = encoder.allocate_scratch_for_shape(&[args.num_routed_experts + 1], DataType::U32)?;
+ let mut total_tiles = encoder.allocate_scratch_for_shape(&[8], DataType::U32)?;
+ self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts, encoder);
- let mut tile_map = encoder.allocate_scratch(size_for_shape(&[args.total_rows * 3], DataType::U32))?;
+ let mut tile_map = encoder.allocate_scratch_for_shape(&[args.total_rows, 3], DataType::U32)?;
self.build.encode(
args.expert_offsets,
&tile_offsets,
&tile_counts,
&mut tile_map,
- args.num_routed_experts as u32,
+ args.num_routed_experts,
encoder,
);
- const COL_TILE_FF: usize = 32; // Must match PASSA_BN in kernel
- let n_tiles_ff = args.d_ff.div_ceil(COL_TILE_FF) as u32;
+ const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in kernel
+ let n_tiles_ff = args.d_ff.div_ceil(COL_TILE_FF);
- let mut pass_a_dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;
+ let mut pass_a_dispatch_args = encoder.allocate_scratch_for_shape(&[3], DataType::U32)?;
self.dispatch.encode(&total_tiles, &mut pass_a_dispatch_args, n_tiles_ff, encoder);
- let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;
+ let mut hidden = encoder.allocate_scratch_for_shape(&[args.total_rows, args.d_ff], DataType::F32)?;
encoder.encode_fill(&mut hidden, 0);
self.pass_a_indirect.encode(
@@ -75,9 +73,9 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
args.w13_all,
args.up_biases,
&mut hidden,
- args.d_model as u32,
- args.d_ff as u32,
- args.num_routed_experts as u32,
+ args.d_model,
+ args.d_ff,
+ args.num_routed_experts,
args.gate_clip_min,
args.gate_clip_max,
args.up_clip_min,
@@ -88,22 +86,22 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
encoder,
);
- const COL_TILE_MODEL: usize = 64;
- let n_tiles_model = args.d_model.div_ceil(COL_TILE_MODEL) as u32;
+ const COL_TILE_MODEL: u32 = 64;
+ let n_tiles_model = args.d_model.div_ceil(COL_TILE_MODEL);
- let mut pass_b_dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;
+ let mut pass_b_dispatch_args = encoder.allocate_scratch_for_shape(&[3], DataType::U32)?;
self.dispatch.encode(&total_tiles, &mut pass_b_dispatch_args, n_tiles_model, encoder);
- let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], self.data_type))?;
+ let mut output = encoder.allocate_scratch_for_shape(&[args.total_rows, args.d_model], self.data_type)?;
self.pass_b_indirect.encode(
&hidden,
args.expert_offsets,
args.w2_all,
args.down_biases,
&mut output,
- args.d_model as u32,
- args.d_ff as u32,
- args.num_routed_experts as u32,
+ args.d_model,
+ args.d_ff,
+ args.num_routed_experts,
&tile_map,
&pass_b_dispatch_args,
encoder,
@@ -119,10 +117,10 @@ pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {
pub w2_all: &'a Allocation<B>,
pub up_biases: &'a Allocation<B>,
pub down_biases: &'a Allocation<B>,
- pub total_rows: usize,
- pub d_model: usize,
- pub d_ff: usize,
- pub num_routed_experts: usize,
+ pub total_rows: u32,
+ pub d_model: u32,
+ pub d_ff: u32,
+ pub num_routed_experts: u32,
pub gate_clip_min: f32,
pub gate_clip_max: f32,
pub up_clip_min: f32,
diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe/gather.rs b/crates/backend-uzu/src/encodable_block/mlp/moe/gather.rs
index 9e72cc33e..7ada8c503 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/moe/gather.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/moe/gather.rs
@@ -1,5 +1,4 @@
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder, Kernels,
kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},
@@ -37,36 +36,22 @@ impl<B: Backend> MoeGather<B> {
input: &Allocation<B>,
bucketed_ids: &Allocation<B>,
sumk: &Allocation<B>,
- batch_dim: usize,
- num_active_experts: usize,
- d_model: usize,
+ batch_dim: u32,
+ num_active_experts: u32,
+ d_model: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
let mut x_perm =
- encoder.allocate_scratch(size_for_shape(&[batch_dim * num_active_experts, d_model], self.data_type))?;
+ encoder.allocate_scratch_for_shape(&[batch_dim, num_active_experts, d_model], self.data_type)?;
encoder.encode_fill(&mut x_perm, 0);
match &self.variant {
- MoeGatherVariant::OneD(kernel) => kernel.encode(
- input,
- bucketed_ids,
- &mut x_perm,
- sumk,
- d_model as u32,
- batch_dim as u32,
- num_active_experts as u32,
- encoder,
- ),
- MoeGatherVariant::TwoD(kernel) => kernel.encode(
- input,
- bucketed_ids,
- &mut x_perm,
- sumk,
- d_model as u32,
- batch_dim as u32,
- num_active_experts as u32,
- encoder,
- ),
+ MoeGatherVariant::OneD(kernel) => {
+ kernel.encode(input, bucketed_ids, &mut x_perm, sumk, d_model, batch_dim, num_active_experts, encoder)
+ },
+ MoeGatherVariant::TwoD(kernel) => {
+ kernel.encode(input, bucketed_ids, &mut x_perm, sumk, d_model, batch_dim, num_active_experts, encoder)
+ },
};
Ok(x_perm)
diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rs
index 48e7333e1..cf43cd025 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rs
@@ -10,7 +10,6 @@ use gather::MoeGather;
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder, Kernels,
gpu_types::ActivationType,
@@ -44,10 +43,10 @@ pub struct MoeBlock<B: Backend> {
w2: Allocation<B>,
up_biases: Allocation<B>,
down_biases: Allocation<B>,
- model_dim: usize,
- hidden_dim: usize,
- num_routed_experts: usize,
- num_active_experts: usize,
+ model_dim: u32,
+ hidden_dim: u32,
+ num_routed_experts: u32,
+ num_active_experts: u32,
gate_clip_min: f32,
gate_clip_max: f32,
up_clip_min: f32,
@@ -84,7 +83,7 @@ impl<B: Backend> MoeBlock<B> {
pub fn new(
context: &B::Context,
moe_config: &MixtureOfExpertsConfig,
- model_dim: usize,
+ model_dim: u32,
data_type: DataType,
parameter_tree: &ParameterTree<B>,
) -> Result<Self, MoeBlockError<B>> {
@@ -213,7 +212,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
fn encode(
&self,
input: Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group("mlp (moe)");
@@ -222,10 +221,9 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
let num_blocks = batch_dim.div_ceil(256);
let num_tiles = self.num_routed_experts.div_ceil(512);
- let mut topk_ids =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.num_active_experts], DataType::I32))?;
+ let mut topk_ids = encoder.allocate_scratch_for_shape(&[batch_dim, self.num_active_experts], DataType::I32)?;
let mut topk_probs =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.num_active_experts], self.data_type))?;
+ encoder.allocate_scratch_for_shape(&[batch_dim, self.num_active_experts], self.data_type)?;
encoder.encode_fill(&mut topk_ids, 0xFF);
@@ -237,36 +235,36 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
None::<&Allocation<B>>,
&mut topk_ids,
&mut topk_probs,
- batch_dim as u32,
- self.model_dim as u32,
- self.num_routed_experts as u32,
- self.num_active_experts as u32,
+ batch_dim,
+ self.model_dim,
+ self.num_routed_experts,
+ self.num_active_experts,
self.router_renorm,
None::<f32>,
None::<f32>,
encoder,
);
- let mut offsets = encoder.allocate_scratch(size_for_shape(&[self.num_routed_experts + 1], DataType::U32))?;
- let mut sumk = encoder.allocate_scratch(size_for_shape(&[1], DataType::U32))?;
+ let mut offsets = encoder.allocate_scratch_for_shape(&[self.num_routed_experts + 1], DataType::U32)?;
+ let mut sumk = encoder.allocate_scratch_for_shape(&[1], DataType::U32)?;
let scatter_entries = num_blocks * num_tiles * 512;
- let mut partials = encoder.allocate_scratch(size_for_shape(&[scatter_entries], DataType::U32))?;
+ let mut partials = encoder.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;
self.counts_offsets_kernel.encode(
&topk_ids,
&mut offsets,
&mut sumk,
&mut partials,
- batch_dim as u32,
- self.num_routed_experts as u32,
- self.num_active_experts as u32,
+ batch_dim,
+ self.num_routed_experts,
+ self.num_active_experts,
encoder,
);
- let mut block_bases = encoder.allocate_scratch(size_for_shape(&[scatter_entries], DataType::U32))?;
- let mut block_alloc = encoder.allocate_scratch(size_for_shape(&[scatter_entries], DataType::U32))?;
- let mut bucketed_ids = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::I32))?;
- let mut bucketed_probs = encoder.allocate_scratch(size_for_shape(&[total_rows], self.data_type))?;
- let mut tok2row = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::I32))?;
+ let mut block_bases = encoder.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;
+ let mut block_alloc = encoder.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;
+ let mut bucketed_ids = encoder.allocate_scratch_for_shape(&[total_rows], DataType::I32)?;
+ let mut bucketed_probs = encoder.allocate_scratch_for_shape(&[total_rows], self.data_type)?;
+ let mut tok2row = encoder.allocate_scratch_for_shape(&[total_rows], DataType::I32)?;
encoder.encode_fill(&mut tok2row, 0xFF);
@@ -274,9 +272,9 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
&partials,
&mut block_bases,
&mut block_alloc,
- self.num_routed_experts as u32,
- num_blocks as u32,
- num_tiles as u32,
+ self.num_routed_experts,
+ num_blocks,
+ num_tiles,
0u32,
encoder,
);
@@ -288,11 +286,11 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
&block_alloc,
&mut bucketed_ids,
&mut bucketed_probs,
- batch_dim as u32,
- self.num_routed_experts as u32,
- self.num_active_experts as u32,
- num_blocks as u32,
- num_tiles as u32,
+ batch_dim,
+ self.num_routed_experts,
+ self.num_active_experts,
+ num_blocks,
+ num_tiles,
&mut tok2row,
encoder,
);
@@ -331,15 +329,15 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
self.experts_two_pass_prefill_block.encode(args, encoder)?
};
- let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.model_dim], self.data_type))?;
+ let mut output = encoder.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;
self.finalize_kernel.encode(
&tok2row,
&topk_probs,
&y_partial,
&mut output,
- batch_dim as u32,
- self.model_dim as u32,
- self.num_active_experts as u32,
+ batch_dim,
+ self.model_dim,
+ self.num_active_experts,
encoder,
);
diff --git a/crates/backend-uzu/src/encodable_block/normalization.rs b/crates/backend-uzu/src/encodable_block/normalization.rs
index 13163763e..24e1a2e93 100644
--- a/crates/backend-uzu/src/encodable_block/normalization.rs
+++ b/crates/backend-uzu/src/encodable_block/normalization.rs
@@ -40,7 +40,7 @@ pub struct Normalization<B: Backend> {
scale_offset: Option<f32>,
scales: Option<Allocation<B>>,
biases: Option<Allocation<B>>,
- element_count: usize,
+ element_count: u32,
hadamard_factors: Option<Allocation<B>>,
post_layer_scalar_value: f32,
data_type: DataType,
@@ -49,7 +49,7 @@ pub struct Normalization<B: Backend> {
impl<B: Backend> Normalization<B> {
pub fn new(
- element_count: usize,
+ element_count: u32,
hadamard_factors: Option<Allocation<B>>,
shortcut_mode: ShortcutMode,
post_layer_scalar: PostLayerScalar,
@@ -114,17 +114,17 @@ impl<B: Backend> Normalization<B> {
pub fn encode(
&self,
input: &Allocation<B>,
- row_offset: usize,
- row_count: usize,
+ row_offset: u32,
+ row_count: u32,
shortcut: Option<&mut Allocation<B>>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group("normalization");
- let row_size = self.element_count * self.data_type.size_in_bytes();
- let row_offset_bytes = row_offset * row_size;
+ let row_size = size_for_shape(&[self.element_count], self.data_type);
+ let row_offset_bytes = row_offset as usize * row_size;
let shortcut = shortcut.map(|shortcut| (shortcut, row_offset_bytes));
- let mut output = encoder.allocate_scratch(size_for_shape(&[row_count, self.element_count], self.data_type))?;
+ let mut output = encoder.allocate_scratch_for_shape(&[row_count, self.element_count], self.data_type)?;
self.kernel.encode(
Some((input, row_offset_bytes)),
self.scales.as_ref(),
@@ -132,8 +132,8 @@ impl<B: Backend> Normalization<B> {
&mut output,
shortcut,
self.hadamard_factors.as_ref(),
- row_count as u32,
- self.element_count as u32,
+ row_count,
+ self.element_count,
self.epsilon,
self.scale_offset.unwrap_or(0.0),
self.post_layer_scalar_value,
diff --git a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
index b03fb0e78..8840eabbc 100644
--- a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
@@ -1,7 +1,6 @@
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder,
kernel::{GatedActMulKernel, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},
@@ -36,9 +35,9 @@ pub struct PerLayerEmbedding<B: Backend> {
model_projection: Box<dyn Linear<B>>,
projection_norm: Normalization<B>,
add_scale: <B::Kernels as Kernels>::TensorAddScaleKernel,
- ple_dim: usize,
- num_layers: usize,
- model_dim: usize,
+ ple_dim: u32,
+ num_layers: u32,
+ model_dim: u32,
fused_token_scale: f32,
data_type: DataType,
}
@@ -47,7 +46,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
pub fn new(
context: &B::Context,
config: &PLEModelConfig,
- model_dim: usize,
+ model_dim: u32,
data_type: DataType,
parameter_tree: &ParameterTree<B>,
) -> Result<Self, PerLayerEmbeddingError<B>> {
@@ -107,7 +106,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
&self,
token_ids: &Allocation<B>,
inner_features: &Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group("per layer embedding");
@@ -116,30 +115,24 @@ impl<B: Backend> PerLayerEmbedding<B> {
let total_rows = batch_dim * self.num_layers;
let total_elements = batch_dim * total_ple_dim;
- let mut token_ple = encoder.allocate_scratch(size_for_shape(&[batch_dim, total_ple_dim], self.data_type))?;
- self.token_embedding.encode_lookup(
- token_ids,
- &mut token_ple,
- batch_dim as u32,
- self.fused_token_scale,
- encoder,
- );
+ let mut token_ple = encoder.allocate_scratch_for_shape(&[batch_dim, total_ple_dim], self.data_type)?;
+ self.token_embedding.encode_lookup(token_ids, &mut token_ple, batch_dim, self.fused_token_scale, encoder);
let mut model_projection_input =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.model_dim], self.data_type))?;
+ encoder.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;
encoder.encode_copy(inner_features, .., &mut model_projection_input, ..);
let model_projected = self.model_projection.encode(model_projection_input, batch_dim, encoder)?;
let model_normed = self.projection_norm.encode(&model_projected, 0, total_rows, None, encoder)?;
let mut per_layer_inputs =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type))?;
+ encoder.allocate_scratch_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type)?;
self.add_scale.encode(
Some(&token_ple),
&model_normed,
&mut per_layer_inputs,
- total_elements as u32,
- total_elements as u32,
+ total_elements,
+ total_elements,
1.0,
encoder,
);
@@ -157,9 +150,9 @@ pub struct PerLayerEmbeddingProjection<B: Backend> {
gate_act_mul: <B::Kernels as Kernels>::GatedActMulKernel,
residual_finalize: <B::Kernels as Kernels>::TensorAddBiasKernel,
residual_combine: <B::Kernels as Kernels>::TensorAddScaleKernel,
- model_dim: usize,
- ple_dim: usize,
- num_layers: usize,
+ model_dim: u32,
+ ple_dim: u32,
+ num_layers: u32,
activation: AnyActivation,
post_layer_scalar: f32,
data_type: DataType,
@@ -169,8 +162,8 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
pub fn new(
context: &B::Context,
config: &PLELayerConfig,
- model_dim: usize,
- num_layers: usize,
+ model_dim: u32,
+ num_layers: u32,
post_layer_scalar: f32,
data_type: DataType,
parameter_tree: &ParameterTree<B>,
@@ -227,40 +220,33 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
pub fn encode(
&self,
- layer_index: usize,
+ layer_index: u32,
per_layer_input: &Allocation<B>,
outputs: &mut Allocation<B>,
hidden: &Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
encoder.push_debug_group("per layer embedding projection");
let length = batch_dim * self.model_dim;
- self.residual_finalize.encode(
- None::<&Allocation<B>>,
- hidden,
- &mut *outputs,
- length as u32,
- length as u32,
- encoder,
- );
+ self.residual_finalize.encode(None::<&Allocation<B>>, hidden, &mut *outputs, length, length, encoder);
- let mut gate_input = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.model_dim], self.data_type))?;
+ let mut gate_input = encoder.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;
encoder.encode_copy(outputs, .., &mut gate_input, ..);
let gate_out = self.gate.encode(gate_input, batch_dim, encoder)?;
- let mut activated = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.ple_dim], self.data_type))?;
+ let mut activated = encoder.allocate_scratch_for_shape(&[batch_dim, self.ple_dim], self.data_type)?;
self.gate_act_mul.encode(
&gate_out,
Some(per_layer_input),
&mut activated,
None::<&Allocation<B>>,
- self.ple_dim as u32,
- batch_dim as u32,
- (layer_index * self.ple_dim) as u32,
- (self.num_layers * self.ple_dim) as u32,
+ self.ple_dim,
+ batch_dim,
+ layer_index * self.ple_dim,
+ self.num_layers * self.ple_dim,
self.activation.act_type(),
encoder,
);
@@ -272,8 +258,8 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
None::<&Allocation<B>>,
&normed,
&mut *outputs,
- length as u32,
- length as u32,
+ length,
+ length,
self.post_layer_scalar,
encoder,
);
diff --git a/crates/backend-uzu/src/encodable_block/prediction_head.rs b/crates/backend-uzu/src/encodable_block/prediction_head.rs
index 3af4a8af5..d79daad53 100644
--- a/crates/backend-uzu/src/encodable_block/prediction_head.rs
+++ b/crates/backend-uzu/src/encodable_block/prediction_head.rs
@@ -26,7 +26,7 @@ pub enum PredictionHeadError<B: Backend> {
}
pub struct PredictionHead<B: Backend> {
- hidden_dim: usize,
+ hidden_dim: u32,
activation: ActivationType,
dense_projection: Box<dyn Linear<B>>,
activation_kernel: <B::Kernels as Kernels>::ActivationKernel,
@@ -36,8 +36,8 @@ pub struct PredictionHead<B: Backend> {
impl<B: Backend> PredictionHead<B> {
pub fn new(
- hidden_dim: usize,
- num_labels: usize,
+ hidden_dim: u32,
+ num_labels: u32,
data_type: DataType,
config: &PredictionHeadConfig,
parameter_tree: &ParameterTree<B>,
@@ -89,19 +89,13 @@ impl<B: Backend> PredictionHead<B> {
pub fn encode(
&self,
input: Allocation<B>,
- batch_dim: usize,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group("prediction head");
let mut hidden = self.dense_projection.encode(input, batch_dim, encoder)?;
- self.activation_kernel.encode(
- None::<&Allocation<B>>,
- &mut hidden,
- self.hidden_dim as u32,
- self.activation,
- encoder,
- );
+ self.activation_kernel.encode(None::<&Allocation<B>>, &mut hidden, self.hidden_dim, self.activation, encoder);
let normalized = self.normalization.encode(&hidden, 0, batch_dim, None, encoder)?;
let logits = self.readout.encode(normalized, batch_dim, encoder)?;
diff --git a/crates/backend-uzu/src/encodable_block/sampling/gumbel.rs b/crates/backend-uzu/src/encodable_block/sampling/gumbel.rs
index ce8d64bdb..710706e81 100644
--- a/crates/backend-uzu/src/encodable_block/sampling/gumbel.rs
+++ b/crates/backend-uzu/src/encodable_block/sampling/gumbel.rs
@@ -53,7 +53,7 @@ fn uniform_float(
/// needs (0,1): u == 1 gives +inf. 24 bits only; the full 32 rounds the top words to 2^32.
#[inline]
fn unit_interval(word: u32) -> f32 {
- ((word >> 8).max(1)) as f32 * (1.0 / 16777216.0)
+ (word >> 8).max(1) as f32 * (1.0 / 16777216.0)
}
pub fn gumbel_float(
diff --git a/crates/backend-uzu/src/encodable_block/sampling/mod.rs b/crates/backend-uzu/src/encodable_block/sampling/mod.rs
index 135f513c0..bd5f99d3c 100644
--- a/crates/backend-uzu/src/encodable_block/sampling/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/sampling/mod.rs
@@ -24,7 +24,7 @@ pub use gumbel::{gumbel_float, revidx};
pub use prng::PRng;
pub struct Sampling<B: Backend> {
- vocab_size: usize,
+ vocab_size: u32,
data_type: DataType,
unified_kernels: Mutex<HashMap<UnifiedSamplingKey, <B::Kernels as Kernels>::UnifiedSamplingKernel>>,
}
@@ -32,7 +32,7 @@ pub struct Sampling<B: Backend> {
impl<B: Backend> Sampling<B> {
pub fn new(
data_type: DataType,
- vocab_size: usize,
+ vocab_size: u32,
) -> Self {
Self {
vocab_size,
@@ -51,12 +51,12 @@ pub enum SamplingMethod {
top_p: Option<f32>,
min_p: Option<f32>,
repetition_penalty: Option<f32>,
- suffix_repetition_length: Option<usize>,
+ suffix_repetition_length: Option<u32>,
},
}
impl SamplingMethod {
- pub fn suffix_repetition_length(&self) -> Option<usize> {
+ pub fn suffix_repetition_length(&self) -> Option<u32> {
match self {
SamplingMethod::Greedy => None,
SamplingMethod::Stochastic {
@@ -89,11 +89,13 @@ impl<B: Backend> Sampling<B> {
token_ids: Option<&Allocation<B>>,
sampling_method: &SamplingMethod,
batch_dim: &BatchTopology,
- sampling_range: Range<usize>,
+ sampling_range: Range<u32>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
encoder.push_debug_group("sampling");
+ let sampling_length = sampling_range.end - sampling_range.start;
+
let (is_stochastic, temperature, top_k, top_p, min_p, repetition_penalty, suffix_repetition_length) =
match sampling_method {
SamplingMethod::Greedy => (false, None, None, None, None, None, None),
@@ -126,7 +128,7 @@ impl<B: Backend> Sampling<B> {
let mut logits_copy = encoder.allocate_scratch(logits.as_buffer_range_ref().range().len())?;
let tensor_copy = <B::Kernels as Kernels>::TensorCopyKernel::new(encoder.context(), self.data_type)?;
- tensor_copy.encode(logits, &mut logits_copy, (self.vocab_size * sampling_range.len()) as u32, encoder);
+ tensor_copy.encode(logits, &mut logits_copy, self.vocab_size * sampling_length, encoder);
let repetition_penalty_kernel =
<B::Kernels as Kernels>::RepetitionPenaltyKernel::new(encoder.context(), self.data_type)?;
@@ -136,10 +138,10 @@ impl<B: Backend> Sampling<B> {
context_ring.expect("context_ring is required for repetition_penalty"),
token_ids.expect("token_ids is required for repetition_penalty"),
repetition_penalty,
- suffix_repetition_length as u32,
- self.vocab_size as u32,
- sampling_range.start as u32,
- sampling_range.len() as u32,
+ suffix_repetition_length,
+ self.vocab_size,
+ sampling_range.start,
+ sampling_length,
encoder,
);
Some(logits_copy)
@@ -171,7 +173,7 @@ impl<B: Backend> Sampling<B> {
};
let mut output =
- encoder.context().create_allocation(sampling_range.len() * size_of::<u32>(), AllocationType::Global)?;
+ encoder.context().create_allocation(sampling_length as usize * size_of::<u32>(), AllocationType::Global)?;
kernel.encode(
logits,
@@ -182,8 +184,8 @@ impl<B: Backend> Sampling<B> {
top_k,
top_p,
min_p,
- self.vocab_size as u32,
- sampling_range.len() as u32,
+ self.vocab_size,
+ sampling_length,
encoder,
);
diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rs
index 75678f97f..d02a9432d 100644
--- a/crates/backend-uzu/src/encodable_block/transformer.rs
+++ b/crates/backend-uzu/src/encodable_block/transformer.rs
@@ -18,12 +18,12 @@ use crate::{
enum TransformerLayerStateType<B: Backend> {
Owned(Box<dyn MixerState<B>>),
- Shared(usize),
+ Shared(u32),
}
pub struct TransformerState<B: Backend> {
layer_states: Box<[TransformerLayerStateType<B>]>,
- context_length: usize,
+ context_length: u32,
}
pub struct TransformerEncodeOutput<B: Backend> {
@@ -32,14 +32,14 @@ pub struct TransformerEncodeOutput<B: Backend> {
}
impl<B: Backend> TransformerState<B> {
- pub fn context_length(&self) -> usize {
+ pub fn context_length(&self) -> u32 {
self.context_length
}
pub fn prepare(
&mut self,
- context_length: usize,
- suffix_length: usize,
+ context_length: u32,
+ suffix_length: u32,
context: &B::Context,
) -> Result<(), B::Error> {
for layer_state in &mut self.layer_states {
@@ -55,7 +55,7 @@ impl<B: Backend> TransformerState<B> {
pub fn encode_accept(
&mut self,
- accepted_indices: &[usize],
+ accepted_indices: &[u32],
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
encoder.push_debug_group("transformer accept");
@@ -68,7 +68,7 @@ impl<B: Backend> TransformerState<B> {
layer_state.encode_accept(accepted_indices, encoder)?;
}
- self.context_length += accepted_indices.len();
+ self.context_length += accepted_indices.len() as u32;
encoder.pop_debug_group();
@@ -90,7 +90,7 @@ pub struct Transformer<B: Backend> {
ropes: Box<[AnyRoPEConfig]>,
layers: Box<[(TransformerLayer<B>, Option<usize>)]>,
output_norm: Normalization<B>,
- model_dim: usize,
+ model_dim: u32,
residual_add: <B::Kernels as Kernels>::TensorAddScaleKernel,
}
@@ -104,11 +104,12 @@ impl<B: Backend> Transformer<B> {
) -> Result<Self, TransformerNewError<B>> {
let mut ropes: Vec<AnyRoPEConfig> = Vec::new();
+ let num_layers = transformer_config.layer_configs.len() as u32;
let layers = transformer_config
.layer_configs
.iter()
- .enumerate()
- .map(|(layer_index, layer_config)| {
+ .zip(0u32..)
+ .map(|(layer_config, layer_index)| {
let layer_loader = parameter_tree.subtree(&format!("layers.{}", layer_index));
let rope = layer_config.rope_config.as_ref().map(|layer_rope_config| {
@@ -122,7 +123,7 @@ impl<B: Backend> Transformer<B> {
context,
transformer_config.model_dim,
transformer_config.hidden_dim,
- transformer_config.layer_configs.len(),
+ num_layers,
layer_config,
layer_index,
&layer_loader,
@@ -160,11 +161,11 @@ impl<B: Backend> Transformer<B> {
&self,
shortcut: &Allocation<B>,
hidden: &Allocation<B>,
- batch_size: usize,
+ batch_size: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
let mut output = encoder.allocate_scratch(hidden.size())?;
- let elements = (batch_size * self.model_dim) as u32;
+ let elements = batch_size * self.model_dim;
self.residual_add.encode(Some(shortcut), hidden, &mut output, elements, elements, 1.0, encoder);
Ok(output)
}
@@ -173,10 +174,10 @@ impl<B: Backend> Transformer<B> {
self.layers.iter().all(|(layer, _rope)| layer.mixer.speculation_supported())
}
- pub fn max_context_length(&self) -> Option<usize> {
+ pub fn max_context_length(&self) -> Option<u32> {
self.layers.iter().map(|(layer, _rope_index)| layer.mixer.max_context_length()).fold(None, |acc, el| {
match (acc, el) {
- (Some(a), Some(b)) => Some(usize::min(a, b)),
+ (Some(a), Some(b)) => Some(u32::min(a, b)),
(Some(x), None) | (None, Some(x)) => Some(x),
(None, None) => None,
}
@@ -200,7 +201,7 @@ impl<B: Backend> Transformer<B> {
pub fn create_empty_state(
&self,
- max_context_length: Option<usize>,
+ max_context_length: Option<u32>,
context: &B::Context,
) -> Result<TransformerState<B>, B::Error> {
let layer_states = self
@@ -227,8 +228,8 @@ impl<B: Backend> Transformer<B> {
input: Allocation<B>,
per_layer_inputs: Option<&Allocation<B>>,
batch_dim: &BatchTopology,
- output_range: Option<Range<usize>>,
- hidden_feature_layer_indices: Option<&[usize]>,
+ output_range: Option<Range<u32>>,
+ hidden_feature_layer_indices: Option<&[u32]>,
mut state: Option<&mut TransformerState<B>>,
encoder: &mut Encoder<B>,
) -> Result<TransformerEncodeOutput<B>, B::Error> {
@@ -244,8 +245,7 @@ impl<B: Backend> Transformer<B> {
hidden_feature_layer_indices.map(|indices| (0..indices.len()).map(|_| None).collect::<Vec<_>>());
let context_length = state.as_ref().map(|state| state.context_length).unwrap_or(0);
- let token_positions =
- batch_dim.heights().map(|rel_pos| context_length + rel_pos as usize).collect::<Box<[usize]>>();
+ let token_positions = batch_dim.heights().map(|rel_pos| context_length + rel_pos).collect::<Box<[u32]>>();
let precalculated_ropes = self
.ropes
@@ -257,10 +257,11 @@ impl<B: Backend> Transformer<B> {
let precalculated_rope = layer_rope_index.map(|i| &precalculated_ropes[i]);
let layer_state = if let Some(state) = &mut state {
- Some(match &mut state.layer_states[layer.layer_index] {
+ Some(match &mut state.layer_states[layer.layer_index as usize] {
TransformerLayerStateType::Owned(layer_state) => MaybeMut::Mut(layer_state.as_mut()),
TransformerLayerStateType::Shared(owned_layer_index) => {
- let TransformerLayerStateType::Owned(owned_layer) = &state.layer_states[*owned_layer_index]
+ let TransformerLayerStateType::Owned(owned_layer) =
+ &state.layer_states[*owned_layer_index as usize]
else {
panic!("shared layer doesn't point to an owned layer");
};
@@ -313,8 +314,13 @@ impl<B: Backend> Transformer<B> {
});
};
- let output_normalized =
- self.output_norm.encode(&hidden, output_range.start, output_range.len(), Some(&mut shortcut), encoder)?;
+ let output_normalized = self.output_norm.encode(
+ &hidden,
+ output_range.start,
+ output_range.end - output_range.start,
+ Some(&mut shortcut),
+ encoder,
+ )?;
Ok(TransformerEncodeOutput {
output: Some(output_normalized),
diff --git a/crates/backend-uzu/src/encodable_block/transformer_layer.rs b/crates/backend-uzu/src/encodable_block/transformer_layer.rs
index 82f55e4d6..180c87113 100644
--- a/crates/backend-uzu/src/encodable_block/transformer_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/transformer_layer.rs
@@ -27,7 +27,7 @@ pub enum TransformerLayerError<B: Backend> {
Normalization(#[from] NormalizationNewError<B>),
#[error("Layer {layer_index} sets post_layer_scalar but has no post_mlp_norm")]
PostLayerScalarWithoutPostMlpNorm {
- layer_index: usize,
+ layer_index: u32,
},
#[error("Transformer layers except the first one if it doesn't have rht in mixer require pre_mixer_norm_config")]
MissingPreMixerNormConfig,
@@ -36,9 +36,9 @@ pub enum TransformerLayerError<B: Backend> {
// TODO: saner shortcut
pub struct TransformerLayer<B: Backend> {
- pub layer_index: usize,
+ pub layer_index: u32,
pub pre_mixer_norm: Option<Normalization<B>>,
- pub kv_source_layer_index: Option<usize>,
+ pub kv_source_layer_index: Option<u32>,
pub mixer: Box<dyn Mixer<B>>,
pub post_mixer_norm: Option<Normalization<B>>,
pub pre_mlp_norm: Normalization<B>,
@@ -50,11 +50,11 @@ pub struct TransformerLayer<B: Backend> {
impl<B: Backend> TransformerLayer<B> {
pub fn new(
context: &B::Context,
- model_dim: usize,
- hidden_dim: usize,
- num_layers: usize,
+ model_dim: u32,
+ hidden_dim: u32,
+ num_layers: u32,
layer_config: &TransformerLayerConfig,
- layer_index: usize,
+ layer_index: u32,
parameter_tree: &ParameterTree<B>,
data_type: DataType,
) -> Result<Self, TransformerLayerError<B>> {
diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rs
index 76f519b6b..e347f15b9 100644
--- a/crates/backend-uzu/src/encodable_block/weaver.rs
+++ b/crates/backend-uzu/src/encodable_block/weaver.rs
@@ -30,10 +30,10 @@ pub(crate) const DATA_TYPE: DataType = DataType::BF16;
pub(crate) const ROPE_DATA_TYPE: DataType = DataType::F32;
pub struct WeaverTreeShape {
- pub budget: usize,
- pub depth: usize,
- pub expand_per_round: usize,
- pub expand_width: usize,
+ pub budget: u32,
+ pub depth: u32,
+ pub expand_per_round: u32,
+ pub expand_width: u32,
}
pub struct EncodedWeaverTree<B: Backend> {
@@ -42,7 +42,7 @@ pub struct EncodedWeaverTree<B: Backend> {
pub struct ProposalNode {
pub token_id: u32,
- pub depth: usize,
+ pub depth: u32,
pub child_indices: Vec<usize>,
}
@@ -71,7 +71,7 @@ impl<B: Backend> EncodedWeaverTree<B> {
}
nodes.push(ProposalNode {
token_id: field(TreeIdx::TokenId, slot),
- depth: field(TreeIdx::Depth, slot) as usize,
+ depth: field(TreeIdx::Depth, slot),
child_indices: Vec::new(),
});
}
@@ -92,10 +92,10 @@ pub struct Weaver<B: Backend> {
top_children: <B::Kernels as Kernels>::WeaverTopChildrenKernel,
frontier_select: <B::Kernels as Kernels>::WeaverFrontierSelectKernel,
frontier_insert_children: <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel,
- model_dim: usize,
- target_model_dim: usize,
- max_depth: usize,
- candidate_pool_size: usize,
+ model_dim: u32,
+ target_model_dim: u32,
+ max_depth: u32,
+ candidate_pool_size: u32,
}
#[derive(Debug, Error)]
@@ -113,16 +113,16 @@ pub enum WeaverNewError<B: Backend> {
#[error("model_dim must be divisible by num_heads")]
InvalidHeadConfig,
#[error("candidate_pool_size must be in 1..={max}, got {0}", max = CANDIDATES_MAX)]
- InvalidCandidatePoolSize(usize),
+ InvalidCandidatePoolSize(u32),
#[error("rope head_dim {actual} does not match model_dim / num_heads = {expected}")]
InvalidRopeHeadDim {
- expected: usize,
- actual: usize,
+ expected: u32,
+ actual: u32,
},
#[error("rope max_sequence_length {actual} is too small for max_depth {max_depth} (needs {max_depth} + 1)")]
InvalidRopeLength {
- max_depth: usize,
- actual: usize,
+ max_depth: u32,
+ actual: u32,
},
}
@@ -251,7 +251,7 @@ impl<B: Backend> Weaver<B> {
})
}
- pub fn max_depth(&self) -> usize {
+ pub fn max_depth(&self) -> u32 {
self.max_depth
}
@@ -269,7 +269,6 @@ impl<B: Backend> Weaver<B> {
encoder.push_debug_group("weaver tree");
let tree_slot_count = shape.budget;
- let frontier_capacity = tree_slot_count * shape.expand_width;
let ancestor_stride = self.max_depth;
if shape.budget == 0
|| shape.depth < 2
@@ -278,11 +277,12 @@ impl<B: Backend> Weaver<B> {
|| shape.expand_per_round > FRONTIER_MAX_WIDTH
|| shape.expand_width == 0
|| shape.expand_width > self.candidate_pool_size
- || frontier_capacity > FRONTIER_MAX_SLOTS
- || depth_seeds.len() != self.max_depth
+ || tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width
+ || depth_seeds.len() as u32 != self.max_depth
{
return Err(WeaverEncodeError::InvalidTreeInput);
}
+ let frontier_capacity = tree_slot_count * shape.expand_width;
// `depth` counts the root: nodes expand up to `depth - 1` edges from it.
let lookahead_count = shape.depth - 1;
@@ -290,7 +290,7 @@ impl<B: Backend> Weaver<B> {
// lookahead row form the candidate pool node expansions draw from.
let vocab_size = target_embedding.vocab_size();
assert!(
- logits.size() >= lookahead_count * vocab_size * DataType::F32.size_in_bytes(),
+ logits.size() >= size_for_shape(&[lookahead_count, vocab_size], DataType::F32),
"draft logits do not cover the lookahead rows"
);
let mut candidate_ids = encoder
@@ -304,8 +304,8 @@ impl<B: Backend> Weaver<B> {
logits,
&mut candidate_ids,
&mut candidate_logits,
- lookahead_count as u32,
- self.candidate_pool_size as u32,
+ lookahead_count,
+ self.candidate_pool_size,
encoder,
)
.map_err(WeaverEncodeError::Backend)?;
@@ -316,16 +316,16 @@ impl<B: Backend> Weaver<B> {
// Prefix pass: run the target token and one draft row per non-root
// chain node through every layer, collecting per-layer KV caches.
- let hidden_row_bytes = self.target_model_dim * DATA_TYPE.size_in_bytes();
+ let hidden_row_bytes = size_for_shape(&[self.target_model_dim], DATA_TYPE);
let mut prefix_hidden = encoder
.allocate_scratch(size_for_shape(&[shape.depth, self.target_model_dim], DATA_TYPE))
.map_err(WeaverEncodeError::Backend)?;
encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_hidden, 0..hidden_row_bytes);
encoder.encode_copy(
draft_hidden,
- hidden_row_bytes..shape.depth * hidden_row_bytes,
+ hidden_row_bytes..shape.depth as usize * hidden_row_bytes,
&mut prefix_hidden,
- hidden_row_bytes..shape.depth * hidden_row_bytes,
+ hidden_row_bytes..shape.depth as usize * hidden_row_bytes,
);
let normalized_prefix = self
.hidden_state_norm
@@ -383,40 +383,43 @@ impl<B: Backend> Weaver<B> {
.collect::<Result<Vec<_>, _>>()
.map_err(WeaverEncodeError::Backend)?;
- let mut tree_init = vec![0u32; TreeIdx::COUNT * tree_slot_count];
- for slot in 0..tree_slot_count {
- tree_init[TreeIdx::ParentSlot as usize * tree_slot_count + slot] = FRONTIER_NO_WINNER;
+ let tree_slots = tree_slot_count as usize;
+ let round_nodes = shape.expand_per_round as usize;
+
+ let mut tree_init = vec![0u32; TreeIdx::COUNT * tree_slots];
+ for slot in 0..tree_slots {
+ tree_init[TreeIdx::ParentSlot as usize * tree_slots + slot] = FRONTIER_NO_WINNER;
}
- tree_init[TreeIdx::TokenId as usize * tree_slot_count] = root_token_id;
- tree_init[TreeIdx::Valid as usize * tree_slot_count] = 1;
+ tree_init[TreeIdx::TokenId as usize * tree_slots] = root_token_id;
+ tree_init[TreeIdx::Valid as usize * tree_slots] = 1;
let mut packed_tree = encoder.allocate_constant_from_slice(&tree_init).map_err(WeaverEncodeError::Backend)?;
let mut frontier = encoder
- .allocate_constant_from_slice(&vec![0u32; FrontierIdx::COUNT * frontier_capacity])
+ .allocate_constant_from_slice(&vec![0u32; FrontierIdx::COUNT * frontier_capacity as usize])
.map_err(WeaverEncodeError::Backend)?;
let mut slot_ancestors = encoder
- .allocate_constant_from_slice(&vec![0u32; tree_slot_count * ancestor_stride])
+ .allocate_constant_from_slice(&vec![0u32; tree_slots * ancestor_stride as usize])
.map_err(WeaverEncodeError::Backend)?;
- let mut initial_node_token_ids = vec![0u32; shape.expand_per_round];
+ let mut initial_node_token_ids = vec![0u32; round_nodes];
initial_node_token_ids[0] = root_token_id;
- let mut initial_node_valid = vec![0u32; shape.expand_per_round];
+ let mut initial_node_valid = vec![0u32; round_nodes];
initial_node_valid[0] = 1;
let mut node_token_ids =
encoder.allocate_constant_from_slice(&initial_node_token_ids).map_err(WeaverEncodeError::Backend)?;
let mut node_metadata = encoder
- .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * shape.expand_per_round])
+ .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * round_nodes])
.map_err(WeaverEncodeError::Backend)?;
let mut node_ancestor_indices = encoder
- .allocate_constant_from_slice(&vec![0u32; shape.expand_per_round * ancestor_stride])
+ .allocate_constant_from_slice(&vec![0u32; round_nodes * ancestor_stride as usize])
.map_err(WeaverEncodeError::Backend)?;
let mut node_valid =
encoder.allocate_constant_from_slice(&initial_node_valid).map_err(WeaverEncodeError::Backend)?;
let mut node_candidate_ids = encoder
- .allocate_constant_from_slice(&vec![0u32; shape.expand_per_round * self.candidate_pool_size])
+ .allocate_constant_from_slice(&vec![0u32; (shape.expand_per_round * self.candidate_pool_size) as usize])
.map_err(WeaverEncodeError::Backend)?;
let mut node_candidate_logits = encoder
- .allocate_constant_from_slice(&vec![0.0f32; shape.expand_per_round * self.candidate_pool_size])
+ .allocate_constant_from_slice(&vec![0.0f32; (shape.expand_per_round * self.candidate_pool_size) as usize])
.map_err(WeaverEncodeError::Backend)?;
let depth_seeds_buffer =
encoder.allocate_constant_from_slice(depth_seeds).map_err(WeaverEncodeError::Backend)?;
@@ -441,15 +444,15 @@ impl<B: Backend> Weaver<B> {
&candidate_logits,
&mut node_candidate_ids,
&mut node_candidate_logits,
- frontier_capacity as u32,
- tree_slot_count as u32,
- batch_node_count as u32,
- batch_start_slot as u32,
- ancestor_stride as u32,
- self.max_depth as u32,
- lookahead_count as u32,
- lookahead_count as u32,
- self.candidate_pool_size as u32,
+ frontier_capacity,
+ tree_slot_count,
+ batch_node_count,
+ batch_start_slot,
+ ancestor_stride,
+ self.max_depth,
+ lookahead_count,
+ lookahead_count,
+ self.candidate_pool_size,
encoder,
);
}
@@ -475,7 +478,7 @@ impl<B: Backend> Weaver<B> {
.map_err(WeaverEncodeError::Backend)?;
let mut residual_state =
encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;
- let metadata_field_bytes = batch_node_count * DataType::U32.size_in_bytes();
+ let metadata_field_bytes = size_for_shape(&[batch_node_count], DataType::U32);
for (layer_index, layer) in self.layers.iter().enumerate() {
let attention_input = layer
.pre_attention_norm
@@ -499,11 +502,11 @@ impl<B: Backend> Weaver<B> {
(&node_metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes),
(&node_metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes),
&mut attention_output,
- batch_node_count as u32,
- shape.depth as u32,
- ancestor_stride as u32,
- tree_slot_count as u32,
- layer.max_depth as u32,
+ batch_node_count,
+ shape.depth,
+ ancestor_stride,
+ tree_slot_count,
+ layer.max_depth,
layer.attention_scale,
encoder,
);
@@ -541,10 +544,10 @@ impl<B: Backend> Weaver<B> {
&node_metadata,
&mut child_token_ids,
&mut child_logprobs,
- batch_node_count as u32,
- self.candidate_pool_size as u32,
- shape.expand_width as u32,
- target_embedding.vocab_size() as u32,
+ batch_node_count,
+ self.candidate_pool_size,
+ shape.expand_width,
+ target_embedding.vocab_size(),
encoder,
);
@@ -555,10 +558,10 @@ impl<B: Backend> Weaver<B> {
&child_token_ids,
&child_logprobs,
&mut frontier,
- frontier_capacity as u32,
- tree_slot_count as u32,
- batch_node_count as u32,
- shape.expand_width as u32,
+ frontier_capacity,
+ tree_slot_count,
+ batch_node_count,
+ shape.expand_width,
encoder,
);
batch_start_slot += batch_node_count;
diff --git a/crates/backend-uzu/src/encodable_block/weaver_layer.rs b/crates/backend-uzu/src/encodable_block/weaver_layer.rs
index 06fe68f1c..139a57834 100644
--- a/crates/backend-uzu/src/encodable_block/weaver_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/weaver_layer.rs
@@ -42,10 +42,10 @@ pub struct WeaverLayer<B: Backend> {
// Geometry
pub attention_scale: f32,
- pub model_dim: usize,
- pub num_heads: usize,
- pub head_dim: usize,
- pub max_depth: usize,
+ pub model_dim: u32,
+ pub num_heads: u32,
+ pub head_dim: u32,
+ pub max_depth: u32,
}
impl<B: Backend> WeaverLayer<B> {
@@ -134,9 +134,8 @@ impl<B: Backend> WeaverLayer<B> {
let (mlp, up_input_hadamard_factors) =
<dyn Mlp<B>>::new(&mlp_config, model_dim, hidden_dim, context, &parameter_tree.subtree("mlp"), DATA_TYPE)?;
assert!(up_input_hadamard_factors.is_none(), "Weaver MLP does not support input Hadamard factors");
- let ancestor_attention =
- <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim as u32, num_heads as u32)
- .map_err(WeaverNewError::Backend)?;
+ let ancestor_attention = <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim, num_heads)
+ .map_err(WeaverNewError::Backend)?;
Ok(Self {
qkv_projection,
out_projection,
@@ -159,16 +158,16 @@ impl<B: Backend> WeaverLayer<B> {
residual_input: &Allocation<B>,
residual_state: &mut Allocation<B>,
rope: &PrecalculatedRoPE<B>,
- token_count: usize,
+ token_count: u32,
encoder: &mut Encoder<B>,
) -> Result<PreparedPrefixAttention<B>, B::Error> {
let attention_input =
self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), encoder)?;
let qkv = self.qkv_projection.encode(attention_input, token_count, encoder)?;
let mut queries =
- encoder.allocate_scratch(size_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE))?;
+ encoder.allocate_scratch_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE)?;
let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);
- let mut kv_cache = encoder.allocate_scratch(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE))?;
+ let mut kv_cache = encoder.allocate_scratch_for_shape(&[2, token_count, self.model_dim], DATA_TYPE)?;
let (keys, values) = kv_cache.as_buffer_range_mut().split_at(kv_plane_bytes);
self.attention_prepare.encode(
&qkv,
@@ -177,12 +176,12 @@ impl<B: Backend> WeaverLayer<B> {
Some(values),
Some(&rope.cosines),
Some(&rope.sines),
- self.num_heads as u32,
- Some(self.num_heads as u32),
- self.head_dim as u32,
- Some(rope.dim as u32),
+ self.num_heads,
+ Some(self.num_heads),
+ self.head_dim,
+ Some(rope.dim),
Some(0),
- token_count as u32,
+ token_count,
encoder,
);
Ok(PreparedPrefixAttention {
@@ -195,7 +194,7 @@ impl<B: Backend> WeaverLayer<B> {
&self,
attention_output: Allocation<B>,
residual_state: &mut Allocation<B>,
- token_count: usize,
+ token_count: u32,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
let projected_attention = self.out_projection.encode(attention_output, token_count, encoder)?;
diff --git a/crates/backend-uzu/src/encodable_block/weight_matrix.rs b/crates/backend-uzu/src/encodable_block/weight_matrix.rs
index cfe153fcc..06c435434 100644
--- a/crates/backend-uzu/src/encodable_block/weight_matrix.rs
+++ b/crates/backend-uzu/src/encodable_block/weight_matrix.rs
@@ -69,7 +69,7 @@ pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeight
Some(QuantizationInfo {
mode,
method,
- group_size: group_size as u32,
+ group_size,
})
},
};
@@ -102,8 +102,8 @@ impl<B: Backend> WeightMatrix<B> {
tree: &ParameterTree<B>,
spec: AnyWeightMatrixSpec,
required_layout: Layout,
- output_dim: usize,
- input_dim: usize,
+ output_dim: u32,
+ input_dim: u32,
data_type: DataType,
) -> Result<Self, WeightMatrixError<B>> {
let ParsedWeightSpec {
@@ -125,7 +125,7 @@ impl<B: Backend> WeightMatrix<B> {
});
};
- let group_size = info.group_size as usize;
+ let group_size = info.group_size;
let packing_divisor = info.mode.packing_divisor();
let storage_data_type = info.mode.storage_type();
if !columns.is_multiple_of(packing_divisor) {
@@ -243,9 +243,9 @@ impl<B: Backend> WeightMatrix<B> {
fn physical_shape(
layout: &Layout,
- output_dim: usize,
- input_dim: usize,
-) -> (usize, usize) {
+ output_dim: u32,
+ input_dim: u32,
+) -> (u32, u32) {
match layout {
Layout::OutputInput => (output_dim, input_dim),
Layout::InputOutput => (input_dim, output_dim),
diff --git a/crates/backend-uzu/src/engine/classifier_model.rs b/crates/backend-uzu/src/engine/classifier_model.rs
index 61bc00a55..324c8ecc5 100644
--- a/crates/backend-uzu/src/engine/classifier_model.rs
+++ b/crates/backend-uzu/src/engine/classifier_model.rs
@@ -64,7 +64,7 @@ impl<B: Backend> Engine<B> {
)?;
let output_labels = if let Some(output_labels) = config.classifier_config.output_labels {
- assert!(output_labels.len() == config.classifier_config.num_labels);
+ assert!(output_labels.len() as u32 == config.classifier_config.num_labels);
output_labels
} else {
(0..config.classifier_config.num_labels).map(|index| format!("class_{index}")).collect()
@@ -102,7 +102,11 @@ impl<B: Backend> ClassifierModel<B> {
return Err(ClassifierModelClassifyError::EmptyInput);
}
- if self.classifier.max_context_length().is_some_and(|max_context_length| input.len() > max_context_length) {
+ if self
+ .classifier
+ .max_context_length()
+ .is_some_and(|max_context_length| input.len() > max_context_length as usize)
+ {
return Err(ClassifierModelClassifyError::ContextOverflow);
}
@@ -113,7 +117,7 @@ impl<B: Backend> ClassifierModel<B> {
.map_err(ClassifierModelClassifyError::Backend)?;
token_ids.copyin(&input.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>());
- let logits = self.classifier.encode(&token_ids, input.len(), &mut encoder)?;
+ let logits = self.classifier.encode(&token_ids, input.len() as u32, &mut encoder)?;
let mut output_buffer = self
.context
diff --git a/crates/backend-uzu/src/engine/language_model/mod.rs b/crates/backend-uzu/src/engine/language_model/mod.rs
index 0ab2a0e40..6148e2379 100644
--- a/crates/backend-uzu/src/engine/language_model/mod.rs
+++ b/crates/backend-uzu/src/engine/language_model/mod.rs
@@ -117,7 +117,7 @@ impl<B: Backend> Engine<B> {
let generation_config = config.generation_config;
#[cfg(grammar)]
- let vocab_size = config.decoder_config.vocab_size;
+ let vocab_size = config.decoder_config.vocab_size as usize;
Ok(LanguageModel {
engine: self.clone(),
@@ -133,11 +133,11 @@ impl<B: Backend> Engine<B> {
}
impl<B: Backend> LanguageModel<B> {
- pub fn max_context_length(&self) -> Option<usize> {
+ pub fn max_context_length(&self) -> Option<u32> {
self.decoder.max_context_length()
}
- pub fn recommended_context_length(&self) -> Option<usize> {
+ pub fn recommended_context_length(&self) -> Option<u32> {
let max_context_length = self.max_context_length();
// TODO: This is not the correct way to do it, there should be a real memory model
@@ -157,7 +157,7 @@ impl<B: Backend> LanguageModel<B> {
16384
};
- Some(usize::min(max_context_length, platform_recommended_context_length))
+ Some(u32::min(max_context_length, platform_recommended_context_length))
} else {
// We just assume that unlimited context means constant state size on all mixers and is thus free
None
diff --git a/crates/backend-uzu/src/engine/language_model/state.rs b/crates/backend-uzu/src/engine/language_model/state.rs
index 86f1c545a..189b43f6e 100644
--- a/crates/backend-uzu/src/engine/language_model/state.rs
+++ b/crates/backend-uzu/src/engine/language_model/state.rs
@@ -12,7 +12,7 @@ pub struct LanguageModelState<B: Backend> {
pub(super) prng: PRng,
pub(super) transformer_state: TransformerState<B>,
pub(super) speculator_state: Option<DFlashState<B>>,
- pub(super) max_context_length: Option<usize>,
+ pub(super) max_context_length: Option<u32>,
}
impl<B: Backend> LanguageModelState<B> {
@@ -30,7 +30,7 @@ pub enum LanguageModelCreateEmptyStateError<B: Backend> {
impl<B: Backend> LanguageModel<B> {
pub fn create_empty_state(
&self,
- max_context_length: Option<usize>,
+ max_context_length: Option<u32>,
) -> Result<LanguageModelState<B>, LanguageModelCreateEmptyStateError<B>> {
let tokens = Vec::new();
let last_output_token = None;
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
index 355c530e7..2bad7060b 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
@@ -9,6 +9,7 @@ use shoji::traits::backend::chat_token::TokenStreamMetrics;
#[cfg(grammar)]
use crate::engine::language_model::grammar::Grammar;
use crate::{
+ array::size_for_shape,
backends::common::{
Allocation, AllocationPool, AllocationType, Backend, Context, Encoder, Pending,
gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,
@@ -141,7 +142,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if model_state
.max_context_length
- .is_some_and(|max_context_length| model_state.tokens.len() + input.len() > max_context_length)
+ .is_some_and(|max_context_length| model_state.tokens.len() + input.len() > max_context_length as usize)
{
return Err(LanguageModelStreamError::ContextOverflow);
}
@@ -162,19 +163,19 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.engine
.context
.create_allocation(
- (2 + suffix_repetition_length) * DataType::U32.size_in_bytes(),
+ size_for_shape(&[2 + suffix_repetition_length], DataType::U32),
AllocationType::Global,
)
.map_err(LanguageModelStreamError::Backend)?;
- let state_tokens_range =
- model_state.tokens.len().saturating_sub(suffix_repetition_length)..model_state.tokens.len();
+ let state_tokens_range = model_state.tokens.len().saturating_sub(suffix_repetition_length as usize)
+ ..model_state.tokens.len();
context_ring.copyin(
&once(0) // offset
.chain(once(state_tokens_range.len() as u64)) // length
.chain(model_state.tokens[state_tokens_range.clone()].iter().copied()) // tokens
- .chain(repeat_n(0, suffix_repetition_length - state_tokens_range.len())) // pad if not full
+ .chain(repeat_n(0, suffix_repetition_length as usize - state_tokens_range.len())) // pad if not full
.map(|x| x as u32)
.collect::<Box<[_]>>(),
);
@@ -196,8 +197,8 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
model_state
.transformer_state
.prepare(
- model_state.transformer_state.context_length() + (number_of_batches - 1) * max_batch_size,
- usize::min(max_batch_size, input.len()),
+ model_state.transformer_state.context_length() + ((number_of_batches - 1) * max_batch_size) as u32,
+ usize::min(max_batch_size, input.len()) as u32,
&model.engine.context,
)
.map_err(LanguageModelStreamError::Backend)?;
@@ -234,7 +235,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let decoder_output = model.decoder.encode(
&token_ids,
&batch_dim,
- sample_last.then(|| (input_chunk.len() - 1)..input_chunk.len()),
+ sample_last.then(|| input_chunk.len() as u32 - 1..input_chunk.len() as u32),
hidden_feature_layer_indices,
&mut model_state.transformer_state,
&mut encoder,
@@ -276,6 +277,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let bitmask = None;
output_norm = decoder_output.final_hidden;
+ let sampled_row = batch_dim.size() - 1;
output_tokens = Some(
model
.sampling
@@ -287,7 +289,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
Some(&token_ids),
&options.sampling_method,
&batch_dim,
- (batch_dim.size() - 1)..batch_dim.size(),
+ sampled_row..sampled_row + 1,
&mut encoder,
)
.map_err(LanguageModelStreamError::Backend)?,
@@ -296,7 +298,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
model_state
.transformer_state
- .encode_accept(&(0..input_chunk.len()).collect::<Box<[usize]>>(), &mut encoder)
+ .encode_accept(&(0..input_chunk.len() as u32).collect::<Box<[u32]>>(), &mut encoder)
.map_err(LanguageModelStreamError::Backend)?;
if let Some(speculator) = model.speculator.as_ref() {
@@ -305,7 +307,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.encode_accept(
speculator_state,
decoder_output.hidden_features.as_ref().unwrap(),
- &(0..input_chunk.len()).collect::<Box<[usize]>>(),
+ &(0..input_chunk.len() as u32).collect::<Box<[u32]>>(),
&mut encoder,
)
.map_err(LanguageModelStreamError::Backend)?;
@@ -315,7 +317,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
model.context_ring_update.encode(
&token_ids,
context_ring.as_mut().unwrap(),
- suffix_repetition_length as u32,
+ suffix_repetition_length,
input_chunk.len() as u32,
&mut encoder,
);
@@ -434,7 +436,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
};
return Ok(Some(output_token_id));
} else {
- let accepted_token_indicies = full.iter().map(|(i, _, _)| *i).collect::<Box<[usize]>>();
+ let accepted_token_indicies = full.iter().map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();
let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();
let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();
let mut encoder = Encoder::<B>::new_with_pool_name(
@@ -481,7 +483,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.model.context_ring_update.encode(
&accepted_input_token_ids_const,
self.context_ring.as_mut().unwrap(),
- suffix_repetition_length as u32,
+ suffix_repetition_length,
full.len() as u32,
&mut encoder,
);
@@ -722,7 +724,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if full_accept {
self.model_state
.transformer_state
- .encode_accept(&(0..batch_dim.size()).collect::<Box<[usize]>>(), &mut encoder)
+ .encode_accept(&(0..batch_dim.size()).collect::<Box<[u32]>>(), &mut encoder)
.map_err(LanguageModelStreamError::Backend)?;
if let Some(speculator) = self.model.speculator.as_ref() {
@@ -731,7 +733,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.encode_accept(
speculator_state,
decoder_output.hidden_features.as_ref().unwrap(),
- &(0..batch_dim.size()).collect::<Box<[usize]>>(),
+ &(0..batch_dim.size()).collect::<Box<[u32]>>(),
&mut encoder,
)
.map_err(LanguageModelStreamError::Backend)?;
@@ -741,8 +743,8 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.model.context_ring_update.encode(
&token_ids,
self.context_ring.as_mut().unwrap(),
- suffix_repetition_length as u32,
- batch_dim.size() as u32,
+ suffix_repetition_length,
+ batch_dim.size(),
&mut encoder,
);
}
@@ -846,7 +848,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
)
.unwrap();
let accepted_token_indicies =
- full.iter().take(num_accepted + 1).map(|(i, _, _)| *i).collect::<Box<[usize]>>();
+ full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();
self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut encoder).unwrap();
if let Some(speculator) = self.model.speculator.as_ref() {
speculator
diff --git a/crates/backend-uzu/src/parameters/loader.rs b/crates/backend-uzu/src/parameters/loader.rs
index 9fae3278f..31e7e564e 100644
--- a/crates/backend-uzu/src/parameters/loader.rs
+++ b/crates/backend-uzu/src/parameters/loader.rs
@@ -17,7 +17,7 @@ use crate::{
};
pub struct ParameterMetadata {
- shape: Box<[usize]>,
+ shape: Box<[u32]>,
data_type: DataType,
offset: usize,
size: usize,
@@ -35,14 +35,14 @@ pub enum ParameterLoaderError<B: Backend> {
MetadataDeserializationError(#[from] serde_json::Error),
#[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]
InvalidTensor {
- shape: Box<[usize]>,
+ shape: Box<[u32]>,
data_type: DataType,
- expected_shape: Box<[usize]>,
+ expected_shape: Box<[u32]>,
expected_data_type: DataType,
},
#[error("Invalid tensor byte size: got {size} bytes for {shape:?} @ {data_type:?}, expected {expected_size} bytes")]
InvalidTensorSize {
- shape: Box<[usize]>,
+ shape: Box<[u32]>,
data_type: DataType,
size: usize,
expected_size: usize,
@@ -142,7 +142,7 @@ pub struct ParameterLeaf<'a, 'leaf, B: Backend, const VALIDATED: bool> {
impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, false> {
pub fn validate(
self,
- expected_shape: &[usize],
+ expected_shape: &[u32],
expected_data_type: DataType,
) -> Result<ParameterLeaf<'a, 'leaf, B, true>, ParameterLoaderError<B>> {
let shape = self.metadata.shape.as_ref();
diff --git a/crates/backend-uzu/src/parameters/safetensors_metadata.rs b/crates/backend-uzu/src/parameters/safetensors_metadata.rs
index 15782bd50..205e5458a 100644
--- a/crates/backend-uzu/src/parameters/safetensors_metadata.rs
+++ b/crates/backend-uzu/src/parameters/safetensors_metadata.rs
@@ -49,7 +49,7 @@ pub struct HashMetadata {
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct TensorInfo {
pub dtype: Dtype,
- pub shape: Vec<usize>,
+ pub shape: Vec<u32>,
pub data_offsets: (usize, usize),
}
diff --git a/crates/backend-uzu/src/speculators/dflash_tfm.rs b/crates/backend-uzu/src/speculators/dflash_tfm.rs
index 6189534a4..1d9a0ae92 100644
--- a/crates/backend-uzu/src/speculators/dflash_tfm.rs
+++ b/crates/backend-uzu/src/speculators/dflash_tfm.rs
@@ -56,14 +56,14 @@ pub enum DFlashSpeculatorLoadError<B: Backend> {
pub enum DFlashTfmTreeConstructionMethod {
Argmax,
Weaver {
- depth: usize,
- expand_per_round: usize,
- expand_width: usize,
+ depth: u32,
+ expand_per_round: u32,
+ expand_width: u32,
},
}
pub struct DFlashTfmTreeShape {
- pub budget: usize,
+ pub budget: u32,
pub construction_method: DFlashTfmTreeConstructionMethod,
}
@@ -98,7 +98,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
Weaver::new(
&*context,
weaver_config,
- config.draft_config.vocab_size as u32,
+ config.draft_config.vocab_size,
&speculator_tree.subtree("weaver"),
)
})
@@ -121,13 +121,13 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
self.weaver.is_some()
}
- pub fn hidden_feature_layer_indices(&self) -> &[usize] {
+ pub fn hidden_feature_layer_indices(&self) -> &[u32] {
&self.config.draft_config.target_layer_ids
}
pub fn empty_state(
&self,
- context_capacity: usize,
+ context_capacity: u32,
) -> Result<DFlashState<B>, B::Error> {
self.dflash.empty_state(context_capacity, &self.context)
}
@@ -136,7 +136,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
&self,
state: &mut DFlashState<B>,
target_features: &[Allocation<B>],
- accepted_indices: &[usize],
+ accepted_indices: &[u32],
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
self.dflash.encode_accept(state, target_features, accepted_indices, encoder)
@@ -171,7 +171,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
)));
}
let chain_length = shape.budget - 1;
- let mut nodes = Vec::with_capacity(shape.budget);
+ let mut nodes = Vec::with_capacity(shape.budget as usize);
nodes.push(ProposalNode {
token_id: target_output_token,
depth: 0,
@@ -186,9 +186,9 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
)?;
let topology_nodes = (0..chain_length)
.map(|index| GpuTrieNode {
- trie_start: index as u32,
- trie_end: (chain_length - 1) as u32,
- height: index as u32,
+ trie_start: index,
+ trie_end: chain_length - 1,
+ height: index,
})
.collect::<Box<[_]>>();
let batch_topology = BatchTopology::new(&topology_nodes, true);
@@ -210,17 +210,14 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
let tokens = sampled.copyout::<u32>();
drop(completed);
- nodes.extend(tokens.into_iter().enumerate().map(|(index, token_id)| {
- let depth = index + 1;
- ProposalNode {
- token_id,
- depth,
- child_indices: if depth < chain_length {
- vec![depth + 1]
- } else {
- Vec::new()
- },
- }
+ nodes.extend(tokens.into_iter().zip(1u32..).map(|(token_id, depth)| ProposalNode {
+ token_id,
+ depth,
+ child_indices: if depth < chain_length {
+ vec![depth as usize + 1]
+ } else {
+ Vec::new()
+ },
}));
nodes
},
@@ -253,7 +250,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
let dflash_output =
self.dflash.encode_draft(state, target_output_token, target_embedding, depth, &mut encoder)?;
let depth_seeds = (0..weaver.max_depth())
- .map(|depth| prng.derive((root_position + depth) as u64))
+ .map(|depth| prng.derive(root_position as u64 + depth as u64))
.collect::<Box<[u64]>>();
let tree = weaver.encode_tree(
target_output_norm,
@@ -281,12 +278,13 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
fn recursive_build(
nodes: &[ProposalNode],
index: usize,
- root_position: usize,
+ root_position: u32,
#[cfg(grammar)] mut grammar: Option<&mut Grammar>,
prng: &PRng,
) -> TrieNode {
let node = &nodes[index];
- let mut trie_node = TrieNode::new(node.token_id as u64, prng.derive((root_position + node.depth) as u64));
+ let mut trie_node =
+ TrieNode::new(node.token_id as u64, prng.derive(root_position as u64 + node.depth as u64));
for &child_index in &node.child_indices {
#[cfg(grammar)]
if let Some(grammar) = grammar.as_mut()
diff --git a/crates/backend-uzu/src/tests/matmul/harness.rs b/crates/backend-uzu/src/tests/matmul/harness.rs
index 8dd541ab7..90900c958 100644
--- a/crates/backend-uzu/src/tests/matmul/harness.rs
+++ b/crates/backend-uzu/src/tests/matmul/harness.rs
@@ -141,7 +141,7 @@ fn run<B: Backend, T: ArrayElement + Float>(
alloc_allocation_with_data::<B, T>(context, prefill)
} else {
context
- .create_allocation(m * n * std::mem::size_of::<T>(), AllocationType::Global)
+ .create_allocation(m as usize * n as usize * std::mem::size_of::<T>(), AllocationType::Global)
.expect("create d allocation")
};
let rht_allocation =
@@ -172,9 +172,9 @@ fn run<B: Backend, T: ArrayElement + Float>(
d: &mut d_allocation,
d_transform,
gather_indices: None,
- m: m as u32,
- n: n as u32,
- k: k as u32,
+ m,
+ n,
+ k,
},
&mut encoder,
);
diff --git a/crates/backend-uzu/src/tests/matmul/quant.rs b/crates/backend-uzu/src/tests/matmul/quant.rs
index 8d81940d4..22b46e5c0 100644
--- a/crates/backend-uzu/src/tests/matmul/quant.rs
+++ b/crates/backend-uzu/src/tests/matmul/quant.rs
@@ -56,21 +56,23 @@ fn mode_for_bits(bits: u32) -> QuantizationMode {
impl<T: ArrayElement + Float> QuantInput<T> {
pub fn new(
- m: usize,
- k: usize,
- n: usize,
+ m: u32,
+ k: u32,
+ n: u32,
group_size: u32,
bits: u32,
quant_method: QuantizationMethod,
seed: u64,
) -> Self {
- let num_groups_k = k.div_ceil(group_size as usize);
+ let num_groups_k = k.div_ceil(group_size) as usize;
+ let (rows, inner, columns) = (m as usize, k as usize, n as usize);
let mut rng = SmallRng::seed_from_u64(seed);
- let w_packed: Vec<u32> = (0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect();
+ let w_packed: Vec<u32> =
+ (0..columns * inner * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect();
let scales: Vec<T> =
- (0..n * num_groups_k).map(|_| T::from(rng.random_range(0.01f32..0.3f32)).unwrap()).collect();
- let x: Vec<T> = (0..m * k).map(|_| T::from(rng.random_range(-0.3f32..0.3f32)).unwrap()).collect();
+ (0..columns * num_groups_k).map(|_| T::from(rng.random_range(0.01f32..0.3f32)).unwrap()).collect();
+ let x: Vec<T> = (0..rows * inner).map(|_| T::from(rng.random_range(-0.3f32..0.3f32)).unwrap()).collect();
let zp_stride = if bits == 4 {
num_groups_k.div_ceil(2)
@@ -80,10 +82,14 @@ impl<T: ArrayElement + Float> QuantInput<T> {
let (zero_points, biases) = match quant_method {
QuantizationMethod::ScaleBias => (
None,
- Some((0..n * num_groups_k).map(|_| T::from(rng.random_range(-0.03f32..0.03f32)).unwrap()).collect()),
+ Some(
+ (0..columns * num_groups_k)
+ .map(|_| T::from(rng.random_range(-0.03f32..0.03f32)).unwrap())
+ .collect(),
+ ),
),
QuantizationMethod::ScaleZeroPoint => {
- (Some((0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)
+ (Some((0..columns * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)
},
QuantizationMethod::ScaleSymmetric => (None, None),
};
@@ -94,9 +100,9 @@ impl<T: ArrayElement + Float> QuantInput<T> {
zero_points,
biases,
x,
- k: k as u32,
- n: n as u32,
- m: m as u32,
+ k,
+ n,
+ m,
group_size,
quant_method,
mode: mode_for_bits(bits),
@@ -112,31 +118,27 @@ impl<T: ArrayElement + Float> QuantInput<T> {
pub fn with_prepared_a(
mut self,
- activation_group_size: usize,
+ activation_group_size: u32,
sum_group_size: Option<u32>,
) -> Self {
self.signed_codes = true;
- let sum_group_size = sum_group_size.map(|group_size| group_size as usize);
- let rows = self.m as usize;
- let columns = self.k as usize;
+ let rows = self.m;
+ let columns = self.k;
assert!(columns.is_multiple_of(activation_group_size));
if let Some(group_size) = sum_group_size {
assert!(columns.is_multiple_of(group_size));
}
let context = <Cpu as Backend>::Context::new().expect("CPU context");
let input = alloc_allocation_with_data::<Cpu, T>(&context, &self.x);
- let factors = alloc_allocation_with_data::<Cpu, i32>(&context, &vec![1; columns]);
- let mut values = alloc_allocation::<Cpu, i8>(&context, rows * columns);
- let mut scales = alloc_allocation::<Cpu, f32>(&context, rows * columns / activation_group_size);
+ let factors = alloc_allocation_with_data::<Cpu, i32>(&context, &vec![1; columns as usize]);
+ let elements = rows as usize * columns as usize;
+ let mut values = alloc_allocation::<Cpu, i8>(&context, elements);
+ let mut scales = alloc_allocation::<Cpu, f32>(&context, elements / activation_group_size as usize);
let mut group_sums =
- sum_group_size.map(|group_size| alloc_allocation::<Cpu, i32>(&context, rows * columns / group_size));
- let transform = ActivationTransform::<Cpu>::quantize(
- &context,
- T::data_type(),
- activation_group_size,
- sum_group_size.map(|group_size| group_size as u32),
- )
- .expect("CPU activation quantization transform");
+ sum_group_size.map(|group_size| alloc_allocation::<Cpu, i32>(&context, elements / group_size as usize));
+ let transform =
+ ActivationTransform::<Cpu>::quantize(&context, T::data_type(), activation_group_size, sum_group_size)
+ .expect("CPU activation quantization transform");
let mut encoder = Encoder::<Cpu>::new(&context).expect("CPU encoder");
transform.encode_quantize(
&input,
@@ -144,8 +146,8 @@ impl<T: ArrayElement + Float> QuantInput<T> {
&mut scales,
group_sums.as_mut(),
&factors,
- rows as u32,
- columns as u32,
+ rows,
+ columns,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().expect("CPU activation quantization");
@@ -154,7 +156,7 @@ impl<T: ArrayElement + Float> QuantInput<T> {
values: allocation_to_vec(&values),
scales: allocation_to_vec(&scales),
group_sums: group_sums.map_or_else(Vec::new, |sums| allocation_to_vec(&sums)),
- activation_scale_group_size: activation_group_size as u32,
+ activation_scale_group_size: activation_group_size,
});
self
}
diff --git a/crates/backend-uzu/src/tests/matmul/shape.rs b/crates/backend-uzu/src/tests/matmul/shape.rs
index 29402382d..0c7a7e866 100644
--- a/crates/backend-uzu/src/tests/matmul/shape.rs
+++ b/crates/backend-uzu/src/tests/matmul/shape.rs
@@ -3,16 +3,16 @@ use derive_more::Display;
#[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
#[display("M[{m}]K[{k}]N[{n}]")]
pub struct Shape {
- pub m: usize,
- pub k: usize,
- pub n: usize,
+ pub m: u32,
+ pub k: u32,
+ pub n: u32,
}
impl Shape {
pub const fn new(
- m: usize,
- k: usize,
- n: usize,
+ m: u32,
+ k: u32,
+ n: u32,
) -> Self {
Self {
m,
@@ -49,16 +49,16 @@ pub fn bench_fp_gemm_shapes() -> impl Iterator<Item = Shape> {
BENCH_FP_GEMM.iter().copied()
}
-const BENCH_NK: &[(usize, usize)] =
+const BENCH_NK: &[(u32, u32)] =
&[(2048, 2048), (2048, 4096), (4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];
pub fn bench_quant_gemm_shapes(bits: u32) -> impl Iterator<Item = Shape> {
- let block_size: usize = if bits == 4 {
+ let block_size: u32 = if bits == 4 {
512
} else {
256
};
- let ms = &[4usize, 5, 6, 7, 8, 16, 32, 48, 64];
+ let ms = &[4u32, 5, 6, 7, 8, 16, 32, 48, 64];
BENCH_NK
.iter()
.filter(move |&&(n, k)| n % 32 == 0 && k % block_size == 0)
@@ -66,19 +66,19 @@ pub fn bench_quant_gemm_shapes(bits: u32) -> impl Iterator<Item = Shape> {
}
pub fn bench_quant_gemv_shapes(bits: u32) -> impl Iterator<Item = Shape> {
- let block_size: usize = if bits == 4 {
+ let block_size: u32 = if bits == 4 {
512
} else {
256
};
- let nk: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];
- let ms = &[1usize, 2, 4];
+ let nk: &[(u32, u32)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];
+ let ms = &[1u32, 2, 4];
nk.iter()
.filter(move |&&(n, k)| n % 8 == 0 && k % block_size == 0)
.flat_map(move |&(n, k)| ms.iter().map(move |&m| Shape::new(m, k, n)))
}
-const QWEN3_LAYERS: &[(&str, usize, usize)] = &[
+const QWEN3_LAYERS: &[(&str, u32, u32)] = &[
("0.8b_qkv", 1024, 3072),
("0.8b_o", 2048, 1024),
("0.8b_gate", 1024, 2048),
@@ -99,12 +99,12 @@ const QWEN3_LAYERS: &[(&str, usize, usize)] = &[
];
pub fn qwen3_layer_shapes(bits: u32) -> impl Iterator<Item = (&'static str, Shape)> {
- let block_size: usize = if bits == 4 {
+ let block_size: u32 = if bits == 4 {
512
} else {
256
};
- let ms = &[1usize, 2, 4, 8, 16, 32, 64];
+ let ms = &[1u32, 2, 4, 8, 16, 32, 64];
QWEN3_LAYERS
.iter()
.filter(move |&&(_, k, _)| k.is_multiple_of(block_size))
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs
index c41ad0ade..f1548ae25 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs
@@ -122,25 +122,21 @@ mod quantize {
factors_data: &[i32],
rows: usize,
columns: usize,
- activation_group_size: usize,
+ activation_group_size: u32,
emit_group_sums: bool,
- sum_group_size: Option<usize>,
+ sum_group_size: Option<u32>,
) -> (Vec<i8>, Vec<f32>, Option<Vec<i32>>) {
- let scale_groups = columns / activation_group_size;
- let sum_groups = sum_group_size.map_or(0, |group_size| columns / group_size);
+ let scale_groups = columns / activation_group_size as usize;
+ let sum_groups = sum_group_size.map_or(0, |group_size| columns / group_size as usize);
let context = B::Context::new().expect("context");
let input = alloc_allocation_with_data::<B, f32>(context.as_ref(), input_data);
let factors = alloc_allocation_with_data::<B, i32>(context.as_ref(), factors_data);
let mut values = alloc_allocation::<B, i8>(context.as_ref(), rows * columns);
let mut scales = alloc_allocation::<B, f32>(context.as_ref(), rows * scale_groups);
let mut group_sums = emit_group_sums.then(|| alloc_allocation::<B, i32>(context.as_ref(), rows * sum_groups));
- let kernel = ActivationTransform::quantize(
- context.as_ref(),
- DataType::F32,
- activation_group_size,
- sum_group_size.map(|size| size as u32),
- )
- .expect("quantize transform");
+ let kernel =
+ ActivationTransform::quantize(context.as_ref(), DataType::F32, activation_group_size, sum_group_size)
+ .expect("quantize transform");
let mut encoder = Encoder::<B>::new(context.as_ref()).expect("encoder");
kernel.encode_quantize(
&input,
@@ -158,9 +154,9 @@ mod quantize {
}
fn check_quantize(
- activation_group_size: usize,
+ activation_group_size: u32,
emit_group_sums: bool,
- sum_group_size: Option<usize>,
+ sum_group_size: Option<u32>,
) {
let rows = 3;
let columns = 256;
@@ -209,7 +205,7 @@ mod quantize {
for (group_index, (&actual, &expected)) in
actual_group_sums.iter().zip(expected_group_sums.as_ref().expect("CPU group sums")).enumerate()
{
- let sum_group_size = sum_group_size.expect("correction group");
+ let sum_group_size = sum_group_size.expect("correction group") as usize;
let start = group_index * sum_group_size;
let sum_from_actual_codes: i32 =
actual_values[start..start + sum_group_size].iter().copied().map(i32::from).sum();
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/embedding/quant_embedding_test.rs
index b6260341f..6b719c5fa 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/embedding/quant_embedding_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/embedding/quant_embedding_test.rs
@@ -45,7 +45,7 @@ fn get_test_data<T: ArrayElement + Float>(quant_mode: QuantizationMode) -> (Inpu
let token_ids: Box<[u32]> = Box::new([2, 5, 0]);
- let packing_divisor = quant_mode.packing_divisor() as u32;
+ let packing_divisor = quant_mode.packing_divisor();
let weights_stride = model_dim / packing_divisor;
let weights: Vec<u8> = (0..vocab_size as usize * weights_stride as usize)
.map(|i| ((i % 16) as u8) | ((((i + 3) % 16) as u8) * 16))
@@ -88,7 +88,7 @@ fn get_test_data_zero_point_group16<T: ArrayElement + Float>() -> (Input<T>, Vec
let token_ids: Box<[u32]> = Box::new([2, 5, 0]);
- let packing_divisor = quant_mode.packing_divisor() as u32;
+ let packing_divisor = quant_mode.packing_divisor();
let weights_stride = model_dim / packing_divisor;
let weights: Vec<u8> = (0..vocab_size as usize * weights_stride as usize)
.map(|i| ((i % 16) as u8) | ((((i + 3) % 16) as u8) * 16))
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs
index df0a7678c..37877e30a 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs
@@ -65,15 +65,15 @@ struct BenchmarkData {
impl BenchmarkData {
fn new(
context: &MetalContext,
- m: usize,
- k: usize,
- n: usize,
+ m: u32,
+ k: u32,
+ n: u32,
bits: u32,
group_size: u32,
seed: u64,
) -> Self {
let input = QuantInput::<bf16>::new(m, k, n, group_size, bits, QuantizationMethod::ScaleSymmetric, seed)
- .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE as usize, None);
+ .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, None);
let unsigned_weights = alloc_allocation_with_data::<Metal, u32>(context, &input.w_packed);
let signed_weights = alloc_allocation_with_data::<Metal, u32>(context, &input.weights_for_upload());
@@ -90,19 +90,20 @@ impl BenchmarkData {
.collect();
let rht_factors = alloc_allocation_with_data::<Metal, i32>(context, &rht);
- let groups = k / group_size as usize;
+ let groups = k / group_size;
+ let a_elements = m as usize * k as usize;
Self {
unsigned_weights,
signed_weights,
weight_scales,
activations,
rht_factors,
- a_working: alloc_allocation::<Metal, bf16>(context, m * k),
- a_int8: alloc_allocation::<Metal, i8>(context, m * k),
- a_scales: alloc_allocation::<Metal, f32>(context, m * groups),
- m: m as u32,
- k: k as u32,
- n: n as u32,
+ a_working: alloc_allocation::<Metal, bf16>(context, a_elements),
+ a_int8: alloc_allocation::<Metal, i8>(context, a_elements),
+ a_scales: alloc_allocation::<Metal, f32>(context, m as usize * groups as usize),
+ m,
+ k,
+ n,
group_size,
mode: if bits == 4 {
QuantizationMode::U4
@@ -237,10 +238,10 @@ fn bench_bits(
k,
n,
bits,
- HADAMARD_TRANSFORM_BLOCK_SIZE as u32,
+ HADAMARD_TRANSFORM_BLOCK_SIZE,
0xA8_00 ^ u64::from(bits) ^ k as u64 ^ n as u64,
);
- let mut output = alloc_allocation::<Metal, bf16>(context, m * n);
+ let mut output = alloc_allocation::<Metal, bf16>(context, m as usize * n as usize);
let shape_label = format!("{layer}_m{m}_k{k}_n{n}");
let gemv_eligible = GemvSpecialization::select_shape(
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemm_bench.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemm_bench.rs
index 7159bbd31..3995d0008 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemm_bench.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemm_bench.rs
@@ -45,10 +45,10 @@ fn bench_gemm(c: &mut Criterion) {
for shape in bench_fp_gemm_shapes() {
let (m, k, n) = (shape.m, shape.k, shape.n);
- let a = alloc_allocation::<Metal, bf16>(&context, m * k);
- let b_weights = alloc_allocation::<Metal, bf16>(&context, n * k);
- let mut d = alloc_allocation::<Metal, bf16>(&context, m * n);
- group.throughput(Throughput::Elements((2 * m * k * n) as u64));
+ let a = alloc_allocation::<Metal, bf16>(&context, m as usize * k as usize);
+ let b_weights = alloc_allocation::<Metal, bf16>(&context, n as usize * k as usize);
+ let mut d = alloc_allocation::<Metal, bf16>(&context, m as usize * n as usize);
+ group.throughput(Throughput::Elements(2 * u64::from(m) * u64::from(k) * u64::from(n)));
group.bench_function(BenchmarkId::new("BF16", shape.to_string()), |b| {
iter_encode_loop::<Metal, _>(&context, b, |encoder| {
kernel
@@ -67,9 +67,9 @@ fn bench_gemm(c: &mut Criterion) {
d: &mut d,
d_transform: MatmulDOps::none(),
gather_indices: None,
- m: m as u32,
- n: n as u32,
- k: k as u32,
+ m,
+ n,
+ k,
},
engine,
encoder,
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rs
index 596a31cca..d069c40a3 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rs
@@ -242,7 +242,7 @@ fn gemv_gather() {
(8, QuantizationMethod::ScaleZeroPoint),
] {
let (m, k, vocab, ids_per_row, group_size) = (8usize, 128usize, 64usize, 8usize, 32u32);
- let input = QuantInput::<bf16>::new(m, k, vocab, group_size, bits, method, 0x5EED);
+ let input = QuantInput::<bf16>::new(m as u32, k as u32, vocab as u32, group_size, bits, method, 0x5EED);
let ids: Vec<u32> = (0..m * ids_per_row).map(|i| ((i * 37 + 11) % vocab) as u32).collect();
// K_SPLIT == 1 keeps k in one reduction, so gather and dense share the exact accumulation.
check_gather!(m, vocab, ids, ids_per_row, 0.05, |B| {
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
index c36aa3a91..f378ebfc6 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
@@ -70,9 +70,9 @@ fn assert_parity<T: ArrayElement + Float + Debug + Display>(
}
fn run_parity<T: ArrayElement + Float + Debug + Display>(
- m: usize,
- k: usize,
- n: usize,
+ m: u32,
+ k: u32,
+ n: u32,
group_size: u32,
bits: u32,
quant_method: QuantizationMethod,
@@ -114,9 +114,9 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(
#[case::gs32_narrow_n_tail_sym(64, 256, 40, 32, 4, QuantizationMethod::ScaleSymmetric)]
#[case::gs16_k_tail_mlx(64, 272, 64, 16, 4, QuantizationMethod::ScaleBias)]
fn parity_bf16(
- #[case] m: usize,
- #[case] k: usize,
- #[case] n: usize,
+ #[case] m: u32,
+ #[case] k: u32,
+ #[case] n: u32,
#[case] gs: u32,
#[case] bits: u32,
#[case] method: QuantizationMethod,
@@ -131,9 +131,9 @@ fn parity_bf16(
#[case::gs128_8bit_zp_prefill(64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]
#[case::gs64_8bit_symmetric_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]
fn parity_bf16_8bit_splitk(
- #[case] m: usize,
- #[case] k: usize,
- #[case] n: usize,
+ #[case] m: u32,
+ #[case] k: u32,
+ #[case] n: u32,
#[case] gs: u32,
#[case] bits: u32,
#[case] method: QuantizationMethod,
@@ -156,9 +156,9 @@ fn parity_bf16_8bit_splitk(
#[case::m8_gs32_4bit_zp(8, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::m8_gs32_4bit_mlx(8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
fn parity_gemv_bf16(
- #[case] m: usize,
- #[case] k: usize,
- #[case] n: usize,
+ #[case] m: u32,
+ #[case] k: u32,
+ #[case] n: u32,
#[case] gs: u32,
#[case] bits: u32,
#[case] method: QuantizationMethod,
@@ -254,9 +254,9 @@ fn parity_bf16_gemv_qmv_fused_scale_bias() {
#[case::gs64_4bit_zp(2, 96, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs128_8bit(1, 192, 64, 128, 8, QuantizationMethod::ScaleBias)]
fn parity_gemv_partial_group_bf16(
- #[case] m: usize,
- #[case] k: usize,
- #[case] n: usize,
+ #[case] m: u32,
+ #[case] k: u32,
+ #[case] n: u32,
#[case] gs: u32,
#[case] bits: u32,
#[case] method: QuantizationMethod,
@@ -272,9 +272,9 @@ fn parity_gemv_partial_group_bf16(
#[case::n20_gs64_4bit_zp(2, 256, 20, 64, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::n36_gs32_8bit(1, 256, 36, 32, 8, QuantizationMethod::ScaleBias)]
fn parity_gemv_unaligned_width_bf16(
- #[case] m: usize,
- #[case] k: usize,
- #[case] n: usize,
+ #[case] m: u32,
+ #[case] k: u32,
+ #[case] n: u32,
#[case] gs: u32,
#[case] bits: u32,
#[case] method: QuantizationMethod,
@@ -452,9 +452,9 @@ fn quant_gemm_accumulate_returns_unsupported_dop() {
#[case::gs64_8bit_zp(128, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
#[case::gs128_8bit_zp(128, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]
fn mxu_quant_parity_bf16(
- #[case] m: usize,
- #[case] k: usize,
- #[case] n: usize,
+ #[case] m: u32,
+ #[case] k: u32,
+ #[case] n: u32,
#[case] gs: u32,
#[case] bits: u32,
#[case] method: QuantizationMethod,
@@ -477,20 +477,20 @@ fn mxu_quant_parity_bf16(
#[rstest]
#[test_attr(uzu_test)]
-#[case::sym_w4_gs32(16usize, 32u32, 4u32, QuantizationMethod::ScaleSymmetric)]
-#[case::sym_w8_gs64(16usize, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
-#[case::sym_w4_gs128(32usize, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]
-#[case::bias_w4_gs32(16usize, 32u32, 4u32, QuantizationMethod::ScaleBias)]
-#[case::bias_w8_gs64(16usize, 64u32, 8u32, QuantizationMethod::ScaleBias)]
-#[case::bias_w4_gs128(32usize, 128u32, 4u32, QuantizationMethod::ScaleBias)]
-#[case::zp_w4_gs32(16usize, 32u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
-#[case::zp_w8_gs64(16usize, 64u32, 8u32, QuantizationMethod::ScaleZeroPoint)]
-#[case::zp_w4_gs128(32usize, 128u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
-#[case::m1_bias_w4_gs64(1usize, 64u32, 4u32, QuantizationMethod::ScaleBias)]
-#[case::m1_zp_w8_gs32(1usize, 32u32, 8u32, QuantizationMethod::ScaleZeroPoint)]
-#[case::m1_sym_w4_gs128(1usize, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]
+#[case::sym_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleSymmetric)]
+#[case::sym_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
+#[case::sym_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]
+#[case::bias_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleBias)]
+#[case::bias_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleBias)]
+#[case::bias_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleBias)]
+#[case::zp_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
+#[case::zp_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleZeroPoint)]
+#[case::zp_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
+#[case::m1_bias_w4_gs64(1u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]
+#[case::m1_zp_w8_gs32(1u32, 32u32, 8u32, QuantizationMethod::ScaleZeroPoint)]
+#[case::m1_sym_w4_gs128(1u32, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]
fn a8w_mxu_parity_bf16(
- #[case] m: usize,
+ #[case] m: u32,
#[case] weight_gs: u32,
#[case] bits: u32,
#[case] method: QuantizationMethod,
@@ -499,9 +499,9 @@ fn a8w_mxu_parity_bf16(
if !context.supports_mxu() {
return;
}
- let (k, n) = (256usize, 128usize);
+ let (k, n) = (256u32, 128u32);
let input = QuantInput::<bf16>::new(m, k, n, weight_gs, bits, method, 0).with_prepared_a(
- ACTIVATION_SCALE_GROUP_SIZE as usize,
+ ACTIVATION_SCALE_GROUP_SIZE,
(method != QuantizationMethod::ScaleSymmetric).then_some(weight_gs),
);
let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmEngine::Mxu));
@@ -517,17 +517,17 @@ fn a8w_mxu_parity_bf16(
#[rstest]
#[test_attr(uzu_test)]
-#[case::m1(1usize)]
-#[case::m16(16usize)]
-#[case::m33(33usize)]
-fn a8w_independent_activation_group_parity_bf16(#[case] m: usize) {
+#[case::m1(1u32)]
+#[case::m16(16u32)]
+#[case::m33(33u32)]
+fn a8w_independent_activation_group_parity_bf16(#[case] m: u32) {
let context = MetalContext::new().expect("Metal context");
if !context.supports_mxu() {
return;
}
for (activation_group_size, weight_group_size) in
- [(32usize, 32u32), (32, 64), (64, 64), (64, 128), (128, 32), (128, 64), (128, 128)]
+ [(32u32, 32u32), (32, 64), (64, 64), (64, 128), (128, 32), (128, 64), (128, 128)]
{
for (bits, method) in [
(4, QuantizationMethod::ScaleSymmetric),
@@ -537,8 +537,7 @@ fn a8w_independent_activation_group_parity_bf16(#[case] m: usize) {
] {
let input = QuantInput::<bf16>::new(m, 256, 72, weight_group_size, bits, method, 0).with_prepared_a(
activation_group_size,
- (method != QuantizationMethod::ScaleSymmetric)
- .then_some((activation_group_size as u32).min(weight_group_size)),
+ (method != QuantizationMethod::ScaleSymmetric).then_some(activation_group_size.min(weight_group_size)),
);
let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmEngine::Mxu));
let reference = run_quant_cpu::<bf16>(&input);
@@ -558,13 +557,13 @@ fn a8w_independent_activation_group_parity_bf16(#[case] m: usize) {
#[case::m1(1)]
#[case::m16(16)]
#[case::m33(33)]
-fn a8w4_zero_point_tail_parity(#[case] m: usize) {
+fn a8w4_zero_point_tail_parity(#[case] m: u32) {
let context = MetalContext::new().expect("Metal context");
if !context.supports_mxu() {
return;
}
let input = QuantInput::<bf16>::new(m, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint, 0)
- .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE as usize, Some(32));
+ .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, Some(32));
let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmEngine::Mxu));
let reference = run_quant_cpu::<bf16>(&input);
assert_parity::<bf16>(&format!("A8W4 ZP N-tail m={m}"), &reference, &actual, 0.08, 0.8);
@@ -577,7 +576,7 @@ fn a8w4_zero_point_tail_signed_codes_parity() {
return;
}
let input = QuantInput::<bf16>::new(33, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint, 0)
- .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE as usize, Some(32))
+ .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, Some(32))
.with_signed_weight_codes();
let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmEngine::Mxu));
let reference = run_quant_cpu::<bf16>(&input);
@@ -586,20 +585,20 @@ fn a8w4_zero_point_tail_signed_codes_parity() {
#[rstest]
#[test_attr(uzu_test)]
-#[case::w4_sym(4u32, QuantizationMethod::ScaleSymmetric, 256usize)]
-#[case::w4_bias(4u32, QuantizationMethod::ScaleBias, 256usize)]
-#[case::w4_zp(4u32, QuantizationMethod::ScaleZeroPoint, 256usize)]
-#[case::w8_sym(8u32, QuantizationMethod::ScaleSymmetric, 256usize)]
-#[case::w8_bias(8u32, QuantizationMethod::ScaleBias, 256usize)]
-#[case::w8_zp(8u32, QuantizationMethod::ScaleZeroPoint, 256usize)]
-#[case::w8_zp_tail(8u32, QuantizationMethod::ScaleZeroPoint, 224usize)]
+#[case::w4_sym(4u32, QuantizationMethod::ScaleSymmetric, 256u32)]
+#[case::w4_bias(4u32, QuantizationMethod::ScaleBias, 256u32)]
+#[case::w4_zp(4u32, QuantizationMethod::ScaleZeroPoint, 256u32)]
+#[case::w8_sym(8u32, QuantizationMethod::ScaleSymmetric, 256u32)]
+#[case::w8_bias(8u32, QuantizationMethod::ScaleBias, 256u32)]
+#[case::w8_zp(8u32, QuantizationMethod::ScaleZeroPoint, 256u32)]
+#[case::w8_zp_tail(8u32, QuantizationMethod::ScaleZeroPoint, 224u32)]
fn signed_weights_full_precision_activations_parity_bf16(
#[case] bits: u32,
#[case] method: QuantizationMethod,
- #[case] k: usize,
+ #[case] k: u32,
) {
let context = MetalContext::new().expect("Metal context");
- let (m, n, group_size) = (2usize, 128usize, 32u32);
+ let (m, n, group_size) = (2u32, 128u32, 32u32);
let input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0).with_signed_weight_codes();
let reference_input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0);
let reference = run_quant_cpu::<bf16>(&reference_input);
@@ -632,16 +631,16 @@ fn a8w_mxu_output_bias_parity_bf16(
}
let (m, k, n) = (
- 16usize,
- 256usize,
+ 16u32,
+ 256u32,
if with_output_hadamard {
- 64usize
+ 64u32
} else {
- 70usize
+ 70u32
},
);
let input = QuantInput::<bf16>::new(m, k, n, 32, bits, QuantizationMethod::ScaleSymmetric, 0)
- .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE as usize, None);
+ .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, None);
let output_bias: Vec<bf16> = (0..n).map(|column| bf16::from_f32(0.25 + 0.05 * (column % 7) as f32)).collect();
let output_hadamard_factors: Option<Vec<i32>> = with_output_hadamard.then(|| {
(0..n)
@@ -676,7 +675,7 @@ fn a8w_mxu_output_bias_parity_bf16(
cpu_matmul.encode(cpu_arguments, &mut cpu_encoder).expect("CPU A8W matmul");
cpu_encoder.end_encoding().submit().wait_until_completed().unwrap();
let mut reference = allocation_to_vec::<Cpu, bf16>(&cpu_buffers.y);
- for row in reference.chunks_exact_mut(n) {
+ for row in reference.chunks_exact_mut(n as usize) {
for (value, bias) in row.iter_mut().zip(&output_bias) {
*value = bf16::from_f32(value.to_f32() + bias.to_f32());
}
@@ -758,10 +757,10 @@ fn run_widened_f32<B: Backend>(
#[case::gemv_m1(1)]
#[case::gemv_m4(4)]
#[case::gemm_m31(31)]
-fn parity_widened_f32_output(#[case] m: usize) {
+fn parity_widened_f32_output(#[case] m: u32) {
// Mirrors the dflash draft-chain readout: quantized LM head (4-bit ZP,
// group 32), BF16 input, widened F32 output.
- let (k, n, gs) = (5120usize, 32768usize, 32u32);
+ let (k, n, gs) = (5120u32, 32768u32, 32u32);
let context = MetalContext::new().expect("Metal context");
let input = QuantInput::<bf16>::new(m, k, n, gs, 4, QuantizationMethod::ScaleZeroPoint, 0);
let cpu_context = <Cpu as Backend>::Context::new().expect("Cpu context");
diff --git a/crates/backend-uzu/tests/unit/backends/metal/kernel/gdn/chunked_test.rs b/crates/backend-uzu/tests/unit/backends/metal/kernel/gdn/chunked_test.rs
index 220c352b2..7e18e2180 100644
--- a/crates/backend-uzu/tests/unit/backends/metal/kernel/gdn/chunked_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/metal/kernel/gdn/chunked_test.rs
@@ -140,7 +140,7 @@ fn run_prefill<T: ArrayElement>(
value_head_dim: HEAD_V_DIM as u32,
key_dim: case.key_dim as u32,
value_dim: case.value_dim as u32,
- suffix_len: case.suffix_len,
+ suffix_len: case.suffix_len as u32,
},
&mut encoder,
)
@@ -185,7 +185,7 @@ fn chunked_prefill_matches_recurrent_prefill() {
}
for suffix_len in [CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, CHUNK_SIZE * 2 + 1] {
- let case = test_case(suffix_len);
+ let case = test_case(suffix_len as usize);
let (recurrent_out, recurrent_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Recurrent);
let (chunked_out, chunked_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Chunked);
diff --git a/crates/backend-uzu/tests/unit/encodable_block/attention_gemm_test.rs b/crates/backend-uzu/tests/unit/encodable_block/attention_gemm_test.rs
index e08fdd295..8c86488e0 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/attention_gemm_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/attention_gemm_test.rs
@@ -83,9 +83,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
let new_arguments = AttentionCoreNewArguments {
- head_dim: input.head_dim,
- num_groups: input.num_kv_heads,
- num_q_heads: input.num_heads,
+ head_dim: input.head_dim as u32,
+ num_groups: input.num_kv_heads as u32,
+ num_q_heads: input.num_heads as u32,
has_sinks: false,
is_kv_cache_ring: false,
is_causal: input.do_causal,
@@ -101,14 +101,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let segment_prefix_length = input.sequence_length - input.suffix_length;
let state_type = AttentionStateType::Full {
- length: segment_prefix_length,
+ length: segment_prefix_length as u32,
};
let args = AttentionCoreEncodeArguments {
queries: &queries_allocation,
keys: &keys_allocation,
values: &values_allocation,
- suffix_length: input.suffix_length,
+ suffix_length: input.suffix_length as u32,
trie: None,
sinks: None,
state_type: &state_type,
diff --git a/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs b/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs
index 8e8fb4702..7e201ac17 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs
@@ -321,9 +321,9 @@ fn run_gemm_attention(
let core = <<Metal as Backend>::Kernels as Kernels>::AttentionGemmCore::new(
context,
&AttentionCoreNewArguments {
- head_dim,
- num_groups: num_kv_heads,
- num_q_heads: num_heads,
+ head_dim: head_dim as u32,
+ num_groups: num_kv_heads as u32,
+ num_q_heads: num_heads as u32,
has_sinks: sinks.is_some(),
is_kv_cache_ring: false,
is_causal,
@@ -349,7 +349,7 @@ fn run_gemm_attention(
queries: &query_allocation,
keys: &key_allocation,
values: &value_allocation,
- suffix_length: seq_len,
+ suffix_length: seq_len as u32,
trie: None,
sinks: sinks_allocation.as_ref(),
state_type: &state_type,
diff --git a/crates/backend-uzu/tests/unit/encodable_block/delta_net_tree_verify_bench.rs b/crates/backend-uzu/tests/unit/encodable_block/delta_net_tree_verify_bench.rs
index bea09ba33..4ff105242 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/delta_net_tree_verify_bench.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/delta_net_tree_verify_bench.rs
@@ -23,10 +23,10 @@ fn bench_delta_net_tree_verify(c: &mut Criterion) {
const HEAD_DIM: usize = 128;
let arguments = TreeVerifyNewArguments {
data_type: DataType::BF16,
- num_k_heads: K_HEADS,
- num_v_heads: V_HEADS,
- head_k_dim: HEAD_DIM,
- head_v_dim: HEAD_DIM,
+ num_k_heads: K_HEADS as u32,
+ num_v_heads: V_HEADS as u32,
+ head_k_dim: HEAD_DIM as u32,
+ head_v_dim: HEAD_DIM as u32,
};
let context = <Metal as Backend>::Context::new().unwrap();
let started = Instant::now();
@@ -69,7 +69,7 @@ fn bench_delta_net_tree_verify(c: &mut Criterion) {
log_decay: &log_decay,
beta: &beta,
h0: &h0,
- tree_size,
+ tree_size: tree_size as u32,
},
encoder,
)
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
index b4c8e67dd..1ba18e376 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
@@ -382,7 +382,9 @@ fn run_moe_parity_test_internal<B: Backend>(
);
let gather = MoeGather::<B>::new(ctx, DataType::BF16).expect("gather");
- let x_perm_buf = gather.encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t, k, d_model, &mut encoder).expect("gather");
+ let x_perm_buf = gather
+ .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut encoder)
+ .expect("gather");
let total_rows = t * k;
@@ -394,10 +396,10 @@ fn run_moe_parity_test_internal<B: Backend>(
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- total_rows,
- d_model,
- d_ff,
- num_routed_experts: e,
+ total_rows: total_rows as u32,
+ d_model: d_model as u32,
+ d_ff: d_ff as u32,
+ num_routed_experts: e as u32,
gate_clip_min: gate_clip.0,
gate_clip_max: gate_clip.1,
up_clip_min: up_clip.0,
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_perf_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_perf_test.rs
index e92ee7dc8..8d843e0dc 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_perf_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_perf_test.rs
@@ -89,10 +89,10 @@ fn run_decode_case<B: Backend>(
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- num_routed_experts: e,
+ total_rows: sum_k as u32,
+ d_model: d_model as u32,
+ d_ff: d_ff as u32,
+ num_routed_experts: e as u32,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
up_clip_min: f32::NEG_INFINITY,
@@ -120,10 +120,10 @@ fn run_decode_case<B: Backend>(
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- num_routed_experts: e,
+ total_rows: sum_k as u32,
+ d_model: d_model as u32,
+ d_ff: d_ff as u32,
+ num_routed_experts: e as u32,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
up_clip_min: f32::NEG_INFINITY,
@@ -208,10 +208,10 @@ fn run_two_pass_prefill_case<B: Backend>(
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- num_routed_experts: e,
+ total_rows: sum_k as u32,
+ d_model: d_model as u32,
+ d_ff: d_ff as u32,
+ num_routed_experts: e as u32,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
up_clip_min: f32::NEG_INFINITY,
@@ -235,10 +235,10 @@ fn run_two_pass_prefill_case<B: Backend>(
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- num_routed_experts: e,
+ total_rows: sum_k as u32,
+ d_model: d_model as u32,
+ d_ff: d_ff as u32,
+ num_routed_experts: e as u32,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
up_clip_min: f32::NEG_INFINITY,
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
index 84adc358b..61d0c7346 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
@@ -381,10 +381,10 @@ fn test_two_pass_decode_correctness() {
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- num_routed_experts: e,
+ total_rows: sum_k as u32,
+ d_model: d_model as u32,
+ d_ff: d_ff as u32,
+ num_routed_experts: e as u32,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
up_clip_min: f32::NEG_INFINITY,
@@ -503,10 +503,10 @@ fn test_two_pass_decode_multi_token() {
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- num_routed_experts: e,
+ total_rows: sum_k as u32,
+ d_model: d_model as u32,
+ d_ff: d_ff as u32,
+ num_routed_experts: e as u32,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
up_clip_min: f32::NEG_INFINITY,
@@ -586,10 +586,10 @@ fn test_two_pass_prefill_correctness() {
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- num_routed_experts: e,
+ total_rows: sum_k as u32,
+ d_model: d_model as u32,
+ d_ff: d_ff as u32,
+ num_routed_experts: e as u32,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
up_clip_min: f32::NEG_INFINITY,
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_gather_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_gather_test.rs
index 4782573a7..de62c28de 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_gather_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_gather_test.rs
@@ -38,9 +38,9 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
&x_allocation,
&ids_allocation,
&sumk_allocation,
- input.t,
- input.sum_k / input.t,
- input.d_model,
+ input.t as u32,
+ (input.sum_k / input.t) as u32,
+ input.d_model as u32,
&mut encoder,
)
.expect("Failed to encode MoE gather");
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_perf_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_perf_test.rs
index 4f84945c7..3c41b99f3 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_perf_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_perf_test.rs
@@ -321,7 +321,7 @@ fn test_moe_pipeline_breakdown_decode() {
let gather_perf = run_perf_with_warmup("Gather", 2, 5, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
let x_perm = gather
- .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t, k, d_model, &mut encoder)
+ .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut encoder)
.expect("failed to encode MoE gather");
let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(x_perm);
@@ -331,7 +331,7 @@ fn test_moe_pipeline_breakdown_decode() {
let (x_perm_buf, x_perm_completed) = {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
let x_perm = gather
- .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t, k, d_model, &mut encoder)
+ .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut encoder)
.expect("failed to encode MoE gather");
let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
(x_perm, completed)
@@ -348,10 +348,10 @@ fn test_moe_pipeline_breakdown_decode() {
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- num_routed_experts: e,
+ total_rows: sum_k as u32,
+ d_model: d_model as u32,
+ d_ff: d_ff as u32,
+ num_routed_experts: e as u32,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: 20.0,
up_clip_min: -19.0,
diff --git a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs
index 6b6e5f80f..0214beb8a 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs
@@ -39,9 +39,9 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(
bitmask: Option<&[u32]>,
vocab_size: usize,
method: &SamplingMethod,
- batch_size: usize,
+ batch_size: u32,
) -> Result<SamplingTestResults, TestCaseError> {
- let sampling = Sampling::new(T::data_type(), vocab_size);
+ let sampling = Sampling::new(T::data_type(), vocab_size as u32);
let mut logits_allocation =
context.create_allocation(logits.len() * T::data_type().size_in_bytes(), AllocationType::Global).unwrap();
@@ -63,9 +63,9 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(
let nodes = (0..batch_size)
.map(|index| TrieNode {
- trie_start: index as u32,
- trie_end: (batch_size - 1) as u32,
- height: index as u32,
+ trie_start: index,
+ trie_end: batch_size - 1,
+ height: index,
})
.collect::<Box<[_]>>();
let batch_topology = BatchTopology::new(&nodes, true);
@@ -182,7 +182,7 @@ fn test_sampling_prop() {
bitmask.as_ref().map(Box::as_ref),
sampling_case.vocab_size,
&sampling_case.method,
- sampling_case.batch_size,
+ sampling_case.batch_size as u32,
))
.compare_results()?
});