A high-performance inference engine for AI models
migrate from size to u32
eugene1bbb59ebe273
diff --git a/Cargo.toml b/Cargo.tomlindex 596283a95..13e51d421 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -231,6 +231,7 @@ uzu_bench = "dinghy_apple_runner_macros::bench_case"[profile.dev]opt-level = 1+overflow-checks = true[profile.dev.package."*"]opt-level = 2@@ -243,3 +244,4 @@ lto = "thin"[profile.test]opt-level = 2+overflow-checks = truediff --git a/crates/backend-uzu/build/metal/gpu_types/mod.rs b/crates/backend-uzu/build/metal/gpu_types/mod.rsindex b5968d062..f36314945 100644--- a/crates/backend-uzu/build/metal/gpu_types/mod.rs+++ b/crates/backend-uzu/build/metal/gpu_types/mod.rs@@ -27,8 +27,6 @@ fn rust_to_metal(ty: &str) -> anyhow::Result<&'static str> {"u64" => Ok("uint64_t"),"f32" => Ok("float"),"bool" => Ok("bool"),- "usize" => Ok("size_t"),- "isize" => Ok("ptrdiff_t"),unknown => anyhow::bail!("Unsupported GPU type: {unknown}"),}}diff --git a/crates/backend-uzu/src/array/util.rs b/crates/backend-uzu/src/array/util.rsindex cfcc03d09..b0c6a5b3b 100644--- a/crates/backend-uzu/src/array/util.rs+++ b/crates/backend-uzu/src/array/util.rs@@ -1,17 +1,17 @@use crate::data_type::DataType;pub fn size_for_shape(- shape: &[usize],+ shape: &[u32],data_type: DataType,) -> usize {let Some(last_dim) = shape.last() else {return data_type.size_in_bytes();};- let bits_per_row = last_dim * data_type.size_in_bits();+ let bits_per_row = *last_dim as usize * data_type.size_in_bits();let padded_bytes_per_row = bits_per_row.div_ceil(8);- let num_rows: usize = shape.iter().rev().skip(1).product();+ let num_rows: usize = shape.iter().rev().skip(1).map(|&dim| dim as usize).product();num_rows * padded_bytes_per_row}diff --git a/crates/backend-uzu/src/backends/common/encoder.rs b/crates/backend-uzu/src/backends/common/encoder.rsindex 995608391..987a37c9c 100644--- a/crates/backend-uzu/src/backends/common/encoder.rs+++ b/crates/backend-uzu/src/backends/common/encoder.rs@@ -7,11 +7,15 @@ use std::{use bytemuck::{AnyBitPattern, NoUninit};-use crate::backends::common::{- AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer,- BufferGpuAddressRangeExt, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,- CommandBufferInitial, CommandBufferPending, Context,- hazard_tracker::{Access, HazardTracker},+use crate::{+ array::size_for_shape,+ backends::common::{+ AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer,+ BufferGpuAddressRangeExt, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,+ CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ hazard_tracker::{Access, HazardTracker},+ },+ data_type::DataType,};fn resolve_copy_range(@@ -83,6 +87,14 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {)}+ pub fn allocate_constant_for_shape(+ &mut self,+ shape: &[u32],+ data_type: DataType,+ ) -> Result<Allocation<B>, B::Error> {+ self.allocate_constant(size_for_shape(shape, data_type))+ }+pub fn allocate_constant_from_slice<T: NoUninit + AnyBitPattern>(&mut self,data: &[T],@@ -106,6 +118,14 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {)}+ pub fn allocate_scratch_for_shape(+ &mut self,+ shape: &[u32],+ data_type: DataType,+ ) -> Result<Allocation<B>, B::Error> {+ self.allocate_scratch(size_for_shape(shape, data_type))+ }+pub fn encode_copy<Src: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,Dst: AsBufferRangeMut<Buffer: Buffer<Backend = B>>,diff --git a/crates/backend-uzu/src/backends/common/gpu_types/attention.rs b/crates/backend-uzu/src/backends/common/gpu_types/attention.rsindex 7df3d5ee6..e0b648ead 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/attention.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/attention.rs@@ -10,10 +10,10 @@#[repr(C)]#[derive(Debug, Default, Copy, Clone)]pub struct AttnParams {- pub q_strides: [u64; 3],- pub k_strides: [u64; 3],- pub v_strides: [u64; 3],- pub o_strides: [u64; 3],+ pub q_strides: [u32; 3],+ pub k_strides: [u32; 3],+ pub v_strides: [u32; 3],+ pub o_strides: [u32; 3],pub gqa_factor: u32,pub scale: f32,/// Query length (suffix length)diff --git a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rsindex 429450dac..a727502c1 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs@@ -1 +1 @@-pub const HADAMARD_TRANSFORM_BLOCK_SIZE: usize = 32;+pub const HADAMARD_TRANSFORM_BLOCK_SIZE: u32 = 32;diff --git a/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs b/crates/backend-uzu/src/backends/common/gpu_types/quantization.rsindex fce644746..9896c7372 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs@@ -16,8 +16,8 @@ pub enum QuantizationMode {}impl QuantizationMode {- pub fn packing_divisor(&self) -> usize {- let bits = DataType::from(*self).size_in_bits();+ pub fn packing_divisor(&self) -> u32 {+ let bits = DataType::from(*self).size_in_bits() as u32;assert_eq!(8 % bits, 0, "QuantizationMode bit width ({bits}) must divide 8 evenly");8 / bits}diff --git a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rsindex 09d0e86cd..bada0a1ec 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs@@ -1,6 +1,6 @@-pub const CANDIDATES_MAX: usize = 512;-pub const TOP_CHILDREN_THREADS: usize = 256;-pub const TOP_CHILDREN_SIMDGROUPS: usize = 8;+pub const CANDIDATES_MAX: u32 = 512;+pub const TOP_CHILDREN_THREADS: u32 = 256;+pub const TOP_CHILDREN_SIMDGROUPS: u32 = 8;#[repr(C)]#[derive(Clone, Copy)]@@ -46,8 +46,8 @@ impl MetadataIdx {}pub const FRONTIER_NO_WINNER: u32 = !0;-pub const FRONTIER_SELECT_THREADS: usize = 256;-pub const FRONTIER_SELECT_SIMDGROUPS: usize = 8;-pub const FRONTIER_ENTRIES_PER_THREAD: usize = 8;-pub const FRONTIER_MAX_SLOTS: usize = 2048;-pub const FRONTIER_MAX_WIDTH: usize = 32;+pub const FRONTIER_SELECT_THREADS: u32 = 256;+pub const FRONTIER_SELECT_SIMDGROUPS: u32 = 8;+pub const FRONTIER_ENTRIES_PER_THREAD: u32 = 8;+pub const FRONTIER_MAX_SLOTS: u32 = 2048;+pub const FRONTIER_MAX_WIDTH: u32 = 32;diff --git a/crates/backend-uzu/src/backends/common/kernel/activation_transform.rs b/crates/backend-uzu/src/backends/common/kernel/activation_transform.rsindex 87b732c43..17cb688c8 100644--- a/crates/backend-uzu/src/backends/common/kernel/activation_transform.rs+++ b/crates/backend-uzu/src/backends/common/kernel/activation_transform.rs@@ -9,7 +9,7 @@ use crate::{fn assert_row_width(element_count: u32) {assert!(- (element_count as usize).is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE),+ element_count.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE),"activation transform requires element_count ({element_count}) to be a multiple of {HADAMARD_TRANSFORM_BLOCK_SIZE}");}@@ -20,7 +20,7 @@ pub struct ActivationTransform<B: Backend> {kernel: <B::Kernels as Kernels>::ActivationTransformKernel,ops: ActivationTransformOp,in_place: bool,- activation_group_size: usize,+ activation_group_size: u32,sum_group_size: Option<u32>,}@@ -30,7 +30,7 @@ impl<B: Backend> ActivationTransform<B> {data_type: DataType,ops: ActivationTransformOp,in_place: bool,- activation_group_size: usize,+ activation_group_size: u32,sum_group_size: Option<u32>,) -> Result<Self, B::Error> {let kernel = <B::Kernels as Kernels>::ActivationTransformKernel::new(@@ -38,8 +38,8 @@ impl<B: Backend> ActivationTransform<B> {data_type,ops,in_place,- activation_group_size as u32,- sum_group_size.unwrap_or(HADAMARD_TRANSFORM_BLOCK_SIZE as u32),+ activation_group_size,+ sum_group_size.unwrap_or(HADAMARD_TRANSFORM_BLOCK_SIZE),)?;Ok(Self {kernel,@@ -69,7 +69,7 @@ impl<B: Backend> ActivationTransform<B> {pub fn quantize(context: &B::Context,data_type: DataType,- activation_group_size: usize,+ activation_group_size: u32,sum_group_size: Option<u32>,) -> Result<Self, B::Error> {let ops = if sum_group_size.is_some() {@@ -149,7 +149,7 @@ impl<B: Backend> ActivationTransform<B> {assert!(self.quantizes());assert_row_width(element_count);assert!(- element_count.is_multiple_of(self.activation_group_size as u32),+ element_count.is_multiple_of(self.activation_group_size),"quantized activation row ({element_count}) must be a multiple of scale group ({})",self.activation_group_size);@@ -177,7 +177,7 @@ impl<B: Backend> ActivationTransform<B> {self.sum_group_size.is_some()}- pub fn activation_group_size(&self) -> usize {+ pub fn activation_group_size(&self) -> u32 {self.activation_group_size}diff --git a/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rsindex 28ec0a2df..9d1e3ec4c 100644--- a/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs+++ b/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs@@ -16,7 +16,7 @@ pub struct DeltaNetChunkedPrefillArgs<'a, B: Backend> {pub value_head_dim: u32,pub key_dim: u32,pub value_dim: u32,- pub suffix_len: usize,+ pub suffix_len: u32,}pub trait DeltaNetChunkedPrefill<B: Backend>: Sized + Send + Sync {@@ -28,7 +28,7 @@ pub trait DeltaNetChunkedPrefill<B: Backend>: Sized + Send + Sync {fn should_use(&self,- suffix_len: usize,+ suffix_len: u32,) -> bool;fn encode(@@ -49,7 +49,7 @@ impl<B: Backend> DeltaNetChunkedPrefill<B> for Infallible {fn should_use(&self,- _suffix_len: usize,+ _suffix_len: u32,) -> bool {match *self {}}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/error.rsindex c98795a5a..96be2b02a 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/error.rs@@ -10,7 +10,7 @@ pub enum MatmulError<B: Backend> {#[error("Unsupported data type: {0:?}")]UnsupportedDataType(DataType),#[error("Unsupported group size: {0}")]- UnsupportedGroupSize(usize),+ UnsupportedGroupSize(u32),#[error("Unsupported D-transform op {bit:?} on path {path}")]UnsupportedDOp {bit: GemmDTransform,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/activation_transform.rs b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/activation_transform.rsindex b91b027d2..df6b0dc5d 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/activation_transform.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/activation_transform.rs@@ -72,9 +72,9 @@ pub fn activation_transform<T: ArrayElement + Float>(let mut transformed = vec![0.0f32; columns];for row in 0..rows {let row_offset = row * columns;- for stripe_start in (0..columns).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE) {- let mut stripe = [0.0f32; HADAMARD_TRANSFORM_BLOCK_SIZE];- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {+ for stripe_start in (0..columns).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {+ let mut stripe = [0.0f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize];+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE as usize {let index = stripe_start + lane;let value: f32 = NumCast::from(unsafe { *input.add(row_offset + index) }).unwrap();let factor = unsafe { *rht_factors.add(index) } as f32;@@ -87,7 +87,7 @@ pub fn activation_transform<T: ArrayElement + Float>(hadamard_transform(&mut stripe);- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE as usize {let index = stripe_start + lane;let factor = unsafe { *rht_factors.add(index) } as f32;transformed[index] = if input_rht {diff --git a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rsindex 3c3defd42..b33ecaff9 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs@@ -22,10 +22,10 @@ pub fn quantize_symmetric_i8((value / divisor).round().clamp(-INT8_SYMMETRIC_QUANTIZATION_MAXIMUM, INT8_SYMMETRIC_QUANTIZATION_MAXIMUM) as i8}-pub(crate) fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZE]) {+pub(crate) fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize]) {let mut stride = 1;- while stride < HADAMARD_TRANSFORM_BLOCK_SIZE {- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {+ while stride < HADAMARD_TRANSFORM_BLOCK_SIZE as usize {+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE as usize {if lane & stride == 0 {let a = values[lane];let b = values[lane | stride];diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rsindex 8a071e503..45cffeb8d 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs@@ -33,7 +33,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(unimplemented!("not supported yet");}- let packing_divisor = quantization_mode.packing_divisor() as u32;+ let packing_divisor = quantization_mode.packing_divisor();let weights_stride = model_dim / packing_divisor;let num_groups = model_dim.div_ceil(group_size);let zero_points_stride = match quantization_mode {diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rsindex 0ea58c3c3..365ce8620 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs@@ -28,9 +28,9 @@ pub fn weaver_frontier_select(candidates_per_depth: u32,) {if frontier_capacity == 0- || frontier_capacity as usize > FRONTIER_MAX_SLOTS+ || frontier_capacity > FRONTIER_MAX_SLOTS|| node_count == 0- || node_count as usize > FRONTIER_MAX_WIDTH+ || node_count > FRONTIER_MAX_WIDTH|| ancestor_stride == 0|| max_depth == 0|| tree_slot_count == 0diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rsindex f4a5bd65c..87ea5135f 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs@@ -23,7 +23,7 @@ pub fn weaver_top_children(let rows = rows as usize;let candidates = candidates as usize;let expand_width = expand_width as usize;- if candidates == 0 || candidates > CANDIDATES_MAX || expand_width == 0 || expand_width > candidates {+ if candidates == 0 || candidates > CANDIDATES_MAX as usize || expand_width == 0 || expand_width > candidates {return;}for row in 0..rows {diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention.rs b/crates/backend-uzu/src/backends/metal/kernel/attention.rsindex cb48dcffb..bd53cdca9 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/attention.rs@@ -3,7 +3,6 @@ use std::collections::{HashMap, hash_map::Entry};use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};use crate::{- array::size_for_shape,backends::{common::{Allocation, BufferArg, Encoder, gpu_types::AttnParams, kernel::attention_gemm::AttentionGemmCore},metal::{Metal, context::MetalContext, error::MetalError, kernel::AttentionGemmMetalKernel},@@ -14,10 +13,10 @@ use crate::{pub struct AttentionGemmMetalCore {kernels: Mutex<HashMap<AttentionGemmKey, AttentionGemmMetalKernel>>,- head_dim: usize,- num_groups: usize,- num_q_heads: usize,- sliding_window_size: Option<usize>,+ head_dim: u32,+ num_groups: u32,+ num_q_heads: u32,+ sliding_window_size: Option<u32>,scale: Option<f32>,data_type: DataType,simd_bk: u32,@@ -65,7 +64,7 @@ impl AttentionGemmMetalCore {context,self.data_type,bk,- self.head_dim as u32,+ self.head_dim,key.use_mxu,key.align_q,key.align_k,@@ -121,10 +120,8 @@ impl AttentionGemmCore<Metal> for AttentionGemmMetalCore {arguments: AttentionCoreEncodeArguments<'a, Metal, KT, VT>,encoder: &mut Encoder<Metal>,) -> Result<Allocation<Metal>, MetalError> {- let mut output = encoder.allocate_constant(size_for_shape(- &[arguments.suffix_length, self.num_q_heads, self.head_dim],- self.data_type,- ))?;+ let mut output = encoder+ .allocate_constant_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;let use_mxu = arguments.suffix_length >= 64&& encoder.context().supports_mxu()@@ -137,15 +134,15 @@ impl AttentionGemmCore<Metal> for AttentionGemmMetalCore {};let params = retile_params(AttnParams {- q_strides: [0, (arguments.suffix_length * self.head_dim) as u64, self.head_dim as u64],- k_strides: [0, self.head_dim as u64, (self.num_groups * self.head_dim) as u64],- v_strides: [0, self.head_dim as u64, (self.num_groups * self.head_dim) as u64],- o_strides: [0, self.head_dim as u64, (self.num_q_heads * self.head_dim) as u64],- gqa_factor: (self.num_q_heads / self.num_groups) as u32,+ q_strides: [0, arguments.suffix_length * self.head_dim, self.head_dim],+ k_strides: [0, self.head_dim, self.num_groups * self.head_dim],+ v_strides: [0, self.head_dim, self.num_groups * self.head_dim],+ o_strides: [0, self.head_dim, self.num_q_heads * self.head_dim],+ gqa_factor: self.num_q_heads / self.num_groups,scale: self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),- q_len: arguments.suffix_length as u32,- k_len: (arguments.state_type.physical_prefix_length() + arguments.suffix_length) as u32,- q_off: arguments.state_type.physical_prefix_length() as u32,+ q_len: arguments.suffix_length,+ k_len: arguments.state_type.physical_prefix_length() + arguments.suffix_length,+ q_off: arguments.state_type.physical_prefix_length(),nq_aligned: 0,q_rem: 0,nk: 0,@@ -170,10 +167,10 @@ impl AttentionGemmCore<Metal> for AttentionGemmMetalCore {params,arguments.state_type.ring_params(),arguments.trie,- self.sliding_window_size.map(|sliding_window_size| sliding_window_size as u32),+ self.sliding_window_size,arguments.sinks,- self.num_q_heads as u32,- arguments.suffix_length as u32,+ self.num_q_heads,+ arguments.suffix_length,encoder,);Ok(output)diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex 728c090e4..b19f733f1 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -82,14 +82,14 @@ KERNEL(AttentionGemm)(using InputType = metal::conditional_t<USE_MXU, T, AccumType>;- q += batch_idx * params.q_strides[0] + head_idx * params.q_strides[1] ++ q += batch_idx * uint64_t(params.q_strides[0]) + head_idx * uint64_t(params.q_strides[1]) +q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.q_strides[2];const int kv_head_idx = int(head_idx) / params.gqa_factor;- k += batch_idx * params.k_strides[0] + int64_t(kv_head_idx) * params.k_strides[1];- v += batch_idx * params.v_strides[0] + int64_t(kv_head_idx) * params.v_strides[1];+ k += batch_idx * uint64_t(params.k_strides[0]) + int64_t(kv_head_idx) * params.k_strides[1];+ v += batch_idx * uint64_t(params.v_strides[0]) + int64_t(kv_head_idx) * params.v_strides[1];- o += batch_idx * params.o_strides[0] + head_idx * params.o_strides[1] ++ o += batch_idx * uint64_t(params.o_strides[0]) + head_idx * uint64_t(params.o_strides[1]) +q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.o_strides[2];if (is_trie) {diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rsindex 716f6086c..b23317a9f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs@@ -3,7 +3,6 @@ use super::super::{DeltaNetChunkedGramMetalKernel, DeltaNetChunkedOutputAndStateMetalKernel, DeltaNetPrefillPrepMetalKernel,};use crate::{- array::size_for_shape,backends::{common::{Backend, Encoder,@@ -17,13 +16,14 @@ use crate::{data_type::DataType,};-const MXU_MIN_T: usize = 256;-const CHUNK_SIZE: usize = 64;-const BLOCK_SIZE: usize = 16;-const VT: usize = 32;+const MXU_MIN_T: u32 = 256;+const CHUNK_SIZE: u32 = 64;+const BLOCK_SIZE: u32 = 16;+const VT: u32 = 32;+const INNER_DATA_TYPE: DataType = DataType::F32;pub struct MetalDeltaNetChunkedPrefill {- min_t: usize,+ min_t: u32,prep: DeltaNetPrefillPrepMetalKernel,cumsum: DeltaNetChunkedCumsumMetalKernel,gram: DeltaNetChunkedGramMetalKernel,@@ -49,23 +49,30 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {let scratch_data_type = if outer_data_type == DataType::BF16 {DataType::BF16} else {- DataType::F32+ INNER_DATA_TYPE};Ok(Some(Self {min_t: MXU_MIN_T,- prep: DeltaNetPrefillPrepMetalKernel::new(context, outer_data_type, DataType::F32, head_dim, true, false)?,- cumsum: DeltaNetChunkedCumsumMetalKernel::new(context, CHUNK_SIZE as u32)?,- gram: DeltaNetChunkedGramMetalKernel::new(context, head_dim, CHUNK_SIZE as u32)?,- a_diag_inv: DeltaNetChunkedADiagInvMetalKernel::new(context, CHUNK_SIZE as u32)?,- causal_inv: DeltaNetChunkedCausalInvMetalKernel::new(context, CHUNK_SIZE as u32, VT as u32)?,+ prep: DeltaNetPrefillPrepMetalKernel::new(+ context,+ outer_data_type,+ INNER_DATA_TYPE,+ head_dim,+ true,+ false,+ )?,+ cumsum: DeltaNetChunkedCumsumMetalKernel::new(context, CHUNK_SIZE)?,+ gram: DeltaNetChunkedGramMetalKernel::new(context, head_dim, CHUNK_SIZE)?,+ a_diag_inv: DeltaNetChunkedADiagInvMetalKernel::new(context, CHUNK_SIZE)?,+ causal_inv: DeltaNetChunkedCausalInvMetalKernel::new(context, CHUNK_SIZE, VT)?,output_and_state: DeltaNetChunkedOutputAndStateMetalKernel::new(context,outer_data_type,outer_data_type,scratch_data_type,head_dim,- VT as u32,+ VT,true,)?,}))@@ -73,7 +80,7 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {fn should_use(&self,- suffix_len: usize,+ suffix_len: u32,) -> bool {suffix_len >= self.min_t}@@ -88,35 +95,25 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {let num_blocks = CHUNK_SIZE.div_ceil(BLOCK_SIZE);let num_col_pairs = num_blocks.div_ceil(2);- let mut q_norm =- encoder.allocate_scratch(size_for_shape(&[suffix_len * args.key_dim as usize], DataType::F32))?;- let mut k_norm =- encoder.allocate_scratch(size_for_shape(&[suffix_len * args.key_dim as usize], DataType::F32))?;- let mut beta =- encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;- let mut log_decay =- encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;- let mut g = encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;- let mut kk = encoder.allocate_scratch(size_for_shape(- &[num_chunks * args.num_groups as usize * CHUNK_SIZE * CHUNK_SIZE],- DataType::F32,- ))?;- let mut qk = encoder.allocate_scratch(size_for_shape(- &[num_chunks * args.num_heads as usize * CHUNK_SIZE * CHUNK_SIZE],- DataType::F32,- ))?;- let mut a_packed = encoder.allocate_scratch(size_for_shape(- &[num_chunks * args.num_heads as usize * num_blocks * num_col_pairs * BLOCK_SIZE * 2 * BLOCK_SIZE],- DataType::F32,- ))?;- let mut a_inv = encoder.allocate_scratch(size_for_shape(- &[num_chunks * args.num_heads as usize * num_blocks * BLOCK_SIZE * BLOCK_SIZE],- DataType::F32,- ))?;- let mut t_mat = encoder.allocate_scratch(size_for_shape(- &[num_chunks * args.num_heads as usize * CHUNK_SIZE * CHUNK_SIZE],- DataType::BF16,- ))?;+ let mut q_norm = encoder.allocate_scratch_for_shape(&[suffix_len, args.key_dim], INNER_DATA_TYPE)?;+ let mut k_norm = encoder.allocate_scratch_for_shape(&[suffix_len, args.key_dim], INNER_DATA_TYPE)?;+ let mut beta = encoder.allocate_scratch_for_shape(&[suffix_len, args.num_heads], INNER_DATA_TYPE)?;+ let mut log_decay = encoder.allocate_scratch_for_shape(&[suffix_len, args.num_heads], INNER_DATA_TYPE)?;+ let mut g = encoder.allocate_scratch_for_shape(&[suffix_len, args.num_heads], INNER_DATA_TYPE)?;+ let mut kk = encoder+ .allocate_scratch_for_shape(&[num_chunks, args.num_groups, CHUNK_SIZE, CHUNK_SIZE], INNER_DATA_TYPE)?;+ let mut qk = encoder+ .allocate_scratch_for_shape(&[num_chunks, args.num_heads, CHUNK_SIZE, CHUNK_SIZE], INNER_DATA_TYPE)?;+ let mut a_packed = encoder.allocate_scratch_for_shape(+ &[num_chunks, args.num_heads, num_blocks, num_col_pairs, BLOCK_SIZE, 2 * BLOCK_SIZE],+ INNER_DATA_TYPE,+ )?;+ let mut a_inv = encoder.allocate_scratch_for_shape(+ &[num_chunks, args.num_heads, num_blocks, BLOCK_SIZE, BLOCK_SIZE],+ INNER_DATA_TYPE,+ )?;+ let mut t_mat = encoder+ .allocate_scratch_for_shape(&[num_chunks, args.num_heads, CHUNK_SIZE, CHUNK_SIZE], DataType::BF16)?;self.prep.encode(args.in_projected,@@ -131,10 +128,10 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {args.num_groups,args.key_dim,args.value_dim,- suffix_len as u32,+ suffix_len,encoder,);- self.cumsum.encode(&log_decay, &mut g, args.num_heads, suffix_len as u32, encoder);+ self.cumsum.encode(&log_decay, &mut g, args.num_heads, suffix_len, encoder);self.gram.encode(&q_norm,&k_norm,@@ -144,7 +141,7 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {args.num_heads,args.num_groups,args.key_dim,- suffix_len as u32,+ suffix_len,encoder,);self.a_diag_inv.encode(@@ -155,10 +152,10 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {&mut a_inv,args.num_heads,args.num_groups,- suffix_len as u32,+ suffix_len,encoder,);- self.causal_inv.encode(&a_packed, &a_inv, &mut t_mat, args.num_heads, suffix_len as u32, encoder);+ self.causal_inv.encode(&a_packed, &a_inv, &mut t_mat, args.num_heads, suffix_len, encoder);self.output_and_state.encode(&q_norm,&k_norm,@@ -174,7 +171,7 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {args.value_head_dim,args.key_dim,args.value_dim,- suffix_len as u32,+ suffix_len,encoder,);Ok(())diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rsindex c786faad6..095d6c161 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs@@ -1,5 +1,4 @@use crate::{- array::size_for_shape,backends::{common::{Allocation, Encoder, Kernels,@@ -14,20 +13,21 @@ use crate::{encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},};-const TOKEN_BLOCK: usize = 16;-const BLOCK_PAIR_WIDTH: usize = 2 * TOKEN_BLOCK;+const TOKEN_BLOCK: u32 = 16;+const BLOCK_PAIR_WIDTH: u32 = 2 * TOKEN_BLOCK;+const INNER_DATA_TYPE: DataType = DataType::F32;struct Layout {- tree_size: usize,- num_blocks: usize,- num_block_pairs: usize,- num_v_heads: usize,- head_v_dim: usize,+ tree_size: u32,+ num_blocks: u32,+ num_block_pairs: u32,+ num_v_heads: u32,+ head_v_dim: u32,}impl Layout {const fn new(- tree_size: usize,+ tree_size: u32,arguments: &TreeVerifyNewArguments,) -> Self {let num_blocks = tree_size.div_ceil(TOKEN_BLOCK);@@ -40,11 +40,11 @@ impl Layout {}}- const fn a_packed_shape(&self) -> [usize; 5] {+ const fn a_packed_shape(&self) -> [u32; 5] {[self.num_v_heads, self.num_blocks, self.num_block_pairs, TOKEN_BLOCK, BLOCK_PAIR_WIDTH]}- const fn a_inverse_shape(&self) -> [usize; 4] {+ const fn a_inverse_shape(&self) -> [u32; 4] {[self.num_v_heads, self.num_blocks, TOKEN_BLOCK, TOKEN_BLOCK]}}@@ -93,21 +93,31 @@ impl DeltaNetTreeVerify<Metal> for MetalDeltaNetTreeVerify {let layout = Layout::new(arguments.tree_size, &self.arguments);let mut h0_indices = encoder.allocate_constant(DataType::I32.size_in_bytes())?;h0_indices.copyin(&[0i32]);- let mut scratch = |shape: &[usize], data_type| encoder.allocate_scratch(size_for_shape(shape, data_type));- let mut prefix = scratch(&[layout.tree_size, layout.num_v_heads], DataType::F32)?;- let mut a_packed = scratch(&layout.a_packed_shape(), DataType::F32)?;- let mut qkd = scratch(&[layout.num_v_heads, layout.tree_size, layout.tree_size], DataType::F32)?;- let mut a_inverse = scratch(&layout.a_inverse_shape(), DataType::F32)?;- let mut kh0 = scratch(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], DataType::F32)?;- let mut u = scratch(&[layout.num_v_heads, layout.tree_size, layout.head_v_dim], DataType::F32)?;- let mut output = scratch(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], self.arguments.data_type)?;- let tree_size = arguments.tree_size.try_into().expect("tree size exceeds u32");- let num_k_heads = self.arguments.num_k_heads.try_into().expect("K head count exceeds u32");- let num_v_heads = self.arguments.num_v_heads.try_into().expect("V head count exceeds u32");- let head_k_dim = self.arguments.head_k_dim.try_into().expect("K head dimension exceeds u32");- let head_v_dim = self.arguments.head_v_dim.try_into().expect("V head dimension exceeds u32");- self.prefix.encode(arguments.trie, arguments.log_decay, &mut prefix, 1, tree_size, num_v_heads, encoder);+ let mut prefix =+ encoder.allocate_scratch_for_shape(&[layout.tree_size, layout.num_v_heads], INNER_DATA_TYPE)?;+ let mut a_packed = encoder.allocate_scratch_for_shape(&layout.a_packed_shape(), INNER_DATA_TYPE)?;+ let mut qkd = encoder+ .allocate_scratch_for_shape(&[layout.num_v_heads, layout.tree_size, layout.tree_size], INNER_DATA_TYPE)?;+ let mut a_inverse = encoder.allocate_scratch_for_shape(&layout.a_inverse_shape(), INNER_DATA_TYPE)?;+ let mut kh0 = encoder+ .allocate_scratch_for_shape(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], INNER_DATA_TYPE)?;+ let mut u = encoder+ .allocate_scratch_for_shape(&[layout.num_v_heads, layout.tree_size, layout.head_v_dim], INNER_DATA_TYPE)?;+ let mut output = encoder.allocate_scratch_for_shape(+ &[layout.tree_size, layout.num_v_heads, layout.head_v_dim],+ self.arguments.data_type,+ )?;++ self.prefix.encode(+ arguments.trie,+ arguments.log_decay,+ &mut prefix,+ 1,+ arguments.tree_size,+ self.arguments.num_v_heads,+ encoder,+ );self.gram.encode(arguments.q,arguments.k,@@ -122,11 +132,11 @@ impl DeltaNetTreeVerify<Metal> for MetalDeltaNetTreeVerify {Some(&mut kh0),1.0,1,- tree_size,- num_k_heads,- num_v_heads,- head_k_dim,- head_v_dim,+ arguments.tree_size,+ self.arguments.num_k_heads,+ self.arguments.num_v_heads,+ self.arguments.head_k_dim,+ self.arguments.head_v_dim,encoder,);self.solve.encode(@@ -139,9 +149,9 @@ impl DeltaNetTreeVerify<Metal> for MetalDeltaNetTreeVerify {Some(&h0_indices),&mut u,1,- tree_size,- num_v_heads,- head_v_dim,+ arguments.tree_size,+ self.arguments.num_v_heads,+ self.arguments.head_v_dim,encoder,);self.out.encode(@@ -154,11 +164,11 @@ impl DeltaNetTreeVerify<Metal> for MetalDeltaNetTreeVerify {&mut output,1.0,1,- tree_size,- num_k_heads,- num_v_heads,- head_k_dim,- head_v_dim,+ arguments.tree_size,+ self.arguments.num_k_heads,+ self.arguments.num_v_heads,+ self.arguments.head_k_dim,+ self.arguments.head_v_dim,encoder,);Ok(output)diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h b/crates/backend-uzu/src/backends/metal/kernel/generated/attention.hindex 55951bc81..55f58b29f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/attention.h@@ -6,10 +6,10 @@ using namespace metal;namespace uzu::attention {typedef struct {- uint64_t q_strides[3];- uint64_t k_strides[3];- uint64_t v_strides[3];- uint64_t o_strides[3];+ uint32_t q_strides[3];+ uint32_t k_strides[3];+ uint32_t v_strides[3];+ uint32_t o_strides[3];uint32_t gqa_factor;float scale;uint32_t q_len;diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.hindex d36ab48c9..aed63aa23 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h@@ -5,5 +5,5 @@using namespace metal;namespace uzu::hadamard_order {-static constant constexpr size_t HADAMARD_TRANSFORM_BLOCK_SIZE = 32;+static constant constexpr uint32_t HADAMARD_TRANSFORM_BLOCK_SIZE = 32;} // namespace uzu::hadamard_orderdiff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.hindex f453dcc9e..fd799486f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h@@ -5,11 +5,11 @@using namespace metal;namespace uzu::weaver {-static constant constexpr size_t CANDIDATES_MAX = 512;+static constant constexpr uint32_t CANDIDATES_MAX = 512;-static constant constexpr size_t TOP_CHILDREN_THREADS = 256;+static constant constexpr uint32_t TOP_CHILDREN_THREADS = 256;-static constant constexpr size_t TOP_CHILDREN_SIMDGROUPS = 8;+static constant constexpr uint32_t TOP_CHILDREN_SIMDGROUPS = 8;enum class FrontierIdx : uint32_t {TokenId = 0,@@ -38,13 +38,13 @@ enum class MetadataIdx : uint32_t {static constant constexpr uint32_t FRONTIER_NO_WINNER = ~0;-static constant constexpr size_t FRONTIER_SELECT_THREADS = 256;+static constant constexpr uint32_t FRONTIER_SELECT_THREADS = 256;-static constant constexpr size_t FRONTIER_SELECT_SIMDGROUPS = 8;+static constant constexpr uint32_t FRONTIER_SELECT_SIMDGROUPS = 8;-static constant constexpr size_t FRONTIER_ENTRIES_PER_THREAD = 8;+static constant constexpr uint32_t FRONTIER_ENTRIES_PER_THREAD = 8;-static constant constexpr size_t FRONTIER_MAX_SLOTS = 2048;+static constant constexpr uint32_t FRONTIER_MAX_SLOTS = 2048;-static constant constexpr size_t FRONTIER_MAX_WIDTH = 32;+static constant constexpr uint32_t FRONTIER_MAX_WIDTH = 32;} // namespace uzu::weaverdiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 2c0a0f998..77ff41cd0 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -66,9 +66,7 @@ impl GemvSpecialization {if shape.d_transform.contains(GemmDTransform::ACCUMULATE) && !shape.n.is_multiple_of(32) {return None;}- if shape.d_transform.contains(GemmDTransform::RHT)- && !shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE as u32)- {+ if shape.d_transform.contains(GemmDTransform::RHT) && !shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE) {return None;}if shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > max_gemv_batch_threshold() {diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rsindex a8917c48a..1d182db33 100644--- a/crates/backend-uzu/src/backends/metal/mod.rs+++ b/crates/backend-uzu/src/backends/metal/mod.rs@@ -14,7 +14,7 @@ use crate::backends::common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE;pub(crate) const METAL_SIMD_SIZE: u32 = 32;const _: () = {- assert!(HADAMARD_TRANSFORM_BLOCK_SIZE == METAL_SIMD_SIZE as usize);+ assert!(HADAMARD_TRANSFORM_BLOCK_SIZE == METAL_SIMD_SIZE);};pub use backend::Metal;diff --git a/crates/backend-uzu/src/bridge/chat_token_backend.rs b/crates/backend-uzu/src/bridge/chat_token_backend.rsindex 5c60f58ae..01fa5dc95 100644--- a/crates/backend-uzu/src/bridge/chat_token_backend.rs+++ b/crates/backend-uzu/src/bridge/chat_token_backend.rs@@ -50,7 +50,7 @@ pub struct UzuChatTokenBackendInstance<B: Backend> {#[cfg(grammar)]tokenizer: Arc<Tokenizer>,stop_token_ids: Vec<i32>,- max_context_length: Option<usize>,+ max_context_length: Option<u32>,}impl<B: Backend> UzuChatTokenBackendInstance<B> {@@ -167,7 +167,7 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {impl<B: Backend> ChatTokenBackendInstance for UzuChatTokenBackendInstance<B> {fn max_context_length(&self) -> Option<usize> {- self.max_context_length+ self.max_context_length.map(|max_context_length| max_context_length as usize)}fn stop_token_ids(&self) -> Option<Box<[u64]>> {diff --git a/crates/backend-uzu/src/bridge/helpers.rs b/crates/backend-uzu/src/bridge/helpers.rsindex c52d2375e..36eb42de9 100644--- a/crates/backend-uzu/src/bridge/helpers.rs+++ b/crates/backend-uzu/src/bridge/helpers.rs@@ -48,7 +48,7 @@ pub fn get_grammar(pub fn get_max_context_length<B: Backend>(model: &LanguageModel<B>,context_length: ContextLength,-) -> Option<usize> {+) -> Option<u32> {match context_length {ContextLength::Default {..@@ -58,7 +58,7 @@ pub fn get_max_context_length<B: Backend>(} => model.max_context_length(),ContextLength::Custom {length,- } => Some(length.max(0) as usize),+ } => Some(length.max(0) as u32),}}@@ -89,7 +89,7 @@ pub fn get_sampling_method<B: Backend>(top_p: top_p.map(|value| value as f32),min_p: min_p.map(|value| value as f32),repetition_penalty: repetition_penalty.map(|value| value as f32),- suffix_repetition_length: suffix_repetition_length.map(|value| value as usize),+ suffix_repetition_length: suffix_repetition_length.map(|value| value as u32),},},}diff --git a/crates/backend-uzu/src/config/classifier.rs b/crates/backend-uzu/src/config/classifier.rsindex 827eab624..5934bf35a 100644--- a/crates/backend-uzu/src/config/classifier.rs+++ b/crates/backend-uzu/src/config/classifier.rs@@ -29,10 +29,10 @@ pub struct ClassifierConfig {pub transformer_config: TransformerConfig,pub prediction_head_config: PredictionHeadConfig,- pub vocab_size: usize,- pub model_dim: usize,- pub hidden_dim: usize,- pub num_labels: usize,+ pub vocab_size: u32,+ pub model_dim: u32,+ pub hidden_dim: u32,+ pub num_labels: u32,pub classifier_pooling: PoolingType,pub output_labels: Option<Box<[String]>>,}diff --git a/crates/backend-uzu/src/config/decoder.rs b/crates/backend-uzu/src/config/decoder.rsindex 1c6d4af35..23ccab305 100644--- a/crates/backend-uzu/src/config/decoder.rs+++ b/crates/backend-uzu/src/config/decoder.rs@@ -10,7 +10,7 @@ pub struct DecoderConfig {pub embedding_config: AnyEmbeddingConfig,pub transformer_config: TransformerConfig,- pub vocab_size: usize,+ pub vocab_size: u32,pub ple_model_config: Option<PLEModelConfig>,pub embedding_norm_config: Option<NormalizationConfig>,}diff --git a/crates/backend-uzu/src/config/dflash.rs b/crates/backend-uzu/src/config/dflash.rsindex 71f06dbd8..c8c4391d6 100644--- a/crates/backend-uzu/src/config/dflash.rs+++ b/crates/backend-uzu/src/config/dflash.rs@@ -7,13 +7,13 @@ use crate::config::{#[uzu_config]pub struct DFlashDraftConfig {- pub model_dim: usize,- pub hidden_dim: usize,- pub block_size: usize,+ pub model_dim: u32,+ pub hidden_dim: u32,+ pub block_size: u32,pub mask_token_id: u64,- pub target_layer_ids: Box<[usize]>,- pub num_target_layers: usize,- pub vocab_size: usize,+ pub target_layer_ids: Box<[u32]>,+ pub num_target_layers: u32,+ pub vocab_size: u32,pub context_projection_config: LinearConfig,pub context_norm_config: NormalizationConfig,pub rope_config: AnyRoPEConfig,diff --git a/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs b/crates/backend-uzu/src/config/mlp/mixture_of_experts.rsindex 8678c539b..624ce4fbf 100644--- a/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs+++ b/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs@@ -11,11 +11,11 @@ pub struct MixtureOfExpertsConfig {pub router_config: LinearConfig,pub routing_function: AnyRoutingFunction,- pub num_routed_experts: usize,- pub num_active_routed_experts: usize,+ pub num_routed_experts: u32,+ pub num_active_routed_experts: u32,pub router_has_biases: bool,- pub num_shared_experts: usize,- pub expert_hidden_dim: usize,+ pub num_shared_experts: u32,+ pub expert_hidden_dim: u32,pub gate_config: Option<LinearConfig>,}diff --git a/crates/backend-uzu/src/config/model/generation.rs b/crates/backend-uzu/src/config/model/generation.rsindex 2be8ff950..e85b2deef 100644--- a/crates/backend-uzu/src/config/model/generation.rs+++ b/crates/backend-uzu/src/config/model/generation.rs@@ -13,5 +13,5 @@ pub struct GenerationConfig {pub repetition_penalty: Option<f32>,pub presence_penalty: Option<Unsupported>,pub frequency_penalty: Option<Unsupported>,- pub suffix_repetition_length: Option<usize>,+ pub suffix_repetition_length: Option<u32>,}diff --git a/crates/backend-uzu/src/config/per_layer_embedding.rs b/crates/backend-uzu/src/config/per_layer_embedding.rsindex dc472f437..adf5034a1 100644--- a/crates/backend-uzu/src/config/per_layer_embedding.rs+++ b/crates/backend-uzu/src/config/per_layer_embedding.rs@@ -4,9 +4,9 @@ use super::{activation::AnyActivation, linear::LinearConfig, normalization::Norm#[uzu_config]pub struct PLEModelConfig {- pub ple_dim: usize,- pub num_layers: usize,- pub ple_vocab_size: usize,+ pub ple_dim: u32,+ pub num_layers: u32,+ pub ple_vocab_size: u32,pub ple_embed_scale: f32,pub model_projection_scale: f32,pub input_scale: f32,@@ -18,6 +18,6 @@ pub struct PLEModelConfig {pub struct PLELayerConfig {pub linear_config: LinearConfig,pub norm_config: NormalizationConfig,- pub ple_dim: usize,+ pub ple_dim: u32,pub activation: AnyActivation,}diff --git a/crates/backend-uzu/src/config/rope/llama_rope.rs b/crates/backend-uzu/src/config/rope/llama_rope.rsindex 8e6ac47ba..9b304b06a 100644--- a/crates/backend-uzu/src/config/rope/llama_rope.rs+++ b/crates/backend-uzu/src/config/rope/llama_rope.rs@@ -3,7 +3,7 @@ use proc_macros::uzu_config;#[uzu_config(super::RoPEConfig)]pub struct LlamaRoPEConfig {pub scaling_factor: f32,- pub original_context_length: usize,+ pub original_context_length: u32,pub low_frequency_factor: f32,pub high_frequency_factor: f32,}diff --git a/crates/backend-uzu/src/config/rope/longrope.rs b/crates/backend-uzu/src/config/rope/longrope.rsindex 403ef4581..f0a65cfd4 100644--- a/crates/backend-uzu/src/config/rope/longrope.rs+++ b/crates/backend-uzu/src/config/rope/longrope.rs@@ -4,6 +4,6 @@ use proc_macros::uzu_config;pub struct LongRoPEConfig {pub short_factor: Box<[f32]>,pub long_factor: Box<[f32]>,- pub original_context_length: usize,+ pub original_context_length: u32,pub scaling_factor: f32,}diff --git a/crates/backend-uzu/src/config/rope/mod.rs b/crates/backend-uzu/src/config/rope/mod.rsindex e493f5ab7..7bf793f99 100644--- a/crates/backend-uzu/src/config/rope/mod.rs+++ b/crates/backend-uzu/src/config/rope/mod.rs@@ -15,6 +15,6 @@ pub mod yarn_rope;)]pub struct RoPEConfig {pub base: f32,- pub max_sequence_length: usize,- pub head_dim: usize,+ pub max_sequence_length: u32,+ pub head_dim: u32,}diff --git a/crates/backend-uzu/src/config/rope/yarn_rope.rs b/crates/backend-uzu/src/config/rope/yarn_rope.rsindex 491b10707..6cb977f3b 100644--- a/crates/backend-uzu/src/config/rope/yarn_rope.rs+++ b/crates/backend-uzu/src/config/rope/yarn_rope.rs@@ -3,7 +3,7 @@ use proc_macros::uzu_config;#[uzu_config(super::RoPEConfig)]pub struct YARNRoPEConfig {pub scaling_factor: f32,- pub original_context_length: usize,+ pub original_context_length: u32,pub beta_fast: f32,pub beta_slow: f32,pub truncate: bool,diff --git a/crates/backend-uzu/src/config/token_mixer/attention.rs b/crates/backend-uzu/src/config/token_mixer/attention.rsindex 9f5f3f09d..96c2b48a1 100644--- a/crates/backend-uzu/src/config/token_mixer/attention.rs+++ b/crates/backend-uzu/src/config/token_mixer/attention.rs@@ -13,13 +13,12 @@ pub struct AttentionConfig {pub query_norm_config: Option<NormalizationConfig>,pub key_norm_config: Option<NormalizationConfig>,- pub num_heads: usize,- pub num_groups: usize,- pub head_dim: usize,+ pub num_heads: u32,+ pub num_groups: u32,+ pub head_dim: u32,pub is_causal: bool,pub scale: Option<f32>,- pub sliding_window_size: Option<usize>,-+ pub sliding_window_size: Option<u32>,pub logit_soft_cap: Option<f32>,pub has_sinks: bool,pub has_qkv_biases: bool,diff --git a/crates/backend-uzu/src/config/token_mixer/delta_net.rs b/crates/backend-uzu/src/config/token_mixer/delta_net.rsindex 960559fc7..d0ae33ab6 100644--- a/crates/backend-uzu/src/config/token_mixer/delta_net.rs+++ b/crates/backend-uzu/src/config/token_mixer/delta_net.rs@@ -11,9 +11,9 @@ pub struct DeltaNetConfig {pub out_proj_config: LinearConfig,pub norm_config: NormalizationConfig,- pub num_heads: usize,- pub num_groups: usize,- pub head_dim: usize,- pub value_head_dim: usize,- pub kernel_size: usize,+ pub num_heads: u32,+ pub num_groups: u32,+ pub head_dim: u32,+ pub value_head_dim: u32,+ pub kernel_size: u32,}diff --git a/crates/backend-uzu/src/config/token_mixer/mamba2.rs b/crates/backend-uzu/src/config/token_mixer/mamba2.rsindex d3139073a..0c17eddf6 100644--- a/crates/backend-uzu/src/config/token_mixer/mamba2.rs+++ b/crates/backend-uzu/src/config/token_mixer/mamba2.rs@@ -11,12 +11,11 @@ pub struct Mamba2Config {pub conv_config: SeparableCausalConvConfig,pub activation: AnyActivation,- pub kernel_size: usize,- pub num_heads: usize,- pub num_groups: usize,- pub head_dim: usize,- pub state_dim: usize,-+ pub kernel_size: u32,+ pub num_heads: u32,+ pub num_groups: u32,+ pub head_dim: u32,+ pub state_dim: u32,pub has_in_biases: bool,pub has_out_biases: bool,}diff --git a/crates/backend-uzu/src/config/token_mixer/short_conv.rs b/crates/backend-uzu/src/config/token_mixer/short_conv.rsindex aa0c11f9c..30c05933d 100644--- a/crates/backend-uzu/src/config/token_mixer/short_conv.rs+++ b/crates/backend-uzu/src/config/token_mixer/short_conv.rs@@ -8,5 +8,5 @@ pub struct ShortConvConfig {pub conv_config: SeparableCausalConvConfig,pub out_projection_config: LinearConfig,- pub kernel_size: usize,+ pub kernel_size: u32,}diff --git a/crates/backend-uzu/src/config/transformer.rs b/crates/backend-uzu/src/config/transformer.rsindex 9bf25d872..d2041d4ec 100644--- a/crates/backend-uzu/src/config/transformer.rs+++ b/crates/backend-uzu/src/config/transformer.rs@@ -6,6 +6,6 @@ use crate::config::{normalization::NormalizationConfig, transformer_layer::Transpub struct TransformerConfig {pub layer_configs: Box<[TransformerLayerConfig]>,pub output_norm_config: NormalizationConfig,- pub model_dim: usize,- pub hidden_dim: usize,+ pub model_dim: u32,+ pub hidden_dim: u32,}diff --git a/crates/backend-uzu/src/config/transformer_layer.rs b/crates/backend-uzu/src/config/transformer_layer.rsindex 7bd395cb6..0fce99a05 100644--- a/crates/backend-uzu/src/config/transformer_layer.rs+++ b/crates/backend-uzu/src/config/transformer_layer.rs@@ -13,9 +13,9 @@ pub struct TransformerLayerConfig {pub pre_mlp_norm_config: NormalizationConfig,pub mlp_config: AnyMLPConfig,pub post_mlp_norm_config: Option<NormalizationConfig>,- pub hidden_dim: Option<usize>,+ pub hidden_dim: Option<u32>,pub ple_config: Option<PLELayerConfig>,pub has_post_layer_scalar: bool,- pub kv_source_layer_index: Option<usize>,+ pub kv_source_layer_index: Option<u32>,pub rope_config: Option<AnyRoPEConfig>,}diff --git a/crates/backend-uzu/src/config/weaver.rs b/crates/backend-uzu/src/config/weaver.rsindex c106830a1..fe2877513 100644--- a/crates/backend-uzu/src/config/weaver.rs+++ b/crates/backend-uzu/src/config/weaver.rs@@ -4,14 +4,14 @@ use crate::config::{linear::LinearConfig, normalization::NormalizationConfig, ro#[uzu_config]pub struct WeaverConfig {- pub model_dim: usize,- pub target_model_dim: usize,- pub target_embedding_dim: usize,- pub num_layers: usize,- pub num_heads: usize,- pub hidden_dim: usize,- pub max_depth: usize,- pub candidate_pool_size: usize,+ pub model_dim: u32,+ pub target_model_dim: u32,+ pub target_embedding_dim: u32,+ pub num_layers: u32,+ pub num_heads: u32,+ pub hidden_dim: u32,+ pub max_depth: u32,+ pub candidate_pool_size: u32,pub linear_config: LinearConfig,pub norm_config: NormalizationConfig,pub rope_config: AnyRoPEConfig,diff --git a/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs b/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rsindex 7c9e08009..cfc1b6a2a 100644--- a/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs+++ b/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs@@ -14,6 +14,6 @@ pub enum IncoherenceProcessingMode {pub struct HybridSpec {pub quantization_spec: Box<AnyWeightMatrixSpec>,pub adapter_spec: Option<Box<AnyWeightMatrixSpec>>,- pub incoherence_block_size: Option<usize>,+ pub incoherence_block_size: Option<u32>,pub incoherence_processing_mode: IncoherenceProcessingMode,}diff --git a/crates/backend-uzu/src/config/weight_matrix/int_spec.rs b/crates/backend-uzu/src/config/weight_matrix/int_spec.rsindex 500478697..ee46d63d2 100644--- a/crates/backend-uzu/src/config/weight_matrix/int_spec.rs+++ b/crates/backend-uzu/src/config/weight_matrix/int_spec.rs@@ -5,7 +5,7 @@ use crate::config::weight_matrix::Layout;#[uzu_config(super::WeightMatrixSpec)]pub struct IntSpec {pub bits: u32,- pub group_size: usize,+ pub group_size: u32,pub is_symmetric: bool,pub layout: Layout,}diff --git a/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs b/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rsindex d056781e4..a4d9d90a3 100644--- a/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs+++ b/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs@@ -2,5 +2,5 @@ use proc_macros::uzu_config;#[uzu_config(super::WeightMatrixSpec)]pub struct LowRankSpec {- pub rank: usize,+ pub rank: u32,}diff --git a/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs b/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rsindex eda4bf6c5..ba60186e0 100644--- a/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs+++ b/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs@@ -5,6 +5,6 @@ use crate::config::weight_matrix::Layout;#[uzu_config(super::WeightMatrixSpec)]pub struct MLXSpec {pub bits: u32,- pub group_size: usize,+ pub group_size: u32,pub layout: Layout,}diff --git a/crates/backend-uzu/src/encodable_block/batch_topology.rs b/crates/backend-uzu/src/encodable_block/batch_topology.rsindex 2c8124247..4606f8562 100644--- a/crates/backend-uzu/src/encodable_block/batch_topology.rs+++ b/crates/backend-uzu/src/encodable_block/batch_topology.rs@@ -17,12 +17,12 @@ impl<'a> BatchTopology<'a> {let mut parents = Box::new_uninit_slice(nodes.len());let mut is_flat = true;- for (index, node) in nodes.iter().enumerate() {+ for (node, index) in nodes.iter().zip(0u32..) {stack.truncate(node.height as usize);- parents[index].write(stack.last().map(|i| *i as i32).unwrap_or(-1));- stack.push(index);+ parents[index as usize].write(stack.last().map(|i| *i as i32).unwrap_or(-1));+ stack.push(index as usize);- is_flat &= node.height == index as u32;+ is_flat &= node.height == index;}let parents = unsafe { parents.assume_init() };@@ -40,8 +40,8 @@ impl<'a> BatchTopology<'a> {self.nodes}- pub fn size(&self) -> usize {- self.nodes.len()+ pub fn size(&self) -> u32 {+ self.nodes.len() as u32}pub fn is_flat(&self) -> bool {diff --git a/crates/backend-uzu/src/encodable_block/classifier.rs b/crates/backend-uzu/src/encodable_block/classifier.rsindex f2af4a3b3..2f5ee208d 100644--- a/crates/backend-uzu/src/encodable_block/classifier.rs+++ b/crates/backend-uzu/src/encodable_block/classifier.rs@@ -1,7 +1,6 @@use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels, gpu_types::trie::TrieNode, kernel::PoolingMeanKernel},config::classifier::{ClassifierConfig, PoolingType},data_type::DataType,@@ -32,7 +31,7 @@ pub enum ClassifierError<B: Backend> {}pub struct Classifier<B: Backend> {- hidden_dim: usize,+ hidden_dim: u32,data_type: DataType,embedding: Embedding<B>,embedding_norm: Normalization<B>,@@ -50,8 +49,8 @@ impl<B: Backend> Classifier<B> {) -> Result<Self, ClassifierError<B>> {let (embedding, _) = Embedding::new(context,- config.vocab_size as u32,- config.transformer_config.model_dim as u32,+ config.vocab_size,+ config.transformer_config.model_dim,&config.embedding_config,¶meter_tree.subtree("embedding"),data_type,@@ -105,14 +104,14 @@ impl<B: Backend> Classifier<B> {})}- pub fn max_context_length(&self) -> Option<usize> {+ pub fn max_context_length(&self) -> Option<u32> {self.transformer.max_context_length()}pub fn encode(&self,token_ids: &Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, ClassifierError<B>> {encoder.push_debug_group("classifier");@@ -124,9 +123,9 @@ impl<B: Backend> Classifier<B> {let nodes = (0..batch_dim).map(|index| TrieNode {- trie_start: index as u32,- trie_end: (batch_dim - 1) as u32,- height: index as u32,+ trie_start: index,+ trie_end: batch_dim - 1,+ height: index,}).collect::<Box<[TrieNode]>>();let hidden = self@@ -136,10 +135,9 @@ impl<B: Backend> Classifier<B> {.output.unwrap();- let mut pooled = encoder- .allocate_scratch(size_for_shape(&[self.hidden_dim], self.data_type))- .map_err(ClassifierError::Backend)?;- self.pooling.encode(&hidden, &mut pooled, batch_dim as u32, self.hidden_dim as u32, 1, encoder);+ let mut pooled =+ encoder.allocate_scratch_for_shape(&[self.hidden_dim], self.data_type).map_err(ClassifierError::Backend)?;+ self.pooling.encode(&hidden, &mut pooled, batch_dim, self.hidden_dim, 1, encoder);let logits = self.prediction_head.encode(pooled, 1, encoder).map_err(ClassifierError::Backend)?;diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rsindex 957ca02d2..6a145194f 100644--- a/crates/backend-uzu/src/encodable_block/decoder.rs+++ b/crates/backend-uzu/src/encodable_block/decoder.rs@@ -58,8 +58,8 @@ impl<B: Backend> Decoder<B> {) -> Result<Self, DecoderError<B>> {let (embedding, readout_input_hadamard_factors) = Embedding::new(context,- config.vocab_size as u32,- config.transformer_config.model_dim as u32,+ config.vocab_size,+ config.transformer_config.model_dim,&config.embedding_config,¶meter_tree.subtree("embedding"),data_type,@@ -85,7 +85,7 @@ impl<B: Backend> Decoder<B> {let per_layer_embedding = if let Some(ple_config) = &config.ple_model_config {assert_eq!(ple_config.num_layers,- config.transformer_config.layer_configs.len(),+ config.transformer_config.layer_configs.len() as u32,"per-layer embedding num_layers must match transformer layer count");Some(PerLayerEmbedding::new(@@ -119,7 +119,7 @@ impl<B: Backend> Decoder<B> {self.transformer.speculation_supported()}- pub fn max_context_length(&self) -> Option<usize> {+ pub fn max_context_length(&self) -> Option<u32> {self.transformer.max_context_length()}@@ -129,7 +129,7 @@ impl<B: Backend> Decoder<B> {pub fn create_empty_state(&self,- max_context_length: Option<usize>,+ max_context_length: Option<u32>,context: &B::Context,) -> Result<TransformerState<B>, B::Error> {self.transformer.create_empty_state(max_context_length, context)@@ -139,8 +139,8 @@ impl<B: Backend> Decoder<B> {&self,token_ids: &Allocation<B>,batch_dim: &BatchTopology,- output_range: Option<Range<usize>>,- hidden_feature_layer_indices: Option<&[usize]>,+ output_range: Option<Range<u32>>,+ hidden_feature_layer_indices: Option<&[u32]>,state: &mut TransformerState<B>,encoder: &mut Encoder<B>,) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {@@ -178,7 +178,12 @@ impl<B: Backend> Decoder<B> {let logits = if let Some(output_range) = output_range {let output = transformer_output.output.as_ref().expect("decoder output range requires transformer output");- Some(self.embedding.encode_readout(output_range.len(), output, self.embedding.data_type(), encoder)?)+ Some(self.embedding.encode_readout(+ output_range.end - output_range.start,+ output,+ self.embedding.data_type(),+ encoder,+ )?)} else {None};diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rsindex 475a0c077..1eba2a76e 100644--- a/crates/backend-uzu/src/encodable_block/dflash.rs+++ b/crates/backend-uzu/src/encodable_block/dflash.rs@@ -3,6 +3,7 @@ use std::any::Any;use thiserror::Error;use crate::{+ array::size_for_shape,backends::common::{Allocation, Backend, Encoder, gpu_types::trie::TrieNode},config::{dflash::DFlashDraftConfig, rope::AnyRoPEConfig, token_mixer::AnyTokenMixerConfig},data_type::DataType,@@ -26,12 +27,12 @@ use crate::{pub struct DFlashState<B: Backend> {layer_states: Box<[Box<dyn MixerState<B>>]>,- context_length: usize,- context_capacity: usize,+ context_length: u32,+ context_capacity: u32,}impl<B: Backend> DFlashState<B> {- pub fn context_length(&self) -> usize {+ pub fn context_length(&self) -> u32 {self.context_length}}@@ -40,15 +41,15 @@ pub struct DFlash<B: Backend> {target_feature_projection: Box<dyn Linear<B>>,projected_feature_norm: Normalization<B>,state_kv_projection: Box<dyn Linear<B>>,- layer_kv_dim: usize,+ layer_kv_dim: u32,layers: Box<[TransformerLayer<B>]>,output_norm: Normalization<B>,rope_config: AnyRoPEConfig,- model_dim: usize,- max_context_length: usize,- block_size: usize,+ model_dim: u32,+ max_context_length: u32,+ block_size: u32,mask_token_id: u32,- target_feature_input_dim: usize,+ target_feature_input_dim: u32,data_type: DataType,}@@ -92,8 +93,13 @@ impl<B: Backend> DFlash<B> {) -> Result<Self, DFlashNewError<B>> {let mask_token_id = config.mask_token_id as u32;assert!(config.block_size <= ATTENTION_SUFFIX_CAPACITY, "DFlash block_size exceeds attention suffix capacity");+ assert_eq!(+ config.num_target_layers,+ config.target_layer_ids.len() as u32,+ "num_target_layers must match target_layer_ids length"+ );let target_feature_projection = <dyn Linear<B>>::new(- config.model_dim * config.target_layer_ids.len(),+ config.model_dim * config.num_target_layers,[config.model_dim],false,context,@@ -116,9 +122,10 @@ impl<B: Backend> DFlash<B> {return Err(DFlashNewError::InvalidAttentionConfig("DFlash layers must use attention mixers"));};let layer_kv_dim = 2 * attention_config.num_groups * attention_config.head_dim;+ let num_layers = config.layer_configs.len() as u32;let state_kv_projection = <dyn Linear<B>>::new(config.model_dim,- [config.layer_configs.len() * layer_kv_dim],+ [num_layers * layer_kv_dim],false,context,data_type,@@ -127,13 +134,13 @@ impl<B: Backend> DFlash<B> {let layers = config.layer_configs.iter()- .enumerate()- .map(|(index, layer_config)| {+ .zip(0u32..)+ .map(|(layer_config, index)| {TransformerLayer::new(context,config.model_dim,config.hidden_dim,- config.layer_configs.len(),+ num_layers,layer_config,index,&layers_tree.subtree(&index.to_string()),@@ -164,18 +171,18 @@ impl<B: Backend> DFlash<B> {max_context_length: *config.rope_config.max_sequence_length(),block_size: config.block_size,mask_token_id,- target_feature_input_dim: config.model_dim * config.target_layer_ids.len(),+ target_feature_input_dim: config.model_dim * config.num_target_layers,data_type,})}- pub fn block_size(&self) -> usize {+ pub fn block_size(&self) -> u32 {self.block_size}pub fn empty_state(&self,- context_capacity: usize,+ context_capacity: u32,context: &B::Context,) -> Result<DFlashState<B>, B::Error> {assert!(context_capacity <= self.max_context_length, "DFlash state capacity exceeds configured RoPE capacity");@@ -195,7 +202,7 @@ impl<B: Backend> DFlash<B> {&self,state: &mut DFlashState<B>,target_features: &[Allocation<B>],- accepted_indices: &[usize],+ accepted_indices: &[u32],encoder: &mut Encoder<B>,) -> Result<(), B::Error> {if accepted_indices.is_empty() {@@ -204,20 +211,20 @@ impl<B: Backend> DFlash<B> {encoder.push_debug_group("dflash accept");- let num_tokens = accepted_indices.len();+ let num_tokens = accepted_indices.len() as u32;let captured_layer_count = self.target_feature_input_dim / self.model_dim;- assert_eq!(target_features.len(), captured_layer_count);- let layer_feature_bytes = self.model_dim * self.data_type.size_in_bytes();+ assert_eq!(target_features.len() as u32, captured_layer_count);+ let layer_feature_bytes = size_for_shape(&[self.model_dim], self.data_type);assert!(target_features.iter().all(|features| features.size() % layer_feature_bytes == 0));let context_length = state.context_length + num_tokens;assert!(context_length <= state.context_capacity, "DFlash state capacity exceeded");assert!(context_length <= self.max_context_length, "DFlash context exceeds configured RoPE capacity");let mut packed_target_features =- encoder.allocate_scratch(num_tokens * self.target_feature_input_dim * self.data_type.size_in_bytes())?;+ encoder.allocate_scratch_for_shape(&[num_tokens, self.target_feature_input_dim], self.data_type)?;for (layer_index, features) in target_features.iter().enumerate() {for (token_index, &accepted_index) in accepted_indices.iter().enumerate() {- let source_start = accepted_index * layer_feature_bytes;+ let source_start = accepted_index as usize * layer_feature_bytes;assert!(source_start + layer_feature_bytes <= features.size(), "accepted index out of feature bounds");let destination_start = (token_index * target_features.len() + layer_index) * layer_feature_bytes;encoder.encode_copy(@@ -235,13 +242,13 @@ impl<B: Backend> DFlash<B> {let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, encoder)?;let projected_kv = self.state_kv_projection.encode(normalized_features, num_tokens, encoder)?;- let layer_kv_bytes = self.layer_kv_dim * self.data_type.size_in_bytes();+ let layer_kv_bytes = size_for_shape(&[self.layer_kv_dim], self.data_type);let kv_chunk = |chunk_index: usize| chunk_index * layer_kv_bytes..(chunk_index + 1) * layer_kv_bytes;let mut layer_key_values = (0..self.layers.len())- .map(|_| encoder.allocate_scratch(num_tokens * layer_kv_bytes))+ .map(|_| encoder.allocate_scratch(num_tokens as usize * layer_kv_bytes)).collect::<Result<Box<[_]>, _>>()?;for (layer_index, key_value) in layer_key_values.iter_mut().enumerate() {- for token_index in 0..num_tokens {+ for token_index in 0..num_tokens as usize {encoder.encode_copy(&projected_kv,kv_chunk(token_index * self.layers.len() + layer_index),@@ -275,7 +282,7 @@ impl<B: Backend> DFlash<B> {state: &mut DFlashState<B>,target_output_token: u32,target_embedding: &Embedding<B>,- batch_size: usize,+ batch_size: u32,encoder: &mut Encoder<B>,) -> Result<DFlashOutput<B>, DFlashEncodeError<B>> {encoder.push_debug_group("dflash draft");@@ -286,7 +293,7 @@ impl<B: Backend> DFlash<B> {"DFlash block positions exceed configured RoPE capacity");- let mut tokens = vec![self.mask_token_id; batch_size];+ let mut tokens = vec![self.mask_token_id; batch_size as usize];tokens[0] = target_output_token;let token_ids = encoder.allocate_constant_from_slice(&tokens).map_err(DFlashEncodeError::Backend)?;@@ -294,9 +301,9 @@ impl<B: Backend> DFlash<B> {let nodes = (0..batch_size).map(|index| TrieNode {- trie_start: index as u32,- trie_end: (batch_size - 1) as u32,- height: index as u32,+ trie_start: index,+ trie_end: batch_size - 1,+ height: index,}).collect::<Box<[_]>>();let batch_topology = BatchTopology::new(&nodes, true);@@ -327,10 +334,11 @@ impl<B: Backend> DFlash<B> {.encode(&hidden, 0, batch_size, Some(&mut residual), encoder).map_err(DFlashEncodeError::Backend)?;- let row_bytes = target_embedding.model_dim() * DataType::BF16.size_in_bytes();+ let row_bytes = size_for_shape(&[target_embedding.model_dim()], DataType::BF16);+ let lookahead_rows = row_bytes..batch_size as usize * row_bytes;let mut lookahead_hidden =- encoder.allocate_scratch((batch_size - 1) * row_bytes).map_err(DFlashEncodeError::Backend)?;- encoder.encode_copy(&draft_hidden, row_bytes..batch_size * row_bytes, &mut lookahead_hidden, ..);+ encoder.allocate_scratch(lookahead_rows.len()).map_err(DFlashEncodeError::Backend)?;+ encoder.encode_copy(&draft_hidden, lookahead_rows, &mut lookahead_hidden, ..);let logits = target_embedding.encode_readout(batch_size - 1, &lookahead_hidden, DataType::F32, encoder)?;encoder.pop_debug_group();diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 13ba866eb..e04b78329 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -2,7 +2,6 @@ use parking_lot::Mutex;use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels,gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,@@ -83,12 +82,12 @@ impl<B: Backend> Embedding<B> {self.data_type}- pub(crate) fn vocab_size(&self) -> usize {- self.vocab_size as usize+ pub(crate) fn vocab_size(&self) -> u32 {+ self.vocab_size}- pub(crate) fn model_dim(&self) -> usize {- self.model_dim as usize+ pub(crate) fn model_dim(&self) -> u32 {+ self.model_dim}fn readout_input_hadamard(&self) -> Option<&InputHadamard<B>> {@@ -136,8 +135,8 @@ impl<B: Backend> Embedding<B> {let table = EmbeddingTable::load_with_spec(context,&embedding_tree,- vocab_size as usize,- model_dim as usize,+ vocab_size,+ model_dim,data_type,spec,None,@@ -162,21 +161,21 @@ impl<B: Backend> Embedding<B> {let output_hadamard_factors = Some(incoherence_signs_tree.leaf("output_signs")?- .validate(&[model_dim as usize], DataType::I32)?+ .validate(&[model_dim], DataType::I32)?.read_allocation()?,);let readout_input_hadamard_factors = Some(incoherence_signs_tree.leaf("output_signs")?- .validate(&[model_dim as usize], DataType::I32)?+ .validate(&[model_dim], DataType::I32)?.read_allocation()?,);let table = EmbeddingTable::load_with_spec(context,&embedding_tree.subtree("quantized"),- vocab_size as usize,- model_dim as usize,+ vocab_size,+ model_dim,data_type,*quantization_spec,output_hadamard_factors,@@ -210,14 +209,14 @@ impl<B: Backend> Embedding<B> {input_embedding_tree.subtree("incoherence_signs").leaf("output_signs")?- .validate(&[model_dim as usize], DataType::I32)?+ .validate(&[model_dim], DataType::I32)?.read_allocation()?,);EmbeddingTable::load_with_spec(context,&input_embedding_tree.subtree("quantized"),- vocab_size as usize,- model_dim as usize,+ vocab_size,+ model_dim,data_type,*quantization_spec,output_hadamard_factors,@@ -226,8 +225,8 @@ impl<B: Backend> Embedding<B> {spec => EmbeddingTable::load_with_spec(context,&input_embedding_tree,- vocab_size as usize,- model_dim as usize,+ vocab_size,+ model_dim,data_type,spec,None,@@ -249,8 +248,8 @@ impl<B: Backend> Embedding<B> {&output_embedding_tree.subtree("quantized"),*quantization_spec,Layout::OutputInput,- vocab_size as usize,- model_dim as usize,+ vocab_size,+ model_dim,data_type,)?;@@ -260,7 +259,7 @@ impl<B: Backend> Embedding<B> {let factors = output_embedding_tree.subtree("incoherence_signs").leaf("input_signs")?- .validate(&[model_dim as usize], DataType::I32)?+ .validate(&[model_dim], DataType::I32)?.read_allocation()?;let kernel = ActivationTransform::input_rht(context, data_type, false).map_err(EmbeddingError::BackendError)?;@@ -279,8 +278,8 @@ impl<B: Backend> Embedding<B> {&output_embedding_tree,spec,Layout::OutputInput,- vocab_size as usize,- model_dim as usize,+ vocab_size,+ model_dim,data_type,)?;UntiedReadout {@@ -346,15 +345,14 @@ impl<B: Backend> Embedding<B> {pub fn encode_lookup(&self,token_ids: &Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, EmbeddingError<B>> {encoder.push_debug_group("embedding lookup");let mut output = encoder- .allocate_scratch(size_for_shape(&[batch_dim, self.model_dim as usize], self.data_type))+ .allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type).map_err(EmbeddingError::BackendError)?;- let batch_dim = batch_dim as u32;let table = match &self.tying {EmbeddingTying::Tied {@@ -375,7 +373,7 @@ impl<B: Backend> Embedding<B> {pub fn encode_readout(&self,- batch_dim: usize,+ batch_dim: u32,input_allocation: &Allocation<B>,output_data_type: DataType,encoder: &mut Encoder<B>,@@ -386,7 +384,7 @@ impl<B: Backend> Embedding<B> {let native_output = output_data_type == self.data_type;let input_hadamard = self.readout_input_hadamard();let mut output_allocation = encoder- .allocate_scratch(size_for_shape(&[batch_dim, self.vocab_size as usize], output_data_type))+ .allocate_scratch_for_shape(&[batch_dim, self.vocab_size], output_data_type).map_err(EmbeddingError::BackendError)?;let (matrix, readout) = self.readout_operands();@@ -399,7 +397,7 @@ impl<B: Backend> Embedding<B> {input_allocation,&mut transformed,&input_hadamard.factors,- batch_dim as u32,+ batch_dim,self.model_dim,encoder,);@@ -418,7 +416,7 @@ impl<B: Backend> Embedding<B> {d: &mut output_allocation,d_transform: MatmulDOps::none(),gather_indices: None,- m: batch_dim as u32,+ m: batch_dim,n: self.vocab_size,k: self.model_dim,};@@ -436,7 +434,7 @@ impl<B: Backend> Embedding<B> {}if let Some(logit_transform) = &self.logit_transform {- let length = (batch_dim * self.vocab_size as usize) as u32;+ let length = batch_dim * self.vocab_size;let kernel = if native_output {&logit_transform.kernel} else {@@ -463,8 +461,8 @@ impl<B: Backend> Embedding<B> {&self,input: &Allocation<B>,token_ids: &Allocation<B>,- rows: usize,- ids_per_row: usize,+ rows: u32,+ ids_per_row: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, EmbeddingError<B>> {encoder.push_debug_group("embedding readout (sparse)");@@ -475,7 +473,7 @@ impl<B: Backend> Embedding<B> {let b = matrix.matmul_b();let mut output = encoder- .allocate_scratch(size_for_shape(&[rows, ids_per_row], self.data_type))+ .allocate_scratch_for_shape(&[rows, ids_per_row], self.data_type).map_err(EmbeddingError::BackendError)?;let mut rht_input: Option<Allocation<B>> = None;@@ -486,7 +484,7 @@ impl<B: Backend> Embedding<B> {input,&mut transformed,&input_hadamard.factors,- rows as u32,+ rows,self.model_dim,encoder,);@@ -517,8 +515,8 @@ impl<B: Backend> Embedding<B> {..MatmulDOps::none()},gather_indices: Some(token_ids),- m: rows as u32,- n: ids_per_row as u32,+ m: rows,+ n: ids_per_row,k: self.model_dim,},encoder,@@ -528,7 +526,7 @@ impl<B: Backend> Embedding<B> {if let Some(logit_transform) = &self.logit_transform&& logit_transform.scale != 1.0{- let length = (rows * ids_per_row) as u32;+ let length = rows * ids_per_row;logit_transform.kernel.encode(&mut output,length,diff --git a/crates/backend-uzu/src/encodable_block/embedding_table.rs b/crates/backend-uzu/src/encodable_block/embedding_table.rsindex 3df7e8e73..250a1e70c 100644--- a/crates/backend-uzu/src/encodable_block/embedding_table.rs+++ b/crates/backend-uzu/src/encodable_block/embedding_table.rs@@ -40,8 +40,8 @@ impl<B: Backend> EmbeddingTable<B> {pub fn load(context: &B::Context,tree: &ParameterTree<B>,- vocab_size: usize,- embedding_dim: usize,+ vocab_size: u32,+ embedding_dim: u32,data_type: DataType,) -> Result<Self, EmbeddingTableError<B>> {let spec = tree.metadata::<AnyWeightMatrixSpec>("spec")?;@@ -51,8 +51,8 @@ impl<B: Backend> EmbeddingTable<B> {pub fn load_with_spec(context: &B::Context,tree: &ParameterTree<B>,- vocab_size: usize,- embedding_dim: usize,+ vocab_size: u32,+ embedding_dim: u32,data_type: DataType,spec: AnyWeightMatrixSpec,output_hadamard_factors: Option<Allocation<B>>,@@ -86,8 +86,8 @@ impl<B: Backend> EmbeddingTable<B> {matrix,lookup,output_hadamard_factors,- vocab_size: vocab_size as u32,- embedding_dim: embedding_dim as u32,+ vocab_size,+ embedding_dim,})}diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rsindex e64f5a23c..213a857ad 100644--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs@@ -2,7 +2,6 @@ use parking_lot::Mutex;use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, Encoder,kernel::{@@ -38,15 +37,15 @@ pub struct LinearMatmul<B: Backend> {matrix: WeightMatrix<B>,biases: Option<Allocation<B>>,output_hadamard_factors: Option<Allocation<B>>,- input_dim: usize,- output_dim: usize,+ input_dim: u32,+ output_dim: u32,output_data_type: DataType,}fn load_biases<B: Backend>(weights_data_type: DataType,output_data_type: DataType,- output_dim: usize,+ output_dim: u32,parameter_tree: Option<&ParameterTree<B>>,) -> Result<Option<Allocation<B>>, LinearMatmulError<B>> {if parameter_tree.is_some() && weights_data_type != output_data_type {@@ -65,8 +64,8 @@ impl<B: Backend> LinearMatmul<B> {pub fn load(context: &B::Context,spec: AnyWeightMatrixSpec,- input_dim: usize,- output_dim: usize,+ input_dim: u32,+ output_dim: u32,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,@@ -112,11 +111,10 @@ impl<B: Backend> LinearMatmul<B> {pub(super) fn encode_with_a(&self,a: MatmulA<'_, B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let mut output =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;+ let mut output = encoder.allocate_scratch_for_shape(&[batch_dim, self.output_dim], self.output_data_type)?;self.kernel.lock().encode(MatmulArguments {@@ -127,9 +125,9 @@ impl<B: Backend> LinearMatmul<B> {d: &mut output,d_transform: self.d_ops(),gather_indices: None,- m: batch_dim as u32,- n: self.output_dim as u32,- k: self.input_dim as u32,+ m: batch_dim,+ n: self.output_dim,+ k: self.input_dim,},encoder,)?;@@ -151,14 +149,14 @@ impl<B: Backend> LinearMatmul<B> {pub(super) fn select_path(&self,- batch_dim: usize,+ batch_dim: u32,context: &B::Context,) -> MatmulPath {let b = self.matmul_b();let shape = MatmulShape {- m: batch_dim as u32,- n: self.output_dim as u32,- k: self.input_dim as u32,+ m: batch_dim,+ n: self.output_dim,+ k: self.input_dim,b_transpose: true,b_leading_dimension: None,b_prologue: b.b_prologue(),@@ -177,7 +175,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {fn encode(&self,input: Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group("matmul");diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rsindex cce6c7ba2..42594c9cf 100644--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs@@ -26,7 +26,7 @@ pub trait Linear<B: Backend>: Send + Sync {fn encode(&self,input: Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error>;}@@ -59,8 +59,8 @@ pub enum OutputHadamardLinearError<B: Backend> {impl<B: Backend> dyn Linear<B> {pub fn new_mixed_precision<const N: usize>(- input_dimension: usize,- output_dimensions: [usize; N],+ input_dimension: u32,+ output_dimensions: [u32; N],has_biases: bool,context: &B::Context,weights_data_type: DataType,@@ -68,7 +68,7 @@ impl<B: Backend> dyn Linear<B> {output_data_type: DataType,parameter_tree: &ParameterTree<B>,) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {- let output_dimension_sum: usize = output_dimensions.iter().sum();+ let output_dimension_sum: u32 = output_dimensions.iter().sum();let weights_tree = parameter_tree.subtree("weights");let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;match spec {@@ -135,8 +135,8 @@ impl<B: Backend> dyn Linear<B> {}pub fn new<const N: usize>(- input_dimension: usize,- output_dimensions: [usize; N],+ input_dimension: u32,+ output_dimensions: [u32; N],has_biases: bool,context: &B::Context,data_type: DataType,@@ -158,8 +158,8 @@ impl<B: Backend> dyn Linear<B> {context: &B::Context,parameter_tree: &ParameterTree<B>,output_factors: Allocation<B>,- input_dim: usize,- output_dim: usize,+ input_dim: u32,+ output_dim: u32,has_biases: bool,weights_data_type: DataType,input_data_type: DataType,@@ -189,8 +189,8 @@ impl<B: Backend> dyn Linear<B> {}pub fn new_extracting_input_hadamard_mixed_precision<const N: usize>(- input_dimension: usize,- output_dimensions: [usize; N],+ input_dimension: u32,+ output_dimensions: [u32; N],has_biases: bool,context: &B::Context,weights_data_type: DataType,@@ -198,7 +198,7 @@ impl<B: Backend> dyn Linear<B> {output_data_type: DataType,parameter_tree: &ParameterTree<B>,) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {- let output_dimension_sum: usize = output_dimensions.iter().sum();+ let output_dimension_sum: u32 = output_dimensions.iter().sum();let weights_tree = parameter_tree.subtree("weights");let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;if let AnyWeightMatrixSpec::HybridSpec(HybridSpec {@@ -217,7 +217,7 @@ impl<B: Backend> dyn Linear<B> {input_dimension,input_data_type,output_data_type,- ACTIVATION_SCALE_GROUP_SIZE as usize,+ ACTIVATION_SCALE_GROUP_SIZE,)}) {let linear = RHTLinearWrapper::new(@@ -269,8 +269,8 @@ impl<B: Backend> dyn Linear<B> {}pub fn new_extracting_input_hadamard<const N: usize>(- input_dimension: usize,- output_dimensions: [usize; N],+ input_dimension: u32,+ output_dimensions: [u32; N],has_biases: bool,context: &B::Context,data_type: DataType,diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rsindex 93d771b76..f5866e5b2 100644--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -2,7 +2,6 @@ use parking_lot::Mutex;use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, Encoder,gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,@@ -42,9 +41,9 @@ pub struct QLoRALinearWrapper<B: Backend> {adapter_up_kernel: Mutex<<B::Kernels as Kernels>::MatmulKernel>,adapter_down: Allocation<B>,adapter_up: Allocation<B>,- input_dim: usize,- output_dim: usize,- lora_rank: usize,+ input_dim: u32,+ output_dim: u32,+ lora_rank: u32,weights_data_type: DataType,input_data_type: DataType,}@@ -54,10 +53,10 @@ impl<B: Backend> QLoRALinearWrapper<B> {context: &B::Context,quantization_spec: AnyWeightMatrixSpec,adapter_spec: LowRankSpec,- incoherence_block_size: Option<usize>,+ incoherence_block_size: Option<u32>,incoherence_processing_mode: IncoherenceProcessingMode,- input_dim: usize,- output_dim: usize,+ input_dim: u32,+ output_dim: u32,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,@@ -168,13 +167,13 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {fn encode(&self,input: Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group("linear (qlora)");let mut intermediate =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type))?;+ encoder.allocate_scratch_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type)?;{let mut adapter_kernel = self.adapter_down_kernel.lock();@@ -192,9 +191,9 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {d: &mut intermediate,d_transform: MatmulDOps::none(),gather_indices: None,- m: batch_dim as u32,- n: self.lora_rank as u32,- k: self.input_dim as u32,+ m: batch_dim,+ n: self.lora_rank,+ k: self.input_dim,},encoder,)?;@@ -202,15 +201,8 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {let base_input = if let Some((input_hadamard_kernel, input_factors)) = &self.input_hadamard {let mut base_input =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dim], self.input_data_type))?;- input_hadamard_kernel.encode_fp(- &input,- &mut base_input,- input_factors,- batch_dim as u32,- self.input_dim as u32,- encoder,- );+ encoder.allocate_scratch_for_shape(&[batch_dim, self.input_dim], self.input_data_type)?;+ input_hadamard_kernel.encode_fp(&input, &mut base_input, input_factors, batch_dim, self.input_dim, encoder);base_input} else {input@@ -237,22 +229,16 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {..MatmulDOps::none()},gather_indices: None,- m: batch_dim as u32,- n: self.output_dim as u32,- k: self.lora_rank as u32,+ m: batch_dim,+ n: self.output_dim,+ k: self.lora_rank,},encoder,)?;}if let Some((output_hadamard_kernel, output_factors)) = &self.output_hadamard {- output_hadamard_kernel.encode_fp_in_place(- &mut output,- output_factors,- batch_dim as u32,- self.output_dim as u32,- encoder,- );+ output_hadamard_kernel.encode_fp_in_place(&mut output, output_factors, batch_dim, self.output_dim, encoder);}encoder.pop_debug_group();diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rsindex 78bc41d01..08488ef70 100644--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs@@ -1,7 +1,6 @@use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, Context, DeviceCapabilities, Encoder,gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod},@@ -30,10 +29,10 @@ const INT8_ACTIVATIONS_ENABLED: bool = true;pub(super) fn int8_activations_eligible<B: Backend>(context: &B::Context,spec: &ParsedWeightSpec,- input_dimension: usize,+ input_dimension: u32,input_data_type: DataType,output_data_type: DataType,- activation_group_size: usize,+ activation_group_size: u32,) -> bool {if !INT8_ACTIVATIONS_ENABLED {return false;@@ -47,7 +46,7 @@ pub(super) fn int8_activations_eligible<B: Backend>(let (Some(info), Layout::OutputInput) = (spec.quantization, &spec.layout) else {return false;};- let group_size = info.group_size as usize;+ let group_size = info.group_size;matches!(group_size, 32 | 64 | 128)&& input_dimension.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE)&& input_dimension.is_multiple_of(group_size)@@ -73,14 +72,14 @@ pub struct RHTLinearWrapper<B: Backend> {quantize_transform: Option<ActivationTransform<B>>,input_factors: Allocation<B>,inner_linear: LinearMatmul<B>,- input_dimension: usize,+ input_dimension: u32,}impl<B: Backend> RHTLinearWrapper<B> {pub fn new(context: &B::Context,- input_dimension: usize,- output_dimension: usize,+ input_dimension: u32,+ output_dimension: u32,has_biases: bool,weights_data_type: DataType,input_data_type: DataType,@@ -113,7 +112,7 @@ impl<B: Backend> RHTLinearWrapper<B> {let Some(quantization) = parsed.quantization else {return Err(RHTLinearWrapperError::UnsupportedConfiguration("RHT requires a quantized inner spec".into()));};- let activation_group_size = ACTIVATION_SCALE_GROUP_SIZE as usize;+ let activation_group_size = ACTIVATION_SCALE_GROUP_SIZE;let input_transform = ActivationTransform::input_rht(context, input_data_type, true).map_err(RHTLinearWrapperError::BackendError)?;@@ -144,7 +143,7 @@ impl<B: Backend> RHTLinearWrapper<B> {context,input_data_type,activation_group_size,- emit_group_sums.then_some(quantization.group_size.min(activation_group_size as u32)),+ emit_group_sums.then_some(quantization.group_size.min(activation_group_size)),).map_err(RHTLinearWrapperError::BackendError)?;inner_linear.make_codes_signed();@@ -167,7 +166,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {fn encode(&self,input: Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group("linear (rht)");@@ -176,15 +175,12 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {&& self.inner_linear.select_path(batch_dim, encoder.context()) == MatmulPath::Gemm{let scale_groups_per_row = self.input_dimension.div_ceil(quantize_transform.activation_group_size());- let sum_groups_per_row = quantize_transform- .sum_group_size()- .map(|group_size| self.input_dimension.div_ceil(group_size as usize));- let mut values =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dimension], DataType::I8))?;- let mut scales =- encoder.allocate_scratch(size_for_shape(&[batch_dim, scale_groups_per_row], DataType::F32))?;+ let sum_groups_per_row =+ quantize_transform.sum_group_size().map(|group_size| self.input_dimension.div_ceil(group_size));+ let mut values = encoder.allocate_scratch_for_shape(&[batch_dim, self.input_dimension], DataType::I8)?;+ let mut scales = encoder.allocate_scratch_for_shape(&[batch_dim, scale_groups_per_row], DataType::F32)?;let mut group_sums = sum_groups_per_row- .map(|groups| encoder.allocate_scratch(size_for_shape(&[batch_dim, groups], DataType::I32)))+ .map(|groups| encoder.allocate_scratch_for_shape(&[batch_dim, groups], DataType::I32)).transpose()?;quantize_transform.encode_quantize(@@ -193,8 +189,8 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {&mut scales,group_sums.as_mut(),&self.input_factors,- batch_dim as u32,- self.input_dimension as u32,+ batch_dim,+ self.input_dimension,encoder,);let output = self.inner_linear.encode_with_a(@@ -202,7 +198,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {values: &values,scales: &scales,group_sums: group_sums.as_ref(),- group_size: quantize_transform.activation_group_size() as u32,+ group_size: quantize_transform.activation_group_size(),},batch_dim,encoder,@@ -217,8 +213,8 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {self.input_transform.encode_fp_in_place(&mut input,&self.input_factors,- batch_dim as u32,- self.input_dimension as u32,+ batch_dim,+ self.input_dimension,encoder,);let output = self.inner_linear.encode(input, batch_dim, encoder)?;diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rsindex c5d5e421b..37fb77d5d 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/fallback.rs@@ -1,7 +1,6 @@use parking_lot::Mutex;use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, BufferArg, Encoder, Kernels,kernel::{@@ -14,10 +13,10 @@ use crate::{};pub struct AttentionFallbackCore<B: Backend> {- head_dim: usize,- num_groups: usize,- num_q_heads: usize,- sliding_window_size: Option<usize>,+ head_dim: u32,+ num_groups: u32,+ num_q_heads: u32,+ sliding_window_size: Option<u32>,scale: Option<f32>,data_type: DataType,scatter_scores: <B::Kernels as Kernels>::AttentionFallbackScatterScoresKernel,@@ -70,35 +69,33 @@ impl<B: Backend> AttentionFallbackCore<B> {arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let sequence_length = arguments.state_type.physical_prefix_length() + arguments.suffix_length;+ let suffix_length = arguments.suffix_length;+ let sequence_length = arguments.state_type.physical_prefix_length() + suffix_length;let gqa_factor = self.num_q_heads / self.num_groups;let scale = self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt());+let dt_bytes = self.data_type.size_in_bytes();+ let head_dim_bytes = self.head_dim as usize * dt_bytes;+ let group_rows = (gqa_factor * suffix_length) as usize;- let mut output = encoder.allocate_constant(size_for_shape(- &[arguments.suffix_length, self.num_q_heads, self.head_dim],- self.data_type,- ))?;- let mut scores = encoder.allocate_scratch(size_for_shape(- &[self.num_q_heads * arguments.suffix_length * sequence_length],- self.data_type,- ))?;- let mut group_scores = encoder.allocate_scratch(size_for_shape(- &[gqa_factor * arguments.suffix_length, sequence_length],- self.data_type,- ))?;+ let mut output =+ encoder.allocate_constant_for_shape(&[suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;+ let mut scores =+ encoder.allocate_scratch_for_shape(&[self.num_q_heads, suffix_length, sequence_length], self.data_type)?;+ let mut group_scores =+ encoder.allocate_scratch_for_shape(&[gqa_factor * suffix_length, sequence_length], self.data_type)?;for group_index in 0..self.num_groups {self.matmul.lock().encode(MatmulArguments {a: MatmulA::FullPrecision {values: arguments.queries,- offset: group_index * gqa_factor * arguments.suffix_length * self.head_dim * dt_bytes,+ offset: group_index as usize * group_rows * head_dim_bytes,},b: MatmulB::FullPrecision {- b: (arguments.keys, group_index * self.head_dim * dt_bytes),+ b: (arguments.keys, group_index as usize * head_dim_bytes),},- b_leading_dimension: Some((self.num_groups * self.head_dim) as u32),+ b_leading_dimension: Some(self.num_groups * self.head_dim),b_transpose: true,d: &mut group_scores,d_transform: MatmulDOps {@@ -106,9 +103,9 @@ impl<B: Backend> AttentionFallbackCore<B> {..MatmulDOps::none()},gather_indices: None,- m: (gqa_factor * arguments.suffix_length) as u32,- n: sequence_length as u32,- k: self.head_dim as u32,+ m: gqa_factor * suffix_length,+ n: sequence_length,+ k: self.head_dim,},encoder,)?;@@ -117,58 +114,51 @@ impl<B: Backend> AttentionFallbackCore<B> {&mut scores,arguments.state_type.ring_params(),None::<&Allocation<B>>,- self.sliding_window_size.map(|sliding_window_size| sliding_window_size as u32),- group_index as u32,- gqa_factor as u32,- sequence_length as u32,- arguments.suffix_length as u32,- (gqa_factor * arguments.suffix_length * sequence_length) as u32,+ self.sliding_window_size,+ group_index,+ gqa_factor,+ sequence_length,+ suffix_length,+ gqa_factor * suffix_length * sequence_length,encoder,);}- self.softmax.encode(- &mut scores,- arguments.sinks,- sequence_length as u32,- self.num_q_heads as u32,- arguments.suffix_length as u32,- encoder,- );+ self.softmax.encode(&mut scores, arguments.sinks, sequence_length, self.num_q_heads, suffix_length, encoder);- let mut group_output = encoder- .allocate_scratch(size_for_shape(&[gqa_factor * arguments.suffix_length, self.head_dim], self.data_type))?;+ let mut group_output =+ encoder.allocate_scratch_for_shape(&[gqa_factor * suffix_length, self.head_dim], self.data_type)?;for group_index in 0..self.num_groups {self.matmul.lock().encode(MatmulArguments {a: MatmulA::FullPrecision {values: &scores,- offset: group_index * gqa_factor * arguments.suffix_length * sequence_length * dt_bytes,+ offset: group_index as usize * group_rows * sequence_length as usize * dt_bytes,},b: MatmulB::FullPrecision {- b: (arguments.values, group_index * self.head_dim * dt_bytes),+ b: (arguments.values, group_index as usize * head_dim_bytes),},- b_leading_dimension: Some((self.num_groups * self.head_dim) as u32),+ b_leading_dimension: Some(self.num_groups * self.head_dim),b_transpose: false,d: &mut group_output,d_transform: MatmulDOps::none(),gather_indices: None,- m: (gqa_factor * arguments.suffix_length) as u32,- n: self.head_dim as u32,- k: sequence_length as u32,+ m: gqa_factor * suffix_length,+ n: self.head_dim,+ k: sequence_length,},encoder,)?;self.scatter_values.encode(&group_output,&mut output,- group_index as u32,- gqa_factor as u32,- arguments.suffix_length as u32,- self.num_q_heads as u32,- self.head_dim as u32,- (gqa_factor * arguments.suffix_length * self.head_dim) as u32,+ group_index,+ gqa_factor,+ suffix_length,+ self.num_q_heads,+ self.head_dim,+ gqa_factor * suffix_length * self.head_dim,encoder,);}diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rsindex 456d02d30..419549143 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs@@ -15,14 +15,14 @@ mod single_pass;mod two_pass;pub struct AttentionCoreNewArguments {- pub head_dim: usize,- pub num_groups: usize,- pub num_q_heads: usize,+ pub head_dim: u32,+ pub num_groups: u32,+ pub num_q_heads: u32,pub has_sinks: bool,pub is_kv_cache_ring: bool,pub is_causal: bool,pub is_trie: bool,- pub sliding_window_size: Option<usize>,+ pub sliding_window_size: Option<u32>,pub scale: Option<f32>,pub data_type: DataType,}@@ -31,7 +31,7 @@ pub struct AttentionCoreEncodeArguments<'a, B: Backend, KT: BufferArg<'a, B>, VTpub queries: &'a Allocation<B>,pub keys: KT,pub values: VT,- pub suffix_length: usize,+ pub suffix_length: u32,pub trie: Option<&'a Allocation<B>>,pub sinks: Option<&'a Allocation<B>>,pub state_type: &'a AttentionStateType,diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rsindex 710fbaf0d..a2e89b301 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/single_pass.rs@@ -1,15 +1,14 @@use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, BufferArg, Encoder, Kernels, kernel::AttentionSinglePassKernel},data_type::DataType,encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},};pub struct AttentionSinglePassCore<B: Backend> {- head_dim: usize,- num_groups: usize,- num_q_heads: usize,- sliding_window_size: Option<usize>,+ head_dim: u32,+ num_groups: u32,+ num_q_heads: u32,+ sliding_window_size: Option<u32>,scale: Option<f32>,data_type: DataType,kernel: <B::Kernels as Kernels>::AttentionSinglePassKernel,@@ -23,7 +22,7 @@ impl<B: Backend> AttentionSinglePassCore<B> {let kernel = <B::Kernels as Kernels>::AttentionSinglePassKernel::new(context,arguments.data_type,- arguments.head_dim as u32,+ arguments.head_dim,arguments.has_sinks,arguments.is_kv_cache_ring,arguments.is_causal,@@ -47,28 +46,26 @@ impl<B: Backend> AttentionSinglePassCore<B> {arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let mut output = encoder.allocate_constant(size_for_shape(- &[arguments.suffix_length, self.num_q_heads, self.head_dim],- self.data_type,- ))?;+ let mut output = encoder+ .allocate_constant_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;self.kernel.encode(arguments.queries,arguments.keys,arguments.values,&mut output,- (self.num_q_heads / self.num_groups) as u32,- (arguments.state_type.physical_prefix_length() + arguments.suffix_length) as u32,- self.head_dim as u32,- (self.num_groups * self.head_dim) as u32,- self.head_dim as u32,- (self.num_groups * self.head_dim) as u32,+ self.num_q_heads / self.num_groups,+ arguments.state_type.physical_prefix_length() + arguments.suffix_length,+ self.head_dim,+ self.num_groups * self.head_dim,+ self.head_dim,+ self.num_groups * self.head_dim,arguments.state_type.ring_params(),self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),arguments.trie,- self.sliding_window_size.map(|sliding_window_size| sliding_window_size as u32),+ self.sliding_window_size,arguments.sinks,- self.num_q_heads as u32,- arguments.suffix_length as u32,+ self.num_q_heads,+ arguments.suffix_length,encoder,);diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rsindex bc905558b..305c184dd 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/two_pass.rs@@ -1,5 +1,4 @@use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, BufferArg, Encoder, Kernels,kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},@@ -8,11 +7,14 @@ use crate::{encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},};+const INNER_DATA_TYPE: DataType = DataType::F32;+const TWO_PASS_BLOCKS: u32 = 32;+pub struct AttentionTwoPassCore<B: Backend> {- head_dim: usize,- num_groups: usize,- num_q_heads: usize,- sliding_window_size: Option<usize>,+ head_dim: u32,+ num_groups: u32,+ num_q_heads: u32,+ sliding_window_size: Option<u32>,scale: Option<f32>,data_type: DataType,pass_1: <B::Kernels as Kernels>::AttentionTwoPass1Kernel,@@ -27,7 +29,7 @@ impl<B: Backend> AttentionTwoPassCore<B> {let pass_1 = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(context,arguments.data_type,- arguments.head_dim as u32,+ arguments.head_dim,arguments.has_sinks,arguments.is_kv_cache_ring,arguments.is_causal,@@ -35,11 +37,8 @@ impl<B: Backend> AttentionTwoPassCore<B> {arguments.sliding_window_size.is_some(),)?;- let pass_2 = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(- context,- arguments.data_type,- arguments.head_dim as u32,- )?;+ let pass_2 =+ <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, arguments.data_type, arguments.head_dim)?;Ok(Self {head_dim: arguments.head_dim,@@ -58,14 +57,18 @@ impl<B: Backend> AttentionTwoPassCore<B> {arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let mut partials = encoder.allocate_scratch(size_for_shape(- &[self.num_q_heads * arguments.suffix_length * 32 * self.head_dim],- DataType::F32,- ))?;- let mut sums = encoder- .allocate_scratch(size_for_shape(&[self.num_q_heads * arguments.suffix_length * 32], DataType::F32))?;- let mut maxs = encoder- .allocate_scratch(size_for_shape(&[self.num_q_heads * arguments.suffix_length * 32], DataType::F32))?;+ let mut partials = encoder.allocate_scratch_for_shape(+ &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS, self.head_dim],+ INNER_DATA_TYPE,+ )?;+ let mut sums = encoder.allocate_scratch_for_shape(+ &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS],+ INNER_DATA_TYPE,+ )?;+ let mut maxs = encoder.allocate_scratch_for_shape(+ &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS],+ INNER_DATA_TYPE,+ )?;self.pass_1.encode(arguments.queries,@@ -74,35 +77,25 @@ impl<B: Backend> AttentionTwoPassCore<B> {&mut partials,&mut sums,&mut maxs,- (self.num_q_heads / self.num_groups) as u32,- (arguments.state_type.physical_prefix_length() + arguments.suffix_length) as u32,- self.head_dim as u32,- (self.num_groups * self.head_dim) as u32,- self.head_dim as u32,- (self.num_groups * self.head_dim) as u32,+ self.num_q_heads / self.num_groups,+ arguments.state_type.physical_prefix_length() + arguments.suffix_length,+ self.head_dim,+ self.num_groups * self.head_dim,+ self.head_dim,+ self.num_groups * self.head_dim,arguments.state_type.ring_params(),self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),- self.num_q_heads as u32,- arguments.suffix_length as u32,+ self.num_q_heads,+ arguments.suffix_length,arguments.trie,- self.sliding_window_size.map(|sliding_window_size| sliding_window_size as u32),+ self.sliding_window_size,arguments.sinks,encoder,);- let mut output = encoder.allocate_constant(size_for_shape(- &[arguments.suffix_length, self.num_q_heads, self.head_dim],- self.data_type,- ))?;- self.pass_2.encode(- &partials,- &sums,- &maxs,- &mut output,- self.num_q_heads as u32,- arguments.suffix_length as u32,- encoder,- );+ let mut output = encoder+ .allocate_constant_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;+ self.pass_2.encode(&partials, &sums, &maxs, &mut output, self.num_q_heads, arguments.suffix_length, encoder);Ok(output)}diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rsindex 276c96338..a95e84210 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs@@ -34,12 +34,12 @@ pub(crate) use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, AttentionStatepub mod rope;pub struct Attention<B: Backend> {- head_dim: usize,- num_q_heads: usize,- num_kv_heads: Option<usize>,+ head_dim: u32,+ num_q_heads: u32,+ num_kv_heads: Option<u32>,is_causal: bool,- sliding_window_size: Option<usize>,- max_rope_length: Option<usize>,+ sliding_window_size: Option<u32>,+ max_rope_length: Option<u32>,data_type: DataType,qkv: LinearProjection<B>,prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,@@ -65,7 +65,7 @@ pub enum AttentionNewError<B: Backend> {impl<B: Backend> Attention<B> {pub fn new(- hidden_dim: usize,+ hidden_dim: u32,data_type: DataType,rope_config: Option<&AnyRoPEConfig>,config: &AttentionConfig,@@ -245,13 +245,13 @@ impl<B: Backend> Mixer<B> for Attention<B> {true}- fn max_context_length(&self) -> Option<usize> {+ fn max_context_length(&self) -> Option<u32> {self.max_rope_length}fn create_empty_state(&self,- max_context_length: Option<usize>,+ max_context_length: Option<u32>,context: &B::Context,) -> Result<Box<dyn MixerState<B>>, B::Error> {Ok(Box::new(AttentionState::create_empty(self, max_context_length, context)?))diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rsindex c0f3a4b51..1fc1284ea 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/mode.rs@@ -1,5 +1,4 @@use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, BufferArgMut, Encoder,gpu_types::trie::TrieNode,@@ -31,7 +30,7 @@ impl<B: Backend> LinearProjection<B> {fn project(&self,hidden: Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {let mut projected = self.lin.encode(hidden, batch_dim, encoder)?;@@ -69,7 +68,7 @@ impl<B: Backend> Attention<B> {state.keys.as_mut(),state.values.as_mut(),state.state_type.physical_prefix_length(),- self.num_q_heads as u32,+ self.num_q_heads,precalculated_rope,batch_dim.size(),encoder,@@ -89,21 +88,17 @@ impl<B: Backend> Attention<B> {assert!(batch_dim.is_flat(), "stateless attention doesn't support trie");let qkv = self.qkv.project(hidden, batch_dim.size(), encoder)?;- let mut keys = encoder.allocate_scratch(size_for_shape(- &[batch_dim.size(), num_kv_heads, self.head_dim],- self.data_type,- ))?;- let mut values = encoder.allocate_scratch(size_for_shape(- &[batch_dim.size(), num_kv_heads, self.head_dim],- self.data_type,- ))?;+ let mut keys = encoder+ .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)?;+ let mut values = encoder+ .allocate_scratch_for_shape(&[batch_dim.size(), num_kv_heads, self.head_dim], self.data_type)?;let queries = self.prepare_kv_and_queries(&qkv,&mut keys,&mut values,0,- self.num_q_heads as u32,+ self.num_q_heads,precalculated_rope,batch_dim.size(),encoder,@@ -141,7 +136,7 @@ impl<B: Backend> Attention<B> {gate_kernel.encode(&gate.unwrap(),&mut attention_output,- (batch_dim.size() * self.num_q_heads * self.head_dim) as u32,+ batch_dim.size() * (self.num_q_heads * self.head_dim),encoder,);}@@ -152,7 +147,7 @@ impl<B: Backend> Attention<B> {&self,mut key_value: Allocation<B>,precalculated_rope: &PrecalculatedRoPE<B>,- batch_dim: usize,+ batch_dim: u32,state: &mut AttentionState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {@@ -169,7 +164,7 @@ impl<B: Backend> Attention<B> {batch_dim,encoder,)?;- state.encode_accept(&(0..batch_dim).collect::<Box<[usize]>>(), encoder)?;+ state.encode_accept(&(0..batch_dim).collect::<Box<[u32]>>(), encoder)?;Ok(())}@@ -183,7 +178,7 @@ impl<B: Backend> Attention<B> {let (core, trie) = if batch_dim.is_flat() {(&self.flat_core, None)} else {- let mut trie = encoder.allocate_constant(batch_dim.size() * size_of::<TrieNode>())?;+ let mut trie = encoder.allocate_constant(batch_dim.size() as usize * size_of::<TrieNode>())?;trie.copyin(batch_dim.nodes());(&self.trie_core, Some(trie))};@@ -207,16 +202,16 @@ impl<B: Backend> Attention<B> {input: &Allocation<B>,keys: impl BufferArgMut<'keys, B>,values: impl BufferArgMut<'values, B>,- kv_token_offset: usize,+ kv_token_offset: u32,num_q_heads: u32,precalculated_rope: Option<&PrecalculatedRoPE<B>>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {let mut queries = if num_q_heads == 0 {encoder.allocate_scratch(self.data_type.size_in_bytes())?} else {- encoder.allocate_scratch(size_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type))?+ encoder.allocate_scratch_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type)?};self.prepare.encode(input,@@ -226,11 +221,11 @@ impl<B: Backend> Attention<B> {precalculated_rope.map(|precalculated_rope| &precalculated_rope.cosines),precalculated_rope.map(|precalculated_rope| &precalculated_rope.sines),num_q_heads,- Some(self.num_kv_heads.expect("KV prepare requires KV heads") as u32),- self.head_dim as u32,- precalculated_rope.map(|precalculated_rope| precalculated_rope.dim as u32),- Some(kv_token_offset as u32),- batch_dim as u32,+ Some(self.num_kv_heads.expect("KV prepare requires KV heads")),+ self.head_dim,+ precalculated_rope.map(|precalculated_rope| precalculated_rope.dim),+ Some(kv_token_offset),+ batch_dim,encoder,);Ok(queries)@@ -240,11 +235,11 @@ impl<B: Backend> Attention<B> {&self,query: &Allocation<B>,precalculated_rope: Option<&PrecalculatedRoPE<B>>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {let mut queries =- encoder.allocate_scratch(size_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type))?;+ encoder.allocate_scratch_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type)?;self.prepare.encode(query,&mut queries,@@ -252,12 +247,12 @@ impl<B: Backend> Attention<B> {None::<&mut Allocation<B>>,precalculated_rope.map(|rope| &rope.cosines),precalculated_rope.map(|rope| &rope.sines),- self.num_q_heads as u32,+ self.num_q_heads,None,- self.head_dim as u32,- precalculated_rope.map(|rope| rope.dim as u32),+ self.head_dim,+ precalculated_rope.map(|rope| rope.dim),None,- batch_dim as u32,+ batch_dim,encoder,);Ok(queries)diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rsindex 27504b525..45eb93a93 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rs@@ -28,9 +28,9 @@ pub struct QKVNorm<B: Backend> {query: Option<Head<B>>,key: Option<Head<B>>,value: Option<Head<B>>,- num_q_heads: usize,- num_kv_heads: usize,- head_dim: usize,+ num_q_heads: u32,+ num_kv_heads: u32,+ head_dim: u32,}impl<B: Backend> QKVNorm<B> {@@ -41,9 +41,9 @@ impl<B: Backend> QKVNorm<B> {key_config: Option<NormalizationConfig>,value_config: Option<NormalizationConfig>,parameter_tree: &ParameterTree<B>,- num_q_heads: usize,- num_kv_heads: usize,- head_dim: usize,+ num_q_heads: u32,+ num_kv_heads: u32,+ head_dim: u32,) -> Result<Self, QKVNormError<B>> {let query = query_config.map(|cfg| {@@ -86,7 +86,7 @@ impl<B: Backend> QKVNorm<B> {intermediate_data_type: DataType,config: NormalizationConfig,parameter_tree: Option<&ParameterTree<B>>,- head_dim: usize,+ head_dim: u32,) -> Result<Head<B>, QKVNormError<B>> {let scales = if config.has_scale {Some(@@ -119,7 +119,7 @@ impl<B: Backend> QKVNorm<B> {pub fn encode(&self,qkv: &mut Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {self.encode_packed(qkv, batch_dim, self.num_q_heads, encoder)@@ -128,7 +128,7 @@ impl<B: Backend> QKVNorm<B> {pub fn encode_key_value(&self,key_value: &mut Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {self.encode_packed(key_value, batch_dim, 0, encoder)@@ -137,8 +137,8 @@ impl<B: Backend> QKVNorm<B> {fn encode_packed(&self,buffer: &mut Allocation<B>,- batch_dim: usize,- q_heads: usize,+ batch_dim: u32,+ q_heads: u32,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {encoder.push_debug_group("qkv norm");@@ -157,13 +157,13 @@ impl<B: Backend> QKVNorm<B> {None::<&Allocation<B>>,head.scales.as_ref(),&mut *buffer,- batch_dim as u32,- total_heads as u32,- self.head_dim as u32,+ batch_dim,+ total_heads,+ self.head_dim,head.config.epsilon,head.config.scale_offset.unwrap_or(0.0),- head_offset as u32,- head_count as u32,+ head_offset,+ head_count,head.config.upcast_mode == UpcastMode::FullLayer,encoder,);diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rsindex b873b2d89..97009a5b1 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs@@ -6,13 +6,13 @@ use crate::{pub struct PrecalculatedRoPE<B: Backend> {pub cosines: Allocation<B>,pub sines: Allocation<B>,- pub dim: usize,+ pub dim: u32,}impl<B: Backend> PrecalculatedRoPE<B> {pub fn precalculate(rope_config: &AnyRoPEConfig,- token_positions: &[usize],+ token_positions: &[u32],encoder: &mut Encoder<B>,) -> Result<Self, B::Error> {let head_dim = *rope_config.head_dim();@@ -26,7 +26,8 @@ impl<B: Backend> PrecalculatedRoPE<B> {_ => 1.0,};- let element_count = token_positions.len() * head_dim;+ let element_count = token_positions.len() * head_dim as usize;+ let half_offset = half_dim as usize;let mut sines = vec![0.0; element_count];let mut cosines = vec![0.0; element_count];for pair_index in 0..half_dim {@@ -84,7 +85,7 @@ impl<B: Backend> PrecalculatedRoPE<B> {} else {&config.short_factor};- inverse_frequency / factors[pair_index]+ inverse_frequency / factors[pair_index as usize]},};@@ -92,11 +93,11 @@ impl<B: Backend> PrecalculatedRoPE<B> {let embedding = *token_position as f32 * inverse_frequency;let sine = embedding.sin() * attention_scaling_factor;let cosine = embedding.cos() * attention_scaling_factor;- let row_offset = token_index * head_dim;- sines[row_offset + pair_index] = sine;- sines[row_offset + half_dim + pair_index] = sine;- cosines[row_offset + pair_index] = cosine;- cosines[row_offset + half_dim + pair_index] = cosine;+ let pair_offset = token_index * head_dim as usize + pair_index as usize;+ sines[pair_offset] = sine;+ sines[pair_offset + half_offset] = sine;+ cosines[pair_offset] = cosine;+ cosines[pair_offset + half_offset] = cosine;}}diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/state.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/state.rsindex 6fb9fc64e..ea48bc691 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/state.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/state.rs@@ -1,6 +1,7 @@use std::any::Any;use crate::{+ array::size_for_shape,backends::common::{Backend, Buffer, Context, DeviceCapabilities, Encoder, Kernels, SparseBuffer,gpu_types::{Copy, ring::RingParams},@@ -10,21 +11,21 @@ use crate::{encodable_block::mixer::{MixerState, attention::Attention},};-pub(crate) const ATTENTION_SUFFIX_CAPACITY: usize = 1024; // TODO: remove hardcoded suffix capacity+pub(crate) const ATTENTION_SUFFIX_CAPACITY: u32 = 1024; // TODO: remove hardcoded suffix capacitypub enum AttentionStateType {Full {- length: usize,+ length: u32,},Ring {- offset: usize,- length: usize,- max_length: usize,+ offset: u32,+ length: u32,+ max_length: u32,},}impl AttentionStateType {- pub fn physical_prefix_length(&self) -> usize {+ pub fn physical_prefix_length(&self) -> u32 {match self {Self::Full {length,@@ -47,16 +48,16 @@ impl AttentionStateType {};Some(RingParams {- ring_offset: *offset as u32,- ring_length: *length as u32,+ ring_offset: *offset,+ ring_length: *length,})}}pub struct AttentionState<B: Backend> {- pub cur_context_length: usize,- pub elements_prepared: usize,- pub element_dim: usize,+ pub cur_context_length: u32,+ pub elements_prepared: u32,+ pub element_dim: u32,pub data_type: DataType,pub state_type: AttentionStateType,pub is_sparse: bool,@@ -68,7 +69,7 @@ pub struct AttentionState<B: Backend> {impl<B: Backend> AttentionState<B> {pub fn create_empty(attention: &Attention<B>,- max_context_length: Option<usize>,+ max_context_length: Option<u32>,context: &B::Context,) -> Result<Self, B::Error> {if let Some(max_context_length) = max_context_length {@@ -106,7 +107,7 @@ impl<B: Backend> AttentionState<B> {let max_elements = max_prefix_elements + ATTENTION_SUFFIX_CAPACITY;let element_size = attention.num_kv_heads.unwrap() * attention.head_dim;- let kv_buffer_bytes = max_elements * element_size * data_type.size_in_bytes();+ let kv_buffer_bytes = size_for_shape(&[max_elements, element_size], data_type);let is_ring = matches!(state_type, AttentionStateType::Ring { .. });let is_sparse = !is_ring && context.device_capabilities().contains(DeviceCapabilities::SPARSE_BUFFERS);@@ -139,8 +140,8 @@ impl<B: Backend> AttentionState<B> {impl<B: Backend> MixerState<B> for AttentionState<B> {fn prepare(&mut self,- context_length: usize,- suffix_length: usize,+ context_length: u32,+ suffix_length: u32,context: &B::Context,) -> Result<(), B::Error> {if !self.is_sparse {@@ -149,8 +150,8 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {assert!(suffix_length <= ATTENTION_SUFFIX_CAPACITY, "attention suffix length exceeds hardcoded capacity");let elements_required = context_length + suffix_length;- let bytes_required = elements_required * self.element_dim * self.data_type.size_in_bytes();- let bytes_prepared = self.elements_prepared * self.element_dim * self.data_type.size_in_bytes();+ let bytes_required = size_for_shape(&[elements_required, self.element_dim], self.data_type);+ let bytes_prepared = size_for_shape(&[self.elements_prepared, self.element_dim], self.data_type);let keys = (self.keys.as_mut() as &mut dyn Any).downcast_mut::<B::SparseBuffer>().unwrap();let values = (self.values.as_mut() as &mut dyn Any).downcast_mut::<B::SparseBuffer>().unwrap();@@ -172,7 +173,7 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {fn encode_accept(&mut self,- accepted_indices: &[usize],+ accepted_indices: &[u32],encoder: &mut Encoder<B>,) -> Result<(), B::Error> {assert!(accepted_indices.is_sorted_by(|a, b| a < b), "invalid accepted indicies");@@ -184,15 +185,15 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {let copies = accepted_indices.iter().copied()- .enumerate()- .filter(|(index, accepted_index)| index != accepted_index)- .map(|(index, accepted_index)| Copy {- source: (*length + accepted_index) as u32,- destination: (*length + index) as u32,+ .zip(0u32..)+ .filter(|&(accepted_index, index)| index != accepted_index)+ .map(|(accepted_index, index)| Copy {+ source: *length + accepted_index,+ destination: *length + index,}).collect::<Vec<Copy>>();- *length += accepted_indices.len();+ *length += accepted_indices.len() as u32;copies},@@ -204,8 +205,8 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {let mut copies = Vec::new();for accepted_index in accepted_indices {copies.push(Copy {- source: (*max_length + accepted_index) as u32,- destination: ((*offset + *length) % *max_length) as u32,+ source: *max_length + *accepted_index,+ destination: (*offset + *length) % *max_length,});if length < max_length {@@ -224,12 +225,12 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {self.values.as_mut(),copies_chunk,copies_chunk.len() as u32,- self.element_dim as u32,+ self.element_dim,encoder,);}- self.cur_context_length += accepted_indices.len();+ self.cur_context_length += accepted_indices.len() as u32;Ok(())}diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rsindex ffaf0e3c0..8e8cfa965 100644--- a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs@@ -1,12 +1,9 @@-use std::mem::size_of;-use thiserror::Error;use crate::{array::size_for_shape,backends::common::{Allocation, AllocationType, Backend, Context, Encoder, Kernels,- gpu_types::trie::TrieNode,kernel::{Conv1dPackKernel, ConvTreeScanKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel,DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,@@ -32,9 +29,11 @@ use crate::{pub(crate) mod tree_verify;+const INNER_DATA_TYPE: DataType = DataType::F32;+enum DeltaNetSuffixStatus<B: Backend> {Flat {- suffix_length: usize,+ suffix_length: u32,},Tree {conv_states: Allocation<B>,@@ -56,8 +55,8 @@ pub struct DeltaNetState<B: Backend> {impl<B: Backend> MixerState<B> for DeltaNetState<B> {fn prepare(&mut self,- _context_length: usize,- _suffix_length: usize,+ _context_length: u32,+ _suffix_length: u32,_context: &B::Context,) -> Result<(), B::Error> {Ok(())@@ -65,7 +64,7 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {fn encode_accept(&mut self,- accepted_indices: &[usize],+ accepted_indices: &[u32],encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let suffix_status = self.suffix_status.take().expect("delta net state has no suffix to accept");@@ -85,22 +84,20 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {beta,parents,} => {- assert!(accepted_indices.iter().all(|&index| index < parents.len()));- assert_eq!(parents[accepted_indices[0]], -1);- assert!(accepted_indices.windows(2).all(|edge| parents[edge[1]] == edge[0] as i32));+ assert!(accepted_indices.iter().all(|&index| (index as usize) < parents.len()));+ assert_eq!(parents[accepted_indices[0] as usize], -1);+ assert!(accepted_indices.windows(2).all(|edge| parents[edge[1] as usize] == edge[0] as i32));let conv_state_size = self.conv_state.size();+ let accepted_offset = accepted_index as usize * conv_state_size;encoder.encode_copy(&conv_states,- accepted_index * conv_state_size..(accepted_index + 1) * conv_state_size,+ accepted_offset..accepted_offset + conv_state_size,&mut self.conv_state,..,);- let accepted_indices = accepted_indices.iter().map(|&index| index as u32).collect::<Box<[_]>>();- let mut accepted_indices_buffer =- encoder.allocate_constant(accepted_indices.len() * DataType::U32.size_in_bytes())?;- accepted_indices_buffer.copyin(&accepted_indices);+ let accepted_indices_buffer = encoder.allocate_constant_from_slice(accepted_indices)?;self.state_advance.encode(&k,&v,@@ -118,15 +115,15 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {}pub struct DeltaNet<B: Backend> {- num_heads: usize,- head_dim: usize,- num_groups: usize,- value_head_dim: usize,- key_dim: usize,- value_dim: usize,- conv_dim: usize,- total_proj_dim: usize,- kernel_size: usize,+ num_heads: u32,+ head_dim: u32,+ num_groups: u32,+ value_head_dim: u32,+ key_dim: u32,+ value_dim: u32,+ conv_dim: u32,+ total_proj_dim: u32,+ kernel_size: u32,outer_data_type: DataType,in_projection: Box<dyn Linear<B>>,conv_weight: Allocation<B>,@@ -163,7 +160,7 @@ pub enum DeltaNetNewError<B: Backend> {impl<B: Backend> DeltaNet<B> {pub fn new(- hidden_dim: usize,+ hidden_dim: u32,outer_data_type: DataType,config: &DeltaNetConfig,parameter_tree: &ParameterTree<B>,@@ -188,8 +185,6 @@ impl<B: Backend> DeltaNet<B> {)));}- let inner_data_type = DataType::F32;-let key_dim = config.num_groups * config.head_dim;let value_dim = config.num_heads * config.value_head_dim;let conv_dim = key_dim * 2 + value_dim;@@ -211,16 +206,16 @@ impl<B: Backend> DeltaNet<B> {let conv_tree = parameter_tree.subtree("conv");let conv_weight =- conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], inner_data_type)?.read_allocation()?;+ conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], INNER_DATA_TYPE)?.read_allocation()?;let conv_bias = if conv_config.has_biases {- Some(conv_tree.leaf("biases")?.validate(&[conv_dim], inner_data_type)?.read_allocation()?)+ Some(conv_tree.leaf("biases")?.validate(&[conv_dim], INNER_DATA_TYPE)?.read_allocation()?)} else {None};let conv_update =<B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, outer_data_type, conv_config.has_biases).map_err(DeltaNetNewError::Backend)?;- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type, outer_data_type)+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, INNER_DATA_TYPE, outer_data_type).map_err(DeltaNetNewError::Backend)?;let conv_scan =<B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, outer_data_type, conv_config.has_biases)@@ -228,27 +223,27 @@ impl<B: Backend> DeltaNet<B> {let conv_tree_scan = <B::Kernels as Kernels>::ConvTreeScanKernel::new(context,outer_data_type,- config.kernel_size as u32,+ config.kernel_size,conv_config.has_biases,).map_err(DeltaNetNewError::Backend)?;- let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], inner_data_type)?.read_allocation()?;+ let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], INNER_DATA_TYPE)?.read_allocation()?;let dt_bias =- parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], inner_data_type)?.read_allocation()?;+ parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], INNER_DATA_TYPE)?.read_allocation()?;let norm_weight = parameter_tree.leaf("norm.scales")?- .validate(&[config.value_head_dim], inner_data_type)?+ .validate(&[config.value_head_dim], INNER_DATA_TYPE)?.read_allocation()?;let norm_epsilon = config.norm_config.epsilon;let delta_net_update =- <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, outer_data_type, config.head_dim as u32)+ <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, outer_data_type, config.head_dim).map_err(DeltaNetNewError::Backend)?;let delta_net_prefill_prep = <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context,outer_data_type,- DataType::F32,- config.head_dim as u32,+ INNER_DATA_TYPE,+ config.head_dim,false,false,)@@ -257,13 +252,13 @@ impl<B: Backend> DeltaNet<B> {context,outer_data_type,outer_data_type,- config.head_dim as u32,+ config.head_dim,true,true,).map_err(DeltaNetNewError::Backend)?;let delta_net_prefill =- <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, outer_data_type, config.head_dim as u32)+ <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, outer_data_type, config.head_dim).map_err(DeltaNetNewError::Backend)?;let delta_net_norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, outer_data_type).map_err(DeltaNetNewError::Backend)?;@@ -286,9 +281,8 @@ impl<B: Backend> DeltaNet<B> {None};- let chunked =- <B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(context, outer_data_type, config.head_dim as u32)- .map_err(DeltaNetNewError::Backend)?;+ let chunked = <B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(context, outer_data_type, config.head_dim)+ .map_err(DeltaNetNewError::Backend)?;let out_projection = <dyn Linear<B>>::new_mixed_precision(value_dim,@@ -346,21 +340,19 @@ impl<B: Backend> DeltaNet<B> {) -> Result<Allocation<B>, B::Error> {let tree_verify = self.tree_verify.as_ref().expect("DeltaNet tree verification is unsupported");let tree_size = batch_dim.size();- let mut parents = encoder.allocate_constant(tree_size * DataType::I32.size_in_bytes())?;- parents.copyin(batch_dim.parents());- let mut trie = encoder.allocate_constant(tree_size * size_of::<TrieNode>())?;- trie.copyin(batch_dim.nodes());-- let mut conv_states = encoder- .allocate_scratch(size_for_shape(&[tree_size, self.conv_dim, self.kernel_size - 1], DataType::F32))?;- let mut k = encoder.allocate_scratch(size_for_shape(&[tree_size, self.key_dim], self.outer_data_type))?;- let mut v = encoder.allocate_scratch(size_for_shape(&[tree_size, self.value_dim], self.outer_data_type))?;- let mut beta = encoder.allocate_scratch(size_for_shape(&[tree_size, self.num_heads], DataType::F32))?;- let mut log_decay = encoder.allocate_scratch(size_for_shape(&[tree_size, self.num_heads], DataType::F32))?;-- let mut scratch = |shape: &[usize], data_type| encoder.allocate_scratch(size_for_shape(shape, data_type));- let mut tree_projected = scratch(&[tree_size, self.total_proj_dim], self.outer_data_type)?;- let mut q = scratch(&[tree_size, self.key_dim], self.outer_data_type)?;+ let parents = encoder.allocate_constant_from_slice(batch_dim.parents())?;+ let trie = encoder.allocate_constant_from_slice(batch_dim.nodes())?;++ let mut conv_states =+ encoder.allocate_scratch_for_shape(&[tree_size, self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE)?;+ let mut k = encoder.allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)?;+ let mut v = encoder.allocate_scratch_for_shape(&[tree_size, self.value_dim], self.outer_data_type)?;+ let mut beta = encoder.allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)?;+ let mut log_decay = encoder.allocate_scratch_for_shape(&[tree_size, self.num_heads], INNER_DATA_TYPE)?;++ let mut tree_projected =+ encoder.allocate_scratch_for_shape(&[tree_size, self.total_proj_dim], self.outer_data_type)?;+ let mut q = encoder.allocate_scratch_for_shape(&[tree_size, self.key_dim], self.outer_data_type)?;self.conv_tree_scan.encode(&in_projected,@@ -370,9 +362,9 @@ impl<B: Backend> DeltaNet<B> {&parents,&mut tree_projected,&mut conv_states,- tree_size as u32,- self.total_proj_dim as u32,- self.conv_dim as u32,+ tree_size,+ self.total_proj_dim,+ self.conv_dim,encoder,);@@ -385,11 +377,11 @@ impl<B: Backend> DeltaNet<B> {Some(&mut v),&mut beta,&mut log_decay,- self.num_heads as u32,- self.num_groups as u32,- self.key_dim as u32,- self.value_dim as u32,- tree_size as u32,+ self.num_heads,+ self.num_groups,+ self.key_dim,+ self.value_dim,+ tree_size,encoder,);@@ -410,13 +402,13 @@ impl<B: Backend> DeltaNet<B> {&mut delta_output,&tree_projected,&self.norm_weight,- self.num_heads as u32,- self.value_head_dim as u32,- self.value_dim as u32,- self.conv_dim as u32,- self.total_proj_dim as u32,+ self.num_heads,+ self.value_head_dim,+ self.value_dim,+ self.conv_dim,+ self.total_proj_dim,self.norm_epsilon,- tree_size as u32,+ tree_size,encoder,);@@ -438,22 +430,22 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {self.tree_verify.is_some()}- fn max_context_length(&self) -> Option<usize> {+ fn max_context_length(&self) -> Option<u32> {None}fn create_empty_state(&self,- _max_context_length: Option<usize>,+ _max_context_length: Option<u32>,context: &B::Context,) -> Result<Box<dyn MixerState<B>>, B::Error> {let mut conv_state = context.create_allocation(- size_for_shape(&[self.conv_dim, self.kernel_size - 1], DataType::F32),+ size_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE),AllocationType::Global,)?;let mut ssm_state = context.create_allocation(- size_for_shape(&[self.num_heads, self.value_head_dim, self.head_dim], DataType::F32),+ size_for_shape(&[self.num_heads, self.value_head_dim, self.head_dim], INNER_DATA_TYPE),AllocationType::Global,)?;@@ -465,9 +457,9 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {let state_advance = <B::Kernels as Kernels>::StateAdvanceKernel::new(context,self.outer_data_type,- self.head_dim as u32,- self.num_heads as u32,- self.num_groups as u32,+ self.head_dim,+ self.num_heads,+ self.num_groups,)?;Ok(Box::new(DeltaNetState {@@ -509,16 +501,16 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {}let mut delta_output =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type))?;+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type)?;if batch_dim.size() == 1 {self.conv_update.encode(&self.conv_weight,self.conv_bias.as_ref(),&mut in_projected,&mut state.conv_state,- self.kernel_size as u32,- self.conv_dim as u32,- (self.kernel_size - 1) as u32,+ self.kernel_size,+ self.conv_dim,+ self.kernel_size - 1,encoder,);@@ -529,27 +521,27 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {&self.norm_weight,&mut state.ssm_state,&mut delta_output,- self.num_heads as u32,- self.num_groups as u32,- self.value_head_dim as u32,- self.key_dim as u32,- self.value_dim as u32,+ self.num_heads,+ self.num_groups,+ self.value_head_dim,+ self.key_dim,+ self.value_dim,self.norm_epsilon,encoder,);} else {- let mut padded = encoder.allocate_scratch(size_for_shape(+ let mut padded = encoder.allocate_scratch_for_shape(&[batch_dim.size() + self.kernel_size - 1, self.total_proj_dim],- DataType::F32,- ))?;+ INNER_DATA_TYPE,+ )?;self.conv_pack.encode(&state.conv_state,&in_projected,&mut padded,- (self.kernel_size - 1) as u32,- self.total_proj_dim as u32,- batch_dim.size() as u32,- self.conv_dim as u32,+ self.kernel_size - 1,+ self.total_proj_dim,+ batch_dim.size(),+ self.conv_dim,encoder,);self.conv_scan.encode(@@ -558,12 +550,12 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {self.conv_bias.as_ref(),&mut in_projected,&mut state.conv_state,- batch_dim.size() as u32,- self.kernel_size as u32,- self.total_proj_dim as u32,- (self.kernel_size - 1) as u32,- self.conv_dim as u32,- self.total_proj_dim as u32,+ batch_dim.size(),+ self.kernel_size,+ self.total_proj_dim,+ self.kernel_size - 1,+ self.conv_dim,+ self.total_proj_dim,encoder,);if let Some(chunked) = self.chunked.as_ref().filter(|chunked| chunked.should_use(batch_dim.size())) {@@ -574,24 +566,24 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {dt_bias: &self.dt_bias,ssm_state: &mut state.ssm_state,delta_output: &mut delta_output,- num_heads: self.num_heads as u32,- num_groups: self.num_groups as u32,- value_head_dim: self.value_head_dim as u32,- key_dim: self.key_dim as u32,- value_dim: self.value_dim as u32,+ num_heads: self.num_heads,+ num_groups: self.num_groups,+ value_head_dim: self.value_head_dim,+ key_dim: self.key_dim,+ value_dim: self.value_dim,suffix_len: batch_dim.size(),},encoder,)?;} else {let mut prep_q_norm =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)?;let mut prep_k_norm =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)?;let mut prep_beta =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;let mut prep_decay =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;self.delta_net_prefill_prep.encode(&in_projected,&self.a_log,@@ -601,11 +593,11 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {None::<&mut Allocation<B>>,&mut prep_beta,&mut prep_decay,- self.num_heads as u32,- self.num_groups as u32,- self.key_dim as u32,- self.value_dim as u32,- batch_dim.size() as u32,+ self.num_heads,+ self.num_groups,+ self.key_dim,+ self.value_dim,+ batch_dim.size(),encoder,);self.delta_net_prefill.encode(@@ -616,13 +608,13 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {&in_projected,&mut state.ssm_state,&mut delta_output,- self.num_heads as u32,- self.num_groups as u32,- self.value_head_dim as u32,- self.key_dim as u32,- self.value_dim as u32,- batch_dim.size() as u32,- self.value_head_dim.div_ceil(16) as u32,+ self.num_heads,+ self.num_groups,+ self.value_head_dim,+ self.key_dim,+ self.value_dim,+ batch_dim.size(),+ self.value_head_dim.div_ceil(16),encoder,);}@@ -630,13 +622,13 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {&mut delta_output,&in_projected,&self.norm_weight,- self.num_heads as u32,- self.value_head_dim as u32,- self.value_dim as u32,- self.conv_dim as u32,- self.total_proj_dim as u32,+ self.num_heads,+ self.value_head_dim,+ self.value_dim,+ self.conv_dim,+ self.total_proj_dim,self.norm_epsilon,- batch_dim.size() as u32,+ batch_dim.size(),encoder,);}diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rsindex 233619b08..92c97af2c 100644--- a/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs@@ -6,10 +6,10 @@ use crate::{#[derive(Clone, Copy)]pub struct TreeVerifyNewArguments {pub data_type: DataType,- pub num_k_heads: usize,- pub num_v_heads: usize,- pub head_k_dim: usize,- pub head_v_dim: usize,+ pub num_k_heads: u32,+ pub num_v_heads: u32,+ pub head_k_dim: u32,+ pub head_v_dim: u32,}#[derive(Clone, Copy)]@@ -21,7 +21,7 @@ pub struct TreeVerifyEncodeArguments<'a, B: Backend> {pub log_decay: &'a Allocation<B>,pub beta: &'a Allocation<B>,pub h0: &'a Allocation<B>,- pub tree_size: usize,+ pub tree_size: u32,}#[cfg(test)]diff --git a/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs b/crates/backend-uzu/src/encodable_block/mixer/mamba2.rsindex d84b58016..01e7e0bf0 100644--- a/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs@@ -21,17 +21,19 @@ use crate::{utils::maybe_mut::MaybeMut,};+const INNER_DATA_TYPE: DataType = DataType::F32;+pub struct Mamba2State<B: Backend> {conv_state: Allocation<B>,ssm_state: Allocation<B>,- suffix_length: Option<usize>,+ suffix_length: Option<u32>,}impl<B: Backend> MixerState<B> for Mamba2State<B> {fn prepare(&mut self,- _context_length: usize,- _suffix_length: usize,+ _context_length: u32,+ _suffix_length: u32,_context: &B::Context,) -> Result<(), B::Error> {Ok(())@@ -39,7 +41,7 @@ impl<B: Backend> MixerState<B> for Mamba2State<B> {fn encode_accept(&mut self,- accepted_indices: &[usize],+ accepted_indices: &[u32],_encoder: &mut Encoder<B>,) -> Result<(), <B as Backend>::Error> {assert!(self.suffix_length.take() == Some(*accepted_indices.last().unwrap() + 1));@@ -53,14 +55,13 @@ enum Mamba2SSDPrefillVariant<B: Backend> {}pub struct Mamba2<B: Backend> {- kernel_size: usize,- num_heads: usize,- num_groups: usize,- head_dim: usize,- state_dim: usize,- inner_dim: usize,- conv_dim: usize,- inner_data_type: DataType,+ kernel_size: u32,+ num_heads: u32,+ num_groups: u32,+ head_dim: u32,+ state_dim: u32,+ inner_dim: u32,+ conv_dim: u32,activation_type: ActivationType,in_projection: Box<dyn Linear<B>>,gate_bias: Allocation<B>,@@ -90,14 +91,12 @@ pub enum Mamba2NewError<B: Backend> {impl<B: Backend> Mamba2<B> {pub fn new(- hidden_dim: usize,+ hidden_dim: u32,outer_data_type: DataType,config: &Mamba2Config,parameter_tree: &ParameterTree<B>,context: &B::Context,) -> Result<(Self, Option<Allocation<B>>), Mamba2NewError<B>> {- let inner_data_type = DataType::F32;-let kernel_size = config.kernel_size;let num_heads = config.num_heads;let num_groups = config.num_groups;@@ -121,48 +120,48 @@ impl<B: Backend> Mamba2<B> {context,outer_data_type,outer_data_type,- inner_data_type,+ INNER_DATA_TYPE,¶meter_tree.subtree("in_projection"),)?;- let gate_bias = parameter_tree.leaf("gate_bias")?.validate(&[inner_dim], inner_data_type)?.read_allocation()?;- let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, inner_data_type)+ let gate_bias = parameter_tree.leaf("gate_bias")?.validate(&[inner_dim], INNER_DATA_TYPE)?.read_allocation()?;+ let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, INNER_DATA_TYPE).map_err(Mamba2NewError::Backend)?;let conv_config = &config.conv_config;let conv_tree = parameter_tree.subtree("conv");let conv_weight =- conv_tree.leaf("weights")?.validate(&[conv_dim, kernel_size], inner_data_type)?.read_allocation()?;+ conv_tree.leaf("weights")?.validate(&[conv_dim, kernel_size], INNER_DATA_TYPE)?.read_allocation()?;let conv_bias = if conv_config.has_biases {- Some(conv_tree.leaf("biases")?.validate(&[conv_dim], inner_data_type)?.read_allocation()?)+ Some(conv_tree.leaf("biases")?.validate(&[conv_dim], INNER_DATA_TYPE)?.read_allocation()?)} else {None};let conv_decode =- <B::Kernels as Kernels>::Conv1dDecodeKernel::new(context, inner_data_type, conv_config.has_biases, true)+ <B::Kernels as Kernels>::Conv1dDecodeKernel::new(context, INNER_DATA_TYPE, conv_config.has_biases, true).map_err(Mamba2NewError::Backend)?;- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type, inner_data_type)+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, INNER_DATA_TYPE, INNER_DATA_TYPE).map_err(Mamba2NewError::Backend)?;let conv_scan =- <B::Kernels as Kernels>::Conv1dScanKernel::new(context, inner_data_type, conv_config.has_biases)+ <B::Kernels as Kernels>::Conv1dScanKernel::new(context, INNER_DATA_TYPE, conv_config.has_biases).map_err(Mamba2NewError::Backend)?;let skip_connection_weight = parameter_tree.leaf("skip_connection_weight")?- .validate(&[num_heads], inner_data_type)?+ .validate(&[num_heads], INNER_DATA_TYPE)?.read_allocation()?;- let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, inner_data_type, true)+ let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, INNER_DATA_TYPE, true).map_err(Mamba2NewError::Backend)?;let ssd_prefill = if state_dim == 64 {Mamba2SSDPrefillVariant::Special64(- <B::Kernels as Kernels>::SSDPrefill64Kernel::new(context, inner_data_type)+ <B::Kernels as Kernels>::SSDPrefill64Kernel::new(context, INNER_DATA_TYPE).map_err(Mamba2NewError::Backend)?,)} else {Mamba2SSDPrefillVariant::Universal(- <B::Kernels as Kernels>::SSDPrefillKernel::new(context, inner_data_type)+ <B::Kernels as Kernels>::SSDPrefillKernel::new(context, INNER_DATA_TYPE).map_err(Mamba2NewError::Backend)?,)};@@ -173,7 +172,7 @@ impl<B: Backend> Mamba2<B> {config.has_out_biases,context,outer_data_type,- inner_data_type,+ INNER_DATA_TYPE,outer_data_type,¶meter_tree.subtree("out_projection"),)?;@@ -187,7 +186,6 @@ impl<B: Backend> Mamba2<B> {state_dim,inner_dim,conv_dim,- inner_data_type,activation_type,in_projection,gate_bias,@@ -212,22 +210,22 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {false}- fn max_context_length(&self) -> Option<usize> {+ fn max_context_length(&self) -> Option<u32> {None}fn create_empty_state(&self,- _max_context_length: Option<usize>,+ _max_context_length: Option<u32>,context: &B::Context,) -> Result<Box<dyn MixerState<B>>, B::Error> {let mut conv_state = context.create_allocation(- size_for_shape(&[self.conv_dim, self.kernel_size - 1], DataType::F32),+ size_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE),AllocationType::Global,)?;let mut ssm_state = context.create_allocation(- size_for_shape(&[self.num_heads, self.head_dim, self.state_dim], DataType::F32),+ size_for_shape(&[self.num_heads, self.head_dim, self.state_dim], INNER_DATA_TYPE),AllocationType::Global,)?;@@ -270,39 +268,30 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {let in_projected = self.in_projection.encode(hidden, batch_dim.size(), encoder)?;let mut conv_inputs =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.conv_dim], self.inner_data_type))?;- let mut gate = encoder.allocate_scratch(size_for_shape(- &[batch_dim.size(), self.num_heads, self.head_dim],- self.inner_data_type,- ))?;- let mut time_step =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], self.inner_data_type))?;+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.conv_dim], INNER_DATA_TYPE)?;+ let mut gate =+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)?;+ let mut time_step = encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)?;self.split_inproj.encode(&in_projected,&mut conv_inputs,&mut gate,&mut time_step,&self.gate_bias,- batch_dim.size() as u32,- (self.conv_dim + self.inner_dim + self.num_heads) as u32,- self.conv_dim as u32,- self.inner_dim as u32,- self.num_heads as u32,+ batch_dim.size(),+ self.conv_dim + self.inner_dim + self.num_heads,+ self.conv_dim,+ self.inner_dim,+ self.num_heads,encoder,);- let mut conv_x = encoder.allocate_scratch(size_for_shape(- &[batch_dim.size(), self.num_heads, self.head_dim],- self.inner_data_type,- ))?;- let mut state_b = encoder.allocate_scratch(size_for_shape(- &[batch_dim.size(), self.num_groups, self.state_dim],- self.inner_data_type,- ))?;- let mut state_c = encoder.allocate_scratch(size_for_shape(- &[batch_dim.size(), self.num_groups, self.state_dim],- self.inner_data_type,- ))?;+ let mut conv_x =+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)?;+ let mut state_b = encoder+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)?;+ let mut state_c = encoder+ .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)?;let state_stride = self.kernel_size - 1;if batch_dim.size() == 1 {self.conv_decode.encode(@@ -314,29 +303,27 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {&mut state_b,&mut state_c,&mut state.conv_state,- self.kernel_size as u32,- self.conv_dim as u32,- state_stride as u32,- self.conv_dim as u32,- batch_dim.size() as u32,- self.inner_dim as u32,- (self.num_groups * self.state_dim) as u32,+ self.kernel_size,+ self.conv_dim,+ state_stride,+ self.conv_dim,+ batch_dim.size(),+ self.inner_dim,+ self.num_groups * self.state_dim,self.activation_type,encoder,);} else {- let mut padded = encoder.allocate_scratch(size_for_shape(- &[batch_dim.size() + state_stride, self.conv_dim],- self.inner_data_type,- ))?;+ let mut padded = encoder+ .allocate_scratch_for_shape(&[batch_dim.size() + state_stride, self.conv_dim], INNER_DATA_TYPE)?;self.conv_pack.encode(&state.conv_state,&conv_inputs,&mut padded,- state_stride as u32,- self.conv_dim as u32,- batch_dim.size() as u32,- self.conv_dim as u32,+ state_stride,+ self.conv_dim,+ batch_dim.size(),+ self.conv_dim,encoder,);self.conv_scan.encode(@@ -347,31 +334,27 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {&mut state_b,&mut state_c,&mut state.conv_state,- batch_dim.size() as u32,- self.kernel_size as u32,- self.conv_dim as u32,- state_stride as u32,- self.conv_dim as u32,- self.inner_dim as u32,- (self.num_groups * self.state_dim) as u32,+ batch_dim.size(),+ self.kernel_size,+ self.conv_dim,+ state_stride,+ self.conv_dim,+ self.inner_dim,+ self.num_groups * self.state_dim,self.activation_type,encoder,);}let mut ssd_output =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.inner_dim], self.inner_data_type))?;- let x_strides = [(self.num_heads * self.head_dim) as u32, self.head_dim as u32, 1];- let dt_strides = [self.num_heads as u32, 1];- let cb_strides = [(self.num_groups * self.state_dim) as u32, self.state_dim as u32, 1];+ encoder.allocate_scratch_for_shape(&[batch_dim.size(), self.inner_dim], INNER_DATA_TYPE)?;+ let x_strides = [self.num_heads * self.head_dim, self.head_dim, 1];+ let dt_strides = [self.num_heads, 1];+ let cb_strides = [self.num_groups * self.state_dim, self.state_dim, 1];let group_size = self.num_heads / self.num_groups;if batch_dim.size() == 1 {- let state_strides = [- (self.num_heads * self.head_dim * self.state_dim) as u32,- (self.head_dim * self.state_dim) as u32,- self.state_dim as u32,- 1,- ];+ let state_strides =+ [self.num_heads * self.head_dim * self.state_dim, self.head_dim * self.state_dim, self.state_dim, 1];self.ssd_update.encode(&conv_x,&time_step,@@ -382,19 +365,19 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {None::<&Allocation<B>>,&mut ssd_output,&mut state.ssm_state,- group_size as u32,- self.state_dim as u32,+ group_size,+ self.state_dim,&x_strides,&dt_strides,&cb_strides,&state_strides,- batch_dim.size() as u32,- self.num_heads as u32,- self.head_dim as u32,+ batch_dim.size(),+ self.num_heads,+ self.head_dim,encoder,);} else {- let state_strides = [(self.head_dim * self.state_dim) as u32, self.state_dim as u32, 1];+ let state_strides = [self.head_dim * self.state_dim, self.state_dim, 1];match &self.ssd_prefill {Mamba2SSDPrefillVariant::Universal(ssd_prefill) => ssd_prefill.encode(&conv_x,@@ -405,15 +388,15 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {&gate,&mut state.ssm_state,&mut ssd_output,- batch_dim.size() as u32,- group_size as u32,- self.state_dim as u32,+ batch_dim.size(),+ group_size,+ self.state_dim,&x_strides,&dt_strides,&cb_strides,&state_strides,- self.num_heads as u32,- self.head_dim as u32,+ self.num_heads,+ self.head_dim,encoder,),Mamba2SSDPrefillVariant::Special64(ssd_prefill) => ssd_prefill.encode(@@ -425,15 +408,15 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {&gate,&mut state.ssm_state,&mut ssd_output,- batch_dim.size() as u32,- group_size as u32,- self.state_dim as u32,+ batch_dim.size(),+ group_size,+ self.state_dim,&x_strides,&dt_strides,&cb_strides,&state_strides,- self.num_heads as u32,- self.head_dim as u32,+ self.num_heads,+ self.head_dim,encoder,),}diff --git a/crates/backend-uzu/src/encodable_block/mixer/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/mod.rsindex c3ae9178c..e38911c01 100644--- a/crates/backend-uzu/src/encodable_block/mixer/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/mod.rs@@ -27,14 +27,14 @@ pub mod short_conv;pub trait MixerState<B: Backend>: Any + Send {fn prepare(&mut self,- context_length: usize,- suffix_length: usize,+ context_length: u32,+ suffix_length: u32,context: &B::Context,) -> Result<(), B::Error>;fn encode_accept(&mut self,- accepted_indices: &[usize],+ accepted_indices: &[u32],encoder: &mut Encoder<B>,) -> Result<(), B::Error>;}@@ -51,11 +51,11 @@ impl<'a, B: Backend> MaybeMut<'a, dyn MixerState<B>> {pub trait Mixer<B: Backend>: Any + Send + Sync {fn speculation_supported(&self) -> bool;- fn max_context_length(&self) -> Option<usize>;+ fn max_context_length(&self) -> Option<u32>;fn create_empty_state(&self,- max_context_length: Option<usize>,+ max_context_length: Option<u32>,context: &B::Context,) -> Result<Box<dyn MixerState<B>>, B::Error>;@@ -83,7 +83,7 @@ pub enum MixerNewError<B: Backend> {impl<B: Backend> dyn Mixer<B> {pub fn new(- hidden_dim: usize,+ hidden_dim: u32,data_type: DataType,rope_config: Option<&AnyRoPEConfig>,config: &AnyTokenMixerConfig,diff --git a/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs b/crates/backend-uzu/src/encodable_block/mixer/short_conv.rsindex 3d285ca7d..bd544b76a 100644--- a/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs@@ -19,7 +19,7 @@ use crate::{enum ShortConvStateSuffixStatus<B: Backend> {Flat {- suffix_length: usize,+ suffix_length: u32,},Trie {conv_states: Allocation<B>,@@ -34,8 +34,8 @@ pub struct ShortConvState<B: Backend> {impl<B: Backend> MixerState<B> for ShortConvState<B> {fn prepare(&mut self,- _context_length: usize,- _suffix_length: usize,+ _context_length: u32,+ _suffix_length: u32,_context: &B::Context,) -> Result<(), B::Error> {Ok(())@@ -43,7 +43,7 @@ impl<B: Backend> MixerState<B> for ShortConvState<B> {fn encode_accept(&mut self,- accepted_indices: &[usize],+ accepted_indices: &[u32],encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let suffix_state_type =@@ -62,9 +62,10 @@ impl<B: Backend> MixerState<B> for ShortConvState<B> {conv_states,} => {let conv_state_size = self.conv_state.size();+ let accepted_offset = accepted_index as usize * conv_state_size;encoder.encode_copy(&conv_states,- (accepted_index * conv_state_size)..((accepted_index + 1) * conv_state_size),+ accepted_offset..accepted_offset + conv_state_size,&mut self.conv_state,..,);@@ -75,9 +76,9 @@ impl<B: Backend> MixerState<B> for ShortConvState<B> {}pub struct ShortConv<B: Backend> {- hidden_dim: usize,+ hidden_dim: u32,data_type: DataType,- kernel_size: usize,+ kernel_size: u32,in_projection: Box<dyn Linear<B>>,out_projection: Box<dyn Linear<B>>,short_conv_pack: <B::Kernels as Kernels>::ShortConvPackKernel,@@ -102,7 +103,7 @@ pub enum ShortConvNewError<B: Backend> {impl<B: Backend> ShortConv<B> {pub fn new(- hidden_dim: usize,+ hidden_dim: u32,data_type: DataType,config: &ShortConvConfig,parameter_tree: &ParameterTree<B>,@@ -183,7 +184,7 @@ impl<B: Backend> ShortConv<B> {state: &mut ShortConvState<B>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let mut conv_output = encoder.allocate_scratch(size_for_shape(&[self.hidden_dim], self.data_type))?;+ let mut conv_output = encoder.allocate_scratch_for_shape(&[self.hidden_dim], self.data_type)?;self.short_conv_decode.encode(in_projected,&self.conv_weight,@@ -192,10 +193,10 @@ impl<B: Backend> ShortConv<B> {&mut conv_output,&mut state.conv_state,1,- self.kernel_size as u32,- self.hidden_dim as u32 * 3,- (self.kernel_size - 1) as u32,- self.hidden_dim as u32,+ self.kernel_size,+ self.hidden_dim * 3,+ self.kernel_size - 1,+ self.hidden_dim,encoder,);Ok(conv_output)@@ -204,27 +205,26 @@ impl<B: Backend> ShortConv<B> {fn encode_prefill_conv(&self,in_projected: &Allocation<B>,- batch_dim: usize,+ batch_dim: u32,state: &mut ShortConvState<B>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {let state_stride = self.kernel_size - 1;let padded_rows = state_stride + batch_dim;- let mut padded = encoder.allocate_scratch(size_for_shape(&[padded_rows, self.hidden_dim], self.data_type))?;+ let mut padded = encoder.allocate_scratch_for_shape(&[padded_rows, self.hidden_dim], self.data_type)?;self.short_conv_pack.encode(&state.conv_state,in_projected,&mut padded,- state_stride as u32,- batch_dim as u32,- self.hidden_dim as u32 * 3,- self.hidden_dim as u32,+ state_stride,+ batch_dim,+ self.hidden_dim * 3,+ self.hidden_dim,encoder,);- let mut conv_output =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;+ let mut conv_output = encoder.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;self.short_conv_prefill.encode(&padded,in_projected,@@ -232,11 +232,11 @@ impl<B: Backend> ShortConv<B> {self.conv_bias.as_ref(),&mut conv_output,&mut state.conv_state,- batch_dim as u32,- self.kernel_size as u32,- self.hidden_dim as u32 * 3,- state_stride as u32,- self.hidden_dim as u32,+ batch_dim,+ self.kernel_size,+ self.hidden_dim * 3,+ state_stride,+ self.hidden_dim,encoder,);Ok(conv_output)@@ -245,15 +245,14 @@ impl<B: Backend> ShortConv<B> {fn encode_trie_conv(&self,in_projected: &Allocation<B>,- batch_dim: usize,+ batch_dim: u32,token_parents: &Allocation<B>,state: &mut ShortConvState<B>,encoder: &mut Encoder<B>,) -> Result<(Allocation<B>, Allocation<B>), B::Error> {- let mut conv_output =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;- let mut conv_states = encoder- .allocate_scratch(size_for_shape(&[batch_dim, self.kernel_size - 1, self.hidden_dim], self.data_type))?;+ let mut conv_output = encoder.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;+ let mut conv_states =+ encoder.allocate_scratch_for_shape(&[batch_dim, self.kernel_size - 1, self.hidden_dim], self.data_type)?;self.short_conv_trie.encode(in_projected,&self.conv_weight,@@ -262,11 +261,11 @@ impl<B: Backend> ShortConv<B> {token_parents,&mut conv_output,&mut conv_states,- batch_dim as u32,- self.kernel_size as u32,- (self.hidden_dim * 3) as u32,- (self.kernel_size - 1) as u32,- self.hidden_dim as u32,+ batch_dim,+ self.kernel_size,+ self.hidden_dim * 3,+ self.kernel_size - 1,+ self.hidden_dim,encoder,);Ok((conv_output, conv_states))@@ -278,13 +277,13 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {true}- fn max_context_length(&self) -> Option<usize> {+ fn max_context_length(&self) -> Option<u32> {None}fn create_empty_state(&self,- _max_context_length: Option<usize>,+ _max_context_length: Option<u32>,context: &B::Context,) -> Result<Box<dyn MixerState<B>>, B::Error> {let mut conv_state = context.create_allocation(@@ -342,8 +341,7 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {});conv_output} else {- let mut token_parents = encoder.allocate_constant(batch_dim.size() * DataType::I32.size_in_bytes())?;- token_parents.copyin(batch_dim.parents());+ let token_parents = encoder.allocate_constant_from_slice(batch_dim.parents())?;let (conv_output, conv_states) =self.encode_trie_conv(&in_projected, batch_dim.size(), &token_parents, state, encoder)?;state.suffix_state = Some(ShortConvStateSuffixStatus::Trie {diff --git a/crates/backend-uzu/src/encodable_block/mlp/dense.rs b/crates/backend-uzu/src/encodable_block/mlp/dense.rsindex eafd4ac4b..a646339f3 100644--- a/crates/backend-uzu/src/encodable_block/mlp/dense.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/dense.rs@@ -32,7 +32,7 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {fn encode(&self,input: Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group("mlp (dense)");diff --git a/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs b/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rsindex 231dcb145..73799c439 100644--- a/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs@@ -1,5 +1,4 @@use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::GatedActMulKernel},config::activation::AnyActivation,data_type::DataType,@@ -8,7 +7,7 @@ use crate::{pub struct MlpGateActMulEncodable<B: Backend> {kernel: <B::Kernels as Kernels>::GatedActMulKernel,activation: AnyActivation,- hidden_dim: usize,+ hidden_dim: u32,data_type: DataType,hadamard_factors: Option<Allocation<B>>,}@@ -18,7 +17,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {context: &B::Context,data_type: DataType,activation: AnyActivation,- hidden_dim: usize,+ hidden_dim: u32,hadamard_factors: Option<Allocation<B>>,) -> Result<Self, B::Error> {let kernel =@@ -36,21 +35,21 @@ impl<B: Backend> MlpGateActMulEncodable<B> {&self,encoder: &mut Encoder<B>,fused_up: &Allocation<B>,- batch_dim: usize,+ batch_dim: u32,) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group("gate act mul");if self.activation.act_type() == ActivationType::IDENTITY {panic!("Identity activation is not supported for kernel")}- let mut hidden = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;+ let mut hidden = encoder.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;self.kernel.encode(fused_up,None::<&Allocation<B>>,&mut hidden,self.hadamard_factors.as_ref(),- self.hidden_dim as u32,- batch_dim as u32,+ self.hidden_dim,+ batch_dim,0,0,self.activation.act_type(),diff --git a/crates/backend-uzu/src/encodable_block/mlp/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/mod.rsindex 73bff433c..57230dd8c 100644--- a/crates/backend-uzu/src/encodable_block/mlp/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/mod.rs@@ -19,7 +19,7 @@ pub trait Mlp<B: Backend>: Send + Sync {fn encode(&self,input: Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error>;}@@ -37,8 +37,8 @@ pub enum MlpBlockError<B: Backend> {impl<B: Backend> dyn Mlp<B> {pub fn new(config: &AnyMLPConfig,- model_dimension: usize,- hidden_dimension: usize,+ model_dimension: u32,+ hidden_dimension: u32,context: &B::Context,parameter_tree: &ParameterTree<B>,data_type: DataType,diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_decode.rs b/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_decode.rsindex 7495f995c..e1ff9e4f7 100644--- a/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_decode.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_decode.rs@@ -1,5 +1,4 @@use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, Encoder,kernel::{@@ -46,42 +45,40 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {args: MoeExpertsTwoPassArguments<B>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- const BLOCK_M: usize = 4;- let h_blocks = args.d_ff.div_ceil(BLOCK_M) as u32;+ const BLOCK_M: u32 = 4;+ let h_blocks = args.d_ff.div_ceil(BLOCK_M);- let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[args.num_routed_experts], DataType::U32))?;- self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts as u32, h_blocks, encoder);+ let mut tile_counts = encoder.allocate_scratch_for_shape(&[args.num_routed_experts], DataType::U32)?;+ self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts, h_blocks, encoder);- let mut tile_offsets =- encoder.allocate_scratch(size_for_shape(&[args.num_routed_experts + 1], DataType::U32))?;- let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[1], DataType::U32))?;- self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts as u32, encoder);+ let mut tile_offsets = encoder.allocate_scratch_for_shape(&[args.num_routed_experts + 1], DataType::U32)?;+ let mut total_tiles = encoder.allocate_scratch_for_shape(&[1], DataType::U32)?;+ self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts, encoder);- let mut row_expert_map = encoder.allocate_scratch(size_for_shape(&[args.total_rows], DataType::U32))?;+ let mut row_expert_map = encoder.allocate_scratch_for_shape(&[args.total_rows], DataType::U32)?;self.row_map.encode(args.expert_offsets,&mut row_expert_map,- args.total_rows as u32,- args.num_routed_experts as u32,+ args.total_rows,+ args.num_routed_experts,encoder,);- let mut tile_map =- encoder.allocate_scratch(size_for_shape(&[args.total_rows * h_blocks as usize * 3], DataType::U32))?;+ let mut tile_map = encoder.allocate_scratch_for_shape(&[args.total_rows, h_blocks, 3], DataType::U32)?;self.build_map.encode(args.expert_offsets,&tile_offsets,&row_expert_map,&mut tile_map,- args.total_rows as u32,+ args.total_rows,h_blocks,encoder,);- let mut dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;+ let mut dispatch_args = encoder.allocate_scratch_for_shape(&[3], DataType::U32)?;self.dispatch.encode(&total_tiles, &mut dispatch_args, 1, encoder);- let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;+ let mut hidden = encoder.allocate_scratch_for_shape(&[args.total_rows, args.d_ff], DataType::F32)?;self.pass_a_indirect.encode(args.x_perm,@@ -89,9 +86,9 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {args.w13_all,&mut hidden,args.up_biases,- args.d_model as u32,- args.d_ff as u32,- args.num_routed_experts as u32,+ args.d_model,+ args.d_ff,+ args.num_routed_experts,args.gate_clip_min,args.gate_clip_max,args.up_clip_min,@@ -102,17 +99,17 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {encoder,);- let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], self.data_type))?;+ let mut output = encoder.allocate_scratch_for_shape(&[args.total_rows, args.d_model], self.data_type)?;self.fused_down.encode(&hidden,&row_expert_map,args.w2_all,args.down_biases,&mut output,- args.total_rows as u32,- args.d_model as u32,- args.d_ff as u32,- args.num_routed_experts as u32,+ args.total_rows,+ args.d_model,+ args.d_ff,+ args.num_routed_experts,encoder,);diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs b/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_prefill.rsindex 08d572785..f53249825 100644--- a/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs@@ -1,5 +1,4 @@use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels,kernel::{@@ -42,31 +41,30 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {args: MoeExpertsTwoPassArguments<B>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[args.num_routed_experts], DataType::U32))?;- self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts as u32, encoder);+ let mut tile_counts = encoder.allocate_scratch_for_shape(&[args.num_routed_experts], DataType::U32)?;+ self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts, encoder);- let mut tile_offsets =- encoder.allocate_scratch(size_for_shape(&[args.num_routed_experts + 1], DataType::U32))?;- let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[8], DataType::U32))?;- self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts as u32, encoder);+ let mut tile_offsets = encoder.allocate_scratch_for_shape(&[args.num_routed_experts + 1], DataType::U32)?;+ let mut total_tiles = encoder.allocate_scratch_for_shape(&[8], DataType::U32)?;+ self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts, encoder);- let mut tile_map = encoder.allocate_scratch(size_for_shape(&[args.total_rows * 3], DataType::U32))?;+ let mut tile_map = encoder.allocate_scratch_for_shape(&[args.total_rows, 3], DataType::U32)?;self.build.encode(args.expert_offsets,&tile_offsets,&tile_counts,&mut tile_map,- args.num_routed_experts as u32,+ args.num_routed_experts,encoder,);- const COL_TILE_FF: usize = 32; // Must match PASSA_BN in kernel- let n_tiles_ff = args.d_ff.div_ceil(COL_TILE_FF) as u32;+ const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in kernel+ let n_tiles_ff = args.d_ff.div_ceil(COL_TILE_FF);- let mut pass_a_dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;+ let mut pass_a_dispatch_args = encoder.allocate_scratch_for_shape(&[3], DataType::U32)?;self.dispatch.encode(&total_tiles, &mut pass_a_dispatch_args, n_tiles_ff, encoder);- let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;+ let mut hidden = encoder.allocate_scratch_for_shape(&[args.total_rows, args.d_ff], DataType::F32)?;encoder.encode_fill(&mut hidden, 0);self.pass_a_indirect.encode(@@ -75,9 +73,9 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {args.w13_all,args.up_biases,&mut hidden,- args.d_model as u32,- args.d_ff as u32,- args.num_routed_experts as u32,+ args.d_model,+ args.d_ff,+ args.num_routed_experts,args.gate_clip_min,args.gate_clip_max,args.up_clip_min,@@ -88,22 +86,22 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {encoder,);- const COL_TILE_MODEL: usize = 64;- let n_tiles_model = args.d_model.div_ceil(COL_TILE_MODEL) as u32;+ const COL_TILE_MODEL: u32 = 64;+ let n_tiles_model = args.d_model.div_ceil(COL_TILE_MODEL);- let mut pass_b_dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;+ let mut pass_b_dispatch_args = encoder.allocate_scratch_for_shape(&[3], DataType::U32)?;self.dispatch.encode(&total_tiles, &mut pass_b_dispatch_args, n_tiles_model, encoder);- let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], self.data_type))?;+ let mut output = encoder.allocate_scratch_for_shape(&[args.total_rows, args.d_model], self.data_type)?;self.pass_b_indirect.encode(&hidden,args.expert_offsets,args.w2_all,args.down_biases,&mut output,- args.d_model as u32,- args.d_ff as u32,- args.num_routed_experts as u32,+ args.d_model,+ args.d_ff,+ args.num_routed_experts,&tile_map,&pass_b_dispatch_args,encoder,@@ -119,10 +117,10 @@ pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {pub w2_all: &'a Allocation<B>,pub up_biases: &'a Allocation<B>,pub down_biases: &'a Allocation<B>,- pub total_rows: usize,- pub d_model: usize,- pub d_ff: usize,- pub num_routed_experts: usize,+ pub total_rows: u32,+ pub d_model: u32,+ pub d_ff: u32,+ pub num_routed_experts: u32,pub gate_clip_min: f32,pub gate_clip_max: f32,pub up_clip_min: f32,diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe/gather.rs b/crates/backend-uzu/src/encodable_block/mlp/moe/gather.rsindex 9e72cc33e..7ada8c503 100644--- a/crates/backend-uzu/src/encodable_block/mlp/moe/gather.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/moe/gather.rs@@ -1,5 +1,4 @@use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels,kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},@@ -37,36 +36,22 @@ impl<B: Backend> MoeGather<B> {input: &Allocation<B>,bucketed_ids: &Allocation<B>,sumk: &Allocation<B>,- batch_dim: usize,- num_active_experts: usize,- d_model: usize,+ batch_dim: u32,+ num_active_experts: u32,+ d_model: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {let mut x_perm =- encoder.allocate_scratch(size_for_shape(&[batch_dim * num_active_experts, d_model], self.data_type))?;+ encoder.allocate_scratch_for_shape(&[batch_dim, num_active_experts, d_model], self.data_type)?;encoder.encode_fill(&mut x_perm, 0);match &self.variant {- MoeGatherVariant::OneD(kernel) => kernel.encode(- input,- bucketed_ids,- &mut x_perm,- sumk,- d_model as u32,- batch_dim as u32,- num_active_experts as u32,- encoder,- ),- MoeGatherVariant::TwoD(kernel) => kernel.encode(- input,- bucketed_ids,- &mut x_perm,- sumk,- d_model as u32,- batch_dim as u32,- num_active_experts as u32,- encoder,- ),+ MoeGatherVariant::OneD(kernel) => {+ kernel.encode(input, bucketed_ids, &mut x_perm, sumk, d_model, batch_dim, num_active_experts, encoder)+ },+ MoeGatherVariant::TwoD(kernel) => {+ kernel.encode(input, bucketed_ids, &mut x_perm, sumk, d_model, batch_dim, num_active_experts, encoder)+ },};Ok(x_perm)diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rsindex 48e7333e1..cf43cd025 100644--- a/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rs@@ -10,7 +10,6 @@ use gather::MoeGather;use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels,gpu_types::ActivationType,@@ -44,10 +43,10 @@ pub struct MoeBlock<B: Backend> {w2: Allocation<B>,up_biases: Allocation<B>,down_biases: Allocation<B>,- model_dim: usize,- hidden_dim: usize,- num_routed_experts: usize,- num_active_experts: usize,+ model_dim: u32,+ hidden_dim: u32,+ num_routed_experts: u32,+ num_active_experts: u32,gate_clip_min: f32,gate_clip_max: f32,up_clip_min: f32,@@ -84,7 +83,7 @@ impl<B: Backend> MoeBlock<B> {pub fn new(context: &B::Context,moe_config: &MixtureOfExpertsConfig,- model_dim: usize,+ model_dim: u32,data_type: DataType,parameter_tree: &ParameterTree<B>,) -> Result<Self, MoeBlockError<B>> {@@ -213,7 +212,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {fn encode(&self,input: Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group("mlp (moe)");@@ -222,10 +221,9 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {let num_blocks = batch_dim.div_ceil(256);let num_tiles = self.num_routed_experts.div_ceil(512);- let mut topk_ids =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.num_active_experts], DataType::I32))?;+ let mut topk_ids = encoder.allocate_scratch_for_shape(&[batch_dim, self.num_active_experts], DataType::I32)?;let mut topk_probs =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.num_active_experts], self.data_type))?;+ encoder.allocate_scratch_for_shape(&[batch_dim, self.num_active_experts], self.data_type)?;encoder.encode_fill(&mut topk_ids, 0xFF);@@ -237,36 +235,36 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {None::<&Allocation<B>>,&mut topk_ids,&mut topk_probs,- batch_dim as u32,- self.model_dim as u32,- self.num_routed_experts as u32,- self.num_active_experts as u32,+ batch_dim,+ self.model_dim,+ self.num_routed_experts,+ self.num_active_experts,self.router_renorm,None::<f32>,None::<f32>,encoder,);- let mut offsets = encoder.allocate_scratch(size_for_shape(&[self.num_routed_experts + 1], DataType::U32))?;- let mut sumk = encoder.allocate_scratch(size_for_shape(&[1], DataType::U32))?;+ let mut offsets = encoder.allocate_scratch_for_shape(&[self.num_routed_experts + 1], DataType::U32)?;+ let mut sumk = encoder.allocate_scratch_for_shape(&[1], DataType::U32)?;let scatter_entries = num_blocks * num_tiles * 512;- let mut partials = encoder.allocate_scratch(size_for_shape(&[scatter_entries], DataType::U32))?;+ let mut partials = encoder.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;self.counts_offsets_kernel.encode(&topk_ids,&mut offsets,&mut sumk,&mut partials,- batch_dim as u32,- self.num_routed_experts as u32,- self.num_active_experts as u32,+ batch_dim,+ self.num_routed_experts,+ self.num_active_experts,encoder,);- let mut block_bases = encoder.allocate_scratch(size_for_shape(&[scatter_entries], DataType::U32))?;- let mut block_alloc = encoder.allocate_scratch(size_for_shape(&[scatter_entries], DataType::U32))?;- let mut bucketed_ids = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::I32))?;- let mut bucketed_probs = encoder.allocate_scratch(size_for_shape(&[total_rows], self.data_type))?;- let mut tok2row = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::I32))?;+ let mut block_bases = encoder.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;+ let mut block_alloc = encoder.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;+ let mut bucketed_ids = encoder.allocate_scratch_for_shape(&[total_rows], DataType::I32)?;+ let mut bucketed_probs = encoder.allocate_scratch_for_shape(&[total_rows], self.data_type)?;+ let mut tok2row = encoder.allocate_scratch_for_shape(&[total_rows], DataType::I32)?;encoder.encode_fill(&mut tok2row, 0xFF);@@ -274,9 +272,9 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {&partials,&mut block_bases,&mut block_alloc,- self.num_routed_experts as u32,- num_blocks as u32,- num_tiles as u32,+ self.num_routed_experts,+ num_blocks,+ num_tiles,0u32,encoder,);@@ -288,11 +286,11 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {&block_alloc,&mut bucketed_ids,&mut bucketed_probs,- batch_dim as u32,- self.num_routed_experts as u32,- self.num_active_experts as u32,- num_blocks as u32,- num_tiles as u32,+ batch_dim,+ self.num_routed_experts,+ self.num_active_experts,+ num_blocks,+ num_tiles,&mut tok2row,encoder,);@@ -331,15 +329,15 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {self.experts_two_pass_prefill_block.encode(args, encoder)?};- let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.model_dim], self.data_type))?;+ let mut output = encoder.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;self.finalize_kernel.encode(&tok2row,&topk_probs,&y_partial,&mut output,- batch_dim as u32,- self.model_dim as u32,- self.num_active_experts as u32,+ batch_dim,+ self.model_dim,+ self.num_active_experts,encoder,);diff --git a/crates/backend-uzu/src/encodable_block/normalization.rs b/crates/backend-uzu/src/encodable_block/normalization.rsindex 13163763e..24e1a2e93 100644--- a/crates/backend-uzu/src/encodable_block/normalization.rs+++ b/crates/backend-uzu/src/encodable_block/normalization.rs@@ -40,7 +40,7 @@ pub struct Normalization<B: Backend> {scale_offset: Option<f32>,scales: Option<Allocation<B>>,biases: Option<Allocation<B>>,- element_count: usize,+ element_count: u32,hadamard_factors: Option<Allocation<B>>,post_layer_scalar_value: f32,data_type: DataType,@@ -49,7 +49,7 @@ pub struct Normalization<B: Backend> {impl<B: Backend> Normalization<B> {pub fn new(- element_count: usize,+ element_count: u32,hadamard_factors: Option<Allocation<B>>,shortcut_mode: ShortcutMode,post_layer_scalar: PostLayerScalar,@@ -114,17 +114,17 @@ impl<B: Backend> Normalization<B> {pub fn encode(&self,input: &Allocation<B>,- row_offset: usize,- row_count: usize,+ row_offset: u32,+ row_count: u32,shortcut: Option<&mut Allocation<B>>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group("normalization");- let row_size = self.element_count * self.data_type.size_in_bytes();- let row_offset_bytes = row_offset * row_size;+ let row_size = size_for_shape(&[self.element_count], self.data_type);+ let row_offset_bytes = row_offset as usize * row_size;let shortcut = shortcut.map(|shortcut| (shortcut, row_offset_bytes));- let mut output = encoder.allocate_scratch(size_for_shape(&[row_count, self.element_count], self.data_type))?;+ let mut output = encoder.allocate_scratch_for_shape(&[row_count, self.element_count], self.data_type)?;self.kernel.encode(Some((input, row_offset_bytes)),self.scales.as_ref(),@@ -132,8 +132,8 @@ impl<B: Backend> Normalization<B> {&mut output,shortcut,self.hadamard_factors.as_ref(),- row_count as u32,- self.element_count as u32,+ row_count,+ self.element_count,self.epsilon,self.scale_offset.unwrap_or(0.0),self.post_layer_scalar_value,diff --git a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rsindex b03fb0e78..8840eabbc 100644--- a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs+++ b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs@@ -1,7 +1,6 @@use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Allocation, Backend, Encoder,kernel::{GatedActMulKernel, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},@@ -36,9 +35,9 @@ pub struct PerLayerEmbedding<B: Backend> {model_projection: Box<dyn Linear<B>>,projection_norm: Normalization<B>,add_scale: <B::Kernels as Kernels>::TensorAddScaleKernel,- ple_dim: usize,- num_layers: usize,- model_dim: usize,+ ple_dim: u32,+ num_layers: u32,+ model_dim: u32,fused_token_scale: f32,data_type: DataType,}@@ -47,7 +46,7 @@ impl<B: Backend> PerLayerEmbedding<B> {pub fn new(context: &B::Context,config: &PLEModelConfig,- model_dim: usize,+ model_dim: u32,data_type: DataType,parameter_tree: &ParameterTree<B>,) -> Result<Self, PerLayerEmbeddingError<B>> {@@ -107,7 +106,7 @@ impl<B: Backend> PerLayerEmbedding<B> {&self,token_ids: &Allocation<B>,inner_features: &Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group("per layer embedding");@@ -116,30 +115,24 @@ impl<B: Backend> PerLayerEmbedding<B> {let total_rows = batch_dim * self.num_layers;let total_elements = batch_dim * total_ple_dim;- let mut token_ple = encoder.allocate_scratch(size_for_shape(&[batch_dim, total_ple_dim], self.data_type))?;- self.token_embedding.encode_lookup(- token_ids,- &mut token_ple,- batch_dim as u32,- self.fused_token_scale,- encoder,- );+ let mut token_ple = encoder.allocate_scratch_for_shape(&[batch_dim, total_ple_dim], self.data_type)?;+ self.token_embedding.encode_lookup(token_ids, &mut token_ple, batch_dim, self.fused_token_scale, encoder);let mut model_projection_input =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.model_dim], self.data_type))?;+ encoder.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;encoder.encode_copy(inner_features, .., &mut model_projection_input, ..);let model_projected = self.model_projection.encode(model_projection_input, batch_dim, encoder)?;let model_normed = self.projection_norm.encode(&model_projected, 0, total_rows, None, encoder)?;let mut per_layer_inputs =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type))?;+ encoder.allocate_scratch_for_shape(&[batch_dim, self.num_layers, self.ple_dim], self.data_type)?;self.add_scale.encode(Some(&token_ple),&model_normed,&mut per_layer_inputs,- total_elements as u32,- total_elements as u32,+ total_elements,+ total_elements,1.0,encoder,);@@ -157,9 +150,9 @@ pub struct PerLayerEmbeddingProjection<B: Backend> {gate_act_mul: <B::Kernels as Kernels>::GatedActMulKernel,residual_finalize: <B::Kernels as Kernels>::TensorAddBiasKernel,residual_combine: <B::Kernels as Kernels>::TensorAddScaleKernel,- model_dim: usize,- ple_dim: usize,- num_layers: usize,+ model_dim: u32,+ ple_dim: u32,+ num_layers: u32,activation: AnyActivation,post_layer_scalar: f32,data_type: DataType,@@ -169,8 +162,8 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {pub fn new(context: &B::Context,config: &PLELayerConfig,- model_dim: usize,- num_layers: usize,+ model_dim: u32,+ num_layers: u32,post_layer_scalar: f32,data_type: DataType,parameter_tree: &ParameterTree<B>,@@ -227,40 +220,33 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {pub fn encode(&self,- layer_index: usize,+ layer_index: u32,per_layer_input: &Allocation<B>,outputs: &mut Allocation<B>,hidden: &Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {encoder.push_debug_group("per layer embedding projection");let length = batch_dim * self.model_dim;- self.residual_finalize.encode(- None::<&Allocation<B>>,- hidden,- &mut *outputs,- length as u32,- length as u32,- encoder,- );+ self.residual_finalize.encode(None::<&Allocation<B>>, hidden, &mut *outputs, length, length, encoder);- let mut gate_input = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.model_dim], self.data_type))?;+ let mut gate_input = encoder.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;encoder.encode_copy(outputs, .., &mut gate_input, ..);let gate_out = self.gate.encode(gate_input, batch_dim, encoder)?;- let mut activated = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.ple_dim], self.data_type))?;+ let mut activated = encoder.allocate_scratch_for_shape(&[batch_dim, self.ple_dim], self.data_type)?;self.gate_act_mul.encode(&gate_out,Some(per_layer_input),&mut activated,None::<&Allocation<B>>,- self.ple_dim as u32,- batch_dim as u32,- (layer_index * self.ple_dim) as u32,- (self.num_layers * self.ple_dim) as u32,+ self.ple_dim,+ batch_dim,+ layer_index * self.ple_dim,+ self.num_layers * self.ple_dim,self.activation.act_type(),encoder,);@@ -272,8 +258,8 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {None::<&Allocation<B>>,&normed,&mut *outputs,- length as u32,- length as u32,+ length,+ length,self.post_layer_scalar,encoder,);diff --git a/crates/backend-uzu/src/encodable_block/prediction_head.rs b/crates/backend-uzu/src/encodable_block/prediction_head.rsindex 3af4a8af5..d79daad53 100644--- a/crates/backend-uzu/src/encodable_block/prediction_head.rs+++ b/crates/backend-uzu/src/encodable_block/prediction_head.rs@@ -26,7 +26,7 @@ pub enum PredictionHeadError<B: Backend> {}pub struct PredictionHead<B: Backend> {- hidden_dim: usize,+ hidden_dim: u32,activation: ActivationType,dense_projection: Box<dyn Linear<B>>,activation_kernel: <B::Kernels as Kernels>::ActivationKernel,@@ -36,8 +36,8 @@ pub struct PredictionHead<B: Backend> {impl<B: Backend> PredictionHead<B> {pub fn new(- hidden_dim: usize,- num_labels: usize,+ hidden_dim: u32,+ num_labels: u32,data_type: DataType,config: &PredictionHeadConfig,parameter_tree: &ParameterTree<B>,@@ -89,19 +89,13 @@ impl<B: Backend> PredictionHead<B> {pub fn encode(&self,input: Allocation<B>,- batch_dim: usize,+ batch_dim: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group("prediction head");let mut hidden = self.dense_projection.encode(input, batch_dim, encoder)?;- self.activation_kernel.encode(- None::<&Allocation<B>>,- &mut hidden,- self.hidden_dim as u32,- self.activation,- encoder,- );+ self.activation_kernel.encode(None::<&Allocation<B>>, &mut hidden, self.hidden_dim, self.activation, encoder);let normalized = self.normalization.encode(&hidden, 0, batch_dim, None, encoder)?;let logits = self.readout.encode(normalized, batch_dim, encoder)?;diff --git a/crates/backend-uzu/src/encodable_block/sampling/gumbel.rs b/crates/backend-uzu/src/encodable_block/sampling/gumbel.rsindex ce8d64bdb..710706e81 100644--- a/crates/backend-uzu/src/encodable_block/sampling/gumbel.rs+++ b/crates/backend-uzu/src/encodable_block/sampling/gumbel.rs@@ -53,7 +53,7 @@ fn uniform_float(/// needs (0,1): u == 1 gives +inf. 24 bits only; the full 32 rounds the top words to 2^32.#[inline]fn unit_interval(word: u32) -> f32 {- ((word >> 8).max(1)) as f32 * (1.0 / 16777216.0)+ (word >> 8).max(1) as f32 * (1.0 / 16777216.0)}pub fn gumbel_float(diff --git a/crates/backend-uzu/src/encodable_block/sampling/mod.rs b/crates/backend-uzu/src/encodable_block/sampling/mod.rsindex 135f513c0..bd5f99d3c 100644--- a/crates/backend-uzu/src/encodable_block/sampling/mod.rs+++ b/crates/backend-uzu/src/encodable_block/sampling/mod.rs@@ -24,7 +24,7 @@ pub use gumbel::{gumbel_float, revidx};pub use prng::PRng;pub struct Sampling<B: Backend> {- vocab_size: usize,+ vocab_size: u32,data_type: DataType,unified_kernels: Mutex<HashMap<UnifiedSamplingKey, <B::Kernels as Kernels>::UnifiedSamplingKernel>>,}@@ -32,7 +32,7 @@ pub struct Sampling<B: Backend> {impl<B: Backend> Sampling<B> {pub fn new(data_type: DataType,- vocab_size: usize,+ vocab_size: u32,) -> Self {Self {vocab_size,@@ -51,12 +51,12 @@ pub enum SamplingMethod {top_p: Option<f32>,min_p: Option<f32>,repetition_penalty: Option<f32>,- suffix_repetition_length: Option<usize>,+ suffix_repetition_length: Option<u32>,},}impl SamplingMethod {- pub fn suffix_repetition_length(&self) -> Option<usize> {+ pub fn suffix_repetition_length(&self) -> Option<u32> {match self {SamplingMethod::Greedy => None,SamplingMethod::Stochastic {@@ -89,11 +89,13 @@ impl<B: Backend> Sampling<B> {token_ids: Option<&Allocation<B>>,sampling_method: &SamplingMethod,batch_dim: &BatchTopology,- sampling_range: Range<usize>,+ sampling_range: Range<u32>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {encoder.push_debug_group("sampling");+ let sampling_length = sampling_range.end - sampling_range.start;+let (is_stochastic, temperature, top_k, top_p, min_p, repetition_penalty, suffix_repetition_length) =match sampling_method {SamplingMethod::Greedy => (false, None, None, None, None, None, None),@@ -126,7 +128,7 @@ impl<B: Backend> Sampling<B> {let mut logits_copy = encoder.allocate_scratch(logits.as_buffer_range_ref().range().len())?;let tensor_copy = <B::Kernels as Kernels>::TensorCopyKernel::new(encoder.context(), self.data_type)?;- tensor_copy.encode(logits, &mut logits_copy, (self.vocab_size * sampling_range.len()) as u32, encoder);+ tensor_copy.encode(logits, &mut logits_copy, self.vocab_size * sampling_length, encoder);let repetition_penalty_kernel =<B::Kernels as Kernels>::RepetitionPenaltyKernel::new(encoder.context(), self.data_type)?;@@ -136,10 +138,10 @@ impl<B: Backend> Sampling<B> {context_ring.expect("context_ring is required for repetition_penalty"),token_ids.expect("token_ids is required for repetition_penalty"),repetition_penalty,- suffix_repetition_length as u32,- self.vocab_size as u32,- sampling_range.start as u32,- sampling_range.len() as u32,+ suffix_repetition_length,+ self.vocab_size,+ sampling_range.start,+ sampling_length,encoder,);Some(logits_copy)@@ -171,7 +173,7 @@ impl<B: Backend> Sampling<B> {};let mut output =- encoder.context().create_allocation(sampling_range.len() * size_of::<u32>(), AllocationType::Global)?;+ encoder.context().create_allocation(sampling_length as usize * size_of::<u32>(), AllocationType::Global)?;kernel.encode(logits,@@ -182,8 +184,8 @@ impl<B: Backend> Sampling<B> {top_k,top_p,min_p,- self.vocab_size as u32,- sampling_range.len() as u32,+ self.vocab_size,+ sampling_length,encoder,);diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rsindex 75678f97f..d02a9432d 100644--- a/crates/backend-uzu/src/encodable_block/transformer.rs+++ b/crates/backend-uzu/src/encodable_block/transformer.rs@@ -18,12 +18,12 @@ use crate::{enum TransformerLayerStateType<B: Backend> {Owned(Box<dyn MixerState<B>>),- Shared(usize),+ Shared(u32),}pub struct TransformerState<B: Backend> {layer_states: Box<[TransformerLayerStateType<B>]>,- context_length: usize,+ context_length: u32,}pub struct TransformerEncodeOutput<B: Backend> {@@ -32,14 +32,14 @@ pub struct TransformerEncodeOutput<B: Backend> {}impl<B: Backend> TransformerState<B> {- pub fn context_length(&self) -> usize {+ pub fn context_length(&self) -> u32 {self.context_length}pub fn prepare(&mut self,- context_length: usize,- suffix_length: usize,+ context_length: u32,+ suffix_length: u32,context: &B::Context,) -> Result<(), B::Error> {for layer_state in &mut self.layer_states {@@ -55,7 +55,7 @@ impl<B: Backend> TransformerState<B> {pub fn encode_accept(&mut self,- accepted_indices: &[usize],+ accepted_indices: &[u32],encoder: &mut Encoder<B>,) -> Result<(), B::Error> {encoder.push_debug_group("transformer accept");@@ -68,7 +68,7 @@ impl<B: Backend> TransformerState<B> {layer_state.encode_accept(accepted_indices, encoder)?;}- self.context_length += accepted_indices.len();+ self.context_length += accepted_indices.len() as u32;encoder.pop_debug_group();@@ -90,7 +90,7 @@ pub struct Transformer<B: Backend> {ropes: Box<[AnyRoPEConfig]>,layers: Box<[(TransformerLayer<B>, Option<usize>)]>,output_norm: Normalization<B>,- model_dim: usize,+ model_dim: u32,residual_add: <B::Kernels as Kernels>::TensorAddScaleKernel,}@@ -104,11 +104,12 @@ impl<B: Backend> Transformer<B> {) -> Result<Self, TransformerNewError<B>> {let mut ropes: Vec<AnyRoPEConfig> = Vec::new();+ let num_layers = transformer_config.layer_configs.len() as u32;let layers = transformer_config.layer_configs.iter()- .enumerate()- .map(|(layer_index, layer_config)| {+ .zip(0u32..)+ .map(|(layer_config, layer_index)| {let layer_loader = parameter_tree.subtree(&format!("layers.{}", layer_index));let rope = layer_config.rope_config.as_ref().map(|layer_rope_config| {@@ -122,7 +123,7 @@ impl<B: Backend> Transformer<B> {context,transformer_config.model_dim,transformer_config.hidden_dim,- transformer_config.layer_configs.len(),+ num_layers,layer_config,layer_index,&layer_loader,@@ -160,11 +161,11 @@ impl<B: Backend> Transformer<B> {&self,shortcut: &Allocation<B>,hidden: &Allocation<B>,- batch_size: usize,+ batch_size: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {let mut output = encoder.allocate_scratch(hidden.size())?;- let elements = (batch_size * self.model_dim) as u32;+ let elements = batch_size * self.model_dim;self.residual_add.encode(Some(shortcut), hidden, &mut output, elements, elements, 1.0, encoder);Ok(output)}@@ -173,10 +174,10 @@ impl<B: Backend> Transformer<B> {self.layers.iter().all(|(layer, _rope)| layer.mixer.speculation_supported())}- pub fn max_context_length(&self) -> Option<usize> {+ pub fn max_context_length(&self) -> Option<u32> {self.layers.iter().map(|(layer, _rope_index)| layer.mixer.max_context_length()).fold(None, |acc, el| {match (acc, el) {- (Some(a), Some(b)) => Some(usize::min(a, b)),+ (Some(a), Some(b)) => Some(u32::min(a, b)),(Some(x), None) | (None, Some(x)) => Some(x),(None, None) => None,}@@ -200,7 +201,7 @@ impl<B: Backend> Transformer<B> {pub fn create_empty_state(&self,- max_context_length: Option<usize>,+ max_context_length: Option<u32>,context: &B::Context,) -> Result<TransformerState<B>, B::Error> {let layer_states = self@@ -227,8 +228,8 @@ impl<B: Backend> Transformer<B> {input: Allocation<B>,per_layer_inputs: Option<&Allocation<B>>,batch_dim: &BatchTopology,- output_range: Option<Range<usize>>,- hidden_feature_layer_indices: Option<&[usize]>,+ output_range: Option<Range<u32>>,+ hidden_feature_layer_indices: Option<&[u32]>,mut state: Option<&mut TransformerState<B>>,encoder: &mut Encoder<B>,) -> Result<TransformerEncodeOutput<B>, B::Error> {@@ -244,8 +245,7 @@ impl<B: Backend> Transformer<B> {hidden_feature_layer_indices.map(|indices| (0..indices.len()).map(|_| None).collect::<Vec<_>>());let context_length = state.as_ref().map(|state| state.context_length).unwrap_or(0);- let token_positions =- batch_dim.heights().map(|rel_pos| context_length + rel_pos as usize).collect::<Box<[usize]>>();+ let token_positions = batch_dim.heights().map(|rel_pos| context_length + rel_pos).collect::<Box<[u32]>>();let precalculated_ropes = self.ropes@@ -257,10 +257,11 @@ impl<B: Backend> Transformer<B> {let precalculated_rope = layer_rope_index.map(|i| &precalculated_ropes[i]);let layer_state = if let Some(state) = &mut state {- Some(match &mut state.layer_states[layer.layer_index] {+ Some(match &mut state.layer_states[layer.layer_index as usize] {TransformerLayerStateType::Owned(layer_state) => MaybeMut::Mut(layer_state.as_mut()),TransformerLayerStateType::Shared(owned_layer_index) => {- let TransformerLayerStateType::Owned(owned_layer) = &state.layer_states[*owned_layer_index]+ let TransformerLayerStateType::Owned(owned_layer) =+ &state.layer_states[*owned_layer_index as usize]else {panic!("shared layer doesn't point to an owned layer");};@@ -313,8 +314,13 @@ impl<B: Backend> Transformer<B> {});};- let output_normalized =- self.output_norm.encode(&hidden, output_range.start, output_range.len(), Some(&mut shortcut), encoder)?;+ let output_normalized = self.output_norm.encode(+ &hidden,+ output_range.start,+ output_range.end - output_range.start,+ Some(&mut shortcut),+ encoder,+ )?;Ok(TransformerEncodeOutput {output: Some(output_normalized),diff --git a/crates/backend-uzu/src/encodable_block/transformer_layer.rs b/crates/backend-uzu/src/encodable_block/transformer_layer.rsindex 82f55e4d6..180c87113 100644--- a/crates/backend-uzu/src/encodable_block/transformer_layer.rs+++ b/crates/backend-uzu/src/encodable_block/transformer_layer.rs@@ -27,7 +27,7 @@ pub enum TransformerLayerError<B: Backend> {Normalization(#[from] NormalizationNewError<B>),#[error("Layer {layer_index} sets post_layer_scalar but has no post_mlp_norm")]PostLayerScalarWithoutPostMlpNorm {- layer_index: usize,+ layer_index: u32,},#[error("Transformer layers except the first one if it doesn't have rht in mixer require pre_mixer_norm_config")]MissingPreMixerNormConfig,@@ -36,9 +36,9 @@ pub enum TransformerLayerError<B: Backend> {// TODO: saner shortcutpub struct TransformerLayer<B: Backend> {- pub layer_index: usize,+ pub layer_index: u32,pub pre_mixer_norm: Option<Normalization<B>>,- pub kv_source_layer_index: Option<usize>,+ pub kv_source_layer_index: Option<u32>,pub mixer: Box<dyn Mixer<B>>,pub post_mixer_norm: Option<Normalization<B>>,pub pre_mlp_norm: Normalization<B>,@@ -50,11 +50,11 @@ pub struct TransformerLayer<B: Backend> {impl<B: Backend> TransformerLayer<B> {pub fn new(context: &B::Context,- model_dim: usize,- hidden_dim: usize,- num_layers: usize,+ model_dim: u32,+ hidden_dim: u32,+ num_layers: u32,layer_config: &TransformerLayerConfig,- layer_index: usize,+ layer_index: u32,parameter_tree: &ParameterTree<B>,data_type: DataType,) -> Result<Self, TransformerLayerError<B>> {diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rsindex 76f519b6b..e347f15b9 100644--- a/crates/backend-uzu/src/encodable_block/weaver.rs+++ b/crates/backend-uzu/src/encodable_block/weaver.rs@@ -30,10 +30,10 @@ pub(crate) const DATA_TYPE: DataType = DataType::BF16;pub(crate) const ROPE_DATA_TYPE: DataType = DataType::F32;pub struct WeaverTreeShape {- pub budget: usize,- pub depth: usize,- pub expand_per_round: usize,- pub expand_width: usize,+ pub budget: u32,+ pub depth: u32,+ pub expand_per_round: u32,+ pub expand_width: u32,}pub struct EncodedWeaverTree<B: Backend> {@@ -42,7 +42,7 @@ pub struct EncodedWeaverTree<B: Backend> {pub struct ProposalNode {pub token_id: u32,- pub depth: usize,+ pub depth: u32,pub child_indices: Vec<usize>,}@@ -71,7 +71,7 @@ impl<B: Backend> EncodedWeaverTree<B> {}nodes.push(ProposalNode {token_id: field(TreeIdx::TokenId, slot),- depth: field(TreeIdx::Depth, slot) as usize,+ depth: field(TreeIdx::Depth, slot),child_indices: Vec::new(),});}@@ -92,10 +92,10 @@ pub struct Weaver<B: Backend> {top_children: <B::Kernels as Kernels>::WeaverTopChildrenKernel,frontier_select: <B::Kernels as Kernels>::WeaverFrontierSelectKernel,frontier_insert_children: <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel,- model_dim: usize,- target_model_dim: usize,- max_depth: usize,- candidate_pool_size: usize,+ model_dim: u32,+ target_model_dim: u32,+ max_depth: u32,+ candidate_pool_size: u32,}#[derive(Debug, Error)]@@ -113,16 +113,16 @@ pub enum WeaverNewError<B: Backend> {#[error("model_dim must be divisible by num_heads")]InvalidHeadConfig,#[error("candidate_pool_size must be in 1..={max}, got {0}", max = CANDIDATES_MAX)]- InvalidCandidatePoolSize(usize),+ InvalidCandidatePoolSize(u32),#[error("rope head_dim {actual} does not match model_dim / num_heads = {expected}")]InvalidRopeHeadDim {- expected: usize,- actual: usize,+ expected: u32,+ actual: u32,},#[error("rope max_sequence_length {actual} is too small for max_depth {max_depth} (needs {max_depth} + 1)")]InvalidRopeLength {- max_depth: usize,- actual: usize,+ max_depth: u32,+ actual: u32,},}@@ -251,7 +251,7 @@ impl<B: Backend> Weaver<B> {})}- pub fn max_depth(&self) -> usize {+ pub fn max_depth(&self) -> u32 {self.max_depth}@@ -269,7 +269,6 @@ impl<B: Backend> Weaver<B> {encoder.push_debug_group("weaver tree");let tree_slot_count = shape.budget;- let frontier_capacity = tree_slot_count * shape.expand_width;let ancestor_stride = self.max_depth;if shape.budget == 0|| shape.depth < 2@@ -278,11 +277,12 @@ impl<B: Backend> Weaver<B> {|| shape.expand_per_round > FRONTIER_MAX_WIDTH|| shape.expand_width == 0|| shape.expand_width > self.candidate_pool_size- || frontier_capacity > FRONTIER_MAX_SLOTS- || depth_seeds.len() != self.max_depth+ || tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width+ || depth_seeds.len() as u32 != self.max_depth{return Err(WeaverEncodeError::InvalidTreeInput);}+ let frontier_capacity = tree_slot_count * shape.expand_width;// `depth` counts the root: nodes expand up to `depth - 1` edges from it.let lookahead_count = shape.depth - 1;@@ -290,7 +290,7 @@ impl<B: Backend> Weaver<B> {// lookahead row form the candidate pool node expansions draw from.let vocab_size = target_embedding.vocab_size();assert!(- logits.size() >= lookahead_count * vocab_size * DataType::F32.size_in_bytes(),+ logits.size() >= size_for_shape(&[lookahead_count, vocab_size], DataType::F32),"draft logits do not cover the lookahead rows");let mut candidate_ids = encoder@@ -304,8 +304,8 @@ impl<B: Backend> Weaver<B> {logits,&mut candidate_ids,&mut candidate_logits,- lookahead_count as u32,- self.candidate_pool_size as u32,+ lookahead_count,+ self.candidate_pool_size,encoder,).map_err(WeaverEncodeError::Backend)?;@@ -316,16 +316,16 @@ impl<B: Backend> Weaver<B> {// Prefix pass: run the target token and one draft row per non-root// chain node through every layer, collecting per-layer KV caches.- let hidden_row_bytes = self.target_model_dim * DATA_TYPE.size_in_bytes();+ let hidden_row_bytes = size_for_shape(&[self.target_model_dim], DATA_TYPE);let mut prefix_hidden = encoder.allocate_scratch(size_for_shape(&[shape.depth, self.target_model_dim], DATA_TYPE)).map_err(WeaverEncodeError::Backend)?;encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_hidden, 0..hidden_row_bytes);encoder.encode_copy(draft_hidden,- hidden_row_bytes..shape.depth * hidden_row_bytes,+ hidden_row_bytes..shape.depth as usize * hidden_row_bytes,&mut prefix_hidden,- hidden_row_bytes..shape.depth * hidden_row_bytes,+ hidden_row_bytes..shape.depth as usize * hidden_row_bytes,);let normalized_prefix = self.hidden_state_norm@@ -383,40 +383,43 @@ impl<B: Backend> Weaver<B> {.collect::<Result<Vec<_>, _>>().map_err(WeaverEncodeError::Backend)?;- let mut tree_init = vec![0u32; TreeIdx::COUNT * tree_slot_count];- for slot in 0..tree_slot_count {- tree_init[TreeIdx::ParentSlot as usize * tree_slot_count + slot] = FRONTIER_NO_WINNER;+ let tree_slots = tree_slot_count as usize;+ let round_nodes = shape.expand_per_round as usize;++ let mut tree_init = vec![0u32; TreeIdx::COUNT * tree_slots];+ for slot in 0..tree_slots {+ tree_init[TreeIdx::ParentSlot as usize * tree_slots + slot] = FRONTIER_NO_WINNER;}- tree_init[TreeIdx::TokenId as usize * tree_slot_count] = root_token_id;- tree_init[TreeIdx::Valid as usize * tree_slot_count] = 1;+ tree_init[TreeIdx::TokenId as usize * tree_slots] = root_token_id;+ tree_init[TreeIdx::Valid as usize * tree_slots] = 1;let mut packed_tree = encoder.allocate_constant_from_slice(&tree_init).map_err(WeaverEncodeError::Backend)?;let mut frontier = encoder- .allocate_constant_from_slice(&vec![0u32; FrontierIdx::COUNT * frontier_capacity])+ .allocate_constant_from_slice(&vec![0u32; FrontierIdx::COUNT * frontier_capacity as usize]).map_err(WeaverEncodeError::Backend)?;let mut slot_ancestors = encoder- .allocate_constant_from_slice(&vec![0u32; tree_slot_count * ancestor_stride])+ .allocate_constant_from_slice(&vec![0u32; tree_slots * ancestor_stride as usize]).map_err(WeaverEncodeError::Backend)?;- let mut initial_node_token_ids = vec![0u32; shape.expand_per_round];+ let mut initial_node_token_ids = vec![0u32; round_nodes];initial_node_token_ids[0] = root_token_id;- let mut initial_node_valid = vec![0u32; shape.expand_per_round];+ let mut initial_node_valid = vec![0u32; round_nodes];initial_node_valid[0] = 1;let mut node_token_ids =encoder.allocate_constant_from_slice(&initial_node_token_ids).map_err(WeaverEncodeError::Backend)?;let mut node_metadata = encoder- .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * shape.expand_per_round])+ .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * round_nodes]).map_err(WeaverEncodeError::Backend)?;let mut node_ancestor_indices = encoder- .allocate_constant_from_slice(&vec![0u32; shape.expand_per_round * ancestor_stride])+ .allocate_constant_from_slice(&vec![0u32; round_nodes * ancestor_stride as usize]).map_err(WeaverEncodeError::Backend)?;let mut node_valid =encoder.allocate_constant_from_slice(&initial_node_valid).map_err(WeaverEncodeError::Backend)?;let mut node_candidate_ids = encoder- .allocate_constant_from_slice(&vec![0u32; shape.expand_per_round * self.candidate_pool_size])+ .allocate_constant_from_slice(&vec![0u32; (shape.expand_per_round * self.candidate_pool_size) as usize]).map_err(WeaverEncodeError::Backend)?;let mut node_candidate_logits = encoder- .allocate_constant_from_slice(&vec![0.0f32; shape.expand_per_round * self.candidate_pool_size])+ .allocate_constant_from_slice(&vec![0.0f32; (shape.expand_per_round * self.candidate_pool_size) as usize]).map_err(WeaverEncodeError::Backend)?;let depth_seeds_buffer =encoder.allocate_constant_from_slice(depth_seeds).map_err(WeaverEncodeError::Backend)?;@@ -441,15 +444,15 @@ impl<B: Backend> Weaver<B> {&candidate_logits,&mut node_candidate_ids,&mut node_candidate_logits,- frontier_capacity as u32,- tree_slot_count as u32,- batch_node_count as u32,- batch_start_slot as u32,- ancestor_stride as u32,- self.max_depth as u32,- lookahead_count as u32,- lookahead_count as u32,- self.candidate_pool_size as u32,+ frontier_capacity,+ tree_slot_count,+ batch_node_count,+ batch_start_slot,+ ancestor_stride,+ self.max_depth,+ lookahead_count,+ lookahead_count,+ self.candidate_pool_size,encoder,);}@@ -475,7 +478,7 @@ impl<B: Backend> Weaver<B> {.map_err(WeaverEncodeError::Backend)?;let mut residual_state =encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;- let metadata_field_bytes = batch_node_count * DataType::U32.size_in_bytes();+ let metadata_field_bytes = size_for_shape(&[batch_node_count], DataType::U32);for (layer_index, layer) in self.layers.iter().enumerate() {let attention_input = layer.pre_attention_norm@@ -499,11 +502,11 @@ impl<B: Backend> Weaver<B> {(&node_metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes),(&node_metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes),&mut attention_output,- batch_node_count as u32,- shape.depth as u32,- ancestor_stride as u32,- tree_slot_count as u32,- layer.max_depth as u32,+ batch_node_count,+ shape.depth,+ ancestor_stride,+ tree_slot_count,+ layer.max_depth,layer.attention_scale,encoder,);@@ -541,10 +544,10 @@ impl<B: Backend> Weaver<B> {&node_metadata,&mut child_token_ids,&mut child_logprobs,- batch_node_count as u32,- self.candidate_pool_size as u32,- shape.expand_width as u32,- target_embedding.vocab_size() as u32,+ batch_node_count,+ self.candidate_pool_size,+ shape.expand_width,+ target_embedding.vocab_size(),encoder,);@@ -555,10 +558,10 @@ impl<B: Backend> Weaver<B> {&child_token_ids,&child_logprobs,&mut frontier,- frontier_capacity as u32,- tree_slot_count as u32,- batch_node_count as u32,- shape.expand_width as u32,+ frontier_capacity,+ tree_slot_count,+ batch_node_count,+ shape.expand_width,encoder,);batch_start_slot += batch_node_count;diff --git a/crates/backend-uzu/src/encodable_block/weaver_layer.rs b/crates/backend-uzu/src/encodable_block/weaver_layer.rsindex 06fe68f1c..139a57834 100644--- a/crates/backend-uzu/src/encodable_block/weaver_layer.rs+++ b/crates/backend-uzu/src/encodable_block/weaver_layer.rs@@ -42,10 +42,10 @@ pub struct WeaverLayer<B: Backend> {// Geometrypub attention_scale: f32,- pub model_dim: usize,- pub num_heads: usize,- pub head_dim: usize,- pub max_depth: usize,+ pub model_dim: u32,+ pub num_heads: u32,+ pub head_dim: u32,+ pub max_depth: u32,}impl<B: Backend> WeaverLayer<B> {@@ -134,9 +134,8 @@ impl<B: Backend> WeaverLayer<B> {let (mlp, up_input_hadamard_factors) =<dyn Mlp<B>>::new(&mlp_config, model_dim, hidden_dim, context, ¶meter_tree.subtree("mlp"), DATA_TYPE)?;assert!(up_input_hadamard_factors.is_none(), "Weaver MLP does not support input Hadamard factors");- let ancestor_attention =- <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim as u32, num_heads as u32)- .map_err(WeaverNewError::Backend)?;+ let ancestor_attention = <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim, num_heads)+ .map_err(WeaverNewError::Backend)?;Ok(Self {qkv_projection,out_projection,@@ -159,16 +158,16 @@ impl<B: Backend> WeaverLayer<B> {residual_input: &Allocation<B>,residual_state: &mut Allocation<B>,rope: &PrecalculatedRoPE<B>,- token_count: usize,+ token_count: u32,encoder: &mut Encoder<B>,) -> Result<PreparedPrefixAttention<B>, B::Error> {let attention_input =self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), encoder)?;let qkv = self.qkv_projection.encode(attention_input, token_count, encoder)?;let mut queries =- encoder.allocate_scratch(size_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE))?;+ encoder.allocate_scratch_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE)?;let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);- let mut kv_cache = encoder.allocate_scratch(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE))?;+ let mut kv_cache = encoder.allocate_scratch_for_shape(&[2, token_count, self.model_dim], DATA_TYPE)?;let (keys, values) = kv_cache.as_buffer_range_mut().split_at(kv_plane_bytes);self.attention_prepare.encode(&qkv,@@ -177,12 +176,12 @@ impl<B: Backend> WeaverLayer<B> {Some(values),Some(&rope.cosines),Some(&rope.sines),- self.num_heads as u32,- Some(self.num_heads as u32),- self.head_dim as u32,- Some(rope.dim as u32),+ self.num_heads,+ Some(self.num_heads),+ self.head_dim,+ Some(rope.dim),Some(0),- token_count as u32,+ token_count,encoder,);Ok(PreparedPrefixAttention {@@ -195,7 +194,7 @@ impl<B: Backend> WeaverLayer<B> {&self,attention_output: Allocation<B>,residual_state: &mut Allocation<B>,- token_count: usize,+ token_count: u32,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {let projected_attention = self.out_projection.encode(attention_output, token_count, encoder)?;diff --git a/crates/backend-uzu/src/encodable_block/weight_matrix.rs b/crates/backend-uzu/src/encodable_block/weight_matrix.rsindex cfe153fcc..06c435434 100644--- a/crates/backend-uzu/src/encodable_block/weight_matrix.rs+++ b/crates/backend-uzu/src/encodable_block/weight_matrix.rs@@ -69,7 +69,7 @@ pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeightSome(QuantizationInfo {mode,method,- group_size: group_size as u32,+ group_size,})},};@@ -102,8 +102,8 @@ impl<B: Backend> WeightMatrix<B> {tree: &ParameterTree<B>,spec: AnyWeightMatrixSpec,required_layout: Layout,- output_dim: usize,- input_dim: usize,+ output_dim: u32,+ input_dim: u32,data_type: DataType,) -> Result<Self, WeightMatrixError<B>> {let ParsedWeightSpec {@@ -125,7 +125,7 @@ impl<B: Backend> WeightMatrix<B> {});};- let group_size = info.group_size as usize;+ let group_size = info.group_size;let packing_divisor = info.mode.packing_divisor();let storage_data_type = info.mode.storage_type();if !columns.is_multiple_of(packing_divisor) {@@ -243,9 +243,9 @@ impl<B: Backend> WeightMatrix<B> {fn physical_shape(layout: &Layout,- output_dim: usize,- input_dim: usize,-) -> (usize, usize) {+ output_dim: u32,+ input_dim: u32,+) -> (u32, u32) {match layout {Layout::OutputInput => (output_dim, input_dim),Layout::InputOutput => (input_dim, output_dim),diff --git a/crates/backend-uzu/src/engine/classifier_model.rs b/crates/backend-uzu/src/engine/classifier_model.rsindex 61bc00a55..324c8ecc5 100644--- a/crates/backend-uzu/src/engine/classifier_model.rs+++ b/crates/backend-uzu/src/engine/classifier_model.rs@@ -64,7 +64,7 @@ impl<B: Backend> Engine<B> {)?;let output_labels = if let Some(output_labels) = config.classifier_config.output_labels {- assert!(output_labels.len() == config.classifier_config.num_labels);+ assert!(output_labels.len() as u32 == config.classifier_config.num_labels);output_labels} else {(0..config.classifier_config.num_labels).map(|index| format!("class_{index}")).collect()@@ -102,7 +102,11 @@ impl<B: Backend> ClassifierModel<B> {return Err(ClassifierModelClassifyError::EmptyInput);}- if self.classifier.max_context_length().is_some_and(|max_context_length| input.len() > max_context_length) {+ if self+ .classifier+ .max_context_length()+ .is_some_and(|max_context_length| input.len() > max_context_length as usize)+ {return Err(ClassifierModelClassifyError::ContextOverflow);}@@ -113,7 +117,7 @@ impl<B: Backend> ClassifierModel<B> {.map_err(ClassifierModelClassifyError::Backend)?;token_ids.copyin(&input.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>());- let logits = self.classifier.encode(&token_ids, input.len(), &mut encoder)?;+ let logits = self.classifier.encode(&token_ids, input.len() as u32, &mut encoder)?;let mut output_buffer = self.contextdiff --git a/crates/backend-uzu/src/engine/language_model/mod.rs b/crates/backend-uzu/src/engine/language_model/mod.rsindex 0ab2a0e40..6148e2379 100644--- a/crates/backend-uzu/src/engine/language_model/mod.rs+++ b/crates/backend-uzu/src/engine/language_model/mod.rs@@ -117,7 +117,7 @@ impl<B: Backend> Engine<B> {let generation_config = config.generation_config;#[cfg(grammar)]- let vocab_size = config.decoder_config.vocab_size;+ let vocab_size = config.decoder_config.vocab_size as usize;Ok(LanguageModel {engine: self.clone(),@@ -133,11 +133,11 @@ impl<B: Backend> Engine<B> {}impl<B: Backend> LanguageModel<B> {- pub fn max_context_length(&self) -> Option<usize> {+ pub fn max_context_length(&self) -> Option<u32> {self.decoder.max_context_length()}- pub fn recommended_context_length(&self) -> Option<usize> {+ pub fn recommended_context_length(&self) -> Option<u32> {let max_context_length = self.max_context_length();// TODO: This is not the correct way to do it, there should be a real memory model@@ -157,7 +157,7 @@ impl<B: Backend> LanguageModel<B> {16384};- Some(usize::min(max_context_length, platform_recommended_context_length))+ Some(u32::min(max_context_length, platform_recommended_context_length))} else {// We just assume that unlimited context means constant state size on all mixers and is thus freeNonediff --git a/crates/backend-uzu/src/engine/language_model/state.rs b/crates/backend-uzu/src/engine/language_model/state.rsindex 86f1c545a..189b43f6e 100644--- a/crates/backend-uzu/src/engine/language_model/state.rs+++ b/crates/backend-uzu/src/engine/language_model/state.rs@@ -12,7 +12,7 @@ pub struct LanguageModelState<B: Backend> {pub(super) prng: PRng,pub(super) transformer_state: TransformerState<B>,pub(super) speculator_state: Option<DFlashState<B>>,- pub(super) max_context_length: Option<usize>,+ pub(super) max_context_length: Option<u32>,}impl<B: Backend> LanguageModelState<B> {@@ -30,7 +30,7 @@ pub enum LanguageModelCreateEmptyStateError<B: Backend> {impl<B: Backend> LanguageModel<B> {pub fn create_empty_state(&self,- max_context_length: Option<usize>,+ max_context_length: Option<u32>,) -> Result<LanguageModelState<B>, LanguageModelCreateEmptyStateError<B>> {let tokens = Vec::new();let last_output_token = None;diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rsindex 355c530e7..2bad7060b 100644--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs@@ -9,6 +9,7 @@ use shoji::traits::backend::chat_token::TokenStreamMetrics;#[cfg(grammar)]use crate::engine::language_model::grammar::Grammar;use crate::{+ array::size_for_shape,backends::common::{Allocation, AllocationPool, AllocationType, Backend, Context, Encoder, Pending,gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,@@ -141,7 +142,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {if model_state.max_context_length- .is_some_and(|max_context_length| model_state.tokens.len() + input.len() > max_context_length)+ .is_some_and(|max_context_length| model_state.tokens.len() + input.len() > max_context_length as usize){return Err(LanguageModelStreamError::ContextOverflow);}@@ -162,19 +163,19 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.engine.context.create_allocation(- (2 + suffix_repetition_length) * DataType::U32.size_in_bytes(),+ size_for_shape(&[2 + suffix_repetition_length], DataType::U32),AllocationType::Global,).map_err(LanguageModelStreamError::Backend)?;- let state_tokens_range =- model_state.tokens.len().saturating_sub(suffix_repetition_length)..model_state.tokens.len();+ let state_tokens_range = model_state.tokens.len().saturating_sub(suffix_repetition_length as usize)+ ..model_state.tokens.len();context_ring.copyin(&once(0) // offset.chain(once(state_tokens_range.len() as u64)) // length.chain(model_state.tokens[state_tokens_range.clone()].iter().copied()) // tokens- .chain(repeat_n(0, suffix_repetition_length - state_tokens_range.len())) // pad if not full+ .chain(repeat_n(0, suffix_repetition_length as usize - state_tokens_range.len())) // pad if not full.map(|x| x as u32).collect::<Box<[_]>>(),);@@ -196,8 +197,8 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {model_state.transformer_state.prepare(- model_state.transformer_state.context_length() + (number_of_batches - 1) * max_batch_size,- usize::min(max_batch_size, input.len()),+ model_state.transformer_state.context_length() + ((number_of_batches - 1) * max_batch_size) as u32,+ usize::min(max_batch_size, input.len()) as u32,&model.engine.context,).map_err(LanguageModelStreamError::Backend)?;@@ -234,7 +235,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let decoder_output = model.decoder.encode(&token_ids,&batch_dim,- sample_last.then(|| (input_chunk.len() - 1)..input_chunk.len()),+ sample_last.then(|| input_chunk.len() as u32 - 1..input_chunk.len() as u32),hidden_feature_layer_indices,&mut model_state.transformer_state,&mut encoder,@@ -276,6 +277,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let bitmask = None;output_norm = decoder_output.final_hidden;+ let sampled_row = batch_dim.size() - 1;output_tokens = Some(model.sampling@@ -287,7 +289,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {Some(&token_ids),&options.sampling_method,&batch_dim,- (batch_dim.size() - 1)..batch_dim.size(),+ sampled_row..sampled_row + 1,&mut encoder,).map_err(LanguageModelStreamError::Backend)?,@@ -296,7 +298,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {model_state.transformer_state- .encode_accept(&(0..input_chunk.len()).collect::<Box<[usize]>>(), &mut encoder)+ .encode_accept(&(0..input_chunk.len() as u32).collect::<Box<[u32]>>(), &mut encoder).map_err(LanguageModelStreamError::Backend)?;if let Some(speculator) = model.speculator.as_ref() {@@ -305,7 +307,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.encode_accept(speculator_state,decoder_output.hidden_features.as_ref().unwrap(),- &(0..input_chunk.len()).collect::<Box<[usize]>>(),+ &(0..input_chunk.len() as u32).collect::<Box<[u32]>>(),&mut encoder,).map_err(LanguageModelStreamError::Backend)?;@@ -315,7 +317,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {model.context_ring_update.encode(&token_ids,context_ring.as_mut().unwrap(),- suffix_repetition_length as u32,+ suffix_repetition_length,input_chunk.len() as u32,&mut encoder,);@@ -434,7 +436,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {};return Ok(Some(output_token_id));} else {- let accepted_token_indicies = full.iter().map(|(i, _, _)| *i).collect::<Box<[usize]>>();+ let accepted_token_indicies = full.iter().map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();let mut encoder = Encoder::<B>::new_with_pool_name(@@ -481,7 +483,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {self.model.context_ring_update.encode(&accepted_input_token_ids_const,self.context_ring.as_mut().unwrap(),- suffix_repetition_length as u32,+ suffix_repetition_length,full.len() as u32,&mut encoder,);@@ -722,7 +724,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {if full_accept {self.model_state.transformer_state- .encode_accept(&(0..batch_dim.size()).collect::<Box<[usize]>>(), &mut encoder)+ .encode_accept(&(0..batch_dim.size()).collect::<Box<[u32]>>(), &mut encoder).map_err(LanguageModelStreamError::Backend)?;if let Some(speculator) = self.model.speculator.as_ref() {@@ -731,7 +733,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.encode_accept(speculator_state,decoder_output.hidden_features.as_ref().unwrap(),- &(0..batch_dim.size()).collect::<Box<[usize]>>(),+ &(0..batch_dim.size()).collect::<Box<[u32]>>(),&mut encoder,).map_err(LanguageModelStreamError::Backend)?;@@ -741,8 +743,8 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {self.model.context_ring_update.encode(&token_ids,self.context_ring.as_mut().unwrap(),- suffix_repetition_length as u32,- batch_dim.size() as u32,+ suffix_repetition_length,+ batch_dim.size(),&mut encoder,);}@@ -846,7 +848,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {).unwrap();let accepted_token_indicies =- full.iter().take(num_accepted + 1).map(|(i, _, _)| *i).collect::<Box<[usize]>>();+ full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut encoder).unwrap();if let Some(speculator) = self.model.speculator.as_ref() {speculatordiff --git a/crates/backend-uzu/src/parameters/loader.rs b/crates/backend-uzu/src/parameters/loader.rsindex 9fae3278f..31e7e564e 100644--- a/crates/backend-uzu/src/parameters/loader.rs+++ b/crates/backend-uzu/src/parameters/loader.rs@@ -17,7 +17,7 @@ use crate::{};pub struct ParameterMetadata {- shape: Box<[usize]>,+ shape: Box<[u32]>,data_type: DataType,offset: usize,size: usize,@@ -35,14 +35,14 @@ pub enum ParameterLoaderError<B: Backend> {MetadataDeserializationError(#[from] serde_json::Error),#[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]InvalidTensor {- shape: Box<[usize]>,+ shape: Box<[u32]>,data_type: DataType,- expected_shape: Box<[usize]>,+ expected_shape: Box<[u32]>,expected_data_type: DataType,},#[error("Invalid tensor byte size: got {size} bytes for {shape:?} @ {data_type:?}, expected {expected_size} bytes")]InvalidTensorSize {- shape: Box<[usize]>,+ shape: Box<[u32]>,data_type: DataType,size: usize,expected_size: usize,@@ -142,7 +142,7 @@ pub struct ParameterLeaf<'a, 'leaf, B: Backend, const VALIDATED: bool> {impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, false> {pub fn validate(self,- expected_shape: &[usize],+ expected_shape: &[u32],expected_data_type: DataType,) -> Result<ParameterLeaf<'a, 'leaf, B, true>, ParameterLoaderError<B>> {let shape = self.metadata.shape.as_ref();diff --git a/crates/backend-uzu/src/parameters/safetensors_metadata.rs b/crates/backend-uzu/src/parameters/safetensors_metadata.rsindex 15782bd50..205e5458a 100644--- a/crates/backend-uzu/src/parameters/safetensors_metadata.rs+++ b/crates/backend-uzu/src/parameters/safetensors_metadata.rs@@ -49,7 +49,7 @@ pub struct HashMetadata {#[derive(Debug, Deserialize, Serialize, Clone)]pub struct TensorInfo {pub dtype: Dtype,- pub shape: Vec<usize>,+ pub shape: Vec<u32>,pub data_offsets: (usize, usize),}diff --git a/crates/backend-uzu/src/speculators/dflash_tfm.rs b/crates/backend-uzu/src/speculators/dflash_tfm.rsindex 6189534a4..1d9a0ae92 100644--- a/crates/backend-uzu/src/speculators/dflash_tfm.rs+++ b/crates/backend-uzu/src/speculators/dflash_tfm.rs@@ -56,14 +56,14 @@ pub enum DFlashSpeculatorLoadError<B: Backend> {pub enum DFlashTfmTreeConstructionMethod {Argmax,Weaver {- depth: usize,- expand_per_round: usize,- expand_width: usize,+ depth: u32,+ expand_per_round: u32,+ expand_width: u32,},}pub struct DFlashTfmTreeShape {- pub budget: usize,+ pub budget: u32,pub construction_method: DFlashTfmTreeConstructionMethod,}@@ -98,7 +98,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {Weaver::new(&*context,weaver_config,- config.draft_config.vocab_size as u32,+ config.draft_config.vocab_size,&speculator_tree.subtree("weaver"),)})@@ -121,13 +121,13 @@ impl<B: Backend> DFlashTfmSpeculator<B> {self.weaver.is_some()}- pub fn hidden_feature_layer_indices(&self) -> &[usize] {+ pub fn hidden_feature_layer_indices(&self) -> &[u32] {&self.config.draft_config.target_layer_ids}pub fn empty_state(&self,- context_capacity: usize,+ context_capacity: u32,) -> Result<DFlashState<B>, B::Error> {self.dflash.empty_state(context_capacity, &self.context)}@@ -136,7 +136,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {&self,state: &mut DFlashState<B>,target_features: &[Allocation<B>],- accepted_indices: &[usize],+ accepted_indices: &[u32],encoder: &mut Encoder<B>,) -> Result<(), B::Error> {self.dflash.encode_accept(state, target_features, accepted_indices, encoder)@@ -171,7 +171,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {)));}let chain_length = shape.budget - 1;- let mut nodes = Vec::with_capacity(shape.budget);+ let mut nodes = Vec::with_capacity(shape.budget as usize);nodes.push(ProposalNode {token_id: target_output_token,depth: 0,@@ -186,9 +186,9 @@ impl<B: Backend> DFlashTfmSpeculator<B> {)?;let topology_nodes = (0..chain_length).map(|index| GpuTrieNode {- trie_start: index as u32,- trie_end: (chain_length - 1) as u32,- height: index as u32,+ trie_start: index,+ trie_end: chain_length - 1,+ height: index,}).collect::<Box<[_]>>();let batch_topology = BatchTopology::new(&topology_nodes, true);@@ -210,17 +210,14 @@ impl<B: Backend> DFlashTfmSpeculator<B> {encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;let tokens = sampled.copyout::<u32>();drop(completed);- nodes.extend(tokens.into_iter().enumerate().map(|(index, token_id)| {- let depth = index + 1;- ProposalNode {- token_id,- depth,- child_indices: if depth < chain_length {- vec![depth + 1]- } else {- Vec::new()- },- }+ nodes.extend(tokens.into_iter().zip(1u32..).map(|(token_id, depth)| ProposalNode {+ token_id,+ depth,+ child_indices: if depth < chain_length {+ vec![depth as usize + 1]+ } else {+ Vec::new()+ },}));nodes},@@ -253,7 +250,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {let dflash_output =self.dflash.encode_draft(state, target_output_token, target_embedding, depth, &mut encoder)?;let depth_seeds = (0..weaver.max_depth())- .map(|depth| prng.derive((root_position + depth) as u64))+ .map(|depth| prng.derive(root_position as u64 + depth as u64)).collect::<Box<[u64]>>();let tree = weaver.encode_tree(target_output_norm,@@ -281,12 +278,13 @@ impl<B: Backend> DFlashTfmSpeculator<B> {fn recursive_build(nodes: &[ProposalNode],index: usize,- root_position: usize,+ root_position: u32,#[cfg(grammar)] mut grammar: Option<&mut Grammar>,prng: &PRng,) -> TrieNode {let node = &nodes[index];- let mut trie_node = TrieNode::new(node.token_id as u64, prng.derive((root_position + node.depth) as u64));+ let mut trie_node =+ TrieNode::new(node.token_id as u64, prng.derive(root_position as u64 + node.depth as u64));for &child_index in &node.child_indices {#[cfg(grammar)]if let Some(grammar) = grammar.as_mut()diff --git a/crates/backend-uzu/src/tests/matmul/harness.rs b/crates/backend-uzu/src/tests/matmul/harness.rsindex 8dd541ab7..90900c958 100644--- a/crates/backend-uzu/src/tests/matmul/harness.rs+++ b/crates/backend-uzu/src/tests/matmul/harness.rs@@ -141,7 +141,7 @@ fn run<B: Backend, T: ArrayElement + Float>(alloc_allocation_with_data::<B, T>(context, prefill)} else {context- .create_allocation(m * n * std::mem::size_of::<T>(), AllocationType::Global)+ .create_allocation(m as usize * n as usize * std::mem::size_of::<T>(), AllocationType::Global).expect("create d allocation")};let rht_allocation =@@ -172,9 +172,9 @@ fn run<B: Backend, T: ArrayElement + Float>(d: &mut d_allocation,d_transform,gather_indices: None,- m: m as u32,- n: n as u32,- k: k as u32,+ m,+ n,+ k,},&mut encoder,);diff --git a/crates/backend-uzu/src/tests/matmul/quant.rs b/crates/backend-uzu/src/tests/matmul/quant.rsindex 8d81940d4..22b46e5c0 100644--- a/crates/backend-uzu/src/tests/matmul/quant.rs+++ b/crates/backend-uzu/src/tests/matmul/quant.rs@@ -56,21 +56,23 @@ fn mode_for_bits(bits: u32) -> QuantizationMode {impl<T: ArrayElement + Float> QuantInput<T> {pub fn new(- m: usize,- k: usize,- n: usize,+ m: u32,+ k: u32,+ n: u32,group_size: u32,bits: u32,quant_method: QuantizationMethod,seed: u64,) -> Self {- let num_groups_k = k.div_ceil(group_size as usize);+ let num_groups_k = k.div_ceil(group_size) as usize;+ let (rows, inner, columns) = (m as usize, k as usize, n as usize);let mut rng = SmallRng::seed_from_u64(seed);- let w_packed: Vec<u32> = (0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect();+ let w_packed: Vec<u32> =+ (0..columns * inner * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect();let scales: Vec<T> =- (0..n * num_groups_k).map(|_| T::from(rng.random_range(0.01f32..0.3f32)).unwrap()).collect();- let x: Vec<T> = (0..m * k).map(|_| T::from(rng.random_range(-0.3f32..0.3f32)).unwrap()).collect();+ (0..columns * num_groups_k).map(|_| T::from(rng.random_range(0.01f32..0.3f32)).unwrap()).collect();+ let x: Vec<T> = (0..rows * inner).map(|_| T::from(rng.random_range(-0.3f32..0.3f32)).unwrap()).collect();let zp_stride = if bits == 4 {num_groups_k.div_ceil(2)@@ -80,10 +82,14 @@ impl<T: ArrayElement + Float> QuantInput<T> {let (zero_points, biases) = match quant_method {QuantizationMethod::ScaleBias => (None,- Some((0..n * num_groups_k).map(|_| T::from(rng.random_range(-0.03f32..0.03f32)).unwrap()).collect()),+ Some(+ (0..columns * num_groups_k)+ .map(|_| T::from(rng.random_range(-0.03f32..0.03f32)).unwrap())+ .collect(),+ ),),QuantizationMethod::ScaleZeroPoint => {- (Some((0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)+ (Some((0..columns * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect()), None)},QuantizationMethod::ScaleSymmetric => (None, None),};@@ -94,9 +100,9 @@ impl<T: ArrayElement + Float> QuantInput<T> {zero_points,biases,x,- k: k as u32,- n: n as u32,- m: m as u32,+ k,+ n,+ m,group_size,quant_method,mode: mode_for_bits(bits),@@ -112,31 +118,27 @@ impl<T: ArrayElement + Float> QuantInput<T> {pub fn with_prepared_a(mut self,- activation_group_size: usize,+ activation_group_size: u32,sum_group_size: Option<u32>,) -> Self {self.signed_codes = true;- let sum_group_size = sum_group_size.map(|group_size| group_size as usize);- let rows = self.m as usize;- let columns = self.k as usize;+ let rows = self.m;+ let columns = self.k;assert!(columns.is_multiple_of(activation_group_size));if let Some(group_size) = sum_group_size {assert!(columns.is_multiple_of(group_size));}let context = <Cpu as Backend>::Context::new().expect("CPU context");let input = alloc_allocation_with_data::<Cpu, T>(&context, &self.x);- let factors = alloc_allocation_with_data::<Cpu, i32>(&context, &vec![1; columns]);- let mut values = alloc_allocation::<Cpu, i8>(&context, rows * columns);- let mut scales = alloc_allocation::<Cpu, f32>(&context, rows * columns / activation_group_size);+ let factors = alloc_allocation_with_data::<Cpu, i32>(&context, &vec![1; columns as usize]);+ let elements = rows as usize * columns as usize;+ let mut values = alloc_allocation::<Cpu, i8>(&context, elements);+ let mut scales = alloc_allocation::<Cpu, f32>(&context, elements / activation_group_size as usize);let mut group_sums =- sum_group_size.map(|group_size| alloc_allocation::<Cpu, i32>(&context, rows * columns / group_size));- let transform = ActivationTransform::<Cpu>::quantize(- &context,- T::data_type(),- activation_group_size,- sum_group_size.map(|group_size| group_size as u32),- )- .expect("CPU activation quantization transform");+ sum_group_size.map(|group_size| alloc_allocation::<Cpu, i32>(&context, elements / group_size as usize));+ let transform =+ ActivationTransform::<Cpu>::quantize(&context, T::data_type(), activation_group_size, sum_group_size)+ .expect("CPU activation quantization transform");let mut encoder = Encoder::<Cpu>::new(&context).expect("CPU encoder");transform.encode_quantize(&input,@@ -144,8 +146,8 @@ impl<T: ArrayElement + Float> QuantInput<T> {&mut scales,group_sums.as_mut(),&factors,- rows as u32,- columns as u32,+ rows,+ columns,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().expect("CPU activation quantization");@@ -154,7 +156,7 @@ impl<T: ArrayElement + Float> QuantInput<T> {values: allocation_to_vec(&values),scales: allocation_to_vec(&scales),group_sums: group_sums.map_or_else(Vec::new, |sums| allocation_to_vec(&sums)),- activation_scale_group_size: activation_group_size as u32,+ activation_scale_group_size: activation_group_size,});self}diff --git a/crates/backend-uzu/src/tests/matmul/shape.rs b/crates/backend-uzu/src/tests/matmul/shape.rsindex 29402382d..0c7a7e866 100644--- a/crates/backend-uzu/src/tests/matmul/shape.rs+++ b/crates/backend-uzu/src/tests/matmul/shape.rs@@ -3,16 +3,16 @@ use derive_more::Display;#[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]#[display("M[{m}]K[{k}]N[{n}]")]pub struct Shape {- pub m: usize,- pub k: usize,- pub n: usize,+ pub m: u32,+ pub k: u32,+ pub n: u32,}impl Shape {pub const fn new(- m: usize,- k: usize,- n: usize,+ m: u32,+ k: u32,+ n: u32,) -> Self {Self {m,@@ -49,16 +49,16 @@ pub fn bench_fp_gemm_shapes() -> impl Iterator<Item = Shape> {BENCH_FP_GEMM.iter().copied()}-const BENCH_NK: &[(usize, usize)] =+const BENCH_NK: &[(u32, u32)] =&[(2048, 2048), (2048, 4096), (4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];pub fn bench_quant_gemm_shapes(bits: u32) -> impl Iterator<Item = Shape> {- let block_size: usize = if bits == 4 {+ let block_size: u32 = if bits == 4 {512} else {256};- let ms = &[4usize, 5, 6, 7, 8, 16, 32, 48, 64];+ let ms = &[4u32, 5, 6, 7, 8, 16, 32, 48, 64];BENCH_NK.iter().filter(move |&&(n, k)| n % 32 == 0 && k % block_size == 0)@@ -66,19 +66,19 @@ pub fn bench_quant_gemm_shapes(bits: u32) -> impl Iterator<Item = Shape> {}pub fn bench_quant_gemv_shapes(bits: u32) -> impl Iterator<Item = Shape> {- let block_size: usize = if bits == 4 {+ let block_size: u32 = if bits == 4 {512} else {256};- let nk: &[(usize, usize)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];- let ms = &[1usize, 2, 4];+ let nk: &[(u32, u32)] = &[(4096, 4096), (4096, 14336), (14336, 4096), (14336, 14336)];+ let ms = &[1u32, 2, 4];nk.iter().filter(move |&&(n, k)| n % 8 == 0 && k % block_size == 0).flat_map(move |&(n, k)| ms.iter().map(move |&m| Shape::new(m, k, n)))}-const QWEN3_LAYERS: &[(&str, usize, usize)] = &[+const QWEN3_LAYERS: &[(&str, u32, u32)] = &[("0.8b_qkv", 1024, 3072),("0.8b_o", 2048, 1024),("0.8b_gate", 1024, 2048),@@ -99,12 +99,12 @@ const QWEN3_LAYERS: &[(&str, usize, usize)] = &[];pub fn qwen3_layer_shapes(bits: u32) -> impl Iterator<Item = (&'static str, Shape)> {- let block_size: usize = if bits == 4 {+ let block_size: u32 = if bits == 4 {512} else {256};- let ms = &[1usize, 2, 4, 8, 16, 32, 64];+ let ms = &[1u32, 2, 4, 8, 16, 32, 64];QWEN3_LAYERS.iter().filter(move |&&(_, k, _)| k.is_multiple_of(block_size))diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rsindex c41ad0ade..f1548ae25 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs@@ -122,25 +122,21 @@ mod quantize {factors_data: &[i32],rows: usize,columns: usize,- activation_group_size: usize,+ activation_group_size: u32,emit_group_sums: bool,- sum_group_size: Option<usize>,+ sum_group_size: Option<u32>,) -> (Vec<i8>, Vec<f32>, Option<Vec<i32>>) {- let scale_groups = columns / activation_group_size;- let sum_groups = sum_group_size.map_or(0, |group_size| columns / group_size);+ let scale_groups = columns / activation_group_size as usize;+ let sum_groups = sum_group_size.map_or(0, |group_size| columns / group_size as usize);let context = B::Context::new().expect("context");let input = alloc_allocation_with_data::<B, f32>(context.as_ref(), input_data);let factors = alloc_allocation_with_data::<B, i32>(context.as_ref(), factors_data);let mut values = alloc_allocation::<B, i8>(context.as_ref(), rows * columns);let mut scales = alloc_allocation::<B, f32>(context.as_ref(), rows * scale_groups);let mut group_sums = emit_group_sums.then(|| alloc_allocation::<B, i32>(context.as_ref(), rows * sum_groups));- let kernel = ActivationTransform::quantize(- context.as_ref(),- DataType::F32,- activation_group_size,- sum_group_size.map(|size| size as u32),- )- .expect("quantize transform");+ let kernel =+ ActivationTransform::quantize(context.as_ref(), DataType::F32, activation_group_size, sum_group_size)+ .expect("quantize transform");let mut encoder = Encoder::<B>::new(context.as_ref()).expect("encoder");kernel.encode_quantize(&input,@@ -158,9 +154,9 @@ mod quantize {}fn check_quantize(- activation_group_size: usize,+ activation_group_size: u32,emit_group_sums: bool,- sum_group_size: Option<usize>,+ sum_group_size: Option<u32>,) {let rows = 3;let columns = 256;@@ -209,7 +205,7 @@ mod quantize {for (group_index, (&actual, &expected)) inactual_group_sums.iter().zip(expected_group_sums.as_ref().expect("CPU group sums")).enumerate(){- let sum_group_size = sum_group_size.expect("correction group");+ let sum_group_size = sum_group_size.expect("correction group") as usize;let start = group_index * sum_group_size;let sum_from_actual_codes: i32 =actual_values[start..start + sum_group_size].iter().copied().map(i32::from).sum();diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/embedding/quant_embedding_test.rsindex b6260341f..6b719c5fa 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/embedding/quant_embedding_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/embedding/quant_embedding_test.rs@@ -45,7 +45,7 @@ fn get_test_data<T: ArrayElement + Float>(quant_mode: QuantizationMode) -> (Inpulet token_ids: Box<[u32]> = Box::new([2, 5, 0]);- let packing_divisor = quant_mode.packing_divisor() as u32;+ let packing_divisor = quant_mode.packing_divisor();let weights_stride = model_dim / packing_divisor;let weights: Vec<u8> = (0..vocab_size as usize * weights_stride as usize).map(|i| ((i % 16) as u8) | ((((i + 3) % 16) as u8) * 16))@@ -88,7 +88,7 @@ fn get_test_data_zero_point_group16<T: ArrayElement + Float>() -> (Input<T>, Veclet token_ids: Box<[u32]> = Box::new([2, 5, 0]);- let packing_divisor = quant_mode.packing_divisor() as u32;+ let packing_divisor = quant_mode.packing_divisor();let weights_stride = model_dim / packing_divisor;let weights: Vec<u8> = (0..vocab_size as usize * weights_stride as usize).map(|i| ((i % 16) as u8) | ((((i + 3) % 16) as u8) * 16))diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rsindex df0a7678c..37877e30a 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs@@ -65,15 +65,15 @@ struct BenchmarkData {impl BenchmarkData {fn new(context: &MetalContext,- m: usize,- k: usize,- n: usize,+ m: u32,+ k: u32,+ n: u32,bits: u32,group_size: u32,seed: u64,) -> Self {let input = QuantInput::<bf16>::new(m, k, n, group_size, bits, QuantizationMethod::ScaleSymmetric, seed)- .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE as usize, None);+ .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, None);let unsigned_weights = alloc_allocation_with_data::<Metal, u32>(context, &input.w_packed);let signed_weights = alloc_allocation_with_data::<Metal, u32>(context, &input.weights_for_upload());@@ -90,19 +90,20 @@ impl BenchmarkData {.collect();let rht_factors = alloc_allocation_with_data::<Metal, i32>(context, &rht);- let groups = k / group_size as usize;+ let groups = k / group_size;+ let a_elements = m as usize * k as usize;Self {unsigned_weights,signed_weights,weight_scales,activations,rht_factors,- a_working: alloc_allocation::<Metal, bf16>(context, m * k),- a_int8: alloc_allocation::<Metal, i8>(context, m * k),- a_scales: alloc_allocation::<Metal, f32>(context, m * groups),- m: m as u32,- k: k as u32,- n: n as u32,+ a_working: alloc_allocation::<Metal, bf16>(context, a_elements),+ a_int8: alloc_allocation::<Metal, i8>(context, a_elements),+ a_scales: alloc_allocation::<Metal, f32>(context, m as usize * groups as usize),+ m,+ k,+ n,group_size,mode: if bits == 4 {QuantizationMode::U4@@ -237,10 +238,10 @@ fn bench_bits(k,n,bits,- HADAMARD_TRANSFORM_BLOCK_SIZE as u32,+ HADAMARD_TRANSFORM_BLOCK_SIZE,0xA8_00 ^ u64::from(bits) ^ k as u64 ^ n as u64,);- let mut output = alloc_allocation::<Metal, bf16>(context, m * n);+ let mut output = alloc_allocation::<Metal, bf16>(context, m as usize * n as usize);let shape_label = format!("{layer}_m{m}_k{k}_n{n}");let gemv_eligible = GemvSpecialization::select_shape(diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemm_bench.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemm_bench.rsindex 7159bbd31..3995d0008 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemm_bench.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemm_bench.rs@@ -45,10 +45,10 @@ fn bench_gemm(c: &mut Criterion) {for shape in bench_fp_gemm_shapes() {let (m, k, n) = (shape.m, shape.k, shape.n);- let a = alloc_allocation::<Metal, bf16>(&context, m * k);- let b_weights = alloc_allocation::<Metal, bf16>(&context, n * k);- let mut d = alloc_allocation::<Metal, bf16>(&context, m * n);- group.throughput(Throughput::Elements((2 * m * k * n) as u64));+ let a = alloc_allocation::<Metal, bf16>(&context, m as usize * k as usize);+ let b_weights = alloc_allocation::<Metal, bf16>(&context, n as usize * k as usize);+ let mut d = alloc_allocation::<Metal, bf16>(&context, m as usize * n as usize);+ group.throughput(Throughput::Elements(2 * u64::from(m) * u64::from(k) * u64::from(n)));group.bench_function(BenchmarkId::new("BF16", shape.to_string()), |b| {iter_encode_loop::<Metal, _>(&context, b, |encoder| {kernel@@ -67,9 +67,9 @@ fn bench_gemm(c: &mut Criterion) {d: &mut d,d_transform: MatmulDOps::none(),gather_indices: None,- m: m as u32,- n: n as u32,- k: k as u32,+ m,+ n,+ k,},engine,encoder,diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rsindex 596a31cca..d069c40a3 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rs@@ -242,7 +242,7 @@ fn gemv_gather() {(8, QuantizationMethod::ScaleZeroPoint),] {let (m, k, vocab, ids_per_row, group_size) = (8usize, 128usize, 64usize, 8usize, 32u32);- let input = QuantInput::<bf16>::new(m, k, vocab, group_size, bits, method, 0x5EED);+ let input = QuantInput::<bf16>::new(m as u32, k as u32, vocab as u32, group_size, bits, method, 0x5EED);let ids: Vec<u32> = (0..m * ids_per_row).map(|i| ((i * 37 + 11) % vocab) as u32).collect();// K_SPLIT == 1 keeps k in one reduction, so gather and dense share the exact accumulation.check_gather!(m, vocab, ids, ids_per_row, 0.05, |B| {diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rsindex c36aa3a91..f378ebfc6 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs@@ -70,9 +70,9 @@ fn assert_parity<T: ArrayElement + Float + Debug + Display>(}fn run_parity<T: ArrayElement + Float + Debug + Display>(- m: usize,- k: usize,- n: usize,+ m: u32,+ k: u32,+ n: u32,group_size: u32,bits: u32,quant_method: QuantizationMethod,@@ -114,9 +114,9 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(#[case::gs32_narrow_n_tail_sym(64, 256, 40, 32, 4, QuantizationMethod::ScaleSymmetric)]#[case::gs16_k_tail_mlx(64, 272, 64, 16, 4, QuantizationMethod::ScaleBias)]fn parity_bf16(- #[case] m: usize,- #[case] k: usize,- #[case] n: usize,+ #[case] m: u32,+ #[case] k: u32,+ #[case] n: u32,#[case] gs: u32,#[case] bits: u32,#[case] method: QuantizationMethod,@@ -131,9 +131,9 @@ fn parity_bf16(#[case::gs128_8bit_zp_prefill(64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]#[case::gs64_8bit_symmetric_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]fn parity_bf16_8bit_splitk(- #[case] m: usize,- #[case] k: usize,- #[case] n: usize,+ #[case] m: u32,+ #[case] k: u32,+ #[case] n: u32,#[case] gs: u32,#[case] bits: u32,#[case] method: QuantizationMethod,@@ -156,9 +156,9 @@ fn parity_bf16_8bit_splitk(#[case::m8_gs32_4bit_zp(8, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]#[case::m8_gs32_4bit_mlx(8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]fn parity_gemv_bf16(- #[case] m: usize,- #[case] k: usize,- #[case] n: usize,+ #[case] m: u32,+ #[case] k: u32,+ #[case] n: u32,#[case] gs: u32,#[case] bits: u32,#[case] method: QuantizationMethod,@@ -254,9 +254,9 @@ fn parity_bf16_gemv_qmv_fused_scale_bias() {#[case::gs64_4bit_zp(2, 96, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]#[case::gs128_8bit(1, 192, 64, 128, 8, QuantizationMethod::ScaleBias)]fn parity_gemv_partial_group_bf16(- #[case] m: usize,- #[case] k: usize,- #[case] n: usize,+ #[case] m: u32,+ #[case] k: u32,+ #[case] n: u32,#[case] gs: u32,#[case] bits: u32,#[case] method: QuantizationMethod,@@ -272,9 +272,9 @@ fn parity_gemv_partial_group_bf16(#[case::n20_gs64_4bit_zp(2, 256, 20, 64, 4, QuantizationMethod::ScaleZeroPoint)]#[case::n36_gs32_8bit(1, 256, 36, 32, 8, QuantizationMethod::ScaleBias)]fn parity_gemv_unaligned_width_bf16(- #[case] m: usize,- #[case] k: usize,- #[case] n: usize,+ #[case] m: u32,+ #[case] k: u32,+ #[case] n: u32,#[case] gs: u32,#[case] bits: u32,#[case] method: QuantizationMethod,@@ -452,9 +452,9 @@ fn quant_gemm_accumulate_returns_unsupported_dop() {#[case::gs64_8bit_zp(128, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]#[case::gs128_8bit_zp(128, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]fn mxu_quant_parity_bf16(- #[case] m: usize,- #[case] k: usize,- #[case] n: usize,+ #[case] m: u32,+ #[case] k: u32,+ #[case] n: u32,#[case] gs: u32,#[case] bits: u32,#[case] method: QuantizationMethod,@@ -477,20 +477,20 @@ fn mxu_quant_parity_bf16(#[rstest]#[test_attr(uzu_test)]-#[case::sym_w4_gs32(16usize, 32u32, 4u32, QuantizationMethod::ScaleSymmetric)]-#[case::sym_w8_gs64(16usize, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]-#[case::sym_w4_gs128(32usize, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]-#[case::bias_w4_gs32(16usize, 32u32, 4u32, QuantizationMethod::ScaleBias)]-#[case::bias_w8_gs64(16usize, 64u32, 8u32, QuantizationMethod::ScaleBias)]-#[case::bias_w4_gs128(32usize, 128u32, 4u32, QuantizationMethod::ScaleBias)]-#[case::zp_w4_gs32(16usize, 32u32, 4u32, QuantizationMethod::ScaleZeroPoint)]-#[case::zp_w8_gs64(16usize, 64u32, 8u32, QuantizationMethod::ScaleZeroPoint)]-#[case::zp_w4_gs128(32usize, 128u32, 4u32, QuantizationMethod::ScaleZeroPoint)]-#[case::m1_bias_w4_gs64(1usize, 64u32, 4u32, QuantizationMethod::ScaleBias)]-#[case::m1_zp_w8_gs32(1usize, 32u32, 8u32, QuantizationMethod::ScaleZeroPoint)]-#[case::m1_sym_w4_gs128(1usize, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]+#[case::sym_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleSymmetric)]+#[case::sym_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]+#[case::sym_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]+#[case::bias_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleBias)]+#[case::bias_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleBias)]+#[case::bias_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleBias)]+#[case::zp_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleZeroPoint)]+#[case::zp_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleZeroPoint)]+#[case::zp_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleZeroPoint)]+#[case::m1_bias_w4_gs64(1u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]+#[case::m1_zp_w8_gs32(1u32, 32u32, 8u32, QuantizationMethod::ScaleZeroPoint)]+#[case::m1_sym_w4_gs128(1u32, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]fn a8w_mxu_parity_bf16(- #[case] m: usize,+ #[case] m: u32,#[case] weight_gs: u32,#[case] bits: u32,#[case] method: QuantizationMethod,@@ -499,9 +499,9 @@ fn a8w_mxu_parity_bf16(if !context.supports_mxu() {return;}- let (k, n) = (256usize, 128usize);+ let (k, n) = (256u32, 128u32);let input = QuantInput::<bf16>::new(m, k, n, weight_gs, bits, method, 0).with_prepared_a(- ACTIVATION_SCALE_GROUP_SIZE as usize,+ ACTIVATION_SCALE_GROUP_SIZE,(method != QuantizationMethod::ScaleSymmetric).then_some(weight_gs),);let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmEngine::Mxu));@@ -517,17 +517,17 @@ fn a8w_mxu_parity_bf16(#[rstest]#[test_attr(uzu_test)]-#[case::m1(1usize)]-#[case::m16(16usize)]-#[case::m33(33usize)]-fn a8w_independent_activation_group_parity_bf16(#[case] m: usize) {+#[case::m1(1u32)]+#[case::m16(16u32)]+#[case::m33(33u32)]+fn a8w_independent_activation_group_parity_bf16(#[case] m: u32) {let context = MetalContext::new().expect("Metal context");if !context.supports_mxu() {return;}for (activation_group_size, weight_group_size) in- [(32usize, 32u32), (32, 64), (64, 64), (64, 128), (128, 32), (128, 64), (128, 128)]+ [(32u32, 32u32), (32, 64), (64, 64), (64, 128), (128, 32), (128, 64), (128, 128)]{for (bits, method) in [(4, QuantizationMethod::ScaleSymmetric),@@ -537,8 +537,7 @@ fn a8w_independent_activation_group_parity_bf16(#[case] m: usize) {] {let input = QuantInput::<bf16>::new(m, 256, 72, weight_group_size, bits, method, 0).with_prepared_a(activation_group_size,- (method != QuantizationMethod::ScaleSymmetric)- .then_some((activation_group_size as u32).min(weight_group_size)),+ (method != QuantizationMethod::ScaleSymmetric).then_some(activation_group_size.min(weight_group_size)),);let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmEngine::Mxu));let reference = run_quant_cpu::<bf16>(&input);@@ -558,13 +557,13 @@ fn a8w_independent_activation_group_parity_bf16(#[case] m: usize) {#[case::m1(1)]#[case::m16(16)]#[case::m33(33)]-fn a8w4_zero_point_tail_parity(#[case] m: usize) {+fn a8w4_zero_point_tail_parity(#[case] m: u32) {let context = MetalContext::new().expect("Metal context");if !context.supports_mxu() {return;}let input = QuantInput::<bf16>::new(m, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint, 0)- .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE as usize, Some(32));+ .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, Some(32));let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmEngine::Mxu));let reference = run_quant_cpu::<bf16>(&input);assert_parity::<bf16>(&format!("A8W4 ZP N-tail m={m}"), &reference, &actual, 0.08, 0.8);@@ -577,7 +576,7 @@ fn a8w4_zero_point_tail_signed_codes_parity() {return;}let input = QuantInput::<bf16>::new(33, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint, 0)- .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE as usize, Some(32))+ .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, Some(32)).with_signed_weight_codes();let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmEngine::Mxu));let reference = run_quant_cpu::<bf16>(&input);@@ -586,20 +585,20 @@ fn a8w4_zero_point_tail_signed_codes_parity() {#[rstest]#[test_attr(uzu_test)]-#[case::w4_sym(4u32, QuantizationMethod::ScaleSymmetric, 256usize)]-#[case::w4_bias(4u32, QuantizationMethod::ScaleBias, 256usize)]-#[case::w4_zp(4u32, QuantizationMethod::ScaleZeroPoint, 256usize)]-#[case::w8_sym(8u32, QuantizationMethod::ScaleSymmetric, 256usize)]-#[case::w8_bias(8u32, QuantizationMethod::ScaleBias, 256usize)]-#[case::w8_zp(8u32, QuantizationMethod::ScaleZeroPoint, 256usize)]-#[case::w8_zp_tail(8u32, QuantizationMethod::ScaleZeroPoint, 224usize)]+#[case::w4_sym(4u32, QuantizationMethod::ScaleSymmetric, 256u32)]+#[case::w4_bias(4u32, QuantizationMethod::ScaleBias, 256u32)]+#[case::w4_zp(4u32, QuantizationMethod::ScaleZeroPoint, 256u32)]+#[case::w8_sym(8u32, QuantizationMethod::ScaleSymmetric, 256u32)]+#[case::w8_bias(8u32, QuantizationMethod::ScaleBias, 256u32)]+#[case::w8_zp(8u32, QuantizationMethod::ScaleZeroPoint, 256u32)]+#[case::w8_zp_tail(8u32, QuantizationMethod::ScaleZeroPoint, 224u32)]fn signed_weights_full_precision_activations_parity_bf16(#[case] bits: u32,#[case] method: QuantizationMethod,- #[case] k: usize,+ #[case] k: u32,) {let context = MetalContext::new().expect("Metal context");- let (m, n, group_size) = (2usize, 128usize, 32u32);+ let (m, n, group_size) = (2u32, 128u32, 32u32);let input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0).with_signed_weight_codes();let reference_input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0);let reference = run_quant_cpu::<bf16>(&reference_input);@@ -632,16 +631,16 @@ fn a8w_mxu_output_bias_parity_bf16(}let (m, k, n) = (- 16usize,- 256usize,+ 16u32,+ 256u32,if with_output_hadamard {- 64usize+ 64u32} else {- 70usize+ 70u32},);let input = QuantInput::<bf16>::new(m, k, n, 32, bits, QuantizationMethod::ScaleSymmetric, 0)- .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE as usize, None);+ .with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, None);let output_bias: Vec<bf16> = (0..n).map(|column| bf16::from_f32(0.25 + 0.05 * (column % 7) as f32)).collect();let output_hadamard_factors: Option<Vec<i32>> = with_output_hadamard.then(|| {(0..n)@@ -676,7 +675,7 @@ fn a8w_mxu_output_bias_parity_bf16(cpu_matmul.encode(cpu_arguments, &mut cpu_encoder).expect("CPU A8W matmul");cpu_encoder.end_encoding().submit().wait_until_completed().unwrap();let mut reference = allocation_to_vec::<Cpu, bf16>(&cpu_buffers.y);- for row in reference.chunks_exact_mut(n) {+ for row in reference.chunks_exact_mut(n as usize) {for (value, bias) in row.iter_mut().zip(&output_bias) {*value = bf16::from_f32(value.to_f32() + bias.to_f32());}@@ -758,10 +757,10 @@ fn run_widened_f32<B: Backend>(#[case::gemv_m1(1)]#[case::gemv_m4(4)]#[case::gemm_m31(31)]-fn parity_widened_f32_output(#[case] m: usize) {+fn parity_widened_f32_output(#[case] m: u32) {// Mirrors the dflash draft-chain readout: quantized LM head (4-bit ZP,// group 32), BF16 input, widened F32 output.- let (k, n, gs) = (5120usize, 32768usize, 32u32);+ let (k, n, gs) = (5120u32, 32768u32, 32u32);let context = MetalContext::new().expect("Metal context");let input = QuantInput::<bf16>::new(m, k, n, gs, 4, QuantizationMethod::ScaleZeroPoint, 0);let cpu_context = <Cpu as Backend>::Context::new().expect("Cpu context");diff --git a/crates/backend-uzu/tests/unit/backends/metal/kernel/gdn/chunked_test.rs b/crates/backend-uzu/tests/unit/backends/metal/kernel/gdn/chunked_test.rsindex 220c352b2..7e18e2180 100644--- a/crates/backend-uzu/tests/unit/backends/metal/kernel/gdn/chunked_test.rs+++ b/crates/backend-uzu/tests/unit/backends/metal/kernel/gdn/chunked_test.rs@@ -140,7 +140,7 @@ fn run_prefill<T: ArrayElement>(value_head_dim: HEAD_V_DIM as u32,key_dim: case.key_dim as u32,value_dim: case.value_dim as u32,- suffix_len: case.suffix_len,+ suffix_len: case.suffix_len as u32,},&mut encoder,)@@ -185,7 +185,7 @@ fn chunked_prefill_matches_recurrent_prefill() {}for suffix_len in [CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, CHUNK_SIZE * 2 + 1] {- let case = test_case(suffix_len);+ let case = test_case(suffix_len as usize);let (recurrent_out, recurrent_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Recurrent);let (chunked_out, chunked_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Chunked);diff --git a/crates/backend-uzu/tests/unit/encodable_block/attention_gemm_test.rs b/crates/backend-uzu/tests/unit/encodable_block/attention_gemm_test.rsindex e08fdd295..8c86488e0 100644--- a/crates/backend-uzu/tests/unit/encodable_block/attention_gemm_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/attention_gemm_test.rs@@ -83,9 +83,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");let new_arguments = AttentionCoreNewArguments {- head_dim: input.head_dim,- num_groups: input.num_kv_heads,- num_q_heads: input.num_heads,+ head_dim: input.head_dim as u32,+ num_groups: input.num_kv_heads as u32,+ num_q_heads: input.num_heads as u32,has_sinks: false,is_kv_cache_ring: false,is_causal: input.do_causal,@@ -101,14 +101,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let segment_prefix_length = input.sequence_length - input.suffix_length;let state_type = AttentionStateType::Full {- length: segment_prefix_length,+ length: segment_prefix_length as u32,};let args = AttentionCoreEncodeArguments {queries: &queries_allocation,keys: &keys_allocation,values: &values_allocation,- suffix_length: input.suffix_length,+ suffix_length: input.suffix_length as u32,trie: None,sinks: None,state_type: &state_type,diff --git a/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs b/crates/backend-uzu/tests/unit/encodable_block/attention_test.rsindex 8e8fb4702..7e201ac17 100644--- a/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs@@ -321,9 +321,9 @@ fn run_gemm_attention(let core = <<Metal as Backend>::Kernels as Kernels>::AttentionGemmCore::new(context,&AttentionCoreNewArguments {- head_dim,- num_groups: num_kv_heads,- num_q_heads: num_heads,+ head_dim: head_dim as u32,+ num_groups: num_kv_heads as u32,+ num_q_heads: num_heads as u32,has_sinks: sinks.is_some(),is_kv_cache_ring: false,is_causal,@@ -349,7 +349,7 @@ fn run_gemm_attention(queries: &query_allocation,keys: &key_allocation,values: &value_allocation,- suffix_length: seq_len,+ suffix_length: seq_len as u32,trie: None,sinks: sinks_allocation.as_ref(),state_type: &state_type,diff --git a/crates/backend-uzu/tests/unit/encodable_block/delta_net_tree_verify_bench.rs b/crates/backend-uzu/tests/unit/encodable_block/delta_net_tree_verify_bench.rsindex bea09ba33..4ff105242 100644--- a/crates/backend-uzu/tests/unit/encodable_block/delta_net_tree_verify_bench.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/delta_net_tree_verify_bench.rs@@ -23,10 +23,10 @@ fn bench_delta_net_tree_verify(c: &mut Criterion) {const HEAD_DIM: usize = 128;let arguments = TreeVerifyNewArguments {data_type: DataType::BF16,- num_k_heads: K_HEADS,- num_v_heads: V_HEADS,- head_k_dim: HEAD_DIM,- head_v_dim: HEAD_DIM,+ num_k_heads: K_HEADS as u32,+ num_v_heads: V_HEADS as u32,+ head_k_dim: HEAD_DIM as u32,+ head_v_dim: HEAD_DIM as u32,};let context = <Metal as Backend>::Context::new().unwrap();let started = Instant::now();@@ -69,7 +69,7 @@ fn bench_delta_net_tree_verify(c: &mut Criterion) {log_decay: &log_decay,beta: &beta,h0: &h0,- tree_size,+ tree_size: tree_size as u32,},encoder,)diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rsindex b4c8e67dd..1ba18e376 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs@@ -382,7 +382,9 @@ fn run_moe_parity_test_internal<B: Backend>();let gather = MoeGather::<B>::new(ctx, DataType::BF16).expect("gather");- let x_perm_buf = gather.encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t, k, d_model, &mut encoder).expect("gather");+ let x_perm_buf = gather+ .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut encoder)+ .expect("gather");let total_rows = t * k;@@ -394,10 +396,10 @@ fn run_moe_parity_test_internal<B: Backend>(w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- total_rows,- d_model,- d_ff,- num_routed_experts: e,+ total_rows: total_rows as u32,+ d_model: d_model as u32,+ d_ff: d_ff as u32,+ num_routed_experts: e as u32,gate_clip_min: gate_clip.0,gate_clip_max: gate_clip.1,up_clip_min: up_clip.0,diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_perf_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_perf_test.rsindex e92ee7dc8..8d843e0dc 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_perf_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_perf_test.rs@@ -89,10 +89,10 @@ fn run_decode_case<B: Backend>(w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- total_rows: sum_k,- d_model,- d_ff,- num_routed_experts: e,+ total_rows: sum_k as u32,+ d_model: d_model as u32,+ d_ff: d_ff as u32,+ num_routed_experts: e as u32,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,up_clip_min: f32::NEG_INFINITY,@@ -120,10 +120,10 @@ fn run_decode_case<B: Backend>(w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- total_rows: sum_k,- d_model,- d_ff,- num_routed_experts: e,+ total_rows: sum_k as u32,+ d_model: d_model as u32,+ d_ff: d_ff as u32,+ num_routed_experts: e as u32,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,up_clip_min: f32::NEG_INFINITY,@@ -208,10 +208,10 @@ fn run_two_pass_prefill_case<B: Backend>(w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- total_rows: sum_k,- d_model,- d_ff,- num_routed_experts: e,+ total_rows: sum_k as u32,+ d_model: d_model as u32,+ d_ff: d_ff as u32,+ num_routed_experts: e as u32,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,up_clip_min: f32::NEG_INFINITY,@@ -235,10 +235,10 @@ fn run_two_pass_prefill_case<B: Backend>(w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- total_rows: sum_k,- d_model,- d_ff,- num_routed_experts: e,+ total_rows: sum_k as u32,+ d_model: d_model as u32,+ d_ff: d_ff as u32,+ num_routed_experts: e as u32,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,up_clip_min: f32::NEG_INFINITY,diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rsindex 84adc358b..61d0c7346 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs@@ -381,10 +381,10 @@ fn test_two_pass_decode_correctness() {w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- total_rows: sum_k,- d_model,- d_ff,- num_routed_experts: e,+ total_rows: sum_k as u32,+ d_model: d_model as u32,+ d_ff: d_ff as u32,+ num_routed_experts: e as u32,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,up_clip_min: f32::NEG_INFINITY,@@ -503,10 +503,10 @@ fn test_two_pass_decode_multi_token() {w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- total_rows: sum_k,- d_model,- d_ff,- num_routed_experts: e,+ total_rows: sum_k as u32,+ d_model: d_model as u32,+ d_ff: d_ff as u32,+ num_routed_experts: e as u32,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,up_clip_min: f32::NEG_INFINITY,@@ -586,10 +586,10 @@ fn test_two_pass_prefill_correctness() {w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- total_rows: sum_k,- d_model,- d_ff,- num_routed_experts: e,+ total_rows: sum_k as u32,+ d_model: d_model as u32,+ d_ff: d_ff as u32,+ num_routed_experts: e as u32,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,up_clip_min: f32::NEG_INFINITY,diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_gather_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_gather_test.rsindex 4782573a7..de62c28de 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_gather_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_gather_test.rs@@ -38,9 +38,9 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {&x_allocation,&ids_allocation,&sumk_allocation,- input.t,- input.sum_k / input.t,- input.d_model,+ input.t as u32,+ (input.sum_k / input.t) as u32,+ input.d_model as u32,&mut encoder,).expect("Failed to encode MoE gather");diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_perf_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_perf_test.rsindex 4f84945c7..3c41b99f3 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_perf_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_perf_test.rs@@ -321,7 +321,7 @@ fn test_moe_pipeline_breakdown_decode() {let gather_perf = run_perf_with_warmup("Gather", 2, 5, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");let x_perm = gather- .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t, k, d_model, &mut encoder)+ .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut encoder).expect("failed to encode MoE gather");let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(x_perm);@@ -331,7 +331,7 @@ fn test_moe_pipeline_breakdown_decode() {let (x_perm_buf, x_perm_completed) = {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");let x_perm = gather- .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t, k, d_model, &mut encoder)+ .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut encoder).expect("failed to encode MoE gather");let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();(x_perm, completed)@@ -348,10 +348,10 @@ fn test_moe_pipeline_breakdown_decode() {w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- total_rows: sum_k,- d_model,- d_ff,- num_routed_experts: e,+ total_rows: sum_k as u32,+ d_model: d_model as u32,+ d_ff: d_ff as u32,+ num_routed_experts: e as u32,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: 20.0,up_clip_min: -19.0,diff --git a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rsindex 6b6e5f80f..0214beb8a 100644--- a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs@@ -39,9 +39,9 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(bitmask: Option<&[u32]>,vocab_size: usize,method: &SamplingMethod,- batch_size: usize,+ batch_size: u32,) -> Result<SamplingTestResults, TestCaseError> {- let sampling = Sampling::new(T::data_type(), vocab_size);+ let sampling = Sampling::new(T::data_type(), vocab_size as u32);let mut logits_allocation =context.create_allocation(logits.len() * T::data_type().size_in_bytes(), AllocationType::Global).unwrap();@@ -63,9 +63,9 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(let nodes = (0..batch_size).map(|index| TrieNode {- trie_start: index as u32,- trie_end: (batch_size - 1) as u32,- height: index as u32,+ trie_start: index,+ trie_end: batch_size - 1,+ height: index,}).collect::<Box<[_]>>();let batch_topology = BatchTopology::new(&nodes, true);@@ -182,7 +182,7 @@ fn test_sampling_prop() {bitmask.as_ref().map(Box::as_ref),sampling_case.vocab_size,&sampling_case.method,- sampling_case.batch_size,+ sampling_case.batch_size as u32,)).compare_results()?});