A high-performance inference engine for AI models
replace suffix length with active_row_count (#305)
eugene002dc5f57ab3
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rsindex f2ab1afda..1204b63e9 100644--- a/crates/uzu/src/encodable_block/embedding.rs+++ b/crates/uzu/src/encodable_block/embedding.rs@@ -491,7 +491,7 @@ impl<B: Backend> Embedding<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), EmbeddingError<B>> {- let batch_size = state.active_suffix_length() as u32;+ let batch_dim = state.active_row_count() as u32;let token_ids_array = state.array(ArrayId::TokenIds);let token_ids_buffer_rc = token_ids_array.buffer();@@ -523,7 +523,7 @@ impl<B: Backend> Embedding<B> {token_ids,weights,output,- batch_size,+ batch_dim,self.vocab_size,self.model_dim,self.input_scale,@@ -555,7 +555,7 @@ impl<B: Backend> Embedding<B> {scales,biases,output,- batch_size,+ batch_dim,self.vocab_size,self.model_dim,self.input_scale,@@ -572,9 +572,9 @@ impl<B: Backend> Embedding<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), EmbeddingError<B>> {- let batch_size = state.sampling_length();+ let batch_dim = state.sampling_length();- if batch_size == 0 {+ if batch_dim == 0 {return Ok(());}@@ -618,7 +618,7 @@ impl<B: Backend> Embedding<B> {ab_scale: 1.0,c: MatmulArgumentC::None,d: output,- batch_dim: batch_size as u32,+ batch_dim: batch_dim as u32,input_dim: input_dim as u32,output_dim: output_dim as u32,},@@ -654,7 +654,7 @@ impl<B: Backend> Embedding<B> {scales_buffer: scales,zero_points_or_biases_buffer: biases,output_buffer: output,- batch: batch_size,+ batch: batch_dim,input_dim: self.model_dim as usize,output_dim: self.vocab_size as usize,quantization_type: QuantizedMatmulType::Mlx,diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rsindex e811bdfb5..765c4d8b4 100644--- a/crates/uzu/src/encodable_block/layer_norm.rs+++ b/crates/uzu/src/encodable_block/layer_norm.rs@@ -76,7 +76,7 @@ impl<B: Backend> LayerNorm<B> {let input_array = state.array(self.input_array_id);let output_array = state.array(self.output_array_id);- let batch_size = input_array.shape()[0] as u32;+ let batch_dim = input_array.shape()[0] as u32;let model_dim = input_array.shape()[1] as u32;let full_layer = if self.config.upcast_mode == UpcastMode::FullLayer {1u32@@ -91,7 +91,7 @@ impl<B: Backend> LayerNorm<B> {input_buffer_borrow.as_deref(),self.scales_buffer.borrow().deref(),output_array.buffer().borrow_mut().deref_mut(),- batch_size,+ batch_dim,model_dim,self.config.epsilon,self.config.scale_offset.unwrap_or(0.0),diff --git a/crates/uzu/src/encodable_block/linear/full_precision.rs b/crates/uzu/src/encodable_block/linear/full_precision.rsindex 9111d4a14..5bd2a58c0 100644--- a/crates/uzu/src/encodable_block/linear/full_precision.rs+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs@@ -134,7 +134,7 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let batch_size = state.active_suffix_length();+ let batch_dim = state.active_row_count();let input_array = state.array(self.input_array_id);let output_array = state.array(self.output_array_id);@@ -151,7 +151,7 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {None => MatmulArgumentC::None,},d: output_array.buffer().borrow_mut().deref_mut(),- batch_dim: batch_size as u32,+ batch_dim: batch_dim as u32,input_dim: self.input_dim as u32,output_dim: self.output_dim as u32,},diff --git a/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rsindex 3552c1564..37d7eecea 100644--- a/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -134,7 +134,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {self.base_linear.encode(state, encoder)?;let mut adapter_kernel = self.adapter_kernel.borrow_mut();- let batch_size = state.active_suffix_length();+ let batch_dim = state.active_row_count();let intermediate_array = state.common_aux.lora_intermediate.as_ref().unwrap();let input_array = state.array(self.input_array_id);@@ -149,7 +149,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {ab_scale: 1.0,c: MatmulArgumentC::None,d: intermediate_array.buffer().borrow_mut().deref_mut(),- batch_dim: batch_size as u32,+ batch_dim: batch_dim as u32,input_dim: self.input_dim as u32,output_dim: self.lora_rank as u32,},@@ -165,7 +165,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {ab_scale: self.lora_scale,c: MatmulArgumentC::Accumulate,d: output_array.buffer().borrow_mut().deref_mut(),- batch_dim: batch_size as u32,+ batch_dim: batch_dim as u32,input_dim: self.lora_rank as u32,output_dim: self.output_dim as u32,},diff --git a/crates/uzu/src/encodable_block/linear/quantized.rs b/crates/uzu/src/encodable_block/linear/quantized.rsindex 360617310..6450399ba 100644--- a/crates/uzu/src/encodable_block/linear/quantized.rs+++ b/crates/uzu/src/encodable_block/linear/quantized.rs@@ -251,7 +251,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let batch_size = state.active_suffix_length();+ let batch_dim = state.active_row_count();let input_array = state.array(self.input_array_id);let output_array = state.array(self.output_array_id);@@ -268,7 +268,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {scales_buffer: self.scales_buffer.borrow().deref(),zero_points_or_biases_buffer: self.zero_points_or_biases_buffer.borrow().deref(),output_buffer: output_buf_borrow.deref_mut(),- batch: batch_size,+ batch: batch_dim,input_dim: self.input_dim,output_dim: self.output_dim,quantization_type: self.quantization_type,@@ -277,7 +277,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {.expect("Failed to encode quantized matmul");if let (Some(bias_add_kernel), Some(biases_buffer)) = (&self.bias_add_kernel, &self.biases_buffer) {- let total_length = batch_size * self.output_dim;+ let total_length = batch_dim * self.output_dim;bias_add_kernel.encode(None::<&B::Buffer>,biases_buffer.borrow().deref(),diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex f7a5862b1..183ba1f7a 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -359,18 +359,18 @@ impl<B: Backend> MambaMixer<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let active_suffix_length = state.active_suffix_length();- if active_suffix_length == 0 {+ let active_row_count = state.active_row_count();+ if active_row_count == 0 {return Ok(());}self.in_projection.encode(state, encoder)?;- self.run_split_inproj(state, encoder, active_suffix_length);- self.run_conv_scan(state, encoder, active_suffix_length);- if active_suffix_length == 1 {- self.run_decode_ssm(state, encoder, active_suffix_length);+ self.run_split_inproj(state, encoder, active_row_count);+ self.run_conv_scan(state, encoder, active_row_count);+ if active_row_count == 1 {+ self.run_decode_ssm(state, encoder, active_row_count);} else {- self.run_prefill_ssm(state, encoder, active_suffix_length);+ self.run_prefill_ssm(state, encoder, active_row_count);}self.out_projection.encode(state, encoder)?;Ok(())diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rsindex 33e9fc6e7..1dbd6a2ae 100644--- a/crates/uzu/src/encodable_block/mlp/moe.rs+++ b/crates/uzu/src/encodable_block/mlp/moe.rs@@ -198,7 +198,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let suffix_length = state.active_suffix_length();+ let suffix_length = state.active_row_count();let main_buf = state.array(ArrayId::Main).buffer();let topk_ids_buf = state.array(ArrayId::MoeTopkIds).buffer();diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rsindex fcf34f55f..fce310b06 100644--- a/crates/uzu/src/encodable_block/pooling.rs+++ b/crates/uzu/src/encodable_block/pooling.rs@@ -24,12 +24,12 @@ impl<B: Backend> PoolingKernel<B> {output: &mut B::Buffer,seq_len: u32,hidden_dim: u32,- batch_size: u32,+ batch_dim: u32,encoder: &mut Encoder<B>,) {match self {- Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),- Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),+ Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_dim, encoder),+ Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_dim, encoder),}}}@@ -63,7 +63,7 @@ impl<B: Backend> Pooling<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let batch_size = 1;+ let batch_dim = 1;let seq_len = state.aux_buffers_suffix_length();let main_array = state.array(ArrayId::Main);@@ -74,7 +74,7 @@ impl<B: Backend> Pooling<B> {pooling_array.buffer().borrow_mut().deref_mut(),seq_len as u32,self.model_dim as u32,- batch_size,+ batch_dim,encoder,);@@ -83,6 +83,8 @@ impl<B: Backend> Pooling<B> {let output_pooling_trace = state.traces().borrow().output_pooling().clone();state.encode_copy_array(encoder, ArrayId::ClassifierPooling, output_pooling_trace);}++ state.set_active_row_count(pooling_array.shape()[0]);Ok(())}}diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rsindex bc1de7ffb..636658dde 100644--- a/crates/uzu/src/encodable_block/qk_norm.rs+++ b/crates/uzu/src/encodable_block/qk_norm.rs@@ -127,7 +127,7 @@ impl<B: Backend> QKNorm<B> {encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let qkv_array = state.array(self.qkv_array_id);- let batch_size = qkv_array.shape()[0] as u32;+ let batch_dim = qkv_array.shape()[0] as u32;// Process query normalization if configuredif let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =@@ -137,7 +137,7 @@ impl<B: Backend> QKNorm<B> {None::<&B::Buffer>,query_scales_buffer.borrow().deref(),qkv_array.buffer().borrow_mut().deref_mut(),- batch_size,+ batch_dim,self.num_q_heads as u32,self.num_kv_heads as u32,self.head_dim as u32,@@ -158,7 +158,7 @@ impl<B: Backend> QKNorm<B> {None::<&B::Buffer>,key_scales_buffer.borrow().deref(),qkv_array.buffer().borrow_mut().deref_mut(),- batch_size,+ batch_dim,self.num_q_heads as u32,self.num_kv_heads as u32,self.head_dim as u32,diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rsindex e70ea0b28..64514c5fa 100644--- a/crates/uzu/src/encodable_block/rms_norm.rs+++ b/crates/uzu/src/encodable_block/rms_norm.rs@@ -99,7 +99,7 @@ impl<B: Backend> RMSNorm<B> {let (batch_start, batch_len) = if self.use_sampling_range {(state.sampling_start(), state.sampling_length())} else {- (0, state.active_suffix_length())+ (0, state.active_row_count())};let batch_len = batch_len.min(suffix_length.saturating_sub(batch_start));diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rsindex 9f0efd154..26fdd9173 100644--- a/crates/uzu/src/encodable_block/sampling.rs+++ b/crates/uzu/src/encodable_block/sampling.rs@@ -33,7 +33,7 @@ impl<B: Backend> Sampling<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let batch_size = state.sampling_length();+ let batch_dim = state.sampling_length();let sampling_start = state.sampling_start();let sampling_method = state.sampling_method().unwrap();@@ -60,7 +60,7 @@ impl<B: Backend> Sampling<B> {bitmask_offset,output.buffer().borrow_mut().deref_mut(),sampling_method,- batch_size,+ batch_dim,vocab_size,encoder,)diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rsindex e9d0b7696..2623867f6 100644--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -131,18 +131,18 @@ impl<B: Backend> ShortConvMixer<B> {&self,state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,- active_suffix_length: usize,+ active_row_count: usize,) {self.clear_suffix_state_valid_range(state);- if active_suffix_length == 1 {+ if active_row_count == 1 {self.run_decode_conv(state, encoder, 1);return;}let sampling_len = state.sampling_length();if sampling_len == 0 {- self.run_prefill_conv(state, encoder, active_suffix_length);+ self.run_prefill_conv(state, encoder, active_row_count);return;}@@ -150,7 +150,7 @@ impl<B: Backend> ShortConvMixer<B> {let trie_len = sampling_len;if trie_len <= 1 {- self.run_prefill_conv(state, encoder, active_suffix_length);+ self.run_prefill_conv(state, encoder, active_row_count);return;}@@ -309,14 +309,14 @@ impl<B: Backend> ShortConvMixer<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let active_suffix_length = state.active_suffix_length();- if active_suffix_length == 0 {+ let active_row_count = state.active_row_count();+ if active_row_count == 0 {return Ok(());}self.in_projection.encode(state, encoder)?;- self.run_conv(state, encoder, active_suffix_length);+ self.run_conv(state, encoder, active_row_count);self.out_projection.encode(state, encoder)?;diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex 8cd427da3..81442956d 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -35,7 +35,7 @@ pub struct LanguageModelGeneratorModeState<B: Backend> {pub sampling_method: Option<SamplingMethod>,#[cfg(feature = "tracing")]pub traces: Rc<RefCell<ActivationTrace<B>>>,- pub active_suffix_length: usize,+ pub active_row_count: usize,pub sampling_start: usize,pub sampling_length: usize,pub is_prefilling: bool,@@ -46,6 +46,7 @@ pub struct ClassifierModeState<B: Backend> {pub dense: Array<B>,pub norm: Array<B>,pub classifier_logits: Array<B>,+ pub active_row_count: usize,#[cfg(feature = "tracing")]pub traces: Rc<RefCell<ActivationTrace<B>>>,}@@ -151,7 +152,7 @@ impl<B: Backend> ForwardPassState<B> {token_positions: &[usize],token_bitmask: Option<&[u32]>,token_seeds: &[u64],- active_suffix_length: usize,+ active_row_count: usize,sampling_start: usize,sampling_length: usize,is_prefilling: bool,@@ -239,7 +240,7 @@ impl<B: Backend> ForwardPassState<B> {sampling_method: None,#[cfg(feature = "tracing")]traces,- active_suffix_length,+ active_row_count,sampling_start,sampling_length,is_prefilling,@@ -312,23 +313,24 @@ impl<B: Backend> ForwardPassState<B> {suffix_length: usize,) -> ClassifierModeState<B> {let data_type = model_shape.activation_data_type();- let batch_size = 1;+ let batch_dim = 1;let create_buffer = |size: usize| -> Array<B> {let buffer_size = size * data_type.size_in_bytes();let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_size, size / batch_size], data_type) }+ unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_dim, size / batch_dim], data_type) }};ClassifierModeState {- pooling: create_buffer(batch_size * model_dim),- dense: create_buffer(batch_size * model_dim),- norm: create_buffer(batch_size * model_dim),+ pooling: create_buffer(batch_dim * model_dim),+ dense: create_buffer(batch_dim * model_dim),+ norm: create_buffer(batch_dim * model_dim),classifier_logits: {- let buffer_size = batch_size * num_labels * data_type.size_in_bytes();+ let buffer_size = batch_dim * num_labels * data_type.size_in_bytes();let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_size, num_labels], data_type) }+ unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_dim, num_labels], data_type) }},+ active_row_count: suffix_length,traces: Rc::new(RefCell::new(ActivationTrace::new_classifier(context,model_shape,@@ -344,10 +346,10 @@ impl<B: Backend> ForwardPassState<B> {model_shape: &ModelShape,model_dim: usize,num_labels: usize,- _suffix_length: usize,+ suffix_length: usize,) -> ClassifierModeState<B> {let data_type = model_shape.activation_data_type();- let batch_size = 1;+ let batch_dim = 1;let create_buffer = |dims: &[usize]| -> Array<B> {let size: usize = dims.iter().product();@@ -357,10 +359,11 @@ impl<B: Backend> ForwardPassState<B> {};ClassifierModeState {- pooling: create_buffer(&[batch_size, model_dim]),- dense: create_buffer(&[batch_size, model_dim]),- norm: create_buffer(&[batch_size, model_dim]),- classifier_logits: create_buffer(&[batch_size, num_labels]),+ pooling: create_buffer(&[batch_dim, model_dim]),+ dense: create_buffer(&[batch_dim, model_dim]),+ norm: create_buffer(&[batch_dim, model_dim]),+ classifier_logits: create_buffer(&[batch_dim, num_labels]),+ active_row_count: suffix_length,}}@@ -594,10 +597,20 @@ impl<B: Backend> ForwardPassState<B> {// Public API Methods (formerly trait methods)// ========================================================================- pub fn active_suffix_length(&self) -> usize {+ pub fn active_row_count(&self) -> usize {match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => state.active_suffix_length,- ForwardPassMode::Classifier(_) => self.common_aux.suffix_length,+ ForwardPassMode::LanguageModelGenerator(state) => state.active_row_count,+ ForwardPassMode::Classifier(state) => state.active_row_count,+ }+ }++ pub fn set_active_row_count(+ &mut self,+ active_row_count: usize,+ ) {+ match &mut self.mode {+ ForwardPassMode::LanguageModelGenerator(state) => state.active_row_count = active_row_count,+ ForwardPassMode::Classifier(state) => state.active_row_count = active_row_count,}}diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex c0f01d1e0..ab5327902 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -45,7 +45,7 @@ struct Task<'a> {token_bitmask: Option<&'a [u32]>,token_seeds: &'a [u64],expected_number_of_new_tokens: usize,- active_suffix_length: usize,+ active_row_count: usize,sampling_start: usize,sampling_length: usize,is_prefilling: bool,@@ -56,7 +56,7 @@ struct TaskEncodingKey {context_len: usize,batch_size: usize,expected_number_of_new_tokens: usize,- active_suffix_len: usize,+ active_row_count: usize,sampling_method: SamplingMethod,sampling_start: usize,sampling_len: usize,@@ -225,7 +225,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let step_token_positions = step_token_positions.collect::<Box<[usize]>>();let step_token_seeds = step_token_seeds.collect::<Box<[u64]>>();- let active_suffix_length = step_token_positions.len();+ let active_row_count = step_token_positions.len();let is_last_prefill_step = step == prefill_steps - 1;let should_sample_after_step = sample_suffix && is_last_prefill_step;@@ -234,7 +234,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {// suffix-root token, which is the last prompt token).let (sampling_start, sampling_length) = if should_sample_after_step {let suffix_root_index_in_step = (tokens_length - 1).saturating_sub(tokens_start_index);- let sampling_length = active_suffix_length.saturating_sub(suffix_root_index_in_step);+ let sampling_length = active_row_count.saturating_sub(suffix_root_index_in_step);debug_assert!(sampling_length > 0, "Expected at least one token to sample on the last prefill step");(suffix_root_index_in_step, sampling_length)} else {@@ -277,7 +277,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {token_bitmask: step_token_bitmask.as_deref(),token_seeds: &step_token_seeds,expected_number_of_new_tokens: step_token_ids.len(),- active_suffix_length,+ active_row_count,sampling_start,sampling_length,is_prefilling: !should_sample_after_step,@@ -358,21 +358,21 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {);let flat_trie = suffix_root.linearize();- let active_suffix_length = flat_trie.len();+ let active_row_count = flat_trie.len();let token_ids =- flat_trie.token_ids().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();+ flat_trie.token_ids().chain(repeat_n(0, suffix_length - active_row_count)).collect::<Box<[u64]>>();let token_subtrie_ranges = flat_trie.token_subtrie_ranges()- .chain(repeat_n([u32::MAX, u32::MAX, u32::MAX], suffix_length - active_suffix_length))+ .chain(repeat_n([u32::MAX, u32::MAX, u32::MAX], suffix_length - active_row_count)).collect::<Box<[[u32; 3]]>>();let token_bitmask: Option<Box<[u32]>> = compiled_grammar.is_some().then(|| {let single_token_bitmask_size = self.context.model_shape.bitmask_shape(1)[1];flat_trie.token_masks()- .chain(repeat_n(None, suffix_length - active_suffix_length))+ .chain(repeat_n(None, suffix_length - active_row_count)).map(|mask| match mask {Some(mask) => Either::Left(mask.iter()@@ -390,11 +390,11 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let token_positions = flat_trie.token_positions().map(|trie_position| start_position + trie_position)- .chain(repeat_n(INVALID_POSITION, suffix_length - active_suffix_length))+ .chain(repeat_n(INVALID_POSITION, suffix_length - active_row_count)).collect::<Box<[usize]>>();let token_seeds =- flat_trie.token_seeds().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();+ flat_trie.token_seeds().chain(repeat_n(0, suffix_length - active_row_count)).collect::<Box<[u64]>>();let task = Task {token_ids: &token_ids,@@ -403,9 +403,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {token_bitmask: token_bitmask.as_deref(),token_seeds: &token_seeds,expected_number_of_new_tokens: 1,- active_suffix_length,+ active_row_count,sampling_start: 0,- sampling_length: active_suffix_length,+ sampling_length: active_row_count,is_prefilling: false,};@@ -415,7 +415,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let (accepted_tokens, accepted_token_indices) =flat_trie.accept(&sampled_tokens, compiled_grammar.as_deref_mut());- let speculator_proposed = active_suffix_length.saturating_sub(1);+ let speculator_proposed = active_row_count.saturating_sub(1);let speculator_accepted = accepted_tokens.len().saturating_sub(1);self.update_cache_layers(&accepted_token_indices, None, false)?;@@ -482,7 +482,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {token_bitmask: None,token_seeds: &[0], // Ignored, using async bufferexpected_number_of_new_tokens: 1,- active_suffix_length: 1,+ active_row_count: 1,sampling_start: 0,sampling_length: 1,is_prefilling: false,@@ -511,9 +511,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {&task.token_positions,task.token_bitmask,&task.token_seeds,- task.active_suffix_length,+ task.active_row_count,/*sampling_start=*/ 0,- /*sampling_length=*/ task.active_suffix_length,+ /*sampling_length=*/ task.active_row_count,task.is_prefilling,skip_token_ids_copy,async_positions,@@ -720,7 +720,7 @@ impl<B: Backend> LanguageModelGenerator<B> {task.token_positions,task.token_bitmask,task.token_seeds,- task.active_suffix_length,+ task.active_row_count,task.sampling_start,task.sampling_length,task.is_prefilling,@@ -737,7 +737,7 @@ impl<B: Backend> LanguageModelGenerator<B> {context_len: self.tokens.len(),batch_size: task.token_ids.len(),expected_number_of_new_tokens: task.expected_number_of_new_tokens,- active_suffix_len: task.active_suffix_length,+ active_row_count: task.active_row_count,sampling_method,sampling_start: task.sampling_start,sampling_len: task.sampling_length,diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex 58d334a5c..a0616de20 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -581,21 +581,6 @@ impl<B: Backend> TraceValidator<B> {) -> Vec<TracerValidationResult> {let mut results = Vec::new();- // Embedding norm (classifier-specific)- if let Some(embedding_norm) = &traces.borrow().embedding_norm {- if traces_view.leaf_array("activation_trace.embedding_norm").is_ok() {- results.push(TracerValidationResult {- name: "activation_trace.embedding_norm".to_string(),- metrics: Self::validate_array_with_name(- data_type,- traces_view,- "activation_trace.embedding_norm",- embedding_norm,- ),- });- }- }-// Output pooling (classifier-specific)if let Some(output_pooling) = &traces.borrow().output_pooling {if traces_view.leaf_array("activation_trace.output_pooling").is_ok() {