Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

replace suffix length with active_row_count (#305)

eugene002dc5f57ab3
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rs
index f2ab1afda..1204b63e9 100644
--- a/crates/uzu/src/encodable_block/embedding.rs
+++ b/crates/uzu/src/encodable_block/embedding.rs
@@ -491,7 +491,7 @@ impl<B: Backend> Embedding<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), EmbeddingError<B>> {
- let batch_size = state.active_suffix_length() as u32;
+ let batch_dim = state.active_row_count() as u32;
let token_ids_array = state.array(ArrayId::TokenIds);
let token_ids_buffer_rc = token_ids_array.buffer();
@@ -523,7 +523,7 @@ impl<B: Backend> Embedding<B> {
token_ids,
weights,
output,
- batch_size,
+ batch_dim,
self.vocab_size,
self.model_dim,
self.input_scale,
@@ -555,7 +555,7 @@ impl<B: Backend> Embedding<B> {
scales,
biases,
output,
- batch_size,
+ batch_dim,
self.vocab_size,
self.model_dim,
self.input_scale,
@@ -572,9 +572,9 @@ impl<B: Backend> Embedding<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), EmbeddingError<B>> {
- let batch_size = state.sampling_length();
+ let batch_dim = state.sampling_length();
- if batch_size == 0 {
+ if batch_dim == 0 {
return Ok(());
}
@@ -618,7 +618,7 @@ impl<B: Backend> Embedding<B> {
ab_scale: 1.0,
c: MatmulArgumentC::None,
d: output,
- batch_dim: batch_size as u32,
+ batch_dim: batch_dim as u32,
input_dim: input_dim as u32,
output_dim: output_dim as u32,
},
@@ -654,7 +654,7 @@ impl<B: Backend> Embedding<B> {
scales_buffer: scales,
zero_points_or_biases_buffer: biases,
output_buffer: output,
- batch: batch_size,
+ batch: batch_dim,
input_dim: self.model_dim as usize,
output_dim: self.vocab_size as usize,
quantization_type: QuantizedMatmulType::Mlx,
diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rs
index e811bdfb5..765c4d8b4 100644
--- a/crates/uzu/src/encodable_block/layer_norm.rs
+++ b/crates/uzu/src/encodable_block/layer_norm.rs
@@ -76,7 +76,7 @@ impl<B: Backend> LayerNorm<B> {
let input_array = state.array(self.input_array_id);
let output_array = state.array(self.output_array_id);
- let batch_size = input_array.shape()[0] as u32;
+ let batch_dim = input_array.shape()[0] as u32;
let model_dim = input_array.shape()[1] as u32;
let full_layer = if self.config.upcast_mode == UpcastMode::FullLayer {
1u32
@@ -91,7 +91,7 @@ impl<B: Backend> LayerNorm<B> {
input_buffer_borrow.as_deref(),
self.scales_buffer.borrow().deref(),
output_array.buffer().borrow_mut().deref_mut(),
- batch_size,
+ batch_dim,
model_dim,
self.config.epsilon,
self.config.scale_offset.unwrap_or(0.0),
diff --git a/crates/uzu/src/encodable_block/linear/full_precision.rs b/crates/uzu/src/encodable_block/linear/full_precision.rs
index 9111d4a14..5bd2a58c0 100644
--- a/crates/uzu/src/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs
@@ -134,7 +134,7 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let batch_size = state.active_suffix_length();
+ let batch_dim = state.active_row_count();
let input_array = state.array(self.input_array_id);
let output_array = state.array(self.output_array_id);
@@ -151,7 +151,7 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {
None => MatmulArgumentC::None,
},
d: output_array.buffer().borrow_mut().deref_mut(),
- batch_dim: batch_size as u32,
+ batch_dim: batch_dim as u32,
input_dim: self.input_dim as u32,
output_dim: self.output_dim as u32,
},
diff --git a/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs
index 3552c1564..37d7eecea 100644
--- a/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/uzu/src/encodable_block/linear/qlora_wrapper.rs
@@ -134,7 +134,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
self.base_linear.encode(state, encoder)?;
let mut adapter_kernel = self.adapter_kernel.borrow_mut();
- let batch_size = state.active_suffix_length();
+ let batch_dim = state.active_row_count();
let intermediate_array = state.common_aux.lora_intermediate.as_ref().unwrap();
let input_array = state.array(self.input_array_id);
@@ -149,7 +149,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
ab_scale: 1.0,
c: MatmulArgumentC::None,
d: intermediate_array.buffer().borrow_mut().deref_mut(),
- batch_dim: batch_size as u32,
+ batch_dim: batch_dim as u32,
input_dim: self.input_dim as u32,
output_dim: self.lora_rank as u32,
},
@@ -165,7 +165,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
ab_scale: self.lora_scale,
c: MatmulArgumentC::Accumulate,
d: output_array.buffer().borrow_mut().deref_mut(),
- batch_dim: batch_size as u32,
+ batch_dim: batch_dim as u32,
input_dim: self.lora_rank as u32,
output_dim: self.output_dim as u32,
},
diff --git a/crates/uzu/src/encodable_block/linear/quantized.rs b/crates/uzu/src/encodable_block/linear/quantized.rs
index 360617310..6450399ba 100644
--- a/crates/uzu/src/encodable_block/linear/quantized.rs
+++ b/crates/uzu/src/encodable_block/linear/quantized.rs
@@ -251,7 +251,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let batch_size = state.active_suffix_length();
+ let batch_dim = state.active_row_count();
let input_array = state.array(self.input_array_id);
let output_array = state.array(self.output_array_id);
@@ -268,7 +268,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {
scales_buffer: self.scales_buffer.borrow().deref(),
zero_points_or_biases_buffer: self.zero_points_or_biases_buffer.borrow().deref(),
output_buffer: output_buf_borrow.deref_mut(),
- batch: batch_size,
+ batch: batch_dim,
input_dim: self.input_dim,
output_dim: self.output_dim,
quantization_type: self.quantization_type,
@@ -277,7 +277,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {
.expect("Failed to encode quantized matmul");
if let (Some(bias_add_kernel), Some(biases_buffer)) = (&self.bias_add_kernel, &self.biases_buffer) {
- let total_length = batch_size * self.output_dim;
+ let total_length = batch_dim * self.output_dim;
bias_add_kernel.encode(
None::<&B::Buffer>,
biases_buffer.borrow().deref(),
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index f7a5862b1..183ba1f7a 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -359,18 +359,18 @@ impl<B: Backend> MambaMixer<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let active_suffix_length = state.active_suffix_length();
- if active_suffix_length == 0 {
+ let active_row_count = state.active_row_count();
+ if active_row_count == 0 {
return Ok(());
}
self.in_projection.encode(state, encoder)?;
- self.run_split_inproj(state, encoder, active_suffix_length);
- self.run_conv_scan(state, encoder, active_suffix_length);
- if active_suffix_length == 1 {
- self.run_decode_ssm(state, encoder, active_suffix_length);
+ self.run_split_inproj(state, encoder, active_row_count);
+ self.run_conv_scan(state, encoder, active_row_count);
+ if active_row_count == 1 {
+ self.run_decode_ssm(state, encoder, active_row_count);
} else {
- self.run_prefill_ssm(state, encoder, active_suffix_length);
+ self.run_prefill_ssm(state, encoder, active_row_count);
}
self.out_projection.encode(state, encoder)?;
Ok(())
diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rs
index 33e9fc6e7..1dbd6a2ae 100644
--- a/crates/uzu/src/encodable_block/mlp/moe.rs
+++ b/crates/uzu/src/encodable_block/mlp/moe.rs
@@ -198,7 +198,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let suffix_length = state.active_suffix_length();
+ let suffix_length = state.active_row_count();
let main_buf = state.array(ArrayId::Main).buffer();
let topk_ids_buf = state.array(ArrayId::MoeTopkIds).buffer();
diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rs
index fcf34f55f..fce310b06 100644
--- a/crates/uzu/src/encodable_block/pooling.rs
+++ b/crates/uzu/src/encodable_block/pooling.rs
@@ -24,12 +24,12 @@ impl<B: Backend> PoolingKernel<B> {
output: &mut B::Buffer,
seq_len: u32,
hidden_dim: u32,
- batch_size: u32,
+ batch_dim: u32,
encoder: &mut Encoder<B>,
) {
match self {
- Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),
- Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),
+ Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_dim, encoder),
+ Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_dim, encoder),
}
}
}
@@ -63,7 +63,7 @@ impl<B: Backend> Pooling<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let batch_size = 1;
+ let batch_dim = 1;
let seq_len = state.aux_buffers_suffix_length();
let main_array = state.array(ArrayId::Main);
@@ -74,7 +74,7 @@ impl<B: Backend> Pooling<B> {
pooling_array.buffer().borrow_mut().deref_mut(),
seq_len as u32,
self.model_dim as u32,
- batch_size,
+ batch_dim,
encoder,
);
@@ -83,6 +83,8 @@ impl<B: Backend> Pooling<B> {
let output_pooling_trace = state.traces().borrow().output_pooling().clone();
state.encode_copy_array(encoder, ArrayId::ClassifierPooling, output_pooling_trace);
}
+
+ state.set_active_row_count(pooling_array.shape()[0]);
Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rs
index bc1de7ffb..636658dde 100644
--- a/crates/uzu/src/encodable_block/qk_norm.rs
+++ b/crates/uzu/src/encodable_block/qk_norm.rs
@@ -127,7 +127,7 @@ impl<B: Backend> QKNorm<B> {
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let qkv_array = state.array(self.qkv_array_id);
- let batch_size = qkv_array.shape()[0] as u32;
+ let batch_dim = qkv_array.shape()[0] as u32;
// Process query normalization if configured
if let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =
@@ -137,7 +137,7 @@ impl<B: Backend> QKNorm<B> {
None::<&B::Buffer>,
query_scales_buffer.borrow().deref(),
qkv_array.buffer().borrow_mut().deref_mut(),
- batch_size,
+ batch_dim,
self.num_q_heads as u32,
self.num_kv_heads as u32,
self.head_dim as u32,
@@ -158,7 +158,7 @@ impl<B: Backend> QKNorm<B> {
None::<&B::Buffer>,
key_scales_buffer.borrow().deref(),
qkv_array.buffer().borrow_mut().deref_mut(),
- batch_size,
+ batch_dim,
self.num_q_heads as u32,
self.num_kv_heads as u32,
self.head_dim as u32,
diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rs
index e70ea0b28..64514c5fa 100644
--- a/crates/uzu/src/encodable_block/rms_norm.rs
+++ b/crates/uzu/src/encodable_block/rms_norm.rs
@@ -99,7 +99,7 @@ impl<B: Backend> RMSNorm<B> {
let (batch_start, batch_len) = if self.use_sampling_range {
(state.sampling_start(), state.sampling_length())
} else {
- (0, state.active_suffix_length())
+ (0, state.active_row_count())
};
let batch_len = batch_len.min(suffix_length.saturating_sub(batch_start));
diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rs
index 9f0efd154..26fdd9173 100644
--- a/crates/uzu/src/encodable_block/sampling.rs
+++ b/crates/uzu/src/encodable_block/sampling.rs
@@ -33,7 +33,7 @@ impl<B: Backend> Sampling<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let batch_size = state.sampling_length();
+ let batch_dim = state.sampling_length();
let sampling_start = state.sampling_start();
let sampling_method = state.sampling_method().unwrap();
@@ -60,7 +60,7 @@ impl<B: Backend> Sampling<B> {
bitmask_offset,
output.buffer().borrow_mut().deref_mut(),
sampling_method,
- batch_size,
+ batch_dim,
vocab_size,
encoder,
)
diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
index e9d0b7696..2623867f6 100644
--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -131,18 +131,18 @@ impl<B: Backend> ShortConvMixer<B> {
&self,
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
- active_suffix_length: usize,
+ active_row_count: usize,
) {
self.clear_suffix_state_valid_range(state);
- if active_suffix_length == 1 {
+ if active_row_count == 1 {
self.run_decode_conv(state, encoder, 1);
return;
}
let sampling_len = state.sampling_length();
if sampling_len == 0 {
- self.run_prefill_conv(state, encoder, active_suffix_length);
+ self.run_prefill_conv(state, encoder, active_row_count);
return;
}
@@ -150,7 +150,7 @@ impl<B: Backend> ShortConvMixer<B> {
let trie_len = sampling_len;
if trie_len <= 1 {
- self.run_prefill_conv(state, encoder, active_suffix_length);
+ self.run_prefill_conv(state, encoder, active_row_count);
return;
}
@@ -309,14 +309,14 @@ impl<B: Backend> ShortConvMixer<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let active_suffix_length = state.active_suffix_length();
- if active_suffix_length == 0 {
+ let active_row_count = state.active_row_count();
+ if active_row_count == 0 {
return Ok(());
}
self.in_projection.encode(state, encoder)?;
- self.run_conv(state, encoder, active_suffix_length);
+ self.run_conv(state, encoder, active_row_count);
self.out_projection.encode(state, encoder)?;
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index 8cd427da3..81442956d 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -35,7 +35,7 @@ pub struct LanguageModelGeneratorModeState<B: Backend> {
pub sampling_method: Option<SamplingMethod>,
#[cfg(feature = "tracing")]
pub traces: Rc<RefCell<ActivationTrace<B>>>,
- pub active_suffix_length: usize,
+ pub active_row_count: usize,
pub sampling_start: usize,
pub sampling_length: usize,
pub is_prefilling: bool,
@@ -46,6 +46,7 @@ pub struct ClassifierModeState<B: Backend> {
pub dense: Array<B>,
pub norm: Array<B>,
pub classifier_logits: Array<B>,
+ pub active_row_count: usize,
#[cfg(feature = "tracing")]
pub traces: Rc<RefCell<ActivationTrace<B>>>,
}
@@ -151,7 +152,7 @@ impl<B: Backend> ForwardPassState<B> {
token_positions: &[usize],
token_bitmask: Option<&[u32]>,
token_seeds: &[u64],
- active_suffix_length: usize,
+ active_row_count: usize,
sampling_start: usize,
sampling_length: usize,
is_prefilling: bool,
@@ -239,7 +240,7 @@ impl<B: Backend> ForwardPassState<B> {
sampling_method: None,
#[cfg(feature = "tracing")]
traces,
- active_suffix_length,
+ active_row_count,
sampling_start,
sampling_length,
is_prefilling,
@@ -312,23 +313,24 @@ impl<B: Backend> ForwardPassState<B> {
suffix_length: usize,
) -> ClassifierModeState<B> {
let data_type = model_shape.activation_data_type();
- let batch_size = 1;
+ let batch_dim = 1;
let create_buffer = |size: usize| -> Array<B> {
let buffer_size = size * data_type.size_in_bytes();
let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");
- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_size, size / batch_size], data_type) }
+ unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_dim, size / batch_dim], data_type) }
};
ClassifierModeState {
- pooling: create_buffer(batch_size * model_dim),
- dense: create_buffer(batch_size * model_dim),
- norm: create_buffer(batch_size * model_dim),
+ pooling: create_buffer(batch_dim * model_dim),
+ dense: create_buffer(batch_dim * model_dim),
+ norm: create_buffer(batch_dim * model_dim),
classifier_logits: {
- let buffer_size = batch_size * num_labels * data_type.size_in_bytes();
+ let buffer_size = batch_dim * num_labels * data_type.size_in_bytes();
let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");
- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_size, num_labels], data_type) }
+ unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_dim, num_labels], data_type) }
},
+ active_row_count: suffix_length,
traces: Rc::new(RefCell::new(ActivationTrace::new_classifier(
context,
model_shape,
@@ -344,10 +346,10 @@ impl<B: Backend> ForwardPassState<B> {
model_shape: &ModelShape,
model_dim: usize,
num_labels: usize,
- _suffix_length: usize,
+ suffix_length: usize,
) -> ClassifierModeState<B> {
let data_type = model_shape.activation_data_type();
- let batch_size = 1;
+ let batch_dim = 1;
let create_buffer = |dims: &[usize]| -> Array<B> {
let size: usize = dims.iter().product();
@@ -357,10 +359,11 @@ impl<B: Backend> ForwardPassState<B> {
};
ClassifierModeState {
- pooling: create_buffer(&[batch_size, model_dim]),
- dense: create_buffer(&[batch_size, model_dim]),
- norm: create_buffer(&[batch_size, model_dim]),
- classifier_logits: create_buffer(&[batch_size, num_labels]),
+ pooling: create_buffer(&[batch_dim, model_dim]),
+ dense: create_buffer(&[batch_dim, model_dim]),
+ norm: create_buffer(&[batch_dim, model_dim]),
+ classifier_logits: create_buffer(&[batch_dim, num_labels]),
+ active_row_count: suffix_length,
}
}
@@ -594,10 +597,20 @@ impl<B: Backend> ForwardPassState<B> {
// Public API Methods (formerly trait methods)
// ========================================================================
- pub fn active_suffix_length(&self) -> usize {
+ pub fn active_row_count(&self) -> usize {
match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => state.active_suffix_length,
- ForwardPassMode::Classifier(_) => self.common_aux.suffix_length,
+ ForwardPassMode::LanguageModelGenerator(state) => state.active_row_count,
+ ForwardPassMode::Classifier(state) => state.active_row_count,
+ }
+ }
+
+ pub fn set_active_row_count(
+ &mut self,
+ active_row_count: usize,
+ ) {
+ match &mut self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => state.active_row_count = active_row_count,
+ ForwardPassMode::Classifier(state) => state.active_row_count = active_row_count,
}
}
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index c0f01d1e0..ab5327902 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -45,7 +45,7 @@ struct Task<'a> {
token_bitmask: Option<&'a [u32]>,
token_seeds: &'a [u64],
expected_number_of_new_tokens: usize,
- active_suffix_length: usize,
+ active_row_count: usize,
sampling_start: usize,
sampling_length: usize,
is_prefilling: bool,
@@ -56,7 +56,7 @@ struct TaskEncodingKey {
context_len: usize,
batch_size: usize,
expected_number_of_new_tokens: usize,
- active_suffix_len: usize,
+ active_row_count: usize,
sampling_method: SamplingMethod,
sampling_start: usize,
sampling_len: usize,
@@ -225,7 +225,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let step_token_positions = step_token_positions.collect::<Box<[usize]>>();
let step_token_seeds = step_token_seeds.collect::<Box<[u64]>>();
- let active_suffix_length = step_token_positions.len();
+ let active_row_count = step_token_positions.len();
let is_last_prefill_step = step == prefill_steps - 1;
let should_sample_after_step = sample_suffix && is_last_prefill_step;
@@ -234,7 +234,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
// suffix-root token, which is the last prompt token).
let (sampling_start, sampling_length) = if should_sample_after_step {
let suffix_root_index_in_step = (tokens_length - 1).saturating_sub(tokens_start_index);
- let sampling_length = active_suffix_length.saturating_sub(suffix_root_index_in_step);
+ let sampling_length = active_row_count.saturating_sub(suffix_root_index_in_step);
debug_assert!(sampling_length > 0, "Expected at least one token to sample on the last prefill step");
(suffix_root_index_in_step, sampling_length)
} else {
@@ -277,7 +277,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
token_bitmask: step_token_bitmask.as_deref(),
token_seeds: &step_token_seeds,
expected_number_of_new_tokens: step_token_ids.len(),
- active_suffix_length,
+ active_row_count,
sampling_start,
sampling_length,
is_prefilling: !should_sample_after_step,
@@ -358,21 +358,21 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
);
let flat_trie = suffix_root.linearize();
- let active_suffix_length = flat_trie.len();
+ let active_row_count = flat_trie.len();
let token_ids =
- flat_trie.token_ids().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();
+ flat_trie.token_ids().chain(repeat_n(0, suffix_length - active_row_count)).collect::<Box<[u64]>>();
let token_subtrie_ranges = flat_trie
.token_subtrie_ranges()
- .chain(repeat_n([u32::MAX, u32::MAX, u32::MAX], suffix_length - active_suffix_length))
+ .chain(repeat_n([u32::MAX, u32::MAX, u32::MAX], suffix_length - active_row_count))
.collect::<Box<[[u32; 3]]>>();
let token_bitmask: Option<Box<[u32]>> = compiled_grammar.is_some().then(|| {
let single_token_bitmask_size = self.context.model_shape.bitmask_shape(1)[1];
flat_trie
.token_masks()
- .chain(repeat_n(None, suffix_length - active_suffix_length))
+ .chain(repeat_n(None, suffix_length - active_row_count))
.map(|mask| match mask {
Some(mask) => Either::Left(
mask.iter()
@@ -390,11 +390,11 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let token_positions = flat_trie
.token_positions()
.map(|trie_position| start_position + trie_position)
- .chain(repeat_n(INVALID_POSITION, suffix_length - active_suffix_length))
+ .chain(repeat_n(INVALID_POSITION, suffix_length - active_row_count))
.collect::<Box<[usize]>>();
let token_seeds =
- flat_trie.token_seeds().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();
+ flat_trie.token_seeds().chain(repeat_n(0, suffix_length - active_row_count)).collect::<Box<[u64]>>();
let task = Task {
token_ids: &token_ids,
@@ -403,9 +403,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
token_bitmask: token_bitmask.as_deref(),
token_seeds: &token_seeds,
expected_number_of_new_tokens: 1,
- active_suffix_length,
+ active_row_count,
sampling_start: 0,
- sampling_length: active_suffix_length,
+ sampling_length: active_row_count,
is_prefilling: false,
};
@@ -415,7 +415,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let (accepted_tokens, accepted_token_indices) =
flat_trie.accept(&sampled_tokens, compiled_grammar.as_deref_mut());
- let speculator_proposed = active_suffix_length.saturating_sub(1);
+ let speculator_proposed = active_row_count.saturating_sub(1);
let speculator_accepted = accepted_tokens.len().saturating_sub(1);
self.update_cache_layers(&accepted_token_indices, None, false)?;
@@ -482,7 +482,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
token_bitmask: None,
token_seeds: &[0], // Ignored, using async buffer
expected_number_of_new_tokens: 1,
- active_suffix_length: 1,
+ active_row_count: 1,
sampling_start: 0,
sampling_length: 1,
is_prefilling: false,
@@ -511,9 +511,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
&task.token_positions,
task.token_bitmask,
&task.token_seeds,
- task.active_suffix_length,
+ task.active_row_count,
/*sampling_start=*/ 0,
- /*sampling_length=*/ task.active_suffix_length,
+ /*sampling_length=*/ task.active_row_count,
task.is_prefilling,
skip_token_ids_copy,
async_positions,
@@ -720,7 +720,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
task.token_positions,
task.token_bitmask,
task.token_seeds,
- task.active_suffix_length,
+ task.active_row_count,
task.sampling_start,
task.sampling_length,
task.is_prefilling,
@@ -737,7 +737,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
context_len: self.tokens.len(),
batch_size: task.token_ids.len(),
expected_number_of_new_tokens: task.expected_number_of_new_tokens,
- active_suffix_len: task.active_suffix_length,
+ active_row_count: task.active_row_count,
sampling_method,
sampling_start: task.sampling_start,
sampling_len: task.sampling_length,
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index 58d334a5c..a0616de20 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -581,21 +581,6 @@ impl<B: Backend> TraceValidator<B> {
) -> Vec<TracerValidationResult> {
let mut results = Vec::new();
- // Embedding norm (classifier-specific)
- if let Some(embedding_norm) = &traces.borrow().embedding_norm {
- if traces_view.leaf_array("activation_trace.embedding_norm").is_ok() {
- results.push(TracerValidationResult {
- name: "activation_trace.embedding_norm".to_string(),
- metrics: Self::validate_array_with_name(
- data_type,
- traces_view,
- "activation_trace.embedding_norm",
- embedding_norm,
- ),
- });
- }
- }
-
// Output pooling (classifier-specific)
if let Some(output_pooling) = &traces.borrow().output_pooling {
if traces_view.leaf_array("activation_trace.output_pooling").is_ok() {