A high-performance inference engine for AI models
Tracer test refactoring (#325)
Aleksandr Golokoz28986c228bfb
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex a0616de20..2b06ba8cf 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -702,7 +702,20 @@ impl<B: Backend> TraceValidator<B> {let result: Vec<f32> = produced_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();- Self::compare_arrays(&reference, expected_data.shape().to_vec(), &result, produced_data.shape().to_vec())+ let (atol, rtol, allowed_voilations_tol) = match expected_array.data_type() {+ DataType::BF16 => (0.04, 0.06, 0.03),+ _ => (0.01, 0.03, 0.01),+ };++ Self::compare_arrays(+ &reference,+ expected_data.shape().to_vec(),+ &result,+ produced_data.shape().to_vec(),+ atol,+ rtol,+ allowed_voilations_tol,+ )}fn compare_arrays(@@ -710,13 +723,12 @@ impl<B: Backend> TraceValidator<B> {reference_shape: Vec<usize>,result: &[f32],result_shape: Vec<usize>,+ atol: f32,+ rtol: f32,+ fraction_of_allowed_violations: f32,) -> TracerValidationMetrics {assert_eq!(result.len(), reference.len());- let atol: f32 = 1e-2;- let rtol: f32 = 0.03;- let fraction_of_allowed_violations: f32 = 0.01;-let mut num_violations = 0;let mut max_err = 0.0f32;let mut max_err_idx = 0;diff --git a/crates/uzu/tests/integration/tracer/trace_validator_test.rs b/crates/uzu/tests/integration/tracer/trace_validator_test.rsindex 477e46063..844c6ae96 100644--- a/crates/uzu/tests/integration/tracer/trace_validator_test.rs+++ b/crates/uzu/tests/integration/tracer/trace_validator_test.rs@@ -1,66 +1,47 @@-#![cfg(all(metal_backend, feature = "tracing"))]+#![cfg(all(feature = "tracing"))]-use uzu::{TraceValidator, backends::metal::Metal};+use uzu::{TraceValidator, backends::common::Backend};-use crate::common::path::{get_test_model_path, get_traces_path};+use crate::common::{+ for_each_non_cpu_backend,+ path::{get_test_model_path, get_traces_path},+};++fn test_tracer_internal<B: Backend>() {+ let model_path = get_test_model_path();+ let mut tracer = TraceValidator::<B>::new(&model_path).expect("Failed to create TraceValidator");+ let results = tracer.run().expect("Failed to run tracer");+ for result in results.results.iter() {+ // this layers contains too many errors+ if result.name == "activation_trace.output_norm" || result.name == "logits" {+ continue;+ }+ assert!(result.metrics.is_valid(), "{} error:\n{}", result.name, result.metrics.message().as_str());+ }++ let total_token_violations = results.number_of_tokens_violations();+ let allowed_token_violations = results.number_of_allowed_tokens_violations();+ assert!(+ total_token_violations < allowed_token_violations,+ "Too much token violations: {} / {}. Indices: {:?}",+ total_token_violations,+ allowed_token_violations,+ results.tokens_violation_indices+ );+}#[test]fn test_tracer() {- let model_path = get_test_model_path();- // Ensure traces file present, otherwise skiplet traces_path = get_traces_path();++ // TODO: when api will return `traces.safetensor` remove `if` and uncomment `assert`if !traces_path.exists() {println!("Skipping tracer test: traces file missing at {:?}", traces_path);return;}+ // assert!(traces_path.exists(), "Traces file missing at {:?}", traces_path);- let mut tracer = match TraceValidator::<Metal>::new(&model_path) {- Ok(t) => t,- Err(e) => {- println!("Failed to create TraceValidator: {:?}", e);- return;- },- };-- let colored_text = |text: &str, valid: bool| {- if valid {- format!("\x1b[32m{}\x1b[0m", text)- } else {- format!("\x1b[31m{}\x1b[0m", text)- }- };-- let results = match tracer.run() {- Ok(r) => r,- Err(e) => {- println!("Tracer run failed: {:?}", e);- return;- },- };-- for result in results.results.iter() {- let valid = result.metrics.is_valid();- let text = colored_text(- if valid {- "ok"- } else {- "error"- },- valid,- );- println!("{}: {}", result.name, text);- if !valid {- println!("{}", colored_text(result.metrics.message().as_str(), false));- }- }- println!("-------------------------");- println!(- "number_of_tokens_violations: {}",- colored_text(- format!("{} / {}", results.number_of_tokens_violations(), results.number_of_allowed_tokens_violations())- .as_str(),- results.is_valid(),- )- );- println!("tokens_violation_indices: {:?}", results.tokens_violation_indices);+ for_each_non_cpu_backend!(|B| {+ test_tracer_internal::<B>();+ })}