Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

First do a zero test, and then a goldens test

laurenspriemd69c8572cba6
diff --git a/rust/crates/photos/src/ml/golden.rs b/rust/crates/photos/src/ml/golden.rs
index 0bde28957d..f32b9fe5d1 100644
--- a/rust/crates/photos/src/ml/golden.rs
+++ b/rust/crates/photos/src/ml/golden.rs
@@ -72,6 +72,14 @@ impl GoldenMetric {
Self::DetectorRelativeL2 { .. } => "detector channel relative L2 error",
}
}
+
+ /// Maximum accepted distance from the committed CPU golden.
+ pub fn threshold(self) -> f64 {
+ match self {
+ Self::CosineDistance => COSINE_DISTANCE_THRESHOLD,
+ Self::DetectorRelativeL2 { .. } => RELATIVE_L2_THRESHOLD,
+ }
+ }
}
/// A committed golden record for one model file.
@@ -145,6 +153,16 @@ pub enum PreparedGoldenInput {
I32(Vec<i32>),
}
+impl PreparedGoldenInput {
+ /// A zero-filled input with the same element type and length.
+ pub fn zeroed(&self) -> Self {
+ match self {
+ Self::F32(data) => Self::F32(vec![0.0; data.len()]),
+ Self::I32(data) => Self::I32(vec![0; data.len()]),
+ }
+ }
+}
+
/// Materializes the entry's fixed input tensor data.
pub fn prepare_input(entry: &GoldenEntry) -> Result<PreparedGoldenInput, String> {
let element_count: i64 = entry.input_shape.iter().product();
@@ -199,6 +217,49 @@ fn sample_stride(output_len: usize) -> usize {
/// entry's metric. Returns the measured distance, or a description of why the
/// output is rejected.
pub fn compare_output(entry: &GoldenEntry, output: &[f32]) -> Result<f64, String> {
+ let distances = measure_output(entry, output)?;
+ match distances {
+ OutputDistances::Cosine(distance) => ensure_within_threshold(
+ "output cosine distance",
+ distance,
+ COSINE_DISTANCE_THRESHOLD,
+ ),
+ OutputDistances::Detector {
+ confidence,
+ remaining,
+ } => {
+ ensure_within_threshold(
+ "detector confidence relative L2 error",
+ confidence,
+ RELATIVE_L2_THRESHOLD,
+ )?;
+ ensure_within_threshold(
+ "detector remaining-output relative L2 error",
+ remaining,
+ RELATIVE_L2_THRESHOLD,
+ )?;
+ Ok(confidence.max(remaining))
+ }
+ }
+}
+
+/// Measures an output's distance from the committed golden without applying
+/// the acceptance threshold. Useful for checking that a warm-up input's
+/// output is well separated from the golden output.
+pub fn output_distance(entry: &GoldenEntry, output: &[f32]) -> Result<f64, String> {
+ Ok(match measure_output(entry, output)? {
+ OutputDistances::Cosine(distance) => distance,
+ OutputDistances::Detector {
+ confidence,
+ remaining,
+ } => confidence.max(remaining),
+ })
+}
+
+/// Validates the properties required of the zero-input warm-up output. The
+/// warm-up has no numeric golden, but it must still produce a complete finite
+/// tensor before the session can proceed to the golden comparison.
+pub fn validate_output(entry: &GoldenEntry, output: &[f32]) -> Result<(), String> {
if output.len() != entry.output_len {
return Err(format!(
"output length {} does not match golden length {}",
@@ -207,6 +268,20 @@ pub fn compare_output(entry: &GoldenEntry, output: &[f32]) -> Result<f64, String
));
}
+ if let Some(bad) = output.iter().find(|value| !value.is_finite()) {
+ return Err(format!("output contains non-finite values ({bad})"));
+ }
+ Ok(())
+}
+
+enum OutputDistances {
+ Cosine(f64),
+ Detector { confidence: f64, remaining: f64 },
+}
+
+fn measure_output(entry: &GoldenEntry, output: &[f32]) -> Result<OutputDistances, String> {
+ validate_output(entry, output)?;
+
let sample = sample_output(output);
if sample.len() != entry.expected_sample.len() {
return Err(format!(
@@ -215,16 +290,12 @@ pub fn compare_output(entry: &GoldenEntry, output: &[f32]) -> Result<f64, String
entry.expected_sample.len()
));
}
- if let Some(bad) = sample.iter().find(|value| !value.is_finite()) {
- return Err(format!("output contains non-finite values ({bad})"));
- }
- let distance = match entry.metric {
- GoldenMetric::CosineDistance => ensure_within_threshold(
- "output cosine distance",
- cosine_distance(&sample, entry.expected_sample)?,
- COSINE_DISTANCE_THRESHOLD,
- )?,
+ match entry.metric {
+ GoldenMetric::CosineDistance => Ok(OutputDistances::Cosine(cosine_distance(
+ &sample,
+ entry.expected_sample,
+ )?)),
GoldenMetric::DetectorRelativeL2 {
row_len,
confidence_offset,
@@ -236,20 +307,12 @@ pub fn compare_output(entry: &GoldenEntry, output: &[f32]) -> Result<f64, String
row_len,
confidence_offset,
)?;
- ensure_within_threshold(
- "detector confidence relative L2 error",
- confidence_distance,
- RELATIVE_L2_THRESHOLD,
- )?;
- ensure_within_threshold(
- "detector remaining-output relative L2 error",
- remaining_distance,
- RELATIVE_L2_THRESHOLD,
- )?;
- confidence_distance.max(remaining_distance)
+ Ok(OutputDistances::Detector {
+ confidence: confidence_distance,
+ remaining: remaining_distance,
+ })
}
- };
- Ok(distance)
+ }
}
fn ensure_within_threshold(label: &str, distance: f64, threshold: f64) -> Result<f64, String> {
@@ -349,7 +412,7 @@ fn relative_l2_error(actual_sample: &[f32], expected_sample: &[f32]) -> Result<f
mod tests {
use super::{
COSINE_DISTANCE_THRESHOLD, GOLDEN_ENTRIES, GoldenEntry, GoldenInput, GoldenMetric,
- compare_output, matches_model_file, seeded_noise,
+ PreparedGoldenInput, compare_output, matches_model_file, seeded_noise, validate_output,
};
fn entry_with_metric(
@@ -428,6 +491,21 @@ mod tests {
assert!(first.iter().any(|value| *value < 0.1));
}
+ #[test]
+ fn zeroed_input_preserves_type_and_length() {
+ let f32_input = PreparedGoldenInput::F32(vec![1.0, 2.0, 3.0]);
+ let i32_input = PreparedGoldenInput::I32(vec![1, 2]);
+
+ let PreparedGoldenInput::F32(f32_zeroes) = f32_input.zeroed() else {
+ panic!("f32 input changed type");
+ };
+ let PreparedGoldenInput::I32(i32_zeroes) = i32_input.zeroed() else {
+ panic!("i32 input changed type");
+ };
+ assert_eq!(f32_zeroes, vec![0.0; 3]);
+ assert_eq!(i32_zeroes, vec![0; 2]);
+ }
+
#[test]
fn accepts_output_close_to_golden() {
static EXPECTED: [f32; 4] = [0.5, -1.0, 2.0, 0.25];
@@ -466,6 +544,19 @@ mod tests {
assert!(error.contains("non-finite"), "{error}");
}
+ #[test]
+ fn warm_up_validation_checks_the_full_output_not_only_the_golden_sample() {
+ static EXPECTED: [f32; 1] = [1.0];
+ let entry = entry_for(&EXPECTED, 10_000);
+ let mut output = vec![1.0; 10_000];
+ // The committed golden would sample every third value for this output
+ // length, so this index demonstrates that validation is full-tensor.
+ output[9_998] = f32::INFINITY;
+
+ let error = validate_output(&entry, &output).unwrap_err();
+ assert!(error.contains("non-finite"), "{error}");
+ }
+
#[test]
fn rejects_zero_output() {
static EXPECTED: [f32; 4] = [0.5, -1.0, 2.0, 0.25];
diff --git a/rust/crates/photos/src/ml/golden_tooling.rs b/rust/crates/photos/src/ml/golden_tooling.rs
index 4e1f7e48f2..81e75a0692 100644
--- a/rust/crates/photos/src/ml/golden_tooling.rs
+++ b/rust/crates/photos/src/ml/golden_tooling.rs
@@ -55,6 +55,16 @@ pub enum GoldenSpecInput {
ClipText { phrase: String, vocab_path: String },
}
+/// Measured separation between a zero-input warm-up and the committed golden
+/// when both are run consecutively on one CPU session.
+pub struct ZeroGoldenSeparation {
+ pub model_file: String,
+ pub metric_label: &'static str,
+ pub golden_distance: f64,
+ pub zero_distance: f64,
+ pub threshold: f64,
+}
+
/// Renders the complete contents of `golden_data.rs` for the given models.
pub fn render_golden_data(models: &[GoldenModelSpec]) -> Result<String, String> {
let mut source = String::from(GENERATED_FILE_HEADER);
@@ -65,6 +75,38 @@ pub fn render_golden_data(models: &[GoldenModelSpec]) -> Result<String, String>
Ok(source)
}
+/// Runs the production self-test sequence on CPU and measures whether a
+/// stale zero-input result would be rejected by the committed golden.
+pub fn measure_zero_golden_separation(model_path: &str) -> Result<ZeroGoldenSeparation, String> {
+ let model_file = model_file_name(model_path)?;
+ let entry = golden::lookup(model_path)
+ .ok_or_else(|| format!("{model_file}: no committed golden entry"))?;
+ let mut session = build_cpu_session(model_path)?;
+ let golden_input = golden::prepare_input(entry)
+ .map_err(|error| format!("{model_file}: preparing golden input: {error}"))?;
+ let zero_input = golden_input.zeroed();
+
+ let zero_output = onnx::run_golden_tensor(&mut session, entry.input_shape, &zero_input)
+ .map_err(|error| format!("{model_file}: zero-input CPU inference failed: {error}"))?;
+ golden::validate_output(entry, &zero_output)
+ .map_err(|error| format!("{model_file}: invalid zero-input output: {error}"))?;
+
+ let golden_output = onnx::run_golden_tensor(&mut session, entry.input_shape, &golden_input)
+ .map_err(|error| format!("{model_file}: golden CPU inference failed: {error}"))?;
+ let golden_distance = golden::compare_output(entry, &golden_output)
+ .map_err(|error| format!("{model_file}: golden output rejected: {error}"))?;
+ let zero_distance = golden::output_distance(entry, &zero_output)
+ .map_err(|error| format!("{model_file}: measuring zero-input output: {error}"))?;
+
+ Ok(ZeroGoldenSeparation {
+ model_file: model_file.to_string(),
+ metric_label: entry.metric.label(),
+ golden_distance,
+ zero_distance,
+ threshold: entry.metric.threshold(),
+ })
+}
+
/// A production model as pinned in the asset lock
/// (`infra/ml/test/ml_indexing/assets.json`).
pub struct PinnedModel {
diff --git a/rust/crates/photos/src/ml/onnx.rs b/rust/crates/photos/src/ml/onnx.rs
index 6500a3dd64..e3afcc3dbe 100644
--- a/rust/crates/photos/src/ml/onnx.rs
+++ b/rust/crates/photos/src/ml/onnx.rs
@@ -325,15 +325,6 @@ fn provider_attempt_failure_message(
))
}
-/// Number of golden inferences a freshly built accelerated session must pass.
-/// Two, because known driver defects (for example on newer Adreno GPUs)
-/// produce a correct first inference and corrupt output from the second
-/// dispatch onwards. The runs double as session warm-up: pipeline creation
-/// and first dispatch happen here (on Android inside the crash-canary
-/// window), not during the first real inference.
-#[cfg(any(target_os = "android", target_os = "ios"))]
-const GOLDEN_SELF_TEST_RUNS: usize = 2;
-
/// Validates a freshly built accelerated session against the model's
/// committed golden output. An error means the session must not be used; the
/// caller falls through to the next execution provider attempt.
@@ -351,51 +342,83 @@ fn run_session_self_test(
"golden self-test entry missing for '{model_file}'"
)));
};
- let input = golden::prepare_input(entry).map_err(|reason| {
+ let golden_input = golden::prepare_input(entry).map_err(|reason| {
MlError::Ort(format!(
"golden self-test input invalid for '{model_file}': {reason}"
))
})?;
+ let zero_input = golden_input.zeroed();
- for run_index in 1..=GOLDEN_SELF_TEST_RUNS {
- let output = match run_golden_tensor(session, entry.input_shape, &input) {
- Ok(output) => output,
- Err(error) => {
- // Surface inference failures too: without this, a provider
- // that cannot even execute the golden input would fall back
- // invisibly (the session-build error is swallowed once the
- // next attempt succeeds).
- events::record(
- events::Severity::Warning,
- format!(
- "{provider_label} golden self-test inference failed for '{model_file}' \
- on run {run_index}: {error}; falling back to the next execution provider"
- ),
- );
- return Err(error);
- }
- };
- match golden::compare_output(entry, &output) {
- Ok(distance) => rt_log(&format!(
- "{provider_label} golden self-test run {run_index} for '{model_file}' passed \
+ // Warm up pipeline creation and the first dispatch with an input that is
+ // deliberately different from the golden. If a later dispatch reuses
+ // this result, the golden comparison below rejects the session.
+ let zero_output = match run_golden_tensor(session, entry.input_shape, &zero_input) {
+ Ok(output) => output,
+ Err(error) => {
+ events::record(
+ events::Severity::Warning,
+ format!(
+ "{provider_label} zero-input warm-up inference failed for '{model_file}': \
+ {error}; falling back to the next execution provider"
+ ),
+ );
+ return Err(error);
+ }
+ };
+ if let Err(reason) = golden::validate_output(entry, &zero_output) {
+ events::record(
+ events::Severity::Severe,
+ format!(
+ "{provider_label} zero-input warm-up failed for '{model_file}': {reason}; \
+ falling back to the next execution provider"
+ ),
+ );
+ return Err(MlError::Ort(format!(
+ "zero-input warm-up failed for '{model_file}': {reason}"
+ )));
+ }
+ rt_log(&format!(
+ "{provider_label} zero-input warm-up for '{model_file}' passed"
+ ));
+
+ let golden_output = match run_golden_tensor(session, entry.input_shape, &golden_input) {
+ Ok(output) => output,
+ Err(error) => {
+ // Surface inference failures too: without this, a provider that
+ // cannot execute the golden input would fall back invisibly (the
+ // session-build error is swallowed once the next attempt succeeds).
+ events::record(
+ events::Severity::Warning,
+ format!(
+ "{provider_label} golden self-test inference failed for '{model_file}': \
+ {error}; falling back to the next execution provider"
+ ),
+ );
+ return Err(error);
+ }
+ };
+ match golden::compare_output(entry, &golden_output) {
+ Ok(distance) => {
+ rt_log(&format!(
+ "{provider_label} golden self-test for '{model_file}' passed \
({} {distance:.2e})",
entry.metric.label()
- )),
- Err(reason) => {
- events::record(
- events::Severity::Severe,
- format!(
- "{provider_label} golden self-test failed for '{model_file}' on run \
- {run_index}: {reason}; falling back to the next execution provider"
- ),
- );
- return Err(MlError::Ort(format!(
- "golden self-test failed for '{model_file}' on run {run_index}: {reason}"
- )));
- }
+ ));
+ Ok(())
+ }
+ Err(reason) => {
+ events::record(
+ events::Severity::Severe,
+ format!(
+ "{provider_label} golden self-test failed for '{model_file}': {reason}; \
+ falling back to the next execution provider"
+ ),
+ );
+ Err(MlError::Ort(format!(
+ "golden self-test failed for '{model_file}': {reason}"
+ )))
}
}
- Ok(())
}
#[cfg(any(target_os = "android", target_os = "ios", test))]
diff --git a/rust/crates/photos/tests/ml_goldens.rs b/rust/crates/photos/tests/ml_goldens.rs
index 359300351a..1838f36e01 100644
--- a/rust/crates/photos/tests/ml_goldens.rs
+++ b/rust/crates/photos/tests/ml_goldens.rs
@@ -17,7 +17,8 @@ mod support;
use anyhow::{Context, bail};
use ente_photos::ml::golden_tooling::{
- GoldenMetric, GoldenModelSpec, GoldenSpecInput, render_golden_data,
+ GoldenMetric, GoldenModelSpec, GoldenSpecInput, measure_zero_golden_separation,
+ render_golden_data,
};
use support::ml_indexing::{GoldenTestAssets, run_with_large_stack};
@@ -88,3 +89,44 @@ fn generate_goldens() {
Ok(())
});
}
+
+#[test]
+#[ignore = "downloads and runs the production models"]
+fn zero_inputs_are_separated_from_committed_goldens() {
+ run_with_large_stack("zero_golden_separation", || {
+ let assets = GoldenTestAssets::load()?;
+ let model_paths = [
+ &assets.face_detection.path,
+ &assets.face_embedding.path,
+ &assets.clip_image.path,
+ &assets.clip_text.path,
+ ];
+
+ for model_path in model_paths {
+ let separation = measure_zero_golden_separation(&model_path.to_string_lossy())
+ .map_err(|error| anyhow::anyhow!(error))?;
+ let threshold_margin = separation.zero_distance / separation.threshold;
+ println!(
+ "{}: metric={}, golden_distance={:.9}, zero_distance={:.9}, threshold={:.9}, \
+ threshold_margin={:.1}x",
+ separation.model_file,
+ separation.metric_label,
+ separation.golden_distance,
+ separation.zero_distance,
+ separation.threshold,
+ threshold_margin,
+ );
+ if separation.zero_distance <= separation.threshold {
+ bail!(
+ "{}: zero-input output is not separated from the golden ({} {:.9} <= \
+ threshold {:.9})",
+ separation.model_file,
+ separation.metric_label,
+ separation.zero_distance,
+ separation.threshold,
+ );
+ }
+ }
+ Ok(())
+ });
+}