π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
First do a zero test, and then a goldens test
laurenspriemd69c8572cba6
diff --git a/rust/crates/photos/src/ml/golden.rs b/rust/crates/photos/src/ml/golden.rsindex 0bde28957d..f32b9fe5d1 100644--- a/rust/crates/photos/src/ml/golden.rs+++ b/rust/crates/photos/src/ml/golden.rs@@ -72,6 +72,14 @@ impl GoldenMetric {Self::DetectorRelativeL2 { .. } => "detector channel relative L2 error",}}++ /// Maximum accepted distance from the committed CPU golden.+ pub fn threshold(self) -> f64 {+ match self {+ Self::CosineDistance => COSINE_DISTANCE_THRESHOLD,+ Self::DetectorRelativeL2 { .. } => RELATIVE_L2_THRESHOLD,+ }+ }}/// A committed golden record for one model file.@@ -145,6 +153,16 @@ pub enum PreparedGoldenInput {I32(Vec<i32>),}+impl PreparedGoldenInput {+ /// A zero-filled input with the same element type and length.+ pub fn zeroed(&self) -> Self {+ match self {+ Self::F32(data) => Self::F32(vec![0.0; data.len()]),+ Self::I32(data) => Self::I32(vec![0; data.len()]),+ }+ }+}+/// Materializes the entry's fixed input tensor data.pub fn prepare_input(entry: &GoldenEntry) -> Result<PreparedGoldenInput, String> {let element_count: i64 = entry.input_shape.iter().product();@@ -199,6 +217,49 @@ fn sample_stride(output_len: usize) -> usize {/// entry's metric. Returns the measured distance, or a description of why the/// output is rejected.pub fn compare_output(entry: &GoldenEntry, output: &[f32]) -> Result<f64, String> {+ let distances = measure_output(entry, output)?;+ match distances {+ OutputDistances::Cosine(distance) => ensure_within_threshold(+ "output cosine distance",+ distance,+ COSINE_DISTANCE_THRESHOLD,+ ),+ OutputDistances::Detector {+ confidence,+ remaining,+ } => {+ ensure_within_threshold(+ "detector confidence relative L2 error",+ confidence,+ RELATIVE_L2_THRESHOLD,+ )?;+ ensure_within_threshold(+ "detector remaining-output relative L2 error",+ remaining,+ RELATIVE_L2_THRESHOLD,+ )?;+ Ok(confidence.max(remaining))+ }+ }+}++/// Measures an output's distance from the committed golden without applying+/// the acceptance threshold. Useful for checking that a warm-up input's+/// output is well separated from the golden output.+pub fn output_distance(entry: &GoldenEntry, output: &[f32]) -> Result<f64, String> {+ Ok(match measure_output(entry, output)? {+ OutputDistances::Cosine(distance) => distance,+ OutputDistances::Detector {+ confidence,+ remaining,+ } => confidence.max(remaining),+ })+}++/// Validates the properties required of the zero-input warm-up output. The+/// warm-up has no numeric golden, but it must still produce a complete finite+/// tensor before the session can proceed to the golden comparison.+pub fn validate_output(entry: &GoldenEntry, output: &[f32]) -> Result<(), String> {if output.len() != entry.output_len {return Err(format!("output length {} does not match golden length {}",@@ -207,6 +268,20 @@ pub fn compare_output(entry: &GoldenEntry, output: &[f32]) -> Result<f64, String));}+ if let Some(bad) = output.iter().find(|value| !value.is_finite()) {+ return Err(format!("output contains non-finite values ({bad})"));+ }+ Ok(())+}++enum OutputDistances {+ Cosine(f64),+ Detector { confidence: f64, remaining: f64 },+}++fn measure_output(entry: &GoldenEntry, output: &[f32]) -> Result<OutputDistances, String> {+ validate_output(entry, output)?;+let sample = sample_output(output);if sample.len() != entry.expected_sample.len() {return Err(format!(@@ -215,16 +290,12 @@ pub fn compare_output(entry: &GoldenEntry, output: &[f32]) -> Result<f64, Stringentry.expected_sample.len()));}- if let Some(bad) = sample.iter().find(|value| !value.is_finite()) {- return Err(format!("output contains non-finite values ({bad})"));- }- let distance = match entry.metric {- GoldenMetric::CosineDistance => ensure_within_threshold(- "output cosine distance",- cosine_distance(&sample, entry.expected_sample)?,- COSINE_DISTANCE_THRESHOLD,- )?,+ match entry.metric {+ GoldenMetric::CosineDistance => Ok(OutputDistances::Cosine(cosine_distance(+ &sample,+ entry.expected_sample,+ )?)),GoldenMetric::DetectorRelativeL2 {row_len,confidence_offset,@@ -236,20 +307,12 @@ pub fn compare_output(entry: &GoldenEntry, output: &[f32]) -> Result<f64, Stringrow_len,confidence_offset,)?;- ensure_within_threshold(- "detector confidence relative L2 error",- confidence_distance,- RELATIVE_L2_THRESHOLD,- )?;- ensure_within_threshold(- "detector remaining-output relative L2 error",- remaining_distance,- RELATIVE_L2_THRESHOLD,- )?;- confidence_distance.max(remaining_distance)+ Ok(OutputDistances::Detector {+ confidence: confidence_distance,+ remaining: remaining_distance,+ })}- };- Ok(distance)+ }}fn ensure_within_threshold(label: &str, distance: f64, threshold: f64) -> Result<f64, String> {@@ -349,7 +412,7 @@ fn relative_l2_error(actual_sample: &[f32], expected_sample: &[f32]) -> Result<fmod tests {use super::{COSINE_DISTANCE_THRESHOLD, GOLDEN_ENTRIES, GoldenEntry, GoldenInput, GoldenMetric,- compare_output, matches_model_file, seeded_noise,+ PreparedGoldenInput, compare_output, matches_model_file, seeded_noise, validate_output,};fn entry_with_metric(@@ -428,6 +491,21 @@ mod tests {assert!(first.iter().any(|value| *value < 0.1));}+ #[test]+ fn zeroed_input_preserves_type_and_length() {+ let f32_input = PreparedGoldenInput::F32(vec![1.0, 2.0, 3.0]);+ let i32_input = PreparedGoldenInput::I32(vec![1, 2]);++ let PreparedGoldenInput::F32(f32_zeroes) = f32_input.zeroed() else {+ panic!("f32 input changed type");+ };+ let PreparedGoldenInput::I32(i32_zeroes) = i32_input.zeroed() else {+ panic!("i32 input changed type");+ };+ assert_eq!(f32_zeroes, vec![0.0; 3]);+ assert_eq!(i32_zeroes, vec![0; 2]);+ }+#[test]fn accepts_output_close_to_golden() {static EXPECTED: [f32; 4] = [0.5, -1.0, 2.0, 0.25];@@ -466,6 +544,19 @@ mod tests {assert!(error.contains("non-finite"), "{error}");}+ #[test]+ fn warm_up_validation_checks_the_full_output_not_only_the_golden_sample() {+ static EXPECTED: [f32; 1] = [1.0];+ let entry = entry_for(&EXPECTED, 10_000);+ let mut output = vec![1.0; 10_000];+ // The committed golden would sample every third value for this output+ // length, so this index demonstrates that validation is full-tensor.+ output[9_998] = f32::INFINITY;++ let error = validate_output(&entry, &output).unwrap_err();+ assert!(error.contains("non-finite"), "{error}");+ }+#[test]fn rejects_zero_output() {static EXPECTED: [f32; 4] = [0.5, -1.0, 2.0, 0.25];diff --git a/rust/crates/photos/src/ml/golden_tooling.rs b/rust/crates/photos/src/ml/golden_tooling.rsindex 4e1f7e48f2..81e75a0692 100644--- a/rust/crates/photos/src/ml/golden_tooling.rs+++ b/rust/crates/photos/src/ml/golden_tooling.rs@@ -55,6 +55,16 @@ pub enum GoldenSpecInput {ClipText { phrase: String, vocab_path: String },}+/// Measured separation between a zero-input warm-up and the committed golden+/// when both are run consecutively on one CPU session.+pub struct ZeroGoldenSeparation {+ pub model_file: String,+ pub metric_label: &'static str,+ pub golden_distance: f64,+ pub zero_distance: f64,+ pub threshold: f64,+}+/// Renders the complete contents of `golden_data.rs` for the given models.pub fn render_golden_data(models: &[GoldenModelSpec]) -> Result<String, String> {let mut source = String::from(GENERATED_FILE_HEADER);@@ -65,6 +75,38 @@ pub fn render_golden_data(models: &[GoldenModelSpec]) -> Result<String, String>Ok(source)}+/// Runs the production self-test sequence on CPU and measures whether a+/// stale zero-input result would be rejected by the committed golden.+pub fn measure_zero_golden_separation(model_path: &str) -> Result<ZeroGoldenSeparation, String> {+ let model_file = model_file_name(model_path)?;+ let entry = golden::lookup(model_path)+ .ok_or_else(|| format!("{model_file}: no committed golden entry"))?;+ let mut session = build_cpu_session(model_path)?;+ let golden_input = golden::prepare_input(entry)+ .map_err(|error| format!("{model_file}: preparing golden input: {error}"))?;+ let zero_input = golden_input.zeroed();++ let zero_output = onnx::run_golden_tensor(&mut session, entry.input_shape, &zero_input)+ .map_err(|error| format!("{model_file}: zero-input CPU inference failed: {error}"))?;+ golden::validate_output(entry, &zero_output)+ .map_err(|error| format!("{model_file}: invalid zero-input output: {error}"))?;++ let golden_output = onnx::run_golden_tensor(&mut session, entry.input_shape, &golden_input)+ .map_err(|error| format!("{model_file}: golden CPU inference failed: {error}"))?;+ let golden_distance = golden::compare_output(entry, &golden_output)+ .map_err(|error| format!("{model_file}: golden output rejected: {error}"))?;+ let zero_distance = golden::output_distance(entry, &zero_output)+ .map_err(|error| format!("{model_file}: measuring zero-input output: {error}"))?;++ Ok(ZeroGoldenSeparation {+ model_file: model_file.to_string(),+ metric_label: entry.metric.label(),+ golden_distance,+ zero_distance,+ threshold: entry.metric.threshold(),+ })+}+/// A production model as pinned in the asset lock/// (`infra/ml/test/ml_indexing/assets.json`).pub struct PinnedModel {diff --git a/rust/crates/photos/src/ml/onnx.rs b/rust/crates/photos/src/ml/onnx.rsindex 6500a3dd64..e3afcc3dbe 100644--- a/rust/crates/photos/src/ml/onnx.rs+++ b/rust/crates/photos/src/ml/onnx.rs@@ -325,15 +325,6 @@ fn provider_attempt_failure_message())}-/// Number of golden inferences a freshly built accelerated session must pass.-/// Two, because known driver defects (for example on newer Adreno GPUs)-/// produce a correct first inference and corrupt output from the second-/// dispatch onwards. The runs double as session warm-up: pipeline creation-/// and first dispatch happen here (on Android inside the crash-canary-/// window), not during the first real inference.-#[cfg(any(target_os = "android", target_os = "ios"))]-const GOLDEN_SELF_TEST_RUNS: usize = 2;-/// Validates a freshly built accelerated session against the model's/// committed golden output. An error means the session must not be used; the/// caller falls through to the next execution provider attempt.@@ -351,51 +342,83 @@ fn run_session_self_test("golden self-test entry missing for '{model_file}'")));};- let input = golden::prepare_input(entry).map_err(|reason| {+ let golden_input = golden::prepare_input(entry).map_err(|reason| {MlError::Ort(format!("golden self-test input invalid for '{model_file}': {reason}"))})?;+ let zero_input = golden_input.zeroed();- for run_index in 1..=GOLDEN_SELF_TEST_RUNS {- let output = match run_golden_tensor(session, entry.input_shape, &input) {- Ok(output) => output,- Err(error) => {- // Surface inference failures too: without this, a provider- // that cannot even execute the golden input would fall back- // invisibly (the session-build error is swallowed once the- // next attempt succeeds).- events::record(- events::Severity::Warning,- format!(- "{provider_label} golden self-test inference failed for '{model_file}' \- on run {run_index}: {error}; falling back to the next execution provider"- ),- );- return Err(error);- }- };- match golden::compare_output(entry, &output) {- Ok(distance) => rt_log(&format!(- "{provider_label} golden self-test run {run_index} for '{model_file}' passed \+ // Warm up pipeline creation and the first dispatch with an input that is+ // deliberately different from the golden. If a later dispatch reuses+ // this result, the golden comparison below rejects the session.+ let zero_output = match run_golden_tensor(session, entry.input_shape, &zero_input) {+ Ok(output) => output,+ Err(error) => {+ events::record(+ events::Severity::Warning,+ format!(+ "{provider_label} zero-input warm-up inference failed for '{model_file}': \+ {error}; falling back to the next execution provider"+ ),+ );+ return Err(error);+ }+ };+ if let Err(reason) = golden::validate_output(entry, &zero_output) {+ events::record(+ events::Severity::Severe,+ format!(+ "{provider_label} zero-input warm-up failed for '{model_file}': {reason}; \+ falling back to the next execution provider"+ ),+ );+ return Err(MlError::Ort(format!(+ "zero-input warm-up failed for '{model_file}': {reason}"+ )));+ }+ rt_log(&format!(+ "{provider_label} zero-input warm-up for '{model_file}' passed"+ ));++ let golden_output = match run_golden_tensor(session, entry.input_shape, &golden_input) {+ Ok(output) => output,+ Err(error) => {+ // Surface inference failures too: without this, a provider that+ // cannot execute the golden input would fall back invisibly (the+ // session-build error is swallowed once the next attempt succeeds).+ events::record(+ events::Severity::Warning,+ format!(+ "{provider_label} golden self-test inference failed for '{model_file}': \+ {error}; falling back to the next execution provider"+ ),+ );+ return Err(error);+ }+ };+ match golden::compare_output(entry, &golden_output) {+ Ok(distance) => {+ rt_log(&format!(+ "{provider_label} golden self-test for '{model_file}' passed \({} {distance:.2e})",entry.metric.label()- )),- Err(reason) => {- events::record(- events::Severity::Severe,- format!(- "{provider_label} golden self-test failed for '{model_file}' on run \- {run_index}: {reason}; falling back to the next execution provider"- ),- );- return Err(MlError::Ort(format!(- "golden self-test failed for '{model_file}' on run {run_index}: {reason}"- )));- }+ ));+ Ok(())+ }+ Err(reason) => {+ events::record(+ events::Severity::Severe,+ format!(+ "{provider_label} golden self-test failed for '{model_file}': {reason}; \+ falling back to the next execution provider"+ ),+ );+ Err(MlError::Ort(format!(+ "golden self-test failed for '{model_file}': {reason}"+ )))}}- Ok(())}#[cfg(any(target_os = "android", target_os = "ios", test))]diff --git a/rust/crates/photos/tests/ml_goldens.rs b/rust/crates/photos/tests/ml_goldens.rsindex 359300351a..1838f36e01 100644--- a/rust/crates/photos/tests/ml_goldens.rs+++ b/rust/crates/photos/tests/ml_goldens.rs@@ -17,7 +17,8 @@ mod support;use anyhow::{Context, bail};use ente_photos::ml::golden_tooling::{- GoldenMetric, GoldenModelSpec, GoldenSpecInput, render_golden_data,+ GoldenMetric, GoldenModelSpec, GoldenSpecInput, measure_zero_golden_separation,+ render_golden_data,};use support::ml_indexing::{GoldenTestAssets, run_with_large_stack};@@ -88,3 +89,44 @@ fn generate_goldens() {Ok(())});}++#[test]+#[ignore = "downloads and runs the production models"]+fn zero_inputs_are_separated_from_committed_goldens() {+ run_with_large_stack("zero_golden_separation", || {+ let assets = GoldenTestAssets::load()?;+ let model_paths = [+ &assets.face_detection.path,+ &assets.face_embedding.path,+ &assets.clip_image.path,+ &assets.clip_text.path,+ ];++ for model_path in model_paths {+ let separation = measure_zero_golden_separation(&model_path.to_string_lossy())+ .map_err(|error| anyhow::anyhow!(error))?;+ let threshold_margin = separation.zero_distance / separation.threshold;+ println!(+ "{}: metric={}, golden_distance={:.9}, zero_distance={:.9}, threshold={:.9}, \+ threshold_margin={:.1}x",+ separation.model_file,+ separation.metric_label,+ separation.golden_distance,+ separation.zero_distance,+ separation.threshold,+ threshold_margin,+ );+ if separation.zero_distance <= separation.threshold {+ bail!(+ "{}: zero-input output is not separated from the golden ({} {:.9} <= \+ threshold {:.9})",+ separation.model_file,+ separation.metric_label,+ separation.zero_distance,+ separation.threshold,+ );+ }+ }+ Ok(())+ });+}