π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Move ML golden generation to developer tooling
laurenspriem33efa5253d72
diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.tomlindex 7f34340fad..2803463a07 100644--- a/rust/crates/photos/Cargo.toml+++ b/rust/crates/photos/Cargo.toml@@ -74,8 +74,3 @@ url = "2.5.8"name = "ml_indexing"path = "tests/ml_indexing.rs"required-features = ["ml-assets"]--[[test]]-name = "ml_goldens"-path = "tests/ml_goldens.rs"-required-features = ["ml-assets"]diff --git a/rust/crates/photos/examples/ml_goldens.rs b/rust/crates/photos/examples/ml_goldens.rsnew file mode 100644index 0000000000..fa561d6bf1--- /dev/null+++ b/rust/crates/photos/examples/ml_goldens.rs@@ -0,0 +1,127 @@+//! Developer tooling for the committed ML self-test data.+//!+//! cargo run -p ente-photos --example ml_goldens -- generate+//! cargo run -p ente-photos --example ml_goldens -- validate-zero-separation++#[allow(dead_code)]+#[path = "../tests/support/mod.rs"]+mod support;++use anyhow::{Context, Result, bail};+use ente_photos::ml::golden_tooling::{+ GoldenMetric, GoldenModelSpec, GoldenSpecInput, measure_zero_golden_separation,+ render_golden_data,+};+use support::ml_indexing::{GoldenTestAssets, run_with_large_stack};++const CLIP_TEXT_GOLDEN_PHRASE: &str = "a photo of a dog playing on a sunny beach";++fn main() -> Result<()> {+ let mut args = std::env::args().skip(1);+ let command = args.next();+ if args.next().is_some() {+ bail!(usage());+ }++ match command.as_deref() {+ Some("generate") => run_with_large_stack("generate_goldens", generate),+ Some("validate-zero-separation") => {+ run_with_large_stack("validate_zero_separation", validate_zero_separation)+ }+ _ => bail!(usage()),+ }+ Ok(())+}++fn usage() -> &'static str {+ "usage: cargo run -p ente-photos --example ml_goldens -- \+ <generate|validate-zero-separation>"+}++fn production_model_specs(assets: &GoldenTestAssets) -> Vec<GoldenModelSpec> {+ vec![+ GoldenModelSpec {+ model_path: assets.face_detection.path.to_string_lossy().into_owned(),+ sha256: assets.face_detection.sha256.clone(),+ input: GoldenSpecInput::SeededNoise,+ metric: GoldenMetric::DetectorRelativeL2 {+ row_len: 16,+ confidence_offset: 4,+ },+ },+ GoldenModelSpec {+ model_path: assets.face_embedding.path.to_string_lossy().into_owned(),+ sha256: assets.face_embedding.sha256.clone(),+ input: GoldenSpecInput::SeededNoise,+ metric: GoldenMetric::CosineDistance,+ },+ GoldenModelSpec {+ model_path: assets.clip_image.path.to_string_lossy().into_owned(),+ sha256: assets.clip_image.sha256.clone(),+ input: GoldenSpecInput::SeededNoise,+ metric: GoldenMetric::CosineDistance,+ },+ GoldenModelSpec {+ model_path: assets.clip_text.path.to_string_lossy().into_owned(),+ sha256: assets.clip_text.sha256.clone(),+ input: GoldenSpecInput::ClipText {+ phrase: CLIP_TEXT_GOLDEN_PHRASE.to_string(),+ vocab_path: assets.clip_text_vocab.to_string_lossy().into_owned(),+ },+ metric: GoldenMetric::CosineDistance,+ },+ ]+}++fn generate() -> Result<()> {+ let assets = GoldenTestAssets::load()?;+ let source = render_golden_data(&production_model_specs(&assets))+ .map_err(|error| anyhow::anyhow!("generating goldens: {error}"))?;+ let path = GoldenTestAssets::golden_data_path()?;+ std::fs::write(&path, source).with_context(|| format!("write {}", path.display()))?;++ let rustfmt = std::process::Command::new("rustfmt")+ .args(["--edition", "2024"])+ .arg(&path)+ .status()+ .with_context(|| format!("run rustfmt on {}", path.display()))?;+ if !rustfmt.success() {+ bail!("rustfmt failed on {}", path.display());+ }+ println!("wrote {}", path.display());+ Ok(())+}++fn validate_zero_separation() -> Result<()> {+ let assets = GoldenTestAssets::load()?;+ for model_path in [+ &assets.face_detection.path,+ &assets.face_embedding.path,+ &assets.clip_image.path,+ &assets.clip_text.path,+ ] {+ let separation = measure_zero_golden_separation(&model_path.to_string_lossy())+ .map_err(|error| anyhow::anyhow!(error))?;+ println!(+ "{}: metric={}, golden_distance={:.9}, zero_distance={:.9}, threshold={:.9}, \+ threshold_margin={:.1}x",+ separation.model_file,+ separation.metric_label,+ separation.golden_distance,+ separation.zero_distance,+ separation.threshold,+ separation.zero_distance / separation.threshold,+ );+ if separation.zero_distance <= separation.threshold {+ bail!(+ "{}: zero-input output is not separated from the golden ({} {:.9} <= threshold \+ {:.9})",+ separation.model_file,+ separation.metric_label,+ separation.zero_distance,+ separation.threshold,+ );+ }+ }+ Ok(())+}diff --git a/rust/crates/photos/src/ml/golden.rs b/rust/crates/photos/src/ml/golden.rsindex 992c39fa8e..ad1845e80e 100644--- a/rust/crates/photos/src/ml/golden.rs+++ b/rust/crates/photos/src/ml/golden.rs@@ -7,7 +7,7 @@//! session (WebGPU on Android, CoreML on iOS) is trusted, it must reproduce a//! committed golden output for a fixed input within a cosine-distance//! threshold. The golden outputs are generated on the CPU execution provider-//! by `cargo test -p ente-photos --features ml-assets --test ml_goldens`+//! by `cargo run -p ente-photos --example ml_goldens -- generate`//! and committed in `golden_data.rs`. A cheap unit test in `golden_tooling`//! keeps the committed entries pinned (by file name and content hash) to the//! production models in `infra/ml/test/ml_indexing/assets.json`, so a modeldiff --git a/rust/crates/photos/src/ml/golden_data.rs b/rust/crates/photos/src/ml/golden_data.rsindex f0082f3cfd..4f85f1501f 100644--- a/rust/crates/photos/src/ml/golden_data.rs+++ b/rust/crates/photos/src/ml/golden_data.rs@@ -1,7 +1,7 @@//! GENERATED FILE β do not edit by hand.//!//! Regenerate with:-//! cargo test -p ente-photos --features ml-assets --test ml_goldens -- --ignored generate_goldens+//! cargo run -p ente-photos --example ml_goldens -- generate//!//! TODO: the pet models are CPU-only and therefore have no golden entries//! yet; they must get entries here before they are moved off CPU-onlydiff --git a/rust/crates/photos/src/ml/golden_tooling.rs b/rust/crates/photos/src/ml/golden_tooling.rsindex 81e75a0692..ec38a4fb84 100644--- a/rust/crates/photos/src/ml/golden_tooling.rs+++ b/rust/crates/photos/src/ml/golden_tooling.rs@@ -1,8 +1,8 @@//! Generation and verification of the committed golden self-test data.//!//! Regeneration (`render_golden_data`) runs the production models on the CPU-//! execution provider and is invoked manually via the `generate_goldens`-//! test in the `ml_goldens` test binary (requires the `ml-assets` feature).+//! execution provider and is invoked manually via the `ml_goldens` developer+//! tool's `generate` command.//! Verification (`verify_goldens_against_pins`) is metadata-only and runs as//! a plain unit test in every CI pass.@@ -24,7 +24,7 @@ const GENERATED_FILE_HEADER: &str = "\//! GENERATED FILE β do not edit by hand.//!//! Regenerate with:-//! cargo test -p ente-photos --features ml-assets --test ml_goldens -- --ignored generate_goldens+//! cargo run -p ente-photos --example ml_goldens -- generate//!//! TODO: the pet models are CPU-only and therefore have no golden entries//! yet; they must get entries here before they are moved off CPU-only@@ -288,8 +288,8 @@ mod tests {/// The CI guard against stale goldens: a model update (new file name) or/// a content change under an unchanged name in the asset lock fails this- /// test until `golden_data.rs` is regenerated (see the `generate_goldens`- /// test in the ml_goldens test binary).+ /// test until `golden_data.rs` is regenerated with the `ml_goldens`+ /// developer tool.#[test]fn committed_goldens_are_pinned_to_the_production_models() {let pins = pinned_production_models();diff --git a/rust/crates/photos/tests/ml_goldens.rs b/rust/crates/photos/tests/ml_goldens.rsdeleted file mode 100644index 1838f36e01..0000000000--- a/rust/crates/photos/tests/ml_goldens.rs+++ /dev/null@@ -1,132 +0,0 @@-//! Manual (re)generation of the committed golden self-test data in-//! `src/ml/golden_data.rs`.-//!-//! `generate_goldens` is `#[ignore]`d; run it after a model update (or an-//! asset-lock change) to regenerate the file:-//! cargo test -p ente-photos --features ml-assets --test ml_goldens -- --ignored generate_goldens-//!-//! CI does not run inference to validate the committed goldens; instead the-//! cheap `committed_goldens_are_pinned_to_the_production_models` unit test in-//! `golden_tooling` checks that every pinned production model in the asset-//! lock has a committed entry generated from the same content hash.--// The support module is shared with the ml_indexing test binary; parts unused-// by this binary are fine.-#[allow(dead_code)]-mod support;--use anyhow::{Context, bail};-use ente_photos::ml::golden_tooling::{- GoldenMetric, GoldenModelSpec, GoldenSpecInput, measure_zero_golden_separation,- render_golden_data,-};-use support::ml_indexing::{GoldenTestAssets, run_with_large_stack};--/// Fixed phrase for the CLIP text golden; tokenized with the production-/// tokenizer at generation time and committed as token ids.-const CLIP_TEXT_GOLDEN_PHRASE: &str = "a photo of a dog playing on a sunny beach";--fn production_model_specs(assets: &GoldenTestAssets) -> Vec<GoldenModelSpec> {- // TODO: add the pet models here (and thereby to golden_data.rs) before- // they are moved off CPU-only execution.- vec![- GoldenModelSpec {- model_path: assets.face_detection.path.to_string_lossy().into_owned(),- sha256: assets.face_detection.sha256.clone(),- input: GoldenSpecInput::SeededNoise,- // Validate confidence independently so large box and landmark- // coordinates cannot hide score corruption.- metric: GoldenMetric::DetectorRelativeL2 {- row_len: 16,- confidence_offset: 4,- },- },- GoldenModelSpec {- model_path: assets.face_embedding.path.to_string_lossy().into_owned(),- sha256: assets.face_embedding.sha256.clone(),- input: GoldenSpecInput::SeededNoise,- metric: GoldenMetric::CosineDistance,- },- GoldenModelSpec {- model_path: assets.clip_image.path.to_string_lossy().into_owned(),- sha256: assets.clip_image.sha256.clone(),- input: GoldenSpecInput::SeededNoise,- metric: GoldenMetric::CosineDistance,- },- GoldenModelSpec {- model_path: assets.clip_text.path.to_string_lossy().into_owned(),- sha256: assets.clip_text.sha256.clone(),- input: GoldenSpecInput::ClipText {- phrase: CLIP_TEXT_GOLDEN_PHRASE.to_string(),- vocab_path: assets.clip_text_vocab.to_string_lossy().into_owned(),- },- metric: GoldenMetric::CosineDistance,- },- ]-}--#[test]-#[ignore = "regenerates src/ml/golden_data.rs from the production models"]-fn generate_goldens() {- run_with_large_stack("generate_goldens", || {- let assets = GoldenTestAssets::load()?;- let specs = production_model_specs(&assets);- let source = render_golden_data(&specs)- .map_err(|error| anyhow::anyhow!("generating goldens: {error}"))?;- let path = GoldenTestAssets::golden_data_path()?;- std::fs::write(&path, source).with_context(|| format!("write {}", path.display()))?;- // Normalize to the repository's rustfmt style so the generated file- // passes format checks without a manual step.- let rustfmt = std::process::Command::new("rustfmt")- .args(["--edition", "2024"])- .arg(&path)- .status()- .with_context(|| format!("run rustfmt on {}", path.display()))?;- if !rustfmt.success() {- bail!("rustfmt failed on {}", path.display());- }- println!("wrote {}", path.display());- Ok(())- });-}--#[test]-#[ignore = "downloads and runs the production models"]-fn zero_inputs_are_separated_from_committed_goldens() {- run_with_large_stack("zero_golden_separation", || {- let assets = GoldenTestAssets::load()?;- let model_paths = [- &assets.face_detection.path,- &assets.face_embedding.path,- &assets.clip_image.path,- &assets.clip_text.path,- ];-- for model_path in model_paths {- let separation = measure_zero_golden_separation(&model_path.to_string_lossy())- .map_err(|error| anyhow::anyhow!(error))?;- let threshold_margin = separation.zero_distance / separation.threshold;- println!(- "{}: metric={}, golden_distance={:.9}, zero_distance={:.9}, threshold={:.9}, \- threshold_margin={:.1}x",- separation.model_file,- separation.metric_label,- separation.golden_distance,- separation.zero_distance,- separation.threshold,- threshold_margin,- );- if separation.zero_distance <= separation.threshold {- bail!(- "{}: zero-input output is not separated from the golden ({} {:.9} <= \- threshold {:.9})",- separation.model_file,- separation.metric_label,- separation.zero_distance,- separation.threshold,- );- }- }- Ok(())- });-}diff --git a/rust/crates/photos/tests/support/ml_indexing.rs b/rust/crates/photos/tests/support/ml_indexing.rsindex df2d7f5953..f9a2d30e62 100644--- a/rust/crates/photos/tests/support/ml_indexing.rs+++ b/rust/crates/photos/tests/support/ml_indexing.rs@@ -389,7 +389,7 @@ pub(crate) struct GoldenModelAsset {}/// Downloaded production model assets for the golden self-test tooling, with-/// ONNX Runtime already initialized. Used by the ml_goldens test binary only.+/// ONNX Runtime already initialized. Used by the ml_goldens developer tool only.#[allow(dead_code)]pub(crate) struct GoldenTestAssets {pub(crate) face_detection: GoldenModelAsset,@@ -492,7 +492,7 @@ struct ModelAssets {face_detection: ModelAsset,face_embedding: ModelAsset,clip_image: ModelAsset,- // Used by the ml_goldens test binary only.+ // Used by the ml_goldens developer tool only.#[allow(dead_code)]clip_text: ModelAsset,#[allow(dead_code)]