Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Move ML golden generation to developer tooling

laurenspriem33efa5253d72
diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.toml
index 7f34340fad..2803463a07 100644
--- a/rust/crates/photos/Cargo.toml
+++ b/rust/crates/photos/Cargo.toml
@@ -74,8 +74,3 @@ url = "2.5.8"
name = "ml_indexing"
path = "tests/ml_indexing.rs"
required-features = ["ml-assets"]
-
-[[test]]
-name = "ml_goldens"
-path = "tests/ml_goldens.rs"
-required-features = ["ml-assets"]
diff --git a/rust/crates/photos/examples/ml_goldens.rs b/rust/crates/photos/examples/ml_goldens.rs
new file mode 100644
index 0000000000..fa561d6bf1
--- /dev/null
+++ b/rust/crates/photos/examples/ml_goldens.rs
@@ -0,0 +1,127 @@
+//! Developer tooling for the committed ML self-test data.
+//!
+//! cargo run -p ente-photos --example ml_goldens -- generate
+//! cargo run -p ente-photos --example ml_goldens -- validate-zero-separation
+
+#[allow(dead_code)]
+#[path = "../tests/support/mod.rs"]
+mod support;
+
+use anyhow::{Context, Result, bail};
+use ente_photos::ml::golden_tooling::{
+ GoldenMetric, GoldenModelSpec, GoldenSpecInput, measure_zero_golden_separation,
+ render_golden_data,
+};
+use support::ml_indexing::{GoldenTestAssets, run_with_large_stack};
+
+const CLIP_TEXT_GOLDEN_PHRASE: &str = "a photo of a dog playing on a sunny beach";
+
+fn main() -> Result<()> {
+ let mut args = std::env::args().skip(1);
+ let command = args.next();
+ if args.next().is_some() {
+ bail!(usage());
+ }
+
+ match command.as_deref() {
+ Some("generate") => run_with_large_stack("generate_goldens", generate),
+ Some("validate-zero-separation") => {
+ run_with_large_stack("validate_zero_separation", validate_zero_separation)
+ }
+ _ => bail!(usage()),
+ }
+ Ok(())
+}
+
+fn usage() -> &'static str {
+ "usage: cargo run -p ente-photos --example ml_goldens -- \
+ <generate|validate-zero-separation>"
+}
+
+fn production_model_specs(assets: &GoldenTestAssets) -> Vec<GoldenModelSpec> {
+ vec![
+ GoldenModelSpec {
+ model_path: assets.face_detection.path.to_string_lossy().into_owned(),
+ sha256: assets.face_detection.sha256.clone(),
+ input: GoldenSpecInput::SeededNoise,
+ metric: GoldenMetric::DetectorRelativeL2 {
+ row_len: 16,
+ confidence_offset: 4,
+ },
+ },
+ GoldenModelSpec {
+ model_path: assets.face_embedding.path.to_string_lossy().into_owned(),
+ sha256: assets.face_embedding.sha256.clone(),
+ input: GoldenSpecInput::SeededNoise,
+ metric: GoldenMetric::CosineDistance,
+ },
+ GoldenModelSpec {
+ model_path: assets.clip_image.path.to_string_lossy().into_owned(),
+ sha256: assets.clip_image.sha256.clone(),
+ input: GoldenSpecInput::SeededNoise,
+ metric: GoldenMetric::CosineDistance,
+ },
+ GoldenModelSpec {
+ model_path: assets.clip_text.path.to_string_lossy().into_owned(),
+ sha256: assets.clip_text.sha256.clone(),
+ input: GoldenSpecInput::ClipText {
+ phrase: CLIP_TEXT_GOLDEN_PHRASE.to_string(),
+ vocab_path: assets.clip_text_vocab.to_string_lossy().into_owned(),
+ },
+ metric: GoldenMetric::CosineDistance,
+ },
+ ]
+}
+
+fn generate() -> Result<()> {
+ let assets = GoldenTestAssets::load()?;
+ let source = render_golden_data(&production_model_specs(&assets))
+ .map_err(|error| anyhow::anyhow!("generating goldens: {error}"))?;
+ let path = GoldenTestAssets::golden_data_path()?;
+ std::fs::write(&path, source).with_context(|| format!("write {}", path.display()))?;
+
+ let rustfmt = std::process::Command::new("rustfmt")
+ .args(["--edition", "2024"])
+ .arg(&path)
+ .status()
+ .with_context(|| format!("run rustfmt on {}", path.display()))?;
+ if !rustfmt.success() {
+ bail!("rustfmt failed on {}", path.display());
+ }
+ println!("wrote {}", path.display());
+ Ok(())
+}
+
+fn validate_zero_separation() -> Result<()> {
+ let assets = GoldenTestAssets::load()?;
+ for model_path in [
+ &assets.face_detection.path,
+ &assets.face_embedding.path,
+ &assets.clip_image.path,
+ &assets.clip_text.path,
+ ] {
+ let separation = measure_zero_golden_separation(&model_path.to_string_lossy())
+ .map_err(|error| anyhow::anyhow!(error))?;
+ println!(
+ "{}: metric={}, golden_distance={:.9}, zero_distance={:.9}, threshold={:.9}, \
+ threshold_margin={:.1}x",
+ separation.model_file,
+ separation.metric_label,
+ separation.golden_distance,
+ separation.zero_distance,
+ separation.threshold,
+ separation.zero_distance / separation.threshold,
+ );
+ if separation.zero_distance <= separation.threshold {
+ bail!(
+ "{}: zero-input output is not separated from the golden ({} {:.9} <= threshold \
+ {:.9})",
+ separation.model_file,
+ separation.metric_label,
+ separation.zero_distance,
+ separation.threshold,
+ );
+ }
+ }
+ Ok(())
+}
diff --git a/rust/crates/photos/src/ml/golden.rs b/rust/crates/photos/src/ml/golden.rs
index 992c39fa8e..ad1845e80e 100644
--- a/rust/crates/photos/src/ml/golden.rs
+++ b/rust/crates/photos/src/ml/golden.rs
@@ -7,7 +7,7 @@
//! session (WebGPU on Android, CoreML on iOS) is trusted, it must reproduce a
//! committed golden output for a fixed input within a cosine-distance
//! threshold. The golden outputs are generated on the CPU execution provider
-//! by `cargo test -p ente-photos --features ml-assets --test ml_goldens`
+//! by `cargo run -p ente-photos --example ml_goldens -- generate`
//! and committed in `golden_data.rs`. A cheap unit test in `golden_tooling`
//! keeps the committed entries pinned (by file name and content hash) to the
//! production models in `infra/ml/test/ml_indexing/assets.json`, so a model
diff --git a/rust/crates/photos/src/ml/golden_data.rs b/rust/crates/photos/src/ml/golden_data.rs
index f0082f3cfd..4f85f1501f 100644
--- a/rust/crates/photos/src/ml/golden_data.rs
+++ b/rust/crates/photos/src/ml/golden_data.rs
@@ -1,7 +1,7 @@
//! GENERATED FILE β€” do not edit by hand.
//!
//! Regenerate with:
-//! cargo test -p ente-photos --features ml-assets --test ml_goldens -- --ignored generate_goldens
+//! cargo run -p ente-photos --example ml_goldens -- generate
//!
//! TODO: the pet models are CPU-only and therefore have no golden entries
//! yet; they must get entries here before they are moved off CPU-only
diff --git a/rust/crates/photos/src/ml/golden_tooling.rs b/rust/crates/photos/src/ml/golden_tooling.rs
index 81e75a0692..ec38a4fb84 100644
--- a/rust/crates/photos/src/ml/golden_tooling.rs
+++ b/rust/crates/photos/src/ml/golden_tooling.rs
@@ -1,8 +1,8 @@
//! Generation and verification of the committed golden self-test data.
//!
//! Regeneration (`render_golden_data`) runs the production models on the CPU
-//! execution provider and is invoked manually via the `generate_goldens`
-//! test in the `ml_goldens` test binary (requires the `ml-assets` feature).
+//! execution provider and is invoked manually via the `ml_goldens` developer
+//! tool's `generate` command.
//! Verification (`verify_goldens_against_pins`) is metadata-only and runs as
//! a plain unit test in every CI pass.
@@ -24,7 +24,7 @@ const GENERATED_FILE_HEADER: &str = "\
//! GENERATED FILE β€” do not edit by hand.
//!
//! Regenerate with:
-//! cargo test -p ente-photos --features ml-assets --test ml_goldens -- --ignored generate_goldens
+//! cargo run -p ente-photos --example ml_goldens -- generate
//!
//! TODO: the pet models are CPU-only and therefore have no golden entries
//! yet; they must get entries here before they are moved off CPU-only
@@ -288,8 +288,8 @@ mod tests {
/// The CI guard against stale goldens: a model update (new file name) or
/// a content change under an unchanged name in the asset lock fails this
- /// test until `golden_data.rs` is regenerated (see the `generate_goldens`
- /// test in the ml_goldens test binary).
+ /// test until `golden_data.rs` is regenerated with the `ml_goldens`
+ /// developer tool.
#[test]
fn committed_goldens_are_pinned_to_the_production_models() {
let pins = pinned_production_models();
diff --git a/rust/crates/photos/tests/ml_goldens.rs b/rust/crates/photos/tests/ml_goldens.rs
deleted file mode 100644
index 1838f36e01..0000000000
--- a/rust/crates/photos/tests/ml_goldens.rs
+++ /dev/null
@@ -1,132 +0,0 @@
-//! Manual (re)generation of the committed golden self-test data in
-//! `src/ml/golden_data.rs`.
-//!
-//! `generate_goldens` is `#[ignore]`d; run it after a model update (or an
-//! asset-lock change) to regenerate the file:
-//! cargo test -p ente-photos --features ml-assets --test ml_goldens -- --ignored generate_goldens
-//!
-//! CI does not run inference to validate the committed goldens; instead the
-//! cheap `committed_goldens_are_pinned_to_the_production_models` unit test in
-//! `golden_tooling` checks that every pinned production model in the asset
-//! lock has a committed entry generated from the same content hash.
-
-// The support module is shared with the ml_indexing test binary; parts unused
-// by this binary are fine.
-#[allow(dead_code)]
-mod support;
-
-use anyhow::{Context, bail};
-use ente_photos::ml::golden_tooling::{
- GoldenMetric, GoldenModelSpec, GoldenSpecInput, measure_zero_golden_separation,
- render_golden_data,
-};
-use support::ml_indexing::{GoldenTestAssets, run_with_large_stack};
-
-/// Fixed phrase for the CLIP text golden; tokenized with the production
-/// tokenizer at generation time and committed as token ids.
-const CLIP_TEXT_GOLDEN_PHRASE: &str = "a photo of a dog playing on a sunny beach";
-
-fn production_model_specs(assets: &GoldenTestAssets) -> Vec<GoldenModelSpec> {
- // TODO: add the pet models here (and thereby to golden_data.rs) before
- // they are moved off CPU-only execution.
- vec![
- GoldenModelSpec {
- model_path: assets.face_detection.path.to_string_lossy().into_owned(),
- sha256: assets.face_detection.sha256.clone(),
- input: GoldenSpecInput::SeededNoise,
- // Validate confidence independently so large box and landmark
- // coordinates cannot hide score corruption.
- metric: GoldenMetric::DetectorRelativeL2 {
- row_len: 16,
- confidence_offset: 4,
- },
- },
- GoldenModelSpec {
- model_path: assets.face_embedding.path.to_string_lossy().into_owned(),
- sha256: assets.face_embedding.sha256.clone(),
- input: GoldenSpecInput::SeededNoise,
- metric: GoldenMetric::CosineDistance,
- },
- GoldenModelSpec {
- model_path: assets.clip_image.path.to_string_lossy().into_owned(),
- sha256: assets.clip_image.sha256.clone(),
- input: GoldenSpecInput::SeededNoise,
- metric: GoldenMetric::CosineDistance,
- },
- GoldenModelSpec {
- model_path: assets.clip_text.path.to_string_lossy().into_owned(),
- sha256: assets.clip_text.sha256.clone(),
- input: GoldenSpecInput::ClipText {
- phrase: CLIP_TEXT_GOLDEN_PHRASE.to_string(),
- vocab_path: assets.clip_text_vocab.to_string_lossy().into_owned(),
- },
- metric: GoldenMetric::CosineDistance,
- },
- ]
-}
-
-#[test]
-#[ignore = "regenerates src/ml/golden_data.rs from the production models"]
-fn generate_goldens() {
- run_with_large_stack("generate_goldens", || {
- let assets = GoldenTestAssets::load()?;
- let specs = production_model_specs(&assets);
- let source = render_golden_data(&specs)
- .map_err(|error| anyhow::anyhow!("generating goldens: {error}"))?;
- let path = GoldenTestAssets::golden_data_path()?;
- std::fs::write(&path, source).with_context(|| format!("write {}", path.display()))?;
- // Normalize to the repository's rustfmt style so the generated file
- // passes format checks without a manual step.
- let rustfmt = std::process::Command::new("rustfmt")
- .args(["--edition", "2024"])
- .arg(&path)
- .status()
- .with_context(|| format!("run rustfmt on {}", path.display()))?;
- if !rustfmt.success() {
- bail!("rustfmt failed on {}", path.display());
- }
- println!("wrote {}", path.display());
- Ok(())
- });
-}
-
-#[test]
-#[ignore = "downloads and runs the production models"]
-fn zero_inputs_are_separated_from_committed_goldens() {
- run_with_large_stack("zero_golden_separation", || {
- let assets = GoldenTestAssets::load()?;
- let model_paths = [
- &assets.face_detection.path,
- &assets.face_embedding.path,
- &assets.clip_image.path,
- &assets.clip_text.path,
- ];
-
- for model_path in model_paths {
- let separation = measure_zero_golden_separation(&model_path.to_string_lossy())
- .map_err(|error| anyhow::anyhow!(error))?;
- let threshold_margin = separation.zero_distance / separation.threshold;
- println!(
- "{}: metric={}, golden_distance={:.9}, zero_distance={:.9}, threshold={:.9}, \
- threshold_margin={:.1}x",
- separation.model_file,
- separation.metric_label,
- separation.golden_distance,
- separation.zero_distance,
- separation.threshold,
- threshold_margin,
- );
- if separation.zero_distance <= separation.threshold {
- bail!(
- "{}: zero-input output is not separated from the golden ({} {:.9} <= \
- threshold {:.9})",
- separation.model_file,
- separation.metric_label,
- separation.zero_distance,
- separation.threshold,
- );
- }
- }
- Ok(())
- });
-}
diff --git a/rust/crates/photos/tests/support/ml_indexing.rs b/rust/crates/photos/tests/support/ml_indexing.rs
index df2d7f5953..f9a2d30e62 100644
--- a/rust/crates/photos/tests/support/ml_indexing.rs
+++ b/rust/crates/photos/tests/support/ml_indexing.rs
@@ -389,7 +389,7 @@ pub(crate) struct GoldenModelAsset {
}
/// Downloaded production model assets for the golden self-test tooling, with
-/// ONNX Runtime already initialized. Used by the ml_goldens test binary only.
+/// ONNX Runtime already initialized. Used by the ml_goldens developer tool only.
#[allow(dead_code)]
pub(crate) struct GoldenTestAssets {
pub(crate) face_detection: GoldenModelAsset,
@@ -492,7 +492,7 @@ struct ModelAssets {
face_detection: ModelAsset,
face_embedding: ModelAsset,
clip_image: ModelAsset,
- // Used by the ml_goldens test binary only.
+ // Used by the ml_goldens developer tool only.
#[allow(dead_code)]
clip_text: ModelAsset,
#[allow(dead_code)]