π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Test ML runtime fallback behavior
laurenspriem969c5349f784
diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rsindex 0acbf635bb..3a54c8ac4f 100644--- a/rust/crates/photos/src/ml/indexing.rs+++ b/rust/crates/photos/src/ml/indexing.rs@@ -301,6 +301,25 @@ fn validate_request_model_paths(req: &AnalyzeImageRequest) -> MlResult<()> {mod tests {use super::*;+ fn request_with_models(run_faces: bool, run_clip: bool, run_pets: bool) -> AnalyzeImageRequest {+ AnalyzeImageRequest {+ file_id: 1,+ source: ImageSource::Bytes(Vec::new()),+ run_faces,+ run_clip,+ run_pets,+ generate_face_crops: false,+ model_paths: ModelPaths::default(),+ }+ }++ fn validation_error(req: &AnalyzeImageRequest) -> String {+ match validate_request_model_paths(req) {+ Err(MlError::InvalidRequest(message)) => message,+ other => panic!("expected an invalid request, got {other:?}"),+ }+ }+fn png_bytes(width: u32, height: u32) -> Vec<u8> {let img =image::RgbImage::from_fn(width, height, |x, y| image::Rgb([x as u8, y as u8, 128]));@@ -348,4 +367,45 @@ mod tests {other => panic!("expected a decode error, got {other:?}"),}}++ #[test]+ fn request_validation_reports_required_models_in_catalog_order() {+ let req = request_with_models(true, true, true);++ assert_eq!(+ validation_error(&req),+ "missing required model paths: faceDetectionModelPath, faceEmbeddingModelPath, \+ clipImageModelPath, petFaceDetectionModelPath, petBodyDetectionModelPath, \+ petFaceEmbeddingDogModelPath, petFaceEmbeddingCatModelPath, \+ petBodyEmbeddingDogModelPath, petBodyEmbeddingCatModelPath"+ );+ }++ #[test]+ fn request_validation_uses_only_enabled_model_groups() {+ assert_eq!(+ validation_error(&request_with_models(true, false, false)),+ "missing required model paths: faceDetectionModelPath, faceEmbeddingModelPath"+ );+ assert_eq!(+ validation_error(&request_with_models(false, true, false)),+ "missing required model paths: clipImageModelPath"+ );+ assert_eq!(+ validation_error(&request_with_models(false, false, true)),+ "missing required model paths: petFaceDetectionModelPath, petBodyDetectionModelPath, \+ petFaceEmbeddingDogModelPath, petFaceEmbeddingCatModelPath, \+ petBodyEmbeddingDogModelPath, petBodyEmbeddingCatModelPath"+ );+ }++ #[test]+ fn request_validation_accepts_all_required_model_paths() {+ let mut req = request_with_models(true, true, true);+ for model in Model::INDEXING {+ *req.model_paths.get_mut(model) = format!("{}.onnx", model.namespace());+ }++ validate_request_model_paths(&req).unwrap();+ }}diff --git a/rust/crates/photos/src/ml/model.rs b/rust/crates/photos/src/ml/model.rsindex b8770a8412..0a280ee974 100644--- a/rust/crates/photos/src/ml/model.rs+++ b/rust/crates/photos/src/ml/model.rs@@ -194,20 +194,94 @@ mod tests {use super::*;#[test]- fn model_catalog_is_array_indexed_and_unique() {+ fn model_catalog_order_and_metadata_are_stable() {+ let expected = [+ (+ Model::FaceDetection,+ "faceDetectionModelPath",+ "face-detection",+ ),+ (+ Model::FaceEmbedding,+ "faceEmbeddingModelPath",+ "face-embedding",+ ),+ (Model::ClipImage, "clipImageModelPath", "clip-image"),+ (Model::ClipText, "clipTextModelPath", "clip-text"),+ (+ Model::PetFaceDetection,+ "petFaceDetectionModelPath",+ "pet-face-detection",+ ),+ (+ Model::PetFaceEmbeddingDog,+ "petFaceEmbeddingDogModelPath",+ "pet-face-embedding-dog",+ ),+ (+ Model::PetFaceEmbeddingCat,+ "petFaceEmbeddingCatModelPath",+ "pet-face-embedding-cat",+ ),+ (+ Model::PetBodyDetection,+ "petBodyDetectionModelPath",+ "pet-body-detection",+ ),+ (+ Model::PetBodyEmbeddingDog,+ "petBodyEmbeddingDogModelPath",+ "pet-body-embedding-dog",+ ),+ (+ Model::PetBodyEmbeddingCat,+ "petBodyEmbeddingCatModelPath",+ "pet-body-embedding-cat",+ ),+ ];let mut path_labels = HashSet::new();let mut namespaces = HashSet::new();- for (index, model) in Model::ALL.into_iter().enumerate() {+ for (index, (model, path_label, namespace)) in expected.into_iter().enumerate() {+ assert_eq!(Model::ALL[index], model);assert_eq!(model.index(), index);- assert!(path_labels.insert(model.path_label()));- assert!(namespaces.insert(model.namespace()));+ assert_eq!(model.path_label(), path_label);+ assert_eq!(model.namespace(), namespace);+ assert!(path_labels.insert(path_label));+ assert!(namespaces.insert(namespace));}}#[test]fn model_paths_are_addressable_by_model() {- let mut paths = ModelPaths::default();+ let mut paths = ModelPaths {+ face_detection: "face-detection.onnx".to_string(),+ face_embedding: "face-embedding.onnx".to_string(),+ clip_image: "clip-image.onnx".to_string(),+ clip_text: "clip-text.onnx".to_string(),+ pet_face_detection: "pet-face-detection.onnx".to_string(),+ pet_face_embedding_dog: "pet-face-embedding-dog.onnx".to_string(),+ pet_face_embedding_cat: "pet-face-embedding-cat.onnx".to_string(),+ pet_body_detection: "pet-body-detection.onnx".to_string(),+ pet_body_embedding_dog: "pet-body-embedding-dog.onnx".to_string(),+ pet_body_embedding_cat: "pet-body-embedding-cat.onnx".to_string(),+ };+ let expected = [+ (Model::FaceDetection, "face-detection.onnx"),+ (Model::FaceEmbedding, "face-embedding.onnx"),+ (Model::ClipImage, "clip-image.onnx"),+ (Model::ClipText, "clip-text.onnx"),+ (Model::PetFaceDetection, "pet-face-detection.onnx"),+ (Model::PetFaceEmbeddingDog, "pet-face-embedding-dog.onnx"),+ (Model::PetFaceEmbeddingCat, "pet-face-embedding-cat.onnx"),+ (Model::PetBodyDetection, "pet-body-detection.onnx"),+ (Model::PetBodyEmbeddingDog, "pet-body-embedding-dog.onnx"),+ (Model::PetBodyEmbeddingCat, "pet-body-embedding-cat.onnx"),+ ];++ for (model, path) in expected {+ assert_eq!(paths.get(model), path);+ }for model in Model::ALL {*paths.get_mut(model) = model.namespace().to_string();@@ -219,7 +293,7 @@ mod tests {}#[test]- fn indexing_catalog_contains_every_model_except_clip_text() {+ fn indexing_catalog_contains_every_model_except_clip_text_once() {let indexes = Model::INDEXING.into_iter().map(Model::index)@@ -236,7 +310,10 @@ mod tests {.collect::<HashSet<_>>(),indexes);+ }+ #[test]+ fn enabled_indexing_models_follow_request_flags() {assert_eq!(enabled_indexing_models(true, false, false).collect::<Vec<_>>(),[Model::FaceDetection, Model::FaceEmbedding]@@ -256,16 +333,5 @@ mod tests {Model::PetBodyEmbeddingCat,]);- assert_eq!(- required_indexing_models(false, false, true).collect::<Vec<_>>(),- [- Model::PetFaceDetection,- Model::PetBodyDetection,- Model::PetFaceEmbeddingDog,- Model::PetFaceEmbeddingCat,- Model::PetBodyEmbeddingDog,- Model::PetBodyEmbeddingCat,- ]- );}}diff --git a/rust/crates/photos/src/ml/onnx.rs b/rust/crates/photos/src/ml/onnx.rsindex fe676a7e7c..83afd9747e 100644--- a/rust/crates/photos/src/ml/onnx.rs+++ b/rust/crates/photos/src/ml/onnx.rs@@ -68,6 +68,10 @@ impl ProviderPlan {ExecutionMode::PlatformDefault => platform_default_providers(model_path),ExecutionMode::CpuOnly => vec![ExecutionProvider::Cpu],};+ Self::from_providers(providers)+ }++ fn from_providers(providers: Vec<ExecutionProvider>) -> Self {Self {providers,next: 0,@@ -99,6 +103,27 @@ impl ProviderPlan {}}+fn run_provider_plan<T, E>(+ plan: &mut ProviderPlan,+ mut attempt: impl FnMut(ExecutionProvider) -> Result<T, E>,+) -> Result<T, Vec<E>> {+ plan.selected = None;++ let mut errors = Vec::new();+ while let Some(provider) = plan.next_provider() {+ match attempt(provider) {+ Ok(value) => {+ plan.select(provider);+ return Ok(value);+ }+ Err(error) => errors.push(error),+ }+ }++ plan.retain_last_provider_for_retry();+ Err(errors)+}+#[derive(Debug)]pub(crate) enum SessionRunError {Retryable(MlError),@@ -159,22 +184,13 @@ pub(crate) fn build_session(plan: &mut ProviderPlan,model_namespace: &str,) -> MlResult<Session> {- plan.selected = None;-- let mut errors = Vec::new();- while let Some(execution_provider) = plan.next_provider() {+ let result = run_provider_plan(plan, |execution_provider| {let attempt = provider_attempt(execution_provider, model_path, model_namespace);if attempt.uses_webgpu {#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]{- match build_webgpu_session_with_canary(model_path, model_namespace, attempt) {- Ok(session) => {- plan.select(execution_provider);- return Ok(session);- }- Err(error) => errors.push(format!("{error}")),- }- continue;+ return build_webgpu_session_with_canary(model_path, model_namespace, attempt)+ .map_err(|error| format!("{error}"));}#[cfg(not(any(target_os = "android", target_os = "linux", target_os = "windows")))]unreachable!("WebGPU provider attempts are not constructed on this platform");@@ -186,8 +202,7 @@ pub(crate) fn build_session(if let Some(cache_dir) = coreml_cache_dir {coreml_cache::finalize(&cache_dir, model_path);}- plan.select(execution_provider);- return Ok(session);+ Ok(session)}Err(error) => {if let Some(cache_dir) = coreml_cache_dir@@ -198,12 +213,15 @@ pub(crate) fn build_session(model_file_label(model_path));}- errors.push(format!("{error}"));+ Err(format!("{error}"))}}- }+ });- plan.retain_last_provider_for_retry();+ let errors = match result {+ Ok(session) => return Ok(session),+ Err(errors) => errors,+ };if has_protobuf_parse_failure(&errors) {return Err(MlError::CorruptModel(model_path.to_string()));@@ -695,62 +713,105 @@ fn build_session_with_providers(model_path: &str, attempt: ProviderAttempt) -> M#[cfg(test)]mod tests {use super::{- ExecutionMode, ExecutionProvider, ProviderPlan, has_protobuf_parse_failure,- is_execution_provider_run_failure, provider_attempt_failure_message,+ ExecutionMode, ExecutionProvider, ProviderPlan, SessionRunError,+ has_protobuf_parse_failure, is_execution_provider_run_failure,+ provider_attempt_failure_message, run_provider_plan,};#[test]- fn provider_plan_resumes_after_the_concrete_selected_provider() {- let mut plan = ProviderPlan {- providers: vec![- ExecutionProvider::WebGpu,- ExecutionProvider::Xnnpack,- ExecutionProvider::Cpu,- ],- next: 0,- selected: None,- };-- assert_eq!(plan.next_provider(), Some(ExecutionProvider::WebGpu));- assert_eq!(plan.next_provider(), Some(ExecutionProvider::Xnnpack));- plan.select(ExecutionProvider::Xnnpack);+ fn construction_falls_through_and_selects_the_successful_provider() {+ let mut plan = accelerated_provider_plan();+ let mut attempted = Vec::new();++ let result = run_provider_plan(&mut plan, |provider| {+ attempted.push(provider);+ match provider {+ ExecutionProvider::Xnnpack => Ok("session"),+ _ => Err(provider),+ }+ });+ assert_eq!(result, Ok("session"));+ assert_eq!(+ attempted,+ [ExecutionProvider::WebGpu, ExecutionProvider::Xnnpack]+ );assert_eq!(plan.selected_provider(), Some(ExecutionProvider::Xnnpack));assert!(plan.has_fallback());- assert_eq!(plan.next_provider(), Some(ExecutionProvider::Cpu));- assert!(!plan.has_fallback());}#[test]- fn cpu_only_provider_plan_is_bounded() {- let mut plan = ProviderPlan::new(ExecutionMode::CpuOnly, "model.onnx");+ fn retry_resumes_strictly_after_the_provider_that_was_selected() {+ let mut plan = accelerated_provider_plan();+ run_provider_plan(&mut plan, |provider| match provider {+ ExecutionProvider::Xnnpack => Ok(()),+ _ => Err(()),+ })+ .unwrap();- assert_eq!(plan.next_provider(), Some(ExecutionProvider::Cpu));- assert_eq!(plan.next_provider(), None);+ let mut attempted = Vec::new();+ let result = run_provider_plan(&mut plan, |provider| {+ attempted.push(provider);+ Ok::<_, ()>("fallback session")+ });++ assert_eq!(result, Ok("fallback session"));+ assert_eq!(attempted, [ExecutionProvider::Cpu]);+ assert_eq!(plan.selected_provider(), Some(ExecutionProvider::Cpu));assert!(!plan.has_fallback());}#[test]- fn exhausted_provider_plan_retries_only_the_final_provider() {- let mut plan = ProviderPlan {- providers: vec![+ fn provider_attempts_are_bounded_and_preserve_each_failure() {+ let mut plan = accelerated_provider_plan();+ let mut attempted = Vec::new();++ let errors = run_provider_plan::<(), _>(&mut plan, |provider| {+ attempted.push(provider);+ Err(provider)+ })+ .unwrap_err();++ assert_eq!(+ attempted,+ [ExecutionProvider::WebGpu,ExecutionProvider::Xnnpack,ExecutionProvider::Cpu,- ],- next: 0,- selected: None,- };+ ]+ );+ assert_eq!(errors, attempted);+ assert_eq!(plan.selected_provider(), None);+ }- while plan.next_provider().is_some() {}- plan.retain_last_provider_for_retry();+ #[test]+ fn exhausted_cpu_is_retained_as_the_only_construction_retry() {+ let mut plan = ProviderPlan::new(ExecutionMode::CpuOnly, "model.onnx");+ let mut attempts = 0;++ let first = run_provider_plan::<(), _>(&mut plan, |provider| {+ attempts += 1;+ Err(provider)+ });++ assert_eq!(first, Err(vec![ExecutionProvider::Cpu]));+ assert_eq!(attempts, 1);+ assert_eq!(plan.selected_provider(), None);+ assert!(plan.has_fallback());++ let retry = run_provider_plan(&mut plan, |provider| {+ attempts += 1;+ Ok::<_, ExecutionProvider>(provider)+ });- assert_eq!(plan.next_provider(), Some(ExecutionProvider::Cpu));- assert_eq!(plan.next_provider(), None);+ assert_eq!(retry, Ok(ExecutionProvider::Cpu));+ assert_eq!(attempts, 2);+ assert_eq!(plan.selected_provider(), Some(ExecutionProvider::Cpu));+ assert!(!plan.has_fallback());}#[test]- fn classifies_only_provider_related_ort_run_failures_as_retryable() {+ fn provider_failure_detection_is_limited_to_ort_execution_errors() {assert!(is_execution_provider_run_failure(&super::MlError::Ort("WebGPU EP error: VK_ERROR_DEVICE_LOST".to_string())));@@ -760,8 +821,15 @@ mod tests {assert!(!is_execution_provider_run_failure(&super::MlError::Postprocess("WebGPU".to_string())));+ }++ #[test]+ fn typed_run_errors_control_retryability() {+ let retryable =+ SessionRunError::retryable(super::MlError::Ort("non-finite output".to_string()));+ assert!(retryable.is_retryable());- let terminal = super::SessionRunError::from(super::MlError::Ort(+ let terminal = SessionRunError::from(super::MlError::Ort("WebGPU text from a typed terminal error".to_string(),));assert!(!terminal.is_retryable());@@ -812,4 +880,12 @@ mod tests {.is_none());}++ fn accelerated_provider_plan() -> ProviderPlan {+ ProviderPlan::from_providers(vec![+ ExecutionProvider::WebGpu,+ ExecutionProvider::Xnnpack,+ ExecutionProvider::Cpu,+ ])+ }}diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rsindex 065a78609e..3b377a5c2f 100644--- a/rust/crates/photos/src/ml/runtime.rs+++ b/rust/crates/photos/src/ml/runtime.rs@@ -22,6 +22,13 @@ struct ModelSlotState {session: Option<Session>,}+#[cfg(test)]+#[derive(Debug, PartialEq, Eq)]+struct ModelSlotSnapshot {+ path: String,+ session_loaded: bool,+}+#[derive(Debug)]struct ModelSlot {model: Model,@@ -71,6 +78,15 @@ impl ModelSlot {}}+ #[cfg(test)]+ fn snapshot(&self) -> ModelSlotSnapshot {+ let state = self.lock_state();+ ModelSlotSnapshot {+ path: state.path.clone(),+ session_loaded: state.session.is_some(),+ }+ }+fn configure_if_requested(&self, path: &str) {if path.trim().is_empty() {return;@@ -328,8 +344,14 @@ mod tests {..ModelPaths::default()});- let clip_text = runtime.slot(Model::ClipText).lock_state();- assert_eq!(clip_text.path, "clip_text.onnx");+ assert_eq!(+ runtime.slot(Model::ClipText).snapshot().path,+ "clip_text.onnx"+ );+ assert_eq!(+ runtime.slot(Model::FaceDetection).snapshot().path,+ "face.onnx"+ );}#[test]@@ -342,34 +364,31 @@ mod tests {}#[test]- fn release_indexing_models_keeps_clip_text_state() {+ fn release_indexing_models_preserves_clip_text_configuration() {let runtime = MlRuntime::new();+ let model_paths = ModelPaths {+ face_detection: "face.onnx".to_string(),+ clip_text: "clip_text.onnx".to_string(),+ ..ModelPaths::default()+ };- {- let mut clip_text = runtime.slot(Model::ClipText).lock_state();- clip_text.path = "clip_text.onnx".to_string();- clip_text.provider_plan = Some(onnx::ProviderPlan::new(- onnx::ExecutionMode::CpuOnly,- "clip_text.onnx",- ));- }- {- let mut face_detection = runtime.slot(Model::FaceDetection).lock_state();- face_detection.path = "face.onnx".to_string();- face_detection.provider_plan = Some(onnx::ProviderPlan::new(- onnx::ExecutionMode::CpuOnly,- "face.onnx",- ));- }-+ runtime.configure_requested_models(&model_paths);runtime.release_indexing_models();- let clip_text = runtime.slot(Model::ClipText).lock_state();- assert_eq!(clip_text.path, "clip_text.onnx");- assert!(clip_text.provider_plan.is_some());-- let face_detection = runtime.slot(Model::FaceDetection).lock_state();- assert!(face_detection.provider_plan.is_none());+ assert_eq!(+ runtime.slot(Model::ClipText).snapshot(),+ ModelSlotSnapshot {+ path: "clip_text.onnx".to_string(),+ session_loaded: false,+ }+ );+ assert_eq!(+ runtime.slot(Model::FaceDetection).snapshot(),+ ModelSlotSnapshot {+ path: "face.onnx".to_string(),+ session_loaded: false,+ }+ );}#[test]@@ -383,116 +402,54 @@ mod tests {..ModelPaths::default()});- let face_detection = runtime.slot(Model::FaceDetection).lock_state();- assert_eq!(face_detection.path, "face.onnx");- assert!(face_detection.session.is_none());-- let face_embedding = runtime.slot(Model::FaceEmbedding).lock_state();- assert_eq!(face_embedding.path, "embed.onnx");- assert!(face_embedding.session.is_none());-- let clip_image = runtime.slot(Model::ClipImage).lock_state();- assert_eq!(clip_image.path, "clip.onnx");- assert!(clip_image.session.is_none());- }-- #[test]- fn sync_indexing_residency_clears_disabled_slots() {- let slot = ModelSlot::new(Model::ClipImage);-- {- let mut state = slot.lock_state();- state.path = "pet.onnx".to_string();- state.provider_plan = Some(onnx::ProviderPlan::new(- onnx::ExecutionMode::CpuOnly,- "pet.onnx",- ));- }-- slot.sync_indexing_residency("");-- let state = slot.lock_state();- assert!(state.path.is_empty());- assert!(state.provider_plan.is_none());- assert!(state.session.is_none());+ assert_eq!(+ runtime.slot(Model::FaceDetection).snapshot(),+ unloaded_snapshot("face.onnx")+ );+ assert_eq!(+ runtime.slot(Model::FaceEmbedding).snapshot(),+ unloaded_snapshot("embed.onnx")+ );+ assert_eq!(+ runtime.slot(Model::ClipImage).snapshot(),+ unloaded_snapshot("clip.onnx")+ );}#[test]- fn release_residency_resets_transient_cpu_fallback_for_any_slot() {- let slot = ModelSlot::new(Model::ClipImage);-- {- let mut state = slot.lock_state();- state.path = "clip_text.onnx".to_string();- state.provider_plan = Some(onnx::ProviderPlan::new(- onnx::ExecutionMode::CpuOnly,- "clip_text.onnx",- ));- }+ fn disabling_an_indexing_model_resets_its_slot() {+ let runtime = MlRuntime::new();+ runtime.prepare_indexing_models(&ModelPaths {+ clip_image: "clip.onnx".to_string(),+ ..ModelPaths::default()+ });- slot.release_residency();+ runtime.prepare_indexing_models(&ModelPaths::default());- let state = slot.lock_state();- assert!(state.provider_plan.is_none());- assert!(state.session.is_none());+ assert_eq!(+ runtime.slot(Model::ClipImage).snapshot(),+ unloaded_snapshot("")+ );}#[test]fn model_execution_modes_match_platform_policy() {- let runtime = MlRuntime::new();- let expected_pet_mode = onnx::ExecutionMode::CpuOnly;+ let accelerated = [Model::FaceDetection, Model::FaceEmbedding, Model::ClipImage];for model in Model::ALL {- assert_eq!(runtime.slot(model).model, model);+ let expected = if accelerated.contains(&model) {+ onnx::ExecutionMode::PlatformDefault+ } else {+ onnx::ExecutionMode::CpuOnly+ };+ assert_eq!(default_execution_mode(model), expected, "{model:?}");}+ }- assert_eq!(- runtime.slot(Model::FaceDetection).default_execution_mode,- onnx::ExecutionMode::PlatformDefault- );- assert_eq!(- runtime.slot(Model::FaceEmbedding).default_execution_mode,- onnx::ExecutionMode::PlatformDefault- );- assert_eq!(- runtime.slot(Model::ClipImage).default_execution_mode,- onnx::ExecutionMode::PlatformDefault- );- assert_eq!(- runtime.slot(Model::ClipText).default_execution_mode,- onnx::ExecutionMode::CpuOnly- );- assert_eq!(- runtime.slot(Model::PetFaceDetection).default_execution_mode,- expected_pet_mode- );- assert_eq!(- runtime- .slot(Model::PetFaceEmbeddingDog)- .default_execution_mode,- expected_pet_mode- );- assert_eq!(- runtime- .slot(Model::PetFaceEmbeddingCat)- .default_execution_mode,- expected_pet_mode- );- assert_eq!(- runtime.slot(Model::PetBodyDetection).default_execution_mode,- expected_pet_mode- );- assert_eq!(- runtime- .slot(Model::PetBodyEmbeddingDog)- .default_execution_mode,- expected_pet_mode- );- assert_eq!(- runtime- .slot(Model::PetBodyEmbeddingCat)- .default_execution_mode,- expected_pet_mode- );+ fn unloaded_snapshot(path: &str) -> ModelSlotSnapshot {+ ModelSlotSnapshot {+ path: path.to_string(),+ session_loaded: false,+ }}}