π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
webgpu gate refactor
laurenspriem575454cf76a5
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dartindex 7c3fa858ed..bb10a77959 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart@@ -6,11 +6,11 @@ import "package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart"import "package:logging/logging.dart";import "package:photos/core/errors.dart";import "package:photos/models/ml/vector.dart";-import "package:photos/service_locator.dart" show flagService;import "package:photos/services/machine_learning/ml_constants.dart";import "package:photos/services/machine_learning/ml_model_assets.dart";import "package:photos/services/machine_learning/ml_model_download_service.dart";import "package:photos/services/machine_learning/semantic_search/query_result.dart";+import "package:photos/services/machine_learning/webgpu_execution_policy.dart";import "package:photos/services/remote_assets_service.dart";import "package:photos/utils/isolate/isolate_operations.dart";import "package:photos/utils/isolate/super_isolate.dart";@@ -83,14 +83,12 @@ class MLComputer extends SuperIsolate {"RustMLMissingModelPath: Missing required model path: clipTextVocabPath",);}+ final enableWebGpu = await webGpuExecutionPolicy.isEligible();final isolateResult = await runInIsolate(IsolateOperation.runClipText, {"text": query,"clipTextModelPath": modelPath,"clipTextVocabPath": vocabPath,- // Keep in sync with MLIndexingIsolate: WebGPU is only enabled for- // internal users while the custom Android ONNX Runtime build is- // being validated. Ignored on other platforms.- "enableWebGpu": flagService.webGPUEnabled,+ "enableWebGpu": enableWebGpu,});if (isolateResult is RustCorruptModelCacheDeletedException) {_clipTextModelPath = null;diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dartindex b136cfd852..7d72a4c7d1 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart@@ -5,6 +5,7 @@ import "package:photos/services/machine_learning/ml_model_assets.dart";import "package:photos/services/machine_learning/ml_model_download_service.dart";import "package:photos/services/machine_learning/ml_models_overview.dart";import "package:photos/services/machine_learning/ml_result.dart";+import "package:photos/services/machine_learning/webgpu_execution_policy.dart";import "package:photos/services/remote_assets_service.dart";import "package:photos/utils/isolate/isolate_operations.dart";import "package:photos/utils/isolate/super_isolate.dart";@@ -47,6 +48,7 @@ class MLIndexingIsolate extends SuperIsolate {) async {try {final rustRuntimeArgs = await _getCachedRustRuntimeArgs();+ final enableWebGpu = await webGpuExecutionPolicy.isEligible();_logger.info("Analyzing image ${instruction.fileKey} with Rust ML");final isolateResult = await runInIsolate(IsolateOperation.analyzeImage, {@@ -56,6 +58,7 @@ class MLIndexingIsolate extends SuperIsolate {"runClip": instruction.shouldRunClip,"runPets": instruction.shouldRunPets,...rustRuntimeArgs,+ "enableWebGpu": enableWebGpu,});if (isolateResult is RustCorruptModelCacheDeletedException) {_logger.warning(@@ -176,9 +179,9 @@ class MLIndexingIsolate extends SuperIsolate {}return {- // WebGPU is only enabled for internal users while the custom Android- // ONNX Runtime build is being validated. Ignored on other platforms.- "enableWebGpu": flagService.webGPUEnabled,+ // Sessions are lazy, so the inference call re-evaluates this app-side+ // policy before Rust applies its own durable crash canary.+ "enableWebGpu": await webGpuExecutionPolicy.isEligible(),"faceDetectionModelPath": faceDetectionPath,"faceEmbeddingModelPath": faceEmbeddingPath,"clipImageModelPath": clipImagePath,diff --git a/mobile/apps/photos/lib/services/machine_learning/webgpu_execution_policy.dart b/mobile/apps/photos/lib/services/machine_learning/webgpu_execution_policy.dartnew file mode 100644index 0000000000..35c10308d6--- /dev/null+++ b/mobile/apps/photos/lib/services/machine_learning/webgpu_execution_policy.dart@@ -0,0 +1,33 @@+import "dart:io" show Platform;++import "package:device_info_plus/device_info_plus.dart";+import "package:photos/service_locator.dart" show flagService;+import "package:photos/utils/ram_check_util.dart";++final webGpuExecutionPolicy = WebGpuExecutionPolicy();++/// Computes the app-owned portion of Android WebGPU eligibility.+///+/// Durable quarantine and GPU adapter compatibility are owned by Rust.+class WebGpuExecutionPolicy {+ Future<int?>? _androidSdkFuture;++ Future<bool> isEligible() async {+ if (!Platform.isAndroid ||+ !flagService.webGPUEnabled ||+ !enoughRamForLocalIndexing) {+ return false;+ }+ final sdk = await (_androidSdkFuture ??= _readAndroidSdk());+ // Android 12++ return sdk != null && sdk >= 31;+ }++ static Future<int?> _readAndroidSdk() async {+ try {+ return (await DeviceInfoPlugin().androidInfo).version.sdkInt;+ } catch (_) {+ return null;+ }+ }+}diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 8bf935c0a2..e2ca22e8c7 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2424,7 +2424,6 @@ dependencies = [name = "ente-photos"version = "0.0.0"dependencies = [- "android_system_properties","anyhow","ente-image","fast_image_resize",diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.tomlindex 6ca0c8c434..a7d6ed0614 100644--- a/rust/crates/photos/Cargo.toml+++ b/rust/crates/photos/Cargo.toml@@ -39,8 +39,6 @@ ort = { workspace = true, features = [] }[target.'cfg(target_os = "android")'.dependencies]-# Used to gate the WebGPU execution provider to Android 12+ (SDK 31).-android_system_properties = "0.1.5"ort = { workspace = true, features = ["ndarray","half",diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rsindex b092da6a0a..b281a37b80 100644--- a/rust/crates/photos/src/ml/indexing.rs+++ b/rust/crates/photos/src/ml/indexing.rs@@ -9,6 +9,7 @@ use crate::ml::{preprocess,runtime::{self, ModelPaths},types::{self, ClipResult, Dimensions, FaceResult, PetBodyResult, PetFaceResult},+ webgpu,};use ente_image::decode::decode_image_from_path;@@ -47,11 +48,11 @@ pub struct RunClipTextResult {/// Configures process-wide ML execution behavior. Must be called before the/// first session is created to take effect for that session.///-/// `enable_webgpu` opts Android into the WebGPU execution provider; it is off-/// by default and additionally requires Android 12+ (SDK 31). It has no-/// effect on other platforms.+/// `enable_webgpu` is the app-side eligibility decision for Android. Rust+/// additionally applies its durable crash canary before attempting the WebGPU+/// execution provider. It has no effect on other platforms.pub fn set_ml_execution_config(enable_webgpu: bool) {- crate::ml::onnx::set_webgpu_enabled(enable_webgpu);+ webgpu::set_enabled(enable_webgpu);}pub fn init_ml_runtime(model_paths: ModelPaths) {diff --git a/rust/crates/photos/src/ml/mod.rs b/rust/crates/photos/src/ml/mod.rsindex 90dcc7acff..2f319e0fa0 100644--- a/rust/crates/photos/src/ml/mod.rs+++ b/rust/crates/photos/src/ml/mod.rs@@ -9,3 +9,4 @@ mod preprocess;pub mod runtime;pub mod types;pub mod vector_db;+mod webgpu;diff --git a/rust/crates/photos/src/ml/onnx.rs b/rust/crates/photos/src/ml/onnx.rsindex ddf1482f21..5b1c61ca96 100644--- a/rust/crates/photos/src/ml/onnx.rs+++ b/rust/crates/photos/src/ml/onnx.rs@@ -22,43 +22,12 @@ use ort::ep::{};#[cfg(target_os = "android")]use std::num::NonZeroUsize;-#[cfg(target_os = "android")]-use std::sync::{- OnceLock,- atomic::{AtomicBool, Ordering},-};use crate::ml::error::{MlError, MlResult};--/// WebGPU is opt-in while the custom Android ONNX Runtime build is being-/// validated: the app currently enables it for internal users only.#[cfg(target_os = "android")]-static WEBGPU_ENABLED: AtomicBool = AtomicBool::new(false);--pub(crate) fn set_webgpu_enabled(enabled: bool) {- #[cfg(target_os = "android")]- WEBGPU_ENABLED.store(enabled, Ordering::Relaxed);- #[cfg(not(target_os = "android"))]- let _ = enabled;-}--#[cfg(target_os = "android")]-fn webgpu_allowed() -> bool {- WEBGPU_ENABLED.load(Ordering::Relaxed) && android_supports_webgpu()-}--/// Dawn's Vulkan backend is only trusted on Android 12+ (SDK 31). This fails-/// closed: if the SDK level cannot be read, WebGPU stays disabled.+use crate::ml::webgpu;#[cfg(target_os = "android")]-fn android_supports_webgpu() -> bool {- static SDK_AT_LEAST_31: OnceLock<bool> = OnceLock::new();- *SDK_AT_LEAST_31.get_or_init(|| {- android_system_properties::AndroidSystemProperties::new()- .get("ro.build.version.sdk")- .and_then(|value| value.trim().parse::<u32>().ok())- .is_some_and(|sdk| sdk >= 31)- })-}+use ort::session::SessionInputValue;#[cfg(any(target_os = "ios", test))]const COREML_CACHE_SCHEMA: &str = "ort-1_27-mlprogram-all-default-v1";@@ -151,6 +120,20 @@ pub(crate) fn build_session(let mut errors = Vec::new();for attempt in attempts {+ if attempt.uses_webgpu {+ #[cfg(target_os = "android")]+ {+ match build_webgpu_session_with_canary(model_path, coreml_cache_namespace, attempt)+ {+ Ok(session) => return Ok(session),+ Err(error) => errors.push(format!("{error}")),+ }+ continue;+ }+ #[cfg(not(target_os = "android"))]+ unreachable!("WebGPU provider attempts are only constructed on Android");+ }+let coreml_cache_dir = attempt.coreml_cache_dir.clone();match build_session_with_providers(model_path, attempt) {Ok(session) => {@@ -188,6 +171,72 @@ pub(crate) fn build_session()))}+/// Builds the WebGPU session under the durable crash canary. The canary is+/// armed before session construction and only disarmed after the session has+/// survived one warm-up inference, so a driver crash anywhere in between is+/// recorded on disk. Soft failures return an error with the canary left armed+/// (the drop keeps the counted attempt), and the caller falls through to the+/// next execution provider.+#[cfg(target_os = "android")]+fn build_webgpu_session_with_canary(+ model_path: &str,+ model_namespace: &str,+ attempt: ProviderAttempt,+) -> MlResult<Session> {+ // Fail closed: without a durable failure record, a crash during the+ // attempt would go unnoticed and the crash loop protection would be lost.+ let canary = webgpu::arm_canary(model_path, model_namespace)+ .map_err(|error| MlError::Ort(format!("failed to arm WebGPU crash canary: {error}")))?;+ let mut session = build_session_with_providers(model_path, attempt)?;+ warmup_session(&mut session)?;+ canary.disarm();+ Ok(session)+}++/// Runs one zeroed inference so that lazy provider work (pipeline creation,+/// first dispatch) happens inside the crash-canary window, and so that the+/// first real inference is not paying the warm-up cost. Output values are+/// irrelevant; only whether the run crashes or errors matters.+#[cfg(target_os = "android")]+fn warmup_session(session: &mut Session) -> MlResult<()> {+ let mut input_specs = Vec::new();+ for input in session.inputs() {+ let ValueType::Tensor { ty, shape, .. } = input.dtype() else {+ // Unsupported input kind: skip the warm-up; the canary then only+ // covers session construction for this model.+ return Ok(());+ };+ // Our models have static shapes; substitute 1 for any dynamic+ // dimension defensively.+ let dims: Vec<i64> = shape.iter().map(|&dim| dim.max(1)).collect();+ input_specs.push((input.name().to_string(), *ty, dims));+ }++ let mut inputs: Vec<(String, SessionInputValue<'_>)> = Vec::new();+ for (name, ty, dims) in input_specs {+ let element_count = dims.iter().product::<i64>() as usize;+ let value: SessionInputValue<'_> = match ty {+ TensorElementType::Float32 => {+ Tensor::<f32>::from_array((dims, vec![0.0f32; element_count]))?.into()+ }+ TensorElementType::Float16 => {+ Tensor::<half::f16>::from_array((dims, vec![half::f16::ZERO; element_count]))?+ .into()+ }+ TensorElementType::Int32 => {+ Tensor::<i32>::from_array((dims, vec![0i32; element_count]))?.into()+ }+ TensorElementType::Int64 => {+ Tensor::<i64>::from_array((dims, vec![0i64; element_count]))?.into()+ }+ _ => return Ok(()),+ };+ inputs.push((name, value));+ }+ session.run(inputs)?;+ Ok(())+}+fn has_protobuf_parse_failure(errors: &[String]) -> bool {errors.iter().any(|error| {error@@ -200,6 +249,7 @@ struct ProviderAttempt {providers: Vec<ExecutionProviderDispatch>,disable_intra_op_spinning: bool,coreml_cache_dir: Option<PathBuf>,+ uses_webgpu: bool,}impl ProviderAttempt {@@ -208,6 +258,7 @@ impl ProviderAttempt {providers: vec![CPU::default().with_arena_allocator(true).build()],disable_intra_op_spinning: false,coreml_cache_dir: None,+ uses_webgpu: false,}}}@@ -241,6 +292,7 @@ fn platform_default_attempts(],disable_intra_op_spinning: false,coreml_cache_dir,+ uses_webgpu: false,},ProviderAttempt::cpu_only(),]@@ -274,11 +326,11 @@ fn coreml_provider(#[cfg(target_os = "android")]fn platform_default_attempts(- _model_path: &str,+ model_path: &str,_coreml_cache_namespace: &str,) -> Vec<ProviderAttempt> {let mut attempts = Vec::new();- if webgpu_allowed() {+ if webgpu::attempt_permitted(model_path) {attempts.push(ProviderAttempt {// EP priority follows registration order. Unsupported WebGPU nodes// fall through to XNNPACK and then CPU in the same session.@@ -289,6 +341,7 @@ fn platform_default_attempts(],disable_intra_op_spinning: true,coreml_cache_dir: None,+ uses_webgpu: true,});}// Clean-session fallbacks for provider or driver failures that prevent@@ -452,6 +505,7 @@ fn xnnpack_attempt() -> ProviderAttempt {],disable_intra_op_spinning: true,coreml_cache_dir: None,+ uses_webgpu: false,}}diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rsindex 63e7d49757..8bd50d0755 100644--- a/rust/crates/photos/src/ml/runtime.rs+++ b/rust/crates/photos/src/ml/runtime.rs@@ -12,7 +12,7 @@ use crate::ml::{};/// Log to Android logcat or stderr.-fn rt_log(msg: &str) {+pub(crate) fn rt_log(msg: &str) {#[cfg(target_os = "android")]{unsafe extern "C" {diff --git a/rust/crates/photos/src/ml/webgpu.rs b/rust/crates/photos/src/ml/webgpu.rsnew file mode 100644index 0000000000..541b15ae34--- /dev/null+++ b/rust/crates/photos/src/ml/webgpu.rs@@ -0,0 +1,285 @@+//! Durable crash canary for the Android WebGPU execution provider.+//!+//! The only goal of this module is to prevent an infinite crash loop on the+//! rare device where Dawn/Vulkan hard-crashes the process while a WebGPU+//! session is built or first dispatched. It intentionally does nothing more.+//!+//! Mechanics: immediately before a WebGPU session is built, a per-model+//! counter file next to the model is incremented and fsynced ("armed"). After+//! the session has been built *and* has survived one warm-up inference, the+//! file is removed ("disarmed"). A crash, kill, or soft failure in between+//! leaves the incremented counter behind. Once any model's counter reaches+//! [`MAX_CONSECUTIVE_FAILURES`], WebGPU is no longer attempted for models in+//! that directory β durably, across process restarts.+//!+//! A single interrupted attempt (e.g. Android killing the app mid-indexing)+//! is therefore tolerated: only consecutive failures with no intervening+//! success trip the breaker, which a genuine crash loop does deterministically+//! within a few launches.+//!+//! Everything here is called from `onnx::build_session` while the caller+//! holds the per-model slot lock, so accesses to one model's counter file are+//! already serialized and no in-memory state is needed beyond the enable flag.++#[cfg(target_os = "android")]+use std::sync::atomic::{AtomicBool, Ordering};+#[cfg(any(target_os = "android", test))]+use std::{+ fs::{self, File, OpenOptions},+ io::{self, Write},+ path::{Path, PathBuf},+};++/// Bump the version to discard all previous canary state, e.g. after a major+/// ONNX Runtime or Dawn upgrade that warrants re-trialing quarantined devices.+#[cfg(any(target_os = "android", test))]+const CANARY_FILE_PREFIX: &str = ".ente-webgpu-canary-v1.";++/// Number of consecutive failed or interrupted WebGPU attempts for one model+/// after which WebGPU is durably disabled.+#[cfg(any(target_os = "android", test))]+const MAX_CONSECUTIVE_FAILURES: u32 = 3;++/// WebGPU is opt-in while the custom Android ONNX Runtime build is being+/// validated; the app-side policy also gates on device eligibility.+#[cfg(target_os = "android")]+static WEBGPU_ENABLED: AtomicBool = AtomicBool::new(false);++pub(crate) fn set_enabled(enabled: bool) {+ #[cfg(target_os = "android")]+ WEBGPU_ENABLED.store(enabled, Ordering::Relaxed);+ #[cfg(not(target_os = "android"))]+ let _ = enabled;+}++/// Whether a WebGPU session may be attempted for this model. False when the+/// app has not opted in, or when any model in the same directory has tripped+/// the crash canary. Fails closed on IO errors.+#[cfg(target_os = "android")]+pub(crate) fn attempt_permitted(model_path: &str) -> bool {+ WEBGPU_ENABLED.load(Ordering::Relaxed)+ && chromium_compatible_adapter_available()+ && model_dir(model_path).is_some_and(|dir| !quarantined(&dir))+}++/// Chromium-compatible Vulkan adapter filtering will be implemented here via+/// `ash`. Keeping this inside the Rust eligibility path ensures an unsupported+/// adapter cannot be bypassed by any Dart inference entry point.+#[cfg(target_os = "android")]+fn chromium_compatible_adapter_available() -> bool {+ true+}++/// A durable record of an in-flight WebGPU attempt for one model. Dropping it+/// without calling [`ArmedCanary::disarm`] leaves the attempt recorded as+/// failed, which is exactly what a crash does implicitly.+#[cfg(any(target_os = "android", test))]+pub(crate) struct ArmedCanary {+ path: PathBuf,+}++/// Increments and fsyncs the model's consecutive-failure counter before a+/// WebGPU attempt. On error the caller must skip WebGPU (fail closed): without+/// a durable record, a crash during the attempt would go unnoticed.+#[cfg(any(target_os = "android", test))]+pub(crate) fn arm_canary(model_path: &str, model_namespace: &str) -> io::Result<ArmedCanary> {+ let dir = model_dir(model_path).ok_or_else(|| {+ io::Error::new(+ io::ErrorKind::InvalidInput,+ "cannot derive WebGPU canary directory from model path",+ )+ })?;+ let path = dir.join(format!("{CANARY_FILE_PREFIX}{model_namespace}"));+ let failures = read_failure_count(&path)?;+ persist_failure_count(&path, failures.saturating_add(1))?;+ Ok(ArmedCanary { path })+}++#[cfg(any(target_os = "android", test))]+impl ArmedCanary {+ /// Marks the attempt as successful by removing the counter file, resetting+ /// the model's consecutive-failure count to zero.+ pub(crate) fn disarm(self) {+ if let Err(error) = remove_file_durably(&self.path) {+ crate::ml::runtime::rt_log(&format!(+ "failed to disarm WebGPU crash canary at '{}': {error}",+ self.path.display()+ ));+ }+ }+}++#[cfg(any(target_os = "android", test))]+fn model_dir(model_path: &str) -> Option<PathBuf> {+ if model_path.trim().is_empty() {+ return None;+ }+ Path::new(model_path)+ .parent()+ .filter(|dir| !dir.as_os_str().is_empty())+ .map(Path::to_path_buf)+}++/// True if any model's canary in this directory records too many consecutive+/// failures. Unreadable state fails closed.+#[cfg(any(target_os = "android", test))]+fn quarantined(dir: &Path) -> bool {+ let entries = match fs::read_dir(dir) {+ Ok(entries) => entries,+ Err(_) => return true,+ };+ for entry in entries {+ let Ok(entry) = entry else {+ return true;+ };+ let file_name = entry.file_name();+ let Some(name) = file_name.to_str() else {+ continue;+ };+ if !name.starts_with(CANARY_FILE_PREFIX) {+ continue;+ }+ match read_failure_count(&entry.path()) {+ Ok(failures) if failures < MAX_CONSECUTIVE_FAILURES => {}+ _ => return true,+ }+ }+ false+}++/// A missing file means zero consecutive failures. Unparseable contents are an+/// error so that callers fail closed.+#[cfg(any(target_os = "android", test))]+fn read_failure_count(path: &Path) -> io::Result<u32> {+ let contents = match fs::read_to_string(path) {+ Ok(contents) => contents,+ Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(0),+ Err(error) => return Err(error),+ };+ contents.trim().parse::<u32>().map_err(|_| {+ io::Error::new(+ io::ErrorKind::InvalidData,+ format!("invalid WebGPU canary contents at '{}'", path.display()),+ )+ })+}++#[cfg(any(target_os = "android", test))]+fn persist_failure_count(path: &Path, failures: u32) -> io::Result<()> {+ let mut file = OpenOptions::new()+ .create(true)+ .truncate(true)+ .write(true)+ .open(path)?;+ file.write_all(format!("{failures}\n").as_bytes())?;+ file.sync_all()?;+ sync_parent(path)+}++#[cfg(any(target_os = "android", test))]+fn remove_file_durably(path: &Path) -> io::Result<()> {+ match fs::remove_file(path) {+ Ok(()) => sync_parent(path),+ Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),+ Err(error) => Err(error),+ }+}++#[cfg(any(target_os = "android", test))]+fn sync_parent(path: &Path) -> io::Result<()> {+ let parent = path.parent().ok_or_else(|| {+ io::Error::new(+ io::ErrorKind::InvalidInput,+ "WebGPU canary path has no parent directory",+ )+ })?;+ File::open(parent)?.sync_all()+}++#[cfg(test)]+mod tests {+ use super::{MAX_CONSECUTIVE_FAILURES, arm_canary, quarantined};++ fn model_path(dir: &std::path::Path) -> String {+ dir.join("model.onnx").to_string_lossy().into_owned()+ }++ #[test]+ fn single_interrupted_attempt_does_not_quarantine() {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());++ // Dropped without disarming, as after a crash or background kill.+ drop(arm_canary(&model, "clip-image").unwrap());++ assert!(!quarantined(dir.path()));+ }++ #[test]+ fn consecutive_interrupted_attempts_quarantine() {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());++ for attempt in 1..=MAX_CONSECUTIVE_FAILURES {+ assert!(+ !quarantined(dir.path()),+ "quarantined before attempt {attempt}"+ );+ drop(arm_canary(&model, "clip-image").unwrap());+ }++ assert!(quarantined(dir.path()));+ }++ #[test]+ fn successful_attempt_resets_consecutive_failures() {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());++ drop(arm_canary(&model, "clip-image").unwrap());+ drop(arm_canary(&model, "clip-image").unwrap());+ arm_canary(&model, "clip-image").unwrap().disarm();++ assert!(!quarantined(dir.path()));+ drop(arm_canary(&model, "clip-image").unwrap());+ assert!(!quarantined(dir.path()));+ }++ #[test]+ fn failures_do_not_accumulate_across_models() {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());++ drop(arm_canary(&model, "clip-image").unwrap());+ drop(arm_canary(&model, "face-detection").unwrap());+ drop(arm_canary(&model, "face-embedding").unwrap());++ assert!(!quarantined(dir.path()));+ }++ #[test]+ fn one_tripped_model_quarantines_the_whole_directory() {+ let dir = tempfile::tempdir().unwrap();+ let model = model_path(dir.path());++ for _ in 0..MAX_CONSECUTIVE_FAILURES {+ drop(arm_canary(&model, "clip-image").unwrap());+ }+ arm_canary(&model, "face-detection").unwrap().disarm();++ assert!(quarantined(dir.path()));+ }++ #[test]+ fn unreadable_canary_state_fails_closed() {+ let dir = tempfile::tempdir().unwrap();+ std::fs::write(+ dir.path()+ .join(format!("{}clip-image", super::CANARY_FILE_PREFIX)),+ b"not a number",+ )+ .unwrap();++ assert!(quarantined(dir.path()));+ }+}