π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Enable guarded WebGPU for desktop ML
laurenspriemfd120281fd44
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.tsindex 88d975860c..01d6976b44 100644--- a/desktop/src/main/services/ml-worker.ts+++ b/desktop/src/main/services/ml-worker.ts@@ -114,6 +114,9 @@ const loadMLNative = (paths: MLNativePaths) => {// eslint-disable-next-line @typescript-eslint/no-require-importsconst native = require(paths.addon) as MLNative;native.initOrt(paths.onnxRuntimeLibrary);+ // Enables the guarded WebGPU path on Linux and Windows. This is a+ // no-op on macOS, where CoreML remains the preferred provider.+ native.setMlExecutionConfig(true);_native = native;log.debugString(`Loaded ML addon at ${paths.addon}`);} catch (e) {@@ -575,7 +578,7 @@ export const computeCLIPTextEmbeddingIfAvailable = async (text: string) => {log.debugString(`Rust ML CLIP text embedding took ${Date.now() - t} ms`,);- return Float32Array.from(embedding);+ return embedding;} finally {logMLRuntimeEvents(native);}diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rsindex 50def15c90..c79e947f23 100644--- a/rust/bindings/napi/photos/src/lib.rs+++ b/rust/bindings/napi/photos/src/lib.rs@@ -23,9 +23,8 @@ pub fn init_ort(dylib_path: String) -> Result<()> {Ok(())}-/// Configures process-wide ML execution behavior. `enable_webgpu` only has an-/// effect on Android; it is a no-op on desktop and exists for surface parity-/// with the mobile bindings.+/// Configures process-wide ML execution behavior. `enable_webgpu` enables the+/// guarded WebGPU path on Linux and Windows; it is a no-op on macOS.#[napi]pub fn set_ml_execution_config(enable_webgpu: bool) {shared_indexing::set_ml_execution_config(enable_webgpu);@@ -94,12 +93,12 @@ pub struct FaceResult {pub face_id: String,pub detection: FaceDetection,pub blur_value: f64,- pub embedding: Vec<f64>,+ pub embedding: Float32Array,}#[napi(object)]pub struct ClipResult {- pub embedding: Vec<f64>,+ pub embedding: Float32Array,}#[napi(object)]@@ -111,7 +110,7 @@ pub struct PetFaceResult {pub keypoints: Vec<Vec<f64>>,/// 0 = dog, 1 = cat.pub species: u32,- pub face_embedding: Vec<f64>,+ pub face_embedding: Float32Array,}#[napi(object)]@@ -121,7 +120,7 @@ pub struct PetBodyResult {pub box_xyxy: Vec<f64>,/// COCO class: 15 = cat, 16 = dog.pub coco_class: u32,- pub body_embedding: Vec<f64>,+ pub body_embedding: Float32Array,}#[napi(object)]@@ -199,7 +198,7 @@ pub struct RunClipTextRequest {#[napi(object)]pub struct RunClipTextResult {- pub embedding: Vec<f64>,+ pub embedding: Float32Array,}pub struct RunClipTextTask {@@ -220,11 +219,7 @@ impl Task for RunClipTextTask {fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {Ok(RunClipTextResult {- embedding: output- .embedding- .into_iter()- .map(|value| value as f64)- .collect(),+ embedding: output.embedding.into(),})}}@@ -325,7 +320,7 @@ fn to_napi_analyze_image_result(result: shared_indexing::AnalyzeImageResult) ->.face_crops.map(|crops| crops.into_iter().map(Buffer::from).collect()),clip: result.clip.map(|clip| ClipResult {- embedding: to_f64_vec(clip.embedding),+ embedding: clip.embedding.into(),}),pet_faces: result.pet_faces@@ -357,7 +352,7 @@ fn to_napi_face_result(result: shared_types::FaceResult) -> FaceResult {.collect(),},blur_value: f64::from(result.blur_value),- embedding: to_f64_vec(result.embedding),+ embedding: result.embedding.into(),}}@@ -378,7 +373,7 @@ fn to_napi_pet_face_result(result: shared_types::PetFaceResult) -> PetFaceResult.map(|point| point.into_iter().map(f64::from).collect()).collect(),species: u32::from(result.species),- face_embedding: to_f64_vec(result.face_embedding),+ face_embedding: result.face_embedding.into(),}}@@ -393,10 +388,6 @@ fn to_napi_pet_body_result(result: shared_types::PetBodyResult) -> PetBodyResult.map(f64::from).collect(),coco_class: u32::from(result.detection.coco_class),- body_embedding: to_f64_vec(result.body_embedding),+ body_embedding: result.body_embedding.into(),}}--fn to_f64_vec(values: Vec<f32>) -> Vec<f64> {- values.into_iter().map(f64::from).collect()-}diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.tomlindex f2144cac69..7d008a55ae 100644--- a/rust/crates/photos/Cargo.toml+++ b/rust/crates/photos/Cargo.toml@@ -62,7 +62,17 @@ ort = { workspace = true, features = ["copy-dylibs",] }-[target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "macos")))'.dependencies]+[target.'cfg(any(target_os = "linux", target_os = "windows"))'.dependencies]+ort = { workspace = true, features = [+ "ndarray",+ "half",+ "api-27",+ "load-dynamic",+ "copy-dylibs",+ "webgpu",+] }++[target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "macos", target_os = "linux", target_os = "windows")))'.dependencies]ort = { workspace = true, features = ["ndarray","half",diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rsindex 4c6b7bffb9..8664d4b529 100644--- a/rust/crates/photos/src/ml/indexing.rs+++ b/rust/crates/photos/src/ml/indexing.rs@@ -67,9 +67,10 @@ pub struct RunClipTextResult {/// Configures process-wide ML execution behavior. Must be called before the/// first session is created to take effect for that session.///-/// `enable_webgpu` is the app-side eligibility decision for Android. Rust-/// additionally applies its durable crash canary before attempting the WebGPU-/// execution provider. It has no effect on other platforms.+/// `enable_webgpu` is the app-side eligibility decision for Android, Linux and+/// Windows. Rust additionally applies its durable crash canary before+/// attempting the WebGPU execution provider. It has no effect on Apple+/// platforms.pub fn set_ml_execution_config(enable_webgpu: bool) {webgpu::set_enabled(enable_webgpu);}diff --git a/rust/crates/photos/src/ml/onnx.rs b/rust/crates/photos/src/ml/onnx.rsindex d066bf1814..e3d886a9ba 100644--- a/rust/crates/photos/src/ml/onnx.rs+++ b/rust/crates/photos/src/ml/onnx.rs@@ -10,14 +10,16 @@ use std::{use half::prelude::{HalfFloatSliceExt, HalfFloatVecExt};+#[cfg(target_os = "android")]+use ort::ep::XNNPACK;#[cfg(any(target_os = "ios", target_os = "macos"))]use ort::ep::{CoreML,coreml::{ComputeUnits, ModelFormat, SpecializationStrategy},};-#[cfg(target_os = "android")]+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]use ort::ep::{- WebGPU, XNNPACK,+ WebGPU,webgpu::{DawnBackendType, PreferredLayout},};#[cfg(target_os = "android")]@@ -26,9 +28,15 @@ use std::num::NonZeroUsize;use crate::ml::error::{MlError, MlResult};use crate::ml::events;use crate::ml::golden;-#[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]+#[cfg(any(+ target_os = "android",+ target_os = "ios",+ target_os = "linux",+ target_os = "macos",+ target_os = "windows"+))]use crate::ml::runtime::rt_log;-#[cfg(target_os = "android")]+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]use crate::ml::webgpu;#[cfg(any(target_os = "ios", target_os = "macos", test))]@@ -148,7 +156,7 @@ pub(crate) fn build_session(for attempt in attempts {let execution_provider = attempt.execution_provider;if attempt.uses_webgpu {- #[cfg(target_os = "android")]+ #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]{match build_webgpu_session_with_canary(model_path, coreml_cache_namespace, attempt){@@ -157,8 +165,8 @@ pub(crate) fn build_session(}continue;}- #[cfg(not(target_os = "android"))]- unreachable!("WebGPU provider attempts are only constructed on Android");+ #[cfg(not(any(target_os = "android", target_os = "linux", target_os = "windows")))]+ unreachable!("WebGPU provider attempts are not constructed on this platform");}let coreml_cache_dir = attempt.coreml_cache_dir.clone();@@ -203,14 +211,26 @@ pub(crate) fn build_session(/// through to the next attempt. WebGPU attempts do not take this path; they/// are validated inside the crash-canary window.fn build_and_validate_session(model_path: &str, attempt: ProviderAttempt) -> MlResult<Session> {- #[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]+ #[cfg(any(+ target_os = "android",+ target_os = "ios",+ target_os = "linux",+ target_os = "macos",+ target_os = "windows"+ ))]let execution_provider = attempt.execution_provider;#[cfg_attr(not(any(target_os = "ios", target_os = "macos")), allow(unused_mut))]let mut session = match build_session_with_providers(model_path, attempt) {Ok(session) => session,Err(error) => {- #[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]+ #[cfg(any(+ target_os = "android",+ target_os = "ios",+ target_os = "linux",+ target_os = "macos",+ target_os = "windows"+ ))]record_provider_attempt_failure(execution_provider,model_path,@@ -235,7 +255,7 @@ fn build_and_validate_session(model_path: &str, attempt: ProviderAttempt) -> MlR/// recorded on disk. Soft failures (including self-test failures) return an/// error with the canary left armed (the drop keeps the counted attempt), and/// the caller falls through to the next execution provider.-#[cfg(target_os = "android")]+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]fn build_webgpu_session_with_canary(model_path: &str,model_namespace: &str,@@ -256,33 +276,36 @@ fn build_webgpu_session_with_canary(return Err(error);}};- // The adapter probe touches the Vulkan driver, so it runs inside the- // armed canary window: a probe crash is recorded like any other WebGPU- // crash.- match webgpu::check_adapter() {- webgpu::AdapterCheck::Allowed => {}- webgpu::AdapterCheck::Denied => {- // A completed probe that denies the adapter is a clean policy- // decision rather than a failed attempt, so the canary is- // disarmed.- canary.disarm();- return Err(MlError::Ort(- "WebGPU skipped: GPU adapter is not on the allowlist".to_string(),- ));- }- webgpu::AdapterCheck::Failed => {- // The canary stays armed so the drop records a failed attempt:- // a driver whose probe keeps failing quarantines like one that- // keeps crashing, and cannot reset the consecutive-failure- // counter of genuine crashes.- let error = MlError::Ort("WebGPU skipped: Vulkan adapter probe failed".to_string());- record_provider_attempt_failure(- ExecutionProvider::WebGpu,- model_path,- "adapter probe",- &error,- );- return Err(error);+ #[cfg(target_os = "android")]+ {+ // The Android adapter probe touches the Vulkan driver, so it runs+ // inside the armed canary window: a probe crash is recorded like any+ // other WebGPU crash.+ match webgpu::check_adapter() {+ webgpu::AdapterCheck::Allowed => {}+ webgpu::AdapterCheck::Denied => {+ // A completed probe that denies the adapter is a clean policy+ // decision rather than a failed attempt, so the canary is+ // disarmed.+ canary.disarm();+ return Err(MlError::Ort(+ "WebGPU skipped: GPU adapter is not on the allowlist".to_string(),+ ));+ }+ webgpu::AdapterCheck::Failed => {+ // The canary stays armed so the drop records a failed attempt:+ // a driver whose probe keeps failing quarantines like one that+ // keeps crashing, and cannot reset the consecutive-failure+ // counter of genuine crashes.+ let error = MlError::Ort("WebGPU skipped: Vulkan adapter probe failed".to_string());+ record_provider_attempt_failure(+ ExecutionProvider::WebGpu,+ model_path,+ "adapter probe",+ &error,+ );+ return Err(error);+ }}}let mut session = match build_session_with_providers(model_path, attempt) {@@ -302,7 +325,13 @@ fn build_webgpu_session_with_canary(Ok(session)}-#[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]+#[cfg(any(+ target_os = "android",+ target_os = "ios",+ target_os = "linux",+ target_os = "macos",+ target_os = "windows"+))]fn record_provider_attempt_failure(provider: ExecutionProvider,model_path: &str,@@ -314,7 +343,14 @@ fn record_provider_attempt_failure(}}-#[cfg(any(target_os = "android", target_os = "ios", target_os = "macos", test))]+#[cfg(any(+ target_os = "android",+ target_os = "ios",+ target_os = "linux",+ target_os = "macos",+ target_os = "windows",+ test+))]fn provider_attempt_failure_message(provider: ExecutionProvider,model_path: &str,@@ -336,7 +372,13 @@ fn provider_attempt_failure_message(/// Validates a freshly built accelerated session against the model's/// committed golden output. An error means the session must not be used; the/// caller falls through to the next execution provider attempt.-#[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]+#[cfg(any(+ target_os = "android",+ target_os = "ios",+ target_os = "linux",+ target_os = "macos",+ target_os = "windows"+))]fn run_session_self_test(model_path: &str,session: &mut Session,@@ -631,10 +673,40 @@ fn platform_default_attempts(attempts}+#[cfg(any(target_os = "linux", target_os = "windows"))]+fn platform_default_attempts(+ model_path: &str,+ _coreml_cache_namespace: &str,+) -> Vec<ProviderAttempt> {+ let mut attempts = Vec::new();+ if webgpu::attempt_permitted(model_path) && golden_entry_required(model_path, "WebGPU") {+ attempts.push(ProviderAttempt {+ // EP priority follows registration order. Unsupported WebGPU nodes+ // fall through to CPU in the same session.+ providers: vec![+ webgpu_provider(),+ CPU::default().with_arena_allocator(true).build(),+ ],+ disable_intra_op_spinning: true,+ coreml_cache_dir: None,+ uses_webgpu: true,+ execution_provider: ExecutionProvider::WebGpu,+ });+ }+ attempts.push(ProviderAttempt::cpu_only());+ attempts+}+/// Accelerated execution providers are only allowed for models with a/// committed golden self-test entry (fail closed). A miss for a production/// model means a model update shipped without regenerating `golden_data.rs`.-#[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]+#[cfg(any(+ target_os = "android",+ target_os = "ios",+ target_os = "linux",+ target_os = "macos",+ target_os = "windows"+))]fn golden_entry_required(model_path: &str, provider_label: &str) -> bool {if golden::lookup(model_path).is_some() {return true;@@ -649,16 +721,23 @@ fn golden_entry_required(model_path: &str, provider_label: &str) -> bool {false}-#[cfg(target_os = "android")]+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]fn webgpu_provider() -> ExecutionProviderDispatch {- WebGPU::default()- .with_dawn_backend_type(DawnBackendType::Vulkan)- .with_preferred_layout(PreferredLayout::NCHW)- .build()- .error_on_failure()+ let provider = WebGPU::default().with_preferred_layout(PreferredLayout::NCHW);+ #[cfg(any(target_os = "android", target_os = "linux"))]+ let provider = provider.with_dawn_backend_type(DawnBackendType::Vulkan);+ #[cfg(target_os = "windows")]+ let provider = provider.with_dawn_backend_type(DawnBackendType::D3D12);+ provider.build().error_on_failure()}-#[cfg(not(any(target_os = "ios", target_os = "android", target_os = "macos")))]+#[cfg(not(any(+ target_os = "ios",+ target_os = "android",+ target_os = "linux",+ target_os = "macos",+ target_os = "windows"+)))]fn platform_default_attempts(_model_path: &str,_coreml_cache_namespace: &str,diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rsindex 4dc2eae9ea..111632f42d 100644--- a/rust/crates/photos/src/ml/runtime.rs+++ b/rust/crates/photos/src/ml/runtime.rs@@ -293,7 +293,7 @@ impl MlRuntime {clip_text: ModelSlot::new(cpu_only, "clip-text"),// Pet models previously had device-specific FP16 driver failures.// Keep them CPU-only until they have been validated on the GPU- // execution providers of supported iOS and Android devices.+ // execution providers of all supported platforms.pet_face_detection: ModelSlot::new(cpu_only, "pet-face-detection"),pet_face_embedding_dog: ModelSlot::new(cpu_only, "pet-face-embedding-dog"),pet_face_embedding_cat: ModelSlot::new(cpu_only, "pet-face-embedding-cat"),@@ -558,7 +558,9 @@ fn should_retry_execution_provider_runtime(error: &MlError) -> bool {cfg!(any(target_os = "ios",target_os = "android",- target_os = "macos"+ target_os = "linux",+ target_os = "macos",+ target_os = "windows")) && is_execution_provider_failure(error)}diff --git a/rust/crates/photos/src/ml/webgpu.rs b/rust/crates/photos/src/ml/webgpu.rsindex 6ef585a219..907076a2a6 100644--- a/rust/crates/photos/src/ml/webgpu.rs+++ b/rust/crates/photos/src/ml/webgpu.rs@@ -1,11 +1,11 @@-//! Device-side guards for the Android WebGPU execution provider: a durable-//! crash canary and a GPU adapter allowlist.+//! Platform guards for the WebGPU execution provider: a durable crash canary+//! on Android, Linux and Windows, plus an Android GPU adapter allowlist.//!-//! The canary prevents an infinite crash loop on the rare device where-//! Dawn/Vulkan hard-crashes the process while a WebGPU session is built or-//! first dispatched. The allowlist only permits WebGPU on GPU vendors that-//! Chromium ships WebGPU to on Android, so Ente never runs Dawn on driver-//! stacks that Google's far larger fleet has not validated.+//! The canary prevents an infinite crash loop on machines where Dawn's native+//! backend hard-crashes the process while a WebGPU session is built or first+//! dispatched. On Android, the allowlist additionally permits WebGPU only on+//! GPU vendors that Chromium ships WebGPU to, so Ente never runs Dawn on+//! mobile driver stacks that Google's far larger fleet has not validated.//!//! Mechanics: immediately before a WebGPU session is built, a per-model//! counter file next to the model is incremented and fsynced ("armed"). After@@ -15,46 +15,65 @@//! [`MAX_CONSECUTIVE_FAILURES`], WebGPU is no longer attempted for models in//! that directory β durably, across process restarts.//!-//! A single interrupted attempt (e.g. Android killing the app mid-indexing)-//! is therefore tolerated: only consecutive failures with no intervening-//! success trip the breaker, which a genuine crash loop does deterministically-//! within a few launches.+//! A single interrupted attempt (e.g. the OS killing the app mid-indexing) is+//! therefore tolerated: only consecutive failures with no intervening success+//! trip the breaker, which a genuine crash loop does deterministically within+//! a few launches.//!//! Everything here is called from `onnx::build_session` while the caller//! holds the per-model slot lock, so accesses to one model's counter file are//! already serialized and no in-memory state is needed beyond the enable flag.+#[cfg(all(+ not(target_os = "windows"),+ any(target_os = "android", target_os = "linux", test)+))]+use std::fs::File;#[cfg(target_os = "android")]-use std::sync::{- OnceLock,- atomic::{AtomicBool, Ordering},-};-#[cfg(any(target_os = "android", test))]+use std::sync::OnceLock;+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]+use std::sync::atomic::{AtomicBool, Ordering};+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]use std::{- fs::{self, File, OpenOptions},+ fs::{self, OpenOptions},io::{self, Write},path::{Path, PathBuf},};/// Bump the version to discard all previous canary state, e.g. after a major/// ONNX Runtime or Dawn upgrade that warrants re-trialing quarantined devices.-#[cfg(any(target_os = "android", test))]+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]const CANARY_FILE_PREFIX: &str = ".ente-webgpu-canary-v1.";/// Number of consecutive failed or interrupted WebGPU attempts for one model/// after which WebGPU is durably disabled.-#[cfg(any(target_os = "android", test))]+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]const MAX_CONSECUTIVE_FAILURES: u32 = 3;-/// WebGPU is opt-in while the custom Android ONNX Runtime build is being-/// validated; the app-side policy also gates on device eligibility.-#[cfg(target_os = "android")]+/// WebGPU is opt-in at the app boundary. Android additionally gates this on+/// device eligibility; desktop enables it when its native addon is loaded.+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]static WEBGPU_ENABLED: AtomicBool = AtomicBool::new(false);pub(crate) fn set_enabled(enabled: bool) {- #[cfg(target_os = "android")]+ #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]WEBGPU_ENABLED.store(enabled, Ordering::Relaxed);- #[cfg(not(target_os = "android"))]+ #[cfg(not(any(target_os = "android", target_os = "linux", target_os = "windows")))]let _ = enabled;}@@ -66,7 +85,7 @@ pub(crate) fn set_enabled(enabled: bool) {/// adapter touches the Vulkan driver, so it must run inside the armed canary/// window (see `onnx::build_webgpu_session_with_canary`), where a probe crash/// is recorded like any other WebGPU crash.-#[cfg(target_os = "android")]+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]pub(crate) fn attempt_permitted(model_path: &str) -> bool {WEBGPU_ENABLED.load(Ordering::Relaxed)&& model_dir(model_path).is_some_and(|dir| !quarantined(&dir))@@ -175,7 +194,12 @@ fn probe_vulkan_vendor_ids() -> Result<Vec<u32>, String> {/// A durable record of an in-flight WebGPU attempt for one model. Dropping it/// without calling [`ArmedCanary::disarm`] leaves the attempt recorded as/// failed, which is exactly what a crash does implicitly.-#[cfg(any(target_os = "android", test))]+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]pub(crate) struct ArmedCanary {path: PathBuf,}@@ -183,7 +207,12 @@ pub(crate) struct ArmedCanary {/// Increments and fsyncs the model's consecutive-failure counter before a/// WebGPU attempt. On error the caller must skip WebGPU (fail closed): without/// a durable record, a crash during the attempt would go unnoticed.-#[cfg(any(target_os = "android", test))]+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]pub(crate) fn arm_canary(model_path: &str, model_namespace: &str) -> io::Result<ArmedCanary> {let dir = model_dir(model_path).ok_or_else(|| {io::Error::new(@@ -197,7 +226,12 @@ pub(crate) fn arm_canary(model_path: &str, model_namespace: &str) -> io::Result<Ok(ArmedCanary { path })}-#[cfg(any(target_os = "android", test))]+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]impl ArmedCanary {/// Marks the attempt as successful by removing the counter file, resetting/// the model's consecutive-failure count to zero.@@ -211,7 +245,12 @@ impl ArmedCanary {}}-#[cfg(any(target_os = "android", test))]+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]fn model_dir(model_path: &str) -> Option<PathBuf> {if model_path.trim().is_empty() {return None;@@ -224,7 +263,12 @@ fn model_dir(model_path: &str) -> Option<PathBuf> {/// True if any model's canary in this directory records too many consecutive/// failures. Unreadable state fails closed.-#[cfg(any(target_os = "android", test))]+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]fn quarantined(dir: &Path) -> bool {let entries = match fs::read_dir(dir) {Ok(entries) => entries,@@ -251,7 +295,12 @@ fn quarantined(dir: &Path) -> bool {/// A missing file means zero consecutive failures. Unparseable contents are an/// error so that callers fail closed.-#[cfg(any(target_os = "android", test))]+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]fn read_failure_count(path: &Path) -> io::Result<u32> {let contents = match fs::read_to_string(path) {Ok(contents) => contents,@@ -266,7 +315,12 @@ fn read_failure_count(path: &Path) -> io::Result<u32> {})}-#[cfg(any(target_os = "android", test))]+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]fn persist_failure_count(path: &Path, failures: u32) -> io::Result<()> {let mut file = OpenOptions::new().create(true)@@ -278,7 +332,12 @@ fn persist_failure_count(path: &Path, failures: u32) -> io::Result<()> {sync_parent(path)}-#[cfg(any(target_os = "android", test))]+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]fn remove_file_durably(path: &Path) -> io::Result<()> {match fs::remove_file(path) {Ok(()) => sync_parent(path),@@ -287,7 +346,12 @@ fn remove_file_durably(path: &Path) -> io::Result<()> {}}-#[cfg(any(target_os = "android", test))]+#[cfg(any(+ target_os = "android",+ target_os = "linux",+ target_os = "windows",+ test+))]fn sync_parent(path: &Path) -> io::Result<()> {let parent = path.parent().ok_or_else(|| {io::Error::new(@@ -295,7 +359,16 @@ fn sync_parent(path: &Path) -> io::Result<()> {"WebGPU canary path has no parent directory",)})?;- File::open(parent)?.sync_all()+ // Rust's standard library can open and fsync directories on Unix. Windows+ // does not expose an equivalent portable operation; the counter file+ // itself is still flushed above, which preserves the breaker semantics.+ #[cfg(not(target_os = "windows"))]+ return File::open(parent)?.sync_all();+ #[cfg(target_os = "windows")]+ {+ let _ = parent;+ Ok(())+ }}#[cfg(test)]diff --git a/web/apps/photos/tests/ml-index-flags.test.ts b/web/apps/photos/tests/ml-index-flags.test.tsnew file mode 100644index 0000000000..800eaac595--- /dev/null+++ b/web/apps/photos/tests/ml-index-flags.test.ts@@ -0,0 +1,18 @@+import { mlIndexFlagsForRustResult } from "ente-new/photos/services/ml/index-flags";+import { describe, expect, test } from "vitest";++describe("ML index runtime flags", () => {+ test.each([+ [false, false, 1],+ [true, false, 3],+ [false, true, 5],+ [true, true, 7],+ ])(+ "encodes CoreML=%s and WebGPU=%s as %d",+ (usedCoreml, usedWebgpu, expected) => {+ expect(mlIndexFlagsForRustResult({ usedCoreml, usedWebgpu })).toBe(+ expected,+ );+ },+ );+});diff --git a/web/packages/base/types/ipc.ts b/web/packages/base/types/ipc.tsindex 2cbe1c6269..27f17ff944 100644--- a/web/packages/base/types/ipc.ts+++ b/web/packages/base/types/ipc.ts@@ -859,7 +859,7 @@ export interface MLWorkerFaceResult {};blurValue: number;/** The face embedding (192 floating point values). */- embedding: number[];+ embedding: Float32Array<ArrayBuffer>;}/**@@ -883,7 +883,7 @@ export interface MLWorkerAnalyzeImageResult {/** Present when the request had {@link runClip} set. */clip?: {/** The (normalized) CLIP image embedding (512 floating point values). */- embedding: number[];+ embedding: Float32Array<ArrayBuffer>;};/** Pet recognition results (feature in development, ignored for now). */petFaces?: unknown;diff --git a/web/packages/new/photos/services/ml/clip.ts b/web/packages/new/photos/services/ml/clip.tsindex 0a6e2e4a29..fc5180dc2a 100644--- a/web/packages/new/photos/services/ml/clip.ts+++ b/web/packages/new/photos/services/ml/clip.ts@@ -72,6 +72,11 @@ export type RemoteCLIPIndex = CLIPIndex & {version: number;/** The UA for the client which generated this embedding. */client: string;+ /**+ * Bitmask describing the runtime that generated this index. An absent+ * value on legacy remote data is parsed as zero.+ */+ flags: number;};export type LocalCLIPIndex = CLIPIndex & {diff --git a/web/packages/new/photos/services/ml/face.ts b/web/packages/new/photos/services/ml/face.tsindex 925b7f3258..15a9f69512 100644--- a/web/packages/new/photos/services/ml/face.ts+++ b/web/packages/new/photos/services/ml/face.ts@@ -84,6 +84,11 @@ export type RemoteFaceIndex = FaceIndex & {* The UA for the client which generated this embedding.*/client: string;+ /**+ * Bitmask describing the runtime that generated this index. An absent+ * value on legacy remote data is parsed as zero.+ */+ flags: number;};export type LocalFaceIndex = FaceIndex & {diff --git a/web/packages/new/photos/services/ml/index-flags.ts b/web/packages/new/photos/services/ml/index-flags.tsnew file mode 100644index 0000000000..b07ec6cbea--- /dev/null+++ b/web/packages/new/photos/services/ml/index-flags.ts@@ -0,0 +1,21 @@+/**+ * Bits for the `flags` bitmask stored on remote face and CLIP indexes.+ *+ * These values are part of the cross-client wire format and must stay aligned+ * with `ml_versions.dart`. Once shipped, a bit's meaning is immutable.+ */+const mlIndexFlagRuntimeRust = 1 << 0;+const mlIndexFlagCoreML = 1 << 1;+const mlIndexFlagWebGPU = 1 << 2;++/** Flags describing the runtime that produced a Rust analysis result. */+export const mlIndexFlagsForRustResult = ({+ usedCoreml,+ usedWebgpu,+}: {+ usedCoreml: boolean;+ usedWebgpu: boolean;+}) =>+ mlIndexFlagRuntimeRust |+ (usedCoreml ? mlIndexFlagCoreML : 0) |+ (usedWebgpu ? mlIndexFlagWebGPU : 0);diff --git a/web/packages/new/photos/services/ml/ml-data.ts b/web/packages/new/photos/services/ml/ml-data.tsindex ab2801be2f..a2d1f40f03 100644--- a/web/packages/new/photos/services/ml/ml-data.ts+++ b/web/packages/new/photos/services/ml/ml-data.ts@@ -94,6 +94,7 @@ export type ParsedRemoteMLData = Partial<{const RemoteFaceIndex = z.object({version: z.number(),client: z.string(),+ flags: z.number().int().default(0),width: z.number(),height: z.number(),faces: z.array(@@ -123,6 +124,7 @@ const RemoteFaceIndex = z.object({const RemoteCLIPIndex = z.object({version: z.number(),client: z.string(),+ flags: z.number().int().default(0),embedding: z.array(z.number()),});diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.tsindex 283459e127..b8d1569c67 100644--- a/web/packages/new/photos/services/ml/worker.ts+++ b/web/packages/new/photos/services/ml/worker.ts@@ -46,6 +46,7 @@ import {updateAssumingLocalFiles,} from "./db";import { faceIndexingVersion, type FaceIndex } from "./face";+import { mlIndexFlagsForRustResult } from "./index-flags";import {fetchMLData,putMLData,@@ -665,27 +666,40 @@ const index = async (const faceIndex = existingFaceIndex ?? faceIndexFromAnalysisResult(result);const clipIndex = existingCLIPIndex ?? clipIndexFromAnalysisResult(result);+ const indexFlags = mlIndexFlagsForRustResult(result);log.debug(() => {const ms = Date.now() - startTime;const msg = [];if (runFaces) msg.push(`${faceIndex.faces.length} faces`);if (runClip) msg.push("clip");- const ep = result.usedCoreml ? " [coreml]" : "";+ const ep = result.usedCoreml+ ? " [coreml]"+ : result.usedWebgpu+ ? " [webgpu]"+ : " [cpu]";return `Indexed ${msg.join(" and ")} in ${f} (${ms} ms)${ep}`;});- const remoteFaceIndex = existingRemoteFaceIndex ?? {- version: faceIndexingVersion,- client: clientIdentifier,- ...faceIndex,- };-- const remoteCLIPIndex = existingRemoteCLIPIndex ?? {- version: clipIndexingVersion,- client: clientIdentifier,- ...clipIndex,- };+ const remoteFaceIndex =+ existingFaceIndex && existingRemoteFaceIndex+ ? existingRemoteFaceIndex+ : {+ version: faceIndexingVersion,+ client: clientIdentifier,+ flags: indexFlags,+ ...faceIndex,+ };++ const remoteCLIPIndex =+ existingCLIPIndex && existingRemoteCLIPIndex+ ? existingRemoteCLIPIndex+ : {+ version: clipIndexingVersion,+ client: clientIdentifier,+ flags: indexFlags,+ ...clipIndex,+ };// Perform an "upsert" by using the existing raw data we got from the// remote as the base, and inserting or overwriting only the parts we@@ -895,7 +909,11 @@ const faceIndexFromAnalysisResult = (},score: face.detection.score,blur: face.blurValue,- embedding: face.embedding,+ // Keep persisted and remote indexes as ordinary arrays. The+ // native/IPC boundary uses Float32Array to avoid widening and an+ // extra allocation, but JSON does not serialize typed arrays as+ // arrays.+ embedding: Array.from(face.embedding),})),};};@@ -911,5 +929,5 @@ const clipIndexFromAnalysisResult = (const embedding = result.clip?.embedding;if (!embedding)throw new Error("Analysis result is missing the CLIP embedding");- return { embedding };+ return { embedding: Array.from(embedding) };};