Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Enable guarded WebGPU for desktop ML

laurenspriemfd120281fd44
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.ts
index 88d975860c..01d6976b44 100644
--- a/desktop/src/main/services/ml-worker.ts
+++ b/desktop/src/main/services/ml-worker.ts
@@ -114,6 +114,9 @@ const loadMLNative = (paths: MLNativePaths) => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const native = require(paths.addon) as MLNative;
native.initOrt(paths.onnxRuntimeLibrary);
+ // Enables the guarded WebGPU path on Linux and Windows. This is a
+ // no-op on macOS, where CoreML remains the preferred provider.
+ native.setMlExecutionConfig(true);
_native = native;
log.debugString(`Loaded ML addon at ${paths.addon}`);
} catch (e) {
@@ -575,7 +578,7 @@ export const computeCLIPTextEmbeddingIfAvailable = async (text: string) => {
log.debugString(
`Rust ML CLIP text embedding took ${Date.now() - t} ms`,
);
- return Float32Array.from(embedding);
+ return embedding;
} finally {
logMLRuntimeEvents(native);
}
diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rs
index 50def15c90..c79e947f23 100644
--- a/rust/bindings/napi/photos/src/lib.rs
+++ b/rust/bindings/napi/photos/src/lib.rs
@@ -23,9 +23,8 @@ pub fn init_ort(dylib_path: String) -> Result<()> {
Ok(())
}
-/// Configures process-wide ML execution behavior. `enable_webgpu` only has an
-/// effect on Android; it is a no-op on desktop and exists for surface parity
-/// with the mobile bindings.
+/// Configures process-wide ML execution behavior. `enable_webgpu` enables the
+/// guarded WebGPU path on Linux and Windows; it is a no-op on macOS.
#[napi]
pub fn set_ml_execution_config(enable_webgpu: bool) {
shared_indexing::set_ml_execution_config(enable_webgpu);
@@ -94,12 +93,12 @@ pub struct FaceResult {
pub face_id: String,
pub detection: FaceDetection,
pub blur_value: f64,
- pub embedding: Vec<f64>,
+ pub embedding: Float32Array,
}
#[napi(object)]
pub struct ClipResult {
- pub embedding: Vec<f64>,
+ pub embedding: Float32Array,
}
#[napi(object)]
@@ -111,7 +110,7 @@ pub struct PetFaceResult {
pub keypoints: Vec<Vec<f64>>,
/// 0 = dog, 1 = cat.
pub species: u32,
- pub face_embedding: Vec<f64>,
+ pub face_embedding: Float32Array,
}
#[napi(object)]
@@ -121,7 +120,7 @@ pub struct PetBodyResult {
pub box_xyxy: Vec<f64>,
/// COCO class: 15 = cat, 16 = dog.
pub coco_class: u32,
- pub body_embedding: Vec<f64>,
+ pub body_embedding: Float32Array,
}
#[napi(object)]
@@ -199,7 +198,7 @@ pub struct RunClipTextRequest {
#[napi(object)]
pub struct RunClipTextResult {
- pub embedding: Vec<f64>,
+ pub embedding: Float32Array,
}
pub struct RunClipTextTask {
@@ -220,11 +219,7 @@ impl Task for RunClipTextTask {
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
Ok(RunClipTextResult {
- embedding: output
- .embedding
- .into_iter()
- .map(|value| value as f64)
- .collect(),
+ embedding: output.embedding.into(),
})
}
}
@@ -325,7 +320,7 @@ fn to_napi_analyze_image_result(result: shared_indexing::AnalyzeImageResult) ->
.face_crops
.map(|crops| crops.into_iter().map(Buffer::from).collect()),
clip: result.clip.map(|clip| ClipResult {
- embedding: to_f64_vec(clip.embedding),
+ embedding: clip.embedding.into(),
}),
pet_faces: result
.pet_faces
@@ -357,7 +352,7 @@ fn to_napi_face_result(result: shared_types::FaceResult) -> FaceResult {
.collect(),
},
blur_value: f64::from(result.blur_value),
- embedding: to_f64_vec(result.embedding),
+ embedding: result.embedding.into(),
}
}
@@ -378,7 +373,7 @@ fn to_napi_pet_face_result(result: shared_types::PetFaceResult) -> PetFaceResult
.map(|point| point.into_iter().map(f64::from).collect())
.collect(),
species: u32::from(result.species),
- face_embedding: to_f64_vec(result.face_embedding),
+ face_embedding: result.face_embedding.into(),
}
}
@@ -393,10 +388,6 @@ fn to_napi_pet_body_result(result: shared_types::PetBodyResult) -> PetBodyResult
.map(f64::from)
.collect(),
coco_class: u32::from(result.detection.coco_class),
- body_embedding: to_f64_vec(result.body_embedding),
+ body_embedding: result.body_embedding.into(),
}
}
-
-fn to_f64_vec(values: Vec<f32>) -> Vec<f64> {
- values.into_iter().map(f64::from).collect()
-}
diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.toml
index f2144cac69..7d008a55ae 100644
--- a/rust/crates/photos/Cargo.toml
+++ b/rust/crates/photos/Cargo.toml
@@ -62,7 +62,17 @@ ort = { workspace = true, features = [
"copy-dylibs",
] }
-[target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "macos")))'.dependencies]
+[target.'cfg(any(target_os = "linux", target_os = "windows"))'.dependencies]
+ort = { workspace = true, features = [
+ "ndarray",
+ "half",
+ "api-27",
+ "load-dynamic",
+ "copy-dylibs",
+ "webgpu",
+] }
+
+[target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "macos", target_os = "linux", target_os = "windows")))'.dependencies]
ort = { workspace = true, features = [
"ndarray",
"half",
diff --git a/rust/crates/photos/src/ml/indexing.rs b/rust/crates/photos/src/ml/indexing.rs
index 4c6b7bffb9..8664d4b529 100644
--- a/rust/crates/photos/src/ml/indexing.rs
+++ b/rust/crates/photos/src/ml/indexing.rs
@@ -67,9 +67,10 @@ pub struct RunClipTextResult {
/// Configures process-wide ML execution behavior. Must be called before the
/// first session is created to take effect for that session.
///
-/// `enable_webgpu` is the app-side eligibility decision for Android. Rust
-/// additionally applies its durable crash canary before attempting the WebGPU
-/// execution provider. It has no effect on other platforms.
+/// `enable_webgpu` is the app-side eligibility decision for Android, Linux and
+/// Windows. Rust additionally applies its durable crash canary before
+/// attempting the WebGPU execution provider. It has no effect on Apple
+/// platforms.
pub fn set_ml_execution_config(enable_webgpu: bool) {
webgpu::set_enabled(enable_webgpu);
}
diff --git a/rust/crates/photos/src/ml/onnx.rs b/rust/crates/photos/src/ml/onnx.rs
index d066bf1814..e3d886a9ba 100644
--- a/rust/crates/photos/src/ml/onnx.rs
+++ b/rust/crates/photos/src/ml/onnx.rs
@@ -10,14 +10,16 @@ use std::{
use half::prelude::{HalfFloatSliceExt, HalfFloatVecExt};
+#[cfg(target_os = "android")]
+use ort::ep::XNNPACK;
#[cfg(any(target_os = "ios", target_os = "macos"))]
use ort::ep::{
CoreML,
coreml::{ComputeUnits, ModelFormat, SpecializationStrategy},
};
-#[cfg(target_os = "android")]
+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
use ort::ep::{
- WebGPU, XNNPACK,
+ WebGPU,
webgpu::{DawnBackendType, PreferredLayout},
};
#[cfg(target_os = "android")]
@@ -26,9 +28,15 @@ use std::num::NonZeroUsize;
use crate::ml::error::{MlError, MlResult};
use crate::ml::events;
use crate::ml::golden;
-#[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "ios",
+ target_os = "linux",
+ target_os = "macos",
+ target_os = "windows"
+))]
use crate::ml::runtime::rt_log;
-#[cfg(target_os = "android")]
+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
use crate::ml::webgpu;
#[cfg(any(target_os = "ios", target_os = "macos", test))]
@@ -148,7 +156,7 @@ pub(crate) fn build_session(
for attempt in attempts {
let execution_provider = attempt.execution_provider;
if attempt.uses_webgpu {
- #[cfg(target_os = "android")]
+ #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
{
match build_webgpu_session_with_canary(model_path, coreml_cache_namespace, attempt)
{
@@ -157,8 +165,8 @@ pub(crate) fn build_session(
}
continue;
}
- #[cfg(not(target_os = "android"))]
- unreachable!("WebGPU provider attempts are only constructed on Android");
+ #[cfg(not(any(target_os = "android", target_os = "linux", target_os = "windows")))]
+ unreachable!("WebGPU provider attempts are not constructed on this platform");
}
let coreml_cache_dir = attempt.coreml_cache_dir.clone();
@@ -203,14 +211,26 @@ pub(crate) fn build_session(
/// through to the next attempt. WebGPU attempts do not take this path; they
/// are validated inside the crash-canary window.
fn build_and_validate_session(model_path: &str, attempt: ProviderAttempt) -> MlResult<Session> {
- #[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]
+ #[cfg(any(
+ target_os = "android",
+ target_os = "ios",
+ target_os = "linux",
+ target_os = "macos",
+ target_os = "windows"
+ ))]
let execution_provider = attempt.execution_provider;
#[cfg_attr(not(any(target_os = "ios", target_os = "macos")), allow(unused_mut))]
let mut session = match build_session_with_providers(model_path, attempt) {
Ok(session) => session,
Err(error) => {
- #[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]
+ #[cfg(any(
+ target_os = "android",
+ target_os = "ios",
+ target_os = "linux",
+ target_os = "macos",
+ target_os = "windows"
+ ))]
record_provider_attempt_failure(
execution_provider,
model_path,
@@ -235,7 +255,7 @@ fn build_and_validate_session(model_path: &str, attempt: ProviderAttempt) -> MlR
/// recorded on disk. Soft failures (including self-test failures) return an
/// error with the canary left armed (the drop keeps the counted attempt), and
/// the caller falls through to the next execution provider.
-#[cfg(target_os = "android")]
+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
fn build_webgpu_session_with_canary(
model_path: &str,
model_namespace: &str,
@@ -256,33 +276,36 @@ fn build_webgpu_session_with_canary(
return Err(error);
}
};
- // The adapter probe touches the Vulkan driver, so it runs inside the
- // armed canary window: a probe crash is recorded like any other WebGPU
- // crash.
- match webgpu::check_adapter() {
- webgpu::AdapterCheck::Allowed => {}
- webgpu::AdapterCheck::Denied => {
- // A completed probe that denies the adapter is a clean policy
- // decision rather than a failed attempt, so the canary is
- // disarmed.
- canary.disarm();
- return Err(MlError::Ort(
- "WebGPU skipped: GPU adapter is not on the allowlist".to_string(),
- ));
- }
- webgpu::AdapterCheck::Failed => {
- // The canary stays armed so the drop records a failed attempt:
- // a driver whose probe keeps failing quarantines like one that
- // keeps crashing, and cannot reset the consecutive-failure
- // counter of genuine crashes.
- let error = MlError::Ort("WebGPU skipped: Vulkan adapter probe failed".to_string());
- record_provider_attempt_failure(
- ExecutionProvider::WebGpu,
- model_path,
- "adapter probe",
- &error,
- );
- return Err(error);
+ #[cfg(target_os = "android")]
+ {
+ // The Android adapter probe touches the Vulkan driver, so it runs
+ // inside the armed canary window: a probe crash is recorded like any
+ // other WebGPU crash.
+ match webgpu::check_adapter() {
+ webgpu::AdapterCheck::Allowed => {}
+ webgpu::AdapterCheck::Denied => {
+ // A completed probe that denies the adapter is a clean policy
+ // decision rather than a failed attempt, so the canary is
+ // disarmed.
+ canary.disarm();
+ return Err(MlError::Ort(
+ "WebGPU skipped: GPU adapter is not on the allowlist".to_string(),
+ ));
+ }
+ webgpu::AdapterCheck::Failed => {
+ // The canary stays armed so the drop records a failed attempt:
+ // a driver whose probe keeps failing quarantines like one that
+ // keeps crashing, and cannot reset the consecutive-failure
+ // counter of genuine crashes.
+ let error = MlError::Ort("WebGPU skipped: Vulkan adapter probe failed".to_string());
+ record_provider_attempt_failure(
+ ExecutionProvider::WebGpu,
+ model_path,
+ "adapter probe",
+ &error,
+ );
+ return Err(error);
+ }
}
}
let mut session = match build_session_with_providers(model_path, attempt) {
@@ -302,7 +325,13 @@ fn build_webgpu_session_with_canary(
Ok(session)
}
-#[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "ios",
+ target_os = "linux",
+ target_os = "macos",
+ target_os = "windows"
+))]
fn record_provider_attempt_failure(
provider: ExecutionProvider,
model_path: &str,
@@ -314,7 +343,14 @@ fn record_provider_attempt_failure(
}
}
-#[cfg(any(target_os = "android", target_os = "ios", target_os = "macos", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "ios",
+ target_os = "linux",
+ target_os = "macos",
+ target_os = "windows",
+ test
+))]
fn provider_attempt_failure_message(
provider: ExecutionProvider,
model_path: &str,
@@ -336,7 +372,13 @@ fn provider_attempt_failure_message(
/// Validates a freshly built accelerated session against the model's
/// committed golden output. An error means the session must not be used; the
/// caller falls through to the next execution provider attempt.
-#[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "ios",
+ target_os = "linux",
+ target_os = "macos",
+ target_os = "windows"
+))]
fn run_session_self_test(
model_path: &str,
session: &mut Session,
@@ -631,10 +673,40 @@ fn platform_default_attempts(
attempts
}
+#[cfg(any(target_os = "linux", target_os = "windows"))]
+fn platform_default_attempts(
+ model_path: &str,
+ _coreml_cache_namespace: &str,
+) -> Vec<ProviderAttempt> {
+ let mut attempts = Vec::new();
+ if webgpu::attempt_permitted(model_path) && golden_entry_required(model_path, "WebGPU") {
+ attempts.push(ProviderAttempt {
+ // EP priority follows registration order. Unsupported WebGPU nodes
+ // fall through to CPU in the same session.
+ providers: vec![
+ webgpu_provider(),
+ CPU::default().with_arena_allocator(true).build(),
+ ],
+ disable_intra_op_spinning: true,
+ coreml_cache_dir: None,
+ uses_webgpu: true,
+ execution_provider: ExecutionProvider::WebGpu,
+ });
+ }
+ attempts.push(ProviderAttempt::cpu_only());
+ attempts
+}
+
/// Accelerated execution providers are only allowed for models with a
/// committed golden self-test entry (fail closed). A miss for a production
/// model means a model update shipped without regenerating `golden_data.rs`.
-#[cfg(any(target_os = "android", target_os = "ios", target_os = "macos"))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "ios",
+ target_os = "linux",
+ target_os = "macos",
+ target_os = "windows"
+))]
fn golden_entry_required(model_path: &str, provider_label: &str) -> bool {
if golden::lookup(model_path).is_some() {
return true;
@@ -649,16 +721,23 @@ fn golden_entry_required(model_path: &str, provider_label: &str) -> bool {
false
}
-#[cfg(target_os = "android")]
+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
fn webgpu_provider() -> ExecutionProviderDispatch {
- WebGPU::default()
- .with_dawn_backend_type(DawnBackendType::Vulkan)
- .with_preferred_layout(PreferredLayout::NCHW)
- .build()
- .error_on_failure()
+ let provider = WebGPU::default().with_preferred_layout(PreferredLayout::NCHW);
+ #[cfg(any(target_os = "android", target_os = "linux"))]
+ let provider = provider.with_dawn_backend_type(DawnBackendType::Vulkan);
+ #[cfg(target_os = "windows")]
+ let provider = provider.with_dawn_backend_type(DawnBackendType::D3D12);
+ provider.build().error_on_failure()
}
-#[cfg(not(any(target_os = "ios", target_os = "android", target_os = "macos")))]
+#[cfg(not(any(
+ target_os = "ios",
+ target_os = "android",
+ target_os = "linux",
+ target_os = "macos",
+ target_os = "windows"
+)))]
fn platform_default_attempts(
_model_path: &str,
_coreml_cache_namespace: &str,
diff --git a/rust/crates/photos/src/ml/runtime.rs b/rust/crates/photos/src/ml/runtime.rs
index 4dc2eae9ea..111632f42d 100644
--- a/rust/crates/photos/src/ml/runtime.rs
+++ b/rust/crates/photos/src/ml/runtime.rs
@@ -293,7 +293,7 @@ impl MlRuntime {
clip_text: ModelSlot::new(cpu_only, "clip-text"),
// Pet models previously had device-specific FP16 driver failures.
// Keep them CPU-only until they have been validated on the GPU
- // execution providers of supported iOS and Android devices.
+ // execution providers of all supported platforms.
pet_face_detection: ModelSlot::new(cpu_only, "pet-face-detection"),
pet_face_embedding_dog: ModelSlot::new(cpu_only, "pet-face-embedding-dog"),
pet_face_embedding_cat: ModelSlot::new(cpu_only, "pet-face-embedding-cat"),
@@ -558,7 +558,9 @@ fn should_retry_execution_provider_runtime(error: &MlError) -> bool {
cfg!(any(
target_os = "ios",
target_os = "android",
- target_os = "macos"
+ target_os = "linux",
+ target_os = "macos",
+ target_os = "windows"
)) && is_execution_provider_failure(error)
}
diff --git a/rust/crates/photos/src/ml/webgpu.rs b/rust/crates/photos/src/ml/webgpu.rs
index 6ef585a219..907076a2a6 100644
--- a/rust/crates/photos/src/ml/webgpu.rs
+++ b/rust/crates/photos/src/ml/webgpu.rs
@@ -1,11 +1,11 @@
-//! Device-side guards for the Android WebGPU execution provider: a durable
-//! crash canary and a GPU adapter allowlist.
+//! Platform guards for the WebGPU execution provider: a durable crash canary
+//! on Android, Linux and Windows, plus an Android GPU adapter allowlist.
//!
-//! The canary prevents an infinite crash loop on the rare device where
-//! Dawn/Vulkan hard-crashes the process while a WebGPU session is built or
-//! first dispatched. The allowlist only permits WebGPU on GPU vendors that
-//! Chromium ships WebGPU to on Android, so Ente never runs Dawn on driver
-//! stacks that Google's far larger fleet has not validated.
+//! The canary prevents an infinite crash loop on machines where Dawn's native
+//! backend hard-crashes the process while a WebGPU session is built or first
+//! dispatched. On Android, the allowlist additionally permits WebGPU only on
+//! GPU vendors that Chromium ships WebGPU to, so Ente never runs Dawn on
+//! mobile driver stacks that Google's far larger fleet has not validated.
//!
//! Mechanics: immediately before a WebGPU session is built, a per-model
//! counter file next to the model is incremented and fsynced ("armed"). After
@@ -15,46 +15,65 @@
//! [`MAX_CONSECUTIVE_FAILURES`], WebGPU is no longer attempted for models in
//! that directory β€” durably, across process restarts.
//!
-//! A single interrupted attempt (e.g. Android killing the app mid-indexing)
-//! is therefore tolerated: only consecutive failures with no intervening
-//! success trip the breaker, which a genuine crash loop does deterministically
-//! within a few launches.
+//! A single interrupted attempt (e.g. the OS killing the app mid-indexing) is
+//! therefore tolerated: only consecutive failures with no intervening success
+//! trip the breaker, which a genuine crash loop does deterministically within
+//! a few launches.
//!
//! Everything here is called from `onnx::build_session` while the caller
//! holds the per-model slot lock, so accesses to one model's counter file are
//! already serialized and no in-memory state is needed beyond the enable flag.
+#[cfg(all(
+ not(target_os = "windows"),
+ any(target_os = "android", target_os = "linux", test)
+))]
+use std::fs::File;
#[cfg(target_os = "android")]
-use std::sync::{
- OnceLock,
- atomic::{AtomicBool, Ordering},
-};
-#[cfg(any(target_os = "android", test))]
+use std::sync::OnceLock;
+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
+use std::sync::atomic::{AtomicBool, Ordering};
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
use std::{
- fs::{self, File, OpenOptions},
+ fs::{self, OpenOptions},
io::{self, Write},
path::{Path, PathBuf},
};
/// Bump the version to discard all previous canary state, e.g. after a major
/// ONNX Runtime or Dawn upgrade that warrants re-trialing quarantined devices.
-#[cfg(any(target_os = "android", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
const CANARY_FILE_PREFIX: &str = ".ente-webgpu-canary-v1.";
/// Number of consecutive failed or interrupted WebGPU attempts for one model
/// after which WebGPU is durably disabled.
-#[cfg(any(target_os = "android", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
const MAX_CONSECUTIVE_FAILURES: u32 = 3;
-/// WebGPU is opt-in while the custom Android ONNX Runtime build is being
-/// validated; the app-side policy also gates on device eligibility.
-#[cfg(target_os = "android")]
+/// WebGPU is opt-in at the app boundary. Android additionally gates this on
+/// device eligibility; desktop enables it when its native addon is loaded.
+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
static WEBGPU_ENABLED: AtomicBool = AtomicBool::new(false);
pub(crate) fn set_enabled(enabled: bool) {
- #[cfg(target_os = "android")]
+ #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
WEBGPU_ENABLED.store(enabled, Ordering::Relaxed);
- #[cfg(not(target_os = "android"))]
+ #[cfg(not(any(target_os = "android", target_os = "linux", target_os = "windows")))]
let _ = enabled;
}
@@ -66,7 +85,7 @@ pub(crate) fn set_enabled(enabled: bool) {
/// adapter touches the Vulkan driver, so it must run inside the armed canary
/// window (see `onnx::build_webgpu_session_with_canary`), where a probe crash
/// is recorded like any other WebGPU crash.
-#[cfg(target_os = "android")]
+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
pub(crate) fn attempt_permitted(model_path: &str) -> bool {
WEBGPU_ENABLED.load(Ordering::Relaxed)
&& model_dir(model_path).is_some_and(|dir| !quarantined(&dir))
@@ -175,7 +194,12 @@ fn probe_vulkan_vendor_ids() -> Result<Vec<u32>, String> {
/// A durable record of an in-flight WebGPU attempt for one model. Dropping it
/// without calling [`ArmedCanary::disarm`] leaves the attempt recorded as
/// failed, which is exactly what a crash does implicitly.
-#[cfg(any(target_os = "android", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
pub(crate) struct ArmedCanary {
path: PathBuf,
}
@@ -183,7 +207,12 @@ pub(crate) struct ArmedCanary {
/// Increments and fsyncs the model's consecutive-failure counter before a
/// WebGPU attempt. On error the caller must skip WebGPU (fail closed): without
/// a durable record, a crash during the attempt would go unnoticed.
-#[cfg(any(target_os = "android", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
pub(crate) fn arm_canary(model_path: &str, model_namespace: &str) -> io::Result<ArmedCanary> {
let dir = model_dir(model_path).ok_or_else(|| {
io::Error::new(
@@ -197,7 +226,12 @@ pub(crate) fn arm_canary(model_path: &str, model_namespace: &str) -> io::Result<
Ok(ArmedCanary { path })
}
-#[cfg(any(target_os = "android", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
impl ArmedCanary {
/// Marks the attempt as successful by removing the counter file, resetting
/// the model's consecutive-failure count to zero.
@@ -211,7 +245,12 @@ impl ArmedCanary {
}
}
-#[cfg(any(target_os = "android", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
fn model_dir(model_path: &str) -> Option<PathBuf> {
if model_path.trim().is_empty() {
return None;
@@ -224,7 +263,12 @@ fn model_dir(model_path: &str) -> Option<PathBuf> {
/// True if any model's canary in this directory records too many consecutive
/// failures. Unreadable state fails closed.
-#[cfg(any(target_os = "android", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
fn quarantined(dir: &Path) -> bool {
let entries = match fs::read_dir(dir) {
Ok(entries) => entries,
@@ -251,7 +295,12 @@ fn quarantined(dir: &Path) -> bool {
/// A missing file means zero consecutive failures. Unparseable contents are an
/// error so that callers fail closed.
-#[cfg(any(target_os = "android", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
fn read_failure_count(path: &Path) -> io::Result<u32> {
let contents = match fs::read_to_string(path) {
Ok(contents) => contents,
@@ -266,7 +315,12 @@ fn read_failure_count(path: &Path) -> io::Result<u32> {
})
}
-#[cfg(any(target_os = "android", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
fn persist_failure_count(path: &Path, failures: u32) -> io::Result<()> {
let mut file = OpenOptions::new()
.create(true)
@@ -278,7 +332,12 @@ fn persist_failure_count(path: &Path, failures: u32) -> io::Result<()> {
sync_parent(path)
}
-#[cfg(any(target_os = "android", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
fn remove_file_durably(path: &Path) -> io::Result<()> {
match fs::remove_file(path) {
Ok(()) => sync_parent(path),
@@ -287,7 +346,12 @@ fn remove_file_durably(path: &Path) -> io::Result<()> {
}
}
-#[cfg(any(target_os = "android", test))]
+#[cfg(any(
+ target_os = "android",
+ target_os = "linux",
+ target_os = "windows",
+ test
+))]
fn sync_parent(path: &Path) -> io::Result<()> {
let parent = path.parent().ok_or_else(|| {
io::Error::new(
@@ -295,7 +359,16 @@ fn sync_parent(path: &Path) -> io::Result<()> {
"WebGPU canary path has no parent directory",
)
})?;
- File::open(parent)?.sync_all()
+ // Rust's standard library can open and fsync directories on Unix. Windows
+ // does not expose an equivalent portable operation; the counter file
+ // itself is still flushed above, which preserves the breaker semantics.
+ #[cfg(not(target_os = "windows"))]
+ return File::open(parent)?.sync_all();
+ #[cfg(target_os = "windows")]
+ {
+ let _ = parent;
+ Ok(())
+ }
}
#[cfg(test)]
diff --git a/web/apps/photos/tests/ml-index-flags.test.ts b/web/apps/photos/tests/ml-index-flags.test.ts
new file mode 100644
index 0000000000..800eaac595
--- /dev/null
+++ b/web/apps/photos/tests/ml-index-flags.test.ts
@@ -0,0 +1,18 @@
+import { mlIndexFlagsForRustResult } from "ente-new/photos/services/ml/index-flags";
+import { describe, expect, test } from "vitest";
+
+describe("ML index runtime flags", () => {
+ test.each([
+ [false, false, 1],
+ [true, false, 3],
+ [false, true, 5],
+ [true, true, 7],
+ ])(
+ "encodes CoreML=%s and WebGPU=%s as %d",
+ (usedCoreml, usedWebgpu, expected) => {
+ expect(mlIndexFlagsForRustResult({ usedCoreml, usedWebgpu })).toBe(
+ expected,
+ );
+ },
+ );
+});
diff --git a/web/packages/base/types/ipc.ts b/web/packages/base/types/ipc.ts
index 2cbe1c6269..27f17ff944 100644
--- a/web/packages/base/types/ipc.ts
+++ b/web/packages/base/types/ipc.ts
@@ -859,7 +859,7 @@ export interface MLWorkerFaceResult {
};
blurValue: number;
/** The face embedding (192 floating point values). */
- embedding: number[];
+ embedding: Float32Array<ArrayBuffer>;
}
/**
@@ -883,7 +883,7 @@ export interface MLWorkerAnalyzeImageResult {
/** Present when the request had {@link runClip} set. */
clip?: {
/** The (normalized) CLIP image embedding (512 floating point values). */
- embedding: number[];
+ embedding: Float32Array<ArrayBuffer>;
};
/** Pet recognition results (feature in development, ignored for now). */
petFaces?: unknown;
diff --git a/web/packages/new/photos/services/ml/clip.ts b/web/packages/new/photos/services/ml/clip.ts
index 0a6e2e4a29..fc5180dc2a 100644
--- a/web/packages/new/photos/services/ml/clip.ts
+++ b/web/packages/new/photos/services/ml/clip.ts
@@ -72,6 +72,11 @@ export type RemoteCLIPIndex = CLIPIndex & {
version: number;
/** The UA for the client which generated this embedding. */
client: string;
+ /**
+ * Bitmask describing the runtime that generated this index. An absent
+ * value on legacy remote data is parsed as zero.
+ */
+ flags: number;
};
export type LocalCLIPIndex = CLIPIndex & {
diff --git a/web/packages/new/photos/services/ml/face.ts b/web/packages/new/photos/services/ml/face.ts
index 925b7f3258..15a9f69512 100644
--- a/web/packages/new/photos/services/ml/face.ts
+++ b/web/packages/new/photos/services/ml/face.ts
@@ -84,6 +84,11 @@ export type RemoteFaceIndex = FaceIndex & {
* The UA for the client which generated this embedding.
*/
client: string;
+ /**
+ * Bitmask describing the runtime that generated this index. An absent
+ * value on legacy remote data is parsed as zero.
+ */
+ flags: number;
};
export type LocalFaceIndex = FaceIndex & {
diff --git a/web/packages/new/photos/services/ml/index-flags.ts b/web/packages/new/photos/services/ml/index-flags.ts
new file mode 100644
index 0000000000..b07ec6cbea
--- /dev/null
+++ b/web/packages/new/photos/services/ml/index-flags.ts
@@ -0,0 +1,21 @@
+/**
+ * Bits for the `flags` bitmask stored on remote face and CLIP indexes.
+ *
+ * These values are part of the cross-client wire format and must stay aligned
+ * with `ml_versions.dart`. Once shipped, a bit's meaning is immutable.
+ */
+const mlIndexFlagRuntimeRust = 1 << 0;
+const mlIndexFlagCoreML = 1 << 1;
+const mlIndexFlagWebGPU = 1 << 2;
+
+/** Flags describing the runtime that produced a Rust analysis result. */
+export const mlIndexFlagsForRustResult = ({
+ usedCoreml,
+ usedWebgpu,
+}: {
+ usedCoreml: boolean;
+ usedWebgpu: boolean;
+}) =>
+ mlIndexFlagRuntimeRust |
+ (usedCoreml ? mlIndexFlagCoreML : 0) |
+ (usedWebgpu ? mlIndexFlagWebGPU : 0);
diff --git a/web/packages/new/photos/services/ml/ml-data.ts b/web/packages/new/photos/services/ml/ml-data.ts
index ab2801be2f..a2d1f40f03 100644
--- a/web/packages/new/photos/services/ml/ml-data.ts
+++ b/web/packages/new/photos/services/ml/ml-data.ts
@@ -94,6 +94,7 @@ export type ParsedRemoteMLData = Partial<{
const RemoteFaceIndex = z.object({
version: z.number(),
client: z.string(),
+ flags: z.number().int().default(0),
width: z.number(),
height: z.number(),
faces: z.array(
@@ -123,6 +124,7 @@ const RemoteFaceIndex = z.object({
const RemoteCLIPIndex = z.object({
version: z.number(),
client: z.string(),
+ flags: z.number().int().default(0),
embedding: z.array(z.number()),
});
diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts
index 283459e127..b8d1569c67 100644
--- a/web/packages/new/photos/services/ml/worker.ts
+++ b/web/packages/new/photos/services/ml/worker.ts
@@ -46,6 +46,7 @@ import {
updateAssumingLocalFiles,
} from "./db";
import { faceIndexingVersion, type FaceIndex } from "./face";
+import { mlIndexFlagsForRustResult } from "./index-flags";
import {
fetchMLData,
putMLData,
@@ -665,27 +666,40 @@ const index = async (
const faceIndex = existingFaceIndex ?? faceIndexFromAnalysisResult(result);
const clipIndex = existingCLIPIndex ?? clipIndexFromAnalysisResult(result);
+ const indexFlags = mlIndexFlagsForRustResult(result);
log.debug(() => {
const ms = Date.now() - startTime;
const msg = [];
if (runFaces) msg.push(`${faceIndex.faces.length} faces`);
if (runClip) msg.push("clip");
- const ep = result.usedCoreml ? " [coreml]" : "";
+ const ep = result.usedCoreml
+ ? " [coreml]"
+ : result.usedWebgpu
+ ? " [webgpu]"
+ : " [cpu]";
return `Indexed ${msg.join(" and ")} in ${f} (${ms} ms)${ep}`;
});
- const remoteFaceIndex = existingRemoteFaceIndex ?? {
- version: faceIndexingVersion,
- client: clientIdentifier,
- ...faceIndex,
- };
-
- const remoteCLIPIndex = existingRemoteCLIPIndex ?? {
- version: clipIndexingVersion,
- client: clientIdentifier,
- ...clipIndex,
- };
+ const remoteFaceIndex =
+ existingFaceIndex && existingRemoteFaceIndex
+ ? existingRemoteFaceIndex
+ : {
+ version: faceIndexingVersion,
+ client: clientIdentifier,
+ flags: indexFlags,
+ ...faceIndex,
+ };
+
+ const remoteCLIPIndex =
+ existingCLIPIndex && existingRemoteCLIPIndex
+ ? existingRemoteCLIPIndex
+ : {
+ version: clipIndexingVersion,
+ client: clientIdentifier,
+ flags: indexFlags,
+ ...clipIndex,
+ };
// Perform an "upsert" by using the existing raw data we got from the
// remote as the base, and inserting or overwriting only the parts we
@@ -895,7 +909,11 @@ const faceIndexFromAnalysisResult = (
},
score: face.detection.score,
blur: face.blurValue,
- embedding: face.embedding,
+ // Keep persisted and remote indexes as ordinary arrays. The
+ // native/IPC boundary uses Float32Array to avoid widening and an
+ // extra allocation, but JSON does not serialize typed arrays as
+ // arrays.
+ embedding: Array.from(face.embedding),
})),
};
};
@@ -911,5 +929,5 @@ const clipIndexFromAnalysisResult = (
const embedding = result.clip?.embedding;
if (!embedding)
throw new Error("Analysis result is missing the CLIP embedding");
- return { embedding };
+ return { embedding: Array.from(embedding) };
};