π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Merge upstream/main into ort_opt_ios
laurenspriem95246d9e5ef0
diff --cc mobile/apps/photos/ios/Podfile.lockindex 32d3589ac8,8785bcd351..883ec2b808--- a/mobile/apps/photos/ios/Podfile.lock+++ b/mobile/apps/photos/ios/Podfile.lock@@@ -542,6 -537,6 +542,6 @@@ SPEC CHECKSUMSwakelock_plus: e29112ab3ef0b318e58cfa5c32326458be66b556workmanager_apple: 904529ae31e97fc5be632cf628507652294a0778-PODFILE CHECKSUM: 980a2602621422bdfd82e34ef370d44ddd600fad+PODFILE CHECKSUM: 7af4c0a38ff14aa9b6ef0b1309156a8d783a20c6- COCOAPODS: 1.16.2+ COCOAPODS: 1.17.0diff --cc mobile/apps/photos/lib/utils/ml_util.dartindex 4e429d51b0,5f6e2e8975..5a6e4947a4--- a/mobile/apps/photos/lib/utils/ml_util.dart+++ b/mobile/apps/photos/lib/utils/ml_util.dart@@@ -872,55 -877,16 +871,22 @@@ Future<MLResult> analyzeImageRust(Map arethrow;}- _logger.warning(- "Rust decode failed for fileID $enteFileID (format: $fileFormat), retrying with JPEG fallback",+ // A Rust decode error may be a resource-limit rejection. Retrying it+ // through an unbounded Dart decoder can exhaust the mobile process.+ throw _asInvalidImageFormatExceptionForRustDecodeFailure(+ enteFileID: enteFileID,+ fileFormat: fileFormat,+ primaryError: e,);- final _DecodeFallbackFile? fallback;- try {- fallback = await _createJpegDecodeFallbackFile(imagePath: imagePath);- } catch (fallbackError, fallbackStack) {- _logger.severe(- "JPEG fallback conversion threw for fileID $enteFileID (format: $fileFormat)",- fallbackError,- fallbackStack,- );- rethrow;- }- if (fallback == null) {- _logger.warning(- "JPEG fallback conversion returned null/empty bytes for fileID $enteFileID (format: $fileFormat); storing empty result instead",- );- throw _asInvalidImageFormatExceptionForRustDecodeFailure(- enteFileID: enteFileID,- fileFormat: fileFormat,- primaryError: e,- );- }-- try {- rustResult = await runRustAnalyzeForPath(fallback.file.path);- _logger.info(- "Rust decode fallback succeeded for fileID $enteFileID (original format: $fileFormat)",- );- } catch (retryError, retryStack) {- _logger.severe(- "Rust decode fallback retry failed for fileID $enteFileID (original format: $fileFormat)",- retryError,- retryStack,- );- rethrow;- } finally {- await _cleanupDecodeFallback(fallback);- }}- final result = MLResult.fromEnteFileID(enteFileID);+ final result = MLResult.fromEnteFileID(+ enteFileID,+ remoteFlags:+ mlIndexFlagRuntimeRust |+ (rustResult.usedCoreml ? mlIndexFlagCoreML : 0) |+ (rustResult.usedWebgpu ? mlIndexFlagWebGPU : 0),+ );result.decodedImageSize = Dimensions(width: rustResult.decodedImageSize.width,height: rustResult.decodedImageSize.height,diff --cc rust/Cargo.lockindex 26d3b71de9,2ca8fc1747..3aa16acda8--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@@ -2385,14 -2378,11 +2387,12 @@@ version = "0.0.0dependencies = ["ente-core","ente-image",- "flate2",- "futures-util",+ "ente-model-download","llama-cpp-2","ort",+ "ort-sys","parking_lot","regex",- "reqwest 0.12.28","rubato","rusqlite","self_cell",@@@ -2435,8 -2454,8 +2464,9 @@@ name = "ente-photosversion = "0.0.0"dependencies = ["anyhow",+ "ash","ente-image",+ "ente-model-download","fast_image_resize","flate2","half",diff --cc rust/Cargo.tomlindex 489ee6f417,d59402326d..3898dec139--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@@ -26,12 -30,15 +30,16 @@@ ente-accounts = { path = "crates/accounente-contacts = { path = "crates/contacts" }ente-core = { path = "crates/core" }ente-image = { path = "crates/image" }+ ente-model-download = { path = "crates/model-download" }+ ente-paste = { path = "crates/paste" }ente-photos = { path = "crates/photos" }ente-rs = { path = "apps/cli" }+ ente-space = { path = "space" }ente-test-support = { path = "crates/test-support" }ort = { git = "https://github.com/pykeio/ort.git", rev = "31c8f1443bb579bfddb82705288347697e468fb3", default-features = false }+ort-sys = { git = "https://github.com/pykeio/ort.git", rev = "31c8f1443bb579bfddb82705288347697e468fb3", default-features = false }serde = { version = "1.0.228", features = ["derive"] }+ serde_json = "1.0.149"thiserror = "2.0.18"uniffi = { version = "0.31.0" }wasm-bindgen = "0.2.125"diff --cc rust/crates/ensu/Cargo.tomlindex 3d9eb1731a,4212755216..f799541413--- a/rust/crates/ensu/Cargo.toml+++ b/rust/crates/ensu/Cargo.toml@@@ -62,12 -55,4 +60,10 @@@ ort-sys = { workspace = true, features[features]default = ["sqlite"]sqlite = ["dep:rusqlite"]-transcription = ["dep:ort", "dep:regex", "dep:rubato", "dep:transcribe-rs"]+transcription = [- "dep:flate2",+ "dep:ort",+ "dep:ort-sys",+ "dep:regex",+ "dep:rubato",- "dep:tar",+ "dep:transcribe-rs",+]diff --cc rust/crates/photos/Cargo.tomlindex 783af6a314,8f6af9bcf2..7f34340fad--- a/rust/crates/photos/Cargo.toml+++ b/rust/crates/photos/Cargo.toml@@@ -50,24 -54,16 +50,25 @@@ ort = { workspace = true, features ="xnnpack",] }+[target.'cfg(not(any(target_os = "ios", target_os = "android")))'.dependencies]+ort = { workspace = true, features = [+ "ndarray",+ "half",+ "api-27",+ "load-dynamic",+ "copy-dylibs",+] }+[dev-dependencies]anyhow = "1"+ ente-model-download.workspace = trueflate2 = "1.1"- reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls-webpki-roots"] }serde = { version = "1", features = ["derive"] }- serde_json = "1"- sha2 = "0.10"+ serde_json.workspace = truetar = "0.4"tempfile = "3.12"+ tokio = { version = "1", features = ["rt"] }+ url = "2.5.8"[[test]]name = "ml_indexing"diff --cc rust/crates/photos/tests/support/ml_indexing.rsindex e1b7b61301,61a4a04cca..df2d7f5953--- a/rust/crates/photos/tests/support/ml_indexing.rs+++ b/rust/crates/photos/tests/support/ml_indexing.rs@@@ -168,12 -164,37 +164,37 @@@ fn should_print_stats() -> boolpub(crate) struct MlIndexingTestContext {asset_lock: AssetLock,cache_dir: PathBuf,- client: Client,golden_results: HashMap<String, ComparableResult>,manifest: FixtureManifest,- runtime_config: MlRuntimeConfig,+ model_paths: ModelPaths,}+ struct AssetFetcher {+ runtime: tokio::runtime::Runtime,+ downloader: download::Downloader,+ }++ impl AssetFetcher {+ fn new() -> Result<Self> {+ Ok(Self {+ runtime: tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .context("build download runtime")?,+ downloader: download::Downloader::new().context("build downloader")?,+ })+ }++ fn fetch(&self, targets: Vec<download::Target>) -> Result<()> {+ self.runtime.block_on(self.downloader.download(+ targets,+ |_| {},+ download::CancellationToken::default(),+ ))?;+ Ok(())+ }+ }+impl MlIndexingTestContext {pub(crate) fn load() -> Result<Self> {let repo_root = repo_root()?;@@@ -201,15 -225,22 +225,14 @@@.context("load ONNX Runtime dynamic library")?.commit();- let model_paths = resolve_model_paths(&client, &cache_dir, &asset_lock.models)?;- let runtime_config = MlRuntimeConfig {- model_paths: resolve_model_paths(&fetcher, &cache_dir, &asset_lock.models)?,- provider_policy: ExecutionProviderPolicy {- prefer_coreml: false,- prefer_nnapi: false,- prefer_xnnpack: false,- allow_cpu_fallback: true,- },- };++ let model_paths = resolve_model_paths(&fetcher, &cache_dir, &asset_lock.models)?;Ok(Self {asset_lock,cache_dir,- client,golden_results,manifest,- runtime_config,+ model_paths,})}@@@ -362,71 -388,8 +380,68 @@@}}+/// A downloaded production model together with its content hash as pinned in+/// the asset lock, for embedding into generated golden entries.+#[allow(dead_code)]+pub(crate) struct GoldenModelAsset {+ pub(crate) path: PathBuf,+ pub(crate) sha256: String,+}++/// Downloaded production model assets for the golden self-test tooling, with+/// ONNX Runtime already initialized. Used by the ml_goldens test binary only.+#[allow(dead_code)]+pub(crate) struct GoldenTestAssets {+ pub(crate) face_detection: GoldenModelAsset,+ pub(crate) face_embedding: GoldenModelAsset,+ pub(crate) clip_image: GoldenModelAsset,+ pub(crate) clip_text: GoldenModelAsset,+ pub(crate) clip_text_vocab: PathBuf,+}++#[allow(dead_code)]+impl GoldenTestAssets {+ pub(crate) fn load() -> Result<Self> {+ let repo_root = repo_root()?;+ let asset_lock = load_asset_lock(&repo_root)?;+ let cache_dir = cache_dir(&repo_root);- let client = Client::builder()- .user_agent("ente-rust-ml-goldens-test")- .build()- .context("build HTTP client")?;++ let fetcher = AssetFetcher::new()?;++ let onnx_runtime_library =- resolve_onnx_runtime_library(&client, &cache_dir, &asset_lock.onnx_runtime)?;++ resolve_onnx_runtime_library(&fetcher, &cache_dir, &asset_lock.onnx_runtime)?;+ let _ = ort::init_from(&onnx_runtime_library)+ .context("load ONNX Runtime dynamic library")?+ .commit();++ let golden_model = |label: &str, model: &ModelAsset| -> Result<GoldenModelAsset> {+ Ok(GoldenModelAsset {- path: resolve_model_asset(&client, &cache_dir, label, model)?,++ path: resolve_model_asset(&fetcher, &cache_dir, label, model)?,+ sha256: model.sha256.clone(),+ })+ };++ let models = &asset_lock.models;+ Ok(Self {+ face_detection: golden_model("face-detection", &models.face_detection)?,+ face_embedding: golden_model("face-embedding", &models.face_embedding)?,+ clip_image: golden_model("clip-image", &models.clip_image)?,+ clip_text: golden_model("clip-text", &models.clip_text)?,+ clip_text_vocab: resolve_model_asset(- &client,++ &fetcher,+ &cache_dir,+ "clip-text-vocab",+ &models.clip_text_vocab,+ )?,+ })+ }++ pub(crate) fn golden_data_path() -> Result<PathBuf> {+ Ok(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/ml/golden_data.rs"))+ }+}+pub(crate) struct PreparedMlRuntime {- config: MlRuntimeConfig,+ model_paths: ModelPaths,}impl PreparedMlRuntime {