Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Merge upstream/main into ort_opt_ios

laurenspriem95246d9e5ef0
diff --cc mobile/apps/photos/ios/Podfile.lock
index 32d3589ac8,8785bcd351..883ec2b808
--- a/mobile/apps/photos/ios/Podfile.lock
+++ b/mobile/apps/photos/ios/Podfile.lock
@@@ -542,6 -537,6 +542,6 @@@ SPEC CHECKSUMS
wakelock_plus: e29112ab3ef0b318e58cfa5c32326458be66b556
workmanager_apple: 904529ae31e97fc5be632cf628507652294a0778
-PODFILE CHECKSUM: 980a2602621422bdfd82e34ef370d44ddd600fad
+PODFILE CHECKSUM: 7af4c0a38ff14aa9b6ef0b1309156a8d783a20c6
- COCOAPODS: 1.16.2
+ COCOAPODS: 1.17.0
diff --cc mobile/apps/photos/lib/utils/ml_util.dart
index 4e429d51b0,5f6e2e8975..5a6e4947a4
--- a/mobile/apps/photos/lib/utils/ml_util.dart
+++ b/mobile/apps/photos/lib/utils/ml_util.dart
@@@ -872,55 -877,16 +871,22 @@@ Future<MLResult> analyzeImageRust(Map a
rethrow;
}
- _logger.warning(
- "Rust decode failed for fileID $enteFileID (format: $fileFormat), retrying with JPEG fallback",
+ // A Rust decode error may be a resource-limit rejection. Retrying it
+ // through an unbounded Dart decoder can exhaust the mobile process.
+ throw _asInvalidImageFormatExceptionForRustDecodeFailure(
+ enteFileID: enteFileID,
+ fileFormat: fileFormat,
+ primaryError: e,
);
- final _DecodeFallbackFile? fallback;
- try {
- fallback = await _createJpegDecodeFallbackFile(imagePath: imagePath);
- } catch (fallbackError, fallbackStack) {
- _logger.severe(
- "JPEG fallback conversion threw for fileID $enteFileID (format: $fileFormat)",
- fallbackError,
- fallbackStack,
- );
- rethrow;
- }
- if (fallback == null) {
- _logger.warning(
- "JPEG fallback conversion returned null/empty bytes for fileID $enteFileID (format: $fileFormat); storing empty result instead",
- );
- throw _asInvalidImageFormatExceptionForRustDecodeFailure(
- enteFileID: enteFileID,
- fileFormat: fileFormat,
- primaryError: e,
- );
- }
-
- try {
- rustResult = await runRustAnalyzeForPath(fallback.file.path);
- _logger.info(
- "Rust decode fallback succeeded for fileID $enteFileID (original format: $fileFormat)",
- );
- } catch (retryError, retryStack) {
- _logger.severe(
- "Rust decode fallback retry failed for fileID $enteFileID (original format: $fileFormat)",
- retryError,
- retryStack,
- );
- rethrow;
- } finally {
- await _cleanupDecodeFallback(fallback);
- }
}
- final result = MLResult.fromEnteFileID(enteFileID);
+ final result = MLResult.fromEnteFileID(
+ enteFileID,
+ remoteFlags:
+ mlIndexFlagRuntimeRust |
+ (rustResult.usedCoreml ? mlIndexFlagCoreML : 0) |
+ (rustResult.usedWebgpu ? mlIndexFlagWebGPU : 0),
+ );
result.decodedImageSize = Dimensions(
width: rustResult.decodedImageSize.width,
height: rustResult.decodedImageSize.height,
diff --cc rust/Cargo.lock
index 26d3b71de9,2ca8fc1747..3aa16acda8
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@@ -2385,14 -2378,11 +2387,12 @@@ version = "0.0.0
dependencies = [
"ente-core",
"ente-image",
- "flate2",
- "futures-util",
+ "ente-model-download",
"llama-cpp-2",
"ort",
+ "ort-sys",
"parking_lot",
"regex",
- "reqwest 0.12.28",
"rubato",
"rusqlite",
"self_cell",
@@@ -2435,8 -2454,8 +2464,9 @@@ name = "ente-photos
version = "0.0.0"
dependencies = [
"anyhow",
+ "ash",
"ente-image",
+ "ente-model-download",
"fast_image_resize",
"flate2",
"half",
diff --cc rust/Cargo.toml
index 489ee6f417,d59402326d..3898dec139
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@@ -26,12 -30,15 +30,16 @@@ ente-accounts = { path = "crates/accoun
ente-contacts = { path = "crates/contacts" }
ente-core = { path = "crates/core" }
ente-image = { path = "crates/image" }
+ ente-model-download = { path = "crates/model-download" }
+ ente-paste = { path = "crates/paste" }
ente-photos = { path = "crates/photos" }
ente-rs = { path = "apps/cli" }
+ ente-space = { path = "space" }
ente-test-support = { path = "crates/test-support" }
ort = { git = "https://github.com/pykeio/ort.git", rev = "31c8f1443bb579bfddb82705288347697e468fb3", default-features = false }
+ort-sys = { git = "https://github.com/pykeio/ort.git", rev = "31c8f1443bb579bfddb82705288347697e468fb3", default-features = false }
serde = { version = "1.0.228", features = ["derive"] }
+ serde_json = "1.0.149"
thiserror = "2.0.18"
uniffi = { version = "0.31.0" }
wasm-bindgen = "0.2.125"
diff --cc rust/crates/ensu/Cargo.toml
index 3d9eb1731a,4212755216..f799541413
--- a/rust/crates/ensu/Cargo.toml
+++ b/rust/crates/ensu/Cargo.toml
@@@ -62,12 -55,4 +60,10 @@@ ort-sys = { workspace = true, features
[features]
default = ["sqlite"]
sqlite = ["dep:rusqlite"]
-transcription = ["dep:ort", "dep:regex", "dep:rubato", "dep:transcribe-rs"]
+transcription = [
- "dep:flate2",
+ "dep:ort",
+ "dep:ort-sys",
+ "dep:regex",
+ "dep:rubato",
- "dep:tar",
+ "dep:transcribe-rs",
+]
diff --cc rust/crates/photos/Cargo.toml
index 783af6a314,8f6af9bcf2..7f34340fad
--- a/rust/crates/photos/Cargo.toml
+++ b/rust/crates/photos/Cargo.toml
@@@ -50,24 -54,16 +50,25 @@@ ort = { workspace = true, features =
"xnnpack",
] }
+[target.'cfg(not(any(target_os = "ios", target_os = "android")))'.dependencies]
+ort = { workspace = true, features = [
+ "ndarray",
+ "half",
+ "api-27",
+ "load-dynamic",
+ "copy-dylibs",
+] }
+
[dev-dependencies]
anyhow = "1"
+ ente-model-download.workspace = true
flate2 = "1.1"
- reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls-webpki-roots"] }
serde = { version = "1", features = ["derive"] }
- serde_json = "1"
- sha2 = "0.10"
+ serde_json.workspace = true
tar = "0.4"
tempfile = "3.12"
+ tokio = { version = "1", features = ["rt"] }
+ url = "2.5.8"
[[test]]
name = "ml_indexing"
diff --cc rust/crates/photos/tests/support/ml_indexing.rs
index e1b7b61301,61a4a04cca..df2d7f5953
--- a/rust/crates/photos/tests/support/ml_indexing.rs
+++ b/rust/crates/photos/tests/support/ml_indexing.rs
@@@ -168,12 -164,37 +164,37 @@@ fn should_print_stats() -> bool
pub(crate) struct MlIndexingTestContext {
asset_lock: AssetLock,
cache_dir: PathBuf,
- client: Client,
golden_results: HashMap<String, ComparableResult>,
manifest: FixtureManifest,
- runtime_config: MlRuntimeConfig,
+ model_paths: ModelPaths,
}
+ struct AssetFetcher {
+ runtime: tokio::runtime::Runtime,
+ downloader: download::Downloader,
+ }
+
+ impl AssetFetcher {
+ fn new() -> Result<Self> {
+ Ok(Self {
+ runtime: tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .context("build download runtime")?,
+ downloader: download::Downloader::new().context("build downloader")?,
+ })
+ }
+
+ fn fetch(&self, targets: Vec<download::Target>) -> Result<()> {
+ self.runtime.block_on(self.downloader.download(
+ targets,
+ |_| {},
+ download::CancellationToken::default(),
+ ))?;
+ Ok(())
+ }
+ }
+
impl MlIndexingTestContext {
pub(crate) fn load() -> Result<Self> {
let repo_root = repo_root()?;
@@@ -201,15 -225,22 +225,14 @@@
.context("load ONNX Runtime dynamic library")?
.commit();
- let model_paths = resolve_model_paths(&client, &cache_dir, &asset_lock.models)?;
- let runtime_config = MlRuntimeConfig {
- model_paths: resolve_model_paths(&fetcher, &cache_dir, &asset_lock.models)?,
- provider_policy: ExecutionProviderPolicy {
- prefer_coreml: false,
- prefer_nnapi: false,
- prefer_xnnpack: false,
- allow_cpu_fallback: true,
- },
- };
++ let model_paths = resolve_model_paths(&fetcher, &cache_dir, &asset_lock.models)?;
Ok(Self {
asset_lock,
cache_dir,
- client,
golden_results,
manifest,
- runtime_config,
+ model_paths,
})
}
@@@ -362,71 -388,8 +380,68 @@@
}
}
+/// A downloaded production model together with its content hash as pinned in
+/// the asset lock, for embedding into generated golden entries.
+#[allow(dead_code)]
+pub(crate) struct GoldenModelAsset {
+ pub(crate) path: PathBuf,
+ pub(crate) sha256: String,
+}
+
+/// Downloaded production model assets for the golden self-test tooling, with
+/// ONNX Runtime already initialized. Used by the ml_goldens test binary only.
+#[allow(dead_code)]
+pub(crate) struct GoldenTestAssets {
+ pub(crate) face_detection: GoldenModelAsset,
+ pub(crate) face_embedding: GoldenModelAsset,
+ pub(crate) clip_image: GoldenModelAsset,
+ pub(crate) clip_text: GoldenModelAsset,
+ pub(crate) clip_text_vocab: PathBuf,
+}
+
+#[allow(dead_code)]
+impl GoldenTestAssets {
+ pub(crate) fn load() -> Result<Self> {
+ let repo_root = repo_root()?;
+ let asset_lock = load_asset_lock(&repo_root)?;
+ let cache_dir = cache_dir(&repo_root);
- let client = Client::builder()
- .user_agent("ente-rust-ml-goldens-test")
- .build()
- .context("build HTTP client")?;
++ let fetcher = AssetFetcher::new()?;
+
+ let onnx_runtime_library =
- resolve_onnx_runtime_library(&client, &cache_dir, &asset_lock.onnx_runtime)?;
++ resolve_onnx_runtime_library(&fetcher, &cache_dir, &asset_lock.onnx_runtime)?;
+ let _ = ort::init_from(&onnx_runtime_library)
+ .context("load ONNX Runtime dynamic library")?
+ .commit();
+
+ let golden_model = |label: &str, model: &ModelAsset| -> Result<GoldenModelAsset> {
+ Ok(GoldenModelAsset {
- path: resolve_model_asset(&client, &cache_dir, label, model)?,
++ path: resolve_model_asset(&fetcher, &cache_dir, label, model)?,
+ sha256: model.sha256.clone(),
+ })
+ };
+
+ let models = &asset_lock.models;
+ Ok(Self {
+ face_detection: golden_model("face-detection", &models.face_detection)?,
+ face_embedding: golden_model("face-embedding", &models.face_embedding)?,
+ clip_image: golden_model("clip-image", &models.clip_image)?,
+ clip_text: golden_model("clip-text", &models.clip_text)?,
+ clip_text_vocab: resolve_model_asset(
- &client,
++ &fetcher,
+ &cache_dir,
+ "clip-text-vocab",
+ &models.clip_text_vocab,
+ )?,
+ })
+ }
+
+ pub(crate) fn golden_data_path() -> Result<PathBuf> {
+ Ok(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/ml/golden_data.rs"))
+ }
+}
+
pub(crate) struct PreparedMlRuntime {
- config: MlRuntimeConfig,
+ model_paths: ModelPaths,
}
impl PreparedMlRuntime {