π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Use UniFFI async runtime for asset downloads
Manav Rathi1498b84aa5da
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.ktindex 48b10b3f00..3a5fa66028 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt@@ -17,10 +17,6 @@ import io.ente.ensu.bindings.reconcileKnowledgePackimport io.ente.ensu.bindings.uniffiEnsureInitializedimport java.io.Fileimport kotlinx.coroutines.CancellationException-import kotlinx.coroutines.Dispatchers-import kotlinx.coroutines.async-import kotlinx.coroutines.coroutineScope-import kotlinx.coroutines.withContextclass AssetStore(context: Context) {private val appContext = context.applicationContext@@ -73,44 +69,35 @@ class AssetStore(context: Context) {internal fun cleanupKnowledgeRevisions(stableId: String, activeIdentity: String) =cleanupObsoleteKnowledgePackRevisions(core, stableId, activeIdentity)- suspend fun estimateDownloadSize(asset: Asset): Long? = withContext(Dispatchers.IO) {- core.estimatedDownloadSize(asset)- }+ suspend fun estimateDownloadSize(asset: Asset): Long? = core.estimatedDownloadSize(asset)suspend fun download(assets: List<Asset>,onProgress: (AssetDownloadProgress) -> Unit- ): Unit = withContext(Dispatchers.IO) {- if (assets.all { core.isDownloaded(it) }) return@withContext+ ) {+ if (assets.all { core.isDownloaded(it) }) returnval token = CancellationToken()val lease = AssetDownloadJobService.begin { token.cancel() }try {- coroutineScope {- val download = async {- core.download(- assets,- object : AssetDownloadCallback {- override fun onProgress(progress: AssetDownloadProgress) {- progress.logLine?.let { Log.i("AssetStore", it) }- AssetDownloadJobService.update(- lease,- progress.percentage.toInt(),- progress.totalBytes == null- )- onProgress(progress)- }- },- token- )- }- try {- download.await()- } catch (e: CancellationException) {- token.cancel()- throw e- }- }+ core.download(+ assets,+ object : AssetDownloadCallback {+ override fun onProgress(progress: AssetDownloadProgress) {+ progress.logLine?.let { Log.i("AssetStore", it) }+ AssetDownloadJobService.update(+ lease,+ progress.percentage.toInt(),+ progress.totalBytes == null+ )+ onProgress(progress)+ }+ },+ token+ )+ } catch (e: CancellationException) {+ token.cancel()+ throw e} finally {AssetDownloadJobService.end(lease)}diff --git a/mobile/native/android/apps/ensu/rust/build.gradle.kts b/mobile/native/android/apps/ensu/rust/build.gradle.ktsindex 06f3e172c7..40fb943b08 100644--- a/mobile/native/android/apps/ensu/rust/build.gradle.kts+++ b/mobile/native/android/apps/ensu/rust/build.gradle.kts@@ -131,6 +131,7 @@ tasks.matching { it.name == "preReleaseBuild" }.configureEach {dependencies {api("net.java.dev.jna:jna:5.18.1@aar")api("androidx.annotation:annotation:1.7.1")+ implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0")// Custom WebGPU/XNNPACK build; the Rust runtime dynamically loads its// libonnxruntime.so. Resolved from the Ivy repository declared in// settings.gradle.kts and SHA-256 pinned in gradle/verification-metadata.xml.diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swift b/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swiftindex 6b7f832866..9b0f9f4414 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swift@@ -81,9 +81,7 @@ final class AssetStore: @unchecked Sendable {}func estimateDownloadSize(_ asset: Asset) async -> Int64? {- await Task.detached(priority: .utility) { [core] in- core.estimatedDownloadSize(asset: asset)- }.value+ await core.estimatedDownloadSize(asset: asset)}func download(@@ -110,29 +108,23 @@ final class AssetStore: @unchecked Sendable {}}- let core = core- let downloadTask = Task.detached(priority: .utility) {- try Task.checkCancellation()- let callback = AssetDownloadCallbackSink { progress in- if let line = progress.logLine {- logger.info(line)- }- if #available(iOS 26.0, *), let leaseId {- AssetDownloadBackgroundTask.update(- id: leaseId,- downloadedBytes: progress.downloadedBytes,- totalBytes: progress.totalBytes- )- }- onProgress(progress)+ let callback = AssetDownloadCallbackSink { progress in+ if let line = progress.logLine {+ logger.info(line)}- try core.download(assets: assets, callback: callback, cancellation: token)+ if #available(iOS 26.0, *), let leaseId {+ AssetDownloadBackgroundTask.update(+ id: leaseId,+ downloadedBytes: progress.downloadedBytes,+ totalBytes: progress.totalBytes+ )+ }+ onProgress(progress)}try await withTaskCancellationHandler {- try await downloadTask.value+ try await core.download(assets: assets, callback: callback, cancellation: token)} onCancel: {- downloadTask.cancel()token.cancel()}succeeded = truediff --git a/rust/Cargo.lock b/rust/Cargo.lockindex a1e81c5e30..93f620b1c0 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -369,6 +369,19 @@ dependencies = ["pin-project-lite",]+[[package]]+name = "async-compat"+version = "0.2.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1ba85bc55464dcbf728b56d97e119d673f4cf9062be330a9a26f3acf504a590"+dependencies = [+ "futures-core",+ "futures-io",+ "once_cell",+ "pin-project-lite",+ "tokio",+]+[[package]]name = "async-executor"version = "1.13.3"@@ -2418,7 +2431,6 @@ dependencies = ["ente-assets","ente-ensu","thiserror 2.0.18",- "tokio","uniffi","uuid",]@@ -9686,6 +9698,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b0ef62e69762fbb9386dcb6c87cd3dd05d525fa8a3a579a290892e60ddbda47e"dependencies = ["anyhow",+ "async-compat","bytes","once_cell","static_assertions",diff --git a/rust/Cargo.toml b/rust/Cargo.tomlindex 1f510a6223..b153e437b4 100644--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@ -44,7 +44,7 @@ serde = { version = "1.0.228", features = ["derive"] }serde_json = "1.0.149"sha2 = "0.11.0"thiserror = "2.0.18"-uniffi = { version = "0.31.0" }+uniffi = { version = "=0.31.0" }uniffi_bindgen = { version = "=0.31.0", default-features = false, features = ["cargo-metadata"] }url = "2.5.8"urlencoding = "2.1.3"diff --git a/rust/bindings/uniffi/ensu/Cargo.toml b/rust/bindings/uniffi/ensu/Cargo.tomlindex 1567596d84..68c2df3b2a 100644--- a/rust/bindings/uniffi/ensu/Cargo.toml+++ b/rust/bindings/uniffi/ensu/Cargo.toml@@ -10,6 +10,5 @@ crate-type = ["staticlib", "cdylib"]ente-assets.workspace = trueente-ensu = { workspace = true, features = ["transcription"] }thiserror.workspace = true-tokio = { version = "1", features = ["rt-multi-thread"] }-uniffi.workspace = true+uniffi = { workspace = true, features = ["tokio"] }uuid = { version = "1" }diff --git a/rust/bindings/uniffi/ensu/src/assets.rs b/rust/bindings/uniffi/ensu/src/assets.rsindex 8d7f7e7b08..24bd9e4846 100644--- a/rust/bindings/uniffi/ensu/src/assets.rs+++ b/rust/bindings/uniffi/ensu/src/assets.rs@@ -1,5 +1,5 @@use std::path::PathBuf;-use std::sync::{Arc, OnceLock};+use std::sync::Arc;use ente_assets::download;use thiserror::Error;@@ -149,7 +149,6 @@ pub fn migrate_ensu_assets(assets_dir: String, legacy: LegacyAssets) -> Option<S#[derive(uniffi::Object)]pub struct AssetStoreCore {inner: ente_assets::AssetStore,- runtime: OnceLock<tokio::runtime::Runtime>,}#[uniffi::export]@@ -158,7 +157,6 @@ impl AssetStoreCore {pub fn new(assets_dir: String) -> Arc<Self> {Arc::new(Self {inner: ente_assets::AssetStore::new(assets_dir),- runtime: OnceLock::new(),})}@@ -186,25 +184,22 @@ impl AssetStoreCore {self.inner.is_downloaded(&asset.inner)}- pub fn estimated_download_size(&self, asset: Arc<Asset>) -> Option<i64> {- self.runtime()- .ok()?- .block_on(- self.inner- .estimated_download_size(std::slice::from_ref(&asset.inner)),- )+ #[uniffi::method(async_runtime = "tokio")]+ pub async fn estimated_download_size(&self, asset: Arc<Asset>) -> Option<i64> {+ self.inner+ .estimated_download_size(std::slice::from_ref(&asset.inner))+ .await.map(|size| i64::try_from(size).unwrap_or(i64::MAX))}pub fn remove_downloaded(&self, asset: Arc<Asset>) -> bool {let existed = self.inner.is_downloaded(&asset.inner);- let removed = self- .runtime()- .is_ok_and(|runtime| runtime.block_on(self.inner.remove(&asset.inner)).is_ok());+ let removed = self.inner.remove(&asset.inner).is_ok();existed && removed}- pub fn download(+ #[uniffi::method(async_runtime = "tokio")]+ pub async fn download(&self,assets: Vec<Arc<Asset>>,callback: Box<dyn AssetDownloadCallback>,@@ -214,16 +209,14 @@ impl AssetStoreCore {.into_iter().map(|asset| asset.inner.clone()).collect::<Vec<_>>();- match self.runtime() {- Ok(runtime) => runtime- .block_on(self.inner.download(- &assets,- |progress| callback.on_progress(progress.into()),- cancellation.inner.clone(),- ))- .map_err(AssetDownloadError::from),- Err(error) => Err(AssetDownloadError::from(error)),- }+ self.inner+ .download(+ &assets,+ |progress| callback.on_progress(progress.into()),+ cancellation.inner.clone(),+ )+ .await+ .map_err(AssetDownloadError::from)}}@@ -231,16 +224,6 @@ impl AssetStoreCore {pub(crate) fn store(&self) -> &ente_assets::AssetStore {&self.inner}-- pub(crate) fn runtime(&self) -> Result<&tokio::runtime::Runtime, ente_assets::download::Error> {- if let Some(runtime) = self.runtime.get() {- return Ok(runtime);- }- let runtime = tokio::runtime::Builder::new_multi_thread()- .enable_all()- .build()?;- Ok(self.runtime.get_or_init(|| runtime))- }}#[derive(Default, uniffi::Object)]diff --git a/rust/crates/assets/src/download.rs b/rust/crates/assets/src/download.rsindex 1c3937065f..7959c1371e 100644--- a/rust/crates/assets/src/download.rs+++ b/rust/crates/assets/src/download.rs@@ -271,7 +271,7 @@ async fn fetch_async(for target in &targets {let destination = &target.destination;- let is_cached = prepare_cached_download(target, destination);+ let is_cached = prepare_cached_download(target, destination).await;cached.push(is_cached);if is_cached {download_probes.push(DownloadProbe {@@ -656,7 +656,7 @@ async fn download_file_single(continue;}- if let Err(err) = check_sha256(&tmp_path, &target.sha256, &target.label) {+ if let Err(err) = check_sha256(&tmp_path, &target.sha256, &target.label).await {let _ = fs::remove_file(&tmp_path);let _ = fs::remove_file(&partial_metadata_path);return Err(err);@@ -773,7 +773,7 @@ async fn download_file_ranged(emit_range_file_progress(total, file_started_at, &range_states, &on_progress);- if let Err(err) = check_sha256(&tmp_path, &target.sha256, &target.label) {+ if let Err(err) = check_sha256(&tmp_path, &target.sha256, &target.label).await {let _ = fs::remove_file(&tmp_path);let _ = fs::remove_file(&range_metadata_path);return Err(err);@@ -1383,8 +1383,12 @@ fn bytes_per_second(bytes: u64, elapsed: Duration) -> f64 {}}-fn prepare_cached_download(target: &DownloadTarget, destination: &Path) -> bool {- if !destination.exists() || check_sha256(destination, &target.sha256, &target.label).is_err() {+async fn prepare_cached_download(target: &DownloadTarget, destination: &Path) -> bool {+ if !destination.exists()+ || check_sha256(destination, &target.sha256, &target.label)+ .await+ .is_err()+ {return false;}cleanup_range_download(destination);@@ -1439,7 +1443,16 @@ pub fn sha256_file(path: &Path) -> Result<String, Error> {Ok(hex)}-fn check_sha256(path: &Path, expected: &str, label: &str) -> Result<(), Error> {+async fn check_sha256(path: &Path, expected: &str, label: &str) -> Result<(), Error> {+ let path = path.to_owned();+ let expected = expected.to_owned();+ let label = label.to_owned();+ tokio::task::spawn_blocking(move || check_sha256_blocking(&path, &expected, &label))+ .await+ .map_err(|error| Error::Io(std::io::Error::other(error)))?+}++fn check_sha256_blocking(path: &Path, expected: &str, label: &str) -> Result<(), Error> {let actual = sha256_file(path)?;if actual != expected.trim().to_ascii_lowercase() {return Err(Error::Validation(format!(@@ -2282,6 +2295,44 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}+ #[tokio::test]+ async fn dropping_download_preserves_partial_without_installing() {+ let bytes = Arc::new(sample_bytes(512));+ let release = Arc::new(AtomicBool::new(false));+ let server = {+ let bytes = Arc::clone(&bytes);+ let release = Arc::clone(&release);+ TestServer::spawn(move |stream| {+ handle_stalled_test_request(stream, Arc::clone(&bytes), Arc::clone(&release));+ })+ };+ let test_dir = scratch_dir("drop-download");+ let destination = test_dir.join("model.bin");+ let tmp = tmp_path_for(&destination);+ let target = DownloadTarget {+ label: "Model".to_string(),+ url: server.url("/model.bin"),+ destination: destination.clone(),+ sha256: sha_hex(&bytes),+ };++ let task = tokio::spawn(download(vec![target], |_| {}, false));+ for _ in 0..10_000 {+ if file_size(&tmp).unwrap_or(0) > 0 {+ break;+ }+ tokio::task::yield_now().await;+ }+ task.abort();+ release.store(true, Ordering::SeqCst);++ assert!(task.await.unwrap_err().is_cancelled());+ assert!(file_size(&tmp).unwrap_or(0) > 0);+ assert!(!destination.exists());+ assert!(partial_metadata_path_for(&destination).is_file());+ let _ = fs::remove_dir_all(test_dir);+ }+#[tokio::test]async fn cached_download_adoption_removes_stale_partials() {let test_dir = scratch_dir("cached-adoption");@@ -2551,30 +2602,47 @@ mod tests {bytes: Arc<Vec<u8>>,get_count: Arc<AtomicUsize>,) {- let mut reader = BufReader::new(stream.try_clone().expect("clone test stream"));- let mut request_line = String::new();- if reader.read_line(&mut request_line).is_err() {+ let Some(request) = read_test_request(&stream) else {return;- }- loop {- let mut line = String::new();- if reader.read_line(&mut line).is_err() || line == "\r\n" || line.is_empty() {- break;- }- }+ };let response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",bytes.len());let _ = stream.write_all(response.as_bytes());- if request_line.starts_with("HEAD ") {+ if request.line.starts_with("HEAD ") {return;}get_count.fetch_add(1, Ordering::SeqCst);let _ = stream.write_all(bytes.as_slice());}+ fn handle_stalled_test_request(+ mut stream: TcpStream,+ bytes: Arc<Vec<u8>>,+ release: Arc<AtomicBool>,+ ) {+ let Some(request) = read_test_request(&stream) else {+ return;+ };+ let response = format!(+ "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nETag: \"test-etag\"\r\nConnection: close\r\n\r\n",+ bytes.len()+ );+ let _ = stream.write_all(response.as_bytes());+ if request.line.starts_with("HEAD ") {+ return;+ }+ let midpoint = bytes.len() / 2;+ let _ = stream.write_all(&bytes[..midpoint]);+ let _ = stream.flush();+ while !release.load(Ordering::SeqCst) {+ thread::sleep(Duration::from_millis(1));+ }+ let _ = stream.write_all(&bytes[midpoint..]);+ }+fn handle_range_test_request(mut stream: TcpStream,bytes: Arc<Vec<u8>>,@@ -2582,29 +2650,11 @@ mod tests {range_get_count: Arc<AtomicUsize>,full_get_count: Arc<AtomicUsize>,) {- let mut reader = BufReader::new(stream.try_clone().expect("clone test stream"));- let mut request_line = String::new();- if reader.read_line(&mut request_line).is_err() {+ let Some(request) = read_test_request(&stream) else {return;- }-- let mut range_header = None;- let mut if_range_header = None;- loop {- let mut line = String::new();- if reader.read_line(&mut line).is_err() || line == "\r\n" || line.is_empty() {- break;- }- if let Some((name, value)) = line.split_once(':') {- if name.eq_ignore_ascii_case("range") {- range_header = Some(value.trim().to_string());- } else if name.eq_ignore_ascii_case("if-range") {- if_range_header = Some(value.trim().to_string());- }- }- }+ };- if request_line.starts_with("HEAD ") {+ if request.line.starts_with("HEAD ") {head_count.fetch_add(1, Ordering::SeqCst);let response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nAccept-Ranges: bytes\r\nETag: \"test-etag\"\r\nConnection: close\r\n\r\n",@@ -2614,10 +2664,11 @@ mod tests {return;}- let resume_allowed = if_range_header+ let resume_allowed = request+ .if_range.as_deref().is_none_or(|value| value == "\"test-etag\"");- if let Some(range_header) = range_header.filter(|_| resume_allowed) {+ if let Some(range_header) = request.range.filter(|_| resume_allowed) {let Some((start, end)) = parse_test_range_header(&range_header, bytes.len() as u64)else {let _ = stream@@ -2646,6 +2697,36 @@ mod tests {}}+ struct TestRequest {+ line: String,+ range: Option<String>,+ if_range: Option<String>,+ }++ fn read_test_request(stream: &TcpStream) -> Option<TestRequest> {+ let mut reader = BufReader::new(stream.try_clone().ok()?);+ let mut request = TestRequest {+ line: String::new(),+ range: None,+ if_range: None,+ };+ reader.read_line(&mut request.line).ok()?;+ loop {+ let mut line = String::new();+ if reader.read_line(&mut line).ok()? == 0 || line == "\r\n" {+ break;+ }+ if let Some((name, value)) = line.split_once(':') {+ if name.eq_ignore_ascii_case("range") {+ request.range = Some(value.trim().to_string());+ } else if name.eq_ignore_ascii_case("if-range") {+ request.if_range = Some(value.trim().to_string());+ }+ }+ }+ Some(request)+ }+fn parse_test_range_header(value: &str, total: u64) -> Option<(u64, u64)> {let value = value.strip_prefix("bytes=")?;let (start, end) = value.split_once('-')?;diff --git a/rust/crates/assets/src/store.rs b/rust/crates/assets/src/store.rsindex 69252d8b55..c8d111d38f 100644--- a/rust/crates/assets/src/store.rs+++ b/rust/crates/assets/src/store.rs@@ -184,7 +184,7 @@ impl AssetStore {Ok(())}- pub async fn remove(&self, asset: &Asset) -> Result<(), AssetStoreError> {+ pub fn remove(&self, asset: &Asset) -> Result<(), AssetStoreError> {let lock = self.asset_lock(asset)?;let _guard = lock.try_lock().map_err(|_| AssetStoreError::Busy)?;remove_path(&self.asset_dir(asset))?;@@ -514,7 +514,6 @@ mod tests {CancellationToken::default(),));assert_send(store.estimated_download_size(std::slice::from_ref(&asset)));- assert_send(store.remove(&asset));}#[test]@@ -640,10 +639,7 @@ mod tests {));let lock = store.asset_lock(&asset).unwrap();let guard = lock.lock().await;- assert!(matches!(- store.remove(&asset).await,- Err(AssetStoreError::Busy)- ));+ assert!(matches!(store.remove(&asset), Err(AssetStoreError::Busy)));let token = CancellationToken::new();let cancellation = token.clone();@@ -799,10 +795,10 @@ mod tests {fs::create_dir_all(&staging).unwrap();fs::write(staging.join("model.onnx.tmp"), b"partial").unwrap();- store.remove(&asset).await.unwrap();+ store.remove(&asset).unwrap();assert!(!final_dir.exists());assert!(!staging.exists());- store.remove(&asset).await.unwrap();+ store.remove(&asset).unwrap();let _ = fs::remove_dir_all(root);}