Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Use UniFFI async runtime for asset downloads

Manav Rathi1498b84aa5da
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt
index 48b10b3f00..3a5fa66028 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt
@@ -17,10 +17,6 @@ import io.ente.ensu.bindings.reconcileKnowledgePack
import io.ente.ensu.bindings.uniffiEnsureInitialized
import java.io.File
import kotlinx.coroutines.CancellationException
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.async
-import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.withContext
class AssetStore(context: Context) {
private val appContext = context.applicationContext
@@ -73,44 +69,35 @@ class AssetStore(context: Context) {
internal fun cleanupKnowledgeRevisions(stableId: String, activeIdentity: String) =
cleanupObsoleteKnowledgePackRevisions(core, stableId, activeIdentity)
- suspend fun estimateDownloadSize(asset: Asset): Long? = withContext(Dispatchers.IO) {
- core.estimatedDownloadSize(asset)
- }
+ suspend fun estimateDownloadSize(asset: Asset): Long? = core.estimatedDownloadSize(asset)
suspend fun download(
assets: List<Asset>,
onProgress: (AssetDownloadProgress) -> Unit
- ): Unit = withContext(Dispatchers.IO) {
- if (assets.all { core.isDownloaded(it) }) return@withContext
+ ) {
+ if (assets.all { core.isDownloaded(it) }) return
val token = CancellationToken()
val lease = AssetDownloadJobService.begin { token.cancel() }
try {
- coroutineScope {
- val download = async {
- core.download(
- assets,
- object : AssetDownloadCallback {
- override fun onProgress(progress: AssetDownloadProgress) {
- progress.logLine?.let { Log.i("AssetStore", it) }
- AssetDownloadJobService.update(
- lease,
- progress.percentage.toInt(),
- progress.totalBytes == null
- )
- onProgress(progress)
- }
- },
- token
- )
- }
- try {
- download.await()
- } catch (e: CancellationException) {
- token.cancel()
- throw e
- }
- }
+ core.download(
+ assets,
+ object : AssetDownloadCallback {
+ override fun onProgress(progress: AssetDownloadProgress) {
+ progress.logLine?.let { Log.i("AssetStore", it) }
+ AssetDownloadJobService.update(
+ lease,
+ progress.percentage.toInt(),
+ progress.totalBytes == null
+ )
+ onProgress(progress)
+ }
+ },
+ token
+ )
+ } catch (e: CancellationException) {
+ token.cancel()
+ throw e
} finally {
AssetDownloadJobService.end(lease)
}
diff --git a/mobile/native/android/apps/ensu/rust/build.gradle.kts b/mobile/native/android/apps/ensu/rust/build.gradle.kts
index 06f3e172c7..40fb943b08 100644
--- a/mobile/native/android/apps/ensu/rust/build.gradle.kts
+++ b/mobile/native/android/apps/ensu/rust/build.gradle.kts
@@ -131,6 +131,7 @@ tasks.matching { it.name == "preReleaseBuild" }.configureEach {
dependencies {
api("net.java.dev.jna:jna:5.18.1@aar")
api("androidx.annotation:annotation:1.7.1")
+ implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0")
// Custom WebGPU/XNNPACK build; the Rust runtime dynamically loads its
// libonnxruntime.so. Resolved from the Ivy repository declared in
// settings.gradle.kts and SHA-256 pinned in gradle/verification-metadata.xml.
diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swift b/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swift
index 6b7f832866..9b0f9f4414 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swift
@@ -81,9 +81,7 @@ final class AssetStore: @unchecked Sendable {
}
func estimateDownloadSize(_ asset: Asset) async -> Int64? {
- await Task.detached(priority: .utility) { [core] in
- core.estimatedDownloadSize(asset: asset)
- }.value
+ await core.estimatedDownloadSize(asset: asset)
}
func download(
@@ -110,29 +108,23 @@ final class AssetStore: @unchecked Sendable {
}
}
- let core = core
- let downloadTask = Task.detached(priority: .utility) {
- try Task.checkCancellation()
- let callback = AssetDownloadCallbackSink { progress in
- if let line = progress.logLine {
- logger.info(line)
- }
- if #available(iOS 26.0, *), let leaseId {
- AssetDownloadBackgroundTask.update(
- id: leaseId,
- downloadedBytes: progress.downloadedBytes,
- totalBytes: progress.totalBytes
- )
- }
- onProgress(progress)
+ let callback = AssetDownloadCallbackSink { progress in
+ if let line = progress.logLine {
+ logger.info(line)
}
- try core.download(assets: assets, callback: callback, cancellation: token)
+ if #available(iOS 26.0, *), let leaseId {
+ AssetDownloadBackgroundTask.update(
+ id: leaseId,
+ downloadedBytes: progress.downloadedBytes,
+ totalBytes: progress.totalBytes
+ )
+ }
+ onProgress(progress)
}
try await withTaskCancellationHandler {
- try await downloadTask.value
+ try await core.download(assets: assets, callback: callback, cancellation: token)
} onCancel: {
- downloadTask.cancel()
token.cancel()
}
succeeded = true
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index a1e81c5e30..93f620b1c0 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -369,6 +369,19 @@ dependencies = [
"pin-project-lite",
]
+[[package]]
+name = "async-compat"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1ba85bc55464dcbf728b56d97e119d673f4cf9062be330a9a26f3acf504a590"
+dependencies = [
+ "futures-core",
+ "futures-io",
+ "once_cell",
+ "pin-project-lite",
+ "tokio",
+]
+
[[package]]
name = "async-executor"
version = "1.13.3"
@@ -2418,7 +2431,6 @@ dependencies = [
"ente-assets",
"ente-ensu",
"thiserror 2.0.18",
- "tokio",
"uniffi",
"uuid",
]
@@ -9686,6 +9698,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0ef62e69762fbb9386dcb6c87cd3dd05d525fa8a3a579a290892e60ddbda47e"
dependencies = [
"anyhow",
+ "async-compat",
"bytes",
"once_cell",
"static_assertions",
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 1f510a6223..b153e437b4 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -44,7 +44,7 @@ serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.149"
sha2 = "0.11.0"
thiserror = "2.0.18"
-uniffi = { version = "0.31.0" }
+uniffi = { version = "=0.31.0" }
uniffi_bindgen = { version = "=0.31.0", default-features = false, features = ["cargo-metadata"] }
url = "2.5.8"
urlencoding = "2.1.3"
diff --git a/rust/bindings/uniffi/ensu/Cargo.toml b/rust/bindings/uniffi/ensu/Cargo.toml
index 1567596d84..68c2df3b2a 100644
--- a/rust/bindings/uniffi/ensu/Cargo.toml
+++ b/rust/bindings/uniffi/ensu/Cargo.toml
@@ -10,6 +10,5 @@ crate-type = ["staticlib", "cdylib"]
ente-assets.workspace = true
ente-ensu = { workspace = true, features = ["transcription"] }
thiserror.workspace = true
-tokio = { version = "1", features = ["rt-multi-thread"] }
-uniffi.workspace = true
+uniffi = { workspace = true, features = ["tokio"] }
uuid = { version = "1" }
diff --git a/rust/bindings/uniffi/ensu/src/assets.rs b/rust/bindings/uniffi/ensu/src/assets.rs
index 8d7f7e7b08..24bd9e4846 100644
--- a/rust/bindings/uniffi/ensu/src/assets.rs
+++ b/rust/bindings/uniffi/ensu/src/assets.rs
@@ -1,5 +1,5 @@
use std::path::PathBuf;
-use std::sync::{Arc, OnceLock};
+use std::sync::Arc;
use ente_assets::download;
use thiserror::Error;
@@ -149,7 +149,6 @@ pub fn migrate_ensu_assets(assets_dir: String, legacy: LegacyAssets) -> Option<S
#[derive(uniffi::Object)]
pub struct AssetStoreCore {
inner: ente_assets::AssetStore,
- runtime: OnceLock<tokio::runtime::Runtime>,
}
#[uniffi::export]
@@ -158,7 +157,6 @@ impl AssetStoreCore {
pub fn new(assets_dir: String) -> Arc<Self> {
Arc::new(Self {
inner: ente_assets::AssetStore::new(assets_dir),
- runtime: OnceLock::new(),
})
}
@@ -186,25 +184,22 @@ impl AssetStoreCore {
self.inner.is_downloaded(&asset.inner)
}
- pub fn estimated_download_size(&self, asset: Arc<Asset>) -> Option<i64> {
- self.runtime()
- .ok()?
- .block_on(
- self.inner
- .estimated_download_size(std::slice::from_ref(&asset.inner)),
- )
+ #[uniffi::method(async_runtime = "tokio")]
+ pub async fn estimated_download_size(&self, asset: Arc<Asset>) -> Option<i64> {
+ self.inner
+ .estimated_download_size(std::slice::from_ref(&asset.inner))
+ .await
.map(|size| i64::try_from(size).unwrap_or(i64::MAX))
}
pub fn remove_downloaded(&self, asset: Arc<Asset>) -> bool {
let existed = self.inner.is_downloaded(&asset.inner);
- let removed = self
- .runtime()
- .is_ok_and(|runtime| runtime.block_on(self.inner.remove(&asset.inner)).is_ok());
+ let removed = self.inner.remove(&asset.inner).is_ok();
existed && removed
}
- pub fn download(
+ #[uniffi::method(async_runtime = "tokio")]
+ pub async fn download(
&self,
assets: Vec<Arc<Asset>>,
callback: Box<dyn AssetDownloadCallback>,
@@ -214,16 +209,14 @@ impl AssetStoreCore {
.into_iter()
.map(|asset| asset.inner.clone())
.collect::<Vec<_>>();
- match self.runtime() {
- Ok(runtime) => runtime
- .block_on(self.inner.download(
- &assets,
- |progress| callback.on_progress(progress.into()),
- cancellation.inner.clone(),
- ))
- .map_err(AssetDownloadError::from),
- Err(error) => Err(AssetDownloadError::from(error)),
- }
+ self.inner
+ .download(
+ &assets,
+ |progress| callback.on_progress(progress.into()),
+ cancellation.inner.clone(),
+ )
+ .await
+ .map_err(AssetDownloadError::from)
}
}
@@ -231,16 +224,6 @@ impl AssetStoreCore {
pub(crate) fn store(&self) -> &ente_assets::AssetStore {
&self.inner
}
-
- pub(crate) fn runtime(&self) -> Result<&tokio::runtime::Runtime, ente_assets::download::Error> {
- if let Some(runtime) = self.runtime.get() {
- return Ok(runtime);
- }
- let runtime = tokio::runtime::Builder::new_multi_thread()
- .enable_all()
- .build()?;
- Ok(self.runtime.get_or_init(|| runtime))
- }
}
#[derive(Default, uniffi::Object)]
diff --git a/rust/crates/assets/src/download.rs b/rust/crates/assets/src/download.rs
index 1c3937065f..7959c1371e 100644
--- a/rust/crates/assets/src/download.rs
+++ b/rust/crates/assets/src/download.rs
@@ -271,7 +271,7 @@ async fn fetch_async(
for target in &targets {
let destination = &target.destination;
- let is_cached = prepare_cached_download(target, destination);
+ let is_cached = prepare_cached_download(target, destination).await;
cached.push(is_cached);
if is_cached {
download_probes.push(DownloadProbe {
@@ -656,7 +656,7 @@ async fn download_file_single(
continue;
}
- if let Err(err) = check_sha256(&tmp_path, &target.sha256, &target.label) {
+ if let Err(err) = check_sha256(&tmp_path, &target.sha256, &target.label).await {
let _ = fs::remove_file(&tmp_path);
let _ = fs::remove_file(&partial_metadata_path);
return Err(err);
@@ -773,7 +773,7 @@ async fn download_file_ranged(
emit_range_file_progress(total, file_started_at, &range_states, &on_progress);
- if let Err(err) = check_sha256(&tmp_path, &target.sha256, &target.label) {
+ if let Err(err) = check_sha256(&tmp_path, &target.sha256, &target.label).await {
let _ = fs::remove_file(&tmp_path);
let _ = fs::remove_file(&range_metadata_path);
return Err(err);
@@ -1383,8 +1383,12 @@ fn bytes_per_second(bytes: u64, elapsed: Duration) -> f64 {
}
}
-fn prepare_cached_download(target: &DownloadTarget, destination: &Path) -> bool {
- if !destination.exists() || check_sha256(destination, &target.sha256, &target.label).is_err() {
+async fn prepare_cached_download(target: &DownloadTarget, destination: &Path) -> bool {
+ if !destination.exists()
+ || check_sha256(destination, &target.sha256, &target.label)
+ .await
+ .is_err()
+ {
return false;
}
cleanup_range_download(destination);
@@ -1439,7 +1443,16 @@ pub fn sha256_file(path: &Path) -> Result<String, Error> {
Ok(hex)
}
-fn check_sha256(path: &Path, expected: &str, label: &str) -> Result<(), Error> {
+async fn check_sha256(path: &Path, expected: &str, label: &str) -> Result<(), Error> {
+ let path = path.to_owned();
+ let expected = expected.to_owned();
+ let label = label.to_owned();
+ tokio::task::spawn_blocking(move || check_sha256_blocking(&path, &expected, &label))
+ .await
+ .map_err(|error| Error::Io(std::io::Error::other(error)))?
+}
+
+fn check_sha256_blocking(path: &Path, expected: &str, label: &str) -> Result<(), Error> {
let actual = sha256_file(path)?;
if actual != expected.trim().to_ascii_lowercase() {
return Err(Error::Validation(format!(
@@ -2282,6 +2295,44 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
+ #[tokio::test]
+ async fn dropping_download_preserves_partial_without_installing() {
+ let bytes = Arc::new(sample_bytes(512));
+ let release = Arc::new(AtomicBool::new(false));
+ let server = {
+ let bytes = Arc::clone(&bytes);
+ let release = Arc::clone(&release);
+ TestServer::spawn(move |stream| {
+ handle_stalled_test_request(stream, Arc::clone(&bytes), Arc::clone(&release));
+ })
+ };
+ let test_dir = scratch_dir("drop-download");
+ let destination = test_dir.join("model.bin");
+ let tmp = tmp_path_for(&destination);
+ let target = DownloadTarget {
+ label: "Model".to_string(),
+ url: server.url("/model.bin"),
+ destination: destination.clone(),
+ sha256: sha_hex(&bytes),
+ };
+
+ let task = tokio::spawn(download(vec![target], |_| {}, false));
+ for _ in 0..10_000 {
+ if file_size(&tmp).unwrap_or(0) > 0 {
+ break;
+ }
+ tokio::task::yield_now().await;
+ }
+ task.abort();
+ release.store(true, Ordering::SeqCst);
+
+ assert!(task.await.unwrap_err().is_cancelled());
+ assert!(file_size(&tmp).unwrap_or(0) > 0);
+ assert!(!destination.exists());
+ assert!(partial_metadata_path_for(&destination).is_file());
+ let _ = fs::remove_dir_all(test_dir);
+ }
+
#[tokio::test]
async fn cached_download_adoption_removes_stale_partials() {
let test_dir = scratch_dir("cached-adoption");
@@ -2551,30 +2602,47 @@ mod tests {
bytes: Arc<Vec<u8>>,
get_count: Arc<AtomicUsize>,
) {
- let mut reader = BufReader::new(stream.try_clone().expect("clone test stream"));
- let mut request_line = String::new();
- if reader.read_line(&mut request_line).is_err() {
+ let Some(request) = read_test_request(&stream) else {
return;
- }
- loop {
- let mut line = String::new();
- if reader.read_line(&mut line).is_err() || line == "\r\n" || line.is_empty() {
- break;
- }
- }
+ };
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
bytes.len()
);
let _ = stream.write_all(response.as_bytes());
- if request_line.starts_with("HEAD ") {
+ if request.line.starts_with("HEAD ") {
return;
}
get_count.fetch_add(1, Ordering::SeqCst);
let _ = stream.write_all(bytes.as_slice());
}
+ fn handle_stalled_test_request(
+ mut stream: TcpStream,
+ bytes: Arc<Vec<u8>>,
+ release: Arc<AtomicBool>,
+ ) {
+ let Some(request) = read_test_request(&stream) else {
+ return;
+ };
+ let response = format!(
+ "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nETag: \"test-etag\"\r\nConnection: close\r\n\r\n",
+ bytes.len()
+ );
+ let _ = stream.write_all(response.as_bytes());
+ if request.line.starts_with("HEAD ") {
+ return;
+ }
+ let midpoint = bytes.len() / 2;
+ let _ = stream.write_all(&bytes[..midpoint]);
+ let _ = stream.flush();
+ while !release.load(Ordering::SeqCst) {
+ thread::sleep(Duration::from_millis(1));
+ }
+ let _ = stream.write_all(&bytes[midpoint..]);
+ }
+
fn handle_range_test_request(
mut stream: TcpStream,
bytes: Arc<Vec<u8>>,
@@ -2582,29 +2650,11 @@ mod tests {
range_get_count: Arc<AtomicUsize>,
full_get_count: Arc<AtomicUsize>,
) {
- let mut reader = BufReader::new(stream.try_clone().expect("clone test stream"));
- let mut request_line = String::new();
- if reader.read_line(&mut request_line).is_err() {
+ let Some(request) = read_test_request(&stream) else {
return;
- }
-
- let mut range_header = None;
- let mut if_range_header = None;
- loop {
- let mut line = String::new();
- if reader.read_line(&mut line).is_err() || line == "\r\n" || line.is_empty() {
- break;
- }
- if let Some((name, value)) = line.split_once(':') {
- if name.eq_ignore_ascii_case("range") {
- range_header = Some(value.trim().to_string());
- } else if name.eq_ignore_ascii_case("if-range") {
- if_range_header = Some(value.trim().to_string());
- }
- }
- }
+ };
- if request_line.starts_with("HEAD ") {
+ if request.line.starts_with("HEAD ") {
head_count.fetch_add(1, Ordering::SeqCst);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nAccept-Ranges: bytes\r\nETag: \"test-etag\"\r\nConnection: close\r\n\r\n",
@@ -2614,10 +2664,11 @@ mod tests {
return;
}
- let resume_allowed = if_range_header
+ let resume_allowed = request
+ .if_range
.as_deref()
.is_none_or(|value| value == "\"test-etag\"");
- if let Some(range_header) = range_header.filter(|_| resume_allowed) {
+ if let Some(range_header) = request.range.filter(|_| resume_allowed) {
let Some((start, end)) = parse_test_range_header(&range_header, bytes.len() as u64)
else {
let _ = stream
@@ -2646,6 +2697,36 @@ mod tests {
}
}
+ struct TestRequest {
+ line: String,
+ range: Option<String>,
+ if_range: Option<String>,
+ }
+
+ fn read_test_request(stream: &TcpStream) -> Option<TestRequest> {
+ let mut reader = BufReader::new(stream.try_clone().ok()?);
+ let mut request = TestRequest {
+ line: String::new(),
+ range: None,
+ if_range: None,
+ };
+ reader.read_line(&mut request.line).ok()?;
+ loop {
+ let mut line = String::new();
+ if reader.read_line(&mut line).ok()? == 0 || line == "\r\n" {
+ break;
+ }
+ if let Some((name, value)) = line.split_once(':') {
+ if name.eq_ignore_ascii_case("range") {
+ request.range = Some(value.trim().to_string());
+ } else if name.eq_ignore_ascii_case("if-range") {
+ request.if_range = Some(value.trim().to_string());
+ }
+ }
+ }
+ Some(request)
+ }
+
fn parse_test_range_header(value: &str, total: u64) -> Option<(u64, u64)> {
let value = value.strip_prefix("bytes=")?;
let (start, end) = value.split_once('-')?;
diff --git a/rust/crates/assets/src/store.rs b/rust/crates/assets/src/store.rs
index 69252d8b55..c8d111d38f 100644
--- a/rust/crates/assets/src/store.rs
+++ b/rust/crates/assets/src/store.rs
@@ -184,7 +184,7 @@ impl AssetStore {
Ok(())
}
- pub async fn remove(&self, asset: &Asset) -> Result<(), AssetStoreError> {
+ pub fn remove(&self, asset: &Asset) -> Result<(), AssetStoreError> {
let lock = self.asset_lock(asset)?;
let _guard = lock.try_lock().map_err(|_| AssetStoreError::Busy)?;
remove_path(&self.asset_dir(asset))?;
@@ -514,7 +514,6 @@ mod tests {
CancellationToken::default(),
));
assert_send(store.estimated_download_size(std::slice::from_ref(&asset)));
- assert_send(store.remove(&asset));
}
#[test]
@@ -640,10 +639,7 @@ mod tests {
));
let lock = store.asset_lock(&asset).unwrap();
let guard = lock.lock().await;
- assert!(matches!(
- store.remove(&asset).await,
- Err(AssetStoreError::Busy)
- ));
+ assert!(matches!(store.remove(&asset), Err(AssetStoreError::Busy)));
let token = CancellationToken::new();
let cancellation = token.clone();
@@ -799,10 +795,10 @@ mod tests {
fs::create_dir_all(&staging).unwrap();
fs::write(staging.join("model.onnx.tmp"), b"partial").unwrap();
- store.remove(&asset).await.unwrap();
+ store.remove(&asset).unwrap();
assert!(!final_dir.exists());
assert!(!staging.exists());
- store.remove(&asset).await.unwrap();
+ store.remove(&asset).unwrap();
let _ = fs::remove_dir_all(root);
}