π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
use std::collections::{HashMap, HashSet};
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::{Arc, OnceLock, Weak};
use futures_util::future::join_all;
use tokio::sync::{Mutex, Semaphore};
use crate::archive;
use crate::download::{self, CancellationToken, DownloadTarget, Downloader, Error, Progress};
const STAGING_DIR: &str = ".staging";
const ARCHIVE_FILE: &str = ".archive.tar.gz";
const EXTRACTION_DIR: &str = ".extracted";
const ASSET_CONCURRENCY: usize = 2;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Asset {
components: Vec<String>,
kind: AssetKind,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum AssetKind {
Files(Vec<AssetFile>),
TarGz {
url: String,
size: u64,
sha256: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AssetFile {
pub name: String,
pub url: String,
pub size: u64,
pub sha256: String,
}
#[derive(Debug, Clone)]
pub struct AssetDownloadProgress {
pub asset_index: usize,
pub asset_progress: Progress,
pub queued: bool,
pub batch_downloaded_bytes: u64,
pub batch_total_bytes: Option<u64>,
pub batch_percentage: f64,
}
#[derive(Debug, thiserror::Error)]
pub enum AssetStoreError {
#[error(transparent)]
Download(#[from] Error),
#[error("asset is busy")]
Busy,
}
impl Asset {
pub fn file(components: Vec<String>, file: AssetFile) -> Result<Self, Error> {
Self::files(components, vec![file])
}
pub fn files(components: Vec<String>, files: Vec<AssetFile>) -> Result<Self, Error> {
validate_components(&components)?;
if files.is_empty() {
return Err(Error::InvalidTarget("asset has no files".to_string()));
}
let mut names = HashSet::new();
for file in &files {
validate_component(&file.name)?;
download::validate_sha256(&file.sha256)?;
if !names.insert(file.name.as_str()) {
return Err(Error::InvalidTarget(format!(
"asset has a duplicate file name {}",
file.name
)));
}
}
Ok(Self {
components,
kind: AssetKind::Files(files),
})
}
pub fn tar_gz(
components: Vec<String>,
url: String,
size: u64,
sha256: String,
) -> Result<Self, Error> {
validate_components(&components)?;
download::validate_sha256(&sha256)?;
Ok(Self {
components,
kind: AssetKind::TarGz { url, size, sha256 },
})
}
}
pub struct AssetStore {
root: PathBuf,
coordination: Arc<StoreCoordination>,
}
struct StoreCoordination {
downloader: OnceLock<Downloader>,
asset_locks: std::sync::Mutex<HashMap<Vec<String>, Arc<Mutex<()>>>>,
asset_slots: Semaphore,
}
impl AssetStore {
pub fn new(root: impl Into<PathBuf>) -> Self {
let root = root.into();
let registry_root = normalized_root(root.clone());
static STORES: OnceLock<std::sync::Mutex<HashMap<PathBuf, Weak<StoreCoordination>>>> =
OnceLock::new();
#[expect(
clippy::expect_used,
reason = "The registry critical section invokes no caller code"
)]
let mut stores = STORES
.get_or_init(Default::default)
.lock()
.expect("store registry");
stores.retain(|_, store| store.strong_count() > 0);
let coordination = stores
.get(®istry_root)
.and_then(Weak::upgrade)
.unwrap_or_else(|| {
let coordination = Arc::new(StoreCoordination {
downloader: OnceLock::new(),
asset_locks: std::sync::Mutex::new(HashMap::new()),
asset_slots: Semaphore::new(ASSET_CONCURRENCY),
});
stores.insert(registry_root, Arc::downgrade(&coordination));
coordination
});
Self { root, coordination }
}
pub fn asset_dir(&self, asset: &Asset) -> PathBuf {
join_components(&self.root, &asset.components)
}
pub fn file_path(&self, asset: &Asset, name: &str) -> Option<PathBuf> {
match &asset.kind {
AssetKind::Files(files) if files.iter().any(|file| file.name == name) => {
Some(self.asset_dir(asset).join(name))
}
_ => None,
}
}
pub fn staged_file_path(&self, asset: &Asset, name: &str) -> Option<PathBuf> {
self.file_path(asset, name)?;
Some(self.staging_dir(asset).join(name))
}
pub fn is_downloaded(&self, asset: &Asset) -> bool {
let directory = self.asset_dir(asset);
match &asset.kind {
AssetKind::Files(files) => files
.iter()
.all(|file| is_regular_file(&directory.join(&file.name))),
AssetKind::TarGz { .. } => is_directory(&directory),
}
}
pub async fn estimated_download_size(&self, assets: &[Asset]) -> Option<u64> {
let mut total = 0u64;
for asset in assets {
if self.is_downloaded(asset) {
continue;
}
total = total.checked_add(
self.downloader()
.ok()?
.estimated_download_size(&self.download_targets(asset))
.await?,
)?;
}
Some(total)
}
pub async fn download(
&self,
assets: &[Asset],
on_progress: impl FnMut(AssetDownloadProgress) + Send,
cancellation: CancellationToken,
) -> Result<(), Error> {
self.download_with_staged_import(assets, on_progress, cancellation, |_, _| Ok(()))
.await
}
pub async fn download_with_staged_import(
&self,
assets: &[Asset],
on_progress: impl FnMut(AssetDownloadProgress) + Send,
cancellation: CancellationToken,
import: impl Fn(&Asset, &Path) -> Result<(), Error> + Sync,
) -> Result<(), Error> {
if cancellation.is_cancelled() {
return Err(Error::Cancelled);
}
let mut keys = HashSet::new();
for asset in assets {
if !keys.insert(&asset.components) {
return Err(Error::InvalidTarget(format!(
"duplicate asset key {}",
asset.components.join("/")
)));
}
}
let locks = assets
.iter()
.map(|asset| self.asset_lock(asset))
.collect::<Result<Vec<_>, _>>()?;
let progress = std::sync::Mutex::new(BatchProgress::new(assets.len(), on_progress));
let progress = &progress;
let import = &import;
let cancellation = &cancellation;
let jobs =
assets
.iter()
.zip(locks)
.enumerate()
.map(|(asset_index, (asset, lock))| async move {
let result = self
.download_asset(
asset_index,
asset,
lock,
progress,
cancellation.clone(),
import,
)
.await;
let state = if result.is_ok() {
AssetDownloadState::Complete
} else {
AssetDownloadState::Failed
};
#[expect(clippy::expect_used, reason = "A panic unwinds the download")]
progress
.lock()
.expect("progress lock")
.set_state(asset_index, state);
result
});
let results = join_all(jobs).await;
if cancellation.is_cancelled() {
return Err(Error::Cancelled);
}
if let Some(error) = results
.into_iter()
.filter_map(Result::err)
.min_by_key(Error::is_retryable)
{
return Err(error);
}
Ok(())
}
pub fn remove(&self, asset: &Asset) -> Result<(), AssetStoreError> {
self.remove_key(&asset.components)
}
pub fn remove_keys(&self, keys: &[Vec<String>]) -> Result<(), AssetStoreError> {
keys.iter().try_for_each(|key| self.remove_key(key))
}
async fn download_asset<F, I>(
&self,
asset_index: usize,
asset: &Asset,
lock: Arc<Mutex<()>>,
progress: &std::sync::Mutex<BatchProgress<F>>,
cancellation: CancellationToken,
import: &I,
) -> Result<(), Error>
where
F: FnMut(AssetDownloadProgress) + Send,
I: Fn(&Asset, &Path) -> Result<(), Error> + Sync,
{
#[expect(clippy::expect_used, reason = "A panic unwinds the download")]
progress
.lock()
.expect("progress lock")
.set_state(asset_index, AssetDownloadState::Queued);
let _guard = tokio::select! {
guard = lock.lock() => guard,
_ = cancellation.cancelled() => return Err(Error::Cancelled),
};
if self.is_downloaded(asset) {
return Ok(());
}
import(asset, &self.staging_dir(asset))?;
#[expect(
clippy::expect_used,
reason = "The private asset semaphore is never closed"
)]
let _slot = tokio::select! {
slot = self.coordination.asset_slots.acquire() => slot.expect("asset semaphore closed"),
_ = cancellation.cancelled() => return Err(Error::Cancelled),
};
#[expect(clippy::expect_used, reason = "A panic unwinds the download")]
progress
.lock()
.expect("progress lock")
.set_state(asset_index, AssetDownloadState::Downloading);
self.downloader()?
.download(
self.download_targets(asset),
|update| {
#[expect(
clippy::expect_used,
reason = "A panic unwinds the download; its local state is not reused"
)]
progress
.lock()
.expect("progress lock")
.update(asset_index, update);
},
cancellation.clone(),
)
.await?;
match &asset.kind {
AssetKind::Files(files) => self.publish_files(asset, files)?,
AssetKind::TarGz { .. } => self.publish_archive(asset, cancellation).await?,
}
Ok(())
}
fn asset_lock(&self, asset: &Asset) -> Result<Arc<Mutex<()>>, Error> {
self.key_lock(&asset.components)
}
fn key_lock(&self, components: &[String]) -> Result<Arc<Mutex<()>>, Error> {
#[expect(
clippy::expect_used,
reason = "The registry critical section invokes no caller code"
)]
let mut locks = self
.coordination
.asset_locks
.lock()
.expect("asset lock registry");
if let Some((key, lock)) = locks.iter().find(|(key, _)| keys_overlap(key, components)) {
if key == components {
return Ok(Arc::clone(lock));
}
return Err(Error::InvalidTarget(format!(
"asset keys {} and {} overlap",
key.join("/"),
components.join("/")
)));
}
let lock = Arc::new(Mutex::new(()));
locks.insert(components.to_vec(), Arc::clone(&lock));
Ok(lock)
}
fn remove_key(&self, key: &[String]) -> Result<(), AssetStoreError> {
validate_components(key)?;
let lock = self.key_lock(key)?;
let _guard = lock.try_lock().map_err(|_| AssetStoreError::Busy)?;
remove_path(&join_components(&self.root, key))?;
remove_path(&join_components(&self.root.join(STAGING_DIR), key))?;
Ok(())
}
fn downloader(&self) -> Result<&Downloader, Error> {
if let Some(downloader) = self.coordination.downloader.get() {
return Ok(downloader);
}
let downloader = Downloader::new()?;
Ok(self.coordination.downloader.get_or_init(|| downloader))
}
fn staging_dir(&self, asset: &Asset) -> PathBuf {
join_components(&self.root.join(STAGING_DIR), &asset.components)
}
fn download_targets(&self, asset: &Asset) -> Vec<DownloadTarget> {
let staging = self.staging_dir(asset);
match &asset.kind {
AssetKind::Files(files) => files
.iter()
.map(|file| DownloadTarget {
label: file.name.clone(),
url: file.url.clone(),
expected_size: file.size,
sha256: file.sha256.clone(),
destination: staging.join(&file.name),
})
.collect(),
AssetKind::TarGz { url, size, sha256 } => vec![DownloadTarget {
label: asset.components.last().cloned().unwrap_or_default(),
url: url.clone(),
expected_size: *size,
sha256: sha256.clone(),
destination: staging.join(ARCHIVE_FILE),
}],
}
}
fn publish_files(&self, asset: &Asset, files: &[AssetFile]) -> Result<(), Error> {
let staging = self.staging_dir(asset);
for file in files {
if !is_regular_file(&staging.join(&file.name)) {
return Err(Error::Validation(format!(
"{} is not a regular file",
staging.join(&file.name).display()
)));
}
}
let expected = files
.iter()
.map(|file| file.name.as_str())
.collect::<HashSet<_>>();
for entry in fs::read_dir(&staging)? {
let entry = entry?;
let name = entry.file_name();
if name.to_str().is_some_and(|name| expected.contains(name)) {
if !is_regular_file(&entry.path()) {
return Err(Error::Validation(format!(
"{} is not a regular file",
entry.path().display()
)));
}
} else {
remove_path(&entry.path())?;
}
}
self.publish_directory(&staging, &self.asset_dir(asset))
}
async fn publish_archive(
&self,
asset: &Asset,
cancellation: CancellationToken,
) -> Result<(), Error> {
let staging = self.staging_dir(asset);
let archive_path = staging.join(ARCHIVE_FILE);
let extraction_dir = staging.join(EXTRACTION_DIR);
let extraction_for_task = extraction_dir.clone();
let extraction_cancellation = cancellation.clone();
let source = tokio::task::spawn_blocking(move || {
let result = archive::extract_tar_gz(
&archive_path,
&extraction_for_task,
&extraction_cancellation,
);
if result.is_err() {
let _ = fs::remove_dir_all(&extraction_for_task);
}
result
})
.await
.map_err(|error| Error::Io(std::io::Error::other(error)))??;
if cancellation.is_cancelled() {
remove_path(&extraction_dir)?;
return Err(Error::Cancelled);
}
self.publish_directory(&source, &self.asset_dir(asset))?;
remove_path(&staging)?;
Ok(())
}
fn publish_directory(&self, source: &Path, destination: &Path) -> Result<(), Error> {
let parent = destination
.parent()
.ok_or_else(|| Error::InvalidTarget(destination.display().to_string()))?;
fs::create_dir_all(parent)?;
remove_path(destination)?;
fs::rename(source, destination)?;
Ok(())
}
}
fn normalized_root(root: PathBuf) -> PathBuf {
let absolute = std::path::absolute(&root).unwrap_or(root);
let mut ancestor = absolute.as_path();
let mut missing = Vec::new();
loop {
if let Ok(mut canonical) = fs::canonicalize(ancestor) {
for component in missing.into_iter().rev() {
canonical.push(component);
}
return canonical;
}
let Some(name) = ancestor.file_name() else {
return absolute;
};
missing.push(name.to_os_string());
let Some(parent) = ancestor.parent() else {
return absolute;
};
ancestor = parent;
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum AssetDownloadState {
Queued,
Downloading,
Complete,
Failed,
}
struct BatchProgress<F> {
assets: Vec<Option<Progress>>,
states: Vec<AssetDownloadState>,
callback: F,
}
impl<F> BatchProgress<F>
where
F: FnMut(AssetDownloadProgress),
{
fn new(asset_count: usize, callback: F) -> Self {
Self {
assets: vec![None; asset_count],
states: vec![AssetDownloadState::Queued; asset_count],
callback,
}
}
fn set_state(&mut self, asset_index: usize, state: AssetDownloadState) {
self.states[asset_index] = state;
self.emit(
asset_index,
self.assets[asset_index].clone().unwrap_or_default(),
);
}
fn update(&mut self, asset_index: usize, asset_progress: Progress) {
self.assets[asset_index] = Some(asset_progress.clone());
self.emit(asset_index, asset_progress);
}
fn emit(&mut self, asset_index: usize, mut asset_progress: Progress) {
let queued = self.states.contains(&AssetDownloadState::Queued)
&& !self.states.contains(&AssetDownloadState::Downloading);
if queued {
asset_progress.label = "Download queued".to_string();
} else if asset_progress.label.is_empty() {
asset_progress.label = "Preparing downloads".to_string();
}
let batch_downloaded_bytes = self
.assets
.iter()
.flatten()
.map(|progress| progress.downloaded_bytes)
.sum();
let batch_total_bytes = self
.assets
.iter()
.zip(&self.states)
.map(|(progress, state)| match progress {
Some(progress) => progress.total_bytes,
None if *state == AssetDownloadState::Complete => Some(0),
None => None,
})
.try_fold(0u64, |total, value| total.checked_add(value?));
let batch_percentage = batch_total_bytes
.filter(|total| *total > 0)
.map(|total| ((batch_downloaded_bytes as f64 / total as f64) * 100.0).clamp(0.0, 100.0))
.unwrap_or(0.0);
(self.callback)(AssetDownloadProgress {
asset_index,
asset_progress,
queued,
batch_downloaded_bytes,
batch_total_bytes,
batch_percentage,
});
}
}
fn keys_overlap(left: &[String], right: &[String]) -> bool {
left.starts_with(right) || right.starts_with(left)
}
fn validate_components(components: &[String]) -> Result<(), Error> {
if components.is_empty() {
return Err(Error::InvalidTarget(
"asset key has no components".to_string(),
));
}
for component in components {
validate_component(component)?;
}
if components[0] == STAGING_DIR {
return Err(Error::InvalidTarget(format!("'{STAGING_DIR}' is reserved")));
}
Ok(())
}
fn validate_component(component: &str) -> Result<(), Error> {
if !component.is_empty()
&& component != "."
&& component != ".."
&& !component.ends_with([' ', '.'])
&& !component.chars().any(|character| {
character.is_ascii_control()
|| matches!(
character,
'<' | '>' | ':' | '"' | '/' | '\\' | '|' | '?' | '*'
)
})
{
return Ok(());
}
Err(Error::InvalidTarget(format!(
"'{component}' is not a safe path component"
)))
}
fn join_components(root: &Path, components: &[String]) -> PathBuf {
components
.iter()
.fold(root.to_path_buf(), |path, component| path.join(component))
}
fn is_regular_file(path: &Path) -> bool {
fs::symlink_metadata(path)
.map(|metadata| metadata.file_type().is_file())
.unwrap_or(false)
}
fn is_directory(path: &Path) -> bool {
fs::symlink_metadata(path)
.map(|metadata| metadata.file_type().is_dir())
.unwrap_or(false)
}
fn remove_path(path: &Path) -> Result<(), Error> {
let metadata = match fs::symlink_metadata(path) {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error.into()),
};
if metadata.file_type().is_dir() {
fs::remove_dir_all(path)?;
} else {
fs::remove_file(path)?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::io::{Cursor, Read, Write};
use std::net::TcpListener;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::thread;
use std::time::Duration;
use std::time::{SystemTime, UNIX_EPOCH};
use flate2::Compression;
use flate2::write::GzEncoder;
use sha2::{Digest, Sha256};
use tar::{Builder, EntryType, Header};
use super::*;
#[tokio::test]
async fn independently_created_stores_share_a_transfer_and_allow_retry() {
let root = scratch_dir("shared-transfer");
let first = AssetStore::new(&root);
fs::create_dir_all(&root).unwrap();
let second = AssetStore::new(root.join("."));
assert!(Arc::ptr_eq(&first.coordination, &second.coordination));
let server = TestServer::new();
let mut model = file("model", b"model");
model.url = server.url.clone();
let asset = Asset::file(key(&["models", "shared"]), model).unwrap();
let (left, right) = tokio::join!(
first.download(
std::slice::from_ref(&asset),
|_| {},
CancellationToken::default()
),
second.download(
std::slice::from_ref(&asset),
|_| {},
CancellationToken::default()
),
);
left.unwrap();
right.unwrap();
assert_eq!(server.transfers.load(Ordering::SeqCst), 1);
assert!(second.is_downloaded(&asset));
fs::write(
first.asset_dir(&asset).join("model"),
b"trusted published bytes",
)
.unwrap();
second
.download(
std::slice::from_ref(&asset),
|_| {},
CancellationToken::default(),
)
.await
.unwrap();
assert_eq!(server.transfers.load(Ordering::SeqCst), 1);
assert_eq!(
fs::read(first.asset_dir(&asset).join("model")).unwrap(),
b"trusted published bytes"
);
first.remove(&asset).unwrap();
server.fail.store(true, Ordering::SeqCst);
assert!(
first
.download(
std::slice::from_ref(&asset),
|_| {},
CancellationToken::default()
)
.await
.is_err()
);
let transfers_after_failure = server.transfers.load(Ordering::SeqCst);
assert!(transfers_after_failure > 1);
server.fail.store(false, Ordering::SeqCst);
second
.download(
std::slice::from_ref(&asset),
|_| {},
CancellationToken::default(),
)
.await
.unwrap();
assert_eq!(
server.transfers.load(Ordering::SeqCst),
transfers_after_failure + 1
);
assert_eq!(
fs::read(first.asset_dir(&asset).join("model")).unwrap(),
b"model"
);
drop(server);
fs::remove_dir_all(root).unwrap();
}
#[tokio::test]
async fn removal_uses_the_shared_download_lock() {
let root = scratch_dir("shared-removal");
let first = AssetStore::new(&root);
let second = AssetStore::new(&root);
let asset = Asset::file(key(&["models", "shared"]), file("model", b"model")).unwrap();
let lock = first.asset_lock(&asset).unwrap();
let guard = lock.lock().await;
assert!(matches!(second.remove(&asset), Err(AssetStoreError::Busy)));
drop(guard);
second.remove(&asset).unwrap();
}
struct TestServer {
url: String,
transfers: Arc<AtomicUsize>,
fail: Arc<AtomicBool>,
running: Arc<AtomicBool>,
worker: Option<thread::JoinHandle<()>>,
}
impl TestServer {
fn new() -> Self {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let url = format!("http://{}/model", listener.local_addr().unwrap());
let transfers = Arc::new(AtomicUsize::new(0));
let fail = Arc::new(AtomicBool::new(false));
let running = Arc::new(AtomicBool::new(true));
let worker = {
let transfers = Arc::clone(&transfers);
let fail = Arc::clone(&fail);
let running = Arc::clone(&running);
thread::spawn(move || {
while running.load(Ordering::SeqCst) {
let (mut stream, _) = match listener.accept() {
Ok(connection) => connection,
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
thread::sleep(Duration::from_millis(2));
continue;
}
Err(error) => panic!("test server: {error}"),
};
stream.set_nonblocking(false).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
let mut request = Vec::new();
while !request.ends_with(b"\r\n\r\n") {
let mut byte = [0];
stream.read_exact(&mut byte).unwrap();
request.push(byte[0]);
}
let get = request.starts_with(b"GET ");
if get {
transfers.fetch_add(1, Ordering::SeqCst);
}
if get && fail.load(Ordering::SeqCst) {
stream.write_all(b"HTTP/1.1 410 Gone\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap();
} else {
stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\n").unwrap();
if get {
thread::sleep(Duration::from_millis(20));
stream.write_all(b"model").unwrap();
}
}
}
})
};
Self {
url,
transfers,
fail,
running,
worker: Some(worker),
}
}
}
impl Drop for TestServer {
fn drop(&mut self) {
self.running.store(false, Ordering::SeqCst);
self.worker.take().unwrap().join().unwrap();
}
}
#[test]
fn store_and_operations_are_send_and_sync() {
fn assert_send<T: Send>(_: T) {}
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<AssetStore>();
assert_send_sync::<CancellationToken>();
let store = AssetStore::new("assets");
let asset =
Asset::files(key(&["models", "clip"]), vec![file("model.onnx", b"model")]).unwrap();
assert_send(store.download(
std::slice::from_ref(&asset),
|_| {},
CancellationToken::default(),
));
assert_send(store.estimated_download_size(std::slice::from_ref(&asset)));
}
#[test]
fn rejects_invalid_assets() {
assert!(Asset::files(Vec::new(), vec![file("model", b"model")]).is_err());
assert!(Asset::files(key(&[".staging", "model"]), vec![file("model", b"model")]).is_err());
assert!(Asset::files(key(&["models", ".."]), vec![file("model", b"model")]).is_err());
for component in [
"less<than",
"greater>than",
"colon:name",
"quoted\"name",
"pipe|name",
"question?",
"star*name",
"nested/name",
"nested\\name",
"control\u{1f}",
"trailing.",
"trailing ",
] {
assert!(Asset::file(key(&["models", component]), file("model", b"model")).is_err());
assert!(Asset::file(key(&["models", "model"]), file(component, b"model")).is_err());
}
assert!(Asset::files(key(&["models", "model"]), Vec::new()).is_err());
assert!(
Asset::files(
key(&["models", "model"]),
vec![file("model", b"a"), file("model", b"b")]
)
.is_err()
);
let mut invalid_sha = file("model", b"model");
invalid_sha.sha256 = "invalid".to_string();
assert!(Asset::files(key(&["models", "model"]), vec![invalid_sha]).is_err());
}
#[tokio::test]
async fn publishes_all_files() {
let root = scratch_dir("files");
let store = AssetStore::new(&root);
let asset = Asset::files(
key(&["models", "clip"]),
vec![file("model.onnx", b"model"), file("vocab.txt", b"vocab")],
)
.unwrap();
stage_files(
&store,
&asset,
&[("model.onnx", b"model"), ("vocab.txt", b"vocab")],
);
let final_dir = store.asset_dir(&asset);
store
.download(
std::slice::from_ref(&asset),
|_| {},
CancellationToken::default(),
)
.await
.unwrap();
assert!(store.is_downloaded(&asset));
assert_eq!(fs::read(final_dir.join("model.onnx")).unwrap(), b"model");
assert_eq!(fs::read(final_dir.join("vocab.txt")).unwrap(), b"vocab");
assert!(!store.staging_dir(&asset).exists());
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn aggregates_progress_across_assets() {
let root = scratch_dir("batch");
let store = AssetStore::new(&root);
let first = Asset::files(key(&["models", "first"]), vec![file("model", b"first")]).unwrap();
let second =
Asset::files(key(&["models", "second"]), vec![file("model", b"second")]).unwrap();
stage_files(&store, &first, &[("model", b"first")]);
stage_files(&store, &second, &[("model", b"second")]);
let slots = store
.coordination
.asset_slots
.acquire_many(ASSET_CONCURRENCY as u32)
.await
.unwrap();
let second_lock = store.asset_lock(&second).unwrap();
let second_guard = second_lock.lock().await;
let (updates, mut receiver) = tokio::sync::mpsc::unbounded_channel();
let assets = [first.clone(), second.clone()];
let download = store.download(
&assets,
|progress| updates.send(progress).unwrap(),
CancellationToken::default(),
);
let mut asset_progress = [0, 0];
let observe_queue = async {
assert!(receiver.recv().await.unwrap().queued);
drop(slots);
let mut started = false;
loop {
let progress = receiver.recv().await.unwrap();
asset_progress[progress.asset_index] = progress.asset_progress.downloaded_bytes;
if !progress.queued {
started = true;
} else if progress.batch_downloaded_bytes == 5 {
assert!(started);
assert_eq!(progress.asset_progress.label, "Download queued");
break;
}
}
drop(second_guard);
};
let (result, ()) = tokio::time::timeout(std::time::Duration::from_secs(5), async {
tokio::join!(download, observe_queue)
})
.await
.unwrap();
result.unwrap();
let mut latest_progress = None;
while let Ok(progress) = receiver.try_recv() {
assert!(!progress.queued);
asset_progress[progress.asset_index] = progress.asset_progress.downloaded_bytes;
latest_progress = Some((
progress.batch_downloaded_bytes,
progress.batch_total_bytes,
progress.batch_percentage,
));
}
assert!(store.is_downloaded(&first));
assert!(store.is_downloaded(&second));
assert_eq!(asset_progress, [5, 6]);
assert_eq!(latest_progress, Some((11, Some(11), 100.0)));
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn publishes_successful_assets_when_a_sibling_fails() {
let root = scratch_dir("batch-failure");
let store = AssetStore::new(&root);
let good = Asset::files(key(&["models", "good"]), vec![file("model", b"good")]).unwrap();
let mut bad_file = file("model", b"bad");
bad_file.url = ":".to_string();
let bad = Asset::files(key(&["models", "bad"]), vec![bad_file]).unwrap();
for fail_before_progress in [false, true] {
stage_files(&store, &good, &[("model", b"good")]);
let mut latest_progress = None;
store
.download_with_staged_import(
&[good.clone(), bad.clone()],
|progress| {
assert!(progress.batch_percentage < 100.0);
latest_progress = Some(progress);
},
CancellationToken::default(),
|asset, _| {
if fail_before_progress && asset == &bad {
Err(Error::StorageFull)
} else {
Ok(())
}
},
)
.await
.unwrap_err();
let progress = latest_progress.unwrap();
assert!(!progress.queued);
assert_eq!(progress.batch_downloaded_bytes, 4);
assert_eq!(
progress.batch_total_bytes,
if fail_before_progress { None } else { Some(7) }
);
assert!(store.is_downloaded(&good));
assert!(!store.is_downloaded(&bad));
store.remove(&good).unwrap();
}
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn asset_locks_are_keyed_and_cancellable() {
let root = scratch_dir("locks");
let store = Arc::new(AssetStore::new(&root));
let asset = Asset::files(key(&["models", "same"]), vec![file("model", b"model")]).unwrap();
assert!(matches!(
store
.download(
&[asset.clone(), asset.clone()],
|_| {},
CancellationToken::default()
)
.await,
Err(Error::InvalidTarget(_))
));
let lock = store.asset_lock(&asset).unwrap();
let guard = lock.lock().await;
assert!(matches!(store.remove(&asset), Err(AssetStoreError::Busy)));
let token = CancellationToken::new();
let cancellation = token.clone();
let waiting_store = Arc::clone(&store);
let waiting_asset = asset.clone();
let waiter = tokio::spawn(async move {
let mut updates = Vec::new();
let result = waiting_store
.download(&[waiting_asset], |progress| updates.push(progress), token)
.await;
(result, updates)
});
tokio::task::yield_now().await;
cancellation.cancel();
let (result, updates) = waiter.await.unwrap();
assert!(matches!(result, Err(Error::Cancelled)));
assert!(!updates.last().unwrap().queued);
assert!(updates.iter().all(|progress| {
progress.batch_total_bytes.is_none() && progress.batch_percentage == 0.0
}));
drop(guard);
stage_files(&store, &asset, &[("model", b"model")]);
let first_assets = [asset.clone()];
let second_assets = [asset.clone()];
let (first, second) = tokio::join!(
store.download(&first_assets, |_| {}, CancellationToken::default()),
store.download(&second_assets, |_| {}, CancellationToken::default())
);
first.unwrap();
second.unwrap();
assert!(store.is_downloaded(&asset));
let nested = Asset::files(
key(&["models", "same", "nested"]),
vec![file("model", b"model")],
)
.unwrap();
assert!(store.asset_lock(&nested).is_err());
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn ignores_derived_files_and_discards_them_when_rebuilding() {
let root = scratch_dir("derived");
let store = AssetStore::new(&root);
let asset = Asset::files(
key(&["onnx-runtime", "macos"]),
vec![file("runtime.tgz", b"archive")],
)
.unwrap();
stage_files(&store, &asset, &[("runtime.tgz", b"archive")]);
store
.download(
std::slice::from_ref(&asset),
|_| {},
CancellationToken::default(),
)
.await
.unwrap();
let directory = store.asset_dir(&asset);
fs::write(directory.join("libonnxruntime.dylib"), b"derived").unwrap();
assert!(store.is_downloaded(&asset));
fs::remove_file(directory.join("runtime.tgz")).unwrap();
assert!(!store.is_downloaded(&asset));
stage_files(&store, &asset, &[("runtime.tgz", b"archive")]);
store
.download(
std::slice::from_ref(&asset),
|_| {},
CancellationToken::default(),
)
.await
.unwrap();
assert!(!directory.join("libonnxruntime.dylib").exists());
assert!(store.is_downloaded(&asset));
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn extracts_and_publishes_tar_gz() {
let root = scratch_dir("archive");
let store = AssetStore::new(&root);
let archive = tar_gz(
"parakeet",
&[("encoder.onnx", b"encoder"), ("tokens.txt", b"tokens")],
);
let asset = Asset::tar_gz(
key(&["models", "parakeet"]),
"https://example.invalid/parakeet.tar.gz".to_string(),
archive.len() as u64,
sha(&archive),
)
.unwrap();
let staging = store.staging_dir(&asset);
fs::create_dir_all(&staging).unwrap();
fs::write(staging.join(ARCHIVE_FILE), archive).unwrap();
store
.download(
std::slice::from_ref(&asset),
|_| {},
CancellationToken::default(),
)
.await
.unwrap();
let directory = store.asset_dir(&asset);
assert_eq!(
fs::read(directory.join("encoder.onnx")).unwrap(),
b"encoder"
);
assert_eq!(fs::read(directory.join("tokens.txt")).unwrap(), b"tokens");
assert!(!staging.exists());
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn rejects_tar_links_without_publishing() {
let root = scratch_dir("archive-link");
let store = AssetStore::new(&root);
let archive = tar_gz_with_link();
let asset = Asset::tar_gz(
key(&["models", "linked"]),
"https://example.invalid/linked.tar.gz".to_string(),
archive.len() as u64,
sha(&archive),
)
.unwrap();
let staging = store.staging_dir(&asset);
fs::create_dir_all(&staging).unwrap();
fs::write(staging.join(ARCHIVE_FILE), archive).unwrap();
assert!(
store
.download(
std::slice::from_ref(&asset),
|_| {},
CancellationToken::default(),
)
.await
.is_err()
);
assert!(!store.asset_dir(&asset).exists());
assert!(staging.join(ARCHIVE_FILE).is_file());
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn remove_keys_deletes_published_and_staged_data() {
let root = scratch_dir("remove");
let store = AssetStore::new(&root);
let asset =
Asset::files(key(&["models", "clip"]), vec![file("model.onnx", b"model")]).unwrap();
let final_dir = store.asset_dir(&asset);
let staging = store.staging_dir(&asset);
fs::create_dir_all(&final_dir).unwrap();
fs::write(final_dir.join("model.onnx"), b"model").unwrap();
fs::create_dir_all(&staging).unwrap();
fs::write(staging.join("model.onnx.tmp"), b"partial").unwrap();
let keys = vec![key(&["models", "clip"])];
store.remove_keys(&keys).unwrap();
assert!(!final_dir.exists());
assert!(!staging.exists());
store.remove_keys(&keys).unwrap();
assert!(matches!(
store.remove_keys(&[key(&[".."])]),
Err(AssetStoreError::Download(Error::InvalidTarget(_)))
));
let _ = fs::remove_dir_all(root);
}
fn key(components: &[&str]) -> Vec<String> {
components.iter().map(ToString::to_string).collect()
}
fn file(name: &str, bytes: &[u8]) -> AssetFile {
AssetFile {
name: name.to_string(),
url: format!("https://example.invalid/{name}"),
size: bytes.len() as u64,
sha256: sha(bytes),
}
}
fn stage_files(store: &AssetStore, asset: &Asset, files: &[(&str, &[u8])]) {
let directory = store.staging_dir(asset);
fs::create_dir_all(&directory).unwrap();
for (name, bytes) in files {
fs::write(directory.join(name), bytes).unwrap();
}
}
fn sha(bytes: &[u8]) -> String {
Sha256::digest(bytes)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
fn tar_gz(root: &str, files: &[(&str, &[u8])]) -> Vec<u8> {
let mut bytes = Vec::new();
{
let encoder = GzEncoder::new(&mut bytes, Compression::default());
let mut archive = Builder::new(encoder);
for (name, contents) in files {
let mut header = Header::new_gnu();
header.set_size(contents.len() as u64);
header.set_mode(0o644);
header.set_cksum();
archive
.append_data(
&mut header,
format!("{root}/{name}"),
Cursor::new(*contents),
)
.unwrap();
}
archive.into_inner().unwrap().finish().unwrap();
}
bytes
}
fn tar_gz_with_link() -> Vec<u8> {
let mut bytes = Vec::new();
{
let encoder = GzEncoder::new(&mut bytes, Compression::default());
let mut archive = Builder::new(encoder);
let mut header = Header::new_gnu();
header.set_entry_type(EntryType::Symlink);
header.set_size(0);
header.set_mode(0o777);
header.set_link_name("../outside").unwrap();
header.set_cksum();
archive
.append_data(&mut header, "model/link", Cursor::new(Vec::<u8>::new()))
.unwrap();
archive.into_inner().unwrap().finish().unwrap();
}
bytes
}
fn scratch_dir(name: &str) -> PathBuf {
let suffix = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
std::env::temp_dir().join(format!("ente-assets-{name}-{suffix}"))
}
}