π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Extract Photos export engine
Manav Rathi0f9daacd844d
diff --git a/.github/scripts/ci/paths.json b/.github/scripts/ci/paths.jsonindex a079752e02..cb48958bea 100644--- a/.github/scripts/ci/paths.json+++ b/.github/scripts/ci/paths.json@@ -114,6 +114,7 @@"rust/crates/exif/**","rust/crates/location/**","rust/crates/photos/**",+ "rust/crates/photos-export/**","rust/crates/test-support/**","rust/crates/vecdb/**"],diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 16d63b62bc..c40c0561bc 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2450,9 +2450,9 @@ dependencies = ["ente-collections","ente-core","ente-photos",+ "ente-photos-export","ente-test-support","futures-util",- "icu_casemap","keyring","mockito","reqwest 0.12.28",@@ -2462,7 +2462,6 @@ dependencies = ["tempfile","thiserror 2.0.18","tokio",- "unicode-normalization","url","uuid","zeroize",@@ -2777,7 +2776,6 @@ dependencies = [name = "ente-photos"version = "0.0.0"dependencies = [- "chrono","ente-assets","ente-collections","ente-core",@@ -2792,6 +2790,25 @@ dependencies = ["serde_json","tempfile","thiserror 2.0.18",+]++[[package]]+name = "ente-photos-export"+version = "0.0.0"+dependencies = [+ "anyhow",+ "chrono",+ "ente-collections",+ "ente-core",+ "ente-photos",+ "icu_casemap",+ "rusqlite",+ "serde",+ "serde_json",+ "thiserror 2.0.18",+ "tokio",+ "unicode-normalization",+ "uuid","zip 8.6.0",]diff --git a/rust/Cargo.toml b/rust/Cargo.tomlindex 9a6bb04bed..055168913f 100644--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@ -38,6 +38,7 @@ members = ["crates/ml","crates/paste","crates/photos",+ "crates/photos-export","crates/space","crates/test-support","crates/vecdb",@@ -72,6 +73,7 @@ ente-locker = { path = "crates/locker" }ente-ml = { path = "crates/ml" }ente-paste = { path = "crates/paste" }ente-photos = { path = "crates/photos" }+ente-photos-export = { path = "crates/photos-export" }ente-space = { path = "crates/space" }ente-test-support = { path = "crates/test-support" }ente-uniffi-log = { path = "bindings/uniffi/log" }diff --git a/rust/apps/cli-next/Cargo.toml b/rust/apps/cli-next/Cargo.tomlindex 3891e79e6a..bb2219360f 100644--- a/rust/apps/cli-next/Cargo.toml+++ b/rust/apps/cli-next/Cargo.toml@@ -17,7 +17,7 @@ ente-accounts.workspace = trueente-collections.workspace = trueente-core.workspace = trueente-photos.workspace = true-icu_casemap.workspace = true+ente-photos-export.workspace = truekeyring = { workspace = true, features = ["apple-native", "windows-native", "linux-native-sync-persistent", "crypto-rust"] }reqwest.workspace = truerusqlite = { workspace = true, features = ["bundled-sqlcipher"] }@@ -26,7 +26,6 @@ serde_json.workspace = truetempfile.workspace = truethiserror.workspace = truetokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal", "sync", "time"] }-unicode-normalization.workspace = trueurl.workspace = trueuuid = { workspace = true, features = ["serde", "v4"] }zeroize.workspace = truediff --git a/rust/apps/cli-next/src/core_db.rs b/rust/apps/cli-next/src/core_db.rsindex a5c8765d7a..b44e22bb33 100644--- a/rust/apps/cli-next/src/core_db.rs+++ b/rust/apps/cli-next/src/core_db.rs@@ -7,7 +7,7 @@ pub use rusqlite::{#[derive(Debug, thiserror::Error)]pub enum Error {- #[error(transparent)]+ #[error("{0}")]Sqlite(#[from] SqliteError),#[error("database version {current} is newer than this build supports ({target})")]Downgrade { current: i64, target: i64 },@@ -22,10 +22,6 @@ impl Db {Self(connection)}- pub fn connection(&self) -> &Connection {- &self.0- }-pub fn migrate(&mut self, scripts: &[&str]) -> Result<()> {let target = scripts.len() as i64;let transaction = self@@ -60,3 +56,15 @@ impl Db {Ok(result)}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn sqlite_errors_preserve_their_source() {+ let error = Error::Sqlite(SqliteError::InvalidQuery);+ let source = std::error::Error::source(&error).unwrap();+ assert!(source.is::<SqliteError>());+ }+}diff --git a/rust/apps/cli-next/src/db.rs b/rust/apps/cli-next/src/db.rsindex 75171d6607..779c6befff 100644--- a/rust/apps/cli-next/src/db.rs+++ b/rust/apps/cli-next/src/db.rs@@ -10,6 +10,9 @@ const MIGRATIONS: &[&str] = &[replica::SCHEMA];pub fn open(account_dir: &Path, key: &DbKey, create: bool) -> Result<Db> {let connection = connect(&account_dir.join("data.db"), key, create)?;+ connection.execute_batch(+ "PRAGMA foreign_keys = ON; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = WAL;",+ )?;let mut db = Db::new(connection);db.migrate(MIGRATIONS)?;Ok(db)@@ -37,8 +40,5 @@ pub fn connect(path: &Path, key: &DbKey, create: bool) -> Result<Connection> {row.get::<_, i64>(0)}).context("cannot open the encrypted database")?;- connection.execute_batch(- "PRAGMA foreign_keys = ON; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = WAL;",- )?;Ok(connection)}diff --git a/rust/apps/cli-next/src/export/mod.rs b/rust/apps/cli-next/src/export/mod.rsindex 400c647303..4c3c767941 100644--- a/rust/apps/cli-next/src/export/mod.rs+++ b/rust/apps/cli-next/src/export/mod.rs@@ -1,41 +1,18 @@-mod adopt;-mod fs;-mod names;-mod reconcile;-mod store;-mod transfer;+mod source;-use std::{- fs::{File, OpenOptions},- io::Write,- path::{Path, PathBuf},- sync::Mutex,- time::{Duration, Instant},-};--use anyhow::{Context, Result, bail, ensure};-use ente_photos::export::Root;-use rusqlite::{OptionalExtension, params};-use serde_json::json;+use std::path::{Path, PathBuf};+use anyhow::{Context, Result, ensure};+use ente_core::crypto::hash;+use ente_photos_export::{AdoptionRequired, Export};use tokio::sync::watch;use crate::{api,args::{ExportArgs, Options, Product},db, home, output,- replica::{AlbumRecord, FileRecord, Replica, canonical_id},vault::State,};-use store::Store;--#[derive(Debug, thiserror::Error)]-#[error("conflict: {0}")]-pub struct Conflict(String);--#[derive(Debug, thiserror::Error)]-#[error("export cancelled")]-pub struct Cancelled;pub async fn run(args: ExportArgs, selected: Option<&str>, options: &Options) -> Result<()> {ensure!(@@ -49,20 +26,30 @@ pub async fn run(args: ExportArgs, selected: Option<&str>, options: &Options) ->runtime.block_on(run_locked(args, selected.as_deref(), as_json))}).await?+ .map_err(|error| {+ if error.is::<AdoptionRequired>() {+ error.context("this export requires --adopt")+ } else {+ error+ }+ })}async fn run_locked(args: ExportArgs, selected: Option<&str>, as_json: bool) -> Result<()> {- let destination = destination(&args.destination)?;let mut state = State::load()?;let index = state.resolve(selected)?;let account = state.accounts.swap_remove(index);let session = api::session(&account, Product::Photos)?;- let expected_root = Root::new(&session.master_key)?;- let mut root_lock = open_root(&destination, &expected_root)?;- ensure!(- !args.adopt || root_lock.is_some(),- "there is no export to adopt"- );+ let mut export = Export::open(+ &args.destination,+ &session.master_key,+ ente_photos_export::Options {+ albums: args.album,+ exclude_albums: args.exclude_album,+ adopt: args.adopt,+ jobs: args.jobs,+ },+ )?;let account_home = home::lock_account(account.storage_id, true)?;let account = State::load()?.accounts@@ -70,88 +57,15 @@ async fn run_locked(args: ExportArgs, selected: Option<&str>, as_json: bool) ->.find(|a| a.storage_id == account.storage_id).context("account was removed while waiting for access")?;let session = api::session(&account, Product::Photos)?;- ensure!(- Root::new(&session.master_key)?.source_verifier == expected_root.source_verifier,- "account keys changed while waiting for access"- );+ export.verify_source(&session.master_key)?;home::create(&account_home.path)?;- let db_path = Store::path(&account_home.path, &destination)?;- let existed = db_path.try_exists()?;+ let db_path = store_path(&account_home.path, export.destination())?;ensure!(- root_lock.is_none() || existed || args.adopt,+ !export.exists() || db_path.try_exists()? || args.adopt,"this export is not associated with this CLI home; use --adopt");- let store = Store::open(&db_path, &account.db_key, &destination, true)?;- let bound: bool =- store- .db- .connection()- .query_row("SELECT bound FROM export WHERE id=1", [], |r| r.get(0))?;- ensure!(- root_lock.is_none() || bound || args.adopt,- "this export requires --adopt"- );- store.reset_desired()?;- if root_lock.is_some() && !bound {- adopt::scan(&destination, &store)?;- }- let mut source_db = db::open(&account_home.path, &account.db_key, true)?;- let mut replica = Replica::new(&mut source_db, session.user_id);- eprintln!("Refreshing album metadata.");- replica.sync_collections(&session).await?;- let mut after = 0;- loop {- let next: Option<(i64, String, Option<String>)> = replica.db.read(|db| {- Ok(db- .query_row(- "SELECT id,record,name FROM photos_collections WHERE id>?1 ORDER BY id LIMIT 1",- [after],- |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),- )- .optional()?)- })?;- let Some((id, record, name)) = next else {- break;- };- after = id;- let record: AlbumRecord = serde_json::from_str(&record)?;- store.desired_album(&record, session.user_id)?;- if record.album(session.user_id).is_err() {- store.db.connection().execute(- "UPDATE desired_albums SET name=?1 WHERE id=?2",- params![name, id],- )?;- }- }- store- .db- .connection()- .execute(- "INSERT OR IGNORE INTO desired_albums(id,name) SELECT s.id,json_extract(s.record,'$.name') FROM album_sources s WHERE EXISTS(SELECT 1 FROM albums a WHERE a.id=s.id)",- [],- )?;- store- .db- .connection()- .execute(- "INSERT OR IGNORE INTO desired_albums(id,name) SELECT id,name FROM albums ORDER BY retained",- [],- )?;- store- .db- .connection()- .execute(- "INSERT OR IGNORE INTO desired_albums(id,name) SELECT album,name FROM pending WHERE file IS NULL AND retained=0",- [],- )?;- select(&store, &args)?;- if root_lock.is_none() {- root_lock = Some(initialize(&destination, &expected_root)?);- }- store- .db- .connection()- .execute("UPDATE export SET bound=1 WHERE id=1 AND bound=0", [])?;+ home::create(db_path.parent().context("export DB has no parent")?)?;+ let connection = db::connect(&db_path, &account.db_key, true)?;let (cancel_send, mut cancel) = watch::channel(false);let signal_send = cancel_send.clone();let signal = tokio::spawn(async move {@@ -159,617 +73,48 @@ async fn run_locked(args: ExportArgs, selected: Option<&str>, as_json: bool) ->signal_send.send_replace(true);}});- let refresh_cancel = cancel.clone();- let refreshed = tokio::select! {- result=refresh(&store,&mut replica,&session,&refresh_cancel)=>result,- _=cancel.changed()=>Err(Cancelled.into()),- };- drop(replica);- drop(source_db);- drop(account_home);- let shared = Mutex::new(store);- let run = reconcile::Run {- root: &destination,- store: &shared,- cancel: &cancel,- };- let result = (|| {- refreshed?;- run.recover()?;- run.albums()?;- transfers(- &destination,- &shared,- &session,- &cancel_send,- &cancel,- args.jobs.get(),- )?;- run.check_cancel()?;- run.retain_removed()- })();+ let report = |line: &str| eprintln!("{line}");+ let result = async {+ let mut source_db = db::open(&account_home.path, &account.db_key, true)?;+ let mut source = source::ReplicaSource::new(&mut source_db, session.user_id);+ export+ .prepare(connection, &mut source, &session, &mut cancel, &report)+ .await?;+ drop(source);+ drop(source_db);+ drop(account_home);+ export.execute(&session, &cancel_send, &cancel, &report)+ }+ .await;signal.abort();- if let Err(error) = result {- run.report("run", &error)?;- }- if let Err(error) = run.cleanup(None) {- run.report("run", &error)?;- }- let store = shared- .into_inner()- .map_err(|_| anyhow::anyhow!("export database mutex poisoned"))?;- let complete = summary(&store, &destination, as_json)?;- drop(root_lock);- ensure!(complete, "export is incomplete");- Ok(())-}--fn destination(path: &Path) -> Result<PathBuf> {- let absolute = if path.is_absolute() {- path.to_owned()+ let summary = result?;+ let progress = if summary.json["copies"].is_null() {+ "inventory incomplete".to_owned()} else {- std::env::current_dir()?.join(path)+ format!("{}/{} copies complete", summary.completed, summary.expected)};- let mut base = absolute.as_path();- let mut tail = Vec::new();- while !base.try_exists()? {- tail.push(- base.file_name()- .context("destination has no directory name")?- .to_owned(),- );- base = base- .parent()- .context("destination has no existing parent")?;- }- if tail.is_empty() {- ensure!(- !std::fs::symlink_metadata(base)?.is_symlink(),- "export destination is a symlink"- );- }- let mut resolved = std::fs::canonicalize(base)?;- for part in tail.into_iter().rev() {- resolved.push(part);- }- Ok(resolved)-}--fn open_root(destination: &Path, expected: &Root) -> Result<Option<File>> {- if !destination.try_exists()? {- return Ok(None);- }- ensure!(- destination.is_dir(),- "export destination is not a directory"- );- let path = names::check(destination, "export.json")?;- if !path.try_exists()? {- ensure!(- std::fs::read_dir(destination)?.next().is_none(),- "nonempty destination has no recognized export.json"- );- return Ok(None);- }- let file = OpenOptions::new().read(true).write(true).open(&path)?;- file.try_lock()- .context("another writer is maintaining this export")?;- let record: Root = serde_json::from_reader(&file).context("invalid export.json")?;- ensure!(- record.format == expected.format && record.version == expected.version,- "unsupported export format"- );- ensure!(- record.source_verifier == expected.source_verifier,- "this export belongs to another source account"- );- Ok(Some(file))-}--fn initialize(destination: &Path, root: &Root) -> Result<File> {- fs::create_directory(destination)?;- let path = names::check(destination, "export.json")?;- let mut file = OpenOptions::new()- .read(true)- .write(true)- .create_new(true)- .open(&path)- .context("destination was initialized by another writer; retry")?;- file.lock().context("cannot lock the new export")?;- ensure!(- std::fs::read_dir(destination)?- .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),- "destination became nonempty during initialization"- );- serde_json::to_writer_pretty(&mut file, root)?;- file.write_all(b"\n")?;- file.sync_all()?;- fs::sync_parent(&path)?;- Ok(file)-}--fn select(store: &Store, args: &ExportArgs) -> Result<()> {- if args.album.is_empty() {- store- .db- .connection()- .execute("UPDATE desired_albums SET selected=1", [])?;- }- for (selectors, include) in [(&args.album, true), (&args.exclude_album, false)] {- for selector in selectors {- let mut query = store.db.connection().prepare(- "SELECT id,name FROM desired_albums WHERE id=?1 OR name=?2 ORDER BY id LIMIT 11",- )?;- let candidates: Vec<(i64, Option<String>)> = query- .query_map(params![canonical_id(selector), selector], |r| {- Ok((r.get(0)?, r.get(1)?))- })?- .collect::<rusqlite::Result<_>>()?;- ensure!(!candidates.is_empty(), "no album matches {selector:?}");- if candidates.len() != 1 {- bail!(- "album {selector:?} is ambiguous: {}",- candidates- .iter()- .map(|(id, name)| format!(- "{id} {:?}",- name.as_deref().unwrap_or("unavailable")- ))- .collect::<Vec<_>>()- .join(", ")- );- }- store.db.connection().execute(- "UPDATE desired_albums SET selected=?1 WHERE id=?2",- params![include, candidates[0].0],- )?;- }- }- Ok(())-}--async fn refresh(- store: &Store,- replica: &mut Replica<'_>,- session: &ente_core::Session,- cancel: &watch::Receiver<bool>,-) -> Result<()> {- let need_favorites: bool = store.db.connection().query_row(- "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND present=1)",- [],- |r| r.get(0),- )?;- let mut favorites_known = true;- let mut after = 0;- eprintln!("Refreshing selected albums and Favorites.");- loop {- ensure!(!*cancel.borrow(), Cancelled);- let next: Option<(i64, bool, bool)> = store- .db- .connection()- .query_row(- "SELECT id,selected,favorites FROM desired_albums WHERE present=1 AND (selected=1 OR (favorites=1 AND ?1)) AND id>?2 ORDER BY id LIMIT 1",- params![need_favorites, after],- |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),- )- .optional()?;- let Some((id, selected, favorites)) = next else {- break;- };- after = id;- let result = async {- replica.retry_album(session, id)?;- let record = replica.album_record(id)?.context("missing album source")?;- let album = record.album(session.user_id)?;- replica.sync_album_files(session, &album).await?;- store.desired_album(&record, session.user_id)?;- if selected {- store.snapshot_album(&album)?;- }- store- .db- .connection()- .execute("UPDATE desired_albums SET ready=1 WHERE id=?1", [id])?;- Ok::<_, anyhow::Error>(())- }- .await;- if let Err(error) = result {- if favorites {- favorites_known = false;- }- store.db.connection().execute(- "UPDATE desired_albums SET failure=?1 WHERE id=?2",- params![error.to_string(), id],- )?;- if selected {- record_outcome(store, &format!("album:{id}"), &error)?;- eprintln!("album:{id}: {error:#}");- }- if fatal(&error) {- return Err(error);- }- }- }- store.db.connection().execute(- "UPDATE desired_albums SET ready=1 WHERE selected=1 AND present=0",- [],- )?;- if need_favorites && favorites_known {- let mut after = 0;- loop {- let ids: Vec<i64> = replica.db.read(|db| {- let mut query = db.prepare(- "SELECT DISTINCT f.id FROM photos_files f JOIN photos_collections c ON c.id=f.collection_id WHERE f.id>?1 AND json_extract(c.record,'$.remote.type')='favorites' AND json_extract(c.record,'$.remote.owner.id')=?2 ORDER BY f.id LIMIT 128",- )?;- Ok(query- .query_map(params![after, session.user_id], |r| r.get(0))?- .collect::<rusqlite::Result<_>>()?)- })?;- if ids.is_empty() {- break;- }- let transaction = store.db.connection().unchecked_transaction()?;- for id in ids {- after = id;- store- .db- .connection()- .execute("INSERT INTO favorites VALUES(?1)", [id])?;- }- transaction.commit()?;- }- }- after = 0;- loop {- let id: Option<i64> = store- .db- .connection()- .query_row(- "SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL AND id>?1 ORDER BY id LIMIT 1",- [after],- |r| r.get(0),- )- .optional()?;- let Some(id) = id else { break };- after = id;- let mut after_file = 0;- loop {- ensure!(!*cancel.borrow(), Cancelled);- let records: Vec<FileRecord> = replica.db.read(|db| {- let mut query = db.prepare(- "SELECT record FROM photos_files WHERE collection_id=?1 AND id>?2 ORDER BY id LIMIT 128",- )?;- Ok(query- .query_map(params![id, after_file], |r| crate::replica::read_json(r, 0))?- .collect::<rusqlite::Result<_>>()?)- })?;- if records.is_empty() {- break;- }- let transaction = store.db.connection().unchecked_transaction()?;- for record in records {- after_file = record.remote.id;- let file = record.file(session.user_id);- let failure = if !favorites_known {- Some("favorite state is incomplete".to_owned())- } else {- file.as_ref().err().map(ToString::to_string)- };- store.db.connection().execute(- "INSERT INTO desired_files(album,file,favorited,failure) VALUES(?1,?2,?3,?4)",- params![- id,- record.remote.id,- if favorites_known { Some(false) } else { None },- failure- ],- )?;- if failure.is_none() {- let file = file?;- store.db.connection().execute(- "INSERT INTO chosen_files VALUES(?1,?2,?3) ON CONFLICT(file) DO UPDATE SET album=excluded.album,version=excluded.version WHERE excluded.version>chosen_files.version OR (excluded.version=chosen_files.version AND excluded.album<chosen_files.album)",- params![file.id, id, file.updated_at_micros],- )?;- }- }- transaction.commit()?;- }- }- store- .db- .connection()- .execute(- "UPDATE desired_files SET favorited=EXISTS(SELECT 1 FROM favorites WHERE favorites.file=desired_files.file) WHERE favorited IS NOT NULL",- [],- )?;- let mut after_file = 0;- loop {- ensure!(!*cancel.borrow(), Cancelled);- let mut query = store- .db- .connection()- .prepare("SELECT album,file FROM chosen_files WHERE file>?1 ORDER BY file LIMIT 128")?;- let chosen: Vec<(i64, i64)> = query- .query_map([after_file], |r| Ok((r.get(0)?, r.get(1)?)))?- .collect::<rusqlite::Result<_>>()?;- drop(query);- if chosen.is_empty() {- break;- }- let transaction = store.db.connection().unchecked_transaction()?;- for (album, id) in chosen {- after_file = id;- let record = replica- .file_record(album, id)?- .context("missing chosen file source")?;- store.snapshot_file(&record.file(session.user_id)?)?;- }- transaction.commit()?;- }- let transaction = store.db.connection().unchecked_transaction()?;- store.db.connection().execute(- "DELETE FROM sources WHERE file NOT IN (SELECT file FROM chosen_files)",- [],- )?;- store- .db- .connection()- .execute(- "DELETE FROM album_sources WHERE id NOT IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM albums WHERE albums.id=album_sources.id) AND NOT EXISTS(SELECT 1 FROM pending WHERE album=album_sources.id)",- [],- )?;- transaction.commit()?;- let mut query = store- .db- .connection()- .prepare("SELECT album,file,failure FROM desired_files WHERE failure IS NOT NULL")?;- let mut rows = query.query([])?;- while let Some(row) = rows.next()? {- let unit = format!("file:{}:{}", row.get::<_, i64>(0)?, row.get::<_, i64>(1)?);- let error = anyhow::anyhow!(row.get::<_, String>(2)?);- record_outcome(store, &unit, &error)?;- eprintln!("{unit}: {error:#}");- }- Ok(())-}--struct CancelWorkers<'a>(&'a watch::Sender<bool>);--impl Drop for CancelWorkers<'_> {- fn drop(&mut self) {- if std::thread::panicking() {- self.0.send_replace(true);- }- }-}--fn transfers(- root: &Path,- store: &Mutex<Store>,- session: &ente_core::Session,- cancel_send: &watch::Sender<bool>,- cancel: &watch::Receiver<bool>,- jobs: usize,-) -> Result<()> {- let (expected, files): (i64, i64) = store::lock(store)?.db.connection().query_row(- "SELECT count(*),count(DISTINCT file) FROM desired_files WHERE failure IS NULL",- [],- |r| Ok((r.get(0)?, r.get(1)?)),- )?;- eprintln!("Maintaining {expected} selected copies.");- let runtime = tokio::runtime::Handle::current();- let last_report = Mutex::new(None::<Instant>);- std::thread::scope(|scope| {- let mut workers = Vec::new();- for _ in 0..jobs.min(files as usize) {- let runtime = &runtime;- let last_report = &last_report;- workers.push(scope.spawn(move || {- let _cancel_on_panic = CancelWorkers(cancel_send);- let run = reconcile::Run { root, store, cancel };- let result = (|| {- loop {- if *cancel.borrow() {- break;- }- let file = {- let store = store::lock(store)?;- let transaction = store.db.connection().unchecked_transaction()?;- let id: Option<i64> = store- .db- .connection()- .query_row(- "SELECT file FROM desired_files WHERE failure IS NULL AND claimed=0 ORDER BY file LIMIT 1",- [],- |r| r.get(0),- )- .optional()?;- let Some(id) = id else {- break;- };- store- .db- .connection()- .execute(- "UPDATE desired_files SET claimed=1 WHERE file=?1",- [id],- )?;- let file: ente_photos::files::File = store- .json("SELECT record FROM sources WHERE file=?1", [id])?- .context("missing file source snapshot")?;- transaction.commit()?;- file- };- let id = file.id;- let result = runtime.block_on(run.file(session, file));- if result.as_ref().err().is_some_and(fatal) {- cancel_send.send_replace(true);- }- let reported = result.or_else(|error| run.block_file(id, error));- let cleaned = run.cleanup(Some(id));- reported?;- cleaned?;- let completed = {- let mut last = last_report.lock().map_err(|_| anyhow::anyhow!("progress mutex poisoned"))?;- if last.is_none_or(|time| time.elapsed() >= Duration::from_secs(2)) {- *last = Some(Instant::now());- Some(store::lock(store)?.db.connection().query_row(- "SELECT count(*) FROM desired_files WHERE completed=1",- [],- |r| r.get::<_, i64>(0),- )?)- } else {- None- }- };- if let Some(completed) = completed {- eprintln!("Completed {completed}/{expected} selected copies.");- }- }- Ok(())- })();- if result.is_err() {- cancel_send.send_replace(true);- }- result- }));- }- let mut result = Ok(());- for worker in workers {- let completed = worker- .join()- .unwrap_or_else(|_| Err(anyhow::anyhow!("export worker panicked")));- if result.is_ok() {- result = completed;- }- }- result- })-}--pub fn record_outcome(store: &Store, unit: &str, error: &anyhow::Error) -> Result<()> {- let conflict = error.downcast_ref::<Conflict>().is_some();- store- .db- .connection()- .execute(- "INSERT INTO outcomes VALUES(?1,?2) ON CONFLICT(unit) DO UPDATE SET conflict=max(conflict,excluded.conflict)",- params![unit, conflict],- )?;- Ok(())-}--pub fn fatal(error: &anyhow::Error) -> bool {- fatal_cause(error.as_ref())-}--fn fatal_cause(cause: &(dyn std::error::Error + 'static)) -> bool {- if cause.is::<rusqlite::Error>()- || cause.is::<Cancelled>()- || matches!(- cause.downcast_ref::<crate::core_db::Error>(),- Some(crate::core_db::Error::Sqlite(_))- )- || cause- .downcast_ref::<ente_core::http::Error>()- .is_some_and(|error| error.status_code() == Some(401))- {- return true;- }- if let Some(error) = cause.downcast_ref::<std::io::Error>() {- return error.kind() == std::io::ErrorKind::StorageFull- || error.get_ref().is_some_and(|inner| fatal_cause(inner));- }- if let Some(error) = cause.downcast_ref::<ente_photos::files::Error>() {- use ente_photos::files::Error;- match error {- Error::Http(error) => return fatal_cause(error),- Error::Io(error) => return fatal_cause(error),- Error::Collections(error) => return fatal_cause(error),- _ => {}- }- }- if let Some(ente_collections::Error::Http(error)) = cause.downcast_ref() {- return fatal_cause(error);- }- if let Some(error) = cause.downcast_ref::<ente_photos::live_photo::Error>() {- match error {- ente_photos::live_photo::Error::Io(error) => return fatal_cause(error),- ente_photos::live_photo::Error::Zip(error) => return fatal_cause(error),- ente_photos::live_photo::Error::Components => {}- }- }- cause.source().is_some_and(fatal_cause)-}--fn summary(store: &Store, destination: &Path, as_json: bool) -> Result<bool> {- let unknown: bool = store.db.connection().query_row(- "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND ready=0)",- [],- |r| r.get(0),- )?;- let (files, expected, completed): (i64, i64, i64) = store.db.connection().query_row(- "SELECT count(DISTINCT file),count(*),coalesce(sum(completed),0) FROM desired_files",- [],- |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),- )?;- let (failures, conflicts): (i64, i64) = store.db.connection().query_row(- "SELECT count(*),coalesce(sum(conflict),0) FROM outcomes",- [],- |r| Ok((r.get(0)?, r.get(1)?)),- )?;- let (exported, metadata_updated, renamed, retained): (i64, i64, i64, i64) = store- .db- .connection()- .query_row(- "SELECT coalesce(sum(exported),0),coalesce(sum(metadata_updated),0),coalesce(sum(renamed),0),coalesce(sum(retained),0) FROM events",- [],- |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),- )?;- let complete = !unknown && failures == 0 && completed == expected;- let result = json!({"destination":destination,"complete":complete,"files":if unknown {None}else{Some(files)},"copies":if unknown {None}else{Some(json!({"expected":expected,"completed":completed,"pending":expected-completed}))},"changes":{"exported":exported,"metadataUpdated":metadata_updated,"renamed":renamed,"retained":retained},"failures":failures,"conflicts":conflicts});+ let changes = &summary.json["changes"];output::action(as_json,- &result,+ &summary.json,&format!(- "{}: {completed}/{expected} copies complete, {exported} exported, {retained} retained, {failures} failures, {conflicts} conflicts.",- destination.display()+ "{}: {progress}, {} exported, {} retained, {} failures, {} conflicts.",+ export.destination().display(),+ changes["exported"],+ changes["retained"],+ summary.json["failures"],+ summary.json["conflicts"],),)?;- Ok(complete)+ ensure!(summary.complete, "export is incomplete");+ Ok(())}-#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn transparent_errors_preserve_fatal_classification() {- for status in [401, 404] {- let error = ente_core::http::Error::Http {- status,- path: "/files".into(),- };- let error =- ente_photos::files::Error::Collections(ente_collections::Error::Http(error));- let error = anyhow::Error::new(error).context("download failed");- assert_eq!(fatal(&error), status == 401);- assert_eq!(- format!("{error:#}"),- format!("download failed: HTTP {status} at /files")- );- }- let cancelled = ente_photos::live_photo::Error::Io(std::io::Error::other(Cancelled));- assert!(fatal(&anyhow::Error::new(cancelled)));- let full = ente_photos::live_photo::Error::Io(std::io::ErrorKind::StorageFull.into());- assert!(fatal(&anyhow::Error::new(full)));- let zip = ente_photos::live_photo::Error::Zip(zip::result::ZipError::Io(- std::io::ErrorKind::StorageFull.into(),- ));- assert!(fatal(&anyhow::Error::new(zip)));- let sqlite = crate::core_db::Error::Sqlite(rusqlite::Error::InvalidQuery);- assert!(fatal(&anyhow::Error::new(sqlite)));- let local = ente_photos::files::Error::Io(std::io::ErrorKind::PermissionDenied.into());- assert!(!fatal(&anyhow::Error::new(local)));- }+fn store_path(account_home: &Path, destination: &Path) -> Result<PathBuf> {+ let bytes = destination.as_os_str().as_encoded_bytes();+ let name: String = hash::hash(bytes, Some(32), None)?+ .iter()+ .map(|b| format!("{b:02x}"))+ .collect();+ Ok(account_home.join("exports").join(format!("{name}.db")))}diff --git a/rust/apps/cli-next/src/export/source.rs b/rust/apps/cli-next/src/export/source.rsnew file mode 100644index 0000000000..bfa687672a--- /dev/null+++ b/rust/apps/cli-next/src/export/source.rs@@ -0,0 +1,120 @@+use anyhow::{Context, Result};+use ente_core::Session;+use ente_photos_export::{CollectionEntry, FileEntry, Source};++use crate::{+ core_db::{Db, OptionalExtension, SqliteResult, params},+ replica::{AlbumRecord, FileRecord, Replica, read_json},+};++pub struct ReplicaSource<'a> {+ replica: Replica<'a>,+ user_id: i64,+}++impl<'a> ReplicaSource<'a> {+ pub fn new(db: &'a mut Db, user_id: i64) -> Self {+ Self {+ replica: Replica::new(db, user_id),+ user_id,+ }+ }++ fn collection(&self, record: AlbumRecord, name: Option<String>) -> CollectionEntry {+ let id = record.remote.id;+ let favorites = record.remote.kind == "favorites" && record.remote.owner.id == self.user_id;+ let value = record+ .album(self.user_id)+ .and_then(|album| Ok((album, record.documents.context("metadata is unavailable")?)));+ let name = value+ .as_ref()+ .ok()+ .map(|(album, _)| album.name.clone())+ .or(name);+ CollectionEntry {+ id,+ name,+ favorites,+ value,+ }+ }++ fn entry(&self, record: FileRecord) -> FileEntry {+ let id = record.remote.id;+ let value = record+ .file(self.user_id)+ .and_then(|file| Ok((file, record.documents.context("metadata is unavailable")?)));+ FileEntry { id, value }+ }+}++impl Source for ReplicaSource<'_> {+ async fn refresh_catalog(&mut self, session: &Session) -> Result<()> {+ self.replica.sync_collections(session).await+ }++ fn catalog_page(&self, after: i64) -> Result<Vec<CollectionEntry>> {+ let next: Option<(AlbumRecord, Option<String>)> = self.replica.db.read(|db| {+ Ok(db+ .query_row(+ "SELECT id,record,name FROM photos_collections WHERE id>?1 ORDER BY id LIMIT 1",+ [after],+ |r| Ok((read_json(r, 1)?, r.get(2)?)),+ )+ .optional()?)+ })?;+ Ok(next+ .into_iter()+ .map(|(record, name)| self.collection(record, name))+ .collect())+ }++ async fn refresh_album(&mut self, session: &Session, id: i64) -> Result<CollectionEntry> {+ self.replica.retry_album(session, id)?;+ let record = self+ .replica+ .album_record(id)?+ .context("missing album source")?;+ let entry = self.collection(record, None);+ let (album, documents) = entry.value?;+ self.replica.sync_album_files(session, &album).await?;+ Ok(CollectionEntry {+ value: Ok((album, documents)),+ ..entry+ })+ }++ fn file_page(&self, album: i64, after: i64) -> Result<Vec<FileEntry>> {+ let records: Vec<FileRecord> = self.replica.db.read(|db| {+ let mut query = db.prepare(+ "SELECT record FROM photos_files WHERE collection_id=?1 AND id>?2 ORDER BY id LIMIT 128",+ )?;+ Ok(query+ .query_map(params![album, after], |r| read_json(r, 0))?+ .collect::<SqliteResult<_>>()?)+ })?;+ Ok(records+ .into_iter()+ .map(|record| self.entry(record))+ .collect())+ }++ fn favorite_page(&self, after: i64) -> Result<Vec<i64>> {+ Ok(self.replica.db.read(|db| {+ let mut query = db.prepare(+ "SELECT DISTINCT f.id FROM photos_files f JOIN photos_collections c ON c.id=f.collection_id WHERE f.id>?1 AND json_extract(c.record,'$.remote.type')='favorites' AND json_extract(c.record,'$.remote.owner.id')=?2 ORDER BY f.id LIMIT 128",+ )?;+ Ok(query+ .query_map(params![after, self.user_id], |r| r.get(0))?+ .collect::<SqliteResult<_>>()?)+ })?)+ }++ fn file(&self, album: i64, id: i64) -> Result<FileEntry> {+ Ok(self.entry(+ self.replica+ .file_record(album, id)?+ .context("missing chosen file source")?,+ ))+ }+}diff --git a/rust/apps/cli-next/src/photos.rs b/rust/apps/cli-next/src/photos.rsindex 7d2235c6c9..201e9fbaad 100644--- a/rust/apps/cli-next/src/photos.rs+++ b/rust/apps/cli-next/src/photos.rs@@ -162,12 +162,18 @@ async fn file(.filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new("."));let mut temporary = tempfile::NamedTempFile::new_in(parent)?;- files::download(session, &entry.file, || {- let file = temporary.as_file_mut();- file.set_len(0)?;- file.rewind()?;- file.try_clone()- })+ files::download(+ session,+ entry.file.id,+ &entry.file.key,+ &entry.file.header,+ || {+ let file = temporary.as_file_mut();+ file.set_len(0)?;+ file.rewind()?;+ file.try_clone()+ },+ ).await?;temporary.as_file().sync_all()?;let bytes = temporary.as_file().metadata()?.len();diff --git a/rust/apps/cli-next/src/vault.rs b/rust/apps/cli-next/src/vault.rsindex fcd3ae2c8e..d83a90c5f9 100644--- a/rust/apps/cli-next/src/vault.rs+++ b/rust/apps/cli-next/src/vault.rs@@ -89,7 +89,7 @@ impl Vault {keyring::Entry::new("io.ente.cli", &home.to_string_lossy())})?;let state = match encrypted {- Some(bytes) => decrypt(&bytes, &key)?,+ Some(bytes) => decrypt(&bytes, &key, &home)?,None => State::default(),};Ok(Self {@@ -144,7 +144,7 @@ impl VaultAccess {let Self { home, key } = self;let lock = lock(&home)?;let state = match read_vault(&home)? {- Some(bytes) => decrypt(&bytes, &key)?,+ Some(bytes) => decrypt(&bytes, &key, &home)?,None => State::default(),};Ok(Vault {@@ -168,7 +168,7 @@ impl State {let key = load_key(override_key, true, || {keyring::Entry::new("io.ente.cli", &home.to_string_lossy())})?;- decrypt(&bytes, &key)+ decrypt(&bytes, &key, &home)}pub fn named(&self, name: &str) -> Result<usize> {@@ -305,7 +305,7 @@ fn keyring_error(error: keyring::Error, exists: bool) -> anyhow::Error {}}-fn decrypt(bytes: &[u8], key: &Key) -> Result<State> {+fn decrypt(bytes: &[u8], key: &Key, home: &Path) -> Result<State> {let unlock_error =|| anyhow::anyhow!("cannot unlock CLI vault: wrong key or damaged ciphertext");let encrypted: EncryptedVault =@@ -318,7 +318,9 @@ fn decrypt(bytes: &[u8], key: &Key) -> Result<State> {parse_json(&plaintext).context("invalid CLI vault contents")?;ensure!(version.schema_version == SCHEMA_VERSION,- "unsupported CLI vault schema"+ "unsupported CLI vault schema {}; this build supports {SCHEMA_VERSION}.\nTo start fresh, move {} aside, then log in again.",+ version.schema_version,+ home.join(VAULT_FILE).display(),);let stored: VersionedState<State> =parse_json(&plaintext).context("invalid CLI vault contents")?;diff --git a/rust/apps/cli-next/tests/cli.rs b/rust/apps/cli-next/tests/cli.rsindex 99e6caab17..4a526708fa 100644--- a/rust/apps/cli-next/tests/cli.rs+++ b/rust/apps/cli-next/tests/cli.rs@@ -231,7 +231,12 @@ fn vault_schema_version_is_checked_after_decryption() {home.seed("http://localhost:8080");home.write_vault_document(&json!({"schema_version": 2, "state": Value::Null}));let original = fs::read(home.dir.path().join("vault.json")).unwrap();- assert!(failure(&home.run(&["account", "list"])).contains("unsupported CLI vault schema"));+ let error = failure(&home.run(&["account", "list"]));+ assert!(error.contains("unsupported CLI vault schema 2; this build supports 1."));+ assert!(error.contains(&format!(+ "To start fresh, move {} aside, then log in again.",+ home.dir.path().canonicalize().unwrap().join("vault.json").display(),+ )));let output = home.command(&["account", "list"]).env(diff --git a/rust/apps/cli-next/tests/support/export.rs b/rust/apps/cli-next/tests/support/export.rsindex 68f0267389..b85d1db778 100644--- a/rust/apps/cli-next/tests/support/export.rs+++ b/rust/apps/cli-next/tests/support/export.rs@@ -132,6 +132,7 @@ fn export_metadata_updates_and_repairs_preserve_source_values() -> TestResult {let record=read_json(&sidecar);assert_eq!(record["ente"]["creationTime"],"2023-11-14T22:13:20.123456Z");assert_eq!(record["photoTakenTime"]["timestamp"],"1700000000");+ assert_eq!(record["photoTakenTime"]["formatted"],"Nov 14, 2023, 10:13:20 PM UTC");assert_eq!(record["ente"]["components"][0]["size"],original.len());assert_eq!(record["ente"]["modificationTime"],"2023-11-14T22:13:21.123456Z");assert_eq!(record["modificationTime"]["timestamp"],"1700000001");@@ -884,10 +885,23 @@ fn export_live_pairs_retry_and_reuse_without_rewriting_unchanged_output() -> Tes}).collect();let db = export_database(&home);- for table in ["albums", "placements", "components", "json_records", "pending"] {+ for table in [+ "albums",+ "placements",+ "components",+ "json_records",+ "pending",+ "sources",+ "album_sources",+ ] {for action in ["INSERT", "UPDATE", "DELETE"] {+ let timing = if ["sources", "album_sources"].contains(&table) {+ "AFTER"+ } else {+ "BEFORE"+ };db.execute_batch(&format!(- "CREATE TRIGGER unchanged_{table}_{action} BEFORE {action} ON {table} BEGIN SELECT RAISE(ABORT,'unchanged association write'); END;",+ "CREATE TRIGGER unchanged_{table}_{action} {timing} {action} ON {table} BEGIN SELECT RAISE(ABORT,'unchanged association write'); END;",))?;}}@@ -911,7 +925,15 @@ fn export_live_pairs_retry_and_reuse_without_rewriting_unchanged_output() -> Teslet path = root.join(path);assert_eq!((fs::read(&path)?, fs::metadata(&path)?.modified()?), before);}- for table in ["albums", "placements", "components", "json_records", "pending"] {+ for table in [+ "albums",+ "placements",+ "components",+ "json_records",+ "pending",+ "sources",+ "album_sources",+ ] {for action in ["INSERT", "UPDATE", "DELETE"] {db.execute_batch(&format!("DROP TRIGGER unchanged_{table}_{action};"))?;}@@ -946,7 +968,8 @@ fn export_excludes_another_writer_while_downloads_are_running() -> TestResult {let other = TestHome::new();login(&other, "photos", &email, &["--host", origin]);let (album, key) = create_album(origin, &account, "Album", "album").await;- for name in ["First.jpg", "Second.jpg"] {+ let names = ["First.jpg", "Second.jpg", "Third.jpg", "Fourth.jpg"];+ for name in names {upload_fixture(origin,&account,@@ -960,6 +983,7 @@ fn export_excludes_another_writer_while_downloads_are_running() -> TestResult {let destination = tempfile::tempdir()?;let root = destination.path().join("photos");let held = objects.hold_reads(0);+ let reads = objects.reads();let child = ExportChild::spawn(&mut home.command(&["photos","export",@@ -969,6 +993,8 @@ fn export_excludes_another_writer_while_downloads_are_running() -> TestResult {"--json",]))?;held.wait_for(2);+ std::thread::sleep(Duration::from_millis(500));+ assert_eq!(objects.reads() - reads, 2);let second = ExportChild::spawn(&mut other.command(&["photos","export",@@ -978,8 +1004,10 @@ fn export_excludes_another_writer_while_downloads_are_running() -> TestResult {assert!(failure(&second.wait_with_output()?).contains("another writer"));drop(held);let result: Value = serde_json::from_slice(&success(child.wait_with_output()?).stdout)?;- assert_eq!(result["copies"]["completed"], 2);- assert_eq!(objects.peak_reads(), 2);+ assert_eq!(result["copies"]["completed"], 4);+ for name in names {+ assert_eq!(fs::read(root.join("Album").join(name))?, name.as_bytes());+ }Ok(())})})diff --git a/rust/apps/cli-next/tests/support/export_maintenance.rs b/rust/apps/cli-next/tests/support/export_maintenance.rsindex bdb82dd45a..b818ee1c73 100644--- a/rust/apps/cli-next/tests/support/export_maintenance.rs+++ b/rust/apps/cli-next/tests/support/export_maintenance.rs@@ -91,6 +91,52 @@ fn export_retries_transient_collection_and_file_refresh_failures() {}}+#[test]+fn export_local_inventory_failure_preserves_output_and_reports_unknown_totals() {+ let mut server = mockito::Server::new();+ let home = TestHome::new();+ home.seed(&server.url());+ let key = Key::generate();+ listing(&mut server, json!([collection(1, "First", &key)]));+ let (file, bytes) = source(10, &key, b"original", "Photo.jpg");+ page(&mut server, 1, 0, json!([file]), false).create();+ let downloaded = download(&mut server, 10, &bytes, 1);+ let destination = tempfile::tempdir().unwrap();+ let root = destination.path().join("photos");+ assert_eq!(run(&home, &root, &[], true)["copies"]["completed"], 1);+ let published = [+ "export.json",+ "First/metadata.json",+ "First/Photo.jpg",+ "First/metadata/Photo.jpg.json",+ ]+ .map(|path| (path, fs::read(root.join(path)).unwrap()));+ database(&home)+ .execute_batch(+ "UPDATE photos_files SET record='{}' WHERE collection_id=1 AND id=10; UPDATE photos_collections SET files_synced_to=NULL WHERE id=1;",+ )+ .unwrap();+ let refreshed = page(&mut server, 1, 10, json!([]), false)+ .expect(1)+ .create();+ let result = run_failure(&home, &root, &[], "missing field `remote`");+ assert!(result["files"].is_null());+ assert!(result["copies"].is_null());+ assert_eq!(result["failures"], 1);+ let output = home.run(&["photos", "export", root.to_str().unwrap()]);+ assert!(failure(&output).contains("missing field `remote`"));+ assert!(+ String::from_utf8(output.stdout)+ .unwrap()+ .contains("inventory incomplete")+ );+ for (path, bytes) in published {+ assert_eq!(fs::read(root.join(path)).unwrap(), bytes);+ }+ refreshed.assert();+ downloaded.assert();+}+#[test]fn export_snapshot_uses_current_healthy_memberships_and_preserves_raw_bytes() {let mut server = mockito::Server::new();@@ -149,6 +195,27 @@ fn export_snapshot_uses_current_healthy_memberships_and_preserves_raw_bytes() {assert_eq!(record["description"], "winner");assert_eq!(record["favorited"], true);}+ let state = export_database(&home);+ for table in ["sources", "album_sources"] {+ for action in ["INSERT", "UPDATE", "DELETE"] {+ state.execute_batch(&format!(+ "CREATE TRIGGER unchanged_{table}_{action} AFTER {action} ON {table} BEGIN SELECT RAISE(ABORT,'unchanged snapshot write'); END;",+ )).unwrap();+ }+ }+ let unchanged = run(&home, &root, &[], false);+ assert_eq!(unchanged["copies"], result["copies"]);+ assert_eq!(+ unchanged["changes"],+ json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})+ );+ for table in ["sources", "album_sources"] {+ for action in ["INSERT", "UPDATE", "DELETE"] {+ state+ .execute_batch(&format!("DROP TRIGGER unchanged_{table}_{action};"))+ .unwrap();+ }+ }let untouched = root.join("Second/metadata/Photo.jpg.json");let before = (fs::read(&untouched).unwrap(),@@ -2205,3 +2272,139 @@ fn export_reports_completion_progress_before_the_command_finishes() {url.assert();slow.assert();}++#[test]+fn export_only_file_metadata_does_not_break_common_commands() {+ let mut server = mockito::Server::new();+ let home = TestHome::new();+ home.seed(&server.url());+ let key = Key::generate();+ listing(+ &mut server,+ json!([+ collection(1, "Affected", &key),+ collection(2, "Healthy", &key)+ ]),+ );+ let (file, _) = source(10, &key, b"unused", "Original.jpg");+ let file = public(+ file,+ &key,+ json!({+ "editedName":"Export metadata.jpg", "cameraMake":17, "cameraModel":17,+ "uploaderName":17, "mediaType":"broken", "mvi":"broken"+ }),+ 10,+ );+ let (healthy, bytes) = source(20, &key, b"healthy", "Healthy.jpg");+ page(&mut server, 1, 0, json!([file]), false).create();+ page(&mut server, 2, 0, json!([healthy]), false).create();+ let no_download = server+ .mock("GET", "/files/download/v3/10")+ .expect(0)+ .create();+ let downloaded = download(&mut server, 20, &bytes, 1);+ let listed = home.json(&["photos", "file", "list"]);+ assert_eq!(listed.as_array().unwrap().len(), 2);+ let viewed = home.json(&["photos", "file", "view", "Export metadata.jpg", "--offline"]);+ assert_eq!(viewed["id"], "10");+ assert_eq!(viewed["name"], "Export metadata.jpg");+ let destination = tempfile::tempdir().unwrap();+ let root = destination.path().join("photos");+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);+ let error = failure(&output);+ assert!(+ error.contains("file:1:10: invalid public file metadata"),+ "{error}"+ );+ let result: Value = serde_json::from_slice(&output.stdout).unwrap();+ assert_eq!(result["complete"], false);+ assert_eq!(+ result["copies"],+ json!({"expected":2,"completed":1,"pending":1})+ );+ assert_eq!(result["failures"], 1);+ assert_component(&root, "Healthy", "Healthy.jpg", b"healthy");+ assert!(!root.join("Affected/Export metadata.jpg").exists());+ no_download.assert();+ downloaded.assert();+}++#[test]+fn export_only_album_metadata_preserves_common_commands_and_favorites() {+ for (document, value, cause) in [+ (+ "magicMetadata",+ json!({"order":"broken"}),+ "invalid album private metadata",+ ),+ (+ "sharedMagicMetadata",+ json!({"order":"broken","mute":"broken"}),+ "invalid album sharee metadata",+ ),+ (+ "pubMagicMetadata",+ json!({"asc":"broken","coverID":"broken","layout":17,"caption":17}),+ "invalid album public metadata",+ ),+ ] {+ let mut server = mockito::Server::new();+ let home = TestHome::new();+ home.seed(&server.url());+ let key = Key::generate();+ let mut favorites = collection(5, "Favorites", &key);+ favorites["type"] = json!("favorites");+ let encrypted = blob::encrypt_json(&value, &key).unwrap();+ favorites[document] = json!({"data":b64::encode(&encrypted.encrypted_data),"header":b64::encode(encrypted.decryption_header.as_bytes())});+ listing(+ &mut server,+ json!([collection(1, "Healthy", &key), favorites]),+ );+ let (file, bytes) = source(10, &key, b"original", "Photo.jpg");+ page(&mut server, 1, 0, json!([file]), false).create();+ let memberships = page(&mut server, 5, 0, json!([file]), false)+ .expect(1)+ .create();+ let downloaded = download(&mut server, 10, &bytes, 1);+ let albums = home.json(&["photos", "album", "list"]);+ assert_eq!(albums.as_array().unwrap().len(), 2);+ let viewed = home.json(&["photos", "album", "view", "Favorites", "--offline"]);+ assert_eq!(viewed["id"], "5");+ assert_eq!(viewed["name"], "Favorites");+ let destination = tempfile::tempdir().unwrap();+ let root = destination.path().join("photos");+ let output = home.run(&[+ "photos",+ "export",+ root.to_str().unwrap(),+ "--album",+ "Healthy",+ "--json",+ ]);+ assert!(+ output.status.success(),+ "{}",+ String::from_utf8_lossy(&output.stderr)+ );+ assert!(!String::from_utf8_lossy(&output.stderr).contains("album:5"));+ let sidecar = root.join("Healthy/metadata/Photo.jpg.json");+ let mut metadata = read_json(&sidecar);+ assert_eq!(metadata["favorited"], true);+ metadata["favorited"] = json!(false);+ fs::write(&sidecar, serde_json::to_vec(&metadata).unwrap()).unwrap();+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);+ let error = failure(&output);+ assert!(error.contains(&format!("album:5: {cause}")), "{error}");+ assert!(!error.contains("favorite state is incomplete"), "{error}");+ let result: Value = serde_json::from_slice(&output.stdout).unwrap();+ assert_eq!(result["complete"], false);+ assert_eq!(result["failures"], 1);+ assert_eq!(result["changes"]["metadataUpdated"], 1);+ assert_eq!(read_json(&sidecar)["favorited"], true);+ assert_component(&root, "Healthy", "Photo.jpg", b"original");+ assert!(!root.join("Favorites").exists());+ memberships.assert();+ downloaded.assert();+ }+}diff --git a/rust/apps/cli-next/tests/support/export_sources.rs b/rust/apps/cli-next/tests/support/export_sources.rsindex 275d62e0ea..8515b41a04 100644--- a/rust/apps/cli-next/tests/support/export_sources.rs+++ b/rust/apps/cli-next/tests/support/export_sources.rs@@ -980,6 +980,7 @@ fn export_page_and_recognized_failures_roll_back_together_on_storage_failure() {broken["metadata"]["encryptedData"] = json!("broken");page(&mut server, 1, 0, json!([good, broken]), false).create();let result = run_failure(&home, &root, &[], "fixture storage failure");+ assert_eq!(result["failures"], 2);assert!(result["files"].is_null());assert_eq!(db.query_row("SELECT count(*) FROM photos_files", [], |r| rdiff --git a/rust/crates/photos-export/Cargo.toml b/rust/crates/photos-export/Cargo.tomlnew file mode 100644index 0000000000..b3d3deb980--- /dev/null+++ b/rust/crates/photos-export/Cargo.toml@@ -0,0 +1,23 @@+[package]+name = "ente-photos-export"+edition.workspace = true+publish = false++[dependencies]+anyhow.workspace = true+chrono = { workspace = true, features = ["std"] }+ente-collections.workspace = true+ente-core.workspace = true+ente-photos.workspace = true+icu_casemap.workspace = true+rusqlite.workspace = true+serde.workspace = true+serde_json.workspace = true+thiserror.workspace = true+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }+unicode-normalization.workspace = true+uuid = { workspace = true, features = ["v4"] }+zip.workspace = true++[lints]+workspace = truediff --git a/rust/apps/cli-next/src/export/adopt.rs b/rust/crates/photos-export/src/adopt.rssimilarity index 99%rename from rust/apps/cli-next/src/export/adopt.rsrename to rust/crates/photos-export/src/adopt.rsindex c784ef680b..a0f7788bff 100644--- a/rust/apps/cli-next/src/export/adopt.rs+++ b/rust/crates/photos-export/src/adopt.rs@@ -1,7 +1,7 @@use std::{fs, path::Path};+use crate::metadata::{Component, Role};use anyhow::{Context, Result, ensure};-use ente_photos::export::{Component, Role};use serde_json::Value;use uuid::Uuid;@@ -13,7 +13,6 @@ use super::{pub fn scan(root: &Path, store: &Store) -> Result<()> {store.db- .connection().execute_batch("DELETE FROM components; DELETE FROM json_records; DELETE FROM placements; DELETE FROM albums; DELETE FROM pending; DELETE FROM temporaries;",)?;@@ -208,7 +207,6 @@ fn album(root: &Path, store: &Store, directory: &Path, retained: bool) -> Resultlet existing = if retained {let mut query = store.db- .connection().prepare("SELECT DISTINCT p.id,p.album,p.file,p.retained,p.name,p.kind FROM placements p JOIN components c ON c.placement=p.id WHERE p.album=?1 AND p.file=?2 AND p.retained=1 AND c.folder=?3",)?;diff --git a/rust/apps/cli-next/src/export/fs.rs b/rust/crates/photos-export/src/fs.rssimilarity index 97%rename from rust/apps/cli-next/src/export/fs.rsrename to rust/crates/photos-export/src/fs.rsindex fc438df966..f12364c424 100644--- a/rust/apps/cli-next/src/export/fs.rs+++ b/rust/crates/photos-export/src/fs.rs@@ -112,9 +112,9 @@ pub fn set_time(path: &Path, micros: i64) -> Result<Properties> {Properties::read(path)}-pub fn sync_parent(path: &Path) -> Result<()> {+pub fn sync_parent(_path: &Path) -> Result<()> {#[cfg(unix)]- if let Some(parent) = path.parent() {+ if let Some(parent) = _path.parent() {File::open(parent)?.sync_all()?;}Ok(())diff --git a/rust/crates/photos-export/src/lib.rs b/rust/crates/photos-export/src/lib.rsnew file mode 100644index 0000000000..bc9427eb7c--- /dev/null+++ b/rust/crates/photos-export/src/lib.rs@@ -0,0 +1,763 @@+mod adopt;+mod fs;+mod live_photo;+mod metadata;+mod names;+mod reconcile;+mod snapshot;+mod source;+mod store;+mod transfer;++pub use source::{CollectionEntry, FileEntry, Source};++use std::{+ fs::{File, OpenOptions},+ io::Write,+ num::NonZeroUsize,+ path::{Path, PathBuf},+ sync::Mutex,+ time::{Duration, Instant},+};++use crate::metadata::Root;+use anyhow::{Context, Result, bail, ensure};+use rusqlite::{OptionalExtension, params};+use serde_json::{Value, json};++use tokio::sync::watch;++use snapshot::FileSnapshot;+use store::Store;++#[derive(Debug, thiserror::Error)]+#[error("conflict: {0}")]+struct Conflict(String);++#[derive(Debug, thiserror::Error)]+#[error("export cancelled")]+struct Cancelled;++pub struct Options {+ pub albums: Vec<String>,+ pub exclude_albums: Vec<String>,+ pub adopt: bool,+ pub jobs: NonZeroUsize,+}++#[derive(Debug, thiserror::Error)]+#[error("{0}")]+pub struct AdoptionRequired(&'static str);++pub struct Summary {+ pub json: Value,+ pub complete: bool,+ pub completed: i64,+ pub expected: i64,+}++pub struct Export {+ destination: PathBuf,+ expected_root: Root,+ root_lock: Option<File>,+ options: Options,+ prepared: Option<(Store, Result<()>)>,+}++impl Export {+ pub fn open(+ path: &Path,+ master_key: &ente_core::crypto::Key,+ options: Options,+ ) -> Result<Self> {+ let destination = destination(path)?;+ let expected_root = Root::new(master_key)?;+ let root_lock = open_root(&destination, &expected_root)?;+ ensure!(+ !options.adopt || root_lock.is_some(),+ "there is no export to adopt"+ );+ Ok(Self {+ destination,+ expected_root,+ root_lock,+ options,+ prepared: None,+ })+ }++ pub fn destination(&self) -> &Path {+ &self.destination+ }++ pub fn exists(&self) -> bool {+ self.root_lock.is_some()+ }++ pub fn verify_source(&self, master_key: &ente_core::crypto::Key) -> Result<()> {+ ensure!(+ Root::new(master_key)?.source_verifier == self.expected_root.source_verifier,+ "account keys changed while waiting for access"+ );+ Ok(())+ }++ pub async fn prepare(+ &mut self,+ connection: rusqlite::Connection,+ source: &mut impl Source,+ session: &ente_core::Session,+ cancel: &mut watch::Receiver<bool>,+ report: &(dyn Fn(&str) + Sync),+ ) -> Result<()> {+ let store = Store::open(connection, &self.destination)?;+ let bound: bool = store+ .db+ .query_row("SELECT bound FROM export WHERE id=1", [], |r| r.get(0))?;+ ensure!(+ self.root_lock.is_none() || bound || self.options.adopt,+ AdoptionRequired("this export is not associated")+ );+ store.reset_desired()?;+ if self.root_lock.is_some() && !bound {+ adopt::scan(&self.destination, &store)?;+ }+ report("Refreshing album metadata.");+ tokio::select! {+ result=source.refresh_catalog(session)=>result?,+ _=cancel.changed()=>return Err(Cancelled.into()),+ }+ let mut after = 0;+ loop {+ ensure!(!*cancel.borrow(), Cancelled);+ let entries = source.catalog_page(after)?;+ if entries.is_empty() {+ break;+ }+ for entry in entries {+ after = entry.id;+ store.desired_album(&entry)?;+ }+ }+ store+ .db+ .execute(+ "INSERT OR IGNORE INTO desired_albums(id,name) SELECT s.id,json_extract(s.record,'$.name') FROM album_sources s WHERE EXISTS(SELECT 1 FROM albums a WHERE a.id=s.id)",+ [],+ )?;+ store+ .db+ .execute(+ "INSERT OR IGNORE INTO desired_albums(id,name) SELECT id,name FROM albums ORDER BY retained",+ [],+ )?;+ store+ .db+ .execute(+ "INSERT OR IGNORE INTO desired_albums(id,name) SELECT album,name FROM pending WHERE file IS NULL AND retained=0",+ [],+ )?;+ select(&store, &self.options)?;+ if self.root_lock.is_none() {+ self.root_lock = Some(initialize(&self.destination, &self.expected_root)?);+ }+ store+ .db+ .execute("UPDATE export SET bound=1 WHERE id=1 AND bound=0", [])?;++ let refresh_cancel = cancel.clone();+ let refreshed = tokio::select! {+ result=refresh(&store,source,session,&refresh_cancel,report)=>result,+ _=cancel.changed()=>Err(Cancelled.into()),+ };+ self.prepared = Some((store, refreshed));+ Ok(())+ }++ pub fn execute(+ &mut self,+ session: &ente_core::Session,+ cancel_send: &watch::Sender<bool>,+ cancel: &watch::Receiver<bool>,+ report: &(dyn Fn(&str) + Sync),+ ) -> Result<Summary> {+ let (store, refreshed) = self.prepared.take().context("export is not prepared")?;+ let prepared = refreshed.is_ok();+ let shared = Mutex::new(store);+ let run = reconcile::Run {+ root: &self.destination,+ store: &shared,+ cancel,+ report,+ };+ let result = (|| {+ refreshed?;+ run.recover()?;+ run.albums()?;+ transfers(+ &self.destination,+ &shared,+ session,+ cancel_send,+ cancel,+ self.options.jobs.get(),+ report,+ )?;+ run.check_cancel()?;+ run.retain_removed()+ })();+ if let Err(error) = result {+ run.report("run", &error)?;+ }+ if let Err(error) = run.cleanup(None) {+ run.report("run", &error)?;+ }+ let store = shared+ .into_inner()+ .map_err(|_| anyhow::anyhow!("export database mutex poisoned"))?;+ summary(&store, &self.destination, prepared)+ }+}++fn destination(path: &Path) -> Result<PathBuf> {+ let absolute = if path.is_absolute() {+ path.to_owned()+ } else {+ std::env::current_dir()?.join(path)+ };+ let mut base = absolute.as_path();+ let mut tail = Vec::new();+ while !base.try_exists()? {+ tail.push(+ base.file_name()+ .context("destination has no directory name")?+ .to_owned(),+ );+ base = base+ .parent()+ .context("destination has no existing parent")?;+ }+ if tail.is_empty() {+ ensure!(+ !std::fs::symlink_metadata(base)?.is_symlink(),+ "export destination is a symlink"+ );+ }+ let mut resolved = std::fs::canonicalize(base)?;+ for part in tail.into_iter().rev() {+ resolved.push(part);+ }+ Ok(resolved)+}++fn open_root(destination: &Path, expected: &Root) -> Result<Option<File>> {+ if !destination.try_exists()? {+ return Ok(None);+ }+ ensure!(+ destination.is_dir(),+ "export destination is not a directory"+ );+ let path = names::check(destination, "export.json")?;+ if !path.try_exists()? {+ ensure!(+ std::fs::read_dir(destination)?.next().is_none(),+ "nonempty destination has no recognized export.json"+ );+ return Ok(None);+ }+ let file = OpenOptions::new().read(true).write(true).open(&path)?;+ file.try_lock()+ .context("another writer is maintaining this export")?;+ let record: Root = serde_json::from_reader(&file).context("invalid export.json")?;+ ensure!(+ record.format == expected.format && record.version == expected.version,+ "unsupported export format"+ );+ ensure!(+ record.source_verifier == expected.source_verifier,+ "this export belongs to another source account"+ );+ Ok(Some(file))+}++fn initialize(destination: &Path, root: &Root) -> Result<File> {+ fs::create_directory(destination)?;+ let path = names::check(destination, "export.json")?;+ let mut file = OpenOptions::new()+ .read(true)+ .write(true)+ .create_new(true)+ .open(&path)+ .context("destination was initialized by another writer; retry")?;+ file.lock().context("cannot lock the new export")?;+ ensure!(+ std::fs::read_dir(destination)?+ .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),+ "destination became nonempty during initialization"+ );+ serde_json::to_writer_pretty(&mut file, root)?;+ file.write_all(b"\n")?;+ file.sync_all()?;+ fs::sync_parent(&path)?;+ Ok(file)+}++fn select(store: &Store, options: &Options) -> Result<()> {+ if options.albums.is_empty() {+ store+ .db+ .execute("UPDATE desired_albums SET selected=1", [])?;+ }+ for (selectors, include) in [(&options.albums, true), (&options.exclude_albums, false)] {+ for selector in selectors {+ let mut query = store.db.prepare(+ "SELECT id,name FROM desired_albums WHERE id=?1 OR name=?2 ORDER BY id LIMIT 11",+ )?;+ let candidates: Vec<(i64, Option<String>)> = query+ .query_map(params![canonical_id(selector), selector], |r| {+ Ok((r.get(0)?, r.get(1)?))+ })?+ .collect::<rusqlite::Result<_>>()?;+ ensure!(!candidates.is_empty(), "no album matches {selector:?}");+ if candidates.len() != 1 {+ bail!(+ "album {selector:?} is ambiguous: {}",+ candidates+ .iter()+ .map(|(id, name)| format!(+ "{id} {:?}",+ name.as_deref().unwrap_or("unavailable")+ ))+ .collect::<Vec<_>>()+ .join(", ")+ );+ }+ store.db.execute(+ "UPDATE desired_albums SET selected=?1 WHERE id=?2",+ params![include, candidates[0].0],+ )?;+ }+ }+ Ok(())+}++async fn refresh(+ store: &Store,+ source: &mut impl Source,+ session: &ente_core::Session,+ cancel: &watch::Receiver<bool>,+ report: &(dyn Fn(&str) + Sync),+) -> Result<()> {+ let need_favorites: bool = store.db.query_row(+ "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND present=1)",+ [],+ |r| r.get(0),+ )?;+ let mut favorites_known = true;+ let mut after = 0;+ report("Refreshing selected albums and Favorites.");+ loop {+ ensure!(!*cancel.borrow(), Cancelled);+ let next: Option<(i64, bool, bool)> = store+ .db+ .query_row(+ "SELECT id,selected,favorites FROM desired_albums WHERE present=1 AND (selected=1 OR (favorites=1 AND ?1)) AND id>?2 ORDER BY id LIMIT 1",+ params![need_favorites, after],+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),+ )+ .optional()?;+ let Some((id, selected, favorites)) = next else {+ break;+ };+ after = id;+ let refreshed = source.refresh_album(session, id).await;+ if refreshed.is_err() && favorites {+ favorites_known = false;+ }+ let result = refreshed.and_then(|entry| {+ store.desired_album(&entry)?;+ let (album, documents) = entry.value?;+ if selected {+ store.snapshot_album(&album, &documents, session.user_id)?;+ }+ store+ .db+ .execute("UPDATE desired_albums SET ready=1 WHERE id=?1", [id])?;+ Ok(())+ });+ if let Err(error) = result {+ store.db.execute(+ "UPDATE desired_albums SET failure=?1 WHERE id=?2",+ params![error.to_string(), id],+ )?;+ if selected {+ record_outcome(store, &format!("album:{id}"), &error)?;+ report(&format!("album:{id}: {error:#}"));+ }+ if fatal(&error) {+ return Err(error);+ }+ }+ }+ store.db.execute(+ "UPDATE desired_albums SET ready=1 WHERE selected=1 AND present=0",+ [],+ )?;+ if need_favorites && favorites_known {+ let mut after = 0;+ loop {+ let ids = source.favorite_page(after)?;+ if ids.is_empty() {+ break;+ }+ let transaction = store.db.unchecked_transaction()?;+ for id in ids {+ after = id;+ store.db.execute("INSERT INTO favorites VALUES(?1)", [id])?;+ }+ transaction.commit()?;+ }+ }+ after = 0;+ loop {+ let id: Option<i64> = store+ .db+ .query_row(+ "SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL AND id>?1 ORDER BY id LIMIT 1",+ [after],+ |r| r.get(0),+ )+ .optional()?;+ let Some(id) = id else { break };+ after = id;+ let mut after_file = 0;+ loop {+ ensure!(!*cancel.borrow(), Cancelled);+ let records = source.file_page(id, after_file)?;+ if records.is_empty() {+ break;+ }+ let transaction = store.db.unchecked_transaction()?;+ for record in records {+ after_file = record.id;+ let file = record+ .value+ .and_then(|(file, documents)| FileSnapshot::new(file, &documents));+ let failure = if !favorites_known {+ Some("favorite state is incomplete".to_owned())+ } else {+ file.as_ref().err().map(ToString::to_string)+ };+ store.db.execute(+ "INSERT INTO desired_files(album,file,favorited,failure) VALUES(?1,?2,?3,?4)",+ params![+ id,+ record.id,+ if favorites_known { Some(false) } else { None },+ failure+ ],+ )?;+ if failure.is_none() {+ let file = file?;+ store.db.execute(+ "INSERT INTO chosen_files VALUES(?1,?2,?3) ON CONFLICT(file) DO UPDATE SET album=excluded.album,version=excluded.version WHERE excluded.version>chosen_files.version OR (excluded.version=chosen_files.version AND excluded.album<chosen_files.album)",+ params![file.id, id, file.updated_at_micros],+ )?;+ }+ }+ transaction.commit()?;+ }+ }+ store+ .db+ .execute(+ "UPDATE desired_files SET favorited=EXISTS(SELECT 1 FROM favorites WHERE favorites.file=desired_files.file) WHERE favorited IS NOT NULL",+ [],+ )?;+ let mut after_file = 0;+ loop {+ ensure!(!*cancel.borrow(), Cancelled);+ let mut query = store+ .db+ .prepare("SELECT album,file FROM chosen_files WHERE file>?1 ORDER BY file LIMIT 128")?;+ let chosen: Vec<(i64, i64)> = query+ .query_map([after_file], |r| Ok((r.get(0)?, r.get(1)?)))?+ .collect::<rusqlite::Result<_>>()?;+ drop(query);+ if chosen.is_empty() {+ break;+ }+ let transaction = store.db.unchecked_transaction()?;+ for (album, id) in chosen {+ after_file = id;+ let (file, documents) = source.file(album, id)?.value?;+ store.snapshot_file(&FileSnapshot::new(file, &documents)?)?;+ }+ transaction.commit()?;+ }+ let transaction = store.db.unchecked_transaction()?;+ store.db.execute(+ "DELETE FROM sources WHERE file NOT IN (SELECT file FROM chosen_files)",+ [],+ )?;+ store+ .db+ .execute(+ "DELETE FROM album_sources WHERE id NOT IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM albums WHERE albums.id=album_sources.id) AND NOT EXISTS(SELECT 1 FROM pending WHERE album=album_sources.id)",+ [],+ )?;+ transaction.commit()?;+ let mut query = store+ .db+ .prepare("SELECT album,file,failure FROM desired_files WHERE failure IS NOT NULL")?;+ let mut rows = query.query([])?;+ while let Some(row) = rows.next()? {+ let unit = format!("file:{}:{}", row.get::<_, i64>(0)?, row.get::<_, i64>(1)?);+ let error = anyhow::anyhow!(row.get::<_, String>(2)?);+ record_outcome(store, &unit, &error)?;+ report(&format!("{unit}: {error:#}"));+ }+ Ok(())+}++struct CancelWorkers<'a>(&'a watch::Sender<bool>);++impl Drop for CancelWorkers<'_> {+ fn drop(&mut self) {+ if std::thread::panicking() {+ self.0.send_replace(true);+ }+ }+}++fn transfers(+ root: &Path,+ store: &Mutex<Store>,+ session: &ente_core::Session,+ cancel_send: &watch::Sender<bool>,+ cancel: &watch::Receiver<bool>,+ jobs: usize,+ report: &(dyn Fn(&str) + Sync),+) -> Result<()> {+ let (expected, files): (i64, i64) = store::lock(store)?.db.query_row(+ "SELECT count(*),count(DISTINCT file) FROM desired_files WHERE failure IS NULL",+ [],+ |r| Ok((r.get(0)?, r.get(1)?)),+ )?;+ report(&format!("Maintaining {expected} selected copies."));+ let runtime = tokio::runtime::Handle::current();+ let last_report = Mutex::new(None::<Instant>);+ std::thread::scope(|scope| {+ let mut workers = Vec::new();+ for _ in 0..jobs.min(files as usize) {+ let runtime = &runtime;+ let last_report = &last_report;+ workers.push(scope.spawn(move || {+ let _cancel_on_panic = CancelWorkers(cancel_send);+ let run = reconcile::Run { root, store, cancel, report };+ let result = (|| {+ loop {+ if *cancel.borrow() {+ break;+ }+ let file = {+ let store = store::lock(store)?;+ let transaction = store.db.unchecked_transaction()?;+ let id: Option<i64> = store+ .db+ .query_row(+ "SELECT file FROM desired_files WHERE failure IS NULL AND claimed=0 ORDER BY file LIMIT 1",+ [],+ |r| r.get(0),+ )+ .optional()?;+ let Some(id) = id else {+ break;+ };+ store+ .db+ .execute(+ "UPDATE desired_files SET claimed=1 WHERE file=?1",+ [id],+ )?;+ let file: FileSnapshot = store+ .json("SELECT record FROM sources WHERE file=?1", [id])?+ .context("missing file source snapshot")?;+ transaction.commit()?;+ file+ };+ let id = file.id;+ let result = runtime.block_on(run.file(session, file));+ if result.as_ref().err().is_some_and(fatal) {+ cancel_send.send_replace(true);+ }+ let reported = result.or_else(|error| run.block_file(id, error));+ let cleaned = run.cleanup(Some(id));+ reported?;+ cleaned?;+ let completed = {+ let mut last = last_report.lock().map_err(|_| anyhow::anyhow!("progress mutex poisoned"))?;+ if last.is_none_or(|time| time.elapsed() >= Duration::from_secs(2)) {+ *last = Some(Instant::now());+ Some(store::lock(store)?.db.query_row(+ "SELECT count(*) FROM desired_files WHERE completed=1",+ [],+ |r| r.get::<_, i64>(0),+ )?)+ } else {+ None+ }+ };+ if let Some(completed) = completed {+ report(&format!("Completed {completed}/{expected} selected copies."));+ }+ }+ Ok(())+ })();+ if result.is_err() {+ cancel_send.send_replace(true);+ }+ result+ }));+ }+ let mut result = Ok(());+ for worker in workers {+ let completed = worker+ .join()+ .unwrap_or_else(|_| Err(anyhow::anyhow!("export worker panicked")));+ if result.is_ok() {+ result = completed;+ }+ }+ result+ })+}++fn record_outcome(store: &Store, unit: &str, error: &anyhow::Error) -> Result<()> {+ let conflict = error.downcast_ref::<Conflict>().is_some();+ store+ .db+ .execute(+ "INSERT INTO outcomes VALUES(?1,?2) ON CONFLICT(unit) DO UPDATE SET conflict=max(conflict,excluded.conflict)",+ params![unit, conflict],+ )?;+ Ok(())+}++fn fatal(error: &anyhow::Error) -> bool {+ fatal_cause(error.as_ref())+}++fn fatal_cause(cause: &(dyn std::error::Error + 'static)) -> bool {+ if cause.is::<rusqlite::Error>()+ || cause.is::<Cancelled>()+ || cause+ .downcast_ref::<ente_core::http::Error>()+ .is_some_and(|error| error.status_code() == Some(401))+ {+ return true;+ }+ if let Some(error) = cause.downcast_ref::<std::io::Error>() {+ return error.kind() == std::io::ErrorKind::StorageFull+ || error.get_ref().is_some_and(|inner| fatal_cause(inner));+ }+ if let Some(error) = cause.downcast_ref::<ente_photos::files::Error>() {+ use ente_photos::files::Error;+ match error {+ Error::Http(error) => return fatal_cause(error),+ Error::Io(error) => return fatal_cause(error),+ Error::Collections(error) => return fatal_cause(error),+ _ => {}+ }+ }+ if let Some(ente_collections::Error::Http(error)) = cause.downcast_ref() {+ return fatal_cause(error);+ }+ if let Some(error) = cause.downcast_ref::<crate::live_photo::Error>() {+ match error {+ crate::live_photo::Error::Io(error) => return fatal_cause(error),+ crate::live_photo::Error::Zip(error) => return fatal_cause(error),+ crate::live_photo::Error::Components => {}+ }+ }+ cause.source().is_some_and(fatal_cause)+}++fn summary(store: &Store, destination: &Path, prepared: bool) -> Result<Summary> {+ let unknown: bool = !prepared+ || store.db.query_row(+ "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND ready=0)",+ [],+ |r| r.get(0),+ )?;+ let (files, expected, completed): (i64, i64, i64) = store.db.query_row(+ "SELECT count(DISTINCT file),count(*),coalesce(sum(completed),0) FROM desired_files",+ [],+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),+ )?;+ let (failures, conflicts): (i64, i64) = store.db.query_row(+ "SELECT count(*),coalesce(sum(conflict),0) FROM outcomes",+ [],+ |r| Ok((r.get(0)?, r.get(1)?)),+ )?;+ let (exported, metadata_updated, renamed, retained): (i64, i64, i64, i64) = store+ .db+ .query_row(+ "SELECT coalesce(sum(exported),0),coalesce(sum(metadata_updated),0),coalesce(sum(renamed),0),coalesce(sum(retained),0) FROM events",+ [],+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),+ )?;+ let complete = !unknown && failures == 0 && completed == expected;+ let result = json!({"destination":destination,"complete":complete,"files":if unknown {None}else{Some(files)},"copies":if unknown {None}else{Some(json!({"expected":expected,"completed":completed,"pending":expected-completed}))},"changes":{"exported":exported,"metadataUpdated":metadata_updated,"renamed":renamed,"retained":retained},"failures":failures,"conflicts":conflicts});+ Ok(Summary {+ json: result,+ complete,+ completed,+ expected,+ })+}++fn canonical_id(selector: &str) -> rusqlite::types::Value {+ selector+ .parse::<i64>()+ .ok()+ .filter(|id| id.to_string() == selector)+ .map(rusqlite::types::Value::Integer)+ .unwrap_or(rusqlite::types::Value::Null)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn transparent_errors_preserve_fatal_classification() {+ for status in [401, 404] {+ let error = ente_core::http::Error::Http {+ status,+ path: "/files".into(),+ };+ let error =+ ente_photos::files::Error::Collections(ente_collections::Error::Http(error));+ let error = anyhow::Error::new(error).context("download failed");+ assert_eq!(fatal(&error), status == 401);+ assert_eq!(+ format!("{error:#}"),+ format!("download failed: HTTP {status} at /files")+ );+ }+ let cancelled = crate::live_photo::Error::Io(std::io::Error::other(Cancelled));+ assert!(fatal(&anyhow::Error::new(cancelled)));+ let full = crate::live_photo::Error::Io(std::io::ErrorKind::StorageFull.into());+ assert!(fatal(&anyhow::Error::new(full)));+ let zip = crate::live_photo::Error::Zip(zip::result::ZipError::Io(+ std::io::ErrorKind::StorageFull.into(),+ ));+ assert!(fatal(&anyhow::Error::new(zip)));+ let sqlite = rusqlite::Error::InvalidQuery;+ assert!(fatal(&anyhow::Error::new(sqlite)));+ let local = ente_photos::files::Error::Io(std::io::ErrorKind::PermissionDenied.into());+ assert!(!fatal(&anyhow::Error::new(local)));+ }+}diff --git a/rust/crates/photos/src/live_photo.rs b/rust/crates/photos-export/src/live_photo.rssimilarity index 98%rename from rust/crates/photos/src/live_photo.rsrename to rust/crates/photos-export/src/live_photo.rsindex b15f93a8f4..daa9832822 100644--- a/rust/crates/photos/src/live_photo.rs+++ b/rust/crates/photos-export/src/live_photo.rs@@ -1,6 +1,6 @@use std::io::{Read, Seek, Write};-use crate::export::Role;+use crate::metadata::Role;#[derive(Debug, thiserror::Error)]pub enum Error {diff --git a/rust/crates/photos-export/src/metadata.rs b/rust/crates/photos-export/src/metadata.rsnew file mode 100644index 0000000000..884d828f7a--- /dev/null+++ b/rust/crates/photos-export/src/metadata.rs@@ -0,0 +1,314 @@+use chrono::{DateTime, SecondsFormat};+use ente_core::{+ b64,+ crypto::{Key, kdf},+};+use serde::{Deserialize, Serialize};+use serde_json::{Value, json};++use ente_photos::{+ collections::{Collection, Visibility},+ files::{File, Kind},+ source::{self, Documents, MetadataError},+};++#[derive(Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+pub struct Root {+ pub format: String,+ pub version: u32,+ pub source_verifier: String,+}++impl Root {+ pub fn new(master_key: &Key) -> ente_core::crypto::Result<Self> {+ Ok(Self {+ format: "ente-photos-export".into(),+ version: 1,+ source_verifier: b64::encode(&kdf::derive_subkey(master_key, 32, 1, b"photoexp")?),+ })+ }+}++#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "lowercase")]+pub enum Role {+ Original,+ Image,+ Video,+}++impl Role {+ pub fn name(&self) -> &'static str {+ match self {+ Self::Original => "original",+ Self::Image => "image",+ Self::Video => "video",+ }+ }+}++#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct Component {+ pub role: Role,+ pub path: String,+ pub size: u64,+ pub hash: String,+}++pub fn album(+ album: &Collection,+ documents: &Documents,+ user_id: i64,+) -> Result<(Value, Vec<String>), MetadataError> {+ let private: AlbumPrivate = source::optional(+ documents.private.as_deref(),+ "invalid album private metadata",+ )?;+ let shared: AlbumShared =+ source::optional(documents.shared.as_deref(), "invalid album sharee metadata")?;+ let public: AlbumPublic =+ source::optional(documents.public.as_deref(), "invalid album public metadata")?;+ let owned = album.owner_id == user_id;+ let mut metadata = json!({+ "albumID": album.id.to_string(), "type": album.kind.name(), "visibility": album.visibility.name(),+ });+ optional(+ &mut metadata,+ "displayOrder",+ if owned { private.order } else { shared.order },+ );+ optional(+ &mut metadata,+ "muted",+ (!owned).then_some(shared.mute).flatten(),+ );+ optional(+ &mut metadata,+ "sortOrder",+ public+ .asc+ .map(|asc| if asc { "ascending" } else { "descending" }),+ );+ optional(&mut metadata, "description", public.caption.as_deref());+ optional(+ &mut metadata,+ "coverFileID",+ public.cover_id.map(|id| id.to_string()),+ );+ optional(&mut metadata, "layout", public.layout.as_deref());+ let mut warnings = Vec::new();+ unfamiliar(+ documents.private.as_deref(),+ &["visibility", "subType", "order"],+ &mut warnings,+ );+ unfamiliar(+ documents.shared.as_deref(),+ &["visibility", "mute", "order"],+ &mut warnings,+ );+ unfamiliar(+ documents.public.as_deref(),+ &["asc", "coverID", "layout", "caption"],+ &mut warnings,+ );+ Ok((json!({"title": album.name, "ente": metadata}), warnings))+}++pub fn file(file: &File, documents: &Documents) -> Result<(Value, Vec<String>), MetadataError> {+ let public: FilePublic =+ source::optional(documents.public.as_deref(), "invalid public file metadata")?;+ let mut metadata = json!({+ "fileID": file.id.to_string(),+ "type": if file.kind == Kind::LivePhoto { "livePhoto" } else { file.kind.name() },+ "name": file.name,+ "creationTime": timestamp(file.created_at_micros)?,+ });+ optional(+ &mut metadata,+ "modificationTime",+ file.modified_at_micros.map(timestamp).transpose()?,+ );+ optional(+ &mut metadata,+ "location",+ file.location+ .as_ref()+ .map(|l| json!({"latitude": l.latitude,"longitude": l.longitude})),+ );+ optional(&mut metadata, "localDateTime", file.date_time.as_deref());+ optional(&mut metadata, "utcOffset", file.offset_time.as_deref());+ optional(&mut metadata, "width", file.width);+ optional(&mut metadata, "height", file.height);+ optional(&mut metadata, "durationSeconds", file.duration_seconds);+ optional(+ &mut metadata,+ "visibility",+ file.visibility.map(Visibility::name),+ );+ optional(&mut metadata, "cameraMake", public.camera_make.as_deref());+ optional(&mut metadata, "cameraModel", public.camera_model.as_deref());+ optional(+ &mut metadata,+ "uploaderName",+ public.uploader_name.as_deref(),+ );+ optional(+ &mut metadata,+ "panorama",+ if file.kind == Kind::Image {+ public.media_type.map(|value| value & 1 != 0)+ } else {+ None+ },+ );+ optional(&mut metadata, "motionPhotoVideoOffset", public.mvi);+ let taken = common_time(file.created_at_micros)?;+ let mut record = json!({+ "photoTakenTime": taken,+ "creationTime": taken,+ "ente": metadata,+ });+ optional(+ &mut record,+ "modificationTime",+ file.modified_at_micros.map(common_time).transpose()?,+ );+ optional(&mut record, "description", file.caption.as_deref());+ optional(+ &mut record,+ "geoData",+ file.location+ .as_ref()+ .map(|l| json!({"latitude":l.latitude,"longitude":l.longitude})),+ );+ let mut warnings = Vec::new();+ unfamiliar(+ Some(&documents.original),+ &[+ "title",+ "fileType",+ "creationTime",+ "modificationTime",+ "latitude",+ "longitude",+ "hash",+ "imageHash",+ "videoHash",+ "duration",+ "hasStaticThumbnail",+ "localID",+ "deviceFolder",+ "subType",+ "version",+ "exif",+ ],+ &mut warnings,+ );+ unfamiliar(+ documents.public.as_deref(),+ &[+ "editedName",+ "editedTime",+ "dateTime",+ "offsetTime",+ "caption",+ "lat",+ "long",+ "w",+ "h",+ "sv",+ "cameraMake",+ "cameraModel",+ "uploaderName",+ "mediaType",+ "mvi",+ "noThumb",+ ],+ &mut warnings,+ );+ unfamiliar(documents.private.as_deref(), &["visibility"], &mut warnings);+ Ok((record, warnings))+}++fn optional(object: &mut Value, name: &str, value: Option<impl Into<Value>>) {+ if let Some(value) = value {+ object[name] = value.into();+ }+}++fn time(micros: i64) -> Result<DateTime<chrono::Utc>, MetadataError> {+ DateTime::from_timestamp_micros(micros)+ .ok_or(MetadataError("timestamp is outside the supported range"))+}++fn timestamp(micros: i64) -> Result<String, MetadataError> {+ Ok(time(micros)?.to_rfc3339_opts(SecondsFormat::Micros, true))+}++fn common_time(micros: i64) -> Result<Value, MetadataError> {+ let time = time(micros)?;+ Ok(+ json!({"timestamp":micros.div_euclid(1_000_000).to_string(),"formatted":time.format("%b %-d, %Y, %-I:%M:%S %p UTC").to_string()}),+ )+}++pub fn publish(+ template: &Value,+ components: &[Component],+ component: usize,+ favorited: bool,+) -> Result<Value, MetadataError> {+ let selected = components+ .get(component)+ .ok_or(MetadataError("missing media component"))?;+ let mut record = template.clone();+ record["title"] = json!(selected.path);+ record["favorited"] = json!(favorited);+ record["ente"]["component"] = json!(selected.role);+ record["ente"]["components"] = json!(components);+ Ok(record)+}++#[derive(Default, Deserialize)]+#[serde(rename_all = "camelCase")]+struct FilePublic {+ camera_make: Option<String>,+ camera_model: Option<String>,+ uploader_name: Option<String>,+ media_type: Option<u64>,+ mvi: Option<u64>,+}++#[derive(Default, Deserialize)]+struct AlbumPrivate {+ order: Option<i64>,+}++#[derive(Default, Deserialize)]+struct AlbumShared {+ order: Option<i64>,+ mute: Option<bool>,+}++#[derive(Default, Deserialize)]+struct AlbumPublic {+ asc: Option<bool>,+ #[serde(rename = "coverID")]+ cover_id: Option<i64>,+ layout: Option<String>,+ caption: Option<String>,+}++fn unfamiliar(bytes: Option<&[u8]>, known: &[&str], warnings: &mut Vec<String>) {+ if let Some(bytes) = bytes+ && let Ok(Value::Object(fields)) = serde_json::from_slice(bytes)+ {+ for key in fields.keys() {+ if !known.contains(&key.as_str()) {+ warnings.push(key.clone());+ }+ }+ }+}diff --git a/rust/apps/cli-next/src/export/names.rs b/rust/crates/photos-export/src/names.rssimilarity index 98%rename from rust/apps/cli-next/src/export/names.rsrename to rust/crates/photos-export/src/names.rsindex 4feb7d3a8a..dd23cf0a9d 100644--- a/rust/apps/cli-next/src/export/names.rs+++ b/rust/crates/photos-export/src/names.rs@@ -131,13 +131,13 @@ pub fn available() -> Result<bool> {for name in names {let occupied: bool = if kind == "album" {- store.db.connection().query_row(+ store.db.query_row("SELECT EXISTS(SELECT 1 FROM albums WHERE parent=?1 AND folded=?2 AND key<>?3 UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND folded1=?2 AND kind='album' AND owner<>?3)",params![folder, folded(name), owner],|r| r.get(0),)?} else {- store.db.connection().query_row(+ store.db.query_row("SELECT EXISTS(SELECT 1 FROM components c JOIN placements p ON p.id=c.placement WHERE c.folder=?1 AND c.folded=?2 AND c.placement<>?3 UNION ALL SELECT 1 FROM components c JOIN placements p ON p.id=c.placement WHERE c.folder=?1 AND c.stem=?4 AND c.placement<>?3 AND NOT(p.kind=?5 AND p.kind IN ('image','video')) UNION ALL SELECT 1 FROM json_records j JOIN placements p ON p.id=j.owner WHERE j.folder=?1 AND j.folded=?2 AND j.owner<>?3 UNION ALL SELECT 1 FROM json_records j JOIN placements p ON p.id=j.owner WHERE j.folder=?1 AND j.stem=?4 AND j.owner<>?3 AND NOT(p.kind=?5 AND p.kind IN ('image','video')) UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND folded1=?2 AND owner<>?3 UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND folded2=?2 AND owner<>?3 UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND stem1=?4 AND owner<>?3 AND NOT(kind=?5 AND kind IN ('image','video')) UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND stem2=?4 AND owner<>?3 AND NOT(kind=?5 AND kind IN ('image','video')))",params![folder,diff --git a/rust/apps/cli-next/src/export/reconcile.rs b/rust/crates/photos-export/src/reconcile.rssimilarity index 95%rename from rust/apps/cli-next/src/export/reconcile.rsrename to rust/crates/photos-export/src/reconcile.rsindex 249a0c6b28..aa51f92e52 100644--- a/rust/apps/cli-next/src/export/reconcile.rs+++ b/rust/crates/photos-export/src/reconcile.rs@@ -6,8 +6,8 @@ use std::{sync::Mutex,};+use crate::{metadata::Role, snapshot::FileSnapshot};use anyhow::{Context, Result, ensure};-use ente_photos::{export::Role, files};use rusqlite::{OptionalExtension, params};use serde_json::Value;use tokio::sync::watch;@@ -26,6 +26,7 @@ pub struct Run<'a> {pub root: &'a Path,pub store: &'a Mutex<Store>,pub cancel: &'a watch::Receiver<bool>,+ pub report: &'a (dyn Fn(&str) + Sync),}impl Run<'_> {@@ -80,7 +81,6 @@ impl Run<'_> {fs::remove(&self.path(location)?)?;store::lock(self.store)?.db- .connection().execute("DELETE FROM temporaries WHERE name=?1", [&location.name])?;Ok(())}@@ -90,7 +90,7 @@ impl Run<'_> {let store = store::lock(self.store)?;super::record_outcome(&store, unit, error)?;}- eprintln!("{unit}: {error:#}");+ (self.report)(&format!("{unit}: {error:#}"));Ok(())}@@ -101,7 +101,7 @@ impl Run<'_> {);self.report(&unit, &error)?;if file.is_none() {- store::lock(self.store)?.db.connection().execute(+ store::lock(self.store)?.db.execute("UPDATE desired_albums SET failure=?1 WHERE id=?2",params![error.to_string(), album],)?;@@ -109,7 +109,6 @@ impl Run<'_> {loop {let next: Option<i64> = store::lock(self.store)?.db- .connection().query_row("SELECT file FROM desired_files WHERE album=?1 AND file>?2 ORDER BY file LIMIT 1",params![album, after],@@ -121,7 +120,7 @@ impl Run<'_> {self.report(&format!("file:{album}:{file}"), &error)?;}}- store::lock(self.store)?.db.connection().execute(+ store::lock(self.store)?.db.execute("UPDATE desired_files SET failure=?1 WHERE album=?2 AND (?3 IS NULL OR file=?3)",params![error.to_string(), album, file],)?;@@ -136,7 +135,6 @@ impl Run<'_> {loop {let next: Option<i64> = store::lock(self.store)?.db- .connection().query_row("SELECT album FROM desired_files WHERE file=?1 AND album>?2 ORDER BY album LIMIT 1",params![file, after],@@ -147,7 +145,7 @@ impl Run<'_> {after = album;self.report(&format!("file:{album}:{file}"), &error)?;}- store::lock(self.store)?.db.connection().execute(+ store::lock(self.store)?.db.execute("UPDATE desired_files SET failure=?1 WHERE file=?2",params![error.to_string(), file],)?;@@ -165,7 +163,7 @@ impl Run<'_> {) -> Result<()> {self.check_cancel()?;let store = store::lock(self.store)?;- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;let published = !store.components(&target.owner)?.is_empty();for (role, value) in snapshots {let previous = store.json_record(&target.owner, role.as_ref())?;@@ -227,7 +225,7 @@ impl Run<'_> {None};let store = store::lock(self.store)?;- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;match output {Output::Media {role,@@ -272,7 +270,6 @@ impl Run<'_> {if let Some(Action::Publish { temporary, .. }) = &target.action {store.db- .connection().execute("DELETE FROM temporaries WHERE name=?1", [&temporary.name])?;}target.action = None;@@ -302,7 +299,6 @@ impl Run<'_> {} else if !directory_exists(&destination)? {let unrepairable: bool = store::lock(self.store)?.db- .connection().query_row("SELECT EXISTS(SELECT 1 FROM placements p WHERE p.album=?1 AND p.retained=0 AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file AND f.failure IS NULL))",[target.album],@@ -334,7 +330,7 @@ impl Run<'_> {}}let store = store::lock(self.store)?;- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;store.save_album(&Album {key: target.owner.clone(),id: target.album,@@ -410,7 +406,7 @@ impl Run<'_> {}fs::sync_move(&from, &to)?;let store = store::lock(self.store)?;- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;let mut component = store.components(&target.owner)?.into_iter()@@ -427,7 +423,7 @@ impl Run<'_> {Action::Remove { location, role } => {fs::remove(&self.path(&location)?)?;let store = store::lock(self.store)?;- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;let mut record = store.json_record(&target.owner, role.as_ref())?.context("missing removed JSON record")?;@@ -618,7 +614,6 @@ impl Run<'_> {let store = store::lock(self.store)?;store.db- .connection().query_row("SELECT p.owner,p.album,p.file FROM pending p JOIN desired_albums a ON a.id=p.album WHERE p.owner>?1 AND (p.file IS NULL)=?2 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file AND f.failure IS NOT NULL) ORDER BY p.owner LIMIT 1",params![after, albums],@@ -646,17 +641,15 @@ impl Run<'_> {}{let store = store::lock(self.store)?;- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;store.db- .connection().execute("DELETE FROM pending WHERE file IS NULL AND action='null' AND album IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL) AND ((retained=0 AND album IN (SELECT id FROM desired_albums WHERE present=0)) OR (retained=1 AND NOT EXISTS(SELECT 1 FROM albums WHERE key=pending.owner) AND NOT EXISTS(SELECT 1 FROM placements p WHERE p.album=pending.album AND p.retained=0 AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file))))",[],)?;store.db- .connection().execute("DELETE FROM json_records WHERE folder IS NULL AND NOT EXISTS(SELECT 1 FROM albums WHERE key=json_records.owner) AND NOT EXISTS(SELECT 1 FROM components WHERE placement=json_records.owner) AND NOT EXISTS(SELECT 1 FROM pending WHERE owner=json_records.owner)",[],@@ -673,7 +666,6 @@ impl Run<'_> {let store = store::lock(self.store)?;store.db- .connection().query_row("SELECT t.name,t.folder,t.album,t.file FROM temporaries t JOIN desired_albums a ON a.id=t.album WHERE t.name>?1 AND (?2 IS NULL OR t.file=?2) AND a.selected=1 ORDER BY t.name LIMIT 1",params![after, file],@@ -699,7 +691,6 @@ impl Run<'_> {let store = store::lock(self.store)?;let owner: Option<String> = store.db- .connection().query_row("SELECT owner FROM pending WHERE json_extract(action,'$.Publish.temporary.name')=?1",[&temporary.location.name],@@ -712,7 +703,7 @@ impl Run<'_> {.flatten()};if let Some(mut pending) = pending {- let selected: bool = store::lock(self.store)?.db.connection().query_row(+ let selected: bool = store::lock(self.store)?.db.query_row("SELECT selected FROM desired_albums WHERE id=?1",[pending.album],|r| r.get(0),@@ -728,7 +719,7 @@ impl Run<'_> {})();if let Err(error) = result {if let Some(file) = temporary.file {- let selected: bool = store::lock(self.store)?.db.connection().query_row(+ let selected: bool = store::lock(self.store)?.db.query_row("SELECT EXISTS(SELECT 1 FROM desired_files WHERE file=?1)",[file],|r| r.get(0),@@ -757,7 +748,7 @@ impl Run<'_> {return Ok(pending);}}- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;let parent = if retained { "Trash" } else { "" };let names = if let Some(existing) = &existing&& (retained || existing.name == name)@@ -846,7 +837,6 @@ impl Run<'_> {let store = store::lock(self.store)?;store.db- .connection().execute("INSERT INTO events(placement,renamed) SELECT id,1 FROM placements WHERE album=?1 AND retained=0 ON CONFLICT(placement) DO UPDATE SET renamed=1",[album],@@ -873,7 +863,6 @@ impl Run<'_> {loop {let source: Option<(i64, String)> = store::lock(self.store)?.db- .connection().query_row("SELECT a.id,s.record FROM desired_albums a JOIN album_sources s ON s.id=a.id WHERE a.id>?1 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND a.present=1 ORDER BY a.id LIMIT 1",[after],@@ -886,9 +875,9 @@ impl Run<'_> {after = id;let source: AlbumSource = serde_json::from_str(&source)?;for field in &source.warnings {- eprintln!(+ (self.report)(&format!("album {id}: unfamiliar metadata field {field:?}; exporting supported fields"- );+ ));}if let Err(error) = self.maintain_album(id, &source.name, false, source.metadata) {self.block(id, None, error)?;@@ -902,7 +891,7 @@ impl Run<'_> {fn file_target(&self,album: &Album,- file: &files::File,+ file: &FileSnapshot,prepared: &transfer::Prepared,) -> Result<Pending> {let store = store::lock(self.store)?;@@ -949,7 +938,7 @@ impl Run<'_> {let same_folder = previous.folder == album.key;if same_folder&& previous.name == file.name- && previous.kind == file.kind.name()+ && previous.kind == file.kind.as_str()&& compatible(&previous.names){return Ok(previous);@@ -960,12 +949,12 @@ impl Run<'_> {.map(|component| component.location.name.clone()).collect();let unchanged = existing.as_ref().is_some_and(|placement| {- placement.name == file.name && placement.kind == file.kind.name()+ placement.name == file.name && placement.kind == file.kind.as_str()}) && components.iter().all(|component| component.location.folder == album.key)&& compatible(&existing_names);- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;let names = if unchanged {existing_names} else {@@ -973,7 +962,7 @@ impl Run<'_> {&store,&album.key,&file.name,- file.kind.name(),+ file.kind.as_str(),if roles.len() == 2 {Some((&extensions[0], &extensions[1]))} else {@@ -988,7 +977,7 @@ impl Run<'_> {file: Some(file.id),retained: false,name: file.name.clone(),- kind: file.kind.name().into(),+ kind: file.kind.clone(),folder: album.key.clone(),names,action: None,@@ -1002,13 +991,12 @@ impl Run<'_> {Ok(target)}- pub async fn file(&self, session: &ente_core::Session, file: files::File) -> Result<()> {+ pub async fn file(&self, session: &ente_core::Session, file: FileSnapshot) -> Result<()> {let mut prepared = transfer::prepare(self, session, &file).await?;let mut after = 0;loop {let album: Option<i64> = store::lock(self.store)?.db- .connection().query_row("SELECT album FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",params![file.id, after],@@ -1039,17 +1027,16 @@ impl Run<'_> {}}for field in &file.warnings {- eprintln!(+ (self.report)(&format!("file {}: unfamiliar metadata field {field:?}; exporting supported fields",file.id- );+ ));}after = 0;loop {self.check_cancel()?;let next: Option<(i64, bool)> = store::lock(self.store)?.db- .connection().query_row("SELECT album,favorited FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",params![file.id, after],@@ -1089,7 +1076,7 @@ impl Run<'_> {fn placement(&self,- file: &files::File,+ file: &FileSnapshot,prepared: &mut transfer::Prepared,album_id: i64,favorited: bool,@@ -1128,7 +1115,7 @@ impl Run<'_> {.context("missing completed original")?;(component.size, component.hash.clone())};- Ok(ente_photos::export::Component {+ Ok(crate::metadata::Component {role,path: name.clone(),size,@@ -1142,7 +1129,7 @@ impl Run<'_> {.map(|(index, component)| {Ok((Some(component.role.clone()),- ente_photos::export::file(file, &intended, index, favorited)?,+ crate::metadata::publish(&file.metadata, &intended, index, favorited)?,))}).collect::<Result<_>>()?;@@ -1185,7 +1172,7 @@ impl Run<'_> {let changed = self.json_changed(previous.as_ref(), &location, &json)?;sidecars.push((location, json, changed, previous));}- let intended_time = file.modified_at_micros.unwrap_or(file.created_at_micros);+ let intended_time = file.intended_time;let mut wrote_media = false;let mut renamed = false;for component in &intended {@@ -1288,7 +1275,7 @@ impl Run<'_> {}}let store = store::lock(self.store)?;- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;if existing.as_ref().is_some_and(|previous| previous.name != target.name || previous.kind != target.kind)@@ -1296,7 +1283,7 @@ impl Run<'_> {store.save_placement(&target.placement()?)?;}store.clear_pending(&target.owner)?;- store.db.connection().execute(+ store.db.execute("UPDATE desired_files SET completed=1 WHERE album=?1 AND file=?2",params![album_id, file.id],)?;@@ -1317,7 +1304,6 @@ impl Run<'_> {fn event(store: &Store, placement: &str, name: &str) -> Result<()> {store.db- .connection().execute(&format!("INSERT INTO events(placement,{name}) VALUES(?1,1) ON CONFLICT(placement) DO UPDATE SET {name}=1",@@ -1367,7 +1353,7 @@ impl Run<'_> {let metadata = store::lock(self.store)?.json_record(&placement.id, Some(&template.role))?.context("missing retained metadata snapshot")?;- let layout: Vec<ente_photos::export::Component> =+ let layout: Vec<crate::metadata::Component> =serde_json::from_value(metadata.value["ente"]["components"].clone())?;let extensions = if placement.kind == "livephoto" {Some((@@ -1395,7 +1381,7 @@ impl Run<'_> {};let mut target = {let store = store::lock(self.store)?;- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;let names = names::allocate(&store,&album.key,@@ -1445,7 +1431,7 @@ impl Run<'_> {};let mut value = previous.value;value["title"] = Value::String(component.location.name.clone());- let mut layout: Vec<ente_photos::export::Component> =+ let mut layout: Vec<crate::metadata::Component> =serde_json::from_value(value["ente"]["components"].clone())?;for identity in &mut layout {identity.path = target_component(target, &identity.role)?.into();@@ -1466,7 +1452,7 @@ impl Run<'_> {)?;}let store = store::lock(self.store)?;- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;store.save_placement(&target.placement()?)?;store.clear_pending(&target.owner)?;event(&store, &target.owner, "retained")?;@@ -1482,7 +1468,6 @@ impl Run<'_> {let store = store::lock(self.store)?;store.db- .connection().query_row("SELECT p.id,p.album,p.file,p.retained,p.name,p.kind FROM placements p JOIN desired_albums a ON a.id=p.album WHERE p.id>?1 AND p.retained=0 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file) AND NOT EXISTS(SELECT 1 FROM outcomes WHERE unit='file:'||p.album||':'||p.file) ORDER BY p.id LIMIT 1",[&after],@@ -1501,17 +1486,15 @@ impl Run<'_> {}{let store = store::lock(self.store)?;- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;store.db- .connection().execute("DELETE FROM pending WHERE file IS NOT NULL AND action='null' AND album IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM placements WHERE id=pending.owner) AND NOT EXISTS(SELECT 1 FROM desired_files WHERE album=pending.album AND file=pending.file)",[],)?;store.db- .connection().execute("DELETE FROM json_records WHERE folder IS NULL AND NOT EXISTS(SELECT 1 FROM albums WHERE key=json_records.owner) AND NOT EXISTS(SELECT 1 FROM components WHERE placement=json_records.owner) AND NOT EXISTS(SELECT 1 FROM pending WHERE owner=json_records.owner)",[],@@ -1523,7 +1506,6 @@ impl Run<'_> {self.check_cancel()?;let id: Option<i64> = store::lock(self.store)?.db- .connection().query_row("SELECT id FROM desired_albums WHERE id>?1 AND selected=1 AND ready=1 AND failure IS NULL AND present=0 ORDER BY id LIMIT 1",[after],@@ -1548,7 +1530,7 @@ impl Run<'_> {store.clear_pending(&format!("active:{id}"))?;return Ok(());};- let incomplete: bool = store::lock(self.store)?.db.connection().query_row(+ let incomplete: bool = store::lock(self.store)?.db.query_row("SELECT EXISTS(SELECT 1 FROM placements WHERE album=?1 AND retained=0) OR EXISTS(SELECT 1 FROM pending WHERE album=?1 AND file IS NOT NULL)",[id],|r| r.get(0),@@ -1578,10 +1560,9 @@ impl Run<'_> {}}let store = store::lock(self.store)?;- let transaction = store.db.connection().unchecked_transaction()?;+ let transaction = store.db.unchecked_transaction()?;store.db- .connection().execute("DELETE FROM albums WHERE key=?1 AND NOT EXISTS(SELECT 1 FROM temporaries WHERE folder=?1)",[&active.key],diff --git a/rust/crates/photos-export/src/snapshot.rs b/rust/crates/photos-export/src/snapshot.rsnew file mode 100644index 0000000000..3ae05e18b2--- /dev/null+++ b/rust/crates/photos-export/src/snapshot.rs@@ -0,0 +1,75 @@+use anyhow::Result;+use ente_core::crypto::{Header, Key};+use ente_photos::{files::File, source::Documents};+use serde::{Deserialize, Serialize};+use serde_json::Value;++use crate::metadata;++#[derive(Serialize, Deserialize)]+pub(crate) struct FileSnapshot {+ pub id: i64,+ pub updated_at_micros: i64,+ pub name: String,+ pub kind: String,+ pub hash: Option<String>,+ #[serde(with = "key")]+ pub key: Key,+ #[serde(with = "header")]+ pub header: Header,+ pub intended_time: i64,+ pub metadata: Value,+ pub warnings: Vec<String>,+}++impl FileSnapshot {+ pub fn new(file: File, documents: &Documents) -> Result<Self> {+ let (metadata, warnings) = metadata::file(&file, documents)?;+ Ok(Self {+ id: file.id,+ updated_at_micros: file.updated_at_micros,+ name: file.name,+ kind: file.kind.name().into(),+ hash: file.hash,+ key: file.key,+ header: file.header,+ intended_time: file.modified_at_micros.unwrap_or(file.created_at_micros),+ metadata,+ warnings,+ })+ }+}++mod key {+ use ente_core::crypto::Key;+ use serde::{Deserialize, Deserializer, Serializer, de::Error};++ pub fn serialize<S: Serializer>(key: &Key, serializer: S) -> Result<S::Ok, S::Error> {+ serializer.serialize_str(&ente_core::b64::encode(key.as_bytes()))+ }++ pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Key, D::Error> {+ Key::try_from_slice(+ &ente_core::b64::decode(&String::deserialize(deserializer)?)+ .map_err(D::Error::custom)?,+ )+ .map_err(D::Error::custom)+ }+}++mod header {+ use ente_core::crypto::Header;+ use serde::{Deserialize, Deserializer, Serializer, de::Error};++ pub fn serialize<S: Serializer>(header: &Header, serializer: S) -> Result<S::Ok, S::Error> {+ serializer.serialize_str(&ente_core::b64::encode(header.as_bytes()))+ }++ pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Header, D::Error> {+ Header::try_from_slice(+ &ente_core::b64::decode(&String::deserialize(deserializer)?)+ .map_err(D::Error::custom)?,+ )+ .map_err(D::Error::custom)+ }+}diff --git a/rust/crates/photos-export/src/source.rs b/rust/crates/photos-export/src/source.rsnew file mode 100644index 0000000000..9170faec91--- /dev/null+++ b/rust/crates/photos-export/src/source.rs@@ -0,0 +1,30 @@+use std::future::Future;++use anyhow::Result;+use ente_core::Session;+use ente_photos::{collections::Collection, files::File, source::Documents};++pub struct CollectionEntry {+ pub id: i64,+ pub name: Option<String>,+ pub favorites: bool,+ pub value: Result<(Collection, Documents)>,+}++pub struct FileEntry {+ pub id: i64,+ pub value: Result<(File, Documents)>,+}++pub trait Source {+ fn refresh_catalog(&mut self, session: &Session) -> impl Future<Output = Result<()>>;+ fn catalog_page(&self, after: i64) -> Result<Vec<CollectionEntry>>;+ fn refresh_album(+ &mut self,+ session: &Session,+ id: i64,+ ) -> impl Future<Output = Result<CollectionEntry>>;+ fn file_page(&self, album: i64, after: i64) -> Result<Vec<FileEntry>>;+ fn favorite_page(&self, after: i64) -> Result<Vec<i64>>;+ fn file(&self, album: i64, id: i64) -> Result<FileEntry>;+}diff --git a/rust/apps/cli-next/src/export/store.rs b/rust/crates/photos-export/src/store.rssimilarity index 89%rename from rust/apps/cli-next/src/export/store.rsrename to rust/crates/photos-export/src/store.rsindex ba069a4f91..e05d491efc 100644--- a/rust/apps/cli-next/src/export/store.rs+++ b/rust/crates/photos-export/src/store.rs@@ -1,25 +1,20 @@use std::{- path::{Path, PathBuf},+ path::Path,sync::{Mutex, MutexGuard},time::{SystemTime, UNIX_EPOCH},};use anyhow::{Context, Result, ensure};-use ente_core::crypto::hash;-use ente_photos::{collections::Collection, export::Role, files::File};-use rusqlite::{OptionalExtension, Row, params};+use ente_photos::{collections::Collection, source::Documents};+use rusqlite::{Connection, OptionalExtension, Row, TransactionBehavior, params};use serde::{Deserialize, Serialize, de::DeserializeOwned};use serde_json::Value;use super::names;-use crate::{- db, home,- replica::{AlbumRecord, read_json},- vault::DbKey,-};+use crate::{AdoptionRequired, CollectionEntry, metadata::Role, snapshot::FileSnapshot};pub struct Store {- pub db: crate::core_db::Db,+ pub db: Connection,}pub fn lock(store: &Mutex<Store>) -> Result<MutexGuard<'_, Store>> {@@ -107,8 +102,8 @@ impl Component {})}- pub fn portable(&self) -> ente_photos::export::Component {- ente_photos::export::Component {+ pub fn portable(&self) -> crate::metadata::Component {+ crate::metadata::Component {role: self.role.clone(),path: self.location.name.clone(),size: self.size,@@ -311,24 +306,26 @@ pub enum Action {}impl Store {- pub fn path(account_home: &Path, destination: &Path) -> Result<PathBuf> {- let bytes = destination.as_os_str().as_encoded_bytes();- let name: String = hash::hash(bytes, Some(32), None)?- .iter()- .map(|b| format!("{b:02x}"))- .collect();- Ok(account_home.join("exports").join(format!("{name}.db")))- }-- pub fn open(path: &Path, key: &DbKey, destination: &Path, create: bool) -> Result<Self> {- if create {- home::create(path.parent().context("export DB has no parent")?)?;- }- let mut db = crate::core_db::Db::new(db::connect(path, key, create)?);- if create {- db.migrate(&[SCHEMA])?;+ pub fn open(mut db: Connection, destination: &Path) -> Result<Self> {+ db.busy_timeout(std::time::Duration::from_secs(5))?;+ db.execute_batch(+ "PRAGMA foreign_keys = ON; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = WAL;",+ )?;+ let transaction = db.transaction_with_behavior(TransactionBehavior::Immediate)?;+ let current: i64 =+ transaction.pragma_query_value(None, "user_version", |row| row.get(0))?;+ ensure!(+ current <= 1,+ "database version {current} is newer than this build supports (1)"+ );+ if current == 0 {+ transaction.execute_batch(SCHEMA)?;+ transaction.pragma_update(None, "user_version", 1)?;+ transaction.commit()?;+ } else {+ drop(transaction);}- let connection = db.connection();+ let connection = &db;let existing: Option<Vec<u8>> = connection.query_row("SELECT destination FROM export WHERE id=1", [], |r| {r.get(0)@@ -337,7 +334,7 @@ impl Store {if let Some(existing) = existing {ensure!(existing == destination.as_os_str().as_encoded_bytes(),- "destination association does not match; use --adopt"+ AdoptionRequired("destination association does not match"));} else {connection.execute(@@ -351,7 +348,6 @@ impl Store {pub fn album(&self, id: i64, retained: bool) -> Result<Option<Album>> {Ok(self.db- .connection().query_row("SELECT key,id,retained,name,path FROM albums WHERE id=?1 AND retained=?2 ORDER BY rowid LIMIT 1",params![id, retained],@@ -361,7 +357,7 @@ impl Store {}pub fn folder(&self, key: &str) -> Result<Album> {- Ok(self.db.connection().query_row(+ Ok(self.db.query_row("SELECT key,id,retained,name,path FROM albums WHERE key=?1",[key],Album::read,@@ -378,7 +374,7 @@ impl Store {pub fn save_album(&self, album: &Album) -> Result<()> {let (parent, name) = album.path.rsplit_once('/').unwrap_or(("", &album.path));- self.db.connection().execute(+ self.db.execute("INSERT INTO albums VALUES(?1,?2,?3,?4,?5,?6,?7) ON CONFLICT(key) DO UPDATE SET name=excluded.name,path=excluded.path,parent=excluded.parent,folded=excluded.folded WHERE albums.name<>excluded.name OR albums.path<>excluded.path",params![album.key,@@ -396,7 +392,6 @@ impl Store {pub fn placement(&self, album: i64, file: i64) -> Result<Option<Placement>> {Ok(self.db- .connection().query_row("SELECT id,album,file,retained,name,kind FROM placements WHERE album=?1 AND file=?2 AND retained=0",params![album, file],@@ -406,7 +401,7 @@ impl Store {}pub fn save_placement(&self, placement: &Placement) -> Result<()> {- self.db.connection().execute(+ self.db.execute("INSERT INTO placements VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(id) DO UPDATE SET retained=excluded.retained,name=excluded.name,kind=excluded.kind WHERE placements.retained<>excluded.retained OR placements.name<>excluded.name OR placements.kind<>excluded.kind",params![placement.id,@@ -421,7 +416,7 @@ impl Store {}pub fn components(&self, owner: &str) -> Result<Vec<Component>> {- let mut query = self.db.connection().prepare(+ let mut query = self.db.prepare("SELECT placement,role,folder,name,size,hash,signature,properties,intended_time FROM components WHERE placement=?1 ORDER BY role",)?;Ok(query@@ -430,7 +425,7 @@ impl Store {}pub fn save_component(&self, component: &Component) -> Result<()> {- self.db.connection().execute(+ self.db.execute("INSERT INTO components VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11) ON CONFLICT(placement,role) DO UPDATE SET folder=excluded.folder,name=excluded.name,folded=excluded.folded,stem=excluded.stem,size=excluded.size,hash=excluded.hash,signature=excluded.signature,properties=excluded.properties,intended_time=excluded.intended_time WHERE components.folder<>excluded.folder OR components.name<>excluded.name OR components.size<>excluded.size OR components.hash<>excluded.hash OR components.signature IS NOT excluded.signature OR components.properties<>excluded.properties OR components.intended_time IS NOT excluded.intended_time",params![component.placement,@@ -452,7 +447,6 @@ impl Store {pub fn json_record(&self, owner: &str, role: Option<&Role>) -> Result<Option<JsonRecord>> {Ok(self.db- .connection().query_row("SELECT owner,role,folder,name,value,hash,properties FROM json_records WHERE owner=?1 AND role=?2",params![owner, role.map_or("", Role::name)],@@ -466,7 +460,7 @@ impl Store {let media = location.and_then(|location| location.name.strip_prefix("metadata/")).and_then(|name| name.strip_suffix(".json"));- self.db.connection().execute(+ self.db.execute("INSERT INTO json_records VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9) ON CONFLICT(owner,role) DO UPDATE SET folder=excluded.folder,name=excluded.name,folded=excluded.folded,stem=excluded.stem,value=excluded.value,hash=excluded.hash,properties=excluded.properties WHERE json_records.folder IS NOT excluded.folder OR json_records.name IS NOT excluded.name OR json_records.value<>excluded.value OR json_records.hash IS NOT excluded.hash OR json_records.properties<>excluded.properties",params![record.owner,@@ -486,7 +480,6 @@ impl Store {pub fn pending(&self, owner: &str) -> Result<Option<Pending>> {Ok(self.db- .connection().query_row("SELECT owner,album,file,retained,name,kind,folder,name1,name2,action FROM pending WHERE owner=?1",[owner],@@ -498,7 +491,6 @@ impl Store {pub fn pending_file(&self, album: i64, file: i64) -> Result<Option<Pending>> {Ok(self.db- .connection().query_row("SELECT owner,album,file,retained,name,kind,folder,name1,name2,action FROM pending WHERE album=?1 AND file=?2 ORDER BY retained DESC LIMIT 1",params![album, file],@@ -510,7 +502,7 @@ impl Store {pub fn save_pending(&self, pending: &Pending) -> Result<()> {let first = &pending.names[0];let second = pending.names.get(1);- self.db.connection().execute(+ self.db.execute("INSERT INTO pending VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14) ON CONFLICT(owner) DO UPDATE SET retained=excluded.retained,name=excluded.name,kind=excluded.kind,folder=excluded.folder,name1=excluded.name1,name2=excluded.name2,folded1=excluded.folded1,folded2=excluded.folded2,stem1=excluded.stem1,stem2=excluded.stem2,action=excluded.action",params![pending.owner,@@ -534,10 +526,8 @@ impl Store {pub fn clear_pending(&self, owner: &str) -> Result<()> {self.db- .connection().execute("DELETE FROM pending WHERE owner=?1", [owner])?;self.db- .connection().execute("DELETE FROM json_records WHERE owner=?1 AND folder IS NULL AND NOT EXISTS(SELECT 1 FROM components WHERE placement=?1)",[owner],@@ -552,7 +542,6 @@ impl Store {) -> Result<Option<T>> {let text: Option<String> = self.db- .connection().query_row(sql, parameters, |r| r.get(0)).optional()?;text.map(|text| serde_json::from_str(&text).map_err(Into::into))@@ -560,7 +549,7 @@ impl Store {}pub fn temporary(&self, temporary: &Temporary) -> Result<()> {- self.db.connection().execute(+ self.db.execute("INSERT INTO temporaries VALUES(?1,?2,?3,?4)",params![temporary.location.name,@@ -572,35 +561,39 @@ impl Store {Ok(())}pub fn reset_desired(&self) -> Result<()> {- self.db.connection().execute_batch(RUN_SCHEMA)?;+ self.db.execute_batch(RUN_SCHEMA)?;Ok(())}- pub fn desired_album(&self, record: &AlbumRecord, user_id: i64) -> Result<()> {- let name = record.album(user_id).ok().map(|a| a.name);- let favorites = record.remote.kind == "favorites" && record.remote.owner.id == user_id;- self.db.connection().execute(+ pub fn desired_album(&self, entry: &CollectionEntry) -> Result<()> {+ self.db.execute("INSERT INTO desired_albums(id,name,present,favorites) VALUES(?1,?2,1,?3) ON CONFLICT(id) DO UPDATE SET name=excluded.name,present=1,favorites=excluded.favorites",- params![record.remote.id, name, favorites],+ params![entry.id, entry.name, entry.favorites],)?;Ok(())}- pub fn snapshot_album(&self, album: &Collection) -> Result<()> {+ pub fn snapshot_album(+ &self,+ album: &Collection,+ documents: &Documents,+ user_id: i64,+ ) -> Result<()> {+ let (metadata, warnings) = crate::metadata::album(album, documents, user_id)?;let source = AlbumSource {name: album.name.clone(),- metadata: ente_photos::export::album(album),- warnings: album.warnings.clone(),+ metadata,+ warnings,};- self.db.connection().execute(+ self.db.execute("INSERT INTO album_sources VALUES(?1,?2) ON CONFLICT(id) DO UPDATE SET record=excluded.record WHERE album_sources.record<>excluded.record",params![album.id, serde_json::to_string(&source)?],)?;Ok(())}- pub fn snapshot_file(&self, file: &File) -> Result<()> {- self.db.connection().execute(+ pub fn snapshot_file(&self, file: &FileSnapshot) -> Result<()> {+ self.db.execute("INSERT INTO sources VALUES(?1,?2) ON CONFLICT(file) DO UPDATE SET record=excluded.record WHERE sources.record<>excluded.record",params![file.id, serde_json::to_string(file)?],)?;@@ -646,3 +639,10 @@ CREATE TEMP TABLE favorites (file INTEGER PRIMARY KEY);CREATE TEMP TABLE outcomes (unit TEXT PRIMARY KEY, conflict INTEGER NOT NULL);CREATE TEMP TABLE events (placement TEXT PRIMARY KEY, exported INTEGER NOT NULL DEFAULT 0, metadata_updated INTEGER NOT NULL DEFAULT 0, renamed INTEGER NOT NULL DEFAULT 0, retained INTEGER NOT NULL DEFAULT 0);";++fn read_json<T: DeserializeOwned>(row: &Row<'_>, column: usize) -> rusqlite::Result<T> {+ let text: String = row.get(column)?;+ serde_json::from_str(&text).map_err(|e| {+ rusqlite::Error::FromSqlConversionFailure(column, rusqlite::types::Type::Text, Box::new(e))+ })+}diff --git a/rust/apps/cli-next/src/export/transfer.rs b/rust/crates/photos-export/src/transfer.rssimilarity index 93%rename from rust/apps/cli-next/src/export/transfer.rsrename to rust/crates/photos-export/src/transfer.rsindex 53fb897020..b5f970e16c 100644--- a/rust/apps/cli-next/src/export/transfer.rs+++ b/rust/crates/photos-export/src/transfer.rs@@ -3,12 +3,10 @@ use std::{io::{Seek, SeekFrom},};+use crate::{metadata::Role, snapshot::FileSnapshot};use anyhow::{Context, Result, ensure};use ente_core::{Session, b64, crypto::hash};-use ente_photos::{- export::Role,- files::{self, Kind},-};+use ente_photos::files;use rusqlite::{OptionalExtension, params};use super::{@@ -33,8 +31,8 @@ pub struct Prepared {pub observed: Vec<(i64, Vec<(Role, Properties)>)>,}-pub fn roles(file: &files::File) -> Vec<Role> {- if file.kind == Kind::LivePhoto {+pub fn roles(file: &FileSnapshot) -> Vec<Role> {+ if file.kind == "livephoto" {vec![Role::Image, Role::Video]} else {vec![Role::Original]@@ -44,7 +42,7 @@ pub fn roles(file: &files::File) -> Vec<Role> {pub fn verify(run: &Run<'_>,component: &Component,- file: &files::File,+ file: &FileSnapshot,signature: &str,) -> Result<Option<Properties>> {let path = run.path(&component.location)?;@@ -68,7 +66,7 @@ pub fn verify(Ok(Some(properties))}-pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Result<Prepared> {+pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> Result<Prepared> {let signature = signature(file)?;let roles = roles(file);let mut observed = Vec::new();@@ -78,7 +76,6 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Rerun.check_cancel()?;let album: Option<i64> = store::lock(run.store)?.db- .connection().query_row("SELECT album FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",params![file.id, after],@@ -139,7 +136,6 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Re}let next: Option<(i64, String)> = store::lock(run.store)?.db- .connection().query_row("SELECT rowid,id FROM placements WHERE file=?1 AND rowid<?2 ORDER BY rowid DESC LIMIT 1",params![file.id, before],@@ -197,7 +193,6 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Relet (folder, album, temporary, output) = loop {let album: Option<i64> = store::lock(run.store)?.db- .connection().query_row("SELECT album FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",params![file.id, after],@@ -224,7 +219,7 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Redrop(output);let path = run.path(&temporary)?;let mut cancel = run.cancel.clone();- let download = files::download(session, file, || {+ let download = files::download(session, file.id, &file.key, &file.header, || {let mut output = File::options().read(true).write(true).open(&path)?;output.set_len(0)?;output.seek(SeekFrom::Start(0))?;@@ -232,9 +227,9 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Re});let writer = tokio::select! { result=download => result?, _=cancel.changed() => return Err(super::Cancelled.into()) };let (archive, size, hash) = writer.finish()?;- let components = if file.kind == Kind::LivePhoto {+ let components = if file.kind == "livephoto" {let mut temporaries = Vec::new();- let extracted = ente_photos::live_photo::extract(archive, |role, extension| {+ let extracted = crate::live_photo::extract(archive, |role, extension| {let (temporary, file) = run.temporary(&folder, album, Some(file.id)).map_err(std::io::Error::other)?;@@ -287,7 +282,7 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Reobserved,})}-pub fn signature(file: &files::File) -> Result<String> {+pub fn signature(file: &FileSnapshot) -> Result<String> {let mut bytes = file.header.as_bytes().to_vec();bytes.extend_from_slice(file.hash.as_deref().unwrap_or_default().as_bytes());Ok(b64::encode(&hash::hash(@@ -297,7 +292,7 @@ pub fn signature(file: &files::File) -> Result<String> {)?))}-pub fn expected_hash<'a>(file: &'a files::File, role: &Role) -> Option<&'a str> {+pub fn expected_hash<'a>(file: &'a FileSnapshot, role: &Role) -> Option<&'a str> {let hash = file.hash.as_deref()?;match role {Role::Original => Some(hash),diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.tomlindex cbcb2d2887..25ca94302a 100644--- a/rust/crates/photos/Cargo.toml+++ b/rust/crates/photos/Cargo.toml@@ -3,7 +3,6 @@ name = "ente-photos"edition.workspace = true[dependencies]-chrono = { workspace = true, features = ["std"] }ente-assets.workspace = trueente-collections.workspace = trueente-core.workspace = true@@ -19,7 +18,6 @@ thiserror.workspace = true[target.'cfg(not(target_arch = "wasm32"))'.dependencies]futures-util.workspace = true-zip.workspace = true[dev-dependencies]tempfile.workspace = truediff --git a/rust/crates/photos/src/collections.rs b/rust/crates/photos/src/collections.rsindex 1a5f7e073f..c98fcfbdd9 100644--- a/rust/crates/photos/src/collections.rs+++ b/rust/crates/photos/src/collections.rs@@ -1,5 +1,5 @@use ente_core::{Session, crypto::Key};-use serde::{Deserialize, Serialize};+use serde::Deserialize;use crate::source::{self, Documents, MetadataError};@@ -14,13 +14,6 @@ pub struct Collection {pub visibility: Visibility,pub owner_id: i64,pub updated_at_micros: i64,- pub display_order: Option<i64>,- pub muted: Option<bool>,- pub ascending: Option<bool>,- pub description: Option<String>,- pub cover_file_id: Option<i64>,- pub layout: Option<String>,- pub warnings: Vec<String>,}#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -46,7 +39,7 @@ impl Kind {}}-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]+#[derive(Debug, Clone, Copy)]pub enum Visibility {Visible,Archived,@@ -112,8 +105,6 @@ pub fn interpret()?;let shared: ShareeMetadata =source::optional(documents.shared.as_deref(), "invalid album sharee metadata")?;- let public: PublicMetadata =- source::optional(documents.public.as_deref(), "invalid album public metadata")?;let kind = match (owned.then_some(private.sub_type).flatten().unwrap_or(0),remote.kind.as_str(),@@ -136,22 +127,6 @@ pub fn interpret(shared.visibility})?};- let mut warnings = Vec::new();- source::unfamiliar(- documents.private.as_deref(),- &["visibility", "subType", "order"],- &mut warnings,- );- source::unfamiliar(- documents.shared.as_deref(),- &["visibility", "mute", "order"],- &mut warnings,- );- source::unfamiliar(- documents.public.as_deref(),- &["asc", "coverID", "layout", "caption"],- &mut warnings,- );Ok(Collection {id: remote.id,key: Key::from_bytes(*key.as_bytes()),@@ -165,13 +140,6 @@ pub fn interpret(visibility,owner_id: remote.owner.id,updated_at_micros: remote.updation_time,- display_order: if owned { private.order } else { shared.order },- muted: (!owned).then_some(shared.mute).flatten(),- ascending: public.asc,- description: public.caption,- cover_file_id: public.cover_id,- layout: public.layout,- warnings,})}@@ -180,21 +148,9 @@ pub fn interpret(struct PrivateMetadata {visibility: Option<u8>,sub_type: Option<u8>,- order: Option<i64>,}#[derive(Default, Deserialize)]struct ShareeMetadata {visibility: Option<u8>,- order: Option<i64>,- mute: Option<bool>,-}--#[derive(Default, Deserialize)]-struct PublicMetadata {- asc: Option<bool>,- #[serde(rename = "coverID")]- cover_id: Option<i64>,- layout: Option<String>,- caption: Option<String>,}diff --git a/rust/crates/photos/src/export.rs b/rust/crates/photos/src/export.rsdeleted file mode 100644index 93139721a8..0000000000--- a/rust/crates/photos/src/export.rs+++ /dev/null@@ -1,174 +0,0 @@-use chrono::{DateTime, SecondsFormat};-use ente_core::{- b64,- crypto::{Key, kdf},-};-use serde::{Deserialize, Serialize};-use serde_json::{Value, json};--use crate::{- collections::{Collection, Visibility},- files::{File, Kind},- source::MetadataError,-};--#[derive(Serialize, Deserialize)]-#[serde(rename_all = "camelCase", deny_unknown_fields)]-pub struct Root {- pub format: String,- pub version: u32,- pub source_verifier: String,-}--impl Root {- pub fn new(master_key: &Key) -> ente_core::crypto::Result<Self> {- Ok(Self {- format: "ente-photos-export".into(),- version: 1,- source_verifier: b64::encode(&kdf::derive_subkey(master_key, 32, 1, b"photoexp")?),- })- }-}--#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]-#[serde(rename_all = "lowercase")]-pub enum Role {- Original,- Image,- Video,-}--impl Role {- pub fn name(&self) -> &'static str {- match self {- Self::Original => "original",- Self::Image => "image",- Self::Video => "video",- }- }-}--#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]-pub struct Component {- pub role: Role,- pub path: String,- pub size: u64,- pub hash: String,-}--pub fn album(album: &Collection) -> Value {- let mut metadata = json!({- "albumID": album.id.to_string(), "type": album.kind.name(), "visibility": album.visibility.name(),- });- optional(&mut metadata, "displayOrder", album.display_order);- optional(&mut metadata, "muted", album.muted);- optional(- &mut metadata,- "sortOrder",- album- .ascending- .map(|asc| if asc { "ascending" } else { "descending" }),- );- optional(&mut metadata, "description", album.description.as_deref());- optional(- &mut metadata,- "coverFileID",- album.cover_file_id.map(|id| id.to_string()),- );- optional(&mut metadata, "layout", album.layout.as_deref());- json!({"title": album.name, "ente": metadata})-}--pub fn file(- file: &File,- components: &[Component],- component: usize,- favorited: bool,-) -> Result<Value, MetadataError> {- let selected = components- .get(component)- .ok_or(MetadataError("missing media component"))?;- let mut metadata = json!({- "fileID": file.id.to_string(),- "type": if file.kind == Kind::LivePhoto { "livePhoto" } else { file.kind.name() },- "name": file.name,- "creationTime": timestamp(file.created_at_micros)?,- "component": selected.role,- "components": components,- });- optional(- &mut metadata,- "modificationTime",- file.modified_at_micros.map(timestamp).transpose()?,- );- optional(- &mut metadata,- "location",- file.location- .as_ref()- .map(|l| json!({"latitude": l.latitude,"longitude": l.longitude})),- );- optional(&mut metadata, "localDateTime", file.date_time.as_deref());- optional(&mut metadata, "utcOffset", file.offset_time.as_deref());- optional(&mut metadata, "width", file.width);- optional(&mut metadata, "height", file.height);- optional(&mut metadata, "durationSeconds", file.duration_seconds);- optional(- &mut metadata,- "visibility",- file.visibility.map(Visibility::name),- );- optional(&mut metadata, "cameraMake", file.camera_make.as_deref());- optional(&mut metadata, "cameraModel", file.camera_model.as_deref());- optional(&mut metadata, "uploaderName", file.uploader_name.as_deref());- optional(&mut metadata, "panorama", file.panorama);- optional(- &mut metadata,- "motionPhotoVideoOffset",- file.motion_video_offset,- );- let taken = common_time(file.created_at_micros)?;- let mut record = json!({- "title": selected.path,- "photoTakenTime": taken,- "creationTime": taken,- "favorited": favorited,- "ente": metadata,- });- optional(- &mut record,- "modificationTime",- file.modified_at_micros.map(common_time).transpose()?,- );- optional(&mut record, "description", file.caption.as_deref());- optional(- &mut record,- "geoData",- file.location- .as_ref()- .map(|l| json!({"latitude":l.latitude,"longitude":l.longitude})),- );- Ok(record)-}--fn optional(object: &mut Value, name: &str, value: Option<impl Into<Value>>) {- if let Some(value) = value {- object[name] = value.into();- }-}--fn time(micros: i64) -> Result<DateTime<chrono::Utc>, MetadataError> {- DateTime::from_timestamp_micros(micros)- .ok_or(MetadataError("timestamp is outside the supported range"))-}--fn timestamp(micros: i64) -> Result<String, MetadataError> {- Ok(time(micros)?.to_rfc3339_opts(SecondsFormat::Micros, true))-}--fn common_time(micros: i64) -> Result<Value, MetadataError> {- let time = time(micros)?;- Ok(- json!({"timestamp":micros.div_euclid(1_000_000).to_string(),"formatted":time.format("%b %-d, %Y, %-I:%M:%S %p UTC").to_string()}),- )-}diff --git a/rust/crates/photos/src/files.rs b/rust/crates/photos/src/files.rsindex 696e2e493c..5da18cfd77 100644--- a/rust/crates/photos/src/files.rs+++ b/rust/crates/photos/src/files.rs@@ -3,7 +3,7 @@ use ente_core::{crypto::{self, Header, Key, blob},http,};-use serde::{Deserialize, Serialize};+use serde::Deserialize;use crate::{collections::Visibility,@@ -12,7 +12,7 @@ use crate::{pub use ente_collections::client::File as RemoteFile;-#[derive(Debug, Serialize, Deserialize)]+#[derive(Debug)]pub struct File {pub id: i64,pub owner_id: i64,@@ -30,19 +30,11 @@ pub struct File {pub width: Option<u32>,pub height: Option<u32>,pub visibility: Option<Visibility>,- pub camera_make: Option<String>,- pub camera_model: Option<String>,- pub uploader_name: Option<String>,- pub panorama: Option<bool>,- pub motion_video_offset: Option<u64>,- pub warnings: Vec<String>,- #[serde(with = "source::key")]pub key: Key,- #[serde(with = "source::header")]pub header: Header,}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum Kind {Image,Video,@@ -61,7 +53,7 @@ impl Kind {}}-#[derive(Debug, Serialize, Deserialize)]+#[derive(Debug)]pub struct Location {pub latitude: f64,pub longitude: f64,@@ -86,7 +78,9 @@ pub enum Error {#[cfg(not(target_arch = "wasm32"))]pub async fn download<W: std::io::Write>(session: &Session,- file: &File,+ id: i64,+ key: &Key,+ header: &Header,mut output: impl FnMut() -> std::io::Result<W>,) -> Result<W, Error> {use ente_core::crypto::stream::DecryptingWriter;@@ -99,7 +93,7 @@ pub async fn download<W: std::io::Write>(let output = output?;let signed: DownloadUrl = session.api- .get(&format!("/files/download/v3/{}", file.id))+ .get(&format!("/files/download/v3/{id}")).send().await?.error_for_code()@@ -114,7 +108,7 @@ pub async fn download<W: std::io::Write>(.await?.error_for_status()?;let mut body = std::pin::pin!(response.bytes_stream());- let mut decryptor = DecryptingWriter::new(&file.header, &file.key, output);+ let mut decryptor = DecryptingWriter::new(header, key, output);while let Some(chunk) = body.next().await {decryptor.write(&chunk?)?;}@@ -193,52 +187,6 @@ pub fn interpret(latitude,longitude,});- let mut warnings = Vec::new();- source::unfamiliar(- Some(&documents.original),- &[- "title",- "fileType",- "creationTime",- "modificationTime",- "latitude",- "longitude",- "hash",- "imageHash",- "videoHash",- "duration",- "hasStaticThumbnail",- "localID",- "deviceFolder",- "subType",- "version",- "exif",- ],- &mut warnings,- );- source::unfamiliar(- documents.public.as_deref(),- &[- "editedName",- "editedTime",- "dateTime",- "offsetTime",- "caption",- "lat",- "long",- "w",- "h",- "sv",- "cameraMake",- "cameraModel",- "uploaderName",- "mediaType",- "mvi",- "noThumb",- ],- &mut warnings,- );- source::unfamiliar(documents.private.as_deref(), &["visibility"], &mut warnings);Ok(File {id: remote.id,owner_id: remote.owner_id,@@ -263,16 +211,6 @@ pub fn interpret(} else {None},- camera_make: public.camera_make,- camera_model: public.camera_model,- uploader_name: public.uploader_name,- panorama: if kind == Kind::Image {- public.media_type.map(|value| value & 1 != 0)- } else {- None- },- motion_video_offset: public.mvi,- warnings,key: Key::from_bytes(*key.as_bytes()),header: Header::try_from_slice(&b64::decode(&remote.file.decryption_header)?)?,})@@ -305,11 +243,6 @@ struct PublicMetadata {long: Option<f64>,w: Option<u32>,h: Option<u32>,- camera_make: Option<String>,- camera_model: Option<String>,- uploader_name: Option<String>,- media_type: Option<u64>,- mvi: Option<u64>,}#[derive(Deserialize)]diff --git a/rust/crates/photos/src/lib.rs b/rust/crates/photos/src/lib.rsindex a718b88631..c472bde578 100644--- a/rust/crates/photos/src/lib.rs+++ b/rust/crates/photos/src/lib.rs@@ -1,6 +1,5 @@pub mod collections;pub mod db;-pub mod export;pub mod files;pub mod location;pub mod metadata;@@ -13,6 +12,3 @@ pub use motion_photo::{MotionPhotoError, VideoIndex, extract_motion_video_file_from_path,extract_motion_video_from_path, extract_xmp_from_path, get_motion_video_index_from_path,};--#[cfg(not(target_arch = "wasm32"))]-pub mod live_photo;diff --git a/rust/crates/photos/src/source.rs b/rust/crates/photos/src/source.rsindex c85c00d147..f3b18b2404 100644--- a/rust/crates/photos/src/source.rs+++ b/rust/crates/photos/src/source.rs@@ -1,5 +1,4 @@use serde::{Deserialize, Serialize, de::DeserializeOwned};-use serde_json::Value;#[derive(Clone, Serialize, Deserialize)]pub struct Documents {@@ -47,32 +46,6 @@ mod optional_bytes {}}-pub(crate) mod key {- use ente_core::crypto::Key;- use serde::{Deserializer, Serializer, de::Error};-- pub fn serialize<S: Serializer>(key: &Key, serializer: S) -> Result<S::Ok, S::Error> {- serializer.serialize_str(&ente_core::b64::encode(key.as_bytes()))- }-- pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Key, D::Error> {- Key::try_from_slice(&super::bytes::deserialize(deserializer)?).map_err(D::Error::custom)- }-}--pub(crate) mod header {- use ente_core::crypto::Header;- use serde::{Deserializer, Serializer, de::Error};-- pub fn serialize<S: Serializer>(header: &Header, serializer: S) -> Result<S::Ok, S::Error> {- serializer.serialize_str(&ente_core::b64::encode(header.as_bytes()))- }-- pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Header, D::Error> {- Header::try_from_slice(&super::bytes::deserialize(deserializer)?).map_err(D::Error::custom)- }-}-#[derive(Debug, thiserror::Error)]#[error("{0}")]pub struct MetadataError(pub &'static str);@@ -90,15 +63,3 @@ pub fn optional<T: DeserializeOwned + Default>(.transpose().map(Option::unwrap_or_default)}--pub fn unfamiliar(bytes: Option<&[u8]>, known: &[&str], warnings: &mut Vec<String>) {- if let Some(bytes) = bytes- && let Ok(Value::Object(fields)) = serde_json::from_slice(bytes)- {- for key in fields.keys() {- if !known.contains(&key.as_str()) {- warnings.push(key.clone());- }- }- }-}diff --git a/rust/crates/test-support/src/object_store.rs b/rust/crates/test-support/src/object_store.rsindex a6f13e3849..b689b53bb8 100644--- a/rust/crates/test-support/src/object_store.rs+++ b/rust/crates/test-support/src/object_store.rs@@ -19,8 +19,6 @@ use crate::{TestResult, net::LOCAL_HOST};#[derive(Default)]pub struct ObjectStoreControl {reads: AtomicUsize,- active: AtomicUsize,- peak: AtomicUsize,interrupted: AtomicUsize,held: Mutex<Option<Arc<ReadGate>>>,}@@ -78,9 +76,6 @@ impl ObjectStoreControl {pub fn reads(&self) -> usize {self.reads.load(Ordering::SeqCst)}- pub fn peak_reads(&self) -> usize {- self.peak.load(Ordering::SeqCst)- }pub fn interrupt_reads(&self, count: usize) {self.interrupted.store(count, Ordering::SeqCst);}@@ -215,40 +210,33 @@ fn handle(stream: TcpStream, root: &Path, control: &ObjectStoreControl) -> TestR{return Ok(());}- let active = control.active.fetch_add(1, Ordering::SeqCst) + 1;- control.peak.fetch_max(active, Ordering::SeqCst);- let result = (|| {- respond(input.get_mut(), "200 OK", file.metadata()?.len())?;- let gate = control- .held- .lock()- .unwrap_or_else(PoisonError::into_inner)- .clone();- if let Some(gate) = gate- && file.metadata()?.len() >= gate.minimum_size- {- let mut state = gate.state.lock().unwrap_or_else(PoisonError::into_inner);- state.0 += 1;- gate.changed.notify_all();- while !state.1 {- state = gate- .changed- .wait(state)- .unwrap_or_else(PoisonError::into_inner);- }+ respond(input.get_mut(), "200 OK", file.metadata()?.len())?;+ let gate = control+ .held+ .lock()+ .unwrap_or_else(PoisonError::into_inner)+ .clone();+ if let Some(gate) = gate+ && file.metadata()?.len() >= gate.minimum_size+ {+ let mut state = gate.state.lock().unwrap_or_else(PoisonError::into_inner);+ state.0 += 1;+ gate.changed.notify_all();+ while !state.1 {+ state = gate+ .changed+ .wait(state)+ .unwrap_or_else(PoisonError::into_inner);}- let mut buffer = vec![0; 64 * 1024];- loop {- let read = file.read(&mut buffer)?;- if read == 0 {- break;- }- input.get_mut().write_all(&buffer[..read])?;+ }+ let mut buffer = vec![0; 64 * 1024];+ loop {+ let read = file.read(&mut buffer)?;+ if read == 0 {+ break;}- Ok::<_, io::Error>(())- })();- control.active.fetch_sub(1, Ordering::SeqCst);- result?;+ input.get_mut().write_all(&buffer[..read])?;+ }}_ => respond(input.get_mut(), "405 Method Not Allowed", 0)?,}