Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Extract Photos export engine

Manav Rathi0f9daacd844d
diff --git a/.github/scripts/ci/paths.json b/.github/scripts/ci/paths.json
index a079752e02..cb48958bea 100644
--- a/.github/scripts/ci/paths.json
+++ b/.github/scripts/ci/paths.json
@@ -114,6 +114,7 @@
"rust/crates/exif/**",
"rust/crates/location/**",
"rust/crates/photos/**",
+ "rust/crates/photos-export/**",
"rust/crates/test-support/**",
"rust/crates/vecdb/**"
],
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 16d63b62bc..c40c0561bc 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2450,9 +2450,9 @@ dependencies = [
"ente-collections",
"ente-core",
"ente-photos",
+ "ente-photos-export",
"ente-test-support",
"futures-util",
- "icu_casemap",
"keyring",
"mockito",
"reqwest 0.12.28",
@@ -2462,7 +2462,6 @@ dependencies = [
"tempfile",
"thiserror 2.0.18",
"tokio",
- "unicode-normalization",
"url",
"uuid",
"zeroize",
@@ -2777,7 +2776,6 @@ dependencies = [
name = "ente-photos"
version = "0.0.0"
dependencies = [
- "chrono",
"ente-assets",
"ente-collections",
"ente-core",
@@ -2792,6 +2790,25 @@ dependencies = [
"serde_json",
"tempfile",
"thiserror 2.0.18",
+]
+
+[[package]]
+name = "ente-photos-export"
+version = "0.0.0"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "ente-collections",
+ "ente-core",
+ "ente-photos",
+ "icu_casemap",
+ "rusqlite",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.18",
+ "tokio",
+ "unicode-normalization",
+ "uuid",
"zip 8.6.0",
]
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 9a6bb04bed..055168913f 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -38,6 +38,7 @@ members = [
"crates/ml",
"crates/paste",
"crates/photos",
+ "crates/photos-export",
"crates/space",
"crates/test-support",
"crates/vecdb",
@@ -72,6 +73,7 @@ ente-locker = { path = "crates/locker" }
ente-ml = { path = "crates/ml" }
ente-paste = { path = "crates/paste" }
ente-photos = { path = "crates/photos" }
+ente-photos-export = { path = "crates/photos-export" }
ente-space = { path = "crates/space" }
ente-test-support = { path = "crates/test-support" }
ente-uniffi-log = { path = "bindings/uniffi/log" }
diff --git a/rust/apps/cli-next/Cargo.toml b/rust/apps/cli-next/Cargo.toml
index 3891e79e6a..bb2219360f 100644
--- a/rust/apps/cli-next/Cargo.toml
+++ b/rust/apps/cli-next/Cargo.toml
@@ -17,7 +17,7 @@ ente-accounts.workspace = true
ente-collections.workspace = true
ente-core.workspace = true
ente-photos.workspace = true
-icu_casemap.workspace = true
+ente-photos-export.workspace = true
keyring = { workspace = true, features = ["apple-native", "windows-native", "linux-native-sync-persistent", "crypto-rust"] }
reqwest.workspace = true
rusqlite = { workspace = true, features = ["bundled-sqlcipher"] }
@@ -26,7 +26,6 @@ serde_json.workspace = true
tempfile.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal", "sync", "time"] }
-unicode-normalization.workspace = true
url.workspace = true
uuid = { workspace = true, features = ["serde", "v4"] }
zeroize.workspace = true
diff --git a/rust/apps/cli-next/src/core_db.rs b/rust/apps/cli-next/src/core_db.rs
index a5c8765d7a..b44e22bb33 100644
--- a/rust/apps/cli-next/src/core_db.rs
+++ b/rust/apps/cli-next/src/core_db.rs
@@ -7,7 +7,7 @@ pub use rusqlite::{
#[derive(Debug, thiserror::Error)]
pub enum Error {
- #[error(transparent)]
+ #[error("{0}")]
Sqlite(#[from] SqliteError),
#[error("database version {current} is newer than this build supports ({target})")]
Downgrade { current: i64, target: i64 },
@@ -22,10 +22,6 @@ impl Db {
Self(connection)
}
- pub fn connection(&self) -> &Connection {
- &self.0
- }
-
pub fn migrate(&mut self, scripts: &[&str]) -> Result<()> {
let target = scripts.len() as i64;
let transaction = self
@@ -60,3 +56,15 @@ impl Db {
Ok(result)
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn sqlite_errors_preserve_their_source() {
+ let error = Error::Sqlite(SqliteError::InvalidQuery);
+ let source = std::error::Error::source(&error).unwrap();
+ assert!(source.is::<SqliteError>());
+ }
+}
diff --git a/rust/apps/cli-next/src/db.rs b/rust/apps/cli-next/src/db.rs
index 75171d6607..779c6befff 100644
--- a/rust/apps/cli-next/src/db.rs
+++ b/rust/apps/cli-next/src/db.rs
@@ -10,6 +10,9 @@ const MIGRATIONS: &[&str] = &[replica::SCHEMA];
pub fn open(account_dir: &Path, key: &DbKey, create: bool) -> Result<Db> {
let connection = connect(&account_dir.join("data.db"), key, create)?;
+ connection.execute_batch(
+ "PRAGMA foreign_keys = ON; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = WAL;",
+ )?;
let mut db = Db::new(connection);
db.migrate(MIGRATIONS)?;
Ok(db)
@@ -37,8 +40,5 @@ pub fn connect(path: &Path, key: &DbKey, create: bool) -> Result<Connection> {
row.get::<_, i64>(0)
})
.context("cannot open the encrypted database")?;
- connection.execute_batch(
- "PRAGMA foreign_keys = ON; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = WAL;",
- )?;
Ok(connection)
}
diff --git a/rust/apps/cli-next/src/export/mod.rs b/rust/apps/cli-next/src/export/mod.rs
index 400c647303..4c3c767941 100644
--- a/rust/apps/cli-next/src/export/mod.rs
+++ b/rust/apps/cli-next/src/export/mod.rs
@@ -1,41 +1,18 @@
-mod adopt;
-mod fs;
-mod names;
-mod reconcile;
-mod store;
-mod transfer;
+mod source;
-use std::{
- fs::{File, OpenOptions},
- io::Write,
- path::{Path, PathBuf},
- sync::Mutex,
- time::{Duration, Instant},
-};
-
-use anyhow::{Context, Result, bail, ensure};
-use ente_photos::export::Root;
-use rusqlite::{OptionalExtension, params};
-use serde_json::json;
+use std::path::{Path, PathBuf};
+use anyhow::{Context, Result, ensure};
+use ente_core::crypto::hash;
+use ente_photos_export::{AdoptionRequired, Export};
use tokio::sync::watch;
use crate::{
api,
args::{ExportArgs, Options, Product},
db, home, output,
- replica::{AlbumRecord, FileRecord, Replica, canonical_id},
vault::State,
};
-use store::Store;
-
-#[derive(Debug, thiserror::Error)]
-#[error("conflict: {0}")]
-pub struct Conflict(String);
-
-#[derive(Debug, thiserror::Error)]
-#[error("export cancelled")]
-pub struct Cancelled;
pub async fn run(args: ExportArgs, selected: Option<&str>, options: &Options) -> Result<()> {
ensure!(
@@ -49,20 +26,30 @@ pub async fn run(args: ExportArgs, selected: Option<&str>, options: &Options) ->
runtime.block_on(run_locked(args, selected.as_deref(), as_json))
})
.await?
+ .map_err(|error| {
+ if error.is::<AdoptionRequired>() {
+ error.context("this export requires --adopt")
+ } else {
+ error
+ }
+ })
}
async fn run_locked(args: ExportArgs, selected: Option<&str>, as_json: bool) -> Result<()> {
- let destination = destination(&args.destination)?;
let mut state = State::load()?;
let index = state.resolve(selected)?;
let account = state.accounts.swap_remove(index);
let session = api::session(&account, Product::Photos)?;
- let expected_root = Root::new(&session.master_key)?;
- let mut root_lock = open_root(&destination, &expected_root)?;
- ensure!(
- !args.adopt || root_lock.is_some(),
- "there is no export to adopt"
- );
+ let mut export = Export::open(
+ &args.destination,
+ &session.master_key,
+ ente_photos_export::Options {
+ albums: args.album,
+ exclude_albums: args.exclude_album,
+ adopt: args.adopt,
+ jobs: args.jobs,
+ },
+ )?;
let account_home = home::lock_account(account.storage_id, true)?;
let account = State::load()?
.accounts
@@ -70,88 +57,15 @@ async fn run_locked(args: ExportArgs, selected: Option<&str>, as_json: bool) ->
.find(|a| a.storage_id == account.storage_id)
.context("account was removed while waiting for access")?;
let session = api::session(&account, Product::Photos)?;
- ensure!(
- Root::new(&session.master_key)?.source_verifier == expected_root.source_verifier,
- "account keys changed while waiting for access"
- );
+ export.verify_source(&session.master_key)?;
home::create(&account_home.path)?;
- let db_path = Store::path(&account_home.path, &destination)?;
- let existed = db_path.try_exists()?;
+ let db_path = store_path(&account_home.path, export.destination())?;
ensure!(
- root_lock.is_none() || existed || args.adopt,
+ !export.exists() || db_path.try_exists()? || args.adopt,
"this export is not associated with this CLI home; use --adopt"
);
- let store = Store::open(&db_path, &account.db_key, &destination, true)?;
- let bound: bool =
- store
- .db
- .connection()
- .query_row("SELECT bound FROM export WHERE id=1", [], |r| r.get(0))?;
- ensure!(
- root_lock.is_none() || bound || args.adopt,
- "this export requires --adopt"
- );
- store.reset_desired()?;
- if root_lock.is_some() && !bound {
- adopt::scan(&destination, &store)?;
- }
- let mut source_db = db::open(&account_home.path, &account.db_key, true)?;
- let mut replica = Replica::new(&mut source_db, session.user_id);
- eprintln!("Refreshing album metadata.");
- replica.sync_collections(&session).await?;
- let mut after = 0;
- loop {
- let next: Option<(i64, String, Option<String>)> = replica.db.read(|db| {
- Ok(db
- .query_row(
- "SELECT id,record,name FROM photos_collections WHERE id>?1 ORDER BY id LIMIT 1",
- [after],
- |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
- )
- .optional()?)
- })?;
- let Some((id, record, name)) = next else {
- break;
- };
- after = id;
- let record: AlbumRecord = serde_json::from_str(&record)?;
- store.desired_album(&record, session.user_id)?;
- if record.album(session.user_id).is_err() {
- store.db.connection().execute(
- "UPDATE desired_albums SET name=?1 WHERE id=?2",
- params![name, id],
- )?;
- }
- }
- store
- .db
- .connection()
- .execute(
- "INSERT OR IGNORE INTO desired_albums(id,name) SELECT s.id,json_extract(s.record,'$.name') FROM album_sources s WHERE EXISTS(SELECT 1 FROM albums a WHERE a.id=s.id)",
- [],
- )?;
- store
- .db
- .connection()
- .execute(
- "INSERT OR IGNORE INTO desired_albums(id,name) SELECT id,name FROM albums ORDER BY retained",
- [],
- )?;
- store
- .db
- .connection()
- .execute(
- "INSERT OR IGNORE INTO desired_albums(id,name) SELECT album,name FROM pending WHERE file IS NULL AND retained=0",
- [],
- )?;
- select(&store, &args)?;
- if root_lock.is_none() {
- root_lock = Some(initialize(&destination, &expected_root)?);
- }
- store
- .db
- .connection()
- .execute("UPDATE export SET bound=1 WHERE id=1 AND bound=0", [])?;
+ home::create(db_path.parent().context("export DB has no parent")?)?;
+ let connection = db::connect(&db_path, &account.db_key, true)?;
let (cancel_send, mut cancel) = watch::channel(false);
let signal_send = cancel_send.clone();
let signal = tokio::spawn(async move {
@@ -159,617 +73,48 @@ async fn run_locked(args: ExportArgs, selected: Option<&str>, as_json: bool) ->
signal_send.send_replace(true);
}
});
- let refresh_cancel = cancel.clone();
- let refreshed = tokio::select! {
- result=refresh(&store,&mut replica,&session,&refresh_cancel)=>result,
- _=cancel.changed()=>Err(Cancelled.into()),
- };
- drop(replica);
- drop(source_db);
- drop(account_home);
- let shared = Mutex::new(store);
- let run = reconcile::Run {
- root: &destination,
- store: &shared,
- cancel: &cancel,
- };
- let result = (|| {
- refreshed?;
- run.recover()?;
- run.albums()?;
- transfers(
- &destination,
- &shared,
- &session,
- &cancel_send,
- &cancel,
- args.jobs.get(),
- )?;
- run.check_cancel()?;
- run.retain_removed()
- })();
+ let report = |line: &str| eprintln!("{line}");
+ let result = async {
+ let mut source_db = db::open(&account_home.path, &account.db_key, true)?;
+ let mut source = source::ReplicaSource::new(&mut source_db, session.user_id);
+ export
+ .prepare(connection, &mut source, &session, &mut cancel, &report)
+ .await?;
+ drop(source);
+ drop(source_db);
+ drop(account_home);
+ export.execute(&session, &cancel_send, &cancel, &report)
+ }
+ .await;
signal.abort();
- if let Err(error) = result {
- run.report("run", &error)?;
- }
- if let Err(error) = run.cleanup(None) {
- run.report("run", &error)?;
- }
- let store = shared
- .into_inner()
- .map_err(|_| anyhow::anyhow!("export database mutex poisoned"))?;
- let complete = summary(&store, &destination, as_json)?;
- drop(root_lock);
- ensure!(complete, "export is incomplete");
- Ok(())
-}
-
-fn destination(path: &Path) -> Result<PathBuf> {
- let absolute = if path.is_absolute() {
- path.to_owned()
+ let summary = result?;
+ let progress = if summary.json["copies"].is_null() {
+ "inventory incomplete".to_owned()
} else {
- std::env::current_dir()?.join(path)
+ format!("{}/{} copies complete", summary.completed, summary.expected)
};
- let mut base = absolute.as_path();
- let mut tail = Vec::new();
- while !base.try_exists()? {
- tail.push(
- base.file_name()
- .context("destination has no directory name")?
- .to_owned(),
- );
- base = base
- .parent()
- .context("destination has no existing parent")?;
- }
- if tail.is_empty() {
- ensure!(
- !std::fs::symlink_metadata(base)?.is_symlink(),
- "export destination is a symlink"
- );
- }
- let mut resolved = std::fs::canonicalize(base)?;
- for part in tail.into_iter().rev() {
- resolved.push(part);
- }
- Ok(resolved)
-}
-
-fn open_root(destination: &Path, expected: &Root) -> Result<Option<File>> {
- if !destination.try_exists()? {
- return Ok(None);
- }
- ensure!(
- destination.is_dir(),
- "export destination is not a directory"
- );
- let path = names::check(destination, "export.json")?;
- if !path.try_exists()? {
- ensure!(
- std::fs::read_dir(destination)?.next().is_none(),
- "nonempty destination has no recognized export.json"
- );
- return Ok(None);
- }
- let file = OpenOptions::new().read(true).write(true).open(&path)?;
- file.try_lock()
- .context("another writer is maintaining this export")?;
- let record: Root = serde_json::from_reader(&file).context("invalid export.json")?;
- ensure!(
- record.format == expected.format && record.version == expected.version,
- "unsupported export format"
- );
- ensure!(
- record.source_verifier == expected.source_verifier,
- "this export belongs to another source account"
- );
- Ok(Some(file))
-}
-
-fn initialize(destination: &Path, root: &Root) -> Result<File> {
- fs::create_directory(destination)?;
- let path = names::check(destination, "export.json")?;
- let mut file = OpenOptions::new()
- .read(true)
- .write(true)
- .create_new(true)
- .open(&path)
- .context("destination was initialized by another writer; retry")?;
- file.lock().context("cannot lock the new export")?;
- ensure!(
- std::fs::read_dir(destination)?
- .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),
- "destination became nonempty during initialization"
- );
- serde_json::to_writer_pretty(&mut file, root)?;
- file.write_all(b"\n")?;
- file.sync_all()?;
- fs::sync_parent(&path)?;
- Ok(file)
-}
-
-fn select(store: &Store, args: &ExportArgs) -> Result<()> {
- if args.album.is_empty() {
- store
- .db
- .connection()
- .execute("UPDATE desired_albums SET selected=1", [])?;
- }
- for (selectors, include) in [(&args.album, true), (&args.exclude_album, false)] {
- for selector in selectors {
- let mut query = store.db.connection().prepare(
- "SELECT id,name FROM desired_albums WHERE id=?1 OR name=?2 ORDER BY id LIMIT 11",
- )?;
- let candidates: Vec<(i64, Option<String>)> = query
- .query_map(params![canonical_id(selector), selector], |r| {
- Ok((r.get(0)?, r.get(1)?))
- })?
- .collect::<rusqlite::Result<_>>()?;
- ensure!(!candidates.is_empty(), "no album matches {selector:?}");
- if candidates.len() != 1 {
- bail!(
- "album {selector:?} is ambiguous: {}",
- candidates
- .iter()
- .map(|(id, name)| format!(
- "{id} {:?}",
- name.as_deref().unwrap_or("unavailable")
- ))
- .collect::<Vec<_>>()
- .join(", ")
- );
- }
- store.db.connection().execute(
- "UPDATE desired_albums SET selected=?1 WHERE id=?2",
- params![include, candidates[0].0],
- )?;
- }
- }
- Ok(())
-}
-
-async fn refresh(
- store: &Store,
- replica: &mut Replica<'_>,
- session: &ente_core::Session,
- cancel: &watch::Receiver<bool>,
-) -> Result<()> {
- let need_favorites: bool = store.db.connection().query_row(
- "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND present=1)",
- [],
- |r| r.get(0),
- )?;
- let mut favorites_known = true;
- let mut after = 0;
- eprintln!("Refreshing selected albums and Favorites.");
- loop {
- ensure!(!*cancel.borrow(), Cancelled);
- let next: Option<(i64, bool, bool)> = store
- .db
- .connection()
- .query_row(
- "SELECT id,selected,favorites FROM desired_albums WHERE present=1 AND (selected=1 OR (favorites=1 AND ?1)) AND id>?2 ORDER BY id LIMIT 1",
- params![need_favorites, after],
- |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
- )
- .optional()?;
- let Some((id, selected, favorites)) = next else {
- break;
- };
- after = id;
- let result = async {
- replica.retry_album(session, id)?;
- let record = replica.album_record(id)?.context("missing album source")?;
- let album = record.album(session.user_id)?;
- replica.sync_album_files(session, &album).await?;
- store.desired_album(&record, session.user_id)?;
- if selected {
- store.snapshot_album(&album)?;
- }
- store
- .db
- .connection()
- .execute("UPDATE desired_albums SET ready=1 WHERE id=?1", [id])?;
- Ok::<_, anyhow::Error>(())
- }
- .await;
- if let Err(error) = result {
- if favorites {
- favorites_known = false;
- }
- store.db.connection().execute(
- "UPDATE desired_albums SET failure=?1 WHERE id=?2",
- params![error.to_string(), id],
- )?;
- if selected {
- record_outcome(store, &format!("album:{id}"), &error)?;
- eprintln!("album:{id}: {error:#}");
- }
- if fatal(&error) {
- return Err(error);
- }
- }
- }
- store.db.connection().execute(
- "UPDATE desired_albums SET ready=1 WHERE selected=1 AND present=0",
- [],
- )?;
- if need_favorites && favorites_known {
- let mut after = 0;
- loop {
- let ids: Vec<i64> = replica.db.read(|db| {
- let mut query = db.prepare(
- "SELECT DISTINCT f.id FROM photos_files f JOIN photos_collections c ON c.id=f.collection_id WHERE f.id>?1 AND json_extract(c.record,'$.remote.type')='favorites' AND json_extract(c.record,'$.remote.owner.id')=?2 ORDER BY f.id LIMIT 128",
- )?;
- Ok(query
- .query_map(params![after, session.user_id], |r| r.get(0))?
- .collect::<rusqlite::Result<_>>()?)
- })?;
- if ids.is_empty() {
- break;
- }
- let transaction = store.db.connection().unchecked_transaction()?;
- for id in ids {
- after = id;
- store
- .db
- .connection()
- .execute("INSERT INTO favorites VALUES(?1)", [id])?;
- }
- transaction.commit()?;
- }
- }
- after = 0;
- loop {
- let id: Option<i64> = store
- .db
- .connection()
- .query_row(
- "SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL AND id>?1 ORDER BY id LIMIT 1",
- [after],
- |r| r.get(0),
- )
- .optional()?;
- let Some(id) = id else { break };
- after = id;
- let mut after_file = 0;
- loop {
- ensure!(!*cancel.borrow(), Cancelled);
- let records: Vec<FileRecord> = replica.db.read(|db| {
- let mut query = db.prepare(
- "SELECT record FROM photos_files WHERE collection_id=?1 AND id>?2 ORDER BY id LIMIT 128",
- )?;
- Ok(query
- .query_map(params![id, after_file], |r| crate::replica::read_json(r, 0))?
- .collect::<rusqlite::Result<_>>()?)
- })?;
- if records.is_empty() {
- break;
- }
- let transaction = store.db.connection().unchecked_transaction()?;
- for record in records {
- after_file = record.remote.id;
- let file = record.file(session.user_id);
- let failure = if !favorites_known {
- Some("favorite state is incomplete".to_owned())
- } else {
- file.as_ref().err().map(ToString::to_string)
- };
- store.db.connection().execute(
- "INSERT INTO desired_files(album,file,favorited,failure) VALUES(?1,?2,?3,?4)",
- params![
- id,
- record.remote.id,
- if favorites_known { Some(false) } else { None },
- failure
- ],
- )?;
- if failure.is_none() {
- let file = file?;
- store.db.connection().execute(
- "INSERT INTO chosen_files VALUES(?1,?2,?3) ON CONFLICT(file) DO UPDATE SET album=excluded.album,version=excluded.version WHERE excluded.version>chosen_files.version OR (excluded.version=chosen_files.version AND excluded.album<chosen_files.album)",
- params![file.id, id, file.updated_at_micros],
- )?;
- }
- }
- transaction.commit()?;
- }
- }
- store
- .db
- .connection()
- .execute(
- "UPDATE desired_files SET favorited=EXISTS(SELECT 1 FROM favorites WHERE favorites.file=desired_files.file) WHERE favorited IS NOT NULL",
- [],
- )?;
- let mut after_file = 0;
- loop {
- ensure!(!*cancel.borrow(), Cancelled);
- let mut query = store
- .db
- .connection()
- .prepare("SELECT album,file FROM chosen_files WHERE file>?1 ORDER BY file LIMIT 128")?;
- let chosen: Vec<(i64, i64)> = query
- .query_map([after_file], |r| Ok((r.get(0)?, r.get(1)?)))?
- .collect::<rusqlite::Result<_>>()?;
- drop(query);
- if chosen.is_empty() {
- break;
- }
- let transaction = store.db.connection().unchecked_transaction()?;
- for (album, id) in chosen {
- after_file = id;
- let record = replica
- .file_record(album, id)?
- .context("missing chosen file source")?;
- store.snapshot_file(&record.file(session.user_id)?)?;
- }
- transaction.commit()?;
- }
- let transaction = store.db.connection().unchecked_transaction()?;
- store.db.connection().execute(
- "DELETE FROM sources WHERE file NOT IN (SELECT file FROM chosen_files)",
- [],
- )?;
- store
- .db
- .connection()
- .execute(
- "DELETE FROM album_sources WHERE id NOT IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM albums WHERE albums.id=album_sources.id) AND NOT EXISTS(SELECT 1 FROM pending WHERE album=album_sources.id)",
- [],
- )?;
- transaction.commit()?;
- let mut query = store
- .db
- .connection()
- .prepare("SELECT album,file,failure FROM desired_files WHERE failure IS NOT NULL")?;
- let mut rows = query.query([])?;
- while let Some(row) = rows.next()? {
- let unit = format!("file:{}:{}", row.get::<_, i64>(0)?, row.get::<_, i64>(1)?);
- let error = anyhow::anyhow!(row.get::<_, String>(2)?);
- record_outcome(store, &unit, &error)?;
- eprintln!("{unit}: {error:#}");
- }
- Ok(())
-}
-
-struct CancelWorkers<'a>(&'a watch::Sender<bool>);
-
-impl Drop for CancelWorkers<'_> {
- fn drop(&mut self) {
- if std::thread::panicking() {
- self.0.send_replace(true);
- }
- }
-}
-
-fn transfers(
- root: &Path,
- store: &Mutex<Store>,
- session: &ente_core::Session,
- cancel_send: &watch::Sender<bool>,
- cancel: &watch::Receiver<bool>,
- jobs: usize,
-) -> Result<()> {
- let (expected, files): (i64, i64) = store::lock(store)?.db.connection().query_row(
- "SELECT count(*),count(DISTINCT file) FROM desired_files WHERE failure IS NULL",
- [],
- |r| Ok((r.get(0)?, r.get(1)?)),
- )?;
- eprintln!("Maintaining {expected} selected copies.");
- let runtime = tokio::runtime::Handle::current();
- let last_report = Mutex::new(None::<Instant>);
- std::thread::scope(|scope| {
- let mut workers = Vec::new();
- for _ in 0..jobs.min(files as usize) {
- let runtime = &runtime;
- let last_report = &last_report;
- workers.push(scope.spawn(move || {
- let _cancel_on_panic = CancelWorkers(cancel_send);
- let run = reconcile::Run { root, store, cancel };
- let result = (|| {
- loop {
- if *cancel.borrow() {
- break;
- }
- let file = {
- let store = store::lock(store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
- let id: Option<i64> = store
- .db
- .connection()
- .query_row(
- "SELECT file FROM desired_files WHERE failure IS NULL AND claimed=0 ORDER BY file LIMIT 1",
- [],
- |r| r.get(0),
- )
- .optional()?;
- let Some(id) = id else {
- break;
- };
- store
- .db
- .connection()
- .execute(
- "UPDATE desired_files SET claimed=1 WHERE file=?1",
- [id],
- )?;
- let file: ente_photos::files::File = store
- .json("SELECT record FROM sources WHERE file=?1", [id])?
- .context("missing file source snapshot")?;
- transaction.commit()?;
- file
- };
- let id = file.id;
- let result = runtime.block_on(run.file(session, file));
- if result.as_ref().err().is_some_and(fatal) {
- cancel_send.send_replace(true);
- }
- let reported = result.or_else(|error| run.block_file(id, error));
- let cleaned = run.cleanup(Some(id));
- reported?;
- cleaned?;
- let completed = {
- let mut last = last_report.lock().map_err(|_| anyhow::anyhow!("progress mutex poisoned"))?;
- if last.is_none_or(|time| time.elapsed() >= Duration::from_secs(2)) {
- *last = Some(Instant::now());
- Some(store::lock(store)?.db.connection().query_row(
- "SELECT count(*) FROM desired_files WHERE completed=1",
- [],
- |r| r.get::<_, i64>(0),
- )?)
- } else {
- None
- }
- };
- if let Some(completed) = completed {
- eprintln!("Completed {completed}/{expected} selected copies.");
- }
- }
- Ok(())
- })();
- if result.is_err() {
- cancel_send.send_replace(true);
- }
- result
- }));
- }
- let mut result = Ok(());
- for worker in workers {
- let completed = worker
- .join()
- .unwrap_or_else(|_| Err(anyhow::anyhow!("export worker panicked")));
- if result.is_ok() {
- result = completed;
- }
- }
- result
- })
-}
-
-pub fn record_outcome(store: &Store, unit: &str, error: &anyhow::Error) -> Result<()> {
- let conflict = error.downcast_ref::<Conflict>().is_some();
- store
- .db
- .connection()
- .execute(
- "INSERT INTO outcomes VALUES(?1,?2) ON CONFLICT(unit) DO UPDATE SET conflict=max(conflict,excluded.conflict)",
- params![unit, conflict],
- )?;
- Ok(())
-}
-
-pub fn fatal(error: &anyhow::Error) -> bool {
- fatal_cause(error.as_ref())
-}
-
-fn fatal_cause(cause: &(dyn std::error::Error + 'static)) -> bool {
- if cause.is::<rusqlite::Error>()
- || cause.is::<Cancelled>()
- || matches!(
- cause.downcast_ref::<crate::core_db::Error>(),
- Some(crate::core_db::Error::Sqlite(_))
- )
- || cause
- .downcast_ref::<ente_core::http::Error>()
- .is_some_and(|error| error.status_code() == Some(401))
- {
- return true;
- }
- if let Some(error) = cause.downcast_ref::<std::io::Error>() {
- return error.kind() == std::io::ErrorKind::StorageFull
- || error.get_ref().is_some_and(|inner| fatal_cause(inner));
- }
- if let Some(error) = cause.downcast_ref::<ente_photos::files::Error>() {
- use ente_photos::files::Error;
- match error {
- Error::Http(error) => return fatal_cause(error),
- Error::Io(error) => return fatal_cause(error),
- Error::Collections(error) => return fatal_cause(error),
- _ => {}
- }
- }
- if let Some(ente_collections::Error::Http(error)) = cause.downcast_ref() {
- return fatal_cause(error);
- }
- if let Some(error) = cause.downcast_ref::<ente_photos::live_photo::Error>() {
- match error {
- ente_photos::live_photo::Error::Io(error) => return fatal_cause(error),
- ente_photos::live_photo::Error::Zip(error) => return fatal_cause(error),
- ente_photos::live_photo::Error::Components => {}
- }
- }
- cause.source().is_some_and(fatal_cause)
-}
-
-fn summary(store: &Store, destination: &Path, as_json: bool) -> Result<bool> {
- let unknown: bool = store.db.connection().query_row(
- "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND ready=0)",
- [],
- |r| r.get(0),
- )?;
- let (files, expected, completed): (i64, i64, i64) = store.db.connection().query_row(
- "SELECT count(DISTINCT file),count(*),coalesce(sum(completed),0) FROM desired_files",
- [],
- |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
- )?;
- let (failures, conflicts): (i64, i64) = store.db.connection().query_row(
- "SELECT count(*),coalesce(sum(conflict),0) FROM outcomes",
- [],
- |r| Ok((r.get(0)?, r.get(1)?)),
- )?;
- let (exported, metadata_updated, renamed, retained): (i64, i64, i64, i64) = store
- .db
- .connection()
- .query_row(
- "SELECT coalesce(sum(exported),0),coalesce(sum(metadata_updated),0),coalesce(sum(renamed),0),coalesce(sum(retained),0) FROM events",
- [],
- |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
- )?;
- let complete = !unknown && failures == 0 && completed == expected;
- let result = json!({"destination":destination,"complete":complete,"files":if unknown {None}else{Some(files)},"copies":if unknown {None}else{Some(json!({"expected":expected,"completed":completed,"pending":expected-completed}))},"changes":{"exported":exported,"metadataUpdated":metadata_updated,"renamed":renamed,"retained":retained},"failures":failures,"conflicts":conflicts});
+ let changes = &summary.json["changes"];
output::action(
as_json,
- &result,
+ &summary.json,
&format!(
- "{}: {completed}/{expected} copies complete, {exported} exported, {retained} retained, {failures} failures, {conflicts} conflicts.",
- destination.display()
+ "{}: {progress}, {} exported, {} retained, {} failures, {} conflicts.",
+ export.destination().display(),
+ changes["exported"],
+ changes["retained"],
+ summary.json["failures"],
+ summary.json["conflicts"],
),
)?;
- Ok(complete)
+ ensure!(summary.complete, "export is incomplete");
+ Ok(())
}
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn transparent_errors_preserve_fatal_classification() {
- for status in [401, 404] {
- let error = ente_core::http::Error::Http {
- status,
- path: "/files".into(),
- };
- let error =
- ente_photos::files::Error::Collections(ente_collections::Error::Http(error));
- let error = anyhow::Error::new(error).context("download failed");
- assert_eq!(fatal(&error), status == 401);
- assert_eq!(
- format!("{error:#}"),
- format!("download failed: HTTP {status} at /files")
- );
- }
- let cancelled = ente_photos::live_photo::Error::Io(std::io::Error::other(Cancelled));
- assert!(fatal(&anyhow::Error::new(cancelled)));
- let full = ente_photos::live_photo::Error::Io(std::io::ErrorKind::StorageFull.into());
- assert!(fatal(&anyhow::Error::new(full)));
- let zip = ente_photos::live_photo::Error::Zip(zip::result::ZipError::Io(
- std::io::ErrorKind::StorageFull.into(),
- ));
- assert!(fatal(&anyhow::Error::new(zip)));
- let sqlite = crate::core_db::Error::Sqlite(rusqlite::Error::InvalidQuery);
- assert!(fatal(&anyhow::Error::new(sqlite)));
- let local = ente_photos::files::Error::Io(std::io::ErrorKind::PermissionDenied.into());
- assert!(!fatal(&anyhow::Error::new(local)));
- }
+fn store_path(account_home: &Path, destination: &Path) -> Result<PathBuf> {
+ let bytes = destination.as_os_str().as_encoded_bytes();
+ let name: String = hash::hash(bytes, Some(32), None)?
+ .iter()
+ .map(|b| format!("{b:02x}"))
+ .collect();
+ Ok(account_home.join("exports").join(format!("{name}.db")))
}
diff --git a/rust/apps/cli-next/src/export/source.rs b/rust/apps/cli-next/src/export/source.rs
new file mode 100644
index 0000000000..bfa687672a
--- /dev/null
+++ b/rust/apps/cli-next/src/export/source.rs
@@ -0,0 +1,120 @@
+use anyhow::{Context, Result};
+use ente_core::Session;
+use ente_photos_export::{CollectionEntry, FileEntry, Source};
+
+use crate::{
+ core_db::{Db, OptionalExtension, SqliteResult, params},
+ replica::{AlbumRecord, FileRecord, Replica, read_json},
+};
+
+pub struct ReplicaSource<'a> {
+ replica: Replica<'a>,
+ user_id: i64,
+}
+
+impl<'a> ReplicaSource<'a> {
+ pub fn new(db: &'a mut Db, user_id: i64) -> Self {
+ Self {
+ replica: Replica::new(db, user_id),
+ user_id,
+ }
+ }
+
+ fn collection(&self, record: AlbumRecord, name: Option<String>) -> CollectionEntry {
+ let id = record.remote.id;
+ let favorites = record.remote.kind == "favorites" && record.remote.owner.id == self.user_id;
+ let value = record
+ .album(self.user_id)
+ .and_then(|album| Ok((album, record.documents.context("metadata is unavailable")?)));
+ let name = value
+ .as_ref()
+ .ok()
+ .map(|(album, _)| album.name.clone())
+ .or(name);
+ CollectionEntry {
+ id,
+ name,
+ favorites,
+ value,
+ }
+ }
+
+ fn entry(&self, record: FileRecord) -> FileEntry {
+ let id = record.remote.id;
+ let value = record
+ .file(self.user_id)
+ .and_then(|file| Ok((file, record.documents.context("metadata is unavailable")?)));
+ FileEntry { id, value }
+ }
+}
+
+impl Source for ReplicaSource<'_> {
+ async fn refresh_catalog(&mut self, session: &Session) -> Result<()> {
+ self.replica.sync_collections(session).await
+ }
+
+ fn catalog_page(&self, after: i64) -> Result<Vec<CollectionEntry>> {
+ let next: Option<(AlbumRecord, Option<String>)> = self.replica.db.read(|db| {
+ Ok(db
+ .query_row(
+ "SELECT id,record,name FROM photos_collections WHERE id>?1 ORDER BY id LIMIT 1",
+ [after],
+ |r| Ok((read_json(r, 1)?, r.get(2)?)),
+ )
+ .optional()?)
+ })?;
+ Ok(next
+ .into_iter()
+ .map(|(record, name)| self.collection(record, name))
+ .collect())
+ }
+
+ async fn refresh_album(&mut self, session: &Session, id: i64) -> Result<CollectionEntry> {
+ self.replica.retry_album(session, id)?;
+ let record = self
+ .replica
+ .album_record(id)?
+ .context("missing album source")?;
+ let entry = self.collection(record, None);
+ let (album, documents) = entry.value?;
+ self.replica.sync_album_files(session, &album).await?;
+ Ok(CollectionEntry {
+ value: Ok((album, documents)),
+ ..entry
+ })
+ }
+
+ fn file_page(&self, album: i64, after: i64) -> Result<Vec<FileEntry>> {
+ let records: Vec<FileRecord> = self.replica.db.read(|db| {
+ let mut query = db.prepare(
+ "SELECT record FROM photos_files WHERE collection_id=?1 AND id>?2 ORDER BY id LIMIT 128",
+ )?;
+ Ok(query
+ .query_map(params![album, after], |r| read_json(r, 0))?
+ .collect::<SqliteResult<_>>()?)
+ })?;
+ Ok(records
+ .into_iter()
+ .map(|record| self.entry(record))
+ .collect())
+ }
+
+ fn favorite_page(&self, after: i64) -> Result<Vec<i64>> {
+ Ok(self.replica.db.read(|db| {
+ let mut query = db.prepare(
+ "SELECT DISTINCT f.id FROM photos_files f JOIN photos_collections c ON c.id=f.collection_id WHERE f.id>?1 AND json_extract(c.record,'$.remote.type')='favorites' AND json_extract(c.record,'$.remote.owner.id')=?2 ORDER BY f.id LIMIT 128",
+ )?;
+ Ok(query
+ .query_map(params![after, self.user_id], |r| r.get(0))?
+ .collect::<SqliteResult<_>>()?)
+ })?)
+ }
+
+ fn file(&self, album: i64, id: i64) -> Result<FileEntry> {
+ Ok(self.entry(
+ self.replica
+ .file_record(album, id)?
+ .context("missing chosen file source")?,
+ ))
+ }
+}
diff --git a/rust/apps/cli-next/src/photos.rs b/rust/apps/cli-next/src/photos.rs
index 7d2235c6c9..201e9fbaad 100644
--- a/rust/apps/cli-next/src/photos.rs
+++ b/rust/apps/cli-next/src/photos.rs
@@ -162,12 +162,18 @@ async fn file(
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or(Path::new("."));
let mut temporary = tempfile::NamedTempFile::new_in(parent)?;
- files::download(session, &entry.file, || {
- let file = temporary.as_file_mut();
- file.set_len(0)?;
- file.rewind()?;
- file.try_clone()
- })
+ files::download(
+ session,
+ entry.file.id,
+ &entry.file.key,
+ &entry.file.header,
+ || {
+ let file = temporary.as_file_mut();
+ file.set_len(0)?;
+ file.rewind()?;
+ file.try_clone()
+ },
+ )
.await?;
temporary.as_file().sync_all()?;
let bytes = temporary.as_file().metadata()?.len();
diff --git a/rust/apps/cli-next/src/vault.rs b/rust/apps/cli-next/src/vault.rs
index fcd3ae2c8e..d83a90c5f9 100644
--- a/rust/apps/cli-next/src/vault.rs
+++ b/rust/apps/cli-next/src/vault.rs
@@ -89,7 +89,7 @@ impl Vault {
keyring::Entry::new("io.ente.cli", &home.to_string_lossy())
})?;
let state = match encrypted {
- Some(bytes) => decrypt(&bytes, &key)?,
+ Some(bytes) => decrypt(&bytes, &key, &home)?,
None => State::default(),
};
Ok(Self {
@@ -144,7 +144,7 @@ impl VaultAccess {
let Self { home, key } = self;
let lock = lock(&home)?;
let state = match read_vault(&home)? {
- Some(bytes) => decrypt(&bytes, &key)?,
+ Some(bytes) => decrypt(&bytes, &key, &home)?,
None => State::default(),
};
Ok(Vault {
@@ -168,7 +168,7 @@ impl State {
let key = load_key(override_key, true, || {
keyring::Entry::new("io.ente.cli", &home.to_string_lossy())
})?;
- decrypt(&bytes, &key)
+ decrypt(&bytes, &key, &home)
}
pub fn named(&self, name: &str) -> Result<usize> {
@@ -305,7 +305,7 @@ fn keyring_error(error: keyring::Error, exists: bool) -> anyhow::Error {
}
}
-fn decrypt(bytes: &[u8], key: &Key) -> Result<State> {
+fn decrypt(bytes: &[u8], key: &Key, home: &Path) -> Result<State> {
let unlock_error =
|| anyhow::anyhow!("cannot unlock CLI vault: wrong key or damaged ciphertext");
let encrypted: EncryptedVault =
@@ -318,7 +318,9 @@ fn decrypt(bytes: &[u8], key: &Key) -> Result<State> {
parse_json(&plaintext).context("invalid CLI vault contents")?;
ensure!(
version.schema_version == SCHEMA_VERSION,
- "unsupported CLI vault schema"
+ "unsupported CLI vault schema {}; this build supports {SCHEMA_VERSION}.\nTo start fresh, move {} aside, then log in again.",
+ version.schema_version,
+ home.join(VAULT_FILE).display(),
);
let stored: VersionedState<State> =
parse_json(&plaintext).context("invalid CLI vault contents")?;
diff --git a/rust/apps/cli-next/tests/cli.rs b/rust/apps/cli-next/tests/cli.rs
index 99e6caab17..4a526708fa 100644
--- a/rust/apps/cli-next/tests/cli.rs
+++ b/rust/apps/cli-next/tests/cli.rs
@@ -231,7 +231,12 @@ fn vault_schema_version_is_checked_after_decryption() {
home.seed("http://localhost:8080");
home.write_vault_document(&json!({"schema_version": 2, "state": Value::Null}));
let original = fs::read(home.dir.path().join("vault.json")).unwrap();
- assert!(failure(&home.run(&["account", "list"])).contains("unsupported CLI vault schema"));
+ let error = failure(&home.run(&["account", "list"]));
+ assert!(error.contains("unsupported CLI vault schema 2; this build supports 1."));
+ assert!(error.contains(&format!(
+ "To start fresh, move {} aside, then log in again.",
+ home.dir.path().canonicalize().unwrap().join("vault.json").display(),
+ )));
let output = home
.command(&["account", "list"])
.env(
diff --git a/rust/apps/cli-next/tests/support/export.rs b/rust/apps/cli-next/tests/support/export.rs
index 68f0267389..b85d1db778 100644
--- a/rust/apps/cli-next/tests/support/export.rs
+++ b/rust/apps/cli-next/tests/support/export.rs
@@ -132,6 +132,7 @@ fn export_metadata_updates_and_repairs_preserve_source_values() -> TestResult {
let record=read_json(&sidecar);
assert_eq!(record["ente"]["creationTime"],"2023-11-14T22:13:20.123456Z");
assert_eq!(record["photoTakenTime"]["timestamp"],"1700000000");
+ assert_eq!(record["photoTakenTime"]["formatted"],"Nov 14, 2023, 10:13:20 PM UTC");
assert_eq!(record["ente"]["components"][0]["size"],original.len());
assert_eq!(record["ente"]["modificationTime"],"2023-11-14T22:13:21.123456Z");
assert_eq!(record["modificationTime"]["timestamp"],"1700000001");
@@ -884,10 +885,23 @@ fn export_live_pairs_retry_and_reuse_without_rewriting_unchanged_output() -> Tes
})
.collect();
let db = export_database(&home);
- for table in ["albums", "placements", "components", "json_records", "pending"] {
+ for table in [
+ "albums",
+ "placements",
+ "components",
+ "json_records",
+ "pending",
+ "sources",
+ "album_sources",
+ ] {
for action in ["INSERT", "UPDATE", "DELETE"] {
+ let timing = if ["sources", "album_sources"].contains(&table) {
+ "AFTER"
+ } else {
+ "BEFORE"
+ };
db.execute_batch(&format!(
- "CREATE TRIGGER unchanged_{table}_{action} BEFORE {action} ON {table} BEGIN SELECT RAISE(ABORT,'unchanged association write'); END;",
+ "CREATE TRIGGER unchanged_{table}_{action} {timing} {action} ON {table} BEGIN SELECT RAISE(ABORT,'unchanged association write'); END;",
))?;
}
}
@@ -911,7 +925,15 @@ fn export_live_pairs_retry_and_reuse_without_rewriting_unchanged_output() -> Tes
let path = root.join(path);
assert_eq!((fs::read(&path)?, fs::metadata(&path)?.modified()?), before);
}
- for table in ["albums", "placements", "components", "json_records", "pending"] {
+ for table in [
+ "albums",
+ "placements",
+ "components",
+ "json_records",
+ "pending",
+ "sources",
+ "album_sources",
+ ] {
for action in ["INSERT", "UPDATE", "DELETE"] {
db.execute_batch(&format!("DROP TRIGGER unchanged_{table}_{action};"))?;
}
@@ -946,7 +968,8 @@ fn export_excludes_another_writer_while_downloads_are_running() -> TestResult {
let other = TestHome::new();
login(&other, "photos", &email, &["--host", origin]);
let (album, key) = create_album(origin, &account, "Album", "album").await;
- for name in ["First.jpg", "Second.jpg"] {
+ let names = ["First.jpg", "Second.jpg", "Third.jpg", "Fourth.jpg"];
+ for name in names {
upload_fixture(
origin,
&account,
@@ -960,6 +983,7 @@ fn export_excludes_another_writer_while_downloads_are_running() -> TestResult {
let destination = tempfile::tempdir()?;
let root = destination.path().join("photos");
let held = objects.hold_reads(0);
+ let reads = objects.reads();
let child = ExportChild::spawn(&mut home.command(&[
"photos",
"export",
@@ -969,6 +993,8 @@ fn export_excludes_another_writer_while_downloads_are_running() -> TestResult {
"--json",
]))?;
held.wait_for(2);
+ std::thread::sleep(Duration::from_millis(500));
+ assert_eq!(objects.reads() - reads, 2);
let second = ExportChild::spawn(&mut other.command(&[
"photos",
"export",
@@ -978,8 +1004,10 @@ fn export_excludes_another_writer_while_downloads_are_running() -> TestResult {
assert!(failure(&second.wait_with_output()?).contains("another writer"));
drop(held);
let result: Value = serde_json::from_slice(&success(child.wait_with_output()?).stdout)?;
- assert_eq!(result["copies"]["completed"], 2);
- assert_eq!(objects.peak_reads(), 2);
+ assert_eq!(result["copies"]["completed"], 4);
+ for name in names {
+ assert_eq!(fs::read(root.join("Album").join(name))?, name.as_bytes());
+ }
Ok(())
})
})
diff --git a/rust/apps/cli-next/tests/support/export_maintenance.rs b/rust/apps/cli-next/tests/support/export_maintenance.rs
index bdb82dd45a..b818ee1c73 100644
--- a/rust/apps/cli-next/tests/support/export_maintenance.rs
+++ b/rust/apps/cli-next/tests/support/export_maintenance.rs
@@ -91,6 +91,52 @@ fn export_retries_transient_collection_and_file_refresh_failures() {
}
}
+#[test]
+fn export_local_inventory_failure_preserves_output_and_reports_unknown_totals() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ listing(&mut server, json!([collection(1, "First", &key)]));
+ let (file, bytes) = source(10, &key, b"original", "Photo.jpg");
+ page(&mut server, 1, 0, json!([file]), false).create();
+ let downloaded = download(&mut server, 10, &bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ assert_eq!(run(&home, &root, &[], true)["copies"]["completed"], 1);
+ let published = [
+ "export.json",
+ "First/metadata.json",
+ "First/Photo.jpg",
+ "First/metadata/Photo.jpg.json",
+ ]
+ .map(|path| (path, fs::read(root.join(path)).unwrap()));
+ database(&home)
+ .execute_batch(
+ "UPDATE photos_files SET record='{}' WHERE collection_id=1 AND id=10; UPDATE photos_collections SET files_synced_to=NULL WHERE id=1;",
+ )
+ .unwrap();
+ let refreshed = page(&mut server, 1, 10, json!([]), false)
+ .expect(1)
+ .create();
+ let result = run_failure(&home, &root, &[], "missing field `remote`");
+ assert!(result["files"].is_null());
+ assert!(result["copies"].is_null());
+ assert_eq!(result["failures"], 1);
+ let output = home.run(&["photos", "export", root.to_str().unwrap()]);
+ assert!(failure(&output).contains("missing field `remote`"));
+ assert!(
+ String::from_utf8(output.stdout)
+ .unwrap()
+ .contains("inventory incomplete")
+ );
+ for (path, bytes) in published {
+ assert_eq!(fs::read(root.join(path)).unwrap(), bytes);
+ }
+ refreshed.assert();
+ downloaded.assert();
+}
+
#[test]
fn export_snapshot_uses_current_healthy_memberships_and_preserves_raw_bytes() {
let mut server = mockito::Server::new();
@@ -149,6 +195,27 @@ fn export_snapshot_uses_current_healthy_memberships_and_preserves_raw_bytes() {
assert_eq!(record["description"], "winner");
assert_eq!(record["favorited"], true);
}
+ let state = export_database(&home);
+ for table in ["sources", "album_sources"] {
+ for action in ["INSERT", "UPDATE", "DELETE"] {
+ state.execute_batch(&format!(
+ "CREATE TRIGGER unchanged_{table}_{action} AFTER {action} ON {table} BEGIN SELECT RAISE(ABORT,'unchanged snapshot write'); END;",
+ )).unwrap();
+ }
+ }
+ let unchanged = run(&home, &root, &[], false);
+ assert_eq!(unchanged["copies"], result["copies"]);
+ assert_eq!(
+ unchanged["changes"],
+ json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})
+ );
+ for table in ["sources", "album_sources"] {
+ for action in ["INSERT", "UPDATE", "DELETE"] {
+ state
+ .execute_batch(&format!("DROP TRIGGER unchanged_{table}_{action};"))
+ .unwrap();
+ }
+ }
let untouched = root.join("Second/metadata/Photo.jpg.json");
let before = (
fs::read(&untouched).unwrap(),
@@ -2205,3 +2272,139 @@ fn export_reports_completion_progress_before_the_command_finishes() {
url.assert();
slow.assert();
}
+
+#[test]
+fn export_only_file_metadata_does_not_break_common_commands() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ listing(
+ &mut server,
+ json!([
+ collection(1, "Affected", &key),
+ collection(2, "Healthy", &key)
+ ]),
+ );
+ let (file, _) = source(10, &key, b"unused", "Original.jpg");
+ let file = public(
+ file,
+ &key,
+ json!({
+ "editedName":"Export metadata.jpg", "cameraMake":17, "cameraModel":17,
+ "uploaderName":17, "mediaType":"broken", "mvi":"broken"
+ }),
+ 10,
+ );
+ let (healthy, bytes) = source(20, &key, b"healthy", "Healthy.jpg");
+ page(&mut server, 1, 0, json!([file]), false).create();
+ page(&mut server, 2, 0, json!([healthy]), false).create();
+ let no_download = server
+ .mock("GET", "/files/download/v3/10")
+ .expect(0)
+ .create();
+ let downloaded = download(&mut server, 20, &bytes, 1);
+ let listed = home.json(&["photos", "file", "list"]);
+ assert_eq!(listed.as_array().unwrap().len(), 2);
+ let viewed = home.json(&["photos", "file", "view", "Export metadata.jpg", "--offline"]);
+ assert_eq!(viewed["id"], "10");
+ assert_eq!(viewed["name"], "Export metadata.jpg");
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
+ let error = failure(&output);
+ assert!(
+ error.contains("file:1:10: invalid public file metadata"),
+ "{error}"
+ );
+ let result: Value = serde_json::from_slice(&output.stdout).unwrap();
+ assert_eq!(result["complete"], false);
+ assert_eq!(
+ result["copies"],
+ json!({"expected":2,"completed":1,"pending":1})
+ );
+ assert_eq!(result["failures"], 1);
+ assert_component(&root, "Healthy", "Healthy.jpg", b"healthy");
+ assert!(!root.join("Affected/Export metadata.jpg").exists());
+ no_download.assert();
+ downloaded.assert();
+}
+
+#[test]
+fn export_only_album_metadata_preserves_common_commands_and_favorites() {
+ for (document, value, cause) in [
+ (
+ "magicMetadata",
+ json!({"order":"broken"}),
+ "invalid album private metadata",
+ ),
+ (
+ "sharedMagicMetadata",
+ json!({"order":"broken","mute":"broken"}),
+ "invalid album sharee metadata",
+ ),
+ (
+ "pubMagicMetadata",
+ json!({"asc":"broken","coverID":"broken","layout":17,"caption":17}),
+ "invalid album public metadata",
+ ),
+ ] {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let mut favorites = collection(5, "Favorites", &key);
+ favorites["type"] = json!("favorites");
+ let encrypted = blob::encrypt_json(&value, &key).unwrap();
+ favorites[document] = json!({"data":b64::encode(&encrypted.encrypted_data),"header":b64::encode(encrypted.decryption_header.as_bytes())});
+ listing(
+ &mut server,
+ json!([collection(1, "Healthy", &key), favorites]),
+ );
+ let (file, bytes) = source(10, &key, b"original", "Photo.jpg");
+ page(&mut server, 1, 0, json!([file]), false).create();
+ let memberships = page(&mut server, 5, 0, json!([file]), false)
+ .expect(1)
+ .create();
+ let downloaded = download(&mut server, 10, &bytes, 1);
+ let albums = home.json(&["photos", "album", "list"]);
+ assert_eq!(albums.as_array().unwrap().len(), 2);
+ let viewed = home.json(&["photos", "album", "view", "Favorites", "--offline"]);
+ assert_eq!(viewed["id"], "5");
+ assert_eq!(viewed["name"], "Favorites");
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let output = home.run(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "--album",
+ "Healthy",
+ "--json",
+ ]);
+ assert!(
+ output.status.success(),
+ "{}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ assert!(!String::from_utf8_lossy(&output.stderr).contains("album:5"));
+ let sidecar = root.join("Healthy/metadata/Photo.jpg.json");
+ let mut metadata = read_json(&sidecar);
+ assert_eq!(metadata["favorited"], true);
+ metadata["favorited"] = json!(false);
+ fs::write(&sidecar, serde_json::to_vec(&metadata).unwrap()).unwrap();
+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
+ let error = failure(&output);
+ assert!(error.contains(&format!("album:5: {cause}")), "{error}");
+ assert!(!error.contains("favorite state is incomplete"), "{error}");
+ let result: Value = serde_json::from_slice(&output.stdout).unwrap();
+ assert_eq!(result["complete"], false);
+ assert_eq!(result["failures"], 1);
+ assert_eq!(result["changes"]["metadataUpdated"], 1);
+ assert_eq!(read_json(&sidecar)["favorited"], true);
+ assert_component(&root, "Healthy", "Photo.jpg", b"original");
+ assert!(!root.join("Favorites").exists());
+ memberships.assert();
+ downloaded.assert();
+ }
+}
diff --git a/rust/apps/cli-next/tests/support/export_sources.rs b/rust/apps/cli-next/tests/support/export_sources.rs
index 275d62e0ea..8515b41a04 100644
--- a/rust/apps/cli-next/tests/support/export_sources.rs
+++ b/rust/apps/cli-next/tests/support/export_sources.rs
@@ -980,6 +980,7 @@ fn export_page_and_recognized_failures_roll_back_together_on_storage_failure() {
broken["metadata"]["encryptedData"] = json!("broken");
page(&mut server, 1, 0, json!([good, broken]), false).create();
let result = run_failure(&home, &root, &[], "fixture storage failure");
+ assert_eq!(result["failures"], 2);
assert!(result["files"].is_null());
assert_eq!(
db.query_row("SELECT count(*) FROM photos_files", [], |r| r
diff --git a/rust/crates/photos-export/Cargo.toml b/rust/crates/photos-export/Cargo.toml
new file mode 100644
index 0000000000..b3d3deb980
--- /dev/null
+++ b/rust/crates/photos-export/Cargo.toml
@@ -0,0 +1,23 @@
+[package]
+name = "ente-photos-export"
+edition.workspace = true
+publish = false
+
+[dependencies]
+anyhow.workspace = true
+chrono = { workspace = true, features = ["std"] }
+ente-collections.workspace = true
+ente-core.workspace = true
+ente-photos.workspace = true
+icu_casemap.workspace = true
+rusqlite.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }
+unicode-normalization.workspace = true
+uuid = { workspace = true, features = ["v4"] }
+zip.workspace = true
+
+[lints]
+workspace = true
diff --git a/rust/apps/cli-next/src/export/adopt.rs b/rust/crates/photos-export/src/adopt.rs
similarity index 99%
rename from rust/apps/cli-next/src/export/adopt.rs
rename to rust/crates/photos-export/src/adopt.rs
index c784ef680b..a0f7788bff 100644
--- a/rust/apps/cli-next/src/export/adopt.rs
+++ b/rust/crates/photos-export/src/adopt.rs
@@ -1,7 +1,7 @@
use std::{fs, path::Path};
+use crate::metadata::{Component, Role};
use anyhow::{Context, Result, ensure};
-use ente_photos::export::{Component, Role};
use serde_json::Value;
use uuid::Uuid;
@@ -13,7 +13,6 @@ use super::{
pub fn scan(root: &Path, store: &Store) -> Result<()> {
store
.db
- .connection()
.execute_batch(
"DELETE FROM components; DELETE FROM json_records; DELETE FROM placements; DELETE FROM albums; DELETE FROM pending; DELETE FROM temporaries;",
)?;
@@ -208,7 +207,6 @@ fn album(root: &Path, store: &Store, directory: &Path, retained: bool) -> Result
let existing = if retained {
let mut query = store
.db
- .connection()
.prepare(
"SELECT DISTINCT p.id,p.album,p.file,p.retained,p.name,p.kind FROM placements p JOIN components c ON c.placement=p.id WHERE p.album=?1 AND p.file=?2 AND p.retained=1 AND c.folder=?3",
)?;
diff --git a/rust/apps/cli-next/src/export/fs.rs b/rust/crates/photos-export/src/fs.rs
similarity index 97%
rename from rust/apps/cli-next/src/export/fs.rs
rename to rust/crates/photos-export/src/fs.rs
index fc438df966..f12364c424 100644
--- a/rust/apps/cli-next/src/export/fs.rs
+++ b/rust/crates/photos-export/src/fs.rs
@@ -112,9 +112,9 @@ pub fn set_time(path: &Path, micros: i64) -> Result<Properties> {
Properties::read(path)
}
-pub fn sync_parent(path: &Path) -> Result<()> {
+pub fn sync_parent(_path: &Path) -> Result<()> {
#[cfg(unix)]
- if let Some(parent) = path.parent() {
+ if let Some(parent) = _path.parent() {
File::open(parent)?.sync_all()?;
}
Ok(())
diff --git a/rust/crates/photos-export/src/lib.rs b/rust/crates/photos-export/src/lib.rs
new file mode 100644
index 0000000000..bc9427eb7c
--- /dev/null
+++ b/rust/crates/photos-export/src/lib.rs
@@ -0,0 +1,763 @@
+mod adopt;
+mod fs;
+mod live_photo;
+mod metadata;
+mod names;
+mod reconcile;
+mod snapshot;
+mod source;
+mod store;
+mod transfer;
+
+pub use source::{CollectionEntry, FileEntry, Source};
+
+use std::{
+ fs::{File, OpenOptions},
+ io::Write,
+ num::NonZeroUsize,
+ path::{Path, PathBuf},
+ sync::Mutex,
+ time::{Duration, Instant},
+};
+
+use crate::metadata::Root;
+use anyhow::{Context, Result, bail, ensure};
+use rusqlite::{OptionalExtension, params};
+use serde_json::{Value, json};
+
+use tokio::sync::watch;
+
+use snapshot::FileSnapshot;
+use store::Store;
+
+#[derive(Debug, thiserror::Error)]
+#[error("conflict: {0}")]
+struct Conflict(String);
+
+#[derive(Debug, thiserror::Error)]
+#[error("export cancelled")]
+struct Cancelled;
+
+pub struct Options {
+ pub albums: Vec<String>,
+ pub exclude_albums: Vec<String>,
+ pub adopt: bool,
+ pub jobs: NonZeroUsize,
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("{0}")]
+pub struct AdoptionRequired(&'static str);
+
+pub struct Summary {
+ pub json: Value,
+ pub complete: bool,
+ pub completed: i64,
+ pub expected: i64,
+}
+
+pub struct Export {
+ destination: PathBuf,
+ expected_root: Root,
+ root_lock: Option<File>,
+ options: Options,
+ prepared: Option<(Store, Result<()>)>,
+}
+
+impl Export {
+ pub fn open(
+ path: &Path,
+ master_key: &ente_core::crypto::Key,
+ options: Options,
+ ) -> Result<Self> {
+ let destination = destination(path)?;
+ let expected_root = Root::new(master_key)?;
+ let root_lock = open_root(&destination, &expected_root)?;
+ ensure!(
+ !options.adopt || root_lock.is_some(),
+ "there is no export to adopt"
+ );
+ Ok(Self {
+ destination,
+ expected_root,
+ root_lock,
+ options,
+ prepared: None,
+ })
+ }
+
+ pub fn destination(&self) -> &Path {
+ &self.destination
+ }
+
+ pub fn exists(&self) -> bool {
+ self.root_lock.is_some()
+ }
+
+ pub fn verify_source(&self, master_key: &ente_core::crypto::Key) -> Result<()> {
+ ensure!(
+ Root::new(master_key)?.source_verifier == self.expected_root.source_verifier,
+ "account keys changed while waiting for access"
+ );
+ Ok(())
+ }
+
+ pub async fn prepare(
+ &mut self,
+ connection: rusqlite::Connection,
+ source: &mut impl Source,
+ session: &ente_core::Session,
+ cancel: &mut watch::Receiver<bool>,
+ report: &(dyn Fn(&str) + Sync),
+ ) -> Result<()> {
+ let store = Store::open(connection, &self.destination)?;
+ let bound: bool = store
+ .db
+ .query_row("SELECT bound FROM export WHERE id=1", [], |r| r.get(0))?;
+ ensure!(
+ self.root_lock.is_none() || bound || self.options.adopt,
+ AdoptionRequired("this export is not associated")
+ );
+ store.reset_desired()?;
+ if self.root_lock.is_some() && !bound {
+ adopt::scan(&self.destination, &store)?;
+ }
+ report("Refreshing album metadata.");
+ tokio::select! {
+ result=source.refresh_catalog(session)=>result?,
+ _=cancel.changed()=>return Err(Cancelled.into()),
+ }
+ let mut after = 0;
+ loop {
+ ensure!(!*cancel.borrow(), Cancelled);
+ let entries = source.catalog_page(after)?;
+ if entries.is_empty() {
+ break;
+ }
+ for entry in entries {
+ after = entry.id;
+ store.desired_album(&entry)?;
+ }
+ }
+ store
+ .db
+ .execute(
+ "INSERT OR IGNORE INTO desired_albums(id,name) SELECT s.id,json_extract(s.record,'$.name') FROM album_sources s WHERE EXISTS(SELECT 1 FROM albums a WHERE a.id=s.id)",
+ [],
+ )?;
+ store
+ .db
+ .execute(
+ "INSERT OR IGNORE INTO desired_albums(id,name) SELECT id,name FROM albums ORDER BY retained",
+ [],
+ )?;
+ store
+ .db
+ .execute(
+ "INSERT OR IGNORE INTO desired_albums(id,name) SELECT album,name FROM pending WHERE file IS NULL AND retained=0",
+ [],
+ )?;
+ select(&store, &self.options)?;
+ if self.root_lock.is_none() {
+ self.root_lock = Some(initialize(&self.destination, &self.expected_root)?);
+ }
+ store
+ .db
+ .execute("UPDATE export SET bound=1 WHERE id=1 AND bound=0", [])?;
+
+ let refresh_cancel = cancel.clone();
+ let refreshed = tokio::select! {
+ result=refresh(&store,source,session,&refresh_cancel,report)=>result,
+ _=cancel.changed()=>Err(Cancelled.into()),
+ };
+ self.prepared = Some((store, refreshed));
+ Ok(())
+ }
+
+ pub fn execute(
+ &mut self,
+ session: &ente_core::Session,
+ cancel_send: &watch::Sender<bool>,
+ cancel: &watch::Receiver<bool>,
+ report: &(dyn Fn(&str) + Sync),
+ ) -> Result<Summary> {
+ let (store, refreshed) = self.prepared.take().context("export is not prepared")?;
+ let prepared = refreshed.is_ok();
+ let shared = Mutex::new(store);
+ let run = reconcile::Run {
+ root: &self.destination,
+ store: &shared,
+ cancel,
+ report,
+ };
+ let result = (|| {
+ refreshed?;
+ run.recover()?;
+ run.albums()?;
+ transfers(
+ &self.destination,
+ &shared,
+ session,
+ cancel_send,
+ cancel,
+ self.options.jobs.get(),
+ report,
+ )?;
+ run.check_cancel()?;
+ run.retain_removed()
+ })();
+ if let Err(error) = result {
+ run.report("run", &error)?;
+ }
+ if let Err(error) = run.cleanup(None) {
+ run.report("run", &error)?;
+ }
+ let store = shared
+ .into_inner()
+ .map_err(|_| anyhow::anyhow!("export database mutex poisoned"))?;
+ summary(&store, &self.destination, prepared)
+ }
+}
+
+fn destination(path: &Path) -> Result<PathBuf> {
+ let absolute = if path.is_absolute() {
+ path.to_owned()
+ } else {
+ std::env::current_dir()?.join(path)
+ };
+ let mut base = absolute.as_path();
+ let mut tail = Vec::new();
+ while !base.try_exists()? {
+ tail.push(
+ base.file_name()
+ .context("destination has no directory name")?
+ .to_owned(),
+ );
+ base = base
+ .parent()
+ .context("destination has no existing parent")?;
+ }
+ if tail.is_empty() {
+ ensure!(
+ !std::fs::symlink_metadata(base)?.is_symlink(),
+ "export destination is a symlink"
+ );
+ }
+ let mut resolved = std::fs::canonicalize(base)?;
+ for part in tail.into_iter().rev() {
+ resolved.push(part);
+ }
+ Ok(resolved)
+}
+
+fn open_root(destination: &Path, expected: &Root) -> Result<Option<File>> {
+ if !destination.try_exists()? {
+ return Ok(None);
+ }
+ ensure!(
+ destination.is_dir(),
+ "export destination is not a directory"
+ );
+ let path = names::check(destination, "export.json")?;
+ if !path.try_exists()? {
+ ensure!(
+ std::fs::read_dir(destination)?.next().is_none(),
+ "nonempty destination has no recognized export.json"
+ );
+ return Ok(None);
+ }
+ let file = OpenOptions::new().read(true).write(true).open(&path)?;
+ file.try_lock()
+ .context("another writer is maintaining this export")?;
+ let record: Root = serde_json::from_reader(&file).context("invalid export.json")?;
+ ensure!(
+ record.format == expected.format && record.version == expected.version,
+ "unsupported export format"
+ );
+ ensure!(
+ record.source_verifier == expected.source_verifier,
+ "this export belongs to another source account"
+ );
+ Ok(Some(file))
+}
+
+fn initialize(destination: &Path, root: &Root) -> Result<File> {
+ fs::create_directory(destination)?;
+ let path = names::check(destination, "export.json")?;
+ let mut file = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create_new(true)
+ .open(&path)
+ .context("destination was initialized by another writer; retry")?;
+ file.lock().context("cannot lock the new export")?;
+ ensure!(
+ std::fs::read_dir(destination)?
+ .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),
+ "destination became nonempty during initialization"
+ );
+ serde_json::to_writer_pretty(&mut file, root)?;
+ file.write_all(b"\n")?;
+ file.sync_all()?;
+ fs::sync_parent(&path)?;
+ Ok(file)
+}
+
+fn select(store: &Store, options: &Options) -> Result<()> {
+ if options.albums.is_empty() {
+ store
+ .db
+ .execute("UPDATE desired_albums SET selected=1", [])?;
+ }
+ for (selectors, include) in [(&options.albums, true), (&options.exclude_albums, false)] {
+ for selector in selectors {
+ let mut query = store.db.prepare(
+ "SELECT id,name FROM desired_albums WHERE id=?1 OR name=?2 ORDER BY id LIMIT 11",
+ )?;
+ let candidates: Vec<(i64, Option<String>)> = query
+ .query_map(params![canonical_id(selector), selector], |r| {
+ Ok((r.get(0)?, r.get(1)?))
+ })?
+ .collect::<rusqlite::Result<_>>()?;
+ ensure!(!candidates.is_empty(), "no album matches {selector:?}");
+ if candidates.len() != 1 {
+ bail!(
+ "album {selector:?} is ambiguous: {}",
+ candidates
+ .iter()
+ .map(|(id, name)| format!(
+ "{id} {:?}",
+ name.as_deref().unwrap_or("unavailable")
+ ))
+ .collect::<Vec<_>>()
+ .join(", ")
+ );
+ }
+ store.db.execute(
+ "UPDATE desired_albums SET selected=?1 WHERE id=?2",
+ params![include, candidates[0].0],
+ )?;
+ }
+ }
+ Ok(())
+}
+
+async fn refresh(
+ store: &Store,
+ source: &mut impl Source,
+ session: &ente_core::Session,
+ cancel: &watch::Receiver<bool>,
+ report: &(dyn Fn(&str) + Sync),
+) -> Result<()> {
+ let need_favorites: bool = store.db.query_row(
+ "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND present=1)",
+ [],
+ |r| r.get(0),
+ )?;
+ let mut favorites_known = true;
+ let mut after = 0;
+ report("Refreshing selected albums and Favorites.");
+ loop {
+ ensure!(!*cancel.borrow(), Cancelled);
+ let next: Option<(i64, bool, bool)> = store
+ .db
+ .query_row(
+ "SELECT id,selected,favorites FROM desired_albums WHERE present=1 AND (selected=1 OR (favorites=1 AND ?1)) AND id>?2 ORDER BY id LIMIT 1",
+ params![need_favorites, after],
+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
+ )
+ .optional()?;
+ let Some((id, selected, favorites)) = next else {
+ break;
+ };
+ after = id;
+ let refreshed = source.refresh_album(session, id).await;
+ if refreshed.is_err() && favorites {
+ favorites_known = false;
+ }
+ let result = refreshed.and_then(|entry| {
+ store.desired_album(&entry)?;
+ let (album, documents) = entry.value?;
+ if selected {
+ store.snapshot_album(&album, &documents, session.user_id)?;
+ }
+ store
+ .db
+ .execute("UPDATE desired_albums SET ready=1 WHERE id=?1", [id])?;
+ Ok(())
+ });
+ if let Err(error) = result {
+ store.db.execute(
+ "UPDATE desired_albums SET failure=?1 WHERE id=?2",
+ params![error.to_string(), id],
+ )?;
+ if selected {
+ record_outcome(store, &format!("album:{id}"), &error)?;
+ report(&format!("album:{id}: {error:#}"));
+ }
+ if fatal(&error) {
+ return Err(error);
+ }
+ }
+ }
+ store.db.execute(
+ "UPDATE desired_albums SET ready=1 WHERE selected=1 AND present=0",
+ [],
+ )?;
+ if need_favorites && favorites_known {
+ let mut after = 0;
+ loop {
+ let ids = source.favorite_page(after)?;
+ if ids.is_empty() {
+ break;
+ }
+ let transaction = store.db.unchecked_transaction()?;
+ for id in ids {
+ after = id;
+ store.db.execute("INSERT INTO favorites VALUES(?1)", [id])?;
+ }
+ transaction.commit()?;
+ }
+ }
+ after = 0;
+ loop {
+ let id: Option<i64> = store
+ .db
+ .query_row(
+ "SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL AND id>?1 ORDER BY id LIMIT 1",
+ [after],
+ |r| r.get(0),
+ )
+ .optional()?;
+ let Some(id) = id else { break };
+ after = id;
+ let mut after_file = 0;
+ loop {
+ ensure!(!*cancel.borrow(), Cancelled);
+ let records = source.file_page(id, after_file)?;
+ if records.is_empty() {
+ break;
+ }
+ let transaction = store.db.unchecked_transaction()?;
+ for record in records {
+ after_file = record.id;
+ let file = record
+ .value
+ .and_then(|(file, documents)| FileSnapshot::new(file, &documents));
+ let failure = if !favorites_known {
+ Some("favorite state is incomplete".to_owned())
+ } else {
+ file.as_ref().err().map(ToString::to_string)
+ };
+ store.db.execute(
+ "INSERT INTO desired_files(album,file,favorited,failure) VALUES(?1,?2,?3,?4)",
+ params![
+ id,
+ record.id,
+ if favorites_known { Some(false) } else { None },
+ failure
+ ],
+ )?;
+ if failure.is_none() {
+ let file = file?;
+ store.db.execute(
+ "INSERT INTO chosen_files VALUES(?1,?2,?3) ON CONFLICT(file) DO UPDATE SET album=excluded.album,version=excluded.version WHERE excluded.version>chosen_files.version OR (excluded.version=chosen_files.version AND excluded.album<chosen_files.album)",
+ params![file.id, id, file.updated_at_micros],
+ )?;
+ }
+ }
+ transaction.commit()?;
+ }
+ }
+ store
+ .db
+ .execute(
+ "UPDATE desired_files SET favorited=EXISTS(SELECT 1 FROM favorites WHERE favorites.file=desired_files.file) WHERE favorited IS NOT NULL",
+ [],
+ )?;
+ let mut after_file = 0;
+ loop {
+ ensure!(!*cancel.borrow(), Cancelled);
+ let mut query = store
+ .db
+ .prepare("SELECT album,file FROM chosen_files WHERE file>?1 ORDER BY file LIMIT 128")?;
+ let chosen: Vec<(i64, i64)> = query
+ .query_map([after_file], |r| Ok((r.get(0)?, r.get(1)?)))?
+ .collect::<rusqlite::Result<_>>()?;
+ drop(query);
+ if chosen.is_empty() {
+ break;
+ }
+ let transaction = store.db.unchecked_transaction()?;
+ for (album, id) in chosen {
+ after_file = id;
+ let (file, documents) = source.file(album, id)?.value?;
+ store.snapshot_file(&FileSnapshot::new(file, &documents)?)?;
+ }
+ transaction.commit()?;
+ }
+ let transaction = store.db.unchecked_transaction()?;
+ store.db.execute(
+ "DELETE FROM sources WHERE file NOT IN (SELECT file FROM chosen_files)",
+ [],
+ )?;
+ store
+ .db
+ .execute(
+ "DELETE FROM album_sources WHERE id NOT IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM albums WHERE albums.id=album_sources.id) AND NOT EXISTS(SELECT 1 FROM pending WHERE album=album_sources.id)",
+ [],
+ )?;
+ transaction.commit()?;
+ let mut query = store
+ .db
+ .prepare("SELECT album,file,failure FROM desired_files WHERE failure IS NOT NULL")?;
+ let mut rows = query.query([])?;
+ while let Some(row) = rows.next()? {
+ let unit = format!("file:{}:{}", row.get::<_, i64>(0)?, row.get::<_, i64>(1)?);
+ let error = anyhow::anyhow!(row.get::<_, String>(2)?);
+ record_outcome(store, &unit, &error)?;
+ report(&format!("{unit}: {error:#}"));
+ }
+ Ok(())
+}
+
+struct CancelWorkers<'a>(&'a watch::Sender<bool>);
+
+impl Drop for CancelWorkers<'_> {
+ fn drop(&mut self) {
+ if std::thread::panicking() {
+ self.0.send_replace(true);
+ }
+ }
+}
+
+fn transfers(
+ root: &Path,
+ store: &Mutex<Store>,
+ session: &ente_core::Session,
+ cancel_send: &watch::Sender<bool>,
+ cancel: &watch::Receiver<bool>,
+ jobs: usize,
+ report: &(dyn Fn(&str) + Sync),
+) -> Result<()> {
+ let (expected, files): (i64, i64) = store::lock(store)?.db.query_row(
+ "SELECT count(*),count(DISTINCT file) FROM desired_files WHERE failure IS NULL",
+ [],
+ |r| Ok((r.get(0)?, r.get(1)?)),
+ )?;
+ report(&format!("Maintaining {expected} selected copies."));
+ let runtime = tokio::runtime::Handle::current();
+ let last_report = Mutex::new(None::<Instant>);
+ std::thread::scope(|scope| {
+ let mut workers = Vec::new();
+ for _ in 0..jobs.min(files as usize) {
+ let runtime = &runtime;
+ let last_report = &last_report;
+ workers.push(scope.spawn(move || {
+ let _cancel_on_panic = CancelWorkers(cancel_send);
+ let run = reconcile::Run { root, store, cancel, report };
+ let result = (|| {
+ loop {
+ if *cancel.borrow() {
+ break;
+ }
+ let file = {
+ let store = store::lock(store)?;
+ let transaction = store.db.unchecked_transaction()?;
+ let id: Option<i64> = store
+ .db
+ .query_row(
+ "SELECT file FROM desired_files WHERE failure IS NULL AND claimed=0 ORDER BY file LIMIT 1",
+ [],
+ |r| r.get(0),
+ )
+ .optional()?;
+ let Some(id) = id else {
+ break;
+ };
+ store
+ .db
+ .execute(
+ "UPDATE desired_files SET claimed=1 WHERE file=?1",
+ [id],
+ )?;
+ let file: FileSnapshot = store
+ .json("SELECT record FROM sources WHERE file=?1", [id])?
+ .context("missing file source snapshot")?;
+ transaction.commit()?;
+ file
+ };
+ let id = file.id;
+ let result = runtime.block_on(run.file(session, file));
+ if result.as_ref().err().is_some_and(fatal) {
+ cancel_send.send_replace(true);
+ }
+ let reported = result.or_else(|error| run.block_file(id, error));
+ let cleaned = run.cleanup(Some(id));
+ reported?;
+ cleaned?;
+ let completed = {
+ let mut last = last_report.lock().map_err(|_| anyhow::anyhow!("progress mutex poisoned"))?;
+ if last.is_none_or(|time| time.elapsed() >= Duration::from_secs(2)) {
+ *last = Some(Instant::now());
+ Some(store::lock(store)?.db.query_row(
+ "SELECT count(*) FROM desired_files WHERE completed=1",
+ [],
+ |r| r.get::<_, i64>(0),
+ )?)
+ } else {
+ None
+ }
+ };
+ if let Some(completed) = completed {
+ report(&format!("Completed {completed}/{expected} selected copies."));
+ }
+ }
+ Ok(())
+ })();
+ if result.is_err() {
+ cancel_send.send_replace(true);
+ }
+ result
+ }));
+ }
+ let mut result = Ok(());
+ for worker in workers {
+ let completed = worker
+ .join()
+ .unwrap_or_else(|_| Err(anyhow::anyhow!("export worker panicked")));
+ if result.is_ok() {
+ result = completed;
+ }
+ }
+ result
+ })
+}
+
+fn record_outcome(store: &Store, unit: &str, error: &anyhow::Error) -> Result<()> {
+ let conflict = error.downcast_ref::<Conflict>().is_some();
+ store
+ .db
+ .execute(
+ "INSERT INTO outcomes VALUES(?1,?2) ON CONFLICT(unit) DO UPDATE SET conflict=max(conflict,excluded.conflict)",
+ params![unit, conflict],
+ )?;
+ Ok(())
+}
+
+fn fatal(error: &anyhow::Error) -> bool {
+ fatal_cause(error.as_ref())
+}
+
+fn fatal_cause(cause: &(dyn std::error::Error + 'static)) -> bool {
+ if cause.is::<rusqlite::Error>()
+ || cause.is::<Cancelled>()
+ || cause
+ .downcast_ref::<ente_core::http::Error>()
+ .is_some_and(|error| error.status_code() == Some(401))
+ {
+ return true;
+ }
+ if let Some(error) = cause.downcast_ref::<std::io::Error>() {
+ return error.kind() == std::io::ErrorKind::StorageFull
+ || error.get_ref().is_some_and(|inner| fatal_cause(inner));
+ }
+ if let Some(error) = cause.downcast_ref::<ente_photos::files::Error>() {
+ use ente_photos::files::Error;
+ match error {
+ Error::Http(error) => return fatal_cause(error),
+ Error::Io(error) => return fatal_cause(error),
+ Error::Collections(error) => return fatal_cause(error),
+ _ => {}
+ }
+ }
+ if let Some(ente_collections::Error::Http(error)) = cause.downcast_ref() {
+ return fatal_cause(error);
+ }
+ if let Some(error) = cause.downcast_ref::<crate::live_photo::Error>() {
+ match error {
+ crate::live_photo::Error::Io(error) => return fatal_cause(error),
+ crate::live_photo::Error::Zip(error) => return fatal_cause(error),
+ crate::live_photo::Error::Components => {}
+ }
+ }
+ cause.source().is_some_and(fatal_cause)
+}
+
+fn summary(store: &Store, destination: &Path, prepared: bool) -> Result<Summary> {
+ let unknown: bool = !prepared
+ || store.db.query_row(
+ "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND ready=0)",
+ [],
+ |r| r.get(0),
+ )?;
+ let (files, expected, completed): (i64, i64, i64) = store.db.query_row(
+ "SELECT count(DISTINCT file),count(*),coalesce(sum(completed),0) FROM desired_files",
+ [],
+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
+ )?;
+ let (failures, conflicts): (i64, i64) = store.db.query_row(
+ "SELECT count(*),coalesce(sum(conflict),0) FROM outcomes",
+ [],
+ |r| Ok((r.get(0)?, r.get(1)?)),
+ )?;
+ let (exported, metadata_updated, renamed, retained): (i64, i64, i64, i64) = store
+ .db
+ .query_row(
+ "SELECT coalesce(sum(exported),0),coalesce(sum(metadata_updated),0),coalesce(sum(renamed),0),coalesce(sum(retained),0) FROM events",
+ [],
+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
+ )?;
+ let complete = !unknown && failures == 0 && completed == expected;
+ let result = json!({"destination":destination,"complete":complete,"files":if unknown {None}else{Some(files)},"copies":if unknown {None}else{Some(json!({"expected":expected,"completed":completed,"pending":expected-completed}))},"changes":{"exported":exported,"metadataUpdated":metadata_updated,"renamed":renamed,"retained":retained},"failures":failures,"conflicts":conflicts});
+ Ok(Summary {
+ json: result,
+ complete,
+ completed,
+ expected,
+ })
+}
+
+fn canonical_id(selector: &str) -> rusqlite::types::Value {
+ selector
+ .parse::<i64>()
+ .ok()
+ .filter(|id| id.to_string() == selector)
+ .map(rusqlite::types::Value::Integer)
+ .unwrap_or(rusqlite::types::Value::Null)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn transparent_errors_preserve_fatal_classification() {
+ for status in [401, 404] {
+ let error = ente_core::http::Error::Http {
+ status,
+ path: "/files".into(),
+ };
+ let error =
+ ente_photos::files::Error::Collections(ente_collections::Error::Http(error));
+ let error = anyhow::Error::new(error).context("download failed");
+ assert_eq!(fatal(&error), status == 401);
+ assert_eq!(
+ format!("{error:#}"),
+ format!("download failed: HTTP {status} at /files")
+ );
+ }
+ let cancelled = crate::live_photo::Error::Io(std::io::Error::other(Cancelled));
+ assert!(fatal(&anyhow::Error::new(cancelled)));
+ let full = crate::live_photo::Error::Io(std::io::ErrorKind::StorageFull.into());
+ assert!(fatal(&anyhow::Error::new(full)));
+ let zip = crate::live_photo::Error::Zip(zip::result::ZipError::Io(
+ std::io::ErrorKind::StorageFull.into(),
+ ));
+ assert!(fatal(&anyhow::Error::new(zip)));
+ let sqlite = rusqlite::Error::InvalidQuery;
+ assert!(fatal(&anyhow::Error::new(sqlite)));
+ let local = ente_photos::files::Error::Io(std::io::ErrorKind::PermissionDenied.into());
+ assert!(!fatal(&anyhow::Error::new(local)));
+ }
+}
diff --git a/rust/crates/photos/src/live_photo.rs b/rust/crates/photos-export/src/live_photo.rs
similarity index 98%
rename from rust/crates/photos/src/live_photo.rs
rename to rust/crates/photos-export/src/live_photo.rs
index b15f93a8f4..daa9832822 100644
--- a/rust/crates/photos/src/live_photo.rs
+++ b/rust/crates/photos-export/src/live_photo.rs
@@ -1,6 +1,6 @@
use std::io::{Read, Seek, Write};
-use crate::export::Role;
+use crate::metadata::Role;
#[derive(Debug, thiserror::Error)]
pub enum Error {
diff --git a/rust/crates/photos-export/src/metadata.rs b/rust/crates/photos-export/src/metadata.rs
new file mode 100644
index 0000000000..884d828f7a
--- /dev/null
+++ b/rust/crates/photos-export/src/metadata.rs
@@ -0,0 +1,314 @@
+use chrono::{DateTime, SecondsFormat};
+use ente_core::{
+ b64,
+ crypto::{Key, kdf},
+};
+use serde::{Deserialize, Serialize};
+use serde_json::{Value, json};
+
+use ente_photos::{
+ collections::{Collection, Visibility},
+ files::{File, Kind},
+ source::{self, Documents, MetadataError},
+};
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct Root {
+ pub format: String,
+ pub version: u32,
+ pub source_verifier: String,
+}
+
+impl Root {
+ pub fn new(master_key: &Key) -> ente_core::crypto::Result<Self> {
+ Ok(Self {
+ format: "ente-photos-export".into(),
+ version: 1,
+ source_verifier: b64::encode(&kdf::derive_subkey(master_key, 32, 1, b"photoexp")?),
+ })
+ }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Role {
+ Original,
+ Image,
+ Video,
+}
+
+impl Role {
+ pub fn name(&self) -> &'static str {
+ match self {
+ Self::Original => "original",
+ Self::Image => "image",
+ Self::Video => "video",
+ }
+ }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Component {
+ pub role: Role,
+ pub path: String,
+ pub size: u64,
+ pub hash: String,
+}
+
+pub fn album(
+ album: &Collection,
+ documents: &Documents,
+ user_id: i64,
+) -> Result<(Value, Vec<String>), MetadataError> {
+ let private: AlbumPrivate = source::optional(
+ documents.private.as_deref(),
+ "invalid album private metadata",
+ )?;
+ let shared: AlbumShared =
+ source::optional(documents.shared.as_deref(), "invalid album sharee metadata")?;
+ let public: AlbumPublic =
+ source::optional(documents.public.as_deref(), "invalid album public metadata")?;
+ let owned = album.owner_id == user_id;
+ let mut metadata = json!({
+ "albumID": album.id.to_string(), "type": album.kind.name(), "visibility": album.visibility.name(),
+ });
+ optional(
+ &mut metadata,
+ "displayOrder",
+ if owned { private.order } else { shared.order },
+ );
+ optional(
+ &mut metadata,
+ "muted",
+ (!owned).then_some(shared.mute).flatten(),
+ );
+ optional(
+ &mut metadata,
+ "sortOrder",
+ public
+ .asc
+ .map(|asc| if asc { "ascending" } else { "descending" }),
+ );
+ optional(&mut metadata, "description", public.caption.as_deref());
+ optional(
+ &mut metadata,
+ "coverFileID",
+ public.cover_id.map(|id| id.to_string()),
+ );
+ optional(&mut metadata, "layout", public.layout.as_deref());
+ let mut warnings = Vec::new();
+ unfamiliar(
+ documents.private.as_deref(),
+ &["visibility", "subType", "order"],
+ &mut warnings,
+ );
+ unfamiliar(
+ documents.shared.as_deref(),
+ &["visibility", "mute", "order"],
+ &mut warnings,
+ );
+ unfamiliar(
+ documents.public.as_deref(),
+ &["asc", "coverID", "layout", "caption"],
+ &mut warnings,
+ );
+ Ok((json!({"title": album.name, "ente": metadata}), warnings))
+}
+
+pub fn file(file: &File, documents: &Documents) -> Result<(Value, Vec<String>), MetadataError> {
+ let public: FilePublic =
+ source::optional(documents.public.as_deref(), "invalid public file metadata")?;
+ let mut metadata = json!({
+ "fileID": file.id.to_string(),
+ "type": if file.kind == Kind::LivePhoto { "livePhoto" } else { file.kind.name() },
+ "name": file.name,
+ "creationTime": timestamp(file.created_at_micros)?,
+ });
+ optional(
+ &mut metadata,
+ "modificationTime",
+ file.modified_at_micros.map(timestamp).transpose()?,
+ );
+ optional(
+ &mut metadata,
+ "location",
+ file.location
+ .as_ref()
+ .map(|l| json!({"latitude": l.latitude,"longitude": l.longitude})),
+ );
+ optional(&mut metadata, "localDateTime", file.date_time.as_deref());
+ optional(&mut metadata, "utcOffset", file.offset_time.as_deref());
+ optional(&mut metadata, "width", file.width);
+ optional(&mut metadata, "height", file.height);
+ optional(&mut metadata, "durationSeconds", file.duration_seconds);
+ optional(
+ &mut metadata,
+ "visibility",
+ file.visibility.map(Visibility::name),
+ );
+ optional(&mut metadata, "cameraMake", public.camera_make.as_deref());
+ optional(&mut metadata, "cameraModel", public.camera_model.as_deref());
+ optional(
+ &mut metadata,
+ "uploaderName",
+ public.uploader_name.as_deref(),
+ );
+ optional(
+ &mut metadata,
+ "panorama",
+ if file.kind == Kind::Image {
+ public.media_type.map(|value| value & 1 != 0)
+ } else {
+ None
+ },
+ );
+ optional(&mut metadata, "motionPhotoVideoOffset", public.mvi);
+ let taken = common_time(file.created_at_micros)?;
+ let mut record = json!({
+ "photoTakenTime": taken,
+ "creationTime": taken,
+ "ente": metadata,
+ });
+ optional(
+ &mut record,
+ "modificationTime",
+ file.modified_at_micros.map(common_time).transpose()?,
+ );
+ optional(&mut record, "description", file.caption.as_deref());
+ optional(
+ &mut record,
+ "geoData",
+ file.location
+ .as_ref()
+ .map(|l| json!({"latitude":l.latitude,"longitude":l.longitude})),
+ );
+ let mut warnings = Vec::new();
+ unfamiliar(
+ Some(&documents.original),
+ &[
+ "title",
+ "fileType",
+ "creationTime",
+ "modificationTime",
+ "latitude",
+ "longitude",
+ "hash",
+ "imageHash",
+ "videoHash",
+ "duration",
+ "hasStaticThumbnail",
+ "localID",
+ "deviceFolder",
+ "subType",
+ "version",
+ "exif",
+ ],
+ &mut warnings,
+ );
+ unfamiliar(
+ documents.public.as_deref(),
+ &[
+ "editedName",
+ "editedTime",
+ "dateTime",
+ "offsetTime",
+ "caption",
+ "lat",
+ "long",
+ "w",
+ "h",
+ "sv",
+ "cameraMake",
+ "cameraModel",
+ "uploaderName",
+ "mediaType",
+ "mvi",
+ "noThumb",
+ ],
+ &mut warnings,
+ );
+ unfamiliar(documents.private.as_deref(), &["visibility"], &mut warnings);
+ Ok((record, warnings))
+}
+
+fn optional(object: &mut Value, name: &str, value: Option<impl Into<Value>>) {
+ if let Some(value) = value {
+ object[name] = value.into();
+ }
+}
+
+fn time(micros: i64) -> Result<DateTime<chrono::Utc>, MetadataError> {
+ DateTime::from_timestamp_micros(micros)
+ .ok_or(MetadataError("timestamp is outside the supported range"))
+}
+
+fn timestamp(micros: i64) -> Result<String, MetadataError> {
+ Ok(time(micros)?.to_rfc3339_opts(SecondsFormat::Micros, true))
+}
+
+fn common_time(micros: i64) -> Result<Value, MetadataError> {
+ let time = time(micros)?;
+ Ok(
+ json!({"timestamp":micros.div_euclid(1_000_000).to_string(),"formatted":time.format("%b %-d, %Y, %-I:%M:%S %p UTC").to_string()}),
+ )
+}
+
+pub fn publish(
+ template: &Value,
+ components: &[Component],
+ component: usize,
+ favorited: bool,
+) -> Result<Value, MetadataError> {
+ let selected = components
+ .get(component)
+ .ok_or(MetadataError("missing media component"))?;
+ let mut record = template.clone();
+ record["title"] = json!(selected.path);
+ record["favorited"] = json!(favorited);
+ record["ente"]["component"] = json!(selected.role);
+ record["ente"]["components"] = json!(components);
+ Ok(record)
+}
+
+#[derive(Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct FilePublic {
+ camera_make: Option<String>,
+ camera_model: Option<String>,
+ uploader_name: Option<String>,
+ media_type: Option<u64>,
+ mvi: Option<u64>,
+}
+
+#[derive(Default, Deserialize)]
+struct AlbumPrivate {
+ order: Option<i64>,
+}
+
+#[derive(Default, Deserialize)]
+struct AlbumShared {
+ order: Option<i64>,
+ mute: Option<bool>,
+}
+
+#[derive(Default, Deserialize)]
+struct AlbumPublic {
+ asc: Option<bool>,
+ #[serde(rename = "coverID")]
+ cover_id: Option<i64>,
+ layout: Option<String>,
+ caption: Option<String>,
+}
+
+fn unfamiliar(bytes: Option<&[u8]>, known: &[&str], warnings: &mut Vec<String>) {
+ if let Some(bytes) = bytes
+ && let Ok(Value::Object(fields)) = serde_json::from_slice(bytes)
+ {
+ for key in fields.keys() {
+ if !known.contains(&key.as_str()) {
+ warnings.push(key.clone());
+ }
+ }
+ }
+}
diff --git a/rust/apps/cli-next/src/export/names.rs b/rust/crates/photos-export/src/names.rs
similarity index 98%
rename from rust/apps/cli-next/src/export/names.rs
rename to rust/crates/photos-export/src/names.rs
index 4feb7d3a8a..dd23cf0a9d 100644
--- a/rust/apps/cli-next/src/export/names.rs
+++ b/rust/crates/photos-export/src/names.rs
@@ -131,13 +131,13 @@ pub fn available(
) -> Result<bool> {
for name in names {
let occupied: bool = if kind == "album" {
- store.db.connection().query_row(
+ store.db.query_row(
"SELECT EXISTS(SELECT 1 FROM albums WHERE parent=?1 AND folded=?2 AND key<>?3 UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND folded1=?2 AND kind='album' AND owner<>?3)",
params![folder, folded(name), owner],
|r| r.get(0),
)?
} else {
- store.db.connection().query_row(
+ store.db.query_row(
"SELECT EXISTS(SELECT 1 FROM components c JOIN placements p ON p.id=c.placement WHERE c.folder=?1 AND c.folded=?2 AND c.placement<>?3 UNION ALL SELECT 1 FROM components c JOIN placements p ON p.id=c.placement WHERE c.folder=?1 AND c.stem=?4 AND c.placement<>?3 AND NOT(p.kind=?5 AND p.kind IN ('image','video')) UNION ALL SELECT 1 FROM json_records j JOIN placements p ON p.id=j.owner WHERE j.folder=?1 AND j.folded=?2 AND j.owner<>?3 UNION ALL SELECT 1 FROM json_records j JOIN placements p ON p.id=j.owner WHERE j.folder=?1 AND j.stem=?4 AND j.owner<>?3 AND NOT(p.kind=?5 AND p.kind IN ('image','video')) UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND folded1=?2 AND owner<>?3 UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND folded2=?2 AND owner<>?3 UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND stem1=?4 AND owner<>?3 AND NOT(kind=?5 AND kind IN ('image','video')) UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND stem2=?4 AND owner<>?3 AND NOT(kind=?5 AND kind IN ('image','video')))",
params![
folder,
diff --git a/rust/apps/cli-next/src/export/reconcile.rs b/rust/crates/photos-export/src/reconcile.rs
similarity index 95%
rename from rust/apps/cli-next/src/export/reconcile.rs
rename to rust/crates/photos-export/src/reconcile.rs
index 249a0c6b28..aa51f92e52 100644
--- a/rust/apps/cli-next/src/export/reconcile.rs
+++ b/rust/crates/photos-export/src/reconcile.rs
@@ -6,8 +6,8 @@ use std::{
sync::Mutex,
};
+use crate::{metadata::Role, snapshot::FileSnapshot};
use anyhow::{Context, Result, ensure};
-use ente_photos::{export::Role, files};
use rusqlite::{OptionalExtension, params};
use serde_json::Value;
use tokio::sync::watch;
@@ -26,6 +26,7 @@ pub struct Run<'a> {
pub root: &'a Path,
pub store: &'a Mutex<Store>,
pub cancel: &'a watch::Receiver<bool>,
+ pub report: &'a (dyn Fn(&str) + Sync),
}
impl Run<'_> {
@@ -80,7 +81,6 @@ impl Run<'_> {
fs::remove(&self.path(location)?)?;
store::lock(self.store)?
.db
- .connection()
.execute("DELETE FROM temporaries WHERE name=?1", [&location.name])?;
Ok(())
}
@@ -90,7 +90,7 @@ impl Run<'_> {
let store = store::lock(self.store)?;
super::record_outcome(&store, unit, error)?;
}
- eprintln!("{unit}: {error:#}");
+ (self.report)(&format!("{unit}: {error:#}"));
Ok(())
}
@@ -101,7 +101,7 @@ impl Run<'_> {
);
self.report(&unit, &error)?;
if file.is_none() {
- store::lock(self.store)?.db.connection().execute(
+ store::lock(self.store)?.db.execute(
"UPDATE desired_albums SET failure=?1 WHERE id=?2",
params![error.to_string(), album],
)?;
@@ -109,7 +109,6 @@ impl Run<'_> {
loop {
let next: Option<i64> = store::lock(self.store)?
.db
- .connection()
.query_row(
"SELECT file FROM desired_files WHERE album=?1 AND file>?2 ORDER BY file LIMIT 1",
params![album, after],
@@ -121,7 +120,7 @@ impl Run<'_> {
self.report(&format!("file:{album}:{file}"), &error)?;
}
}
- store::lock(self.store)?.db.connection().execute(
+ store::lock(self.store)?.db.execute(
"UPDATE desired_files SET failure=?1 WHERE album=?2 AND (?3 IS NULL OR file=?3)",
params![error.to_string(), album, file],
)?;
@@ -136,7 +135,6 @@ impl Run<'_> {
loop {
let next: Option<i64> = store::lock(self.store)?
.db
- .connection()
.query_row(
"SELECT album FROM desired_files WHERE file=?1 AND album>?2 ORDER BY album LIMIT 1",
params![file, after],
@@ -147,7 +145,7 @@ impl Run<'_> {
after = album;
self.report(&format!("file:{album}:{file}"), &error)?;
}
- store::lock(self.store)?.db.connection().execute(
+ store::lock(self.store)?.db.execute(
"UPDATE desired_files SET failure=?1 WHERE file=?2",
params![error.to_string(), file],
)?;
@@ -165,7 +163,7 @@ impl Run<'_> {
) -> Result<()> {
self.check_cancel()?;
let store = store::lock(self.store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
let published = !store.components(&target.owner)?.is_empty();
for (role, value) in snapshots {
let previous = store.json_record(&target.owner, role.as_ref())?;
@@ -227,7 +225,7 @@ impl Run<'_> {
None
};
let store = store::lock(self.store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
match output {
Output::Media {
role,
@@ -272,7 +270,6 @@ impl Run<'_> {
if let Some(Action::Publish { temporary, .. }) = &target.action {
store
.db
- .connection()
.execute("DELETE FROM temporaries WHERE name=?1", [&temporary.name])?;
}
target.action = None;
@@ -302,7 +299,6 @@ impl Run<'_> {
} else if !directory_exists(&destination)? {
let unrepairable: bool = store::lock(self.store)?
.db
- .connection()
.query_row(
"SELECT EXISTS(SELECT 1 FROM placements p WHERE p.album=?1 AND p.retained=0 AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file AND f.failure IS NULL))",
[target.album],
@@ -334,7 +330,7 @@ impl Run<'_> {
}
}
let store = store::lock(self.store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
store.save_album(&Album {
key: target.owner.clone(),
id: target.album,
@@ -410,7 +406,7 @@ impl Run<'_> {
}
fs::sync_move(&from, &to)?;
let store = store::lock(self.store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
let mut component = store
.components(&target.owner)?
.into_iter()
@@ -427,7 +423,7 @@ impl Run<'_> {
Action::Remove { location, role } => {
fs::remove(&self.path(&location)?)?;
let store = store::lock(self.store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
let mut record = store
.json_record(&target.owner, role.as_ref())?
.context("missing removed JSON record")?;
@@ -618,7 +614,6 @@ impl Run<'_> {
let store = store::lock(self.store)?;
store
.db
- .connection()
.query_row(
"SELECT p.owner,p.album,p.file FROM pending p JOIN desired_albums a ON a.id=p.album WHERE p.owner>?1 AND (p.file IS NULL)=?2 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file AND f.failure IS NOT NULL) ORDER BY p.owner LIMIT 1",
params![after, albums],
@@ -646,17 +641,15 @@ impl Run<'_> {
}
{
let store = store::lock(self.store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
store
.db
- .connection()
.execute(
"DELETE FROM pending WHERE file IS NULL AND action='null' AND album IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL) AND ((retained=0 AND album IN (SELECT id FROM desired_albums WHERE present=0)) OR (retained=1 AND NOT EXISTS(SELECT 1 FROM albums WHERE key=pending.owner) AND NOT EXISTS(SELECT 1 FROM placements p WHERE p.album=pending.album AND p.retained=0 AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file))))",
[],
)?;
store
.db
- .connection()
.execute(
"DELETE FROM json_records WHERE folder IS NULL AND NOT EXISTS(SELECT 1 FROM albums WHERE key=json_records.owner) AND NOT EXISTS(SELECT 1 FROM components WHERE placement=json_records.owner) AND NOT EXISTS(SELECT 1 FROM pending WHERE owner=json_records.owner)",
[],
@@ -673,7 +666,6 @@ impl Run<'_> {
let store = store::lock(self.store)?;
store
.db
- .connection()
.query_row(
"SELECT t.name,t.folder,t.album,t.file FROM temporaries t JOIN desired_albums a ON a.id=t.album WHERE t.name>?1 AND (?2 IS NULL OR t.file=?2) AND a.selected=1 ORDER BY t.name LIMIT 1",
params![after, file],
@@ -699,7 +691,6 @@ impl Run<'_> {
let store = store::lock(self.store)?;
let owner: Option<String> = store
.db
- .connection()
.query_row(
"SELECT owner FROM pending WHERE json_extract(action,'$.Publish.temporary.name')=?1",
[&temporary.location.name],
@@ -712,7 +703,7 @@ impl Run<'_> {
.flatten()
};
if let Some(mut pending) = pending {
- let selected: bool = store::lock(self.store)?.db.connection().query_row(
+ let selected: bool = store::lock(self.store)?.db.query_row(
"SELECT selected FROM desired_albums WHERE id=?1",
[pending.album],
|r| r.get(0),
@@ -728,7 +719,7 @@ impl Run<'_> {
})();
if let Err(error) = result {
if let Some(file) = temporary.file {
- let selected: bool = store::lock(self.store)?.db.connection().query_row(
+ let selected: bool = store::lock(self.store)?.db.query_row(
"SELECT EXISTS(SELECT 1 FROM desired_files WHERE file=?1)",
[file],
|r| r.get(0),
@@ -757,7 +748,7 @@ impl Run<'_> {
return Ok(pending);
}
}
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
let parent = if retained { "Trash" } else { "" };
let names = if let Some(existing) = &existing
&& (retained || existing.name == name)
@@ -846,7 +837,6 @@ impl Run<'_> {
let store = store::lock(self.store)?;
store
.db
- .connection()
.execute(
"INSERT INTO events(placement,renamed) SELECT id,1 FROM placements WHERE album=?1 AND retained=0 ON CONFLICT(placement) DO UPDATE SET renamed=1",
[album],
@@ -873,7 +863,6 @@ impl Run<'_> {
loop {
let source: Option<(i64, String)> = store::lock(self.store)?
.db
- .connection()
.query_row(
"SELECT a.id,s.record FROM desired_albums a JOIN album_sources s ON s.id=a.id WHERE a.id>?1 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND a.present=1 ORDER BY a.id LIMIT 1",
[after],
@@ -886,9 +875,9 @@ impl Run<'_> {
after = id;
let source: AlbumSource = serde_json::from_str(&source)?;
for field in &source.warnings {
- eprintln!(
+ (self.report)(&format!(
"album {id}: unfamiliar metadata field {field:?}; exporting supported fields"
- );
+ ));
}
if let Err(error) = self.maintain_album(id, &source.name, false, source.metadata) {
self.block(id, None, error)?;
@@ -902,7 +891,7 @@ impl Run<'_> {
fn file_target(
&self,
album: &Album,
- file: &files::File,
+ file: &FileSnapshot,
prepared: &transfer::Prepared,
) -> Result<Pending> {
let store = store::lock(self.store)?;
@@ -949,7 +938,7 @@ impl Run<'_> {
let same_folder = previous.folder == album.key;
if same_folder
&& previous.name == file.name
- && previous.kind == file.kind.name()
+ && previous.kind == file.kind.as_str()
&& compatible(&previous.names)
{
return Ok(previous);
@@ -960,12 +949,12 @@ impl Run<'_> {
.map(|component| component.location.name.clone())
.collect();
let unchanged = existing.as_ref().is_some_and(|placement| {
- placement.name == file.name && placement.kind == file.kind.name()
+ placement.name == file.name && placement.kind == file.kind.as_str()
}) && components
.iter()
.all(|component| component.location.folder == album.key)
&& compatible(&existing_names);
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
let names = if unchanged {
existing_names
} else {
@@ -973,7 +962,7 @@ impl Run<'_> {
&store,
&album.key,
&file.name,
- file.kind.name(),
+ file.kind.as_str(),
if roles.len() == 2 {
Some((&extensions[0], &extensions[1]))
} else {
@@ -988,7 +977,7 @@ impl Run<'_> {
file: Some(file.id),
retained: false,
name: file.name.clone(),
- kind: file.kind.name().into(),
+ kind: file.kind.clone(),
folder: album.key.clone(),
names,
action: None,
@@ -1002,13 +991,12 @@ impl Run<'_> {
Ok(target)
}
- pub async fn file(&self, session: &ente_core::Session, file: files::File) -> Result<()> {
+ pub async fn file(&self, session: &ente_core::Session, file: FileSnapshot) -> Result<()> {
let mut prepared = transfer::prepare(self, session, &file).await?;
let mut after = 0;
loop {
let album: Option<i64> = store::lock(self.store)?
.db
- .connection()
.query_row(
"SELECT album FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",
params![file.id, after],
@@ -1039,17 +1027,16 @@ impl Run<'_> {
}
}
for field in &file.warnings {
- eprintln!(
+ (self.report)(&format!(
"file {}: unfamiliar metadata field {field:?}; exporting supported fields",
file.id
- );
+ ));
}
after = 0;
loop {
self.check_cancel()?;
let next: Option<(i64, bool)> = store::lock(self.store)?
.db
- .connection()
.query_row(
"SELECT album,favorited FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",
params![file.id, after],
@@ -1089,7 +1076,7 @@ impl Run<'_> {
fn placement(
&self,
- file: &files::File,
+ file: &FileSnapshot,
prepared: &mut transfer::Prepared,
album_id: i64,
favorited: bool,
@@ -1128,7 +1115,7 @@ impl Run<'_> {
.context("missing completed original")?;
(component.size, component.hash.clone())
};
- Ok(ente_photos::export::Component {
+ Ok(crate::metadata::Component {
role,
path: name.clone(),
size,
@@ -1142,7 +1129,7 @@ impl Run<'_> {
.map(|(index, component)| {
Ok((
Some(component.role.clone()),
- ente_photos::export::file(file, &intended, index, favorited)?,
+ crate::metadata::publish(&file.metadata, &intended, index, favorited)?,
))
})
.collect::<Result<_>>()?;
@@ -1185,7 +1172,7 @@ impl Run<'_> {
let changed = self.json_changed(previous.as_ref(), &location, &json)?;
sidecars.push((location, json, changed, previous));
}
- let intended_time = file.modified_at_micros.unwrap_or(file.created_at_micros);
+ let intended_time = file.intended_time;
let mut wrote_media = false;
let mut renamed = false;
for component in &intended {
@@ -1288,7 +1275,7 @@ impl Run<'_> {
}
}
let store = store::lock(self.store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
if existing
.as_ref()
.is_some_and(|previous| previous.name != target.name || previous.kind != target.kind)
@@ -1296,7 +1283,7 @@ impl Run<'_> {
store.save_placement(&target.placement()?)?;
}
store.clear_pending(&target.owner)?;
- store.db.connection().execute(
+ store.db.execute(
"UPDATE desired_files SET completed=1 WHERE album=?1 AND file=?2",
params![album_id, file.id],
)?;
@@ -1317,7 +1304,6 @@ impl Run<'_> {
fn event(store: &Store, placement: &str, name: &str) -> Result<()> {
store
.db
- .connection()
.execute(
&format!(
"INSERT INTO events(placement,{name}) VALUES(?1,1) ON CONFLICT(placement) DO UPDATE SET {name}=1",
@@ -1367,7 +1353,7 @@ impl Run<'_> {
let metadata = store::lock(self.store)?
.json_record(&placement.id, Some(&template.role))?
.context("missing retained metadata snapshot")?;
- let layout: Vec<ente_photos::export::Component> =
+ let layout: Vec<crate::metadata::Component> =
serde_json::from_value(metadata.value["ente"]["components"].clone())?;
let extensions = if placement.kind == "livephoto" {
Some((
@@ -1395,7 +1381,7 @@ impl Run<'_> {
};
let mut target = {
let store = store::lock(self.store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
let names = names::allocate(
&store,
&album.key,
@@ -1445,7 +1431,7 @@ impl Run<'_> {
};
let mut value = previous.value;
value["title"] = Value::String(component.location.name.clone());
- let mut layout: Vec<ente_photos::export::Component> =
+ let mut layout: Vec<crate::metadata::Component> =
serde_json::from_value(value["ente"]["components"].clone())?;
for identity in &mut layout {
identity.path = target_component(target, &identity.role)?.into();
@@ -1466,7 +1452,7 @@ impl Run<'_> {
)?;
}
let store = store::lock(self.store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
store.save_placement(&target.placement()?)?;
store.clear_pending(&target.owner)?;
event(&store, &target.owner, "retained")?;
@@ -1482,7 +1468,6 @@ impl Run<'_> {
let store = store::lock(self.store)?;
store
.db
- .connection()
.query_row(
"SELECT p.id,p.album,p.file,p.retained,p.name,p.kind FROM placements p JOIN desired_albums a ON a.id=p.album WHERE p.id>?1 AND p.retained=0 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file) AND NOT EXISTS(SELECT 1 FROM outcomes WHERE unit='file:'||p.album||':'||p.file) ORDER BY p.id LIMIT 1",
[&after],
@@ -1501,17 +1486,15 @@ impl Run<'_> {
}
{
let store = store::lock(self.store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
store
.db
- .connection()
.execute(
"DELETE FROM pending WHERE file IS NOT NULL AND action='null' AND album IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM placements WHERE id=pending.owner) AND NOT EXISTS(SELECT 1 FROM desired_files WHERE album=pending.album AND file=pending.file)",
[],
)?;
store
.db
- .connection()
.execute(
"DELETE FROM json_records WHERE folder IS NULL AND NOT EXISTS(SELECT 1 FROM albums WHERE key=json_records.owner) AND NOT EXISTS(SELECT 1 FROM components WHERE placement=json_records.owner) AND NOT EXISTS(SELECT 1 FROM pending WHERE owner=json_records.owner)",
[],
@@ -1523,7 +1506,6 @@ impl Run<'_> {
self.check_cancel()?;
let id: Option<i64> = store::lock(self.store)?
.db
- .connection()
.query_row(
"SELECT id FROM desired_albums WHERE id>?1 AND selected=1 AND ready=1 AND failure IS NULL AND present=0 ORDER BY id LIMIT 1",
[after],
@@ -1548,7 +1530,7 @@ impl Run<'_> {
store.clear_pending(&format!("active:{id}"))?;
return Ok(());
};
- let incomplete: bool = store::lock(self.store)?.db.connection().query_row(
+ let incomplete: bool = store::lock(self.store)?.db.query_row(
"SELECT EXISTS(SELECT 1 FROM placements WHERE album=?1 AND retained=0) OR EXISTS(SELECT 1 FROM pending WHERE album=?1 AND file IS NOT NULL)",
[id],
|r| r.get(0),
@@ -1578,10 +1560,9 @@ impl Run<'_> {
}
}
let store = store::lock(self.store)?;
- let transaction = store.db.connection().unchecked_transaction()?;
+ let transaction = store.db.unchecked_transaction()?;
store
.db
- .connection()
.execute(
"DELETE FROM albums WHERE key=?1 AND NOT EXISTS(SELECT 1 FROM temporaries WHERE folder=?1)",
[&active.key],
diff --git a/rust/crates/photos-export/src/snapshot.rs b/rust/crates/photos-export/src/snapshot.rs
new file mode 100644
index 0000000000..3ae05e18b2
--- /dev/null
+++ b/rust/crates/photos-export/src/snapshot.rs
@@ -0,0 +1,75 @@
+use anyhow::Result;
+use ente_core::crypto::{Header, Key};
+use ente_photos::{files::File, source::Documents};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use crate::metadata;
+
+#[derive(Serialize, Deserialize)]
+pub(crate) struct FileSnapshot {
+ pub id: i64,
+ pub updated_at_micros: i64,
+ pub name: String,
+ pub kind: String,
+ pub hash: Option<String>,
+ #[serde(with = "key")]
+ pub key: Key,
+ #[serde(with = "header")]
+ pub header: Header,
+ pub intended_time: i64,
+ pub metadata: Value,
+ pub warnings: Vec<String>,
+}
+
+impl FileSnapshot {
+ pub fn new(file: File, documents: &Documents) -> Result<Self> {
+ let (metadata, warnings) = metadata::file(&file, documents)?;
+ Ok(Self {
+ id: file.id,
+ updated_at_micros: file.updated_at_micros,
+ name: file.name,
+ kind: file.kind.name().into(),
+ hash: file.hash,
+ key: file.key,
+ header: file.header,
+ intended_time: file.modified_at_micros.unwrap_or(file.created_at_micros),
+ metadata,
+ warnings,
+ })
+ }
+}
+
+mod key {
+ use ente_core::crypto::Key;
+ use serde::{Deserialize, Deserializer, Serializer, de::Error};
+
+ pub fn serialize<S: Serializer>(key: &Key, serializer: S) -> Result<S::Ok, S::Error> {
+ serializer.serialize_str(&ente_core::b64::encode(key.as_bytes()))
+ }
+
+ pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Key, D::Error> {
+ Key::try_from_slice(
+ &ente_core::b64::decode(&String::deserialize(deserializer)?)
+ .map_err(D::Error::custom)?,
+ )
+ .map_err(D::Error::custom)
+ }
+}
+
+mod header {
+ use ente_core::crypto::Header;
+ use serde::{Deserialize, Deserializer, Serializer, de::Error};
+
+ pub fn serialize<S: Serializer>(header: &Header, serializer: S) -> Result<S::Ok, S::Error> {
+ serializer.serialize_str(&ente_core::b64::encode(header.as_bytes()))
+ }
+
+ pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Header, D::Error> {
+ Header::try_from_slice(
+ &ente_core::b64::decode(&String::deserialize(deserializer)?)
+ .map_err(D::Error::custom)?,
+ )
+ .map_err(D::Error::custom)
+ }
+}
diff --git a/rust/crates/photos-export/src/source.rs b/rust/crates/photos-export/src/source.rs
new file mode 100644
index 0000000000..9170faec91
--- /dev/null
+++ b/rust/crates/photos-export/src/source.rs
@@ -0,0 +1,30 @@
+use std::future::Future;
+
+use anyhow::Result;
+use ente_core::Session;
+use ente_photos::{collections::Collection, files::File, source::Documents};
+
+pub struct CollectionEntry {
+ pub id: i64,
+ pub name: Option<String>,
+ pub favorites: bool,
+ pub value: Result<(Collection, Documents)>,
+}
+
+pub struct FileEntry {
+ pub id: i64,
+ pub value: Result<(File, Documents)>,
+}
+
+pub trait Source {
+ fn refresh_catalog(&mut self, session: &Session) -> impl Future<Output = Result<()>>;
+ fn catalog_page(&self, after: i64) -> Result<Vec<CollectionEntry>>;
+ fn refresh_album(
+ &mut self,
+ session: &Session,
+ id: i64,
+ ) -> impl Future<Output = Result<CollectionEntry>>;
+ fn file_page(&self, album: i64, after: i64) -> Result<Vec<FileEntry>>;
+ fn favorite_page(&self, after: i64) -> Result<Vec<i64>>;
+ fn file(&self, album: i64, id: i64) -> Result<FileEntry>;
+}
diff --git a/rust/apps/cli-next/src/export/store.rs b/rust/crates/photos-export/src/store.rs
similarity index 89%
rename from rust/apps/cli-next/src/export/store.rs
rename to rust/crates/photos-export/src/store.rs
index ba069a4f91..e05d491efc 100644
--- a/rust/apps/cli-next/src/export/store.rs
+++ b/rust/crates/photos-export/src/store.rs
@@ -1,25 +1,20 @@
use std::{
- path::{Path, PathBuf},
+ path::Path,
sync::{Mutex, MutexGuard},
time::{SystemTime, UNIX_EPOCH},
};
use anyhow::{Context, Result, ensure};
-use ente_core::crypto::hash;
-use ente_photos::{collections::Collection, export::Role, files::File};
-use rusqlite::{OptionalExtension, Row, params};
+use ente_photos::{collections::Collection, source::Documents};
+use rusqlite::{Connection, OptionalExtension, Row, TransactionBehavior, params};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::Value;
use super::names;
-use crate::{
- db, home,
- replica::{AlbumRecord, read_json},
- vault::DbKey,
-};
+use crate::{AdoptionRequired, CollectionEntry, metadata::Role, snapshot::FileSnapshot};
pub struct Store {
- pub db: crate::core_db::Db,
+ pub db: Connection,
}
pub fn lock(store: &Mutex<Store>) -> Result<MutexGuard<'_, Store>> {
@@ -107,8 +102,8 @@ impl Component {
})
}
- pub fn portable(&self) -> ente_photos::export::Component {
- ente_photos::export::Component {
+ pub fn portable(&self) -> crate::metadata::Component {
+ crate::metadata::Component {
role: self.role.clone(),
path: self.location.name.clone(),
size: self.size,
@@ -311,24 +306,26 @@ pub enum Action {
}
impl Store {
- pub fn path(account_home: &Path, destination: &Path) -> Result<PathBuf> {
- let bytes = destination.as_os_str().as_encoded_bytes();
- let name: String = hash::hash(bytes, Some(32), None)?
- .iter()
- .map(|b| format!("{b:02x}"))
- .collect();
- Ok(account_home.join("exports").join(format!("{name}.db")))
- }
-
- pub fn open(path: &Path, key: &DbKey, destination: &Path, create: bool) -> Result<Self> {
- if create {
- home::create(path.parent().context("export DB has no parent")?)?;
- }
- let mut db = crate::core_db::Db::new(db::connect(path, key, create)?);
- if create {
- db.migrate(&[SCHEMA])?;
+ pub fn open(mut db: Connection, destination: &Path) -> Result<Self> {
+ db.busy_timeout(std::time::Duration::from_secs(5))?;
+ db.execute_batch(
+ "PRAGMA foreign_keys = ON; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = WAL;",
+ )?;
+ let transaction = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
+ let current: i64 =
+ transaction.pragma_query_value(None, "user_version", |row| row.get(0))?;
+ ensure!(
+ current <= 1,
+ "database version {current} is newer than this build supports (1)"
+ );
+ if current == 0 {
+ transaction.execute_batch(SCHEMA)?;
+ transaction.pragma_update(None, "user_version", 1)?;
+ transaction.commit()?;
+ } else {
+ drop(transaction);
}
- let connection = db.connection();
+ let connection = &db;
let existing: Option<Vec<u8>> = connection
.query_row("SELECT destination FROM export WHERE id=1", [], |r| {
r.get(0)
@@ -337,7 +334,7 @@ impl Store {
if let Some(existing) = existing {
ensure!(
existing == destination.as_os_str().as_encoded_bytes(),
- "destination association does not match; use --adopt"
+ AdoptionRequired("destination association does not match")
);
} else {
connection.execute(
@@ -351,7 +348,6 @@ impl Store {
pub fn album(&self, id: i64, retained: bool) -> Result<Option<Album>> {
Ok(self
.db
- .connection()
.query_row(
"SELECT key,id,retained,name,path FROM albums WHERE id=?1 AND retained=?2 ORDER BY rowid LIMIT 1",
params![id, retained],
@@ -361,7 +357,7 @@ impl Store {
}
pub fn folder(&self, key: &str) -> Result<Album> {
- Ok(self.db.connection().query_row(
+ Ok(self.db.query_row(
"SELECT key,id,retained,name,path FROM albums WHERE key=?1",
[key],
Album::read,
@@ -378,7 +374,7 @@ impl Store {
pub fn save_album(&self, album: &Album) -> Result<()> {
let (parent, name) = album.path.rsplit_once('/').unwrap_or(("", &album.path));
- self.db.connection().execute(
+ self.db.execute(
"INSERT INTO albums VALUES(?1,?2,?3,?4,?5,?6,?7) ON CONFLICT(key) DO UPDATE SET name=excluded.name,path=excluded.path,parent=excluded.parent,folded=excluded.folded WHERE albums.name<>excluded.name OR albums.path<>excluded.path",
params![
album.key,
@@ -396,7 +392,6 @@ impl Store {
pub fn placement(&self, album: i64, file: i64) -> Result<Option<Placement>> {
Ok(self
.db
- .connection()
.query_row(
"SELECT id,album,file,retained,name,kind FROM placements WHERE album=?1 AND file=?2 AND retained=0",
params![album, file],
@@ -406,7 +401,7 @@ impl Store {
}
pub fn save_placement(&self, placement: &Placement) -> Result<()> {
- self.db.connection().execute(
+ self.db.execute(
"INSERT INTO placements VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(id) DO UPDATE SET retained=excluded.retained,name=excluded.name,kind=excluded.kind WHERE placements.retained<>excluded.retained OR placements.name<>excluded.name OR placements.kind<>excluded.kind",
params![
placement.id,
@@ -421,7 +416,7 @@ impl Store {
}
pub fn components(&self, owner: &str) -> Result<Vec<Component>> {
- let mut query = self.db.connection().prepare(
+ let mut query = self.db.prepare(
"SELECT placement,role,folder,name,size,hash,signature,properties,intended_time FROM components WHERE placement=?1 ORDER BY role",
)?;
Ok(query
@@ -430,7 +425,7 @@ impl Store {
}
pub fn save_component(&self, component: &Component) -> Result<()> {
- self.db.connection().execute(
+ self.db.execute(
"INSERT INTO components VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11) ON CONFLICT(placement,role) DO UPDATE SET folder=excluded.folder,name=excluded.name,folded=excluded.folded,stem=excluded.stem,size=excluded.size,hash=excluded.hash,signature=excluded.signature,properties=excluded.properties,intended_time=excluded.intended_time WHERE components.folder<>excluded.folder OR components.name<>excluded.name OR components.size<>excluded.size OR components.hash<>excluded.hash OR components.signature IS NOT excluded.signature OR components.properties<>excluded.properties OR components.intended_time IS NOT excluded.intended_time",
params![
component.placement,
@@ -452,7 +447,6 @@ impl Store {
pub fn json_record(&self, owner: &str, role: Option<&Role>) -> Result<Option<JsonRecord>> {
Ok(self
.db
- .connection()
.query_row(
"SELECT owner,role,folder,name,value,hash,properties FROM json_records WHERE owner=?1 AND role=?2",
params![owner, role.map_or("", Role::name)],
@@ -466,7 +460,7 @@ impl Store {
let media = location
.and_then(|location| location.name.strip_prefix("metadata/"))
.and_then(|name| name.strip_suffix(".json"));
- self.db.connection().execute(
+ self.db.execute(
"INSERT INTO json_records VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9) ON CONFLICT(owner,role) DO UPDATE SET folder=excluded.folder,name=excluded.name,folded=excluded.folded,stem=excluded.stem,value=excluded.value,hash=excluded.hash,properties=excluded.properties WHERE json_records.folder IS NOT excluded.folder OR json_records.name IS NOT excluded.name OR json_records.value<>excluded.value OR json_records.hash IS NOT excluded.hash OR json_records.properties<>excluded.properties",
params![
record.owner,
@@ -486,7 +480,6 @@ impl Store {
pub fn pending(&self, owner: &str) -> Result<Option<Pending>> {
Ok(self
.db
- .connection()
.query_row(
"SELECT owner,album,file,retained,name,kind,folder,name1,name2,action FROM pending WHERE owner=?1",
[owner],
@@ -498,7 +491,6 @@ impl Store {
pub fn pending_file(&self, album: i64, file: i64) -> Result<Option<Pending>> {
Ok(self
.db
- .connection()
.query_row(
"SELECT owner,album,file,retained,name,kind,folder,name1,name2,action FROM pending WHERE album=?1 AND file=?2 ORDER BY retained DESC LIMIT 1",
params![album, file],
@@ -510,7 +502,7 @@ impl Store {
pub fn save_pending(&self, pending: &Pending) -> Result<()> {
let first = &pending.names[0];
let second = pending.names.get(1);
- self.db.connection().execute(
+ self.db.execute(
"INSERT INTO pending VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14) ON CONFLICT(owner) DO UPDATE SET retained=excluded.retained,name=excluded.name,kind=excluded.kind,folder=excluded.folder,name1=excluded.name1,name2=excluded.name2,folded1=excluded.folded1,folded2=excluded.folded2,stem1=excluded.stem1,stem2=excluded.stem2,action=excluded.action",
params![
pending.owner,
@@ -534,10 +526,8 @@ impl Store {
pub fn clear_pending(&self, owner: &str) -> Result<()> {
self.db
- .connection()
.execute("DELETE FROM pending WHERE owner=?1", [owner])?;
self.db
- .connection()
.execute(
"DELETE FROM json_records WHERE owner=?1 AND folder IS NULL AND NOT EXISTS(SELECT 1 FROM components WHERE placement=?1)",
[owner],
@@ -552,7 +542,6 @@ impl Store {
) -> Result<Option<T>> {
let text: Option<String> = self
.db
- .connection()
.query_row(sql, parameters, |r| r.get(0))
.optional()?;
text.map(|text| serde_json::from_str(&text).map_err(Into::into))
@@ -560,7 +549,7 @@ impl Store {
}
pub fn temporary(&self, temporary: &Temporary) -> Result<()> {
- self.db.connection().execute(
+ self.db.execute(
"INSERT INTO temporaries VALUES(?1,?2,?3,?4)",
params![
temporary.location.name,
@@ -572,35 +561,39 @@ impl Store {
Ok(())
}
pub fn reset_desired(&self) -> Result<()> {
- self.db.connection().execute_batch(RUN_SCHEMA)?;
+ self.db.execute_batch(RUN_SCHEMA)?;
Ok(())
}
- pub fn desired_album(&self, record: &AlbumRecord, user_id: i64) -> Result<()> {
- let name = record.album(user_id).ok().map(|a| a.name);
- let favorites = record.remote.kind == "favorites" && record.remote.owner.id == user_id;
- self.db.connection().execute(
+ pub fn desired_album(&self, entry: &CollectionEntry) -> Result<()> {
+ self.db.execute(
"INSERT INTO desired_albums(id,name,present,favorites) VALUES(?1,?2,1,?3) ON CONFLICT(id) DO UPDATE SET name=excluded.name,present=1,favorites=excluded.favorites",
- params![record.remote.id, name, favorites],
+ params![entry.id, entry.name, entry.favorites],
)?;
Ok(())
}
- pub fn snapshot_album(&self, album: &Collection) -> Result<()> {
+ pub fn snapshot_album(
+ &self,
+ album: &Collection,
+ documents: &Documents,
+ user_id: i64,
+ ) -> Result<()> {
+ let (metadata, warnings) = crate::metadata::album(album, documents, user_id)?;
let source = AlbumSource {
name: album.name.clone(),
- metadata: ente_photos::export::album(album),
- warnings: album.warnings.clone(),
+ metadata,
+ warnings,
};
- self.db.connection().execute(
+ self.db.execute(
"INSERT INTO album_sources VALUES(?1,?2) ON CONFLICT(id) DO UPDATE SET record=excluded.record WHERE album_sources.record<>excluded.record",
params![album.id, serde_json::to_string(&source)?],
)?;
Ok(())
}
- pub fn snapshot_file(&self, file: &File) -> Result<()> {
- self.db.connection().execute(
+ pub fn snapshot_file(&self, file: &FileSnapshot) -> Result<()> {
+ self.db.execute(
"INSERT INTO sources VALUES(?1,?2) ON CONFLICT(file) DO UPDATE SET record=excluded.record WHERE sources.record<>excluded.record",
params![file.id, serde_json::to_string(file)?],
)?;
@@ -646,3 +639,10 @@ CREATE TEMP TABLE favorites (file INTEGER PRIMARY KEY);
CREATE TEMP TABLE outcomes (unit TEXT PRIMARY KEY, conflict INTEGER NOT NULL);
CREATE TEMP TABLE events (placement TEXT PRIMARY KEY, exported INTEGER NOT NULL DEFAULT 0, metadata_updated INTEGER NOT NULL DEFAULT 0, renamed INTEGER NOT NULL DEFAULT 0, retained INTEGER NOT NULL DEFAULT 0);
";
+
+fn read_json<T: DeserializeOwned>(row: &Row<'_>, column: usize) -> rusqlite::Result<T> {
+ let text: String = row.get(column)?;
+ serde_json::from_str(&text).map_err(|e| {
+ rusqlite::Error::FromSqlConversionFailure(column, rusqlite::types::Type::Text, Box::new(e))
+ })
+}
diff --git a/rust/apps/cli-next/src/export/transfer.rs b/rust/crates/photos-export/src/transfer.rs
similarity index 93%
rename from rust/apps/cli-next/src/export/transfer.rs
rename to rust/crates/photos-export/src/transfer.rs
index 53fb897020..b5f970e16c 100644
--- a/rust/apps/cli-next/src/export/transfer.rs
+++ b/rust/crates/photos-export/src/transfer.rs
@@ -3,12 +3,10 @@ use std::{
io::{Seek, SeekFrom},
};
+use crate::{metadata::Role, snapshot::FileSnapshot};
use anyhow::{Context, Result, ensure};
use ente_core::{Session, b64, crypto::hash};
-use ente_photos::{
- export::Role,
- files::{self, Kind},
-};
+use ente_photos::files;
use rusqlite::{OptionalExtension, params};
use super::{
@@ -33,8 +31,8 @@ pub struct Prepared {
pub observed: Vec<(i64, Vec<(Role, Properties)>)>,
}
-pub fn roles(file: &files::File) -> Vec<Role> {
- if file.kind == Kind::LivePhoto {
+pub fn roles(file: &FileSnapshot) -> Vec<Role> {
+ if file.kind == "livephoto" {
vec![Role::Image, Role::Video]
} else {
vec![Role::Original]
@@ -44,7 +42,7 @@ pub fn roles(file: &files::File) -> Vec<Role> {
pub fn verify(
run: &Run<'_>,
component: &Component,
- file: &files::File,
+ file: &FileSnapshot,
signature: &str,
) -> Result<Option<Properties>> {
let path = run.path(&component.location)?;
@@ -68,7 +66,7 @@ pub fn verify(
Ok(Some(properties))
}
-pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Result<Prepared> {
+pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> Result<Prepared> {
let signature = signature(file)?;
let roles = roles(file);
let mut observed = Vec::new();
@@ -78,7 +76,6 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Re
run.check_cancel()?;
let album: Option<i64> = store::lock(run.store)?
.db
- .connection()
.query_row(
"SELECT album FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",
params![file.id, after],
@@ -139,7 +136,6 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Re
}
let next: Option<(i64, String)> = store::lock(run.store)?
.db
- .connection()
.query_row(
"SELECT rowid,id FROM placements WHERE file=?1 AND rowid<?2 ORDER BY rowid DESC LIMIT 1",
params![file.id, before],
@@ -197,7 +193,6 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Re
let (folder, album, temporary, output) = loop {
let album: Option<i64> = store::lock(run.store)?
.db
- .connection()
.query_row(
"SELECT album FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",
params![file.id, after],
@@ -224,7 +219,7 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Re
drop(output);
let path = run.path(&temporary)?;
let mut cancel = run.cancel.clone();
- let download = files::download(session, file, || {
+ let download = files::download(session, file.id, &file.key, &file.header, || {
let mut output = File::options().read(true).write(true).open(&path)?;
output.set_len(0)?;
output.seek(SeekFrom::Start(0))?;
@@ -232,9 +227,9 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Re
});
let writer = tokio::select! { result=download => result?, _=cancel.changed() => return Err(super::Cancelled.into()) };
let (archive, size, hash) = writer.finish()?;
- let components = if file.kind == Kind::LivePhoto {
+ let components = if file.kind == "livephoto" {
let mut temporaries = Vec::new();
- let extracted = ente_photos::live_photo::extract(archive, |role, extension| {
+ let extracted = crate::live_photo::extract(archive, |role, extension| {
let (temporary, file) = run
.temporary(&folder, album, Some(file.id))
.map_err(std::io::Error::other)?;
@@ -287,7 +282,7 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Re
observed,
})
}
-pub fn signature(file: &files::File) -> Result<String> {
+pub fn signature(file: &FileSnapshot) -> Result<String> {
let mut bytes = file.header.as_bytes().to_vec();
bytes.extend_from_slice(file.hash.as_deref().unwrap_or_default().as_bytes());
Ok(b64::encode(&hash::hash(
@@ -297,7 +292,7 @@ pub fn signature(file: &files::File) -> Result<String> {
)?))
}
-pub fn expected_hash<'a>(file: &'a files::File, role: &Role) -> Option<&'a str> {
+pub fn expected_hash<'a>(file: &'a FileSnapshot, role: &Role) -> Option<&'a str> {
let hash = file.hash.as_deref()?;
match role {
Role::Original => Some(hash),
diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.toml
index cbcb2d2887..25ca94302a 100644
--- a/rust/crates/photos/Cargo.toml
+++ b/rust/crates/photos/Cargo.toml
@@ -3,7 +3,6 @@ name = "ente-photos"
edition.workspace = true
[dependencies]
-chrono = { workspace = true, features = ["std"] }
ente-assets.workspace = true
ente-collections.workspace = true
ente-core.workspace = true
@@ -19,7 +18,6 @@ thiserror.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
futures-util.workspace = true
-zip.workspace = true
[dev-dependencies]
tempfile.workspace = true
diff --git a/rust/crates/photos/src/collections.rs b/rust/crates/photos/src/collections.rs
index 1a5f7e073f..c98fcfbdd9 100644
--- a/rust/crates/photos/src/collections.rs
+++ b/rust/crates/photos/src/collections.rs
@@ -1,5 +1,5 @@
use ente_core::{Session, crypto::Key};
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
use crate::source::{self, Documents, MetadataError};
@@ -14,13 +14,6 @@ pub struct Collection {
pub visibility: Visibility,
pub owner_id: i64,
pub updated_at_micros: i64,
- pub display_order: Option<i64>,
- pub muted: Option<bool>,
- pub ascending: Option<bool>,
- pub description: Option<String>,
- pub cover_file_id: Option<i64>,
- pub layout: Option<String>,
- pub warnings: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -46,7 +39,7 @@ impl Kind {
}
}
-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+#[derive(Debug, Clone, Copy)]
pub enum Visibility {
Visible,
Archived,
@@ -112,8 +105,6 @@ pub fn interpret(
)?;
let shared: ShareeMetadata =
source::optional(documents.shared.as_deref(), "invalid album sharee metadata")?;
- let public: PublicMetadata =
- source::optional(documents.public.as_deref(), "invalid album public metadata")?;
let kind = match (
owned.then_some(private.sub_type).flatten().unwrap_or(0),
remote.kind.as_str(),
@@ -136,22 +127,6 @@ pub fn interpret(
shared.visibility
})?
};
- let mut warnings = Vec::new();
- source::unfamiliar(
- documents.private.as_deref(),
- &["visibility", "subType", "order"],
- &mut warnings,
- );
- source::unfamiliar(
- documents.shared.as_deref(),
- &["visibility", "mute", "order"],
- &mut warnings,
- );
- source::unfamiliar(
- documents.public.as_deref(),
- &["asc", "coverID", "layout", "caption"],
- &mut warnings,
- );
Ok(Collection {
id: remote.id,
key: Key::from_bytes(*key.as_bytes()),
@@ -165,13 +140,6 @@ pub fn interpret(
visibility,
owner_id: remote.owner.id,
updated_at_micros: remote.updation_time,
- display_order: if owned { private.order } else { shared.order },
- muted: (!owned).then_some(shared.mute).flatten(),
- ascending: public.asc,
- description: public.caption,
- cover_file_id: public.cover_id,
- layout: public.layout,
- warnings,
})
}
@@ -180,21 +148,9 @@ pub fn interpret(
struct PrivateMetadata {
visibility: Option<u8>,
sub_type: Option<u8>,
- order: Option<i64>,
}
#[derive(Default, Deserialize)]
struct ShareeMetadata {
visibility: Option<u8>,
- order: Option<i64>,
- mute: Option<bool>,
-}
-
-#[derive(Default, Deserialize)]
-struct PublicMetadata {
- asc: Option<bool>,
- #[serde(rename = "coverID")]
- cover_id: Option<i64>,
- layout: Option<String>,
- caption: Option<String>,
}
diff --git a/rust/crates/photos/src/export.rs b/rust/crates/photos/src/export.rs
deleted file mode 100644
index 93139721a8..0000000000
--- a/rust/crates/photos/src/export.rs
+++ /dev/null
@@ -1,174 +0,0 @@
-use chrono::{DateTime, SecondsFormat};
-use ente_core::{
- b64,
- crypto::{Key, kdf},
-};
-use serde::{Deserialize, Serialize};
-use serde_json::{Value, json};
-
-use crate::{
- collections::{Collection, Visibility},
- files::{File, Kind},
- source::MetadataError,
-};
-
-#[derive(Serialize, Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-pub struct Root {
- pub format: String,
- pub version: u32,
- pub source_verifier: String,
-}
-
-impl Root {
- pub fn new(master_key: &Key) -> ente_core::crypto::Result<Self> {
- Ok(Self {
- format: "ente-photos-export".into(),
- version: 1,
- source_verifier: b64::encode(&kdf::derive_subkey(master_key, 32, 1, b"photoexp")?),
- })
- }
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase")]
-pub enum Role {
- Original,
- Image,
- Video,
-}
-
-impl Role {
- pub fn name(&self) -> &'static str {
- match self {
- Self::Original => "original",
- Self::Image => "image",
- Self::Video => "video",
- }
- }
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Component {
- pub role: Role,
- pub path: String,
- pub size: u64,
- pub hash: String,
-}
-
-pub fn album(album: &Collection) -> Value {
- let mut metadata = json!({
- "albumID": album.id.to_string(), "type": album.kind.name(), "visibility": album.visibility.name(),
- });
- optional(&mut metadata, "displayOrder", album.display_order);
- optional(&mut metadata, "muted", album.muted);
- optional(
- &mut metadata,
- "sortOrder",
- album
- .ascending
- .map(|asc| if asc { "ascending" } else { "descending" }),
- );
- optional(&mut metadata, "description", album.description.as_deref());
- optional(
- &mut metadata,
- "coverFileID",
- album.cover_file_id.map(|id| id.to_string()),
- );
- optional(&mut metadata, "layout", album.layout.as_deref());
- json!({"title": album.name, "ente": metadata})
-}
-
-pub fn file(
- file: &File,
- components: &[Component],
- component: usize,
- favorited: bool,
-) -> Result<Value, MetadataError> {
- let selected = components
- .get(component)
- .ok_or(MetadataError("missing media component"))?;
- let mut metadata = json!({
- "fileID": file.id.to_string(),
- "type": if file.kind == Kind::LivePhoto { "livePhoto" } else { file.kind.name() },
- "name": file.name,
- "creationTime": timestamp(file.created_at_micros)?,
- "component": selected.role,
- "components": components,
- });
- optional(
- &mut metadata,
- "modificationTime",
- file.modified_at_micros.map(timestamp).transpose()?,
- );
- optional(
- &mut metadata,
- "location",
- file.location
- .as_ref()
- .map(|l| json!({"latitude": l.latitude,"longitude": l.longitude})),
- );
- optional(&mut metadata, "localDateTime", file.date_time.as_deref());
- optional(&mut metadata, "utcOffset", file.offset_time.as_deref());
- optional(&mut metadata, "width", file.width);
- optional(&mut metadata, "height", file.height);
- optional(&mut metadata, "durationSeconds", file.duration_seconds);
- optional(
- &mut metadata,
- "visibility",
- file.visibility.map(Visibility::name),
- );
- optional(&mut metadata, "cameraMake", file.camera_make.as_deref());
- optional(&mut metadata, "cameraModel", file.camera_model.as_deref());
- optional(&mut metadata, "uploaderName", file.uploader_name.as_deref());
- optional(&mut metadata, "panorama", file.panorama);
- optional(
- &mut metadata,
- "motionPhotoVideoOffset",
- file.motion_video_offset,
- );
- let taken = common_time(file.created_at_micros)?;
- let mut record = json!({
- "title": selected.path,
- "photoTakenTime": taken,
- "creationTime": taken,
- "favorited": favorited,
- "ente": metadata,
- });
- optional(
- &mut record,
- "modificationTime",
- file.modified_at_micros.map(common_time).transpose()?,
- );
- optional(&mut record, "description", file.caption.as_deref());
- optional(
- &mut record,
- "geoData",
- file.location
- .as_ref()
- .map(|l| json!({"latitude":l.latitude,"longitude":l.longitude})),
- );
- Ok(record)
-}
-
-fn optional(object: &mut Value, name: &str, value: Option<impl Into<Value>>) {
- if let Some(value) = value {
- object[name] = value.into();
- }
-}
-
-fn time(micros: i64) -> Result<DateTime<chrono::Utc>, MetadataError> {
- DateTime::from_timestamp_micros(micros)
- .ok_or(MetadataError("timestamp is outside the supported range"))
-}
-
-fn timestamp(micros: i64) -> Result<String, MetadataError> {
- Ok(time(micros)?.to_rfc3339_opts(SecondsFormat::Micros, true))
-}
-
-fn common_time(micros: i64) -> Result<Value, MetadataError> {
- let time = time(micros)?;
- Ok(
- json!({"timestamp":micros.div_euclid(1_000_000).to_string(),"formatted":time.format("%b %-d, %Y, %-I:%M:%S %p UTC").to_string()}),
- )
-}
diff --git a/rust/crates/photos/src/files.rs b/rust/crates/photos/src/files.rs
index 696e2e493c..5da18cfd77 100644
--- a/rust/crates/photos/src/files.rs
+++ b/rust/crates/photos/src/files.rs
@@ -3,7 +3,7 @@ use ente_core::{
crypto::{self, Header, Key, blob},
http,
};
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
use crate::{
collections::Visibility,
@@ -12,7 +12,7 @@ use crate::{
pub use ente_collections::client::File as RemoteFile;
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug)]
pub struct File {
pub id: i64,
pub owner_id: i64,
@@ -30,19 +30,11 @@ pub struct File {
pub width: Option<u32>,
pub height: Option<u32>,
pub visibility: Option<Visibility>,
- pub camera_make: Option<String>,
- pub camera_model: Option<String>,
- pub uploader_name: Option<String>,
- pub panorama: Option<bool>,
- pub motion_video_offset: Option<u64>,
- pub warnings: Vec<String>,
- #[serde(with = "source::key")]
pub key: Key,
- #[serde(with = "source::header")]
pub header: Header,
}
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Kind {
Image,
Video,
@@ -61,7 +53,7 @@ impl Kind {
}
}
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug)]
pub struct Location {
pub latitude: f64,
pub longitude: f64,
@@ -86,7 +78,9 @@ pub enum Error {
#[cfg(not(target_arch = "wasm32"))]
pub async fn download<W: std::io::Write>(
session: &Session,
- file: &File,
+ id: i64,
+ key: &Key,
+ header: &Header,
mut output: impl FnMut() -> std::io::Result<W>,
) -> Result<W, Error> {
use ente_core::crypto::stream::DecryptingWriter;
@@ -99,7 +93,7 @@ pub async fn download<W: std::io::Write>(
let output = output?;
let signed: DownloadUrl = session
.api
- .get(&format!("/files/download/v3/{}", file.id))
+ .get(&format!("/files/download/v3/{id}"))
.send()
.await?
.error_for_code()
@@ -114,7 +108,7 @@ pub async fn download<W: std::io::Write>(
.await?
.error_for_status()?;
let mut body = std::pin::pin!(response.bytes_stream());
- let mut decryptor = DecryptingWriter::new(&file.header, &file.key, output);
+ let mut decryptor = DecryptingWriter::new(header, key, output);
while let Some(chunk) = body.next().await {
decryptor.write(&chunk?)?;
}
@@ -193,52 +187,6 @@ pub fn interpret(
latitude,
longitude,
});
- let mut warnings = Vec::new();
- source::unfamiliar(
- Some(&documents.original),
- &[
- "title",
- "fileType",
- "creationTime",
- "modificationTime",
- "latitude",
- "longitude",
- "hash",
- "imageHash",
- "videoHash",
- "duration",
- "hasStaticThumbnail",
- "localID",
- "deviceFolder",
- "subType",
- "version",
- "exif",
- ],
- &mut warnings,
- );
- source::unfamiliar(
- documents.public.as_deref(),
- &[
- "editedName",
- "editedTime",
- "dateTime",
- "offsetTime",
- "caption",
- "lat",
- "long",
- "w",
- "h",
- "sv",
- "cameraMake",
- "cameraModel",
- "uploaderName",
- "mediaType",
- "mvi",
- "noThumb",
- ],
- &mut warnings,
- );
- source::unfamiliar(documents.private.as_deref(), &["visibility"], &mut warnings);
Ok(File {
id: remote.id,
owner_id: remote.owner_id,
@@ -263,16 +211,6 @@ pub fn interpret(
} else {
None
},
- camera_make: public.camera_make,
- camera_model: public.camera_model,
- uploader_name: public.uploader_name,
- panorama: if kind == Kind::Image {
- public.media_type.map(|value| value & 1 != 0)
- } else {
- None
- },
- motion_video_offset: public.mvi,
- warnings,
key: Key::from_bytes(*key.as_bytes()),
header: Header::try_from_slice(&b64::decode(&remote.file.decryption_header)?)?,
})
@@ -305,11 +243,6 @@ struct PublicMetadata {
long: Option<f64>,
w: Option<u32>,
h: Option<u32>,
- camera_make: Option<String>,
- camera_model: Option<String>,
- uploader_name: Option<String>,
- media_type: Option<u64>,
- mvi: Option<u64>,
}
#[derive(Deserialize)]
diff --git a/rust/crates/photos/src/lib.rs b/rust/crates/photos/src/lib.rs
index a718b88631..c472bde578 100644
--- a/rust/crates/photos/src/lib.rs
+++ b/rust/crates/photos/src/lib.rs
@@ -1,6 +1,5 @@
pub mod collections;
pub mod db;
-pub mod export;
pub mod files;
pub mod location;
pub mod metadata;
@@ -13,6 +12,3 @@ pub use motion_photo::{
MotionPhotoError, VideoIndex, extract_motion_video_file_from_path,
extract_motion_video_from_path, extract_xmp_from_path, get_motion_video_index_from_path,
};
-
-#[cfg(not(target_arch = "wasm32"))]
-pub mod live_photo;
diff --git a/rust/crates/photos/src/source.rs b/rust/crates/photos/src/source.rs
index c85c00d147..f3b18b2404 100644
--- a/rust/crates/photos/src/source.rs
+++ b/rust/crates/photos/src/source.rs
@@ -1,5 +1,4 @@
use serde::{Deserialize, Serialize, de::DeserializeOwned};
-use serde_json::Value;
#[derive(Clone, Serialize, Deserialize)]
pub struct Documents {
@@ -47,32 +46,6 @@ mod optional_bytes {
}
}
-pub(crate) mod key {
- use ente_core::crypto::Key;
- use serde::{Deserializer, Serializer, de::Error};
-
- pub fn serialize<S: Serializer>(key: &Key, serializer: S) -> Result<S::Ok, S::Error> {
- serializer.serialize_str(&ente_core::b64::encode(key.as_bytes()))
- }
-
- pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Key, D::Error> {
- Key::try_from_slice(&super::bytes::deserialize(deserializer)?).map_err(D::Error::custom)
- }
-}
-
-pub(crate) mod header {
- use ente_core::crypto::Header;
- use serde::{Deserializer, Serializer, de::Error};
-
- pub fn serialize<S: Serializer>(header: &Header, serializer: S) -> Result<S::Ok, S::Error> {
- serializer.serialize_str(&ente_core::b64::encode(header.as_bytes()))
- }
-
- pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Header, D::Error> {
- Header::try_from_slice(&super::bytes::deserialize(deserializer)?).map_err(D::Error::custom)
- }
-}
-
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
pub struct MetadataError(pub &'static str);
@@ -90,15 +63,3 @@ pub fn optional<T: DeserializeOwned + Default>(
.transpose()
.map(Option::unwrap_or_default)
}
-
-pub fn unfamiliar(bytes: Option<&[u8]>, known: &[&str], warnings: &mut Vec<String>) {
- if let Some(bytes) = bytes
- && let Ok(Value::Object(fields)) = serde_json::from_slice(bytes)
- {
- for key in fields.keys() {
- if !known.contains(&key.as_str()) {
- warnings.push(key.clone());
- }
- }
- }
-}
diff --git a/rust/crates/test-support/src/object_store.rs b/rust/crates/test-support/src/object_store.rs
index a6f13e3849..b689b53bb8 100644
--- a/rust/crates/test-support/src/object_store.rs
+++ b/rust/crates/test-support/src/object_store.rs
@@ -19,8 +19,6 @@ use crate::{TestResult, net::LOCAL_HOST};
#[derive(Default)]
pub struct ObjectStoreControl {
reads: AtomicUsize,
- active: AtomicUsize,
- peak: AtomicUsize,
interrupted: AtomicUsize,
held: Mutex<Option<Arc<ReadGate>>>,
}
@@ -78,9 +76,6 @@ impl ObjectStoreControl {
pub fn reads(&self) -> usize {
self.reads.load(Ordering::SeqCst)
}
- pub fn peak_reads(&self) -> usize {
- self.peak.load(Ordering::SeqCst)
- }
pub fn interrupt_reads(&self, count: usize) {
self.interrupted.store(count, Ordering::SeqCst);
}
@@ -215,40 +210,33 @@ fn handle(stream: TcpStream, root: &Path, control: &ObjectStoreControl) -> TestR
{
return Ok(());
}
- let active = control.active.fetch_add(1, Ordering::SeqCst) + 1;
- control.peak.fetch_max(active, Ordering::SeqCst);
- let result = (|| {
- respond(input.get_mut(), "200 OK", file.metadata()?.len())?;
- let gate = control
- .held
- .lock()
- .unwrap_or_else(PoisonError::into_inner)
- .clone();
- if let Some(gate) = gate
- && file.metadata()?.len() >= gate.minimum_size
- {
- let mut state = gate.state.lock().unwrap_or_else(PoisonError::into_inner);
- state.0 += 1;
- gate.changed.notify_all();
- while !state.1 {
- state = gate
- .changed
- .wait(state)
- .unwrap_or_else(PoisonError::into_inner);
- }
+ respond(input.get_mut(), "200 OK", file.metadata()?.len())?;
+ let gate = control
+ .held
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .clone();
+ if let Some(gate) = gate
+ && file.metadata()?.len() >= gate.minimum_size
+ {
+ let mut state = gate.state.lock().unwrap_or_else(PoisonError::into_inner);
+ state.0 += 1;
+ gate.changed.notify_all();
+ while !state.1 {
+ state = gate
+ .changed
+ .wait(state)
+ .unwrap_or_else(PoisonError::into_inner);
}
- let mut buffer = vec![0; 64 * 1024];
- loop {
- let read = file.read(&mut buffer)?;
- if read == 0 {
- break;
- }
- input.get_mut().write_all(&buffer[..read])?;
+ }
+ let mut buffer = vec![0; 64 * 1024];
+ loop {
+ let read = file.read(&mut buffer)?;
+ if read == 0 {
+ break;
}
- Ok::<_, io::Error>(())
- })();
- control.active.fetch_sub(1, Ordering::SeqCst);
- result?;
+ input.get_mut().write_all(&buffer[..read])?;
+ }
}
_ => respond(input.get_mut(), "405 Method Not Allowed", 0)?,
}