Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Add Photos export

Manav Rathif122ce531944
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index c97d1165bb..16d63b62bc 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2447,9 +2447,12 @@ dependencies = [
"dialoguer",
"dirs 5.0.1",
"ente-accounts",
+ "ente-collections",
"ente-core",
"ente-photos",
"ente-test-support",
+ "futures-util",
+ "icu_casemap",
"keyring",
"mockito",
"reqwest 0.12.28",
@@ -2459,9 +2462,11 @@ dependencies = [
"tempfile",
"thiserror 2.0.18",
"tokio",
+ "unicode-normalization",
"url",
"uuid",
"zeroize",
+ "zip 8.6.0",
]
[[package]]
@@ -2470,6 +2475,7 @@ version = "0.0.0"
dependencies = [
"ente-core",
"serde",
+ "serde_json",
"thiserror 2.0.18",
]
@@ -2771,6 +2777,7 @@ dependencies = [
name = "ente-photos"
version = "0.0.0"
dependencies = [
+ "chrono",
"ente-assets",
"ente-collections",
"ente-core",
@@ -2785,6 +2792,7 @@ dependencies = [
"serde_json",
"tempfile",
"thiserror 2.0.18",
+ "zip 8.6.0",
]
[[package]]
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index f22f09f726..9a6bb04bed 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -85,6 +85,7 @@ futures-util = "0.3.32"
getrandom = { version = "0.4.0", features = ["wasm_js"] }
hkdf = "0.13.0"
hmac = "0.13.0"
+icu_casemap = "1.5.1"
image = "0.25.10"
js-sys = "0.3.102"
keyring = { version = "3.6.3", default-features = false }
@@ -109,6 +110,7 @@ tempfile = "3.27.0"
thiserror = "2.0.18"
tokio = "1.41.0"
tsify = { version = "0.5.8", default-features = false, features = ["js"] }
+unicode-normalization = "0.1.25"
uniffi = { version = "=0.31.2" }
uniffi_bindgen = { version = "=0.31.2", default-features = false, features = ["cargo-metadata"] }
url = "2.5.8"
diff --git a/rust/apps/cli-next/Cargo.toml b/rust/apps/cli-next/Cargo.toml
index 3761152ba1..3891e79e6a 100644
--- a/rust/apps/cli-next/Cargo.toml
+++ b/rust/apps/cli-next/Cargo.toml
@@ -14,8 +14,10 @@ clap = { workspace = true, features = ["derive"] }
dialoguer = { workspace = true, features = ["password"] }
dirs.workspace = true
ente-accounts.workspace = true
+ente-collections.workspace = true
ente-core.workspace = true
ente-photos.workspace = true
+icu_casemap.workspace = true
keyring = { workspace = true, features = ["apple-native", "windows-native", "linux-native-sync-persistent", "crypto-rust"] }
reqwest.workspace = true
rusqlite = { workspace = true, features = ["bundled-sqlcipher"] }
@@ -23,7 +25,8 @@ serde.workspace = true
serde_json.workspace = true
tempfile.workspace = true
thiserror.workspace = true
-tokio = { workspace = true, features = ["macros", "rt"] }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal", "sync", "time"] }
+unicode-normalization.workspace = true
url.workspace = true
uuid = { workspace = true, features = ["serde", "v4"] }
zeroize.workspace = true
@@ -33,7 +36,9 @@ rusqlite = { workspace = true, features = ["bundled-sqlcipher-vendored-openssl"]
[dev-dependencies]
ente-test-support.workspace = true
+futures-util.workspace = true
mockito.workspace = true
+zip.workspace = true
[lints]
workspace = true
diff --git a/rust/apps/cli-next/src/args.rs b/rust/apps/cli-next/src/args.rs
index a755e6afc4..613f6e93d5 100644
--- a/rust/apps/cli-next/src/args.rs
+++ b/rust/apps/cli-next/src/args.rs
@@ -108,6 +108,8 @@ pub enum PhotosCommand {
#[derive(Subcommand)]
pub enum PhotosLibraryCommand {
+ #[command(about = "Maintain an independent copy of your Photos library")]
+ Export(ExportArgs),
#[command(about = "Manage albums")]
Album {
#[command(subcommand)]
@@ -127,6 +129,27 @@ pub enum PhotosLibraryCommand {
},
}
+#[derive(Args)]
+pub struct ExportArgs {
+ #[arg(
+ long,
+ value_name = "ALBUM",
+ help = "Include an album ID or exact name; repeat for multiple albums"
+ )]
+ pub album: Vec<String>,
+ #[arg(long, value_name = "ALBUM", help = "Exclude an album ID or exact name")]
+ pub exclude_album: Vec<String>,
+ #[arg(
+ long,
+ help = "Reconnect an existing export after moving it or losing local state"
+ )]
+ pub adopt: bool,
+ #[arg(long, help = "Attempt selected saved decryption failures once more")]
+ pub retry_failed: bool,
+ #[arg(value_name = "DESTINATION")]
+ pub destination: PathBuf,
+}
+
#[derive(Subcommand)]
pub enum SessionCommand {
#[command(
diff --git a/rust/apps/cli-next/src/core_db.rs b/rust/apps/cli-next/src/core_db.rs
index acb18bfe60..701e76136e 100644
--- a/rust/apps/cli-next/src/core_db.rs
+++ b/rust/apps/cli-next/src/core_db.rs
@@ -7,7 +7,7 @@ pub use rusqlite::{
#[derive(Debug, thiserror::Error)]
pub enum Error {
- #[error(transparent)]
+ #[error("{0}")]
Sqlite(#[from] SqliteError),
#[error("database version {current} is newer than this build supports ({target})")]
Downgrade { current: i64, target: i64 },
@@ -22,6 +22,10 @@ impl Db {
Self(connection)
}
+ pub fn connection(&self) -> &Connection {
+ &self.0
+ }
+
pub fn migrate(&mut self, scripts: &[&str]) -> Result<()> {
let target = scripts.len() as i64;
let transaction = self
diff --git a/rust/apps/cli-next/src/db.rs b/rust/apps/cli-next/src/db.rs
index 87def43cf9..75171d6607 100644
--- a/rust/apps/cli-next/src/db.rs
+++ b/rust/apps/cli-next/src/db.rs
@@ -9,11 +9,19 @@ use crate::{core_db::Db, replica, vault::DbKey};
const MIGRATIONS: &[&str] = &[replica::SCHEMA];
pub fn open(account_dir: &Path, key: &DbKey, create: bool) -> Result<Db> {
+ let connection = connect(&account_dir.join("data.db"), key, create)?;
+ let mut db = Db::new(connection);
+ db.migrate(MIGRATIONS)?;
+ Ok(db)
+}
+
+pub fn connect(path: &Path, key: &DbKey, create: bool) -> Result<Connection> {
let mut flags = OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX;
if create {
flags |= OpenFlags::SQLITE_OPEN_CREATE;
}
- let connection = Connection::open_with_flags(account_dir.join("data.db"), flags)?;
+ let connection = Connection::open_with_flags(path, flags)?;
+ connection.busy_timeout(std::time::Duration::from_secs(5))?;
connection
.query_row("PRAGMA cipher_version", [], |row| row.get::<_, String>(0))
.context("this CLI build does not support encrypted databases")?;
@@ -32,7 +40,5 @@ pub fn open(account_dir: &Path, key: &DbKey, create: bool) -> Result<Db> {
connection.execute_batch(
"PRAGMA foreign_keys = ON; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = WAL;",
)?;
- let mut db = Db::new(connection);
- db.migrate(MIGRATIONS)?;
- Ok(db)
+ Ok(connection)
}
diff --git a/rust/apps/cli-next/src/export/adopt.rs b/rust/apps/cli-next/src/export/adopt.rs
new file mode 100644
index 0000000000..14013c37f2
--- /dev/null
+++ b/rust/apps/cli-next/src/export/adopt.rs
@@ -0,0 +1,333 @@
+use std::{fs, path::Path};
+
+use anyhow::{Context, Result, ensure};
+use ente_photos::export::{Component, Role};
+use serde_json::Value;
+use uuid::Uuid;
+
+use super::{
+ names,
+ store::{Album, JsonRecord, Media, Placement, Properties, Store},
+};
+
+pub fn scan(root: &Path, store: &Store) -> Result<()> {
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store.db.connection().execute_batch(
+ "DELETE FROM placements; DELETE FROM albums; DELETE FROM json_records; DELETE FROM names; DELETE FROM allocations;",
+ )?;
+ for entry in fs::read_dir(root)? {
+ let entry = entry?;
+ let name = entry
+ .file_name()
+ .into_string()
+ .map_err(|_| anyhow::anyhow!("export contains a non-UTF-8 directory name"))?;
+ if name == "Trash" {
+ ensure!(
+ entry.file_type()?.is_dir(),
+ super::Conflict("at Trash: expected a directory".into())
+ );
+ for entry in fs::read_dir(entry.path())? {
+ let entry = entry?;
+ if entry.file_type()?.is_dir() {
+ let name = entry
+ .file_name()
+ .into_string()
+ .map_err(|_| anyhow::anyhow!("retained album name is not UTF-8"))?;
+ album(root, store, &format!("Trash/{name}"), true)?;
+ }
+ }
+ } else if entry.file_type()?.is_dir() {
+ album(root, store, &name, false)?;
+ }
+ }
+ let mut query = store
+ .db
+ .connection()
+ .prepare("SELECT record FROM placements WHERE json_extract(record,'$.complete')=0")?;
+ let mut rows = query.query([])?;
+ while let Some(row) = rows.next()? {
+ let mut placement: Placement = serde_json::from_str(&row.get::<_, String>(0)?)?;
+ let template = placement
+ .metadata
+ .iter()
+ .find(|record| !record.is_null())
+ .context("placement has no metadata")?
+ .clone();
+ for (media, record) in placement.media.iter().zip(&mut placement.metadata) {
+ if record.is_null() {
+ *record = template.clone();
+ record["title"] = Value::String(media.component.path.clone());
+ record["ente"]["component"] = serde_json::to_value(&media.component.role)?;
+ }
+ }
+ store.save_placement(&placement)?;
+ }
+ drop(rows);
+ drop(query);
+ transaction.commit()?;
+ Ok(())
+}
+
+fn album(root: &Path, store: &Store, folder: &str, retained: bool) -> Result<()> {
+ let path = names::check(root, &format!("{folder}/metadata.json"))?;
+ if !path.try_exists()? {
+ return Ok(());
+ }
+ let record = read_record(root, store, &format!("{folder}/metadata.json"))?;
+ let Some(identity) = record.get("ente") else {
+ return Ok(());
+ };
+ let id = parse_id(identity, "albumID")?;
+ let name = record["title"]
+ .as_str()
+ .context("album record has no title")?
+ .to_owned();
+ ensure!(
+ [
+ "album",
+ "folder",
+ "favorites",
+ "uncategorized",
+ "defaultHidden",
+ "quicklink"
+ ]
+ .contains(&identity["type"].as_str().unwrap_or_default()),
+ "unsupported album type in {folder}"
+ );
+ ensure!(
+ ["visible", "archived", "hidden"]
+ .contains(&identity["visibility"].as_str().unwrap_or_default()),
+ "unsupported album visibility in {folder}"
+ );
+ ensure!(
+ retained || store.album(id, false)?.is_none(),
+ super::Conflict(format!("active album ID {id}"))
+ );
+ let album = Album {
+ key: Uuid::new_v4().to_string(),
+ id,
+ name,
+ path: folder.into(),
+ retained,
+ metadata: record,
+ initialized: true,
+ };
+ let (parent, name) = folder.rsplit_once('/').unwrap_or(("", folder));
+ ensure!(
+ retained || !names::reserved(name, true),
+ "reserved active album path {folder}"
+ );
+ ensure!(
+ names::reserve(store, parent, name, &album.key, "album")?,
+ super::Conflict(format!("overlapping album paths at {folder}"))
+ );
+ store.save_album(&album)?;
+ let metadata = names::check(root, &format!("{folder}/metadata"))?;
+ if !metadata.try_exists()? {
+ return Ok(());
+ }
+ ensure!(
+ metadata.is_dir(),
+ super::Conflict(format!(
+ "at {}: expected metadata directory",
+ metadata.display()
+ ))
+ );
+ for entry in fs::read_dir(metadata)? {
+ let entry = entry?;
+ if entry.path().extension().is_none_or(|ext| ext != "json") {
+ continue;
+ }
+ let name = entry
+ .file_name()
+ .into_string()
+ .map_err(|_| anyhow::anyhow!("sidecar name is not UTF-8"))?;
+ let path = names::check(root, &format!("{folder}/metadata/{name}"))?;
+ ensure!(
+ entry.file_type()?.is_file(),
+ super::Conflict(format!("at {}: expected a sidecar", path.display()))
+ );
+ let record = read_record(root, store, &format!("{folder}/metadata/{name}"))?;
+ let Some(identity) = record.get("ente") else {
+ continue;
+ };
+ let file = parse_id(identity, "fileID")?;
+ let display_name = identity["name"]
+ .as_str()
+ .context("file record has no name")?;
+ chrono::DateTime::parse_from_rfc3339(
+ identity["creationTime"]
+ .as_str()
+ .context("file record has no creationTime")?,
+ )?;
+ let kind = identity["type"]
+ .as_str()
+ .context("file record has no type")?;
+ ensure!(
+ ["image", "video", "livePhoto", "unknown"].contains(&kind),
+ "unsupported file type for {file}"
+ );
+ let mut components: Vec<Component> =
+ serde_json::from_value(identity["components"].clone())?;
+ ensure!(
+ if kind == "livePhoto" {
+ components.len() == 2
+ && components.iter().any(|c| c.role == Role::Image)
+ && components.iter().any(|c| c.role == Role::Video)
+ } else {
+ components.len() == 1 && components[0].role == Role::Original
+ },
+ "invalid components for file {file}"
+ );
+ components.sort_by_key(|component| match component.role {
+ Role::Original => 0,
+ Role::Image => 1,
+ Role::Video => 2,
+ });
+ for component in &components {
+ names::relative(&component.path)?;
+ ensure!(
+ !component.path.contains('/') && !names::reserved(&component.path, false),
+ "invalid component path for file {file}"
+ );
+ ensure!(
+ ente_core::b64::decode(&component.hash)?.len() == 64,
+ "invalid component hash for file {file}"
+ );
+ }
+ ensure!(
+ components.len() == 1
+ || names::folded(&components[0].path) != names::folded(&components[1].path),
+ "overlapping Live Photo components for file {file}"
+ );
+ let own: Role = serde_json::from_value(identity["component"].clone())?;
+ let index = components
+ .iter()
+ .position(|c| c.role == own)
+ .context("undeclared sidecar component")?;
+ ensure!(
+ record["title"].as_str() == Some(components[index].path.as_str())
+ && name == format!("{}.json", components[index].path),
+ "contradictory sidecar location for file {file}"
+ );
+ let existing = if retained {
+ let mut query = store.db.connection().prepare(
+ "SELECT record FROM placements WHERE album=?1 AND file=?2 AND retained=1",
+ )?;
+ let mut found = None;
+ let mut rows = query.query(rusqlite::params![id, file])?;
+ while let Some(row) = rows.next()? {
+ let candidate: Placement = serde_json::from_str(&row.get::<_, String>(0)?)?;
+ if candidate.folder == folder
+ && candidate
+ .media
+ .iter()
+ .map(|m| &m.component)
+ .eq(components.iter())
+ {
+ found = Some(candidate);
+ break;
+ }
+ }
+ found
+ } else {
+ store.placement(id, file)?
+ };
+ let mut placement = existing.unwrap_or_else(|| Placement {
+ id: Uuid::new_v4().to_string(),
+ album: id,
+ folder: folder.into(),
+ file,
+ retained,
+ name: display_name.into(),
+ original: String::new(),
+ media: components
+ .iter()
+ .cloned()
+ .map(|component| Media {
+ component,
+ properties: None,
+ intended_time: None,
+ })
+ .collect(),
+ metadata: vec![Value::Null; components.len()],
+ complete: false,
+ retaining: false,
+ });
+ let owner = if retained {
+ placement.id.clone()
+ } else {
+ format!("file:{id}:{file}")
+ };
+ for component in &components {
+ ensure!(
+ names::reserve(store, folder, &component.path, &owner, kind)?,
+ super::Conflict(format!("overlapping media paths in {folder}"))
+ );
+ }
+ ensure!(
+ placement.folder == folder
+ && placement
+ .media
+ .iter()
+ .map(|m| &m.component)
+ .eq(components.iter()),
+ super::Conflict(format!("active placement for file {file} in album {id}"))
+ );
+ ensure!(
+ placement.metadata[index].is_null(),
+ "duplicate component identity for file {file}"
+ );
+ if let Some(other) = placement.metadata.iter().find(|value| !value.is_null()) {
+ let mut a = other.clone();
+ let mut b = record.clone();
+ a.as_object_mut()
+ .context("invalid sidecar")?
+ .remove("title");
+ b.as_object_mut()
+ .context("invalid sidecar")?
+ .remove("title");
+ a["ente"]
+ .as_object_mut()
+ .context("invalid Ente record")?
+ .remove("component");
+ b["ente"]
+ .as_object_mut()
+ .context("invalid Ente record")?
+ .remove("component");
+ a["ente"]["components"] = serde_json::to_value(&components)?;
+ b["ente"]["components"] = a["ente"]["components"].clone();
+ ensure!(a == b, "contradictory Live Photo metadata for file {file}");
+ }
+ placement.metadata[index] = record;
+ placement.complete = placement.metadata.iter().all(|record| !record.is_null());
+ store.save_placement(&placement)?;
+ }
+ Ok(())
+}
+
+fn read_record(root: &Path, store: &Store, relative: &str) -> Result<Value> {
+ let path = names::check(root, relative)?;
+ let bytes = fs::read(&path)?;
+ let record: Value = serde_json::from_slice(&bytes)?;
+ if record.get("ente").is_some() {
+ let cached = JsonRecord {
+ hash: ente_core::b64::encode(&ente_core::crypto::hash::hash(&bytes, Some(64), None)?),
+ properties: Properties::read(&path)?,
+ };
+ store.db.connection().execute(
+ "INSERT INTO json_records VALUES(?1,?2)",
+ rusqlite::params![relative, serde_json::to_string(&cached)?],
+ )?;
+ }
+ Ok(record)
+}
+
+fn parse_id(value: &Value, field: &str) -> Result<i64> {
+ let text = value[field]
+ .as_str()
+ .with_context(|| format!("missing {field}"))?;
+ let id: i64 = text.parse().with_context(|| format!("invalid {field}"))?;
+ ensure!(id > 0 && id.to_string() == text, "invalid {field}");
+ Ok(id)
+}
diff --git a/rust/apps/cli-next/src/export/allocation.rs b/rust/apps/cli-next/src/export/allocation.rs
new file mode 100644
index 0000000000..a07125eb1f
--- /dev/null
+++ b/rust/apps/cli-next/src/export/allocation.rs
@@ -0,0 +1,156 @@
+use anyhow::{Result, ensure};
+use rusqlite::params;
+use serde::{Deserialize, Serialize};
+
+use super::{
+ names,
+ store::{Album, Placement, Store},
+};
+
+#[derive(Serialize, Deserialize)]
+struct Allocation {
+ parent: String,
+ name: String,
+ kind: String,
+ extensions: Option<(String, String)>,
+ paths: Vec<String>,
+}
+
+pub fn seed(store: &Store) -> Result<()> {
+ let transaction = store.db.connection().unchecked_transaction()?;
+ names::reserve(store, "", "export.json", "root", "album")?;
+ names::reserve(store, "", "Trash", "root", "album")?;
+ let mut after = String::new();
+ loop {
+ let album: Option<Album> = store.json(
+ "SELECT record FROM albums WHERE key>?1 ORDER BY key LIMIT 1",
+ [&after],
+ )?;
+ let Some(album) = album else { break };
+ after = album.key.clone();
+ let (parent, name) = album.path.rsplit_once('/').unwrap_or(("", &album.path));
+ ensure!(
+ names::reserve(store, parent, name, &album.key, "album")?,
+ super::Conflict(format!("overlapping album paths at {}", album.path))
+ );
+ }
+ after.clear();
+ loop {
+ let placement: Option<Placement> = store.json(
+ "SELECT record FROM placements WHERE id>?1 ORDER BY id LIMIT 1",
+ [&after],
+ )?;
+ let Some(placement) = placement else { break };
+ after = placement.id.clone();
+ let owner = if placement.retained {
+ placement.id.clone()
+ } else {
+ format!("file:{}:{}", placement.album, placement.file)
+ };
+ for media in &placement.media {
+ ensure!(
+ names::reserve(
+ store,
+ &placement.folder,
+ &media.component.path,
+ &owner,
+ placement.kind()?
+ )?,
+ super::Conflict(format!("overlapping media in {}", placement.folder))
+ );
+ }
+ }
+ store.db.connection().execute("DELETE FROM allocations WHERE EXISTS(SELECT 1 FROM desired_albums a WHERE a.selected=1 AND a.ready=1 AND a.failure IS NULL AND allocations.owner GLOB 'file:'||a.id||':*' AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE allocations.owner='file:'||f.album||':'||f.file))", [])?;
+ store.db.connection().execute("DELETE FROM allocations WHERE owner IN (SELECT coalesce(b.key,'active:'||a.id) FROM desired_albums a LEFT JOIN albums b ON b.id=a.id AND b.retained=0 WHERE a.selected=1 AND a.ready=1 AND a.failure IS NULL AND a.record IS NULL)", [])?;
+ after.clear();
+ loop {
+ let next: Option<(String, String)> = store
+ .db
+ .connection()
+ .query_row(
+ "SELECT owner,record FROM allocations WHERE owner>?1 ORDER BY owner LIMIT 1",
+ [&after],
+ |r| Ok((r.get(0)?, r.get(1)?)),
+ )
+ .optional()?;
+ let Some((owner, record)) = next else { break };
+ after = owner.clone();
+ let allocation: Allocation = serde_json::from_str(&record)?;
+ for path in &allocation.paths {
+ ensure!(
+ names::reserve(store, &allocation.parent, path, &owner, &allocation.kind)?,
+ super::Conflict(format!(
+ "overlapping pending names in {}",
+ allocation.parent
+ ))
+ );
+ }
+ }
+ transaction.commit()?;
+ Ok(())
+}
+
+pub fn choose(
+ store: &Store,
+ parent: &str,
+ name: &str,
+ kind: &str,
+ extensions: Option<(&str, &str)>,
+ owner: &str,
+) -> Result<Vec<String>> {
+ let previous: Option<Allocation> =
+ store.json("SELECT record FROM allocations WHERE owner=?1", [owner])?;
+ let extensions =
+ extensions.map(|(image, video)| (names::portable(image), names::portable(video)));
+ if let Some(previous) = previous
+ && previous.parent == parent
+ && previous.name == name
+ && previous.kind == kind
+ && previous.extensions == extensions
+ {
+ return Ok(previous.paths);
+ }
+ let transaction = rusqlite::Transaction::new_unchecked(
+ store.db.connection(),
+ rusqlite::TransactionBehavior::Immediate,
+ )?;
+ let paths = names::allocate(
+ store,
+ parent,
+ name,
+ kind,
+ extensions
+ .as_ref()
+ .map(|(image, video)| (image.as_str(), video.as_str())),
+ owner,
+ )?;
+ let allocation = Allocation {
+ parent: parent.into(),
+ name: name.into(),
+ kind: kind.into(),
+ extensions,
+ paths: paths.clone(),
+ };
+ store.db.connection().execute("INSERT INTO allocations VALUES(?1,?2) ON CONFLICT(owner) DO UPDATE SET record=excluded.record", params![owner,serde_json::to_string(&allocation)?])?;
+ transaction.commit()?;
+ Ok(paths)
+}
+
+pub fn clear(store: &Store, owner: &str) -> Result<()> {
+ store
+ .db
+ .connection()
+ .execute("DELETE FROM allocations WHERE owner=?1", [owner])?;
+ Ok(())
+}
+
+pub fn relocate(store: &Store, old: &str, new: &str) -> Result<()> {
+ store.db.connection().execute(
+ "UPDATE names SET parent=?1 WHERE parent=?2",
+ params![new, old],
+ )?;
+ store.db.connection().execute("UPDATE allocations SET record=json_set(record,'$.parent',?1) WHERE json_extract(record,'$.parent')=?2", params![new,old])?;
+ Ok(())
+}
+
+use rusqlite::OptionalExtension;
diff --git a/rust/apps/cli-next/src/export/fs.rs b/rust/apps/cli-next/src/export/fs.rs
new file mode 100644
index 0000000000..d5ed27d212
--- /dev/null
+++ b/rust/apps/cli-next/src/export/fs.rs
@@ -0,0 +1,381 @@
+use std::{
+ fs::{self, File, OpenOptions},
+ io::{self, Read, Write},
+ path::Path,
+ time::{Duration, UNIX_EPOCH},
+};
+
+use anyhow::{Context, Result, ensure};
+use ente_core::{
+ b64,
+ crypto::hash::{self, HashState},
+};
+use rusqlite::params;
+use serde_json::Value;
+use tokio::sync::watch;
+use uuid::Uuid;
+
+use super::{
+ names,
+ store::{JsonRecord, Operation, Properties, Store, Temporary},
+};
+
+pub struct HashWriter {
+ pub file: File,
+ digest: HashState,
+ pub size: u64,
+ cancel: watch::Receiver<bool>,
+}
+
+impl HashWriter {
+ pub fn new(file: File, cancel: &watch::Receiver<bool>) -> Result<Self> {
+ Ok(Self {
+ file,
+ digest: HashState::new(Some(64), None)?,
+ size: 0,
+ cancel: cancel.clone(),
+ })
+ }
+
+ pub fn finish(self) -> Result<(File, u64, String)> {
+ self.file.sync_all()?;
+ Ok((self.file, self.size, b64::encode(&self.digest.finalize()?)))
+ }
+}
+
+impl Write for HashWriter {
+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+ if *self.cancel.borrow() {
+ return Err(io::Error::other(super::Cancelled));
+ }
+ let written = self.file.write(bytes)?;
+ self.digest
+ .update(&bytes[..written])
+ .map_err(io::Error::other)?;
+ self.size += written as u64;
+ Ok(written)
+ }
+ fn flush(&mut self) -> io::Result<()> {
+ self.file.flush()
+ }
+}
+
+pub fn hash_cancellable(path: &Path, cancel: &watch::Receiver<bool>) -> Result<(u64, String)> {
+ let mut file = File::open(path)?;
+ let mut digest = HashState::new(Some(64), None)?;
+ let mut bytes = vec![0u8; 64 * 1024];
+ let mut size = 0;
+ loop {
+ ensure!(!*cancel.borrow(), super::Cancelled);
+ let read = file.read(&mut bytes)?;
+ if read == 0 {
+ break;
+ }
+ digest.update(&bytes[..read])?;
+ size += read as u64;
+ }
+ Ok((size, b64::encode(&digest.finalize()?)))
+}
+
+pub fn directory(root: &Path, relative: &str) -> Result<()> {
+ let path = names::check(root, relative)?;
+ match fs::symlink_metadata(&path) {
+ Ok(meta) => ensure!(
+ meta.is_dir(),
+ super::Conflict(format!("expected directory at {}", path.display()))
+ ),
+ Err(error) if error.kind() == io::ErrorKind::NotFound => fs::create_dir_all(&path)?,
+ Err(error) => return Err(error.into()),
+ }
+ Ok(())
+}
+
+pub fn temporary(
+ root: &Path,
+ store: &Store,
+ directory: &str,
+ file_id: Option<i64>,
+) -> Result<(Temporary, File)> {
+ self::directory(root, directory)?;
+ let path = format!("{directory}/.ente-{}.part", Uuid::new_v4().simple());
+ let album:i64=store.db.connection().query_row("SELECT id FROM albums WHERE ?1=path OR substr(?1,1,length(path)+1)=path||'/' ORDER BY length(path) DESC LIMIT 1",[directory],|r|r.get(0))?;
+ let temporary = Temporary {
+ album,
+ path,
+ file: file_id,
+ original: None,
+ role: None,
+ extension: None,
+ destination: None,
+ hash: None,
+ size: None,
+ };
+ store.temporary(&temporary)?;
+ let file = OpenOptions::new()
+ .write(true)
+ .read(true)
+ .create_new(true)
+ .open(names::check(root, &temporary.path)?);
+ match file {
+ Ok(file) => Ok((temporary, file)),
+ Err(error) => {
+ store
+ .db
+ .connection()
+ .execute("DELETE FROM temporaries WHERE path=?1", [&temporary.path])?;
+ Err(error.into())
+ }
+ }
+}
+
+pub fn publish(
+ root: &Path,
+ store: &Store,
+ mut temporary: Temporary,
+ destination: &str,
+ size: u64,
+ hash: String,
+) -> Result<()> {
+ temporary.destination = Some(destination.into());
+ temporary.size = Some(size);
+ temporary.hash = Some(hash);
+ store.temporary(&temporary)?;
+ finish_publication(root, store, &temporary, None)
+}
+
+fn finish_publication(
+ root: &Path,
+ store: &Store,
+ temporary: &Temporary,
+ verify: Option<&watch::Receiver<bool>>,
+) -> Result<()> {
+ let source = names::check(root, &temporary.path)?;
+ let destination = names::check(
+ root,
+ temporary
+ .destination
+ .as_deref()
+ .context("publication has no destination")?,
+ )?;
+ match fs::symlink_metadata(&source) {
+ Ok(metadata) => {
+ ensure!(
+ metadata.is_file(),
+ super::Conflict(format!("at {}: expected temporary file", source.display()))
+ );
+ if let Some(cancel) = verify {
+ let (size, hash) = hash_cancellable(&source, cancel)?;
+ if Some(size) != temporary.size || Some(&hash) != temporary.hash.as_ref() {
+ return discard(root, store, &temporary.path);
+ }
+ }
+ if let Ok(meta) = fs::symlink_metadata(&destination) {
+ ensure!(
+ meta.is_file(),
+ super::Conflict(format!(
+ "at {}: expected a regular file",
+ destination.display()
+ ))
+ );
+ }
+ fs::rename(&source, &destination)?;
+ sync_parent(&destination)?;
+ }
+ Err(error) if error.kind() == io::ErrorKind::NotFound => {
+ if destination.try_exists()? {
+ Properties::read(&destination)?;
+ }
+ }
+ Err(error) => return Err(error.into()),
+ }
+ store
+ .db
+ .connection()
+ .execute("DELETE FROM temporaries WHERE path=?1", [&temporary.path])?;
+ Ok(())
+}
+
+pub fn json(
+ root: &Path,
+ store: &Store,
+ path: &str,
+ value: &Value,
+ associated: bool,
+ file_id: Option<i64>,
+) -> Result<bool> {
+ let mut bytes = serde_json::to_vec_pretty(value)?;
+ bytes.push(b'\n');
+ let hash = b64::encode(&hash::hash(&bytes, Some(64), None)?);
+ let destination = names::check(root, path)?;
+ let recorded: Option<JsonRecord> =
+ store.json("SELECT record FROM json_records WHERE path=?1", [path])?;
+ let properties = match fs::symlink_metadata(&destination) {
+ Ok(_) => Some(Properties::read(&destination)?),
+ Err(error) if error.kind() == io::ErrorKind::NotFound => None,
+ Err(error) => return Err(error.into()),
+ };
+ ensure!(
+ associated || properties.is_none(),
+ super::Conflict(format!("at {}: unrelated JSON", destination.display()))
+ );
+ if let (Some(recorded), Some(properties)) = (&recorded, &properties)
+ && recorded.hash == hash
+ && recorded.properties == *properties
+ {
+ return Ok(false);
+ }
+ let parent = path
+ .rsplit_once('/')
+ .context("JSON has no album directory")?
+ .0;
+ let (temporary, mut file) = self::temporary(root, store, parent, file_id)?;
+ file.write_all(&bytes)?;
+ file.sync_all()?;
+ drop(file);
+ publish(
+ root,
+ store,
+ temporary,
+ path,
+ bytes.len() as u64,
+ hash.clone(),
+ )?;
+ let record = JsonRecord {
+ hash,
+ properties: Properties::read(&destination)?,
+ };
+ store.db.connection().execute("INSERT INTO json_records VALUES(?1,?2) ON CONFLICT(path) DO UPDATE SET record=excluded.record",params![path,serde_json::to_string(&record)?])?;
+ Ok(true)
+}
+
+pub fn set_time(path: &Path, micros: i64) -> Result<Properties> {
+ let duration = Duration::from_micros(micros.unsigned_abs());
+ let time = if micros >= 0 {
+ UNIX_EPOCH.checked_add(duration)
+ } else {
+ UNIX_EPOCH.checked_sub(duration)
+ }
+ .context("media timestamp is outside filesystem range")?;
+ File::options().write(true).open(path)?.set_modified(time)?;
+ Properties::read(path)
+}
+
+pub fn operations(root: &Path, store: &Store, album: i64, file: Option<i64>) -> Result<()> {
+ loop {
+ let operation:Option<(i64,String)>=store.db.connection().query_row("SELECT id,record FROM operations WHERE album=?1 AND file IS ?2 ORDER BY id LIMIT 1",params![album,file],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
+ let Some((id, json)) = operation else { break };
+ match serde_json::from_str::<Operation>(&json)? {
+ Operation::Move {
+ source,
+ destination,
+ ..
+ } => {
+ let source = names::check(root, &source)?;
+ let destination = names::check(root, &destination)?;
+ if source.try_exists()? {
+ ensure!(
+ !destination.try_exists()? || names::same_path(&source, &destination),
+ super::Conflict(format!(
+ "at {}: occupied rename target",
+ destination.display()
+ ))
+ );
+ if let Some(parent) = destination.parent() {
+ fs::create_dir_all(parent)?;
+ }
+ fs::rename(&source, &destination)?;
+ sync_parent(&destination)?;
+ }
+ }
+ Operation::Remove { path, .. } => {
+ let path = names::check(root, &path)?;
+ match fs::remove_file(&path) {
+ Ok(()) => {}
+ Err(e) if e.kind() == io::ErrorKind::NotFound => {}
+ Err(e) => return Err(e.into()),
+ }
+ }
+ }
+ store
+ .db
+ .connection()
+ .execute("DELETE FROM operations WHERE id=?1", [id])?;
+ }
+ Ok(())
+}
+
+pub fn recover(root: &Path, store: &Store, cancel: &watch::Receiver<bool>) -> Result<()> {
+ let mut query=store.db.connection().prepare("SELECT DISTINCT o.album,o.file FROM operations o JOIN desired_albums a ON a.id=o.album WHERE a.selected=1 AND a.ready=1 AND a.failure IS NULL AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=o.album AND f.file=o.file AND f.failure IS NOT NULL) ORDER BY o.album,o.file")?;
+ let mut rows = query.query([])?;
+ while let Some(row) = rows.next()? {
+ ensure!(!*cancel.borrow(), super::Cancelled);
+ let album = row.get(0)?;
+ let file = row.get(1)?;
+ if let Err(error) = operations(root, store, album, file) {
+ block(store, album, file, error)?;
+ }
+ }
+ let mut after = String::new();
+ loop {
+ ensure!(!*cancel.borrow(), super::Cancelled);
+ let temporary:Option<Temporary>=store.json("SELECT record FROM temporaries WHERE path>?1 AND (json_extract(record,'$.destination') IS NOT NULL OR json_extract(record,'$.hash') IS NULL) AND json_extract(record,'$.album') IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=json_extract(temporaries.record,'$.album') AND f.file=json_extract(temporaries.record,'$.file') AND f.failure IS NOT NULL) ORDER BY path LIMIT 1",[&after])?;
+ let Some(temporary) = temporary else { break };
+ after = temporary.path.clone();
+ let result = if temporary.destination.is_some() {
+ finish_publication(root, store, &temporary, Some(cancel))
+ } else {
+ discard(root, store, &temporary.path)
+ };
+ if let Err(error) = result {
+ block(store, temporary.album, temporary.file, error)?;
+ }
+ }
+ Ok(())
+}
+
+fn block(store: &Store, album: i64, file: Option<i64>, error: anyhow::Error) -> Result<()> {
+ let unit = if let Some(file) = file {
+ format!("file:{album}:{file}")
+ } else {
+ format!("album:{album}")
+ };
+ super::report(store, &unit, &error)?;
+ if file.is_none() {
+ store.db.connection().execute(
+ "UPDATE desired_albums SET failure=?1 WHERE id=?2",
+ params![error.to_string(), album],
+ )?;
+ }
+ store.db.connection().execute(
+ "UPDATE desired_files SET failure=?1 WHERE album=?2 AND (?3 IS NULL OR file=?3)",
+ params![error.to_string(), album, file],
+ )?;
+ if super::fatal(&error) {
+ return Err(error);
+ }
+ Ok(())
+}
+
+pub fn discard(root: &Path, store: &Store, path: &str) -> Result<()> {
+ let absolute = names::check(root, path)?;
+ match fs::remove_file(absolute) {
+ Ok(()) => {}
+ Err(e) if e.kind() == io::ErrorKind::NotFound => {}
+ Err(e) => return Err(e.into()),
+ }
+ store
+ .db
+ .connection()
+ .execute("DELETE FROM temporaries WHERE path=?1", [path])?;
+ Ok(())
+}
+
+pub fn sync_parent(path: &Path) -> Result<()> {
+ #[cfg(unix)]
+ if let Some(parent) = path.parent() {
+ File::open(parent)?.sync_all()?;
+ }
+ Ok(())
+}
+
+use rusqlite::OptionalExtension;
diff --git a/rust/apps/cli-next/src/export/mod.rs b/rust/apps/cli-next/src/export/mod.rs
new file mode 100644
index 0000000000..c9be2cb10e
--- /dev/null
+++ b/rust/apps/cli-next/src/export/mod.rs
@@ -0,0 +1,567 @@
+mod adopt;
+mod allocation;
+mod fs;
+mod names;
+mod reconcile;
+mod store;
+mod transfer;
+
+use std::{
+ fs::{File, OpenOptions},
+ io::Write,
+ path::{Path, PathBuf},
+ sync::Arc,
+ time::Instant,
+};
+
+use anyhow::{Context, Result, bail, ensure};
+use ente_photos::export::Root;
+use rusqlite::{OptionalExtension, params};
+use serde_json::json;
+
+use tokio::sync::watch;
+
+use crate::{
+ api,
+ args::{ExportArgs, Options, Product},
+ db, home, output,
+ replica::{AlbumRecord, FileRecord, Replica, canonical_id},
+ vault::{DbKey, State},
+};
+use store::Store;
+
+#[derive(Debug, thiserror::Error)]
+#[error("conflict: {0}")]
+pub struct Conflict(String);
+
+#[derive(Debug, thiserror::Error)]
+#[error("export cancelled")]
+pub struct Cancelled;
+
+pub async fn run(args: ExportArgs, selected: Option<&str>, options: &Options) -> Result<()> {
+ ensure!(
+ !options.offline,
+ "export requires network access; remove --offline"
+ );
+ let destination = destination(&args.destination)?;
+ let mut state = State::load()?;
+ let index = state.resolve(selected)?;
+ let account = state.accounts.swap_remove(index);
+ let session = api::session(&account, Product::Photos)?;
+ let expected_root = Root::new(&session.master_key)?;
+ let mut root_lock = open_root(&destination, &expected_root)?;
+ ensure!(
+ !args.adopt || root_lock.is_some(),
+ "there is no export to adopt"
+ );
+ let account_home = home::lock_account(account.storage_id, true)?;
+ let account = State::load()?
+ .accounts
+ .into_iter()
+ .find(|a| a.storage_id == account.storage_id)
+ .context("account was removed while waiting for access")?;
+ let session = api::session(&account, Product::Photos)?;
+ ensure!(
+ Root::new(&session.master_key)?.source_verifier == expected_root.source_verifier,
+ "account keys changed while waiting for access"
+ );
+ home::create(&account_home.path)?;
+ let db_path = Store::path(&account_home.path, &destination)?;
+ let existed = db_path.try_exists()?;
+ ensure!(
+ root_lock.is_none() || existed || args.adopt,
+ "this export is not associated with this CLI home; use --adopt"
+ );
+ let store = Store::open(&db_path, &account.db_key, &destination, true)?;
+ let bound: bool =
+ store
+ .db
+ .connection()
+ .query_row("SELECT bound FROM export WHERE id=1", [], |r| r.get(0))?;
+ ensure!(
+ root_lock.is_none() || bound || args.adopt,
+ "this export requires --adopt"
+ );
+ store.reset_desired()?;
+ if root_lock.is_some() && !bound {
+ adopt::scan(&destination, &store)?;
+ }
+ let mut source_db = db::open(&account_home.path, &account.db_key, true)?;
+ let mut replica = Replica::new(&mut source_db, session.user_id);
+ eprintln!("Refreshing album metadata.");
+ replica.sync_collections(&session).await?;
+ let mut after = 0;
+ loop {
+ let next: Option<(i64, String, Option<String>)> = replica.db.read(|db| {
+ Ok(db
+ .query_row(
+ "SELECT id,record,name FROM photos_collections WHERE id>?1 ORDER BY id LIMIT 1",
+ [after],
+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
+ )
+ .optional()?)
+ })?;
+ let Some((id, record, name)) = next else {
+ break;
+ };
+ after = id;
+ let record: AlbumRecord = serde_json::from_str(&record)?;
+ store.desired_album(&record, session.user_id)?;
+ if record.album(session.user_id).is_err() {
+ store.db.connection().execute(
+ "UPDATE desired_albums SET name=?1 WHERE id=?2",
+ params![name, id],
+ )?;
+ }
+ }
+ store.db.connection().execute("INSERT OR IGNORE INTO desired_albums(id,name) SELECT id,name FROM albums ORDER BY retained",[])?;
+ store.db.connection().execute("INSERT OR IGNORE INTO desired_albums(id,name) SELECT CAST(substr(owner,8) AS INTEGER),json_extract(record,'$.name') FROM allocations WHERE owner GLOB 'active:*'",[])?;
+ select(&store, &args)?;
+ if root_lock.is_none() {
+ root_lock = Some(initialize(&destination, &expected_root)?);
+ }
+ store
+ .db
+ .connection()
+ .execute("UPDATE export SET bound=1 WHERE id=1", [])?;
+ let (cancel_send, mut cancel) = watch::channel(false);
+ let signal = tokio::spawn(async move {
+ if tokio::signal::ctrl_c().await.is_ok() {
+ cancel_send.send_replace(true);
+ }
+ });
+ let refresh_cancel = cancel.clone();
+ let refreshed = tokio::select! {
+ result=refresh(&store,&mut replica,&session,args.retry_failed,&refresh_cancel)=>result,
+ _=cancel.changed()=>Err(Cancelled.into()),
+ };
+ drop(replica);
+ drop(source_db);
+ drop(account_home);
+ let context = Arc::new(transfer::Context {
+ root: destination.clone(),
+ db_path,
+ db_key: DbKey(account.db_key.0),
+ session,
+ });
+ let result = async {
+ refreshed?;
+ fs::recover(&destination, &store, &cancel)?;
+ allocation::seed(&store)?;
+ reconcile::albums(&destination, &store, context.session.user_id)?;
+ transfers(context.clone(), &store, cancel.clone()).await?;
+ if *cancel.borrow() {
+ bail!(Cancelled)
+ }
+ reconcile::retain_removed(&destination, &store)?;
+ Ok::<(), anyhow::Error>(())
+ }
+ .await;
+ signal.abort();
+ if let Err(error) = result {
+ report(&store, "run", &error)?;
+ }
+ cleanup(&destination, &store)?;
+ let complete = summary(&store, &destination, options.json)?;
+ drop(root_lock);
+ ensure!(complete, "export is incomplete");
+ Ok(())
+}
+
+fn destination(path: &Path) -> Result<PathBuf> {
+ let absolute = if path.is_absolute() {
+ path.to_owned()
+ } else {
+ std::env::current_dir()?.join(path)
+ };
+ let mut base = absolute.as_path();
+ let mut tail = Vec::new();
+ while !base.try_exists()? {
+ tail.push(
+ base.file_name()
+ .context("destination has no directory name")?
+ .to_owned(),
+ );
+ base = base
+ .parent()
+ .context("destination has no existing parent")?;
+ }
+ if tail.is_empty() {
+ ensure!(
+ !std::fs::symlink_metadata(base)?.is_symlink(),
+ "export destination is a symlink"
+ );
+ }
+ let mut resolved = std::fs::canonicalize(base)?;
+ for part in tail.into_iter().rev() {
+ resolved.push(part);
+ }
+ Ok(resolved)
+}
+
+fn open_root(destination: &Path, expected: &Root) -> Result<Option<File>> {
+ if !destination.try_exists()? {
+ return Ok(None);
+ }
+ ensure!(
+ destination.is_dir(),
+ "export destination is not a directory"
+ );
+ let path = names::check(destination, "export.json")?;
+ if !path.try_exists()? {
+ ensure!(
+ std::fs::read_dir(destination)?.next().is_none(),
+ "nonempty destination has no recognized export.json"
+ );
+ return Ok(None);
+ }
+ let file = OpenOptions::new().read(true).write(true).open(&path)?;
+ file.try_lock()
+ .context("another writer is maintaining this export")?;
+ let record: Root = serde_json::from_reader(&file).context("invalid export.json")?;
+ ensure!(
+ record.format == expected.format && record.version == expected.version,
+ "unsupported export format"
+ );
+ ensure!(
+ record.source_verifier == expected.source_verifier,
+ "this export belongs to another source account"
+ );
+ Ok(Some(file))
+}
+
+fn initialize(destination: &Path, root: &Root) -> Result<File> {
+ std::fs::create_dir_all(destination)?;
+ let path = names::check(destination, "export.json")?;
+ let mut file = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create_new(true)
+ .open(&path)
+ .context("destination was initialized by another writer; retry")?;
+ file.lock().context("cannot lock the new export")?;
+ ensure!(
+ std::fs::read_dir(destination)?
+ .all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),
+ "destination became nonempty during initialization"
+ );
+ serde_json::to_writer_pretty(&mut file, root)?;
+ file.write_all(b"\n")?;
+ file.sync_all()?;
+ fs::sync_parent(&path)?;
+ Ok(file)
+}
+
+fn select(store: &Store, args: &ExportArgs) -> Result<()> {
+ if args.album.is_empty() {
+ store
+ .db
+ .connection()
+ .execute("UPDATE desired_albums SET selected=1", [])?;
+ }
+ for (selectors, include) in [(&args.album, true), (&args.exclude_album, false)] {
+ for selector in selectors {
+ let mut query = store.db.connection().prepare(
+ "SELECT id,name FROM desired_albums WHERE id=?1 OR name=?2 ORDER BY id LIMIT 11",
+ )?;
+ let candidates: Vec<(i64, Option<String>)> = query
+ .query_map(params![canonical_id(selector), selector], |r| {
+ Ok((r.get(0)?, r.get(1)?))
+ })?
+ .collect::<rusqlite::Result<_>>()?;
+ ensure!(!candidates.is_empty(), "no album matches {selector:?}");
+ if candidates.len() != 1 {
+ bail!(
+ "album {selector:?} is ambiguous: {}",
+ candidates
+ .iter()
+ .map(|(id, name)| format!(
+ "{id} {:?}",
+ name.as_deref().unwrap_or("unavailable")
+ ))
+ .collect::<Vec<_>>()
+ .join(", ")
+ );
+ }
+ store.db.connection().execute(
+ "UPDATE desired_albums SET selected=?1 WHERE id=?2",
+ params![include, candidates[0].0],
+ )?;
+ }
+ }
+ Ok(())
+}
+
+async fn refresh(
+ store: &Store,
+ replica: &mut Replica<'_>,
+ session: &ente_core::Session,
+ retry: bool,
+ cancel: &watch::Receiver<bool>,
+) -> Result<()> {
+ let need_favorites: bool = store.db.connection().query_row(
+ "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND record IS NOT NULL)",
+ [],
+ |r| r.get(0),
+ )?;
+ let mut favorites_known = true;
+ let mut after = 0;
+ loop {
+ let next:Option<(i64,String,bool)>=store.db.connection().query_row("SELECT id,record,selected FROM desired_albums WHERE id>?1 AND record IS NOT NULL ORDER BY id LIMIT 1",[after],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?;
+ let Some((id, record, selected)) = next else {
+ break;
+ };
+ after = id;
+ let record: AlbumRecord = serde_json::from_str(&record)?;
+ if !need_favorites
+ || record.remote.kind != "favorites"
+ || record.remote.owner.id != session.user_id
+ {
+ continue;
+ }
+ if *cancel.borrow() {
+ bail!(Cancelled)
+ }
+ replica.retry_album(session, id, retry)?;
+ let result = async {
+ let record = replica
+ .album_record(id)?
+ .context("missing favorites source")?;
+ let album = record.album(session.user_id)?;
+ replica.sync_album_files(session, &album, retry).await?;
+ store.desired_album(&record, session.user_id)?;
+ Ok::<_, anyhow::Error>(())
+ }
+ .await;
+ if let Err(error) = result {
+ favorites_known = false;
+ if selected {
+ report(store, &format!("album:{id}"), &error)?;
+ }
+ if fatal(&error) {
+ return Err(error);
+ }
+ }
+ }
+ let selected: i64 = store.db.connection().query_row(
+ "SELECT count(*) FROM desired_albums WHERE selected=1",
+ [],
+ |r| r.get(0),
+ )?;
+ eprintln!("Refreshing files in {selected} selected albums.");
+ let mut progress = Instant::now();
+ let mut refreshed = 0;
+ after = 0;
+ loop {
+ let next:Option<(i64,Option<String>)>=store.db.connection().query_row("SELECT id,record FROM desired_albums WHERE selected=1 AND id>?1 ORDER BY id LIMIT 1",[after],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
+ let Some((id, record)) = next else { break };
+ after = id;
+ if *cancel.borrow() {
+ bail!(Cancelled)
+ }
+ let Some(_) = record else {
+ store
+ .db
+ .connection()
+ .execute("UPDATE desired_albums SET ready=1 WHERE id=?1", [id])?;
+ continue;
+ };
+ let result=async {
+ replica.retry_album(session,id,retry)?;
+ let record=replica.album_record(id)?.context("missing album source")?;
+ let album=record.album(session.user_id)?;
+ replica.sync_album_files(session,&album,retry).await?;
+ store.desired_album(&record,session.user_id)?;
+ let mut after_file=0;
+ loop {
+ let records:Vec<FileRecord>=replica.db.read(|db| {
+ let mut query=db.prepare("SELECT record FROM photos_files WHERE collection_id=?1 AND id>?2 ORDER BY id LIMIT 128")?;
+ Ok(query.query_map(params![id,after_file],|r|crate::replica::read_json(r,0))?.collect::<rusqlite::Result<_>>()?)
+ })?;
+ if records.is_empty() {break;}
+ let transaction=store.db.connection().unchecked_transaction()?;
+ for record in records {
+ after_file=record.remote.id;
+ let favorited=if favorites_known {Some(replica.db.read(|db|Ok(db.query_row("SELECT EXISTS(SELECT 1 FROM photos_files f JOIN photos_collections c ON c.id=f.collection_id WHERE f.id=?1 AND json_extract(c.record,'$.remote.type')='favorites' AND json_extract(c.record,'$.remote.owner.id')=?2)",params![record.remote.id,session.user_id],|r|r.get::<_,bool>(0))?))?)} else {None};
+ store.desired_file(id,&record,session.user_id,favorited)?;
+ if !favorites_known {store.db.connection().execute("UPDATE desired_files SET failure='favorite state is incomplete' WHERE album=?1 AND file=?2",params![id,record.remote.id])?;}
+ }
+ transaction.commit()?;
+ }
+ store.db.connection().execute("UPDATE desired_albums SET ready=1 WHERE id=?1",[id])?;
+ Ok::<_,anyhow::Error>(())
+ }.await;
+ if let Err(error) = result {
+ store.db.connection().execute(
+ "UPDATE desired_albums SET failure=?1 WHERE id=?2",
+ params![error.to_string(), id],
+ )?;
+ report(store, &format!("album:{id}"), &error)?;
+ if fatal(&error) {
+ return Err(error);
+ }
+ }
+ refreshed += 1;
+ if progress.elapsed().as_secs() >= 2 {
+ eprintln!("Refreshed {refreshed}/{selected} albums.");
+ progress = Instant::now();
+ }
+ }
+ let mut query = store
+ .db
+ .connection()
+ .prepare("SELECT album,file,failure FROM desired_files WHERE failure IS NOT NULL")?;
+ let mut rows = query.query([])?;
+ while let Some(row) = rows.next()? {
+ report(
+ store,
+ &format!("file:{}:{}", row.get::<_, i64>(0)?, row.get::<_, i64>(1)?),
+ &anyhow::anyhow!(row.get::<_, String>(2)?),
+ )?;
+ }
+ Ok(())
+}
+
+async fn transfers(
+ context: Arc<transfer::Context>,
+ store: &Store,
+ cancel: watch::Receiver<bool>,
+) -> Result<()> {
+ let expected: i64 =
+ store
+ .db
+ .connection()
+ .query_row("SELECT count(*) FROM desired_files", [], |r| r.get(0))?;
+ eprintln!("Maintaining {expected} selected copies.");
+ let mut work = tokio::task::JoinSet::new();
+ let result=async {
+ let ceiling=(256*1024*1024/ente_core::crypto::stream::DECRYPTION_CHUNK_SIZE).max(1);
+ let mut admission=std::thread::available_parallelism().map(usize::from).unwrap_or(1).min(ceiling);
+ let mut measured=Instant::now();let mut completed_bytes=0u64;let mut previous_rate=0.0;
+ loop {
+ while work.len()<admission && !*cancel.borrow() {
+ let staged:i64=store.db.connection().query_row("SELECT count(DISTINCT json_extract(t.record,'$.file')) FROM temporaries t WHERE NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.file=json_extract(t.record,'$.file') AND f.running=1)",[],|r|r.get(0))?;
+ let available=staged as usize+work.len()<ceiling;
+ let staged:Option<i64>=store.db.connection().query_row("SELECT json_extract(t.record,'$.file') FROM temporaries t WHERE EXISTS(SELECT 1 FROM desired_files f WHERE f.file=json_extract(t.record,'$.file') AND f.failure IS NULL AND f.attempted=0 AND f.running=0) ORDER BY json_extract(t.record,'$.file') LIMIT 1",[],|r|r.get(0)).optional()?;
+ let next=if let Some(id)=staged {Some((id,true))} else {
+ store.db.connection().query_row("SELECT file FROM desired_files WHERE failure IS NULL AND attempted=0 AND running=0 AND deferred<=?1 ORDER BY deferred,file LIMIT 1",[available],|r|Ok((r.get(0)?,false))).optional()?
+ };
+ let Some((id,staged))=next else {break};
+ store.db.connection().execute("UPDATE desired_files SET running=1 WHERE file=?1",[id])?;
+ let context=context.clone();let cancel=cancel.clone();
+ work.spawn(async move {(id,transfer::prepare(context,id,cancel,available || staged).await)});
+ }
+ let Some(result)=work.join_next().await else {break};
+ let (id,result)=result?;
+ store.db.connection().execute("UPDATE desired_files SET running=0 WHERE file=?1",[id])?;
+ transfer::discard_incomplete(&context.root,store,id)?;
+ let result=match result {
+ Ok(transfer::Preparation::Ready(mut prepared))=>{
+ completed_bytes+=prepared.components.iter().filter(|c|c.temporary).map(|c|c.size).sum::<u64>();
+ if !*cancel.borrow() {reconcile::file(&context.root,store,&mut prepared,&cancel)?;}
+ transfer::finish_source(&context.root,store,&prepared.components)
+ },
+ Ok(transfer::Preparation::AwaitingCapacity)=>{
+ store.db.connection().execute("UPDATE desired_files SET deferred=1 WHERE file=?1",[id])?;
+ Ok(())
+ },
+ Err(error)=>Err(error),
+ };
+ if let Err(error)=result {
+ let mut query=store.db.connection().prepare("SELECT album FROM desired_files WHERE file=?1 AND attempted=0 AND failure IS NULL")?;
+ let albums:Vec<i64>=query.query_map([id],|r|r.get(0))?.collect::<rusqlite::Result<_>>()?;drop(query);
+ for album in albums {report(store,&format!("file:{album}:{id}"),&error)?;}
+ store.db.connection().execute("UPDATE desired_files SET failure=?1 WHERE file=?2 AND attempted=0",params![error.to_string(),id])?;
+ if fatal(&error) {return Err(error)}
+ }
+ if measured.elapsed().as_secs_f64()>=2.0 {
+ let completed:i64=store.db.connection().query_row("SELECT count(*) FROM desired_files WHERE completed=1",[],|r|r.get(0))?;
+ eprintln!("Exporting: {completed}/{expected} copies complete.");
+ let rate=completed_bytes as f64/measured.elapsed().as_secs_f64();
+ if completed_bytes>0 && (previous_rate==0.0 || rate>previous_rate*1.1) {admission=(admission*2).min(ceiling);}
+ previous_rate=rate;measured=Instant::now();completed_bytes=0;
+ }
+ }
+ let mut query=store.db.connection().prepare("SELECT album,file FROM desired_files WHERE deferred=1 AND attempted=0 AND failure IS NULL")?;
+ let mut rows=query.query([])?;
+ while let Some(row)=rows.next()? {
+ let album:i64=row.get(0)?;let file:i64=row.get(1)?;
+ report(store,&format!("file:{album}:{file}"),&anyhow::anyhow!("completed transfers have filled export staging; resolve the reported destination failures and retry"))?;
+ }
+ Ok(())
+ }.await;
+ work.abort_all();
+ while work.join_next().await.is_some() {}
+ result
+}
+
+pub fn report(store: &Store, unit: &str, error: &anyhow::Error) -> Result<()> {
+ let conflict = error.downcast_ref::<Conflict>().is_some();
+ store.db.connection().execute("INSERT INTO outcomes VALUES(?1,?2) ON CONFLICT(unit) DO UPDATE SET conflict=max(conflict,excluded.conflict)",params![unit,conflict])?;
+ eprintln!("{unit}: {error:#}");
+ Ok(())
+}
+
+pub fn fatal(error: &anyhow::Error) -> bool {
+ error.chain().any(|cause| {
+ cause.downcast_ref::<rusqlite::Error>().is_some()
+ || cause.downcast_ref::<Cancelled>().is_some()
+ || cause
+ .downcast_ref::<std::io::Error>()
+ .is_some_and(|e| matches!(e.kind(), std::io::ErrorKind::StorageFull))
+ || cause
+ .downcast_ref::<ente_core::http::Error>()
+ .is_some_and(|e| e.status_code() == Some(401))
+ })
+}
+
+fn cleanup(root: &Path, store: &Store) -> Result<()> {
+ let mut after = String::new();
+ loop {
+ let temporary:Option<store::Temporary>=store.json("SELECT record FROM temporaries WHERE path>?1 AND json_extract(record,'$.destination') IS NULL AND (json_extract(record,'$.hash') IS NULL OR (EXISTS(SELECT 1 FROM desired_files WHERE file=json_extract(temporaries.record,'$.file')) AND NOT EXISTS(SELECT 1 FROM desired_files WHERE file=json_extract(temporaries.record,'$.file') AND completed=0))) AND json_extract(record,'$.album') IN (SELECT id FROM desired_albums WHERE selected=1) ORDER BY path LIMIT 1",[&after])?;
+ let Some(temporary) = temporary else { break };
+ after = temporary.path.clone();
+ if let Err(error) = fs::discard(root, store, &temporary.path) {
+ let unit = if let Some(file) = temporary.file {
+ format!("file:{}:{file}", temporary.album)
+ } else {
+ format!("album:{}", temporary.album)
+ };
+ report(store, &unit, &error)?;
+ if fatal(&error) {
+ return Err(error);
+ }
+ }
+ }
+ Ok(())
+}
+
+fn summary(store: &Store, destination: &Path, as_json: bool) -> Result<bool> {
+ let unknown: bool = store.db.connection().query_row(
+ "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND ready=0)",
+ [],
+ |r| r.get(0),
+ )?;
+ let (files, expected, completed): (i64, i64, i64) = store.db.connection().query_row(
+ "SELECT count(DISTINCT file),count(*),coalesce(sum(completed),0) FROM desired_files",
+ [],
+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
+ )?;
+ let (failures, conflicts): (i64, i64) = store.db.connection().query_row(
+ "SELECT count(*),coalesce(sum(conflict),0) FROM outcomes",
+ [],
+ |r| Ok((r.get(0)?, r.get(1)?)),
+ )?;
+ let (exported,metadata_updated,renamed,retained):(i64,i64,i64,i64)=store.db.connection().query_row("SELECT coalesce(sum(exported),0),coalesce(sum(metadata_updated),0),coalesce(sum(renamed),0),coalesce(sum(retained),0) FROM events",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?)))?;
+ let complete = !unknown && failures == 0 && completed == expected;
+ let result = json!({"destination":destination,"complete":complete,"files":if unknown {None}else{Some(files)},"copies":if unknown {None}else{Some(json!({"expected":expected,"completed":completed,"pending":expected-completed}))},"changes":{"exported":exported,"metadataUpdated":metadata_updated,"renamed":renamed,"retained":retained},"failures":failures,"conflicts":conflicts});
+ output::action(
+ as_json,
+ &result,
+ &format!(
+ "{}: {completed}/{expected} copies complete, {exported} exported, {retained} retained, {failures} failures, {conflicts} conflicts.",
+ destination.display()
+ ),
+ )?;
+ Ok(complete)
+}
diff --git a/rust/apps/cli-next/src/export/names.rs b/rust/apps/cli-next/src/export/names.rs
new file mode 100644
index 0000000000..9703007a10
--- /dev/null
+++ b/rust/apps/cli-next/src/export/names.rs
@@ -0,0 +1,245 @@
+use std::{
+ fs,
+ path::{Component, Path},
+};
+
+use anyhow::{Result, ensure};
+use icu_casemap::CaseMapper;
+use rusqlite::params;
+use unicode_normalization::UnicodeNormalization;
+
+use super::store::Store;
+
+pub fn folded(name: &str) -> String {
+ CaseMapper::new()
+ .fold_string(&name.nfc().collect::<String>())
+ .nfc()
+ .collect()
+}
+
+pub fn relative(path: &str) -> Result<()> {
+ ensure!(
+ !path.is_empty()
+ && !path.contains('\\')
+ && Path::new(path)
+ .components()
+ .all(|part| matches!(part, Component::Normal(_))),
+ "invalid export path {path:?}"
+ );
+ Ok(())
+}
+
+pub fn check(root: &Path, path: &str) -> Result<std::path::PathBuf> {
+ relative(path)?;
+ let mut current = root.to_path_buf();
+ let mut parts = Path::new(path).components().peekable();
+ while let Some(part) = parts.next() {
+ current.push(part);
+ match fs::symlink_metadata(&current) {
+ Ok(metadata) => {
+ ensure!(
+ !metadata.is_symlink(),
+ super::Conflict(format!("symlink at {}", current.display()))
+ );
+ ensure!(
+ parts.peek().is_none() || metadata.is_dir(),
+ super::Conflict(format!("expected directory at {}", current.display()))
+ );
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+ Err(error) => return Err(error.into()),
+ }
+ }
+ Ok(current)
+}
+
+pub fn portable(name: &str) -> String {
+ let converted: String = name
+ .chars()
+ .map(|c| {
+ if c.is_control() || "/\\<>:\"|?*".contains(c) {
+ '_'
+ } else {
+ c
+ }
+ })
+ .collect();
+ converted.trim_end_matches(['.', ' ']).to_owned()
+}
+
+fn shorten(stem: &str, bytes: usize) -> &str {
+ let mut end = stem.len().min(bytes);
+ while !stem.is_char_boundary(end) {
+ end -= 1;
+ }
+ &stem[..end]
+}
+
+pub fn split(name: &str, album: bool) -> (&str, &str) {
+ if !album
+ && let Some(index) = name.rfind('.')
+ && index > 0
+ {
+ (&name[..index], &name[index..])
+ } else {
+ (name, "")
+ }
+}
+
+pub fn reserved(name: &str, album: bool) -> bool {
+ let folded = folded(name);
+ if if album {
+ ["trash", "export.json"].contains(&folded.as_str())
+ } else {
+ ["metadata", "metadata.json"].contains(&folded.as_str())
+ } {
+ return true;
+ }
+ device(&folded)
+}
+
+fn device(name: &str) -> bool {
+ let folded = folded(name);
+ let device = folded.split('.').next().unwrap_or_default();
+ ["con", "prn", "aux", "nul", "clock$"].contains(&device)
+ || ["com", "lpt"].iter().any(|prefix| {
+ device.strip_prefix(prefix).is_some_and(|n| {
+ matches!(
+ n,
+ "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "ΒΉ" | "Β²" | "Β³"
+ )
+ })
+ })
+}
+
+fn numbered(stem: &str, suffix: &str) -> String {
+ if device(stem)
+ && let Some((head, tail)) = stem.split_once('.')
+ {
+ format!("{head}{suffix}.{tail}")
+ } else {
+ format!("{stem}{suffix}")
+ }
+}
+
+pub fn reserve(store: &Store, parent: &str, name: &str, owner: &str, kind: &str) -> Result<bool> {
+ Ok(store.db.connection().execute(
+ "INSERT INTO names VALUES(?1,?2,?3,?4,?5) ON CONFLICT(parent,folded) DO UPDATE SET kind=excluded.kind WHERE names.owner=excluded.owner",
+ params![parent, folded(name), owner, folded(split(name, kind == "album").0), kind],
+ )? > 0)
+}
+
+pub fn available(
+ store: &Store,
+ parent: &str,
+ names: &[String],
+ owner: &str,
+ kind: &str,
+) -> Result<bool> {
+ for name in names {
+ let occupied: bool = store.db.connection().query_row(
+ "SELECT EXISTS(SELECT 1 FROM names WHERE parent=?1 AND owner<>?2 AND folded=?3) OR (?4<>'album' AND EXISTS(SELECT 1 FROM names WHERE parent=?1 AND owner<>?2 AND stem=?5 AND NOT(kind=?4 AND kind IN ('image','video'))))",
+ params![parent,owner,folded(name),kind,folded(split(name,false).0)], |r| r.get(0))?;
+ if occupied {
+ return Ok(false);
+ }
+ }
+ Ok(true)
+}
+
+pub fn allocate(
+ store: &Store,
+ parent: &str,
+ name: &str,
+ kind: &str,
+ extensions: Option<(&str, &str)>,
+ owner: &str,
+) -> Result<Vec<String>> {
+ for suffix in 0u64.. {
+ let Some(candidates) = candidates(name, kind == "album", extensions, suffix)? else {
+ continue;
+ };
+ if available(store, parent, &candidates, owner, kind)? {
+ for candidate in &candidates {
+ ensure!(
+ reserve(store, parent, candidate, owner, kind)?,
+ "name allocation changed"
+ );
+ }
+ return Ok(candidates);
+ }
+ }
+ unreachable!()
+}
+
+pub fn candidates(
+ name: &str,
+ album: bool,
+ extensions: Option<(&str, &str)>,
+ suffix: u64,
+) -> Result<Option<Vec<String>>> {
+ let mut base = portable(name);
+ if base.is_empty() {
+ base = if album { "album" } else { "file" }.into();
+ }
+ let (stem, extension) = split(&base, album);
+ let limit = if album { 255usize } else { 250usize };
+ let suffix = if suffix == 0 {
+ String::new()
+ } else {
+ format!("-{suffix}")
+ };
+ let candidates = if let Some((image, video)) = extensions {
+ let image = portable(image);
+ let video = portable(video);
+ let equal = folded(&image) == folded(&video);
+ let tail = image.len().max(video.len()) + suffix.len() + if equal { 6 } else { 0 };
+ ensure!(tail < limit, "Live Photo extension is too long");
+ let stem = numbered(shorten(stem, limit - tail), &suffix);
+ let stem = if !equal && (image.is_empty() || video.is_empty()) {
+ stem.trim_end_matches(['.', ' '])
+ } else {
+ &stem
+ };
+ ensure!(
+ !stem.is_empty(),
+ "Live Photo name cannot fit its extensions"
+ );
+ if equal {
+ vec![
+ format!("{stem}-image{image}"),
+ format!("{stem}-video{video}"),
+ ]
+ } else {
+ vec![format!("{stem}{image}"), format!("{stem}{video}")]
+ }
+ } else {
+ ensure!(
+ extension.len() + suffix.len() < limit,
+ "file extension is too long"
+ );
+ vec![format!(
+ "{}{extension}",
+ numbered(
+ shorten(stem, limit - extension.len() - suffix.len()),
+ &suffix
+ )
+ )]
+ };
+ let candidates: Vec<_> = candidates.into_iter().map(|name| portable(&name)).collect();
+ ensure!(
+ candidates.iter().all(|name| !name.is_empty()),
+ "export name is empty after shortening"
+ );
+ if candidates.iter().any(|name| reserved(name, album)) {
+ return Ok(None);
+ }
+ Ok(Some(candidates))
+}
+
+pub fn same_path(a: &Path, b: &Path) -> bool {
+ match (fs::canonicalize(a), fs::canonicalize(b)) {
+ (Ok(a), Ok(b)) => a == b,
+ _ => false,
+ }
+}
diff --git a/rust/apps/cli-next/src/export/reconcile.rs b/rust/apps/cli-next/src/export/reconcile.rs
new file mode 100644
index 0000000000..11f1d2afd5
--- /dev/null
+++ b/rust/apps/cli-next/src/export/reconcile.rs
@@ -0,0 +1,781 @@
+use std::{fs as disk, path::Path};
+
+use anyhow::{Context, Result, ensure};
+use ente_photos::export;
+use rusqlite::{OptionalExtension, params};
+use serde_json::Value;
+use uuid::Uuid;
+
+use super::{
+ allocation, fatal, fs, names, report,
+ store::{Album, Media, Operation, Placement, Properties, Store},
+ transfer::{self, Prepared},
+};
+use crate::replica::AlbumRecord;
+
+pub fn albums(root: &Path, store: &Store, user_id: i64) -> Result<()> {
+ let mut after = 0;
+ loop {
+ let next:Option<(i64,String)>=store.db.connection().query_row("SELECT id,record FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL AND record IS NOT NULL AND id>?1 ORDER BY id LIMIT 1",[after],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
+ let Some((id, record)) = next else { break };
+ after = id;
+ let result = (|| {
+ let record: AlbumRecord = serde_json::from_str(&record)?;
+ let source = record.album(user_id)?;
+ for field in &source.warnings {
+ eprintln!(
+ "album {id}: unfamiliar metadata field {field:?}; exporting supported fields"
+ );
+ }
+ let existing = store.album(id, false)?;
+ let key = existing
+ .as_ref()
+ .map(|a| a.key.clone())
+ .unwrap_or_else(|| format!("active:{id}"));
+ let path = if let Some(existing) = &existing
+ && existing.name == source.name
+ {
+ existing.path.clone()
+ } else {
+ allocation::choose(store, "", &source.name, "album", None, &key)?.remove(0)
+ };
+ let album = Album {
+ key,
+ id,
+ name: source.name.clone(),
+ path: path.clone(),
+ retained: false,
+ metadata: export::album(&source),
+ initialized: true,
+ };
+ if let Some(existing) = &existing
+ && existing.path != path
+ {
+ rename_target(root, &existing.path, &path)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store.operation(&Operation::Move {
+ album: id,
+ file: None,
+ source: existing.path.clone(),
+ destination: path.clone(),
+ })?;
+ let mut query = store
+ .db
+ .connection()
+ .prepare("SELECT record FROM placements WHERE album=?1 AND retained=0")?;
+ let mut rows = query.query([id])?;
+ while let Some(row) = rows.next()? {
+ let mut placement: Placement = serde_json::from_str(&row.get::<_, String>(0)?)?;
+ placement.folder = path.clone();
+ store.save_placement(&placement)?;
+ event(store, &placement.id, "renamed")?;
+ }
+ store.db.connection().execute(
+ "UPDATE json_records SET path=?1||substr(path,?2) WHERE substr(path,1,?3)=?4",
+ params![
+ path,
+ existing.path.chars().count() as i64 + 1,
+ existing.path.chars().count() as i64 + 1,
+ format!("{}/", existing.path)
+ ],
+ )?;
+ relocate_work(store, &existing.path, &path)?;
+ allocation::relocate(store, &existing.path, &path)?;
+ store.save_album(&album)?;
+ allocation::clear(store, &album.key)?;
+ transaction.commit()?;
+ fs::operations(root, store, id, None)?;
+ } else {
+ if existing.is_none() {
+ free_target(root, &path)?;
+ }
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store.save_album(&album)?;
+ allocation::clear(store, &album.key)?;
+ transaction.commit()?;
+ }
+ fs::directory(root, &path)?;
+ fs::json(
+ root,
+ store,
+ &format!("{path}/metadata.json"),
+ &album.metadata,
+ true,
+ None,
+ )?;
+ Ok::<_, anyhow::Error>(())
+ })();
+ if let Err(error) = result {
+ report(store, &format!("album:{id}"), &error)?;
+ let mut query = store
+ .db
+ .connection()
+ .prepare("SELECT file FROM desired_files WHERE album=?1")?;
+ let mut rows = query.query([id])?;
+ while let Some(row) = rows.next()? {
+ report(
+ store,
+ &format!("file:{id}:{}", row.get::<_, i64>(0)?),
+ &error,
+ )?;
+ }
+ store.db.connection().execute(
+ "UPDATE desired_files SET failure=?1 WHERE album=?2",
+ params![error.to_string(), id],
+ )?;
+ if fatal(&error) {
+ return Err(error);
+ }
+ }
+ }
+ Ok(())
+}
+
+pub fn file(
+ root: &Path,
+ store: &Store,
+ prepared: &mut Prepared,
+ cancel: &tokio::sync::watch::Receiver<bool>,
+) -> Result<()> {
+ let mut after = 0;
+ loop {
+ let next:Option<(i64,bool)>=store.db.connection().query_row("SELECT album,favorited FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL AND attempted=0 ORDER BY album LIMIT 1",params![prepared.file.id,after],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
+ let Some((album, favorited)) = next else {
+ break;
+ };
+ after = album;
+ store.db.connection().execute(
+ "UPDATE desired_files SET attempted=1 WHERE album=?1 AND file=?2",
+ params![album, prepared.file.id],
+ )?;
+ let result = placement(root, store, prepared, album, favorited, cancel);
+ if let Err(error) = result {
+ report(store, &format!("file:{album}:{}", prepared.file.id), &error)?;
+ transfer::discard_incomplete(root, store, prepared.file.id)?;
+ if fatal(&error) {
+ return Err(error);
+ }
+ }
+ }
+ Ok(())
+}
+
+fn placement(
+ root: &Path,
+ store: &Store,
+ prepared: &mut Prepared,
+ album_id: i64,
+ favorited: bool,
+ cancel: &tokio::sync::watch::Receiver<bool>,
+) -> Result<()> {
+ let album = store
+ .album(album_id, false)?
+ .context("missing album association")?;
+ let file = &prepared.file;
+ for field in &file.warnings {
+ eprintln!(
+ "file {}: unfamiliar metadata field {field:?}; exporting supported fields",
+ file.id
+ );
+ }
+ let existing = store.placement(album_id, file.id)?;
+ let old_complete = existing.as_ref().is_some_and(|p| p.complete);
+ let owner = format!("file:{album_id}:{}", file.id);
+ let extensions = if file.kind == ente_photos::files::Kind::LivePhoto {
+ if prepared.components.len() == 2 {
+ Some((
+ prepared.components[0].extension.as_str(),
+ prepared.components[1].extension.as_str(),
+ ))
+ } else {
+ let media = &existing
+ .as_ref()
+ .context("missing Live Photo original")?
+ .media;
+ Some((
+ names::split(&media[0].component.path, false).1,
+ names::split(&media[1].component.path, false).1,
+ ))
+ }
+ } else {
+ None
+ };
+ let allocated = if let Some(existing) = &existing
+ && existing.name == file.name
+ && existing.media.len() == if extensions.is_some() { 2 } else { 1 }
+ && (prepared.components.is_empty()
+ || extensions.is_none()
+ || existing
+ .media
+ .iter()
+ .zip(&prepared.components)
+ .all(|(old, new)| {
+ names::split(&old.component.path, false).1 == names::portable(&new.extension)
+ })) {
+ existing
+ .media
+ .iter()
+ .map(|m| m.component.path.clone())
+ .collect()
+ } else {
+ allocation::choose(
+ store,
+ &album.path,
+ &file.name,
+ file.kind.name(),
+ extensions,
+ &owner,
+ )?
+ };
+ let mut placement = Placement {
+ id: existing
+ .as_ref()
+ .map(|p| p.id.clone())
+ .unwrap_or_else(|| Uuid::new_v4().to_string()),
+ album: album_id,
+ folder: album.path.clone(),
+ file: file.id,
+ retained: false,
+ name: file.name.clone(),
+ original: prepared.signature.clone(),
+ media: Vec::new(),
+ metadata: Vec::new(),
+ complete: false,
+ retaining: false,
+ };
+ for (index, path) in allocated.iter().enumerate() {
+ let mut component = if let Some(component) = prepared.components.get(index) {
+ export::Component {
+ role: component.role.clone(),
+ path: path.clone(),
+ size: component.size,
+ hash: component.hash.clone(),
+ }
+ } else {
+ existing
+ .as_ref()
+ .and_then(|p| p.media.get(index))
+ .context("missing completed original")?
+ .component
+ .clone()
+ };
+ component.path = path.clone();
+ let previous = existing
+ .as_ref()
+ .and_then(|p| p.media.iter().find(|m| m.component.role == component.role));
+ placement.media.push(Media {
+ component,
+ properties: previous.and_then(|m| m.properties.clone()),
+ intended_time: previous.and_then(|m| m.intended_time),
+ });
+ }
+ let intended_components: Vec<_> = placement
+ .media
+ .iter()
+ .map(|m| m.component.clone())
+ .collect();
+ placement.metadata = intended_components
+ .iter()
+ .enumerate()
+ .map(|(index, _)| export::file(file, &intended_components, index, favorited))
+ .collect::<std::result::Result<_, _>>()?;
+ let renamed = existing.as_ref().is_some_and(|p| {
+ p.media
+ .iter()
+ .map(|m| &m.component.path)
+ .ne(placement.media.iter().map(|m| &m.component.path))
+ });
+ if renamed {
+ let previous = existing.as_ref().context("missing rename association")?;
+ for (old, new) in previous.media.iter().zip(&placement.media) {
+ if old.component.path != new.component.path {
+ rename_target(
+ root,
+ &format!("{}/{}", album.path, old.component.path),
+ &format!("{}/{}", album.path, new.component.path),
+ )?;
+ rename_target(
+ root,
+ &format!("{}/metadata/{}.json", album.path, old.component.path),
+ &format!("{}/metadata/{}.json", album.path, new.component.path),
+ )?;
+ }
+ }
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store.save_placement(&placement)?;
+ allocation::clear(store, &owner)?;
+ for (old, new) in previous.media.iter().zip(&placement.media) {
+ if old.component.path != new.component.path {
+ let source = format!("{}/{}", album.path, old.component.path);
+ let destination = format!("{}/{}", album.path, new.component.path);
+ store.operation(&Operation::Move {
+ album: album_id,
+ file: Some(file.id),
+ source: source.clone(),
+ destination: destination.clone(),
+ })?;
+ for component in &mut prepared.components {
+ if component.path == source {
+ component.path = destination.clone();
+ }
+ }
+ }
+ }
+ for (old, new) in previous.media.iter().zip(&placement.media) {
+ if old.component.path != new.component.path {
+ store.operation(&Operation::Move {
+ album: album_id,
+ file: Some(file.id),
+ source: format!("{}/metadata/{}.json", album.path, old.component.path),
+ destination: format!("{}/metadata/{}.json", album.path, new.component.path),
+ })?;
+ store.db.connection().execute(
+ "DELETE FROM json_records WHERE path=?1",
+ [format!(
+ "{}/metadata/{}.json",
+ album.path, old.component.path
+ )],
+ )?;
+ }
+ }
+ transaction.commit()?;
+ fs::operations(root, store, album_id, Some(file.id))?;
+ } else {
+ if existing.is_none() {
+ for media in &placement.media {
+ free_target(root, &format!("{}/{}", album.path, media.component.path))?;
+ free_target(
+ root,
+ &format!("{}/metadata/{}.json", album.path, media.component.path),
+ )?;
+ }
+ }
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store.save_placement(&placement)?;
+ allocation::clear(store, &owner)?;
+ transaction.commit()?;
+ }
+ for media in &placement.media {
+ names::reserve(
+ store,
+ &placement.folder,
+ &media.component.path,
+ &owner,
+ file.kind.name(),
+ )?;
+ }
+ let mut wrote_media = false;
+ for media in &mut placement.media {
+ let relative = format!("{}/{}", album.path, media.component.path);
+ let path = names::check(root, &relative)?;
+ let properties = if path.try_exists()? {
+ Some(Properties::read(&path)?)
+ } else {
+ None
+ };
+ let valid = if let Some(properties) = &properties {
+ if media.properties.as_ref() == Some(properties)
+ && existing.as_ref().is_some_and(|old| {
+ old.media.iter().any(|m| {
+ m.component.role == media.component.role
+ && m.component.hash == media.component.hash
+ })
+ })
+ {
+ true
+ } else {
+ let (size, hash) = fs::hash_cancellable(&path, cancel)?;
+ size == media.component.size && hash == media.component.hash
+ }
+ } else {
+ false
+ };
+ if !valid {
+ let source = prepared
+ .components
+ .iter_mut()
+ .find(|c| c.role == media.component.role)
+ .context("no verified original is available")?;
+ if source.temporary {
+ let temporary: super::store::Temporary = store
+ .json(
+ "SELECT record FROM temporaries WHERE path=?1",
+ [&source.path],
+ )?
+ .context("missing transfer ownership")?;
+ fs::publish(
+ root,
+ store,
+ temporary,
+ &relative,
+ source.size,
+ source.hash.clone(),
+ )?;
+ source.path = relative.clone();
+ source.temporary = false;
+ } else {
+ transfer::materialize(
+ root,
+ store,
+ source,
+ (&album.path, &media.component.path),
+ file.id,
+ &prepared.signature,
+ cancel,
+ )?;
+ }
+ wrote_media = true;
+ }
+ let observed = Properties::read(&path)?;
+ let intended = file.modified_at_micros.unwrap_or(file.created_at_micros);
+ media.properties = Some(
+ if media.intended_time == Some(intended) && media.properties.as_ref() == Some(&observed)
+ {
+ observed
+ } else {
+ fs::set_time(&path, intended)?
+ },
+ );
+ media.intended_time = Some(intended);
+ }
+ store.save_placement(&placement)?;
+ let mut metadata_changed = false;
+ for (media, record) in placement.media.iter().zip(&placement.metadata) {
+ metadata_changed |= fs::json(
+ root,
+ store,
+ &format!("{}/metadata/{}.json", album.path, media.component.path),
+ record,
+ true,
+ Some(file.id),
+ )?;
+ }
+ placement.complete = true;
+ store.save_placement(&placement)?;
+ store.db.connection().execute(
+ "UPDATE desired_files SET completed=1 WHERE album=?1 AND file=?2",
+ params![album_id, file.id],
+ )?;
+ if wrote_media || !old_complete {
+ event(store, &placement.id, "exported")?;
+ }
+ if existing.is_some() && metadata_changed {
+ event(store, &placement.id, "metadata_updated")?;
+ }
+ if renamed {
+ event(store, &placement.id, "renamed")?;
+ }
+ Ok(())
+}
+
+pub fn retain_removed(root: &Path, store: &Store) -> Result<()> {
+ let mut after = String::new();
+ loop {
+ let next:Option<Placement>=store.json("SELECT p.record FROM placements p JOIN desired_albums a ON a.id=p.album WHERE p.retained=0 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND p.id>?1 AND NOT EXISTS(SELECT 1 FROM desired_files d WHERE d.album=p.album AND d.file=p.file) ORDER BY p.id LIMIT 1",[&after])?;
+ let Some(placement) = next else { break };
+ after = placement.id.clone();
+ let result = retain(root, store, &placement);
+ if let Err(error) = result {
+ report(
+ store,
+ &format!("file:{}:{}", placement.album, placement.file),
+ &error,
+ )?;
+ if fatal(&error) {
+ return Err(error);
+ }
+ }
+ }
+ let mut after = String::new();
+ loop {
+ let next:Option<Placement>=store.json("SELECT p.record FROM placements p JOIN desired_albums a ON a.id=p.album WHERE p.retained=1 AND json_extract(p.record,'$.retaining')=1 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND NOT EXISTS(SELECT 1 FROM operations o WHERE o.album=p.album AND o.file=p.file) AND p.id>?1 ORDER BY p.id LIMIT 1",[&after])?;
+ let Some(mut placement) = next else { break };
+ after = placement.id.clone();
+ let result = (|| {
+ for (component, record) in placement.media.iter().zip(&placement.metadata) {
+ fs::json(
+ root,
+ store,
+ &format!(
+ "{}/metadata/{}.json",
+ placement.folder, component.component.path
+ ),
+ record,
+ true,
+ Some(placement.file),
+ )?;
+ }
+ placement.retaining = false;
+ placement.complete = true;
+ store.save_placement(&placement)?;
+ event(store, &placement.id, "retained")?;
+ Ok::<_, anyhow::Error>(())
+ })();
+ if let Err(error) = result {
+ report(
+ store,
+ &format!("file:{}:{}", placement.album, placement.file),
+ &error,
+ )?;
+ if fatal(&error) {
+ return Err(error);
+ }
+ }
+ }
+ let mut after = 0;
+ loop {
+ let id:Option<i64>=store.db.connection().query_row("SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL AND record IS NULL AND id>?1 ORDER BY id LIMIT 1",[after],|r|r.get(0)).optional()?;
+ let Some(id) = id else { break };
+ after = id;
+ let result = (|| {
+ let Some(album) = store.album(id, false)? else {
+ return Ok(());
+ };
+ retained_album(root, store, &album)?;
+ store.operation(&Operation::Remove {
+ album: id,
+ path: format!("{}/metadata.json", album.path),
+ })?;
+ fs::operations(root, store, id, None)?;
+ for relative in [format!("{}/metadata", album.path), album.path] {
+ let path = names::check(root, &relative)?;
+ if path.is_dir() && disk::read_dir(&path)?.next().is_none() {
+ disk::remove_dir(path)?;
+ }
+ }
+ Ok::<_, anyhow::Error>(())
+ })();
+ if let Err(error) = result {
+ report(store, &format!("album:{id}"), &error)?;
+ if fatal(&error) {
+ return Err(error);
+ }
+ }
+ }
+ Ok(())
+}
+
+fn retain(root: &Path, store: &Store, active: &Placement) -> Result<()> {
+ let album = store
+ .album(active.album, false)?
+ .context("missing removed album")?;
+ let retained_album = retained_album(root, store, &album)?;
+ let missing_folder = !names::check(root, &retained_album.path)?.try_exists()?;
+ fs::directory(root, &retained_album.path)?;
+ if missing_folder {
+ fs::json(
+ root,
+ store,
+ &format!("{}/metadata.json", retained_album.path),
+ &retained_album.metadata,
+ true,
+ None,
+ )?;
+ }
+ let mut retained = active.clone();
+ retained.id = format!("retain:{}", active.id);
+ retained.retained = true;
+ retained.folder = retained_album.path;
+ retained.complete = false;
+ retained.retaining = true;
+ let extensions = if active.media.len() == 2 {
+ Some((
+ names::split(&active.media[0].component.path, false).1,
+ names::split(&active.media[1].component.path, false).1,
+ ))
+ } else {
+ None
+ };
+ let names = allocation::choose(
+ store,
+ &retained.folder,
+ &active.name,
+ active.kind()?,
+ extensions,
+ &retained.id,
+ )?;
+ for (media, name) in retained.media.iter_mut().zip(names) {
+ media.component.path = name;
+ media.properties = None;
+ }
+ let components: Vec<_> = retained.media.iter().map(|m| m.component.clone()).collect();
+ for (index, record) in retained.metadata.iter_mut().enumerate() {
+ if record.is_null() {
+ continue;
+ }
+ record["title"] = json_string(&components[index].path);
+ record["ente"]["components"] = serde_json::to_value(&components)?;
+ }
+ for media in &retained.media {
+ free_target(
+ root,
+ &format!("{}/{}", retained.folder, media.component.path),
+ )?;
+ free_target(
+ root,
+ &format!("{}/metadata/{}.json", retained.folder, media.component.path),
+ )?;
+ }
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store.save_placement(&retained)?;
+ allocation::clear(store, &retained.id)?;
+ allocation::clear(store, &format!("file:{}:{}", active.album, active.file))?;
+ store
+ .db
+ .connection()
+ .execute("DELETE FROM placements WHERE id=?1", [&active.id])?;
+ for (old, new) in active.media.iter().zip(&retained.media) {
+ store.operation(&Operation::Move {
+ album: active.album,
+ file: Some(active.file),
+ source: format!("{}/{}", active.folder, old.component.path),
+ destination: format!("{}/{}", retained.folder, new.component.path),
+ })?;
+ }
+ for (old, new) in active.media.iter().zip(&retained.media) {
+ store.operation(&Operation::Move {
+ album: active.album,
+ file: Some(active.file),
+ source: format!("{}/metadata/{}.json", active.folder, old.component.path),
+ destination: format!("{}/metadata/{}.json", retained.folder, new.component.path),
+ })?;
+ store.db.connection().execute(
+ "DELETE FROM json_records WHERE path=?1",
+ [format!(
+ "{}/metadata/{}.json",
+ active.folder, old.component.path
+ )],
+ )?;
+ }
+ transaction.commit()?;
+ fs::operations(root, store, active.album, Some(active.file))?;
+ Ok(())
+}
+
+fn retained_album(root: &Path, store: &Store, active: &Album) -> Result<Album> {
+ if let Some(mut album) = store.album(active.id, true)? {
+ if !album.initialized {
+ initialize_retained(root, store, &mut album)?;
+ }
+ return Ok(album);
+ }
+ let key = format!("retained:{}", active.id);
+ let path = format!(
+ "Trash/{}",
+ allocation::choose(store, "Trash", &active.name, "album", None, &key)?.remove(0)
+ );
+ free_target(root, &path)?;
+ let mut album = Album {
+ key,
+ id: active.id,
+ name: active.name.clone(),
+ path,
+ retained: true,
+ metadata: active.metadata.clone(),
+ initialized: false,
+ };
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store.save_album(&album)?;
+ allocation::clear(store, &album.key)?;
+ transaction.commit()?;
+ initialize_retained(root, store, &mut album)?;
+ Ok(album)
+}
+
+fn initialize_retained(root: &Path, store: &Store, album: &mut Album) -> Result<()> {
+ fs::directory(root, &album.path)?;
+ fs::json(
+ root,
+ store,
+ &format!("{}/metadata.json", album.path),
+ &album.metadata,
+ true,
+ None,
+ )?;
+ album.initialized = true;
+ store.save_album(album)
+}
+
+fn relocate_work(store: &Store, old: &str, new: &str) -> Result<()> {
+ let prefix = format!("{old}/");
+ let relocate = |path: &mut String| {
+ if let Some(tail) = path.strip_prefix(&prefix) {
+ *path = format!("{new}/{tail}");
+ }
+ };
+ let mut query = store
+ .db
+ .connection()
+ .prepare("SELECT record FROM temporaries WHERE substr(path,1,?1)=?2")?;
+ let mut rows = query.query(params![prefix.chars().count() as i64, prefix])?;
+ while let Some(row) = rows.next()? {
+ let mut temporary: super::store::Temporary =
+ serde_json::from_str(&row.get::<_, String>(0)?)?;
+ store
+ .db
+ .connection()
+ .execute("DELETE FROM temporaries WHERE path=?1", [&temporary.path])?;
+ relocate(&mut temporary.path);
+ if let Some(destination) = &mut temporary.destination {
+ relocate(destination);
+ }
+ store.temporary(&temporary)?;
+ }
+ let mut query = store
+ .db
+ .connection()
+ .prepare("SELECT id,record FROM operations ORDER BY id")?;
+ let mut rows = query.query([])?;
+ while let Some(row) = rows.next()? {
+ let id: i64 = row.get(0)?;
+ let mut operation: Operation = serde_json::from_str(&row.get::<_, String>(1)?)?;
+ match &mut operation {
+ Operation::Move {
+ source,
+ destination,
+ ..
+ } => {
+ relocate(source);
+ relocate(destination);
+ }
+ Operation::Remove { path, .. } => relocate(path),
+ }
+ store.db.connection().execute(
+ "UPDATE operations SET record=?1 WHERE id=?2",
+ params![serde_json::to_string(&operation)?, id],
+ )?;
+ }
+ Ok(())
+}
+
+fn rename_target(root: &Path, source: &str, destination: &str) -> Result<()> {
+ let source = names::check(root, source)?;
+ let destination = names::check(root, destination)?;
+ ensure!(
+ !destination.try_exists()? || names::same_path(&source, &destination),
+ super::Conflict(format!("occupied rename target {}", destination.display()))
+ );
+ Ok(())
+}
+
+fn free_target(root: &Path, relative: &str) -> Result<()> {
+ let path = names::check(root, relative)?;
+ ensure!(
+ !path.try_exists()?,
+ super::Conflict(format!("at {}: unrelated or occupied path", path.display()))
+ );
+ Ok(())
+}
+
+fn event(store: &Store, placement: &str, event: &str) -> Result<()> {
+ ensure!(
+ ["exported", "metadata_updated", "renamed", "retained"].contains(&event),
+ "unknown export event"
+ );
+ store.db.connection().execute(&format!("INSERT INTO events(placement,{event}) VALUES(?1,1) ON CONFLICT(placement) DO UPDATE SET {event}=1"),[placement])?;
+ Ok(())
+}
+
+fn json_string(value: &str) -> Value {
+ Value::String(value.into())
+}
diff --git a/rust/apps/cli-next/src/export/store.rs b/rust/apps/cli-next/src/export/store.rs
new file mode 100644
index 0000000000..490aac4598
--- /dev/null
+++ b/rust/apps/cli-next/src/export/store.rs
@@ -0,0 +1,264 @@
+use std::{
+ path::{Path, PathBuf},
+ time::{SystemTime, UNIX_EPOCH},
+};
+
+use anyhow::{Context, Result, ensure};
+use ente_core::crypto::hash;
+use ente_photos::export::Component;
+use rusqlite::{OptionalExtension, params};
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
+use serde_json::Value;
+
+use crate::{
+ db, home,
+ replica::{AlbumRecord, FileRecord},
+ vault::DbKey,
+};
+
+pub struct Store {
+ pub db: crate::core_db::Db,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct Album {
+ pub key: String,
+ pub id: i64,
+ pub name: String,
+ pub path: String,
+ pub retained: bool,
+ pub metadata: Value,
+ pub initialized: bool,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct Media {
+ #[serde(flatten)]
+ pub component: Component,
+ pub properties: Option<Properties>,
+ pub intended_time: Option<i64>,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct Placement {
+ pub id: String,
+ pub album: i64,
+ pub folder: String,
+ pub file: i64,
+ pub retained: bool,
+ pub name: String,
+ pub original: String,
+ pub media: Vec<Media>,
+ pub metadata: Vec<Value>,
+ pub complete: bool,
+ pub retaining: bool,
+}
+
+impl Placement {
+ pub fn kind(&self) -> Result<&str> {
+ self.metadata
+ .first()
+ .and_then(|record| record["ente"]["type"].as_str())
+ .context("missing placement media type")
+ }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Properties {
+ pub size: u64,
+ pub modified: i128,
+}
+
+impl Properties {
+ pub fn read(path: &Path) -> Result<Self> {
+ let metadata = std::fs::symlink_metadata(path)?;
+ ensure!(
+ metadata.is_file() && !metadata.is_symlink(),
+ super::Conflict(format!("at {}: expected a regular file", path.display()))
+ );
+ Ok(Self {
+ size: metadata.len(),
+ modified: system_time(metadata.modified()?),
+ })
+ }
+}
+
+pub fn system_time(time: SystemTime) -> i128 {
+ match time.duration_since(UNIX_EPOCH) {
+ Ok(duration) => duration.as_nanos() as i128,
+ Err(error) => -(error.duration().as_nanos() as i128),
+ }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct JsonRecord {
+ pub hash: String,
+ pub properties: Properties,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct Temporary {
+ pub album: i64,
+ pub path: String,
+ pub file: Option<i64>,
+ pub original: Option<String>,
+ pub role: Option<ente_photos::export::Role>,
+ pub extension: Option<String>,
+ pub destination: Option<String>,
+ pub hash: Option<String>,
+ pub size: Option<u64>,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub enum Operation {
+ Move {
+ album: i64,
+ file: Option<i64>,
+ source: String,
+ destination: String,
+ },
+ Remove {
+ album: i64,
+ path: String,
+ },
+}
+
+impl Store {
+ pub fn path(account_home: &Path, destination: &Path) -> Result<PathBuf> {
+ let bytes = destination.as_os_str().as_encoded_bytes();
+ let name: String = hash::hash(bytes, Some(32), None)?
+ .iter()
+ .map(|b| format!("{b:02x}"))
+ .collect();
+ Ok(account_home.join("exports").join(format!("{name}.db")))
+ }
+
+ pub fn open(path: &Path, key: &DbKey, destination: &Path, create: bool) -> Result<Self> {
+ if create {
+ home::create(path.parent().context("export DB has no parent")?)?;
+ }
+ let mut db = crate::core_db::Db::new(db::connect(path, key, create)?);
+ if create {
+ db.migrate(&[SCHEMA])?;
+ }
+ let connection = db.connection();
+ let existing: Option<Vec<u8>> = connection
+ .query_row("SELECT destination FROM export WHERE id=1", [], |r| {
+ r.get(0)
+ })
+ .optional()?;
+ if let Some(existing) = existing {
+ ensure!(
+ existing == destination.as_os_str().as_encoded_bytes(),
+ "destination association does not match; use --adopt"
+ );
+ } else {
+ connection.execute(
+ "INSERT INTO export(id,destination) VALUES(1,?1)",
+ [destination.as_os_str().as_encoded_bytes()],
+ )?;
+ }
+ Ok(Self { db })
+ }
+
+ pub fn album(&self, id: i64, retained: bool) -> Result<Option<Album>> {
+ self.json(
+ "SELECT record FROM albums WHERE id=?1 AND retained=?2 ORDER BY rowid LIMIT 1",
+ params![id, retained],
+ )
+ }
+
+ pub fn save_album(&self, album: &Album) -> Result<()> {
+ self.db.connection().execute("INSERT INTO albums VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(key) DO UPDATE SET name=excluded.name,path=excluded.path,record=excluded.record",params![album.key,album.id,album.retained,album.name,album.path,serde_json::to_string(album)?])?;
+ Ok(())
+ }
+
+ pub fn placement(&self, album: i64, file: i64) -> Result<Option<Placement>> {
+ self.json(
+ "SELECT record FROM placements WHERE album=?1 AND file=?2 AND retained=0",
+ params![album, file],
+ )
+ }
+
+ pub fn save_placement(&self, placement: &Placement) -> Result<()> {
+ self.db.connection().execute("INSERT INTO placements VALUES(?1,?2,?3,?4,?5) ON CONFLICT(id) DO UPDATE SET album=excluded.album,file=excluded.file,retained=excluded.retained,record=excluded.record",params![placement.id,placement.album,placement.file,placement.retained,serde_json::to_string(placement)?])?;
+ Ok(())
+ }
+
+ pub fn json<T: DeserializeOwned>(
+ &self,
+ sql: &str,
+ parameters: impl rusqlite::Params,
+ ) -> Result<Option<T>> {
+ let text: Option<String> = self
+ .db
+ .connection()
+ .query_row(sql, parameters, |r| r.get(0))
+ .optional()?;
+ text.map(|text| serde_json::from_str(&text).map_err(Into::into))
+ .transpose()
+ }
+
+ pub fn temporary(&self, temporary: &Temporary) -> Result<()> {
+ self.db.connection().execute("INSERT INTO temporaries VALUES(?1,?2) ON CONFLICT(path) DO UPDATE SET record=excluded.record",params![temporary.path,serde_json::to_string(temporary)?])?;
+ Ok(())
+ }
+
+ pub fn operation(&self, operation: &Operation) -> Result<i64> {
+ let (album, file) = match operation {
+ Operation::Move { album, file, .. } => (*album, *file),
+ Operation::Remove { album, .. } => (*album, None),
+ };
+ self.db.connection().execute(
+ "INSERT INTO operations(album,file,record) VALUES(?1,?2,?3)",
+ params![album, file, serde_json::to_string(operation)?],
+ )?;
+ Ok(self.db.connection().last_insert_rowid())
+ }
+
+ pub fn reset_desired(&self) -> Result<()> {
+ self.db.connection().execute_batch("DELETE FROM desired_albums; DELETE FROM desired_files; DELETE FROM names; DELETE FROM outcomes; DELETE FROM events;")?;
+ Ok(())
+ }
+
+ pub fn desired_album(&self, record: &AlbumRecord, user_id: i64) -> Result<()> {
+ let name = record.album(user_id).ok().map(|a| a.name);
+ self.db.connection().execute("INSERT INTO desired_albums(id,name,record) VALUES(?1,?2,?3) ON CONFLICT(id) DO UPDATE SET name=excluded.name,record=excluded.record",params![record.remote.id,name,serde_json::to_string(record)?])?;
+ Ok(())
+ }
+
+ pub fn desired_file(
+ &self,
+ album: i64,
+ record: &FileRecord,
+ user_id: i64,
+ favorited: Option<bool>,
+ ) -> Result<()> {
+ let result = record.file(user_id);
+ let failure = result.as_ref().err().map(ToString::to_string);
+ self.db.connection().execute("INSERT INTO desired_files(album,file,version,record,favorited,failure) VALUES(?1,?2,?3,?4,?5,?6)",params![album,record.remote.id,record.remote.updation_time,serde_json::to_string(record)?,favorited,failure])?;
+ Ok(())
+ }
+}
+
+const SCHEMA: &str = "
+CREATE TABLE export (id INTEGER PRIMARY KEY CHECK(id=1), destination BLOB NOT NULL, bound INTEGER NOT NULL DEFAULT 0);
+ CREATE TABLE albums (key TEXT PRIMARY KEY, id INTEGER NOT NULL, retained INTEGER NOT NULL, name TEXT NOT NULL, path TEXT NOT NULL UNIQUE, record TEXT NOT NULL);
+ CREATE UNIQUE INDEX active_album ON albums(id) WHERE retained=0;
+ CREATE TABLE placements (id TEXT PRIMARY KEY, album INTEGER NOT NULL, file INTEGER NOT NULL, retained INTEGER NOT NULL, record TEXT NOT NULL);
+ CREATE UNIQUE INDEX active_placement ON placements(album,file) WHERE retained=0;
+ CREATE INDEX reusable_media ON placements(file,retained);
+ CREATE TABLE json_records (path TEXT PRIMARY KEY, record TEXT NOT NULL);
+ CREATE TABLE temporaries (path TEXT PRIMARY KEY, record TEXT NOT NULL);
+ CREATE INDEX temporary_file ON temporaries(json_extract(record,'$.file'));
+ CREATE TABLE operations (id INTEGER PRIMARY KEY, album INTEGER NOT NULL, file INTEGER, record TEXT NOT NULL);
+ CREATE TABLE desired_albums (id INTEGER PRIMARY KEY, name TEXT, record TEXT, selected INTEGER NOT NULL DEFAULT 0, ready INTEGER NOT NULL DEFAULT 0, failure TEXT);
+ CREATE TABLE desired_files (album INTEGER NOT NULL, file INTEGER NOT NULL, version INTEGER NOT NULL, record TEXT NOT NULL, favorited INTEGER, completed INTEGER NOT NULL DEFAULT 0, failure TEXT, running INTEGER NOT NULL DEFAULT 0, attempted INTEGER NOT NULL DEFAULT 0, deferred INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(album,file));
+ CREATE INDEX desired_file_id ON desired_files(file,album);
+ CREATE INDEX eligible_files ON desired_files(deferred,file) WHERE failure IS NULL AND attempted=0 AND running=0;
+ CREATE TABLE outcomes (unit TEXT PRIMARY KEY, conflict INTEGER NOT NULL);
+ CREATE TABLE events (placement TEXT PRIMARY KEY, exported INTEGER NOT NULL DEFAULT 0, metadata_updated INTEGER NOT NULL DEFAULT 0, renamed INTEGER NOT NULL DEFAULT 0, retained INTEGER NOT NULL DEFAULT 0);
+ CREATE TABLE allocations (owner TEXT PRIMARY KEY, record TEXT NOT NULL);
+ CREATE TABLE names (parent TEXT NOT NULL, folded TEXT NOT NULL, owner TEXT NOT NULL, stem TEXT NOT NULL, kind TEXT NOT NULL, PRIMARY KEY(parent,folded));
+ CREATE INDEX named_stems ON names(parent,stem);
+";
diff --git a/rust/apps/cli-next/src/export/transfer.rs b/rust/apps/cli-next/src/export/transfer.rs
new file mode 100644
index 0000000000..7549824553
--- /dev/null
+++ b/rust/apps/cli-next/src/export/transfer.rs
@@ -0,0 +1,475 @@
+use std::{
+ fs::File,
+ io::{Seek, SeekFrom},
+ path::{Path, PathBuf},
+ sync::Arc,
+};
+
+use anyhow::{Context as _, Result, bail, ensure};
+use ente_core::{Session, b64, crypto::hash};
+use ente_photos::{
+ export::Role,
+ files::{self, Kind},
+};
+use rusqlite::OptionalExtension;
+use tokio::sync::watch;
+
+use super::{
+ fs::{self, HashWriter},
+ names,
+ store::{Placement, Properties, Store},
+};
+use crate::{replica::FileRecord, vault::DbKey};
+
+pub struct Component {
+ pub role: Role,
+ pub extension: String,
+ pub path: String,
+ pub size: u64,
+ pub hash: String,
+ pub temporary: bool,
+}
+
+pub struct Prepared {
+ pub file: files::File,
+ pub signature: String,
+ pub components: Vec<Component>,
+}
+
+pub enum Preparation {
+ Ready(Box<Prepared>),
+ AwaitingCapacity,
+}
+
+pub struct Context {
+ pub root: PathBuf,
+ pub db_path: PathBuf,
+ pub db_key: DbKey,
+ pub session: Session,
+}
+
+pub async fn prepare(
+ context: Arc<Context>,
+ id: i64,
+ mut cancel: watch::Receiver<bool>,
+ capacity: bool,
+) -> Result<Preparation> {
+ let store = Store::open(&context.db_path, &context.db_key, &context.root, false)?;
+ let record:FileRecord=store.json("SELECT record FROM desired_files WHERE file=?1 AND failure IS NULL ORDER BY version DESC,album LIMIT 1",[id])?.context("file has no usable source metadata")?;
+ let file = record.file(context.session.user_id)?;
+ let signature = signature(&file)?;
+ let mut needs_bytes = false;
+ {
+ let mut after = 0;
+ loop {
+ let album:Option<i64>=store.db.connection().query_row("SELECT album FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",rusqlite::params![id,after],|r|r.get(0)).optional()?;
+ let Some(album) = album else { break };
+ after = album;
+ let existing = store.placement(album, id)?;
+ if let Some(mut placement) = existing {
+ match valid_placement(&context.root, &placement, &file, &signature, &cancel) {
+ Ok(true) => {
+ for media in &mut placement.media {
+ let observed = Properties::read(&names::check(
+ &context.root,
+ &format!("{}/{}", placement.folder, media.component.path),
+ )?)?;
+ if media.properties.as_ref() != Some(&observed) {
+ media.intended_time = None;
+ }
+ media.properties = Some(observed);
+ }
+ store.save_placement(&placement)?;
+ }
+ Ok(false) => needs_bytes = true,
+ Err(error) => {
+ super::report(&store, &format!("file:{album}:{id}"), &error)?;
+ store.db.connection().execute(
+ "UPDATE desired_files SET failure=?1 WHERE album=?2 AND file=?3",
+ rusqlite::params![error.to_string(), album, id],
+ )?;
+ if super::fatal(&error) {
+ return Err(error);
+ }
+ }
+ }
+ } else {
+ needs_bytes = true;
+ }
+ }
+ }
+ if !needs_bytes {
+ return Ok(Preparation::Ready(Box::new(Prepared {
+ file,
+ signature,
+ components: Vec::new(),
+ })));
+ }
+ if !capacity {
+ return Ok(Preparation::AwaitingCapacity);
+ }
+ let roles = if file.kind == Kind::LivePhoto {
+ vec![Role::Image, Role::Video]
+ } else {
+ vec![Role::Original]
+ };
+ let mut reusable = Vec::new();
+ {
+ let mut after = String::new();
+ loop {
+ let temporary:Option<super::store::Temporary>=store.json("SELECT record FROM temporaries WHERE json_extract(record,'$.file')=?1 AND json_extract(record,'$.destination') IS NULL AND path>?2 ORDER BY path LIMIT 1",rusqlite::params![id,after])?;
+ let Some(temporary) = temporary else { break };
+ after = temporary.path.clone();
+ if temporary.original.as_deref() != Some(&signature) {
+ fs::discard(&context.root, &store, &temporary.path)?;
+ continue;
+ }
+ if let (Some(role), Some(size), Some(hash), Some(extension)) = (
+ temporary.role,
+ temporary.size,
+ temporary.hash,
+ temporary.extension,
+ ) {
+ let path = names::check(&context.root, &temporary.path)?;
+ if fs::hash_cancellable(&path, &cancel).is_ok_and(|(actual_size, actual_hash)| {
+ actual_size == size && actual_hash == hash
+ }) {
+ reusable.push(Component {
+ role,
+ extension,
+ path: temporary.path,
+ size,
+ hash,
+ temporary: true,
+ });
+ }
+ }
+ }
+ }
+ for retained in [false, true] {
+ let mut before = i64::MAX;
+ loop {
+ if roles
+ .iter()
+ .all(|role| reusable.iter().any(|c| c.role == *role))
+ {
+ break;
+ }
+ if *cancel.borrow() {
+ bail!(super::Cancelled)
+ }
+ let next:Option<(i64,String)>=store.db.connection().query_row("SELECT rowid,record FROM placements WHERE file=?1 AND retained=?2 AND rowid<?3 ORDER BY rowid DESC LIMIT 1",rusqlite::params![id,retained,before],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
+ let Some((rowid, record)) = next else { break };
+ before = rowid;
+ let candidate: Placement = serde_json::from_str(&record)?;
+ for media in &candidate.media {
+ if reusable.iter().any(|c| c.role == media.component.role) {
+ continue;
+ }
+ if expected_hash(&file, &media.component.role)
+ .is_some_and(|hash| hash != media.component.hash)
+ {
+ continue;
+ }
+ if expected_hash(&file, &media.component.role).is_none()
+ && !candidate.original.is_empty()
+ && candidate.original != signature
+ {
+ continue;
+ }
+ let relative = format!("{}/{}", candidate.folder, media.component.path);
+ let valid = (|| {
+ let path = names::check(&context.root, &relative)?;
+ Properties::read(&path)?;
+ let (size, hash) = fs::hash_cancellable(&path, &cancel)?;
+ Ok::<_, anyhow::Error>(
+ size == media.component.size && hash == media.component.hash,
+ )
+ })()
+ .unwrap_or(false);
+ if valid {
+ reusable.push(Component {
+ role: media.component.role.clone(),
+ extension: names::split(&media.component.path, false).1.into(),
+ path: relative,
+ size: media.component.size,
+ hash: media.component.hash.clone(),
+ temporary: false,
+ });
+ }
+ }
+ }
+ }
+ if roles
+ .iter()
+ .all(|role| reusable.iter().any(|c| c.role == *role))
+ {
+ let mut components = Vec::with_capacity(roles.len());
+ for role in roles {
+ let index = reusable
+ .iter()
+ .position(|c| c.role == role)
+ .context("missing verified component")?;
+ components.push(reusable.remove(index));
+ }
+ return Ok(Preparation::Ready(Box::new(Prepared {
+ file,
+ signature,
+ components,
+ })));
+ }
+ let (folder, temporary, output) = loop {
+ let album:Option<i64>=store.db.connection().query_row("SELECT album FROM desired_files WHERE file=?1 AND failure IS NULL ORDER BY album LIMIT 1",[id],|r|r.get(0)).optional()?;
+ let Some(album) = album else {
+ return Ok(Preparation::Ready(Box::new(Prepared {
+ file,
+ signature,
+ components: Vec::new(),
+ })));
+ };
+ let folder = store
+ .album(album, false)?
+ .context("missing destination album")?
+ .path;
+ match fs::temporary(&context.root, &store, &folder, Some(id)) {
+ Ok((temporary, output)) => break (folder, temporary, output),
+ Err(error) => {
+ super::report(&store, &format!("file:{album}:{id}"), &error)?;
+ store.db.connection().execute(
+ "UPDATE desired_files SET failure=?1 WHERE album=?2 AND file=?3",
+ rusqlite::params![error.to_string(), album, id],
+ )?;
+ if super::fatal(&error) {
+ return Err(error);
+ }
+ }
+ }
+ };
+ drop(output);
+ let path = names::check(&context.root, &temporary.path)?;
+ let write_cancel = cancel.clone();
+ let download = files::download(&context.session, &file, || {
+ let mut output = File::options().read(true).write(true).open(&path)?;
+ output.set_len(0)?;
+ output.seek(SeekFrom::Start(0))?;
+ HashWriter::new(output, &write_cancel).map_err(std::io::Error::other)
+ });
+ let writer = tokio::select! {
+ result=download=>result?,
+ _=cancel.changed()=>bail!(super::Cancelled),
+ };
+ let (archive, size, hash) = writer.finish()?;
+ let components = if file.kind == Kind::LivePhoto {
+ let mut temporary_files = Vec::new();
+ let extracted = ente_photos::live_photo::extract(archive, |role, extension| {
+ if *cancel.borrow() {
+ return Err(std::io::Error::other(super::Cancelled));
+ }
+ let (temporary, file) = fs::temporary(&context.root, &store, &folder, Some(id))
+ .map_err(std::io::Error::other)?;
+ temporary_files.push((role, extension.to_owned(), temporary));
+ HashWriter::new(file, &cancel).map_err(std::io::Error::other)
+ })?;
+ let mut components = Vec::with_capacity(2);
+ for ((role, extension, writer), (_, _, temporary)) in
+ extracted.into_iter().zip(temporary_files)
+ {
+ let (_, size, hash) = writer.finish()?;
+ if let Some(expected) = expected_hash(&file, &role) {
+ ensure!(
+ hash == expected,
+ "original component hash does not match source for file {}",
+ file.id
+ );
+ }
+ stage(
+ &store, &temporary, &signature, &role, &extension, size, &hash,
+ )?;
+ components.push(Component {
+ role,
+ extension,
+ path: temporary.path,
+ size,
+ hash,
+ temporary: true,
+ });
+ }
+ fs::discard(&context.root, &store, &temporary.path)?;
+ components
+ } else {
+ drop(archive);
+ if let Some(expected) = expected_hash(&file, &Role::Original) {
+ ensure!(
+ hash == expected,
+ "original hash does not match source for file {}",
+ file.id
+ );
+ }
+ stage(
+ &store,
+ &temporary,
+ &signature,
+ &Role::Original,
+ names::split(&file.name, false).1,
+ size,
+ &hash,
+ )?;
+ vec![Component {
+ role: Role::Original,
+ extension: names::split(&file.name, false).1.into(),
+ path: temporary.path,
+ size,
+ hash,
+ temporary: true,
+ }]
+ };
+ Ok(Preparation::Ready(Box::new(Prepared {
+ file,
+ signature,
+ components,
+ })))
+}
+
+pub fn signature(file: &files::File) -> Result<String> {
+ let mut bytes = file.header.as_bytes().to_vec();
+ bytes.extend_from_slice(file.hash.as_deref().unwrap_or_default().as_bytes());
+ Ok(b64::encode(&hash::hash(
+ &bytes,
+ Some(32),
+ Some(file.key.as_bytes()),
+ )?))
+}
+
+pub fn expected_hash<'a>(file: &'a files::File, role: &Role) -> Option<&'a str> {
+ let hash = file.hash.as_deref()?;
+ match role {
+ Role::Original => Some(hash),
+ Role::Image => hash.split_once(':').map(|pair| pair.0),
+ Role::Video => hash.split_once(':').map(|pair| pair.1),
+ }
+}
+
+pub fn valid_placement(
+ root: &Path,
+ placement: &Placement,
+ file: &files::File,
+ signature: &str,
+ cancel: &watch::Receiver<bool>,
+) -> Result<bool> {
+ if placement.media.len() != if file.kind == Kind::LivePhoto { 2 } else { 1 } {
+ return Ok(false);
+ }
+ for media in &placement.media {
+ if expected_hash(file, &media.component.role)
+ .is_some_and(|hash| hash != media.component.hash)
+ {
+ return Ok(false);
+ }
+ if expected_hash(file, &media.component.role).is_none()
+ && !placement.original.is_empty()
+ && placement.original != signature
+ {
+ return Ok(false);
+ }
+ let path = names::check(
+ root,
+ &format!("{}/{}", placement.folder, media.component.path),
+ )?;
+ if !path.try_exists()? {
+ return Ok(false);
+ }
+ let properties = Properties::read(&path)?;
+ if media.properties.as_ref() != Some(&properties) {
+ let (size, hash) = fs::hash_cancellable(&path, cancel)?;
+ if size != media.component.size || hash != media.component.hash {
+ return Ok(false);
+ }
+ }
+ }
+ Ok(true)
+}
+
+pub fn materialize(
+ root: &Path,
+ store: &Store,
+ source: &mut Component,
+ (folder, name): (&str, &str),
+ file_id: i64,
+ signature: &str,
+ cancel: &watch::Receiver<bool>,
+) -> Result<()> {
+ let destination = format!("{folder}/{name}");
+ let (mut temporary, output) = fs::temporary(root, store, folder, Some(file_id))?;
+ let mut writer = HashWriter::new(output, cancel)?;
+ std::io::copy(
+ &mut File::open(names::check(root, &source.path)?)?,
+ &mut writer,
+ )?;
+ let (_, size, hash) = writer.finish()?;
+ ensure!(
+ size == source.size && hash == source.hash,
+ "local original changed while copying"
+ );
+ temporary.original = Some(signature.into());
+ temporary.role = Some(source.role.clone());
+ temporary.extension = Some(source.extension.clone());
+ temporary.size = Some(size);
+ temporary.hash = Some(hash.clone());
+ store.temporary(&temporary)?;
+ source.path = temporary.path.clone();
+ source.temporary = true;
+ fs::publish(root, store, temporary, &destination, size, hash)?;
+ source.path = destination;
+ source.temporary = false;
+ Ok(())
+}
+
+pub fn discard_incomplete(root: &Path, store: &Store, file: i64) -> Result<()> {
+ let mut after = String::new();
+ loop {
+ let next: Option<(String, i64)> = store.db.connection().query_row("SELECT path,json_extract(record,'$.album') FROM temporaries WHERE json_extract(record,'$.file')=?1 AND json_extract(record,'$.hash') IS NULL AND path>?2 ORDER BY path LIMIT 1", rusqlite::params![file, after], |r| Ok((r.get(0)?, r.get(1)?))).optional()?;
+ let Some((path, album)) = next else { break };
+ after = path.clone();
+ if let Err(error) = fs::discard(root, store, &path) {
+ super::report(store, &format!("file:{album}:{file}"), &error)?;
+ store.db.connection().execute(
+ "UPDATE desired_files SET failure=?1 WHERE album=?2 AND file=?3",
+ rusqlite::params![error.to_string(), album, file],
+ )?;
+ if super::fatal(&error) {
+ return Err(error);
+ }
+ }
+ }
+ Ok(())
+}
+
+pub fn finish_source(root: &Path, store: &Store, components: &[Component]) -> Result<()> {
+ for component in components {
+ if component.temporary {
+ let needed:bool=store.db.connection().query_row("SELECT EXISTS(SELECT 1 FROM desired_files WHERE file=(SELECT json_extract(record,'$.file') FROM temporaries WHERE path=?1) AND completed=0)",[&component.path],|r|r.get(0))?;
+ if !needed {
+ fs::discard(root, store, &component.path)?;
+ }
+ }
+ }
+ Ok(())
+}
+
+fn stage(
+ store: &Store,
+ temporary: &super::store::Temporary,
+ signature: &str,
+ role: &Role,
+ extension: &str,
+ size: u64,
+ hash: &str,
+) -> Result<()> {
+ let mut temporary = temporary.clone();
+ temporary.original = Some(signature.into());
+ temporary.role = Some(role.clone());
+ temporary.extension = Some(extension.into());
+ temporary.size = Some(size);
+ temporary.hash = Some(hash.into());
+ store.temporary(&temporary)
+}
diff --git a/rust/apps/cli-next/src/main.rs b/rust/apps/cli-next/src/main.rs
index fb0d4645b6..e6df40e49b 100644
--- a/rust/apps/cli-next/src/main.rs
+++ b/rust/apps/cli-next/src/main.rs
@@ -2,6 +2,7 @@ mod api;
mod args;
mod core_db;
mod db;
+mod export;
mod home;
mod login;
mod output;
@@ -28,7 +29,7 @@ use args::{
use output::AccountView;
use vault::{State, Vault};
-#[tokio::main(flavor = "current_thread")]
+#[tokio::main]
async fn main() {
if let Err(error) = run(Cli::parse()).await {
eprintln!("Error: {error:#}");
diff --git a/rust/apps/cli-next/src/output.rs b/rust/apps/cli-next/src/output.rs
index 6c679ee3d0..dee0265d5c 100644
--- a/rust/apps/cli-next/src/output.rs
+++ b/rust/apps/cli-next/src/output.rs
@@ -3,7 +3,10 @@ use std::io::Write;
use anyhow::{Context, Result};
use chrono::{DateTime, SecondsFormat};
use dialoguer::console::{Alignment, measure_text_width, pad_str};
-use ente_photos::{collections::Collection, files::File};
+use ente_photos::{
+ collections::{Collection, Visibility},
+ files::File,
+};
use serde::Serialize;
use uuid::Uuid;
@@ -73,7 +76,7 @@ pub struct FileView {
owner_id: String,
album_ids: Vec<String>,
created_at: String,
- modified_at: String,
+ modified_at: Option<String>,
updated_at: String,
location: Option<LocationView>,
caption: Option<String>,
@@ -83,7 +86,7 @@ pub struct FileView {
duration_seconds: Option<u64>,
width: Option<u32>,
height: Option<u32>,
- visibility: &'static str,
+ visibility: Option<&'static str>,
}
#[derive(Serialize)]
@@ -101,7 +104,7 @@ impl FileView {
owner_id: file.owner_id.to_string(),
album_ids: album_ids.iter().map(i64::to_string).collect(),
created_at: timestamp(file.created_at_micros)?,
- modified_at: timestamp(file.modified_at_micros)?,
+ modified_at: file.modified_at_micros.map(timestamp).transpose()?,
updated_at: timestamp(file.updated_at_micros)?,
location: file.location.as_ref().map(|location| LocationView {
latitude: location.latitude,
@@ -114,7 +117,7 @@ impl FileView {
duration_seconds: file.duration_seconds,
width: file.width,
height: file.height,
- visibility: file.visibility.name(),
+ visibility: file.visibility.map(Visibility::name),
})
}
}
@@ -236,9 +239,12 @@ pub fn file(file: &FileView) -> Result<()> {
let mut fields = vec![
("Name", escape_controls(&file.name)),
("Type", file.kind.to_owned()),
- ("Visibility", file.visibility.to_owned()),
+ (
+ "Visibility",
+ file.visibility.unwrap_or("unavailable").to_owned(),
+ ),
("Created", file.created_at.clone()),
- ("Modified", file.modified_at.clone()),
+ ("Modified", file.modified_at.clone().unwrap_or_default()),
("Updated", file.updated_at.clone()),
("Owner", file.owner_id.clone()),
("Albums", file.album_ids.join(", ")),
diff --git a/rust/apps/cli-next/src/photos.rs b/rust/apps/cli-next/src/photos.rs
index 6aa4e19a37..7d2235c6c9 100644
--- a/rust/apps/cli-next/src/photos.rs
+++ b/rust/apps/cli-next/src/photos.rs
@@ -21,6 +21,9 @@ pub async fn run(
selected: Option<&str>,
options: &Options,
) -> Result<()> {
+ if let PhotosLibraryCommand::Export(args) = command {
+ return crate::export::run(args, selected, options).await;
+ }
ensure!(
!options.offline
|| !matches!(
@@ -35,7 +38,7 @@ pub async fn run(
let (account, home) = open_account(selected, !options.offline)?;
let session = api::session(&account, Product::Photos)?;
let mut db = db::open(&home.path, &account.db_key, !options.offline)?;
- let mut replica = Replica::new(&mut db);
+ let mut replica = Replica::new(&mut db, session.user_id);
if !options.offline {
replica.sync_collections(&session).await?;
} else {
@@ -45,6 +48,7 @@ pub async fn run(
);
}
match command {
+ PhotosLibraryCommand::Export(_) => unreachable!(),
PhotosLibraryCommand::Album { command } => album(command, &replica, options),
PhotosLibraryCommand::File { album, command } => {
file(command, album.as_deref(), &session, &mut replica, options).await
diff --git a/rust/apps/cli-next/src/replica.rs b/rust/apps/cli-next/src/replica.rs
index 3804524d62..c23a6b75f8 100644
--- a/rust/apps/cli-next/src/replica.rs
+++ b/rust/apps/cli-next/src/replica.rs
@@ -1,122 +1,270 @@
-use anyhow::Result;
+use anyhow::{Context, Result};
use ente_core::{
- Session,
- crypto::{Header, Key},
+ Session, b64,
+ crypto::{Key, hash},
};
-
use ente_photos::{
- collections::{self, Collection, Visibility},
- files::{self, File, Location},
+ collections::{self, Collection, RemoteCollection},
+ files::{self, File, RemoteFile},
+ source::Documents,
};
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
use crate::core_db::{
- self, Connection, Db, OptionalExtension, Row, SqliteError, SqliteResult, params,
- params_from_iter,
+ self, Db, OptionalExtension, Row, SqliteError, SqliteResult, params, params_from_iter,
types::{Type, Value},
};
-pub struct Entry {
- pub file: File,
- pub album_ids: Vec<i64>,
-}
-
-pub struct Replica<'a> {
- db: &'a mut Db,
-}
-
-const COLLECTION_COLUMNS: &str = "id, name, kind, visibility, owner_id, updated_at, key";
-const FILE_COLUMNS: &str = "f.id, f.owner_id, f.updated_at, f.name, f.kind, f.created_at, f.modified_at,
- f.latitude, f.longitude, f.caption, f.hash, f.date_time, f.offset_time, f.duration, f.width, f.height,
- f.visibility, f.key, f.header";
-
pub const SCHEMA: &str = "
- CREATE TABLE photos_sync (
- id INTEGER PRIMARY KEY CHECK (id = 1),
- classical_collections_cursor INTEGER NOT NULL
- );
+ CREATE TABLE photos_sync (id INTEGER PRIMARY KEY CHECK(id=1), cursor INTEGER NOT NULL);
CREATE TABLE photos_collections (
- id INTEGER PRIMARY KEY, name TEXT NOT NULL, kind TEXT NOT NULL,
- visibility TEXT NOT NULL, owner_id INTEGER NOT NULL, updated_at INTEGER NOT NULL,
- key BLOB NOT NULL, files_cursor INTEGER NOT NULL DEFAULT 0, files_synced_to INTEGER
+ id INTEGER PRIMARY KEY, name TEXT, updated_at INTEGER NOT NULL,
+ record TEXT NOT NULL, files_cursor INTEGER NOT NULL DEFAULT 0, files_synced_to INTEGER
);
CREATE INDEX photos_collections_name ON photos_collections(name);
CREATE TABLE photos_files (
- id INTEGER PRIMARY KEY, owner_id INTEGER NOT NULL, updated_at INTEGER NOT NULL,
- name TEXT NOT NULL, kind TEXT NOT NULL, created_at INTEGER NOT NULL, modified_at INTEGER NOT NULL,
- latitude REAL, longitude REAL, caption TEXT, hash TEXT, date_time TEXT, offset_time TEXT,
- duration INTEGER, width INTEGER, height INTEGER, visibility TEXT NOT NULL, key BLOB NOT NULL, header BLOB NOT NULL
- );
- CREATE INDEX photos_files_name ON photos_files(name);
- CREATE TABLE photos_memberships (
collection_id INTEGER NOT NULL REFERENCES photos_collections(id) ON DELETE CASCADE,
- file_id INTEGER NOT NULL REFERENCES photos_files(id) ON DELETE CASCADE,
- PRIMARY KEY(collection_id, file_id)
+ id INTEGER NOT NULL, name TEXT, updated_at INTEGER NOT NULL,
+ record TEXT NOT NULL, decrypt_failed INTEGER NOT NULL,
+ PRIMARY KEY(collection_id,id)
);
- CREATE INDEX photos_memberships_file ON photos_memberships(file_id);
+ CREATE INDEX photos_files_id ON photos_files(id,updated_at DESC);
+ CREATE INDEX photos_files_name ON photos_files(name);
";
+#[derive(Clone, Serialize, Deserialize)]
+pub struct Record<R> {
+ pub remote: R,
+ pub documents: Option<Documents>,
+ pub key: Option<Vec<u8>>,
+ pub input_hash: String,
+ pub failure: Option<String>,
+ pub attempt: String,
+}
+
+pub type AlbumRecord = Record<RemoteCollection>;
+pub type FileRecord = Record<RemoteFile>;
+
+impl<R: Serialize> Record<R> {
+ fn receive<E: RecordError>(
+ remote: R,
+ previous: Option<Self>,
+ parent_key: &[u8],
+ retry: bool,
+ attempt: &str,
+ decrypt: impl FnOnce(&R) -> std::result::Result<(Key, Documents), E>,
+ ) -> Result<Self> {
+ let mut input = serde_json::to_value(&remote)?;
+ if let Some(object) = input.as_object_mut() {
+ object.retain(|name, _| {
+ [
+ "encryptedKey",
+ "keyDecryptionNonce",
+ "name",
+ "encryptedName",
+ "nameDecryptionNonce",
+ "metadata",
+ "magicMetadata",
+ "pubMagicMetadata",
+ "sharedMagicMetadata",
+ ]
+ .contains(&name.as_str())
+ });
+ for name in [
+ "metadata",
+ "magicMetadata",
+ "pubMagicMetadata",
+ "sharedMagicMetadata",
+ ] {
+ if let Some(object) = object
+ .get_mut(name)
+ .and_then(serde_json::Value::as_object_mut)
+ {
+ object.retain(|name, _| {
+ ["encryptedData", "decryptionHeader", "data", "header"]
+ .contains(&name.as_str())
+ });
+ }
+ }
+ }
+ let input_hash = b64::encode(&hash::hash(
+ &serde_json::to_vec(&input)?,
+ Some(32),
+ Some(parent_key),
+ )?);
+ if let Some(mut previous) = previous
+ && previous.input_hash == input_hash
+ && !(retry && previous.documents.is_none())
+ {
+ previous.remote = remote;
+ return Ok(previous);
+ }
+ let (key, documents, failure) = match decrypt(&remote) {
+ Ok((key, documents)) => (Some(key.as_bytes().to_vec()), Some(documents), None),
+ Err(error) if error.is_record_failure() => (None, None, Some(error.to_string())),
+ Err(error) => return Err(error.into()),
+ };
+ Ok(Self {
+ remote,
+ key,
+ documents,
+ input_hash,
+ failure,
+ attempt: attempt.to_owned(),
+ })
+ }
+
+ fn opened(&self) -> Result<(Key, &Documents)> {
+ let documents = self.documents.as_ref().with_context(|| {
+ self.failure
+ .clone()
+ .unwrap_or_else(|| "metadata is unavailable".into())
+ })?;
+ let key = Key::try_from_slice(self.key.as_deref().context("decrypted record has no key")?)?;
+ Ok((key, documents))
+ }
+}
+
+trait RecordError: std::error::Error + Send + Sync + 'static {
+ fn is_record_failure(&self) -> bool;
+}
+
+fn decryption_failure(error: &ente_core::crypto::Error) -> bool {
+ use ente_core::crypto::Error::*;
+ matches!(
+ error,
+ InvalidKeyLength { .. }
+ | InvalidNonceLength { .. }
+ | InvalidHeaderLength { .. }
+ | CiphertextTooShort { .. }
+ | DecryptionFailed
+ | StreamPullFailed
+ | StreamTruncated
+ | StreamTrailingData
+ | SealedBoxOpenFailed
+ )
+}
+
+impl RecordError for ente_collections::Error {
+ fn is_record_failure(&self) -> bool {
+ use ente_collections::Error::*;
+ match self {
+ Decode(_) | MissingKeyDecryptionNonce | InvalidCollection { .. } => true,
+ Crypto(error) => decryption_failure(error),
+ _ => false,
+ }
+ }
+}
+
+impl RecordError for files::Error {
+ fn is_record_failure(&self) -> bool {
+ match self {
+ Self::Base64(_) => true,
+ Self::Crypto(error) => decryption_failure(error),
+ Self::Collections(error) => error.is_record_failure(),
+ _ => false,
+ }
+ }
+}
+
+impl AlbumRecord {
+ pub fn album(&self, user_id: i64) -> Result<Collection> {
+ let (key, documents) = self.opened()?;
+ Ok(collections::interpret(
+ &self.remote,
+ &key,
+ documents,
+ user_id,
+ )?)
+ }
+}
+
+impl FileRecord {
+ pub fn file(&self, user_id: i64) -> Result<File> {
+ let (key, documents) = self.opened()?;
+ Ok(files::interpret(&self.remote, &key, documents, user_id)?)
+ }
+}
+
+pub struct Entry {
+ pub file: File,
+ pub album_ids: Vec<i64>,
+}
+
+pub struct Replica<'a> {
+ pub db: &'a mut Db,
+ user_id: i64,
+ attempt: String,
+}
+
impl<'a> Replica<'a> {
- pub fn new(db: &'a mut Db) -> Self {
- Self { db }
+ pub fn new(db: &'a mut Db, user_id: i64) -> Self {
+ Self {
+ db,
+ user_id,
+ attempt: uuid::Uuid::new_v4().to_string(),
+ }
}
pub async fn sync_collections(&mut self, session: &Session) -> Result<()> {
- let since = self.collections_cursor()?.unwrap_or(0);
- let page = collections::diff(session, since).await?;
+ let page = ente_collections::client::diff(session, self.collections_cursor()?.unwrap_or(0))
+ .await?;
+ let mut changes = Vec::with_capacity(page.collections.len());
+ for remote in page.collections {
+ let id = remote.id;
+ if remote.is_deleted == Some(true) {
+ changes.push((id, None));
+ continue;
+ }
+ let previous = self.album_record(id)?;
+ let parent_key = if remote.owner.id == session.user_id {
+ session.master_key.as_bytes().as_slice()
+ } else {
+ session.secret_key.as_bytes().as_slice()
+ };
+ let record =
+ Record::receive(remote, previous, parent_key, false, &self.attempt, |r| {
+ collections::decrypt(r, session)
+ })?;
+ changes.push((id, Some(record)));
+ }
self.db.write(|tx| {
- for change in page.changes {
- match change.collection {
- Some(c) => {
- tx.execute("
- INSERT INTO photos_collections (id, name, kind, visibility, owner_id, updated_at, key)
- VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
- ON CONFLICT(id) DO UPDATE SET name=excluded.name, kind=excluded.kind,
- visibility=excluded.visibility, owner_id=excluded.owner_id,
- updated_at=excluded.updated_at, key=excluded.key
- ", params![c.id, c.name, c.kind.name(), c.visibility.name(), c.owner_id, c.updated_at_micros, c.key.as_bytes()])?;
- }
- None => {
- tx.execute(
- "DELETE FROM photos_files WHERE id IN
- (SELECT file_id FROM photos_memberships WHERE collection_id=?1)
- AND NOT EXISTS (SELECT 1 FROM photos_memberships
- WHERE file_id=photos_files.id AND collection_id!=?1)",
- [change.id],
- )?;
- tx.execute("DELETE FROM photos_collections WHERE id=?1", [change.id])?;
- }
+ for (id, record) in changes {
+ if let Some(record) = record {
+ save_album(tx, record, self.user_id)?;
+ } else {
+ tx.execute("DELETE FROM photos_collections WHERE id=?1", [id])?;
}
}
- tx.execute("INSERT INTO photos_sync VALUES (1, ?1)
- ON CONFLICT(id) DO UPDATE SET classical_collections_cursor=excluded.classical_collections_cursor", [page.cursor])?;
+ tx.execute("INSERT INTO photos_sync VALUES(1,?1) ON CONFLICT(id) DO UPDATE SET cursor=excluded.cursor", [page.cursor])?;
Ok(())
})?;
Ok(())
}
- pub fn collections<T>(
- &self,
- selector: Option<&str>,
- limit: Option<i64>,
- read: impl FnOnce(&mut dyn Iterator<Item = SqliteResult<Collection>>) -> Result<T>,
- ) -> Result<T> {
- let mut sql = format!("SELECT {COLLECTION_COLUMNS} FROM photos_collections");
- let mut parameters = Vec::new();
- if let Some(selector) = selector {
- sql.push_str(" WHERE id=? OR name=?");
- parameters.push(canonical_id(selector));
- parameters.push(Value::Text(selector.to_owned()));
- }
- sql.push_str(" ORDER BY id");
- if let Some(limit) = limit {
- sql.push_str(" LIMIT ?");
- parameters.push(Value::Integer(limit));
+ pub fn retry_album(&mut self, session: &Session, id: i64, force: bool) -> Result<()> {
+ let Some(previous) = self.album_record(id)? else {
+ return Ok(());
+ };
+ if previous.documents.is_some() {
+ return Ok(());
}
- self.db.read(|conn| {
- let mut statement = conn.prepare(&sql)?;
- let mut rows = statement.query_map(params_from_iter(parameters), read_collection)?;
- Ok(read(&mut rows))
- })?
+ let parent_key = if previous.remote.owner.id == session.user_id {
+ session.master_key.as_bytes().as_slice()
+ } else {
+ session.secret_key.as_bytes().as_slice()
+ };
+ let retry = force && previous.attempt != self.attempt;
+ let record = Record::receive(
+ previous.remote.clone(),
+ Some(previous),
+ parent_key,
+ retry,
+ &self.attempt,
+ |r| collections::decrypt(r, session),
+ )?;
+ self.db.write(|tx| save_album(tx, record, self.user_id))?;
+ Ok(())
}
pub async fn sync_files(
@@ -125,50 +273,51 @@ impl<'a> Replica<'a> {
collections: &[Collection],
) -> Result<()> {
for collection in collections {
- let (mut cursor, synced_to): (i64, Option<i64>) = self.db.read(|conn| {
- Ok(conn.query_row(
- "SELECT files_cursor, files_synced_to FROM photos_collections WHERE id=?1",
- [collection.id],
- |row| Ok((row.get(0)?, row.get(1)?)),
- )?)
- })?;
- // Museum advances the collection timestamp on file and membership changes.
- if synced_to.is_some_and(|time| time >= collection.updated_at_micros) {
- continue;
- }
+ self.sync_album_files(session, collection, false).await?;
+ }
+ Ok(())
+ }
+
+ pub async fn sync_album_files(
+ &mut self,
+ session: &Session,
+ album: &Collection,
+ retry_failed: bool,
+ ) -> Result<()> {
+ let (mut cursor, synced_to): (i64, Option<i64>) = self.db.read(|db| {
+ Ok(db.query_row(
+ "SELECT files_cursor,files_synced_to FROM photos_collections WHERE id=?1",
+ [album.id],
+ |r| Ok((r.get(0)?, r.get(1)?)),
+ )?)
+ })?;
+ if synced_to.is_none_or(|time| time < album.updated_at_micros) {
loop {
- let page = files::diff(session, collection, cursor).await?;
- self.db.write(|tx| {
- for change in page.changes {
- match change.file {
- Some(file) => {
- save_file(tx, &file)?;
- tx.execute(
- "INSERT OR IGNORE INTO photos_memberships VALUES (?1, ?2)",
- params![collection.id, file.id],
- )?;
- }
- None => {
- tx.execute("DELETE FROM photos_memberships WHERE collection_id=?1 AND file_id=?2",
- params![collection.id, change.id])?;
- tx.execute(
- "DELETE FROM photos_files WHERE id=?1 AND NOT EXISTS
- (SELECT 1 FROM photos_memberships WHERE file_id=?1)",
- [change.id],
- )?;
- }
- }
+ let page = ente_collections::client::files_diff(session, album.id, cursor).await?;
+ let mut changes = Vec::with_capacity(page.files.len());
+ for remote in page.files {
+ let id = remote.id;
+ if remote.is_deleted() {
+ changes.push((id, None));
+ continue;
}
- tx.execute(
- "UPDATE photos_collections SET files_cursor=?1,
- files_synced_to=CASE WHEN ?2 THEN files_synced_to ELSE ?3 END WHERE id=?4",
- params![
- page.cursor,
- page.has_more,
- collection.updated_at_micros,
- collection.id
- ],
+ let previous = self.file_record(album.id, id)?;
+ let record = Record::receive(
+ remote,
+ previous,
+ album.key.as_bytes(),
+ false,
+ &self.attempt,
+ |r| files::decrypt(r, &album.key),
)?;
+ changes.push((id, Some(record)));
+ }
+ self.db.write(|tx| {
+ for (id, record) in changes {
+ if let Some(record) = record { save_file(tx, album.id, record, self.user_id)?; }
+ else { tx.execute("DELETE FROM photos_files WHERE collection_id=?1 AND id=?2", params![album.id,id])?; }
+ }
+ tx.execute("UPDATE photos_collections SET files_cursor=?1,files_synced_to=CASE WHEN ?2 THEN files_synced_to ELSE ?3 END WHERE id=?4", params![page.cursor,page.has_more,album.updated_at_micros,album.id])?;
Ok(())
})?;
cursor = page.cursor;
@@ -177,17 +326,82 @@ impl<'a> Replica<'a> {
}
}
}
+ let mut after = 0;
+ loop {
+ let records: Vec<FileRecord> = self.db.read(|db| {
+ let mut query = db.prepare("SELECT record FROM photos_files WHERE collection_id=?1 AND decrypt_failed=1 AND id>?2 ORDER BY id LIMIT 128")?;
+ Ok(query.query_map(params![album.id,after], |r| read_json(r,0))?.collect::<SqliteResult<_>>()?)
+ })?;
+ if records.is_empty() {
+ break;
+ }
+ for previous in records {
+ after = previous.remote.id;
+ let retry = retry_failed && previous.attempt != self.attempt;
+ let record = Record::receive(
+ previous.remote.clone(),
+ Some(previous),
+ album.key.as_bytes(),
+ retry,
+ &self.attempt,
+ |r| files::decrypt(r, &album.key),
+ )?;
+ self.db
+ .write(|tx| save_file(tx, album.id, record, self.user_id))?;
+ }
+ }
Ok(())
}
- pub fn files_synced_to(&self, collection_id: i64) -> core_db::Result<Option<i64>> {
- self.db.read(|conn| {
- Ok(conn.query_row(
- "SELECT files_synced_to FROM photos_collections WHERE id=?1",
- [collection_id],
- |row| row.get(0),
- )?)
- })
+ pub fn album_record(&self, id: i64) -> Result<Option<AlbumRecord>> {
+ Ok(self.db.read(|db| {
+ Ok(db
+ .query_row(
+ "SELECT record FROM photos_collections WHERE id=?1",
+ [id],
+ |r| read_json(r, 0),
+ )
+ .optional()?)
+ })?)
+ }
+
+ pub fn file_record(&self, album: i64, id: i64) -> Result<Option<FileRecord>> {
+ Ok(self.db.read(|db| {
+ Ok(db
+ .query_row(
+ "SELECT record FROM photos_files WHERE collection_id=?1 AND id=?2",
+ params![album, id],
+ |r| read_json(r, 0),
+ )
+ .optional()?)
+ })?)
+ }
+
+ pub fn collections<T>(
+ &self,
+ selector: Option<&str>,
+ limit: Option<i64>,
+ read: impl FnOnce(&mut dyn Iterator<Item = SqliteResult<Collection>>) -> Result<T>,
+ ) -> Result<T> {
+ let mut sql = "SELECT record FROM photos_collections".to_owned();
+ let mut parameters = Vec::new();
+ if let Some(selector) = selector {
+ sql.push_str(" WHERE id=? OR name=?");
+ parameters.extend([canonical_id(selector), Value::Text(selector.to_owned())]);
+ }
+ sql.push_str(" ORDER BY id");
+ if let Some(limit) = limit {
+ sql.push_str(" LIMIT ?");
+ parameters.push(Value::Integer(limit));
+ }
+ self.db.read(|db| {
+ let mut statement = db.prepare(&sql)?;
+ let mut rows = statement.query_map(params_from_iter(parameters), |row| {
+ let record: AlbumRecord = read_json(row, 0)?;
+ record.album(self.user_id).map_err(conversion)
+ })?;
+ Ok(read(&mut rows))
+ })?
}
pub fn files<T>(
@@ -197,168 +411,120 @@ impl<'a> Replica<'a> {
limit: Option<i64>,
read: impl FnOnce(&mut dyn Iterator<Item = SqliteResult<Entry>>) -> Result<T>,
) -> Result<T> {
- let album_ids = if album.is_some() {
- "json_array(m.collection_id)"
- } else {
- "json_group_array(m.collection_id ORDER BY m.collection_id)"
- };
- let mut sql = format!(
- "SELECT {FILE_COLUMNS}, {album_ids}
- FROM photos_files f JOIN photos_memberships m ON m.file_id=f.id"
- );
- let mut parameters = Vec::new();
let mut conditions = Vec::new();
- if let Some(selector) = selector {
- conditions.push("(f.id=? OR f.name=?)");
- parameters.push(canonical_id(selector));
- parameters.push(Value::Text(selector.to_owned()));
- }
+ let mut parameters = Vec::new();
if let Some(album) = album {
- conditions.push("m.collection_id=?");
+ conditions.push("collection_id=?");
parameters.push(Value::Integer(album));
}
+ if let Some(selector) = selector {
+ conditions.push("(id=? OR name=?)");
+ parameters.extend([canonical_id(selector), Value::Text(selector.into())]);
+ }
+ let mut sql = "WITH matched AS (SELECT * FROM photos_files".to_owned();
if !conditions.is_empty() {
sql.push_str(" WHERE ");
sql.push_str(&conditions.join(" AND "));
}
- if album.is_some() {
- sql.push_str(" ORDER BY m.file_id");
- } else {
- sql.push_str(" GROUP BY f.id ORDER BY f.id");
- }
+ sql.push_str(") SELECT record,(SELECT json_group_array(collection_id ORDER BY collection_id) FROM matched m WHERE m.id=f.id) FROM (SELECT *,row_number() OVER(PARTITION BY id ORDER BY updated_at DESC,collection_id) AS rank FROM matched) f WHERE rank=1 ORDER BY id");
if let Some(limit) = limit {
sql.push_str(" LIMIT ?");
parameters.push(Value::Integer(limit));
}
- self.db.read(|conn| {
- let mut statement = conn.prepare(&sql)?;
+ self.db.read(|db| {
+ let mut statement = db.prepare(&sql)?;
let mut rows = statement.query_map(params_from_iter(parameters), |row| {
- let ids: String = row.get(19)?;
- let album_ids = serde_json::from_str(&ids).map_err(|error| {
- SqliteError::FromSqlConversionFailure(19, Type::Text, Box::new(error))
- })?;
+ let record: FileRecord = read_json(row, 0)?;
Ok(Entry {
- file: read_file(row)?,
- album_ids,
+ file: record.file(self.user_id).map_err(conversion)?,
+ album_ids: read_json(row, 1)?,
})
})?;
Ok(read(&mut rows))
})?
}
+ pub fn files_synced_to(&self, id: i64) -> core_db::Result<Option<i64>> {
+ self.db.read(|db| {
+ Ok(db.query_row(
+ "SELECT files_synced_to FROM photos_collections WHERE id=?1",
+ [id],
+ |r| r.get(0),
+ )?)
+ })
+ }
+
pub fn collections_cursor(&self) -> core_db::Result<Option<i64>> {
- self.db.read(|conn| {
- Ok(conn
- .query_row(
- "SELECT classical_collections_cursor FROM photos_sync WHERE id=1",
- [],
- |row| row.get(0),
- )
+ self.db.read(|db| {
+ Ok(db
+ .query_row("SELECT cursor FROM photos_sync WHERE id=1", [], |r| {
+ r.get(0)
+ })
.optional()?)
})
}
}
-fn canonical_id(selector: &str) -> Value {
- selector
- .parse::<i64>()
- .ok()
- .filter(|id| id.to_string() == selector)
- .map(Value::Integer)
- .unwrap_or(Value::Null)
+fn save_album(
+ tx: &crate::core_db::Connection,
+ mut record: AlbumRecord,
+ user_id: i64,
+) -> core_db::Result<()> {
+ let name = match record.album(user_id) {
+ Ok(album) => {
+ record.failure = None;
+ Some(album.name)
+ }
+ Err(error) => {
+ record.failure = Some(error.to_string());
+ None
+ }
+ };
+ let json = encode(&record)?;
+ tx.execute("INSERT INTO photos_collections(id,name,updated_at,record) VALUES(?1,?2,?3,?4) ON CONFLICT(id) DO UPDATE SET name=coalesce(excluded.name,photos_collections.name),updated_at=excluded.updated_at,record=excluded.record", params![record.remote.id,name,record.remote.updation_time,json])?;
+ Ok(())
}
-fn read_collection(row: &Row<'_>) -> SqliteResult<Collection> {
- let key: Vec<u8> = row.get(6)?;
- Ok(Collection {
- id: row.get(0)?,
- name: row.get(1)?,
- kind: match row.get::<_, String>(2)?.as_str() {
- "favorites" => collections::Kind::Favorites,
- "uncategorized" => collections::Kind::Uncategorized,
- _ => collections::Kind::Album,
- },
- visibility: visibility(&row.get::<_, String>(3)?),
- owner_id: row.get(4)?,
- updated_at_micros: row.get(5)?,
- key: Key::try_from_slice(&key).map_err(|error| {
- SqliteError::FromSqlConversionFailure(6, Type::Blob, Box::new(error))
- })?,
- })
+fn save_file(
+ tx: &crate::core_db::Connection,
+ album: i64,
+ mut record: FileRecord,
+ user_id: i64,
+) -> core_db::Result<()> {
+ let name = match record.file(user_id) {
+ Ok(file) => {
+ record.failure = None;
+ Some(file.name)
+ }
+ Err(error) => {
+ record.failure = Some(error.to_string());
+ None
+ }
+ };
+ let json = encode(&record)?;
+ tx.execute("INSERT INTO photos_files VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(collection_id,id) DO UPDATE SET name=coalesce(excluded.name,photos_files.name),updated_at=excluded.updated_at,record=excluded.record,decrypt_failed=excluded.decrypt_failed WHERE excluded.updated_at>=photos_files.updated_at", params![album,record.remote.id,name,record.remote.updation_time,json,record.documents.is_none()])?;
+ Ok(())
}
-fn visibility(name: &str) -> Visibility {
- match name {
- "archived" => Visibility::Archived,
- "hidden" => Visibility::Hidden,
- _ => Visibility::Visible,
- }
+pub fn encode(value: &impl Serialize) -> SqliteResult<String> {
+ serde_json::to_string(value).map_err(|e| SqliteError::ToSqlConversionFailure(Box::new(e)))
}
-fn read_file(row: &Row<'_>) -> SqliteResult<File> {
- let key: Vec<u8> = row.get(17)?;
- let header: Vec<u8> = row.get(18)?;
- let latitude: Option<f64> = row.get(7)?;
- let longitude: Option<f64> = row.get(8)?;
- Ok(File {
- id: row.get(0)?,
- owner_id: row.get(1)?,
- updated_at_micros: row.get(2)?,
- name: row.get(3)?,
- kind: match row.get::<_, String>(4)?.as_str() {
- "image" => files::Kind::Image,
- "video" => files::Kind::Video,
- "livephoto" => files::Kind::LivePhoto,
- _ => files::Kind::Unknown,
- },
- created_at_micros: row.get(5)?,
- modified_at_micros: row.get(6)?,
- location: latitude
- .zip(longitude)
- .map(|(latitude, longitude)| Location {
- latitude,
- longitude,
- }),
- caption: row.get(9)?,
- hash: row.get(10)?,
- date_time: row.get(11)?,
- offset_time: row.get(12)?,
- duration_seconds: row
- .get::<_, Option<i64>>(13)?
- .map(u64::try_from)
- .transpose()
- .map_err(|error| {
- SqliteError::FromSqlConversionFailure(13, Type::Integer, Box::new(error))
- })?,
- width: row.get(14)?,
- height: row.get(15)?,
- visibility: visibility(&row.get::<_, String>(16)?),
- key: Key::try_from_slice(&key).map_err(|error| {
- SqliteError::FromSqlConversionFailure(17, Type::Blob, Box::new(error))
- })?,
- header: Header::try_from_slice(&header).map_err(|error| {
- SqliteError::FromSqlConversionFailure(18, Type::Blob, Box::new(error))
- })?,
- })
+pub fn read_json<T: DeserializeOwned>(row: &Row<'_>, column: usize) -> SqliteResult<T> {
+ let text: String = row.get(column)?;
+ serde_json::from_str(&text)
+ .map_err(|e| SqliteError::FromSqlConversionFailure(column, Type::Text, Box::new(e)))
}
-fn save_file(connection: &Connection, file: &File) -> SqliteResult<()> {
- let duration = file
- .duration_seconds
- .map(i64::try_from)
- .transpose()
- .map_err(|error| SqliteError::ToSqlConversionFailure(Box::new(error)))?;
- connection.execute("INSERT INTO photos_files VALUES
- (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19)
- ON CONFLICT(id) DO UPDATE SET owner_id=excluded.owner_id, updated_at=excluded.updated_at,
- name=excluded.name, kind=excluded.kind, created_at=excluded.created_at, modified_at=excluded.modified_at,
- latitude=excluded.latitude, longitude=excluded.longitude, caption=excluded.caption, hash=excluded.hash,
- date_time=excluded.date_time, offset_time=excluded.offset_time, duration=excluded.duration,
- width=excluded.width, height=excluded.height, visibility=excluded.visibility, key=excluded.key, header=excluded.header
- WHERE excluded.updated_at >= photos_files.updated_at",
- params![file.id, file.owner_id, file.updated_at_micros, file.name, file.kind.name(), file.created_at_micros,
- file.modified_at_micros, file.location.as_ref().map(|l| l.latitude), file.location.as_ref().map(|l| l.longitude),
- file.caption, file.hash, file.date_time, file.offset_time, duration, file.width, file.height,
- file.visibility.name(), file.key.as_bytes(), file.header.as_bytes()])?;
- Ok(())
+fn conversion(error: anyhow::Error) -> SqliteError {
+ SqliteError::FromSqlConversionFailure(0, Type::Text, error.into_boxed_dyn_error())
+}
+
+pub fn canonical_id(selector: &str) -> Value {
+ selector
+ .parse::<i64>()
+ .ok()
+ .filter(|id| id.to_string() == selector)
+ .map(Value::Integer)
+ .unwrap_or(Value::Null)
}
diff --git a/rust/apps/cli-next/tests/cli.rs b/rust/apps/cli-next/tests/cli.rs
index 3d28cb0e33..cd53081da6 100644
--- a/rust/apps/cli-next/tests/cli.rs
+++ b/rust/apps/cli-next/tests/cli.rs
@@ -742,3 +742,6 @@ mod files;
#[cfg(feature = "museum")]
#[path = "support/museum.rs"]
mod museum;
+
+#[path = "support/export_sources.rs"]
+mod export_sources;
diff --git a/rust/apps/cli-next/tests/support/export.rs b/rust/apps/cli-next/tests/support/export.rs
new file mode 100644
index 0000000000..098f0e03a0
--- /dev/null
+++ b/rust/apps/cli-next/tests/support/export.rs
@@ -0,0 +1,1101 @@
+use super::*;
+use ente_core::crypto::hash;
+use std::{
+ path::Path,
+ time::{Duration, UNIX_EPOCH},
+};
+
+#[cfg(unix)]
+#[path = "export_resources.rs"]
+mod resources;
+
+#[test]
+fn export_lifecycle_and_adoption() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin=museum.endpoint();
+ let objects=museum.object_store();
+ let email=format!("export-{}@example.org",Uuid::new_v4());
+ let account=create_account(origin,&email).await;
+ let home=TestHome::new();login(&home,"photos",&email,&["--host",origin]);
+ let destination=tempfile::tempdir()?;
+ let root=destination.path().join("photos");
+ let (family,family_key)=create_album(origin,&account,"Family","album").await;
+ let (travel,travel_key)=create_album(origin,&account,"Travel","folder").await;
+ let (work,work_key)=create_album(origin,&account,"Work","album").await;
+ let (favorites,favorites_key)=create_album(origin,&account,"Favorites","favorites").await;
+ let (empty,_)=create_album(origin,&account,"Trash","album").await;
+ let original=b"unmodified original bytes";
+ let (file,key)=upload_fixture(origin,&account,family,&family_key,original,metadata("A.jpg",original)).await;
+ place(origin,&account,travel,&travel_key,file,&key,"add-files").await;
+ let (work_file,work_file_key)=upload_fixture(origin,&account,work,&work_key,b"work",metadata("Work.jpg",b"work")).await;
+ let initial_reads=objects.reads();
+ let result=export(&home,&root,&["--album","Family"]);
+ assert_eq!(result["copies"]["completed"],1);
+ assert_eq!(objects.reads(),initial_reads+1);
+ assert!(!root.join("Travel").exists());
+ let root_bytes=fs::read(root.join("export.json"))?;
+ let original_time=fs::metadata(root.join("Family/A.jpg"))?.modified()?;
+ assert_eq!(original_time,UNIX_EPOCH+Duration::from_micros(1_700_000_001_123_456));
+ let sidecar=root.join("Family/metadata/A.jpg.json");
+ let sidecar_time=fs::metadata(&sidecar)?.modified()?;
+ let record=read_json(&sidecar);
+ assert_eq!(record["ente"]["creationTime"],"2023-11-14T22:13:20.123456Z");
+ assert_eq!(record["photoTakenTime"]["timestamp"],"1700000000");
+ assert_eq!(record["ente"]["components"][0]["size"],original.len());
+ assert_eq!(record["ente"]["modificationTime"],"2023-11-14T22:13:21.123456Z");
+ assert_eq!(record["modificationTime"]["timestamp"],"1700000001");
+ for field in ["deviceFolder","localID","editedName","editedTime","updationTime","pubMagicMetadata"] {
+ assert!(record.get(field).is_none());
+ assert!(record["ente"].get(field).is_none());
+ }
+ assert_eq!(fs::read(root.join("Family/A.jpg"))?,original);
+ let unchanged=export(&home,&root,&["--album","Family"]);
+ assert_eq!(unchanged["changes"],json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0}));
+ assert_eq!(fs::metadata(&sidecar)?.modified()?,sidecar_time);
+ assert_eq!(fs::metadata(root.join("Family/A.jpg"))?.modified()?,original_time);
+ let before_locale=fs::read(&sidecar)?;
+ let localized=success(home.command(&["photos","export",root.to_str().unwrap(),"--album","Family","--json"]).env("LC_ALL","tr_TR.UTF-8").env("TZ","Pacific/Honolulu").output()?);
+ assert_eq!(serde_json::from_slice::<Value>(&localized.stdout)?["changes"],json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0}));
+ assert_eq!(fs::read(&sidecar)?,before_locale);
+ assert_eq!(fs::metadata(&sidecar)?.modified()?,sidecar_time);
+ fs::File::options().write(true).open(root.join("Family/A.jpg"))?.set_modified(UNIX_EPOCH+Duration::from_secs(99))?;
+ export(&home,&root,&["--album","Family"]);
+ assert_eq!(fs::metadata(root.join("Family/A.jpg"))?.modified()?,original_time);
+ #[cfg(unix)] {
+ use std::os::unix::fs::MetadataExt;
+ let before=fs::metadata(root.join("Family/A.jpg"))?;
+ export(&home,&root,&["--album","Family"]);
+ let after=fs::metadata(root.join("Family/A.jpg"))?;
+ assert_eq!((before.ctime(),before.ctime_nsec()),(after.ctime(),after.ctime_nsec()));
+ }
+ let result=export(&home,&root,&["--album","Family","--album","Travel"]);
+ assert_eq!(result["copies"]["expected"],2);
+ assert_eq!(objects.reads(),initial_reads+1);
+ assert_eq!(fs::read(root.join("Travel/A.jpg"))?,original);
+ #[cfg(unix)] {
+ use std::os::unix::fs::MetadataExt;
+ assert_ne!(fs::metadata(root.join("Travel/A.jpg"))?.ino(),fs::metadata(root.join("Family/A.jpg"))?.ino());
+ }
+ let travel_time=fs::metadata(root.join("Travel/A.jpg"))?.modified()?;
+ export(&home,&root,&["--album","Family"]);
+ assert_eq!(fs::metadata(root.join("Travel/A.jpg"))?.modified()?,travel_time);
+ let scoped=export(&home,&root,&["--album","Family","--album","Family","--album","Travel","--album","Work","--album",&empty.to_string(),"--exclude-album","Work"]);
+ assert_eq!(scoped["copies"]["expected"],2);
+ assert!(root.join("Trash-1/metadata.json").exists());
+ assert_eq!(read_json(&root.join("Trash-1/metadata.json"))["ente"]["albumID"],empty.to_string());
+ assert!(!root.join("Work").exists());
+ export(&home,&root,&[]);
+ assert_eq!(objects.reads(),initial_reads+2);
+ assert_eq!(fs::read(root.join("Work/Work.jpg"))?,b"work");
+
+ #[cfg(unix)] {
+ use std::os::unix::fs::PermissionsExt;
+ fs::write(root.join("Family/metadata/A.jpg.json"),b"{\"custom\":true}")?;
+ fs::write(root.join("Work/metadata/Work.jpg.json"),b"{\"custom\":true}")?;
+ let protected=root.join("Work/metadata");let permissions=fs::metadata(&protected)?.permissions();
+ fs::set_permissions(&protected,fs::Permissions::from_mode(0o500))?;
+ let output=home.run(&["photos","export",root.to_str().unwrap(),"--json"]);
+ fs::set_permissions(&protected,permissions)?;
+ failure(&output);
+ let result:Value=serde_json::from_slice(&output.stdout)?;
+ assert_eq!(result["copies"],json!({"expected":3,"completed":2,"pending":1}));
+ assert_eq!(result["failures"],1);
+ assert!(read_json(&root.join("Family/metadata/A.jpg.json")).get("custom").is_none());
+ export(&home,&root,&["--album","Work"]);
+ }
+ place(origin,&account,favorites,&favorites_key,file,&key,"add-files").await;
+ for (version,lat,long) in [(1,0,77),(2,12,0),(3,0,0)] {
+ edit_file(origin,&account,file,&key,version,json!({"editedName":"Renamed.jpg","editedTime":1_700_000_002_654_321i64,"caption":"","lat":lat,"long":long,"dateTime":"2023:11:15 03:43:22","offsetTime":"+05:30","w":4096,"h":3072,"cameraMake":"Example","cameraModel":"Camera","uploaderName":"Guest","mediaType":1,"mvi":123})).await;
+ export(&home,&root,&["--album","Family"]);
+ let record=read_json(&root.join("Family/metadata/Renamed.jpg.json"));
+ assert_eq!(record["description"],"");assert_eq!(record["favorited"],true);
+ assert_eq!(record["ente"]["creationTime"],"2023-11-14T22:13:22.654321Z");
+ assert_eq!(record["ente"]["name"],"Renamed.jpg");
+ assert_eq!(record["photoTakenTime"]["timestamp"],"1700000002");
+ assert!(!record.to_string().contains("2023-11-14T22:13:20.123456Z"));
+ assert!(!record.to_string().contains("not portable"));
+ for (field,value) in [("localDateTime",json!("2023:11:15 03:43:22")),("utcOffset",json!("+05:30")),("width",json!(4096)),("height",json!(3072)),("cameraMake",json!("Example")),("cameraModel",json!("Camera")),("uploaderName",json!("Guest")),("panorama",json!(true)),("motionPhotoVideoOffset",json!(123))] { assert_eq!(record["ente"][field],value); }
+ if lat==0 && long==0 {assert!(record.get("geoData").is_none());assert!(record["ente"].get("location").is_none());}
+ else {assert_eq!(record["geoData"],json!({"latitude":lat as f64,"longitude":long as f64}));assert_eq!(record["ente"]["location"],record["geoData"]);}
+ }
+ assert!(root.join("Travel/A.jpg").exists());
+ fs::write(root.join("Family/metadata/Renamed.jpg.json"),b"{\"custom\":true}")?;
+ export(&home,&root,&["--album","Family"]);
+ assert!(read_json(&root.join("Family/metadata/Renamed.jpg.json")).get("custom").is_none());
+ fs::write(root.join("Family/Renamed.jpg"),vec![0;original.len()])?;
+ export(&home,&root,&["--album","Family"]);
+ assert_eq!(fs::read(root.join("Family/Renamed.jpg"))?,original);
+ assert_eq!(objects.reads(),initial_reads+2);
+
+ move_file(origin,&account,family,(travel,&travel_key),(file,&key)).await;
+ let result=export(&home,&root,&["--album","Family"]);
+ assert_eq!(result["changes"]["retained"],1);
+ let retained=root.join("Trash/Family/Renamed.jpg");
+ assert_eq!(fs::read(&retained)?,original);
+ let retained_record=fs::read(root.join("Trash/Family/metadata/Renamed.jpg.json"))?;
+ let retained_time=fs::metadata(&retained)?.modified()?;
+ assert_eq!(export(&home,&root,&["--album","Family"])["changes"]["retained"],0);
+ place(origin,&account,family,&family_key,file,&key,"add-files").await;
+ export(&home,&root,&["--album","Family"]);
+ assert_eq!(objects.reads(),initial_reads+2);
+ assert_eq!(fs::read(root.join("Trash/Family/metadata/Renamed.jpg.json"))?,retained_record);
+ assert_eq!(fs::metadata(&retained)?.modified()?,retained_time);
+ fs::remove_dir_all(root.join("Trash"))?;
+ export(&home,&root,&["--album","Family"]);
+ assert!(!root.join("Trash").exists());
+ move_file(origin,&account,family,(travel,&travel_key),(file,&key)).await;
+ export(&home,&root,&["--album","Family"]);
+ assert_eq!(read_json(&root.join("Trash/Family/metadata.json"))["ente"]["albumID"],family.to_string());
+ assert_eq!(fs::read(root.join("Trash/Family/Renamed-1.jpg"))?,original);
+ let recovery=TestHome::new();login(&recovery,"photos",&email,&["--host",origin]);
+ export(&recovery,&root,&["--adopt","--album","Family"]);
+ place(origin,&account,family,&family_key,file,&key,"add-files").await;
+ export(&home,&root,&["--album","Family"]);
+ fs::write(root.join("Family/notes.txt"),b"local notes")?;
+ let encrypted=secretbox::encrypt(b"Family renamed",&family_key);
+ request(origin,&account,reqwest::Method::POST,"/collections/rename",json!({"collectionID":family,"encryptedName":b64::encode(&encrypted.encrypted_data),"nameDecryptionNonce":b64::encode(encrypted.nonce.as_bytes())})).await;
+ export(&home,&root,&["--album",&family.to_string()]);
+ assert_eq!(fs::read(root.join("Family renamed/notes.txt"))?,b"local notes");
+ move_file(origin,&account,family,(travel,&travel_key),(file,&key)).await;
+ request(origin,&account,reqwest::Method::DELETE,&format!("/collections/v3/{family}?collectionID={family}&keepFiles=true"),json!({})).await;
+ export(&home,&root,&["--album",&family.to_string()]);
+ assert_eq!(fs::read(root.join("Family renamed/notes.txt"))?,b"local notes");
+ assert!(root.join("Trash/Family/Renamed-2.jpg").exists());
+ fs::remove_dir_all(root.join("Trash"))?;
+ export(&home,&root,&["--album",&family.to_string()]);
+ assert!(!root.join("Trash").exists());
+
+ let relocated=destination.path().join("relocated");fs::rename(&root,&relocated)?;
+ let fresh=TestHome::new();login(&fresh,"photos",&email,&["--host",origin]);
+ assert!(failure(&fresh.run(&["photos","export",relocated.to_str().unwrap()])).contains("--adopt"));
+ let reads=objects.reads();
+ export(&fresh,&relocated,&["--adopt","--album","Travel"]);
+ assert_eq!(objects.reads(),reads);
+ assert_eq!(fs::read(relocated.join("export.json"))?,root_bytes);
+ fs::write(relocated.join("Work/Work.jpg"),b"xxxx")?;
+ export(&fresh,&relocated,&["--album","Work"]);
+ assert_eq!(objects.reads(),reads+1);
+ assert_eq!(fs::read(relocated.join("Work/Work.jpg"))?,b"work");
+ request(origin,&account,reqwest::Method::POST,"/files/trash",json!({"items":[{"fileID":work_file,"collectionID":work}]})).await;
+ let trashed=export(&fresh,&relocated,&["--album","Work"]);
+ assert_eq!(trashed["copies"]["expected"],0);
+ assert_eq!(trashed["changes"]["retained"],1);
+ assert!(!relocated.join("Work/Work.jpg").exists());
+ assert!(!relocated.join("Work/metadata/Work.jpg.json").exists());
+ assert_eq!(fs::read(relocated.join("Trash/Work/Work.jpg"))?,b"work");
+ assert_eq!(read_json(&relocated.join("Trash/Work/metadata/Work.jpg.json"))["ente"]["fileID"],work_file.to_string());
+ place(origin,&account,work,&work_key,work_file,&work_file_key,"restore-files").await;
+ let reads=objects.reads();
+ export(&fresh,&relocated,&["--album","Work"]);
+ assert_eq!(objects.reads(),reads);
+ Ok(())
+ })
+ })
+}
+
+#[test]
+fn export_bounds_failed_stages_and_keeps_servicing_existing_output() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let email = format!("export-staging-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ let (blocked, blocked_key) = create_album(origin, &account, "Blocked", "album").await;
+ let (healthy, healthy_key) = create_album(origin, &account, "Healthy", "album").await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("export");
+ export(&home, &root, &[]);
+ let limit = 256 * 1024 * 1024 / stream::DECRYPTION_CHUNK_SIZE;
+ let total = limit + 2;
+ for index in 0..total {
+ let name = format!("{index}.jpg");
+ upload_fixture(
+ origin,
+ &account,
+ blocked,
+ &blocked_key,
+ b"original",
+ metadata(&name, b"original"),
+ )
+ .await;
+ fs::write(root.join("Blocked").join(name), b"unrelated")?;
+ }
+ let (file, key) = upload_fixture(
+ origin,
+ &account,
+ healthy,
+ &healthy_key,
+ b"healthy",
+ metadata("Healthy.jpg", b"healthy"),
+ )
+ .await;
+ export(&home, &root, &["--album", "Healthy"]);
+ let reads = objects.reads();
+ edit_file(
+ origin,
+ &account,
+ file,
+ &key,
+ 1,
+ json!({"caption":"updated while staging is full"}),
+ )
+ .await;
+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
+ assert!(failure(&output).contains("filled export staging"));
+ let result: Value = serde_json::from_slice(&output.stdout)?;
+ assert_eq!(result["copies"]["completed"], 1);
+ assert_eq!(result["copies"]["pending"], total);
+ assert_eq!(objects.reads(), reads + limit);
+ assert_eq!(
+ read_json(&root.join("Healthy/metadata/Healthy.jpg.json"))["description"],
+ "updated while staging is full"
+ );
+ let db = export_database(&home);
+ let staged: i64 = db.query_row(
+ "SELECT count(DISTINCT json_extract(record,'$.file')) FROM temporaries",
+ [],
+ |r| r.get(0),
+ )?;
+ assert_eq!(staged, limit as i64);
+ for index in 0..total {
+ let path = root.join("Blocked").join(format!("{index}.jpg"));
+ assert_eq!(fs::read(&path)?, b"unrelated");
+ fs::remove_file(path)?;
+ }
+ assert_eq!(export(&home, &root, &[])["copies"]["completed"], total + 1);
+ assert_eq!(objects.reads(), reads + total);
+ assert_eq!(
+ db.query_row("SELECT count(*) FROM temporaries", [], |r| r
+ .get::<_, i64>(0))?,
+ 0
+ );
+ Ok(())
+ })
+ })
+}
+
+#[test]
+fn export_preserves_album_types_visibility_and_settings() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let email = format!("export-albums-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ let cases = [
+ ("Archive", "album", "album", 1, 0, "archived"),
+ ("Private", "album", "album", 2, 0, "hidden"),
+ ("Hidden", "album", "defaultHidden", 0, 1, "hidden"),
+ ("Quick link", "album", "quicklink", 0, 2, "visible"),
+ ("Folder", "folder", "folder", 0, 0, "visible"),
+ ("Uncategorized", "uncategorized", "uncategorized", 0, 0, "visible"),
+ ];
+ for (name, kind, _, visibility, subtype, _) in cases {
+ let (album, key) = create_album(origin, &account, name, kind).await;
+ let (file, file_key) = upload_fixture(origin, &account, album, &key, name.as_bytes(), metadata("original.jpg", name.as_bytes())).await;
+ set_album_metadata(origin, &account, album, &key, "magic-metadata", json!({"visibility":visibility,"subType":subtype,"order":7})).await;
+ set_album_metadata(origin, &account, album, &key, "public-magic-metadata", json!({"asc":true,"coverID":file,"layout":"trip","caption":"album description"})).await;
+ let private = blob::encrypt_json(&json!({"visibility":visibility}), &file_key)?;
+ request(origin, &account, reqwest::Method::PUT, "/files/magic-metadata", json!({"metadataList":[{"id":file,"magicMetadata":{"version":1,"count":1,"data":b64::encode(&private.encrypted_data),"header":b64::encode(private.decryption_header.as_bytes())}}]})).await;
+ }
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("photos");
+ let result = export(&home, &root, &[]);
+ assert_eq!(result["copies"]["completed"], cases.len());
+ for (name, _, kind, file_visibility, _, visibility) in cases {
+ let folder = root.join(name);
+ let album = read_json(&folder.join("metadata.json"));
+ let file = read_json(&folder.join("metadata/original.jpg.json"));
+ assert_eq!(fs::read(folder.join("original.jpg"))?, name.as_bytes());
+ assert_eq!(album["ente"]["type"], kind);
+ assert_eq!(album["ente"]["visibility"], visibility);
+ assert_eq!(album["ente"]["displayOrder"], 7);
+ assert_eq!(album["ente"]["sortOrder"], "ascending");
+ assert_eq!(album["ente"]["description"], "album description");
+ assert_eq!(album["ente"]["layout"], "trip");
+ assert_eq!(album["ente"]["coverFileID"], file["ente"]["fileID"]);
+ assert_eq!(file["ente"]["visibility"], ["visible", "archived", "hidden"][file_visibility]);
+ }
+ Ok(())
+ })
+ })
+}
+
+#[test]
+fn export_resumes_a_live_pair_after_publication_failure() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let email = format!("live-publication-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ let (album, key) = create_album(origin, &account, "Live", "album").await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("export");
+ export(&home, &root, &[]);
+ let db = export_database(&home);
+ db.execute_batch("CREATE TRIGGER fail_video_publication BEFORE UPDATE ON temporaries WHEN json_extract(new.record,'$.destination')='Live/Motion.mov' BEGIN SELECT RAISE(ABORT,'injected publication failure'); END;")?;
+ upload_live(origin, &account, album, &key, "Motion.heic", b"image", b"video").await;
+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
+ assert!(failure(&output).contains("injected publication failure"));
+ let result: Value = serde_json::from_slice(&output.stdout)?;
+ assert_eq!(result["copies"]["pending"], 1);
+ assert_eq!(fs::read(root.join("Live/Motion.heic"))?, b"image");
+ assert!(!root.join("Live/Motion.mov").exists());
+ assert!(!root.join("Live/metadata/Motion.heic.json").exists());
+ let pending: String = db.query_row("SELECT path FROM temporaries WHERE json_extract(record,'$.role')='video' AND json_extract(record,'$.hash') IS NOT NULL", [], |r| r.get(0))?;
+ assert_eq!(fs::read(root.join(&pending))?, b"video");
+ let reads = objects.reads();
+ db.execute_batch("DROP TRIGGER fail_video_publication;")?;
+ export(&home, &root, &[]);
+ assert_eq!(objects.reads(), reads);
+ assert!(!root.join(pending).exists());
+ assert_eq!(fs::read(root.join("Live/Motion.heic"))?, b"image");
+ assert_eq!(fs::read(root.join("Live/Motion.mov"))?, b"video");
+ for name in ["Motion.heic", "Motion.mov"] {
+ assert!(root.join("Live/metadata").join(format!("{name}.json")).exists());
+ }
+ let sidecar = root.join("Live/metadata/Motion.mov.json");
+ let mut record = read_json(&sidecar);
+ record["ente"]["components"].as_array_mut().unwrap().reverse();
+ fs::write(&sidecar, serde_json::to_vec(&record)?)?;
+ let fresh = TestHome::new();
+ fresh.write_vault(&home.read_vault());
+ export(&fresh, &root, &["--adopt"]);
+ assert_eq!(objects.reads(), reads);
+ assert_eq!(read_json(&sidecar)["ente"]["components"][0]["role"], "image");
+ Ok(())
+ })
+ })
+}
+
+#[test]
+fn export_live_photos_parallel_transfers_and_locking() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let email = format!("live-export-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ let other = TestHome::new();
+ login(&other, "photos", &email, &["--host", origin]);
+ let (album, key) = create_album(origin, &account, "Live", "album").await;
+ let (copy, copy_key) = create_album(origin, &account, "Copies", "album").await;
+ let image = vec![61u8; stream::ENCRYPTION_CHUNK_SIZE + 17];
+ let video = vec![92u8; stream::ENCRYPTION_CHUNK_SIZE * 2 + 31];
+ let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
+ for (name, bytes) in [("image.heic", &image), ("video.mov", &video)] {
+ zip.start_file(
+ name,
+ zip::write::SimpleFileOptions::default()
+ .compression_method(zip::CompressionMethod::Stored),
+ )?;
+ zip.write_all(bytes)?;
+ }
+ let archive = zip.finish()?.into_inner();
+ let mut data = metadata("Motion.heic", &archive);
+ data["fileType"] = json!(2);
+ data.as_object_mut().unwrap().remove("hash");
+ data["imageHash"] = json!(digest(&image));
+ data["videoHash"] = json!(digest(&video));
+ let (file, file_key) =
+ upload_fixture(origin, &account, album, &key, &archive, data).await;
+ place(
+ origin,
+ &account,
+ copy,
+ &copy_key,
+ file,
+ &file_key,
+ "add-files",
+ )
+ .await;
+ for index in 0..4 {
+ let bytes = vec![index as u8; stream::ENCRYPTION_CHUNK_SIZE + index + 100];
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ &bytes,
+ metadata(&format!("{index}.jpg"), &bytes),
+ )
+ .await;
+ }
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("export");
+ objects.delay_chunks(Duration::from_millis(15));
+ objects.interrupt_reads(1);
+ let child = home
+ .command(&["photos", "export", root.to_str().unwrap(), "--json"])
+ .spawn()?;
+ let deadline = std::time::Instant::now() + Duration::from_secs(20);
+ while !root.join("export.json").exists() && std::time::Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ let started = std::time::Instant::now();
+ assert!(
+ failure(&other.run(&["photos", "export", root.to_str().unwrap(), "--adopt"]))
+ .contains("another writer")
+ );
+ assert!(started.elapsed() < Duration::from_secs(3));
+ let result: Value = serde_json::from_slice(&success(child.wait_with_output()?).stdout)?;
+ assert_eq!(result["copies"]["completed"], 6);
+ assert!(objects.peak_reads() > 1);
+ assert_eq!(objects.reads(), 6);
+ for folder in ["Live", "Copies"] {
+ assert_eq!(fs::read(root.join(folder).join("Motion.heic"))?, image);
+ assert_eq!(fs::read(root.join(folder).join("Motion.mov"))?, video);
+ let record = read_json(&root.join(folder).join("metadata/Motion.heic.json"));
+ assert_eq!(record["ente"]["components"][0]["hash"], digest(&image));
+ assert_eq!(record["ente"]["components"][1]["hash"], digest(&video));
+ }
+ let reads = objects.reads();
+ export(&home, &root, &[]);
+ for entry in fs::read_dir(root.join("Live"))? {
+ let entry = entry?;
+ if entry.file_type()?.is_file() && entry.file_name() != "metadata.json" {
+ fs::File::options()
+ .write(true)
+ .open(entry.path())?
+ .set_modified(UNIX_EPOCH + Duration::from_secs(19))?;
+ }
+ }
+ export(&home, &root, &[]);
+ assert_eq!(objects.reads(), reads);
+ fs::remove_file(root.join("Live/Motion.mov"))?;
+ export(&home, &root, &[]);
+ assert_eq!(objects.reads(), reads);
+ assert_eq!(fs::read(root.join("Live/Motion.mov"))?, video);
+ let unrelated = root.join("Live/.ente-11111111111111111111111111111111.part");
+ fs::write(&unrelated, b"unassociated")?;
+ let unchanged_root = fs::read(root.join("export.json"))?;
+ for graceful in [true, false] {
+ #[cfg(not(unix))]
+ if graceful {
+ continue;
+ }
+ for index in 0..4 {
+ fs::remove_file(root.join(format!("Live/{index}.jpg")))?;
+ }
+ objects.delay_chunks(Duration::from_millis(30));
+ let before = objects.reads();
+ let mut child = home
+ .command(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "--album",
+ "Live",
+ "--json",
+ ])
+ .spawn()?;
+ let deadline = std::time::Instant::now() + Duration::from_secs(20);
+ while objects.reads() == before && std::time::Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ assert!(objects.reads() > before);
+ if graceful {
+ assert!(
+ Command::new("kill")
+ .args(["-INT", &child.id().to_string()])
+ .status()?
+ .success()
+ );
+ } else {
+ child.kill()?;
+ }
+ let interrupted = child.wait_with_output()?;
+ failure(&interrupted);
+ if graceful {
+ let result: Value = serde_json::from_slice(&interrupted.stdout)?;
+ assert_eq!(result["complete"], false);
+ }
+ objects.delay_chunks(Duration::ZERO);
+ export(&home, &root, &["--album", "Live"]);
+ assert_eq!(fs::read(&unrelated)?, b"unassociated");
+ assert_eq!(fs::read(root.join("export.json"))?, unchanged_root);
+ }
+ Ok(())
+ })
+ })
+}
+
+#[test]
+fn export_rejects_invalid_live_archives_and_component_hashes() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let email = format!("invalid-live-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ let (album, key) = create_album(origin, &account, "Album", "album").await;
+ let mut archives = vec![b"not a ZIP".to_vec()];
+ for names in [
+ vec!["image.jpg"],
+ vec!["image.jpg", "image.png", "video.mov"],
+ vec!["image.jpg", "video.mov", "other.jpg"],
+ vec!["image/../outside.jpg", "video.mov"],
+ vec!["image.jpg", "video.mov"],
+ ] {
+ let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
+ for name in names {
+ zip.start_file(name, zip::write::SimpleFileOptions::default())?;
+ zip.write_all(b"component")?;
+ }
+ archives.push(zip.finish()?.into_inner());
+ }
+ for (index, archive) in archives.iter().enumerate() {
+ let mut data = metadata(&format!("Invalid-{index}.jpg"), archive);
+ data["fileType"] = json!(2);
+ data["hash"] = json!(format!(
+ "{}:{}",
+ digest(b"component"),
+ digest(if index + 1 == archives.len() {
+ b"wrong video"
+ } else {
+ b"component"
+ })
+ ));
+ upload_fixture(origin, &account, album, &key, archive, data).await;
+ }
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ b"independent",
+ metadata("Good.jpg", b"independent"),
+ )
+ .await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("export");
+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
+ failure(&output);
+ let result: Value = serde_json::from_slice(&output.stdout)?;
+ assert_eq!(
+ result["copies"],
+ json!({"expected":7,"completed":1,"pending":6})
+ );
+ assert_eq!(result["failures"], 6);
+ assert_eq!(fs::read(root.join("Album/Good.jpg"))?, b"independent");
+ assert_eq!(fs::read_dir(root.join("Album/metadata"))?.count(), 1);
+ for index in 0..archives.len() {
+ assert!(!root.join(format!("Album/Invalid-{index}.jpg")).exists());
+ assert!(!root.join(format!("Album/Invalid-{index}.mov")).exists());
+ }
+ assert!(!root.join("outside.jpg").exists());
+ Ok(())
+ })
+ })
+}
+
+fn export(home: &TestHome, destination: &Path, options: &[&str]) -> Value {
+ let mut args = vec!["photos", "export", destination.to_str().unwrap()];
+ args.extend_from_slice(options);
+ home.json(&args)
+}
+fn read_json(path: &Path) -> Value {
+ serde_json::from_slice(&fs::read(path).unwrap()).unwrap()
+}
+
+fn export_database(home: &TestHome) -> rusqlite::Connection {
+ let state = home.read_vault();
+ let account = &state["accounts"][0];
+ let directory = home
+ .dir
+ .path()
+ .join("accounts")
+ .join(account["storage_id"].as_str().unwrap())
+ .join("exports");
+ let path = fs::read_dir(directory)
+ .unwrap()
+ .map(|entry| entry.unwrap().path())
+ .find(|path| path.extension().is_some_and(|ext| ext == "db"))
+ .unwrap();
+ let key: Vec<u8> = serde_json::from_value(account["db_key"].clone()).unwrap();
+ let hex: String = key.iter().map(|byte| format!("{byte:02x}")).collect();
+ let db = rusqlite::Connection::open(path).unwrap();
+ db.pragma_update(None, "key", format!("x'{hex}'")).unwrap();
+ db
+}
+fn digest(bytes: &[u8]) -> String {
+ b64::encode(&hash::hash(bytes, Some(64), None).unwrap())
+}
+fn metadata(name: &str, original: &[u8]) -> Value {
+ json!({"title":name,"fileType":0,"creationTime":1_700_000_000_123_456i64,"modificationTime":1_700_000_001_123_456i64,"latitude":10.0,"longitude":20.0,"hash":digest(original),"deviceFolder":"not portable"})
+}
+async fn request(
+ origin: &str,
+ account: &AuthenticatedAccount,
+ method: reqwest::Method,
+ path: &str,
+ body: Value,
+) {
+ let response = reqwest::Client::new()
+ .request(method, format!("{origin}{path}"))
+ .header("x-auth-token", b64::encode_url_safe(&account.secrets.token))
+ .header("x-client-package", "io.ente.photos")
+ .json(&body)
+ .send()
+ .await
+ .unwrap();
+ assert!(
+ response.status().is_success(),
+ "{path}: {} {}",
+ response.status(),
+ response.text().await.unwrap()
+ );
+}
+async fn place(
+ origin: &str,
+ account: &AuthenticatedAccount,
+ album: i64,
+ album_key: &Key,
+ file: i64,
+ file_key: &Key,
+ operation: &str,
+) {
+ let wrapped = secretbox::encrypt(file_key.as_bytes(), album_key);
+ request(origin,account,reqwest::Method::POST,&format!("/collections/{operation}"),json!({"collectionID":album,"files":[{"id":file,"encryptedKey":b64::encode(&wrapped.encrypted_data),"keyDecryptionNonce":b64::encode(wrapped.nonce.as_bytes())}]})).await;
+}
+async fn move_file(
+ origin: &str,
+ account: &AuthenticatedAccount,
+ from: i64,
+ (to, album_key): (i64, &Key),
+ (file, file_key): (i64, &Key),
+) {
+ let wrapped = secretbox::encrypt(file_key.as_bytes(), album_key);
+ request(
+ origin,
+ account,
+ reqwest::Method::POST,
+ "/collections/move-files",
+ json!({"fromCollectionID":from,"toCollectionID":to,"files":[{"id":file,"encryptedKey":b64::encode(&wrapped.encrypted_data),"keyDecryptionNonce":b64::encode(wrapped.nonce.as_bytes())}]}),
+ )
+ .await;
+}
+async fn edit_file(
+ origin: &str,
+ account: &AuthenticatedAccount,
+ file: i64,
+ key: &Key,
+ version: i64,
+ data: Value,
+) {
+ let encrypted = blob::encrypt_json(&data, key).unwrap();
+ request(origin,account,reqwest::Method::PUT,"/files/public-magic-metadata",json!({"metadataList":[{"id":file,"magicMetadata":{"version":version,"count":data.as_object().unwrap().len(),"data":b64::encode(&encrypted.encrypted_data),"header":b64::encode(encrypted.decryption_header.as_bytes())}}]})).await;
+}
+
+#[test]
+fn export_names_are_portable_and_stable() -> TestResult {
+ Museum::run_async(|origin| async move {
+ let email = format!("export-names-{}@example.org", Uuid::new_v4());
+ let account = create_account(&origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", &origin]);
+ let (album, key) = create_album(&origin, &account, "Names, commas", "album").await;
+ create_album(&origin, &account, "Album:", "album").await;
+ create_album(&origin, &account, "Album_", "album").await;
+ let long_album = format!("{}.tail", "a".repeat(254));
+ create_album(&origin, &account, &long_album, "album").await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("photos");
+ let cases = [
+ ("IMG.jpg", Some(0), "IMG.jpg"),
+ ("IMG.jpg", Some(0), "IMG-1.jpg"),
+ ("IMG-1.jpg", Some(0), "IMG-1-1.jpg"),
+ ("CON.jpg", Some(0), "CON-1.jpg"),
+ ("CON.extra.jpg", Some(0), "CON-1.extra.jpg"),
+ ("bad:name.jpg", Some(0), "bad_name.jpg"),
+ ("bad_name.jpg", Some(0), "bad_name-1.jpg"),
+ ("Straße.jpg", Some(0), "Straße.jpg"),
+ ("STRASSE.jpg", Some(0), "STRASSE-1.jpg"),
+ ("Γ©.jpg", Some(0), "Γ©.jpg"),
+ ("e\u{301}.jpg", Some(0), "e\u{301}-1.jpg"),
+ ("metadata.json", Some(0), "metadata-1.json"),
+ ("same.jpg", Some(0), "same.jpg"),
+ ("same.png", Some(0), "same.png"),
+ ("same.mov", Some(1), "same-1.mov"),
+ ("same.mp4", Some(1), "same-1.mp4"),
+ ("same.raw", None, "same-2.raw"),
+ ("same.other", None, "same-3.other"),
+ ];
+ for (name, kind, expected) in cases {
+ let mut data = metadata(name, name.as_bytes());
+ data["fileType"] = json!(kind);
+ let (id, _) =
+ upload_fixture(&origin, &account, album, &key, name.as_bytes(), data).await;
+ export(&home, &root, &["--album", "Names, commas"]);
+ let path = root.join("Names, commas").join(expected);
+ assert_eq!(fs::read(&path)?, name.as_bytes());
+ assert_eq!(
+ read_json(
+ &root
+ .join("Names, commas/metadata")
+ .join(format!("{expected}.json"))
+ )["ente"]["fileID"],
+ id.to_string()
+ );
+ }
+ let long = format!("{}.jpg", "あ".repeat(100));
+ upload_fixture(
+ &origin,
+ &account,
+ album,
+ &key,
+ b"long",
+ metadata(&long, b"long"),
+ )
+ .await;
+ export(&home, &root, &[]);
+ let shortened = format!("{}.jpg", "あ".repeat(82));
+ assert_eq!(
+ fs::read(root.join("Names, commas").join(&shortened))?,
+ b"long"
+ );
+ assert_eq!(
+ read_json(&root.join("a".repeat(254)).join("metadata.json"))["title"],
+ long_album
+ );
+ assert_eq!(
+ read_json(&root.join("Album_/metadata.json"))["title"],
+ "Album:"
+ );
+ assert_eq!(
+ read_json(&root.join("Album_-1/metadata.json"))["title"],
+ "Album_"
+ );
+ assert_eq!(
+ export(&home, &root, &[])["changes"],
+ json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})
+ );
+ let fresh = TestHome::new();
+ fresh.write_vault(&home.read_vault());
+ export(&fresh, &root, &["--adopt"]);
+ for (name, _, expected) in cases {
+ assert_eq!(
+ fs::read(root.join("Names, commas").join(expected))?,
+ name.as_bytes()
+ );
+ }
+ let mut data = metadata("same.avi", b"new video");
+ data["fileType"] = json!(1);
+ upload_fixture(&origin, &account, album, &key, b"new video", data).await;
+ export(&fresh, &root, &[]);
+ assert_eq!(
+ fs::read(root.join("Names, commas/same-1.avi"))?,
+ b"new video"
+ );
+ assert_eq!(
+ export(&fresh, &root, &[])["changes"],
+ json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})
+ );
+ Ok(())
+ })
+}
+
+#[test]
+fn export_allocates_live_pairs_from_their_actual_extensions() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let email = format!("live-names-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ let (album, key) = create_album(origin, &account, "Live names", "album").await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("photos");
+ let folder = root.join("Live names");
+ let long_video = format!("{}.mp4", "a".repeat(245));
+ let long_image = format!("{}-1.png", "a".repeat(243));
+ for (name, kind) in [
+ ("pair.jpg", 0),
+ ("same-image.png", 0),
+ (&long_video, 1),
+ (&long_image, 0),
+ ] {
+ let mut data = metadata(name, name.as_bytes());
+ data["fileType"] = json!(kind);
+ upload_fixture(origin, &account, album, &key, name.as_bytes(), data).await;
+ export(&home, &root, &[]);
+ assert_eq!(fs::read(folder.join(name))?, name.as_bytes());
+ }
+ let long_source = format!("{}.jpg", "a".repeat(246));
+ let long_outputs = [
+ format!("{}-2.jpeg", "a".repeat(243)),
+ format!("{}-2.mov", "a".repeat(243)),
+ ];
+ let cases = [
+ (
+ "pair.heic",
+ [".heic", ".mov"],
+ ["pair-1.heic", "pair-1.mov"],
+ ),
+ (
+ "same.heic",
+ [".jpg", ".JPG"],
+ ["same-1-image.jpg", "same-1-video.JPG"],
+ ),
+ ("absent.heic", ["", ""], ["absent-image", "absent-video"]),
+ ("half.heic", ["", ".mov"], ["half", "half.mov"]),
+ ("dotted..heic", ["", ".mov"], ["dotted", "dotted.mov"]),
+ (
+ "metadata.heic",
+ ["", ".mov"],
+ ["metadata-1", "metadata-1.mov"],
+ ),
+ (
+ long_source.as_str(),
+ [".jpeg", ".mov"],
+ [long_outputs[0].as_str(), long_outputs[1].as_str()],
+ ),
+ (
+ "converted.heic",
+ [".j?g ", ".m:v"],
+ ["converted.j_g", "converted.m_v"],
+ ),
+ ];
+ let mut snapshots = Vec::new();
+ for (index, (name, extensions, paths)) in cases.into_iter().enumerate() {
+ let image = format!("image {index}");
+ let video = format!("video {index}");
+ let id = upload_live_parts(
+ origin,
+ &account,
+ album,
+ &key,
+ name,
+ [
+ (extensions[0], image.as_bytes()),
+ (extensions[1], video.as_bytes()),
+ ],
+ )
+ .await;
+ export(&home, &root, &[]);
+ for (path, bytes) in paths.into_iter().zip([image.as_bytes(), video.as_bytes()]) {
+ let media = folder.join(path);
+ assert_eq!(fs::read(&media)?, bytes);
+ let sidecar = folder.join("metadata").join(format!("{path}.json"));
+ let record = read_json(&sidecar);
+ assert_eq!(record["ente"]["fileID"], id.to_string());
+ assert_eq!(record["ente"]["name"], name);
+ assert_eq!(record["ente"]["components"][0]["path"], paths[0]);
+ assert_eq!(record["ente"]["components"][1]["path"], paths[1]);
+ snapshots.push((media, bytes.to_vec()));
+ snapshots.push((sidecar.clone(), fs::read(sidecar)?));
+ }
+ }
+ let reads = objects.reads();
+ assert_eq!(
+ export(&home, &root, &[])["changes"],
+ json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})
+ );
+ let fresh = TestHome::new();
+ fresh.write_vault(&home.read_vault());
+ export(&fresh, &root, &["--adopt"]);
+ assert_eq!(objects.reads(), reads);
+ for (path, bytes) in snapshots {
+ assert_eq!(fs::read(path)?, bytes);
+ }
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ b"later image",
+ metadata("half.jpg", b"later image"),
+ )
+ .await;
+ export(&fresh, &root, &[]);
+ assert_eq!(fs::read(folder.join("half-1.jpg"))?, b"later image");
+ Ok(())
+ })
+ })
+}
+
+#[test]
+fn export_refills_transfers_while_a_large_original_is_pending() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let email = format!("skewed-export-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ let small_count = std::thread::available_parallelism()
+ .map(usize::from)
+ .unwrap_or(1)
+ .min(63)
+ + 2;
+ for live in [false, true] {
+ let (album, key) = create_album(
+ origin,
+ &account,
+ if live { "Live" } else { "Ordinary" },
+ "album",
+ )
+ .await;
+ let large = vec![31u8; 32 * 1024 * 1024];
+ if live {
+ upload_live(
+ origin,
+ &account,
+ album,
+ &key,
+ "IMG:1.heic",
+ &large,
+ b"video",
+ )
+ .await;
+ let mut contender = metadata("IMG?1.mov", b"ordinary contender");
+ contender["fileType"] = json!(1);
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ b"ordinary contender",
+ contender,
+ )
+ .await;
+ upload_live(
+ origin,
+ &account,
+ album,
+ &key,
+ "IMG?1-1.heic",
+ b"small image",
+ b"small video",
+ )
+ .await;
+ } else {
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ &large,
+ metadata("Slow.jpg", &large),
+ )
+ .await;
+ }
+ for index in 0..small_count {
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ b"independent",
+ metadata(&format!("Small-{index}.jpg"), b"independent"),
+ )
+ .await;
+ }
+ objects.delay_chunks(Duration::from_millis(30));
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("export");
+ let folder = root.join(if live { "Live" } else { "Ordinary" });
+ let child = home
+ .command(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "--album",
+ &album.to_string(),
+ "--json",
+ ])
+ .spawn()?;
+ let last = folder.join(format!("Small-{}.jpg", small_count - 1));
+ let deadline = std::time::Instant::now() + Duration::from_secs(40);
+ while !last.exists() && std::time::Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ assert!(last.exists(), "independent transfers did not make progress");
+ assert!(
+ !folder
+ .join(if live { "IMG_1-2.heic" } else { "Slow.jpg" })
+ .exists(),
+ "a slow first transfer blocked later placements"
+ );
+ success(child.wait_with_output()?);
+ objects.delay_chunks(Duration::ZERO);
+ if live {
+ assert_eq!(fs::read(folder.join("IMG_1-2.heic"))?, large);
+ assert_eq!(fs::read(folder.join("IMG_1-2.mov"))?, b"video");
+ assert_eq!(fs::read(folder.join("IMG_1-1.heic"))?, b"small image");
+ assert_eq!(fs::read(folder.join("IMG_1-1.mov"))?, b"small video");
+ assert_eq!(fs::read(folder.join("IMG_1.mov"))?, b"ordinary contender");
+ assert!(!folder.join("IMG_1.heic").exists());
+ } else {
+ assert_eq!(fs::read(folder.join("Slow.jpg"))?, large);
+ }
+ assert_eq!(
+ export(&home, &root, &["--album", &album.to_string()])["changes"],
+ json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})
+ );
+ }
+ Ok(())
+ })
+ })
+}
+
+async fn upload_live(
+ origin: &str,
+ account: &AuthenticatedAccount,
+ album: i64,
+ key: &Key,
+ name: &str,
+ image: &[u8],
+ video: &[u8],
+) {
+ upload_live_parts(
+ origin,
+ account,
+ album,
+ key,
+ name,
+ [(".heic", image), (".mov", video)],
+ )
+ .await;
+}
+
+async fn upload_live_parts(
+ origin: &str,
+ account: &AuthenticatedAccount,
+ album: i64,
+ key: &Key,
+ name: &str,
+ parts: [(&str, &[u8]); 2],
+) -> i64 {
+ let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
+ for (role, (extension, bytes)) in ["image", "video"].into_iter().zip(parts) {
+ zip.start_file(
+ format!("{role}{extension}"),
+ zip::write::SimpleFileOptions::default()
+ .compression_method(zip::CompressionMethod::Stored),
+ )
+ .unwrap();
+ zip.write_all(bytes).unwrap();
+ }
+ let archive = zip.finish().unwrap().into_inner();
+ let mut data = metadata(name, &archive);
+ data["fileType"] = json!(2);
+ data["hash"] = json!(format!("{}:{}", digest(parts[0].1), digest(parts[1].1)));
+ upload_fixture(origin, account, album, key, &archive, data)
+ .await
+ .0
+}
diff --git a/rust/apps/cli-next/tests/support/export_resources.rs b/rust/apps/cli-next/tests/support/export_resources.rs
new file mode 100644
index 0000000000..605968a20e
--- /dev/null
+++ b/rust/apps/cli-next/tests/support/export_resources.rs
@@ -0,0 +1,278 @@
+use super::*;
+use std::{
+ io::{Read, Seek},
+ time::Instant,
+};
+
+#[test]
+#[ignore = "requires ENTE_EXPORT_TEST_VOLUME pointing to a disposable volume smaller than 512 MiB"]
+fn export_recovers_after_real_disk_full() -> TestResult {
+ let volume = std::env::var("ENTE_EXPORT_TEST_VOLUME")?;
+ let destination = tempfile::tempdir_in(volume)?;
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let email = format!("export-full-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ let (album, key) = create_album(origin, &account, "Album", "album").await;
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ b"completed",
+ metadata("Before.jpg", b"completed"),
+ )
+ .await;
+ let root = destination.path().join("export");
+ export(&home, &root, &[]);
+ let prior_time = fs::metadata(root.join("Album/Before.jpg"))?.modified()?;
+ let root_bytes = fs::read(root.join("export.json"))?;
+ let size = 32 * 1024 * 1024;
+ let mut data = metadata("After.bin", b"");
+ data["hash"] = json!(repeat_hash(31, size));
+ upload_reader(
+ origin,
+ &account,
+ album,
+ &key,
+ std::io::repeat(31).take(size),
+ data,
+ )
+ .await;
+ let filler_path = destination.path().join("filler");
+ let mut filler = fs::File::create(&filler_path)?;
+ let block = vec![61; 64 * 1024];
+ let mut full = false;
+ for _ in 0..8192 {
+ match filler.write_all(&block) {
+ Ok(()) => {}
+ Err(error) if error.kind() == std::io::ErrorKind::StorageFull => {
+ full = true;
+ break;
+ }
+ Err(error) => return Err(error.into()),
+ }
+ }
+ assert!(full, "fixture volume did not fill within 512 MiB");
+ filler.set_len(filler.metadata()?.len().saturating_sub(256 * 1024))?;
+ filler.sync_all()?;
+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
+ failure(&output);
+ let result: Value = serde_json::from_slice(&output.stdout)?;
+ assert_eq!(result["complete"], false);
+ assert_eq!(result["copies"]["pending"], 1);
+ assert_eq!(fs::read(root.join("Album/Before.jpg"))?, b"completed");
+ assert_eq!(
+ fs::metadata(root.join("Album/Before.jpg"))?.modified()?,
+ prior_time
+ );
+ assert_eq!(fs::read(root.join("export.json"))?, root_bytes);
+ drop(filler);
+ fs::remove_file(filler_path)?;
+ export(&home, &root, &[]);
+ assert_eq!(fs::metadata(root.join("Album/After.bin"))?.len(), size);
+ assert_eq!(
+ b64::encode(&hash::hash_reader(
+ &mut fs::File::open(root.join("Album/After.bin"))?,
+ Some(64)
+ )?),
+ repeat_hash(31, size)
+ );
+ assert_eq!(
+ fs::metadata(root.join("Album/Before.jpg"))?.modified()?,
+ prior_time
+ );
+ Ok(())
+ })
+ })
+}
+
+#[test]
+#[ignore = "resource check: run in release mode with at least 20 GiB free"]
+fn export_large_original_and_live_photo_with_bounded_memory() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let email = format!("export-large-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ let (album, key) = create_album(origin, &account, "Large", "album").await;
+ let size = 2 * 1024 * 1024 * 1024 + 53;
+ let original_hash = repeat_hash(29, size);
+ let mut data = metadata("Original.bin", b"");
+ data["hash"] = json!(original_hash);
+ upload_reader(
+ origin,
+ &account,
+ album,
+ &key,
+ std::io::repeat(29).take(size),
+ data,
+ )
+ .await;
+ let image_size = 16 * 1024 * 1024;
+ let image_hash = repeat_hash(17, image_size);
+ let video_hash = repeat_hash(73, size);
+ let mut archive = zip::ZipWriter::new(tempfile::tempfile()?);
+ for (name, byte, length) in [("image.heic", 17, image_size), ("video.mov", 73, size)] {
+ archive.start_file(
+ name,
+ zip::write::SimpleFileOptions::default()
+ .large_file(true)
+ .compression_method(zip::CompressionMethod::Stored),
+ )?;
+ std::io::copy(&mut std::io::repeat(byte).take(length), &mut archive)?;
+ }
+ let mut archive = archive.finish()?;
+ archive.rewind()?;
+ let mut data = metadata("Motion.heic", b"");
+ data["fileType"] = json!(2);
+ data["hash"] = json!(format!("{image_hash}:{video_hash}"));
+ upload_reader(origin, &account, album, &key, archive, data).await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("export");
+ let result = measure(&home, &root, "large media")?;
+ assert_eq!(result["copies"]["completed"], 2);
+ for (name, length, expected) in [
+ ("Original.bin", size, original_hash),
+ ("Motion.heic", image_size, image_hash),
+ ("Motion.mov", size, video_hash),
+ ] {
+ let path = root.join("Large").join(name);
+ assert_eq!(fs::metadata(&path)?.len(), length);
+ let actual = b64::encode(&hash::hash_reader(&mut fs::File::open(path)?, Some(64))?);
+ assert_eq!(actual, expected);
+ }
+ let reads = museum.object_store().reads();
+ assert_eq!(reads, 2);
+ assert_eq!(
+ measure(&home, &root, "large media unchanged")?["changes"],
+ json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})
+ );
+ assert_eq!(museum.object_store().reads(), reads);
+ Ok(())
+ })
+ })
+}
+
+#[test]
+#[ignore = "resource check: builds a 2,000-record Museum library"]
+fn export_many_records_unchanged_resource_check() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let email = format!("export-many-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ let (album, key) = create_album(origin, &account, "Many", "album").await;
+ for index in 0..2000 {
+ let name = format!("{index}.jpg");
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ b"original",
+ metadata(&name, b"original"),
+ )
+ .await;
+ // Each file needs two URLs; Museum allows 500 per minute.
+ tokio::time::sleep(Duration::from_millis(300)).await;
+ if index % 250 == 249 {
+ eprintln!("Prepared {} of 2,000 resource-fixture files", index + 1);
+ }
+ }
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("export");
+ assert_eq!(
+ measure(&home, &root, "2,000 records initial")?["copies"]["completed"],
+ 2000
+ );
+ let mut observed = Vec::new();
+ for directory in [root.join("Many"), root.join("Many/metadata")] {
+ for entry in fs::read_dir(directory)? {
+ let entry = entry?;
+ if entry.file_type()?.is_file() {
+ observed.push((entry.path(), entry.metadata()?.modified()?));
+ }
+ }
+ }
+ let reads = museum.object_store().reads();
+ let result = measure(&home, &root, "2,000 records unchanged")?;
+ assert_eq!(result["copies"]["completed"], 2000);
+ assert_eq!(
+ result["changes"],
+ json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})
+ );
+ assert_eq!(museum.object_store().reads(), reads);
+ for (path, modified) in observed {
+ assert_eq!(fs::metadata(path)?.modified()?, modified);
+ }
+ Ok(())
+ })
+ })
+}
+
+fn repeat_hash(byte: u8, size: u64) -> String {
+ b64::encode(&hash::hash_reader(&mut std::io::repeat(byte).take(size), Some(64)).unwrap())
+}
+
+fn measure(home: &TestHome, root: &Path, label: &str) -> TestResult<Value> {
+ let started = Instant::now();
+ let mut child = home
+ .command(&["photos", "export", root.to_str().unwrap(), "--json"])
+ .spawn()?;
+ let output_readers = [
+ child
+ .stdout
+ .take()
+ .map(|pipe| Box::new(pipe) as Box<dyn Read + Send>),
+ child
+ .stderr
+ .take()
+ .map(|pipe| Box::new(pipe) as Box<dyn Read + Send>),
+ ]
+ .map(|pipe| {
+ std::thread::spawn(move || {
+ let mut bytes = Vec::new();
+ pipe.unwrap().read_to_end(&mut bytes).map(|_| bytes)
+ })
+ });
+ let mut peak_kib = 0;
+ while child.try_wait()?.is_none() {
+ let sampled = Command::new("ps")
+ .args(["-o", "rss=", "-p", &child.id().to_string()])
+ .output()?;
+ if sampled.status.success() {
+ peak_kib = peak_kib.max(String::from_utf8(sampled.stdout)?.trim().parse::<u64>()?);
+ }
+ if started.elapsed() > Duration::from_secs(1800) {
+ child.kill()?;
+ child.wait()?;
+ return Err("export resource check timed out".into());
+ }
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ let status = child.wait()?;
+ let [stdout, stderr] = output_readers.map(|reader| reader.join().unwrap().unwrap());
+ let result = success(std::process::Output {
+ status,
+ stdout,
+ stderr,
+ });
+ eprintln!(
+ "{label}: {:.3}s, peak sampled RSS {:.1} MiB",
+ started.elapsed().as_secs_f64(),
+ peak_kib as f64 / 1024.0
+ );
+ assert!(
+ peak_kib < 512 * 1024,
+ "export RSS exceeded 512 MiB: {peak_kib} KiB"
+ );
+ Ok(serde_json::from_slice(&result.stdout)?)
+}
diff --git a/rust/apps/cli-next/tests/support/export_sources.rs b/rust/apps/cli-next/tests/support/export_sources.rs
new file mode 100644
index 0000000000..d39a5246a7
--- /dev/null
+++ b/rust/apps/cli-next/tests/support/export_sources.rs
@@ -0,0 +1,1077 @@
+use super::files::{collection, page, remote_file};
+use super::*;
+use ente_core::crypto::{Nonce, hash, secretbox};
+use std::path::Path;
+
+#[test]
+fn export_rejects_wrong_accounts_and_unsupported_roots_before_refresh() {
+ let mut server = mockito::Server::new();
+ let albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[]}).to_string())
+ .expect(1)
+ .create();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let root_path = root.join("export.json");
+ let original = fs::read(&root_path).unwrap();
+ let wrong = TestHome::new();
+ wrong.seed(&server.url());
+ let mut state = wrong.read_vault();
+ state["accounts"][0]["identity"]["master_key"] = json!(vec![1u8; 32]);
+ wrong.write_vault(&state);
+ assert!(
+ failure(&wrong.run(&["photos", "export", root.to_str().unwrap(), "--adopt"]))
+ .contains("another source account")
+ );
+ assert_eq!(fs::read(&root_path).unwrap(), original);
+ for (field, value) in [("version", json!(2)), ("format", json!("other-export"))] {
+ let mut record: Value = serde_json::from_slice(&original).unwrap();
+ record[field] = value;
+ let unsupported = serde_json::to_vec(&record).unwrap();
+ fs::write(&root_path, &unsupported).unwrap();
+ assert!(
+ failure(&home.run(&["photos", "export", root.to_str().unwrap()]))
+ .contains("unsupported export format")
+ );
+ assert_eq!(fs::read(&root_path).unwrap(), unsupported);
+ }
+ fs::write(&root_path, b"{malformed").unwrap();
+ assert!(
+ failure(&home.run(&["photos", "export", root.to_str().unwrap()]))
+ .contains("invalid export.json")
+ );
+ assert_eq!(fs::read_dir(&root).unwrap().count(), 1);
+ albums.assert();
+}
+
+#[test]
+fn export_preserves_failed_records_and_advances_the_page_atomically() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let key = Key::generate();
+ let albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(
+ json!({"collections":[collection(1,"First",&key),collection(2,"Second",&key)]})
+ .to_string(),
+ )
+ .expect_at_least(1)
+ .create();
+ let (good, encrypted) = source(10, &key, b"good", "Good.jpg");
+ let (mut broken, broken_bytes) = source(11, &key, b"broken", "Broken.jpg");
+ let mut repaired = broken.clone();
+ broken["metadata"]["encryptedData"] = json!("invalid ciphertext");
+ let first = page(&mut server, 1, 0, json!([good, broken]), true)
+ .expect(1)
+ .create();
+ let (later, later_bytes) = source(12, &key, b"later", "Later.jpg");
+ let mut later = later;
+ later["updationTime"] = json!(15);
+ let last = page(&mut server, 1, 10, json!([later]), false)
+ .expect(1)
+ .create();
+ page(&mut server, 2, 0, json!([broken]), false)
+ .expect(1)
+ .create();
+ let good_download = download(&mut server, 10, &encrypted, 1);
+ let later_download = download(&mut server, 12, &later_bytes, 1);
+ let initial = run(&home, &root, &[], false);
+ assert_eq!(
+ initial["copies"],
+ json!({"expected":4,"completed":2,"pending":2})
+ );
+ assert_eq!(initial["failures"], 2);
+ assert_eq!(fs::read(root.join("First/Good.jpg")).unwrap(), b"good");
+ assert_eq!(fs::read(root.join("First/Later.jpg")).unwrap(), b"later");
+ let db = database(&home);
+ let failed = record(&db, 1, 11);
+ assert!(failed["documents"].is_null());
+ assert_eq!(
+ db.query_row(
+ "SELECT files_cursor FROM photos_collections WHERE id=1",
+ [],
+ |r| r.get::<_, i64>(0)
+ )
+ .unwrap(),
+ 15
+ );
+ run(&home, &root, &[], false);
+ assert_eq!(record(&db, 1, 11)["attempt"], failed["attempt"]);
+ run(&home, &root, &["--album", "First", "--retry-failed"], false);
+ assert_ne!(record(&db, 1, 11)["attempt"], failed["attempt"]);
+ assert_eq!(record(&db, 2, 11)["attempt"], failed["attempt"]);
+ first.assert();
+ last.assert();
+
+ let before = record(&db, 1, 10);
+ let mut docs: Value = serde_json::from_slice(
+ &serde_json::from_value::<Vec<u8>>(before["documents"]["public"].clone()).unwrap(),
+ )
+ .unwrap();
+ docs["caption"] = json!("newly supported from saved metadata");
+ let mut updated = before;
+ updated["documents"]["public"] = json!(serde_json::to_vec(&docs).unwrap());
+ db.execute(
+ "UPDATE photos_files SET record=?1 WHERE collection_id=1 AND id=10",
+ [updated.to_string()],
+ )
+ .unwrap();
+ run(&home, &root, &["--album", "First"], false);
+ assert_eq!(
+ read_json(&root.join("First/metadata/Good.jpg.json"))["description"],
+ "newly supported from saved metadata"
+ );
+ let previous_attempt = record(&db, 1, 11)["attempt"].clone();
+ albums.remove();
+ let mut changed = collection(1, "First", &key);
+ changed["updationTime"] = json!(40);
+ let changed_albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[changed]}).to_string())
+ .create();
+ repaired["updationTime"] = json!(30);
+ page(&mut server, 1, 15, json!([repaired]), false).create();
+ let repaired_download = download(&mut server, 11, &broken_bytes, 1);
+ run(&home, &root, &["--album", "First"], true);
+ assert_ne!(record(&db, 1, 11)["attempt"], previous_attempt);
+ assert_eq!(fs::read(root.join("First/Broken.jpg")).unwrap(), b"broken");
+ good_download.assert();
+ later_download.assert();
+ repaired_download.assert();
+ changed_albums.remove();
+ let mut second = collection(2, "Second", &key);
+ second["updationTime"] = json!(50);
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[second]}).to_string())
+ .create();
+ let mut removed = broken;
+ removed["updationTime"] = json!(50);
+ removed["isDeleted"] = json!(true);
+ page(&mut server, 2, 10, json!([removed]), false).create();
+ let result = run(&home, &root, &["--album", "Second"], true);
+ assert_eq!(result["copies"]["expected"], 0);
+ assert_eq!(result["failures"], 0);
+ assert_eq!(
+ db.query_row(
+ "SELECT count(*) FROM photos_files WHERE collection_id=2 AND id=11",
+ [],
+ |r| r.get::<_, i64>(0)
+ )
+ .unwrap(),
+ 0
+ );
+ assert!(!root.join("Second/Broken.jpg").exists());
+ assert_eq!(
+ run(&home, &root, &["--album", "Second", "--retry-failed"], true)["failures"],
+ 0
+ );
+}
+
+#[test]
+fn export_retries_saved_file_decryption_when_the_album_key_changes() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let wrong_key = Key::generate();
+ let albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[collection(1,"First",&wrong_key)]}).to_string())
+ .create();
+ let (file, bytes) = source(10, &key, b"original", "Original.jpg");
+ let received = page(&mut server, 1, 0, json!([file]), false)
+ .expect(1)
+ .create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], false);
+ let db = database(&home);
+ let failed = record(&db, 1, 10);
+ assert!(failed["documents"].is_null());
+ albums.remove();
+ let mut changed = collection(1, "First", &key);
+ changed["updationTime"] = json!(20);
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[changed]}).to_string())
+ .create();
+ page(&mut server, 1, 10, json!([]), false)
+ .expect(1)
+ .create();
+ let downloaded = download(&mut server, 10, &bytes, 1);
+ run(&home, &root, &[], true);
+ assert_ne!(record(&db, 1, 10)["attempt"], failed["attempt"]);
+ assert_eq!(
+ fs::read(root.join("First/Original.jpg")).unwrap(),
+ b"original"
+ );
+ received.assert();
+ downloaded.assert();
+}
+
+#[test]
+fn export_keeps_unresolved_output_and_rejects_ambiguous_adoption() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let key = Key::generate();
+ let albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[collection(1,"First",&key)]}).to_string())
+ .create();
+ let (good, encrypted) = source(10, &key, b"good", "Good.jpg");
+ page(&mut server, 1, 0, json!([good]), false).create();
+ download(&mut server, 10, &encrypted, 1);
+ run(&home, &root, &[], true);
+ let before = fs::read(root.join("First/metadata/Good.jpg.json")).unwrap();
+ let mut broken = good;
+ broken["updationTime"] = json!(30);
+ broken["pubMagicMetadata"]["data"] = json!("broken");
+ albums.remove();
+ let mut album = collection(1, "First", &key);
+ album["updationTime"] = json!(40);
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[album]}).to_string())
+ .create();
+ page(&mut server, 1, 10, json!([broken]), false).create();
+ run(&home, &root, &[], false);
+ assert_eq!(
+ fs::read(root.join("First/metadata/Good.jpg.json")).unwrap(),
+ before
+ );
+ assert_eq!(fs::read(root.join("First/Good.jpg")).unwrap(), b"good");
+ let unassociated = root.join("First/unassociated.jpg");
+ fs::write(&unassociated, b"keep me").unwrap();
+ fs::create_dir(root.join("Other")).unwrap();
+ fs::copy(
+ root.join("First/metadata.json"),
+ root.join("Other/metadata.json"),
+ )
+ .unwrap();
+ let fresh = TestHome::new();
+ fresh.seed(&server.url());
+ let output = fresh.run(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "--adopt",
+ "--json",
+ ]);
+ assert!(failure(&output).contains("active album ID"));
+ assert!(output.stdout.is_empty());
+ assert_eq!(fs::read(unassociated).unwrap(), b"keep me");
+ assert_eq!(
+ fs::read(root.join("First/metadata/Good.jpg.json")).unwrap(),
+ before
+ );
+}
+
+#[test]
+fn export_releases_a_deleted_album_reservation_before_its_first_association() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let empty = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[]}).to_string())
+ .create();
+ run(&home, &root, &[], true);
+ empty.remove();
+ fs::create_dir(root.join("Family")).unwrap();
+ let unrelated = root.join("Family/notes.txt");
+ fs::write(&unrelated, b"unrelated").unwrap();
+ let key = Key::generate();
+ let album = collection(1, "Family", &key);
+ let albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[album]}).to_string())
+ .create();
+ page(&mut server, 1, 0, json!([]), false).create();
+ assert_eq!(run(&home, &root, &["--album", "1"], false)["conflicts"], 1);
+ let db = export_database(&home);
+ let reservation = || {
+ db.query_row(
+ "SELECT record FROM allocations WHERE owner='active:1'",
+ [],
+ |r| r.get::<_, String>(0),
+ )
+ .unwrap()
+ };
+ let pending = reservation();
+ assert_eq!(
+ serde_json::from_str::<Value>(&pending).unwrap()["paths"],
+ json!(["Family"])
+ );
+ assert_eq!(
+ db.query_row("SELECT count(*) FROM albums", [], |r| r.get::<_, i64>(0))
+ .unwrap(),
+ 0
+ );
+ assert_eq!(
+ run(&home, &root, &["--album", "Family"], false)["conflicts"],
+ 1
+ );
+ run(&home, &root, &["--exclude-album", "Family"], true);
+ assert_eq!(reservation(), pending);
+ assert_eq!(fs::read(&unrelated).unwrap(), b"unrelated");
+ assert!(!root.join("Family/metadata.json").exists());
+ albums.remove();
+ let malformed = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body("{")
+ .create();
+ failure(&home.run(&["photos", "export", root.to_str().unwrap(), "--album", "1"]));
+ assert_eq!(reservation(), pending);
+ malformed.remove();
+ let mut deleted = album;
+ deleted["updationTime"] = json!(30);
+ deleted["isDeleted"] = json!(true);
+ let deletion = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[deleted]}).to_string())
+ .create();
+ for selector in ["1", "Family"] {
+ assert_eq!(
+ run(&home, &root, &["--exclude-album", selector], true)["copies"]["expected"],
+ 0
+ );
+ assert_eq!(reservation(), pending);
+ }
+ run(&home, &root, &["--album", "1"], true);
+ assert_eq!(
+ db.query_row("SELECT count(*) FROM allocations", [], |r| r
+ .get::<_, i64>(0))
+ .unwrap(),
+ 0
+ );
+ assert_eq!(
+ db.query_row("SELECT count(*) FROM albums", [], |r| r.get::<_, i64>(0))
+ .unwrap(),
+ 0
+ );
+ assert_eq!(fs::read(&unrelated).unwrap(), b"unrelated");
+ assert!(!root.join("Family/metadata.json").exists());
+ assert!(!root.join("Trash").exists());
+ deletion.remove();
+ fs::remove_file(unrelated).unwrap();
+ fs::remove_dir(root.join("Family")).unwrap();
+ let mut replacement = collection(2, "Family", &key);
+ replacement["updationTime"] = json!(40);
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[replacement]}).to_string())
+ .create();
+ page(&mut server, 2, 0, json!([]), false).create();
+ run(&home, &root, &[], true);
+ assert_eq!(
+ read_json(&root.join("Family/metadata.json"))["ente"]["albumID"],
+ "2"
+ );
+ assert!(!root.join("Family-1").exists());
+}
+
+#[test]
+fn export_clears_deleted_adopted_album_rename_reservations() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let key = Key::generate();
+ let albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[collection(1,"First",&key)]}).to_string())
+ .create();
+ let (file, encrypted) = source(10, &key, b"original", "Original.jpg");
+ page(&mut server, 1, 0, json!([file]), false).create();
+ page(&mut server, 1, 10, json!([]), false).create();
+ let downloaded = download(&mut server, 10, &encrypted, 1);
+ run(&home, &root, &[], true);
+ let fresh = TestHome::new();
+ fresh.seed(&server.url());
+ run(&fresh, &root, &["--adopt"], true);
+ let db = export_database(&fresh);
+ let owner: String = db
+ .query_row(
+ "SELECT key FROM albums WHERE id=1 AND retained=0",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ uuid::Uuid::parse_str(&owner).unwrap();
+ fs::create_dir(root.join("Blocked")).unwrap();
+ let unrelated = root.join("Blocked/notes.txt");
+ fs::write(&unrelated, b"unrelated rename target").unwrap();
+ albums.remove();
+ let mut renamed = collection(1, "Blocked", &key);
+ renamed["updationTime"] = json!(30);
+ let rename = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[renamed]}).to_string())
+ .create();
+ assert_eq!(run(&fresh, &root, &[], false)["conflicts"], 2);
+ let reservation = || {
+ db.query_row(
+ "SELECT record FROM allocations WHERE owner=?1",
+ [&owner],
+ |r| r.get::<_, String>(0),
+ )
+ .unwrap()
+ };
+ let pending = reservation();
+ assert_eq!(
+ serde_json::from_str::<Value>(&pending).unwrap()["paths"],
+ json!(["Blocked"])
+ );
+ assert_eq!(
+ fs::read(root.join("First/Original.jpg")).unwrap(),
+ b"original"
+ );
+ rename.remove();
+ let mut deleted = renamed;
+ deleted["updationTime"] = json!(40);
+ deleted["isDeleted"] = json!(true);
+ let deletion = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[deleted]}).to_string())
+ .create();
+ run(&fresh, &root, &["--exclude-album", "1"], true);
+ assert_eq!(reservation(), pending);
+ fs::create_dir_all(root.join("Trash/First")).unwrap();
+ let retained_blocker = root.join("Trash/First/notes.txt");
+ fs::write(&retained_blocker, b"unrelated retention target").unwrap();
+ run(&fresh, &root, &["--album", "1"], false);
+ assert_eq!(
+ db.query_row(
+ "SELECT count(*) FROM allocations WHERE owner=?1",
+ [&owner],
+ |r| r.get::<_, i64>(0)
+ )
+ .unwrap(),
+ 0
+ );
+ let retained = || {
+ db.query_row(
+ "SELECT record FROM allocations WHERE owner='retained:1'",
+ [],
+ |r| r.get::<_, String>(0),
+ )
+ .unwrap()
+ };
+ let pending_retention = retained();
+ assert_eq!(
+ serde_json::from_str::<Value>(&pending_retention).unwrap()["paths"],
+ json!(["First"])
+ );
+ run(&fresh, &root, &["--album", "1"], false);
+ assert_eq!(retained(), pending_retention);
+ assert!(!root.join("Trash/First-1").exists());
+ assert_eq!(
+ fs::read(root.join("First/Original.jpg")).unwrap(),
+ b"original"
+ );
+ assert_eq!(
+ fs::read(&retained_blocker).unwrap(),
+ b"unrelated retention target"
+ );
+ assert_eq!(fs::read(&unrelated).unwrap(), b"unrelated rename target");
+ fs::remove_file(retained_blocker).unwrap();
+ fs::remove_dir(root.join("Trash/First")).unwrap();
+ run(&fresh, &root, &["--album", "1"], true);
+ assert_eq!(
+ fs::read(root.join("Trash/First/Original.jpg")).unwrap(),
+ b"original"
+ );
+ assert_eq!(
+ read_json(&root.join("Trash/First/metadata/Original.jpg.json"))["ente"]["fileID"],
+ "10"
+ );
+ assert_eq!(
+ db.query_row("SELECT count(*) FROM allocations", [], |r| r
+ .get::<_, i64>(0))
+ .unwrap(),
+ 0
+ );
+ deletion.remove();
+ fs::remove_file(unrelated).unwrap();
+ fs::remove_dir(root.join("Blocked")).unwrap();
+ let mut replacement = collection(2, "Blocked", &key);
+ replacement["updationTime"] = json!(50);
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[replacement]}).to_string())
+ .create();
+ page(&mut server, 2, 0, json!([]), false).create();
+ run(&fresh, &root, &[], true);
+ assert_eq!(
+ read_json(&root.join("Blocked/metadata.json"))["ente"]["albumID"],
+ "2"
+ );
+ assert!(!root.join("Blocked-1").exists());
+ downloaded.assert();
+}
+
+#[test]
+fn export_rejects_offline_early_and_reports_path_conflicts() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ assert!(
+ failure(&home.run(&["photos", "export", root.to_str().unwrap(), "--offline"]))
+ .contains("requires network")
+ );
+ assert!(!root.exists());
+ let key = Key::generate();
+ let albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[collection(1,"First",&key)]}).to_string())
+ .create();
+ let empty = page(&mut server, 1, 0, json!([]), false).create();
+ run(&home, &root, &[], true);
+ empty.remove();
+ fs::write(root.join("First/Good.jpg"), b"unrelated").unwrap();
+ let db = database(&home);
+ db.execute("UPDATE photos_collections SET files_synced_to=NULL", [])
+ .unwrap();
+ let (good, encrypted) = source(10, &key, b"good", "Good.jpg");
+ page(&mut server, 1, 0, json!([good]), false).create();
+ let downloaded = download(&mut server, 10, &encrypted, 1);
+ #[cfg(unix)]
+ {
+ let target = destination.path().join("outside.jpg");
+ fs::write(&target, b"outside export").unwrap();
+ fs::remove_file(root.join("First/Good.jpg")).unwrap();
+ std::os::unix::fs::symlink(&target, root.join("First/Good.jpg")).unwrap();
+ assert_eq!(run(&home, &root, &[], false)["conflicts"], 1);
+ assert_eq!(fs::read(&target).unwrap(), b"outside export");
+ assert!(
+ fs::symlink_metadata(root.join("First/Good.jpg"))
+ .unwrap()
+ .is_symlink()
+ );
+ fs::remove_file(root.join("First/Good.jpg")).unwrap();
+ let external_metadata = tempfile::tempdir().unwrap();
+ std::os::unix::fs::symlink(external_metadata.path(), root.join("First/metadata")).unwrap();
+ assert_eq!(run(&home, &root, &[], false)["conflicts"], 1);
+ assert_eq!(fs::read_dir(external_metadata.path()).unwrap().count(), 0);
+ fs::remove_file(root.join("First/metadata")).unwrap();
+ fs::write(root.join("First/Good.jpg"), b"unrelated").unwrap();
+ }
+ let result = run(&home, &root, &[], false);
+ assert_eq!(result["conflicts"], 1);
+ assert_eq!(result["copies"]["pending"], 1);
+ assert_eq!(fs::read(root.join("First/Good.jpg")).unwrap(), b"unrelated");
+ assert!(!root.join("First/Good-1.jpg").exists());
+ let reservation = || {
+ let db = export_database(&home);
+ assert_eq!(
+ db.query_row("SELECT count(*) FROM placements WHERE file=10", [], |r| r
+ .get::<_, i64>(
+ 0
+ ))
+ .unwrap(),
+ 0
+ );
+ let record: String = db
+ .query_row(
+ "SELECT record FROM allocations WHERE owner='file:1:10'",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ serde_json::from_str::<Value>(&record).unwrap()
+ };
+ let allocated = reservation();
+ assert_eq!(allocated["paths"], json!(["Good.jpg"]));
+ let unchanged = page(&mut server, 1, 10, json!([]), false).create();
+ assert_eq!(run(&home, &root, &[], false)["conflicts"], 1);
+ assert_eq!(reservation(), allocated);
+ assert_eq!(fs::read(root.join("First/Good.jpg")).unwrap(), b"unrelated");
+ unchanged.remove();
+ let (mut contender, contender_bytes) = source(11, &key, b"contender", "Good.jpg");
+ contender["updationTime"] = json!(20);
+ page(&mut server, 1, 10, json!([contender]), false).create();
+ page(&mut server, 1, 20, json!([]), false).create();
+ let contender_download = download(&mut server, 11, &contender_bytes, 1);
+ albums.remove();
+ let mut changed = collection(1, "First", &key);
+ changed["updationTime"] = json!(30);
+ let albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[changed]}).to_string())
+ .create();
+ let result = run(&home, &root, &[], false);
+ assert_eq!(
+ result["copies"],
+ json!({"expected":2,"completed":1,"pending":1})
+ );
+ assert_eq!(result["conflicts"], 1);
+ assert_eq!(reservation(), allocated);
+ assert_eq!(fs::read(root.join("First/Good.jpg")).unwrap(), b"unrelated");
+ assert_eq!(
+ fs::read(root.join("First/Good-1.jpg")).unwrap(),
+ b"contender"
+ );
+ assert_eq!(
+ read_json(&root.join("First/metadata/Good-1.jpg.json"))["ente"]["fileID"],
+ "11"
+ );
+ assert_eq!(
+ run(&home, &root, &["--exclude-album", "First"], true)["copies"]["expected"],
+ 0
+ );
+ let export_db = export_database(&home);
+ let pending: String = export_db
+ .query_row(
+ "SELECT path FROM temporaries WHERE json_extract(record,'$.file')=10 AND json_extract(record,'$.hash') IS NOT NULL",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ assert_eq!(fs::read(root.join(&pending)).unwrap(), b"good");
+ drop(export_db);
+ albums.remove();
+ let mut renamed = collection(1, "Renamed", &key);
+ renamed["updationTime"] = json!(40);
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[renamed]}).to_string())
+ .create();
+ fs::remove_file(root.join("First/Good.jpg")).unwrap();
+ let unrelated_part = root.join("First/.ente-11111111111111111111111111111111.part");
+ fs::write(&unrelated_part, b"unrelated temporary").unwrap();
+ run(&home, &root, &[], true);
+ assert_eq!(fs::read(root.join("Renamed/Good.jpg")).unwrap(), b"good");
+ assert_eq!(
+ fs::read(root.join("Renamed/Good-1.jpg")).unwrap(),
+ b"contender"
+ );
+ assert_eq!(
+ export_database(&home)
+ .query_row("SELECT count(*) FROM allocations", [], |r| r
+ .get::<_, i64>(0))
+ .unwrap(),
+ 0
+ );
+ assert!(!root.join(pending.replace("First/", "Renamed/")).exists());
+ assert_eq!(
+ fs::read(root.join("Renamed/.ente-11111111111111111111111111111111.part")).unwrap(),
+ b"unrelated temporary"
+ );
+ downloaded.assert();
+ contender_download.assert();
+}
+
+fn run(home: &TestHome, root: &Path, options: &[&str], complete: bool) -> Value {
+ let mut args = vec!["photos", "export", root.to_str().unwrap(), "--json"];
+ args.extend_from_slice(options);
+ let output = home.run(&args);
+ if complete {
+ assert!(
+ output.status.success(),
+ "{}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ } else {
+ failure(&output);
+ }
+ let result: Value = serde_json::from_slice(&output.stdout).unwrap();
+ assert_eq!(result["complete"], complete);
+ result
+}
+fn source(id: i64, collection: &Key, bytes: &[u8], name: &str) -> (Value, Vec<u8>) {
+ let (mut remote, encrypted) = remote_file(id, collection, bytes);
+ let key = secretbox::decrypt(
+ &b64::decode(remote["encryptedKey"].as_str().unwrap()).unwrap(),
+ &Nonce::try_from_slice(
+ &b64::decode(remote["keyDecryptionNonce"].as_str().unwrap()).unwrap(),
+ )
+ .unwrap(),
+ collection,
+ )
+ .unwrap();
+ let key = Key::try_from_slice(&key).unwrap();
+ let metadata=blob::encrypt_json(&json!({"title":name,"fileType":0,"creationTime":1_700_000_000_123_456i64,"hash":b64::encode(&hash::hash(bytes,Some(64),None).unwrap()),"newAdditiveField":{"opaque":"kept in replica"}}),&key).unwrap();
+ remote["metadata"] = json!({"encryptedData":b64::encode(&metadata.encrypted_data),"decryptionHeader":b64::encode(metadata.decryption_header.as_bytes())});
+ let public = blob::encrypt_json(&json!({}), &key).unwrap();
+ remote["pubMagicMetadata"] = json!({"data":b64::encode(&public.encrypted_data),"header":b64::encode(public.decryption_header.as_bytes())});
+ (remote, encrypted)
+}
+fn download(
+ server: &mut mockito::ServerGuard,
+ id: i64,
+ bytes: &[u8],
+ times: usize,
+) -> mockito::Mock {
+ server
+ .mock("GET", format!("/files/download/v3/{id}").as_str())
+ .with_body(json!({"url":format!("{}/media/{id}",server.url())}).to_string())
+ .expect(times)
+ .create();
+ server
+ .mock("GET", format!("/media/{id}").as_str())
+ .with_body(bytes)
+ .expect(times)
+ .create()
+}
+fn database(home: &TestHome) -> rusqlite::Connection {
+ let db = rusqlite::Connection::open(
+ home.dir
+ .path()
+ .join("accounts/e6b355f0-5d4d-4fc3-b32c-a6c8f52b8210/data.db"),
+ )
+ .unwrap();
+ db.pragma_update(None, "key", format!("x'{}'", "2a".repeat(32)))
+ .unwrap();
+ db
+}
+fn export_database(home: &TestHome) -> rusqlite::Connection {
+ let directory = home
+ .dir
+ .path()
+ .join("accounts/e6b355f0-5d4d-4fc3-b32c-a6c8f52b8210/exports");
+ let path = fs::read_dir(directory)
+ .unwrap()
+ .map(|entry| entry.unwrap().path())
+ .find(|path| path.extension().is_some_and(|extension| extension == "db"))
+ .unwrap();
+ let db = rusqlite::Connection::open(path).unwrap();
+ db.pragma_update(None, "key", format!("x'{}'", "2a".repeat(32)))
+ .unwrap();
+ db
+}
+fn record(db: &rusqlite::Connection, album: i64, file: i64) -> Value {
+ let encoded: String = db
+ .query_row(
+ "SELECT record FROM photos_files WHERE collection_id=?1 AND id=?2",
+ rusqlite::params![album, file],
+ |r| r.get(0),
+ )
+ .unwrap();
+ serde_json::from_str(&encoded).unwrap()
+}
+fn read_json(path: &Path) -> Value {
+ serde_json::from_slice(&fs::read(path).unwrap()).unwrap()
+}
+
+#[test]
+#[cfg(unix)]
+fn export_continues_when_failed_transfer_cleanup_becomes_unwritable() {
+ use std::os::unix::fs::PermissionsExt;
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(
+ json!({"collections":[collection(1,"Blocked",&key),collection(2,"Healthy",&key)]})
+ .to_string(),
+ )
+ .create();
+ let (broken, mut encrypted) = source(
+ 10,
+ &key,
+ &vec![61; ente_core::crypto::stream::ENCRYPTION_CHUNK_SIZE + 20],
+ "Broken.jpg",
+ );
+ let intact = encrypted.clone();
+ encrypted.truncate(encrypted.len() - 1);
+ let (good, original) = source(20, &key, b"good", "Good.jpg");
+ page(&mut server, 1, 0, json!([broken]), false).create();
+ page(&mut server, 2, 0, json!([good]), false).create();
+ for id in [10, 20] {
+ server
+ .mock("GET", format!("/files/download/v3/{id}").as_str())
+ .with_body(json!({"url":format!("{}/media/{id}",server.url())}).to_string())
+ .create();
+ }
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let protected = root.join("Blocked");
+ let folder = protected.clone();
+ let denied = server
+ .mock("GET", "/media/10")
+ .with_body_from_request(move |_| {
+ fs::set_permissions(&folder, fs::Permissions::from_mode(0o500)).unwrap();
+ encrypted.clone()
+ })
+ .create();
+ let healthy = server
+ .mock("GET", "/media/20")
+ .with_body_from_request(move |_| {
+ std::thread::sleep(std::time::Duration::from_millis(500));
+ original.clone()
+ })
+ .create();
+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
+ fs::set_permissions(&protected, fs::Permissions::from_mode(0o700)).unwrap();
+ assert!(
+ failure(&output)
+ .to_lowercase()
+ .contains("permission denied")
+ );
+ let result: Value = serde_json::from_slice(&output.stdout).unwrap();
+ assert_eq!(
+ result["copies"],
+ json!({"expected":2,"completed":1,"pending":1})
+ );
+ assert_eq!(fs::read(root.join("Healthy/Good.jpg")).unwrap(), b"good");
+ let db = export_database(&home);
+ let partial: String = db.query_row("SELECT path FROM temporaries WHERE json_extract(record,'$.album')=1 AND json_extract(record,'$.file')=10 AND json_extract(record,'$.hash') IS NULL", [], |r| r.get(0)).unwrap();
+ assert!(root.join(&partial).is_file());
+ denied.remove();
+ let retry = server
+ .mock("GET", "/media/10")
+ .with_body(intact)
+ .expect(1)
+ .create();
+ run(&home, &root, &[], true);
+ assert!(!root.join(partial).exists());
+ assert_eq!(
+ db.query_row("SELECT count(*) FROM temporaries", [], |r| r
+ .get::<_, i64>(0))
+ .unwrap(),
+ 0
+ );
+ assert_eq!(
+ fs::read(root.join("Blocked/Broken.jpg")).unwrap(),
+ vec![61; ente_core::crypto::stream::ENCRYPTION_CHUNK_SIZE + 20]
+ );
+ healthy.assert();
+ retry.assert();
+}
+
+#[test]
+fn export_failed_stream_does_not_block_independent_album_or_advance_cursor() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(
+ json!({"collections":[collection(1,"Blocked",&key),collection(2,"Independent",&key)]})
+ .to_string(),
+ )
+ .create();
+ let (good, encrypted) = source(10, &key, b"good", "Good.jpg");
+ let malformed = page(&mut server, 1, 0, json!([{ "id":"not an integer" }]), false).create();
+ page(&mut server, 2, 0, json!([good]), false).create();
+ let downloaded = download(&mut server, 10, &encrypted, 1);
+ let result = run(&home, &root, &[], false);
+ assert!(result["files"].is_null());
+ assert!(result["copies"].is_null());
+ assert_eq!(
+ fs::read(root.join("Independent/Good.jpg")).unwrap(),
+ b"good"
+ );
+ let db = database(&home);
+ assert_eq!(
+ db.query_row(
+ "SELECT files_cursor FROM photos_collections WHERE id=1",
+ [],
+ |r| r.get::<_, i64>(0)
+ )
+ .unwrap(),
+ 0
+ );
+ malformed.remove();
+ let stuck = page(&mut server, 1, 0, json!([]), true).create();
+ run(&home, &root, &[], false);
+ stuck.remove();
+ page(&mut server, 1, 0, json!([good]), false).create();
+ run(&home, &root, &[], true);
+ downloaded.assert();
+ assert_eq!(fs::read(root.join("Blocked/Good.jpg")).unwrap(), b"good");
+}
+
+#[test]
+fn export_page_and_recognized_failures_roll_back_together_on_storage_failure() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[collection(1,"First",&key)]}).to_string())
+ .create();
+ let empty = page(&mut server, 1, 0, json!([]), false).create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ empty.remove();
+ let db = database(&home);
+ db.execute_batch("UPDATE photos_collections SET files_synced_to=NULL; CREATE TRIGGER fail_second BEFORE INSERT ON photos_files WHEN new.id=11 BEGIN SELECT RAISE(ABORT,'fixture storage failure'); END;").unwrap();
+ let (good, encrypted) = source(10, &key, b"good", "Good.jpg");
+ let (mut broken, _) = source(11, &key, b"broken", "Broken.jpg");
+ broken["metadata"]["encryptedData"] = json!("broken");
+ page(&mut server, 1, 0, json!([good, broken]), false).create();
+ let result = run(&home, &root, &[], false);
+ assert!(result["files"].is_null());
+ assert_eq!(
+ db.query_row("SELECT count(*) FROM photos_files", [], |r| r
+ .get::<_, i64>(0))
+ .unwrap(),
+ 0
+ );
+ assert_eq!(
+ db.query_row(
+ "SELECT files_cursor FROM photos_collections WHERE id=1",
+ [],
+ |r| r.get::<_, i64>(0)
+ )
+ .unwrap(),
+ 0
+ );
+ db.execute_batch("DROP TRIGGER fail_second;").unwrap();
+ download(&mut server, 10, &encrypted, 1);
+ let result = run(&home, &root, &[], false);
+ assert_eq!(result["copies"]["completed"], 1);
+ assert_eq!(result["copies"]["pending"], 1);
+ assert_eq!(
+ db.query_row(
+ "SELECT files_cursor FROM photos_collections WHERE id=1",
+ [],
+ |r| r.get::<_, i64>(0)
+ )
+ .unwrap(),
+ 10
+ );
+ assert!(!record(&db, 1, 11)["remote"].is_null());
+ assert!(record(&db, 1, 11)["documents"].is_null());
+}
+
+#[test]
+fn export_saves_interpretation_failures_and_omits_unknown_field_values_from_diagnostics() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[collection(1,"First",&key)]}).to_string())
+ .create();
+ let (good, encrypted) = source(10, &key, b"good", "Good.jpg");
+ let (mut invalid, _) = source(11, &key, b"invalid", "Invalid.jpg");
+ let file_key = secretbox::decrypt(
+ &b64::decode(invalid["encryptedKey"].as_str().unwrap()).unwrap(),
+ &Nonce::try_from_slice(
+ &b64::decode(invalid["keyDecryptionNonce"].as_str().unwrap()).unwrap(),
+ )
+ .unwrap(),
+ &key,
+ )
+ .unwrap();
+ let private = blob::encrypt_json(
+ &json!({"visibility":99}),
+ &Key::try_from_slice(&file_key).unwrap(),
+ )
+ .unwrap();
+ invalid["magicMetadata"] = json!({"data":b64::encode(&private.encrypted_data),"header":b64::encode(private.decryption_header.as_bytes())});
+ page(&mut server, 1, 0, json!([good, invalid]), false).create();
+ download(&mut server, 10, &encrypted, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
+ let diagnostics = failure(&output);
+ assert!(diagnostics.contains("newAdditiveField"));
+ assert!(!diagnostics.contains("kept in replica"));
+ let result: Value = serde_json::from_slice(&output.stdout).unwrap();
+ assert_eq!(
+ result["copies"],
+ json!({"expected":2,"completed":1,"pending":1})
+ );
+ let db = database(&home);
+ let failure = record(&db, 1, 11);
+ assert!(!failure["documents"].is_null());
+ let original: Vec<u8> =
+ serde_json::from_value(record(&db, 1, 10)["documents"]["original"].clone()).unwrap();
+ let future_reader: Value = serde_json::from_slice(&original).unwrap();
+ assert_eq!(
+ future_reader["newAdditiveField"]["opaque"],
+ "kept in replica"
+ );
+ run(&home, &root, &["--retry-failed"], false);
+ assert_eq!(record(&db, 1, 11)["attempt"], failure["attempt"]);
+}
+
+#[test]
+fn export_does_not_reuse_a_copy_with_a_different_current_source_hash() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[collection(1,"First",&key)]}).to_string())
+ .create();
+ let (first, first_bytes) = source(10, &key, b"old original", "Original.jpg");
+ page(&mut server, 1, 0, json!([first]), false).create();
+ let downloaded = download(&mut server, 10, &first_bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ downloaded.assert();
+ downloaded.remove();
+ albums.remove();
+ let mut album = collection(1, "First", &key);
+ album["updationTime"] = json!(40);
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[album]}).to_string())
+ .create();
+ let (mut current, current_bytes) = source(10, &key, b"new original", "Original.jpg");
+ current["updationTime"] = json!(30);
+ page(&mut server, 1, 10, json!([current]), false).create();
+ let downloaded = download(&mut server, 10, &current_bytes, 1);
+ run(&home, &root, &[], true);
+ downloaded.assert();
+ assert_eq!(
+ fs::read(root.join("First/Original.jpg")).unwrap(),
+ b"new original"
+ );
+}
diff --git a/rust/apps/cli-next/tests/support/files.rs b/rust/apps/cli-next/tests/support/files.rs
index 229c972c01..31ca5298d8 100644
--- a/rust/apps/cli-next/tests/support/files.rs
+++ b/rust/apps/cli-next/tests/support/files.rs
@@ -521,10 +521,18 @@ fn photos_lists_limit_output_and_stream_complete_results() {
connection
.pragma_update(None, "key", format!("x'{}'", "2a".repeat(32)))
.unwrap();
+ let encoded: String = connection
+ .query_row("SELECT record FROM photos_files WHERE id=150", [], |r| {
+ r.get(0)
+ })
+ .unwrap();
+ let mut record: Value = serde_json::from_str(&encoded).unwrap();
+ let public = serde_json::to_vec(&json!({"editedTime":i64::MAX})).unwrap();
+ record["documents"]["public"] = json!(public);
connection
.execute(
- "UPDATE photos_files SET created_at=?1 WHERE id=150",
- [i64::MAX],
+ "UPDATE photos_files SET record=?1 WHERE id=150",
+ [record.to_string()],
)
.unwrap();
drop(connection);
@@ -547,7 +555,7 @@ fn photos_lists_limit_output_and_stream_complete_results() {
}
}
-fn collection(id: i64, name: &str, key: &Key) -> Value {
+pub(super) fn collection(id: i64, name: &str, key: &Key) -> Value {
let wrapped = secretbox::encrypt(key.as_bytes(), &Key::from_bytes([0; 32]));
json!({
"id": id, "owner": {"id": 9007199254740993i64}, "name": name, "type": "album",
@@ -556,7 +564,7 @@ fn collection(id: i64, name: &str, key: &Key) -> Value {
})
}
-fn remote_file(id: i64, collection_key: &Key, original: &[u8]) -> (Value, Vec<u8>) {
+pub(super) fn remote_file(id: i64, collection_key: &Key, original: &[u8]) -> (Value, Vec<u8>) {
let key = Key::generate();
let wrapped = secretbox::encrypt(key.as_bytes(), collection_key);
let metadata = blob::encrypt_json(
@@ -600,7 +608,7 @@ fn remote_file(id: i64, collection_key: &Key, original: &[u8]) -> (Value, Vec<u8
)
}
-fn page(
+pub(super) fn page(
server: &mut mockito::ServerGuard,
collection: i64,
since: i64,
diff --git a/rust/apps/cli-next/tests/support/museum.rs b/rust/apps/cli-next/tests/support/museum.rs
index eee9774716..bd16aa7d92 100644
--- a/rust/apps/cli-next/tests/support/museum.rs
+++ b/rust/apps/cli-next/tests/support/museum.rs
@@ -218,7 +218,7 @@ async fn exercise(origin: String) -> TestResult {
listed.sort();
let expected = [
("Archive", "archived"),
- ("Default hidden", "hidden"),
+ ("Hidden", "hidden"),
("Monsoon 🌧", "visible"),
("Private album", "hidden"),
];
@@ -231,7 +231,7 @@ async fn exercise(origin: String) -> TestResult {
.unwrap();
assert_eq!(monsoon["id"], album.to_string());
assert_eq!(monsoon["ownerId"], alice_id);
- assert_eq!(monsoon["type"], "album");
+ assert_eq!(monsoon["type"], "folder");
assert_eq!(
chrono::DateTime::parse_from_rfc3339(monsoon["updatedAt"].as_str().unwrap())
.unwrap()
@@ -251,7 +251,7 @@ async fn exercise(origin: String) -> TestResult {
String::from_utf8(success(home.run(&["photos", "album", "view", "Monsoon 🌧"])).stdout)
.unwrap(),
format!(
- "Name Monsoon 🌧\nType album\nVisibility visible\nOwner {alice_id}\nUpdated {}\nID {album}\n",
+ "Name Monsoon 🌧\nType folder\nVisibility visible\nOwner {alice_id}\nUpdated {}\nID {album}\n",
monsoon["updatedAt"].as_str().unwrap(),
),
);
@@ -358,6 +358,38 @@ async fn exercise(origin: String) -> TestResult {
);
let shared_file = home.json(&["photos", "file", "view", &file_id]);
assert_eq!(shared_file["ownerId"], alice_id);
+ let shared_export = output_dir.path().join("shared-export");
+ home.json(&[
+ "photos",
+ "export",
+ shared_export.to_str().unwrap(),
+ "--album",
+ "Monsoon 🌧",
+ ]);
+ let album_record: Value =
+ serde_json::from_slice(&fs::read(shared_export.join("Monsoon 🌧/metadata.json")).unwrap())
+ .unwrap();
+ assert_eq!(album_record["ente"]["visibility"], "hidden");
+ let shared_sidecar = shared_export.join("Monsoon 🌧/metadata/original.jpg.json");
+ let file_record: Value = serde_json::from_slice(&fs::read(&shared_sidecar).unwrap()).unwrap();
+ assert!(file_record["ente"].get("visibility").is_none());
+ let adopted = TestHome::new();
+ adopted.write_vault(&home.read_vault());
+ fs::write(
+ shared_export.join("Monsoon 🌧/original.jpg"),
+ vec![0; original.len()],
+ )
+ .unwrap();
+ adopted.json(&[
+ "photos",
+ "export",
+ shared_export.to_str().unwrap(),
+ "--adopt",
+ ]);
+ assert_eq!(
+ fs::read(shared_export.join("Monsoon 🌧/original.jpg")).unwrap(),
+ original
+ );
let output_path = output_dir.path().join("shared.jpg");
success(home.run(&[
"photos",
@@ -369,6 +401,57 @@ async fn exercise(origin: String) -> TestResult {
]));
assert_eq!(fs::read(output_path).unwrap(), original);
+ reqwest::Client::new()
+ .post(format!("{origin}/collections/unshare"))
+ .header("x-auth-token", b64::encode_url_safe(&alice.secrets.token))
+ .header("x-client-package", "io.ente.photos")
+ .json(&json!({"collectionID":album,"email":bob_email}))
+ .send()
+ .await
+ .unwrap()
+ .error_for_status()
+ .unwrap();
+ adopted.json(&[
+ "photos",
+ "export",
+ shared_export.to_str().unwrap(),
+ "--exclude-album",
+ &album.to_string(),
+ ]);
+ assert_eq!(
+ fs::read(shared_export.join("Monsoon 🌧/original.jpg")).unwrap(),
+ original
+ );
+ let revoked = adopted.json(&[
+ "photos",
+ "export",
+ shared_export.to_str().unwrap(),
+ "--album",
+ &album.to_string(),
+ ]);
+ assert_eq!(revoked["copies"]["expected"], 0);
+ assert_eq!(revoked["changes"]["retained"], 1);
+ assert!(!shared_export.join("Monsoon 🌧/original.jpg").exists());
+ assert_eq!(
+ fs::read(shared_export.join("Trash/Monsoon 🌧/original.jpg")).unwrap(),
+ original
+ );
+ let retained_record: Value = serde_json::from_slice(
+ &fs::read(shared_export.join("Trash/Monsoon 🌧/metadata/original.jpg.json")).unwrap(),
+ )
+ .unwrap();
+ assert_eq!(retained_record["ente"]["fileID"], file_id);
+ assert_eq!(
+ adopted.json(&[
+ "photos",
+ "export",
+ shared_export.to_str().unwrap(),
+ "--album",
+ &album.to_string()
+ ])["changes"]["retained"],
+ 0
+ );
+
login(&home, "photos", &alice_email, &["--host", &origin]);
let added_original = b"added after the first sync";
let added_file = upload_file(&origin, &alice, album, &key, added_original, None)
@@ -412,6 +495,18 @@ async fn exercise(origin: String) -> TestResult {
assert!(error.contains(&format!("{album} \"Monsoon 🌧\"")));
assert!(error.contains(&format!("{conflict} {selector:?}")));
assert!(output.stdout.is_empty());
+ let ambiguous_export = output_dir.path().join("ambiguous-export");
+ let output = home.run(&[
+ "photos",
+ "export",
+ ambiguous_export.to_str().unwrap(),
+ "--album",
+ selector,
+ "--json",
+ ]);
+ assert!(failure(&output).contains("is ambiguous"));
+ assert!(output.stdout.is_empty());
+ assert!(!ambiguous_export.exists());
assert_eq!(
home.json(&["photos", "album", "view", &conflict.to_string()])["id"],
conflict.to_string(),
@@ -565,24 +660,66 @@ async fn upload_file(
original: &[u8],
file_type: Option<i32>,
) -> i64 {
+ let mut metadata = json!({"title":"original.jpg","creationTime":1_700_000_000_000_000i64,"modificationTime":1_700_000_000_000_000i64});
+ if let Some(kind) = file_type {
+ metadata["fileType"] = json!(kind);
+ }
+ upload_fixture(origin, owner, album, collection_key, original, metadata)
+ .await
+ .0
+}
+
+async fn upload_fixture(
+ origin: &str,
+ owner: &AuthenticatedAccount,
+ album: i64,
+ collection_key: &Key,
+ original: &[u8],
+ metadata: Value,
+) -> (i64, Key) {
+ upload_reader(origin, owner, album, collection_key, original, metadata).await
+}
+
+async fn upload_reader(
+ origin: &str,
+ owner: &AuthenticatedAccount,
+ album: i64,
+ collection_key: &Key,
+ mut original: impl std::io::Read,
+ metadata: Value,
+) -> (i64, Key) {
let client = reqwest::Client::new();
let token = b64::encode_url_safe(&owner.secrets.token);
let key = Key::generate();
let wrapped = secretbox::encrypt(key.as_bytes(), collection_key);
- let mut encrypted = Vec::new();
- let header =
- stream::encrypt_file(&mut std::io::Cursor::new(original), &mut encrypted, &key).unwrap();
+ let mut encrypted = Md5Writer::new(tempfile::tempfile().unwrap());
+ let header = stream::encrypt_file(&mut original, &mut encrypted, &key).unwrap();
+ let (mut encrypted, checksum) = encrypted.finalize();
+ std::io::Seek::rewind(&mut encrypted).unwrap();
let thumbnail = blob::encrypt(b"thumbnail", &key).unwrap();
let mut object_keys = Vec::new();
- for bytes in [&encrypted, &thumbnail.encrypted_data] {
- let mut checksum = Md5Writer::new(std::io::sink());
- checksum.write_all(bytes).unwrap();
- let checksum = b64::encode(&checksum.finalize().1);
+ let mut thumbnail_checksum = Md5Writer::new(std::io::sink());
+ thumbnail_checksum
+ .write_all(&thumbnail.encrypted_data)
+ .unwrap();
+ let objects: [(Box<dyn std::io::Read + Send>, u64, String); 2] = [
+ (
+ Box::new(encrypted.try_clone().unwrap()),
+ encrypted.metadata().unwrap().len(),
+ b64::encode(&checksum),
+ ),
+ (
+ Box::new(std::io::Cursor::new(thumbnail.encrypted_data.clone())),
+ thumbnail.encrypted_data.len() as u64,
+ b64::encode(&thumbnail_checksum.finalize().1),
+ ),
+ ];
+ for (bytes, size, checksum) in objects {
let upload: Value = client
.post(format!("{origin}/files/upload-url"))
.header("x-auth-token", &token)
.header("x-client-package", "io.ente.photos")
- .json(&json!({"contentLength": bytes.len(), "contentMD5": checksum}))
+ .json(&json!({"contentLength": size, "contentMD5": checksum}))
.send()
.await
.unwrap()
@@ -591,10 +728,17 @@ async fn upload_file(
.json()
.await
.unwrap();
+ let body = futures_util::stream::try_unfold(bytes, |mut input| async move {
+ let mut chunk = vec![0; 64 * 1024];
+ let count = input.read(&mut chunk)?;
+ chunk.truncate(count);
+ Ok::<_, std::io::Error>((count > 0).then_some((chunk, input)))
+ });
client
.put(upload["url"].as_str().unwrap())
.header("content-md5", checksum)
- .body(bytes.clone())
+ .header("content-length", size)
+ .body(reqwest::Body::wrap_stream(body))
.send()
.await
.unwrap()
@@ -602,13 +746,6 @@ async fn upload_file(
.unwrap();
object_keys.push(upload["objectKey"].as_str().unwrap().to_owned());
}
- let mut metadata = json!({
- "title": "original.jpg", "creationTime": 1_700_000_000_000_000i64,
- "modificationTime": 1_700_000_000_000_000i64
- });
- if let Some(file_type) = file_type {
- metadata["fileType"] = json!(file_type);
- }
let metadata = blob::encrypt_json(&metadata, &key).unwrap();
let result: Value = client.post(format!("{origin}/files"))
.header("x-auth-token", &token).header("x-client-package", "io.ente.photos")
@@ -620,7 +757,7 @@ async fn upload_file(
"metadata": {"encryptedData": b64::encode(&metadata.encrypted_data),
"decryptionHeader": b64::encode(metadata.decryption_header.as_bytes())}
})).send().await.unwrap().error_for_status().unwrap().json().await.unwrap();
- result["id"].as_i64().unwrap()
+ (result["id"].as_i64().unwrap(), key)
}
async fn set_album_metadata(
@@ -676,3 +813,6 @@ impl AuthFlowUi for NoPrompts {
unreachable!()
}
}
+
+#[path = "export.rs"]
+mod export;
diff --git a/rust/crates/collections/Cargo.toml b/rust/crates/collections/Cargo.toml
index 71a7ba6428..c0338492c3 100644
--- a/rust/crates/collections/Cargo.toml
+++ b/rust/crates/collections/Cargo.toml
@@ -5,6 +5,7 @@ edition.workspace = true
[dependencies]
ente-core.workspace = true
serde.workspace = true
+serde_json.workspace = true
thiserror.workspace = true
[lints]
diff --git a/rust/crates/collections/src/client.rs b/rust/crates/collections/src/client.rs
index 3ec6ee0c87..b433d5bcb7 100644
--- a/rust/crates/collections/src/client.rs
+++ b/rust/crates/collections/src/client.rs
@@ -3,7 +3,7 @@ use ente_core::{
crypto::{Header, Key, Nonce, blob, secretbox},
http,
};
-use serde::{Deserialize, de::DeserializeOwned};
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
use crate::{Error, Result, open_collection_key};
@@ -30,6 +30,15 @@ pub async fn diff(session: &Session, since: i64) -> Result<CollectionPage> {
.await
})
.await?;
+ if response
+ .collections
+ .iter()
+ .any(|c| c.id <= 0 || c.owner.id <= 0 || c.updation_time < since)
+ {
+ return Err(Error::InvalidPage(
+ "invalid collection identity or timestamp",
+ ));
+ }
let cursor = response
.collections
.iter()
@@ -68,6 +77,13 @@ pub async fn files_diff(session: &Session, collection_id: i64, since: i64) -> Re
.await
})
.await?;
+ if response
+ .diff
+ .iter()
+ .any(|f| f.id <= 0 || f.owner_id <= 0 || f.updation_time < since)
+ {
+ return Err(Error::InvalidPage("invalid file identity or timestamp"));
+ }
let cursor = response
.diff
.iter()
@@ -84,7 +100,7 @@ pub async fn files_diff(session: &Session, collection_id: i64, since: i64) -> Re
})
}
-#[derive(Deserialize)]
+#[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Collection {
pub id: i64,
@@ -99,7 +115,10 @@ pub struct Collection {
pub updation_time: i64,
pub is_deleted: Option<bool>,
pub magic_metadata: Option<EncryptedMetadata>,
+ pub pub_magic_metadata: Option<EncryptedMetadata>,
pub shared_magic_metadata: Option<EncryptedMetadata>,
+ #[serde(flatten)]
+ pub extra: serde_json::Map<String, serde_json::Value>,
}
impl Collection {
@@ -113,8 +132,15 @@ impl Collection {
}
pub fn open_name(&self, key: &Key) -> Result<String> {
+ String::from_utf8(self.open_name_bytes(key)?).map_err(|_| Error::InvalidCollection {
+ id: self.id,
+ reason: "name is not UTF-8",
+ })
+ }
+
+ pub fn open_name_bytes(&self, key: &Key) -> Result<Vec<u8>> {
if let Some(name) = self.name.as_ref().filter(|name| !name.is_empty()) {
- return Ok(name.clone());
+ return Ok(name.as_bytes().to_vec());
}
let encrypted = self
.encrypted_name
@@ -130,24 +156,22 @@ impl Collection {
id: self.id,
reason: "missing name nonce",
})?;
- let bytes = secretbox::decrypt(
+ Ok(secretbox::decrypt(
&b64::decode(encrypted)?,
&Nonce::try_from_slice(&b64::decode(nonce)?)?,
key,
- )?;
- String::from_utf8(bytes).map_err(|_| Error::InvalidCollection {
- id: self.id,
- reason: "name is not UTF-8",
- })
+ )?)
}
}
-#[derive(Deserialize)]
+#[derive(Clone, Serialize, Deserialize)]
pub struct Owner {
pub id: i64,
+ #[serde(flatten)]
+ pub extra: serde_json::Map<String, serde_json::Value>,
}
-#[derive(Deserialize)]
+#[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct File {
pub id: i64,
@@ -161,6 +185,8 @@ pub struct File {
pub updation_time: i64,
pub magic_metadata: Option<EncryptedMetadata>,
pub pub_magic_metadata: Option<EncryptedMetadata>,
+ #[serde(flatten)]
+ pub extra: serde_json::Map<String, serde_json::Value>,
}
impl File {
@@ -178,20 +204,32 @@ impl File {
}
}
-#[derive(Deserialize)]
+#[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct FileAttributes {
pub encrypted_data: Option<String>,
pub decryption_header: String,
+ #[serde(flatten)]
+ pub extra: serde_json::Map<String, serde_json::Value>,
}
-#[derive(Deserialize)]
+#[derive(Clone, Serialize, Deserialize)]
pub struct EncryptedMetadata {
pub data: String,
pub header: String,
+ #[serde(flatten)]
+ pub extra: serde_json::Map<String, serde_json::Value>,
}
impl EncryptedMetadata {
+ pub fn decrypt(&self, key: &Key) -> Result<Vec<u8>> {
+ Ok(blob::decrypt(
+ &b64::decode(&self.data)?,
+ &Header::try_from_slice(&b64::decode(&self.header)?)?,
+ key,
+ )?)
+ }
+
pub fn open<T: DeserializeOwned>(&self, key: &Key) -> Result<T> {
Ok(blob::decrypt_json(
&blob::EncryptedBlob {
diff --git a/rust/crates/collections/src/lib.rs b/rust/crates/collections/src/lib.rs
index f932f360dc..120a99fc48 100644
--- a/rust/crates/collections/src/lib.rs
+++ b/rust/crates/collections/src/lib.rs
@@ -33,11 +33,11 @@ pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug, thiserror::Error)]
pub enum Error {
- #[error(transparent)]
+ #[error("{0}")]
Http(#[from] ente_core::http::Error),
- #[error(transparent)]
+ #[error("{0}")]
Decode(#[from] b64::DecodeError),
- #[error(transparent)]
+ #[error("{0}")]
Crypto(#[from] crypto::Error),
#[error("Owned collection is missing its key decryption nonce")]
MissingKeyDecryptionNonce,
@@ -45,6 +45,8 @@ pub enum Error {
InvalidCollection { id: i64, reason: &'static str },
#[error("file changes did not advance the cursor for collection {0}")]
CursorDidNotAdvance(i64),
+ #[error("invalid source page: {0}")]
+ InvalidPage(&'static str),
}
#[cfg(test)]
diff --git a/rust/crates/location/Cargo.toml b/rust/crates/location/Cargo.toml
index 5bbec5b652..4623b34563 100644
--- a/rust/crates/location/Cargo.toml
+++ b/rust/crates/location/Cargo.toml
@@ -4,7 +4,7 @@ edition.workspace = true
[dependencies]
thiserror.workspace = true
-unicode-normalization = "0.1.25"
+unicode-normalization.workspace = true
[lints]
workspace = true
diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.toml
index 25ca94302a..cbcb2d2887 100644
--- a/rust/crates/photos/Cargo.toml
+++ b/rust/crates/photos/Cargo.toml
@@ -3,6 +3,7 @@ name = "ente-photos"
edition.workspace = true
[dependencies]
+chrono = { workspace = true, features = ["std"] }
ente-assets.workspace = true
ente-collections.workspace = true
ente-core.workspace = true
@@ -18,6 +19,7 @@ thiserror.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
futures-util.workspace = true
+zip.workspace = true
[dev-dependencies]
tempfile.workspace = true
diff --git a/rust/crates/photos/src/collections.rs b/rust/crates/photos/src/collections.rs
index 3719a17a21..85e5f34ffe 100644
--- a/rust/crates/photos/src/collections.rs
+++ b/rust/crates/photos/src/collections.rs
@@ -1,9 +1,9 @@
use ente_core::{Session, crypto::Key};
use serde::Deserialize;
-const DEFAULT_HIDDEN_SUBTYPE: u8 = 1;
-const ARCHIVED_VISIBILITY: u8 = 1;
-const HIDDEN_VISIBILITY: u8 = 2;
+use crate::source::{self, Documents, MetadataError};
+
+pub use ente_collections::client::Collection as RemoteCollection;
#[derive(Debug)]
pub struct Collection {
@@ -14,26 +14,39 @@ pub struct Collection {
pub visibility: Visibility,
pub owner_id: i64,
pub updated_at_micros: i64,
+ pub display_order: Option<i64>,
+ pub muted: Option<bool>,
+ pub ascending: Option<bool>,
+ pub description: Option<String>,
+ pub cover_file_id: Option<i64>,
+ pub layout: Option<String>,
+ pub warnings: Vec<String>,
}
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Kind {
Album,
+ Folder,
Favorites,
Uncategorized,
+ DefaultHidden,
+ Quicklink,
}
impl Kind {
pub fn name(self) -> &'static str {
match self {
Self::Album => "album",
+ Self::Folder => "folder",
Self::Favorites => "favorites",
Self::Uncategorized => "uncategorized",
+ Self::DefaultHidden => "defaultHidden",
+ Self::Quicklink => "quicklink",
}
}
}
-#[derive(Debug)]
+#[derive(Debug, Clone, Copy)]
pub enum Visibility {
Visible,
Archived,
@@ -41,16 +54,31 @@ pub enum Visibility {
}
impl Visibility {
- pub fn name(&self) -> &'static str {
+ pub fn name(self) -> &'static str {
match self {
Self::Visible => "visible",
Self::Archived => "archived",
Self::Hidden => "hidden",
}
}
+
+ pub fn from_value(value: Option<u8>) -> Result<Self, MetadataError> {
+ match value.unwrap_or(0) {
+ 0 => Ok(Self::Visible),
+ 1 => Ok(Self::Archived),
+ 2 => Ok(Self::Hidden),
+ _ => Err(MetadataError("unsupported visibility")),
+ }
+ }
}
-pub type Error = ente_collections::Error;
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error(transparent)]
+ Collections(#[from] ente_collections::Error),
+ #[error(transparent)]
+ Metadata(#[from] MetadataError),
+}
pub struct Change {
pub id: i64,
@@ -68,13 +96,15 @@ pub async fn diff(session: &Session, since: i64) -> Result<Page, Error> {
.collections
.into_iter()
.map(|remote| {
+ let collection = if remote.is_deleted == Some(true) {
+ None
+ } else {
+ let (key, documents) = decrypt(&remote, session)?;
+ Some(interpret(&remote, &key, &documents, session.user_id)?)
+ };
Ok(Change {
id: remote.id,
- collection: if remote.is_deleted == Some(true) {
- None
- } else {
- Some(Collection::open(remote, session)?)
- },
+ collection,
})
})
.collect::<Result<_, Error>>()?;
@@ -89,57 +119,133 @@ pub async fn list(session: &Session) -> Result<Vec<Collection>, Error> {
.await?
.changes
.into_iter()
- .filter_map(|change| change.collection)
+ .filter_map(|c| c.collection)
.collect())
}
-impl Collection {
- fn open(
- remote: ente_collections::client::Collection,
- session: &Session,
- ) -> Result<Self, Error> {
- let key = remote.open_key(session)?;
- let name = remote.open_name(&key)?;
- let owned = remote.owner.id == session.user_id;
- let metadata = if owned {
- remote.magic_metadata.as_ref()
+pub fn decrypt(
+ remote: &RemoteCollection,
+ session: &Session,
+) -> Result<(Key, Documents), ente_collections::Error> {
+ let key = remote.open_key(session)?;
+ let original = remote.open_name_bytes(&key)?;
+ let documents = Documents {
+ original,
+ public: remote
+ .pub_magic_metadata
+ .as_ref()
+ .map(|m| m.decrypt(&key))
+ .transpose()?,
+ private: remote
+ .magic_metadata
+ .as_ref()
+ .map(|m| m.decrypt(&key))
+ .transpose()?,
+ shared: remote
+ .shared_magic_metadata
+ .as_ref()
+ .map(|m| m.decrypt(&key))
+ .transpose()?,
+ };
+ Ok((key, documents))
+}
+
+pub fn interpret(
+ remote: &RemoteCollection,
+ key: &Key,
+ documents: &Documents,
+ user_id: i64,
+) -> Result<Collection, MetadataError> {
+ let owned = remote.owner.id == user_id;
+ let private: PrivateMetadata = source::optional(
+ documents.private.as_deref(),
+ "invalid album private metadata",
+ )?;
+ let shared: ShareeMetadata =
+ source::optional(documents.shared.as_deref(), "invalid album sharee metadata")?;
+ let public: PublicMetadata =
+ source::optional(documents.public.as_deref(), "invalid album public metadata")?;
+ let kind = match (
+ owned.then_some(private.sub_type).flatten().unwrap_or(0),
+ remote.kind.as_str(),
+ ) {
+ (1, _) => Kind::DefaultHidden,
+ (2, _) => Kind::Quicklink,
+ (0, "album") => Kind::Album,
+ (0, "folder") => Kind::Folder,
+ (0, "favorites") => Kind::Favorites,
+ (0, "uncategorized") => Kind::Uncategorized,
+ (0, "quicklink") => Kind::Quicklink,
+ _ => return Err(MetadataError("unsupported album type")),
+ };
+ let visibility = if kind == Kind::DefaultHidden {
+ Visibility::Hidden
+ } else {
+ Visibility::from_value(if owned {
+ private.visibility
} else {
- remote.shared_magic_metadata.as_ref()
- };
- let visibility = match metadata {
- Some(metadata) => {
- let metadata: Metadata = metadata.open(&key)?;
- if owned && metadata.sub_type == Some(DEFAULT_HIDDEN_SUBTYPE) {
- HIDDEN_VISIBILITY
- } else {
- metadata.visibility.unwrap_or(0)
- }
- }
- None => 0,
- };
- Ok(Self {
- id: remote.id,
- key,
- name,
- kind: match remote.kind.as_str() {
- "favorites" => Kind::Favorites,
- "uncategorized" => Kind::Uncategorized,
- _ => Kind::Album,
- },
- visibility: match visibility {
- ARCHIVED_VISIBILITY => Visibility::Archived,
- HIDDEN_VISIBILITY => Visibility::Hidden,
- _ => Visibility::Visible,
- },
- owner_id: remote.owner.id,
- updated_at_micros: remote.updation_time,
- })
- }
+ shared.visibility
+ })?
+ };
+ let mut warnings = Vec::new();
+ source::unfamiliar(
+ documents.private.as_deref(),
+ &["visibility", "subType", "order"],
+ &mut warnings,
+ );
+ source::unfamiliar(
+ documents.shared.as_deref(),
+ &["visibility", "mute", "order"],
+ &mut warnings,
+ );
+ source::unfamiliar(
+ documents.public.as_deref(),
+ &["asc", "coverID", "layout", "caption"],
+ &mut warnings,
+ );
+ Ok(Collection {
+ id: remote.id,
+ key: Key::from_bytes(*key.as_bytes()),
+ name: if kind == Kind::DefaultHidden {
+ "Hidden".to_owned()
+ } else {
+ String::from_utf8(documents.original.clone())
+ .map_err(|_| MetadataError("album name is not UTF-8"))?
+ },
+ kind,
+ visibility,
+ owner_id: remote.owner.id,
+ updated_at_micros: remote.updation_time,
+ display_order: if owned { private.order } else { shared.order },
+ muted: (!owned).then_some(shared.mute).flatten(),
+ ascending: public.asc,
+ description: public.caption,
+ cover_file_id: public.cover_id,
+ layout: public.layout,
+ warnings,
+ })
}
-#[derive(Deserialize)]
+#[derive(Default, Deserialize)]
#[serde(rename_all = "camelCase")]
-struct Metadata {
+struct PrivateMetadata {
visibility: Option<u8>,
sub_type: Option<u8>,
+ order: Option<i64>,
+}
+
+#[derive(Default, Deserialize)]
+struct ShareeMetadata {
+ visibility: Option<u8>,
+ order: Option<i64>,
+ mute: Option<bool>,
+}
+
+#[derive(Default, Deserialize)]
+struct PublicMetadata {
+ asc: Option<bool>,
+ #[serde(rename = "coverID")]
+ cover_id: Option<i64>,
+ layout: Option<String>,
+ caption: Option<String>,
}
diff --git a/rust/crates/photos/src/export.rs b/rust/crates/photos/src/export.rs
new file mode 100644
index 0000000000..3faabb1e0a
--- /dev/null
+++ b/rust/crates/photos/src/export.rs
@@ -0,0 +1,180 @@
+use chrono::{DateTime, SecondsFormat};
+use ente_core::{
+ b64,
+ crypto::{Key, kdf},
+};
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value, json};
+
+use crate::{
+ collections::{Collection, Visibility},
+ files::{File, Kind},
+ source::MetadataError,
+};
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct Root {
+ pub format: String,
+ pub version: u32,
+ pub source_verifier: String,
+}
+
+impl Root {
+ pub fn new(master_key: &Key) -> ente_core::crypto::Result<Self> {
+ Ok(Self {
+ format: "ente-photos-export".into(),
+ version: 1,
+ source_verifier: b64::encode(&kdf::derive_subkey(master_key, 32, 1, b"photoexp")?),
+ })
+ }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Role {
+ Original,
+ Image,
+ Video,
+}
+
+impl Role {
+ pub fn name(&self) -> &'static str {
+ match self {
+ Self::Original => "original",
+ Self::Image => "image",
+ Self::Video => "video",
+ }
+ }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Component {
+ pub role: Role,
+ pub path: String,
+ pub size: u64,
+ pub hash: String,
+}
+
+pub fn album(album: &Collection) -> Value {
+ let mut metadata = json!({
+ "albumID": album.id.to_string(), "type": album.kind.name(), "visibility": album.visibility.name(),
+ }).as_object().cloned().unwrap_or_default();
+ optional(&mut metadata, "displayOrder", album.display_order);
+ optional(&mut metadata, "muted", album.muted);
+ optional(
+ &mut metadata,
+ "sortOrder",
+ album
+ .ascending
+ .map(|asc| if asc { "ascending" } else { "descending" }),
+ );
+ optional(&mut metadata, "description", album.description.as_deref());
+ optional(
+ &mut metadata,
+ "coverFileID",
+ album.cover_file_id.map(|id| id.to_string()),
+ );
+ optional(&mut metadata, "layout", album.layout.as_deref());
+ json!({"title": album.name, "ente": metadata})
+}
+
+pub fn file(
+ file: &File,
+ components: &[Component],
+ component: usize,
+ favorited: bool,
+) -> Result<Value, MetadataError> {
+ let selected = components
+ .get(component)
+ .ok_or(MetadataError("missing media component"))?;
+ let mut metadata = json!({
+ "fileID": file.id.to_string(),
+ "type": if file.kind == Kind::LivePhoto { "livePhoto" } else { file.kind.name() },
+ "name": file.name,
+ "creationTime": timestamp(file.created_at_micros)?,
+ "component": selected.role,
+ "components": components,
+ })
+ .as_object()
+ .cloned()
+ .unwrap_or_default();
+ optional(
+ &mut metadata,
+ "modificationTime",
+ file.modified_at_micros.map(timestamp).transpose()?,
+ );
+ optional(
+ &mut metadata,
+ "location",
+ file.location
+ .as_ref()
+ .map(|l| json!({"latitude": l.latitude,"longitude": l.longitude})),
+ );
+ optional(&mut metadata, "localDateTime", file.date_time.as_deref());
+ optional(&mut metadata, "utcOffset", file.offset_time.as_deref());
+ optional(&mut metadata, "width", file.width);
+ optional(&mut metadata, "height", file.height);
+ optional(&mut metadata, "durationSeconds", file.duration_seconds);
+ optional(
+ &mut metadata,
+ "visibility",
+ file.visibility.map(Visibility::name),
+ );
+ optional(&mut metadata, "cameraMake", file.camera_make.as_deref());
+ optional(&mut metadata, "cameraModel", file.camera_model.as_deref());
+ optional(&mut metadata, "uploaderName", file.uploader_name.as_deref());
+ optional(&mut metadata, "panorama", file.panorama);
+ optional(
+ &mut metadata,
+ "motionPhotoVideoOffset",
+ file.motion_video_offset,
+ );
+ let taken = common_time(file.created_at_micros)?;
+ let mut record = json!({
+ "title": selected.path,
+ "photoTakenTime": taken,
+ "creationTime": taken,
+ "favorited": favorited,
+ "ente": metadata,
+ });
+ let object = record
+ .as_object_mut()
+ .ok_or(MetadataError("invalid file record"))?;
+ optional(
+ object,
+ "modificationTime",
+ file.modified_at_micros.map(common_time).transpose()?,
+ );
+ optional(object, "description", file.caption.as_deref());
+ optional(
+ object,
+ "geoData",
+ file.location
+ .as_ref()
+ .map(|l| json!({"latitude":l.latitude,"longitude":l.longitude})),
+ );
+ Ok(record)
+}
+
+fn optional(object: &mut Map<String, Value>, name: &str, value: Option<impl Into<Value>>) {
+ if let Some(value) = value {
+ object.insert(name.into(), value.into());
+ }
+}
+
+fn time(micros: i64) -> Result<DateTime<chrono::Utc>, MetadataError> {
+ DateTime::from_timestamp_micros(micros)
+ .ok_or(MetadataError("timestamp is outside the supported range"))
+}
+
+fn timestamp(micros: i64) -> Result<String, MetadataError> {
+ Ok(time(micros)?.to_rfc3339_opts(SecondsFormat::Micros, true))
+}
+
+fn common_time(micros: i64) -> Result<Value, MetadataError> {
+ let time = time(micros)?;
+ Ok(
+ json!({"timestamp":micros.div_euclid(1_000_000).to_string(),"formatted":time.format("%b %-d, %Y, %-I:%M:%S %p UTC").to_string()}),
+ )
+}
diff --git a/rust/crates/photos/src/files.rs b/rust/crates/photos/src/files.rs
index b97d2ffb55..23aa4ad0dc 100644
--- a/rust/crates/photos/src/files.rs
+++ b/rust/crates/photos/src/files.rs
@@ -5,7 +5,12 @@ use ente_core::{
};
use serde::Deserialize;
-use crate::collections::{Collection, Visibility};
+use crate::{
+ collections::{Collection, Visibility},
+ source::{self, Documents, MetadataError},
+};
+
+pub use ente_collections::client::File as RemoteFile;
#[derive(Debug)]
pub struct File {
@@ -15,7 +20,7 @@ pub struct File {
pub name: String,
pub kind: Kind,
pub created_at_micros: i64,
- pub modified_at_micros: i64,
+ pub modified_at_micros: Option<i64>,
pub location: Option<Location>,
pub caption: Option<String>,
pub hash: Option<String>,
@@ -24,12 +29,18 @@ pub struct File {
pub duration_seconds: Option<u64>,
pub width: Option<u32>,
pub height: Option<u32>,
- pub visibility: Visibility,
+ pub visibility: Option<Visibility>,
+ pub camera_make: Option<String>,
+ pub camera_model: Option<String>,
+ pub uploader_name: Option<String>,
+ pub panorama: Option<bool>,
+ pub motion_video_offset: Option<u64>,
+ pub warnings: Vec<String>,
pub key: Key,
pub header: Header,
}
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Kind {
Image,
Video,
@@ -67,16 +78,18 @@ pub struct Page {
#[derive(Debug, thiserror::Error)]
pub enum Error {
- #[error(transparent)]
+ #[error("{0}")]
Http(#[from] http::Error),
- #[error(transparent)]
+ #[error("{0}")]
Crypto(#[from] crypto::Error),
- #[error(transparent)]
+ #[error("{0}")]
Base64(#[from] b64::DecodeError),
- #[error(transparent)]
+ #[error("{0}")]
Io(#[from] std::io::Error),
- #[error(transparent)]
+ #[error("{0}")]
Collections(#[from] ente_collections::Error),
+ #[error("{0}")]
+ Metadata(#[from] MetadataError),
}
pub async fn diff(session: &Session, collection: &Collection, since: i64) -> Result<Page, Error> {
@@ -90,7 +103,10 @@ pub async fn diff(session: &Session, collection: &Collection, since: i64) -> Res
file: if remote.is_deleted() {
None
} else {
- Some(File::open(remote, &collection.key)?)
+ {
+ let (key, documents) = decrypt(&remote, &collection.key)?;
+ Some(interpret(&remote, &key, &documents, session.user_id)?)
+ }
},
})
})
@@ -107,7 +123,7 @@ pub async fn download<W: std::io::Write>(
session: &Session,
file: &File,
mut output: impl FnMut() -> std::io::Result<W>,
-) -> Result<(), Error> {
+) -> Result<W, Error> {
use ente_core::crypto::stream::DecryptingWriter;
use futures_util::StreamExt;
@@ -137,8 +153,7 @@ pub async fn download<W: std::io::Write>(
while let Some(chunk) = body.next().await {
decryptor.write(&chunk?)?;
}
- decryptor.finish()?;
- Ok(())
+ Ok(decryptor.finish()?)
}
},
|error| matches!(error, Error::Http(error) if error.is_retryable()),
@@ -152,81 +167,150 @@ struct DownloadUrl {
url: String,
}
-impl File {
- fn open(remote: ente_collections::client::File, collection_key: &Key) -> Result<Self, Error> {
- let key = remote.open_key(collection_key)?;
- let metadata: Metadata = blob::decrypt_json(
- &blob::EncryptedBlob {
- encrypted_data: b64::decode(
- remote.metadata.encrypted_data.as_deref().unwrap_or(""),
- )?,
- decryption_header: Header::try_from_slice(&b64::decode(
- &remote.metadata.decryption_header,
- )?)?,
- },
+pub fn decrypt(remote: &RemoteFile, collection_key: &Key) -> Result<(Key, Documents), Error> {
+ let key = remote.open_key(collection_key)?;
+ let documents = Documents {
+ original: blob::decrypt(
+ &b64::decode(remote.metadata.encrypted_data.as_deref().unwrap_or(""))?,
+ &Header::try_from_slice(&b64::decode(&remote.metadata.decryption_header)?)?,
&key,
- )?;
- let public: PublicMetadata = match remote.pub_magic_metadata {
- Some(encrypted) => encrypted.open(&key)?,
- None => PublicMetadata::default(),
- };
- let private: PrivateMetadata = match remote.magic_metadata {
- Some(encrypted) => encrypted.open(&key)?,
- None => PrivateMetadata::default(),
- };
- let kind = match metadata.file_type {
- Some(0) => Kind::Image,
- Some(1) => Kind::Video,
- Some(2) => Kind::LivePhoto,
- _ => Kind::Unknown,
- };
- let hash = metadata.hash.filter(|hash| !hash.is_empty()).or_else(|| {
- if matches!(kind, Kind::LivePhoto) {
- metadata
- .image_hash
- .zip(metadata.video_hash)
- .map(|(image, video)| format!("{image}:{video}"))
- } else {
- None
- }
+ )?,
+ public: remote
+ .pub_magic_metadata
+ .as_ref()
+ .map(|m| m.decrypt(&key))
+ .transpose()?,
+ private: remote
+ .magic_metadata
+ .as_ref()
+ .map(|m| m.decrypt(&key))
+ .transpose()?,
+ shared: None,
+ };
+ Ok((key, documents))
+}
+
+pub fn interpret(
+ remote: &RemoteFile,
+ key: &Key,
+ documents: &Documents,
+ user_id: i64,
+) -> Result<File, Error> {
+ let metadata: Metadata = source::parse(&documents.original, "invalid original file metadata")?;
+ let public: PublicMetadata =
+ source::optional(documents.public.as_deref(), "invalid public file metadata")?;
+ let private: PrivateMetadata = source::optional(
+ documents.private.as_deref(),
+ "invalid private file metadata",
+ )?;
+ let kind = match metadata.file_type {
+ Some(0) => Kind::Image,
+ Some(1) => Kind::Video,
+ Some(2) => Kind::LivePhoto,
+ _ => Kind::Unknown,
+ };
+ let hash = metadata.hash.filter(|hash| !hash.is_empty()).or_else(|| {
+ if kind == Kind::LivePhoto {
+ metadata
+ .image_hash
+ .zip(metadata.video_hash)
+ .map(|(image, video)| format!("{image}:{video}"))
+ } else {
+ None
+ }
+ });
+ let location = public
+ .lat
+ .zip(public.long)
+ .or(metadata.latitude.zip(metadata.longitude))
+ .filter(|&(lat, long)| lat != 0.0 || long != 0.0)
+ .map(|(latitude, longitude)| Location {
+ latitude,
+ longitude,
});
- let location = public
- .lat
- .zip(public.long)
- .or(metadata.latitude.zip(metadata.longitude))
- .filter(|&(lat, long)| lat != 0.0 || long != 0.0)
- .map(|(latitude, longitude)| Location {
- latitude,
- longitude,
- });
- Ok(File {
- id: remote.id,
- owner_id: remote.owner_id,
- updated_at_micros: remote.updation_time,
- name: public.edited_name.unwrap_or(metadata.title),
- kind,
- created_at_micros: public.edited_time.unwrap_or(metadata.creation_time),
- modified_at_micros: metadata.modification_time.unwrap_or(metadata.creation_time),
- location,
- caption: public.caption.map(|caption| match caption {
- Caption::Text(text) => text,
- Caption::Number(number) => number.to_string(),
- }),
- hash,
- date_time: public.date_time,
- offset_time: public.offset_time,
- duration_seconds: metadata.duration,
- width: public.w,
- height: public.h,
- visibility: match private.visibility {
- Some(1) => Visibility::Archived,
- Some(2) => Visibility::Hidden,
- _ => Visibility::Visible,
- },
- key,
- header: Header::try_from_slice(&b64::decode(&remote.file.decryption_header)?)?,
- })
- }
+ let mut warnings = Vec::new();
+ source::unfamiliar(
+ Some(&documents.original),
+ &[
+ "title",
+ "fileType",
+ "creationTime",
+ "modificationTime",
+ "latitude",
+ "longitude",
+ "hash",
+ "imageHash",
+ "videoHash",
+ "duration",
+ "hasStaticThumbnail",
+ "localID",
+ "deviceFolder",
+ "subType",
+ "version",
+ "exif",
+ ],
+ &mut warnings,
+ );
+ source::unfamiliar(
+ documents.public.as_deref(),
+ &[
+ "editedName",
+ "editedTime",
+ "dateTime",
+ "offsetTime",
+ "caption",
+ "lat",
+ "long",
+ "w",
+ "h",
+ "sv",
+ "cameraMake",
+ "cameraModel",
+ "uploaderName",
+ "mediaType",
+ "mvi",
+ "noThumb",
+ ],
+ &mut warnings,
+ );
+ source::unfamiliar(documents.private.as_deref(), &["visibility"], &mut warnings);
+ Ok(File {
+ id: remote.id,
+ owner_id: remote.owner_id,
+ updated_at_micros: remote.updation_time,
+ name: public.edited_name.unwrap_or(metadata.title),
+ kind,
+ created_at_micros: public.edited_time.unwrap_or(metadata.creation_time),
+ modified_at_micros: metadata.modification_time,
+ location,
+ caption: public.caption.map(|caption| match caption {
+ Caption::Text(text) => text,
+ Caption::Number(number) => number.to_string(),
+ }),
+ hash,
+ date_time: public.date_time,
+ offset_time: public.offset_time,
+ duration_seconds: metadata.duration,
+ width: public.w,
+ height: public.h,
+ visibility: if remote.owner_id == user_id {
+ Some(Visibility::from_value(private.visibility)?)
+ } else {
+ None
+ },
+ camera_make: public.camera_make,
+ camera_model: public.camera_model,
+ uploader_name: public.uploader_name,
+ panorama: if kind == Kind::Image {
+ public.media_type.map(|value| value & 1 != 0)
+ } else {
+ None
+ },
+ motion_video_offset: public.mvi,
+ warnings,
+ key: Key::from_bytes(*key.as_bytes()),
+ header: Header::try_from_slice(&b64::decode(&remote.file.decryption_header)?)?,
+ })
}
#[derive(Deserialize)]
@@ -256,6 +340,11 @@ struct PublicMetadata {
long: Option<f64>,
w: Option<u32>,
h: Option<u32>,
+ camera_make: Option<String>,
+ camera_model: Option<String>,
+ uploader_name: Option<String>,
+ media_type: Option<u64>,
+ mvi: Option<u64>,
}
#[derive(Deserialize)]
diff --git a/rust/crates/photos/src/lib.rs b/rust/crates/photos/src/lib.rs
index 08e1912399..a718b88631 100644
--- a/rust/crates/photos/src/lib.rs
+++ b/rust/crates/photos/src/lib.rs
@@ -1,13 +1,18 @@
pub mod collections;
pub mod db;
+pub mod export;
pub mod files;
pub mod location;
pub mod metadata;
pub mod ml_db;
pub mod ml_store;
pub mod motion_photo;
+pub mod source;
pub use motion_photo::{
MotionPhotoError, VideoIndex, extract_motion_video_file_from_path,
extract_motion_video_from_path, extract_xmp_from_path, get_motion_video_index_from_path,
};
+
+#[cfg(not(target_arch = "wasm32"))]
+pub mod live_photo;
diff --git a/rust/crates/photos/src/live_photo.rs b/rust/crates/photos/src/live_photo.rs
new file mode 100644
index 0000000000..54bc4ba2b3
--- /dev/null
+++ b/rust/crates/photos/src/live_photo.rs
@@ -0,0 +1,60 @@
+use std::io::{Read, Seek, Write};
+
+use crate::export::Role;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error("{0}")]
+ Zip(#[from] zip::result::ZipError),
+ #[error("{0}")]
+ Io(#[from] std::io::Error),
+ #[error("Live Photo must contain one image and one video")]
+ Components,
+}
+
+pub fn extract<R: Read + Seek, W: Write>(
+ reader: R,
+ mut output: impl FnMut(Role, &str) -> std::io::Result<W>,
+) -> Result<Vec<(Role, String, W)>, Error> {
+ let mut archive = zip::ZipArchive::new(reader)?;
+ let mut image = None;
+ let mut video = None;
+ for index in 0..archive.len() {
+ let entry = archive.by_index(index)?;
+ if entry.is_dir() {
+ continue;
+ }
+ let name = entry.name();
+ let (role, extension) = if let Some(extension) = name.strip_prefix("image") {
+ (Role::Image, extension)
+ } else if let Some(extension) = name.strip_prefix("video") {
+ (Role::Video, extension)
+ } else {
+ return Err(Error::Components);
+ };
+ if (!extension.is_empty() && (!extension.starts_with('.') || extension.len() == 1))
+ || extension.contains(['/', '\\'])
+ {
+ return Err(Error::Components);
+ }
+ let slot = if role == Role::Image {
+ &mut image
+ } else {
+ &mut video
+ };
+ if slot.replace((index, extension.to_owned())).is_some() {
+ return Err(Error::Components);
+ }
+ }
+ let (Some(image), Some(video)) = (image, video) else {
+ return Err(Error::Components);
+ };
+ let mut results = Vec::with_capacity(2);
+ for (role, (index, extension)) in [(Role::Image, image), (Role::Video, video)] {
+ let mut writer = output(role.clone(), &extension)?;
+ std::io::copy(&mut archive.by_index(index)?, &mut writer)?;
+ writer.flush()?;
+ results.push((role, extension, writer));
+ }
+ Ok(results)
+}
diff --git a/rust/crates/photos/src/source.rs b/rust/crates/photos/src/source.rs
new file mode 100644
index 0000000000..00d9188b5d
--- /dev/null
+++ b/rust/crates/photos/src/source.rs
@@ -0,0 +1,40 @@
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
+use serde_json::Value;
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct Documents {
+ pub original: Vec<u8>,
+ pub public: Option<Vec<u8>>,
+ pub private: Option<Vec<u8>>,
+ pub shared: Option<Vec<u8>>,
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("{0}")]
+pub struct MetadataError(pub &'static str);
+
+pub fn parse<T: DeserializeOwned>(bytes: &[u8], context: &'static str) -> Result<T, MetadataError> {
+ serde_json::from_slice(bytes).map_err(|_| MetadataError(context))
+}
+
+pub fn optional<T: DeserializeOwned + Default>(
+ bytes: Option<&[u8]>,
+ context: &'static str,
+) -> Result<T, MetadataError> {
+ bytes
+ .map(|bytes| parse(bytes, context))
+ .transpose()
+ .map(Option::unwrap_or_default)
+}
+
+pub fn unfamiliar(bytes: Option<&[u8]>, known: &[&str], warnings: &mut Vec<String>) {
+ if let Some(bytes) = bytes
+ && let Ok(Value::Object(fields)) = serde_json::from_slice(bytes)
+ {
+ for key in fields.keys() {
+ if !known.contains(&key.as_str()) {
+ warnings.push(key.clone());
+ }
+ }
+ }
+}
diff --git a/rust/crates/test-support/src/lib.rs b/rust/crates/test-support/src/lib.rs
index 9f5b536032..834105a2e2 100644
--- a/rust/crates/test-support/src/lib.rs
+++ b/rust/crates/test-support/src/lib.rs
@@ -7,6 +7,7 @@ mod process;
mod server;
pub use museum::Museum;
+pub use object_store::ObjectStoreControl;
pub type TestResult<T = ()> = Result<T, Box<dyn std::error::Error>>;
diff --git a/rust/crates/test-support/src/museum.rs b/rust/crates/test-support/src/museum.rs
index c121c7687e..3983c1283e 100644
--- a/rust/crates/test-support/src/museum.rs
+++ b/rust/crates/test-support/src/museum.rs
@@ -96,6 +96,10 @@ impl Museum {
pub fn temp_dir(&self) -> &Path {
self.temp_dir.path()
}
+
+ pub fn object_store(&self) -> std::sync::Arc<crate::ObjectStoreControl> {
+ self._object_store.control.clone()
+ }
}
fn server_dir() -> TestResult<PathBuf> {
diff --git a/rust/crates/test-support/src/object_store.rs b/rust/crates/test-support/src/object_store.rs
index 2abb23e7c3..4034d405a5 100644
--- a/rust/crates/test-support/src/object_store.rs
+++ b/rust/crates/test-support/src/object_store.rs
@@ -1,41 +1,88 @@
use std::{
- collections::HashMap,
- io::{Read, Write},
+ fs::{self, File},
+ io::{self, BufRead, BufReader, Read, Write},
net::{TcpListener, TcpStream},
+ path::{Path, PathBuf},
sync::{
Arc,
- atomic::{AtomicBool, Ordering},
+ atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
},
thread::{self, JoinHandle},
+ time::Duration,
};
+use ente_core::crypto::hash;
+use uuid::Uuid;
+
use crate::{TestResult, net::LOCAL_HOST};
+#[derive(Default)]
+pub struct ObjectStoreControl {
+ reads: AtomicUsize,
+ active: AtomicUsize,
+ peak: AtomicUsize,
+ interrupted: AtomicUsize,
+ delay_millis: AtomicU64,
+}
+
+impl ObjectStoreControl {
+ pub fn reads(&self) -> usize {
+ self.reads.load(Ordering::SeqCst)
+ }
+ pub fn peak_reads(&self) -> usize {
+ self.peak.load(Ordering::SeqCst)
+ }
+ pub fn interrupt_reads(&self, count: usize) {
+ self.interrupted.store(count, Ordering::SeqCst);
+ }
+ pub fn delay_chunks(&self, delay: Duration) {
+ self.delay_millis
+ .store(delay.as_millis() as u64, Ordering::SeqCst);
+ }
+}
+
pub struct ObjectStore {
port: u16,
stop: Arc<AtomicBool>,
worker: Option<JoinHandle<()>>,
+ directory: PathBuf,
+ pub control: Arc<ObjectStoreControl>,
}
impl ObjectStore {
pub fn start() -> TestResult<Self> {
let listener = TcpListener::bind((LOCAL_HOST, 0))?;
let port = listener.local_addr()?.port();
+ let directory = std::env::temp_dir().join(format!("ente-objects-{}", Uuid::new_v4()));
+ fs::create_dir(&directory)?;
let stop = Arc::new(AtomicBool::new(false));
- let worker_stop = Arc::clone(&stop);
+ let worker_stop = stop.clone();
+ let root = directory.clone();
+ let control = Arc::new(ObjectStoreControl::default());
+ let worker_control = control.clone();
let worker = thread::spawn(move || {
- let mut objects = HashMap::new();
+ let mut requests: Vec<JoinHandle<()>> = Vec::new();
while let Ok((stream, _)) = listener.accept() {
if worker_stop.load(Ordering::Relaxed) {
break;
}
- handle(stream, &mut objects);
+ let root = root.clone();
+ let control = worker_control.clone();
+ requests.retain(|request| !request.is_finished());
+ requests.push(thread::spawn(move || {
+ let _ = handle(stream, &root, &control);
+ }));
+ }
+ for request in requests {
+ let _ = request.join();
}
});
Ok(Self {
port,
stop,
worker: Some(worker),
+ directory,
+ control,
})
}
@@ -51,77 +98,107 @@ impl Drop for ObjectStore {
if let Some(worker) = self.worker.take() {
let _ = worker.join();
}
+ let _ = fs::remove_dir_all(&self.directory);
}
}
-fn handle(mut stream: TcpStream, objects: &mut HashMap<String, Vec<u8>>) {
- let mut request = Vec::new();
- let mut buffer = [0; 4096];
- let header_end = loop {
- let Ok(read) = stream.read(&mut buffer) else {
- return;
- };
- if read == 0 {
- return;
+fn handle(stream: TcpStream, root: &Path, control: &ObjectStoreControl) -> TestResult {
+ stream.set_read_timeout(Some(Duration::from_secs(30)))?;
+ stream.set_write_timeout(Some(Duration::from_secs(30)))?;
+ let mut input = BufReader::new(stream);
+ let mut line = String::new();
+ input.read_line(&mut line)?;
+ let mut words = line.split_whitespace();
+ let method = words.next().ok_or("missing method")?.to_owned();
+ let target = words
+ .next()
+ .ok_or("missing target")?
+ .split('?')
+ .next()
+ .ok_or("missing path")?;
+ let name: String = hash::hash(target.as_bytes(), Some(32), None)?
+ .iter()
+ .map(|byte| format!("{byte:02x}"))
+ .collect();
+ let path = root.join(&name);
+ let mut length = 0u64;
+ loop {
+ line.clear();
+ if input.read_line(&mut line)? == 0 {
+ return Err("incomplete headers".into());
}
- request.extend_from_slice(&buffer[..read]);
- if let Some(end) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") {
- break end + 4;
+ if line == "\r\n" {
+ break;
}
- };
- let Ok(headers) = std::str::from_utf8(&request[..header_end]) else {
- return;
- };
- let mut lines = headers.lines();
- let Some((method, target)) = lines.next().and_then(|line| {
- let mut parts = line.split_whitespace();
- Some((parts.next()?.to_owned(), parts.next()?.to_owned()))
- }) else {
- return;
- };
- let content_length = lines
- .find_map(|line| {
- let (name, value) = line.split_once(':')?;
- if name.eq_ignore_ascii_case("content-length") {
- value.trim().parse().ok()
- } else {
- None
- }
- })
- .unwrap_or(0);
- while request.len() < header_end + content_length {
- let Ok(read) = stream.read(&mut buffer) else {
- return;
- };
- if read == 0 {
- return;
+ if let Some((name, value)) = line.split_once(':')
+ && name.eq_ignore_ascii_case("content-length")
+ {
+ length = value.trim().parse()?;
}
- request.extend_from_slice(&buffer[..read]);
}
- let path = target.split('?').next().unwrap_or(&target).to_owned();
match method.as_str() {
+ "OPTIONS" => respond(input.get_mut(), "204 No Content", 0)?,
"PUT" => {
- objects.insert(path, request[header_end..].to_vec());
- respond(&mut stream, "200 OK", 0);
+ let temporary = root.join(format!("{name}.{}.part", Uuid::new_v4()));
+ let mut file = File::create(&temporary)?;
+ let received = io::copy(&mut (&mut input).take(length), &mut file)?;
+ if received != length {
+ fs::remove_file(temporary)?;
+ return Err("incomplete upload".into());
+ }
+ file.sync_all()?;
+ drop(file);
+ fs::rename(temporary, path)?;
+ respond(input.get_mut(), "200 OK", 0)?;
}
- "HEAD" => match objects.get(&path) {
- Some(body) => respond(&mut stream, "200 OK", body.len()),
- None => respond(&mut stream, "404 Not Found", 0),
+ "HEAD" => match fs::metadata(path) {
+ Ok(meta) => respond(input.get_mut(), "200 OK", meta.len())?,
+ Err(_) => respond(input.get_mut(), "404 Not Found", 0)?,
},
- "GET" => match objects.get(&path) {
- Some(body) => {
- respond(&mut stream, "200 OK", body.len());
- let _ = stream.write_all(body);
+ "GET" => {
+ control.reads.fetch_add(1, Ordering::SeqCst);
+ let Ok(mut file) = File::open(path) else {
+ respond(input.get_mut(), "404 Not Found", 0)?;
+ return Ok(());
+ };
+ if control
+ .interrupted
+ .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| {
+ left.checked_sub(1)
+ })
+ .is_ok()
+ {
+ return Ok(());
}
- None => respond(&mut stream, "404 Not Found", 0),
- },
- _ => respond(&mut stream, "405 Method Not Allowed", 0),
+ let active = control.active.fetch_add(1, Ordering::SeqCst) + 1;
+ control.peak.fetch_max(active, Ordering::SeqCst);
+ let result = (|| {
+ respond(input.get_mut(), "200 OK", file.metadata()?.len())?;
+ let mut buffer = vec![0; 64 * 1024];
+ loop {
+ let read = file.read(&mut buffer)?;
+ if read == 0 {
+ break;
+ }
+ let delay = control.delay_millis.load(Ordering::SeqCst);
+ if delay > 0 {
+ thread::sleep(Duration::from_millis(delay));
+ }
+ input.get_mut().write_all(&buffer[..read])?;
+ }
+ Ok::<_, io::Error>(())
+ })();
+ control.active.fetch_sub(1, Ordering::SeqCst);
+ result?;
+ }
+ _ => respond(input.get_mut(), "405 Method Not Allowed", 0)?,
}
+ Ok(())
}
-fn respond(stream: &mut TcpStream, status: &str, content_length: usize) {
- let _ = write!(
+fn respond(stream: &mut TcpStream, status: &str, length: u64) -> io::Result<()> {
+ write!(
stream,
- "HTTP/1.1 {status}\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n"
- );
+ "HTTP/1.1 {status}\r\nContent-Length: {length}\r\nAccess-Control-Allow-Origin: *\r\nAccess-Control-Allow-Methods: GET, HEAD, PUT\r\nAccess-Control-Allow-Headers: Content-Type, Content-MD5, X-Client-Package, X-Client-Version\r\nConnection: close\r\n\r\n"
+ )
}