Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Improve Photos export scheduling and recovery

Manav Rathi43f90d62273e
diff --git a/rust/apps/cli-next/src/args.rs b/rust/apps/cli-next/src/args.rs
index 613f6e93d5..27b8d4cb52 100644
--- a/rust/apps/cli-next/src/args.rs
+++ b/rust/apps/cli-next/src/args.rs
@@ -1,4 +1,4 @@
-use std::path::PathBuf;
+use std::{num::NonZeroUsize, path::PathBuf};
use clap::{Args, Parser, Subcommand};
use serde::{Deserialize, Serialize};
@@ -144,8 +144,14 @@ pub struct ExportArgs {
help = "Reconnect an existing export after moving it or losing local state"
)]
pub adopt: bool,
- #[arg(long, help = "Attempt selected saved decryption failures once more")]
- pub retry_failed: bool,
+ #[arg(
+ long,
+ short = 'j',
+ value_name = "N",
+ default_value = "16",
+ help = "Maximum number of files processed concurrently"
+ )]
+ pub jobs: NonZeroUsize,
#[arg(value_name = "DESTINATION")]
pub destination: PathBuf,
}
diff --git a/rust/apps/cli-next/src/core_db.rs b/rust/apps/cli-next/src/core_db.rs
index 701e76136e..a5c8765d7a 100644
--- a/rust/apps/cli-next/src/core_db.rs
+++ b/rust/apps/cli-next/src/core_db.rs
@@ -7,7 +7,7 @@ pub use rusqlite::{
#[derive(Debug, thiserror::Error)]
pub enum Error {
- #[error("{0}")]
+ #[error(transparent)]
Sqlite(#[from] SqliteError),
#[error("database version {current} is newer than this build supports ({target})")]
Downgrade { current: i64, target: i64 },
diff --git a/rust/apps/cli-next/src/export/adopt.rs b/rust/apps/cli-next/src/export/adopt.rs
index 14013c37f2..c784ef680b 100644
--- a/rust/apps/cli-next/src/export/adopt.rs
+++ b/rust/apps/cli-next/src/export/adopt.rs
@@ -7,76 +7,51 @@ use uuid::Uuid;
use super::{
names,
- store::{Album, JsonRecord, Media, Placement, Properties, Store},
+ store::{Album, JsonRecord, Location, Placement, Properties, Store},
};
pub fn scan(root: &Path, store: &Store) -> Result<()> {
- let transaction = store.db.connection().unchecked_transaction()?;
- store.db.connection().execute_batch(
- "DELETE FROM placements; DELETE FROM albums; DELETE FROM json_records; DELETE FROM names; DELETE FROM allocations;",
- )?;
+ store
+ .db
+ .connection()
+ .execute_batch(
+ "DELETE FROM components; DELETE FROM json_records; DELETE FROM placements; DELETE FROM albums; DELETE FROM pending; DELETE FROM temporaries;",
+ )?;
for entry in fs::read_dir(root)? {
let entry = entry?;
- let name = entry
- .file_name()
- .into_string()
- .map_err(|_| anyhow::anyhow!("export contains a non-UTF-8 directory name"))?;
- if name == "Trash" {
+ if entry.file_name() == "Trash" {
ensure!(
entry.file_type()?.is_dir(),
- super::Conflict("at Trash: expected a directory".into())
+ super::Conflict("at Trash: expected directory".into())
);
for entry in fs::read_dir(entry.path())? {
let entry = entry?;
if entry.file_type()?.is_dir() {
- let name = entry
- .file_name()
- .into_string()
- .map_err(|_| anyhow::anyhow!("retained album name is not UTF-8"))?;
- album(root, store, &format!("Trash/{name}"), true)?;
+ album(root, store, &entry.path(), true)?;
}
}
} else if entry.file_type()?.is_dir() {
- album(root, store, &name, false)?;
- }
- }
- let mut query = store
- .db
- .connection()
- .prepare("SELECT record FROM placements WHERE json_extract(record,'$.complete')=0")?;
- let mut rows = query.query([])?;
- while let Some(row) = rows.next()? {
- let mut placement: Placement = serde_json::from_str(&row.get::<_, String>(0)?)?;
- let template = placement
- .metadata
- .iter()
- .find(|record| !record.is_null())
- .context("placement has no metadata")?
- .clone();
- for (media, record) in placement.media.iter().zip(&mut placement.metadata) {
- if record.is_null() {
- *record = template.clone();
- record["title"] = Value::String(media.component.path.clone());
- record["ente"]["component"] = serde_json::to_value(&media.component.role)?;
- }
+ album(root, store, &entry.path(), false)?;
}
- store.save_placement(&placement)?;
}
- drop(rows);
- drop(query);
- transaction.commit()?;
Ok(())
}
-fn album(root: &Path, store: &Store, folder: &str, retained: bool) -> Result<()> {
- let path = names::check(root, &format!("{folder}/metadata.json"))?;
- if !path.try_exists()? {
+fn album(root: &Path, store: &Store, directory: &Path, retained: bool) -> Result<()> {
+ let path = directory.join("metadata.json");
+ let Some(properties) = Properties::optional(&path)? else {
return Ok(());
- }
- let record = read_record(root, store, &format!("{folder}/metadata.json"))?;
+ };
+ let bytes = fs::read(&path)?;
+ let record: Value = serde_json::from_slice(&bytes)?;
let Some(identity) = record.get("ente") else {
return Ok(());
};
+ let folder = directory
+ .strip_prefix(root)?
+ .to_str()
+ .context("claimed Ente album path is not UTF-8")?;
+ names::check(root, &format!("{folder}/metadata.json"))?;
let id = parse_id(identity, "albumID")?;
let name = record["title"]
.as_str()
@@ -104,13 +79,15 @@ fn album(root: &Path, store: &Store, folder: &str, retained: bool) -> Result<()>
super::Conflict(format!("active album ID {id}"))
);
let album = Album {
- key: Uuid::new_v4().to_string(),
+ key: if retained {
+ Uuid::new_v4().to_string()
+ } else {
+ format!("active:{id}")
+ },
id,
+ retained,
name,
path: folder.into(),
- retained,
- metadata: record,
- initialized: true,
};
let (parent, name) = folder.rsplit_once('/').unwrap_or(("", folder));
ensure!(
@@ -118,10 +95,21 @@ fn album(root: &Path, store: &Store, folder: &str, retained: bool) -> Result<()>
"reserved active album path {folder}"
);
ensure!(
- names::reserve(store, parent, name, &album.key, "album")?,
+ names::available(store, parent, &[name.into()], &album.key, "album")?,
super::Conflict(format!("overlapping album paths at {folder}"))
);
store.save_album(&album)?;
+ store.save_json(&JsonRecord {
+ owner: album.key.clone(),
+ role: None,
+ location: Some(Location {
+ folder: album.key.clone(),
+ name: "metadata.json".into(),
+ }),
+ value: record,
+ hash: Some(digest(&bytes)?),
+ properties: Some(properties),
+ })?;
let metadata = names::check(root, &format!("{folder}/metadata"))?;
if !metadata.try_exists()? {
return Ok(());
@@ -135,22 +123,24 @@ fn album(root: &Path, store: &Store, folder: &str, retained: bool) -> Result<()>
);
for entry in fs::read_dir(metadata)? {
let entry = entry?;
- if entry.path().extension().is_none_or(|ext| ext != "json") {
+ if entry
+ .path()
+ .extension()
+ .is_none_or(|extension| extension != "json")
+ {
continue;
}
- let name = entry
- .file_name()
- .into_string()
- .map_err(|_| anyhow::anyhow!("sidecar name is not UTF-8"))?;
- let path = names::check(root, &format!("{folder}/metadata/{name}"))?;
- ensure!(
- entry.file_type()?.is_file(),
- super::Conflict(format!("at {}: expected a sidecar", path.display()))
- );
- let record = read_record(root, store, &format!("{folder}/metadata/{name}"))?;
+ let properties = Properties::read(&entry.path())?;
+ let bytes = fs::read(entry.path())?;
+ let record: Value = serde_json::from_slice(&bytes)?;
let Some(identity) = record.get("ente") else {
continue;
};
+ let name = entry
+ .file_name()
+ .into_string()
+ .map_err(|_| anyhow::anyhow!("claimed Ente sidecar name is not UTF-8"))?;
+ names::check(root, &format!("{folder}/metadata/{name}"))?;
let file = parse_id(identity, "fileID")?;
let display_name = identity["name"]
.as_str()
@@ -210,20 +200,27 @@ fn album(root: &Path, store: &Store, folder: &str, retained: bool) -> Result<()>
&& name == format!("{}.json", components[index].path),
"contradictory sidecar location for file {file}"
);
+ let kind = if kind == "livePhoto" {
+ "livephoto"
+ } else {
+ kind
+ };
let existing = if retained {
- let mut query = store.db.connection().prepare(
- "SELECT record FROM placements WHERE album=?1 AND file=?2 AND retained=1",
- )?;
+ let mut query = store
+ .db
+ .connection()
+ .prepare(
+ "SELECT DISTINCT p.id,p.album,p.file,p.retained,p.name,p.kind FROM placements p JOIN components c ON c.placement=p.id WHERE p.album=?1 AND p.file=?2 AND p.retained=1 AND c.folder=?3",
+ )?;
+ let mut rows = query.query(rusqlite::params![id, file, album.key])?;
let mut found = None;
- let mut rows = query.query(rusqlite::params![id, file])?;
while let Some(row) = rows.next()? {
- let candidate: Placement = serde_json::from_str(&row.get::<_, String>(0)?)?;
- if candidate.folder == folder
- && candidate
- .media
- .iter()
- .map(|m| &m.component)
- .eq(components.iter())
+ let candidate = Placement::read(row)?;
+ if store
+ .components(&candidate.id)?
+ .iter()
+ .map(super::store::Component::portable)
+ .eq(components.iter().cloned())
{
found = Some(candidate);
break;
@@ -233,94 +230,94 @@ fn album(root: &Path, store: &Store, folder: &str, retained: bool) -> Result<()>
} else {
store.placement(id, file)?
};
- let mut placement = existing.unwrap_or_else(|| Placement {
+ let fresh = existing.is_none();
+ let placement = existing.unwrap_or_else(|| Placement {
id: Uuid::new_v4().to_string(),
album: id,
- folder: folder.into(),
file,
retained,
name: display_name.into(),
- original: String::new(),
- media: components
- .iter()
- .cloned()
- .map(|component| Media {
- component,
- properties: None,
- intended_time: None,
- })
- .collect(),
- metadata: vec![Value::Null; components.len()],
- complete: false,
- retaining: false,
+ kind: kind.into(),
});
- let owner = if retained {
- placement.id.clone()
- } else {
- format!("file:{id}:{file}")
- };
- for component in &components {
+ if !fresh {
ensure!(
- names::reserve(store, folder, &component.path, &owner, kind)?,
- super::Conflict(format!("overlapping media paths in {folder}"))
+ placement.kind == kind
+ && store
+ .components(&placement.id)?
+ .iter()
+ .all(|component| component.location.folder == album.key)
+ && store
+ .components(&placement.id)?
+ .iter()
+ .map(super::store::Component::portable)
+ .eq(components.iter().cloned()),
+ super::Conflict(format!("active placement for file {file} in album {id}"))
);
}
ensure!(
- placement.folder == folder
- && placement
- .media
- .iter()
- .map(|m| &m.component)
- .eq(components.iter()),
- super::Conflict(format!("active placement for file {file} in album {id}"))
+ !store
+ .json_record(&placement.id, Some(&own))?
+ .is_some_and(|record| record.location.is_some()),
+ "duplicate component identity for file {file}"
);
+ let media_names: Vec<_> = components
+ .iter()
+ .map(|component| component.path.clone())
+ .collect();
ensure!(
- placement.metadata[index].is_null(),
- "duplicate component identity for file {file}"
+ names::available(store, &album.key, &media_names, &placement.id, kind)?,
+ super::Conflict(format!("overlapping media paths in {folder}"))
);
- if let Some(other) = placement.metadata.iter().find(|value| !value.is_null()) {
- let mut a = other.clone();
- let mut b = record.clone();
- a.as_object_mut()
- .context("invalid sidecar")?
- .remove("title");
- b.as_object_mut()
- .context("invalid sidecar")?
- .remove("title");
- a["ente"]
- .as_object_mut()
- .context("invalid Ente record")?
- .remove("component");
- b["ente"]
- .as_object_mut()
- .context("invalid Ente record")?
- .remove("component");
- a["ente"]["components"] = serde_json::to_value(&components)?;
- b["ente"]["components"] = a["ente"]["components"].clone();
- ensure!(a == b, "contradictory Live Photo metadata for file {file}");
- }
- placement.metadata[index] = record;
- placement.complete = placement.metadata.iter().all(|record| !record.is_null());
store.save_placement(&placement)?;
+ if fresh {
+ for component in &components {
+ store.save_component(&super::store::Component {
+ placement: placement.id.clone(),
+ role: component.role.clone(),
+ location: Location {
+ folder: album.key.clone(),
+ name: component.path.clone(),
+ },
+ size: component.size,
+ hash: component.hash.clone(),
+ signature: None,
+ properties: None,
+ intended_time: None,
+ })?;
+ let mut fallback = record.clone();
+ fallback["title"] = Value::String(component.path.clone());
+ fallback["ente"]["component"] = serde_json::to_value(&component.role)?;
+ store.save_json(&JsonRecord {
+ owner: placement.id.clone(),
+ role: Some(component.role.clone()),
+ location: None,
+ value: fallback,
+ hash: None,
+ properties: None,
+ })?;
+ }
+ }
+ store.save_json(&JsonRecord {
+ owner: placement.id,
+ role: Some(own),
+ location: Some(Location {
+ folder: album.key.clone(),
+ name: format!("metadata/{name}"),
+ }),
+ value: record,
+ hash: Some(digest(&bytes)?),
+ properties: Some(properties),
+ })?;
}
Ok(())
}
-fn read_record(root: &Path, store: &Store, relative: &str) -> Result<Value> {
- let path = names::check(root, relative)?;
- let bytes = fs::read(&path)?;
- let record: Value = serde_json::from_slice(&bytes)?;
- if record.get("ente").is_some() {
- let cached = JsonRecord {
- hash: ente_core::b64::encode(&ente_core::crypto::hash::hash(&bytes, Some(64), None)?),
- properties: Properties::read(&path)?,
- };
- store.db.connection().execute(
- "INSERT INTO json_records VALUES(?1,?2)",
- rusqlite::params![relative, serde_json::to_string(&cached)?],
- )?;
- }
- Ok(record)
+fn digest(bytes: &[u8]) -> Result<String> {
+ Ok(ente_core::b64::encode(&ente_core::crypto::hash::hash(
+ bytes,
+ Some(64),
+ None,
+ )?))
}
fn parse_id(value: &Value, field: &str) -> Result<i64> {
diff --git a/rust/apps/cli-next/src/export/allocation.rs b/rust/apps/cli-next/src/export/allocation.rs
deleted file mode 100644
index a07125eb1f..0000000000
--- a/rust/apps/cli-next/src/export/allocation.rs
+++ /dev/null
@@ -1,156 +0,0 @@
-use anyhow::{Result, ensure};
-use rusqlite::params;
-use serde::{Deserialize, Serialize};
-
-use super::{
- names,
- store::{Album, Placement, Store},
-};
-
-#[derive(Serialize, Deserialize)]
-struct Allocation {
- parent: String,
- name: String,
- kind: String,
- extensions: Option<(String, String)>,
- paths: Vec<String>,
-}
-
-pub fn seed(store: &Store) -> Result<()> {
- let transaction = store.db.connection().unchecked_transaction()?;
- names::reserve(store, "", "export.json", "root", "album")?;
- names::reserve(store, "", "Trash", "root", "album")?;
- let mut after = String::new();
- loop {
- let album: Option<Album> = store.json(
- "SELECT record FROM albums WHERE key>?1 ORDER BY key LIMIT 1",
- [&after],
- )?;
- let Some(album) = album else { break };
- after = album.key.clone();
- let (parent, name) = album.path.rsplit_once('/').unwrap_or(("", &album.path));
- ensure!(
- names::reserve(store, parent, name, &album.key, "album")?,
- super::Conflict(format!("overlapping album paths at {}", album.path))
- );
- }
- after.clear();
- loop {
- let placement: Option<Placement> = store.json(
- "SELECT record FROM placements WHERE id>?1 ORDER BY id LIMIT 1",
- [&after],
- )?;
- let Some(placement) = placement else { break };
- after = placement.id.clone();
- let owner = if placement.retained {
- placement.id.clone()
- } else {
- format!("file:{}:{}", placement.album, placement.file)
- };
- for media in &placement.media {
- ensure!(
- names::reserve(
- store,
- &placement.folder,
- &media.component.path,
- &owner,
- placement.kind()?
- )?,
- super::Conflict(format!("overlapping media in {}", placement.folder))
- );
- }
- }
- store.db.connection().execute("DELETE FROM allocations WHERE EXISTS(SELECT 1 FROM desired_albums a WHERE a.selected=1 AND a.ready=1 AND a.failure IS NULL AND allocations.owner GLOB 'file:'||a.id||':*' AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE allocations.owner='file:'||f.album||':'||f.file))", [])?;
- store.db.connection().execute("DELETE FROM allocations WHERE owner IN (SELECT coalesce(b.key,'active:'||a.id) FROM desired_albums a LEFT JOIN albums b ON b.id=a.id AND b.retained=0 WHERE a.selected=1 AND a.ready=1 AND a.failure IS NULL AND a.record IS NULL)", [])?;
- after.clear();
- loop {
- let next: Option<(String, String)> = store
- .db
- .connection()
- .query_row(
- "SELECT owner,record FROM allocations WHERE owner>?1 ORDER BY owner LIMIT 1",
- [&after],
- |r| Ok((r.get(0)?, r.get(1)?)),
- )
- .optional()?;
- let Some((owner, record)) = next else { break };
- after = owner.clone();
- let allocation: Allocation = serde_json::from_str(&record)?;
- for path in &allocation.paths {
- ensure!(
- names::reserve(store, &allocation.parent, path, &owner, &allocation.kind)?,
- super::Conflict(format!(
- "overlapping pending names in {}",
- allocation.parent
- ))
- );
- }
- }
- transaction.commit()?;
- Ok(())
-}
-
-pub fn choose(
- store: &Store,
- parent: &str,
- name: &str,
- kind: &str,
- extensions: Option<(&str, &str)>,
- owner: &str,
-) -> Result<Vec<String>> {
- let previous: Option<Allocation> =
- store.json("SELECT record FROM allocations WHERE owner=?1", [owner])?;
- let extensions =
- extensions.map(|(image, video)| (names::portable(image), names::portable(video)));
- if let Some(previous) = previous
- && previous.parent == parent
- && previous.name == name
- && previous.kind == kind
- && previous.extensions == extensions
- {
- return Ok(previous.paths);
- }
- let transaction = rusqlite::Transaction::new_unchecked(
- store.db.connection(),
- rusqlite::TransactionBehavior::Immediate,
- )?;
- let paths = names::allocate(
- store,
- parent,
- name,
- kind,
- extensions
- .as_ref()
- .map(|(image, video)| (image.as_str(), video.as_str())),
- owner,
- )?;
- let allocation = Allocation {
- parent: parent.into(),
- name: name.into(),
- kind: kind.into(),
- extensions,
- paths: paths.clone(),
- };
- store.db.connection().execute("INSERT INTO allocations VALUES(?1,?2) ON CONFLICT(owner) DO UPDATE SET record=excluded.record", params![owner,serde_json::to_string(&allocation)?])?;
- transaction.commit()?;
- Ok(paths)
-}
-
-pub fn clear(store: &Store, owner: &str) -> Result<()> {
- store
- .db
- .connection()
- .execute("DELETE FROM allocations WHERE owner=?1", [owner])?;
- Ok(())
-}
-
-pub fn relocate(store: &Store, old: &str, new: &str) -> Result<()> {
- store.db.connection().execute(
- "UPDATE names SET parent=?1 WHERE parent=?2",
- params![new, old],
- )?;
- store.db.connection().execute("UPDATE allocations SET record=json_set(record,'$.parent',?1) WHERE json_extract(record,'$.parent')=?2", params![new,old])?;
- Ok(())
-}
-
-use rusqlite::OptionalExtension;
diff --git a/rust/apps/cli-next/src/export/fs.rs b/rust/apps/cli-next/src/export/fs.rs
index d5ed27d212..fc438df966 100644
--- a/rust/apps/cli-next/src/export/fs.rs
+++ b/rust/apps/cli-next/src/export/fs.rs
@@ -1,5 +1,5 @@
use std::{
- fs::{self, File, OpenOptions},
+ fs::{self, File},
io::{self, Read, Write},
path::Path,
time::{Duration, UNIX_EPOCH},
@@ -10,15 +10,10 @@ use ente_core::{
b64,
crypto::hash::{self, HashState},
};
-use rusqlite::params;
use serde_json::Value;
use tokio::sync::watch;
-use uuid::Uuid;
-use super::{
- names,
- store::{JsonRecord, Operation, Properties, Store, Temporary},
-};
+use super::{names, store::Properties};
pub struct HashWriter {
pub file: File,
@@ -84,170 +79,27 @@ pub fn directory(root: &Path, relative: &str) -> Result<()> {
meta.is_dir(),
super::Conflict(format!("expected directory at {}", path.display()))
),
- Err(error) if error.kind() == io::ErrorKind::NotFound => fs::create_dir_all(&path)?,
+ Err(error) if error.kind() == io::ErrorKind::NotFound => create_directory(&path)?,
Err(error) => return Err(error.into()),
}
Ok(())
}
-pub fn temporary(
- root: &Path,
- store: &Store,
- directory: &str,
- file_id: Option<i64>,
-) -> Result<(Temporary, File)> {
- self::directory(root, directory)?;
- let path = format!("{directory}/.ente-{}.part", Uuid::new_v4().simple());
- let album:i64=store.db.connection().query_row("SELECT id FROM albums WHERE ?1=path OR substr(?1,1,length(path)+1)=path||'/' ORDER BY length(path) DESC LIMIT 1",[directory],|r|r.get(0))?;
- let temporary = Temporary {
- album,
- path,
- file: file_id,
- original: None,
- role: None,
- extension: None,
- destination: None,
- hash: None,
- size: None,
- };
- store.temporary(&temporary)?;
- let file = OpenOptions::new()
- .write(true)
- .read(true)
- .create_new(true)
- .open(names::check(root, &temporary.path)?);
- match file {
- Ok(file) => Ok((temporary, file)),
- Err(error) => {
- store
- .db
- .connection()
- .execute("DELETE FROM temporaries WHERE path=?1", [&temporary.path])?;
- Err(error.into())
+pub fn create_directory(path: &Path) -> Result<()> {
+ let mut missing = Vec::new();
+ for directory in path.ancestors() {
+ if directory.try_exists()? {
+ break;
}
+ missing.push(directory);
}
-}
-
-pub fn publish(
- root: &Path,
- store: &Store,
- mut temporary: Temporary,
- destination: &str,
- size: u64,
- hash: String,
-) -> Result<()> {
- temporary.destination = Some(destination.into());
- temporary.size = Some(size);
- temporary.hash = Some(hash);
- store.temporary(&temporary)?;
- finish_publication(root, store, &temporary, None)
-}
-
-fn finish_publication(
- root: &Path,
- store: &Store,
- temporary: &Temporary,
- verify: Option<&watch::Receiver<bool>>,
-) -> Result<()> {
- let source = names::check(root, &temporary.path)?;
- let destination = names::check(
- root,
- temporary
- .destination
- .as_deref()
- .context("publication has no destination")?,
- )?;
- match fs::symlink_metadata(&source) {
- Ok(metadata) => {
- ensure!(
- metadata.is_file(),
- super::Conflict(format!("at {}: expected temporary file", source.display()))
- );
- if let Some(cancel) = verify {
- let (size, hash) = hash_cancellable(&source, cancel)?;
- if Some(size) != temporary.size || Some(&hash) != temporary.hash.as_ref() {
- return discard(root, store, &temporary.path);
- }
- }
- if let Ok(meta) = fs::symlink_metadata(&destination) {
- ensure!(
- meta.is_file(),
- super::Conflict(format!(
- "at {}: expected a regular file",
- destination.display()
- ))
- );
- }
- fs::rename(&source, &destination)?;
- sync_parent(&destination)?;
- }
- Err(error) if error.kind() == io::ErrorKind::NotFound => {
- if destination.try_exists()? {
- Properties::read(&destination)?;
- }
- }
- Err(error) => return Err(error.into()),
+ fs::create_dir_all(path)?;
+ for directory in missing.into_iter().rev() {
+ sync_parent(directory)?;
}
- store
- .db
- .connection()
- .execute("DELETE FROM temporaries WHERE path=?1", [&temporary.path])?;
Ok(())
}
-pub fn json(
- root: &Path,
- store: &Store,
- path: &str,
- value: &Value,
- associated: bool,
- file_id: Option<i64>,
-) -> Result<bool> {
- let mut bytes = serde_json::to_vec_pretty(value)?;
- bytes.push(b'\n');
- let hash = b64::encode(&hash::hash(&bytes, Some(64), None)?);
- let destination = names::check(root, path)?;
- let recorded: Option<JsonRecord> =
- store.json("SELECT record FROM json_records WHERE path=?1", [path])?;
- let properties = match fs::symlink_metadata(&destination) {
- Ok(_) => Some(Properties::read(&destination)?),
- Err(error) if error.kind() == io::ErrorKind::NotFound => None,
- Err(error) => return Err(error.into()),
- };
- ensure!(
- associated || properties.is_none(),
- super::Conflict(format!("at {}: unrelated JSON", destination.display()))
- );
- if let (Some(recorded), Some(properties)) = (&recorded, &properties)
- && recorded.hash == hash
- && recorded.properties == *properties
- {
- return Ok(false);
- }
- let parent = path
- .rsplit_once('/')
- .context("JSON has no album directory")?
- .0;
- let (temporary, mut file) = self::temporary(root, store, parent, file_id)?;
- file.write_all(&bytes)?;
- file.sync_all()?;
- drop(file);
- publish(
- root,
- store,
- temporary,
- path,
- bytes.len() as u64,
- hash.clone(),
- )?;
- let record = JsonRecord {
- hash,
- properties: Properties::read(&destination)?,
- };
- store.db.connection().execute("INSERT INTO json_records VALUES(?1,?2) ON CONFLICT(path) DO UPDATE SET record=excluded.record",params![path,serde_json::to_string(&record)?])?;
- Ok(true)
-}
-
pub fn set_time(path: &Path, micros: i64) -> Result<Properties> {
let duration = Duration::from_micros(micros.unsigned_abs());
let time = if micros >= 0 {
@@ -260,122 +112,52 @@ pub fn set_time(path: &Path, micros: i64) -> Result<Properties> {
Properties::read(path)
}
-pub fn operations(root: &Path, store: &Store, album: i64, file: Option<i64>) -> Result<()> {
- loop {
- let operation:Option<(i64,String)>=store.db.connection().query_row("SELECT id,record FROM operations WHERE album=?1 AND file IS ?2 ORDER BY id LIMIT 1",params![album,file],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
- let Some((id, json)) = operation else { break };
- match serde_json::from_str::<Operation>(&json)? {
- Operation::Move {
- source,
- destination,
- ..
- } => {
- let source = names::check(root, &source)?;
- let destination = names::check(root, &destination)?;
- if source.try_exists()? {
- ensure!(
- !destination.try_exists()? || names::same_path(&source, &destination),
- super::Conflict(format!(
- "at {}: occupied rename target",
- destination.display()
- ))
- );
- if let Some(parent) = destination.parent() {
- fs::create_dir_all(parent)?;
- }
- fs::rename(&source, &destination)?;
- sync_parent(&destination)?;
- }
- }
- Operation::Remove { path, .. } => {
- let path = names::check(root, &path)?;
- match fs::remove_file(&path) {
- Ok(()) => {}
- Err(e) if e.kind() == io::ErrorKind::NotFound => {}
- Err(e) => return Err(e.into()),
- }
- }
- }
- store
- .db
- .connection()
- .execute("DELETE FROM operations WHERE id=?1", [id])?;
+pub fn sync_parent(path: &Path) -> Result<()> {
+ #[cfg(unix)]
+ if let Some(parent) = path.parent() {
+ File::open(parent)?.sync_all()?;
}
Ok(())
}
-pub fn recover(root: &Path, store: &Store, cancel: &watch::Receiver<bool>) -> Result<()> {
- let mut query=store.db.connection().prepare("SELECT DISTINCT o.album,o.file FROM operations o JOIN desired_albums a ON a.id=o.album WHERE a.selected=1 AND a.ready=1 AND a.failure IS NULL AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=o.album AND f.file=o.file AND f.failure IS NOT NULL) ORDER BY o.album,o.file")?;
- let mut rows = query.query([])?;
- while let Some(row) = rows.next()? {
- ensure!(!*cancel.borrow(), super::Cancelled);
- let album = row.get(0)?;
- let file = row.get(1)?;
- if let Err(error) = operations(root, store, album, file) {
- block(store, album, file, error)?;
- }
+pub fn sync_move(source: &Path, destination: &Path) -> Result<()> {
+ if destination.parent().is_some_and(Path::exists) {
+ sync_parent(destination)?;
}
- let mut after = String::new();
- loop {
- ensure!(!*cancel.borrow(), super::Cancelled);
- let temporary:Option<Temporary>=store.json("SELECT record FROM temporaries WHERE path>?1 AND (json_extract(record,'$.destination') IS NOT NULL OR json_extract(record,'$.hash') IS NULL) AND json_extract(record,'$.album') IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=json_extract(temporaries.record,'$.album') AND f.file=json_extract(temporaries.record,'$.file') AND f.failure IS NOT NULL) ORDER BY path LIMIT 1",[&after])?;
- let Some(temporary) = temporary else { break };
- after = temporary.path.clone();
- let result = if temporary.destination.is_some() {
- finish_publication(root, store, &temporary, Some(cancel))
- } else {
- discard(root, store, &temporary.path)
- };
- if let Err(error) = result {
- block(store, temporary.album, temporary.file, error)?;
- }
+ if source.parent() != destination.parent() && source.parent().is_some_and(Path::exists) {
+ sync_parent(source)?;
}
Ok(())
}
-fn block(store: &Store, album: i64, file: Option<i64>, error: anyhow::Error) -> Result<()> {
- let unit = if let Some(file) = file {
- format!("file:{album}:{file}")
- } else {
- format!("album:{album}")
- };
- super::report(store, &unit, &error)?;
- if file.is_none() {
- store.db.connection().execute(
- "UPDATE desired_albums SET failure=?1 WHERE id=?2",
- params![error.to_string(), album],
- )?;
- }
- store.db.connection().execute(
- "UPDATE desired_files SET failure=?1 WHERE album=?2 AND (?3 IS NULL OR file=?3)",
- params![error.to_string(), album, file],
- )?;
- if super::fatal(&error) {
- return Err(error);
- }
- Ok(())
+pub struct Json {
+ pub value: Value,
+ pub bytes: Vec<u8>,
+ pub hash: String,
}
-pub fn discard(root: &Path, store: &Store, path: &str) -> Result<()> {
- let absolute = names::check(root, path)?;
- match fs::remove_file(absolute) {
- Ok(()) => {}
- Err(e) if e.kind() == io::ErrorKind::NotFound => {}
- Err(e) => return Err(e.into()),
+impl Json {
+ pub fn new(value: Value) -> Result<Self> {
+ let mut bytes = serde_json::to_vec_pretty(&value)?;
+ bytes.push(b'\n');
+ let hash = b64::encode(&hash::hash(&bytes, Some(64), None)?);
+ Ok(Self { value, bytes, hash })
}
- store
- .db
- .connection()
- .execute("DELETE FROM temporaries WHERE path=?1", [path])?;
- Ok(())
}
-pub fn sync_parent(path: &Path) -> Result<()> {
- #[cfg(unix)]
- if let Some(parent) = path.parent() {
- File::open(parent)?.sync_all()?;
+pub fn matches(path: &Path, size: u64, hash: &str, cancel: &watch::Receiver<bool>) -> Result<bool> {
+ let actual = hash_cancellable(path, cancel)?;
+ Ok(actual.0 == size && actual.1 == hash)
+}
+
+pub fn remove(path: &Path) -> Result<()> {
+ match fs::remove_file(path) {
+ Ok(()) => {}
+ Err(error) if error.kind() == io::ErrorKind::NotFound => {}
+ Err(error) => return Err(error.into()),
+ }
+ if path.parent().is_some_and(Path::exists) {
+ sync_parent(path)?;
}
Ok(())
}
-
-use rusqlite::OptionalExtension;
diff --git a/rust/apps/cli-next/src/export/mod.rs b/rust/apps/cli-next/src/export/mod.rs
index c9be2cb10e..400c647303 100644
--- a/rust/apps/cli-next/src/export/mod.rs
+++ b/rust/apps/cli-next/src/export/mod.rs
@@ -1,5 +1,4 @@
mod adopt;
-mod allocation;
mod fs;
mod names;
mod reconcile;
@@ -10,8 +9,8 @@ use std::{
fs::{File, OpenOptions},
io::Write,
path::{Path, PathBuf},
- sync::Arc,
- time::Instant,
+ sync::Mutex,
+ time::{Duration, Instant},
};
use anyhow::{Context, Result, bail, ensure};
@@ -26,7 +25,7 @@ use crate::{
args::{ExportArgs, Options, Product},
db, home, output,
replica::{AlbumRecord, FileRecord, Replica, canonical_id},
- vault::{DbKey, State},
+ vault::State,
};
use store::Store;
@@ -43,6 +42,16 @@ pub async fn run(args: ExportArgs, selected: Option<&str>, options: &Options) ->
!options.offline,
"export requires network access; remove --offline"
);
+ let selected = selected.map(str::to_owned);
+ let as_json = options.json;
+ let runtime = tokio::runtime::Handle::current();
+ tokio::task::spawn_blocking(move || {
+ runtime.block_on(run_locked(args, selected.as_deref(), as_json))
+ })
+ .await?
+}
+
+async fn run_locked(args: ExportArgs, selected: Option<&str>, as_json: bool) -> Result<()> {
let destination = destination(&args.destination)?;
let mut state = State::load()?;
let index = state.resolve(selected)?;
@@ -114,8 +123,27 @@ pub async fn run(args: ExportArgs, selected: Option<&str>, options: &Options) ->
)?;
}
}
- store.db.connection().execute("INSERT OR IGNORE INTO desired_albums(id,name) SELECT id,name FROM albums ORDER BY retained",[])?;
- store.db.connection().execute("INSERT OR IGNORE INTO desired_albums(id,name) SELECT CAST(substr(owner,8) AS INTEGER),json_extract(record,'$.name') FROM allocations WHERE owner GLOB 'active:*'",[])?;
+ store
+ .db
+ .connection()
+ .execute(
+ "INSERT OR IGNORE INTO desired_albums(id,name) SELECT s.id,json_extract(s.record,'$.name') FROM album_sources s WHERE EXISTS(SELECT 1 FROM albums a WHERE a.id=s.id)",
+ [],
+ )?;
+ store
+ .db
+ .connection()
+ .execute(
+ "INSERT OR IGNORE INTO desired_albums(id,name) SELECT id,name FROM albums ORDER BY retained",
+ [],
+ )?;
+ store
+ .db
+ .connection()
+ .execute(
+ "INSERT OR IGNORE INTO desired_albums(id,name) SELECT album,name FROM pending WHERE file IS NULL AND retained=0",
+ [],
+ )?;
select(&store, &args)?;
if root_lock.is_none() {
root_lock = Some(initialize(&destination, &expected_root)?);
@@ -123,46 +151,54 @@ pub async fn run(args: ExportArgs, selected: Option<&str>, options: &Options) ->
store
.db
.connection()
- .execute("UPDATE export SET bound=1 WHERE id=1", [])?;
+ .execute("UPDATE export SET bound=1 WHERE id=1 AND bound=0", [])?;
let (cancel_send, mut cancel) = watch::channel(false);
+ let signal_send = cancel_send.clone();
let signal = tokio::spawn(async move {
if tokio::signal::ctrl_c().await.is_ok() {
- cancel_send.send_replace(true);
+ signal_send.send_replace(true);
}
});
let refresh_cancel = cancel.clone();
let refreshed = tokio::select! {
- result=refresh(&store,&mut replica,&session,args.retry_failed,&refresh_cancel)=>result,
+ result=refresh(&store,&mut replica,&session,&refresh_cancel)=>result,
_=cancel.changed()=>Err(Cancelled.into()),
};
drop(replica);
drop(source_db);
drop(account_home);
- let context = Arc::new(transfer::Context {
- root: destination.clone(),
- db_path,
- db_key: DbKey(account.db_key.0),
- session,
- });
- let result = async {
+ let shared = Mutex::new(store);
+ let run = reconcile::Run {
+ root: &destination,
+ store: &shared,
+ cancel: &cancel,
+ };
+ let result = (|| {
refreshed?;
- fs::recover(&destination, &store, &cancel)?;
- allocation::seed(&store)?;
- reconcile::albums(&destination, &store, context.session.user_id)?;
- transfers(context.clone(), &store, cancel.clone()).await?;
- if *cancel.borrow() {
- bail!(Cancelled)
- }
- reconcile::retain_removed(&destination, &store)?;
- Ok::<(), anyhow::Error>(())
- }
- .await;
+ run.recover()?;
+ run.albums()?;
+ transfers(
+ &destination,
+ &shared,
+ &session,
+ &cancel_send,
+ &cancel,
+ args.jobs.get(),
+ )?;
+ run.check_cancel()?;
+ run.retain_removed()
+ })();
signal.abort();
if let Err(error) = result {
- report(&store, "run", &error)?;
+ run.report("run", &error)?;
+ }
+ if let Err(error) = run.cleanup(None) {
+ run.report("run", &error)?;
}
- cleanup(&destination, &store)?;
- let complete = summary(&store, &destination, options.json)?;
+ let store = shared
+ .into_inner()
+ .map_err(|_| anyhow::anyhow!("export database mutex poisoned"))?;
+ let complete = summary(&store, &destination, as_json)?;
drop(root_lock);
ensure!(complete, "export is incomplete");
Ok(())
@@ -231,7 +267,7 @@ fn open_root(destination: &Path, expected: &Root) -> Result<Option<File>> {
}
fn initialize(destination: &Path, root: &Root) -> Result<File> {
- std::fs::create_dir_all(destination)?;
+ fs::create_directory(destination)?;
let path = names::check(destination, "export.json")?;
let mut file = OpenOptions::new()
.read(true)
@@ -296,244 +332,374 @@ async fn refresh(
store: &Store,
replica: &mut Replica<'_>,
session: &ente_core::Session,
- retry: bool,
cancel: &watch::Receiver<bool>,
) -> Result<()> {
let need_favorites: bool = store.db.connection().query_row(
- "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND record IS NOT NULL)",
+ "SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND present=1)",
[],
|r| r.get(0),
)?;
let mut favorites_known = true;
let mut after = 0;
+ eprintln!("Refreshing selected albums and Favorites.");
loop {
- let next:Option<(i64,String,bool)>=store.db.connection().query_row("SELECT id,record,selected FROM desired_albums WHERE id>?1 AND record IS NOT NULL ORDER BY id LIMIT 1",[after],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?;
- let Some((id, record, selected)) = next else {
+ ensure!(!*cancel.borrow(), Cancelled);
+ let next: Option<(i64, bool, bool)> = store
+ .db
+ .connection()
+ .query_row(
+ "SELECT id,selected,favorites FROM desired_albums WHERE present=1 AND (selected=1 OR (favorites=1 AND ?1)) AND id>?2 ORDER BY id LIMIT 1",
+ params![need_favorites, after],
+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
+ )
+ .optional()?;
+ let Some((id, selected, favorites)) = next else {
break;
};
after = id;
- let record: AlbumRecord = serde_json::from_str(&record)?;
- if !need_favorites
- || record.remote.kind != "favorites"
- || record.remote.owner.id != session.user_id
- {
- continue;
- }
- if *cancel.borrow() {
- bail!(Cancelled)
- }
- replica.retry_album(session, id, retry)?;
let result = async {
- let record = replica
- .album_record(id)?
- .context("missing favorites source")?;
+ replica.retry_album(session, id)?;
+ let record = replica.album_record(id)?.context("missing album source")?;
let album = record.album(session.user_id)?;
- replica.sync_album_files(session, &album, retry).await?;
+ replica.sync_album_files(session, &album).await?;
store.desired_album(&record, session.user_id)?;
+ if selected {
+ store.snapshot_album(&album)?;
+ }
+ store
+ .db
+ .connection()
+ .execute("UPDATE desired_albums SET ready=1 WHERE id=?1", [id])?;
Ok::<_, anyhow::Error>(())
}
.await;
if let Err(error) = result {
- favorites_known = false;
+ if favorites {
+ favorites_known = false;
+ }
+ store.db.connection().execute(
+ "UPDATE desired_albums SET failure=?1 WHERE id=?2",
+ params![error.to_string(), id],
+ )?;
if selected {
- report(store, &format!("album:{id}"), &error)?;
+ record_outcome(store, &format!("album:{id}"), &error)?;
+ eprintln!("album:{id}: {error:#}");
}
if fatal(&error) {
return Err(error);
}
}
}
- let selected: i64 = store.db.connection().query_row(
- "SELECT count(*) FROM desired_albums WHERE selected=1",
+ store.db.connection().execute(
+ "UPDATE desired_albums SET ready=1 WHERE selected=1 AND present=0",
[],
- |r| r.get(0),
)?;
- eprintln!("Refreshing files in {selected} selected albums.");
- let mut progress = Instant::now();
- let mut refreshed = 0;
+ if need_favorites && favorites_known {
+ let mut after = 0;
+ loop {
+ let ids: Vec<i64> = replica.db.read(|db| {
+ let mut query = db.prepare(
+ "SELECT DISTINCT f.id FROM photos_files f JOIN photos_collections c ON c.id=f.collection_id WHERE f.id>?1 AND json_extract(c.record,'$.remote.type')='favorites' AND json_extract(c.record,'$.remote.owner.id')=?2 ORDER BY f.id LIMIT 128",
+ )?;
+ Ok(query
+ .query_map(params![after, session.user_id], |r| r.get(0))?
+ .collect::<rusqlite::Result<_>>()?)
+ })?;
+ if ids.is_empty() {
+ break;
+ }
+ let transaction = store.db.connection().unchecked_transaction()?;
+ for id in ids {
+ after = id;
+ store
+ .db
+ .connection()
+ .execute("INSERT INTO favorites VALUES(?1)", [id])?;
+ }
+ transaction.commit()?;
+ }
+ }
after = 0;
loop {
- let next:Option<(i64,Option<String>)>=store.db.connection().query_row("SELECT id,record FROM desired_albums WHERE selected=1 AND id>?1 ORDER BY id LIMIT 1",[after],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
- let Some((id, record)) = next else { break };
+ let id: Option<i64> = store
+ .db
+ .connection()
+ .query_row(
+ "SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL AND id>?1 ORDER BY id LIMIT 1",
+ [after],
+ |r| r.get(0),
+ )
+ .optional()?;
+ let Some(id) = id else { break };
after = id;
- if *cancel.borrow() {
- bail!(Cancelled)
- }
- let Some(_) = record else {
- store
- .db
- .connection()
- .execute("UPDATE desired_albums SET ready=1 WHERE id=?1", [id])?;
- continue;
- };
- let result=async {
- replica.retry_album(session,id,retry)?;
- let record=replica.album_record(id)?.context("missing album source")?;
- let album=record.album(session.user_id)?;
- replica.sync_album_files(session,&album,retry).await?;
- store.desired_album(&record,session.user_id)?;
- let mut after_file=0;
- loop {
- let records:Vec<FileRecord>=replica.db.read(|db| {
- let mut query=db.prepare("SELECT record FROM photos_files WHERE collection_id=?1 AND id>?2 ORDER BY id LIMIT 128")?;
- Ok(query.query_map(params![id,after_file],|r|crate::replica::read_json(r,0))?.collect::<rusqlite::Result<_>>()?)
- })?;
- if records.is_empty() {break;}
- let transaction=store.db.connection().unchecked_transaction()?;
- for record in records {
- after_file=record.remote.id;
- let favorited=if favorites_known {Some(replica.db.read(|db|Ok(db.query_row("SELECT EXISTS(SELECT 1 FROM photos_files f JOIN photos_collections c ON c.id=f.collection_id WHERE f.id=?1 AND json_extract(c.record,'$.remote.type')='favorites' AND json_extract(c.record,'$.remote.owner.id')=?2)",params![record.remote.id,session.user_id],|r|r.get::<_,bool>(0))?))?)} else {None};
- store.desired_file(id,&record,session.user_id,favorited)?;
- if !favorites_known {store.db.connection().execute("UPDATE desired_files SET failure='favorite state is incomplete' WHERE album=?1 AND file=?2",params![id,record.remote.id])?;}
- }
- transaction.commit()?;
+ let mut after_file = 0;
+ loop {
+ ensure!(!*cancel.borrow(), Cancelled);
+ let records: Vec<FileRecord> = replica.db.read(|db| {
+ let mut query = db.prepare(
+ "SELECT record FROM photos_files WHERE collection_id=?1 AND id>?2 ORDER BY id LIMIT 128",
+ )?;
+ Ok(query
+ .query_map(params![id, after_file], |r| crate::replica::read_json(r, 0))?
+ .collect::<rusqlite::Result<_>>()?)
+ })?;
+ if records.is_empty() {
+ break;
}
- store.db.connection().execute("UPDATE desired_albums SET ready=1 WHERE id=?1",[id])?;
- Ok::<_,anyhow::Error>(())
- }.await;
- if let Err(error) = result {
- store.db.connection().execute(
- "UPDATE desired_albums SET failure=?1 WHERE id=?2",
- params![error.to_string(), id],
- )?;
- report(store, &format!("album:{id}"), &error)?;
- if fatal(&error) {
- return Err(error);
+ let transaction = store.db.connection().unchecked_transaction()?;
+ for record in records {
+ after_file = record.remote.id;
+ let file = record.file(session.user_id);
+ let failure = if !favorites_known {
+ Some("favorite state is incomplete".to_owned())
+ } else {
+ file.as_ref().err().map(ToString::to_string)
+ };
+ store.db.connection().execute(
+ "INSERT INTO desired_files(album,file,favorited,failure) VALUES(?1,?2,?3,?4)",
+ params![
+ id,
+ record.remote.id,
+ if favorites_known { Some(false) } else { None },
+ failure
+ ],
+ )?;
+ if failure.is_none() {
+ let file = file?;
+ store.db.connection().execute(
+ "INSERT INTO chosen_files VALUES(?1,?2,?3) ON CONFLICT(file) DO UPDATE SET album=excluded.album,version=excluded.version WHERE excluded.version>chosen_files.version OR (excluded.version=chosen_files.version AND excluded.album<chosen_files.album)",
+ params![file.id, id, file.updated_at_micros],
+ )?;
+ }
}
+ transaction.commit()?;
}
- refreshed += 1;
- if progress.elapsed().as_secs() >= 2 {
- eprintln!("Refreshed {refreshed}/{selected} albums.");
- progress = Instant::now();
+ }
+ store
+ .db
+ .connection()
+ .execute(
+ "UPDATE desired_files SET favorited=EXISTS(SELECT 1 FROM favorites WHERE favorites.file=desired_files.file) WHERE favorited IS NOT NULL",
+ [],
+ )?;
+ let mut after_file = 0;
+ loop {
+ ensure!(!*cancel.borrow(), Cancelled);
+ let mut query = store
+ .db
+ .connection()
+ .prepare("SELECT album,file FROM chosen_files WHERE file>?1 ORDER BY file LIMIT 128")?;
+ let chosen: Vec<(i64, i64)> = query
+ .query_map([after_file], |r| Ok((r.get(0)?, r.get(1)?)))?
+ .collect::<rusqlite::Result<_>>()?;
+ drop(query);
+ if chosen.is_empty() {
+ break;
+ }
+ let transaction = store.db.connection().unchecked_transaction()?;
+ for (album, id) in chosen {
+ after_file = id;
+ let record = replica
+ .file_record(album, id)?
+ .context("missing chosen file source")?;
+ store.snapshot_file(&record.file(session.user_id)?)?;
}
+ transaction.commit()?;
}
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store.db.connection().execute(
+ "DELETE FROM sources WHERE file NOT IN (SELECT file FROM chosen_files)",
+ [],
+ )?;
+ store
+ .db
+ .connection()
+ .execute(
+ "DELETE FROM album_sources WHERE id NOT IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM albums WHERE albums.id=album_sources.id) AND NOT EXISTS(SELECT 1 FROM pending WHERE album=album_sources.id)",
+ [],
+ )?;
+ transaction.commit()?;
let mut query = store
.db
.connection()
.prepare("SELECT album,file,failure FROM desired_files WHERE failure IS NOT NULL")?;
let mut rows = query.query([])?;
while let Some(row) = rows.next()? {
- report(
- store,
- &format!("file:{}:{}", row.get::<_, i64>(0)?, row.get::<_, i64>(1)?),
- &anyhow::anyhow!(row.get::<_, String>(2)?),
- )?;
+ let unit = format!("file:{}:{}", row.get::<_, i64>(0)?, row.get::<_, i64>(1)?);
+ let error = anyhow::anyhow!(row.get::<_, String>(2)?);
+ record_outcome(store, &unit, &error)?;
+ eprintln!("{unit}: {error:#}");
}
Ok(())
}
-async fn transfers(
- context: Arc<transfer::Context>,
- store: &Store,
- cancel: watch::Receiver<bool>,
+struct CancelWorkers<'a>(&'a watch::Sender<bool>);
+
+impl Drop for CancelWorkers<'_> {
+ fn drop(&mut self) {
+ if std::thread::panicking() {
+ self.0.send_replace(true);
+ }
+ }
+}
+
+fn transfers(
+ root: &Path,
+ store: &Mutex<Store>,
+ session: &ente_core::Session,
+ cancel_send: &watch::Sender<bool>,
+ cancel: &watch::Receiver<bool>,
+ jobs: usize,
) -> Result<()> {
- let expected: i64 =
- store
- .db
- .connection()
- .query_row("SELECT count(*) FROM desired_files", [], |r| r.get(0))?;
+ let (expected, files): (i64, i64) = store::lock(store)?.db.connection().query_row(
+ "SELECT count(*),count(DISTINCT file) FROM desired_files WHERE failure IS NULL",
+ [],
+ |r| Ok((r.get(0)?, r.get(1)?)),
+ )?;
eprintln!("Maintaining {expected} selected copies.");
- let mut work = tokio::task::JoinSet::new();
- let result=async {
- let ceiling=(256*1024*1024/ente_core::crypto::stream::DECRYPTION_CHUNK_SIZE).max(1);
- let mut admission=std::thread::available_parallelism().map(usize::from).unwrap_or(1).min(ceiling);
- let mut measured=Instant::now();let mut completed_bytes=0u64;let mut previous_rate=0.0;
- loop {
- while work.len()<admission && !*cancel.borrow() {
- let staged:i64=store.db.connection().query_row("SELECT count(DISTINCT json_extract(t.record,'$.file')) FROM temporaries t WHERE NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.file=json_extract(t.record,'$.file') AND f.running=1)",[],|r|r.get(0))?;
- let available=staged as usize+work.len()<ceiling;
- let staged:Option<i64>=store.db.connection().query_row("SELECT json_extract(t.record,'$.file') FROM temporaries t WHERE EXISTS(SELECT 1 FROM desired_files f WHERE f.file=json_extract(t.record,'$.file') AND f.failure IS NULL AND f.attempted=0 AND f.running=0) ORDER BY json_extract(t.record,'$.file') LIMIT 1",[],|r|r.get(0)).optional()?;
- let next=if let Some(id)=staged {Some((id,true))} else {
- store.db.connection().query_row("SELECT file FROM desired_files WHERE failure IS NULL AND attempted=0 AND running=0 AND deferred<=?1 ORDER BY deferred,file LIMIT 1",[available],|r|Ok((r.get(0)?,false))).optional()?
- };
- let Some((id,staged))=next else {break};
- store.db.connection().execute("UPDATE desired_files SET running=1 WHERE file=?1",[id])?;
- let context=context.clone();let cancel=cancel.clone();
- work.spawn(async move {(id,transfer::prepare(context,id,cancel,available || staged).await)});
- }
- let Some(result)=work.join_next().await else {break};
- let (id,result)=result?;
- store.db.connection().execute("UPDATE desired_files SET running=0 WHERE file=?1",[id])?;
- transfer::discard_incomplete(&context.root,store,id)?;
- let result=match result {
- Ok(transfer::Preparation::Ready(mut prepared))=>{
- completed_bytes+=prepared.components.iter().filter(|c|c.temporary).map(|c|c.size).sum::<u64>();
- if !*cancel.borrow() {reconcile::file(&context.root,store,&mut prepared,&cancel)?;}
- transfer::finish_source(&context.root,store,&prepared.components)
- },
- Ok(transfer::Preparation::AwaitingCapacity)=>{
- store.db.connection().execute("UPDATE desired_files SET deferred=1 WHERE file=?1",[id])?;
+ let runtime = tokio::runtime::Handle::current();
+ let last_report = Mutex::new(None::<Instant>);
+ std::thread::scope(|scope| {
+ let mut workers = Vec::new();
+ for _ in 0..jobs.min(files as usize) {
+ let runtime = &runtime;
+ let last_report = &last_report;
+ workers.push(scope.spawn(move || {
+ let _cancel_on_panic = CancelWorkers(cancel_send);
+ let run = reconcile::Run { root, store, cancel };
+ let result = (|| {
+ loop {
+ if *cancel.borrow() {
+ break;
+ }
+ let file = {
+ let store = store::lock(store)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ let id: Option<i64> = store
+ .db
+ .connection()
+ .query_row(
+ "SELECT file FROM desired_files WHERE failure IS NULL AND claimed=0 ORDER BY file LIMIT 1",
+ [],
+ |r| r.get(0),
+ )
+ .optional()?;
+ let Some(id) = id else {
+ break;
+ };
+ store
+ .db
+ .connection()
+ .execute(
+ "UPDATE desired_files SET claimed=1 WHERE file=?1",
+ [id],
+ )?;
+ let file: ente_photos::files::File = store
+ .json("SELECT record FROM sources WHERE file=?1", [id])?
+ .context("missing file source snapshot")?;
+ transaction.commit()?;
+ file
+ };
+ let id = file.id;
+ let result = runtime.block_on(run.file(session, file));
+ if result.as_ref().err().is_some_and(fatal) {
+ cancel_send.send_replace(true);
+ }
+ let reported = result.or_else(|error| run.block_file(id, error));
+ let cleaned = run.cleanup(Some(id));
+ reported?;
+ cleaned?;
+ let completed = {
+ let mut last = last_report.lock().map_err(|_| anyhow::anyhow!("progress mutex poisoned"))?;
+ if last.is_none_or(|time| time.elapsed() >= Duration::from_secs(2)) {
+ *last = Some(Instant::now());
+ Some(store::lock(store)?.db.connection().query_row(
+ "SELECT count(*) FROM desired_files WHERE completed=1",
+ [],
+ |r| r.get::<_, i64>(0),
+ )?)
+ } else {
+ None
+ }
+ };
+ if let Some(completed) = completed {
+ eprintln!("Completed {completed}/{expected} selected copies.");
+ }
+ }
Ok(())
- },
- Err(error)=>Err(error),
- };
- if let Err(error)=result {
- let mut query=store.db.connection().prepare("SELECT album FROM desired_files WHERE file=?1 AND attempted=0 AND failure IS NULL")?;
- let albums:Vec<i64>=query.query_map([id],|r|r.get(0))?.collect::<rusqlite::Result<_>>()?;drop(query);
- for album in albums {report(store,&format!("file:{album}:{id}"),&error)?;}
- store.db.connection().execute("UPDATE desired_files SET failure=?1 WHERE file=?2 AND attempted=0",params![error.to_string(),id])?;
- if fatal(&error) {return Err(error)}
- }
- if measured.elapsed().as_secs_f64()>=2.0 {
- let completed:i64=store.db.connection().query_row("SELECT count(*) FROM desired_files WHERE completed=1",[],|r|r.get(0))?;
- eprintln!("Exporting: {completed}/{expected} copies complete.");
- let rate=completed_bytes as f64/measured.elapsed().as_secs_f64();
- if completed_bytes>0 && (previous_rate==0.0 || rate>previous_rate*1.1) {admission=(admission*2).min(ceiling);}
- previous_rate=rate;measured=Instant::now();completed_bytes=0;
- }
+ })();
+ if result.is_err() {
+ cancel_send.send_replace(true);
+ }
+ result
+ }));
}
- let mut query=store.db.connection().prepare("SELECT album,file FROM desired_files WHERE deferred=1 AND attempted=0 AND failure IS NULL")?;
- let mut rows=query.query([])?;
- while let Some(row)=rows.next()? {
- let album:i64=row.get(0)?;let file:i64=row.get(1)?;
- report(store,&format!("file:{album}:{file}"),&anyhow::anyhow!("completed transfers have filled export staging; resolve the reported destination failures and retry"))?;
+ let mut result = Ok(());
+ for worker in workers {
+ let completed = worker
+ .join()
+ .unwrap_or_else(|_| Err(anyhow::anyhow!("export worker panicked")));
+ if result.is_ok() {
+ result = completed;
+ }
}
- Ok(())
- }.await;
- work.abort_all();
- while work.join_next().await.is_some() {}
- result
+ result
+ })
}
-pub fn report(store: &Store, unit: &str, error: &anyhow::Error) -> Result<()> {
+pub fn record_outcome(store: &Store, unit: &str, error: &anyhow::Error) -> Result<()> {
let conflict = error.downcast_ref::<Conflict>().is_some();
- store.db.connection().execute("INSERT INTO outcomes VALUES(?1,?2) ON CONFLICT(unit) DO UPDATE SET conflict=max(conflict,excluded.conflict)",params![unit,conflict])?;
- eprintln!("{unit}: {error:#}");
+ store
+ .db
+ .connection()
+ .execute(
+ "INSERT INTO outcomes VALUES(?1,?2) ON CONFLICT(unit) DO UPDATE SET conflict=max(conflict,excluded.conflict)",
+ params![unit, conflict],
+ )?;
Ok(())
}
pub fn fatal(error: &anyhow::Error) -> bool {
- error.chain().any(|cause| {
- cause.downcast_ref::<rusqlite::Error>().is_some()
- || cause.downcast_ref::<Cancelled>().is_some()
- || cause
- .downcast_ref::<std::io::Error>()
- .is_some_and(|e| matches!(e.kind(), std::io::ErrorKind::StorageFull))
- || cause
- .downcast_ref::<ente_core::http::Error>()
- .is_some_and(|e| e.status_code() == Some(401))
- })
+ fatal_cause(error.as_ref())
}
-fn cleanup(root: &Path, store: &Store) -> Result<()> {
- let mut after = String::new();
- loop {
- let temporary:Option<store::Temporary>=store.json("SELECT record FROM temporaries WHERE path>?1 AND json_extract(record,'$.destination') IS NULL AND (json_extract(record,'$.hash') IS NULL OR (EXISTS(SELECT 1 FROM desired_files WHERE file=json_extract(temporaries.record,'$.file')) AND NOT EXISTS(SELECT 1 FROM desired_files WHERE file=json_extract(temporaries.record,'$.file') AND completed=0))) AND json_extract(record,'$.album') IN (SELECT id FROM desired_albums WHERE selected=1) ORDER BY path LIMIT 1",[&after])?;
- let Some(temporary) = temporary else { break };
- after = temporary.path.clone();
- if let Err(error) = fs::discard(root, store, &temporary.path) {
- let unit = if let Some(file) = temporary.file {
- format!("file:{}:{file}", temporary.album)
- } else {
- format!("album:{}", temporary.album)
- };
- report(store, &unit, &error)?;
- if fatal(&error) {
- return Err(error);
- }
+fn fatal_cause(cause: &(dyn std::error::Error + 'static)) -> bool {
+ if cause.is::<rusqlite::Error>()
+ || cause.is::<Cancelled>()
+ || matches!(
+ cause.downcast_ref::<crate::core_db::Error>(),
+ Some(crate::core_db::Error::Sqlite(_))
+ )
+ || cause
+ .downcast_ref::<ente_core::http::Error>()
+ .is_some_and(|error| error.status_code() == Some(401))
+ {
+ return true;
+ }
+ if let Some(error) = cause.downcast_ref::<std::io::Error>() {
+ return error.kind() == std::io::ErrorKind::StorageFull
+ || error.get_ref().is_some_and(|inner| fatal_cause(inner));
+ }
+ if let Some(error) = cause.downcast_ref::<ente_photos::files::Error>() {
+ use ente_photos::files::Error;
+ match error {
+ Error::Http(error) => return fatal_cause(error),
+ Error::Io(error) => return fatal_cause(error),
+ Error::Collections(error) => return fatal_cause(error),
+ _ => {}
}
}
- Ok(())
+ if let Some(ente_collections::Error::Http(error)) = cause.downcast_ref() {
+ return fatal_cause(error);
+ }
+ if let Some(error) = cause.downcast_ref::<ente_photos::live_photo::Error>() {
+ match error {
+ ente_photos::live_photo::Error::Io(error) => return fatal_cause(error),
+ ente_photos::live_photo::Error::Zip(error) => return fatal_cause(error),
+ ente_photos::live_photo::Error::Components => {}
+ }
+ }
+ cause.source().is_some_and(fatal_cause)
}
fn summary(store: &Store, destination: &Path, as_json: bool) -> Result<bool> {
@@ -552,7 +718,14 @@ fn summary(store: &Store, destination: &Path, as_json: bool) -> Result<bool> {
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
- let (exported,metadata_updated,renamed,retained):(i64,i64,i64,i64)=store.db.connection().query_row("SELECT coalesce(sum(exported),0),coalesce(sum(metadata_updated),0),coalesce(sum(renamed),0),coalesce(sum(retained),0) FROM events",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?)))?;
+ let (exported, metadata_updated, renamed, retained): (i64, i64, i64, i64) = store
+ .db
+ .connection()
+ .query_row(
+ "SELECT coalesce(sum(exported),0),coalesce(sum(metadata_updated),0),coalesce(sum(renamed),0),coalesce(sum(retained),0) FROM events",
+ [],
+ |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
+ )?;
let complete = !unknown && failures == 0 && completed == expected;
let result = json!({"destination":destination,"complete":complete,"files":if unknown {None}else{Some(files)},"copies":if unknown {None}else{Some(json!({"expected":expected,"completed":completed,"pending":expected-completed}))},"changes":{"exported":exported,"metadataUpdated":metadata_updated,"renamed":renamed,"retained":retained},"failures":failures,"conflicts":conflicts});
output::action(
@@ -565,3 +738,38 @@ fn summary(store: &Store, destination: &Path, as_json: bool) -> Result<bool> {
)?;
Ok(complete)
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn transparent_errors_preserve_fatal_classification() {
+ for status in [401, 404] {
+ let error = ente_core::http::Error::Http {
+ status,
+ path: "/files".into(),
+ };
+ let error =
+ ente_photos::files::Error::Collections(ente_collections::Error::Http(error));
+ let error = anyhow::Error::new(error).context("download failed");
+ assert_eq!(fatal(&error), status == 401);
+ assert_eq!(
+ format!("{error:#}"),
+ format!("download failed: HTTP {status} at /files")
+ );
+ }
+ let cancelled = ente_photos::live_photo::Error::Io(std::io::Error::other(Cancelled));
+ assert!(fatal(&anyhow::Error::new(cancelled)));
+ let full = ente_photos::live_photo::Error::Io(std::io::ErrorKind::StorageFull.into());
+ assert!(fatal(&anyhow::Error::new(full)));
+ let zip = ente_photos::live_photo::Error::Zip(zip::result::ZipError::Io(
+ std::io::ErrorKind::StorageFull.into(),
+ ));
+ assert!(fatal(&anyhow::Error::new(zip)));
+ let sqlite = crate::core_db::Error::Sqlite(rusqlite::Error::InvalidQuery);
+ assert!(fatal(&anyhow::Error::new(sqlite)));
+ let local = ente_photos::files::Error::Io(std::io::ErrorKind::PermissionDenied.into());
+ assert!(!fatal(&anyhow::Error::new(local)));
+ }
+}
diff --git a/rust/apps/cli-next/src/export/names.rs b/rust/apps/cli-next/src/export/names.rs
index 9703007a10..4feb7d3a8a 100644
--- a/rust/apps/cli-next/src/export/names.rs
+++ b/rust/apps/cli-next/src/export/names.rs
@@ -122,24 +122,33 @@ fn numbered(stem: &str, suffix: &str) -> String {
}
}
-pub fn reserve(store: &Store, parent: &str, name: &str, owner: &str, kind: &str) -> Result<bool> {
- Ok(store.db.connection().execute(
- "INSERT INTO names VALUES(?1,?2,?3,?4,?5) ON CONFLICT(parent,folded) DO UPDATE SET kind=excluded.kind WHERE names.owner=excluded.owner",
- params![parent, folded(name), owner, folded(split(name, kind == "album").0), kind],
- )? > 0)
-}
-
pub fn available(
store: &Store,
- parent: &str,
+ folder: &str,
names: &[String],
owner: &str,
kind: &str,
) -> Result<bool> {
for name in names {
- let occupied: bool = store.db.connection().query_row(
- "SELECT EXISTS(SELECT 1 FROM names WHERE parent=?1 AND owner<>?2 AND folded=?3) OR (?4<>'album' AND EXISTS(SELECT 1 FROM names WHERE parent=?1 AND owner<>?2 AND stem=?5 AND NOT(kind=?4 AND kind IN ('image','video'))))",
- params![parent,owner,folded(name),kind,folded(split(name,false).0)], |r| r.get(0))?;
+ let occupied: bool = if kind == "album" {
+ store.db.connection().query_row(
+ "SELECT EXISTS(SELECT 1 FROM albums WHERE parent=?1 AND folded=?2 AND key<>?3 UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND folded1=?2 AND kind='album' AND owner<>?3)",
+ params![folder, folded(name), owner],
+ |r| r.get(0),
+ )?
+ } else {
+ store.db.connection().query_row(
+ "SELECT EXISTS(SELECT 1 FROM components c JOIN placements p ON p.id=c.placement WHERE c.folder=?1 AND c.folded=?2 AND c.placement<>?3 UNION ALL SELECT 1 FROM components c JOIN placements p ON p.id=c.placement WHERE c.folder=?1 AND c.stem=?4 AND c.placement<>?3 AND NOT(p.kind=?5 AND p.kind IN ('image','video')) UNION ALL SELECT 1 FROM json_records j JOIN placements p ON p.id=j.owner WHERE j.folder=?1 AND j.folded=?2 AND j.owner<>?3 UNION ALL SELECT 1 FROM json_records j JOIN placements p ON p.id=j.owner WHERE j.folder=?1 AND j.stem=?4 AND j.owner<>?3 AND NOT(p.kind=?5 AND p.kind IN ('image','video')) UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND folded1=?2 AND owner<>?3 UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND folded2=?2 AND owner<>?3 UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND stem1=?4 AND owner<>?3 AND NOT(kind=?5 AND kind IN ('image','video')) UNION ALL SELECT 1 FROM pending WHERE folder=?1 AND stem2=?4 AND owner<>?3 AND NOT(kind=?5 AND kind IN ('image','video')))",
+ params![
+ folder,
+ folded(name),
+ owner,
+ folded(split(name, false).0),
+ kind
+ ],
+ |r| r.get(0),
+ )?
+ };
if occupied {
return Ok(false);
}
@@ -149,7 +158,7 @@ pub fn available(
pub fn allocate(
store: &Store,
- parent: &str,
+ folder: &str,
name: &str,
kind: &str,
extensions: Option<(&str, &str)>,
@@ -159,13 +168,7 @@ pub fn allocate(
let Some(candidates) = candidates(name, kind == "album", extensions, suffix)? else {
continue;
};
- if available(store, parent, &candidates, owner, kind)? {
- for candidate in &candidates {
- ensure!(
- reserve(store, parent, candidate, owner, kind)?,
- "name allocation changed"
- );
- }
+ if available(store, folder, &candidates, owner, kind)? {
return Ok(candidates);
}
}
diff --git a/rust/apps/cli-next/src/export/reconcile.rs b/rust/apps/cli-next/src/export/reconcile.rs
index 11f1d2afd5..249a0c6b28 100644
--- a/rust/apps/cli-next/src/export/reconcile.rs
+++ b/rust/apps/cli-next/src/export/reconcile.rs
@@ -1,781 +1,1605 @@
-use std::{fs as disk, path::Path};
+use std::{
+ fs as disk,
+ fs::{File, OpenOptions},
+ io::Write,
+ path::{Path, PathBuf},
+ sync::Mutex,
+};
use anyhow::{Context, Result, ensure};
-use ente_photos::export;
+use ente_photos::{export::Role, files};
use rusqlite::{OptionalExtension, params};
use serde_json::Value;
+use tokio::sync::watch;
use uuid::Uuid;
use super::{
- allocation, fatal, fs, names, report,
- store::{Album, Media, Operation, Placement, Properties, Store},
- transfer::{self, Prepared},
+ fs, names,
+ store::{
+ self, Action, Album, AlbumSource, Component, JsonRecord, Location, Output, Pending,
+ Placement, Properties, Store, Temporary,
+ },
+ transfer,
};
-use crate::replica::AlbumRecord;
-
-pub fn albums(root: &Path, store: &Store, user_id: i64) -> Result<()> {
- let mut after = 0;
- loop {
- let next:Option<(i64,String)>=store.db.connection().query_row("SELECT id,record FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL AND record IS NOT NULL AND id>?1 ORDER BY id LIMIT 1",[after],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
- let Some((id, record)) = next else { break };
- after = id;
- let result = (|| {
- let record: AlbumRecord = serde_json::from_str(&record)?;
- let source = record.album(user_id)?;
- for field in &source.warnings {
- eprintln!(
- "album {id}: unfamiliar metadata field {field:?}; exporting supported fields"
- );
+
+pub struct Run<'a> {
+ pub root: &'a Path,
+ pub store: &'a Mutex<Store>,
+ pub cancel: &'a watch::Receiver<bool>,
+}
+
+impl Run<'_> {
+ pub fn path(&self, location: &Location) -> Result<PathBuf> {
+ let relative = {
+ let store = store::lock(self.store)?;
+ ensure!(
+ !store
+ .pending(&location.folder)?
+ .is_some_and(|pending| matches!(
+ pending.action,
+ Some(Action::Directory { .. })
+ )),
+ "album directory has an unfinished transition"
+ );
+ store.relative(location)?
+ };
+ names::check(self.root, &relative)
+ }
+
+ pub fn check_cancel(&self) -> Result<()> {
+ ensure!(!*self.cancel.borrow(), super::Cancelled);
+ Ok(())
+ }
+
+ pub fn temporary(
+ &self,
+ folder: &str,
+ album: i64,
+ file: Option<i64>,
+ ) -> Result<(Location, File)> {
+ self.check_cancel()?;
+ let temporary = Temporary {
+ location: Location {
+ folder: folder.into(),
+ name: format!(".ente-{}.part", Uuid::new_v4().simple()),
+ },
+ album,
+ file,
+ };
+ store::lock(self.store)?.temporary(&temporary)?;
+ let path = self.path(&temporary.location)?;
+ let file = OpenOptions::new()
+ .write(true)
+ .read(true)
+ .create_new(true)
+ .open(&path)?;
+ Ok((temporary.location, file))
+ }
+
+ pub fn discard(&self, location: &Location) -> Result<()> {
+ fs::remove(&self.path(location)?)?;
+ store::lock(self.store)?
+ .db
+ .connection()
+ .execute("DELETE FROM temporaries WHERE name=?1", [&location.name])?;
+ Ok(())
+ }
+
+ pub fn report(&self, unit: &str, error: &anyhow::Error) -> Result<()> {
+ {
+ let store = store::lock(self.store)?;
+ super::record_outcome(&store, unit, error)?;
+ }
+ eprintln!("{unit}: {error:#}");
+ Ok(())
+ }
+
+ pub fn block(&self, album: i64, file: Option<i64>, error: anyhow::Error) -> Result<()> {
+ let unit = file.map_or_else(
+ || format!("album:{album}"),
+ |file| format!("file:{album}:{file}"),
+ );
+ self.report(&unit, &error)?;
+ if file.is_none() {
+ store::lock(self.store)?.db.connection().execute(
+ "UPDATE desired_albums SET failure=?1 WHERE id=?2",
+ params![error.to_string(), album],
+ )?;
+ let mut after = 0;
+ loop {
+ let next: Option<i64> = store::lock(self.store)?
+ .db
+ .connection()
+ .query_row(
+ "SELECT file FROM desired_files WHERE album=?1 AND file>?2 ORDER BY file LIMIT 1",
+ params![album, after],
+ |r| r.get(0),
+ )
+ .optional()?;
+ let Some(file) = next else { break };
+ after = file;
+ self.report(&format!("file:{album}:{file}"), &error)?;
}
- let existing = store.album(id, false)?;
- let key = existing
- .as_ref()
- .map(|a| a.key.clone())
- .unwrap_or_else(|| format!("active:{id}"));
- let path = if let Some(existing) = &existing
- && existing.name == source.name
- {
- existing.path.clone()
- } else {
- allocation::choose(store, "", &source.name, "album", None, &key)?.remove(0)
- };
- let album = Album {
- key,
- id,
- name: source.name.clone(),
- path: path.clone(),
- retained: false,
- metadata: export::album(&source),
- initialized: true,
- };
- if let Some(existing) = &existing
- && existing.path != path
+ }
+ store::lock(self.store)?.db.connection().execute(
+ "UPDATE desired_files SET failure=?1 WHERE album=?2 AND (?3 IS NULL OR file=?3)",
+ params![error.to_string(), album, file],
+ )?;
+ if super::fatal(&error) {
+ return Err(error);
+ }
+ Ok(())
+ }
+
+ pub fn block_file(&self, file: i64, error: anyhow::Error) -> Result<()> {
+ let mut after = 0;
+ loop {
+ let next: Option<i64> = store::lock(self.store)?
+ .db
+ .connection()
+ .query_row(
+ "SELECT album FROM desired_files WHERE file=?1 AND album>?2 ORDER BY album LIMIT 1",
+ params![file, after],
+ |r| r.get(0),
+ )
+ .optional()?;
+ let Some(album) = next else { break };
+ after = album;
+ self.report(&format!("file:{album}:{file}"), &error)?;
+ }
+ store::lock(self.store)?.db.connection().execute(
+ "UPDATE desired_files SET failure=?1 WHERE file=?2",
+ params![error.to_string(), file],
+ )?;
+ if super::fatal(&error) {
+ return Err(error);
+ }
+ Ok(())
+ }
+
+ fn start(
+ &self,
+ target: &mut Pending,
+ action: Action,
+ snapshots: &[(Option<Role>, Value)],
+ ) -> Result<()> {
+ self.check_cancel()?;
+ let store = store::lock(self.store)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ let published = !store.components(&target.owner)?.is_empty();
+ for (role, value) in snapshots {
+ let previous = store.json_record(&target.owner, role.as_ref())?;
+ if previous.is_none()
+ || (previous
+ .as_ref()
+ .is_some_and(|record| record.location.is_none())
+ && !published)
{
- rename_target(root, &existing.path, &path)?;
- let transaction = store.db.connection().unchecked_transaction()?;
- store.operation(&Operation::Move {
- album: id,
- file: None,
- source: existing.path.clone(),
- destination: path.clone(),
+ store.save_json(&JsonRecord {
+ owner: target.owner.clone(),
+ role: role.clone(),
+ location: None,
+ value: value.clone(),
+ hash: None,
+ properties: None,
})?;
- let mut query = store
- .db
- .connection()
- .prepare("SELECT record FROM placements WHERE album=?1 AND retained=0")?;
- let mut rows = query.query([id])?;
- while let Some(row) = rows.next()? {
- let mut placement: Placement = serde_json::from_str(&row.get::<_, String>(0)?)?;
- placement.folder = path.clone();
- store.save_placement(&placement)?;
- event(store, &placement.id, "renamed")?;
+ }
+ }
+ target.action = Some(action);
+ store.save_pending(target)?;
+ transaction.commit()?;
+ Ok(())
+ }
+
+ fn owns(&self, owner: &str, location: &Location, output: &Output) -> Result<bool> {
+ let store = store::lock(self.store)?;
+ Ok(match output {
+ Output::Media { role, .. } => store
+ .components(owner)?
+ .iter()
+ .any(|component| component.role == *role && component.location == *location),
+ Output::Json { role, .. } => store
+ .json_record(owner, role.as_ref())?
+ .is_some_and(|record| record.location.as_ref() == Some(location)),
+ })
+ }
+
+ fn clear_action(&self, target: &mut Pending) -> Result<()> {
+ target.action = None;
+ store::lock(self.store)?.save_pending(target)
+ }
+
+ fn acknowledge(
+ &self,
+ target: &mut Pending,
+ destination: &Location,
+ output: &Output,
+ value: Option<Value>,
+ ) -> Result<()> {
+ let path = self.path(destination)?;
+ let properties = Properties::read(&path)?;
+ let value = if matches!(output, Output::Json { .. }) {
+ Some(match value {
+ Some(value) => value,
+ None => serde_json::from_slice(&disk::read(&path)?)?,
+ })
+ } else {
+ None
+ };
+ let store = store::lock(self.store)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ match output {
+ Output::Media {
+ role,
+ size,
+ hash,
+ signature,
+ } => {
+ if store
+ .placement(
+ target.album,
+ target.file.context("missing publication file")?,
+ )?
+ .is_none()
+ {
+ store.save_placement(&target.placement()?)?;
}
- store.db.connection().execute(
- "UPDATE json_records SET path=?1||substr(path,?2) WHERE substr(path,1,?3)=?4",
- params![
- path,
- existing.path.chars().count() as i64 + 1,
- existing.path.chars().count() as i64 + 1,
- format!("{}/", existing.path)
- ],
- )?;
- relocate_work(store, &existing.path, &path)?;
- allocation::relocate(store, &existing.path, &path)?;
- store.save_album(&album)?;
- allocation::clear(store, &album.key)?;
+ let component = Component {
+ placement: target.owner.clone(),
+ role: role.clone(),
+ location: destination.clone(),
+ size: *size,
+ hash: hash.clone(),
+ signature: signature.clone(),
+ properties: Some(properties),
+ intended_time: None,
+ };
+ store.save_component(&component)?;
+ }
+ Output::Json { role, hash, .. } => {
+ let mut record = store
+ .json_record(&target.owner, role.as_ref())?
+ .context("missing metadata snapshot")?;
+ record.location = Some(destination.clone());
+ record.properties = Some(properties);
+ if let Some(value) = value {
+ record.value = value;
+ record.hash = Some(hash.clone());
+ }
+ store.save_json(&record)?;
+ }
+ }
+ if let Some(Action::Publish { temporary, .. }) = &target.action {
+ store
+ .db
+ .connection()
+ .execute("DELETE FROM temporaries WHERE name=?1", [&temporary.name])?;
+ }
+ target.action = None;
+ store.save_pending(target)?;
+ transaction.commit()?;
+ Ok(())
+ }
+
+ fn finish_action(
+ &self,
+ target: &mut Pending,
+ recovering: bool,
+ value: Option<Value>,
+ ) -> Result<()> {
+ let Some(action) = target.action.clone() else {
+ return Ok(());
+ };
+ self.check_cancel()?;
+ match action {
+ Action::Directory { source } => {
+ let destination = names::check(self.root, &target.album_path())?;
+ if let Some(source) = source {
+ let source = names::check(self.root, &source)?;
+ if directory_exists(&source)? {
+ free_rename(&source, &destination)?;
+ disk::rename(&source, &destination)?;
+ } else if !directory_exists(&destination)? {
+ let unrepairable: bool = store::lock(self.store)?
+ .db
+ .connection()
+ .query_row(
+ "SELECT EXISTS(SELECT 1 FROM placements p WHERE p.album=?1 AND p.retained=0 AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file AND f.failure IS NULL))",
+ [target.album],
+ |r| r.get(0),
+ )?;
+ ensure!(
+ !unrepairable,
+ "missing album contains originals that cannot be repaired"
+ );
+ fs::create_directory(&destination)?;
+ }
+ fs::sync_move(&source, &destination)?;
+ } else {
+ fs::create_directory(&destination)?;
+ ensure!(
+ disk::read_dir(&destination)?.next().is_none(),
+ super::Conflict(format!(
+ "at {}: unrelated directory contents",
+ destination.display()
+ ))
+ );
+ fs::sync_parent(&destination)?;
+ if target.retained {
+ fs::sync_parent(
+ destination
+ .parent()
+ .context("retained album has no parent")?,
+ )?;
+ }
+ }
+ let store = store::lock(self.store)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store.save_album(&Album {
+ key: target.owner.clone(),
+ id: target.album,
+ retained: target.retained,
+ name: target.name.clone(),
+ path: target.album_path(),
+ })?;
+ target.action = None;
+ store.save_pending(target)?;
transaction.commit()?;
- fs::operations(root, store, id, None)?;
- } else {
- if existing.is_none() {
- free_target(root, &path)?;
+ }
+ Action::Publish {
+ temporary,
+ destination,
+ output,
+ } => {
+ let source = self.path(&temporary)?;
+ let path = self.path(&destination)?;
+ let owned = self.owns(&target.owner, &destination, &output)?;
+ if Properties::optional(&source)?.is_some() {
+ if recovering {
+ self.clear_action(target)?;
+ return self.discard(&temporary);
+ }
+ ensure!(
+ Properties::optional(&path)?.is_none() || owned,
+ super::Conflict(format!("at {}: unrelated occupant", path.display()))
+ );
+ fs::create_directory(path.parent().context("publication has no parent")?)?;
+ disk::rename(&source, &path)?;
+ } else {
+ let (size, hash) = output.identity();
+ let matches = Properties::optional(&path)?.is_some()
+ && fs::matches(&path, size, hash, self.cancel)?;
+ if !matches {
+ ensure!(
+ !path.try_exists()? || owned,
+ super::Conflict(format!("at {}: unrelated occupant", path.display()))
+ );
+ self.clear_action(target)?;
+ self.discard(&temporary)?;
+ ensure!(recovering, "publication temporary disappeared");
+ return Ok(());
+ }
+ }
+ fs::sync_move(&source, &path)?;
+ self.acknowledge(target, &destination, &output, value)?;
+ }
+ Action::Move {
+ source,
+ destination,
+ role,
+ size,
+ hash,
+ } => {
+ let from = self.path(&source)?;
+ let to = self.path(&destination)?;
+ if Properties::optional(&from)?.is_some() {
+ free_rename(&from, &to)?;
+ fs::create_directory(to.parent().context("move has no parent")?)?;
+ disk::rename(&from, &to)?;
+ } else if Properties::optional(&to)?.is_some() {
+ ensure!(
+ fs::matches(&to, size, &hash, self.cancel)?,
+ super::Conflict(format!("at {}: moved bytes differ", to.display()))
+ );
+ } else {
+ ensure!(
+ !target.retained,
+ "retained original is missing at both move endpoints"
+ );
+ return self.clear_action(target);
}
+ fs::sync_move(&from, &to)?;
+ let store = store::lock(self.store)?;
let transaction = store.db.connection().unchecked_transaction()?;
- store.save_album(&album)?;
- allocation::clear(store, &album.key)?;
+ let mut component = store
+ .components(&target.owner)?
+ .into_iter()
+ .find(|component| component.role == role)
+ .context("missing moved component")?;
+ component.location = destination;
+ component.properties = None;
+ component.intended_time = None;
+ store.save_component(&component)?;
+ target.action = None;
+ store.save_pending(target)?;
transaction.commit()?;
}
- fs::directory(root, &path)?;
- fs::json(
- root,
- store,
- &format!("{path}/metadata.json"),
- &album.metadata,
- true,
- None,
- )?;
- Ok::<_, anyhow::Error>(())
- })();
- if let Err(error) = result {
- report(store, &format!("album:{id}"), &error)?;
- let mut query = store
- .db
- .connection()
- .prepare("SELECT file FROM desired_files WHERE album=?1")?;
- let mut rows = query.query([id])?;
- while let Some(row) = rows.next()? {
- report(
- store,
- &format!("file:{id}:{}", row.get::<_, i64>(0)?),
- &error,
- )?;
+ Action::Remove { location, role } => {
+ fs::remove(&self.path(&location)?)?;
+ let store = store::lock(self.store)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ let mut record = store
+ .json_record(&target.owner, role.as_ref())?
+ .context("missing removed JSON record")?;
+ record.location = None;
+ record.hash = None;
+ record.properties = None;
+ store.save_json(&record)?;
+ target.action = None;
+ store.save_pending(target)?;
+ transaction.commit()?;
}
- store.db.connection().execute(
- "UPDATE desired_files SET failure=?1 WHERE album=?2",
- params![error.to_string(), id],
+ }
+ Ok(())
+ }
+
+ fn json_changed(
+ &self,
+ previous: Option<&JsonRecord>,
+ location: &Location,
+ json: &fs::Json,
+ ) -> Result<bool> {
+ let path = self.path(location)?;
+ let properties = Properties::optional(&path)?;
+ let owned = previous
+ .as_ref()
+ .is_some_and(|record| record.location.as_ref() == Some(location));
+ let renamed = previous
+ .as_ref()
+ .and_then(|record| record.location.as_ref())
+ .map(|source| {
+ self.path(source)
+ .map(|source| names::same_path(&source, &path))
+ })
+ .transpose()?
+ .unwrap_or(false);
+ ensure!(
+ properties.is_none() || owned || renamed,
+ super::Conflict(format!("at {}: unrelated JSON", path.display()))
+ );
+ Ok(!previous.as_ref().is_some_and(|record| {
+ owned
+ && record.hash.as_ref() == Some(&json.hash)
+ && record.properties == properties
+ && properties.is_some()
+ }))
+ }
+
+ fn write_json(
+ &self,
+ target: &mut Pending,
+ role: Option<Role>,
+ location: Location,
+ json: fs::Json,
+ ) -> Result<bool> {
+ let previous = store::lock(self.store)?.json_record(&target.owner, role.as_ref())?;
+ if !self.json_changed(previous.as_ref(), &location, &json)? {
+ return Ok(false);
+ }
+ self.replace_json(target, role, location, json, previous)?;
+ Ok(true)
+ }
+
+ fn replace_json(
+ &self,
+ target: &mut Pending,
+ role: Option<Role>,
+ location: Location,
+ json: fs::Json,
+ previous: Option<JsonRecord>,
+ ) -> Result<bool> {
+ let previous = previous
+ .and_then(|record| record.location)
+ .filter(|old| *old != location);
+ let renamed = previous.is_some();
+ if let Some(previous) = previous {
+ self.start(
+ target,
+ Action::Remove {
+ location: previous,
+ role: role.clone(),
+ },
+ &[],
)?;
- if fatal(&error) {
- return Err(error);
- }
+ self.finish_action(target, false, None)?;
}
+ self.publish_json(target, role, location, json)?;
+ Ok(renamed)
}
- Ok(())
-}
-pub fn file(
- root: &Path,
- store: &Store,
- prepared: &mut Prepared,
- cancel: &tokio::sync::watch::Receiver<bool>,
-) -> Result<()> {
- let mut after = 0;
- loop {
- let next:Option<(i64,bool)>=store.db.connection().query_row("SELECT album,favorited FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL AND attempted=0 ORDER BY album LIMIT 1",params![prepared.file.id,after],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
- let Some((album, favorited)) = next else {
- break;
+ fn publish_json(
+ &self,
+ target: &mut Pending,
+ role: Option<Role>,
+ location: Location,
+ json: fs::Json,
+ ) -> Result<()> {
+ let (temporary, mut file) = self.temporary(&location.folder, target.album, target.file)?;
+ file.write_all(&json.bytes)?;
+ file.sync_all()?;
+ drop(file);
+ let output = Output::Json {
+ role: role.clone(),
+ size: json.bytes.len() as u64,
+ hash: json.hash,
};
- after = album;
- store.db.connection().execute(
- "UPDATE desired_files SET attempted=1 WHERE album=?1 AND file=?2",
- params![album, prepared.file.id],
+ self.start(
+ target,
+ Action::Publish {
+ temporary,
+ destination: location,
+ output,
+ },
+ &[(role, json.value.clone())],
)?;
- let result = placement(root, store, prepared, album, favorited, cancel);
- if let Err(error) = result {
- report(store, &format!("file:{album}:{}", prepared.file.id), &error)?;
- transfer::discard_incomplete(root, store, prepared.file.id)?;
- if fatal(&error) {
- return Err(error);
- }
+ self.finish_action(target, false, Some(json.value))
+ }
+
+ fn move_media(
+ &self,
+ target: &mut Pending,
+ component: &Component,
+ destination: Location,
+ verified: Option<&Properties>,
+ ) -> Result<bool> {
+ if component.location == destination {
+ return Ok(false);
+ }
+ let from = self.path(&component.location)?;
+ let to = self.path(&destination)?;
+ free_rename(&from, &to)?;
+ let properties = Properties::optional(&from)?;
+ if properties.is_none() {
+ ensure!(!target.retained, "previously published original is missing");
+ return Ok(false);
}
+ let (size, hash) = if verified == properties.as_ref() {
+ (component.size, component.hash.clone())
+ } else {
+ fs::hash_cancellable(&from, self.cancel)?
+ };
+ self.start(
+ target,
+ Action::Move {
+ source: component.location.clone(),
+ destination,
+ role: component.role.clone(),
+ size,
+ hash,
+ },
+ &[],
+ )?;
+ self.finish_action(target, false, None)?;
+ Ok(true)
}
- Ok(())
}
-fn placement(
- root: &Path,
- store: &Store,
- prepared: &mut Prepared,
- album_id: i64,
- favorited: bool,
- cancel: &tokio::sync::watch::Receiver<bool>,
-) -> Result<()> {
- let album = store
- .album(album_id, false)?
- .context("missing album association")?;
- let file = &prepared.file;
- for field in &file.warnings {
- eprintln!(
- "file {}: unfamiliar metadata field {field:?}; exporting supported fields",
- file.id
- );
- }
- let existing = store.placement(album_id, file.id)?;
- let old_complete = existing.as_ref().is_some_and(|p| p.complete);
- let owner = format!("file:{album_id}:{}", file.id);
- let extensions = if file.kind == ente_photos::files::Kind::LivePhoto {
- if prepared.components.len() == 2 {
- Some((
- prepared.components[0].extension.as_str(),
- prepared.components[1].extension.as_str(),
- ))
- } else {
- let media = &existing
- .as_ref()
- .context("missing Live Photo original")?
- .media;
- Some((
- names::split(&media[0].component.path, false).1,
- names::split(&media[1].component.path, false).1,
- ))
+fn directory_exists(path: &Path) -> Result<bool> {
+ match disk::symlink_metadata(path) {
+ Ok(metadata) => {
+ ensure!(
+ metadata.is_dir(),
+ super::Conflict(format!("at {}: expected directory", path.display()))
+ );
+ Ok(true)
}
- } else {
- None
- };
- let allocated = if let Some(existing) = &existing
- && existing.name == file.name
- && existing.media.len() == if extensions.is_some() { 2 } else { 1 }
- && (prepared.components.is_empty()
- || extensions.is_none()
- || existing
- .media
- .iter()
- .zip(&prepared.components)
- .all(|(old, new)| {
- names::split(&old.component.path, false).1 == names::portable(&new.extension)
- })) {
- existing
- .media
- .iter()
- .map(|m| m.component.path.clone())
- .collect()
- } else {
- allocation::choose(
- store,
- &album.path,
- &file.name,
- file.kind.name(),
- extensions,
- &owner,
- )?
- };
- let mut placement = Placement {
- id: existing
- .as_ref()
- .map(|p| p.id.clone())
- .unwrap_or_else(|| Uuid::new_v4().to_string()),
- album: album_id,
- folder: album.path.clone(),
- file: file.id,
- retained: false,
- name: file.name.clone(),
- original: prepared.signature.clone(),
- media: Vec::new(),
- metadata: Vec::new(),
- complete: false,
- retaining: false,
- };
- for (index, path) in allocated.iter().enumerate() {
- let mut component = if let Some(component) = prepared.components.get(index) {
- export::Component {
- role: component.role.clone(),
- path: path.clone(),
- size: component.size,
- hash: component.hash.clone(),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
+ Err(error) => Err(error.into()),
+ }
+}
+
+fn free_rename(source: &Path, destination: &Path) -> Result<()> {
+ ensure!(
+ !destination.try_exists()? || names::same_path(source, destination),
+ super::Conflict(format!(
+ "at {}: occupied rename target",
+ destination.display()
+ ))
+ );
+ Ok(())
+}
+
+impl Run<'_> {
+ pub fn recover(&self) -> Result<()> {
+ for albums in [true, false] {
+ let mut after = String::new();
+ loop {
+ let next = {
+ let store = store::lock(self.store)?;
+ store
+ .db
+ .connection()
+ .query_row(
+ "SELECT p.owner,p.album,p.file FROM pending p JOIN desired_albums a ON a.id=p.album WHERE p.owner>?1 AND (p.file IS NULL)=?2 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file AND f.failure IS NOT NULL) ORDER BY p.owner LIMIT 1",
+ params![after, albums],
+ |r| {
+ Ok((
+ r.get::<_, String>(0)?,
+ r.get::<_, i64>(1)?,
+ r.get::<_, Option<i64>>(2)?,
+ ))
+ },
+ )
+ .optional()?
+ };
+ let Some((owner, album, file)) = next else {
+ break;
+ };
+ after = owner.clone();
+ let mut pending = store::lock(self.store)?
+ .pending(&owner)?
+ .context("missing pending transition")?;
+ if let Err(error) = self.finish_action(&mut pending, true, None) {
+ self.block(album, file, error)?;
+ }
}
- } else {
- existing
- .as_ref()
- .and_then(|p| p.media.get(index))
- .context("missing completed original")?
- .component
- .clone()
- };
- component.path = path.clone();
- let previous = existing
- .as_ref()
- .and_then(|p| p.media.iter().find(|m| m.component.role == component.role));
- placement.media.push(Media {
- component,
- properties: previous.and_then(|m| m.properties.clone()),
- intended_time: previous.and_then(|m| m.intended_time),
- });
- }
- let intended_components: Vec<_> = placement
- .media
- .iter()
- .map(|m| m.component.clone())
- .collect();
- placement.metadata = intended_components
- .iter()
- .enumerate()
- .map(|(index, _)| export::file(file, &intended_components, index, favorited))
- .collect::<std::result::Result<_, _>>()?;
- let renamed = existing.as_ref().is_some_and(|p| {
- p.media
- .iter()
- .map(|m| &m.component.path)
- .ne(placement.media.iter().map(|m| &m.component.path))
- });
- if renamed {
- let previous = existing.as_ref().context("missing rename association")?;
- for (old, new) in previous.media.iter().zip(&placement.media) {
- if old.component.path != new.component.path {
- rename_target(
- root,
- &format!("{}/{}", album.path, old.component.path),
- &format!("{}/{}", album.path, new.component.path),
+ }
+ {
+ let store = store::lock(self.store)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store
+ .db
+ .connection()
+ .execute(
+ "DELETE FROM pending WHERE file IS NULL AND action='null' AND album IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL) AND ((retained=0 AND album IN (SELECT id FROM desired_albums WHERE present=0)) OR (retained=1 AND NOT EXISTS(SELECT 1 FROM albums WHERE key=pending.owner) AND NOT EXISTS(SELECT 1 FROM placements p WHERE p.album=pending.album AND p.retained=0 AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file))))",
+ [],
)?;
- rename_target(
- root,
- &format!("{}/metadata/{}.json", album.path, old.component.path),
- &format!("{}/metadata/{}.json", album.path, new.component.path),
+ store
+ .db
+ .connection()
+ .execute(
+ "DELETE FROM json_records WHERE folder IS NULL AND NOT EXISTS(SELECT 1 FROM albums WHERE key=json_records.owner) AND NOT EXISTS(SELECT 1 FROM components WHERE placement=json_records.owner) AND NOT EXISTS(SELECT 1 FROM pending WHERE owner=json_records.owner)",
+ [],
)?;
- }
+ transaction.commit()?;
}
- let transaction = store.db.connection().unchecked_transaction()?;
- store.save_placement(&placement)?;
- allocation::clear(store, &owner)?;
- for (old, new) in previous.media.iter().zip(&placement.media) {
- if old.component.path != new.component.path {
- let source = format!("{}/{}", album.path, old.component.path);
- let destination = format!("{}/{}", album.path, new.component.path);
- store.operation(&Operation::Move {
- album: album_id,
- file: Some(file.id),
- source: source.clone(),
- destination: destination.clone(),
- })?;
- for component in &mut prepared.components {
- if component.path == source {
- component.path = destination.clone();
+ self.cleanup(None)
+ }
+
+ pub fn cleanup(&self, file: Option<i64>) -> Result<()> {
+ let mut after = String::new();
+ loop {
+ let next = {
+ let store = store::lock(self.store)?;
+ store
+ .db
+ .connection()
+ .query_row(
+ "SELECT t.name,t.folder,t.album,t.file FROM temporaries t JOIN desired_albums a ON a.id=t.album WHERE t.name>?1 AND (?2 IS NULL OR t.file=?2) AND a.selected=1 ORDER BY t.name LIMIT 1",
+ params![after, file],
+ |r| {
+ Ok(Temporary {
+ location: Location {
+ name: r.get(0)?,
+ folder: r.get(1)?,
+ },
+ album: r.get(2)?,
+ file: r.get(3)?,
+ })
+ },
+ )
+ .optional()?
+ };
+ let Some(temporary) = next else {
+ break;
+ };
+ after = temporary.location.name.clone();
+ let result = (|| {
+ let pending = {
+ let store = store::lock(self.store)?;
+ let owner: Option<String> = store
+ .db
+ .connection()
+ .query_row(
+ "SELECT owner FROM pending WHERE json_extract(action,'$.Publish.temporary.name')=?1",
+ [&temporary.location.name],
+ |r| r.get(0),
+ )
+ .optional()?;
+ owner
+ .map(|owner| store.pending(&owner))
+ .transpose()?
+ .flatten()
+ };
+ if let Some(mut pending) = pending {
+ let selected: bool = store::lock(self.store)?.db.connection().query_row(
+ "SELECT selected FROM desired_albums WHERE id=?1",
+ [pending.album],
+ |r| r.get(0),
+ )?;
+ if !selected
+ || Properties::optional(&self.path(&temporary.location)?)?.is_none()
+ {
+ return Ok(());
}
+ self.clear_action(&mut pending)?;
}
+ self.discard(&temporary.location)
+ })();
+ if let Err(error) = result {
+ if let Some(file) = temporary.file {
+ let selected: bool = store::lock(self.store)?.db.connection().query_row(
+ "SELECT EXISTS(SELECT 1 FROM desired_files WHERE file=?1)",
+ [file],
+ |r| r.get(0),
+ )?;
+ if selected {
+ self.block_file(file, error)?;
+ continue;
+ }
+ }
+ self.block(temporary.album, temporary.file, error)?;
}
}
- for (old, new) in previous.media.iter().zip(&placement.media) {
- if old.component.path != new.component.path {
- store.operation(&Operation::Move {
- album: album_id,
- file: Some(file.id),
- source: format!("{}/metadata/{}.json", album.path, old.component.path),
- destination: format!("{}/metadata/{}.json", album.path, new.component.path),
- })?;
- store.db.connection().execute(
- "DELETE FROM json_records WHERE path=?1",
- [format!(
- "{}/metadata/{}.json",
- album.path, old.component.path
- )],
- )?;
- }
- }
- transaction.commit()?;
- fs::operations(root, store, album_id, Some(file.id))?;
- } else {
- if existing.is_none() {
- for media in &placement.media {
- free_target(root, &format!("{}/{}", album.path, media.component.path))?;
- free_target(
- root,
- &format!("{}/metadata/{}.json", album.path, media.component.path),
- )?;
+ Ok(())
+ }
+
+ fn album_target(&self, album: i64, name: &str, retained: bool) -> Result<Pending> {
+ let store = store::lock(self.store)?;
+ let existing = store.album(album, retained)?;
+ let owner = existing
+ .as_ref()
+ .map(|album| album.key.clone())
+ .unwrap_or_else(|| format!("{}:{album}", if retained { "retained" } else { "active" }));
+ if let Some(pending) = store.pending(&owner)? {
+ ensure!(pending.action.is_none(), "album has unfinished action");
+ if pending.name == name || retained {
+ return Ok(pending);
}
}
let transaction = store.db.connection().unchecked_transaction()?;
- store.save_placement(&placement)?;
- allocation::clear(store, &owner)?;
- transaction.commit()?;
- }
- for media in &placement.media {
- names::reserve(
- store,
- &placement.folder,
- &media.component.path,
- &owner,
- file.kind.name(),
- )?;
- }
- let mut wrote_media = false;
- for media in &mut placement.media {
- let relative = format!("{}/{}", album.path, media.component.path);
- let path = names::check(root, &relative)?;
- let properties = if path.try_exists()? {
- Some(Properties::read(&path)?)
+ let parent = if retained { "Trash" } else { "" };
+ let names = if let Some(existing) = &existing
+ && (retained || existing.name == name)
+ {
+ vec![
+ existing
+ .path
+ .rsplit('/')
+ .next()
+ .context("empty album path")?
+ .to_owned(),
+ ]
} else {
- None
+ names::allocate(&store, parent, name, "album", None, &owner)?
};
- let valid = if let Some(properties) = &properties {
- if media.properties.as_ref() == Some(properties)
- && existing.as_ref().is_some_and(|old| {
- old.media.iter().any(|m| {
- m.component.role == media.component.role
- && m.component.hash == media.component.hash
- })
- })
+ let target = Pending {
+ owner,
+ album,
+ file: None,
+ retained,
+ name: name.into(),
+ kind: "album".into(),
+ folder: parent.into(),
+ names,
+ action: None,
+ };
+ store.save_pending(&target)?;
+ transaction.commit()?;
+ Ok(target)
+ }
+
+ fn maintain_album(
+ &self,
+ album: i64,
+ name: &str,
+ retained: bool,
+ metadata: Value,
+ ) -> Result<Album> {
+ let existing = store::lock(self.store)?.album(album, retained)?;
+ let json = fs::Json::new(metadata)?;
+ if let Some(existing) = &existing
+ && (retained || existing.name == name)
+ {
+ let pending = store::lock(self.store)?.pending(&existing.key)?;
+ let cached = store::lock(self.store)?.json_record(&existing.key, None)?;
+ let location = Location {
+ folder: existing.key.clone(),
+ name: "metadata.json".into(),
+ };
+ if pending.is_none()
+ && let Some(cached) = cached
{
- true
+ let properties = Properties::optional(&self.path(&location)?)?;
+ if cached.location.as_ref() == Some(&location)
+ && cached.hash.as_ref() == Some(&json.hash)
+ && properties.is_some()
+ && cached.properties == properties
+ {
+ return Ok(existing.clone());
+ }
+ }
+ }
+ let mut target = self.album_target(album, name, retained)?;
+ let target_path = target.album_path();
+ if existing
+ .as_ref()
+ .is_none_or(|album| album.path != target_path)
+ {
+ let source = existing.as_ref().map(|album| album.path.clone());
+ let destination = names::check(self.root, &target_path)?;
+ if let Some(source) = &source {
+ free_rename(&names::check(self.root, source)?, &destination)?;
} else {
- let (size, hash) = fs::hash_cancellable(&path, cancel)?;
- size == media.component.size && hash == media.component.hash
+ ensure!(
+ !destination.try_exists()?,
+ super::Conflict(format!("at {}: unrelated directory", destination.display()))
+ );
+ }
+ self.start(
+ &mut target,
+ Action::Directory { source },
+ &[(None, json.value.clone())],
+ )?;
+ self.finish_action(&mut target, false, None)?;
+ if existing.is_some() {
+ let store = store::lock(self.store)?;
+ store
+ .db
+ .connection()
+ .execute(
+ "INSERT INTO events(placement,renamed) SELECT id,1 FROM placements WHERE album=?1 AND retained=0 ON CONFLICT(placement) DO UPDATE SET renamed=1",
+ [album],
+ )?;
}
} else {
- false
+ fs::directory(self.root, &target_path)?;
+ }
+ let location = Location {
+ folder: target.owner.clone(),
+ name: "metadata.json".into(),
};
- if !valid {
- let source = prepared
- .components
- .iter_mut()
- .find(|c| c.role == media.component.role)
- .context("no verified original is available")?;
- if source.temporary {
- let temporary: super::store::Temporary = store
- .json(
- "SELECT record FROM temporaries WHERE path=?1",
- [&source.path],
- )?
- .context("missing transfer ownership")?;
- fs::publish(
- root,
- store,
- temporary,
- &relative,
- source.size,
- source.hash.clone(),
- )?;
- source.path = relative.clone();
- source.temporary = false;
- } else {
- transfer::materialize(
- root,
- store,
- source,
- (&album.path, &media.component.path),
- file.id,
- &prepared.signature,
- cancel,
- )?;
+ self.write_json(&mut target, None, location, json)?;
+ let store = store::lock(self.store)?;
+ let mut album = store.folder(&target.owner)?;
+ album.name = target.name;
+ store.save_album(&album)?;
+ store.clear_pending(&target.owner)?;
+ Ok(album)
+ }
+
+ pub fn albums(&self) -> Result<()> {
+ let mut after = 0;
+ loop {
+ let source: Option<(i64, String)> = store::lock(self.store)?
+ .db
+ .connection()
+ .query_row(
+ "SELECT a.id,s.record FROM desired_albums a JOIN album_sources s ON s.id=a.id WHERE a.id>?1 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND a.present=1 ORDER BY a.id LIMIT 1",
+ [after],
+ |r| Ok((r.get(0)?, r.get(1)?)),
+ )
+ .optional()?;
+ let Some((id, source)) = source else {
+ break;
+ };
+ after = id;
+ let source: AlbumSource = serde_json::from_str(&source)?;
+ for field in &source.warnings {
+ eprintln!(
+ "album {id}: unfamiliar metadata field {field:?}; exporting supported fields"
+ );
+ }
+ if let Err(error) = self.maintain_album(id, &source.name, false, source.metadata) {
+ self.block(id, None, error)?;
}
- wrote_media = true;
}
- let observed = Properties::read(&path)?;
- let intended = file.modified_at_micros.unwrap_or(file.created_at_micros);
- media.properties = Some(
- if media.intended_time == Some(intended) && media.properties.as_ref() == Some(&observed)
- {
- observed
- } else {
- fs::set_time(&path, intended)?
- },
- );
- media.intended_time = Some(intended);
- }
- store.save_placement(&placement)?;
- let mut metadata_changed = false;
- for (media, record) in placement.media.iter().zip(&placement.metadata) {
- metadata_changed |= fs::json(
- root,
- store,
- &format!("{}/metadata/{}.json", album.path, media.component.path),
- record,
- true,
- Some(file.id),
- )?;
- }
- placement.complete = true;
- store.save_placement(&placement)?;
- store.db.connection().execute(
- "UPDATE desired_files SET completed=1 WHERE album=?1 AND file=?2",
- params![album_id, file.id],
- )?;
- if wrote_media || !old_complete {
- event(store, &placement.id, "exported")?;
- }
- if existing.is_some() && metadata_changed {
- event(store, &placement.id, "metadata_updated")?;
+ Ok(())
}
- if renamed {
- event(store, &placement.id, "renamed")?;
- }
- Ok(())
}
-pub fn retain_removed(root: &Path, store: &Store) -> Result<()> {
- let mut after = String::new();
- loop {
- let next:Option<Placement>=store.json("SELECT p.record FROM placements p JOIN desired_albums a ON a.id=p.album WHERE p.retained=0 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND p.id>?1 AND NOT EXISTS(SELECT 1 FROM desired_files d WHERE d.album=p.album AND d.file=p.file) ORDER BY p.id LIMIT 1",[&after])?;
- let Some(placement) = next else { break };
- after = placement.id.clone();
- let result = retain(root, store, &placement);
- if let Err(error) = result {
- report(
- store,
- &format!("file:{}:{}", placement.album, placement.file),
- &error,
- )?;
- if fatal(&error) {
- return Err(error);
- }
- }
- }
- let mut after = String::new();
- loop {
- let next:Option<Placement>=store.json("SELECT p.record FROM placements p JOIN desired_albums a ON a.id=p.album WHERE p.retained=1 AND json_extract(p.record,'$.retaining')=1 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND NOT EXISTS(SELECT 1 FROM operations o WHERE o.album=p.album AND o.file=p.file) AND p.id>?1 ORDER BY p.id LIMIT 1",[&after])?;
- let Some(mut placement) = next else { break };
- after = placement.id.clone();
- let result = (|| {
- for (component, record) in placement.media.iter().zip(&placement.metadata) {
- fs::json(
- root,
- store,
- &format!(
- "{}/metadata/{}.json",
- placement.folder, component.component.path
- ),
- record,
- true,
- Some(placement.file),
- )?;
- }
- placement.retaining = false;
- placement.complete = true;
- store.save_placement(&placement)?;
- event(store, &placement.id, "retained")?;
- Ok::<_, anyhow::Error>(())
- })();
- if let Err(error) = result {
- report(
- store,
- &format!("file:{}:{}", placement.album, placement.file),
- &error,
- )?;
- if fatal(&error) {
- return Err(error);
+impl Run<'_> {
+ fn file_target(
+ &self,
+ album: &Album,
+ file: &files::File,
+ prepared: &transfer::Prepared,
+ ) -> Result<Pending> {
+ let store = store::lock(self.store)?;
+ let existing = store.placement(album.id, file.id)?;
+ let previous = store.pending_file(album.id, file.id)?;
+ let owner = existing
+ .as_ref()
+ .map(|placement| placement.id.clone())
+ .or_else(|| previous.as_ref().map(|pending| pending.owner.clone()))
+ .unwrap_or_else(|| Uuid::new_v4().to_string());
+ let components = store.components(&owner)?;
+ let roles = transfer::roles(file);
+ let extensions: Vec<String> = roles
+ .iter()
+ .map(|role| {
+ prepared
+ .components
+ .iter()
+ .find(|source| source.role == *role)
+ .map(|source| source.extension.clone())
+ .or_else(|| {
+ components
+ .iter()
+ .find(|component| component.role == *role)
+ .map(|component| {
+ names::split(&component.location.name, false).1.to_owned()
+ })
+ })
+ .context("missing verified original")
+ })
+ .collect::<Result<_>>()?;
+ let compatible = |names: &[String]| {
+ names.len() == roles.len()
+ && (roles.len() == 1
+ || names.iter().zip(&extensions).all(|(name, extension)| {
+ names::split(name, false).1 == names::portable(extension)
+ }))
+ };
+ if let Some(previous) = previous {
+ ensure!(
+ previous.action.is_none() && !previous.retained,
+ "file has unfinished transition"
+ );
+ let same_folder = previous.folder == album.key;
+ if same_folder
+ && previous.name == file.name
+ && previous.kind == file.kind.name()
+ && compatible(&previous.names)
+ {
+ return Ok(previous);
}
}
+ let existing_names: Vec<_> = components
+ .iter()
+ .map(|component| component.location.name.clone())
+ .collect();
+ let unchanged = existing.as_ref().is_some_and(|placement| {
+ placement.name == file.name && placement.kind == file.kind.name()
+ }) && components
+ .iter()
+ .all(|component| component.location.folder == album.key)
+ && compatible(&existing_names);
+ let transaction = store.db.connection().unchecked_transaction()?;
+ let names = if unchanged {
+ existing_names
+ } else {
+ names::allocate(
+ &store,
+ &album.key,
+ &file.name,
+ file.kind.name(),
+ if roles.len() == 2 {
+ Some((&extensions[0], &extensions[1]))
+ } else {
+ None
+ },
+ &owner,
+ )?
+ };
+ let target = Pending {
+ owner,
+ album: album.id,
+ file: Some(file.id),
+ retained: false,
+ name: file.name.clone(),
+ kind: file.kind.name().into(),
+ folder: album.key.clone(),
+ names,
+ action: None,
+ };
+ if !unchanged {
+ store.save_pending(&target)?;
+ } else if store.pending(&target.owner)?.is_some() {
+ store.clear_pending(&target.owner)?;
+ }
+ transaction.commit()?;
+ Ok(target)
}
- let mut after = 0;
- loop {
- let id:Option<i64>=store.db.connection().query_row("SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL AND record IS NULL AND id>?1 ORDER BY id LIMIT 1",[after],|r|r.get(0)).optional()?;
- let Some(id) = id else { break };
- after = id;
- let result = (|| {
- let Some(album) = store.album(id, false)? else {
- return Ok(());
+
+ pub async fn file(&self, session: &ente_core::Session, file: files::File) -> Result<()> {
+ let mut prepared = transfer::prepare(self, session, &file).await?;
+ let mut after = 0;
+ loop {
+ let album: Option<i64> = store::lock(self.store)?
+ .db
+ .connection()
+ .query_row(
+ "SELECT album FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",
+ params![file.id, after],
+ |r| r.get(0),
+ )
+ .optional()?;
+ let Some(album) = album else {
+ break;
};
- retained_album(root, store, &album)?;
- store.operation(&Operation::Remove {
- album: id,
- path: format!("{}/metadata.json", album.path),
- })?;
- fs::operations(root, store, id, None)?;
- for relative in [format!("{}/metadata", album.path), album.path] {
- let path = names::check(root, &relative)?;
- if path.is_dir() && disk::read_dir(&path)?.next().is_none() {
- disk::remove_dir(path)?;
+ after = album;
+ let pending = store::lock(self.store)?.pending_file(album, file.id)?;
+ if let Some(mut pending) = pending
+ && pending.retained
+ {
+ let components = store::lock(self.store)?.components(&pending.owner)?;
+ if components
+ .iter()
+ .any(|component| component.location.folder == pending.folder)
+ {
+ if let Err(error) = self.finish_retention(&mut pending) {
+ self.block(album, Some(file.id), error)?;
+ } else {
+ self.retired_sources(&components, &mut prepared, album)?;
+ }
+ } else {
+ store::lock(self.store)?.clear_pending(&pending.owner)?;
}
}
- Ok::<_, anyhow::Error>(())
- })();
- if let Err(error) = result {
- report(store, &format!("album:{id}"), &error)?;
- if fatal(&error) {
- return Err(error);
+ }
+ for field in &file.warnings {
+ eprintln!(
+ "file {}: unfamiliar metadata field {field:?}; exporting supported fields",
+ file.id
+ );
+ }
+ after = 0;
+ loop {
+ self.check_cancel()?;
+ let next: Option<(i64, bool)> = store::lock(self.store)?
+ .db
+ .connection()
+ .query_row(
+ "SELECT album,favorited FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",
+ params![file.id, after],
+ |r| Ok((r.get(0)?, r.get(1)?)),
+ )
+ .optional()?;
+ let Some((album, favorited)) = next else {
+ break;
+ };
+ after = album;
+ if let Err(error) = self.placement(&file, &mut prepared, album, favorited) {
+ self.block(album, Some(file.id), error)?;
}
}
+ Ok(())
}
- Ok(())
-}
-fn retain(root: &Path, store: &Store, active: &Placement) -> Result<()> {
- let album = store
- .album(active.album, false)?
- .context("missing removed album")?;
- let retained_album = retained_album(root, store, &album)?;
- let missing_folder = !names::check(root, &retained_album.path)?.try_exists()?;
- fs::directory(root, &retained_album.path)?;
- if missing_folder {
- fs::json(
- root,
- store,
- &format!("{}/metadata.json", retained_album.path),
- &retained_album.metadata,
- true,
- None,
- )?;
+ fn retired_sources(
+ &self,
+ previous: &[Component],
+ prepared: &mut transfer::Prepared,
+ album: i64,
+ ) -> Result<()> {
+ for previous in previous {
+ let current = store::lock(self.store)?.components(&previous.placement)?;
+ if let Some(current) = current.iter().find(|current| current.role == previous.role) {
+ for source in &mut prepared.components {
+ if source.location == previous.location {
+ source.location = current.location.clone();
+ }
+ }
+ }
+ }
+ prepared.observed.retain(|(id, _)| *id != album);
+ Ok(())
}
- let mut retained = active.clone();
- retained.id = format!("retain:{}", active.id);
- retained.retained = true;
- retained.folder = retained_album.path;
- retained.complete = false;
- retained.retaining = true;
- let extensions = if active.media.len() == 2 {
- Some((
- names::split(&active.media[0].component.path, false).1,
- names::split(&active.media[1].component.path, false).1,
- ))
- } else {
- None
- };
- let names = allocation::choose(
- store,
- &retained.folder,
- &active.name,
- active.kind()?,
- extensions,
- &retained.id,
- )?;
- for (media, name) in retained.media.iter_mut().zip(names) {
- media.component.path = name;
- media.properties = None;
- }
- let components: Vec<_> = retained.media.iter().map(|m| m.component.clone()).collect();
- for (index, record) in retained.metadata.iter_mut().enumerate() {
- if record.is_null() {
- continue;
- }
- record["title"] = json_string(&components[index].path);
- record["ente"]["components"] = serde_json::to_value(&components)?;
- }
- for media in &retained.media {
- free_target(
- root,
- &format!("{}/{}", retained.folder, media.component.path),
- )?;
- free_target(
- root,
- &format!("{}/metadata/{}.json", retained.folder, media.component.path),
+
+ fn placement(
+ &self,
+ file: &files::File,
+ prepared: &mut transfer::Prepared,
+ album_id: i64,
+ favorited: bool,
+ ) -> Result<()> {
+ let album = store::lock(self.store)?
+ .album(album_id, false)?
+ .context("missing destination album")?;
+ let existing = store::lock(self.store)?.placement(album_id, file.id)?;
+ if let Some(existing) = existing {
+ let components = store::lock(self.store)?.components(&existing.id)?;
+ if components
+ .iter()
+ .any(|component| !transfer::roles(file).contains(&component.role))
+ {
+ self.retain(existing)?;
+ self.retired_sources(&components, prepared, album_id)?;
+ }
+ }
+ let existing = store::lock(self.store)?.placement(album_id, file.id)?;
+ let mut target = self.file_target(&album, file, prepared)?;
+ let old = store::lock(self.store)?.components(&target.owner)?;
+ let intended: Vec<_> = transfer::roles(file)
+ .into_iter()
+ .zip(&target.names)
+ .map(|(role, name)| {
+ let (size, hash) = if let Some(source) = prepared
+ .components
+ .iter()
+ .find(|source| source.role == role)
+ {
+ (source.size, source.hash.clone())
+ } else {
+ let component = old
+ .iter()
+ .find(|component| component.role == role)
+ .context("missing completed original")?;
+ (component.size, component.hash.clone())
+ };
+ Ok(ente_photos::export::Component {
+ role,
+ path: name.clone(),
+ size,
+ hash,
+ })
+ })
+ .collect::<Result<_>>()?;
+ let metadata: Vec<_> = intended
+ .iter()
+ .enumerate()
+ .map(|(index, component)| {
+ Ok((
+ Some(component.role.clone()),
+ ente_photos::export::file(file, &intended, index, favorited)?,
+ ))
+ })
+ .collect::<Result<_>>()?;
+ let mut sidecars = Vec::new();
+ for (component, (_, value)) in intended.iter().zip(&metadata) {
+ let location = Location {
+ folder: album.key.clone(),
+ name: component.path.clone(),
+ };
+ let previous = old.iter().find(|previous| previous.role == component.role);
+ let observed = prepared
+ .observed
+ .iter()
+ .find(|(id, _)| *id == album_id)
+ .is_some_and(|(_, observed)| {
+ observed.iter().any(|(role, _)| *role == component.role)
+ });
+ if !observed || previous.is_none_or(|previous| previous.location != location) {
+ let path = self.path(&location)?;
+ let owned = previous.is_some_and(|previous| previous.location == location);
+ let renamed = previous
+ .map(|previous| {
+ self.path(&previous.location)
+ .map(|source| names::same_path(&source, &path))
+ })
+ .transpose()?
+ .unwrap_or(false);
+ ensure!(
+ Properties::optional(&path)?.is_none() || owned || renamed,
+ super::Conflict(format!("at {}: unrelated occupant", path.display()))
+ );
+ }
+ let location = Location {
+ folder: album.key.clone(),
+ name: format!("metadata/{}.json", component.path),
+ };
+ let json = fs::Json::new(value.clone())?;
+ let previous =
+ store::lock(self.store)?.json_record(&target.owner, Some(&component.role))?;
+ let changed = self.json_changed(previous.as_ref(), &location, &json)?;
+ sidecars.push((location, json, changed, previous));
+ }
+ let intended_time = file.modified_at_micros.unwrap_or(file.created_at_micros);
+ let mut wrote_media = false;
+ let mut renamed = false;
+ for component in &intended {
+ self.check_cancel()?;
+ let destination = Location {
+ folder: album.key.clone(),
+ name: component.path.clone(),
+ };
+ let previous = old.iter().find(|previous| previous.role == component.role);
+ let observed = prepared
+ .observed
+ .iter()
+ .find(|(id, _)| *id == album_id)
+ .and_then(|(_, observed)| observed.iter().find(|(role, _)| *role == component.role))
+ .map(|(_, properties)| properties);
+ let valid = observed.is_some();
+ if let Some(previous) = previous
+ && previous.location != destination
+ {
+ let moved =
+ self.move_media(&mut target, previous, destination.clone(), observed)?;
+ if moved {
+ renamed = true;
+ for source in &mut prepared.components {
+ if source.location == previous.location {
+ source.location = destination.clone();
+ }
+ }
+ }
+ }
+ if !valid {
+ let source = prepared
+ .components
+ .iter_mut()
+ .find(|source| source.role == component.role)
+ .context("no verified original is available")?;
+ let temporary = if source.temporary {
+ source.location.clone()
+ } else {
+ let (temporary, file) = self.temporary(&album.key, album_id, Some(file.id))?;
+ let mut writer = fs::HashWriter::new(file, self.cancel)?;
+ std::io::copy(&mut File::open(self.path(&source.location)?)?, &mut writer)?;
+ let (_, size, hash) = writer.finish()?;
+ ensure!(
+ size == source.size && hash == source.hash,
+ "local original changed while copying"
+ );
+ temporary
+ };
+ let output = Output::Media {
+ role: component.role.clone(),
+ size: source.size,
+ hash: source.hash.clone(),
+ signature: Some(prepared.signature.clone()),
+ };
+ self.start(
+ &mut target,
+ Action::Publish {
+ temporary,
+ destination: destination.clone(),
+ output,
+ },
+ &metadata,
+ )?;
+ self.finish_action(&mut target, false, None)?;
+ source.location = destination.clone();
+ source.temporary = false;
+ wrote_media = true;
+ }
+ let mut published = store::lock(self.store)?
+ .components(&target.owner)?
+ .into_iter()
+ .find(|published| published.role == component.role)
+ .context("missing published component")?;
+ let previous = published.clone();
+ if published.intended_time != Some(intended_time)
+ || observed != published.properties.as_ref()
+ || renamed
+ {
+ published.properties =
+ Some(fs::set_time(&self.path(&destination)?, intended_time)?);
+ }
+ published.intended_time = Some(intended_time);
+ published.signature = Some(prepared.signature.clone());
+ if published != previous {
+ store::lock(self.store)?.save_component(&published)?;
+ }
+ }
+ let mut metadata_changed = false;
+ for (component, (location, json, changed, previous)) in intended.iter().zip(sidecars) {
+ if changed {
+ renamed |= self.replace_json(
+ &mut target,
+ Some(component.role.clone()),
+ location,
+ json,
+ previous,
+ )?;
+ metadata_changed = true;
+ }
+ }
+ let store = store::lock(self.store)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ if existing
+ .as_ref()
+ .is_some_and(|previous| previous.name != target.name || previous.kind != target.kind)
+ {
+ store.save_placement(&target.placement()?)?;
+ }
+ store.clear_pending(&target.owner)?;
+ store.db.connection().execute(
+ "UPDATE desired_files SET completed=1 WHERE album=?1 AND file=?2",
+ params![album_id, file.id],
)?;
+ if wrote_media || existing.is_none() {
+ event(&store, &target.owner, "exported")?;
+ }
+ if existing.is_some() && metadata_changed {
+ event(&store, &target.owner, "metadata_updated")?;
+ }
+ if renamed {
+ event(&store, &target.owner, "renamed")?;
+ }
+ transaction.commit()?;
+ Ok(())
}
- let transaction = store.db.connection().unchecked_transaction()?;
- store.save_placement(&retained)?;
- allocation::clear(store, &retained.id)?;
- allocation::clear(store, &format!("file:{}:{}", active.album, active.file))?;
+}
+
+fn event(store: &Store, placement: &str, name: &str) -> Result<()> {
store
.db
.connection()
- .execute("DELETE FROM placements WHERE id=?1", [&active.id])?;
- for (old, new) in active.media.iter().zip(&retained.media) {
- store.operation(&Operation::Move {
- album: active.album,
- file: Some(active.file),
- source: format!("{}/{}", active.folder, old.component.path),
- destination: format!("{}/{}", retained.folder, new.component.path),
- })?;
- }
- for (old, new) in active.media.iter().zip(&retained.media) {
- store.operation(&Operation::Move {
- album: active.album,
- file: Some(active.file),
- source: format!("{}/metadata/{}.json", active.folder, old.component.path),
- destination: format!("{}/metadata/{}.json", retained.folder, new.component.path),
- })?;
- store.db.connection().execute(
- "DELETE FROM json_records WHERE path=?1",
- [format!(
- "{}/metadata/{}.json",
- active.folder, old.component.path
- )],
+ .execute(
+ &format!(
+ "INSERT INTO events(placement,{name}) VALUES(?1,1) ON CONFLICT(placement) DO UPDATE SET {name}=1",
+ ),
+ [placement],
)?;
- }
- transaction.commit()?;
- fs::operations(root, store, active.album, Some(active.file))?;
Ok(())
}
-fn retained_album(root: &Path, store: &Store, active: &Album) -> Result<Album> {
- if let Some(mut album) = store.album(active.id, true)? {
- if !album.initialized {
- initialize_retained(root, store, &mut album)?;
- }
- return Ok(album);
+impl Run<'_> {
+ fn retained_album(&self, active: &Album) -> Result<Album> {
+ let existing = store::lock(self.store)?.album(active.id, true)?;
+ let (name, value) = if let Some(existing) = existing {
+ let record = store::lock(self.store)?
+ .json_record(&existing.key, None)?
+ .context("missing retained album snapshot")?;
+ (existing.name, record.value)
+ } else {
+ let record = store::lock(self.store)?
+ .json_record(&active.key, None)?
+ .context("missing album snapshot")?;
+ (active.name.clone(), record.value)
+ };
+ self.maintain_album(active.id, &name, true, value)
}
- let key = format!("retained:{}", active.id);
- let path = format!(
- "Trash/{}",
- allocation::choose(store, "Trash", &active.name, "album", None, &key)?.remove(0)
- );
- free_target(root, &path)?;
- let mut album = Album {
- key,
- id: active.id,
- name: active.name.clone(),
- path,
- retained: true,
- metadata: active.metadata.clone(),
- initialized: false,
- };
- let transaction = store.db.connection().unchecked_transaction()?;
- store.save_album(&album)?;
- allocation::clear(store, &album.key)?;
- transaction.commit()?;
- initialize_retained(root, store, &mut album)?;
- Ok(album)
-}
-
-fn initialize_retained(root: &Path, store: &Store, album: &mut Album) -> Result<()> {
- fs::directory(root, &album.path)?;
- fs::json(
- root,
- store,
- &format!("{}/metadata.json", album.path),
- &album.metadata,
- true,
- None,
- )?;
- album.initialized = true;
- store.save_album(album)
-}
-fn relocate_work(store: &Store, old: &str, new: &str) -> Result<()> {
- let prefix = format!("{old}/");
- let relocate = |path: &mut String| {
- if let Some(tail) = path.strip_prefix(&prefix) {
- *path = format!("{new}/{tail}");
+ fn retain(&self, placement: Placement) -> Result<()> {
+ let mut pending = store::lock(self.store)?.pending(&placement.id)?;
+ if let Some(target) = &mut pending {
+ self.finish_action(target, true, None)?;
+ if target.retained {
+ return self.finish_retention(target);
+ }
}
- };
- let mut query = store
- .db
- .connection()
- .prepare("SELECT record FROM temporaries WHERE substr(path,1,?1)=?2")?;
- let mut rows = query.query(params![prefix.chars().count() as i64, prefix])?;
- while let Some(row) = rows.next()? {
- let mut temporary: super::store::Temporary =
- serde_json::from_str(&row.get::<_, String>(0)?)?;
- store
- .db
- .connection()
- .execute("DELETE FROM temporaries WHERE path=?1", [&temporary.path])?;
- relocate(&mut temporary.path);
- if let Some(destination) = &mut temporary.destination {
- relocate(destination);
+ let components = store::lock(self.store)?.components(&placement.id)?;
+ for component in &components {
+ Properties::read(&self.path(&component.location)?)
+ .context("previously published original is missing")?;
}
- store.temporary(&temporary)?;
+ let active = store::lock(self.store)?
+ .album(placement.album, false)?
+ .context("missing active album")?;
+ let album = self.retained_album(&active)?;
+ let template = components
+ .first()
+ .context("placement has no published components")?;
+ let metadata = store::lock(self.store)?
+ .json_record(&placement.id, Some(&template.role))?
+ .context("missing retained metadata snapshot")?;
+ let layout: Vec<ente_photos::export::Component> =
+ serde_json::from_value(metadata.value["ente"]["components"].clone())?;
+ let extensions = if placement.kind == "livephoto" {
+ Some((
+ names::split(
+ &layout
+ .iter()
+ .find(|component| component.role == Role::Image)
+ .context("missing image identity")?
+ .path,
+ false,
+ )
+ .1,
+ names::split(
+ &layout
+ .iter()
+ .find(|component| component.role == Role::Video)
+ .context("missing video identity")?
+ .path,
+ false,
+ )
+ .1,
+ ))
+ } else {
+ None
+ };
+ let mut target = {
+ let store = store::lock(self.store)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ let names = names::allocate(
+ &store,
+ &album.key,
+ &placement.name,
+ &placement.kind,
+ extensions,
+ &placement.id,
+ )?;
+ let target = Pending {
+ owner: placement.id,
+ album: placement.album,
+ file: Some(placement.file),
+ retained: true,
+ name: placement.name,
+ kind: placement.kind,
+ folder: album.key,
+ names,
+ action: None,
+ };
+ store.save_pending(&target)?;
+ transaction.commit()?;
+ target
+ };
+ self.finish_retention(&mut target)
}
- let mut query = store
- .db
- .connection()
- .prepare("SELECT id,record FROM operations ORDER BY id")?;
- let mut rows = query.query([])?;
- while let Some(row) = rows.next()? {
- let id: i64 = row.get(0)?;
- let mut operation: Operation = serde_json::from_str(&row.get::<_, String>(1)?)?;
- match &mut operation {
- Operation::Move {
- source,
- destination,
- ..
- } => {
- relocate(source);
- relocate(destination);
+
+ fn finish_retention(&self, target: &mut Pending) -> Result<()> {
+ let components = store::lock(self.store)?.components(&target.owner)?;
+ for component in &components {
+ let name = target_component(target, &component.role)?;
+ let location = Location {
+ folder: target.folder.clone(),
+ name: name.into(),
+ };
+ self.move_media(target, component, location, component.properties.as_ref())?;
+ }
+ let components = store::lock(self.store)?.components(&target.owner)?;
+ for component in &components {
+ Properties::read(&self.path(&component.location)?)
+ .context("retained original is missing")?;
+ let previous = store::lock(self.store)?
+ .json_record(&target.owner, Some(&component.role))?
+ .context("missing retained metadata")?;
+ let location = Location {
+ folder: target.folder.clone(),
+ name: format!("metadata/{}.json", component.location.name),
+ };
+ let mut value = previous.value;
+ value["title"] = Value::String(component.location.name.clone());
+ let mut layout: Vec<ente_photos::export::Component> =
+ serde_json::from_value(value["ente"]["components"].clone())?;
+ for identity in &mut layout {
+ identity.path = target_component(target, &identity.role)?.into();
+ if let Some(recorded) = components
+ .iter()
+ .find(|component| component.role == identity.role)
+ {
+ identity.size = recorded.size;
+ identity.hash = recorded.hash.clone();
+ }
}
- Operation::Remove { path, .. } => relocate(path),
+ value["ente"]["components"] = serde_json::to_value(layout)?;
+ self.write_json(
+ target,
+ Some(component.role.clone()),
+ location,
+ fs::Json::new(value)?,
+ )?;
}
- store.db.connection().execute(
- "UPDATE operations SET record=?1 WHERE id=?2",
- params![serde_json::to_string(&operation)?, id],
- )?;
+ let store = store::lock(self.store)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store.save_placement(&target.placement()?)?;
+ store.clear_pending(&target.owner)?;
+ event(&store, &target.owner, "retained")?;
+ transaction.commit()?;
+ Ok(())
}
- Ok(())
-}
-
-fn rename_target(root: &Path, source: &str, destination: &str) -> Result<()> {
- let source = names::check(root, source)?;
- let destination = names::check(root, destination)?;
- ensure!(
- !destination.try_exists()? || names::same_path(&source, &destination),
- super::Conflict(format!("occupied rename target {}", destination.display()))
- );
- Ok(())
-}
-fn free_target(root: &Path, relative: &str) -> Result<()> {
- let path = names::check(root, relative)?;
- ensure!(
- !path.try_exists()?,
- super::Conflict(format!("at {}: unrelated or occupied path", path.display()))
- );
- Ok(())
-}
+ pub fn retain_removed(&self) -> Result<()> {
+ let mut after = String::new();
+ loop {
+ self.check_cancel()?;
+ let next = {
+ let store = store::lock(self.store)?;
+ store
+ .db
+ .connection()
+ .query_row(
+ "SELECT p.id,p.album,p.file,p.retained,p.name,p.kind FROM placements p JOIN desired_albums a ON a.id=p.album WHERE p.id>?1 AND p.retained=0 AND a.selected=1 AND a.ready=1 AND a.failure IS NULL AND NOT EXISTS(SELECT 1 FROM desired_files f WHERE f.album=p.album AND f.file=p.file) AND NOT EXISTS(SELECT 1 FROM outcomes WHERE unit='file:'||p.album||':'||p.file) ORDER BY p.id LIMIT 1",
+ [&after],
+ Placement::read,
+ )
+ .optional()?
+ };
+ let Some(placement) = next else {
+ break;
+ };
+ after = placement.id.clone();
+ let (album, file) = (placement.album, placement.file);
+ if let Err(error) = self.retain(placement) {
+ self.block(album, Some(file), error)?;
+ }
+ }
+ {
+ let store = store::lock(self.store)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store
+ .db
+ .connection()
+ .execute(
+ "DELETE FROM pending WHERE file IS NOT NULL AND action='null' AND album IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM placements WHERE id=pending.owner) AND NOT EXISTS(SELECT 1 FROM desired_files WHERE album=pending.album AND file=pending.file)",
+ [],
+ )?;
+ store
+ .db
+ .connection()
+ .execute(
+ "DELETE FROM json_records WHERE folder IS NULL AND NOT EXISTS(SELECT 1 FROM albums WHERE key=json_records.owner) AND NOT EXISTS(SELECT 1 FROM components WHERE placement=json_records.owner) AND NOT EXISTS(SELECT 1 FROM pending WHERE owner=json_records.owner)",
+ [],
+ )?;
+ transaction.commit()?;
+ }
+ let mut after = 0;
+ loop {
+ self.check_cancel()?;
+ let id: Option<i64> = store::lock(self.store)?
+ .db
+ .connection()
+ .query_row(
+ "SELECT id FROM desired_albums WHERE id>?1 AND selected=1 AND ready=1 AND failure IS NULL AND present=0 ORDER BY id LIMIT 1",
+ [after],
+ |r| r.get(0),
+ )
+ .optional()?;
+ let Some(id) = id else {
+ break;
+ };
+ after = id;
+ if let Err(error) = self.remove_album(id) {
+ self.block(id, None, error)?;
+ }
+ }
+ Ok(())
+ }
-fn event(store: &Store, placement: &str, event: &str) -> Result<()> {
- ensure!(
- ["exported", "metadata_updated", "renamed", "retained"].contains(&event),
- "unknown export event"
- );
- store.db.connection().execute(&format!("INSERT INTO events(placement,{event}) VALUES(?1,1) ON CONFLICT(placement) DO UPDATE SET {event}=1"),[placement])?;
- Ok(())
+ fn remove_album(&self, id: i64) -> Result<()> {
+ let active = store::lock(self.store)?.album(id, false)?;
+ let Some(active) = active else {
+ let store = store::lock(self.store)?;
+ store.clear_pending(&format!("active:{id}"))?;
+ return Ok(());
+ };
+ let incomplete: bool = store::lock(self.store)?.db.connection().query_row(
+ "SELECT EXISTS(SELECT 1 FROM placements WHERE album=?1 AND retained=0) OR EXISTS(SELECT 1 FROM pending WHERE album=?1 AND file IS NOT NULL)",
+ [id],
+ |r| r.get(0),
+ )?;
+ ensure!(!incomplete, "album has unfinished retention");
+ self.retained_album(&active)?;
+ let mut target = self.album_target(id, &active.name, false)?;
+ let record = store::lock(self.store)?.json_record(&active.key, None)?;
+ if let Some(record) = record
+ && let Some(location) = record.location
+ {
+ self.start(
+ &mut target,
+ Action::Remove {
+ location,
+ role: None,
+ },
+ &[],
+ )?;
+ self.finish_action(&mut target, false, None)?;
+ }
+ for relative in [format!("{}/metadata", active.path), active.path.clone()] {
+ let path = names::check(self.root, &relative)?;
+ if directory_exists(&path)? && disk::read_dir(&path)?.next().is_none() {
+ disk::remove_dir(&path)?;
+ fs::sync_parent(&path)?;
+ }
+ }
+ let store = store::lock(self.store)?;
+ let transaction = store.db.connection().unchecked_transaction()?;
+ store
+ .db
+ .connection()
+ .execute(
+ "DELETE FROM albums WHERE key=?1 AND NOT EXISTS(SELECT 1 FROM temporaries WHERE folder=?1)",
+ [&active.key],
+ )?;
+ store.clear_pending(&target.owner)?;
+ transaction.commit()?;
+ Ok(())
+ }
}
-fn json_string(value: &str) -> Value {
- Value::String(value.into())
+fn target_component<'a>(target: &'a Pending, role: &Role) -> Result<&'a str> {
+ let index = match role {
+ Role::Original | Role::Image => 0,
+ Role::Video => 1,
+ };
+ target
+ .names
+ .get(index)
+ .map(String::as_str)
+ .context("missing target component")
}
diff --git a/rust/apps/cli-next/src/export/store.rs b/rust/apps/cli-next/src/export/store.rs
index 490aac4598..ba069a4f91 100644
--- a/rust/apps/cli-next/src/export/store.rs
+++ b/rust/apps/cli-next/src/export/store.rs
@@ -1,18 +1,20 @@
use std::{
path::{Path, PathBuf},
+ sync::{Mutex, MutexGuard},
time::{SystemTime, UNIX_EPOCH},
};
use anyhow::{Context, Result, ensure};
use ente_core::crypto::hash;
-use ente_photos::export::Component;
-use rusqlite::{OptionalExtension, params};
+use ente_photos::{collections::Collection, export::Role, files::File};
+use rusqlite::{OptionalExtension, Row, params};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::Value;
+use super::names;
use crate::{
db, home,
- replica::{AlbumRecord, FileRecord},
+ replica::{AlbumRecord, read_json},
vault::DbKey,
};
@@ -20,47 +22,110 @@ pub struct Store {
pub db: crate::core_db::Db,
}
-#[derive(Clone, Serialize, Deserialize)]
+pub fn lock(store: &Mutex<Store>) -> Result<MutexGuard<'_, Store>> {
+ store
+ .lock()
+ .map_err(|_| anyhow::anyhow!("export database mutex poisoned"))
+}
+
+#[derive(Clone)]
pub struct Album {
pub key: String,
pub id: i64,
+ pub retained: bool,
pub name: String,
pub path: String,
- pub retained: bool,
- pub metadata: Value,
- pub initialized: bool,
}
-#[derive(Clone, Serialize, Deserialize)]
-pub struct Media {
- #[serde(flatten)]
- pub component: Component,
- pub properties: Option<Properties>,
- pub intended_time: Option<i64>,
+impl Album {
+ pub fn read(row: &Row<'_>) -> rusqlite::Result<Self> {
+ Ok(Self {
+ key: row.get(0)?,
+ id: row.get(1)?,
+ retained: row.get(2)?,
+ name: row.get(3)?,
+ path: row.get(4)?,
+ })
+ }
}
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Clone)]
pub struct Placement {
pub id: String,
pub album: i64,
- pub folder: String,
pub file: i64,
pub retained: bool,
pub name: String,
- pub original: String,
- pub media: Vec<Media>,
- pub metadata: Vec<Value>,
- pub complete: bool,
- pub retaining: bool,
+ pub kind: String,
}
impl Placement {
- pub fn kind(&self) -> Result<&str> {
- self.metadata
- .first()
- .and_then(|record| record["ente"]["type"].as_str())
- .context("missing placement media type")
+ pub fn read(row: &Row<'_>) -> rusqlite::Result<Self> {
+ Ok(Self {
+ id: row.get(0)?,
+ album: row.get(1)?,
+ file: row.get(2)?,
+ retained: row.get(3)?,
+ name: row.get(4)?,
+ kind: row.get(5)?,
+ })
+ }
+}
+
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Location {
+ pub folder: String,
+ pub name: String,
+}
+
+#[derive(Clone, PartialEq, Eq)]
+pub struct Component {
+ pub placement: String,
+ pub role: Role,
+ pub location: Location,
+ pub size: u64,
+ pub hash: String,
+ pub signature: Option<String>,
+ pub properties: Option<Properties>,
+ pub intended_time: Option<i64>,
+}
+
+impl Component {
+ pub fn read(row: &Row<'_>) -> rusqlite::Result<Self> {
+ Ok(Self {
+ placement: row.get(0)?,
+ role: read_role(row, 1)?,
+ location: Location {
+ folder: row.get(2)?,
+ name: row.get(3)?,
+ },
+ size: row.get::<_, i64>(4)? as u64,
+ hash: row.get(5)?,
+ signature: row.get(6)?,
+ properties: read_json(row, 7)?,
+ intended_time: row.get(8)?,
+ })
}
+
+ pub fn portable(&self) -> ente_photos::export::Component {
+ ente_photos::export::Component {
+ role: self.role.clone(),
+ path: self.location.name.clone(),
+ size: self.size,
+ hash: self.hash.clone(),
+ }
+ }
+}
+
+fn read_role(row: &Row<'_>, index: usize) -> rusqlite::Result<Role> {
+ let role: String = row.get(index)?;
+ serde_json::from_value(Value::String(role)).map_err(|error| {
+ rusqlite::Error::FromSqlConversionFailure(
+ index,
+ rusqlite::types::Type::Text,
+ Box::new(error),
+ )
+ })
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -71,15 +136,23 @@ pub struct Properties {
impl Properties {
pub fn read(path: &Path) -> Result<Self> {
- let metadata = std::fs::symlink_metadata(path)?;
+ Self::optional(path)?.with_context(|| format!("missing file at {}", path.display()))
+ }
+
+ pub fn optional(path: &Path) -> Result<Option<Self>> {
+ let metadata = match std::fs::symlink_metadata(path) {
+ Ok(metadata) => metadata,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
+ Err(error) => return Err(error.into()),
+ };
ensure!(
- metadata.is_file() && !metadata.is_symlink(),
+ metadata.is_file(),
super::Conflict(format!("at {}: expected a regular file", path.display()))
);
- Ok(Self {
+ Ok(Some(Self {
size: metadata.len(),
modified: system_time(metadata.modified()?),
- })
+ }))
}
}
@@ -90,36 +163,150 @@ pub fn system_time(time: SystemTime) -> i128 {
}
}
-#[derive(Serialize, Deserialize)]
+#[derive(Clone)]
pub struct JsonRecord {
- pub hash: String,
- pub properties: Properties,
+ pub owner: String,
+ pub role: Option<Role>,
+ pub location: Option<Location>,
+ pub value: Value,
+ pub hash: Option<String>,
+ pub properties: Option<Properties>,
}
-#[derive(Clone, Serialize, Deserialize)]
+impl JsonRecord {
+ pub fn read(row: &Row<'_>) -> rusqlite::Result<Self> {
+ Ok(Self {
+ owner: row.get(0)?,
+ role: if row.get::<_, String>(1)?.is_empty() {
+ None
+ } else {
+ Some(read_role(row, 1)?)
+ },
+ location: row
+ .get::<_, Option<String>>(2)?
+ .map(|folder| {
+ Ok::<_, rusqlite::Error>(Location {
+ folder,
+ name: row.get(3)?,
+ })
+ })
+ .transpose()?,
+ value: read_json(row, 4)?,
+ hash: row.get(5)?,
+ properties: read_json(row, 6)?,
+ })
+ }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct AlbumSource {
+ pub name: String,
+ pub metadata: Value,
+ pub warnings: Vec<String>,
+}
+
+#[derive(Clone)]
pub struct Temporary {
+ pub location: Location,
pub album: i64,
- pub path: String,
pub file: Option<i64>,
- pub original: Option<String>,
- pub role: Option<ente_photos::export::Role>,
- pub extension: Option<String>,
- pub destination: Option<String>,
- pub hash: Option<String>,
- pub size: Option<u64>,
+}
+
+#[derive(Clone)]
+pub struct Pending {
+ pub owner: String,
+ pub album: i64,
+ pub file: Option<i64>,
+ pub retained: bool,
+ pub name: String,
+ pub kind: String,
+ pub folder: String,
+ pub names: Vec<String>,
+ pub action: Option<Action>,
+}
+
+impl Pending {
+ pub fn read(row: &Row<'_>) -> rusqlite::Result<Self> {
+ let mut names = vec![row.get(7)?];
+ if let Some(name) = row.get(8)? {
+ names.push(name);
+ }
+ Ok(Self {
+ owner: row.get(0)?,
+ album: row.get(1)?,
+ file: row.get(2)?,
+ retained: row.get(3)?,
+ name: row.get(4)?,
+ kind: row.get(5)?,
+ folder: row.get(6)?,
+ names,
+ action: read_json(row, 9)?,
+ })
+ }
+
+ pub fn placement(&self) -> Result<Placement> {
+ Ok(Placement {
+ id: self.owner.clone(),
+ album: self.album,
+ file: self.file.context("album target is not a placement")?,
+ retained: self.retained,
+ name: self.name.clone(),
+ kind: self.kind.clone(),
+ })
+ }
+
+ pub fn album_path(&self) -> String {
+ if self.folder.is_empty() {
+ self.names[0].clone()
+ } else {
+ format!("{}/{}", self.folder, self.names[0])
+ }
+ }
}
#[derive(Clone, Serialize, Deserialize)]
-pub enum Operation {
+pub enum Output {
+ Media {
+ role: Role,
+ size: u64,
+ hash: String,
+ signature: Option<String>,
+ },
+ Json {
+ role: Option<Role>,
+ size: u64,
+ hash: String,
+ },
+}
+
+impl Output {
+ pub fn identity(&self) -> (u64, &str) {
+ match self {
+ Self::Media { size, hash, .. } | Self::Json { size, hash, .. } => (*size, hash),
+ }
+ }
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub enum Action {
+ Directory {
+ source: Option<String>,
+ },
+ Publish {
+ temporary: Location,
+ destination: Location,
+ output: Output,
+ },
Move {
- album: i64,
- file: Option<i64>,
- source: String,
- destination: String,
+ source: Location,
+ destination: Location,
+ role: Role,
+ size: u64,
+ hash: String,
},
Remove {
- album: i64,
- path: String,
+ location: Location,
+ role: Option<Role>,
},
}
@@ -162,26 +349,199 @@ impl Store {
}
pub fn album(&self, id: i64, retained: bool) -> Result<Option<Album>> {
- self.json(
- "SELECT record FROM albums WHERE id=?1 AND retained=?2 ORDER BY rowid LIMIT 1",
- params![id, retained],
- )
+ Ok(self
+ .db
+ .connection()
+ .query_row(
+ "SELECT key,id,retained,name,path FROM albums WHERE id=?1 AND retained=?2 ORDER BY rowid LIMIT 1",
+ params![id, retained],
+ Album::read,
+ )
+ .optional()?)
+ }
+
+ pub fn folder(&self, key: &str) -> Result<Album> {
+ Ok(self.db.connection().query_row(
+ "SELECT key,id,retained,name,path FROM albums WHERE key=?1",
+ [key],
+ Album::read,
+ )?)
+ }
+
+ pub fn relative(&self, location: &Location) -> Result<String> {
+ Ok(format!(
+ "{}/{}",
+ self.folder(&location.folder)?.path,
+ location.name
+ ))
}
pub fn save_album(&self, album: &Album) -> Result<()> {
- self.db.connection().execute("INSERT INTO albums VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(key) DO UPDATE SET name=excluded.name,path=excluded.path,record=excluded.record",params![album.key,album.id,album.retained,album.name,album.path,serde_json::to_string(album)?])?;
+ let (parent, name) = album.path.rsplit_once('/').unwrap_or(("", &album.path));
+ self.db.connection().execute(
+ "INSERT INTO albums VALUES(?1,?2,?3,?4,?5,?6,?7) ON CONFLICT(key) DO UPDATE SET name=excluded.name,path=excluded.path,parent=excluded.parent,folded=excluded.folded WHERE albums.name<>excluded.name OR albums.path<>excluded.path",
+ params![
+ album.key,
+ album.id,
+ album.retained,
+ album.name,
+ album.path,
+ parent,
+ names::folded(name)
+ ],
+ )?;
Ok(())
}
pub fn placement(&self, album: i64, file: i64) -> Result<Option<Placement>> {
- self.json(
- "SELECT record FROM placements WHERE album=?1 AND file=?2 AND retained=0",
- params![album, file],
- )
+ Ok(self
+ .db
+ .connection()
+ .query_row(
+ "SELECT id,album,file,retained,name,kind FROM placements WHERE album=?1 AND file=?2 AND retained=0",
+ params![album, file],
+ Placement::read,
+ )
+ .optional()?)
}
pub fn save_placement(&self, placement: &Placement) -> Result<()> {
- self.db.connection().execute("INSERT INTO placements VALUES(?1,?2,?3,?4,?5) ON CONFLICT(id) DO UPDATE SET album=excluded.album,file=excluded.file,retained=excluded.retained,record=excluded.record",params![placement.id,placement.album,placement.file,placement.retained,serde_json::to_string(placement)?])?;
+ self.db.connection().execute(
+ "INSERT INTO placements VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(id) DO UPDATE SET retained=excluded.retained,name=excluded.name,kind=excluded.kind WHERE placements.retained<>excluded.retained OR placements.name<>excluded.name OR placements.kind<>excluded.kind",
+ params![
+ placement.id,
+ placement.album,
+ placement.file,
+ placement.retained,
+ placement.name,
+ placement.kind
+ ],
+ )?;
+ Ok(())
+ }
+
+ pub fn components(&self, owner: &str) -> Result<Vec<Component>> {
+ let mut query = self.db.connection().prepare(
+ "SELECT placement,role,folder,name,size,hash,signature,properties,intended_time FROM components WHERE placement=?1 ORDER BY role",
+ )?;
+ Ok(query
+ .query_map([owner], Component::read)?
+ .collect::<rusqlite::Result<_>>()?)
+ }
+
+ pub fn save_component(&self, component: &Component) -> Result<()> {
+ self.db.connection().execute(
+ "INSERT INTO components VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11) ON CONFLICT(placement,role) DO UPDATE SET folder=excluded.folder,name=excluded.name,folded=excluded.folded,stem=excluded.stem,size=excluded.size,hash=excluded.hash,signature=excluded.signature,properties=excluded.properties,intended_time=excluded.intended_time WHERE components.folder<>excluded.folder OR components.name<>excluded.name OR components.size<>excluded.size OR components.hash<>excluded.hash OR components.signature IS NOT excluded.signature OR components.properties<>excluded.properties OR components.intended_time IS NOT excluded.intended_time",
+ params![
+ component.placement,
+ component.role.name(),
+ component.location.folder,
+ component.location.name,
+ names::folded(&component.location.name),
+ names::folded(names::split(&component.location.name, false).0),
+ component.size as i64,
+ component.hash,
+ component.signature,
+ serde_json::to_string(&component.properties)?,
+ component.intended_time
+ ],
+ )?;
+ Ok(())
+ }
+
+ pub fn json_record(&self, owner: &str, role: Option<&Role>) -> Result<Option<JsonRecord>> {
+ Ok(self
+ .db
+ .connection()
+ .query_row(
+ "SELECT owner,role,folder,name,value,hash,properties FROM json_records WHERE owner=?1 AND role=?2",
+ params![owner, role.map_or("", Role::name)],
+ JsonRecord::read,
+ )
+ .optional()?)
+ }
+
+ pub fn save_json(&self, record: &JsonRecord) -> Result<()> {
+ let location = record.location.as_ref();
+ let media = location
+ .and_then(|location| location.name.strip_prefix("metadata/"))
+ .and_then(|name| name.strip_suffix(".json"));
+ self.db.connection().execute(
+ "INSERT INTO json_records VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9) ON CONFLICT(owner,role) DO UPDATE SET folder=excluded.folder,name=excluded.name,folded=excluded.folded,stem=excluded.stem,value=excluded.value,hash=excluded.hash,properties=excluded.properties WHERE json_records.folder IS NOT excluded.folder OR json_records.name IS NOT excluded.name OR json_records.value<>excluded.value OR json_records.hash IS NOT excluded.hash OR json_records.properties<>excluded.properties",
+ params![
+ record.owner,
+ record.role.as_ref().map_or("", Role::name),
+ location.map(|l| &l.folder),
+ location.map(|l| &l.name),
+ media.map(names::folded),
+ media.map(|name| names::folded(names::split(name, false).0)),
+ serde_json::to_string(&record.value)?,
+ record.hash,
+ serde_json::to_string(&record.properties)?
+ ],
+ )?;
+ Ok(())
+ }
+
+ pub fn pending(&self, owner: &str) -> Result<Option<Pending>> {
+ Ok(self
+ .db
+ .connection()
+ .query_row(
+ "SELECT owner,album,file,retained,name,kind,folder,name1,name2,action FROM pending WHERE owner=?1",
+ [owner],
+ Pending::read,
+ )
+ .optional()?)
+ }
+
+ pub fn pending_file(&self, album: i64, file: i64) -> Result<Option<Pending>> {
+ Ok(self
+ .db
+ .connection()
+ .query_row(
+ "SELECT owner,album,file,retained,name,kind,folder,name1,name2,action FROM pending WHERE album=?1 AND file=?2 ORDER BY retained DESC LIMIT 1",
+ params![album, file],
+ Pending::read,
+ )
+ .optional()?)
+ }
+
+ pub fn save_pending(&self, pending: &Pending) -> Result<()> {
+ let first = &pending.names[0];
+ let second = pending.names.get(1);
+ self.db.connection().execute(
+ "INSERT INTO pending VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14) ON CONFLICT(owner) DO UPDATE SET retained=excluded.retained,name=excluded.name,kind=excluded.kind,folder=excluded.folder,name1=excluded.name1,name2=excluded.name2,folded1=excluded.folded1,folded2=excluded.folded2,stem1=excluded.stem1,stem2=excluded.stem2,action=excluded.action",
+ params![
+ pending.owner,
+ pending.album,
+ pending.file,
+ pending.retained,
+ pending.name,
+ pending.kind,
+ pending.folder,
+ first,
+ second,
+ names::folded(first),
+ second.map(|name| names::folded(name)),
+ names::folded(names::split(first, pending.file.is_none()).0),
+ second.map(|name| names::folded(names::split(name, false).0)),
+ serde_json::to_string(&pending.action)?
+ ],
+ )?;
+ Ok(())
+ }
+
+ pub fn clear_pending(&self, owner: &str) -> Result<()> {
+ self.db
+ .connection()
+ .execute("DELETE FROM pending WHERE owner=?1", [owner])?;
+ self.db
+ .connection()
+ .execute(
+ "DELETE FROM json_records WHERE owner=?1 AND folder IS NULL AND NOT EXISTS(SELECT 1 FROM components WHERE placement=?1)",
+ [owner],
+ )?;
Ok(())
}
@@ -200,65 +560,89 @@ impl Store {
}
pub fn temporary(&self, temporary: &Temporary) -> Result<()> {
- self.db.connection().execute("INSERT INTO temporaries VALUES(?1,?2) ON CONFLICT(path) DO UPDATE SET record=excluded.record",params![temporary.path,serde_json::to_string(temporary)?])?;
- Ok(())
- }
-
- pub fn operation(&self, operation: &Operation) -> Result<i64> {
- let (album, file) = match operation {
- Operation::Move { album, file, .. } => (*album, *file),
- Operation::Remove { album, .. } => (*album, None),
- };
self.db.connection().execute(
- "INSERT INTO operations(album,file,record) VALUES(?1,?2,?3)",
- params![album, file, serde_json::to_string(operation)?],
+ "INSERT INTO temporaries VALUES(?1,?2,?3,?4)",
+ params![
+ temporary.location.name,
+ temporary.location.folder,
+ temporary.album,
+ temporary.file
+ ],
)?;
- Ok(self.db.connection().last_insert_rowid())
+ Ok(())
}
-
pub fn reset_desired(&self) -> Result<()> {
- self.db.connection().execute_batch("DELETE FROM desired_albums; DELETE FROM desired_files; DELETE FROM names; DELETE FROM outcomes; DELETE FROM events;")?;
+ self.db.connection().execute_batch(RUN_SCHEMA)?;
Ok(())
}
pub fn desired_album(&self, record: &AlbumRecord, user_id: i64) -> Result<()> {
let name = record.album(user_id).ok().map(|a| a.name);
- self.db.connection().execute("INSERT INTO desired_albums(id,name,record) VALUES(?1,?2,?3) ON CONFLICT(id) DO UPDATE SET name=excluded.name,record=excluded.record",params![record.remote.id,name,serde_json::to_string(record)?])?;
+ let favorites = record.remote.kind == "favorites" && record.remote.owner.id == user_id;
+ self.db.connection().execute(
+ "INSERT INTO desired_albums(id,name,present,favorites) VALUES(?1,?2,1,?3) ON CONFLICT(id) DO UPDATE SET name=excluded.name,present=1,favorites=excluded.favorites",
+ params![record.remote.id, name, favorites],
+ )?;
Ok(())
}
- pub fn desired_file(
- &self,
- album: i64,
- record: &FileRecord,
- user_id: i64,
- favorited: Option<bool>,
- ) -> Result<()> {
- let result = record.file(user_id);
- let failure = result.as_ref().err().map(ToString::to_string);
- self.db.connection().execute("INSERT INTO desired_files(album,file,version,record,favorited,failure) VALUES(?1,?2,?3,?4,?5,?6)",params![album,record.remote.id,record.remote.updation_time,serde_json::to_string(record)?,favorited,failure])?;
+ pub fn snapshot_album(&self, album: &Collection) -> Result<()> {
+ let source = AlbumSource {
+ name: album.name.clone(),
+ metadata: ente_photos::export::album(album),
+ warnings: album.warnings.clone(),
+ };
+ self.db.connection().execute(
+ "INSERT INTO album_sources VALUES(?1,?2) ON CONFLICT(id) DO UPDATE SET record=excluded.record WHERE album_sources.record<>excluded.record",
+ params![album.id, serde_json::to_string(&source)?],
+ )?;
+ Ok(())
+ }
+
+ pub fn snapshot_file(&self, file: &File) -> Result<()> {
+ self.db.connection().execute(
+ "INSERT INTO sources VALUES(?1,?2) ON CONFLICT(file) DO UPDATE SET record=excluded.record WHERE sources.record<>excluded.record",
+ params![file.id, serde_json::to_string(file)?],
+ )?;
Ok(())
}
}
const SCHEMA: &str = "
CREATE TABLE export (id INTEGER PRIMARY KEY CHECK(id=1), destination BLOB NOT NULL, bound INTEGER NOT NULL DEFAULT 0);
- CREATE TABLE albums (key TEXT PRIMARY KEY, id INTEGER NOT NULL, retained INTEGER NOT NULL, name TEXT NOT NULL, path TEXT NOT NULL UNIQUE, record TEXT NOT NULL);
- CREATE UNIQUE INDEX active_album ON albums(id) WHERE retained=0;
- CREATE TABLE placements (id TEXT PRIMARY KEY, album INTEGER NOT NULL, file INTEGER NOT NULL, retained INTEGER NOT NULL, record TEXT NOT NULL);
- CREATE UNIQUE INDEX active_placement ON placements(album,file) WHERE retained=0;
- CREATE INDEX reusable_media ON placements(file,retained);
- CREATE TABLE json_records (path TEXT PRIMARY KEY, record TEXT NOT NULL);
- CREATE TABLE temporaries (path TEXT PRIMARY KEY, record TEXT NOT NULL);
- CREATE INDEX temporary_file ON temporaries(json_extract(record,'$.file'));
- CREATE TABLE operations (id INTEGER PRIMARY KEY, album INTEGER NOT NULL, file INTEGER, record TEXT NOT NULL);
- CREATE TABLE desired_albums (id INTEGER PRIMARY KEY, name TEXT, record TEXT, selected INTEGER NOT NULL DEFAULT 0, ready INTEGER NOT NULL DEFAULT 0, failure TEXT);
- CREATE TABLE desired_files (album INTEGER NOT NULL, file INTEGER NOT NULL, version INTEGER NOT NULL, record TEXT NOT NULL, favorited INTEGER, completed INTEGER NOT NULL DEFAULT 0, failure TEXT, running INTEGER NOT NULL DEFAULT 0, attempted INTEGER NOT NULL DEFAULT 0, deferred INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(album,file));
- CREATE INDEX desired_file_id ON desired_files(file,album);
- CREATE INDEX eligible_files ON desired_files(deferred,file) WHERE failure IS NULL AND attempted=0 AND running=0;
- CREATE TABLE outcomes (unit TEXT PRIMARY KEY, conflict INTEGER NOT NULL);
- CREATE TABLE events (placement TEXT PRIMARY KEY, exported INTEGER NOT NULL DEFAULT 0, metadata_updated INTEGER NOT NULL DEFAULT 0, renamed INTEGER NOT NULL DEFAULT 0, retained INTEGER NOT NULL DEFAULT 0);
- CREATE TABLE allocations (owner TEXT PRIMARY KEY, record TEXT NOT NULL);
- CREATE TABLE names (parent TEXT NOT NULL, folded TEXT NOT NULL, owner TEXT NOT NULL, stem TEXT NOT NULL, kind TEXT NOT NULL, PRIMARY KEY(parent,folded));
- CREATE INDEX named_stems ON names(parent,stem);
+CREATE TABLE albums (key TEXT PRIMARY KEY, id INTEGER NOT NULL, retained INTEGER NOT NULL, name TEXT NOT NULL, path TEXT NOT NULL UNIQUE, parent TEXT NOT NULL, folded TEXT NOT NULL);
+CREATE UNIQUE INDEX active_album ON albums(id) WHERE retained=0;
+CREATE UNIQUE INDEX album_names ON albums(parent,folded);
+CREATE TABLE placements (id TEXT PRIMARY KEY, album INTEGER NOT NULL, file INTEGER NOT NULL, retained INTEGER NOT NULL, name TEXT NOT NULL, kind TEXT NOT NULL);
+CREATE UNIQUE INDEX active_placement ON placements(album,file) WHERE retained=0;
+CREATE INDEX reusable_media ON placements(file,retained);
+CREATE TABLE components (placement TEXT NOT NULL, role TEXT NOT NULL, folder TEXT NOT NULL, name TEXT NOT NULL, folded TEXT NOT NULL, stem TEXT NOT NULL, size INTEGER NOT NULL, hash TEXT NOT NULL, signature TEXT, properties TEXT NOT NULL, intended_time INTEGER, PRIMARY KEY(placement,role));
+CREATE INDEX component_names ON components(folder,folded);
+CREATE INDEX component_stems ON components(folder,stem);
+CREATE TABLE json_records (owner TEXT NOT NULL, role TEXT NOT NULL, folder TEXT, name TEXT, folded TEXT, stem TEXT, value TEXT NOT NULL, hash TEXT, properties TEXT NOT NULL, PRIMARY KEY(owner,role));
+CREATE INDEX json_names ON json_records(folder,folded);
+CREATE INDEX json_stems ON json_records(folder,stem);
+CREATE TABLE pending (owner TEXT PRIMARY KEY, album INTEGER NOT NULL, file INTEGER, retained INTEGER NOT NULL, name TEXT NOT NULL, kind TEXT NOT NULL, folder TEXT NOT NULL, name1 TEXT NOT NULL, name2 TEXT, folded1 TEXT NOT NULL, folded2 TEXT, stem1 TEXT NOT NULL, stem2 TEXT, action TEXT NOT NULL);
+CREATE INDEX pending_file ON pending(album,file);
+CREATE INDEX pending_temporary ON pending(json_extract(action,'$.Publish.temporary.name'));
+CREATE INDEX pending_name1 ON pending(folder,folded1);
+CREATE INDEX pending_name2 ON pending(folder,folded2);
+CREATE INDEX pending_stem1 ON pending(folder,stem1);
+CREATE INDEX pending_stem2 ON pending(folder,stem2);
+CREATE TABLE temporaries (name TEXT PRIMARY KEY, folder TEXT NOT NULL, album INTEGER NOT NULL, file INTEGER);
+CREATE INDEX temporary_file ON temporaries(file);
+CREATE INDEX temporary_folder ON temporaries(folder);
+CREATE TABLE album_sources (id INTEGER PRIMARY KEY, record TEXT NOT NULL);
+CREATE TABLE sources (file INTEGER PRIMARY KEY, record TEXT NOT NULL);
+";
+
+const RUN_SCHEMA: &str = "
+CREATE TEMP TABLE desired_albums (id INTEGER PRIMARY KEY, name TEXT, present INTEGER NOT NULL DEFAULT 0, favorites INTEGER NOT NULL DEFAULT 0, selected INTEGER NOT NULL DEFAULT 0, ready INTEGER NOT NULL DEFAULT 0, failure TEXT);
+CREATE TEMP TABLE desired_files (album INTEGER NOT NULL, file INTEGER NOT NULL, favorited INTEGER, completed INTEGER NOT NULL DEFAULT 0, failure TEXT, claimed INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(album,file));
+CREATE INDEX desired_file_id ON desired_files(file,album);
+CREATE INDEX eligible_files ON desired_files(file) WHERE failure IS NULL AND claimed=0;
+CREATE TEMP TABLE chosen_files (file INTEGER PRIMARY KEY, album INTEGER NOT NULL, version INTEGER NOT NULL);
+CREATE TEMP TABLE favorites (file INTEGER PRIMARY KEY);
+CREATE TEMP TABLE outcomes (unit TEXT PRIMARY KEY, conflict INTEGER NOT NULL);
+CREATE TEMP TABLE events (placement TEXT PRIMARY KEY, exported INTEGER NOT NULL DEFAULT 0, metadata_updated INTEGER NOT NULL DEFAULT 0, renamed INTEGER NOT NULL DEFAULT 0, retained INTEGER NOT NULL DEFAULT 0);
";
diff --git a/rust/apps/cli-next/src/export/transfer.rs b/rust/apps/cli-next/src/export/transfer.rs
index 7549824553..53fb897020 100644
--- a/rust/apps/cli-next/src/export/transfer.rs
+++ b/rust/apps/cli-next/src/export/transfer.rs
@@ -1,335 +1,292 @@
use std::{
fs::File,
io::{Seek, SeekFrom},
- path::{Path, PathBuf},
- sync::Arc,
};
-use anyhow::{Context as _, Result, bail, ensure};
+use anyhow::{Context, Result, ensure};
use ente_core::{Session, b64, crypto::hash};
use ente_photos::{
export::Role,
files::{self, Kind},
};
-use rusqlite::OptionalExtension;
-use tokio::sync::watch;
+use rusqlite::{OptionalExtension, params};
use super::{
fs::{self, HashWriter},
names,
- store::{Placement, Properties, Store},
+ reconcile::Run,
+ store::{self, Component, Location, Properties},
};
-use crate::{replica::FileRecord, vault::DbKey};
-pub struct Component {
+pub struct Source {
pub role: Role,
pub extension: String,
- pub path: String,
+ pub location: Location,
pub size: u64,
pub hash: String,
pub temporary: bool,
}
pub struct Prepared {
- pub file: files::File,
pub signature: String,
- pub components: Vec<Component>,
+ pub components: Vec<Source>,
+ pub observed: Vec<(i64, Vec<(Role, Properties)>)>,
}
-pub enum Preparation {
- Ready(Box<Prepared>),
- AwaitingCapacity,
+pub fn roles(file: &files::File) -> Vec<Role> {
+ if file.kind == Kind::LivePhoto {
+ vec![Role::Image, Role::Video]
+ } else {
+ vec![Role::Original]
+ }
}
-pub struct Context {
- pub root: PathBuf,
- pub db_path: PathBuf,
- pub db_key: DbKey,
- pub session: Session,
+pub fn verify(
+ run: &Run<'_>,
+ component: &Component,
+ file: &files::File,
+ signature: &str,
+) -> Result<Option<Properties>> {
+ let path = run.path(&component.location)?;
+ let Some(properties) = Properties::optional(&path)? else {
+ return Ok(None);
+ };
+ if expected_hash(file, &component.role).is_some_and(|hash| hash != component.hash)
+ || (expected_hash(file, &component.role).is_none()
+ && component
+ .signature
+ .as_ref()
+ .is_some_and(|original| original != signature))
+ {
+ return Ok(None);
+ }
+ if component.properties.as_ref() != Some(&properties)
+ && !fs::matches(&path, component.size, &component.hash, run.cancel)?
+ {
+ return Ok(None);
+ }
+ Ok(Some(properties))
}
-pub async fn prepare(
- context: Arc<Context>,
- id: i64,
- mut cancel: watch::Receiver<bool>,
- capacity: bool,
-) -> Result<Preparation> {
- let store = Store::open(&context.db_path, &context.db_key, &context.root, false)?;
- let record:FileRecord=store.json("SELECT record FROM desired_files WHERE file=?1 AND failure IS NULL ORDER BY version DESC,album LIMIT 1",[id])?.context("file has no usable source metadata")?;
- let file = record.file(context.session.user_id)?;
- let signature = signature(&file)?;
+pub async fn prepare(run: &Run<'_>, session: &Session, file: &files::File) -> Result<Prepared> {
+ let signature = signature(file)?;
+ let roles = roles(file);
+ let mut observed = Vec::new();
let mut needs_bytes = false;
- {
- let mut after = 0;
- loop {
- let album:Option<i64>=store.db.connection().query_row("SELECT album FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",rusqlite::params![id,after],|r|r.get(0)).optional()?;
- let Some(album) = album else { break };
- after = album;
- let existing = store.placement(album, id)?;
- if let Some(mut placement) = existing {
- match valid_placement(&context.root, &placement, &file, &signature, &cancel) {
- Ok(true) => {
- for media in &mut placement.media {
- let observed = Properties::read(&names::check(
- &context.root,
- &format!("{}/{}", placement.folder, media.component.path),
- )?)?;
- if media.properties.as_ref() != Some(&observed) {
- media.intended_time = None;
- }
- media.properties = Some(observed);
- }
- store.save_placement(&placement)?;
- }
- Ok(false) => needs_bytes = true,
- Err(error) => {
- super::report(&store, &format!("file:{album}:{id}"), &error)?;
- store.db.connection().execute(
- "UPDATE desired_files SET failure=?1 WHERE album=?2 AND file=?3",
- rusqlite::params![error.to_string(), album, id],
- )?;
- if super::fatal(&error) {
- return Err(error);
- }
- }
+ let mut after = 0;
+ loop {
+ run.check_cancel()?;
+ let album: Option<i64> = store::lock(run.store)?
+ .db
+ .connection()
+ .query_row(
+ "SELECT album FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",
+ params![file.id, after],
+ |r| r.get(0),
+ )
+ .optional()?;
+ let Some(album) = album else {
+ break;
+ };
+ after = album;
+ let result = (|| {
+ let components = {
+ let store = store::lock(run.store)?;
+ needs_bytes |= store
+ .pending_file(album, file.id)?
+ .is_some_and(|pending| pending.retained);
+ store
+ .placement(album, file.id)?
+ .map(|placement| store.components(&placement.id))
+ .transpose()?
+ .unwrap_or_default()
+ };
+ let mut verified = Vec::new();
+ for component in &components {
+ if !roles.contains(&component.role) {
+ continue;
+ }
+ if let Some(properties) = verify(run, component, file, &signature)? {
+ verified.push((component.role.clone(), properties));
}
- } else {
- needs_bytes = true;
}
+ needs_bytes |= roles
+ .iter()
+ .any(|role| !verified.iter().any(|(verified, _)| role == verified));
+ observed.push((album, verified));
+ Ok(())
+ })();
+ if let Err(error) = result {
+ run.block(album, Some(file.id), error)?;
}
}
if !needs_bytes {
- return Ok(Preparation::Ready(Box::new(Prepared {
- file,
+ return Ok(Prepared {
signature,
components: Vec::new(),
- })));
- }
- if !capacity {
- return Ok(Preparation::AwaitingCapacity);
+ observed,
+ });
}
- let roles = if file.kind == Kind::LivePhoto {
- vec![Role::Image, Role::Video]
- } else {
- vec![Role::Original]
- };
let mut reusable = Vec::new();
- {
- let mut after = String::new();
- loop {
- let temporary:Option<super::store::Temporary>=store.json("SELECT record FROM temporaries WHERE json_extract(record,'$.file')=?1 AND json_extract(record,'$.destination') IS NULL AND path>?2 ORDER BY path LIMIT 1",rusqlite::params![id,after])?;
- let Some(temporary) = temporary else { break };
- after = temporary.path.clone();
- if temporary.original.as_deref() != Some(&signature) {
- fs::discard(&context.root, &store, &temporary.path)?;
- continue;
- }
- if let (Some(role), Some(size), Some(hash), Some(extension)) = (
- temporary.role,
- temporary.size,
- temporary.hash,
- temporary.extension,
- ) {
- let path = names::check(&context.root, &temporary.path)?;
- if fs::hash_cancellable(&path, &cancel).is_ok_and(|(actual_size, actual_hash)| {
- actual_size == size && actual_hash == hash
- }) {
- reusable.push(Component {
- role,
- extension,
- path: temporary.path,
- size,
- hash,
- temporary: true,
- });
- }
- }
+ let mut before = i64::MAX;
+ loop {
+ run.check_cancel()?;
+ if roles
+ .iter()
+ .all(|role| reusable.iter().any(|source: &Source| source.role == *role))
+ {
+ break;
}
- }
- for retained in [false, true] {
- let mut before = i64::MAX;
- loop {
- if roles
- .iter()
- .all(|role| reusable.iter().any(|c| c.role == *role))
+ let next: Option<(i64, String)> = store::lock(run.store)?
+ .db
+ .connection()
+ .query_row(
+ "SELECT rowid,id FROM placements WHERE file=?1 AND rowid<?2 ORDER BY rowid DESC LIMIT 1",
+ params![file.id, before],
+ |r| Ok((r.get(0)?, r.get(1)?)),
+ )
+ .optional()?;
+ let Some((rowid, id)) = next else {
+ break;
+ };
+ before = rowid;
+ let components = store::lock(run.store)?.components(&id)?;
+ for component in components {
+ if !roles.contains(&component.role)
+ || reusable.iter().any(|source| source.role == component.role)
+ || expected_hash(file, &component.role).is_some_and(|hash| hash != component.hash)
+ || (expected_hash(file, &component.role).is_none()
+ && component
+ .signature
+ .as_ref()
+ .is_some_and(|original| original != &signature))
{
- break;
+ continue;
}
- if *cancel.borrow() {
- bail!(super::Cancelled)
- }
- let next:Option<(i64,String)>=store.db.connection().query_row("SELECT rowid,record FROM placements WHERE file=?1 AND retained=?2 AND rowid<?3 ORDER BY rowid DESC LIMIT 1",rusqlite::params![id,retained,before],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
- let Some((rowid, record)) = next else { break };
- before = rowid;
- let candidate: Placement = serde_json::from_str(&record)?;
- for media in &candidate.media {
- if reusable.iter().any(|c| c.role == media.component.role) {
- continue;
- }
- if expected_hash(&file, &media.component.role)
- .is_some_and(|hash| hash != media.component.hash)
- {
- continue;
- }
- if expected_hash(&file, &media.component.role).is_none()
- && !candidate.original.is_empty()
- && candidate.original != signature
- {
- continue;
- }
- let relative = format!("{}/{}", candidate.folder, media.component.path);
- let valid = (|| {
- let path = names::check(&context.root, &relative)?;
- Properties::read(&path)?;
- let (size, hash) = fs::hash_cancellable(&path, &cancel)?;
- Ok::<_, anyhow::Error>(
- size == media.component.size && hash == media.component.hash,
- )
- })()
- .unwrap_or(false);
- if valid {
- reusable.push(Component {
- role: media.component.role.clone(),
- extension: names::split(&media.component.path, false).1.into(),
- path: relative,
- size: media.component.size,
- hash: media.component.hash.clone(),
- temporary: false,
- });
- }
+ let valid = (|| {
+ let path = run.path(&component.location)?;
+ Properties::read(&path)?;
+ fs::matches(&path, component.size, &component.hash, run.cancel)
+ })();
+ match valid {
+ Ok(true) => reusable.push(Source {
+ role: component.role,
+ extension: names::split(&component.location.name, false).1.into(),
+ location: component.location,
+ size: component.size,
+ hash: component.hash,
+ temporary: false,
+ }),
+ Err(error) if super::fatal(&error) => return Err(error),
+ _ => {}
}
}
}
if roles
.iter()
- .all(|role| reusable.iter().any(|c| c.role == *role))
+ .all(|role| reusable.iter().any(|source| source.role == *role))
{
- let mut components = Vec::with_capacity(roles.len());
- for role in roles {
- let index = reusable
- .iter()
- .position(|c| c.role == role)
- .context("missing verified component")?;
- components.push(reusable.remove(index));
- }
- return Ok(Preparation::Ready(Box::new(Prepared {
- file,
+ reusable.sort_by_key(|source| roles.iter().position(|role| *role == source.role));
+ return Ok(Prepared {
signature,
- components,
- })));
+ components: reusable,
+ observed,
+ });
}
- let (folder, temporary, output) = loop {
- let album:Option<i64>=store.db.connection().query_row("SELECT album FROM desired_files WHERE file=?1 AND failure IS NULL ORDER BY album LIMIT 1",[id],|r|r.get(0)).optional()?;
+ after = 0;
+ let (folder, album, temporary, output) = loop {
+ let album: Option<i64> = store::lock(run.store)?
+ .db
+ .connection()
+ .query_row(
+ "SELECT album FROM desired_files WHERE file=?1 AND album>?2 AND failure IS NULL ORDER BY album LIMIT 1",
+ params![file.id, after],
+ |r| r.get(0),
+ )
+ .optional()?;
let Some(album) = album else {
- return Ok(Preparation::Ready(Box::new(Prepared {
- file,
+ return Ok(Prepared {
signature,
components: Vec::new(),
- })));
+ observed,
+ });
};
- let folder = store
+ after = album;
+ let folder = store::lock(run.store)?
.album(album, false)?
.context("missing destination album")?
- .path;
- match fs::temporary(&context.root, &store, &folder, Some(id)) {
- Ok((temporary, output)) => break (folder, temporary, output),
- Err(error) => {
- super::report(&store, &format!("file:{album}:{id}"), &error)?;
- store.db.connection().execute(
- "UPDATE desired_files SET failure=?1 WHERE album=?2 AND file=?3",
- rusqlite::params![error.to_string(), album, id],
- )?;
- if super::fatal(&error) {
- return Err(error);
- }
- }
+ .key;
+ match run.temporary(&folder, album, Some(file.id)) {
+ Ok((temporary, output)) => break (folder, album, temporary, output),
+ Err(error) => run.block(album, Some(file.id), error)?,
}
};
drop(output);
- let path = names::check(&context.root, &temporary.path)?;
- let write_cancel = cancel.clone();
- let download = files::download(&context.session, &file, || {
+ let path = run.path(&temporary)?;
+ let mut cancel = run.cancel.clone();
+ let download = files::download(session, file, || {
let mut output = File::options().read(true).write(true).open(&path)?;
output.set_len(0)?;
output.seek(SeekFrom::Start(0))?;
- HashWriter::new(output, &write_cancel).map_err(std::io::Error::other)
+ HashWriter::new(output, run.cancel).map_err(std::io::Error::other)
});
- let writer = tokio::select! {
- result=download=>result?,
- _=cancel.changed()=>bail!(super::Cancelled),
- };
+ let writer = tokio::select! { result=download => result?, _=cancel.changed() => return Err(super::Cancelled.into()) };
let (archive, size, hash) = writer.finish()?;
let components = if file.kind == Kind::LivePhoto {
- let mut temporary_files = Vec::new();
+ let mut temporaries = Vec::new();
let extracted = ente_photos::live_photo::extract(archive, |role, extension| {
- if *cancel.borrow() {
- return Err(std::io::Error::other(super::Cancelled));
- }
- let (temporary, file) = fs::temporary(&context.root, &store, &folder, Some(id))
+ let (temporary, file) = run
+ .temporary(&folder, album, Some(file.id))
.map_err(std::io::Error::other)?;
- temporary_files.push((role, extension.to_owned(), temporary));
- HashWriter::new(file, &cancel).map_err(std::io::Error::other)
+ temporaries.push((role, extension.to_owned(), temporary));
+ HashWriter::new(file, run.cancel).map_err(std::io::Error::other)
})?;
let mut components = Vec::with_capacity(2);
- for ((role, extension, writer), (_, _, temporary)) in
- extracted.into_iter().zip(temporary_files)
+ for ((role, extension, writer), (_, _, temporary)) in extracted.into_iter().zip(temporaries)
{
let (_, size, hash) = writer.finish()?;
- if let Some(expected) = expected_hash(&file, &role) {
+ if let Some(expected) = expected_hash(file, &role) {
ensure!(
hash == expected,
"original component hash does not match source for file {}",
file.id
);
}
- stage(
- &store, &temporary, &signature, &role, &extension, size, &hash,
- )?;
- components.push(Component {
+ components.push(Source {
role,
extension,
- path: temporary.path,
+ location: temporary,
size,
hash,
temporary: true,
});
}
- fs::discard(&context.root, &store, &temporary.path)?;
+ run.discard(&temporary)?;
components
} else {
drop(archive);
- if let Some(expected) = expected_hash(&file, &Role::Original) {
+ if let Some(expected) = &file.hash {
ensure!(
- hash == expected,
+ hash == *expected,
"original hash does not match source for file {}",
file.id
);
}
- stage(
- &store,
- &temporary,
- &signature,
- &Role::Original,
- names::split(&file.name, false).1,
- size,
- &hash,
- )?;
- vec![Component {
+ vec![Source {
role: Role::Original,
extension: names::split(&file.name, false).1.into(),
- path: temporary.path,
+ location: temporary,
size,
hash,
temporary: true,
}]
};
- Ok(Preparation::Ready(Box::new(Prepared {
- file,
+ Ok(Prepared {
signature,
components,
- })))
+ observed,
+ })
}
-
pub fn signature(file: &files::File) -> Result<String> {
let mut bytes = file.header.as_bytes().to_vec();
bytes.extend_from_slice(file.hash.as_deref().unwrap_or_default().as_bytes());
@@ -348,128 +305,3 @@ pub fn expected_hash<'a>(file: &'a files::File, role: &Role) -> Option<&'a str>
Role::Video => hash.split_once(':').map(|pair| pair.1),
}
}
-
-pub fn valid_placement(
- root: &Path,
- placement: &Placement,
- file: &files::File,
- signature: &str,
- cancel: &watch::Receiver<bool>,
-) -> Result<bool> {
- if placement.media.len() != if file.kind == Kind::LivePhoto { 2 } else { 1 } {
- return Ok(false);
- }
- for media in &placement.media {
- if expected_hash(file, &media.component.role)
- .is_some_and(|hash| hash != media.component.hash)
- {
- return Ok(false);
- }
- if expected_hash(file, &media.component.role).is_none()
- && !placement.original.is_empty()
- && placement.original != signature
- {
- return Ok(false);
- }
- let path = names::check(
- root,
- &format!("{}/{}", placement.folder, media.component.path),
- )?;
- if !path.try_exists()? {
- return Ok(false);
- }
- let properties = Properties::read(&path)?;
- if media.properties.as_ref() != Some(&properties) {
- let (size, hash) = fs::hash_cancellable(&path, cancel)?;
- if size != media.component.size || hash != media.component.hash {
- return Ok(false);
- }
- }
- }
- Ok(true)
-}
-
-pub fn materialize(
- root: &Path,
- store: &Store,
- source: &mut Component,
- (folder, name): (&str, &str),
- file_id: i64,
- signature: &str,
- cancel: &watch::Receiver<bool>,
-) -> Result<()> {
- let destination = format!("{folder}/{name}");
- let (mut temporary, output) = fs::temporary(root, store, folder, Some(file_id))?;
- let mut writer = HashWriter::new(output, cancel)?;
- std::io::copy(
- &mut File::open(names::check(root, &source.path)?)?,
- &mut writer,
- )?;
- let (_, size, hash) = writer.finish()?;
- ensure!(
- size == source.size && hash == source.hash,
- "local original changed while copying"
- );
- temporary.original = Some(signature.into());
- temporary.role = Some(source.role.clone());
- temporary.extension = Some(source.extension.clone());
- temporary.size = Some(size);
- temporary.hash = Some(hash.clone());
- store.temporary(&temporary)?;
- source.path = temporary.path.clone();
- source.temporary = true;
- fs::publish(root, store, temporary, &destination, size, hash)?;
- source.path = destination;
- source.temporary = false;
- Ok(())
-}
-
-pub fn discard_incomplete(root: &Path, store: &Store, file: i64) -> Result<()> {
- let mut after = String::new();
- loop {
- let next: Option<(String, i64)> = store.db.connection().query_row("SELECT path,json_extract(record,'$.album') FROM temporaries WHERE json_extract(record,'$.file')=?1 AND json_extract(record,'$.hash') IS NULL AND path>?2 ORDER BY path LIMIT 1", rusqlite::params![file, after], |r| Ok((r.get(0)?, r.get(1)?))).optional()?;
- let Some((path, album)) = next else { break };
- after = path.clone();
- if let Err(error) = fs::discard(root, store, &path) {
- super::report(store, &format!("file:{album}:{file}"), &error)?;
- store.db.connection().execute(
- "UPDATE desired_files SET failure=?1 WHERE album=?2 AND file=?3",
- rusqlite::params![error.to_string(), album, file],
- )?;
- if super::fatal(&error) {
- return Err(error);
- }
- }
- }
- Ok(())
-}
-
-pub fn finish_source(root: &Path, store: &Store, components: &[Component]) -> Result<()> {
- for component in components {
- if component.temporary {
- let needed:bool=store.db.connection().query_row("SELECT EXISTS(SELECT 1 FROM desired_files WHERE file=(SELECT json_extract(record,'$.file') FROM temporaries WHERE path=?1) AND completed=0)",[&component.path],|r|r.get(0))?;
- if !needed {
- fs::discard(root, store, &component.path)?;
- }
- }
- }
- Ok(())
-}
-
-fn stage(
- store: &Store,
- temporary: &super::store::Temporary,
- signature: &str,
- role: &Role,
- extension: &str,
- size: u64,
- hash: &str,
-) -> Result<()> {
- let mut temporary = temporary.clone();
- temporary.original = Some(signature.into());
- temporary.role = Some(role.clone());
- temporary.extension = Some(extension.into());
- temporary.size = Some(size);
- temporary.hash = Some(hash.into());
- store.temporary(&temporary)
-}
diff --git a/rust/apps/cli-next/src/replica.rs b/rust/apps/cli-next/src/replica.rs
index c23a6b75f8..49705e857e 100644
--- a/rust/apps/cli-next/src/replica.rs
+++ b/rust/apps/cli-next/src/replica.rs
@@ -1,3 +1,5 @@
+use std::collections::HashSet;
+
use anyhow::{Context, Result};
use ente_core::{
Session, b64,
@@ -36,10 +38,9 @@ pub const SCHEMA: &str = "
pub struct Record<R> {
pub remote: R,
pub documents: Option<Documents>,
- pub key: Option<Vec<u8>>,
+ pub key: Option<String>,
pub input_hash: String,
pub failure: Option<String>,
- pub attempt: String,
}
pub type AlbumRecord = Record<RemoteCollection>;
@@ -50,8 +51,7 @@ impl<R: Serialize> Record<R> {
remote: R,
previous: Option<Self>,
parent_key: &[u8],
- retry: bool,
- attempt: &str,
+ failed_inputs: &mut HashSet<String>,
decrypt: impl FnOnce(&R) -> std::result::Result<(Key, Documents), E>,
) -> Result<Self> {
let mut input = serde_json::to_value(&remote)?;
@@ -94,14 +94,17 @@ impl<R: Serialize> Record<R> {
)?);
if let Some(mut previous) = previous
&& previous.input_hash == input_hash
- && !(retry && previous.documents.is_none())
+ && (previous.documents.is_some() || failed_inputs.contains(&input_hash))
{
previous.remote = remote;
return Ok(previous);
}
let (key, documents, failure) = match decrypt(&remote) {
- Ok((key, documents)) => (Some(key.as_bytes().to_vec()), Some(documents), None),
- Err(error) if error.is_record_failure() => (None, None, Some(error.to_string())),
+ Ok((key, documents)) => (Some(b64::encode(key.as_bytes())), Some(documents), None),
+ Err(error) if error.is_record_failure() => {
+ failed_inputs.insert(input_hash.clone());
+ (None, None, Some(error.to_string()))
+ }
Err(error) => return Err(error.into()),
};
Ok(Self {
@@ -110,7 +113,6 @@ impl<R: Serialize> Record<R> {
documents,
input_hash,
failure,
- attempt: attempt.to_owned(),
})
}
@@ -120,7 +122,9 @@ impl<R: Serialize> Record<R> {
.clone()
.unwrap_or_else(|| "metadata is unavailable".into())
})?;
- let key = Key::try_from_slice(self.key.as_deref().context("decrypted record has no key")?)?;
+ let key = Key::try_from_slice(&b64::decode(
+ self.key.as_deref().context("decrypted record has no key")?,
+ )?)?;
Ok((key, documents))
}
}
@@ -194,7 +198,7 @@ pub struct Entry {
pub struct Replica<'a> {
pub db: &'a mut Db,
user_id: i64,
- attempt: String,
+ failed_inputs: HashSet<String>,
}
impl<'a> Replica<'a> {
@@ -202,7 +206,7 @@ impl<'a> Replica<'a> {
Self {
db,
user_id,
- attempt: uuid::Uuid::new_v4().to_string(),
+ failed_inputs: HashSet::new(),
}
}
@@ -223,7 +227,7 @@ impl<'a> Replica<'a> {
session.secret_key.as_bytes().as_slice()
};
let record =
- Record::receive(remote, previous, parent_key, false, &self.attempt, |r| {
+ Record::receive(remote, previous, parent_key, &mut self.failed_inputs, |r| {
collections::decrypt(r, session)
})?;
changes.push((id, Some(record)));
@@ -236,13 +240,16 @@ impl<'a> Replica<'a> {
tx.execute("DELETE FROM photos_collections WHERE id=?1", [id])?;
}
}
- tx.execute("INSERT INTO photos_sync VALUES(1,?1) ON CONFLICT(id) DO UPDATE SET cursor=excluded.cursor", [page.cursor])?;
+ tx.execute(
+ "INSERT INTO photos_sync VALUES(1,?1) ON CONFLICT(id) DO UPDATE SET cursor=excluded.cursor",
+ [page.cursor],
+ )?;
Ok(())
})?;
Ok(())
}
- pub fn retry_album(&mut self, session: &Session, id: i64, force: bool) -> Result<()> {
+ pub fn retry_album(&mut self, session: &Session, id: i64) -> Result<()> {
let Some(previous) = self.album_record(id)? else {
return Ok(());
};
@@ -254,13 +261,11 @@ impl<'a> Replica<'a> {
} else {
session.secret_key.as_bytes().as_slice()
};
- let retry = force && previous.attempt != self.attempt;
let record = Record::receive(
previous.remote.clone(),
Some(previous),
parent_key,
- retry,
- &self.attempt,
+ &mut self.failed_inputs,
|r| collections::decrypt(r, session),
)?;
self.db.write(|tx| save_album(tx, record, self.user_id))?;
@@ -273,17 +278,12 @@ impl<'a> Replica<'a> {
collections: &[Collection],
) -> Result<()> {
for collection in collections {
- self.sync_album_files(session, collection, false).await?;
+ self.sync_album_files(session, collection).await?;
}
Ok(())
}
- pub async fn sync_album_files(
- &mut self,
- session: &Session,
- album: &Collection,
- retry_failed: bool,
- ) -> Result<()> {
+ pub async fn sync_album_files(&mut self, session: &Session, album: &Collection) -> Result<()> {
let (mut cursor, synced_to): (i64, Option<i64>) = self.db.read(|db| {
Ok(db.query_row(
"SELECT files_cursor,files_synced_to FROM photos_collections WHERE id=?1",
@@ -306,18 +306,26 @@ impl<'a> Replica<'a> {
remote,
previous,
album.key.as_bytes(),
- false,
- &self.attempt,
+ &mut self.failed_inputs,
|r| files::decrypt(r, &album.key),
)?;
changes.push((id, Some(record)));
}
self.db.write(|tx| {
for (id, record) in changes {
- if let Some(record) = record { save_file(tx, album.id, record, self.user_id)?; }
- else { tx.execute("DELETE FROM photos_files WHERE collection_id=?1 AND id=?2", params![album.id,id])?; }
+ if let Some(record) = record {
+ save_file(tx, album.id, record, self.user_id)?;
+ } else {
+ tx.execute(
+ "DELETE FROM photos_files WHERE collection_id=?1 AND id=?2",
+ params![album.id, id],
+ )?;
+ }
}
- tx.execute("UPDATE photos_collections SET files_cursor=?1,files_synced_to=CASE WHEN ?2 THEN files_synced_to ELSE ?3 END WHERE id=?4", params![page.cursor,page.has_more,album.updated_at_micros,album.id])?;
+ tx.execute(
+ "UPDATE photos_collections SET files_cursor=?1,files_synced_to=CASE WHEN ?2 THEN files_synced_to ELSE ?3 END WHERE id=?4",
+ params![page.cursor, page.has_more, album.updated_at_micros, album.id],
+ )?;
Ok(())
})?;
cursor = page.cursor;
@@ -337,13 +345,11 @@ impl<'a> Replica<'a> {
}
for previous in records {
after = previous.remote.id;
- let retry = retry_failed && previous.attempt != self.attempt;
let record = Record::receive(
previous.remote.clone(),
Some(previous),
album.key.as_bytes(),
- retry,
- &self.attempt,
+ &mut self.failed_inputs,
|r| files::decrypt(r, &album.key),
)?;
self.db
@@ -481,7 +487,10 @@ fn save_album(
}
};
let json = encode(&record)?;
- tx.execute("INSERT INTO photos_collections(id,name,updated_at,record) VALUES(?1,?2,?3,?4) ON CONFLICT(id) DO UPDATE SET name=coalesce(excluded.name,photos_collections.name),updated_at=excluded.updated_at,record=excluded.record", params![record.remote.id,name,record.remote.updation_time,json])?;
+ tx.execute(
+ "INSERT INTO photos_collections(id,name,updated_at,record) VALUES(?1,?2,?3,?4) ON CONFLICT(id) DO UPDATE SET name=coalesce(excluded.name,photos_collections.name),updated_at=excluded.updated_at,record=excluded.record",
+ params![record.remote.id, name, record.remote.updation_time, json],
+ )?;
Ok(())
}
@@ -502,7 +511,17 @@ fn save_file(
}
};
let json = encode(&record)?;
- tx.execute("INSERT INTO photos_files VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(collection_id,id) DO UPDATE SET name=coalesce(excluded.name,photos_files.name),updated_at=excluded.updated_at,record=excluded.record,decrypt_failed=excluded.decrypt_failed WHERE excluded.updated_at>=photos_files.updated_at", params![album,record.remote.id,name,record.remote.updation_time,json,record.documents.is_none()])?;
+ tx.execute(
+ "INSERT INTO photos_files VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(collection_id,id) DO UPDATE SET name=coalesce(excluded.name,photos_files.name),updated_at=excluded.updated_at,record=excluded.record,decrypt_failed=excluded.decrypt_failed WHERE excluded.updated_at>=photos_files.updated_at",
+ params![
+ album,
+ record.remote.id,
+ name,
+ record.remote.updation_time,
+ json,
+ record.documents.is_none()
+ ],
+ )?;
Ok(())
}
diff --git a/rust/apps/cli-next/tests/cli.rs b/rust/apps/cli-next/tests/cli.rs
index cd53081da6..99e6caab17 100644
--- a/rust/apps/cli-next/tests/cli.rs
+++ b/rust/apps/cli-next/tests/cli.rs
@@ -745,3 +745,6 @@ mod museum;
#[path = "support/export_sources.rs"]
mod export_sources;
+
+#[path = "support/export_process.rs"]
+mod export_process;
diff --git a/rust/apps/cli-next/tests/support/export.rs b/rust/apps/cli-next/tests/support/export.rs
index 098f0e03a0..68f0267389 100644
--- a/rust/apps/cli-next/tests/support/export.rs
+++ b/rust/apps/cli-next/tests/support/export.rs
@@ -1,4 +1,5 @@
use super::*;
+use crate::export_process::ExportChild;
use ente_core::crypto::hash;
use std::{
path::Path,
@@ -10,31 +11,120 @@ use std::{
mod resources;
#[test]
-fn export_lifecycle_and_adoption() -> TestResult {
+fn export_selection_reuses_independent_album_copies() -> TestResult {
Museum::run(|museum| {
tokio::runtime::Runtime::new()?.block_on(async {
- let origin=museum.endpoint();
- let objects=museum.object_store();
- let email=format!("export-{}@example.org",Uuid::new_v4());
- let account=create_account(origin,&email).await;
- let home=TestHome::new();login(&home,"photos",&email,&["--host",origin]);
- let destination=tempfile::tempdir()?;
- let root=destination.path().join("photos");
- let (family,family_key)=create_album(origin,&account,"Family","album").await;
- let (travel,travel_key)=create_album(origin,&account,"Travel","folder").await;
- let (work,work_key)=create_album(origin,&account,"Work","album").await;
- let (favorites,favorites_key)=create_album(origin,&account,"Favorites","favorites").await;
- let (empty,_)=create_album(origin,&account,"Trash","album").await;
- let original=b"unmodified original bytes";
- let (file,key)=upload_fixture(origin,&account,family,&family_key,original,metadata("A.jpg",original)).await;
- place(origin,&account,travel,&travel_key,file,&key,"add-files").await;
- let (work_file,work_file_key)=upload_fixture(origin,&account,work,&work_key,b"work",metadata("Work.jpg",b"work")).await;
- let initial_reads=objects.reads();
- let result=export(&home,&root,&["--album","Family"]);
- assert_eq!(result["copies"]["completed"],1);
- assert_eq!(objects.reads(),initial_reads+1);
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let (account, home, _) = export_account(origin).await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("photos");
+ let (family, family_key) = create_album(origin, &account, "Family", "album").await;
+ let (travel, travel_key) = create_album(origin, &account, "Travel", "folder").await;
+ let (work, work_key) = create_album(origin, &account, "Work", "album").await;
+ let (empty, _) = create_album(origin, &account, "Trash", "album").await;
+ let original = b"unmodified original bytes";
+ let (file, key) = upload_fixture(
+ origin,
+ &account,
+ family,
+ &family_key,
+ original,
+ metadata("A.jpg", original),
+ )
+ .await;
+ place(
+ origin,
+ &account,
+ travel,
+ &travel_key,
+ file,
+ &key,
+ "add-files",
+ )
+ .await;
+ upload_fixture(
+ origin,
+ &account,
+ work,
+ &work_key,
+ b"work",
+ metadata("Work.jpg", b"work"),
+ )
+ .await;
+ let initial_reads = objects.reads();
+ let result = export(&home, &root, &["--album", "Family"]);
+ assert_eq!(result["copies"]["completed"], 1);
+ assert_eq!(objects.reads(), initial_reads + 1);
assert!(!root.join("Travel").exists());
- let root_bytes=fs::read(root.join("export.json"))?;
+ let result = export(&home, &root, &["--album", "Family", "--album", "Travel"]);
+ assert_eq!(result["copies"]["expected"], 2);
+ assert_eq!(objects.reads(), initial_reads + 1);
+ assert_eq!(fs::read(root.join("Travel/A.jpg"))?, original);
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::MetadataExt;
+ assert_ne!(
+ fs::metadata(root.join("Travel/A.jpg"))?.ino(),
+ fs::metadata(root.join("Family/A.jpg"))?.ino()
+ );
+ }
+ let travel_time = fs::metadata(root.join("Travel/A.jpg"))?.modified()?;
+ export(&home, &root, &["--album", "Family"]);
+ assert_eq!(
+ fs::metadata(root.join("Travel/A.jpg"))?.modified()?,
+ travel_time
+ );
+ let scoped = export(
+ &home,
+ &root,
+ &[
+ "--album",
+ "Family",
+ "--album",
+ "Family",
+ "--album",
+ "Travel",
+ "--album",
+ "Work",
+ "--album",
+ &empty.to_string(),
+ "--exclude-album",
+ "Work",
+ ],
+ );
+ assert_eq!(scoped["copies"]["expected"], 2);
+ assert!(root.join("Trash-1/metadata.json").exists());
+ assert_eq!(
+ read_json(&root.join("Trash-1/metadata.json"))["ente"]["albumID"],
+ empty.to_string()
+ );
+ assert!(!root.join("Work").exists());
+ export(&home, &root, &[]);
+ assert_eq!(objects.reads(), initial_reads + 2);
+ assert_eq!(fs::read(root.join("Work/Work.jpg"))?, b"work");
+ Ok(())
+ })
+ })
+}
+
+#[test]
+fn export_metadata_updates_and_repairs_preserve_source_values() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let (account, home, _) = export_account(origin).await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("photos");
+ let (family, family_key) = create_album(origin, &account, "Family", "album").await;
+ let (travel, travel_key) = create_album(origin, &account, "Travel", "folder").await;
+ let original = b"unmodified original bytes";
+ let (file, key) = upload_fixture(origin, &account, family, &family_key, original, metadata("A.jpg", original)).await;
+ place(origin, &account, travel, &travel_key, file, &key, "add-files").await;
+ let initial_reads = objects.reads();
+ export(&home, &root, &["--album", "Family"]);
+ let (favorites,favorites_key)=create_album(origin,&account,"Favorites","favorites").await;
let original_time=fs::metadata(root.join("Family/A.jpg"))?.modified()?;
assert_eq!(original_time,UNIX_EPOCH+Duration::from_micros(1_700_000_001_123_456));
let sidecar=root.join("Family/metadata/A.jpg.json");
@@ -69,55 +159,27 @@ fn export_lifecycle_and_adoption() -> TestResult {
let after=fs::metadata(root.join("Family/A.jpg"))?;
assert_eq!((before.ctime(),before.ctime_nsec()),(after.ctime(),after.ctime_nsec()));
}
- let result=export(&home,&root,&["--album","Family","--album","Travel"]);
- assert_eq!(result["copies"]["expected"],2);
- assert_eq!(objects.reads(),initial_reads+1);
- assert_eq!(fs::read(root.join("Travel/A.jpg"))?,original);
- #[cfg(unix)] {
- use std::os::unix::fs::MetadataExt;
- assert_ne!(fs::metadata(root.join("Travel/A.jpg"))?.ino(),fs::metadata(root.join("Family/A.jpg"))?.ino());
- }
- let travel_time=fs::metadata(root.join("Travel/A.jpg"))?.modified()?;
- export(&home,&root,&["--album","Family"]);
- assert_eq!(fs::metadata(root.join("Travel/A.jpg"))?.modified()?,travel_time);
- let scoped=export(&home,&root,&["--album","Family","--album","Family","--album","Travel","--album","Work","--album",&empty.to_string(),"--exclude-album","Work"]);
- assert_eq!(scoped["copies"]["expected"],2);
- assert!(root.join("Trash-1/metadata.json").exists());
- assert_eq!(read_json(&root.join("Trash-1/metadata.json"))["ente"]["albumID"],empty.to_string());
- assert!(!root.join("Work").exists());
- export(&home,&root,&[]);
- assert_eq!(objects.reads(),initial_reads+2);
- assert_eq!(fs::read(root.join("Work/Work.jpg"))?,b"work");
-
- #[cfg(unix)] {
- use std::os::unix::fs::PermissionsExt;
- fs::write(root.join("Family/metadata/A.jpg.json"),b"{\"custom\":true}")?;
- fs::write(root.join("Work/metadata/Work.jpg.json"),b"{\"custom\":true}")?;
- let protected=root.join("Work/metadata");let permissions=fs::metadata(&protected)?.permissions();
- fs::set_permissions(&protected,fs::Permissions::from_mode(0o500))?;
- let output=home.run(&["photos","export",root.to_str().unwrap(),"--json"]);
- fs::set_permissions(&protected,permissions)?;
- failure(&output);
- let result:Value=serde_json::from_slice(&output.stdout)?;
- assert_eq!(result["copies"],json!({"expected":3,"completed":2,"pending":1}));
- assert_eq!(result["failures"],1);
- assert!(read_json(&root.join("Family/metadata/A.jpg.json")).get("custom").is_none());
- export(&home,&root,&["--album","Work"]);
- }
+ export(&home, &root, &["--album", "Travel"]);
place(origin,&account,favorites,&favorites_key,file,&key,"add-files").await;
for (version,lat,long) in [(1,0,77),(2,12,0),(3,0,0)] {
edit_file(origin,&account,file,&key,version,json!({"editedName":"Renamed.jpg","editedTime":1_700_000_002_654_321i64,"caption":"","lat":lat,"long":long,"dateTime":"2023:11:15 03:43:22","offsetTime":"+05:30","w":4096,"h":3072,"cameraMake":"Example","cameraModel":"Camera","uploaderName":"Guest","mediaType":1,"mvi":123})).await;
export(&home,&root,&["--album","Family"]);
let record=read_json(&root.join("Family/metadata/Renamed.jpg.json"));
- assert_eq!(record["description"],"");assert_eq!(record["favorited"],true);
+ assert_eq!(record["description"],"");
+ assert_eq!(record["favorited"],true);
assert_eq!(record["ente"]["creationTime"],"2023-11-14T22:13:22.654321Z");
assert_eq!(record["ente"]["name"],"Renamed.jpg");
assert_eq!(record["photoTakenTime"]["timestamp"],"1700000002");
assert!(!record.to_string().contains("2023-11-14T22:13:20.123456Z"));
assert!(!record.to_string().contains("not portable"));
for (field,value) in [("localDateTime",json!("2023:11:15 03:43:22")),("utcOffset",json!("+05:30")),("width",json!(4096)),("height",json!(3072)),("cameraMake",json!("Example")),("cameraModel",json!("Camera")),("uploaderName",json!("Guest")),("panorama",json!(true)),("motionPhotoVideoOffset",json!(123))] { assert_eq!(record["ente"][field],value); }
- if lat==0 && long==0 {assert!(record.get("geoData").is_none());assert!(record["ente"].get("location").is_none());}
- else {assert_eq!(record["geoData"],json!({"latitude":lat as f64,"longitude":long as f64}));assert_eq!(record["ente"]["location"],record["geoData"]);}
+ if lat==0 && long==0 {
+ assert!(record.get("geoData").is_none());
+ assert!(record["ente"].get("location").is_none());
+ } else {
+ assert_eq!(record["geoData"],json!({"latitude":lat as f64,"longitude":long as f64}));
+ assert_eq!(record["ente"]["location"],record["geoData"]);
+ }
}
assert!(root.join("Travel/A.jpg").exists());
fs::write(root.join("Family/metadata/Renamed.jpg.json"),b"{\"custom\":true}")?;
@@ -126,8 +188,95 @@ fn export_lifecycle_and_adoption() -> TestResult {
fs::write(root.join("Family/Renamed.jpg"),vec![0;original.len()])?;
export(&home,&root,&["--album","Family"]);
assert_eq!(fs::read(root.join("Family/Renamed.jpg"))?,original);
- assert_eq!(objects.reads(),initial_reads+2);
+ assert_eq!(objects.reads(),initial_reads+1);
+ Ok(())
+ })
+ })
+}
+
+#[cfg(unix)]
+#[test]
+fn export_metadata_failure_does_not_block_an_independent_album() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let (account, home, _) = export_account(origin).await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("photos");
+ use std::os::unix::fs::PermissionsExt;
+ let (family, family_key) = create_album(origin, &account, "Family", "album").await;
+ upload_fixture(
+ origin,
+ &account,
+ family,
+ &family_key,
+ b"family",
+ metadata("A.jpg", b"family"),
+ )
+ .await;
+ let (work, work_key) = create_album(origin, &account, "Work", "album").await;
+ upload_fixture(
+ origin,
+ &account,
+ work,
+ &work_key,
+ b"work",
+ metadata("Work.jpg", b"work"),
+ )
+ .await;
+ export(&home, &root, &[]);
+ fs::write(
+ root.join("Family/metadata/A.jpg.json"),
+ b"{\"custom\":true}",
+ )?;
+ fs::write(
+ root.join("Work/metadata/Work.jpg.json"),
+ b"{\"custom\":true}",
+ )?;
+ let protected = root.join("Work/metadata");
+ let permissions = fs::metadata(&protected)?.permissions();
+ fs::set_permissions(&protected, fs::Permissions::from_mode(0o500))?;
+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
+ fs::set_permissions(&protected, permissions)?;
+ assert!(
+ failure(&output)
+ .to_lowercase()
+ .contains("permission denied")
+ );
+ let result: Value = serde_json::from_slice(&output.stdout)?;
+ assert_eq!(
+ result["copies"],
+ json!({"expected":2,"completed":1,"pending":1})
+ );
+ assert_eq!(result["failures"], 1);
+ assert!(
+ read_json(&root.join("Family/metadata/A.jpg.json"))
+ .get("custom")
+ .is_none()
+ );
+ export(&home, &root, &["--album", "Work"]);
+ Ok(())
+ })
+ })
+}
+#[test]
+fn export_retention_restoration_and_album_removal_preserve_history() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let (account, home, email) = export_account(origin).await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("photos");
+ let (family, family_key) = create_album(origin, &account, "Family", "album").await;
+ let (travel, travel_key) = create_album(origin, &account, "Travel", "folder").await;
+ let original = b"unmodified original bytes";
+ let (file, key) = upload_fixture(origin, &account, family, &family_key, original, metadata("Renamed.jpg", original)).await;
+ place(origin, &account, travel, &travel_key, file, &key, "add-files").await;
+ let initial_reads = objects.reads();
+ export(&home, &root, &["--album", "Family"]);
+ export(&home, &root, &["--album", "Travel"]);
move_file(origin,&account,family,(travel,&travel_key),(file,&key)).await;
let result=export(&home,&root,&["--album","Family"]);
assert_eq!(result["changes"]["retained"],1);
@@ -138,7 +287,7 @@ fn export_lifecycle_and_adoption() -> TestResult {
assert_eq!(export(&home,&root,&["--album","Family"])["changes"]["retained"],0);
place(origin,&account,family,&family_key,file,&key,"add-files").await;
export(&home,&root,&["--album","Family"]);
- assert_eq!(objects.reads(),initial_reads+2);
+ assert_eq!(objects.reads(),initial_reads+1);
assert_eq!(fs::read(root.join("Trash/Family/metadata/Renamed.jpg.json"))?,retained_record);
assert_eq!(fs::metadata(&retained)?.modified()?,retained_time);
fs::remove_dir_all(root.join("Trash"))?;
@@ -148,7 +297,8 @@ fn export_lifecycle_and_adoption() -> TestResult {
export(&home,&root,&["--album","Family"]);
assert_eq!(read_json(&root.join("Trash/Family/metadata.json"))["ente"]["albumID"],family.to_string());
assert_eq!(fs::read(root.join("Trash/Family/Renamed-1.jpg"))?,original);
- let recovery=TestHome::new();login(&recovery,"photos",&email,&["--host",origin]);
+ let recovery=TestHome::new();
+ login(&recovery,"photos",&email,&["--host",origin]);
export(&recovery,&root,&["--adopt","--album","Family"]);
place(origin,&account,family,&family_key,file,&key,"add-files").await;
export(&home,&root,&["--album","Family"]);
@@ -165,37 +315,130 @@ fn export_lifecycle_and_adoption() -> TestResult {
fs::remove_dir_all(root.join("Trash"))?;
export(&home,&root,&["--album",&family.to_string()]);
assert!(!root.join("Trash").exists());
+ Ok(())
+ })
+ })
+}
+
+#[test]
+fn export_adopts_relocated_output_and_verifies_later_selections() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let (account, home, email) = export_account(origin).await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("photos");
+ let (travel, travel_key) = create_album(origin, &account, "Travel", "album").await;
+ upload_fixture(
+ origin,
+ &account,
+ travel,
+ &travel_key,
+ b"travel",
+ metadata("A.jpg", b"travel"),
+ )
+ .await;
+ let (work, work_key) = create_album(origin, &account, "Work", "album").await;
+ upload_fixture(
+ origin,
+ &account,
+ work,
+ &work_key,
+ b"work",
+ metadata("Work.jpg", b"work"),
+ )
+ .await;
+ export(&home, &root, &[]);
+ let root_bytes = fs::read(root.join("export.json"))?;
+ let relocated = destination.path().join("relocated");
+ fs::rename(&root, &relocated)?;
+ let fresh = TestHome::new();
+ login(&fresh, "photos", &email, &["--host", origin]);
+ assert!(
+ failure(&fresh.run(&["photos", "export", relocated.to_str().unwrap()]))
+ .contains("--adopt")
+ );
+ let reads = objects.reads();
+ export(&fresh, &relocated, &["--adopt", "--album", "Travel"]);
+ assert_eq!(objects.reads(), reads);
+ assert_eq!(fs::read(relocated.join("export.json"))?, root_bytes);
+ fs::write(relocated.join("Work/Work.jpg"), b"xxxx")?;
+ export(&fresh, &relocated, &["--album", "Work"]);
+ assert_eq!(objects.reads(), reads + 1);
+ assert_eq!(fs::read(relocated.join("Work/Work.jpg"))?, b"work");
+ Ok(())
+ })
+ })
+}
- let relocated=destination.path().join("relocated");fs::rename(&root,&relocated)?;
- let fresh=TestHome::new();login(&fresh,"photos",&email,&["--host",origin]);
- assert!(failure(&fresh.run(&["photos","export",relocated.to_str().unwrap()])).contains("--adopt"));
- let reads=objects.reads();
- export(&fresh,&relocated,&["--adopt","--album","Travel"]);
- assert_eq!(objects.reads(),reads);
- assert_eq!(fs::read(relocated.join("export.json"))?,root_bytes);
- fs::write(relocated.join("Work/Work.jpg"),b"xxxx")?;
- export(&fresh,&relocated,&["--album","Work"]);
- assert_eq!(objects.reads(),reads+1);
- assert_eq!(fs::read(relocated.join("Work/Work.jpg"))?,b"work");
- request(origin,&account,reqwest::Method::POST,"/files/trash",json!({"items":[{"fileID":work_file,"collectionID":work}]})).await;
- let trashed=export(&fresh,&relocated,&["--album","Work"]);
- assert_eq!(trashed["copies"]["expected"],0);
- assert_eq!(trashed["changes"]["retained"],1);
- assert!(!relocated.join("Work/Work.jpg").exists());
- assert!(!relocated.join("Work/metadata/Work.jpg.json").exists());
- assert_eq!(fs::read(relocated.join("Trash/Work/Work.jpg"))?,b"work");
- assert_eq!(read_json(&relocated.join("Trash/Work/metadata/Work.jpg.json"))["ente"]["fileID"],work_file.to_string());
- place(origin,&account,work,&work_key,work_file,&work_file_key,"restore-files").await;
- let reads=objects.reads();
- export(&fresh,&relocated,&["--album","Work"]);
- assert_eq!(objects.reads(),reads);
+#[test]
+fn export_restores_a_trashed_original_from_retained_output() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let (account, home, _) = export_account(origin).await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("photos");
+ let (work, work_key) = create_album(origin, &account, "Work", "album").await;
+ let (work_file, work_file_key) = upload_fixture(
+ origin,
+ &account,
+ work,
+ &work_key,
+ b"work",
+ metadata("Work.jpg", b"work"),
+ )
+ .await;
+ export(&home, &root, &[]);
+ request(
+ origin,
+ &account,
+ reqwest::Method::POST,
+ "/files/trash",
+ json!({"items":[{"fileID":work_file,"collectionID":work}]}),
+ )
+ .await;
+ let trashed = export(&home, &root, &["--album", "Work"]);
+ assert_eq!(trashed["copies"]["expected"], 0);
+ assert_eq!(trashed["changes"]["retained"], 1);
+ assert!(!root.join("Work/Work.jpg").exists());
+ assert!(!root.join("Work/metadata/Work.jpg.json").exists());
+ assert_eq!(fs::read(root.join("Trash/Work/Work.jpg"))?, b"work");
+ assert_eq!(
+ read_json(&root.join("Trash/Work/metadata/Work.jpg.json"))["ente"]["fileID"],
+ work_file.to_string()
+ );
+ place(
+ origin,
+ &account,
+ work,
+ &work_key,
+ work_file,
+ &work_file_key,
+ "restore-files",
+ )
+ .await;
+ let reads = objects.reads();
+ let restored = export(&home, &root, &["--album", "Work"]);
+ assert_eq!(
+ restored["copies"],
+ json!({"expected":1,"completed":1,"pending":0})
+ );
+ assert_eq!(fs::read(root.join("Work/Work.jpg"))?, b"work");
+ assert_eq!(
+ read_json(&root.join("Work/metadata/Work.jpg.json"))["ente"]["fileID"],
+ work_file.to_string()
+ );
+ assert_eq!(objects.reads(), reads);
Ok(())
})
})
}
#[test]
-fn export_bounds_failed_stages_and_keeps_servicing_existing_output() -> TestResult {
+fn export_destination_conflicts_do_not_block_other_jobs() -> TestResult {
Museum::run(|museum| {
tokio::runtime::Runtime::new()?.block_on(async {
let origin = museum.endpoint();
@@ -209,8 +452,7 @@ fn export_bounds_failed_stages_and_keeps_servicing_existing_output() -> TestResu
let destination = tempfile::tempdir()?;
let root = destination.path().join("export");
export(&home, &root, &[]);
- let limit = 256 * 1024 * 1024 / stream::DECRYPTION_CHUNK_SIZE;
- let total = limit + 2;
+ let total = 4;
for index in 0..total {
let name = format!("{index}.jpg");
upload_fixture(
@@ -241,33 +483,39 @@ fn export_bounds_failed_stages_and_keeps_servicing_existing_output() -> TestResu
file,
&key,
1,
- json!({"caption":"updated while staging is full"}),
+ json!({"caption":"updated despite other conflicts"}),
)
.await;
- let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
- assert!(failure(&output).contains("filled export staging"));
+ let output = home.run(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "--jobs",
+ "2",
+ "--json",
+ ]);
+ assert!(failure(&output).contains("unrelated occupant"));
let result: Value = serde_json::from_slice(&output.stdout)?;
assert_eq!(result["copies"]["completed"], 1);
assert_eq!(result["copies"]["pending"], total);
- assert_eq!(objects.reads(), reads + limit);
+ assert_eq!(objects.reads(), reads + total);
assert_eq!(
read_json(&root.join("Healthy/metadata/Healthy.jpg.json"))["description"],
- "updated while staging is full"
+ "updated despite other conflicts"
);
let db = export_database(&home);
- let staged: i64 = db.query_row(
- "SELECT count(DISTINCT json_extract(record,'$.file')) FROM temporaries",
- [],
- |r| r.get(0),
- )?;
- assert_eq!(staged, limit as i64);
+ let staged: i64 = db.query_row("SELECT count(*) FROM temporaries", [], |r| r.get(0))?;
+ assert_eq!(staged, 0);
for index in 0..total {
let path = root.join("Blocked").join(format!("{index}.jpg"));
assert_eq!(fs::read(&path)?, b"unrelated");
fs::remove_file(path)?;
}
- assert_eq!(export(&home, &root, &[])["copies"]["completed"], total + 1);
- assert_eq!(objects.reads(), reads + total);
+ assert_eq!(
+ export(&home, &root, &["--jobs", "1"])["copies"]["completed"],
+ total + 1
+ );
+ assert_eq!(objects.reads(), reads + 2 * total);
assert_eq!(
db.query_row("SELECT count(*) FROM temporaries", [], |r| r
.get::<_, i64>(0))?,
@@ -341,7 +589,9 @@ fn export_resumes_a_live_pair_after_publication_failure() -> TestResult {
let root = destination.path().join("export");
export(&home, &root, &[]);
let db = export_database(&home);
- db.execute_batch("CREATE TRIGGER fail_video_publication BEFORE UPDATE ON temporaries WHEN json_extract(new.record,'$.destination')='Live/Motion.mov' BEGIN SELECT RAISE(ABORT,'injected publication failure'); END;")?;
+ db.execute_batch(
+ "CREATE TRIGGER fail_video_publication BEFORE UPDATE ON pending WHEN json_extract(new.action,'$.Publish.output.Media.role')='video' BEGIN SELECT RAISE(ABORT,'injected publication failure'); END;",
+ )?;
upload_live(origin, &account, album, &key, "Motion.heic", b"image", b"video").await;
let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
assert!(failure(&output).contains("injected publication failure"));
@@ -350,13 +600,13 @@ fn export_resumes_a_live_pair_after_publication_failure() -> TestResult {
assert_eq!(fs::read(root.join("Live/Motion.heic"))?, b"image");
assert!(!root.join("Live/Motion.mov").exists());
assert!(!root.join("Live/metadata/Motion.heic.json").exists());
- let pending: String = db.query_row("SELECT path FROM temporaries WHERE json_extract(record,'$.role')='video' AND json_extract(record,'$.hash') IS NOT NULL", [], |r| r.get(0))?;
- assert_eq!(fs::read(root.join(&pending))?, b"video");
+ assert_eq!(db.query_row("SELECT count(*) FROM temporaries", [], |r| r.get::<_, i64>(0))?, 0);
+ let image_time = fs::metadata(root.join("Live/Motion.heic"))?.modified()?;
let reads = objects.reads();
db.execute_batch("DROP TRIGGER fail_video_publication;")?;
export(&home, &root, &[]);
- assert_eq!(objects.reads(), reads);
- assert!(!root.join(pending).exists());
+ assert_eq!(objects.reads(), reads + 1);
+ assert_eq!(fs::metadata(root.join("Live/Motion.heic"))?.modified()?, image_time);
assert_eq!(fs::read(root.join("Live/Motion.heic"))?, b"image");
assert_eq!(fs::read(root.join("Live/Motion.mov"))?, b"video");
for name in ["Motion.heic", "Motion.mov"] {
@@ -369,7 +619,7 @@ fn export_resumes_a_live_pair_after_publication_failure() -> TestResult {
let fresh = TestHome::new();
fresh.write_vault(&home.read_vault());
export(&fresh, &root, &["--adopt"]);
- assert_eq!(objects.reads(), reads);
+ assert_eq!(objects.reads(), reads + 1);
assert_eq!(read_json(&sidecar)["ente"]["components"][0]["role"], "image");
Ok(())
})
@@ -377,7 +627,173 @@ fn export_resumes_a_live_pair_after_publication_failure() -> TestResult {
}
#[test]
-fn export_live_photos_parallel_transfers_and_locking() -> TestResult {
+fn export_adopts_live_sidecars_with_different_metadata() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let email = format!("live-adoption-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ let (active, active_key) = create_album(origin, &account, "Active", "album").await;
+ let (unselected, unselected_key) =
+ create_album(origin, &account, "Unselected", "album").await;
+ let (history, history_key) = create_album(origin, &account, "History", "album").await;
+ let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
+ for (name, bytes) in [("image.heic", b"image"), ("video.mov", b"video")] {
+ zip.start_file(
+ name,
+ zip::write::SimpleFileOptions::default()
+ .compression_method(zip::CompressionMethod::Stored),
+ )?;
+ zip.write_all(bytes)?;
+ }
+ let archive = zip.finish()?.into_inner();
+ let mut data = metadata("Motion.heic", &archive);
+ data["fileType"] = json!(2);
+ data["hash"] = json!(format!("{}:{}", digest(b"image"), digest(b"video")));
+ let (file, key) =
+ upload_fixture(origin, &account, active, &active_key, &archive, data).await;
+ for (album, album_key) in [(unselected, &unselected_key), (history, &history_key)] {
+ place(origin, &account, album, album_key, file, &key, "add-files").await;
+ }
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("export");
+ export(&home, &root, &[]);
+ move_file(
+ origin,
+ &account,
+ history,
+ (active, &active_key),
+ (file, &key),
+ )
+ .await;
+ request(
+ origin,
+ &account,
+ reqwest::Method::DELETE,
+ &format!("/collections/v3/{history}?collectionID={history}&keepFiles=true"),
+ json!({}),
+ )
+ .await;
+ export(&home, &root, &[]);
+ edit_file(
+ origin,
+ &account,
+ file,
+ &key,
+ 1,
+ json!({"caption":"current remote caption","w":640}),
+ )
+ .await;
+ let reads = objects.reads();
+ let mut preserved = Vec::new();
+ for folder in ["Active", "Unselected", "Trash/History"] {
+ for (index, name) in ["Motion.heic", "Motion.mov"].into_iter().enumerate() {
+ let sidecar = root
+ .join(folder)
+ .join("metadata")
+ .join(format!("{name}.json"));
+ let mut record = read_json(&sidecar);
+ record["description"] = json!(format!("{folder} component {index}"));
+ record["ente"]["width"] = json!(1024 + index);
+ fs::write(&sidecar, serde_json::to_vec(&record)?)?;
+ if folder != "Active" {
+ preserved.push((
+ sidecar.clone(),
+ fs::read(&sidecar)?,
+ fs::metadata(&sidecar)?.modified()?,
+ ));
+ }
+ let media = root.join(folder).join(name);
+ preserved.push((
+ media.clone(),
+ fs::read(&media)?,
+ fs::metadata(&media)?.modified()?,
+ ));
+ }
+ }
+ let fresh = TestHome::new();
+ fresh.write_vault(&home.read_vault());
+ export(&fresh, &root, &["--adopt", "--album", "Active"]);
+ assert_eq!(objects.reads(), reads);
+ for name in ["Motion.heic", "Motion.mov"] {
+ let record = read_json(&root.join("Active/metadata").join(format!("{name}.json")));
+ assert_eq!(record["description"], "current remote caption");
+ assert_eq!(record["ente"]["width"], 640);
+ }
+ for (path, bytes, modified) in preserved {
+ assert_eq!(fs::read(&path)?, bytes);
+ assert_eq!(fs::metadata(path)?.modified()?, modified);
+ }
+ move_file(
+ origin,
+ &account,
+ unselected,
+ (active, &active_key),
+ (file, &key),
+ )
+ .await;
+ for name in ["Motion.heic", "Motion.mov"] {
+ fs::remove_file(
+ root.join("Unselected/metadata")
+ .join(format!("{name}.json")),
+ )?;
+ }
+ export(&fresh, &root, &["--album", "Unselected"]);
+ for (index, name) in ["Motion.heic", "Motion.mov"].into_iter().enumerate() {
+ let record = read_json(
+ &root
+ .join("Trash/Unselected/metadata")
+ .join(format!("{name}.json")),
+ );
+ assert_eq!(
+ record["description"],
+ format!("Unselected component {index}")
+ );
+ assert_eq!(record["ente"]["width"], 1024 + index);
+ }
+ for folder in ["Active", "Trash/History"] {
+ let sidecar = root.join(folder).join("metadata/Motion.mov.json");
+ let original = fs::read(&sidecar)?;
+ for (pointer, value) in [
+ ("/ente/fileID", json!((file + 1).to_string())),
+ ("/ente/components/0/path", json!("Other.heic")),
+ ("/ente/components/0/size", json!(100)),
+ ("/ente/components/0/hash", json!(digest(b"other"))),
+ ("/ente/component", json!("image")),
+ ] {
+ let mut record: Value = serde_json::from_slice(&original)?;
+ *record.pointer_mut(pointer).unwrap() = value;
+ fs::write(&sidecar, serde_json::to_vec(&record)?)?;
+ let ambiguous = TestHome::new();
+ ambiguous.write_vault(&home.read_vault());
+ let output = ambiguous.run(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "--adopt",
+ "--album",
+ "Active",
+ ]);
+ let error = failure(&output);
+ assert!(
+ error.contains("conflict:")
+ || error.contains("contradictory sidecar location"),
+ "{folder} {pointer}: {error}"
+ );
+ fs::write(&sidecar, &original)?;
+ }
+ }
+ assert_eq!(objects.reads(), reads);
+ Ok(())
+ })
+ })
+}
+
+#[test]
+fn export_live_pairs_retry_and_reuse_without_rewriting_unchanged_output() -> TestResult {
Museum::run(|museum| {
tokio::runtime::Runtime::new()?.block_on(async {
let origin = museum.endpoint();
@@ -386,8 +802,6 @@ fn export_live_photos_parallel_transfers_and_locking() -> TestResult {
let account = create_account(origin, &email).await;
let home = TestHome::new();
login(&home, "photos", &email, &["--host", origin]);
- let other = TestHome::new();
- login(&other, "photos", &email, &["--host", origin]);
let (album, key) = create_album(origin, &account, "Live", "album").await;
let (copy, copy_key) = create_album(origin, &account, "Copies", "album").await;
let image = vec![61u8; stream::ENCRYPTION_CHUNK_SIZE + 17];
@@ -419,39 +833,22 @@ fn export_live_photos_parallel_transfers_and_locking() -> TestResult {
"add-files",
)
.await;
- for index in 0..4 {
- let bytes = vec![index as u8; stream::ENCRYPTION_CHUNK_SIZE + index + 100];
- upload_fixture(
- origin,
- &account,
- album,
- &key,
- &bytes,
- metadata(&format!("{index}.jpg"), &bytes),
- )
- .await;
- }
+ let original = vec![0; stream::ENCRYPTION_CHUNK_SIZE + 100];
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ &original,
+ metadata("0.jpg", &original),
+ )
+ .await;
let destination = tempfile::tempdir()?;
let root = destination.path().join("export");
- objects.delay_chunks(Duration::from_millis(15));
objects.interrupt_reads(1);
- let child = home
- .command(&["photos", "export", root.to_str().unwrap(), "--json"])
- .spawn()?;
- let deadline = std::time::Instant::now() + Duration::from_secs(20);
- while !root.join("export.json").exists() && std::time::Instant::now() < deadline {
- std::thread::sleep(Duration::from_millis(10));
- }
- let started = std::time::Instant::now();
- assert!(
- failure(&other.run(&["photos", "export", root.to_str().unwrap(), "--adopt"]))
- .contains("another writer")
- );
- assert!(started.elapsed() < Duration::from_secs(3));
- let result: Value = serde_json::from_slice(&success(child.wait_with_output()?).stdout)?;
- assert_eq!(result["copies"]["completed"], 6);
- assert!(objects.peak_reads() > 1);
- assert_eq!(objects.reads(), 6);
+ let result = export(&home, &root, &[]);
+ assert_eq!(result["copies"]["completed"], 3);
+ assert_eq!(objects.reads(), 3);
for folder in ["Live", "Copies"] {
assert_eq!(fs::read(root.join(folder).join("Motion.heic"))?, image);
assert_eq!(fs::read(root.join(folder).join("Motion.mov"))?, video);
@@ -461,6 +858,64 @@ fn export_live_photos_parallel_transfers_and_locking() -> TestResult {
}
let reads = objects.reads();
export(&home, &root, &[]);
+ let paths = [
+ "export.json",
+ "Live/metadata.json",
+ "Live/0.jpg",
+ "Live/metadata/0.jpg.json",
+ "Live/Motion.heic",
+ "Live/Motion.mov",
+ "Live/metadata/Motion.heic.json",
+ "Live/metadata/Motion.mov.json",
+ "Copies/metadata.json",
+ "Copies/Motion.heic",
+ "Copies/Motion.mov",
+ "Copies/metadata/Motion.heic.json",
+ "Copies/metadata/Motion.mov.json",
+ ];
+ let unchanged: Vec<_> = paths
+ .iter()
+ .map(|path| {
+ let path = root.join(path);
+ (
+ fs::read(&path).unwrap(),
+ fs::metadata(&path).unwrap().modified().unwrap(),
+ )
+ })
+ .collect();
+ let db = export_database(&home);
+ for table in ["albums", "placements", "components", "json_records", "pending"] {
+ for action in ["INSERT", "UPDATE", "DELETE"] {
+ db.execute_batch(&format!(
+ "CREATE TRIGGER unchanged_{table}_{action} BEFORE {action} ON {table} BEGIN SELECT RAISE(ABORT,'unchanged association write'); END;",
+ ))?;
+ }
+ }
+ let result = success(
+ home.command(&["photos", "export", root.to_str().unwrap(), "--json"])
+ .env("TOKIO_WORKER_THREADS", "1")
+ .output()?,
+ );
+ let result: Value = serde_json::from_slice(&result.stdout)?;
+ assert_eq!(result["complete"], true);
+ assert_eq!(
+ result["copies"],
+ json!({"expected":3,"completed":3,"pending":0})
+ );
+ assert_eq!(
+ result["changes"],
+ json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})
+ );
+ assert_eq!(objects.reads(), reads);
+ for (path, before) in paths.iter().zip(unchanged) {
+ let path = root.join(path);
+ assert_eq!((fs::read(&path)?, fs::metadata(&path)?.modified()?), before);
+ }
+ for table in ["albums", "placements", "components", "json_records", "pending"] {
+ for action in ["INSERT", "UPDATE", "DELETE"] {
+ db.execute_batch(&format!("DROP TRIGGER unchanged_{table}_{action};"))?;
+ }
+ }
for entry in fs::read_dir(root.join("Live"))? {
let entry = entry?;
if entry.file_type()?.is_file() && entry.file_name() != "metadata.json" {
@@ -476,7 +931,81 @@ fn export_live_photos_parallel_transfers_and_locking() -> TestResult {
export(&home, &root, &[]);
assert_eq!(objects.reads(), reads);
assert_eq!(fs::read(root.join("Live/Motion.mov"))?, video);
- let unrelated = root.join("Live/.ente-11111111111111111111111111111111.part");
+ Ok(())
+ })
+ })
+}
+
+#[test]
+fn export_excludes_another_writer_while_downloads_are_running() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let (account, home, email) = export_account(origin).await;
+ let other = TestHome::new();
+ login(&other, "photos", &email, &["--host", origin]);
+ let (album, key) = create_album(origin, &account, "Album", "album").await;
+ for name in ["First.jpg", "Second.jpg"] {
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ name.as_bytes(),
+ metadata(name, name.as_bytes()),
+ )
+ .await;
+ }
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("photos");
+ let held = objects.hold_reads(0);
+ let child = ExportChild::spawn(&mut home.command(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "-j",
+ "2",
+ "--json",
+ ]))?;
+ held.wait_for(2);
+ let second = ExportChild::spawn(&mut other.command(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "--adopt",
+ ]))?;
+ assert!(failure(&second.wait_with_output()?).contains("another writer"));
+ drop(held);
+ let result: Value = serde_json::from_slice(&success(child.wait_with_output()?).stdout)?;
+ assert_eq!(result["copies"]["completed"], 2);
+ assert_eq!(objects.peak_reads(), 2);
+ Ok(())
+ })
+ })
+}
+
+#[test]
+fn export_recovers_after_cancelled_and_killed_downloads() -> TestResult {
+ Museum::run(|museum| {
+ tokio::runtime::Runtime::new()?.block_on(async {
+ let origin = museum.endpoint();
+ let objects = museum.object_store();
+ let (account, home, _) = export_account(origin).await;
+ let (album, key) = create_album(origin, &account, "Album", "album").await;
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ b"original",
+ metadata("Photo.jpg", b"original"),
+ )
+ .await;
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("photos");
+ export(&home, &root, &[]);
+ let unrelated = root.join("Album/.ente-11111111111111111111111111111111.part");
fs::write(&unrelated, b"unassociated")?;
let unchanged_root = fs::read(root.join("export.json"))?;
for graceful in [true, false] {
@@ -484,26 +1013,13 @@ fn export_live_photos_parallel_transfers_and_locking() -> TestResult {
if graceful {
continue;
}
- for index in 0..4 {
- fs::remove_file(root.join(format!("Live/{index}.jpg")))?;
- }
- objects.delay_chunks(Duration::from_millis(30));
- let before = objects.reads();
- let mut child = home
- .command(&[
- "photos",
- "export",
- root.to_str().unwrap(),
- "--album",
- "Live",
- "--json",
- ])
- .spawn()?;
- let deadline = std::time::Instant::now() + Duration::from_secs(20);
- while objects.reads() == before && std::time::Instant::now() < deadline {
- std::thread::sleep(Duration::from_millis(10));
- }
- assert!(objects.reads() > before);
+ fs::remove_file(root.join("Album/Photo.jpg"))?;
+ let held = objects.hold_reads(0);
+ let mut child = ExportChild::spawn(
+ home.command(&["photos", "export", root.to_str().unwrap(), "--json"])
+ .env("TOKIO_WORKER_THREADS", "1"),
+ )?;
+ held.wait_for(1);
if graceful {
assert!(
Command::new("kill")
@@ -515,13 +1031,17 @@ fn export_live_photos_parallel_transfers_and_locking() -> TestResult {
child.kill()?;
}
let interrupted = child.wait_with_output()?;
- failure(&interrupted);
+ let error = failure(&interrupted);
if graceful {
- let result: Value = serde_json::from_slice(&interrupted.stdout)?;
- assert_eq!(result["complete"], false);
+ assert!(error.contains("cancelled"), "{error}");
+ assert_eq!(
+ serde_json::from_slice::<Value>(&interrupted.stdout)?["complete"],
+ false
+ );
}
- objects.delay_chunks(Duration::ZERO);
- export(&home, &root, &["--album", "Live"]);
+ drop(held);
+ export(&home, &root, &[]);
+ assert_eq!(fs::read(root.join("Album/Photo.jpg"))?, b"original");
assert_eq!(fs::read(&unrelated)?, b"unassociated");
assert_eq!(fs::read(root.join("export.json"))?, unchanged_root);
}
@@ -928,7 +1448,16 @@ fn export_allocates_live_pairs_from_their_actual_extensions() -> TestResult {
}
#[test]
-fn export_refills_transfers_while_a_large_original_is_pending() -> TestResult {
+fn export_refills_transfers_while_an_ordinary_original_is_held() -> TestResult {
+ export_refills_transfers(false)
+}
+
+#[test]
+fn export_refills_transfers_while_a_live_original_is_held() -> TestResult {
+ export_refills_transfers(true)
+}
+
+fn export_refills_transfers(live: bool) -> TestResult {
Museum::run(|museum| {
tokio::runtime::Runtime::new()?.block_on(async {
let origin = museum.endpoint();
@@ -937,117 +1466,112 @@ fn export_refills_transfers_while_a_large_original_is_pending() -> TestResult {
let account = create_account(origin, &email).await;
let home = TestHome::new();
login(&home, "photos", &email, &["--host", origin]);
- let small_count = std::thread::available_parallelism()
- .map(usize::from)
- .unwrap_or(1)
- .min(63)
- + 2;
- for live in [false, true] {
- let (album, key) = create_album(
+ let small_count = 4;
+ let (album, key) = create_album(
+ origin,
+ &account,
+ if live { "Live" } else { "Ordinary" },
+ "album",
+ )
+ .await;
+ let large = vec![31u8; stream::ENCRYPTION_CHUNK_SIZE + 17];
+ if live {
+ upload_live(
origin,
&account,
- if live { "Live" } else { "Ordinary" },
- "album",
+ album,
+ &key,
+ "IMG:1.heic",
+ &large,
+ b"video",
+ )
+ .await;
+ let mut contender = metadata("IMG?1.mov", b"ordinary contender");
+ contender["fileType"] = json!(1);
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ b"ordinary contender",
+ contender,
+ )
+ .await;
+ upload_live(
+ origin,
+ &account,
+ album,
+ &key,
+ "IMG?1-1.heic",
+ b"small image",
+ b"small video",
+ )
+ .await;
+ } else {
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ &large,
+ metadata("Slow.jpg", &large),
+ )
+ .await;
+ }
+ for index in 0..small_count {
+ upload_fixture(
+ origin,
+ &account,
+ album,
+ &key,
+ b"independent",
+ metadata(&format!("Small-{index}.jpg"), b"independent"),
)
.await;
- let large = vec![31u8; 32 * 1024 * 1024];
- if live {
- upload_live(
- origin,
- &account,
- album,
- &key,
- "IMG:1.heic",
- &large,
- b"video",
- )
- .await;
- let mut contender = metadata("IMG?1.mov", b"ordinary contender");
- contender["fileType"] = json!(1);
- upload_fixture(
- origin,
- &account,
- album,
- &key,
- b"ordinary contender",
- contender,
- )
- .await;
- upload_live(
- origin,
- &account,
- album,
- &key,
- "IMG?1-1.heic",
- b"small image",
- b"small video",
- )
- .await;
- } else {
- upload_fixture(
- origin,
- &account,
- album,
- &key,
- &large,
- metadata("Slow.jpg", &large),
- )
- .await;
- }
- for index in 0..small_count {
- upload_fixture(
- origin,
- &account,
- album,
- &key,
- b"independent",
- metadata(&format!("Small-{index}.jpg"), b"independent"),
- )
- .await;
- }
- objects.delay_chunks(Duration::from_millis(30));
- let destination = tempfile::tempdir()?;
- let root = destination.path().join("export");
- let folder = root.join(if live { "Live" } else { "Ordinary" });
- let child = home
- .command(&[
- "photos",
- "export",
- root.to_str().unwrap(),
- "--album",
- &album.to_string(),
- "--json",
- ])
- .spawn()?;
- let last = folder.join(format!("Small-{}.jpg", small_count - 1));
- let deadline = std::time::Instant::now() + Duration::from_secs(40);
- while !last.exists() && std::time::Instant::now() < deadline {
- std::thread::sleep(Duration::from_millis(10));
- }
- assert!(last.exists(), "independent transfers did not make progress");
- assert!(
- !folder
- .join(if live { "IMG_1-2.heic" } else { "Slow.jpg" })
- .exists(),
- "a slow first transfer blocked later placements"
- );
- success(child.wait_with_output()?);
- objects.delay_chunks(Duration::ZERO);
- if live {
- assert_eq!(fs::read(folder.join("IMG_1-2.heic"))?, large);
- assert_eq!(fs::read(folder.join("IMG_1-2.mov"))?, b"video");
- assert_eq!(fs::read(folder.join("IMG_1-1.heic"))?, b"small image");
- assert_eq!(fs::read(folder.join("IMG_1-1.mov"))?, b"small video");
- assert_eq!(fs::read(folder.join("IMG_1.mov"))?, b"ordinary contender");
- assert!(!folder.join("IMG_1.heic").exists());
- } else {
- assert_eq!(fs::read(folder.join("Slow.jpg"))?, large);
- }
- assert_eq!(
- export(&home, &root, &["--album", &album.to_string()])["changes"],
- json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})
- );
}
+ let held = objects.hold_reads(1024 * 1024);
+ let destination = tempfile::tempdir()?;
+ let root = destination.path().join("export");
+ let folder = root.join(if live { "Live" } else { "Ordinary" });
+ let child = ExportChild::spawn(&mut home.command(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "--album",
+ &album.to_string(),
+ "-j",
+ "2",
+ "--json",
+ ]))?;
+ held.wait_for(1);
+ let last = folder.join(format!("Small-{}.jpg", small_count - 1));
+ let deadline = std::time::Instant::now() + Duration::from_secs(60);
+ while !last.exists() && std::time::Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ assert!(last.exists(), "independent transfers did not make progress");
+ assert!(
+ !folder
+ .join(if live { "IMG_1-2.heic" } else { "Slow.jpg" })
+ .exists(),
+ "a slow first transfer blocked later placements"
+ );
+ drop(held);
+ success(child.wait_with_output()?);
+ if live {
+ assert_eq!(fs::read(folder.join("IMG_1-2.heic"))?, large);
+ assert_eq!(fs::read(folder.join("IMG_1-2.mov"))?, b"video");
+ assert_eq!(fs::read(folder.join("IMG_1-1.heic"))?, b"small image");
+ assert_eq!(fs::read(folder.join("IMG_1-1.mov"))?, b"small video");
+ assert_eq!(fs::read(folder.join("IMG_1.mov"))?, b"ordinary contender");
+ assert!(!folder.join("IMG_1.heic").exists());
+ } else {
+ assert_eq!(fs::read(folder.join("Slow.jpg"))?, large);
+ }
+ assert_eq!(
+ export(&home, &root, &["--album", &album.to_string()])["changes"],
+ json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})
+ );
Ok(())
})
})
@@ -1099,3 +1623,11 @@ async fn upload_live_parts(
.await
.0
}
+
+async fn export_account(origin: &str) -> (AuthenticatedAccount, TestHome, String) {
+ let email = format!("export-{}@example.org", Uuid::new_v4());
+ let account = create_account(origin, &email).await;
+ let home = TestHome::new();
+ login(&home, "photos", &email, &["--host", origin]);
+ (account, home, email)
+}
diff --git a/rust/apps/cli-next/tests/support/export_maintenance.rs b/rust/apps/cli-next/tests/support/export_maintenance.rs
new file mode 100644
index 0000000000..bdb82dd45a
--- /dev/null
+++ b/rust/apps/cli-next/tests/support/export_maintenance.rs
@@ -0,0 +1,2207 @@
+use super::*;
+use crate::export_process::ExportChild;
+
+fn listing(server: &mut mockito::ServerGuard, albums: Value) -> mockito::Mock {
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":albums}).to_string())
+ .create()
+}
+
+fn revision(mut record: Value, version: i64) -> Value {
+ record["updationTime"] = json!(version);
+ record
+}
+
+fn removed(mut file: Value, version: i64) -> Value {
+ file["updationTime"] = json!(version);
+ file["file"]["encryptedData"] = json!("-");
+ file["metadata"]["encryptedData"] = json!("-");
+ file
+}
+
+fn file_key(file: &Value, album_key: &Key) -> Key {
+ Key::try_from_slice(
+ &secretbox::decrypt(
+ &b64::decode(file["encryptedKey"].as_str().unwrap()).unwrap(),
+ &Nonce::try_from_slice(
+ &b64::decode(file["keyDecryptionNonce"].as_str().unwrap()).unwrap(),
+ )
+ .unwrap(),
+ album_key,
+ )
+ .unwrap(),
+ )
+ .unwrap()
+}
+
+fn public(mut file: Value, album_key: &Key, value: Value, version: i64) -> Value {
+ let encrypted = blob::encrypt_json(&value, &file_key(&file, album_key)).unwrap();
+ file["pubMagicMetadata"] = json!({"data":b64::encode(&encrypted.encrypted_data),"header":b64::encode(encrypted.decryption_header.as_bytes())});
+ revision(file, version)
+}
+
+fn count(db: &rusqlite::Connection, sql: &str) -> i64 {
+ db.query_row(sql, [], |r| r.get(0)).unwrap()
+}
+
+#[test]
+fn export_retries_transient_collection_and_file_refresh_failures() {
+ for route in ["/collections/v2", "/collections/v2/diff"] {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let failed = server
+ .mock("GET", route)
+ .match_query(mockito::Matcher::Any)
+ .with_status(503)
+ .expect(1)
+ .create();
+ let albums = listing(&mut server, json!([collection(1, "First", &key)]));
+ let (file, bytes) = source(10, &key, b"original", "Photo.jpg");
+ let files = page(&mut server, 1, 0, json!([file]), false)
+ .expect(1)
+ .create();
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let result = run(&home, &root, &[], true);
+ assert_eq!(
+ result["copies"],
+ json!({"expected":1,"completed":1,"pending":0})
+ );
+ assert_component(&root, "First", "Photo.jpg", b"original");
+ let db = database(&home);
+ assert_eq!(count(&db, "SELECT cursor FROM photos_sync"), 20);
+ assert_eq!(
+ db.query_row(
+ "SELECT files_cursor,files_synced_to FROM photos_collections WHERE id=1",
+ [],
+ |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?))
+ )
+ .unwrap(),
+ (10, 20)
+ );
+ failed.assert();
+ albums.assert();
+ files.assert();
+ fetched.assert();
+ }
+}
+
+#[test]
+fn export_snapshot_uses_current_healthy_memberships_and_preserves_raw_bytes() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let mut favorites = collection(5, "Favorites", &key);
+ favorites["type"] = json!("favorites");
+ listing(
+ &mut server,
+ json!([
+ collection(1, "First", &key),
+ collection(2, "Second", &key),
+ collection(3, "Third", &key),
+ collection(4, "Broken", &key),
+ favorites
+ ]),
+ );
+ let (base, encrypted) = source(10, &key, b"shared original", "Photo.jpg");
+ let first = public(base.clone(), &key, json!({"caption":"older"}), 10);
+ let second = public(base.clone(), &key, json!({"caption":"winner"}), 30);
+ let third = public(base.clone(), &key, json!({"caption":"losing tie"}), 30);
+ let mut broken = revision(base.clone(), 40);
+ broken["metadata"]["encryptedData"] = json!("invalid ciphertext");
+ let mut invalid = Vec::new();
+ for (id, bytes) in [(11, b"\xff\0{".as_slice()), (12, b"{broken".as_slice())] {
+ let (mut file, _) = source(id, &key, b"unused", "Invalid.jpg");
+ let encrypted = blob::encrypt(bytes, &file_key(&file, &key)).unwrap();
+ file["metadata"] = json!({"encryptedData":b64::encode(&encrypted.encrypted_data),"decryptionHeader":b64::encode(encrypted.decryption_header.as_bytes())});
+ invalid.push(revision(file, 50));
+ }
+ page(&mut server, 1, 0, json!([first]), false).create();
+ page(&mut server, 2, 0, json!([second]), false).create();
+ page(&mut server, 3, 0, json!([third]), false).create();
+ page(
+ &mut server,
+ 4,
+ 0,
+ json!([broken, invalid[0], invalid[1]]),
+ false,
+ )
+ .create();
+ let favorite_page = page(&mut server, 5, 0, json!([base]), false)
+ .expect(1)
+ .create();
+ let downloaded = download(&mut server, 10, &encrypted, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let result = run(&home, &root, &[], false);
+ assert_eq!(
+ result["copies"],
+ json!({"expected":7,"completed":4,"pending":3})
+ );
+ for folder in ["First", "Second", "Third", "Favorites"] {
+ let record = read_json(&root.join(format!("{folder}/metadata/Photo.jpg.json")));
+ assert_eq!(record["description"], "winner");
+ assert_eq!(record["favorited"], true);
+ }
+ let untouched = root.join("Second/metadata/Photo.jpg.json");
+ let before = (
+ fs::read(&untouched).unwrap(),
+ fs::metadata(&untouched).unwrap().modified().unwrap(),
+ );
+ run(&home, &root, &["--album", "First"], true);
+ assert_eq!(
+ read_json(&root.join("First/metadata/Photo.jpg.json"))["description"],
+ "older"
+ );
+ assert_eq!(
+ (
+ fs::read(&untouched).unwrap(),
+ fs::metadata(&untouched).unwrap().modified().unwrap()
+ ),
+ before
+ );
+ run(&home, &root, &["--album", "Broken"], false);
+ let replica = database(&home);
+ for (id, bytes) in [(11, b"\xff\0{".as_slice()), (12, b"{broken".as_slice())] {
+ let raw = record(&replica, 4, id);
+ assert_eq!(
+ b64::decode(raw["documents"]["original"].as_str().unwrap()).unwrap(),
+ bytes
+ );
+ }
+ favorite_page.assert();
+ downloaded.assert();
+}
+
+#[test]
+fn export_keeps_latest_deleted_album_selector_without_rewriting_retained_history() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(&mut server, json!([collection(1, "Family", &key)]));
+ let (gone, gone_bytes) = source(10, &key, b"gone", "Gone.jpg");
+ let (kept, kept_bytes) = source(11, &key, b"kept", "Kept.jpg");
+ page(&mut server, 1, 0, json!([gone, kept]), false).create();
+ download(&mut server, 10, &gone_bytes, 1);
+ download(&mut server, 11, &kept_bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ albums.remove();
+ let albums = listing(
+ &mut server,
+ json!([revision(collection(1, "Family", &key), 40)]),
+ );
+ page(&mut server, 1, 10, json!([removed(gone, 30)]), false).create();
+ run(&home, &root, &[], true);
+ let history = root.join("Trash/Family/metadata.json");
+ let retained = (
+ fs::read(&history).unwrap(),
+ fs::metadata(&history).unwrap().modified().unwrap(),
+ );
+ albums.remove();
+ let travel = revision(collection(1, "Travel", &key), 50);
+ let albums = listing(&mut server, json!([travel]));
+ page(&mut server, 1, 30, json!([]), false).create();
+ run(&home, &root, &[], true);
+ fs::write(root.join("Travel/notes.txt"), b"local addition").unwrap();
+ albums.remove();
+ let mut deleted = revision(travel, 60);
+ deleted["isDeleted"] = json!(true);
+ let albums = listing(&mut server, json!([deleted]));
+ run(&home, &root, &["--album", "Travel"], true);
+ assert_eq!(
+ fs::read(root.join("Trash/Family/Gone.jpg")).unwrap(),
+ b"gone"
+ );
+ assert_eq!(
+ fs::read(root.join("Trash/Family/Kept.jpg")).unwrap(),
+ b"kept"
+ );
+ assert_eq!(
+ (
+ fs::read(&history).unwrap(),
+ fs::metadata(&history).unwrap().modified().unwrap()
+ ),
+ retained
+ );
+ run(&home, &root, &["--album", "Travel"], true);
+ run(&home, &root, &["--album", "1"], true);
+ albums.remove();
+ listing(
+ &mut server,
+ json!([revision(collection(2, "Travel", &key), 70)]),
+ );
+ page(&mut server, 2, 0, json!([]), false).create();
+ assert_eq!(run(&home, &root, &["--album", "2"], false)["conflicts"], 1);
+ assert_eq!(
+ fs::read(root.join("Travel/notes.txt")).unwrap(),
+ b"local addition"
+ );
+ assert!(!root.join("Travel-1").exists());
+}
+
+#[test]
+fn export_discards_failed_downloads_and_services_other_albums() {
+ for delete_album in [false, true] {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(
+ &mut server,
+ json!([collection(1, "First", &key), collection(2, "Second", &key)]),
+ );
+ let empty = page(&mut server, 1, 0, json!([]), false).create();
+ let second_empty = page(&mut server, 2, 0, json!([]), false).create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &["-j", "1"], true);
+ fs::write(root.join("First/Blocked.jpg"), b"unrelated").unwrap();
+ albums.remove();
+ empty.remove();
+ let albums = listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), 40)]),
+ );
+ let (file, bytes) = source(10, &key, b"staged", "Blocked.jpg");
+ let file = revision(file, 30);
+ page(&mut server, 1, 0, json!([file]), false).create();
+ let staged = download(&mut server, 10, &bytes, 1);
+ assert_eq!(run(&home, &root, &["-j", "1"], false)["conflicts"], 1);
+ let db = export_database(&home);
+ assert_eq!(count(&db, "SELECT count(*) FROM temporaries"), 0);
+ run(&home, &root, &["--album", "Second", "-j", "1"], true);
+ assert_eq!(count(&db, "SELECT count(*) FROM pending WHERE file=10"), 1);
+ albums.remove();
+ second_empty.remove();
+ let mut first = revision(collection(1, "First", &key), 60);
+ if delete_album {
+ first["isDeleted"] = json!(true);
+ }
+ listing(
+ &mut server,
+ json!([first, revision(collection(2, "Second", &key), 60)]),
+ );
+ if !delete_album {
+ page(&mut server, 1, 30, json!([removed(file, 50)]), false).create();
+ }
+ let (new_file, new_bytes) = source(20, &key, b"new download", "New.jpg");
+ page(&mut server, 2, 0, json!([revision(new_file, 50)]), false).create();
+ let downloaded = download(&mut server, 20, &new_bytes, 1);
+ let result = run(&home, &root, &["-j", "1"], true);
+ assert_eq!(
+ result["copies"],
+ json!({"expected":1,"completed":1,"pending":0})
+ );
+ assert_eq!(count(&db, "SELECT count(*) FROM temporaries"), 0);
+ assert_eq!(count(&db, "SELECT count(*) FROM pending"), 0);
+ assert_eq!(
+ fs::read(root.join("Second/New.jpg")).unwrap(),
+ b"new download"
+ );
+ assert_eq!(
+ fs::read(root.join("First/Blocked.jpg")).unwrap(),
+ b"unrelated"
+ );
+ if delete_album {
+ assert_eq!(
+ read_json(&root.join("Trash/First/metadata.json"))["ente"]["albumID"],
+ "1"
+ );
+ }
+ staged.assert();
+ downloaded.assert();
+ }
+}
+
+#[test]
+fn export_releases_replaced_and_cancelled_pending_names() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let mut albums = listing(&mut server, json!([collection(1, "First", &key)]));
+ let (base, bytes) = source(10, &key, b"original", "Original.jpg");
+ page(&mut server, 1, 0, json!([base]), false).create();
+ let downloaded = download(&mut server, 10, &bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let db = export_database(&home);
+ for name in ["Blocked.jpg", "Another.jpg"] {
+ fs::write(root.join("First").join(name), b"unrelated").unwrap();
+ }
+ let mut version = 10;
+ for (name, complete) in [
+ ("Blocked.jpg", false),
+ ("Another.jpg", false),
+ ("Original.jpg", true),
+ ("Renamed.jpg", true),
+ ("RENAMED.jpg", true),
+ ] {
+ albums.remove();
+ albums = listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), version + 20)]),
+ );
+ page(
+ &mut server,
+ 1,
+ version,
+ json!([public(
+ base.clone(),
+ &key,
+ json!({"editedName":name}),
+ version + 10
+ )]),
+ false,
+ )
+ .create();
+ version += 10;
+ run(&home, &root, &[], complete);
+ if complete {
+ assert_eq!(
+ fs::read(root.join("First").join(name)).unwrap(),
+ b"original"
+ );
+ } else {
+ let pending: String = db
+ .query_row(
+ "SELECT folded1 FROM pending WHERE album=1 AND file=10",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ assert_eq!(pending, name.to_lowercase());
+ }
+ }
+ for name in ["Blocked album", "Another album"] {
+ fs::create_dir(root.join(name)).unwrap();
+ fs::write(root.join(name).join("notes"), b"unrelated").unwrap();
+ }
+ page(&mut server, 1, version, json!([]), false).create();
+ for (index, name) in ["Blocked album", "Another album", "First"]
+ .into_iter()
+ .enumerate()
+ {
+ albums.remove();
+ albums = listing(
+ &mut server,
+ json!([revision(collection(1, name, &key), 100 + index as i64)]),
+ );
+ run(&home, &root, &[], name == "First");
+ if name != "First" {
+ let pending: String = db
+ .query_row(
+ "SELECT folded1 FROM pending WHERE owner='active:1'",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ assert_eq!(pending, name.to_lowercase());
+ }
+ }
+ albums.remove();
+ listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), 120)]),
+ );
+ let (new_file, new_bytes) = source(11, &key, b"new original", "Original.jpg");
+ page(
+ &mut server,
+ 1,
+ version,
+ json!([revision(new_file, 110)]),
+ false,
+ )
+ .create();
+ let new_download = download(&mut server, 11, &new_bytes, 1);
+ run(&home, &root, &[], true);
+ assert_eq!(
+ fs::read(root.join("First/Original.jpg")).unwrap(),
+ b"new original"
+ );
+ assert_eq!(
+ fs::read(root.join("First/RENAMED.jpg")).unwrap(),
+ b"original"
+ );
+ assert!(!root.join("First/Original-1.jpg").exists());
+ downloaded.assert();
+ new_download.assert();
+}
+
+#[test]
+fn export_keeps_live_names_occupied_until_every_component_and_sidecar_moves() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(&mut server, json!([collection(1, "Live", &key)]));
+ let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
+ for (name, bytes) in [
+ ("image.heic", b"image".as_slice()),
+ ("video.mov", b"video".as_slice()),
+ ] {
+ zip.start_file(
+ name,
+ zip::write::SimpleFileOptions::default()
+ .compression_method(zip::CompressionMethod::Stored),
+ )
+ .unwrap();
+ zip.write_all(bytes).unwrap();
+ }
+ let archive = zip.finish().unwrap().into_inner();
+ let (mut base, bytes) = source(10, &key, &archive, "Motion.heic");
+ let data = json!({"title":"Motion.heic","fileType":2,"creationTime":1_700_000_000_123_456i64,"hash":format!("{}:{}",b64::encode(&hash::hash(b"image",Some(64),None).unwrap()),b64::encode(&hash::hash(b"video",Some(64),None).unwrap()))});
+ let encrypted = blob::encrypt_json(&data, &file_key(&base, &key)).unwrap();
+ base["metadata"] = json!({"encryptedData":b64::encode(&encrypted.encrypted_data),"decryptionHeader":b64::encode(encrypted.decryption_header.as_bytes())});
+ page(&mut server, 1, 0, json!([base]), false).create();
+ let downloaded = download(&mut server, 10, &bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let adopted = TestHome::new();
+ adopted.seed(&server.url());
+ run(&adopted, &root, &["--adopt"], true);
+ let db = export_database(&adopted);
+ db.execute_batch(
+ "CREATE TRIGGER stop_move BEFORE UPDATE ON pending WHEN json_extract(old.action,'$.Move.source.name')='Motion.heic' AND new.action='null' BEGIN SELECT RAISE(ABORT,'stop after image move'); END;",
+ )
+ .unwrap();
+ albums.remove();
+ let albums = listing(
+ &mut server,
+ json!([revision(collection(1, "Live", &key), 40)]),
+ );
+ let renamed = public(base.clone(), &key, json!({"editedName":"Renamed.heic"}), 30);
+ page(&mut server, 1, 10, json!([renamed]), false).create();
+ run_failure(&adopted, &root, &[], "stop after image move");
+ assert_eq!(fs::read(root.join("Live/Renamed.heic")).unwrap(), b"image");
+ assert_eq!(fs::read(root.join("Live/Motion.mov")).unwrap(), b"video");
+ db.execute_batch("DROP TRIGGER stop_move;").unwrap();
+ fs::write(root.join("Live/Renamed.mov"), b"unrelated").unwrap();
+ albums.remove();
+ let albums = listing(
+ &mut server,
+ json!([revision(collection(1, "Live", &key), 60)]),
+ );
+ let (contender, contender_bytes) = source(11, &key, b"contender", "Motion.heic");
+ page(&mut server, 1, 30, json!([revision(contender, 50)]), false).create();
+ let new_download = download(&mut server, 11, &contender_bytes, 1);
+ run(&adopted, &root, &[], false);
+ assert_eq!(
+ fs::read(root.join("Live/Motion-1.heic")).unwrap(),
+ b"contender"
+ );
+ fs::remove_file(root.join("Live/Renamed.mov")).unwrap();
+ run(&adopted, &root, &[], true);
+ albums.remove();
+ let albums = listing(
+ &mut server,
+ json!([revision(collection(1, "Live", &key), 80)]),
+ );
+ let (later, later_bytes) = source(12, &key, b"new original", "Motion.heic");
+ page(&mut server, 1, 50, json!([revision(later, 70)]), false).create();
+ let later_download = download(&mut server, 12, &later_bytes, 1);
+ run(&adopted, &root, &[], true);
+ assert_component(&root, "Live", "Motion.heic", b"new original");
+ assert_eq!(
+ read_json(&root.join("Live/metadata/Motion.heic.json"))["ente"]["fileID"],
+ "12"
+ );
+ assert_eq!(
+ fs::read(root.join("Live/Motion-1.heic")).unwrap(),
+ b"contender"
+ );
+ db.execute_batch(
+ "CREATE TRIGGER stop_sidecar BEFORE UPDATE ON pending WHEN json_extract(old.action,'$.Remove.location.name')='metadata/Renamed.mov.json' AND new.action='null' BEGIN SELECT RAISE(ABORT,'stop after final sidecar removal'); END;",
+ )
+ .unwrap();
+ albums.remove();
+ listing(
+ &mut server,
+ json!([revision(collection(1, "Live", &key), 100)]),
+ );
+ page(
+ &mut server,
+ 1,
+ 70,
+ json!([public(base, &key, json!({"editedName":"RENAMED.heic"}), 90)]),
+ false,
+ )
+ .create();
+ run_failure(&adopted, &root, &[], "stop after final sidecar removal");
+ db.execute_batch("DROP TRIGGER stop_sidecar;").unwrap();
+ assert!(!root.join("Live/metadata/Renamed.mov.json").exists());
+ assert_eq!(
+ count(
+ &db,
+ "SELECT count(*) FROM json_records WHERE name='metadata/Renamed.mov.json' AND hash IS NOT NULL"
+ ),
+ 1
+ );
+ fs::remove_file(root.join("Live/metadata/RENAMED.heic.json")).unwrap();
+ run(&adopted, &root, &[], true);
+ for (name, bytes) in [
+ ("RENAMED.heic", b"image".as_slice()),
+ ("RENAMED.mov", b"video".as_slice()),
+ ] {
+ assert_eq!(fs::read(root.join("Live").join(name)).unwrap(), bytes);
+ assert_eq!(
+ read_json(&root.join(format!("Live/metadata/{name}.json")))["title"],
+ name
+ );
+ }
+ downloaded.assert();
+ new_download.assert();
+ later_download.assert();
+}
+
+#[test]
+fn export_stops_dispatch_and_joins_workers_after_a_fatal_transfer() {
+ use std::sync::{Mutex, mpsc};
+ use std::time::Duration;
+
+ let mut server = mockito::Server::new();
+ let mut slow_server = mockito::Server::new();
+ let mut fatal_server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ listing(&mut server, json!([collection(1, "First", &key)]));
+ let original = vec![73; 8 * 1024 * 1024];
+ let (slow_file, slow_bytes) = source(10, &key, &original, "Slow.jpg");
+ let (fatal_file, fatal_bytes) = source(11, &key, b"recovered", "Failed.jpg");
+ let (next_file, next_bytes) = source(12, &key, b"next", "Next.jpg");
+ page(
+ &mut server,
+ 1,
+ 0,
+ json!([slow_file, fatal_file, next_file]),
+ false,
+ )
+ .create();
+ let (started, reading) = mpsc::channel();
+ let (fatal_release, fatal_wait) = mpsc::channel::<()>();
+ let fatal_wait = Mutex::new(fatal_wait);
+ let (release, held) = mpsc::channel::<()>();
+ let held = Mutex::new(held);
+ let response_bytes = slow_bytes.clone();
+ let slow = slow_server
+ .mock("GET", "/slow")
+ .with_chunked_body(move |writer| {
+ writer.write_all(&response_bytes[..64 * 1024])?;
+ let _ = started.send(());
+ let _ = held.lock().unwrap().recv();
+ writer.write_all(&response_bytes[64 * 1024..])
+ })
+ .create();
+ let fatal_response = fatal_server
+ .mock("GET", "/fatal")
+ .with_status(401)
+ .with_body_from_request(move |_| {
+ let _ = fatal_wait.lock().unwrap().recv();
+ b"unauthorized".to_vec()
+ })
+ .create();
+ let slow_url = server
+ .mock("GET", "/files/download/v3/10")
+ .with_body(json!({"url":format!("{}/slow",slow_server.url())}).to_string())
+ .create();
+ let fatal_url = server
+ .mock("GET", "/files/download/v3/11")
+ .with_body(json!({"url":format!("{}/fatal",fatal_server.url())}).to_string())
+ .create();
+ let undispatched = server
+ .mock("GET", "/files/download/v3/12")
+ .with_status(500)
+ .expect(0)
+ .create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let child = ExportChild::spawn(
+ home.command(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "-j",
+ "2",
+ "--json",
+ ])
+ .env("TOKIO_WORKER_THREADS", "1"),
+ )
+ .unwrap();
+ reading
+ .recv_timeout(Duration::from_secs(60))
+ .expect("the independent download never started");
+ drop(fatal_release);
+ let output = child.wait_with_output().unwrap();
+ drop(release);
+ let error = failure(&output);
+ assert!(error.contains("401"));
+ assert!(error.contains("cancelled"), "{error}");
+ for line in error.lines().filter(|line| line.contains("401")) {
+ assert_eq!(line.matches("401").count(), 1, "{line}");
+ }
+ assert_eq!(
+ serde_json::from_slice::<Value>(&output.stdout).unwrap()["complete"],
+ false
+ );
+ undispatched.assert();
+ let lock = fs::File::options()
+ .read(true)
+ .write(true)
+ .open(root.join("export.json"))
+ .unwrap();
+ lock.try_lock().unwrap();
+ assert_eq!(fs::read_dir(root.join("First")).unwrap().count(), 1);
+ assert!(!root.join("First/Slow.jpg").exists());
+ drop(lock);
+ slow_url.remove();
+ fatal_url.remove();
+ undispatched.remove();
+ slow.assert();
+ fatal_response.assert();
+ download(&mut server, 10, &slow_bytes, 1);
+ download(&mut server, 11, &fatal_bytes, 1);
+ download(&mut server, 12, &next_bytes, 1);
+ run(&home, &root, &["-j", "2"], true);
+ assert_eq!(fs::read(root.join("First/Slow.jpg")).unwrap(), original);
+ assert_eq!(
+ fs::read(root.join("First/Failed.jpg")).unwrap(),
+ b"recovered"
+ );
+ assert_eq!(fs::read(root.join("First/Next.jpg")).unwrap(), b"next");
+}
+
+#[test]
+fn export_reports_shared_store_failure_and_joins_workers() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(&mut server, json!([collection(1, "First", &key)]));
+ let empty = page(&mut server, 1, 0, json!([]), false).create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let db = export_database(&home);
+ db.execute_batch(
+ "CREATE TRIGGER fail_component BEFORE INSERT ON components BEGIN SELECT RAISE(ABORT,'injected shared store failure'); END;",
+ )
+ .unwrap();
+ albums.remove();
+ empty.remove();
+ listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), 30)]),
+ );
+ let (file, bytes) = source(10, &key, b"original", "Original.jpg");
+ page(&mut server, 1, 0, json!([file]), false).create();
+ let downloaded = download(&mut server, 10, &bytes, 1);
+ let child = ExportChild::spawn(
+ home.command(&["photos", "export", root.to_str().unwrap(), "--json"])
+ .env("TOKIO_WORKER_THREADS", "1"),
+ )
+ .unwrap();
+ let output = child.wait_with_output().unwrap();
+ assert!(failure(&output).contains("injected shared store failure"));
+ assert_eq!(
+ serde_json::from_slice::<Value>(&output.stdout).unwrap()["complete"],
+ false
+ );
+ let lock = fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .open(root.join("export.json"))
+ .unwrap();
+ lock.try_lock().unwrap();
+ drop(lock);
+ db.execute_batch("DROP TRIGGER fail_component;").unwrap();
+ run(&home, &root, &[], true);
+ assert_eq!(
+ fs::read(root.join("First/Original.jpg")).unwrap(),
+ b"original"
+ );
+ downloaded.assert();
+}
+
+#[test]
+fn export_cancels_pending_retention_when_selected_membership_returns() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let mut albums = listing(&mut server, json!([collection(1, "First", &key)]));
+ let (base, bytes) = source(10, &key, b"original", "Original.jpg");
+ page(&mut server, 1, 0, json!([base]), false).create();
+ let downloaded = download(&mut server, 10, &bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let db = export_database(&home);
+ fs::create_dir_all(root.join("Trash/First")).unwrap();
+ fs::write(root.join("Trash/First/notes"), b"unrelated album").unwrap();
+ let mut version = 10;
+ for (step, remove, complete) in [
+ (0, true, false),
+ (1, false, true),
+ (2, true, true),
+ (3, false, true),
+ (4, true, false),
+ (5, false, true),
+ ] {
+ if step == 2 {
+ fs::remove_file(root.join("Trash/First/notes")).unwrap();
+ fs::remove_dir(root.join("Trash/First")).unwrap();
+ }
+ if step == 4 {
+ fs::write(root.join("Trash/First/Original-1.jpg"), b"unrelated file").unwrap();
+ }
+ albums.remove();
+ albums = listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), version + 30)]),
+ );
+ let next = if remove {
+ removed(base.clone(), version + 20)
+ } else {
+ revision(base.clone(), version + 20)
+ };
+ page(&mut server, 1, version, json!([next]), false).create();
+ version += 20;
+ run(&home, &root, &[], complete);
+ if !complete {
+ assert_eq!(count(&db, "SELECT count(*) FROM pending"), 1);
+ assert_eq!(
+ count(&db, "SELECT count(*) FROM pending WHERE retained=1"),
+ 1
+ );
+ assert_eq!(
+ fs::read(root.join("First/Original.jpg")).unwrap(),
+ b"original"
+ );
+ }
+ if !remove {
+ assert_eq!(count(&db, "SELECT count(*) FROM pending"), 0);
+ assert_eq!(
+ count(&db, "SELECT count(*) FROM pending WHERE retained=1"),
+ 0
+ );
+ assert_eq!(
+ fs::read(root.join("First/Original.jpg")).unwrap(),
+ b"original"
+ );
+ }
+ }
+ assert_eq!(
+ fs::read(root.join("Trash/First/Original.jpg")).unwrap(),
+ b"original"
+ );
+ assert_eq!(
+ fs::read(root.join("Trash/First/Original-1.jpg")).unwrap(),
+ b"unrelated file"
+ );
+ downloaded.assert();
+}
+
+#[test]
+fn export_preserves_pending_names_when_a_later_source_refresh_is_fatal() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(
+ &mut server,
+ json!([collection(1, "First", &key), collection(2, "Second", &key)]),
+ );
+ let first_empty = page(&mut server, 1, 0, json!([]), false).create();
+ page(&mut server, 2, 0, json!([]), false).create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &["-j", "1"], true);
+ fs::write(root.join("First/Blocked.jpg"), b"unrelated").unwrap();
+ albums.remove();
+ first_empty.remove();
+ let albums = listing(
+ &mut server,
+ json!([
+ revision(collection(1, "First", &key), 40),
+ collection(2, "Second", &key)
+ ]),
+ );
+ let (file, bytes) = source(10, &key, b"completed original", "Blocked.jpg");
+ page(&mut server, 1, 0, json!([revision(file, 30)]), false).create();
+ let downloaded = download(&mut server, 10, &bytes, 2);
+ run(&home, &root, &["-j", "1"], false);
+ let db = export_database(&home);
+ let saved: String = db
+ .query_row("SELECT name1 FROM pending WHERE file=10", [], |r| r.get(0))
+ .unwrap();
+ assert_eq!(count(&db, "SELECT count(*) FROM temporaries"), 0);
+ albums.remove();
+ listing(
+ &mut server,
+ json!([
+ revision(collection(1, "First", &key), 60),
+ revision(collection(2, "Second", &key), 60)
+ ]),
+ );
+ let first_success = page(&mut server, 1, 30, json!([]), false)
+ .expect(1)
+ .create();
+ let second_fatal = page(&mut server, 2, 0, json!([]), false)
+ .with_status(401)
+ .expect(1)
+ .create();
+ let output = home.run(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "-j",
+ "1",
+ "--json",
+ ]);
+ assert!(failure(&output).contains("401"));
+ assert_eq!(
+ serde_json::from_slice::<Value>(&output.stdout).unwrap()["complete"],
+ false
+ );
+ first_success.assert();
+ second_fatal.assert();
+ assert_eq!(
+ fs::read(root.join("First/Blocked.jpg")).unwrap(),
+ b"unrelated"
+ );
+ assert_eq!(
+ db.query_row("SELECT name1 FROM pending WHERE file=10", [], |r| r
+ .get::<_, String>(0))
+ .unwrap(),
+ saved
+ );
+ second_fatal.remove();
+ page(&mut server, 2, 0, json!([]), false).create();
+ fs::remove_file(root.join("First/Blocked.jpg")).unwrap();
+ run(&home, &root, &["-j", "1"], true);
+ assert_eq!(count(&db, "SELECT count(*) FROM temporaries"), 0);
+ assert_eq!(
+ fs::read(root.join("First/Blocked.jpg")).unwrap(),
+ b"completed original"
+ );
+ downloaded.assert();
+}
+
+fn original_fixture(
+ key: &Key,
+ bytes: &[u8],
+ live: bool,
+ hashed: bool,
+ version: i64,
+) -> (Value, Vec<u8>) {
+ let name = if live { "Motion.heic" } else { "Photo.jpg" };
+ let original = if live {
+ let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
+ for name in ["image.heic", "video.mov"] {
+ zip.start_file(
+ name,
+ zip::write::SimpleFileOptions::default()
+ .compression_method(zip::CompressionMethod::Stored),
+ )
+ .unwrap();
+ zip.write_all(bytes).unwrap();
+ }
+ zip.finish().unwrap().into_inner()
+ } else {
+ bytes.to_vec()
+ };
+ let (mut remote, encrypted) = source(10, key, &original, name);
+ let mut metadata = json!({"title":name,"fileType":if live {2} else {0},"creationTime":1_700_000_000_123_456i64});
+ if hashed {
+ let hash = b64::encode(&hash::hash(bytes, Some(64), None).unwrap());
+ metadata["hash"] = json!(if live { format!("{hash}:{hash}") } else { hash });
+ }
+ let metadata = blob::encrypt_json(&metadata, &file_key(&remote, key)).unwrap();
+ remote["metadata"] = json!({"encryptedData":b64::encode(&metadata.encrypted_data),"decryptionHeader":b64::encode(metadata.decryption_header.as_bytes())});
+ (revision(remote, version), encrypted)
+}
+
+fn assert_component(root: &Path, folder: &str, name: &str, bytes: &[u8]) {
+ assert_eq!(fs::read(root.join(folder).join(name)).unwrap(), bytes);
+ let record = read_json(&root.join(folder).join(format!("metadata/{name}.json")));
+ let component = record["ente"]["components"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .find(|component| component["path"] == name)
+ .unwrap();
+ assert_eq!(component["size"], bytes.len());
+ assert_eq!(
+ component["hash"],
+ b64::encode(&hash::hash(bytes, Some(64), None).unwrap())
+ );
+}
+
+#[test]
+fn export_interrupted_replacement_never_binds_old_bytes_to_new_verification() {
+ for (live, hashed) in [(false, true), (false, false), (true, true)] {
+ for remove_after in [false, true] {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(
+ &mut server,
+ json!([collection(1, "First", &key), collection(2, "Second", &key)]),
+ );
+ let (old, bytes) = original_fixture(&key, b"old", live, hashed, 10);
+ for album in [1, 2] {
+ page(&mut server, album, 0, json!([old]), false).create();
+ }
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ fetched.assert();
+ fetched.remove();
+ let name = if live { "Motion.heic" } else { "Photo.jpg" };
+ let before = fs::metadata(root.join("Second").join(name))
+ .unwrap()
+ .modified()
+ .unwrap();
+ let db = export_database(&home);
+ db.execute_batch(
+ "CREATE TRIGGER stop_second BEFORE INSERT ON pending WHEN new.album=2 AND json_type(new.action,'$.Publish.output.Media') IS NOT NULL BEGIN SELECT RAISE(ABORT,'stop before second publication'); END;",
+ )
+ .unwrap();
+ albums.remove();
+ let albums = listing(
+ &mut server,
+ json!([
+ revision(collection(1, "First", &key), 40),
+ revision(collection(2, "Second", &key), 40)
+ ]),
+ );
+ let (new, bytes) = original_fixture(&key, b"new", live, hashed, 30);
+ for album in [1, 2] {
+ page(&mut server, album, 10, json!([new]), false).create();
+ }
+ let fetched = download(&mut server, 10, &bytes, 1);
+ run_failure(&home, &root, &[], "stop before second publication");
+ assert_eq!(fs::read(root.join("First").join(name)).unwrap(), b"new");
+ assert_eq!(fs::read(root.join("Second").join(name)).unwrap(), b"old");
+ assert_eq!(
+ fs::metadata(root.join("Second").join(name))
+ .unwrap()
+ .modified()
+ .unwrap(),
+ before
+ );
+ assert_component(&root, "Second", name, b"old");
+ if live {
+ assert_component(&root, "Second", "Motion.mov", b"old");
+ }
+ db.execute_batch("DROP TRIGGER stop_second;").unwrap();
+ if remove_after {
+ albums.remove();
+ listing(
+ &mut server,
+ json!([
+ revision(collection(1, "First", &key), 60),
+ revision(collection(2, "Second", &key), 60)
+ ]),
+ );
+ for album in [1, 2] {
+ page(
+ &mut server,
+ album,
+ 30,
+ json!([removed(new.clone(), 50)]),
+ false,
+ )
+ .create();
+ }
+ run(&home, &root, &[], true);
+ for (folder, bytes) in [("Trash/First", b"new"), ("Trash/Second", b"old")] {
+ assert_component(&root, folder, name, bytes);
+ if live {
+ assert_component(&root, folder, "Motion.mov", bytes);
+ }
+ }
+ fs::remove_file(root.join("Trash/First").join(name)).unwrap();
+ if live {
+ fs::remove_file(root.join("Trash/First/Motion.mov")).unwrap();
+ }
+ fetched.assert();
+ fetched.remove();
+ listing(
+ &mut server,
+ json!([revision(collection(3, "Restored", &key), 80)]),
+ );
+ let restored = revision(if hashed { old.clone() } else { new.clone() }, 70);
+ page(&mut server, 3, 0, json!([restored]), false).create();
+ let restore_get = download(&mut server, 10, &bytes, usize::from(!hashed));
+ run(&home, &root, &["--album", "Restored"], true);
+ let expected = if hashed { b"old" } else { b"new" };
+ assert_component(&root, "Restored", name, expected);
+ if live {
+ assert_component(&root, "Restored", "Motion.mov", expected);
+ }
+ restore_get.assert();
+ } else {
+ run(&home, &root, &[], true);
+ for folder in ["First", "Second"] {
+ assert_component(&root, folder, name, b"new");
+ if live {
+ assert_component(&root, folder, "Motion.mov", b"new");
+ }
+ }
+ }
+ if !remove_after {
+ fetched.assert();
+ }
+ }
+ }
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn export_adoption_ignores_unrelated_non_utf8_entries_but_rejects_claimed_paths() {
+ use std::os::unix::ffi::OsStringExt;
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ listing(&mut server, json!([collection(1, "First", &key)]));
+ let (file, bytes) = original_fixture(&key, b"original", false, true, 10);
+ page(&mut server, 1, 0, json!([file]), false).create();
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let unrelated = root.join(std::ffi::OsString::from_vec(b"note-\xff".to_vec()));
+ let directory = root.join(std::ffi::OsString::from_vec(b"folder-\xfe".to_vec()));
+ let trash = root
+ .join("Trash")
+ .join(std::ffi::OsString::from_vec(b"folder-\xfd".to_vec()));
+ fs::write(&unrelated, b"unrelated").unwrap();
+ for directory in [&directory, &trash] {
+ fs::create_dir_all(directory).unwrap();
+ fs::write(directory.join("metadata.json"), br#"{"title":"unrelated"}"#).unwrap();
+ }
+ let adopted = TestHome::new();
+ adopted.seed(&server.url());
+ run(&adopted, &root, &["--adopt"], true);
+ assert_component(&root, "First", "Photo.jpg", b"original");
+ assert_eq!(fs::read(&unrelated).unwrap(), b"unrelated");
+ fs::copy(
+ root.join("First/metadata.json"),
+ directory.join("metadata.json"),
+ )
+ .unwrap();
+ let rejected = TestHome::new();
+ rejected.seed(&server.url());
+ let output = rejected.run(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "--adopt",
+ "--json",
+ ]);
+ assert!(failure(&output).contains("claimed Ente album path is not UTF-8"));
+ assert_component(&root, "First", "Photo.jpg", b"original");
+ fetched.assert();
+}
+
+#[test]
+fn export_missing_move_endpoints_require_active_repair_or_retained_recovery() {
+ for retaining in [false, true] {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(&mut server, json!([collection(1, "First", &key)]));
+ let (file, bytes) = original_fixture(&key, b"original", false, true, 10);
+ page(&mut server, 1, 0, json!([file]), false).create();
+ let fetched = download(&mut server, 10, &bytes, if retaining { 1 } else { 2 });
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let db = export_database(&home);
+ db.execute_batch(
+ "CREATE TRIGGER stop_move BEFORE UPDATE ON pending WHEN json_extract(old.action,'$.Move.source.name')='Photo.jpg' AND new.action='null' BEGIN SELECT RAISE(ABORT,'stop after media move'); END;",
+ )
+ .unwrap();
+ albums.remove();
+ listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), 40)]),
+ );
+ let changed = if retaining {
+ removed(file, 30)
+ } else {
+ public(file, &key, json!({"editedName":"Renamed.jpg"}), 30)
+ };
+ page(&mut server, 1, 10, json!([changed]), false).create();
+ run_failure(&home, &root, &[], "stop after media move");
+ let moved = if retaining {
+ "Trash/First/Photo.jpg"
+ } else {
+ "First/Renamed.jpg"
+ };
+ assert_eq!(fs::read(root.join(moved)).unwrap(), b"original");
+ fs::remove_file(root.join(moved)).unwrap();
+ assert!(!root.join("First/Photo.jpg").exists());
+ db.execute_batch("DROP TRIGGER stop_move;").unwrap();
+ if retaining {
+ run(&home, &root, &[], false);
+ assert!(!root.join("Trash/First/metadata/Photo.jpg.json").exists());
+ fs::write(root.join(moved), b"original").unwrap();
+ run(&home, &root, &[], true);
+ assert_component(&root, "Trash/First", "Photo.jpg", b"original");
+ } else {
+ run(&home, &root, &[], true);
+ assert_component(&root, "First", "Renamed.jpg", b"original");
+ }
+ fetched.assert();
+ }
+}
+
+#[test]
+#[cfg(unix)]
+fn export_failed_album_move_blocks_retention_until_recovery() {
+ use std::os::unix::fs::PermissionsExt;
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(
+ &mut server,
+ json!([
+ collection(1, "Family", &key),
+ collection(2, "Healthy", &key)
+ ]),
+ );
+ let (file, bytes) = source(10, &key, b"retained", "Photo.jpg");
+ let (healthy, healthy_bytes) = source(20, &key, b"healthy", "Good.jpg");
+ page(&mut server, 1, 0, json!([file]), false).create();
+ page(&mut server, 2, 0, json!([healthy]), false).create();
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let healthy_get = download(&mut server, 20, &healthy_bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ albums.remove();
+ listing(
+ &mut server,
+ json!([
+ revision(collection(1, "Kin", &key), 40),
+ revision(collection(2, "Healthy", &key), 40)
+ ]),
+ );
+ page(&mut server, 1, 10, json!([removed(file, 30)]), false).create();
+ page(
+ &mut server,
+ 2,
+ 10,
+ json!([public(healthy, &key, json!({"caption":"independent"}), 30)]),
+ false,
+ )
+ .create();
+ fs::set_permissions(&root, fs::Permissions::from_mode(0o500)).unwrap();
+ let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
+ fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap();
+ failure(&output);
+ let result: Value = serde_json::from_slice(&output.stdout).unwrap();
+ assert_eq!(result["complete"], false);
+ assert_eq!(result["copies"]["completed"], 1);
+ assert_eq!(
+ read_json(&root.join("Healthy/metadata/Good.jpg.json"))["description"],
+ "independent"
+ );
+ assert_eq!(
+ fs::read(root.join("Family/Photo.jpg")).unwrap(),
+ b"retained"
+ );
+ assert!(!root.join("Trash/Kin/metadata/Photo.jpg.json").exists());
+ run(&home, &root, &[], true);
+ assert_component(&root, "Trash/Kin", "Photo.jpg", b"retained");
+ assert!(!root.join("Kin/Photo.jpg").exists());
+ fetched.assert();
+ healthy_get.assert();
+}
+
+#[test]
+fn export_first_publication_requires_ownership_and_recovers_renamed_bytes() {
+ for media in [false, true] {
+ for after_rename in [false, true] {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(&mut server, json!([collection(1, "First", &key)]));
+ let empty = page(&mut server, 1, 0, json!([]), false).create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let db = export_database(&home);
+ let output = if media { "Media" } else { "Json" };
+ let (operation, record, cleared) = if after_rename {
+ ("UPDATE", "old", "AND new.action='null'")
+ } else {
+ ("INSERT", "new", "")
+ };
+ db.execute_batch(&format!(
+ "CREATE TRIGGER stop_publication BEFORE {operation} ON pending WHEN {record}.file=10 AND json_type({record}.action,'$.Publish.output.{output}') IS NOT NULL {cleared} BEGIN SELECT RAISE(ABORT,'publication interruption'); END;",
+ ))
+ .unwrap();
+ albums.remove();
+ empty.remove();
+ listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), 40)]),
+ );
+ let (file, bytes) = original_fixture(&key, b"original", false, true, 30);
+ page(&mut server, 1, 0, json!([file]), false).create();
+ let fetched = download(
+ &mut server,
+ 10,
+ &bytes,
+ if media && !after_rename { 3 } else { 1 },
+ );
+ run_failure(&home, &root, &[], "publication interruption");
+ assert_eq!(
+ count(&db, "SELECT count(*) FROM components"),
+ i64::from(!media)
+ );
+ assert_eq!(
+ count(
+ &db,
+ "SELECT count(*) FROM json_records WHERE role!='' AND folder IS NOT NULL"
+ ),
+ 0
+ );
+ db.execute_batch("DROP TRIGGER stop_publication;").unwrap();
+ let path = root.join(if media {
+ "First/Photo.jpg"
+ } else {
+ "First/metadata/Photo.jpg.json"
+ });
+ if after_rename {
+ assert!(path.is_file());
+ } else {
+ fs::create_dir_all(path.parent().unwrap()).unwrap();
+ fs::write(&path, b"unrelated").unwrap();
+ assert!(run(&home, &root, &[], false)["conflicts"].as_i64().unwrap() > 0);
+ assert_eq!(fs::read(&path).unwrap(), b"unrelated");
+ fs::remove_file(&path).unwrap();
+ }
+ run(&home, &root, &[], true);
+ assert_component(&root, "First", "Photo.jpg", b"original");
+ fetched.assert();
+ }
+ }
+}
+
+#[test]
+fn export_removal_accounts_for_first_media_before_acknowledgement() {
+ for (live, after_rename) in [(false, false), (true, false), (false, true), (true, true)] {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(&mut server, json!([collection(1, "First", &key)]));
+ let empty = page(&mut server, 1, 0, json!([]), false).create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let db = export_database(&home);
+ let role = if live {
+ if after_rename { "image" } else { "video" }
+ } else {
+ "original"
+ };
+ let (operation, record, cleared) = if after_rename {
+ ("UPDATE", "old", "AND new.action='null'")
+ } else {
+ ("INSERT", "new", "")
+ };
+ db.execute_batch(&format!(
+ "CREATE TRIGGER stop_component BEFORE {operation} ON pending WHEN json_extract({record}.action,'$.Publish.output.Media.role')='{role}' {cleared} BEGIN SELECT RAISE(ABORT,'stop first publication'); END;",
+ ))
+ .unwrap();
+ albums.remove();
+ empty.remove();
+ let albums = listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), 40)]),
+ );
+ let (file, bytes) = original_fixture(&key, b"original", live, true, 30);
+ page(&mut server, 1, 0, json!([file]), false).create();
+ let fetched = download(&mut server, 10, &bytes, 1);
+ run_failure(&home, &root, &[], "stop first publication");
+ assert_eq!(
+ count(
+ &db,
+ "SELECT count(*) FROM json_records WHERE role!='' AND folder IS NOT NULL"
+ ),
+ 0
+ );
+ if after_rename {
+ assert_eq!(count(&db, "SELECT count(*) FROM components"), 0);
+ assert_eq!(
+ count(
+ &db,
+ "SELECT count(*) FROM json_records WHERE role!='' AND folder IS NULL"
+ ),
+ if live { 2 } else { 1 }
+ );
+ }
+ let unrelated = root.join(if live {
+ "First/Motion.mov"
+ } else {
+ "First/Photo.jpg"
+ });
+ if live || !after_rename {
+ fs::write(&unrelated, b"unrelated").unwrap();
+ }
+ db.execute_batch("DROP TRIGGER stop_component;").unwrap();
+ albums.remove();
+ listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), 60)]),
+ );
+ page(&mut server, 1, 30, json!([removed(file, 50)]), false).create();
+ run(&home, &root, &[], true);
+ if live || after_rename {
+ assert_component(
+ &root,
+ "Trash/First",
+ if live { "Motion.heic" } else { "Photo.jpg" },
+ b"original",
+ );
+ } else {
+ assert!(!root.join("Trash/First/metadata/Photo.jpg.json").exists());
+ }
+ if live {
+ assert!(!root.join("Trash/First/Motion.mov").exists());
+ assert!(!root.join("Trash/First/metadata/Motion.mov.json").exists());
+ assert_eq!(read_json(&root.join("Trash/First/metadata/Motion.heic.json"))["ente"]["components"].as_array().unwrap().len(), 2);
+ }
+ if live || !after_rename {
+ assert_eq!(fs::read(unrelated).unwrap(), b"unrelated");
+ }
+ fetched.assert();
+ }
+}
+
+#[test]
+fn export_converts_ordinary_and_live_layouts_without_losing_old_originals() {
+ for (old_live, adopt) in [(false, false), (true, false), (true, true)] {
+ let mut server = mockito::Server::new();
+ let original_home = TestHome::new();
+ original_home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(&mut server, json!([collection(1, "First", &key)]));
+ let (old, old_bytes) = original_fixture(&key, b"old", old_live, true, 10);
+ page(&mut server, 1, 0, json!([old]), false).create();
+ let old_get = download(&mut server, 10, &old_bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&original_home, &root, &[], true);
+ let adopted = TestHome::new();
+ let home = if adopt {
+ adopted.seed(&server.url());
+ run(&adopted, &root, &["--adopt"], true);
+ &adopted
+ } else {
+ &original_home
+ };
+ old_get.assert();
+ old_get.remove();
+ let db = export_database(home);
+ albums.remove();
+ listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), 40)]),
+ );
+ let (new, bytes) = original_fixture(&key, b"new", !old_live, true, 30);
+ page(&mut server, 1, 10, json!([new]), false).create();
+ let denied = server
+ .mock("GET", "/files/download/v3/10")
+ .with_status(403)
+ .expect(1)
+ .create();
+ run(home, &root, &[], false);
+ assert_component(
+ &root,
+ "First",
+ if old_live { "Motion.heic" } else { "Photo.jpg" },
+ b"old",
+ );
+ if old_live {
+ assert_component(&root, "First", "Motion.mov", b"old");
+ }
+ assert!(!root.join("Trash").exists());
+ denied.assert();
+ denied.remove();
+ let boundary = if old_live {
+ "json_type(old.action,'$.Publish.output.Media') IS NOT NULL AND old.retained=0"
+ } else {
+ "json_type(old.action,'$.Move') IS NOT NULL AND old.retained=1"
+ };
+ db.execute_batch(&format!(
+ "CREATE TRIGGER stop_conversion BEFORE UPDATE ON pending WHEN {boundary} AND new.action='null' BEGIN SELECT RAISE(ABORT,'conversion interruption'); END;",
+ ))
+ .unwrap();
+ let fetched = download(&mut server, 10, &bytes, if old_live { 1 } else { 2 });
+ run_failure(home, &root, &[], "conversion interruption");
+ db.execute_batch("DROP TRIGGER stop_conversion;").unwrap();
+ run(home, &root, &[], true);
+ run(home, &root, &[], true);
+ assert_component(
+ &root,
+ "Trash/First",
+ if old_live { "Motion.heic" } else { "Photo.jpg" },
+ b"old",
+ );
+ assert_component(
+ &root,
+ "First",
+ if old_live { "Photo.jpg" } else { "Motion.heic" },
+ b"new",
+ );
+ if old_live {
+ assert_component(&root, "Trash/First", "Motion.mov", b"old");
+ for path in [
+ "Motion.heic",
+ "Motion.mov",
+ "metadata/Motion.heic.json",
+ "metadata/Motion.mov.json",
+ ] {
+ assert!(!root.join("First").join(path).exists());
+ }
+ } else {
+ assert_component(&root, "First", "Motion.mov", b"new");
+ assert!(!root.join("First/Photo.jpg").exists());
+ assert!(!root.join("First/metadata/Photo.jpg.json").exists());
+ }
+ assert!(
+ !root
+ .join("Trash/First")
+ .join(if old_live {
+ "Motion-1.heic"
+ } else {
+ "Photo-1.jpg"
+ })
+ .exists()
+ );
+ fetched.assert();
+ }
+}
+
+struct CrossAlbumPublication {
+ db: rusqlite::Connection,
+ albums: mockito::Mock,
+ fetched: mockito::Mock,
+ server: mockito::ServerGuard,
+ home: TestHome,
+ key: Key,
+ root: tempfile::TempDir,
+}
+
+fn interrupted_cross_album_publication(owned: bool, downloads: usize) -> CrossAlbumPublication {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(
+ &mut server,
+ json!([collection(1, "First", &key), collection(2, "Second", &key)]),
+ );
+ let empty_first = page(&mut server, 1, 0, json!([]), false).create();
+ let empty_second = page(&mut server, 2, 0, json!([]), false).create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path();
+ run(&home, root, &[], true);
+ let db = export_database(&home);
+ fs::write(root.join("First/Photo.jpg"), b"unrelated first").unwrap();
+ albums.remove();
+ empty_first.remove();
+ empty_second.remove();
+ let albums = listing(
+ &mut server,
+ json!([
+ revision(collection(1, "First", &key), 40),
+ revision(collection(2, "Second", &key), 40)
+ ]),
+ );
+ let (old, old_bytes) = original_fixture(&key, b"old", false, true, 30);
+ let cursor = if owned {
+ for album in [1, 2] {
+ page(&mut server, album, 0, json!([old]), false).create();
+ }
+ let fetched = download(&mut server, 10, &old_bytes, 1);
+ run(&home, root, &[], false);
+ fetched.assert();
+ fetched.remove();
+ 30
+ } else {
+ 0
+ };
+ db.execute_batch(
+ "CREATE TRIGGER stop_ack BEFORE UPDATE ON pending WHEN old.album=2 AND json_type(old.action,'$.Publish.output.Media') IS NOT NULL AND new.action='null' BEGIN SELECT RAISE(ABORT,'stop after cross-album rename'); END;",
+ )
+ .unwrap();
+ albums.remove();
+ let albums = listing(
+ &mut server,
+ json!([
+ revision(collection(1, "First", &key), 60),
+ revision(collection(2, "Second", &key), 60)
+ ]),
+ );
+ let (file, bytes) = original_fixture(&key, b"new", false, true, 50);
+ for album in [1, 2] {
+ page(&mut server, album, cursor, json!([file]), false).create();
+ }
+ let fetched = download(&mut server, 10, &bytes, downloads);
+ run_failure(&home, root, &[], "stop after cross-album rename");
+
+ assert_eq!(fs::read(root.join("Second/Photo.jpg")).unwrap(), b"new");
+ db.execute_batch("DROP TRIGGER stop_ack;").unwrap();
+ CrossAlbumPublication {
+ server,
+ home,
+ key,
+ root: destination,
+ db,
+ albums,
+ fetched,
+ }
+}
+
+#[test]
+fn export_recovers_cross_album_publication_after_album_rename() {
+ let mut fixture = interrupted_cross_album_publication(false, 1);
+ let root = fixture.root.path();
+ fixture.albums.remove();
+ listing(
+ &mut fixture.server,
+ json!([revision(collection(2, "Renamed", &fixture.key), 80)]),
+ );
+ page(&mut fixture.server, 2, 50, json!([]), false).create();
+ run(&fixture.home, root, &["--album", "Renamed"], true);
+ assert_component(root, "Renamed", "Photo.jpg", b"new");
+ assert!(!root.join("Second").exists());
+ assert_eq!(
+ fs::read(root.join("First/Photo.jpg")).unwrap(),
+ b"unrelated first"
+ );
+ fixture.fetched.assert();
+}
+
+#[test]
+fn export_repairs_missing_or_changed_cross_album_publications() {
+ for (owned, missing) in [(false, true), (false, false), (true, true), (true, false)] {
+ let mut fixture = interrupted_cross_album_publication(owned, 2);
+ let root = fixture.root.path();
+ let photo = root.join("Second/Photo.jpg");
+ if missing {
+ fs::remove_file(&photo).unwrap();
+ } else {
+ fs::write(&photo, b"wrong").unwrap();
+ }
+ fixture.albums.remove();
+ listing(
+ &mut fixture.server,
+ json!([revision(collection(2, "Renamed", &fixture.key), 80)]),
+ );
+ page(&mut fixture.server, 2, 50, json!([]), false).create();
+ if !missing && !owned {
+ let result = run(&fixture.home, root, &["--album", "Renamed"], false);
+ assert!(result["conflicts"].as_i64().unwrap() > 0);
+ assert_eq!(fs::read(root.join("Renamed/Photo.jpg")).unwrap(), b"wrong");
+ fs::remove_file(root.join("Renamed/Photo.jpg")).unwrap();
+ }
+ run(&fixture.home, root, &["--album", "Renamed"], true);
+ assert_component(root, "Renamed", "Photo.jpg", b"new");
+ assert!(!root.join("Second").exists());
+ assert_eq!(
+ fs::read(root.join("First/Photo.jpg")).unwrap(),
+ b"unrelated first"
+ );
+ fixture.fetched.assert();
+ }
+}
+
+#[test]
+fn export_preserves_an_excluded_temporary_after_its_source_album_is_deleted() {
+ let mut fixture = interrupted_cross_album_publication(false, 2);
+ let root = fixture.root.path();
+ let pending: String = fixture
+ .db
+ .query_row(
+ "SELECT action FROM pending WHERE album=2 AND file=10",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ let action: Value = serde_json::from_str(&pending).unwrap();
+ assert_eq!(action["Publish"]["temporary"]["folder"], "active:1");
+ assert_eq!(
+ action["Publish"]["destination"],
+ json!({"folder":"active:2","name":"Photo.jpg"})
+ );
+ let temporary = root
+ .join("First")
+ .join(action["Publish"]["temporary"]["name"].as_str().unwrap());
+ assert!(!temporary.exists());
+ fs::rename(root.join("Second/Photo.jpg"), &temporary).unwrap();
+ fixture.albums.remove();
+ let mut deleted = revision(collection(1, "First", &fixture.key), 70);
+ deleted["isDeleted"] = json!(true);
+ let deleted = listing(&mut fixture.server, json!([deleted]));
+ run(&fixture.home, root, &["--album", "First"], true);
+ assert!(!root.join("First/metadata.json").exists());
+ assert_eq!(fs::read(&temporary).unwrap(), b"new");
+ let unchanged: String = fixture
+ .db
+ .query_row(
+ "SELECT action FROM pending WHERE album=2 AND file=10",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ assert_eq!(unchanged, pending);
+ deleted.remove();
+ listing(
+ &mut fixture.server,
+ json!([revision(collection(2, "Renamed", &fixture.key), 80)]),
+ );
+ page(&mut fixture.server, 2, 50, json!([]), false).create();
+ run(&fixture.home, root, &["--album", "Renamed"], true);
+ assert_component(root, "Renamed", "Photo.jpg", b"new");
+ assert!(!temporary.exists());
+ run(&fixture.home, root, &["--album", "First"], true);
+ assert_eq!(
+ count(
+ &fixture.db,
+ "SELECT count(*) FROM albums WHERE key='active:1'"
+ ),
+ 0
+ );
+ assert_eq!(
+ fs::read(root.join("First/Photo.jpg")).unwrap(),
+ b"unrelated first"
+ );
+ fixture.fetched.assert();
+}
+
+#[test]
+fn export_resumed_retention_preserves_original_hashes_after_local_edits() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(&mut server, json!([collection(1, "First", &key)]));
+ let (file, bytes) = original_fixture(&key, b"original", false, false, 10);
+ page(&mut server, 1, 0, json!([file]), false).create();
+ let fetched = download(&mut server, 10, &bytes, 2);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let db = export_database(&home);
+ let original_hash =
+ read_json(&root.join("First/metadata/Photo.jpg.json"))["ente"]["components"][0]["hash"]
+ .clone();
+ fs::remove_file(root.join("First/metadata/Photo.jpg.json")).unwrap();
+ db.execute_batch(
+ "CREATE TRIGGER stop_sidecar BEFORE UPDATE ON pending WHEN json_extract(old.action,'$.Remove.location.name')='metadata/Photo.jpg.json' AND new.action='null' BEGIN SELECT RAISE(ABORT,'stop retention acknowledgement'); END;",
+ )
+ .unwrap();
+ albums.remove();
+ let albums = listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), 40)]),
+ );
+ page(
+ &mut server,
+ 1,
+ 10,
+ json!([removed(file.clone(), 30)]),
+ false,
+ )
+ .create();
+ run_failure(&home, &root, &[], "stop retention acknowledgement");
+ assert_eq!(
+ count(
+ &db,
+ "SELECT count(*) FROM json_records WHERE role='original' AND name='metadata/Photo.jpg.json'"
+ ),
+ 1
+ );
+ assert!(!root.join("Trash/First/metadata/Photo.jpg.json").exists());
+ fs::write(root.join("Trash/First/Photo.jpg"), b"local edit").unwrap();
+ db.execute_batch("DROP TRIGGER stop_sidecar;").unwrap();
+ run(&home, &root, &[], true);
+ assert_eq!(
+ fs::read(root.join("Trash/First/Photo.jpg")).unwrap(),
+ b"local edit"
+ );
+ assert_eq!(
+ read_json(&root.join("Trash/First/metadata/Photo.jpg.json"))["ente"]["components"][0]["hash"],
+ original_hash
+ );
+ albums.remove();
+ listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), 60)]),
+ );
+ page(&mut server, 1, 30, json!([revision(file, 50)]), false).create();
+ run(&home, &root, &[], true);
+ assert_component(&root, "First", "Photo.jpg", b"original");
+ assert_eq!(
+ fs::read(root.join("Trash/First/Photo.jpg")).unwrap(),
+ b"local edit"
+ );
+ fetched.assert();
+}
+
+#[test]
+#[cfg(unix)]
+fn export_reports_selected_cleanup_failures_without_changing_completed_counts() {
+ use std::os::unix::fs::PermissionsExt;
+ for selected_owner in [false, true] {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ listing(
+ &mut server,
+ json!([collection(1, "First", &key), collection(2, "Second", &key)]),
+ );
+ let (file, bytes) = original_fixture(&key, b"original", false, true, 10);
+ for album in [1, 2] {
+ page(&mut server, album, 0, json!([file]), false).create();
+ }
+ let fetched = download(&mut server, 10, &bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let db = export_database(&home);
+ fs::write(root.join("First/.ente-cleanup.part"), b"disposable").unwrap();
+ fs::write(root.join("Second/metadata/Photo.jpg.json"), b"changed").unwrap();
+ db.execute_batch(
+ "CREATE TRIGGER own_cleanup AFTER UPDATE ON json_records WHEN new.folder='active:2' AND new.role='original' BEGIN INSERT INTO temporaries VALUES('.ente-cleanup.part','active:1',1,10); END;",
+ )
+ .unwrap();
+ let options: &[&str] = if selected_owner {
+ &[]
+ } else {
+ &["--album", "Second"]
+ };
+ let mut args = vec!["photos", "export", root.to_str().unwrap(), "--json"];
+ args.extend_from_slice(options);
+ fs::set_permissions(root.join("First"), fs::Permissions::from_mode(0o500)).unwrap();
+ let output = home.run(&args);
+ fs::set_permissions(root.join("First"), fs::Permissions::from_mode(0o700)).unwrap();
+ assert_eq!(
+ output.status.success(),
+ !selected_owner,
+ "{}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let result: Value = serde_json::from_slice(&output.stdout).unwrap();
+ assert_eq!(result["complete"], !selected_owner);
+ assert_eq!(result["copies"]["pending"], 0);
+ assert_eq!(
+ result["copies"]["completed"],
+ if selected_owner { 2 } else { 1 }
+ );
+ assert_eq!(result["failures"].as_i64().unwrap() > 0, selected_owner);
+ assert_eq!(count(&db, "SELECT count(*) FROM temporaries"), 1);
+ assert_eq!(
+ fs::read(root.join("First/.ente-cleanup.part")).unwrap(),
+ b"disposable"
+ );
+ assert_component(&root, "Second", "Photo.jpg", b"original");
+ db.execute_batch("DROP TRIGGER own_cleanup;").unwrap();
+ run(&home, &root, &[], true);
+ assert_eq!(count(&db, "SELECT count(*) FROM temporaries"), 0);
+ fetched.assert();
+ }
+}
+
+#[test]
+fn export_leaves_unselected_pending_and_temporary_rows_untouched() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = listing(
+ &mut server,
+ json!([collection(1, "First", &key), collection(2, "Second", &key)]),
+ );
+ let empty = page(&mut server, 1, 0, json!([]), false).create();
+ page(&mut server, 2, 0, json!([]), false).create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let db = export_database(&home);
+ db.execute_batch(
+ "CREATE TRIGGER stop_ack BEFORE UPDATE ON pending WHEN old.file=10 AND json_type(old.action,'$.Publish.output.Media') IS NOT NULL AND new.action='null' BEGIN SELECT RAISE(ABORT,'publication acknowledgement'); END;",
+ )
+ .unwrap();
+ empty.remove();
+ albums.remove();
+ let albums = listing(
+ &mut server,
+ json!([revision(collection(1, "First", &key), 40)]),
+ );
+ let (file, bytes) = original_fixture(&key, b"original", false, true, 30);
+ page(&mut server, 1, 0, json!([file]), false).create();
+ let fetched = download(&mut server, 10, &bytes, 1);
+ run_failure(&home, &root, &[], "publication acknowledgement");
+ db.execute_batch("DROP TRIGGER stop_ack;").unwrap();
+ let pending: String = db
+ .query_row("SELECT action FROM pending WHERE file=10", [], |r| r.get(0))
+ .unwrap();
+ for index in 0..3 {
+ let name = format!(".ente-disposable-{index}.part");
+ fs::write(root.join("First").join(&name), b"unselected").unwrap();
+ db.execute(
+ "INSERT INTO temporaries VALUES(?1,'active:1',1,?2)",
+ rusqlite::params![name, 100 + index],
+ )
+ .unwrap();
+ }
+ albums.remove();
+ listing(
+ &mut server,
+ json!([revision(collection(2, "Renamed", &key), 60)]),
+ );
+ let (other, other_bytes) = source(20, &key, b"independent", "Other.jpg");
+ page(&mut server, 2, 0, json!([revision(other, 50)]), false).create();
+ let other_get = download(&mut server, 20, &other_bytes, 1);
+ let result = run(&home, &root, &["--album", "Renamed", "-j", "1"], true);
+ assert_eq!(result["failures"], 0);
+ assert_eq!(count(&db, "SELECT count(*) FROM temporaries"), 4);
+ assert_eq!(
+ db.query_row("SELECT action FROM pending WHERE file=10", [], |r| r
+ .get::<_, String>(0))
+ .unwrap(),
+ pending
+ );
+ for index in 0..3 {
+ assert_eq!(
+ fs::read(root.join(format!("First/.ente-disposable-{index}.part"))).unwrap(),
+ b"unselected"
+ );
+ }
+ run(&home, &root, &["--album", "First"], true);
+ assert_component(&root, "First", "Photo.jpg", b"original");
+ assert_eq!(count(&db, "SELECT count(*) FROM temporaries"), 0);
+ fetched.assert();
+ other_get.assert();
+}
+
+#[test]
+fn export_automatically_retries_saved_selected_album_and_file_failures() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let good_album = collection(1, "First", &key);
+ let mut first = good_album.clone();
+ first["encryptedKey"] = json!("invalid");
+ let mut second = collection(2, "Second", &key);
+ second["encryptedKey"] = json!("invalid");
+ let albums = listing(
+ &mut server,
+ json!([first, second, collection(3, "Third", &key)]),
+ );
+ let (good_file, bytes) = original_fixture(&key, b"original", false, true, 10);
+ let mut broken = good_file.clone();
+ broken["metadata"]["encryptedData"] = json!("invalid");
+ let first_page = page(&mut server, 3, 0, json!([broken]), false)
+ .expect(1)
+ .create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], false);
+ let db = database(&home);
+ db.execute(
+ "UPDATE photos_collections SET record=json_set(record,'$.failure','saved failure') WHERE id IN (1,2)",
+ [],
+ )
+ .unwrap();
+ let mut saved: Value = serde_json::from_str(
+ &db.query_row(
+ "SELECT record FROM photos_collections WHERE id=1",
+ [],
+ |r| r.get::<_, String>(0),
+ )
+ .unwrap(),
+ )
+ .unwrap();
+ saved["remote"] = good_album;
+ db.execute(
+ "UPDATE photos_collections SET record=?1 WHERE id=1",
+ [saved.to_string()],
+ )
+ .unwrap();
+ let mut saved = record(&db, 3, 10);
+ saved["remote"] = good_file;
+ db.execute(
+ "UPDATE photos_files SET record=?1 WHERE collection_id=3 AND id=10",
+ [saved.to_string()],
+ )
+ .unwrap();
+ albums.remove();
+ listing(&mut server, json!([]));
+ page(&mut server, 1, 0, json!([]), false).create();
+ let fetched = download(&mut server, 10, &bytes, 1);
+ run(&home, &root, &["--album", "1", "--album", "3"], true);
+ assert_component(&root, "Third", "Photo.jpg", b"original");
+ assert!(record(&db, 3, 10)["documents"].is_object());
+ assert_eq!(
+ db.query_row(
+ "SELECT json_extract(record,'$.failure') FROM photos_collections WHERE id=2",
+ [],
+ |r| r.get::<_, String>(0)
+ )
+ .unwrap(),
+ "saved failure"
+ );
+ db.execute(
+ "INSERT INTO photos_files SELECT 1,id,name,updated_at,record,decrypt_failed FROM photos_files WHERE collection_id=3 AND id=10",
+ [],
+ )
+ .unwrap();
+ db.execute(
+ "UPDATE photos_files SET record=json_set(record,'$.documents',NULL,'$.key',NULL,'$.failure','saved failure'),decrypt_failed=1 WHERE id=10",
+ [],
+ )
+ .unwrap();
+ run(&home, &root, &["--album", "1", "--album", "3"], true);
+ for album in [1, 3] {
+ assert!(record(&db, album, 10)["documents"].is_object());
+ }
+ assert_component(&root, "First", "Photo.jpg", b"original");
+ assert!(
+ !home
+ .run(&["photos", "export", root.to_str().unwrap(), "--retry-failed"])
+ .status
+ .success()
+ );
+ first_page.assert();
+ fetched.assert();
+}
+
+#[test]
+fn export_recovers_empty_initial_directories_without_claiming_unrelated_contents() {
+ for scenario in ["active", "retained", "removed"] {
+ let retained = scenario == "retained";
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let album = collection(1, "First", &key);
+ let initial = listing(
+ &mut server,
+ if retained { json!([album]) } else { json!([]) },
+ );
+ let (file, bytes) = source(10, &key, b"original", "Photo.jpg");
+ if retained {
+ page(&mut server, 1, 0, json!([file]), false).create();
+ }
+ let fetched = download(&mut server, 10, &bytes, usize::from(retained));
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let db = export_database(&home);
+ if scenario == "removed" {
+ db.execute_batch(
+ "CREATE TRIGGER stop_directory BEFORE INSERT ON temporaries WHEN new.file IS NULL BEGIN SELECT RAISE(ABORT,'directory acknowledged before metadata'); END;",
+ )
+ .unwrap();
+ } else {
+ db.execute_batch(&format!(
+ "CREATE TRIGGER stop_directory BEFORE INSERT ON albums WHEN new.retained={} BEGIN SELECT RAISE(ABORT,'directory before acknowledgement'); END;",
+ i64::from(retained),
+ ))
+ .unwrap();
+ }
+ initial.remove();
+ let current = listing(&mut server, json!([revision(album.clone(), 40)]));
+ if retained {
+ page(&mut server, 1, 10, json!([removed(file, 30)]), false).create();
+ } else {
+ page(&mut server, 1, 0, json!([]), false).create();
+ }
+ run_failure(
+ &home,
+ &root,
+ &[],
+ if scenario == "removed" {
+ "directory acknowledged before metadata"
+ } else {
+ "directory before acknowledgement"
+ },
+ );
+ db.execute_batch("DROP TRIGGER stop_directory;").unwrap();
+ let folder = root.join(if retained { "Trash/First" } else { "First" });
+ assert_eq!(fs::read_dir(&folder).unwrap().count(), 0);
+ if scenario == "removed" {
+ assert_eq!(
+ count(&db, "SELECT count(*) FROM albums WHERE id=1 AND retained=0"),
+ 1
+ );
+ let cached: String = db
+ .query_row(
+ "SELECT value FROM json_records WHERE owner='active:1' AND folder IS NULL",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ let cached: Value = serde_json::from_str(&cached).unwrap();
+ assert_eq!(cached["ente"]["albumID"], "1");
+ assert_eq!(cached["title"], "First");
+ current.remove();
+ let mut deleted = revision(album, 60);
+ deleted["isDeleted"] = json!(true);
+ listing(&mut server, json!([deleted]));
+ run(&home, &root, &[], true);
+ assert!(!folder.exists());
+ let history = root.join("Trash/First/metadata.json");
+ assert_eq!(read_json(&history), cached);
+ assert_eq!(fs::read_dir(history.parent().unwrap()).unwrap().count(), 1);
+ let bytes = fs::read(&history).unwrap();
+ let rerun = run(&home, &root, &[], true);
+ assert_eq!(
+ rerun["changes"],
+ json!({"exported":0,"metadataUpdated":0,"renamed":0,"retained":0})
+ );
+ assert_eq!(fs::read(&history).unwrap(), bytes);
+ fetched.assert();
+ continue;
+ }
+ let notes = folder.join("notes.txt");
+ fs::write(&notes, b"unrelated").unwrap();
+ let result = run(&home, &root, &[], false);
+ assert!(result["conflicts"].as_i64().unwrap() > 0);
+ assert_eq!(fs::read(&notes).unwrap(), b"unrelated");
+ assert!(!folder.join("metadata.json").exists());
+ assert_eq!(
+ db.query_row(
+ "SELECT count(*) FROM albums WHERE id=1 AND retained=?1",
+ [retained],
+ |r| r.get::<_, i64>(0)
+ )
+ .unwrap(),
+ 0
+ );
+ assert_eq!(
+ count(
+ &db,
+ "SELECT count(*) FROM pending WHERE json_type(action,'$.Directory') IS NOT NULL"
+ ),
+ 1
+ );
+ fs::remove_file(notes).unwrap();
+ run(&home, &root, &[], true);
+ assert_eq!(
+ read_json(&folder.join("metadata.json"))["ente"]["albumID"],
+ "1"
+ );
+ if retained {
+ assert_component(&root, "Trash/First", "Photo.jpg", b"original");
+ }
+ fetched.assert();
+ }
+}
+
+#[test]
+fn export_reports_completion_progress_before_the_command_finishes() {
+ use std::sync::{Mutex, mpsc};
+ use std::time::{Duration, Instant};
+
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ listing(&mut server, json!([collection(1, "Album", &key)]));
+ let (first, first_bytes) = source(10, &key, b"first", "First.jpg");
+ let (second, second_bytes) = source(11, &key, b"second", "Second.jpg");
+ page(&mut server, 1, 0, json!([first, second]), false).create();
+ let fetched = download(&mut server, 10, &first_bytes, 1);
+ let url = server
+ .mock("GET", "/files/download/v3/11")
+ .with_body(json!({"url":format!("{}/slow",server.url())}).to_string())
+ .expect(1)
+ .create();
+ let (started, reading) = mpsc::channel();
+ let (release, receiver) = mpsc::channel::<()>();
+ let receiver = Mutex::new(receiver);
+ let slow = server
+ .mock("GET", "/slow")
+ .with_body_from_request(move |_| {
+ let _ = started.send(());
+ let _ = receiver.lock().unwrap().recv();
+ second_bytes.clone()
+ })
+ .expect(1)
+ .create();
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ let diagnostics = destination.path().join("stderr");
+ let mut child = ExportChild::spawn(
+ home.command(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "-j",
+ "1",
+ "--json",
+ ])
+ .stderr(fs::File::create(&diagnostics).unwrap()),
+ )
+ .unwrap();
+ reading
+ .recv_timeout(Duration::from_secs(60))
+ .expect("second download never started");
+ let deadline = Instant::now() + Duration::from_secs(60);
+ while !fs::read_to_string(&diagnostics)
+ .unwrap()
+ .contains("Completed 1/2 selected copies.")
+ {
+ assert!(
+ child.try_wait().unwrap().is_none(),
+ "export exited before reporting progress"
+ );
+ assert!(
+ Instant::now() < deadline,
+ "progress was not reported while the response was held"
+ );
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ assert!(child.try_wait().unwrap().is_none());
+ drop(release);
+ let output = child.wait_with_output().unwrap();
+ assert!(output.status.success());
+ assert_eq!(
+ serde_json::from_slice::<Value>(&output.stdout).unwrap()["copies"]["completed"],
+ 2
+ );
+ fetched.assert();
+ url.assert();
+ slow.assert();
+}
diff --git a/rust/apps/cli-next/tests/support/export_process.rs b/rust/apps/cli-next/tests/support/export_process.rs
new file mode 100644
index 0000000000..87d8339907
--- /dev/null
+++ b/rust/apps/cli-next/tests/support/export_process.rs
@@ -0,0 +1,51 @@
+use std::{
+ io,
+ ops::{Deref, DerefMut},
+ process::{Child, Command, Output},
+ time::{Duration, Instant},
+};
+
+pub(crate) struct ExportChild(Option<Child>);
+
+impl ExportChild {
+ pub fn spawn(command: &mut Command) -> io::Result<Self> {
+ Ok(Self(Some(command.spawn()?)))
+ }
+
+ pub fn wait_with_output(mut self) -> io::Result<Output> {
+ let deadline = Instant::now() + Duration::from_secs(60);
+ while self.try_wait()?.is_none() {
+ if Instant::now() >= deadline {
+ return Err(io::Error::new(
+ io::ErrorKind::TimedOut,
+ "export child did not exit",
+ ));
+ }
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ self.0.take().unwrap().wait_with_output()
+ }
+}
+
+impl Deref for ExportChild {
+ type Target = Child;
+
+ fn deref(&self) -> &Child {
+ self.0.as_ref().unwrap()
+ }
+}
+
+impl DerefMut for ExportChild {
+ fn deref_mut(&mut self) -> &mut Child {
+ self.0.as_mut().unwrap()
+ }
+}
+
+impl Drop for ExportChild {
+ fn drop(&mut self) {
+ if let Some(child) = &mut self.0 {
+ let _ = child.kill();
+ let _ = child.wait();
+ }
+ }
+}
diff --git a/rust/apps/cli-next/tests/support/export_resources.rs b/rust/apps/cli-next/tests/support/export_resources.rs
index 605968a20e..e666645f3d 100644
--- a/rust/apps/cli-next/tests/support/export_resources.rs
+++ b/rust/apps/cli-next/tests/support/export_resources.rs
@@ -224,9 +224,12 @@ fn repeat_hash(byte: u8, size: u64) -> String {
fn measure(home: &TestHome, root: &Path, label: &str) -> TestResult<Value> {
let started = Instant::now();
- let mut child = home
- .command(&["photos", "export", root.to_str().unwrap(), "--json"])
- .spawn()?;
+ let mut child = ExportChild::spawn(&mut home.command(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "--json",
+ ]))?;
let output_readers = [
child
.stdout
diff --git a/rust/apps/cli-next/tests/support/export_sources.rs b/rust/apps/cli-next/tests/support/export_sources.rs
index d39a5246a7..275d62e0ea 100644
--- a/rust/apps/cli-next/tests/support/export_sources.rs
+++ b/rust/apps/cli-next/tests/support/export_sources.rs
@@ -3,6 +3,31 @@ use super::*;
use ente_core::crypto::{Nonce, hash, secretbox};
use std::path::Path;
+#[path = "export_maintenance.rs"]
+mod maintenance;
+
+#[test]
+fn export_rejects_zero_jobs_before_creating_storage() {
+ let home = TestHome::new();
+ let unused = home.dir.path().join("unused");
+ let destination = home.dir.path().join("photos");
+ let output = home
+ .command(&[
+ "photos",
+ "export",
+ destination.to_str().unwrap(),
+ "--jobs",
+ "0",
+ ])
+ .env("ENTE_CLI_HOME", &unused)
+ .output()
+ .unwrap();
+ assert!(failure(&output).contains("invalid value '0'"));
+ assert!(output.stdout.is_empty());
+ assert!(!unused.exists());
+ assert!(!destination.exists());
+}
+
#[test]
fn export_rejects_wrong_accounts_and_unsupported_roots_before_refresh() {
let mut server = mockito::Server::new();
@@ -105,32 +130,18 @@ fn export_preserves_failed_records_and_advances_the_page_atomically() {
15
);
run(&home, &root, &[], false);
- assert_eq!(record(&db, 1, 11)["attempt"], failed["attempt"]);
- run(&home, &root, &["--album", "First", "--retry-failed"], false);
- assert_ne!(record(&db, 1, 11)["attempt"], failed["attempt"]);
- assert_eq!(record(&db, 2, 11)["attempt"], failed["attempt"]);
- first.assert();
- last.assert();
-
- let before = record(&db, 1, 10);
- let mut docs: Value = serde_json::from_slice(
- &serde_json::from_value::<Vec<u8>>(before["documents"]["public"].clone()).unwrap(),
- )
- .unwrap();
- docs["caption"] = json!("newly supported from saved metadata");
- let mut updated = before;
- updated["documents"]["public"] = json!(serde_json::to_vec(&docs).unwrap());
db.execute(
- "UPDATE photos_files SET record=?1 WHERE collection_id=1 AND id=10",
- [updated.to_string()],
+ "UPDATE photos_files SET record=json_set(record,'$.failure','saved failure') WHERE id=11",
+ [],
)
.unwrap();
run(&home, &root, &["--album", "First"], false);
- assert_eq!(
- read_json(&root.join("First/metadata/Good.jpg.json"))["description"],
- "newly supported from saved metadata"
- );
- let previous_attempt = record(&db, 1, 11)["attempt"].clone();
+ assert_ne!(record(&db, 1, 11)["failure"], "saved failure");
+ assert_eq!(record(&db, 2, 11)["failure"], "saved failure");
+ first.assert();
+ last.assert();
+
+ let previous_input = record(&db, 1, 11)["input_hash"].clone();
albums.remove();
let mut changed = collection(1, "First", &key);
changed["updationTime"] = json!(40);
@@ -143,7 +154,8 @@ fn export_preserves_failed_records_and_advances_the_page_atomically() {
page(&mut server, 1, 15, json!([repaired]), false).create();
let repaired_download = download(&mut server, 11, &broken_bytes, 1);
run(&home, &root, &["--album", "First"], true);
- assert_ne!(record(&db, 1, 11)["attempt"], previous_attempt);
+ assert_ne!(record(&db, 1, 11)["input_hash"], previous_input);
+ assert!(record(&db, 1, 11)["documents"].is_object());
assert_eq!(fs::read(root.join("First/Broken.jpg")).unwrap(), b"broken");
good_download.assert();
later_download.assert();
@@ -174,11 +186,55 @@ fn export_preserves_failed_records_and_advances_the_page_atomically() {
);
assert!(!root.join("Second/Broken.jpg").exists());
assert_eq!(
- run(&home, &root, &["--album", "Second", "--retry-failed"], true)["failures"],
+ run(&home, &root, &["--album", "Second"], true)["failures"],
0
);
}
+#[test]
+fn export_reevaluates_saved_metadata_without_an_upstream_change() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections":[collection(1,"First",&key)]}).to_string())
+ .create();
+ let (file, bytes) = source(10, &key, b"original", "Good.jpg");
+ let page = page(&mut server, 1, 0, json!([file]), false)
+ .expect(1)
+ .create();
+ let downloaded = download(&mut server, 10, &bytes, 1);
+ let destination = tempfile::tempdir().unwrap();
+ let root = destination.path().join("photos");
+ run(&home, &root, &[], true);
+ let db = database(&home);
+ let before = record(&db, 1, 10);
+ let mut docs: Value = serde_json::from_slice(
+ &ente_core::b64::decode(before["documents"]["public"].as_str().unwrap()).unwrap(),
+ )
+ .unwrap();
+ docs["caption"] = json!("reevaluated stored caption");
+ let mut updated = before;
+ updated["documents"]["public"] =
+ json!(ente_core::b64::encode(&serde_json::to_vec(&docs).unwrap()));
+ db.execute(
+ "UPDATE photos_files SET record=?1 WHERE collection_id=1 AND id=10",
+ [updated.to_string()],
+ )
+ .unwrap();
+ run(&home, &root, &["--album", "First"], true);
+ assert_eq!(
+ read_json(&root.join("First/metadata/Good.jpg.json"))["description"],
+ "reevaluated stored caption"
+ );
+
+ page.assert();
+ downloaded.assert();
+}
+
#[test]
fn export_retries_saved_file_decryption_when_the_album_key_changes() {
let mut server = mockito::Server::new();
@@ -214,7 +270,8 @@ fn export_retries_saved_file_decryption_when_the_album_key_changes() {
.create();
let downloaded = download(&mut server, 10, &bytes, 1);
run(&home, &root, &[], true);
- assert_ne!(record(&db, 1, 10)["attempt"], failed["attempt"]);
+ assert_ne!(record(&db, 1, 10)["input_hash"], failed["input_hash"]);
+ assert!(record(&db, 1, 10)["documents"].is_object());
assert_eq!(
fs::read(root.join("First/Original.jpg")).unwrap(),
b"original"
@@ -314,22 +371,14 @@ fn export_releases_a_deleted_album_reservation_before_its_first_association() {
let db = export_database(&home);
let reservation = || {
db.query_row(
- "SELECT record FROM allocations WHERE owner='active:1'",
+ "SELECT name1 FROM pending WHERE owner='active:1'",
[],
|r| r.get::<_, String>(0),
)
.unwrap()
};
let pending = reservation();
- assert_eq!(
- serde_json::from_str::<Value>(&pending).unwrap()["paths"],
- json!(["Family"])
- );
- assert_eq!(
- db.query_row("SELECT count(*) FROM albums", [], |r| r.get::<_, i64>(0))
- .unwrap(),
- 0
- );
+ assert_eq!(pending, "Family");
assert_eq!(
run(&home, &root, &["--album", "Family"], false)["conflicts"],
1
@@ -363,17 +412,6 @@ fn export_releases_a_deleted_album_reservation_before_its_first_association() {
assert_eq!(reservation(), pending);
}
run(&home, &root, &["--album", "1"], true);
- assert_eq!(
- db.query_row("SELECT count(*) FROM allocations", [], |r| r
- .get::<_, i64>(0))
- .unwrap(),
- 0
- );
- assert_eq!(
- db.query_row("SELECT count(*) FROM albums", [], |r| r.get::<_, i64>(0))
- .unwrap(),
- 0
- );
assert_eq!(fs::read(&unrelated).unwrap(), b"unrelated");
assert!(!root.join("Family/metadata.json").exists());
assert!(!root.join("Trash").exists());
@@ -425,7 +463,6 @@ fn export_clears_deleted_adopted_album_rename_reservations() {
|r| r.get(0),
)
.unwrap();
- uuid::Uuid::parse_str(&owner).unwrap();
fs::create_dir(root.join("Blocked")).unwrap();
let unrelated = root.join("Blocked/notes.txt");
fs::write(&unrelated, b"unrelated rename target").unwrap();
@@ -439,18 +476,13 @@ fn export_clears_deleted_adopted_album_rename_reservations() {
.create();
assert_eq!(run(&fresh, &root, &[], false)["conflicts"], 2);
let reservation = || {
- db.query_row(
- "SELECT record FROM allocations WHERE owner=?1",
- [&owner],
- |r| r.get::<_, String>(0),
- )
+ db.query_row("SELECT name1 FROM pending WHERE owner=?1", [&owner], |r| {
+ r.get::<_, String>(0)
+ })
.unwrap()
};
let pending = reservation();
- assert_eq!(
- serde_json::from_str::<Value>(&pending).unwrap()["paths"],
- json!(["Blocked"])
- );
+ assert_eq!(pending, "Blocked");
assert_eq!(
fs::read(root.join("First/Original.jpg")).unwrap(),
b"original"
@@ -470,28 +502,17 @@ fn export_clears_deleted_adopted_album_rename_reservations() {
let retained_blocker = root.join("Trash/First/notes.txt");
fs::write(&retained_blocker, b"unrelated retention target").unwrap();
run(&fresh, &root, &["--album", "1"], false);
- assert_eq!(
- db.query_row(
- "SELECT count(*) FROM allocations WHERE owner=?1",
- [&owner],
- |r| r.get::<_, i64>(0)
- )
- .unwrap(),
- 0
- );
let retained = || {
db.query_row(
- "SELECT record FROM allocations WHERE owner='retained:1'",
+ "SELECT name1 FROM pending WHERE owner='retained:1'",
[],
|r| r.get::<_, String>(0),
)
.unwrap()
};
let pending_retention = retained();
- assert_eq!(
- serde_json::from_str::<Value>(&pending_retention).unwrap()["paths"],
- json!(["First"])
- );
+ assert!(root.join("First/metadata.json").exists());
+ assert_eq!(pending_retention, "First");
run(&fresh, &root, &["--album", "1"], false);
assert_eq!(retained(), pending_retention);
assert!(!root.join("Trash/First-1").exists());
@@ -515,12 +536,6 @@ fn export_clears_deleted_adopted_album_rename_reservations() {
read_json(&root.join("Trash/First/metadata/Original.jpg.json"))["ente"]["fileID"],
"10"
);
- assert_eq!(
- db.query_row("SELECT count(*) FROM allocations", [], |r| r
- .get::<_, i64>(0))
- .unwrap(),
- 0
- );
deletion.remove();
fs::remove_file(unrelated).unwrap();
fs::remove_dir(root.join("Blocked")).unwrap();
@@ -568,7 +583,7 @@ fn export_rejects_offline_early_and_reports_path_conflicts() {
.unwrap();
let (good, encrypted) = source(10, &key, b"good", "Good.jpg");
page(&mut server, 1, 0, json!([good]), false).create();
- let downloaded = download(&mut server, 10, &encrypted, 1);
+ let downloaded = download(&mut server, 10, &encrypted, if cfg!(unix) { 6 } else { 4 });
#[cfg(unix)]
{
let target = destination.path().join("outside.jpg");
@@ -607,15 +622,15 @@ fn export_rejects_offline_early_and_reports_path_conflicts() {
);
let record: String = db
.query_row(
- "SELECT record FROM allocations WHERE owner='file:1:10'",
+ "SELECT name1 FROM pending WHERE album=1 AND file=10",
[],
|r| r.get(0),
)
.unwrap();
- serde_json::from_str::<Value>(&record).unwrap()
+ record
};
let allocated = reservation();
- assert_eq!(allocated["paths"], json!(["Good.jpg"]));
+ assert_eq!(allocated, "Good.jpg");
let unchanged = page(&mut server, 1, 10, json!([]), false).create();
assert_eq!(run(&home, &root, &[], false)["conflicts"], 1);
assert_eq!(reservation(), allocated);
@@ -655,14 +670,12 @@ fn export_rejects_offline_early_and_reports_path_conflicts() {
0
);
let export_db = export_database(&home);
- let pending: String = export_db
- .query_row(
- "SELECT path FROM temporaries WHERE json_extract(record,'$.file')=10 AND json_extract(record,'$.hash') IS NOT NULL",
- [],
- |r| r.get(0),
- )
+ let pending: i64 = export_db
+ .query_row("SELECT count(*) FROM temporaries WHERE file=10", [], |r| {
+ r.get(0)
+ })
.unwrap();
- assert_eq!(fs::read(root.join(&pending)).unwrap(), b"good");
+ assert_eq!(pending, 0);
drop(export_db);
albums.remove();
let mut renamed = collection(1, "Renamed", &key);
@@ -683,12 +696,10 @@ fn export_rejects_offline_early_and_reports_path_conflicts() {
);
assert_eq!(
export_database(&home)
- .query_row("SELECT count(*) FROM allocations", [], |r| r
- .get::<_, i64>(0))
+ .query_row("SELECT count(*) FROM pending", [], |r| r.get::<_, i64>(0))
.unwrap(),
0
);
- assert!(!root.join(pending.replace("First/", "Renamed/")).exists());
assert_eq!(
fs::read(root.join("Renamed/.ente-11111111111111111111111111111111.part")).unwrap(),
b"unrelated temporary"
@@ -697,6 +708,17 @@ fn export_rejects_offline_early_and_reports_path_conflicts() {
contender_download.assert();
}
+fn run_failure(home: &TestHome, root: &Path, options: &[&str], cause: &str) -> Value {
+ let mut args = vec!["photos", "export", root.to_str().unwrap(), "--json"];
+ args.extend_from_slice(options);
+ let output = home.run(&args);
+ let error = failure(&output);
+ assert!(error.contains(cause), "expected {cause:?}: {error}");
+ let result: Value = serde_json::from_slice(&output.stdout).unwrap();
+ assert_eq!(result["complete"], false);
+ result
+}
+
fn run(home: &TestHome, root: &Path, options: &[&str], complete: bool) -> Value {
let mut args = vec!["photos", "export", root.to_str().unwrap(), "--json"];
args.extend_from_slice(options);
@@ -833,14 +855,15 @@ fn export_continues_when_failed_transfer_cleanup_becomes_unwritable() {
encrypted.clone()
})
.create();
- let healthy = server
- .mock("GET", "/media/20")
- .with_body_from_request(move |_| {
- std::thread::sleep(std::time::Duration::from_millis(500));
- original.clone()
- })
- .create();
- let output = home.run(&["photos", "export", root.to_str().unwrap(), "--json"]);
+ let healthy = server.mock("GET", "/media/20").with_body(original).create();
+ let output = home.run(&[
+ "photos",
+ "export",
+ root.to_str().unwrap(),
+ "--json",
+ "--jobs",
+ "1",
+ ]);
fs::set_permissions(&protected, fs::Permissions::from_mode(0o700)).unwrap();
assert!(
failure(&output)
@@ -854,7 +877,13 @@ fn export_continues_when_failed_transfer_cleanup_becomes_unwritable() {
);
assert_eq!(fs::read(root.join("Healthy/Good.jpg")).unwrap(), b"good");
let db = export_database(&home);
- let partial: String = db.query_row("SELECT path FROM temporaries WHERE json_extract(record,'$.album')=1 AND json_extract(record,'$.file')=10 AND json_extract(record,'$.hash') IS NULL", [], |r| r.get(0)).unwrap();
+ let partial: String = db
+ .query_row(
+ "SELECT a.path||'/'||t.name FROM temporaries t JOIN albums a ON a.key=t.folder WHERE t.album=1 AND t.file=10",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
assert!(root.join(&partial).is_file());
denied.remove();
let retry = server
@@ -942,12 +971,15 @@ fn export_page_and_recognized_failures_roll_back_together_on_storage_failure() {
run(&home, &root, &[], true);
empty.remove();
let db = database(&home);
- db.execute_batch("UPDATE photos_collections SET files_synced_to=NULL; CREATE TRIGGER fail_second BEFORE INSERT ON photos_files WHEN new.id=11 BEGIN SELECT RAISE(ABORT,'fixture storage failure'); END;").unwrap();
+ db.execute_batch(
+ "UPDATE photos_collections SET files_synced_to=NULL; CREATE TRIGGER fail_second BEFORE INSERT ON photos_files WHEN new.id=11 BEGIN SELECT RAISE(ABORT,'fixture storage failure'); END;",
+ )
+ .unwrap();
let (good, encrypted) = source(10, &key, b"good", "Good.jpg");
let (mut broken, _) = source(11, &key, b"broken", "Broken.jpg");
broken["metadata"]["encryptedData"] = json!("broken");
page(&mut server, 1, 0, json!([good, broken]), false).create();
- let result = run(&home, &root, &[], false);
+ let result = run_failure(&home, &root, &[], "fixture storage failure");
assert!(result["files"].is_null());
assert_eq!(
db.query_row("SELECT count(*) FROM photos_files", [], |r| r
@@ -1026,15 +1058,19 @@ fn export_saves_interpretation_failures_and_omits_unknown_field_values_from_diag
let db = database(&home);
let failure = record(&db, 1, 11);
assert!(!failure["documents"].is_null());
- let original: Vec<u8> =
- serde_json::from_value(record(&db, 1, 10)["documents"]["original"].clone()).unwrap();
+ let original: Vec<u8> = ente_core::b64::decode(
+ record(&db, 1, 10)["documents"]["original"]
+ .as_str()
+ .unwrap(),
+ )
+ .unwrap();
let future_reader: Value = serde_json::from_slice(&original).unwrap();
assert_eq!(
future_reader["newAdditiveField"]["opaque"],
"kept in replica"
);
- run(&home, &root, &["--retry-failed"], false);
- assert_eq!(record(&db, 1, 11)["attempt"], failure["attempt"]);
+ run(&home, &root, &[], false);
+ assert_eq!(record(&db, 1, 11), failure);
}
#[test]
diff --git a/rust/apps/cli-next/tests/support/files.rs b/rust/apps/cli-next/tests/support/files.rs
index 31ca5298d8..2cd0f40e7d 100644
--- a/rust/apps/cli-next/tests/support/files.rs
+++ b/rust/apps/cli-next/tests/support/files.rs
@@ -528,7 +528,7 @@ fn photos_lists_limit_output_and_stream_complete_results() {
.unwrap();
let mut record: Value = serde_json::from_str(&encoded).unwrap();
let public = serde_json::to_vec(&json!({"editedTime":i64::MAX})).unwrap();
- record["documents"]["public"] = json!(public);
+ record["documents"]["public"] = json!(ente_core::b64::encode(&public));
connection
.execute(
"UPDATE photos_files SET record=?1 WHERE id=150",
diff --git a/rust/crates/collections/src/client.rs b/rust/crates/collections/src/client.rs
index b433d5bcb7..6ced8e45e8 100644
--- a/rust/crates/collections/src/client.rs
+++ b/rust/crates/collections/src/client.rs
@@ -3,7 +3,7 @@ use ente_core::{
crypto::{Header, Key, Nonce, blob, secretbox},
http,
};
-use serde::{Deserialize, Serialize, de::DeserializeOwned};
+use serde::{Deserialize, Serialize};
use crate::{Error, Result, open_collection_key};
@@ -131,13 +131,6 @@ impl Collection {
)
}
- pub fn open_name(&self, key: &Key) -> Result<String> {
- String::from_utf8(self.open_name_bytes(key)?).map_err(|_| Error::InvalidCollection {
- id: self.id,
- reason: "name is not UTF-8",
- })
- }
-
pub fn open_name_bytes(&self, key: &Key) -> Result<Vec<u8>> {
if let Some(name) = self.name.as_ref().filter(|name| !name.is_empty()) {
return Ok(name.as_bytes().to_vec());
@@ -229,14 +222,4 @@ impl EncryptedMetadata {
key,
)?)
}
-
- pub fn open<T: DeserializeOwned>(&self, key: &Key) -> Result<T> {
- Ok(blob::decrypt_json(
- &blob::EncryptedBlob {
- encrypted_data: b64::decode(&self.data)?,
- decryption_header: Header::try_from_slice(&b64::decode(&self.header)?)?,
- },
- key,
- )?)
- }
}
diff --git a/rust/crates/collections/src/lib.rs b/rust/crates/collections/src/lib.rs
index 120a99fc48..ad90e03344 100644
--- a/rust/crates/collections/src/lib.rs
+++ b/rust/crates/collections/src/lib.rs
@@ -33,11 +33,11 @@ pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug, thiserror::Error)]
pub enum Error {
- #[error("{0}")]
+ #[error(transparent)]
Http(#[from] ente_core::http::Error),
- #[error("{0}")]
+ #[error(transparent)]
Decode(#[from] b64::DecodeError),
- #[error("{0}")]
+ #[error(transparent)]
Crypto(#[from] crypto::Error),
#[error("Owned collection is missing its key decryption nonce")]
MissingKeyDecryptionNonce,
diff --git a/rust/crates/photos/src/collections.rs b/rust/crates/photos/src/collections.rs
index 85e5f34ffe..1a5f7e073f 100644
--- a/rust/crates/photos/src/collections.rs
+++ b/rust/crates/photos/src/collections.rs
@@ -1,5 +1,5 @@
use ente_core::{Session, crypto::Key};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
use crate::source::{self, Documents, MetadataError};
@@ -46,7 +46,7 @@ impl Kind {
}
}
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub enum Visibility {
Visible,
Archived,
@@ -72,57 +72,6 @@ impl Visibility {
}
}
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
- #[error(transparent)]
- Collections(#[from] ente_collections::Error),
- #[error(transparent)]
- Metadata(#[from] MetadataError),
-}
-
-pub struct Change {
- pub id: i64,
- pub collection: Option<Collection>,
-}
-
-pub struct Page {
- pub changes: Vec<Change>,
- pub cursor: i64,
-}
-
-pub async fn diff(session: &Session, since: i64) -> Result<Page, Error> {
- let page = ente_collections::client::diff(session, since).await?;
- let changes = page
- .collections
- .into_iter()
- .map(|remote| {
- let collection = if remote.is_deleted == Some(true) {
- None
- } else {
- let (key, documents) = decrypt(&remote, session)?;
- Some(interpret(&remote, &key, &documents, session.user_id)?)
- };
- Ok(Change {
- id: remote.id,
- collection,
- })
- })
- .collect::<Result<_, Error>>()?;
- Ok(Page {
- changes,
- cursor: page.cursor,
- })
-}
-
-pub async fn list(session: &Session) -> Result<Vec<Collection>, Error> {
- Ok(diff(session, 0)
- .await?
- .changes
- .into_iter()
- .filter_map(|c| c.collection)
- .collect())
-}
-
pub fn decrypt(
remote: &RemoteCollection,
session: &Session,
diff --git a/rust/crates/photos/src/export.rs b/rust/crates/photos/src/export.rs
index 3faabb1e0a..93139721a8 100644
--- a/rust/crates/photos/src/export.rs
+++ b/rust/crates/photos/src/export.rs
@@ -4,7 +4,7 @@ use ente_core::{
crypto::{Key, kdf},
};
use serde::{Deserialize, Serialize};
-use serde_json::{Map, Value, json};
+use serde_json::{Value, json};
use crate::{
collections::{Collection, Visibility},
@@ -59,7 +59,7 @@ pub struct Component {
pub fn album(album: &Collection) -> Value {
let mut metadata = json!({
"albumID": album.id.to_string(), "type": album.kind.name(), "visibility": album.visibility.name(),
- }).as_object().cloned().unwrap_or_default();
+ });
optional(&mut metadata, "displayOrder", album.display_order);
optional(&mut metadata, "muted", album.muted);
optional(
@@ -95,10 +95,7 @@ pub fn file(
"creationTime": timestamp(file.created_at_micros)?,
"component": selected.role,
"components": components,
- })
- .as_object()
- .cloned()
- .unwrap_or_default();
+ });
optional(
&mut metadata,
"modificationTime",
@@ -138,17 +135,14 @@ pub fn file(
"favorited": favorited,
"ente": metadata,
});
- let object = record
- .as_object_mut()
- .ok_or(MetadataError("invalid file record"))?;
optional(
- object,
+ &mut record,
"modificationTime",
file.modified_at_micros.map(common_time).transpose()?,
);
- optional(object, "description", file.caption.as_deref());
+ optional(&mut record, "description", file.caption.as_deref());
optional(
- object,
+ &mut record,
"geoData",
file.location
.as_ref()
@@ -157,9 +151,9 @@ pub fn file(
Ok(record)
}
-fn optional(object: &mut Map<String, Value>, name: &str, value: Option<impl Into<Value>>) {
+fn optional(object: &mut Value, name: &str, value: Option<impl Into<Value>>) {
if let Some(value) = value {
- object.insert(name.into(), value.into());
+ object[name] = value.into();
}
}
diff --git a/rust/crates/photos/src/files.rs b/rust/crates/photos/src/files.rs
index 23aa4ad0dc..696e2e493c 100644
--- a/rust/crates/photos/src/files.rs
+++ b/rust/crates/photos/src/files.rs
@@ -3,16 +3,16 @@ use ente_core::{
crypto::{self, Header, Key, blob},
http,
};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
use crate::{
- collections::{Collection, Visibility},
+ collections::Visibility,
source::{self, Documents, MetadataError},
};
pub use ente_collections::client::File as RemoteFile;
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
pub struct File {
pub id: i64,
pub owner_id: i64,
@@ -36,11 +36,13 @@ pub struct File {
pub panorama: Option<bool>,
pub motion_video_offset: Option<u64>,
pub warnings: Vec<String>,
+ #[serde(with = "source::key")]
pub key: Key,
+ #[serde(with = "source::header")]
pub header: Header,
}
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum Kind {
Image,
Video,
@@ -59,65 +61,28 @@ impl Kind {
}
}
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
pub struct Location {
pub latitude: f64,
pub longitude: f64,
}
-pub struct Change {
- pub id: i64,
- pub file: Option<File>,
-}
-
-pub struct Page {
- pub changes: Vec<Change>,
- pub cursor: i64,
- pub has_more: bool,
-}
-
#[derive(Debug, thiserror::Error)]
pub enum Error {
- #[error("{0}")]
+ #[error(transparent)]
Http(#[from] http::Error),
- #[error("{0}")]
+ #[error(transparent)]
Crypto(#[from] crypto::Error),
- #[error("{0}")]
+ #[error(transparent)]
Base64(#[from] b64::DecodeError),
- #[error("{0}")]
+ #[error(transparent)]
Io(#[from] std::io::Error),
- #[error("{0}")]
+ #[error(transparent)]
Collections(#[from] ente_collections::Error),
- #[error("{0}")]
+ #[error(transparent)]
Metadata(#[from] MetadataError),
}
-pub async fn diff(session: &Session, collection: &Collection, since: i64) -> Result<Page, Error> {
- let page = ente_collections::client::files_diff(session, collection.id, since).await?;
- let changes = page
- .files
- .into_iter()
- .map(|remote| {
- Ok(Change {
- id: remote.id,
- file: if remote.is_deleted() {
- None
- } else {
- {
- let (key, documents) = decrypt(&remote, &collection.key)?;
- Some(interpret(&remote, &key, &documents, session.user_id)?)
- }
- },
- })
- })
- .collect::<Result<_, Error>>()?;
- Ok(Page {
- changes,
- cursor: page.cursor,
- has_more: page.has_more,
- })
-}
-
#[cfg(not(target_arch = "wasm32"))]
pub async fn download<W: std::io::Write>(
session: &Session,
diff --git a/rust/crates/photos/src/live_photo.rs b/rust/crates/photos/src/live_photo.rs
index 54bc4ba2b3..b15f93a8f4 100644
--- a/rust/crates/photos/src/live_photo.rs
+++ b/rust/crates/photos/src/live_photo.rs
@@ -4,9 +4,9 @@ use crate::export::Role;
#[derive(Debug, thiserror::Error)]
pub enum Error {
- #[error("{0}")]
+ #[error(transparent)]
Zip(#[from] zip::result::ZipError),
- #[error("{0}")]
+ #[error(transparent)]
Io(#[from] std::io::Error),
#[error("Live Photo must contain one image and one video")]
Components,
diff --git a/rust/crates/photos/src/source.rs b/rust/crates/photos/src/source.rs
index 00d9188b5d..c85c00d147 100644
--- a/rust/crates/photos/src/source.rs
+++ b/rust/crates/photos/src/source.rs
@@ -3,12 +3,76 @@ use serde_json::Value;
#[derive(Clone, Serialize, Deserialize)]
pub struct Documents {
+ #[serde(with = "bytes")]
pub original: Vec<u8>,
+ #[serde(with = "optional_bytes")]
pub public: Option<Vec<u8>>,
+ #[serde(with = "optional_bytes")]
pub private: Option<Vec<u8>>,
+ #[serde(with = "optional_bytes")]
pub shared: Option<Vec<u8>>,
}
+mod bytes {
+ use serde::{Deserialize, Deserializer, Serializer, de::Error};
+
+ pub fn serialize<S: Serializer>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error> {
+ serializer.serialize_str(&ente_core::b64::encode(bytes))
+ }
+
+ pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Vec<u8>, D::Error> {
+ ente_core::b64::decode(&String::deserialize(deserializer)?).map_err(D::Error::custom)
+ }
+}
+
+mod optional_bytes {
+ use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error};
+
+ pub fn serialize<S: Serializer>(
+ bytes: &Option<Vec<u8>>,
+ serializer: S,
+ ) -> Result<S::Ok, S::Error> {
+ bytes
+ .as_deref()
+ .map(ente_core::b64::encode)
+ .serialize(serializer)
+ }
+
+ pub fn deserialize<'de, D: Deserializer<'de>>(
+ deserializer: D,
+ ) -> Result<Option<Vec<u8>>, D::Error> {
+ Option::<String>::deserialize(deserializer)?
+ .map(|text| ente_core::b64::decode(&text).map_err(D::Error::custom))
+ .transpose()
+ }
+}
+
+pub(crate) mod key {
+ use ente_core::crypto::Key;
+ use serde::{Deserializer, Serializer, de::Error};
+
+ pub fn serialize<S: Serializer>(key: &Key, serializer: S) -> Result<S::Ok, S::Error> {
+ serializer.serialize_str(&ente_core::b64::encode(key.as_bytes()))
+ }
+
+ pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Key, D::Error> {
+ Key::try_from_slice(&super::bytes::deserialize(deserializer)?).map_err(D::Error::custom)
+ }
+}
+
+pub(crate) mod header {
+ use ente_core::crypto::Header;
+ use serde::{Deserializer, Serializer, de::Error};
+
+ pub fn serialize<S: Serializer>(header: &Header, serializer: S) -> Result<S::Ok, S::Error> {
+ serializer.serialize_str(&ente_core::b64::encode(header.as_bytes()))
+ }
+
+ pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Header, D::Error> {
+ Header::try_from_slice(&super::bytes::deserialize(deserializer)?).map_err(D::Error::custom)
+ }
+}
+
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
pub struct MetadataError(pub &'static str);
diff --git a/rust/crates/test-support/src/lib.rs b/rust/crates/test-support/src/lib.rs
index 834105a2e2..1d80e88450 100644
--- a/rust/crates/test-support/src/lib.rs
+++ b/rust/crates/test-support/src/lib.rs
@@ -7,7 +7,7 @@ mod process;
mod server;
pub use museum::Museum;
-pub use object_store::ObjectStoreControl;
+pub use object_store::{HeldReads, ObjectStoreControl};
pub type TestResult<T = ()> = Result<T, Box<dyn std::error::Error>>;
diff --git a/rust/crates/test-support/src/object_store.rs b/rust/crates/test-support/src/object_store.rs
index 4034d405a5..a6f13e3849 100644
--- a/rust/crates/test-support/src/object_store.rs
+++ b/rust/crates/test-support/src/object_store.rs
@@ -4,8 +4,8 @@ use std::{
net::{TcpListener, TcpStream},
path::{Path, PathBuf},
sync::{
- Arc,
- atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
+ Arc, Condvar, Mutex, PoisonError,
+ atomic::{AtomicBool, AtomicUsize, Ordering},
},
thread::{self, JoinHandle},
time::Duration,
@@ -22,10 +22,59 @@ pub struct ObjectStoreControl {
active: AtomicUsize,
peak: AtomicUsize,
interrupted: AtomicUsize,
- delay_millis: AtomicU64,
+ held: Mutex<Option<Arc<ReadGate>>>,
+}
+
+struct ReadGate {
+ minimum_size: u64,
+ state: Mutex<(usize, bool)>,
+ changed: Condvar,
+}
+
+pub struct HeldReads(Arc<ReadGate>);
+
+impl HeldReads {
+ pub fn wait_for(&self, count: usize) {
+ let (state, _) = self
+ .0
+ .changed
+ .wait_timeout_while(
+ self.0.state.lock().unwrap_or_else(PoisonError::into_inner),
+ Duration::from_secs(60),
+ |(started, _)| *started < count,
+ )
+ .unwrap_or_else(PoisonError::into_inner);
+ let started = state.0;
+ drop(state);
+ assert!(
+ started >= count,
+ "expected {count} held downloads, got {started}"
+ );
+ }
+}
+
+impl Drop for HeldReads {
+ fn drop(&mut self) {
+ self.0
+ .state
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .1 = true;
+ self.0.changed.notify_all();
+ }
}
impl ObjectStoreControl {
+ pub fn hold_reads(&self, minimum_size: u64) -> HeldReads {
+ let gate = Arc::new(ReadGate {
+ minimum_size,
+ state: Mutex::new((0, false)),
+ changed: Condvar::new(),
+ });
+ *self.held.lock().unwrap_or_else(PoisonError::into_inner) = Some(gate.clone());
+ HeldReads(gate)
+ }
+
pub fn reads(&self) -> usize {
self.reads.load(Ordering::SeqCst)
}
@@ -35,10 +84,6 @@ impl ObjectStoreControl {
pub fn interrupt_reads(&self, count: usize) {
self.interrupted.store(count, Ordering::SeqCst);
}
- pub fn delay_chunks(&self, delay: Duration) {
- self.delay_millis
- .store(delay.as_millis() as u64, Ordering::SeqCst);
- }
}
pub struct ObjectStore {
@@ -174,16 +219,30 @@ fn handle(stream: TcpStream, root: &Path, control: &ObjectStoreControl) -> TestR
control.peak.fetch_max(active, Ordering::SeqCst);
let result = (|| {
respond(input.get_mut(), "200 OK", file.metadata()?.len())?;
+ let gate = control
+ .held
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .clone();
+ if let Some(gate) = gate
+ && file.metadata()?.len() >= gate.minimum_size
+ {
+ let mut state = gate.state.lock().unwrap_or_else(PoisonError::into_inner);
+ state.0 += 1;
+ gate.changed.notify_all();
+ while !state.1 {
+ state = gate
+ .changed
+ .wait(state)
+ .unwrap_or_else(PoisonError::into_inner);
+ }
+ }
let mut buffer = vec![0; 64 * 1024];
loop {
let read = file.read(&mut buffer)?;
if read == 0 {
break;
}
- let delay = control.delay_millis.load(Ordering::SeqCst);
- if delay > 0 {
- thread::sleep(Duration::from_millis(delay));
- }
input.get_mut().write_all(&buffer[..read])?;
}
Ok::<_, io::Error>(())