π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
mod adopt;
mod fs;
mod legacy;
mod live_photo;
mod metadata;
mod names;
mod reconcile;
mod snapshot;
mod source;
mod store;
mod transfer;
pub use source::{CollectionEntry, FileEntry, Source};
use std::{
fs::{File, OpenOptions},
io::Write,
num::NonZeroUsize,
path::{Path, PathBuf},
sync::Mutex,
time::{Duration, Instant},
};
use crate::metadata::Root;
use anyhow::{Context, Result, bail, ensure};
use rusqlite::{OptionalExtension, params};
use serde_json::{Value, json};
use tokio::sync::watch;
use snapshot::FileSnapshot;
use store::Store;
#[derive(Debug, thiserror::Error)]
#[error("conflict: {0}")]
struct Conflict(String);
#[derive(Debug, thiserror::Error)]
#[error("export cancelled")]
struct Cancelled;
pub struct Options {
pub albums: Vec<String>,
pub exclude_albums: Vec<String>,
pub exclude_shared: bool,
pub exclude_hidden: bool,
pub adopt: bool,
pub hardlinks: bool,
pub jobs: NonZeroUsize,
}
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
pub struct AdoptionRequired(&'static str);
pub struct Summary {
pub json: Value,
pub complete: bool,
pub completed: i64,
pub expected: i64,
}
struct RootLock(File);
impl Drop for RootLock {
fn drop(&mut self) {
let _ = self.0.unlock();
}
}
pub struct Export {
destination: PathBuf,
expected_root: Root,
prepared: Option<(Store, Result<()>)>,
root_lock: Option<RootLock>,
options: Options,
adopting: bool,
}
impl Export {
pub fn open(path: &Path, session: &ente_core::Session, options: Options) -> Result<Self> {
let destination = destination(path)?;
let expected_root = Root::new(&session.master_key, session.user_id)?;
let root_lock = open_root(&destination, &expected_root)?;
let adopting = root_lock.is_none() && legacy::read::recognized(&destination)?;
ensure!(
!adopting || options.adopt,
AdoptionRequired(
"This folder contains an export from the old CLI or Ente Desktop. Use --adopt if you want to take it over."
)
);
ensure!(
!options.adopt || root_lock.is_some() || adopting,
"there is no export to adopt"
);
ensure!(
root_lock.is_some()
|| adopting
|| !destination.try_exists()?
|| std::fs::read_dir(&destination)?.next().is_none(),
"nonempty destination has no recognized export.json"
);
Ok(Self {
destination,
expected_root,
root_lock,
options,
prepared: None,
adopting,
})
}
pub fn destination(&self) -> &Path {
&self.destination
}
pub fn exists(&self) -> bool {
self.root_lock.is_some()
}
pub fn verify_source(&self, session: &ente_core::Session) -> Result<()> {
ensure!(
Root::new(&session.master_key, session.user_id)?.source_verifier
== self.expected_root.source_verifier,
"account keys changed while waiting for access"
);
Ok(())
}
pub async fn prepare(
&mut self,
connection: rusqlite::Connection,
open_index: impl FnOnce() -> Result<(tempfile::TempDir, rusqlite::Connection)>,
source: &mut impl Source,
session: &ente_core::Session,
cancel: &mut watch::Receiver<bool>,
report: &(dyn Fn(&str) + Sync),
) -> Result<()> {
let store = Store::open(connection, &self.destination)?;
let bound: bool = store
.db
.query_row("SELECT bound FROM export WHERE id=1", [], |r| r.get(0))?;
let accepted = !bound
&& store.db.query_row(
"SELECT EXISTS(SELECT 1 FROM albums WHERE retained=0)",
[],
|r| r.get::<_, bool>(0),
)?;
ensure!(
self.root_lock.is_none() || bound || accepted || self.options.adopt,
AdoptionRequired("this export is not associated; use --adopt")
);
let initial = !bound && !accepted && self.options.adopt;
let index = initial.then(|| adopt::Index::new(open_index)).transpose()?;
store.reset_desired()?;
let mut failures = legacy::Failures::default();
self.adopting = !bound
&& (accepted
|| (initial && (self.adopting || legacy::read::recognized(&self.destination)?)));
if let Some(index) = &index {
let mut desktop = None;
failures.record(
legacy::read::desktop(&self.destination).map(|value| desktop = value),
"export_status.json",
)?;
if self.root_lock.is_some() {
failures.record(
adopt::scan(&self.destination, index, desktop.as_ref(), session.user_id),
"this CLI's export records",
)?;
}
if self.adopting {
legacy::read::scan(
&self.destination,
index,
session.user_id,
desktop.as_ref(),
&mut failures,
)?;
}
}
let mut albums = store.db.prepare("SELECT a.id,a.name,j.value FROM albums a JOIN json_records j ON j.owner=a.key AND j.role='' LEFT JOIN album_sources s ON s.id=a.id WHERE json_extract(s.record,'$.shared') IS NULL OR json_extract(s.record,'$.hidden') IS NULL ORDER BY a.retained")?;
for row in albums.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
))
})? {
let (id, name, metadata) = row?;
store.recover_album(
id,
&name,
serde_json::from_str(&metadata)?,
None,
session.user_id,
)?;
}
drop(albums);
if let Some(index) = &index {
let mut albums = index.db.prepare("SELECT id,json_extract(record,'$.album.name'),json_extract(record,'$.json.value'),json_extract(record,'$.legacy.owner') FROM adoption_album ORDER BY retained")?;
for row in albums.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, Option<String>>(2)?,
r.get::<_, Option<i64>>(3)?,
))
})? {
let (id, name, metadata, owner) = row?;
store.recover_album(
id,
&name,
metadata
.map(|metadata| serde_json::from_str(&metadata))
.transpose()?
.unwrap_or(Value::Null),
owner.map(|owner| owner != session.user_id),
session.user_id,
)?;
store.db.execute(
"INSERT OR IGNORE INTO desired_albums(id,name) VALUES(?1,?2)",
params![id, name],
)?;
}
}
report("Refreshing album metadata.");
tokio::select! {
result=source.refresh_catalog(session)=>result?,
_=cancel.changed()=>return Err(Cancelled.into()),
}
let mut after = 0;
loop {
ensure!(!*cancel.borrow(), Cancelled);
let entries = source.catalog_page(after)?;
if entries.is_empty() {
break;
}
for entry in entries {
after = entry.id;
store.desired_album(&entry, session.user_id)?;
}
}
store
.db
.execute(
"INSERT OR IGNORE INTO desired_albums(id,name) SELECT s.id,json_extract(s.record,'$.name') FROM album_sources s WHERE EXISTS(SELECT 1 FROM albums a WHERE a.id=s.id)",
[],
)?;
store
.db
.execute(
"INSERT OR IGNORE INTO desired_albums(id,name) SELECT id,name FROM albums ORDER BY retained",
[],
)?;
store
.db
.execute(
"INSERT OR IGNORE INTO desired_albums(id,name) SELECT album,name FROM pending WHERE file IS NULL AND retained=0",
[],
)?;
store.db.execute("UPDATE desired_albums SET shared=(SELECT json_extract(record,'$.shared') FROM album_sources WHERE id=desired_albums.id),hidden=(SELECT json_extract(record,'$.hidden') FROM album_sources WHERE id=desired_albums.id) WHERE present=0", [])?;
if self.adopting {
legacy::scope(&store, &self.options, index.as_ref(), report)?;
} else {
failures.finish()?;
select(&store, &self.options, index.as_ref(), report)?;
}
if !self.adopting {
if self.root_lock.is_none() {
self.root_lock = Some(initialize(&self.destination, &self.expected_root, false)?);
}
if !initial {
store
.db
.execute("UPDATE export SET bound=1 WHERE id=1 AND bound=0", [])?;
}
}
let refresh_cancel = cancel.clone();
let mut refreshed = tokio::select! {
result=refresh(&store,source,session,&self.options,&refresh_cancel,report)=>result,
_=cancel.changed()=>Err(Cancelled.into()),
};
if self.adopting {
refreshed?;
legacy::scope(&store, &self.options, index.as_ref(), report)?;
if let Some(index) = &index {
report("Checking the existing files before taking over this export.");
legacy::Preflight {
root: &self.destination,
store: &store,
session,
index,
cancel,
matching_root: self.root_lock.is_some(),
}
.run(source, failures)
.await?;
if self.root_lock.is_none() {
self.root_lock =
Some(initialize(&self.destination, &self.expected_root, true)?);
}
legacy::move_leftovers(&self.destination, index, cancel)?;
adopt::accept(index, &store, false)?;
}
refreshed = Ok(());
} else if let Some(index) = &index {
adopt::accept(index, &store, true)?;
}
let refreshed = refreshed.and_then(|()| {
store.db.execute("DELETE FROM album_sources WHERE id NOT IN (SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL) AND NOT EXISTS(SELECT 1 FROM albums WHERE albums.id=album_sources.id) AND NOT EXISTS(SELECT 1 FROM pending WHERE album=album_sources.id)", [])?;
Ok(())
});
self.prepared = Some((store, refreshed));
Ok(())
}
pub fn execute(
&mut self,
session: &ente_core::Session,
cancel_send: &watch::Sender<bool>,
cancel: &watch::Receiver<bool>,
report: &(dyn Fn(&str) + Sync),
) -> Result<Summary> {
let (store, refreshed) = self.prepared.take().context("export is not prepared")?;
let prepared = refreshed.is_ok();
let shared = Mutex::new(store);
let run = reconcile::Run {
root: &self.destination,
store: &shared,
cancel,
report,
hardlinks: self.options.hardlinks,
};
let result = (|| {
refreshed?;
run.recover()?;
run.albums()?;
transfers(&run, session, cancel_send, self.options.jobs.get())?;
run.check_cancel()?;
run.retain_removed()
})();
if let Err(error) = result {
run.report("run", &error)?;
}
if let Err(error) = run.cleanup(None) {
run.report("run", &error)?;
}
let store = shared
.into_inner()
.map_err(|_| anyhow::anyhow!("export database mutex poisoned"))?;
if self.adopting
&& work_complete(&store, prepared)?
&& let Err(error) = legacy::finish(&self.destination, &store, cancel)
{
record_outcome(&store, "run", &error)?;
report(&format!("run: {error:#}"));
}
summary(&store, &self.destination, prepared)
}
}
fn destination(path: &Path) -> Result<PathBuf> {
let absolute = if path.is_absolute() {
path.to_owned()
} else {
std::env::current_dir()?.join(path)
};
let mut base = absolute.as_path();
let mut tail = Vec::new();
while !base.try_exists()? {
tail.push(
base.file_name()
.context("destination has no directory name")?
.to_owned(),
);
base = base
.parent()
.context("destination has no existing parent")?;
}
if tail.is_empty() {
ensure!(
!std::fs::symlink_metadata(base)?.is_symlink(),
"export destination is a symlink"
);
}
let mut resolved = std::fs::canonicalize(base)?;
for part in tail.into_iter().rev() {
resolved.push(part);
}
Ok(resolved)
}
fn open_root(destination: &Path, expected: &Root) -> Result<Option<RootLock>> {
if !destination.try_exists()? {
return Ok(None);
}
ensure!(
destination.is_dir(),
"export destination is not a directory"
);
let path = names::check(destination, "export.json")?;
if !path.try_exists()? {
return Ok(None);
}
let file = OpenOptions::new().read(true).write(true).open(&path)?;
file.try_lock()
.context("another writer is maintaining this export")?;
let lock = RootLock(file);
let record: Root = serde_json::from_reader(&lock.0).context("invalid export.json")?;
ensure!(
record.format == expected.format && record.version == expected.version,
"unsupported export format"
);
ensure!(
record.source_verifier == expected.source_verifier,
"this export belongs to another source account"
);
Ok(Some(lock))
}
fn initialize(destination: &Path, root: &Root, legacy: bool) -> Result<RootLock> {
std::fs::create_dir_all(destination)?;
let path = names::check(destination, "export.json")?;
let file = OpenOptions::new()
.read(true)
.write(true)
.create_new(true)
.open(&path)
.context("destination was initialized by another writer; retry")?;
let result = (|| {
file.lock().context("cannot lock the new export")?;
let mut lock = RootLock(file);
ensure!(
legacy
|| std::fs::read_dir(destination)?
.all(|entry| entry.is_ok_and(|entry| entry.file_name() == "export.json")),
"destination became nonempty during initialization"
);
serde_json::to_writer_pretty(&mut lock.0, root)?;
lock.0.write_all(b"\n")?;
Ok(lock)
})();
if result.is_err()
&& let Err(error) = std::fs::remove_file(&path)
{
return result.with_context(|| format!("cannot remove incomplete export.json: {error}"));
}
result
}
fn select(
store: &Store,
options: &Options,
index: Option<&adopt::Index>,
report: &(dyn Fn(&str) + Sync),
) -> Result<()> {
if options.albums.is_empty() {
store
.db
.execute("UPDATE desired_albums SET selected=1", [])?;
}
for (selectors, include) in [(&options.albums, true), (&options.exclude_albums, false)] {
for selector in selectors {
let mut query = store.db.prepare(
"SELECT id,name FROM desired_albums WHERE id=?1 OR name=?2 ORDER BY id LIMIT 11",
)?;
let candidates: Vec<(i64, Option<String>)> = query
.query_map(params![canonical_id(selector), selector], |r| {
Ok((r.get(0)?, r.get(1)?))
})?
.collect::<rusqlite::Result<_>>()?;
ensure!(!candidates.is_empty(), "no album matches {selector:?}");
if candidates.len() != 1 {
bail!(
"album {selector:?} is ambiguous: {}",
candidates
.iter()
.map(|(id, name)| format!(
"{id} {:?}",
name.as_deref().unwrap_or("unavailable")
))
.collect::<Vec<_>>()
.join(", ")
);
}
store.db.execute(
"UPDATE desired_albums SET selected=?1 WHERE id=?2",
params![include, candidates[0].0],
)?;
}
}
store.db.execute(
"UPDATE desired_albums SET selected=0 WHERE (?1 AND shared=1) OR (?2 AND hidden=1)",
params![options.exclude_shared, options.exclude_hidden],
)?;
let mut unknown = store.db.prepare("SELECT id,CASE WHEN ?1 AND shared IS NULL AND ?2 AND hidden IS NULL THEN '--exclude-shared and --exclude-hidden' WHEN ?1 AND shared IS NULL THEN '--exclude-shared' ELSE '--exclude-hidden' END FROM desired_albums WHERE selected=1 AND present=0 AND ((?1 AND shared IS NULL) OR (?2 AND hidden IS NULL))")?;
for row in unknown.query_map(
params![options.exclude_shared, options.exclude_hidden],
|r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)),
)? {
let (id, flags) = row?;
store
.db
.execute("UPDATE desired_albums SET selected=0 WHERE id=?1", [id])?;
let work: bool = store.db.query_row("SELECT EXISTS(SELECT 1 FROM albums WHERE id=?1 AND retained=0) OR EXISTS(SELECT 1 FROM pending WHERE album=?1) OR EXISTS(SELECT 1 FROM temporaries WHERE album=?1)", [id], |r| r.get(0))?;
let importing = index
.map(|index| {
index.db.query_row(
"SELECT EXISTS(SELECT 1 FROM adoption_album WHERE id=?1 AND retained=0)",
[id],
|r| r.get::<_, bool>(0),
)
})
.transpose()?
.unwrap_or(false);
let unit = format!("album:{id}");
if (work || importing)
&& store
.db
.execute("INSERT OR IGNORE INTO outcomes VALUES(?1,0)", [&unit])?
!= 0
{
report(&format!(
"{unit}: cannot determine whether this album is excluded by {flags}; use --exclude-album {id} or remove {flags}"
));
}
}
Ok(())
}
async fn refresh(
store: &Store,
source: &mut impl Source,
session: &ente_core::Session,
options: &Options,
cancel: &watch::Receiver<bool>,
report: &(dyn Fn(&str) + Sync),
) -> Result<()> {
let need_favorites: bool = store.db.query_row(
"SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND present=1)",
[],
|r| r.get(0),
)?;
let mut favorites_known = true;
let mut after = 0;
report("Refreshing selected albums and Favorites.");
loop {
ensure!(!*cancel.borrow(), Cancelled);
let next: Option<(i64, bool, bool)> = store
.db
.query_row(
"SELECT id,selected,favorites FROM desired_albums WHERE present=1 AND (selected=1 OR (favorites=1 AND ?1)) AND id>?2 ORDER BY id LIMIT 1",
params![need_favorites, after],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)
.optional()?;
let Some((id, mut selected, favorites)) = next else {
break;
};
after = id;
let refreshed = source.refresh_album(session, id).await;
if refreshed.is_err() && favorites {
favorites_known = false;
}
let result = refreshed.and_then(|entry| {
store.desired_album(&entry, session.user_id)?;
let (album, documents) = entry.value?;
selected &= !(options.exclude_shared && album.owner_id != session.user_id
|| options.exclude_hidden
&& matches!(
album.visibility,
ente_photos::collections::Visibility::Hidden
));
store.db.execute(
"UPDATE desired_albums SET selected=?1 WHERE id=?2",
params![selected, id],
)?;
if selected {
store.snapshot_album(&album, &documents, session.user_id)?;
}
store
.db
.execute("UPDATE desired_albums SET ready=1 WHERE id=?1", [id])?;
Ok(())
});
if let Err(error) = result {
store.db.execute(
"UPDATE desired_albums SET failure=?1 WHERE id=?2",
params![error.to_string(), id],
)?;
if selected {
record_outcome(store, &format!("album:{id}"), &error)?;
report(&format!("album:{id}: {error:#}"));
}
if fatal(&error) {
return Err(error);
}
}
}
store.db.execute(
"UPDATE desired_albums SET ready=1 WHERE selected=1 AND present=0",
[],
)?;
if need_favorites && favorites_known {
let mut after = 0;
loop {
let ids = source.favorite_page(after)?;
if ids.is_empty() {
break;
}
let transaction = store.db.unchecked_transaction()?;
for id in ids {
after = id;
store.db.execute("INSERT INTO favorites VALUES(?1)", [id])?;
}
transaction.commit()?;
}
}
after = 0;
loop {
let id: Option<i64> = store
.db
.query_row(
"SELECT id FROM desired_albums WHERE selected=1 AND ready=1 AND present=1 AND failure IS NULL AND id>?1 ORDER BY id LIMIT 1",
[after],
|r| r.get(0),
)
.optional()?;
let Some(id) = id else { break };
after = id;
let mut after_file = 0;
loop {
ensure!(!*cancel.borrow(), Cancelled);
let records = source.file_page(id, after_file)?;
if records.is_empty() {
break;
}
let transaction = store.db.unchecked_transaction()?;
for record in records {
after_file = record.id;
let file = record
.value
.and_then(|(file, documents)| FileSnapshot::new(file, &documents));
let failure = if !favorites_known {
Some("favorite state is incomplete".to_owned())
} else {
file.as_ref().err().map(ToString::to_string)
};
store.db.execute(
"INSERT INTO desired_files(album,file,favorited,failure) VALUES(?1,?2,?3,?4)",
params![
id,
record.id,
if favorites_known { Some(false) } else { None },
failure
],
)?;
if failure.is_none() {
let file = file?;
store.db.execute(
"INSERT INTO chosen_files VALUES(?1,?2,?3) ON CONFLICT(file) DO UPDATE SET album=excluded.album,version=excluded.version WHERE excluded.version>chosen_files.version OR (excluded.version=chosen_files.version AND excluded.album<chosen_files.album)",
params![file.id, id, file.updated_at_micros],
)?;
}
}
transaction.commit()?;
}
}
store
.db
.execute(
"UPDATE desired_files SET favorited=EXISTS(SELECT 1 FROM favorites WHERE favorites.file=desired_files.file) WHERE favorited IS NOT NULL",
[],
)?;
let mut after_file = 0;
loop {
ensure!(!*cancel.borrow(), Cancelled);
let mut query = store
.db
.prepare("SELECT album,file FROM chosen_files WHERE file>?1 ORDER BY file LIMIT 128")?;
let chosen: Vec<(i64, i64)> = query
.query_map([after_file], |r| Ok((r.get(0)?, r.get(1)?)))?
.collect::<rusqlite::Result<_>>()?;
drop(query);
if chosen.is_empty() {
break;
}
let transaction = store.db.unchecked_transaction()?;
for (album, id) in chosen {
after_file = id;
let (file, documents) = source.file(album, id)?.value?;
store.snapshot_file(&FileSnapshot::new(file, &documents)?)?;
}
transaction.commit()?;
}
store.db.execute(
"DELETE FROM sources WHERE file NOT IN (SELECT file FROM chosen_files)",
[],
)?;
let mut query = store
.db
.prepare("SELECT album,file,failure FROM desired_files WHERE failure IS NOT NULL")?;
let mut rows = query.query([])?;
while let Some(row) = rows.next()? {
let unit = format!("file:{}:{}", row.get::<_, i64>(0)?, row.get::<_, i64>(1)?);
let error = anyhow::anyhow!(row.get::<_, String>(2)?);
record_outcome(store, &unit, &error)?;
report(&format!("{unit}: {error:#}"));
}
Ok(())
}
struct CancelWorkers<'a>(&'a watch::Sender<bool>);
impl Drop for CancelWorkers<'_> {
fn drop(&mut self) {
if std::thread::panicking() {
self.0.send_replace(true);
}
}
}
fn transfers(
run: &reconcile::Run<'_>,
session: &ente_core::Session,
cancel_send: &watch::Sender<bool>,
jobs: usize,
) -> Result<()> {
let (store, cancel, report) = (run.store, run.cancel, run.report);
let (expected, files): (i64, i64) = store::lock(store)?.db.query_row(
"SELECT count(*),count(DISTINCT file) FROM desired_files WHERE failure IS NULL",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
report(&format!("Maintaining {expected} selected copies."));
let runtime = tokio::runtime::Handle::current();
let last_report = Mutex::new(None::<Instant>);
std::thread::scope(|scope| {
let mut workers = Vec::new();
for _ in 0..jobs.min(files as usize) {
let runtime = &runtime;
let last_report = &last_report;
workers.push(scope.spawn(move || {
let _cancel_on_panic = CancelWorkers(cancel_send);
let result = (|| {
loop {
if *cancel.borrow() {
break;
}
let file = {
let store = store::lock(store)?;
let transaction = store.db.unchecked_transaction()?;
let id: Option<i64> = store
.db
.query_row(
"SELECT file FROM desired_files WHERE failure IS NULL AND claimed=0 ORDER BY file LIMIT 1",
[],
|r| r.get(0),
)
.optional()?;
let Some(id) = id else {
break;
};
store
.db
.execute(
"UPDATE desired_files SET claimed=1 WHERE file=?1",
[id],
)?;
let file: FileSnapshot = store::json(&store.db,"SELECT record FROM sources WHERE file=?1", [id])?
.context("missing file source snapshot")?;
transaction.commit()?;
file
};
let id = file.id;
let result = runtime.block_on(run.file(session, file));
if result.as_ref().err().is_some_and(fatal) {
cancel_send.send_replace(true);
}
let reported = result.or_else(|error| run.block_file(id, error));
let cleaned = run.cleanup(Some(id));
reported?;
cleaned?;
let completed = {
let mut last = last_report.lock().map_err(|_| anyhow::anyhow!("progress mutex poisoned"))?;
if last.is_none_or(|time| time.elapsed() >= Duration::from_secs(2)) {
*last = Some(Instant::now());
Some(store::lock(store)?.db.query_row(
"SELECT count(*) FROM desired_files WHERE completed=1",
[],
|r| r.get::<_, i64>(0),
)?)
} else {
None
}
};
if let Some(completed) = completed {
report(&format!("Completed {completed}/{expected} selected copies."));
}
}
Ok(())
})();
if result.is_err() {
cancel_send.send_replace(true);
}
result
}));
}
let mut result = Ok(());
for worker in workers {
let completed = worker
.join()
.unwrap_or_else(|_| Err(anyhow::anyhow!("export worker panicked")));
if result.is_ok() {
result = completed;
}
}
result
})
}
fn record_outcome(store: &Store, unit: &str, error: &anyhow::Error) -> Result<()> {
let conflict = error.downcast_ref::<Conflict>().is_some();
store
.db
.execute(
"INSERT INTO outcomes VALUES(?1,?2) ON CONFLICT(unit) DO UPDATE SET conflict=max(conflict,excluded.conflict)",
params![unit, conflict],
)?;
Ok(())
}
fn fatal(error: &anyhow::Error) -> bool {
fatal_cause(error.as_ref())
}
fn fatal_cause(cause: &(dyn std::error::Error + 'static)) -> bool {
if cause.is::<rusqlite::Error>()
|| cause.is::<Cancelled>()
|| cause
.downcast_ref::<ente_core::http::Error>()
.is_some_and(|error| error.status_code() == Some(401))
{
return true;
}
if let Some(error) = cause.downcast_ref::<std::io::Error>() {
return error.kind() == std::io::ErrorKind::StorageFull
|| error.get_ref().is_some_and(|inner| fatal_cause(inner));
}
if let Some(error) = cause.downcast_ref::<ente_photos::files::Error>() {
use ente_photos::files::Error;
match error {
Error::Http(error) => return fatal_cause(error),
Error::Io(error) => return fatal_cause(error),
Error::Collections(error) => return fatal_cause(error),
_ => {}
}
}
if let Some(ente_collections::Error::Http(error)) = cause.downcast_ref() {
return fatal_cause(error);
}
if let Some(error) = cause.downcast_ref::<crate::live_photo::Error>() {
match error {
crate::live_photo::Error::Io(error) => return fatal_cause(error),
crate::live_photo::Error::Zip(error) => return fatal_cause(error),
crate::live_photo::Error::Components | crate::live_photo::Error::ExpansionLimit => {}
}
}
cause.source().is_some_and(fatal_cause)
}
fn work_complete(store: &Store, prepared: bool) -> Result<bool> {
Ok(prepared && store.db.query_row("SELECT NOT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND (ready=0 OR failure IS NOT NULL)) AND NOT EXISTS(SELECT 1 FROM desired_files WHERE completed=0 OR failure IS NOT NULL) AND NOT EXISTS(SELECT 1 FROM outcomes) AND NOT EXISTS(SELECT 1 FROM pending p JOIN desired_albums a ON a.id=p.album WHERE a.selected=1)",[],|r|r.get::<_,bool>(0))?)
}
fn summary(store: &Store, destination: &Path, prepared: bool) -> Result<Summary> {
let unknown: bool = !prepared
|| store.db.query_row(
"SELECT EXISTS(SELECT 1 FROM desired_albums WHERE selected=1 AND ready=0)",
[],
|r| r.get(0),
)?;
let (files, expected, completed): (i64, i64, i64) = store.db.query_row(
"SELECT count(DISTINCT file),count(*),coalesce(sum(completed),0) FROM desired_files",
[],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)?;
let (failures, conflicts): (i64, i64) = store.db.query_row(
"SELECT count(*),coalesce(sum(conflict),0) FROM outcomes",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
let (exported, metadata_updated, renamed, retained): (i64, i64, i64, i64) = store
.db
.query_row(
"SELECT coalesce(sum(exported),0),coalesce(sum(metadata_updated),0),coalesce(sum(renamed),0),coalesce(sum(retained),0) FROM events",
[],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
)?;
let bound: bool = store
.db
.query_row("SELECT bound FROM export WHERE id=1", [], |r| r.get(0))?;
let complete = bound && work_complete(store, prepared)?;
let result = json!({"output":destination.to_string_lossy(),"complete":complete,"files":if unknown {None}else{Some(files)},"copies":if unknown {None}else{Some(json!({"expected":expected,"completed":completed,"pending":expected-completed}))},"changes":{"exported":exported,"metadataUpdated":metadata_updated,"renamed":renamed,"movedToTrash":retained},"failures":{"total":failures,"conflicts":conflicts}});
Ok(Summary {
json: result,
complete,
completed,
expected,
})
}
fn canonical_id(selector: &str) -> rusqlite::types::Value {
selector
.parse::<i64>()
.ok()
.filter(|id| id.to_string() == selector)
.map(rusqlite::types::Value::Integer)
.unwrap_or(rusqlite::types::Value::Null)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn initialization_removes_its_marker_on_failure_and_retries() {
let root = Root::new(&ente_core::crypto::Key::generate(), 1).unwrap();
let directory = tempfile::tempdir().unwrap();
let stray = directory.path().join("photo.jpg");
std::fs::write(&stray, b"unrelated").unwrap();
let error = initialize(directory.path(), &root, false).err().unwrap();
assert!(error.to_string().contains("destination became nonempty"));
assert!(!directory.path().join("export.json").exists());
assert_eq!(std::fs::read(&stray).unwrap(), b"unrelated");
std::fs::remove_file(stray).unwrap();
drop(initialize(directory.path(), &root, false).unwrap());
assert!(open_root(directory.path(), &root).unwrap().is_some());
}
#[test]
fn transparent_errors_preserve_fatal_classification() {
for status in [401, 404] {
let error = ente_core::http::Error::Http {
status,
path: "/files".into(),
};
let error =
ente_photos::files::Error::Collections(ente_collections::Error::Http(error));
let error = anyhow::Error::new(error).context("download failed");
assert_eq!(fatal(&error), status == 401);
assert_eq!(
format!("{error:#}"),
format!("download failed: HTTP {status} at /files")
);
}
let cancelled = crate::live_photo::Error::Io(std::io::Error::other(Cancelled));
assert!(fatal(&anyhow::Error::new(cancelled)));
let full = crate::live_photo::Error::Io(std::io::ErrorKind::StorageFull.into());
assert!(fatal(&anyhow::Error::new(full)));
let zip = crate::live_photo::Error::Zip(zip::result::ZipError::Io(
std::io::ErrorKind::StorageFull.into(),
));
assert!(fatal(&anyhow::Error::new(zip)));
let sqlite = rusqlite::Error::InvalidQuery;
assert!(fatal(&anyhow::Error::new(sqlite)));
let local = ente_photos::files::Error::Io(std::io::ErrorKind::PermissionDenied.into());
assert!(!fatal(&anyhow::Error::new(local)));
}
}