Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Simplify linked export preparation and verification

Manav Rathi37a00540df9c
diff --git a/rust/apps/cli-next/tests/support/export_media.rs b/rust/apps/cli-next/tests/support/export_media.rs
index 8a46c92b71..93b3de49f5 100644
--- a/rust/apps/cli-next/tests/support/export_media.rs
+++ b/rust/apps/cli-next/tests/support/export_media.rs
@@ -442,6 +442,30 @@ fn engine_consolidates_copies_and_keeps_linked_history() {
readonly.set_readonly(true);
fs::set_permissions(&first, readonly).unwrap();
engine.hardlinks = true;
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt;
+ let directory = root.join("Second");
+ let permissions = fs::metadata(&directory).unwrap().permissions();
+ fs::set_permissions(&directory, fs::Permissions::from_mode(0o500)).unwrap();
+ let probe = directory.join("permission-probe");
+ let result = fs::hard_link(&first, &probe)
+ .is_err()
+ .then(|| engine.run(false, &["First", "Second"]));
+ fs::set_permissions(&directory, permissions).unwrap();
+ if let Some(result) = result {
+ let result = result.unwrap();
+ assert!(!result.complete);
+ assert_eq!(result.json["copies"]["completed"], 2);
+ assert_eq!(fs::read(&second).unwrap(), b"old");
+ assert!(!same_file::is_same_file(&first, &second).unwrap());
+ } else {
+ fs::remove_file(probe).unwrap();
+ eprintln!(
+ "skipping permission failure: directory permissions do not prevent hard links"
+ );
+ }
+ }
assert_eq!(run(&mut engine, true)["changes"]["exported"], 2);
assert!(same_file::is_same_file(&first, &second).unwrap());
assert!(!same_file::is_same_file(&first, root.join("Second/Other.jpg")).unwrap());
@@ -587,50 +611,7 @@ fn engine_recovers_interrupted_link_publication() {
}
#[test]
-#[cfg(unix)]
-fn engine_link_failure_preserves_existing_copy() {
- use std::os::unix::fs::PermissionsExt;
- let mut server = mockito::Server::new();
- let key = Key::generate();
- let (record, bytes) = original_fixture(&key, b"original", false, true, 10);
- let fetched = download(&mut server, 10, &bytes, 1);
- let mut engine = new_engine(&server, key, record);
- run(&mut engine, true);
- engine.hardlinks = true;
- let directory = engine.root().join("Second");
- let permissions = fs::metadata(&directory).unwrap().permissions();
- fs::set_permissions(&directory, fs::Permissions::from_mode(0o500)).unwrap();
- let probe = directory.join("permission-probe");
- if fs::hard_link(directory.join("Photo.jpg"), &probe).is_ok() {
- fs::set_permissions(&directory, permissions).unwrap();
- fs::remove_file(probe).unwrap();
- eprintln!("skipping: directory permissions do not prevent hard links");
- return;
- }
- let result = engine.run(false, &[]);
- fs::set_permissions(&directory, permissions).unwrap();
- let result = result.unwrap();
- assert!(!result.complete);
- assert_eq!(result.json["copies"]["completed"], 1);
- assert_eq!(fs::read(directory.join("Photo.jpg")).unwrap(), b"original");
- assert!(!linked(
- &engine.root(),
- "First/Photo.jpg",
- "Second/Photo.jpg"
- ));
- run(&mut engine, true);
- assert!(linked(
- &engine.root(),
- "First/Photo.jpg",
- "Second/Photo.jpg"
- ));
- fetched.assert();
-}
-
-#[test]
-#[cfg(unix)]
-fn engine_rechecks_link_source_before_resetting_timestamp() {
- use std::os::unix::fs::PermissionsExt;
+fn engine_rechecks_link_source_before_completion() {
use std::time::{Duration, UNIX_EPOCH};
let mut server = mockito::Server::new();
let key = Key::generate();
@@ -658,20 +639,14 @@ fn engine_rechecks_link_source_before_resetting_timestamp() {
let modified = UNIX_EPOCH + Duration::from_secs(1_700_000_100);
engine.hardlinks = true;
let export = engine.prepare(engine.options(false, &[])).unwrap();
- let directory = root.join("Third");
- let permissions = fs::metadata(&directory).unwrap().permissions();
- fs::set_permissions(&directory, fs::Permissions::from_mode(0o500)).unwrap();
- let probe = directory.join("permission-probe");
- if fs::hard_link(&first, &probe).is_ok() {
- fs::set_permissions(&directory, permissions).unwrap();
- fs::remove_file(probe).unwrap();
- eprintln!("skipping: directory permissions do not prevent hard links");
- return;
- }
+ let blocked = root.join("Third/metadata/Photo.jpg.json");
+ fs::remove_file(&blocked).unwrap();
+ fs::create_dir(&blocked).unwrap();
let result = engine.execute_with_report(export, &|line| {
eprintln!("{line}");
- if line.starts_with("file:3:10:") && line.contains("cannot link") {
+ if line.starts_with("file:3:10:") && line.contains("expected a regular file") {
assert!(linked(&root, "First/Photo.jpg", "Second/Photo.jpg"));
+ assert_eq!(fs::metadata(&first).unwrap().modified().unwrap(), modified);
fs::write(&first, b"modified").unwrap();
fs::File::options()
.write(true)
@@ -681,10 +656,9 @@ fn engine_rechecks_link_source_before_resetting_timestamp() {
.unwrap();
}
});
- fs::set_permissions(&directory, permissions).unwrap();
let result = result.unwrap();
assert!(!result.complete);
- assert_eq!(result.json["conflicts"], 1);
+ assert_eq!(result.json["conflicts"], 2);
assert_eq!(result.json["copies"]["completed"], 2);
assert_eq!(fs::read(&first).unwrap(), b"modified");
assert_eq!(
diff --git a/rust/crates/photos-export/src/reconcile.rs b/rust/crates/photos-export/src/reconcile.rs
index b788d2f23f..88160ef69f 100644
--- a/rust/crates/photos-export/src/reconcile.rs
+++ b/rust/crates/photos-export/src/reconcile.rs
@@ -1262,20 +1262,16 @@ impl Run<'_> {
}
}
let path = self.path(&destination)?;
- let same_source = self.hardlinks
+ let mut source = prepared
+ .components
+ .iter_mut()
+ .find(|source| source.role == component.role);
+ let consolidate = self.hardlinks
&& valid
- && same_file::is_same_file(
- self.path(
- &prepared
- .components
- .iter()
- .find(|source| source.role == component.role)
- .context("no original is available for linking")?
- .location,
- )?,
+ && !same_file::is_same_file(
+ self.path(&source.as_ref().context("missing original")?.location)?,
&path,
)?;
- let consolidate = self.hardlinks && valid && !same_source;
if consolidate {
let previous = previous.context("missing completed original")?;
ensure!(
@@ -1284,11 +1280,7 @@ impl Run<'_> {
);
}
if !valid || consolidate {
- let source = prepared
- .components
- .iter_mut()
- .find(|source| source.role == component.role)
- .context("no verified original is available")?;
+ let source = source.as_deref_mut().context("missing original")?;
let temporary = if source.temporary {
source.location.clone()
} else if self.hardlinks {
@@ -1298,17 +1290,7 @@ impl Run<'_> {
disk::hard_link(&from, &to).with_context(|| {
format!("cannot link {} to {}", from.display(), path.display())
})?;
- let properties = Properties::read(&to)?;
- if source.verified_properties.as_ref() != Some(&properties) {
- ensure!(
- fs::matches(&to, source.size, &source.hash, self.cancel)?,
- super::Conflict(format!(
- "at {}: locally changed media",
- from.display()
- ))
- );
- source.verified_properties = Some(properties);
- }
+ source.verify(self, &to)?;
temporary
} else {
let (temporary, file) = self.temporary(&album.key, album_id, Some(file.id))?;
@@ -1351,32 +1333,11 @@ impl Run<'_> {
|| observed != published.properties.as_ref()
|| renamed
{
- let verified = if self.hardlinks {
- let properties = Properties::read(&path)?;
- let source = prepared
- .components
- .iter_mut()
- .find(|source| source.role == component.role)
- .context("no original is available for verification")?;
- if source.verified_properties.as_ref().or(observed) != Some(&properties) {
- ensure!(
- fs::matches(&path, published.size, &published.hash, self.cancel)?,
- super::Conflict(format!(
- "at {}: locally changed media",
- path.display()
- ))
- );
- source.verified_properties = Some(properties);
- }
- source.verified_properties.as_mut()
+ published.properties = Some(if self.hardlinks {
+ source.context("missing original")?.verify(self, &path)?
} else {
- None
- };
- let updated = fs::set_time(&path, intended_time)?;
- if let Some(verified) = verified {
- verified.modified = updated.modified;
- }
- published.properties = Some(updated);
+ fs::set_time(&path, intended_time)?
+ });
}
published.intended_time = Some(intended_time);
published.signature = Some(prepared.signature.clone());
diff --git a/rust/crates/photos-export/src/transfer.rs b/rust/crates/photos-export/src/transfer.rs
index 3b687ec1f4..e8c1c62373 100644
--- a/rust/crates/photos-export/src/transfer.rs
+++ b/rust/crates/photos-export/src/transfer.rs
@@ -1,6 +1,7 @@
use std::{
fs::File,
io::{Seek, SeekFrom},
+ path::Path,
};
use crate::{metadata::Role, snapshot::FileSnapshot};
@@ -27,16 +28,39 @@ pub struct Source {
}
impl Source {
- fn existing(component: &Component, verified_properties: Option<Properties>) -> Self {
- Self {
+ fn existing(
+ run: &Run<'_>,
+ component: &Component,
+ (mut properties, hashed): (Properties, bool),
+ intended_time: i64,
+ ) -> Result<Self> {
+ if run.hardlinks
+ && (component.intended_time != Some(intended_time)
+ || component.properties.as_ref() != Some(&properties))
+ {
+ properties = fs::set_time(&run.path(&component.location)?, intended_time)?;
+ }
+ Ok(Self {
role: component.role.clone(),
extension: names::split(&component.location.name, false).1.into(),
location: component.location.clone(),
size: component.size,
hash: component.hash.clone(),
temporary: false,
- verified_properties,
+ verified_properties: hashed.then_some(properties),
+ })
+ }
+
+ pub fn verify(&mut self, run: &Run<'_>, path: &Path) -> Result<Properties> {
+ let properties = Properties::read(path)?;
+ if self.verified_properties.as_ref() != Some(&properties) {
+ ensure!(
+ fs::matches(path, self.size, &self.hash, run.cancel)?,
+ super::Conflict(format!("at {}: locally changed media", path.display()))
+ );
+ self.verified_properties = Some(properties.clone());
}
+ Ok(properties)
}
}
@@ -126,7 +150,7 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> R
if retained_folder.as_ref() == Some(&component.location.folder) {
continue;
}
- if let Some((properties, hashed)) = verify(run, component, replacing)?
+ if let Some(verification) = verify(run, component, replacing)?
&& !source_changed(component, file, &signature)
{
if run.hardlinks
@@ -135,11 +159,13 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> R
.any(|source: &Source| source.role == component.role)
{
reusable.push(Source::existing(
+ run,
component,
- hashed.then(|| properties.clone()),
- ));
+ verification.clone(),
+ file.intended_time,
+ )?);
}
- verified.push((component.role.clone(), properties));
+ verified.push((component.role.clone(), verification.0));
}
}
needs_bytes |= roles
@@ -194,9 +220,12 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> R
continue;
}
match verify(run, &component, true) {
- Ok(Some((properties, _))) => {
- reusable.push(Source::existing(&component, Some(properties)))
- }
+ Ok(Some(verification)) => reusable.push(Source::existing(
+ run,
+ &component,
+ verification,
+ file.intended_time,
+ )?),
Err(error) if super::fatal(&error) => return Err(error),
_ => {}
}
@@ -276,7 +305,7 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> R
extension,
verified_properties: run
.hardlinks
- .then(|| Properties::read(&run.path(&temporary)?))
+ .then(|| fs::set_time(&run.path(&temporary)?, file.intended_time))
.transpose()?,
location: temporary,
size,
@@ -298,7 +327,10 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> R
vec![Source {
role: Role::Original,
extension: names::split(&file.name, false).1.into(),
- verified_properties: run.hardlinks.then(|| Properties::read(&path)).transpose()?,
+ verified_properties: run
+ .hardlinks
+ .then(|| fs::set_time(&path, file.intended_time))
+ .transpose()?,
location: temporary,
size,
hash,