π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Simplify linked export preparation and verification
Manav Rathi37a00540df9c
diff --git a/rust/apps/cli-next/tests/support/export_media.rs b/rust/apps/cli-next/tests/support/export_media.rsindex 8a46c92b71..93b3de49f5 100644--- a/rust/apps/cli-next/tests/support/export_media.rs+++ b/rust/apps/cli-next/tests/support/export_media.rs@@ -442,6 +442,30 @@ fn engine_consolidates_copies_and_keeps_linked_history() {readonly.set_readonly(true);fs::set_permissions(&first, readonly).unwrap();engine.hardlinks = true;+ #[cfg(unix)]+ {+ use std::os::unix::fs::PermissionsExt;+ let directory = root.join("Second");+ let permissions = fs::metadata(&directory).unwrap().permissions();+ fs::set_permissions(&directory, fs::Permissions::from_mode(0o500)).unwrap();+ let probe = directory.join("permission-probe");+ let result = fs::hard_link(&first, &probe)+ .is_err()+ .then(|| engine.run(false, &["First", "Second"]));+ fs::set_permissions(&directory, permissions).unwrap();+ if let Some(result) = result {+ let result = result.unwrap();+ assert!(!result.complete);+ assert_eq!(result.json["copies"]["completed"], 2);+ assert_eq!(fs::read(&second).unwrap(), b"old");+ assert!(!same_file::is_same_file(&first, &second).unwrap());+ } else {+ fs::remove_file(probe).unwrap();+ eprintln!(+ "skipping permission failure: directory permissions do not prevent hard links"+ );+ }+ }assert_eq!(run(&mut engine, true)["changes"]["exported"], 2);assert!(same_file::is_same_file(&first, &second).unwrap());assert!(!same_file::is_same_file(&first, root.join("Second/Other.jpg")).unwrap());@@ -587,50 +611,7 @@ fn engine_recovers_interrupted_link_publication() {}#[test]-#[cfg(unix)]-fn engine_link_failure_preserves_existing_copy() {- use std::os::unix::fs::PermissionsExt;- let mut server = mockito::Server::new();- let key = Key::generate();- let (record, bytes) = original_fixture(&key, b"original", false, true, 10);- let fetched = download(&mut server, 10, &bytes, 1);- let mut engine = new_engine(&server, key, record);- run(&mut engine, true);- engine.hardlinks = true;- let directory = engine.root().join("Second");- let permissions = fs::metadata(&directory).unwrap().permissions();- fs::set_permissions(&directory, fs::Permissions::from_mode(0o500)).unwrap();- let probe = directory.join("permission-probe");- if fs::hard_link(directory.join("Photo.jpg"), &probe).is_ok() {- fs::set_permissions(&directory, permissions).unwrap();- fs::remove_file(probe).unwrap();- eprintln!("skipping: directory permissions do not prevent hard links");- return;- }- let result = engine.run(false, &[]);- fs::set_permissions(&directory, permissions).unwrap();- let result = result.unwrap();- assert!(!result.complete);- assert_eq!(result.json["copies"]["completed"], 1);- assert_eq!(fs::read(directory.join("Photo.jpg")).unwrap(), b"original");- assert!(!linked(- &engine.root(),- "First/Photo.jpg",- "Second/Photo.jpg"- ));- run(&mut engine, true);- assert!(linked(- &engine.root(),- "First/Photo.jpg",- "Second/Photo.jpg"- ));- fetched.assert();-}--#[test]-#[cfg(unix)]-fn engine_rechecks_link_source_before_resetting_timestamp() {- use std::os::unix::fs::PermissionsExt;+fn engine_rechecks_link_source_before_completion() {use std::time::{Duration, UNIX_EPOCH};let mut server = mockito::Server::new();let key = Key::generate();@@ -658,20 +639,14 @@ fn engine_rechecks_link_source_before_resetting_timestamp() {let modified = UNIX_EPOCH + Duration::from_secs(1_700_000_100);engine.hardlinks = true;let export = engine.prepare(engine.options(false, &[])).unwrap();- let directory = root.join("Third");- let permissions = fs::metadata(&directory).unwrap().permissions();- fs::set_permissions(&directory, fs::Permissions::from_mode(0o500)).unwrap();- let probe = directory.join("permission-probe");- if fs::hard_link(&first, &probe).is_ok() {- fs::set_permissions(&directory, permissions).unwrap();- fs::remove_file(probe).unwrap();- eprintln!("skipping: directory permissions do not prevent hard links");- return;- }+ let blocked = root.join("Third/metadata/Photo.jpg.json");+ fs::remove_file(&blocked).unwrap();+ fs::create_dir(&blocked).unwrap();let result = engine.execute_with_report(export, &|line| {eprintln!("{line}");- if line.starts_with("file:3:10:") && line.contains("cannot link") {+ if line.starts_with("file:3:10:") && line.contains("expected a regular file") {assert!(linked(&root, "First/Photo.jpg", "Second/Photo.jpg"));+ assert_eq!(fs::metadata(&first).unwrap().modified().unwrap(), modified);fs::write(&first, b"modified").unwrap();fs::File::options().write(true)@@ -681,10 +656,9 @@ fn engine_rechecks_link_source_before_resetting_timestamp() {.unwrap();}});- fs::set_permissions(&directory, permissions).unwrap();let result = result.unwrap();assert!(!result.complete);- assert_eq!(result.json["conflicts"], 1);+ assert_eq!(result.json["conflicts"], 2);assert_eq!(result.json["copies"]["completed"], 2);assert_eq!(fs::read(&first).unwrap(), b"modified");assert_eq!(diff --git a/rust/crates/photos-export/src/reconcile.rs b/rust/crates/photos-export/src/reconcile.rsindex b788d2f23f..88160ef69f 100644--- a/rust/crates/photos-export/src/reconcile.rs+++ b/rust/crates/photos-export/src/reconcile.rs@@ -1262,20 +1262,16 @@ impl Run<'_> {}}let path = self.path(&destination)?;- let same_source = self.hardlinks+ let mut source = prepared+ .components+ .iter_mut()+ .find(|source| source.role == component.role);+ let consolidate = self.hardlinks&& valid- && same_file::is_same_file(- self.path(- &prepared- .components- .iter()- .find(|source| source.role == component.role)- .context("no original is available for linking")?- .location,- )?,+ && !same_file::is_same_file(+ self.path(&source.as_ref().context("missing original")?.location)?,&path,)?;- let consolidate = self.hardlinks && valid && !same_source;if consolidate {let previous = previous.context("missing completed original")?;ensure!(@@ -1284,11 +1280,7 @@ impl Run<'_> {);}if !valid || consolidate {- let source = prepared- .components- .iter_mut()- .find(|source| source.role == component.role)- .context("no verified original is available")?;+ let source = source.as_deref_mut().context("missing original")?;let temporary = if source.temporary {source.location.clone()} else if self.hardlinks {@@ -1298,17 +1290,7 @@ impl Run<'_> {disk::hard_link(&from, &to).with_context(|| {format!("cannot link {} to {}", from.display(), path.display())})?;- let properties = Properties::read(&to)?;- if source.verified_properties.as_ref() != Some(&properties) {- ensure!(- fs::matches(&to, source.size, &source.hash, self.cancel)?,- super::Conflict(format!(- "at {}: locally changed media",- from.display()- ))- );- source.verified_properties = Some(properties);- }+ source.verify(self, &to)?;temporary} else {let (temporary, file) = self.temporary(&album.key, album_id, Some(file.id))?;@@ -1351,32 +1333,11 @@ impl Run<'_> {|| observed != published.properties.as_ref()|| renamed{- let verified = if self.hardlinks {- let properties = Properties::read(&path)?;- let source = prepared- .components- .iter_mut()- .find(|source| source.role == component.role)- .context("no original is available for verification")?;- if source.verified_properties.as_ref().or(observed) != Some(&properties) {- ensure!(- fs::matches(&path, published.size, &published.hash, self.cancel)?,- super::Conflict(format!(- "at {}: locally changed media",- path.display()- ))- );- source.verified_properties = Some(properties);- }- source.verified_properties.as_mut()+ published.properties = Some(if self.hardlinks {+ source.context("missing original")?.verify(self, &path)?} else {- None- };- let updated = fs::set_time(&path, intended_time)?;- if let Some(verified) = verified {- verified.modified = updated.modified;- }- published.properties = Some(updated);+ fs::set_time(&path, intended_time)?+ });}published.intended_time = Some(intended_time);published.signature = Some(prepared.signature.clone());diff --git a/rust/crates/photos-export/src/transfer.rs b/rust/crates/photos-export/src/transfer.rsindex 3b687ec1f4..e8c1c62373 100644--- a/rust/crates/photos-export/src/transfer.rs+++ b/rust/crates/photos-export/src/transfer.rs@@ -1,6 +1,7 @@use std::{fs::File,io::{Seek, SeekFrom},+ path::Path,};use crate::{metadata::Role, snapshot::FileSnapshot};@@ -27,16 +28,39 @@ pub struct Source {}impl Source {- fn existing(component: &Component, verified_properties: Option<Properties>) -> Self {- Self {+ fn existing(+ run: &Run<'_>,+ component: &Component,+ (mut properties, hashed): (Properties, bool),+ intended_time: i64,+ ) -> Result<Self> {+ if run.hardlinks+ && (component.intended_time != Some(intended_time)+ || component.properties.as_ref() != Some(&properties))+ {+ properties = fs::set_time(&run.path(&component.location)?, intended_time)?;+ }+ Ok(Self {role: component.role.clone(),extension: names::split(&component.location.name, false).1.into(),location: component.location.clone(),size: component.size,hash: component.hash.clone(),temporary: false,- verified_properties,+ verified_properties: hashed.then_some(properties),+ })+ }++ pub fn verify(&mut self, run: &Run<'_>, path: &Path) -> Result<Properties> {+ let properties = Properties::read(path)?;+ if self.verified_properties.as_ref() != Some(&properties) {+ ensure!(+ fs::matches(path, self.size, &self.hash, run.cancel)?,+ super::Conflict(format!("at {}: locally changed media", path.display()))+ );+ self.verified_properties = Some(properties.clone());}+ Ok(properties)}}@@ -126,7 +150,7 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> Rif retained_folder.as_ref() == Some(&component.location.folder) {continue;}- if let Some((properties, hashed)) = verify(run, component, replacing)?+ if let Some(verification) = verify(run, component, replacing)?&& !source_changed(component, file, &signature){if run.hardlinks@@ -135,11 +159,13 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> R.any(|source: &Source| source.role == component.role){reusable.push(Source::existing(+ run,component,- hashed.then(|| properties.clone()),- ));+ verification.clone(),+ file.intended_time,+ )?);}- verified.push((component.role.clone(), properties));+ verified.push((component.role.clone(), verification.0));}}needs_bytes |= roles@@ -194,9 +220,12 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> Rcontinue;}match verify(run, &component, true) {- Ok(Some((properties, _))) => {- reusable.push(Source::existing(&component, Some(properties)))- }+ Ok(Some(verification)) => reusable.push(Source::existing(+ run,+ &component,+ verification,+ file.intended_time,+ )?),Err(error) if super::fatal(&error) => return Err(error),_ => {}}@@ -276,7 +305,7 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> Rextension,verified_properties: run.hardlinks- .then(|| Properties::read(&run.path(&temporary)?))+ .then(|| fs::set_time(&run.path(&temporary)?, file.intended_time)).transpose()?,location: temporary,size,@@ -298,7 +327,10 @@ pub async fn prepare(run: &Run<'_>, session: &Session, file: &FileSnapshot) -> Rvec![Source {role: Role::Original,extension: names::split(&file.name, false).1.into(),- verified_properties: run.hardlinks.then(|| Properties::read(&path)).transpose()?,+ verified_properties: run+ .hardlinks+ .then(|| fs::set_time(&path, file.intended_time))+ .transpose()?,location: temporary,size,hash,