π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Persist Photos metadata in an encrypted CLI replica
Manav Rathifa78665e0dc3
diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.ymlindex 4b7d842240..834e4d2995 100644--- a/.github/workflows/rust-test.yml+++ b/.github/workflows/rust-test.yml@@ -60,10 +60,15 @@ jobs:~/.cache/go-build~/.cache/.theseus/postgresql- - run: cargo test --locked --features museum,ente-ml/ml-assets+ - run: cargo test --locked --workspace --exclude ente-cli-next --features museum,ente-ml/ml-assets# postgresql_embedded downloads its binaries from GitHub.# GITHUB_TOKEN raises the API rate limit for those downloads.env:GITHUB_TOKEN: ${{ github.token }}+ # Avoid unifying SQLCipher into other packages' SQLite tests.+ - run: cargo test --locked -p ente-cli-next --features museum+ env:+ GITHUB_TOKEN: ${{ github.token }}+- run: apps/cli-next/scripts/test-with-keyring-linux.sh --features museumdiff --git a/.github/workflows/warm-caches.yml b/.github/workflows/warm-caches.ymlindex ba5a05df13..927940d91b 100644--- a/.github/workflows/warm-caches.yml+++ b/.github/workflows/warm-caches.yml@@ -71,11 +71,11 @@ jobs:~/.cache/go-build~/.cache/.theseus/postgresql- - run: cargo test --locked --features museum,ente-ml/ml-assets+ - run: cargo test --locked --workspace --exclude ente-cli-next --features museum,ente-ml/ml-assetsenv:GITHUB_TOKEN: ${{ github.token }}- - run: cargo test --locked -p ente-cli-next --test cli --features museum --no-run+ - run: cargo test --locked -p ente-cli-next --features museum --no-run- run: node .github/actions/restore-rust-target/mtimes.mjs saveworking-directory: ${{ github.workspace }}diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex dca96818ef..1903e011d8 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2453,9 +2453,11 @@ dependencies = ["keyring","mockito","reqwest 0.12.28",+ "rusqlite","serde","serde_json","tempfile",+ "thiserror 2.0.18","tokio","url","uuid",@@ -2467,6 +2469,7 @@ name = "ente-collections"version = "0.0.0"dependencies = ["ente-core",+ "serde","thiserror 2.0.18",]@@ -5689,6 +5692,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8"dependencies = ["cc",+ "openssl-sys","pkg-config","vcpkg",]@@ -6694,6 +6698,15 @@ version = "0.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"+[[package]]+name = "openssl-src"+version = "300.6.1+3.6.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846"+dependencies = [+ "cc",+]+[[package]]name = "openssl-sys"version = "0.9.116"@@ -6702,6 +6715,7 @@ checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4"dependencies = ["cc","libc",+ "openssl-src","pkg-config","vcpkg",]diff --git a/rust/apps/cli-next/Cargo.toml b/rust/apps/cli-next/Cargo.tomlindex abfd781438..3761152ba1 100644--- a/rust/apps/cli-next/Cargo.toml+++ b/rust/apps/cli-next/Cargo.toml@@ -18,14 +18,19 @@ ente-core.workspace = trueente-photos.workspace = truekeyring = { workspace = true, features = ["apple-native", "windows-native", "linux-native-sync-persistent", "crypto-rust"] }reqwest.workspace = true+rusqlite = { workspace = true, features = ["bundled-sqlcipher"] }serde.workspace = trueserde_json.workspace = truetempfile.workspace = true+thiserror.workspace = truetokio = { workspace = true, features = ["macros", "rt"] }url.workspace = trueuuid = { workspace = true, features = ["serde", "v4"] }zeroize.workspace = true+[target.'cfg(windows)'.dependencies]+rusqlite = { workspace = true, features = ["bundled-sqlcipher-vendored-openssl"] }+[dev-dependencies]ente-test-support.workspace = truemockito.workspace = truediff --git a/rust/apps/cli-next/src/args.rs b/rust/apps/cli-next/src/args.rsindex 18a72ad333..a755e6afc4 100644--- a/rust/apps/cli-next/src/args.rs+++ b/rust/apps/cli-next/src/args.rs@@ -10,12 +10,41 @@ use serde::{Deserialize, Serialize};after_help = "Start with: ente-cli-next photos login")]pub struct Cli {- #[arg(long, global = true, help = "Print the result as JSON")]- pub json: bool,+ #[command(flatten)]+ pub options: Options,#[command(subcommand)]pub command: Command,}+#[derive(Args)]+pub struct Options {+ #[arg(long, global = true, help = "Print the result as JSON")]+ pub json: bool,+ #[arg(long, global = true, help = "Use local data without network access")]+ pub offline: bool,+}++pub const DEFAULT_LIST_LIMIT: u32 = 100;++#[derive(Args)]+pub struct ListArgs {+ #[arg(+ long,+ default_value_t = DEFAULT_LIST_LIMIT,+ value_parser = clap::value_parser!(u32).range(1..),+ help = "Maximum number of results"+ )]+ pub limit: u32,+ #[arg(long, conflicts_with = "limit", help = "List every result")]+ pub all: bool,+}++impl ListArgs {+ pub fn limit(&self) -> Option<u32> {+ (!self.all).then_some(self.limit)+ }+}+#[derive(Args)]pub struct AccountSelector {#[arg(@@ -73,6 +102,12 @@ Keep the same home and key across unattended invocations."pub enum PhotosCommand {#[command(flatten)]Session(SessionCommand),+ #[command(flatten)]+ Library(PhotosLibraryCommand),+}++#[derive(Subcommand)]+pub enum PhotosLibraryCommand {#[command(about = "Manage albums")]Album {#[command(subcommand)]@@ -159,7 +194,7 @@ pub struct ApiArgs {#[derive(Subcommand)]pub enum AlbumCommand {#[command(about = "List your albums, including shared and hidden ones")]- List,+ List(ListArgs),#[command(about = "Show an album")]View {#[arg(help = "Album ID or exact name")]@@ -170,7 +205,7 @@ pub enum AlbumCommand {#[derive(Subcommand)]pub enum FileCommand {#[command(about = "List files, including shared and hidden ones")]- List,+ List(ListArgs),#[command(about = "Show a file's metadata")]View {#[arg(help = "File ID or exact name")]diff --git a/rust/apps/cli-next/src/core_db.rs b/rust/apps/cli-next/src/core_db.rsnew file mode 100644index 0000000000..acb18bfe60--- /dev/null+++ b/rust/apps/cli-next/src/core_db.rs@@ -0,0 +1,58 @@+use rusqlite::TransactionBehavior;++pub use rusqlite::{+ Connection, Error as SqliteError, OptionalExtension, Result as SqliteResult, Row, Transaction,+ params, params_from_iter, types,+};++#[derive(Debug, thiserror::Error)]+pub enum Error {+ #[error(transparent)]+ Sqlite(#[from] SqliteError),+ #[error("database version {current} is newer than this build supports ({target})")]+ Downgrade { current: i64, target: i64 },+}++pub type Result<T> = std::result::Result<T, Error>;++pub struct Db(Connection);++impl Db {+ pub fn new(connection: Connection) -> Self {+ Self(connection)+ }++ pub fn migrate(&mut self, scripts: &[&str]) -> Result<()> {+ let target = scripts.len() as i64;+ let transaction = self+ .0+ .transaction_with_behavior(TransactionBehavior::Immediate)?;+ let current: i64 =+ transaction.pragma_query_value(None, "user_version", |row| row.get(0))?;+ if current > target {+ return Err(Error::Downgrade { current, target });+ }+ if current == target {+ return Ok(());+ }+ for script in &scripts[current as usize..] {+ transaction.execute_batch(script)?;+ }+ transaction.pragma_update(None, "user_version", target)?;+ transaction.commit()?;+ Ok(())+ }++ pub fn read<T>(&self, read: impl FnOnce(&Connection) -> Result<T>) -> Result<T> {+ read(&self.0)+ }++ pub fn write<T>(&mut self, write: impl FnOnce(&Transaction<'_>) -> Result<T>) -> Result<T> {+ let transaction = self+ .0+ .transaction_with_behavior(TransactionBehavior::Immediate)?;+ let result = write(&transaction)?;+ transaction.commit()?;+ Ok(result)+ }+}diff --git a/rust/apps/cli-next/src/db.rs b/rust/apps/cli-next/src/db.rsnew file mode 100644index 0000000000..87def43cf9--- /dev/null+++ b/rust/apps/cli-next/src/db.rs@@ -0,0 +1,38 @@+use std::path::Path;++use anyhow::{Context, Result};+use rusqlite::{Connection, OpenFlags};+use zeroize::Zeroizing;++use crate::{core_db::Db, replica, vault::DbKey};++const MIGRATIONS: &[&str] = &[replica::SCHEMA];++pub fn open(account_dir: &Path, key: &DbKey, create: bool) -> Result<Db> {+ let mut flags = OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX;+ if create {+ flags |= OpenFlags::SQLITE_OPEN_CREATE;+ }+ let connection = Connection::open_with_flags(account_dir.join("data.db"), flags)?;+ connection+ .query_row("PRAGMA cipher_version", [], |row| row.get::<_, String>(0))+ .context("this CLI build does not support encrypted databases")?;+ let mut encoded_key = Zeroizing::new(String::from("x'"));+ for byte in &key.0 {+ use std::fmt::Write;+ write!(encoded_key, "{byte:02x}")?;+ }+ encoded_key.push('\'');+ connection.pragma_update(None, "key", encoded_key.as_str())?;+ connection+ .query_row("SELECT count(*) FROM sqlite_master", [], |row| {+ row.get::<_, i64>(0)+ })+ .context("cannot open the encrypted database")?;+ connection.execute_batch(+ "PRAGMA foreign_keys = ON; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = WAL;",+ )?;+ let mut db = Db::new(connection);+ db.migrate(MIGRATIONS)?;+ Ok(db)+}diff --git a/rust/apps/cli-next/src/home.rs b/rust/apps/cli-next/src/home.rsnew file mode 100644index 0000000000..b2815f9256--- /dev/null+++ b/rust/apps/cli-next/src/home.rs@@ -0,0 +1,78 @@+use std::{+ env,+ fs::{self, File, OpenOptions},+ io,+ path::{Path, PathBuf},+};++use anyhow::{Context, Result, ensure};+use uuid::Uuid;++pub struct AccountHome {+ pub path: PathBuf,+ _lock: File,+}++pub fn application_home() -> Result<PathBuf> {+ if let Some(home) = env::var_os("ENTE_CLI_HOME") {+ ensure!(!home.is_empty(), "ENTE_CLI_HOME cannot be empty");+ return Ok(home.into());+ }+ Ok(dirs::data_local_dir()+ .context("cannot find the application data directory; set ENTE_CLI_HOME")?+ .join("ente-cli"))+}++pub fn create(path: &Path) -> io::Result<()> {+ #[cfg(unix)]+ use std::os::unix::fs::{DirBuilderExt, PermissionsExt};++ let mut builder = fs::DirBuilder::new();+ builder.recursive(true);+ #[cfg(unix)]+ builder.mode(0o700);+ builder.create(path)?;+ #[cfg(unix)]+ fs::set_permissions(path, fs::Permissions::from_mode(0o700))?;+ Ok(())+}++pub fn lock_account(id: Uuid, create: bool) -> Result<AccountHome> {+ let accounts = application_home()?.join("accounts");+ let path = accounts.join(id.to_string());+ let lock_path = accounts.join(format!("{id}.lock"));+ if create {+ self::create(&accounts)?;+ } else {+ ensure!(path.is_dir(), "no local data; run the command online first");+ }+ let mut options = OpenOptions::new();+ options.read(true).write(true).create(true);+ #[cfg(unix)]+ {+ use std::os::unix::fs::OpenOptionsExt;+ options.mode(0o600);+ }+ let lock = options+ .open(lock_path)+ .context("cannot open the account lock")?;+ lock.lock().context("cannot lock the account home")?;+ Ok(AccountHome { path, _lock: lock })+}++impl AccountHome {+ pub fn remove(self) -> Result<()> {+ if let Err(error) = fs::remove_dir_all(&self.path)+ && error.kind() != io::ErrorKind::NotFound+ {+ return Err(error.into());+ }+ // The vault must exclude this account before its lock is unlinked.+ if let Err(error) = fs::remove_file(self.path.with_extension("lock"))+ && error.kind() != io::ErrorKind::NotFound+ {+ return Err(error.into());+ }+ Ok(())+ }+}diff --git a/rust/apps/cli-next/src/login.rs b/rust/apps/cli-next/src/login.rsindex a9e57dc3f5..77bd1a4618 100644--- a/rust/apps/cli-next/src/login.rs+++ b/rust/apps/cli-next/src/login.rs@@ -20,7 +20,7 @@ use crate::{api,args::{LoginArgs, Product},parse_json, read_input,- vault::{Account, AccountKeys, State, StoredSession, Vault},+ vault::{Account, AccountKeys, DbKey, State, StoredSession, Vault},};pub async fn login(@@ -143,6 +143,7 @@ pub async fn login(vault.state.check_name(&new_name)?;let index = vault.state.accounts.len();vault.state.accounts.push(Account {+ db_key: DbKey(*ente_core::crypto::Key::generate().as_bytes()),storage_id: Uuid::new_v4(),name: new_name,email,diff --git a/rust/apps/cli-next/src/main.rs b/rust/apps/cli-next/src/main.rsindex cd24731607..fb0d4645b6 100644--- a/rust/apps/cli-next/src/main.rs+++ b/rust/apps/cli-next/src/main.rs@@ -1,8 +1,12 @@mod api;mod args;+mod core_db;+mod db;+mod home;mod login;mod output;mod photos;+mod replica;mod vault;use std::{@@ -10,7 +14,7 @@ use std::{path::Path,};-use anyhow::{Context, Result, bail};+use anyhow::{Context, Result, bail, ensure};use clap::Parser;use ente_core::{b64, crypto::Key};use serde::de::DeserializeOwned;@@ -18,7 +22,8 @@ use serde_json::json;use zeroize::Zeroizing;use args::{- AccountCommand, Cli, Command, KeyCommand, PhotosCommand, Product, SessionCommand, VaultCommand,+ AccountCommand, Cli, Command, KeyCommand, Options, PhotosCommand, Product, SessionCommand,+ VaultCommand,};use output::AccountView;use vault::{State, Vault};@@ -32,33 +37,49 @@ async fn main() {}async fn run(cli: Cli) -> Result<()> {- let Cli { json, command } = cli;+ let Cli { options, command } = cli;match command {Command::Photos {selector,command: PhotosCommand::Session(command),- } => session(Product::Photos, command, selector.account.as_deref(), json).await,+ } => {+ session(+ Product::Photos,+ command,+ selector.account.as_deref(),+ &options,+ )+ .await+ }+ Command::Photos {+ selector,+ command: PhotosCommand::Library(command),+ } => photos::run(command, selector.account.as_deref(), &options).await,Command::Locker { selector, command } => {- session(Product::Locker, command, selector.account.as_deref(), json).await+ session(+ Product::Locker,+ command,+ selector.account.as_deref(),+ &options,+ )+ .await}Command::Auth { selector, command } => {- session(Product::Auth, command, selector.account.as_deref(), json).await+ session(+ Product::Auth,+ command,+ selector.account.as_deref(),+ &options,+ )+ .await}- Command::Photos {- selector,- command: PhotosCommand::Album { command },- } => photos::album(command, selector.account.as_deref(), json).await,- Command::Photos {- selector,- command: PhotosCommand::File { album, command },- } => photos::file(command, album.as_deref(), selector.account.as_deref(), json).await,- Command::Accounts { command } => account_command(command, json).await,+ Command::Accounts { command } => account_command(command, &options).await,Command::Vault {command:VaultCommand::Key {command: KeyCommand::Generate,},- } => vault_key(json),+ } => vault_key(options.json),}}@@ -75,8 +96,9 @@ async fn session(product: Product,command: SessionCommand,selected: Option<&str>,- json_output: bool,+ options: &Options,) -> Result<()> {+ ensure!(!options.offline, "this command requires network access");match command {SessionCommand::Api(args) => {let state = State::load()?;@@ -86,7 +108,7 @@ async fn session(SessionCommand::Login(args) => {let (state, index) = login::login(product, args, selected).await?;let account = AccountView::new(&state.accounts[index], state.selected);- if json_output {+ if options.json {output::json(&json!({ "account": account, "product": product }))} else {output::account(&account)@@ -95,16 +117,21 @@ async fn session(SessionCommand::Logout => {let mut vault = Vault::open()?;let index = vault.state.resolve(selected)?;+ let home = home::lock_account(vault.state.accounts[index].storage_id, true)?;let name = vault.state.accounts[index].name.clone();api::logout(&vault.state.accounts[index], product).await?;vault.state.accounts[index].sessions.remove(&product);- if vault.state.accounts[index].sessions.is_empty() {+ let removed = vault.state.accounts[index].sessions.is_empty();+ if removed {remove_account(&mut vault.state, index);}vault.save()?;+ if removed {+ home.remove()?;+ }drop(vault);output::action(- json_output,+ options.json,&json!({ "account": name, "product": product, "loggedOut": true }),&format!("Logged out of {} for {:?}.", product.display_name(), name),)@@ -112,7 +139,7 @@ async fn session(}}-async fn account_command(command: AccountCommand, json_output: bool) -> Result<()> {+async fn account_command(command: AccountCommand, options: &Options) -> Result<()> {match command {AccountCommand::List => {let state = State::load()?;@@ -121,7 +148,7 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(.iter().map(|a| AccountView::new(a, state.selected)).collect();- if json_output {+ if options.json {output::json(&accounts)} else {output::accounts(&accounts)@@ -130,7 +157,7 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(AccountCommand::View { name } => {let state = State::load()?;let account = AccountView::new(&state.accounts[state.named(&name)?], state.selected);- if json_output {+ if options.json {output::json(&account)} else {output::account(&account)@@ -145,7 +172,7 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(state.selected = selected;vault.save()?;}- output_account(vault.into_state(), index, json_output)+ output_account(vault.into_state(), index, options.json)}AccountCommand::Rename { name, new_name } => {let mut vault = Vault::open()?;@@ -156,11 +183,16 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(state.accounts[index].name = new_name;vault.save()?;}- output_account(vault.into_state(), index, json_output)+ output_account(vault.into_state(), index, options.json)}AccountCommand::Logout { name, local } => {+ ensure!(+ !options.offline || local,+ "logging out on the server requires network access; use --local to forget the account on this device"+ );let mut vault = Vault::open()?;let index = vault.state.named(&name)?;+ let home = home::lock_account(vault.state.accounts[index].storage_id, true)?;let products = vault.state.accounts[index].sessions.keys()@@ -188,6 +220,7 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(}remove_account(&mut vault.state, index);vault.save()?;+ home.remove()?;drop(vault);let message = if local && !products.is_empty() {format!(@@ -200,7 +233,7 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(format!("Logged out of {} for {name:?}.", product_names(&products))};output::action(- json_output,+ options.json,&json!({"account": name,"products": products,diff --git a/rust/apps/cli-next/src/output.rs b/rust/apps/cli-next/src/output.rsindex db4168a686..6c679ee3d0 100644--- a/rust/apps/cli-next/src/output.rs+++ b/rust/apps/cli-next/src/output.rs@@ -7,7 +7,10 @@ use ente_photos::{collections::Collection, files::File};use serde::Serialize;use uuid::Uuid;-use crate::{args::Product, vault::Account};+use crate::{+ args::{DEFAULT_LIST_LIMIT, ListArgs, Product},+ vault::Account,+};#[derive(Serialize)]pub struct AccountView<'a> {@@ -62,9 +65,9 @@ impl TryFrom<Collection> for AlbumView {#[derive(Serialize)]#[serde(rename_all = "camelCase")]-pub struct FileView<'a> {+pub struct FileView {id: String,- name: &'a str,+ name: String,#[serde(rename = "type")]kind: &'static str,owner_id: String,@@ -73,10 +76,10 @@ pub struct FileView<'a> {modified_at: String,updated_at: String,location: Option<LocationView>,- caption: Option<&'a str>,- hash: Option<&'a str>,- date_time: Option<&'a str>,- offset_time: Option<&'a str>,+ caption: Option<String>,+ hash: Option<String>,+ date_time: Option<String>,+ offset_time: Option<String>,duration_seconds: Option<u64>,width: Option<u32>,height: Option<u32>,@@ -89,11 +92,11 @@ struct LocationView {longitude: f64,}-impl<'a> FileView<'a> {- pub fn new(file: &'a File, album_ids: &[i64]) -> Result<Self> {+impl FileView {+ pub fn new(file: File, album_ids: &[i64]) -> Result<Self> {Ok(Self {id: file.id.to_string(),- name: &file.name,+ name: file.name,kind: file.kind.name(),owner_id: file.owner_id.to_string(),album_ids: album_ids.iter().map(i64::to_string).collect(),@@ -104,10 +107,10 @@ impl<'a> FileView<'a> {latitude: location.latitude,longitude: location.longitude,}),- caption: file.caption.as_deref(),- hash: file.hash.as_deref(),- date_time: file.date_time.as_deref(),- offset_time: file.offset_time.as_deref(),+ caption: file.caption,+ hash: file.hash,+ date_time: file.date_time,+ offset_time: file.offset_time,duration_seconds: file.duration_seconds,width: file.width,height: file.height,@@ -153,7 +156,8 @@ pub fn accounts(accounts: &[AccountView<'_>]) -> Result<()> {&mut stdout,["SELECTED", "NAME", "EMAIL", "HOST", "LOGGED IN", "ID"],&rows,- )+ )?;+ Ok(())}pub fn account(account: &AccountView<'_>) -> Result<()> {@@ -170,28 +174,27 @@ pub fn account(account: &AccountView<'_>) -> Result<()> {])}-pub fn albums(albums: &[AlbumView]) -> Result<()> {- let mut stdout = std::io::stdout().lock();- if albums.is_empty() {- return writeln!(stdout, "No albums.").map_err(Into::into);- }- let rows = albums- .iter()- .map(|album| {+pub fn albums(+ albums: impl Iterator<Item = Result<AlbumView>>,+ args: &ListArgs,+ as_json: bool,+) -> Result<()> {+ list(+ albums,+ args,+ as_json,+ ["ID", "NAME", "TYPE", "VISIBILITY", "OWNER", "UPDATED"],+ "No albums.",+ |album| {[- album.id.clone(),+ album.id,escape_controls(&album.name),album.kind.to_owned(),album.visibility.to_owned(),- album.owner_id.clone(),- album.updated_at.clone(),+ album.owner_id,+ album.updated_at,]- })- .collect::<Vec<_>>();- write_table(- &mut stdout,- ["ID", "NAME", "TYPE", "VISIBILITY", "OWNER", "UPDATED"],- &rows,+ },)}@@ -206,33 +209,32 @@ pub fn album(album: &AlbumView) -> Result<()> {])}-pub fn files(files: &[FileView<'_>]) -> Result<()> {- let mut stdout = std::io::stdout().lock();- if files.is_empty() {- return writeln!(stdout, "No files.").map_err(Into::into);- }- let rows = files- .iter()- .map(|file| {+pub fn files(+ files: impl Iterator<Item = Result<FileView>>,+ args: &ListArgs,+ as_json: bool,+) -> Result<()> {+ list(+ files,+ args,+ as_json,+ ["ID", "NAME", "TYPE", "CREATED", "ALBUMS"],+ "No files.",+ |file| {[- file.id.clone(),- escape_controls(file.name),+ file.id,+ escape_controls(&file.name),file.kind.to_owned(),- file.created_at.clone(),+ file.created_at,file.album_ids.join(","),]- })- .collect::<Vec<_>>();- write_table(- &mut stdout,- ["ID", "NAME", "TYPE", "CREATED", "ALBUMS"],- &rows,+ },)}-pub fn file(file: &FileView<'_>) -> Result<()> {+pub fn file(file: &FileView) -> Result<()> {let mut fields = vec![- ("Name", escape_controls(file.name)),+ ("Name", escape_controls(&file.name)),("Type", file.kind.to_owned()),("Visibility", file.visibility.to_owned()),("Created", file.created_at.clone()),@@ -243,19 +245,22 @@ pub fn file(file: &FileView<'_>) -> Result<()> {("ID", file.id.clone()),];for (label, value) in [- ("Caption", file.caption.map(escape_controls)),+ ("Caption", file.caption.as_deref().map(escape_controls)),("Location",file.location.as_ref().map(|l| format!("{}, {}", l.latitude, l.longitude)),),- ("Date/time", file.date_time.map(escape_controls)),- ("UTC offset", file.offset_time.map(escape_controls)),+ ("Date/time", file.date_time.as_deref().map(escape_controls)),+ (+ "UTC offset",+ file.offset_time.as_deref().map(escape_controls),+ ),("Duration", file.duration_seconds.map(|s| format!("{s} s"))),("Width", file.width.map(|w| w.to_string())),("Height", file.height.map(|h| h.to_string())),- ("Hash", file.hash.map(escape_controls)),+ ("Hash", file.hash.as_deref().map(escape_controls)),] {if let Some(value) = value {fields.push((label, value));@@ -301,11 +306,56 @@ fn write_fields(fields: &[(&str, String)]) -> Result<()> {Ok(())}+fn list<T: Serialize, const N: usize>(+ mut items: impl Iterator<Item = Result<T>>,+ args: &ListArgs,+ as_json: bool,+ header: [&str; N],+ empty: &str,+ row: impl Fn(T) -> [String; N],+) -> Result<()> {+ let mut stdout = std::io::stdout().lock();+ let limit = args.limit().map(|limit| limit as usize);+ let values = items.by_ref().take(limit.unwrap_or(usize::MAX));+ if as_json {+ write!(stdout, "[")?;+ for (index, value) in values.enumerate() {+ let value = value?;+ if index > 0 {+ write!(stdout, ",")?;+ }+ serde_json::to_writer(&mut stdout, &value)?;+ }+ writeln!(stdout, "]")?;+ } else {+ let mut rows = values.map(|value| value.map(&row));+ let first = rows+ .by_ref()+ .take(DEFAULT_LIST_LIMIT as usize)+ .collect::<Result<Vec<_>>>()?;+ if first.is_empty() {+ writeln!(stdout, "{empty}")?;+ } else {+ let widths = write_table(&mut stdout, header, &first)?;+ for row in rows {+ write_row(&mut stdout, &row?, &widths)?;+ }+ }+ }+ stdout.flush()?;+ if let Some(limit) = limit+ && items.next().transpose()?.is_some()+ {+ eprintln!("Showing first {limit} results; use --limit N or --all to see more.");+ }+ Ok(())+}+fn write_table<const N: usize>(out: &mut impl Write,header: [&str; N],rows: &[[String; N]],-) -> Result<()> {+) -> Result<[usize; N]> {let header = header.map(str::to_owned);let widths: [usize; N] = std::array::from_fn(|column| {std::iter::once(&header)@@ -315,22 +365,31 @@ fn write_table<const N: usize>(.unwrap_or(0)});for row in std::iter::once(&header).chain(rows) {- for (column, cell) in row.iter().enumerate() {- if column > 0 {- write!(out, " ")?;- }- if column + 1 == N {- write!(out, "{cell}")?;- } else {- write!(- out,- "{}",- pad_str(cell, widths[column], Alignment::Left, None)- )?;- }+ write_row(out, row, &widths)?;+ }+ Ok(widths)+}++fn write_row<const N: usize>(+ out: &mut impl Write,+ row: &[String; N],+ widths: &[usize; N],+) -> Result<()> {+ for (column, cell) in row.iter().enumerate() {+ if column > 0 {+ write!(out, " ")?;+ }+ if column + 1 == N {+ write!(out, "{cell}")?;+ } else {+ write!(+ out,+ "{}",+ pad_str(cell, widths[column], Alignment::Left, None)+ )?;}- writeln!(out)?;}+ writeln!(out)?;Ok(())}diff --git a/rust/apps/cli-next/src/photos.rs b/rust/apps/cli-next/src/photos.rsindex 4d181459f6..6aa4e19a37 100644--- a/rust/apps/cli-next/src/photos.rs+++ b/rust/apps/cli-next/src/photos.rs@@ -1,37 +1,79 @@-use std::{collections::BTreeMap, path::Path};+use std::{io::Seek, path::Path};-use anyhow::{Context, Result, bail};-use ente_photos::{collections, files};+use anyhow::{Context, Result, bail, ensure};+use ente_core::Session;+use ente_photos::files;use serde_json::json;use crate::{api,- args::{AlbumCommand, FileCommand, Product},+ args::{AlbumCommand, FileCommand, Options, PhotosLibraryCommand, Product},+ db, home,output::{self, AlbumView, FileView},- vault::State,+ replica::Replica,+ vault::{Account, State},};-pub async fn album(command: AlbumCommand, selected: Option<&str>, json_output: bool) -> Result<()> {- let state = State::load()?;- let account = &state.accounts[state.resolve(selected)?];- let session = api::session(account, Product::Photos)?;- let albums = collections::list(&session).await?;+const MAX_CANDIDATES: usize = 10;++pub async fn run(+ command: PhotosLibraryCommand,+ selected: Option<&str>,+ options: &Options,+) -> Result<()> {+ ensure!(+ !options.offline+ || !matches!(+ &command,+ PhotosLibraryCommand::File {+ command: FileCommand::Download { .. },+ ..+ }+ ),+ "original files are not stored locally; download requires network access"+ );+ let (account, home) = open_account(selected, !options.offline)?;+ let session = api::session(&account, Product::Photos)?;+ let mut db = db::open(&home.path, &account.db_key, !options.offline)?;+ let mut replica = Replica::new(&mut db);+ if !options.offline {+ replica.sync_collections(&session).await?;+ } else {+ ensure!(+ replica.collections_cursor()?.is_some(),+ "no local Photos data; run the command online first"+ );+ }match command {- AlbumCommand::List => {- let albums = albums- .into_iter()- .map(AlbumView::try_from)- .collect::<Result<Vec<_>>>()?;- if json_output {- output::json(&albums)- } else {- output::albums(&albums)- }+ PhotosLibraryCommand::Album { command } => album(command, &replica, options),+ PhotosLibraryCommand::File { album, command } => {+ file(command, album.as_deref(), &session, &mut replica, options).await}+ }+}++fn album(command: AlbumCommand, replica: &Replica<'_>, options: &Options) -> Result<()> {+ match command {+ AlbumCommand::List(args) => replica.collections(+ None,+ args.limit().map(|limit| i64::from(limit) + 1),+ |rows| {+ output::albums(+ rows.map(|album| AlbumView::try_from(album?)),+ &args,+ options.json,+ )+ },+ ),AlbumCommand::View { album } => {- let album = select(albums, &album, "album", |a| (a.id, &a.name))?;+ let album =+ replica.collections(Some(&album), Some((MAX_CANDIDATES + 1) as i64), |rows| {+ select(rows.map(|album| Ok(album?)), &album, "album", |a| {+ (a.id, &a.name)+ })+ })?;let album = AlbumView::try_from(album)?;- if json_output {+ if options.json {output::json(&album)} else {output::album(&album)@@ -40,72 +82,89 @@ pub async fn album(command: AlbumCommand, selected: Option<&str>, json_output: b}}-pub async fn file(+async fn file(command: FileCommand,album: Option<&str>,- selected: Option<&str>,- json_output: bool,+ session: &Session,+ replica: &mut Replica<'_>,+ options: &Options,) -> Result<()> {- let state = State::load()?;- let account = &state.accounts[state.resolve(selected)?];- let session = api::session(account, Product::Photos)?;- let mut albums = collections::list(&session).await?;- if let Some(selector) = album {- albums = vec![select(albums, selector, "album", |a| (a.id, &a.name))?];- }- let mut entries: BTreeMap<i64, Entry> = BTreeMap::new();- for album in albums {- for file in files::list(&session, &album).await? {- match entries.entry(file.id) {- std::collections::btree_map::Entry::Vacant(entry) => {- entry.insert(Entry {- file,- album_ids: vec![album.id],- });- }- std::collections::btree_map::Entry::Occupied(mut entry) => {- let entry = entry.get_mut();- entry.album_ids.push(album.id);- if file.updated_at_micros > entry.file.updated_at_micros {- entry.file = file;- }- }+ let albums =+ replica.collections(album, album.map(|_| (MAX_CANDIDATES + 1) as i64), |rows| {+ if let Some(selector) = album {+ Ok(vec![select(+ rows.map(|album| Ok(album?)),+ selector,+ "album",+ |a| (a.id, &a.name),+ )?])+ } else {+ rows.map(|album| Ok(album?)).collect::<Result<Vec<_>>>()}+ })?;+ if options.offline {+ for album in &albums {+ ensure!(+ replica.files_synced_to(album.id)?.is_some(),+ "files for album {:?} have not finished syncing; run the command online first",+ album.name+ );}+ } else {+ replica.sync_files(session, &albums).await?;}- let mut entries = entries.into_values().collect::<Vec<_>>();- for entry in &mut entries {- entry.album_ids.sort_unstable();- }+ let album_id = album.map(|_| albums[0].id);+ let find = |selector: &str| {+ replica.files(+ album_id,+ Some(selector),+ Some((MAX_CANDIDATES + 1) as i64),+ |rows| {+ select(rows.map(|entry| Ok(entry?)), selector, "file", |e| {+ (e.file.id, &e.file.name)+ })+ },+ )+ };match command {- FileCommand::List => {- let views = entries- .iter()- .map(|entry| FileView::new(&entry.file, &entry.album_ids))- .collect::<Result<Vec<_>>>()?;- if json_output {- output::json(&views)- } else {- output::files(&views)- }- }+ FileCommand::List(args) => replica.files(+ album_id,+ None,+ args.limit().map(|limit| i64::from(limit) + 1),+ |rows| {+ output::files(+ rows.map(|entry| {+ let entry = entry?;+ FileView::new(entry.file, &entry.album_ids)+ }),+ &args,+ options.json,+ )+ },+ ),FileCommand::View { file } => {- let entry = select(entries, &file, "file", |e| (e.file.id, &e.file.name))?;- let view = FileView::new(&entry.file, &entry.album_ids)?;- if json_output {+ let entry = find(&file)?;+ let view = FileView::new(entry.file, &entry.album_ids)?;+ if options.json {output::json(&view)} else {output::file(&view)}}FileCommand::Download { file, output } => {- let entry = select(entries, &file, "file", |e| (e.file.id, &e.file.name))?;+ let entry = find(&file)?;let parent = output.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new("."));let mut temporary = tempfile::NamedTempFile::new_in(parent)?;- files::download(&session, &entry.file, temporary.as_file_mut()).await?;+ files::download(session, &entry.file, || {+ let file = temporary.as_file_mut();+ file.set_len(0)?;+ file.rewind()?;+ file.try_clone()+ })+ .await?;temporary.as_file().sync_all()?;let bytes = temporary.as_file().metadata()?.len();temporary@@ -113,7 +172,7 @@ pub async fn file(.map_err(|error| error.error).with_context(|| format!("cannot save download to {}", output.display()))?;output::action(- json_output,+ options.json,&json!({ "id": entry.file.id.to_string(), "output": output, "bytes": bytes }),&format!("Downloaded {:?} to {}.", entry.file.name, output.display()),)@@ -121,34 +180,47 @@ pub async fn file(}}-struct Entry {- file: files::File,- album_ids: Vec<i64>,+fn open_account(selected: Option<&str>, create: bool) -> Result<(Account, home::AccountHome)> {+ let state = State::load()?;+ let id = state.accounts[state.resolve(selected)?].storage_id;+ let home = home::lock_account(id, create)?;+ let Some(account) = State::load()?+ .accounts+ .into_iter()+ .find(|account| account.storage_id == id)+ else {+ home.remove()?;+ bail!("account was removed while waiting for access");+ };+ if create {+ home::create(&home.path)?;+ }+ Ok((account, home))}fn select<T>(- items: Vec<T>,+ items: impl Iterator<Item = Result<T>>,selector: &str,kind: &str,identity: impl Fn(&T) -> (i64, &str),) -> Result<T> {- let mut matches = items- .into_iter()- .filter(|item| {- let (id, name) = identity(item);- id.to_string() == selector || name == selector- })- .collect::<Vec<_>>();+ let mut matches = items.take(MAX_CANDIDATES + 1).collect::<Result<Vec<_>>>()?;if matches.len() > 1 {let candidates = matches.iter()+ .take(MAX_CANDIDATES).map(|item| {let (id, name) = identity(item);format!(" {id} {name:?}")}).collect::<Vec<_>>().join("\n");- bail!("{kind} {selector:?} is ambiguous:\n{candidates}");+ let more = if matches.len() > MAX_CANDIDATES {+ "\nMore matches omitted; use an ID."+ } else {+ ""+ };+ bail!("{kind} {selector:?} is ambiguous:\n{candidates}{more}");}matches.pop()diff --git a/rust/apps/cli-next/src/replica.rs b/rust/apps/cli-next/src/replica.rsnew file mode 100644index 0000000000..3804524d62--- /dev/null+++ b/rust/apps/cli-next/src/replica.rs@@ -0,0 +1,364 @@+use anyhow::Result;+use ente_core::{+ Session,+ crypto::{Header, Key},+};++use ente_photos::{+ collections::{self, Collection, Visibility},+ files::{self, File, Location},+};++use crate::core_db::{+ self, Connection, Db, OptionalExtension, Row, SqliteError, SqliteResult, params,+ params_from_iter,+ types::{Type, Value},+};++pub struct Entry {+ pub file: File,+ pub album_ids: Vec<i64>,+}++pub struct Replica<'a> {+ db: &'a mut Db,+}++const COLLECTION_COLUMNS: &str = "id, name, kind, visibility, owner_id, updated_at, key";+const FILE_COLUMNS: &str = "f.id, f.owner_id, f.updated_at, f.name, f.kind, f.created_at, f.modified_at,+ f.latitude, f.longitude, f.caption, f.hash, f.date_time, f.offset_time, f.duration, f.width, f.height,+ f.visibility, f.key, f.header";++pub const SCHEMA: &str = "+ CREATE TABLE photos_sync (+ id INTEGER PRIMARY KEY CHECK (id = 1),+ classical_collections_cursor INTEGER NOT NULL+ );+ CREATE TABLE photos_collections (+ id INTEGER PRIMARY KEY, name TEXT NOT NULL, kind TEXT NOT NULL,+ visibility TEXT NOT NULL, owner_id INTEGER NOT NULL, updated_at INTEGER NOT NULL,+ key BLOB NOT NULL, files_cursor INTEGER NOT NULL DEFAULT 0, files_synced_to INTEGER+ );+ CREATE INDEX photos_collections_name ON photos_collections(name);+ CREATE TABLE photos_files (+ id INTEGER PRIMARY KEY, owner_id INTEGER NOT NULL, updated_at INTEGER NOT NULL,+ name TEXT NOT NULL, kind TEXT NOT NULL, created_at INTEGER NOT NULL, modified_at INTEGER NOT NULL,+ latitude REAL, longitude REAL, caption TEXT, hash TEXT, date_time TEXT, offset_time TEXT,+ duration INTEGER, width INTEGER, height INTEGER, visibility TEXT NOT NULL, key BLOB NOT NULL, header BLOB NOT NULL+ );+ CREATE INDEX photos_files_name ON photos_files(name);+ CREATE TABLE photos_memberships (+ collection_id INTEGER NOT NULL REFERENCES photos_collections(id) ON DELETE CASCADE,+ file_id INTEGER NOT NULL REFERENCES photos_files(id) ON DELETE CASCADE,+ PRIMARY KEY(collection_id, file_id)+ );+ CREATE INDEX photos_memberships_file ON photos_memberships(file_id);+";++impl<'a> Replica<'a> {+ pub fn new(db: &'a mut Db) -> Self {+ Self { db }+ }++ pub async fn sync_collections(&mut self, session: &Session) -> Result<()> {+ let since = self.collections_cursor()?.unwrap_or(0);+ let page = collections::diff(session, since).await?;+ self.db.write(|tx| {+ for change in page.changes {+ match change.collection {+ Some(c) => {+ tx.execute("+ INSERT INTO photos_collections (id, name, kind, visibility, owner_id, updated_at, key)+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)+ ON CONFLICT(id) DO UPDATE SET name=excluded.name, kind=excluded.kind,+ visibility=excluded.visibility, owner_id=excluded.owner_id,+ updated_at=excluded.updated_at, key=excluded.key+ ", params![c.id, c.name, c.kind.name(), c.visibility.name(), c.owner_id, c.updated_at_micros, c.key.as_bytes()])?;+ }+ None => {+ tx.execute(+ "DELETE FROM photos_files WHERE id IN+ (SELECT file_id FROM photos_memberships WHERE collection_id=?1)+ AND NOT EXISTS (SELECT 1 FROM photos_memberships+ WHERE file_id=photos_files.id AND collection_id!=?1)",+ [change.id],+ )?;+ tx.execute("DELETE FROM photos_collections WHERE id=?1", [change.id])?;+ }+ }+ }+ tx.execute("INSERT INTO photos_sync VALUES (1, ?1)+ ON CONFLICT(id) DO UPDATE SET classical_collections_cursor=excluded.classical_collections_cursor", [page.cursor])?;+ Ok(())+ })?;+ Ok(())+ }++ pub fn collections<T>(+ &self,+ selector: Option<&str>,+ limit: Option<i64>,+ read: impl FnOnce(&mut dyn Iterator<Item = SqliteResult<Collection>>) -> Result<T>,+ ) -> Result<T> {+ let mut sql = format!("SELECT {COLLECTION_COLUMNS} FROM photos_collections");+ let mut parameters = Vec::new();+ if let Some(selector) = selector {+ sql.push_str(" WHERE id=? OR name=?");+ parameters.push(canonical_id(selector));+ parameters.push(Value::Text(selector.to_owned()));+ }+ sql.push_str(" ORDER BY id");+ if let Some(limit) = limit {+ sql.push_str(" LIMIT ?");+ parameters.push(Value::Integer(limit));+ }+ self.db.read(|conn| {+ let mut statement = conn.prepare(&sql)?;+ let mut rows = statement.query_map(params_from_iter(parameters), read_collection)?;+ Ok(read(&mut rows))+ })?+ }++ pub async fn sync_files(+ &mut self,+ session: &Session,+ collections: &[Collection],+ ) -> Result<()> {+ for collection in collections {+ let (mut cursor, synced_to): (i64, Option<i64>) = self.db.read(|conn| {+ Ok(conn.query_row(+ "SELECT files_cursor, files_synced_to FROM photos_collections WHERE id=?1",+ [collection.id],+ |row| Ok((row.get(0)?, row.get(1)?)),+ )?)+ })?;+ // Museum advances the collection timestamp on file and membership changes.+ if synced_to.is_some_and(|time| time >= collection.updated_at_micros) {+ continue;+ }+ loop {+ let page = files::diff(session, collection, cursor).await?;+ self.db.write(|tx| {+ for change in page.changes {+ match change.file {+ Some(file) => {+ save_file(tx, &file)?;+ tx.execute(+ "INSERT OR IGNORE INTO photos_memberships VALUES (?1, ?2)",+ params![collection.id, file.id],+ )?;+ }+ None => {+ tx.execute("DELETE FROM photos_memberships WHERE collection_id=?1 AND file_id=?2",+ params![collection.id, change.id])?;+ tx.execute(+ "DELETE FROM photos_files WHERE id=?1 AND NOT EXISTS+ (SELECT 1 FROM photos_memberships WHERE file_id=?1)",+ [change.id],+ )?;+ }+ }+ }+ tx.execute(+ "UPDATE photos_collections SET files_cursor=?1,+ files_synced_to=CASE WHEN ?2 THEN files_synced_to ELSE ?3 END WHERE id=?4",+ params![+ page.cursor,+ page.has_more,+ collection.updated_at_micros,+ collection.id+ ],+ )?;+ Ok(())+ })?;+ cursor = page.cursor;+ if !page.has_more {+ break;+ }+ }+ }+ Ok(())+ }++ pub fn files_synced_to(&self, collection_id: i64) -> core_db::Result<Option<i64>> {+ self.db.read(|conn| {+ Ok(conn.query_row(+ "SELECT files_synced_to FROM photos_collections WHERE id=?1",+ [collection_id],+ |row| row.get(0),+ )?)+ })+ }++ pub fn files<T>(+ &self,+ album: Option<i64>,+ selector: Option<&str>,+ limit: Option<i64>,+ read: impl FnOnce(&mut dyn Iterator<Item = SqliteResult<Entry>>) -> Result<T>,+ ) -> Result<T> {+ let album_ids = if album.is_some() {+ "json_array(m.collection_id)"+ } else {+ "json_group_array(m.collection_id ORDER BY m.collection_id)"+ };+ let mut sql = format!(+ "SELECT {FILE_COLUMNS}, {album_ids}+ FROM photos_files f JOIN photos_memberships m ON m.file_id=f.id"+ );+ let mut parameters = Vec::new();+ let mut conditions = Vec::new();+ if let Some(selector) = selector {+ conditions.push("(f.id=? OR f.name=?)");+ parameters.push(canonical_id(selector));+ parameters.push(Value::Text(selector.to_owned()));+ }+ if let Some(album) = album {+ conditions.push("m.collection_id=?");+ parameters.push(Value::Integer(album));+ }+ if !conditions.is_empty() {+ sql.push_str(" WHERE ");+ sql.push_str(&conditions.join(" AND "));+ }+ if album.is_some() {+ sql.push_str(" ORDER BY m.file_id");+ } else {+ sql.push_str(" GROUP BY f.id ORDER BY f.id");+ }+ if let Some(limit) = limit {+ sql.push_str(" LIMIT ?");+ parameters.push(Value::Integer(limit));+ }+ self.db.read(|conn| {+ let mut statement = conn.prepare(&sql)?;+ let mut rows = statement.query_map(params_from_iter(parameters), |row| {+ let ids: String = row.get(19)?;+ let album_ids = serde_json::from_str(&ids).map_err(|error| {+ SqliteError::FromSqlConversionFailure(19, Type::Text, Box::new(error))+ })?;+ Ok(Entry {+ file: read_file(row)?,+ album_ids,+ })+ })?;+ Ok(read(&mut rows))+ })?+ }++ pub fn collections_cursor(&self) -> core_db::Result<Option<i64>> {+ self.db.read(|conn| {+ Ok(conn+ .query_row(+ "SELECT classical_collections_cursor FROM photos_sync WHERE id=1",+ [],+ |row| row.get(0),+ )+ .optional()?)+ })+ }+}++fn canonical_id(selector: &str) -> Value {+ selector+ .parse::<i64>()+ .ok()+ .filter(|id| id.to_string() == selector)+ .map(Value::Integer)+ .unwrap_or(Value::Null)+}++fn read_collection(row: &Row<'_>) -> SqliteResult<Collection> {+ let key: Vec<u8> = row.get(6)?;+ Ok(Collection {+ id: row.get(0)?,+ name: row.get(1)?,+ kind: match row.get::<_, String>(2)?.as_str() {+ "favorites" => collections::Kind::Favorites,+ "uncategorized" => collections::Kind::Uncategorized,+ _ => collections::Kind::Album,+ },+ visibility: visibility(&row.get::<_, String>(3)?),+ owner_id: row.get(4)?,+ updated_at_micros: row.get(5)?,+ key: Key::try_from_slice(&key).map_err(|error| {+ SqliteError::FromSqlConversionFailure(6, Type::Blob, Box::new(error))+ })?,+ })+}++fn visibility(name: &str) -> Visibility {+ match name {+ "archived" => Visibility::Archived,+ "hidden" => Visibility::Hidden,+ _ => Visibility::Visible,+ }+}++fn read_file(row: &Row<'_>) -> SqliteResult<File> {+ let key: Vec<u8> = row.get(17)?;+ let header: Vec<u8> = row.get(18)?;+ let latitude: Option<f64> = row.get(7)?;+ let longitude: Option<f64> = row.get(8)?;+ Ok(File {+ id: row.get(0)?,+ owner_id: row.get(1)?,+ updated_at_micros: row.get(2)?,+ name: row.get(3)?,+ kind: match row.get::<_, String>(4)?.as_str() {+ "image" => files::Kind::Image,+ "video" => files::Kind::Video,+ "livephoto" => files::Kind::LivePhoto,+ _ => files::Kind::Unknown,+ },+ created_at_micros: row.get(5)?,+ modified_at_micros: row.get(6)?,+ location: latitude+ .zip(longitude)+ .map(|(latitude, longitude)| Location {+ latitude,+ longitude,+ }),+ caption: row.get(9)?,+ hash: row.get(10)?,+ date_time: row.get(11)?,+ offset_time: row.get(12)?,+ duration_seconds: row+ .get::<_, Option<i64>>(13)?+ .map(u64::try_from)+ .transpose()+ .map_err(|error| {+ SqliteError::FromSqlConversionFailure(13, Type::Integer, Box::new(error))+ })?,+ width: row.get(14)?,+ height: row.get(15)?,+ visibility: visibility(&row.get::<_, String>(16)?),+ key: Key::try_from_slice(&key).map_err(|error| {+ SqliteError::FromSqlConversionFailure(17, Type::Blob, Box::new(error))+ })?,+ header: Header::try_from_slice(&header).map_err(|error| {+ SqliteError::FromSqlConversionFailure(18, Type::Blob, Box::new(error))+ })?,+ })+}++fn save_file(connection: &Connection, file: &File) -> SqliteResult<()> {+ let duration = file+ .duration_seconds+ .map(i64::try_from)+ .transpose()+ .map_err(|error| SqliteError::ToSqlConversionFailure(Box::new(error)))?;+ connection.execute("INSERT INTO photos_files VALUES+ (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19)+ ON CONFLICT(id) DO UPDATE SET owner_id=excluded.owner_id, updated_at=excluded.updated_at,+ name=excluded.name, kind=excluded.kind, created_at=excluded.created_at, modified_at=excluded.modified_at,+ latitude=excluded.latitude, longitude=excluded.longitude, caption=excluded.caption, hash=excluded.hash,+ date_time=excluded.date_time, offset_time=excluded.offset_time, duration=excluded.duration,+ width=excluded.width, height=excluded.height, visibility=excluded.visibility, key=excluded.key, header=excluded.header+ WHERE excluded.updated_at >= photos_files.updated_at",+ params![file.id, file.owner_id, file.updated_at_micros, file.name, file.kind.name(), file.created_at_micros,+ file.modified_at_micros, file.location.as_ref().map(|l| l.latitude), file.location.as_ref().map(|l| l.longitude),+ file.caption, file.hash, file.date_time, file.offset_time, duration, file.width, file.height,+ file.visibility.name(), file.key.as_bytes(), file.header.as_bytes()])?;+ Ok(())+}diff --git a/rust/apps/cli-next/src/vault.rs b/rust/apps/cli-next/src/vault.rsindex 843744f233..fcd3ae2c8e 100644--- a/rust/apps/cli-next/src/vault.rs+++ b/rust/apps/cli-next/src/vault.rs@@ -2,7 +2,7 @@ use std::{collections::BTreeMap,env,fs::{self, File, OpenOptions},- io::{self, ErrorKind},+ io::ErrorKind,path::{Path, PathBuf},};@@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize};use uuid::Uuid;use zeroize::{ZeroizeOnDrop, Zeroizing};-use crate::{args::Product, parse_json};+use crate::{args::Product, home, parse_json};const SCHEMA_VERSION: u8 = 1;const VAULT_FILE: &str = "vault.json";@@ -53,6 +53,7 @@ struct EncryptedVault {#[derive(Serialize, Deserialize)]pub struct Account {pub storage_id: Uuid,+ pub db_key: DbKey,pub name: String,pub email: String,pub origin: String,@@ -61,6 +62,9 @@ pub struct Account {pub sessions: BTreeMap<Product, StoredSession>,}+#[derive(Serialize, Deserialize, ZeroizeOnDrop)]+pub struct DbKey(pub [u8; Key::BYTES]);+#[derive(Serialize, Deserialize, ZeroizeOnDrop)]pub struct AccountKeys {pub master_key: Vec<u8>,@@ -76,8 +80,8 @@ pub struct StoredSession {impl Vault {pub fn open() -> Result<Self> {let override_key = environment_key()?;- let home = application_home()?;- create_home(&home).context("cannot create private CLI home")?;+ let home = home::application_home()?;+ home::create(&home).context("cannot create private CLI home")?;let home = fs::canonicalize(home)?;let lock = lock(&home)?;let encrypted = read_vault(&home)?;@@ -155,7 +159,7 @@ impl VaultAccess {impl State {pub fn load() -> Result<Self> {let override_key = environment_key()?;- let home = application_home()?;+ let home = home::application_home()?;// Atomic replacement makes the file a complete snapshot without a read lock.let Some(bytes) = read_vault(&home)? else {return Ok(Self::default());@@ -227,20 +231,6 @@ fn read_vault(home: &Path) -> Result<Option<Vec<u8>>> {}}-fn create_home(path: &Path) -> io::Result<()> {- #[cfg(unix)]- use std::os::unix::fs::{DirBuilderExt, PermissionsExt};-- let mut builder = fs::DirBuilder::new();- builder.recursive(true);- #[cfg(unix)]- builder.mode(0o700);- builder.create(path)?;- #[cfg(unix)]- fs::set_permissions(path, fs::Permissions::from_mode(0o700))?;- Ok(())-}-fn lock(home: &Path) -> Result<File> {let mut options = OpenOptions::new();options.read(true).write(true).create(true);@@ -254,16 +244,6 @@ fn lock(home: &Path) -> Result<File> {Ok(lock)}-fn application_home() -> Result<PathBuf> {- if let Some(home) = env::var_os("ENTE_CLI_HOME") {- ensure!(!home.is_empty(), "ENTE_CLI_HOME cannot be empty");- return Ok(home.into());- }- Ok(dirs::data_local_dir()- .context("cannot find the application data directory; set ENTE_CLI_HOME")?- .join("ente-cli"))-}-fn environment_key() -> Result<Option<Key>> {match env::var("ENTE_CLI_VAULT_KEY") {Ok(value) => decode_key(&Zeroizing::new(value)).map(Some),diff --git a/rust/apps/cli-next/tests/cli.rs b/rust/apps/cli-next/tests/cli.rsindex 4e008bc530..3d28cb0e33 100644--- a/rust/apps/cli-next/tests/cli.rs+++ b/rust/apps/cli-next/tests/cli.rs@@ -105,7 +105,13 @@ fn deleted_collections_are_filtered_before_decryption() {}]}).to_string(),)- .expect(3)+ .expect(1)+ .create();+ let subsequent = server+ .mock("GET", "/collections/v2")+ .match_query(mockito::Matcher::UrlEncoded("sinceTime".into(), "1".into()))+ .with_body(json!({"collections": []}).to_string())+ .expect(2).create();let home = TestHome::new();home.seed(&server.url());@@ -116,6 +122,7 @@ fn deleted_collections_are_filtered_before_decryption() {);assert!(failure(&home.run(&["photos", "album", "view", "12"])).contains("no album matches"));request.assert();+ subsequent.assert();}#[cfg(unix)]@@ -413,6 +420,12 @@ fn logout_reconciles_remote_and_local_session_state() {);assert_eq!(home.json(&["account", "list"]), json!([]));assert_eq!(home.read_vault(), json!({"accounts": [], "selected": null}));+ assert_eq!(+ fs::read_dir(home.dir.path().join("accounts"))+ .unwrap()+ .count(),+ 0+ );revoked.assert();let unavailable = server@@ -667,7 +680,7 @@ impl TestHome {self.write_vault(&json!({"selected": id,"accounts": [{- "storage_id": id, "name": "fixture", "email": "[email protected]",+ "storage_id": id, "db_key": vec![42u8; 32], "name": "fixture", "email": "[email protected]","origin": origin, "user_id": 9007199254740993i64,"identity": {"master_key": vec![0u8; 32], "recovery_key": vec![0u8; 32],diff --git a/rust/apps/cli-next/tests/support/files.rs b/rust/apps/cli-next/tests/support/files.rsindex a23a30b5b4..229c972c01 100644--- a/rust/apps/cli-next/tests/support/files.rs+++ b/rust/apps/cli-next/tests/support/files.rs@@ -1,4 +1,8 @@use ente_core::crypto::{secretbox, stream};+use std::sync::{+ Arc,+ atomic::{AtomicUsize, Ordering},+};use super::*;@@ -27,9 +31,9 @@ fn file_pages_preserve_membership_changes_and_metadata() {removed["updationTime"] = json!(20);removed["file"]["encryptedData"] = json!("-");removed["metadata"]["encryptedData"] = json!("-");- let page_one = page(&mut server, 1, 0, json!([first, second]), true);- let page_two = page(&mut server, 1, 10, json!([removed]), false);- let duplicate = page(&mut server, 2, 0, json!([first]), false);+ let page_one = page(&mut server, 1, 0, json!([first, second]), true).create();+ let page_two = page(&mut server, 1, 10, json!([removed]), false).create();+ let duplicate = page(&mut server, 2, 0, json!([first]), false).create();let listed = home.json(&["photos", "file", "list"]);assert_eq!(listed.as_array().unwrap().len(), 1);let file = &listed[0];@@ -83,7 +87,7 @@ fn downloads_publish_only_complete_authenticated_originals() {.create();let original = vec![37u8; stream::ENCRYPTION_CHUNK_SIZE + 321];let (file, encrypted) = remote_file(10, &collection_key, &original);- page(&mut server, 1, 0, json!([file]), false);+ page(&mut server, 1, 0, json!([file]), false).create();let signed = server.mock("GET", "/files/download/v3/10").match_header("x-auth-token", "--8=")@@ -117,11 +121,19 @@ fn downloads_publish_only_complete_authenticated_originals() {object.assert();object.remove();}+ let attempts = Arc::new(AtomicUsize::new(0));+ let count = Arc::clone(&attempts);let object = server.mock("GET", "/original").match_header("x-auth-token", mockito::Matcher::Missing).match_header("x-client-package", mockito::Matcher::Missing)- .with_body(encrypted)+ .with_chunked_body(move |writer| {+ if count.fetch_add(1, Ordering::SeqCst) == 0 {+ writer.write_all(&encrypted[..stream::DECRYPTION_CHUNK_SIZE + 1])?;+ return Err(std::io::Error::from(std::io::ErrorKind::ConnectionReset));+ }+ writer.write_all(&encrypted)+ }).expect_at_least(1).create();let result: Value = serde_json::from_slice(&success(home.run(&args)).stdout).unwrap();@@ -130,6 +142,7 @@ fn downloads_publish_only_complete_authenticated_originals() {json!({"id": "10", "output": path, "bytes": original.len()}));assert_eq!(fs::read(&path).unwrap(), original);+ assert_eq!(attempts.load(Ordering::SeqCst), 2);fs::write(&path, b"existing user file").unwrap();failure(&home.run(&args));assert_eq!(fs::read(&path).unwrap(), b"existing user file");@@ -138,6 +151,402 @@ fn downloads_publish_only_complete_authenticated_originals() {object.assert();}+#[test]+fn replica_resumes_failed_pages_and_only_refreshes_changed_albums() {+ let mut server = mockito::Server::new();+ let home = TestHome::new();+ home.seed(&server.url());+ let key = Key::generate();+ let initial = server+ .mock("GET", "/collections/v2")+ .match_query(mockito::Matcher::UrlEncoded("sinceTime".into(), "0".into()))+ .with_body(+ json!({"collections": [collection(1, "First", &key), collection(2, "Second", &key)]})+ .to_string(),+ )+ .expect(1)+ .create();+ let unchanged = server+ .mock("GET", "/collections/v2")+ .match_query(mockito::Matcher::UrlEncoded(+ "sinceTime".into(),+ "20".into(),+ ))+ .with_body(json!({"collections": []}).to_string())+ .expect(2)+ .create();+ let (first, _) = remote_file(10, &key, b"first");+ let (second, _) = remote_file(11, &key, b"second");+ let first_page = page(&mut server, 1, 0, json!([first]), true)+ .expect(1)+ .create();+ let broken_page = server+ .mock("GET", "/collections/v2/diff")+ .match_query(mockito::Matcher::AllOf(vec![+ mockito::Matcher::UrlEncoded("collectionID".into(), "1".into()),+ mockito::Matcher::UrlEncoded("sinceTime".into(), "10".into()),+ ]))+ .with_status(400)+ .expect(1)+ .create();+ failure(&home.run(&["photos", "file", "list"]));+ assert!(+ failure(&home.run(&["--offline", "photos", "file", "list", "--album", "First"]))+ .contains("have not finished syncing")+ );+ first_page.assert();+ broken_page.assert();+ broken_page.remove();+ let mut second = second;+ second["updationTime"] = json!(20);+ let final_page = page(&mut server, 1, 10, json!([second]), false)+ .expect(1)+ .create();+ let other_album = page(&mut server, 2, 0, json!([first]), false)+ .expect(1)+ .create();+ let files = home.json(&["photos", "file", "list"]);+ assert_eq!(files.as_array().unwrap().len(), 2);+ assert_eq!(files[0]["albumIds"], json!(["1", "2"]));+ assert_eq!(home.json(&["photos", "file", "list"]), files);+ assert_eq!(home.json(&["--offline", "photos", "file", "list"]), files);+ initial.assert();+ unchanged.assert();+ first_page.assert();+ final_page.assert();+ other_album.assert();+ unchanged.remove();++ let mut deleted_album = collection(1, "First", &key);+ deleted_album["updationTime"] = json!(30);+ deleted_album["isDeleted"] = json!(true);+ let deletion = server+ .mock("GET", "/collections/v2")+ .match_query(mockito::Matcher::UrlEncoded(+ "sinceTime".into(),+ "20".into(),+ ))+ .with_body(json!({"collections": [deleted_album]}).to_string())+ .expect(1)+ .create();+ let files = home.json(&["photos", "file", "list"]);+ assert_eq!(files.as_array().unwrap().len(), 1);+ assert_eq!(files[0]["albumIds"], json!(["2"]));+ deletion.assert();+ other_album.assert();++ let mut updated_album = collection(2, "Second", &key);+ updated_album["updationTime"] = json!(40);+ let updated = server+ .mock("GET", "/collections/v2")+ .match_query(mockito::Matcher::UrlEncoded(+ "sinceTime".into(),+ "30".into(),+ ))+ .with_body(json!({"collections": [updated_album]}).to_string())+ .expect(1)+ .create();+ let mut removed = first;+ removed["updationTime"] = json!(40);+ removed["isDeleted"] = json!(true);+ let removal = page(&mut server, 2, 10, json!([removed]), false)+ .expect(1)+ .create();+ assert_eq!(home.json(&["photos", "file", "list"]), json!([]));+ updated.assert();+ removal.assert();+}++#[test]+fn account_replica_is_encrypted_and_available_offline() {+ let mut server = mockito::Server::new();+ let home = TestHome::new();+ home.seed(&server.url());+ assert!(+ failure(&home.run(&["photos", "album", "list", "--offline"])).contains("no local data")+ );+ assert!(!home.dir.path().join("accounts").exists());+ let failed_sync = server+ .mock("GET", "/collections/v2")+ .match_query(mockito::Matcher::Any)+ .with_status(400)+ .create();+ failure(&home.run(&["photos", "album", "list"]));+ assert!(+ failure(&home.run(&["photos", "album", "list", "--offline"]))+ .contains("no local Photos data")+ );+ failed_sync.assert();+ failed_sync.remove();+ let key = Key::generate();+ let albums = server+ .mock("GET", "/collections/v2")+ .match_query(mockito::Matcher::Any)+ .with_body(json!({"collections": [collection(1, "Private album", &key)]}).to_string())+ .expect(2)+ .create();+ home.json(&["photos", "album", "list"]);+ let state = home.read_vault();+ let directory = home+ .dir+ .path()+ .join("accounts")+ .join(state["accounts"][0]["storage_id"].as_str().unwrap());+ let path = directory.join("data.db");+ let lock_path = directory.with_extension("lock");+ let connection = rusqlite::Connection::open(&path).unwrap();+ connection+ .pragma_update(None, "key", format!("x'{}'", "2a".repeat(32)))+ .unwrap();+ connection+ .query_row("SELECT count(*) FROM sqlite_master", [], |row| {+ row.get::<_, i64>(0)+ })+ .unwrap();+ let (file, _) = remote_file(10, &key, b"original");+ let files_page = page(&mut server, 1, 0, json!([file]), false)+ .expect(1)+ .create();+ let files = home.json(&["photos", "file", "list"]);+ assert!(directory.join("data.db-wal").exists());+ for filename in ["data.db", "data.db-wal"] {+ let bytes = fs::read(directory.join(filename)).unwrap();+ assert!(!bytes.is_empty());+ for plaintext in ["SQLite format 3", "Private album", "Edited π§.jpg"] {+ assert!(+ !bytes+ .windows(plaintext.len())+ .any(|window| window == plaintext.as_bytes())+ );+ }+ }+ drop(connection);+ let unkeyed = rusqlite::Connection::open(&path).unwrap();+ assert!(+ unkeyed+ .query_row("SELECT count(*) FROM sqlite_master", [], |row| row+ .get::<_, i64>(0))+ .is_err()+ );+ drop(unkeyed);+ albums.assert();+ files_page.assert();+ drop(server);+ assert_eq!(home.json(&["photos", "file", "list", "--offline"]), files);+ assert!(+ failure(&home.run(&[+ "photos",+ "file",+ "download",+ "10",+ "--output",+ "unused.jpg",+ "--offline"+ ]))+ .contains("download requires network access")+ );++ let mut wrong_key = state.clone();+ wrong_key["accounts"][0]["db_key"] = json!(vec![43u8; 32]);+ home.write_vault(&wrong_key);+ assert!(+ failure(&home.run(&["photos", "album", "list", "--offline"]))+ .contains("cannot open the encrypted database")+ );+ home.write_vault(&state);+ fs::remove_file(&lock_path).unwrap();+ assert_eq!(home.json(&["photos", "file", "list", "--offline"]), files);+ assert!(lock_path.is_file());+ home.json(&["accounts", "rename", "fixture", "renamed"]);+ assert_eq!(+ home.json(&[+ "photos",+ "file",+ "list",+ "--account",+ "renamed",+ "--offline"+ ]),+ files+ );+ let mut two_accounts = home.read_vault();+ let mut other = two_accounts["accounts"][0].clone();+ other["storage_id"] = json!("45b3495e-92de-43a1-a58b-e0b1e57d8c68");+ other["name"] = json!("other");+ other["user_id"] = json!(123);+ other["db_key"] = json!(vec![43u8; 32]);+ two_accounts["accounts"].as_array_mut().unwrap().push(other);+ home.write_vault(&two_accounts);+ assert!(+ failure(&home.run(&["photos", "album", "list", "--account", "other", "--offline"]))+ .contains("no local data")+ );+ assert!(+ !home+ .dir+ .path()+ .join("accounts")+ .join("45b3495e-92de-43a1-a58b-e0b1e57d8c68.lock")+ .exists()+ );+ let other_directory = home+ .dir+ .path()+ .join("accounts/45b3495e-92de-43a1-a58b-e0b1e57d8c68");+ fs::create_dir(&other_directory).unwrap();+ assert!(+ failure(&home.run(&["photos", "album", "list", "--account", "other", "--offline"]))+ .contains("unable to open database file")+ );+ assert!(!other_directory.join("data.db").exists());+ home.json(&["accounts", "logout", "renamed", "--local", "--offline"]);+ assert!(!directory.exists());+ assert!(!lock_path.exists());+}++#[test]+fn photos_lists_limit_output_and_stream_complete_results() {+ let mut server = mockito::Server::new();+ let home = TestHome::new();+ home.seed(&server.url());+ let key = Key::generate();+ let albums = (1..=150)+ .map(|id| collection(id, "Same album", &key))+ .collect::<Vec<_>>();+ let albums = server+ .mock("GET", "/collections/v2")+ .match_query(mockito::Matcher::Any)+ .with_body(json!({"collections": albums}).to_string())+ .expect(2)+ .create();+ let (file, _) = remote_file(1, &key, b"original");+ let files = (1..=150)+ .map(|id| {+ let mut file = file.clone();+ file["id"] = json!(id);+ file+ })+ .collect::<Vec<_>>();+ let files = page(&mut server, 1, 0, json!(files), false)+ .expect(1)+ .create();+ let output = success(home.run(&["photos", "album", "list", "--json"]));+ let listed: Vec<Value> = serde_json::from_slice(&output.stdout).unwrap();+ assert_eq!(listed.len(), 100);+ assert!(String::from_utf8_lossy(&output.stderr).contains("Showing first 100"));+ assert_eq!(+ home.json(&["photos", "file", "list", "--album", "1", "--limit", "1"])+ .as_array()+ .unwrap()+ .len(),+ 1+ );+ albums.assert();+ files.assert();+ drop(server);++ for resource in ["album", "file"] {+ let mut base = vec!["--offline", "photos", resource, "list"];+ if resource == "file" {+ base.extend(["--album", "1"]);+ }+ for (extra, count, truncated) in [+ (vec![], 100, true),+ (vec!["--limit", "1"], 1, true),+ (vec!["--limit", "150"], 150, false),+ (vec!["--limit", "4294967295"], 150, false),+ (vec!["--all"], 150, false),+ ] {+ for as_json in [false, true] {+ let mut args = base.clone();+ args.extend(&extra);+ if as_json {+ args.push("--json");+ }+ let output = success(home.run(&args));+ if as_json {+ let values: Vec<Value> = serde_json::from_slice(&output.stdout).unwrap();+ assert_eq!(values.len(), count);+ assert_eq!(values[0]["id"], "1");+ assert_eq!(values[count - 1]["id"], count.to_string());+ } else {+ assert_eq!(+ String::from_utf8_lossy(&output.stdout).lines().count(),+ count + 1+ );+ }+ assert_eq!(!output.stderr.is_empty(), truncated);+ }+ }+ for extra in [+ vec!["--limit", "0"],+ vec!["--limit", "-1"],+ vec!["--all", "--limit", "1"],+ ] {+ let mut args = base.clone();+ args.extend(extra);+ failure(&home.run(&args));+ }+ }+ for args in [+ vec!["--offline", "photos", "album", "view", "Same album"],+ vec![+ "--offline",+ "photos",+ "file",+ "view",+ "Edited π§.jpg",+ "--album",+ "1",+ ],+ ] {+ let output = home.run(&args);+ let error = failure(&output);+ assert!(output.stdout.is_empty());+ assert_eq!(+ error.lines().filter(|line| line.starts_with(" ")).count(),+ 10+ );+ assert!(error.contains("More matches omitted"));+ }++ let state = home.read_vault();+ let database = home+ .dir+ .path()+ .join("accounts")+ .join(state["accounts"][0]["storage_id"].as_str().unwrap())+ .join("data.db");+ let connection = rusqlite::Connection::open(database).unwrap();+ connection+ .pragma_update(None, "key", format!("x'{}'", "2a".repeat(32)))+ .unwrap();+ connection+ .execute(+ "UPDATE photos_files SET created_at=?1 WHERE id=150",+ [i64::MAX],+ )+ .unwrap();+ drop(connection);+ for as_json in [false, true] {+ let mut args = vec![+ "--offline",+ "photos",+ "file",+ "list",+ "--album",+ "1",+ "--all",+ ];+ if as_json {+ args.push("--json");+ }+ let output = home.run(&args);+ assert!(failure(&output).contains("timestamp is outside the supported range"));+ assert!(String::from_utf8_lossy(&output.stdout).contains("Edited π§.jpg"));+ }+}+fn collection(id: i64, name: &str, key: &Key) -> Value {let wrapped = secretbox::encrypt(key.as_bytes(), &Key::from_bytes([0; 32]));json!({@@ -205,6 +614,4 @@ fn page(mockito::Matcher::UrlEncoded("sinceTime".into(), since.to_string()),])).with_body(json!({"diff": diff, "hasMore": more}).to_string())- .expect_at_least(1)- .create()}diff --git a/rust/apps/cli-next/tests/support/museum.rs b/rust/apps/cli-next/tests/support/museum.rsindex 04ff51b1f7..73ba8ae9ae 100644--- a/rust/apps/cli-next/tests/support/museum.rs+++ b/rust/apps/cli-next/tests/support/museum.rs@@ -2,7 +2,10 @@ use ente_accounts::{AccountsClient, AccountsClientConfig, AuthFlow, AuthFlowUi, AuthenticatedAccount,CreateAccountParams, OtpPurpose, SecondFactorMethod, TotpPurpose,};-use ente_core::crypto::{PublicKey, blob, sealed, secretbox, stream};+use ente_core::{+ crypto::{PublicKey, blob, sealed, secretbox, stream},+ io::Md5Writer,+};use ente_test_support::{HARDCODED_OTT, Museum, TestResult};use uuid::Uuid;use zeroize::Zeroizing;@@ -366,6 +369,28 @@ async fn exercise(origin: String) -> TestResult {assert_eq!(fs::read(output_path).unwrap(), original);login(&home, "photos", &alice_email, &["--host", &origin]);+ let added_file = upload_file(&origin, &alice, album, &key, b"added after the first sync")+ .await+ .to_string();+ assert_eq!(home.json(&["photos", "file", "list", "--offline"]), files);+ let refreshed = home.json(&["photos", "file", "list", "--all"]);+ assert_eq!(refreshed.as_array().unwrap().len(), 2);+ assert!(+ refreshed+ .as_array()+ .unwrap()+ .iter()+ .any(|file| file["id"] == added_file)+ );+ assert_eq!(+ home.json(&["photos", "file", "list", "--offline", "--all"]),+ refreshed+ );+ assert_eq!(+ home.json(&["photos", "file", "list", "--offline", "--limit", "1"]),+ json!([refreshed[0]])+ );+for selector in ["Monsoon π§", &album.to_string()] {let (conflict, _) = create_album(&origin, &alice, selector, "album").await;let output = home.run(&["photos", "album", "view", selector, "--json"]);@@ -414,9 +439,26 @@ async fn exercise(origin: String) -> TestResult {home.json(&["account", "view", "work"])["products"],json!(["locker"]));+ let account_dir = home+ .dir+ .path()+ .join("accounts")+ .join(first_id.as_str().unwrap());+ assert!(account_dir.join("data.db").is_file());+ success(home.run(&["account", "logout", "work", "--local"]));+ assert!(!account_dir.exists());+ assert!(!account_dir.with_extension("lock").exists());for entry in fs::read_dir(home.dir.path()).unwrap() {let entry = entry.unwrap();+ #[cfg(unix)]+ {+ use std::os::unix::fs::PermissionsExt;+ assert_eq!(entry.metadata().unwrap().permissions().mode() & 0o077, 0);+ }+ if entry.file_type().unwrap().is_dir() {+ continue;+ }let bytes = fs::read(entry.path()).unwrap();assert!(!bytes@@ -428,11 +470,6 @@ async fn exercise(origin: String) -> TestResult {.windows(PASSWORD.len()).any(|b| b == PASSWORD.as_bytes()));- #[cfg(unix)]- {- use std::os::unix::fs::PermissionsExt;- assert_eq!(entry.metadata().unwrap().permissions().mode() & 0o077, 0);- }}Ok(())}@@ -516,37 +553,40 @@ async fn upload_file() -> i64 {let client = reqwest::Client::new();let token = b64::encode_url_safe(&owner.secrets.token);- let urls: Value = client- .get(format!("{origin}/files/upload-urls"))- .header("x-auth-token", &token)- .header("x-client-package", "io.ente.photos")- .query(&[("count", 2)])- .send()- .await- .unwrap()- .error_for_status()- .unwrap()- .json()- .await- .unwrap();let key = Key::generate();let wrapped = secretbox::encrypt(key.as_bytes(), collection_key);let mut encrypted = Vec::new();let header =stream::encrypt_file(&mut std::io::Cursor::new(original), &mut encrypted, &key).unwrap();let thumbnail = blob::encrypt(b"thumbnail", &key).unwrap();- for (index, bytes) in [&encrypted, &thumbnail.encrypted_data]- .into_iter()- .enumerate()- {+ let mut object_keys = Vec::new();+ for bytes in [&encrypted, &thumbnail.encrypted_data] {+ let mut checksum = Md5Writer::new(std::io::sink());+ checksum.write_all(bytes).unwrap();+ let checksum = b64::encode(&checksum.finalize().1);+ let upload: Value = client+ .post(format!("{origin}/files/upload-url"))+ .header("x-auth-token", &token)+ .header("x-client-package", "io.ente.photos")+ .json(&json!({"contentLength": bytes.len(), "contentMD5": checksum}))+ .send()+ .await+ .unwrap()+ .error_for_status()+ .unwrap()+ .json()+ .await+ .unwrap();client- .put(urls["urls"][index]["url"].as_str().unwrap())+ .put(upload["url"].as_str().unwrap())+ .header("content-md5", checksum).body(bytes.clone()).send().await.unwrap().error_for_status().unwrap();+ object_keys.push(upload["objectKey"].as_str().unwrap().to_owned());}let metadata = blob::encrypt_json(&json!({@@ -561,8 +601,8 @@ async fn upload_file(.json(&json!({"collectionID": album, "encryptedKey": b64::encode(&wrapped.encrypted_data),"keyDecryptionNonce": b64::encode(wrapped.nonce.as_bytes()),- "file": {"objectKey": urls["urls"][0]["objectKey"], "decryptionHeader": b64::encode(header.as_bytes())},- "thumbnail": {"objectKey": urls["urls"][1]["objectKey"], "decryptionHeader": b64::encode(thumbnail.decryption_header.as_bytes())},+ "file": {"objectKey": object_keys[0], "decryptionHeader": b64::encode(header.as_bytes())},+ "thumbnail": {"objectKey": object_keys[1], "decryptionHeader": b64::encode(thumbnail.decryption_header.as_bytes())},"metadata": {"encryptedData": b64::encode(&metadata.encrypted_data),"decryptionHeader": b64::encode(metadata.decryption_header.as_bytes())}})).send().await.unwrap().error_for_status().unwrap().json().await.unwrap();diff --git a/rust/crates/collections/Cargo.toml b/rust/crates/collections/Cargo.tomlindex cc24b50c20..71a7ba6428 100644--- a/rust/crates/collections/Cargo.toml+++ b/rust/crates/collections/Cargo.toml@@ -4,6 +4,7 @@ edition.workspace = true[dependencies]ente-core.workspace = true+serde.workspace = truethiserror.workspace = true[lints]diff --git a/rust/crates/collections/src/client.rs b/rust/crates/collections/src/client.rsnew file mode 100644index 0000000000..3ec6ee0c87--- /dev/null+++ b/rust/crates/collections/src/client.rs@@ -0,0 +1,204 @@+use ente_core::{+ Session, b64,+ crypto::{Header, Key, Nonce, blob, secretbox},+ http,+};+use serde::{Deserialize, de::DeserializeOwned};++use crate::{Error, Result, open_collection_key};++pub struct CollectionPage {+ pub collections: Vec<Collection>,+ pub cursor: i64,+}++pub async fn diff(session: &Session, since: i64) -> Result<CollectionPage> {+ #[derive(Deserialize)]+ struct Response {+ collections: Vec<Collection>,+ }+ let response: Response = http::retry(|| async {+ session+ .api+ .get("/collections/v2")+ .query(&[("sinceTime", since)])+ .send()+ .await?+ .error_for_code()+ .await?+ .json()+ .await+ })+ .await?;+ let cursor = response+ .collections+ .iter()+ .map(|c| c.updation_time)+ .max()+ .unwrap_or(since);+ Ok(CollectionPage {+ collections: response.collections,+ cursor,+ })+}++pub struct FilePage {+ pub files: Vec<File>,+ pub cursor: i64,+ pub has_more: bool,+}++pub async fn files_diff(session: &Session, collection_id: i64, since: i64) -> Result<FilePage> {+ #[derive(Deserialize)]+ #[serde(rename_all = "camelCase")]+ struct Response {+ diff: Vec<File>,+ has_more: bool,+ }+ let response: Response = http::retry(|| async {+ session+ .api+ .get("/collections/v2/diff")+ .query(&[("collectionID", collection_id), ("sinceTime", since)])+ .send()+ .await?+ .error_for_code()+ .await?+ .json()+ .await+ })+ .await?;+ let cursor = response+ .diff+ .iter()+ .map(|f| f.updation_time)+ .max()+ .unwrap_or(since);+ if response.has_more && cursor <= since {+ return Err(Error::CursorDidNotAdvance(collection_id));+ }+ Ok(FilePage {+ files: response.diff,+ cursor,+ has_more: response.has_more,+ })+}++#[derive(Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct Collection {+ pub id: i64,+ pub owner: Owner,+ pub encrypted_key: String,+ pub key_decryption_nonce: Option<String>,+ pub encrypted_name: Option<String>,+ pub name_decryption_nonce: Option<String>,+ pub name: Option<String>,+ #[serde(rename = "type")]+ pub kind: String,+ pub updation_time: i64,+ pub is_deleted: Option<bool>,+ pub magic_metadata: Option<EncryptedMetadata>,+ pub shared_magic_metadata: Option<EncryptedMetadata>,+}++impl Collection {+ pub fn open_key(&self, session: &Session) -> Result<Key> {+ open_collection_key(+ session,+ self.owner.id,+ &self.encrypted_key,+ self.key_decryption_nonce.as_deref(),+ )+ }++ pub fn open_name(&self, key: &Key) -> Result<String> {+ if let Some(name) = self.name.as_ref().filter(|name| !name.is_empty()) {+ return Ok(name.clone());+ }+ let encrypted = self+ .encrypted_name+ .as_deref()+ .ok_or(Error::InvalidCollection {+ id: self.id,+ reason: "missing name",+ })?;+ let nonce = self+ .name_decryption_nonce+ .as_deref()+ .ok_or(Error::InvalidCollection {+ id: self.id,+ reason: "missing name nonce",+ })?;+ let bytes = secretbox::decrypt(+ &b64::decode(encrypted)?,+ &Nonce::try_from_slice(&b64::decode(nonce)?)?,+ key,+ )?;+ String::from_utf8(bytes).map_err(|_| Error::InvalidCollection {+ id: self.id,+ reason: "name is not UTF-8",+ })+ }+}++#[derive(Deserialize)]+pub struct Owner {+ pub id: i64,+}++#[derive(Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct File {+ pub id: i64,+ #[serde(rename = "ownerID")]+ pub owner_id: i64,+ pub encrypted_key: String,+ pub key_decryption_nonce: String,+ pub file: FileAttributes,+ pub metadata: FileAttributes,+ pub is_deleted: bool,+ pub updation_time: i64,+ pub magic_metadata: Option<EncryptedMetadata>,+ pub pub_magic_metadata: Option<EncryptedMetadata>,+}++impl File {+ pub fn is_deleted(&self) -> bool {+ self.is_deleted || self.metadata.encrypted_data.as_deref() == Some("-")+ }++ pub fn open_key(&self, collection_key: &Key) -> Result<Key> {+ let bytes = secretbox::decrypt(+ &b64::decode(&self.encrypted_key)?,+ &Nonce::try_from_slice(&b64::decode(&self.key_decryption_nonce)?)?,+ collection_key,+ )?;+ Ok(Key::try_from_slice(&bytes)?)+ }+}++#[derive(Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct FileAttributes {+ pub encrypted_data: Option<String>,+ pub decryption_header: String,+}++#[derive(Deserialize)]+pub struct EncryptedMetadata {+ pub data: String,+ pub header: String,+}++impl EncryptedMetadata {+ pub fn open<T: DeserializeOwned>(&self, key: &Key) -> Result<T> {+ Ok(blob::decrypt_json(+ &blob::EncryptedBlob {+ encrypted_data: b64::decode(&self.data)?,+ decryption_header: Header::try_from_slice(&b64::decode(&self.header)?)?,+ },+ key,+ )?)+ }+}diff --git a/rust/crates/collections/src/lib.rs b/rust/crates/collections/src/lib.rsindex 06ee4aec41..f932f360dc 100644--- a/rust/crates/collections/src/lib.rs+++ b/rust/crates/collections/src/lib.rs@@ -1,3 +1,5 @@+pub mod client;+use ente_core::{Session, b64,crypto::{self, Key, Nonce, sealed, secretbox},@@ -31,12 +33,18 @@ pub type Result<T> = std::result::Result<T, Error>;#[derive(Debug, thiserror::Error)]pub enum Error {+ #[error(transparent)]+ Http(#[from] ente_core::http::Error),#[error(transparent)]Decode(#[from] b64::DecodeError),#[error(transparent)]Crypto(#[from] crypto::Error),#[error("Owned collection is missing its key decryption nonce")]MissingKeyDecryptionNonce,+ #[error("invalid collection {id}: {reason}")]+ InvalidCollection { id: i64, reason: &'static str },+ #[error("file changes did not advance the cursor for collection {0}")]+ CursorDidNotAdvance(i64),}#[cfg(test)]diff --git a/rust/crates/core/src/crypto/stream.rs b/rust/crates/core/src/crypto/stream.rsindex def2e15bdf..4a05459df9 100644--- a/rust/crates/core/src/crypto/stream.rs+++ b/rust/crates/core/src/crypto/stream.rs@@ -238,6 +238,52 @@ pub struct StreamingDecryptor<R: Read> {seen_final: bool,}+pub struct DecryptingWriter<W: Write> {+ decryptor: Decryptor,+ writer: W,+ buffer: Vec<u8>,+}++impl<W: Write> DecryptingWriter<W> {+ pub fn new(header: &Header, key: &Key, writer: W) -> Self {+ Self {+ decryptor: Decryptor::new(header, key),+ writer,+ buffer: Vec::with_capacity(DECRYPTION_CHUNK_SIZE),+ }+ }++ pub fn write(&mut self, mut data: &[u8]) -> Result<()> {+ while !data.is_empty() {+ if self.decryptor.seen_final {+ return Err(Error::StreamTrailingData);+ }+ let count = data.len().min(DECRYPTION_CHUNK_SIZE - self.buffer.len());+ self.buffer.extend_from_slice(&data[..count]);+ data = &data[count..];+ if self.buffer.len() == DECRYPTION_CHUNK_SIZE {+ self.flush_chunk()?;+ }+ }+ Ok(())+ }++ fn flush_chunk(&mut self) -> Result<()> {+ self.decryptor.pull_in_place(&mut self.buffer)?;+ self.writer.write_all(&self.buffer)?;+ self.buffer.clear();+ Ok(())+ }++ pub fn finish(mut self) -> Result<W> {+ if !self.buffer.is_empty() {+ self.flush_chunk()?;+ }+ self.decryptor.finish()?;+ Ok(self.writer)+ }+}+fn ensure_reader_exhausted<R: Read>(reader: &mut R) -> Result<()> {let mut extra = [0u8; 1];@@ -865,6 +911,22 @@ mod tests {assert_eq!(ABYTES, 17);}+ #[test]+ fn decrypting_writer_accepts_arbitrary_chunk_boundaries() {+ let key = Key::generate();+ for size in [0, 2 * ENCRYPTION_CHUNK_SIZE + 71] {+ let plaintext = vec![37; size];+ let mut ciphertext = Vec::new();+ let header = encrypt_file(&mut plaintext.as_slice(), &mut ciphertext, &key).unwrap();+ let mut writer = DecryptingWriter::new(&header, &key, Vec::new());+ writer.write(&ciphertext[..13]).unwrap();+ for chunk in ciphertext[13..].chunks(DECRYPTION_CHUNK_SIZE + 5) {+ writer.write(chunk).unwrap();+ }+ assert_eq!(writer.finish().unwrap(), plaintext);+ }+ }+#[test]fn test_estimate_encrypted_size() {assert_eq!(estimate_encrypted_size(0), ABYTES);diff --git a/rust/crates/core/src/http.rs b/rust/crates/core/src/http.rsindex 37a6f6c81c..b18f68aa58 100644--- a/rust/crates/core/src/http.rs+++ b/rust/crates/core/src/http.rs@@ -77,7 +77,8 @@ impl Error {pub fn is_retryable(&self) -> bool {match self {- Error::Network(e) => e.0.is_request() || e.0.is_body(),+ // reqwest also reports interrupted bodies as decoding errors.+ Error::Network(e) => e.0.is_request() || e.0.is_body() || e.0.is_decode(),Error::Http { status, .. } | Error::Api { status, .. } => {*status == 429 || *status >= 500}@@ -450,16 +451,38 @@ whereretry_with_delays(&profile.delays(), operation).await}+pub async fn retry_if<T, E, F, Fut>(operation: F, should_retry: impl Fn(&E) -> bool) -> Result<T, E>+where+ E: std::fmt::Display,+ F: FnMut() -> Fut,+ Fut: Future<Output = Result<T, E>>,+{+ retry_operation(&RetryProfile::Interactive.delays(), operation, should_retry).await+}+async fn retry_with_delays<T, F, Fut>(delays: &[Duration], mut operation: F) -> Result<T, Error>whereF: FnMut() -> Fut,Fut: Future<Output = Result<T, Error>>,+{+ retry_operation(delays, &mut operation, Error::is_retryable).await+}++async fn retry_operation<T, E, F, Fut>(+ delays: &[Duration],+ mut operation: F,+ should_retry: impl Fn(&E) -> bool,+) -> Result<T, E>+where+ E: std::fmt::Display,+ F: FnMut() -> Fut,+ Fut: Future<Output = Result<T, E>>,{let mut delays = delays.iter();loop {match operation().await {Ok(value) => return Ok(value),- Err(error) if error.is_retryable() => match delays.next() {+ Err(error) if should_retry(&error) => match delays.next() {Some(delay) => {log::warn!("retrying in {delay:?}: {error}");sleep(*delay).await;@@ -745,6 +768,7 @@ mod tests {let api = api(&server, None);let err = api.ping().await.unwrap_err();assert!(matches!(err, Error::Parse(_)));+ assert!(!err.is_retryable());}#[tokio::test]diff --git a/rust/crates/photos/src/collections.rs b/rust/crates/photos/src/collections.rsindex 7623721ff5..3719a17a21 100644--- a/rust/crates/photos/src/collections.rs+++ b/rust/crates/photos/src/collections.rs@@ -1,9 +1,4 @@-use ente_collections::open_collection_key;-use ente_core::{- Session, b64,- crypto::{self, Header, Key, Nonce, blob, secretbox},- http,-};+use ente_core::{Session, crypto::Key};use serde::Deserialize;const DEFAULT_HIDDEN_SUBTYPE: u8 = 1;@@ -55,151 +50,93 @@ impl Visibility {}}-#[derive(Debug, thiserror::Error)]-pub enum Error {- #[error(transparent)]- Http(#[from] http::Error),- #[error(transparent)]- Crypto(#[from] crypto::Error),- #[error(transparent)]- Base64(#[from] b64::DecodeError),- #[error(transparent)]- Collections(#[from] ente_collections::Error),- #[error("invalid collection {id}: {reason}")]- InvalidCollection { id: i64, reason: &'static str },+pub type Error = ente_collections::Error;++pub struct Change {+ pub id: i64,+ pub collection: Option<Collection>,}-pub async fn list(session: &Session) -> Result<Vec<Collection>, Error> {- let response: CollectionsResponse = session- .api- .get("/collections/v2")- .query(&[("sinceTime", 0)])- .send()- .await?- .error_for_code()- .await?- .json()- .await?;- response+pub struct Page {+ pub changes: Vec<Change>,+ pub cursor: i64,+}++pub async fn diff(session: &Session, since: i64) -> Result<Page, Error> {+ let page = ente_collections::client::diff(session, since).await?;+ let changes = page.collections.into_iter()- .filter(|c| c.is_deleted != Some(true))- .map(|c| {- let owned = c.owner.id == session.user_id;- let key = open_collection_key(- session,- c.owner.id,- &c.encrypted_key,- c.key_decryption_nonce.as_deref(),- )?;- let name = match c.name.as_deref().filter(|name| !name.is_empty()) {- Some(name) => name.to_owned(),- None => {- let encrypted_name = c.required(&c.encrypted_name, "missing name")?;- let nonce = c.required(&c.name_decryption_nonce, "missing name nonce")?;- let bytes = secretbox::decrypt(- &b64::decode(encrypted_name)?,- &Nonce::try_from_slice(&b64::decode(nonce)?)?,- &key,- )?;- String::from_utf8(bytes).map_err(|_| Error::InvalidCollection {- id: c.id,- reason: "name is not UTF-8",- })?- }- };- let metadata = if owned {- c.magic_metadata.as_ref()- } else {- c.shared_magic_metadata.as_ref()- };- let visibility = match metadata {- Some(metadata) => {- let metadata = blob::decrypt_json::<Metadata>(- &blob::EncryptedBlob {- encrypted_data: b64::decode(&metadata.data)?,- decryption_header: Header::try_from_slice(&b64::decode(- &metadata.header,- )?)?,- },- &key,- )?;- if owned && metadata.sub_type == Some(DEFAULT_HIDDEN_SUBTYPE) {- HIDDEN_VISIBILITY- } else {- metadata.visibility.unwrap_or(0)- }- }- None => 0,- };- Ok(Collection {- id: c.id,- key,- name,- kind: match c.kind.as_str() {- "favorites" => Kind::Favorites,- "uncategorized" => Kind::Uncategorized,- _ => Kind::Album,- },- visibility: match visibility {- ARCHIVED_VISIBILITY => Visibility::Archived,- HIDDEN_VISIBILITY => Visibility::Hidden,- _ => Visibility::Visible,+ .map(|remote| {+ Ok(Change {+ id: remote.id,+ collection: if remote.is_deleted == Some(true) {+ None+ } else {+ Some(Collection::open(remote, session)?)},- owner_id: c.owner.id,- updated_at_micros: c.updation_time,})})- .collect()-}--#[derive(Deserialize)]-struct CollectionsResponse {- collections: Vec<RemoteCollection>,+ .collect::<Result<_, Error>>()?;+ Ok(Page {+ changes,+ cursor: page.cursor,+ })}-#[derive(Deserialize)]-#[serde(rename_all = "camelCase")]-struct RemoteCollection {- id: i64,- owner: Owner,- encrypted_key: String,- key_decryption_nonce: Option<String>,- encrypted_name: Option<String>,- name_decryption_nonce: Option<String>,- name: Option<String>,- #[serde(rename = "type")]- kind: String,- updation_time: i64,- is_deleted: Option<bool>,- magic_metadata: Option<EncryptedMetadata>,- shared_magic_metadata: Option<EncryptedMetadata>,+pub async fn list(session: &Session) -> Result<Vec<Collection>, Error> {+ Ok(diff(session, 0)+ .await?+ .changes+ .into_iter()+ .filter_map(|change| change.collection)+ .collect())}-impl RemoteCollection {- fn required<'a>(- &self,- value: &'a Option<String>,- reason: &'static str,- ) -> Result<&'a str, Error> {- value.as_deref().ok_or(Error::InvalidCollection {- id: self.id,- reason,+impl Collection {+ fn open(+ remote: ente_collections::client::Collection,+ session: &Session,+ ) -> Result<Self, Error> {+ let key = remote.open_key(session)?;+ let name = remote.open_name(&key)?;+ let owned = remote.owner.id == session.user_id;+ let metadata = if owned {+ remote.magic_metadata.as_ref()+ } else {+ remote.shared_magic_metadata.as_ref()+ };+ let visibility = match metadata {+ Some(metadata) => {+ let metadata: Metadata = metadata.open(&key)?;+ if owned && metadata.sub_type == Some(DEFAULT_HIDDEN_SUBTYPE) {+ HIDDEN_VISIBILITY+ } else {+ metadata.visibility.unwrap_or(0)+ }+ }+ None => 0,+ };+ Ok(Self {+ id: remote.id,+ key,+ name,+ kind: match remote.kind.as_str() {+ "favorites" => Kind::Favorites,+ "uncategorized" => Kind::Uncategorized,+ _ => Kind::Album,+ },+ visibility: match visibility {+ ARCHIVED_VISIBILITY => Visibility::Archived,+ HIDDEN_VISIBILITY => Visibility::Hidden,+ _ => Visibility::Visible,+ },+ owner_id: remote.owner.id,+ updated_at_micros: remote.updation_time,})}}-#[derive(Deserialize)]-struct Owner {- id: i64,-}--#[derive(Deserialize)]-struct EncryptedMetadata {- data: String,- header: String,-}-#[derive(Deserialize)]#[serde(rename_all = "camelCase")]struct Metadata {diff --git a/rust/crates/photos/src/files.rs b/rust/crates/photos/src/files.rsindex 1b73c05b0e..56b82b7c8e 100644--- a/rust/crates/photos/src/files.rs+++ b/rust/crates/photos/src/files.rs@@ -1,8 +1,6 @@-use std::collections::BTreeMap;-use ente_core::{Session, b64,- crypto::{self, Header, Key, Nonce, blob, secretbox},+ crypto::{self, Header, Key, blob},http,};use serde::Deserialize;@@ -77,130 +75,75 @@ pub enum Error {Base64(#[from] b64::DecodeError),#[error(transparent)]Io(#[from] std::io::Error),- #[error("file changes did not advance the cursor for collection {0}")]- CursorDidNotAdvance(i64),+ #[error(transparent)]+ Collections(#[from] ente_collections::Error),}pub async fn diff(session: &Session, collection: &Collection, since: i64) -> Result<Page, Error> {- let response: DiffResponse = http::retry(|| async {- session- .api- .get("/collections/v2/diff")- .query(&[("collectionID", collection.id), ("sinceTime", since)])- .send()- .await?- .error_for_code()- .await?- .json()- .await- })- .await?;- let mut cursor = since;- let mut changes = Vec::with_capacity(response.diff.len());- for remote in response.diff {- cursor = cursor.max(remote.updation_time);- changes.push(Change {- id: remote.id,- file: if remote.is_deleted || remote.file.encrypted_data.as_deref() == Some("-") {- None- } else {- Some(remote.open(&collection.key)?)- },- });- }- if response.has_more && cursor <= since {- return Err(Error::CursorDidNotAdvance(collection.id));- }+ let page = ente_collections::client::files_diff(session, collection.id, since).await?;+ let changes = page+ .files+ .into_iter()+ .map(|remote| {+ Ok(Change {+ id: remote.id,+ file: if remote.is_deleted() {+ None+ } else {+ Some(File::open(remote, &collection.key)?)+ },+ })+ })+ .collect::<Result<_, Error>>()?;Ok(Page {changes,- cursor,- has_more: response.has_more,+ cursor: page.cursor,+ has_more: page.has_more,})}-pub async fn list(session: &Session, collection: &Collection) -> Result<Vec<File>, Error> {- let mut files = BTreeMap::new();- let mut cursor = 0;- loop {- let page = diff(session, collection, cursor).await?;- for change in page.changes {- if let Some(file) = change.file {- files.insert(change.id, file);- } else {- files.remove(&change.id);- }- }- cursor = page.cursor;- if !page.has_more {- return Ok(files.into_values().collect());- }- }-}-#[cfg(not(target_arch = "wasm32"))]-pub async fn download(+pub async fn download<W: std::io::Write>(session: &Session,file: &File,- output: &mut impl std::io::Write,+ mut output: impl FnMut() -> std::io::Result<W>,) -> Result<(), Error> {- use ente_core::crypto::stream::{DECRYPTION_CHUNK_SIZE, Decryptor};+ use ente_core::crypto::stream::DecryptingWriter;use futures_util::StreamExt;- let response = http::retry(|| async {- let signed: DownloadUrl = session- .api- .get(&format!("/files/download/v3/{}", file.id))- .send()- .await?- .error_for_code()- .await?- .json()- .await?;- session- .api- .http()- .get(&signed.url)- .send()- .await?- .error_for_status()- })- .await?;- let body = response.bytes_stream();- let mut body = std::pin::pin!(body);- let mut decryptor = Decryptor::new(&file.header, &file.key);- let mut buffer = Vec::with_capacity(DECRYPTION_CHUNK_SIZE);- let mut seen_final = false;- while let Some(chunk) = body.next().await {- let chunk = chunk?;- let mut remaining = chunk.as_ref();- while !remaining.is_empty() {- if seen_final {- return Err(crypto::Error::StreamTrailingData.into());- }- let count = remaining.len().min(DECRYPTION_CHUNK_SIZE - buffer.len());- buffer.extend_from_slice(&remaining[..count]);- remaining = &remaining[count..];- if buffer.len() == DECRYPTION_CHUNK_SIZE {- let (plaintext, is_final) = decryptor.pull(&buffer)?;- output.write_all(&plaintext)?;- seen_final = is_final;- buffer.clear();+ http::retry_if(+ || {+ let output = output();+ async move {+ let output = output?;+ let signed: DownloadUrl = session+ .api+ .get(&format!("/files/download/v3/{}", file.id))+ .send()+ .await?+ .error_for_code()+ .await?+ .json()+ .await?;+ let response = session+ .api+ .http()+ .get(&signed.url)+ .send()+ .await?+ .error_for_status()?;+ let mut body = std::pin::pin!(response.bytes_stream());+ let mut decryptor = DecryptingWriter::new(&file.header, &file.key, output);+ while let Some(chunk) = body.next().await {+ decryptor.write(&chunk?)?;+ }+ decryptor.finish()?;+ Ok(())}- }- }- if !buffer.is_empty() {- let (plaintext, _) = decryptor.pull(&buffer)?;- output.write_all(&plaintext)?;- }- decryptor.finish()?;- Ok(())-}--#[derive(Deserialize)]-#[serde(rename_all = "camelCase")]-struct DiffResponse {- diff: Vec<RemoteFile>,- has_more: bool,+ },+ |error| matches!(error, Error::Http(error) if error.is_retryable()),+ )+ .await}#[cfg(not(target_arch = "wasm32"))]@@ -209,43 +152,25 @@ struct DownloadUrl {url: String,}-#[derive(Deserialize)]-#[serde(rename_all = "camelCase")]-struct RemoteFile {- id: i64,- #[serde(rename = "ownerID")]- owner_id: i64,- encrypted_key: String,- key_decryption_nonce: String,- file: FileAttributes,- metadata: FileAttributes,- is_deleted: bool,- updation_time: i64,- magic_metadata: Option<EncryptedMetadata>,- pub_magic_metadata: Option<EncryptedMetadata>,-}--impl RemoteFile {- fn open(self, collection_key: &Key) -> Result<File, Error> {- let key = Key::try_from_slice(&secretbox::decrypt(- &b64::decode(&self.encrypted_key)?,- &Nonce::try_from_slice(&b64::decode(&self.key_decryption_nonce)?)?,- collection_key,- )?)?;+impl File {+ fn open(remote: ente_collections::client::File, collection_key: &Key) -> Result<Self, Error> {+ let key = remote.open_key(collection_key)?;let metadata: Metadata = blob::decrypt_json(&blob::EncryptedBlob {- encrypted_data: b64::decode(self.metadata.encrypted_data.as_deref().unwrap_or(""))?,+ encrypted_data: b64::decode(+ remote.metadata.encrypted_data.as_deref().unwrap_or(""),+ )?,decryption_header: Header::try_from_slice(&b64::decode(- &self.metadata.decryption_header,+ &remote.metadata.decryption_header,)?)?,},&key,)?;- let public: PublicMetadata = match self.pub_magic_metadata {+ let public: PublicMetadata = match remote.pub_magic_metadata {Some(encrypted) => encrypted.open(&key)?,None => PublicMetadata::default(),};- let private: PrivateMetadata = match self.magic_metadata {+ let private: PrivateMetadata = match remote.magic_metadata {Some(encrypted) => encrypted.open(&key)?,None => PrivateMetadata::default(),};@@ -275,9 +200,9 @@ impl RemoteFile {longitude,});Ok(File {- id: self.id,- owner_id: self.owner_id,- updated_at_micros: self.updation_time,+ id: remote.id,+ owner_id: remote.owner_id,+ updated_at_micros: remote.updation_time,name: public.edited_name.unwrap_or(metadata.title),kind,created_at_micros: public.edited_time.unwrap_or(metadata.creation_time),@@ -299,36 +224,11 @@ impl RemoteFile {_ => Visibility::Visible,},key,- header: Header::try_from_slice(&b64::decode(&self.file.decryption_header)?)?,+ header: Header::try_from_slice(&b64::decode(&remote.file.decryption_header)?)?,})}}-#[derive(Deserialize)]-#[serde(rename_all = "camelCase")]-struct FileAttributes {- encrypted_data: Option<String>,- decryption_header: String,-}--#[derive(Deserialize)]-struct EncryptedMetadata {- data: String,- header: String,-}--impl EncryptedMetadata {- fn open<T: serde::de::DeserializeOwned>(&self, key: &Key) -> Result<T, Error> {- Ok(blob::decrypt_json(- &blob::EncryptedBlob {- encrypted_data: b64::decode(&self.data)?,- decryption_header: Header::try_from_slice(&b64::decode(&self.header)?)?,- },- key,- )?)- }-}-#[derive(Deserialize)]#[serde(rename_all = "camelCase")]struct Metadata {