Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Persist Photos metadata in an encrypted CLI replica

Manav Rathifa78665e0dc3
diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml
index 4b7d842240..834e4d2995 100644
--- a/.github/workflows/rust-test.yml
+++ b/.github/workflows/rust-test.yml
@@ -60,10 +60,15 @@ jobs:
~/.cache/go-build
~/.cache/.theseus/postgresql
- - run: cargo test --locked --features museum,ente-ml/ml-assets
+ - run: cargo test --locked --workspace --exclude ente-cli-next --features museum,ente-ml/ml-assets
# postgresql_embedded downloads its binaries from GitHub.
# GITHUB_TOKEN raises the API rate limit for those downloads.
env:
GITHUB_TOKEN: ${{ github.token }}
+ # Avoid unifying SQLCipher into other packages' SQLite tests.
+ - run: cargo test --locked -p ente-cli-next --features museum
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+
- run: apps/cli-next/scripts/test-with-keyring-linux.sh --features museum
diff --git a/.github/workflows/warm-caches.yml b/.github/workflows/warm-caches.yml
index ba5a05df13..927940d91b 100644
--- a/.github/workflows/warm-caches.yml
+++ b/.github/workflows/warm-caches.yml
@@ -71,11 +71,11 @@ jobs:
~/.cache/go-build
~/.cache/.theseus/postgresql
- - run: cargo test --locked --features museum,ente-ml/ml-assets
+ - run: cargo test --locked --workspace --exclude ente-cli-next --features museum,ente-ml/ml-assets
env:
GITHUB_TOKEN: ${{ github.token }}
- - run: cargo test --locked -p ente-cli-next --test cli --features museum --no-run
+ - run: cargo test --locked -p ente-cli-next --features museum --no-run
- run: node .github/actions/restore-rust-target/mtimes.mjs save
working-directory: ${{ github.workspace }}
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index dca96818ef..1903e011d8 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2453,9 +2453,11 @@ dependencies = [
"keyring",
"mockito",
"reqwest 0.12.28",
+ "rusqlite",
"serde",
"serde_json",
"tempfile",
+ "thiserror 2.0.18",
"tokio",
"url",
"uuid",
@@ -2467,6 +2469,7 @@ name = "ente-collections"
version = "0.0.0"
dependencies = [
"ente-core",
+ "serde",
"thiserror 2.0.18",
]
@@ -5689,6 +5692,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8"
dependencies = [
"cc",
+ "openssl-sys",
"pkg-config",
"vcpkg",
]
@@ -6694,6 +6698,15 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+[[package]]
+name = "openssl-src"
+version = "300.6.1+3.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "openssl-sys"
version = "0.9.116"
@@ -6702,6 +6715,7 @@ checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4"
dependencies = [
"cc",
"libc",
+ "openssl-src",
"pkg-config",
"vcpkg",
]
diff --git a/rust/apps/cli-next/Cargo.toml b/rust/apps/cli-next/Cargo.toml
index abfd781438..3761152ba1 100644
--- a/rust/apps/cli-next/Cargo.toml
+++ b/rust/apps/cli-next/Cargo.toml
@@ -18,14 +18,19 @@ ente-core.workspace = true
ente-photos.workspace = true
keyring = { workspace = true, features = ["apple-native", "windows-native", "linux-native-sync-persistent", "crypto-rust"] }
reqwest.workspace = true
+rusqlite = { workspace = true, features = ["bundled-sqlcipher"] }
serde.workspace = true
serde_json.workspace = true
tempfile.workspace = true
+thiserror.workspace = true
tokio = { workspace = true, features = ["macros", "rt"] }
url.workspace = true
uuid = { workspace = true, features = ["serde", "v4"] }
zeroize.workspace = true
+[target.'cfg(windows)'.dependencies]
+rusqlite = { workspace = true, features = ["bundled-sqlcipher-vendored-openssl"] }
+
[dev-dependencies]
ente-test-support.workspace = true
mockito.workspace = true
diff --git a/rust/apps/cli-next/src/args.rs b/rust/apps/cli-next/src/args.rs
index 18a72ad333..a755e6afc4 100644
--- a/rust/apps/cli-next/src/args.rs
+++ b/rust/apps/cli-next/src/args.rs
@@ -10,12 +10,41 @@ use serde::{Deserialize, Serialize};
after_help = "Start with: ente-cli-next photos login"
)]
pub struct Cli {
- #[arg(long, global = true, help = "Print the result as JSON")]
- pub json: bool,
+ #[command(flatten)]
+ pub options: Options,
#[command(subcommand)]
pub command: Command,
}
+#[derive(Args)]
+pub struct Options {
+ #[arg(long, global = true, help = "Print the result as JSON")]
+ pub json: bool,
+ #[arg(long, global = true, help = "Use local data without network access")]
+ pub offline: bool,
+}
+
+pub const DEFAULT_LIST_LIMIT: u32 = 100;
+
+#[derive(Args)]
+pub struct ListArgs {
+ #[arg(
+ long,
+ default_value_t = DEFAULT_LIST_LIMIT,
+ value_parser = clap::value_parser!(u32).range(1..),
+ help = "Maximum number of results"
+ )]
+ pub limit: u32,
+ #[arg(long, conflicts_with = "limit", help = "List every result")]
+ pub all: bool,
+}
+
+impl ListArgs {
+ pub fn limit(&self) -> Option<u32> {
+ (!self.all).then_some(self.limit)
+ }
+}
+
#[derive(Args)]
pub struct AccountSelector {
#[arg(
@@ -73,6 +102,12 @@ Keep the same home and key across unattended invocations."
pub enum PhotosCommand {
#[command(flatten)]
Session(SessionCommand),
+ #[command(flatten)]
+ Library(PhotosLibraryCommand),
+}
+
+#[derive(Subcommand)]
+pub enum PhotosLibraryCommand {
#[command(about = "Manage albums")]
Album {
#[command(subcommand)]
@@ -159,7 +194,7 @@ pub struct ApiArgs {
#[derive(Subcommand)]
pub enum AlbumCommand {
#[command(about = "List your albums, including shared and hidden ones")]
- List,
+ List(ListArgs),
#[command(about = "Show an album")]
View {
#[arg(help = "Album ID or exact name")]
@@ -170,7 +205,7 @@ pub enum AlbumCommand {
#[derive(Subcommand)]
pub enum FileCommand {
#[command(about = "List files, including shared and hidden ones")]
- List,
+ List(ListArgs),
#[command(about = "Show a file's metadata")]
View {
#[arg(help = "File ID or exact name")]
diff --git a/rust/apps/cli-next/src/core_db.rs b/rust/apps/cli-next/src/core_db.rs
new file mode 100644
index 0000000000..acb18bfe60
--- /dev/null
+++ b/rust/apps/cli-next/src/core_db.rs
@@ -0,0 +1,58 @@
+use rusqlite::TransactionBehavior;
+
+pub use rusqlite::{
+ Connection, Error as SqliteError, OptionalExtension, Result as SqliteResult, Row, Transaction,
+ params, params_from_iter, types,
+};
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error(transparent)]
+ Sqlite(#[from] SqliteError),
+ #[error("database version {current} is newer than this build supports ({target})")]
+ Downgrade { current: i64, target: i64 },
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+pub struct Db(Connection);
+
+impl Db {
+ pub fn new(connection: Connection) -> Self {
+ Self(connection)
+ }
+
+ pub fn migrate(&mut self, scripts: &[&str]) -> Result<()> {
+ let target = scripts.len() as i64;
+ let transaction = self
+ .0
+ .transaction_with_behavior(TransactionBehavior::Immediate)?;
+ let current: i64 =
+ transaction.pragma_query_value(None, "user_version", |row| row.get(0))?;
+ if current > target {
+ return Err(Error::Downgrade { current, target });
+ }
+ if current == target {
+ return Ok(());
+ }
+ for script in &scripts[current as usize..] {
+ transaction.execute_batch(script)?;
+ }
+ transaction.pragma_update(None, "user_version", target)?;
+ transaction.commit()?;
+ Ok(())
+ }
+
+ pub fn read<T>(&self, read: impl FnOnce(&Connection) -> Result<T>) -> Result<T> {
+ read(&self.0)
+ }
+
+ pub fn write<T>(&mut self, write: impl FnOnce(&Transaction<'_>) -> Result<T>) -> Result<T> {
+ let transaction = self
+ .0
+ .transaction_with_behavior(TransactionBehavior::Immediate)?;
+ let result = write(&transaction)?;
+ transaction.commit()?;
+ Ok(result)
+ }
+}
diff --git a/rust/apps/cli-next/src/db.rs b/rust/apps/cli-next/src/db.rs
new file mode 100644
index 0000000000..87def43cf9
--- /dev/null
+++ b/rust/apps/cli-next/src/db.rs
@@ -0,0 +1,38 @@
+use std::path::Path;
+
+use anyhow::{Context, Result};
+use rusqlite::{Connection, OpenFlags};
+use zeroize::Zeroizing;
+
+use crate::{core_db::Db, replica, vault::DbKey};
+
+const MIGRATIONS: &[&str] = &[replica::SCHEMA];
+
+pub fn open(account_dir: &Path, key: &DbKey, create: bool) -> Result<Db> {
+ let mut flags = OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX;
+ if create {
+ flags |= OpenFlags::SQLITE_OPEN_CREATE;
+ }
+ let connection = Connection::open_with_flags(account_dir.join("data.db"), flags)?;
+ connection
+ .query_row("PRAGMA cipher_version", [], |row| row.get::<_, String>(0))
+ .context("this CLI build does not support encrypted databases")?;
+ let mut encoded_key = Zeroizing::new(String::from("x'"));
+ for byte in &key.0 {
+ use std::fmt::Write;
+ write!(encoded_key, "{byte:02x}")?;
+ }
+ encoded_key.push('\'');
+ connection.pragma_update(None, "key", encoded_key.as_str())?;
+ connection
+ .query_row("SELECT count(*) FROM sqlite_master", [], |row| {
+ row.get::<_, i64>(0)
+ })
+ .context("cannot open the encrypted database")?;
+ connection.execute_batch(
+ "PRAGMA foreign_keys = ON; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = WAL;",
+ )?;
+ let mut db = Db::new(connection);
+ db.migrate(MIGRATIONS)?;
+ Ok(db)
+}
diff --git a/rust/apps/cli-next/src/home.rs b/rust/apps/cli-next/src/home.rs
new file mode 100644
index 0000000000..b2815f9256
--- /dev/null
+++ b/rust/apps/cli-next/src/home.rs
@@ -0,0 +1,78 @@
+use std::{
+ env,
+ fs::{self, File, OpenOptions},
+ io,
+ path::{Path, PathBuf},
+};
+
+use anyhow::{Context, Result, ensure};
+use uuid::Uuid;
+
+pub struct AccountHome {
+ pub path: PathBuf,
+ _lock: File,
+}
+
+pub fn application_home() -> Result<PathBuf> {
+ if let Some(home) = env::var_os("ENTE_CLI_HOME") {
+ ensure!(!home.is_empty(), "ENTE_CLI_HOME cannot be empty");
+ return Ok(home.into());
+ }
+ Ok(dirs::data_local_dir()
+ .context("cannot find the application data directory; set ENTE_CLI_HOME")?
+ .join("ente-cli"))
+}
+
+pub fn create(path: &Path) -> io::Result<()> {
+ #[cfg(unix)]
+ use std::os::unix::fs::{DirBuilderExt, PermissionsExt};
+
+ let mut builder = fs::DirBuilder::new();
+ builder.recursive(true);
+ #[cfg(unix)]
+ builder.mode(0o700);
+ builder.create(path)?;
+ #[cfg(unix)]
+ fs::set_permissions(path, fs::Permissions::from_mode(0o700))?;
+ Ok(())
+}
+
+pub fn lock_account(id: Uuid, create: bool) -> Result<AccountHome> {
+ let accounts = application_home()?.join("accounts");
+ let path = accounts.join(id.to_string());
+ let lock_path = accounts.join(format!("{id}.lock"));
+ if create {
+ self::create(&accounts)?;
+ } else {
+ ensure!(path.is_dir(), "no local data; run the command online first");
+ }
+ let mut options = OpenOptions::new();
+ options.read(true).write(true).create(true);
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::OpenOptionsExt;
+ options.mode(0o600);
+ }
+ let lock = options
+ .open(lock_path)
+ .context("cannot open the account lock")?;
+ lock.lock().context("cannot lock the account home")?;
+ Ok(AccountHome { path, _lock: lock })
+}
+
+impl AccountHome {
+ pub fn remove(self) -> Result<()> {
+ if let Err(error) = fs::remove_dir_all(&self.path)
+ && error.kind() != io::ErrorKind::NotFound
+ {
+ return Err(error.into());
+ }
+ // The vault must exclude this account before its lock is unlinked.
+ if let Err(error) = fs::remove_file(self.path.with_extension("lock"))
+ && error.kind() != io::ErrorKind::NotFound
+ {
+ return Err(error.into());
+ }
+ Ok(())
+ }
+}
diff --git a/rust/apps/cli-next/src/login.rs b/rust/apps/cli-next/src/login.rs
index a9e57dc3f5..77bd1a4618 100644
--- a/rust/apps/cli-next/src/login.rs
+++ b/rust/apps/cli-next/src/login.rs
@@ -20,7 +20,7 @@ use crate::{
api,
args::{LoginArgs, Product},
parse_json, read_input,
- vault::{Account, AccountKeys, State, StoredSession, Vault},
+ vault::{Account, AccountKeys, DbKey, State, StoredSession, Vault},
};
pub async fn login(
@@ -143,6 +143,7 @@ pub async fn login(
vault.state.check_name(&new_name)?;
let index = vault.state.accounts.len();
vault.state.accounts.push(Account {
+ db_key: DbKey(*ente_core::crypto::Key::generate().as_bytes()),
storage_id: Uuid::new_v4(),
name: new_name,
email,
diff --git a/rust/apps/cli-next/src/main.rs b/rust/apps/cli-next/src/main.rs
index cd24731607..fb0d4645b6 100644
--- a/rust/apps/cli-next/src/main.rs
+++ b/rust/apps/cli-next/src/main.rs
@@ -1,8 +1,12 @@
mod api;
mod args;
+mod core_db;
+mod db;
+mod home;
mod login;
mod output;
mod photos;
+mod replica;
mod vault;
use std::{
@@ -10,7 +14,7 @@ use std::{
path::Path,
};
-use anyhow::{Context, Result, bail};
+use anyhow::{Context, Result, bail, ensure};
use clap::Parser;
use ente_core::{b64, crypto::Key};
use serde::de::DeserializeOwned;
@@ -18,7 +22,8 @@ use serde_json::json;
use zeroize::Zeroizing;
use args::{
- AccountCommand, Cli, Command, KeyCommand, PhotosCommand, Product, SessionCommand, VaultCommand,
+ AccountCommand, Cli, Command, KeyCommand, Options, PhotosCommand, Product, SessionCommand,
+ VaultCommand,
};
use output::AccountView;
use vault::{State, Vault};
@@ -32,33 +37,49 @@ async fn main() {
}
async fn run(cli: Cli) -> Result<()> {
- let Cli { json, command } = cli;
+ let Cli { options, command } = cli;
match command {
Command::Photos {
selector,
command: PhotosCommand::Session(command),
- } => session(Product::Photos, command, selector.account.as_deref(), json).await,
+ } => {
+ session(
+ Product::Photos,
+ command,
+ selector.account.as_deref(),
+ &options,
+ )
+ .await
+ }
+ Command::Photos {
+ selector,
+ command: PhotosCommand::Library(command),
+ } => photos::run(command, selector.account.as_deref(), &options).await,
Command::Locker { selector, command } => {
- session(Product::Locker, command, selector.account.as_deref(), json).await
+ session(
+ Product::Locker,
+ command,
+ selector.account.as_deref(),
+ &options,
+ )
+ .await
}
Command::Auth { selector, command } => {
- session(Product::Auth, command, selector.account.as_deref(), json).await
+ session(
+ Product::Auth,
+ command,
+ selector.account.as_deref(),
+ &options,
+ )
+ .await
}
- Command::Photos {
- selector,
- command: PhotosCommand::Album { command },
- } => photos::album(command, selector.account.as_deref(), json).await,
- Command::Photos {
- selector,
- command: PhotosCommand::File { album, command },
- } => photos::file(command, album.as_deref(), selector.account.as_deref(), json).await,
- Command::Accounts { command } => account_command(command, json).await,
+ Command::Accounts { command } => account_command(command, &options).await,
Command::Vault {
command:
VaultCommand::Key {
command: KeyCommand::Generate,
},
- } => vault_key(json),
+ } => vault_key(options.json),
}
}
@@ -75,8 +96,9 @@ async fn session(
product: Product,
command: SessionCommand,
selected: Option<&str>,
- json_output: bool,
+ options: &Options,
) -> Result<()> {
+ ensure!(!options.offline, "this command requires network access");
match command {
SessionCommand::Api(args) => {
let state = State::load()?;
@@ -86,7 +108,7 @@ async fn session(
SessionCommand::Login(args) => {
let (state, index) = login::login(product, args, selected).await?;
let account = AccountView::new(&state.accounts[index], state.selected);
- if json_output {
+ if options.json {
output::json(&json!({ "account": account, "product": product }))
} else {
output::account(&account)
@@ -95,16 +117,21 @@ async fn session(
SessionCommand::Logout => {
let mut vault = Vault::open()?;
let index = vault.state.resolve(selected)?;
+ let home = home::lock_account(vault.state.accounts[index].storage_id, true)?;
let name = vault.state.accounts[index].name.clone();
api::logout(&vault.state.accounts[index], product).await?;
vault.state.accounts[index].sessions.remove(&product);
- if vault.state.accounts[index].sessions.is_empty() {
+ let removed = vault.state.accounts[index].sessions.is_empty();
+ if removed {
remove_account(&mut vault.state, index);
}
vault.save()?;
+ if removed {
+ home.remove()?;
+ }
drop(vault);
output::action(
- json_output,
+ options.json,
&json!({ "account": name, "product": product, "loggedOut": true }),
&format!("Logged out of {} for {:?}.", product.display_name(), name),
)
@@ -112,7 +139,7 @@ async fn session(
}
}
-async fn account_command(command: AccountCommand, json_output: bool) -> Result<()> {
+async fn account_command(command: AccountCommand, options: &Options) -> Result<()> {
match command {
AccountCommand::List => {
let state = State::load()?;
@@ -121,7 +148,7 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(
.iter()
.map(|a| AccountView::new(a, state.selected))
.collect();
- if json_output {
+ if options.json {
output::json(&accounts)
} else {
output::accounts(&accounts)
@@ -130,7 +157,7 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(
AccountCommand::View { name } => {
let state = State::load()?;
let account = AccountView::new(&state.accounts[state.named(&name)?], state.selected);
- if json_output {
+ if options.json {
output::json(&account)
} else {
output::account(&account)
@@ -145,7 +172,7 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(
state.selected = selected;
vault.save()?;
}
- output_account(vault.into_state(), index, json_output)
+ output_account(vault.into_state(), index, options.json)
}
AccountCommand::Rename { name, new_name } => {
let mut vault = Vault::open()?;
@@ -156,11 +183,16 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(
state.accounts[index].name = new_name;
vault.save()?;
}
- output_account(vault.into_state(), index, json_output)
+ output_account(vault.into_state(), index, options.json)
}
AccountCommand::Logout { name, local } => {
+ ensure!(
+ !options.offline || local,
+ "logging out on the server requires network access; use --local to forget the account on this device"
+ );
let mut vault = Vault::open()?;
let index = vault.state.named(&name)?;
+ let home = home::lock_account(vault.state.accounts[index].storage_id, true)?;
let products = vault.state.accounts[index]
.sessions
.keys()
@@ -188,6 +220,7 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(
}
remove_account(&mut vault.state, index);
vault.save()?;
+ home.remove()?;
drop(vault);
let message = if local && !products.is_empty() {
format!(
@@ -200,7 +233,7 @@ async fn account_command(command: AccountCommand, json_output: bool) -> Result<(
format!("Logged out of {} for {name:?}.", product_names(&products))
};
output::action(
- json_output,
+ options.json,
&json!({
"account": name,
"products": products,
diff --git a/rust/apps/cli-next/src/output.rs b/rust/apps/cli-next/src/output.rs
index db4168a686..6c679ee3d0 100644
--- a/rust/apps/cli-next/src/output.rs
+++ b/rust/apps/cli-next/src/output.rs
@@ -7,7 +7,10 @@ use ente_photos::{collections::Collection, files::File};
use serde::Serialize;
use uuid::Uuid;
-use crate::{args::Product, vault::Account};
+use crate::{
+ args::{DEFAULT_LIST_LIMIT, ListArgs, Product},
+ vault::Account,
+};
#[derive(Serialize)]
pub struct AccountView<'a> {
@@ -62,9 +65,9 @@ impl TryFrom<Collection> for AlbumView {
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
-pub struct FileView<'a> {
+pub struct FileView {
id: String,
- name: &'a str,
+ name: String,
#[serde(rename = "type")]
kind: &'static str,
owner_id: String,
@@ -73,10 +76,10 @@ pub struct FileView<'a> {
modified_at: String,
updated_at: String,
location: Option<LocationView>,
- caption: Option<&'a str>,
- hash: Option<&'a str>,
- date_time: Option<&'a str>,
- offset_time: Option<&'a str>,
+ caption: Option<String>,
+ hash: Option<String>,
+ date_time: Option<String>,
+ offset_time: Option<String>,
duration_seconds: Option<u64>,
width: Option<u32>,
height: Option<u32>,
@@ -89,11 +92,11 @@ struct LocationView {
longitude: f64,
}
-impl<'a> FileView<'a> {
- pub fn new(file: &'a File, album_ids: &[i64]) -> Result<Self> {
+impl FileView {
+ pub fn new(file: File, album_ids: &[i64]) -> Result<Self> {
Ok(Self {
id: file.id.to_string(),
- name: &file.name,
+ name: file.name,
kind: file.kind.name(),
owner_id: file.owner_id.to_string(),
album_ids: album_ids.iter().map(i64::to_string).collect(),
@@ -104,10 +107,10 @@ impl<'a> FileView<'a> {
latitude: location.latitude,
longitude: location.longitude,
}),
- caption: file.caption.as_deref(),
- hash: file.hash.as_deref(),
- date_time: file.date_time.as_deref(),
- offset_time: file.offset_time.as_deref(),
+ caption: file.caption,
+ hash: file.hash,
+ date_time: file.date_time,
+ offset_time: file.offset_time,
duration_seconds: file.duration_seconds,
width: file.width,
height: file.height,
@@ -153,7 +156,8 @@ pub fn accounts(accounts: &[AccountView<'_>]) -> Result<()> {
&mut stdout,
["SELECTED", "NAME", "EMAIL", "HOST", "LOGGED IN", "ID"],
&rows,
- )
+ )?;
+ Ok(())
}
pub fn account(account: &AccountView<'_>) -> Result<()> {
@@ -170,28 +174,27 @@ pub fn account(account: &AccountView<'_>) -> Result<()> {
])
}
-pub fn albums(albums: &[AlbumView]) -> Result<()> {
- let mut stdout = std::io::stdout().lock();
- if albums.is_empty() {
- return writeln!(stdout, "No albums.").map_err(Into::into);
- }
- let rows = albums
- .iter()
- .map(|album| {
+pub fn albums(
+ albums: impl Iterator<Item = Result<AlbumView>>,
+ args: &ListArgs,
+ as_json: bool,
+) -> Result<()> {
+ list(
+ albums,
+ args,
+ as_json,
+ ["ID", "NAME", "TYPE", "VISIBILITY", "OWNER", "UPDATED"],
+ "No albums.",
+ |album| {
[
- album.id.clone(),
+ album.id,
escape_controls(&album.name),
album.kind.to_owned(),
album.visibility.to_owned(),
- album.owner_id.clone(),
- album.updated_at.clone(),
+ album.owner_id,
+ album.updated_at,
]
- })
- .collect::<Vec<_>>();
- write_table(
- &mut stdout,
- ["ID", "NAME", "TYPE", "VISIBILITY", "OWNER", "UPDATED"],
- &rows,
+ },
)
}
@@ -206,33 +209,32 @@ pub fn album(album: &AlbumView) -> Result<()> {
])
}
-pub fn files(files: &[FileView<'_>]) -> Result<()> {
- let mut stdout = std::io::stdout().lock();
- if files.is_empty() {
- return writeln!(stdout, "No files.").map_err(Into::into);
- }
- let rows = files
- .iter()
- .map(|file| {
+pub fn files(
+ files: impl Iterator<Item = Result<FileView>>,
+ args: &ListArgs,
+ as_json: bool,
+) -> Result<()> {
+ list(
+ files,
+ args,
+ as_json,
+ ["ID", "NAME", "TYPE", "CREATED", "ALBUMS"],
+ "No files.",
+ |file| {
[
- file.id.clone(),
- escape_controls(file.name),
+ file.id,
+ escape_controls(&file.name),
file.kind.to_owned(),
- file.created_at.clone(),
+ file.created_at,
file.album_ids.join(","),
]
- })
- .collect::<Vec<_>>();
- write_table(
- &mut stdout,
- ["ID", "NAME", "TYPE", "CREATED", "ALBUMS"],
- &rows,
+ },
)
}
-pub fn file(file: &FileView<'_>) -> Result<()> {
+pub fn file(file: &FileView) -> Result<()> {
let mut fields = vec![
- ("Name", escape_controls(file.name)),
+ ("Name", escape_controls(&file.name)),
("Type", file.kind.to_owned()),
("Visibility", file.visibility.to_owned()),
("Created", file.created_at.clone()),
@@ -243,19 +245,22 @@ pub fn file(file: &FileView<'_>) -> Result<()> {
("ID", file.id.clone()),
];
for (label, value) in [
- ("Caption", file.caption.map(escape_controls)),
+ ("Caption", file.caption.as_deref().map(escape_controls)),
(
"Location",
file.location
.as_ref()
.map(|l| format!("{}, {}", l.latitude, l.longitude)),
),
- ("Date/time", file.date_time.map(escape_controls)),
- ("UTC offset", file.offset_time.map(escape_controls)),
+ ("Date/time", file.date_time.as_deref().map(escape_controls)),
+ (
+ "UTC offset",
+ file.offset_time.as_deref().map(escape_controls),
+ ),
("Duration", file.duration_seconds.map(|s| format!("{s} s"))),
("Width", file.width.map(|w| w.to_string())),
("Height", file.height.map(|h| h.to_string())),
- ("Hash", file.hash.map(escape_controls)),
+ ("Hash", file.hash.as_deref().map(escape_controls)),
] {
if let Some(value) = value {
fields.push((label, value));
@@ -301,11 +306,56 @@ fn write_fields(fields: &[(&str, String)]) -> Result<()> {
Ok(())
}
+fn list<T: Serialize, const N: usize>(
+ mut items: impl Iterator<Item = Result<T>>,
+ args: &ListArgs,
+ as_json: bool,
+ header: [&str; N],
+ empty: &str,
+ row: impl Fn(T) -> [String; N],
+) -> Result<()> {
+ let mut stdout = std::io::stdout().lock();
+ let limit = args.limit().map(|limit| limit as usize);
+ let values = items.by_ref().take(limit.unwrap_or(usize::MAX));
+ if as_json {
+ write!(stdout, "[")?;
+ for (index, value) in values.enumerate() {
+ let value = value?;
+ if index > 0 {
+ write!(stdout, ",")?;
+ }
+ serde_json::to_writer(&mut stdout, &value)?;
+ }
+ writeln!(stdout, "]")?;
+ } else {
+ let mut rows = values.map(|value| value.map(&row));
+ let first = rows
+ .by_ref()
+ .take(DEFAULT_LIST_LIMIT as usize)
+ .collect::<Result<Vec<_>>>()?;
+ if first.is_empty() {
+ writeln!(stdout, "{empty}")?;
+ } else {
+ let widths = write_table(&mut stdout, header, &first)?;
+ for row in rows {
+ write_row(&mut stdout, &row?, &widths)?;
+ }
+ }
+ }
+ stdout.flush()?;
+ if let Some(limit) = limit
+ && items.next().transpose()?.is_some()
+ {
+ eprintln!("Showing first {limit} results; use --limit N or --all to see more.");
+ }
+ Ok(())
+}
+
fn write_table<const N: usize>(
out: &mut impl Write,
header: [&str; N],
rows: &[[String; N]],
-) -> Result<()> {
+) -> Result<[usize; N]> {
let header = header.map(str::to_owned);
let widths: [usize; N] = std::array::from_fn(|column| {
std::iter::once(&header)
@@ -315,22 +365,31 @@ fn write_table<const N: usize>(
.unwrap_or(0)
});
for row in std::iter::once(&header).chain(rows) {
- for (column, cell) in row.iter().enumerate() {
- if column > 0 {
- write!(out, " ")?;
- }
- if column + 1 == N {
- write!(out, "{cell}")?;
- } else {
- write!(
- out,
- "{}",
- pad_str(cell, widths[column], Alignment::Left, None)
- )?;
- }
+ write_row(out, row, &widths)?;
+ }
+ Ok(widths)
+}
+
+fn write_row<const N: usize>(
+ out: &mut impl Write,
+ row: &[String; N],
+ widths: &[usize; N],
+) -> Result<()> {
+ for (column, cell) in row.iter().enumerate() {
+ if column > 0 {
+ write!(out, " ")?;
+ }
+ if column + 1 == N {
+ write!(out, "{cell}")?;
+ } else {
+ write!(
+ out,
+ "{}",
+ pad_str(cell, widths[column], Alignment::Left, None)
+ )?;
}
- writeln!(out)?;
}
+ writeln!(out)?;
Ok(())
}
diff --git a/rust/apps/cli-next/src/photos.rs b/rust/apps/cli-next/src/photos.rs
index 4d181459f6..6aa4e19a37 100644
--- a/rust/apps/cli-next/src/photos.rs
+++ b/rust/apps/cli-next/src/photos.rs
@@ -1,37 +1,79 @@
-use std::{collections::BTreeMap, path::Path};
+use std::{io::Seek, path::Path};
-use anyhow::{Context, Result, bail};
-use ente_photos::{collections, files};
+use anyhow::{Context, Result, bail, ensure};
+use ente_core::Session;
+use ente_photos::files;
use serde_json::json;
use crate::{
api,
- args::{AlbumCommand, FileCommand, Product},
+ args::{AlbumCommand, FileCommand, Options, PhotosLibraryCommand, Product},
+ db, home,
output::{self, AlbumView, FileView},
- vault::State,
+ replica::Replica,
+ vault::{Account, State},
};
-pub async fn album(command: AlbumCommand, selected: Option<&str>, json_output: bool) -> Result<()> {
- let state = State::load()?;
- let account = &state.accounts[state.resolve(selected)?];
- let session = api::session(account, Product::Photos)?;
- let albums = collections::list(&session).await?;
+const MAX_CANDIDATES: usize = 10;
+
+pub async fn run(
+ command: PhotosLibraryCommand,
+ selected: Option<&str>,
+ options: &Options,
+) -> Result<()> {
+ ensure!(
+ !options.offline
+ || !matches!(
+ &command,
+ PhotosLibraryCommand::File {
+ command: FileCommand::Download { .. },
+ ..
+ }
+ ),
+ "original files are not stored locally; download requires network access"
+ );
+ let (account, home) = open_account(selected, !options.offline)?;
+ let session = api::session(&account, Product::Photos)?;
+ let mut db = db::open(&home.path, &account.db_key, !options.offline)?;
+ let mut replica = Replica::new(&mut db);
+ if !options.offline {
+ replica.sync_collections(&session).await?;
+ } else {
+ ensure!(
+ replica.collections_cursor()?.is_some(),
+ "no local Photos data; run the command online first"
+ );
+ }
match command {
- AlbumCommand::List => {
- let albums = albums
- .into_iter()
- .map(AlbumView::try_from)
- .collect::<Result<Vec<_>>>()?;
- if json_output {
- output::json(&albums)
- } else {
- output::albums(&albums)
- }
+ PhotosLibraryCommand::Album { command } => album(command, &replica, options),
+ PhotosLibraryCommand::File { album, command } => {
+ file(command, album.as_deref(), &session, &mut replica, options).await
}
+ }
+}
+
+fn album(command: AlbumCommand, replica: &Replica<'_>, options: &Options) -> Result<()> {
+ match command {
+ AlbumCommand::List(args) => replica.collections(
+ None,
+ args.limit().map(|limit| i64::from(limit) + 1),
+ |rows| {
+ output::albums(
+ rows.map(|album| AlbumView::try_from(album?)),
+ &args,
+ options.json,
+ )
+ },
+ ),
AlbumCommand::View { album } => {
- let album = select(albums, &album, "album", |a| (a.id, &a.name))?;
+ let album =
+ replica.collections(Some(&album), Some((MAX_CANDIDATES + 1) as i64), |rows| {
+ select(rows.map(|album| Ok(album?)), &album, "album", |a| {
+ (a.id, &a.name)
+ })
+ })?;
let album = AlbumView::try_from(album)?;
- if json_output {
+ if options.json {
output::json(&album)
} else {
output::album(&album)
@@ -40,72 +82,89 @@ pub async fn album(command: AlbumCommand, selected: Option<&str>, json_output: b
}
}
-pub async fn file(
+async fn file(
command: FileCommand,
album: Option<&str>,
- selected: Option<&str>,
- json_output: bool,
+ session: &Session,
+ replica: &mut Replica<'_>,
+ options: &Options,
) -> Result<()> {
- let state = State::load()?;
- let account = &state.accounts[state.resolve(selected)?];
- let session = api::session(account, Product::Photos)?;
- let mut albums = collections::list(&session).await?;
- if let Some(selector) = album {
- albums = vec![select(albums, selector, "album", |a| (a.id, &a.name))?];
- }
- let mut entries: BTreeMap<i64, Entry> = BTreeMap::new();
- for album in albums {
- for file in files::list(&session, &album).await? {
- match entries.entry(file.id) {
- std::collections::btree_map::Entry::Vacant(entry) => {
- entry.insert(Entry {
- file,
- album_ids: vec![album.id],
- });
- }
- std::collections::btree_map::Entry::Occupied(mut entry) => {
- let entry = entry.get_mut();
- entry.album_ids.push(album.id);
- if file.updated_at_micros > entry.file.updated_at_micros {
- entry.file = file;
- }
- }
+ let albums =
+ replica.collections(album, album.map(|_| (MAX_CANDIDATES + 1) as i64), |rows| {
+ if let Some(selector) = album {
+ Ok(vec![select(
+ rows.map(|album| Ok(album?)),
+ selector,
+ "album",
+ |a| (a.id, &a.name),
+ )?])
+ } else {
+ rows.map(|album| Ok(album?)).collect::<Result<Vec<_>>>()
}
+ })?;
+ if options.offline {
+ for album in &albums {
+ ensure!(
+ replica.files_synced_to(album.id)?.is_some(),
+ "files for album {:?} have not finished syncing; run the command online first",
+ album.name
+ );
}
+ } else {
+ replica.sync_files(session, &albums).await?;
}
- let mut entries = entries.into_values().collect::<Vec<_>>();
- for entry in &mut entries {
- entry.album_ids.sort_unstable();
- }
+ let album_id = album.map(|_| albums[0].id);
+ let find = |selector: &str| {
+ replica.files(
+ album_id,
+ Some(selector),
+ Some((MAX_CANDIDATES + 1) as i64),
+ |rows| {
+ select(rows.map(|entry| Ok(entry?)), selector, "file", |e| {
+ (e.file.id, &e.file.name)
+ })
+ },
+ )
+ };
match command {
- FileCommand::List => {
- let views = entries
- .iter()
- .map(|entry| FileView::new(&entry.file, &entry.album_ids))
- .collect::<Result<Vec<_>>>()?;
- if json_output {
- output::json(&views)
- } else {
- output::files(&views)
- }
- }
+ FileCommand::List(args) => replica.files(
+ album_id,
+ None,
+ args.limit().map(|limit| i64::from(limit) + 1),
+ |rows| {
+ output::files(
+ rows.map(|entry| {
+ let entry = entry?;
+ FileView::new(entry.file, &entry.album_ids)
+ }),
+ &args,
+ options.json,
+ )
+ },
+ ),
FileCommand::View { file } => {
- let entry = select(entries, &file, "file", |e| (e.file.id, &e.file.name))?;
- let view = FileView::new(&entry.file, &entry.album_ids)?;
- if json_output {
+ let entry = find(&file)?;
+ let view = FileView::new(entry.file, &entry.album_ids)?;
+ if options.json {
output::json(&view)
} else {
output::file(&view)
}
}
FileCommand::Download { file, output } => {
- let entry = select(entries, &file, "file", |e| (e.file.id, &e.file.name))?;
+ let entry = find(&file)?;
let parent = output
.parent()
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or(Path::new("."));
let mut temporary = tempfile::NamedTempFile::new_in(parent)?;
- files::download(&session, &entry.file, temporary.as_file_mut()).await?;
+ files::download(session, &entry.file, || {
+ let file = temporary.as_file_mut();
+ file.set_len(0)?;
+ file.rewind()?;
+ file.try_clone()
+ })
+ .await?;
temporary.as_file().sync_all()?;
let bytes = temporary.as_file().metadata()?.len();
temporary
@@ -113,7 +172,7 @@ pub async fn file(
.map_err(|error| error.error)
.with_context(|| format!("cannot save download to {}", output.display()))?;
output::action(
- json_output,
+ options.json,
&json!({ "id": entry.file.id.to_string(), "output": output, "bytes": bytes }),
&format!("Downloaded {:?} to {}.", entry.file.name, output.display()),
)
@@ -121,34 +180,47 @@ pub async fn file(
}
}
-struct Entry {
- file: files::File,
- album_ids: Vec<i64>,
+fn open_account(selected: Option<&str>, create: bool) -> Result<(Account, home::AccountHome)> {
+ let state = State::load()?;
+ let id = state.accounts[state.resolve(selected)?].storage_id;
+ let home = home::lock_account(id, create)?;
+ let Some(account) = State::load()?
+ .accounts
+ .into_iter()
+ .find(|account| account.storage_id == id)
+ else {
+ home.remove()?;
+ bail!("account was removed while waiting for access");
+ };
+ if create {
+ home::create(&home.path)?;
+ }
+ Ok((account, home))
}
fn select<T>(
- items: Vec<T>,
+ items: impl Iterator<Item = Result<T>>,
selector: &str,
kind: &str,
identity: impl Fn(&T) -> (i64, &str),
) -> Result<T> {
- let mut matches = items
- .into_iter()
- .filter(|item| {
- let (id, name) = identity(item);
- id.to_string() == selector || name == selector
- })
- .collect::<Vec<_>>();
+ let mut matches = items.take(MAX_CANDIDATES + 1).collect::<Result<Vec<_>>>()?;
if matches.len() > 1 {
let candidates = matches
.iter()
+ .take(MAX_CANDIDATES)
.map(|item| {
let (id, name) = identity(item);
format!(" {id} {name:?}")
})
.collect::<Vec<_>>()
.join("\n");
- bail!("{kind} {selector:?} is ambiguous:\n{candidates}");
+ let more = if matches.len() > MAX_CANDIDATES {
+ "\nMore matches omitted; use an ID."
+ } else {
+ ""
+ };
+ bail!("{kind} {selector:?} is ambiguous:\n{candidates}{more}");
}
matches
.pop()
diff --git a/rust/apps/cli-next/src/replica.rs b/rust/apps/cli-next/src/replica.rs
new file mode 100644
index 0000000000..3804524d62
--- /dev/null
+++ b/rust/apps/cli-next/src/replica.rs
@@ -0,0 +1,364 @@
+use anyhow::Result;
+use ente_core::{
+ Session,
+ crypto::{Header, Key},
+};
+
+use ente_photos::{
+ collections::{self, Collection, Visibility},
+ files::{self, File, Location},
+};
+
+use crate::core_db::{
+ self, Connection, Db, OptionalExtension, Row, SqliteError, SqliteResult, params,
+ params_from_iter,
+ types::{Type, Value},
+};
+
+pub struct Entry {
+ pub file: File,
+ pub album_ids: Vec<i64>,
+}
+
+pub struct Replica<'a> {
+ db: &'a mut Db,
+}
+
+const COLLECTION_COLUMNS: &str = "id, name, kind, visibility, owner_id, updated_at, key";
+const FILE_COLUMNS: &str = "f.id, f.owner_id, f.updated_at, f.name, f.kind, f.created_at, f.modified_at,
+ f.latitude, f.longitude, f.caption, f.hash, f.date_time, f.offset_time, f.duration, f.width, f.height,
+ f.visibility, f.key, f.header";
+
+pub const SCHEMA: &str = "
+ CREATE TABLE photos_sync (
+ id INTEGER PRIMARY KEY CHECK (id = 1),
+ classical_collections_cursor INTEGER NOT NULL
+ );
+ CREATE TABLE photos_collections (
+ id INTEGER PRIMARY KEY, name TEXT NOT NULL, kind TEXT NOT NULL,
+ visibility TEXT NOT NULL, owner_id INTEGER NOT NULL, updated_at INTEGER NOT NULL,
+ key BLOB NOT NULL, files_cursor INTEGER NOT NULL DEFAULT 0, files_synced_to INTEGER
+ );
+ CREATE INDEX photos_collections_name ON photos_collections(name);
+ CREATE TABLE photos_files (
+ id INTEGER PRIMARY KEY, owner_id INTEGER NOT NULL, updated_at INTEGER NOT NULL,
+ name TEXT NOT NULL, kind TEXT NOT NULL, created_at INTEGER NOT NULL, modified_at INTEGER NOT NULL,
+ latitude REAL, longitude REAL, caption TEXT, hash TEXT, date_time TEXT, offset_time TEXT,
+ duration INTEGER, width INTEGER, height INTEGER, visibility TEXT NOT NULL, key BLOB NOT NULL, header BLOB NOT NULL
+ );
+ CREATE INDEX photos_files_name ON photos_files(name);
+ CREATE TABLE photos_memberships (
+ collection_id INTEGER NOT NULL REFERENCES photos_collections(id) ON DELETE CASCADE,
+ file_id INTEGER NOT NULL REFERENCES photos_files(id) ON DELETE CASCADE,
+ PRIMARY KEY(collection_id, file_id)
+ );
+ CREATE INDEX photos_memberships_file ON photos_memberships(file_id);
+";
+
+impl<'a> Replica<'a> {
+ pub fn new(db: &'a mut Db) -> Self {
+ Self { db }
+ }
+
+ pub async fn sync_collections(&mut self, session: &Session) -> Result<()> {
+ let since = self.collections_cursor()?.unwrap_or(0);
+ let page = collections::diff(session, since).await?;
+ self.db.write(|tx| {
+ for change in page.changes {
+ match change.collection {
+ Some(c) => {
+ tx.execute("
+ INSERT INTO photos_collections (id, name, kind, visibility, owner_id, updated_at, key)
+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
+ ON CONFLICT(id) DO UPDATE SET name=excluded.name, kind=excluded.kind,
+ visibility=excluded.visibility, owner_id=excluded.owner_id,
+ updated_at=excluded.updated_at, key=excluded.key
+ ", params![c.id, c.name, c.kind.name(), c.visibility.name(), c.owner_id, c.updated_at_micros, c.key.as_bytes()])?;
+ }
+ None => {
+ tx.execute(
+ "DELETE FROM photos_files WHERE id IN
+ (SELECT file_id FROM photos_memberships WHERE collection_id=?1)
+ AND NOT EXISTS (SELECT 1 FROM photos_memberships
+ WHERE file_id=photos_files.id AND collection_id!=?1)",
+ [change.id],
+ )?;
+ tx.execute("DELETE FROM photos_collections WHERE id=?1", [change.id])?;
+ }
+ }
+ }
+ tx.execute("INSERT INTO photos_sync VALUES (1, ?1)
+ ON CONFLICT(id) DO UPDATE SET classical_collections_cursor=excluded.classical_collections_cursor", [page.cursor])?;
+ Ok(())
+ })?;
+ Ok(())
+ }
+
+ pub fn collections<T>(
+ &self,
+ selector: Option<&str>,
+ limit: Option<i64>,
+ read: impl FnOnce(&mut dyn Iterator<Item = SqliteResult<Collection>>) -> Result<T>,
+ ) -> Result<T> {
+ let mut sql = format!("SELECT {COLLECTION_COLUMNS} FROM photos_collections");
+ let mut parameters = Vec::new();
+ if let Some(selector) = selector {
+ sql.push_str(" WHERE id=? OR name=?");
+ parameters.push(canonical_id(selector));
+ parameters.push(Value::Text(selector.to_owned()));
+ }
+ sql.push_str(" ORDER BY id");
+ if let Some(limit) = limit {
+ sql.push_str(" LIMIT ?");
+ parameters.push(Value::Integer(limit));
+ }
+ self.db.read(|conn| {
+ let mut statement = conn.prepare(&sql)?;
+ let mut rows = statement.query_map(params_from_iter(parameters), read_collection)?;
+ Ok(read(&mut rows))
+ })?
+ }
+
+ pub async fn sync_files(
+ &mut self,
+ session: &Session,
+ collections: &[Collection],
+ ) -> Result<()> {
+ for collection in collections {
+ let (mut cursor, synced_to): (i64, Option<i64>) = self.db.read(|conn| {
+ Ok(conn.query_row(
+ "SELECT files_cursor, files_synced_to FROM photos_collections WHERE id=?1",
+ [collection.id],
+ |row| Ok((row.get(0)?, row.get(1)?)),
+ )?)
+ })?;
+ // Museum advances the collection timestamp on file and membership changes.
+ if synced_to.is_some_and(|time| time >= collection.updated_at_micros) {
+ continue;
+ }
+ loop {
+ let page = files::diff(session, collection, cursor).await?;
+ self.db.write(|tx| {
+ for change in page.changes {
+ match change.file {
+ Some(file) => {
+ save_file(tx, &file)?;
+ tx.execute(
+ "INSERT OR IGNORE INTO photos_memberships VALUES (?1, ?2)",
+ params![collection.id, file.id],
+ )?;
+ }
+ None => {
+ tx.execute("DELETE FROM photos_memberships WHERE collection_id=?1 AND file_id=?2",
+ params![collection.id, change.id])?;
+ tx.execute(
+ "DELETE FROM photos_files WHERE id=?1 AND NOT EXISTS
+ (SELECT 1 FROM photos_memberships WHERE file_id=?1)",
+ [change.id],
+ )?;
+ }
+ }
+ }
+ tx.execute(
+ "UPDATE photos_collections SET files_cursor=?1,
+ files_synced_to=CASE WHEN ?2 THEN files_synced_to ELSE ?3 END WHERE id=?4",
+ params![
+ page.cursor,
+ page.has_more,
+ collection.updated_at_micros,
+ collection.id
+ ],
+ )?;
+ Ok(())
+ })?;
+ cursor = page.cursor;
+ if !page.has_more {
+ break;
+ }
+ }
+ }
+ Ok(())
+ }
+
+ pub fn files_synced_to(&self, collection_id: i64) -> core_db::Result<Option<i64>> {
+ self.db.read(|conn| {
+ Ok(conn.query_row(
+ "SELECT files_synced_to FROM photos_collections WHERE id=?1",
+ [collection_id],
+ |row| row.get(0),
+ )?)
+ })
+ }
+
+ pub fn files<T>(
+ &self,
+ album: Option<i64>,
+ selector: Option<&str>,
+ limit: Option<i64>,
+ read: impl FnOnce(&mut dyn Iterator<Item = SqliteResult<Entry>>) -> Result<T>,
+ ) -> Result<T> {
+ let album_ids = if album.is_some() {
+ "json_array(m.collection_id)"
+ } else {
+ "json_group_array(m.collection_id ORDER BY m.collection_id)"
+ };
+ let mut sql = format!(
+ "SELECT {FILE_COLUMNS}, {album_ids}
+ FROM photos_files f JOIN photos_memberships m ON m.file_id=f.id"
+ );
+ let mut parameters = Vec::new();
+ let mut conditions = Vec::new();
+ if let Some(selector) = selector {
+ conditions.push("(f.id=? OR f.name=?)");
+ parameters.push(canonical_id(selector));
+ parameters.push(Value::Text(selector.to_owned()));
+ }
+ if let Some(album) = album {
+ conditions.push("m.collection_id=?");
+ parameters.push(Value::Integer(album));
+ }
+ if !conditions.is_empty() {
+ sql.push_str(" WHERE ");
+ sql.push_str(&conditions.join(" AND "));
+ }
+ if album.is_some() {
+ sql.push_str(" ORDER BY m.file_id");
+ } else {
+ sql.push_str(" GROUP BY f.id ORDER BY f.id");
+ }
+ if let Some(limit) = limit {
+ sql.push_str(" LIMIT ?");
+ parameters.push(Value::Integer(limit));
+ }
+ self.db.read(|conn| {
+ let mut statement = conn.prepare(&sql)?;
+ let mut rows = statement.query_map(params_from_iter(parameters), |row| {
+ let ids: String = row.get(19)?;
+ let album_ids = serde_json::from_str(&ids).map_err(|error| {
+ SqliteError::FromSqlConversionFailure(19, Type::Text, Box::new(error))
+ })?;
+ Ok(Entry {
+ file: read_file(row)?,
+ album_ids,
+ })
+ })?;
+ Ok(read(&mut rows))
+ })?
+ }
+
+ pub fn collections_cursor(&self) -> core_db::Result<Option<i64>> {
+ self.db.read(|conn| {
+ Ok(conn
+ .query_row(
+ "SELECT classical_collections_cursor FROM photos_sync WHERE id=1",
+ [],
+ |row| row.get(0),
+ )
+ .optional()?)
+ })
+ }
+}
+
+fn canonical_id(selector: &str) -> Value {
+ selector
+ .parse::<i64>()
+ .ok()
+ .filter(|id| id.to_string() == selector)
+ .map(Value::Integer)
+ .unwrap_or(Value::Null)
+}
+
+fn read_collection(row: &Row<'_>) -> SqliteResult<Collection> {
+ let key: Vec<u8> = row.get(6)?;
+ Ok(Collection {
+ id: row.get(0)?,
+ name: row.get(1)?,
+ kind: match row.get::<_, String>(2)?.as_str() {
+ "favorites" => collections::Kind::Favorites,
+ "uncategorized" => collections::Kind::Uncategorized,
+ _ => collections::Kind::Album,
+ },
+ visibility: visibility(&row.get::<_, String>(3)?),
+ owner_id: row.get(4)?,
+ updated_at_micros: row.get(5)?,
+ key: Key::try_from_slice(&key).map_err(|error| {
+ SqliteError::FromSqlConversionFailure(6, Type::Blob, Box::new(error))
+ })?,
+ })
+}
+
+fn visibility(name: &str) -> Visibility {
+ match name {
+ "archived" => Visibility::Archived,
+ "hidden" => Visibility::Hidden,
+ _ => Visibility::Visible,
+ }
+}
+
+fn read_file(row: &Row<'_>) -> SqliteResult<File> {
+ let key: Vec<u8> = row.get(17)?;
+ let header: Vec<u8> = row.get(18)?;
+ let latitude: Option<f64> = row.get(7)?;
+ let longitude: Option<f64> = row.get(8)?;
+ Ok(File {
+ id: row.get(0)?,
+ owner_id: row.get(1)?,
+ updated_at_micros: row.get(2)?,
+ name: row.get(3)?,
+ kind: match row.get::<_, String>(4)?.as_str() {
+ "image" => files::Kind::Image,
+ "video" => files::Kind::Video,
+ "livephoto" => files::Kind::LivePhoto,
+ _ => files::Kind::Unknown,
+ },
+ created_at_micros: row.get(5)?,
+ modified_at_micros: row.get(6)?,
+ location: latitude
+ .zip(longitude)
+ .map(|(latitude, longitude)| Location {
+ latitude,
+ longitude,
+ }),
+ caption: row.get(9)?,
+ hash: row.get(10)?,
+ date_time: row.get(11)?,
+ offset_time: row.get(12)?,
+ duration_seconds: row
+ .get::<_, Option<i64>>(13)?
+ .map(u64::try_from)
+ .transpose()
+ .map_err(|error| {
+ SqliteError::FromSqlConversionFailure(13, Type::Integer, Box::new(error))
+ })?,
+ width: row.get(14)?,
+ height: row.get(15)?,
+ visibility: visibility(&row.get::<_, String>(16)?),
+ key: Key::try_from_slice(&key).map_err(|error| {
+ SqliteError::FromSqlConversionFailure(17, Type::Blob, Box::new(error))
+ })?,
+ header: Header::try_from_slice(&header).map_err(|error| {
+ SqliteError::FromSqlConversionFailure(18, Type::Blob, Box::new(error))
+ })?,
+ })
+}
+
+fn save_file(connection: &Connection, file: &File) -> SqliteResult<()> {
+ let duration = file
+ .duration_seconds
+ .map(i64::try_from)
+ .transpose()
+ .map_err(|error| SqliteError::ToSqlConversionFailure(Box::new(error)))?;
+ connection.execute("INSERT INTO photos_files VALUES
+ (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19)
+ ON CONFLICT(id) DO UPDATE SET owner_id=excluded.owner_id, updated_at=excluded.updated_at,
+ name=excluded.name, kind=excluded.kind, created_at=excluded.created_at, modified_at=excluded.modified_at,
+ latitude=excluded.latitude, longitude=excluded.longitude, caption=excluded.caption, hash=excluded.hash,
+ date_time=excluded.date_time, offset_time=excluded.offset_time, duration=excluded.duration,
+ width=excluded.width, height=excluded.height, visibility=excluded.visibility, key=excluded.key, header=excluded.header
+ WHERE excluded.updated_at >= photos_files.updated_at",
+ params![file.id, file.owner_id, file.updated_at_micros, file.name, file.kind.name(), file.created_at_micros,
+ file.modified_at_micros, file.location.as_ref().map(|l| l.latitude), file.location.as_ref().map(|l| l.longitude),
+ file.caption, file.hash, file.date_time, file.offset_time, duration, file.width, file.height,
+ file.visibility.name(), file.key.as_bytes(), file.header.as_bytes()])?;
+ Ok(())
+}
diff --git a/rust/apps/cli-next/src/vault.rs b/rust/apps/cli-next/src/vault.rs
index 843744f233..fcd3ae2c8e 100644
--- a/rust/apps/cli-next/src/vault.rs
+++ b/rust/apps/cli-next/src/vault.rs
@@ -2,7 +2,7 @@ use std::{
collections::BTreeMap,
env,
fs::{self, File, OpenOptions},
- io::{self, ErrorKind},
+ io::ErrorKind,
path::{Path, PathBuf},
};
@@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize};
use uuid::Uuid;
use zeroize::{ZeroizeOnDrop, Zeroizing};
-use crate::{args::Product, parse_json};
+use crate::{args::Product, home, parse_json};
const SCHEMA_VERSION: u8 = 1;
const VAULT_FILE: &str = "vault.json";
@@ -53,6 +53,7 @@ struct EncryptedVault {
#[derive(Serialize, Deserialize)]
pub struct Account {
pub storage_id: Uuid,
+ pub db_key: DbKey,
pub name: String,
pub email: String,
pub origin: String,
@@ -61,6 +62,9 @@ pub struct Account {
pub sessions: BTreeMap<Product, StoredSession>,
}
+#[derive(Serialize, Deserialize, ZeroizeOnDrop)]
+pub struct DbKey(pub [u8; Key::BYTES]);
+
#[derive(Serialize, Deserialize, ZeroizeOnDrop)]
pub struct AccountKeys {
pub master_key: Vec<u8>,
@@ -76,8 +80,8 @@ pub struct StoredSession {
impl Vault {
pub fn open() -> Result<Self> {
let override_key = environment_key()?;
- let home = application_home()?;
- create_home(&home).context("cannot create private CLI home")?;
+ let home = home::application_home()?;
+ home::create(&home).context("cannot create private CLI home")?;
let home = fs::canonicalize(home)?;
let lock = lock(&home)?;
let encrypted = read_vault(&home)?;
@@ -155,7 +159,7 @@ impl VaultAccess {
impl State {
pub fn load() -> Result<Self> {
let override_key = environment_key()?;
- let home = application_home()?;
+ let home = home::application_home()?;
// Atomic replacement makes the file a complete snapshot without a read lock.
let Some(bytes) = read_vault(&home)? else {
return Ok(Self::default());
@@ -227,20 +231,6 @@ fn read_vault(home: &Path) -> Result<Option<Vec<u8>>> {
}
}
-fn create_home(path: &Path) -> io::Result<()> {
- #[cfg(unix)]
- use std::os::unix::fs::{DirBuilderExt, PermissionsExt};
-
- let mut builder = fs::DirBuilder::new();
- builder.recursive(true);
- #[cfg(unix)]
- builder.mode(0o700);
- builder.create(path)?;
- #[cfg(unix)]
- fs::set_permissions(path, fs::Permissions::from_mode(0o700))?;
- Ok(())
-}
-
fn lock(home: &Path) -> Result<File> {
let mut options = OpenOptions::new();
options.read(true).write(true).create(true);
@@ -254,16 +244,6 @@ fn lock(home: &Path) -> Result<File> {
Ok(lock)
}
-fn application_home() -> Result<PathBuf> {
- if let Some(home) = env::var_os("ENTE_CLI_HOME") {
- ensure!(!home.is_empty(), "ENTE_CLI_HOME cannot be empty");
- return Ok(home.into());
- }
- Ok(dirs::data_local_dir()
- .context("cannot find the application data directory; set ENTE_CLI_HOME")?
- .join("ente-cli"))
-}
-
fn environment_key() -> Result<Option<Key>> {
match env::var("ENTE_CLI_VAULT_KEY") {
Ok(value) => decode_key(&Zeroizing::new(value)).map(Some),
diff --git a/rust/apps/cli-next/tests/cli.rs b/rust/apps/cli-next/tests/cli.rs
index 4e008bc530..3d28cb0e33 100644
--- a/rust/apps/cli-next/tests/cli.rs
+++ b/rust/apps/cli-next/tests/cli.rs
@@ -105,7 +105,13 @@ fn deleted_collections_are_filtered_before_decryption() {
}]})
.to_string(),
)
- .expect(3)
+ .expect(1)
+ .create();
+ let subsequent = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::UrlEncoded("sinceTime".into(), "1".into()))
+ .with_body(json!({"collections": []}).to_string())
+ .expect(2)
.create();
let home = TestHome::new();
home.seed(&server.url());
@@ -116,6 +122,7 @@ fn deleted_collections_are_filtered_before_decryption() {
);
assert!(failure(&home.run(&["photos", "album", "view", "12"])).contains("no album matches"));
request.assert();
+ subsequent.assert();
}
#[cfg(unix)]
@@ -413,6 +420,12 @@ fn logout_reconciles_remote_and_local_session_state() {
);
assert_eq!(home.json(&["account", "list"]), json!([]));
assert_eq!(home.read_vault(), json!({"accounts": [], "selected": null}));
+ assert_eq!(
+ fs::read_dir(home.dir.path().join("accounts"))
+ .unwrap()
+ .count(),
+ 0
+ );
revoked.assert();
let unavailable = server
@@ -667,7 +680,7 @@ impl TestHome {
self.write_vault(&json!({
"selected": id,
"accounts": [{
- "storage_id": id, "name": "fixture", "email": "[email protected]",
+ "storage_id": id, "db_key": vec![42u8; 32], "name": "fixture", "email": "[email protected]",
"origin": origin, "user_id": 9007199254740993i64,
"identity": {
"master_key": vec![0u8; 32], "recovery_key": vec![0u8; 32],
diff --git a/rust/apps/cli-next/tests/support/files.rs b/rust/apps/cli-next/tests/support/files.rs
index a23a30b5b4..229c972c01 100644
--- a/rust/apps/cli-next/tests/support/files.rs
+++ b/rust/apps/cli-next/tests/support/files.rs
@@ -1,4 +1,8 @@
use ente_core::crypto::{secretbox, stream};
+use std::sync::{
+ Arc,
+ atomic::{AtomicUsize, Ordering},
+};
use super::*;
@@ -27,9 +31,9 @@ fn file_pages_preserve_membership_changes_and_metadata() {
removed["updationTime"] = json!(20);
removed["file"]["encryptedData"] = json!("-");
removed["metadata"]["encryptedData"] = json!("-");
- let page_one = page(&mut server, 1, 0, json!([first, second]), true);
- let page_two = page(&mut server, 1, 10, json!([removed]), false);
- let duplicate = page(&mut server, 2, 0, json!([first]), false);
+ let page_one = page(&mut server, 1, 0, json!([first, second]), true).create();
+ let page_two = page(&mut server, 1, 10, json!([removed]), false).create();
+ let duplicate = page(&mut server, 2, 0, json!([first]), false).create();
let listed = home.json(&["photos", "file", "list"]);
assert_eq!(listed.as_array().unwrap().len(), 1);
let file = &listed[0];
@@ -83,7 +87,7 @@ fn downloads_publish_only_complete_authenticated_originals() {
.create();
let original = vec![37u8; stream::ENCRYPTION_CHUNK_SIZE + 321];
let (file, encrypted) = remote_file(10, &collection_key, &original);
- page(&mut server, 1, 0, json!([file]), false);
+ page(&mut server, 1, 0, json!([file]), false).create();
let signed = server
.mock("GET", "/files/download/v3/10")
.match_header("x-auth-token", "--8=")
@@ -117,11 +121,19 @@ fn downloads_publish_only_complete_authenticated_originals() {
object.assert();
object.remove();
}
+ let attempts = Arc::new(AtomicUsize::new(0));
+ let count = Arc::clone(&attempts);
let object = server
.mock("GET", "/original")
.match_header("x-auth-token", mockito::Matcher::Missing)
.match_header("x-client-package", mockito::Matcher::Missing)
- .with_body(encrypted)
+ .with_chunked_body(move |writer| {
+ if count.fetch_add(1, Ordering::SeqCst) == 0 {
+ writer.write_all(&encrypted[..stream::DECRYPTION_CHUNK_SIZE + 1])?;
+ return Err(std::io::Error::from(std::io::ErrorKind::ConnectionReset));
+ }
+ writer.write_all(&encrypted)
+ })
.expect_at_least(1)
.create();
let result: Value = serde_json::from_slice(&success(home.run(&args)).stdout).unwrap();
@@ -130,6 +142,7 @@ fn downloads_publish_only_complete_authenticated_originals() {
json!({"id": "10", "output": path, "bytes": original.len()})
);
assert_eq!(fs::read(&path).unwrap(), original);
+ assert_eq!(attempts.load(Ordering::SeqCst), 2);
fs::write(&path, b"existing user file").unwrap();
failure(&home.run(&args));
assert_eq!(fs::read(&path).unwrap(), b"existing user file");
@@ -138,6 +151,402 @@ fn downloads_publish_only_complete_authenticated_originals() {
object.assert();
}
+#[test]
+fn replica_resumes_failed_pages_and_only_refreshes_changed_albums() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let initial = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::UrlEncoded("sinceTime".into(), "0".into()))
+ .with_body(
+ json!({"collections": [collection(1, "First", &key), collection(2, "Second", &key)]})
+ .to_string(),
+ )
+ .expect(1)
+ .create();
+ let unchanged = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::UrlEncoded(
+ "sinceTime".into(),
+ "20".into(),
+ ))
+ .with_body(json!({"collections": []}).to_string())
+ .expect(2)
+ .create();
+ let (first, _) = remote_file(10, &key, b"first");
+ let (second, _) = remote_file(11, &key, b"second");
+ let first_page = page(&mut server, 1, 0, json!([first]), true)
+ .expect(1)
+ .create();
+ let broken_page = server
+ .mock("GET", "/collections/v2/diff")
+ .match_query(mockito::Matcher::AllOf(vec![
+ mockito::Matcher::UrlEncoded("collectionID".into(), "1".into()),
+ mockito::Matcher::UrlEncoded("sinceTime".into(), "10".into()),
+ ]))
+ .with_status(400)
+ .expect(1)
+ .create();
+ failure(&home.run(&["photos", "file", "list"]));
+ assert!(
+ failure(&home.run(&["--offline", "photos", "file", "list", "--album", "First"]))
+ .contains("have not finished syncing")
+ );
+ first_page.assert();
+ broken_page.assert();
+ broken_page.remove();
+ let mut second = second;
+ second["updationTime"] = json!(20);
+ let final_page = page(&mut server, 1, 10, json!([second]), false)
+ .expect(1)
+ .create();
+ let other_album = page(&mut server, 2, 0, json!([first]), false)
+ .expect(1)
+ .create();
+ let files = home.json(&["photos", "file", "list"]);
+ assert_eq!(files.as_array().unwrap().len(), 2);
+ assert_eq!(files[0]["albumIds"], json!(["1", "2"]));
+ assert_eq!(home.json(&["photos", "file", "list"]), files);
+ assert_eq!(home.json(&["--offline", "photos", "file", "list"]), files);
+ initial.assert();
+ unchanged.assert();
+ first_page.assert();
+ final_page.assert();
+ other_album.assert();
+ unchanged.remove();
+
+ let mut deleted_album = collection(1, "First", &key);
+ deleted_album["updationTime"] = json!(30);
+ deleted_album["isDeleted"] = json!(true);
+ let deletion = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::UrlEncoded(
+ "sinceTime".into(),
+ "20".into(),
+ ))
+ .with_body(json!({"collections": [deleted_album]}).to_string())
+ .expect(1)
+ .create();
+ let files = home.json(&["photos", "file", "list"]);
+ assert_eq!(files.as_array().unwrap().len(), 1);
+ assert_eq!(files[0]["albumIds"], json!(["2"]));
+ deletion.assert();
+ other_album.assert();
+
+ let mut updated_album = collection(2, "Second", &key);
+ updated_album["updationTime"] = json!(40);
+ let updated = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::UrlEncoded(
+ "sinceTime".into(),
+ "30".into(),
+ ))
+ .with_body(json!({"collections": [updated_album]}).to_string())
+ .expect(1)
+ .create();
+ let mut removed = first;
+ removed["updationTime"] = json!(40);
+ removed["isDeleted"] = json!(true);
+ let removal = page(&mut server, 2, 10, json!([removed]), false)
+ .expect(1)
+ .create();
+ assert_eq!(home.json(&["photos", "file", "list"]), json!([]));
+ updated.assert();
+ removal.assert();
+}
+
+#[test]
+fn account_replica_is_encrypted_and_available_offline() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ assert!(
+ failure(&home.run(&["photos", "album", "list", "--offline"])).contains("no local data")
+ );
+ assert!(!home.dir.path().join("accounts").exists());
+ let failed_sync = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_status(400)
+ .create();
+ failure(&home.run(&["photos", "album", "list"]));
+ assert!(
+ failure(&home.run(&["photos", "album", "list", "--offline"]))
+ .contains("no local Photos data")
+ );
+ failed_sync.assert();
+ failed_sync.remove();
+ let key = Key::generate();
+ let albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections": [collection(1, "Private album", &key)]}).to_string())
+ .expect(2)
+ .create();
+ home.json(&["photos", "album", "list"]);
+ let state = home.read_vault();
+ let directory = home
+ .dir
+ .path()
+ .join("accounts")
+ .join(state["accounts"][0]["storage_id"].as_str().unwrap());
+ let path = directory.join("data.db");
+ let lock_path = directory.with_extension("lock");
+ let connection = rusqlite::Connection::open(&path).unwrap();
+ connection
+ .pragma_update(None, "key", format!("x'{}'", "2a".repeat(32)))
+ .unwrap();
+ connection
+ .query_row("SELECT count(*) FROM sqlite_master", [], |row| {
+ row.get::<_, i64>(0)
+ })
+ .unwrap();
+ let (file, _) = remote_file(10, &key, b"original");
+ let files_page = page(&mut server, 1, 0, json!([file]), false)
+ .expect(1)
+ .create();
+ let files = home.json(&["photos", "file", "list"]);
+ assert!(directory.join("data.db-wal").exists());
+ for filename in ["data.db", "data.db-wal"] {
+ let bytes = fs::read(directory.join(filename)).unwrap();
+ assert!(!bytes.is_empty());
+ for plaintext in ["SQLite format 3", "Private album", "Edited 🌧.jpg"] {
+ assert!(
+ !bytes
+ .windows(plaintext.len())
+ .any(|window| window == plaintext.as_bytes())
+ );
+ }
+ }
+ drop(connection);
+ let unkeyed = rusqlite::Connection::open(&path).unwrap();
+ assert!(
+ unkeyed
+ .query_row("SELECT count(*) FROM sqlite_master", [], |row| row
+ .get::<_, i64>(0))
+ .is_err()
+ );
+ drop(unkeyed);
+ albums.assert();
+ files_page.assert();
+ drop(server);
+ assert_eq!(home.json(&["photos", "file", "list", "--offline"]), files);
+ assert!(
+ failure(&home.run(&[
+ "photos",
+ "file",
+ "download",
+ "10",
+ "--output",
+ "unused.jpg",
+ "--offline"
+ ]))
+ .contains("download requires network access")
+ );
+
+ let mut wrong_key = state.clone();
+ wrong_key["accounts"][0]["db_key"] = json!(vec![43u8; 32]);
+ home.write_vault(&wrong_key);
+ assert!(
+ failure(&home.run(&["photos", "album", "list", "--offline"]))
+ .contains("cannot open the encrypted database")
+ );
+ home.write_vault(&state);
+ fs::remove_file(&lock_path).unwrap();
+ assert_eq!(home.json(&["photos", "file", "list", "--offline"]), files);
+ assert!(lock_path.is_file());
+ home.json(&["accounts", "rename", "fixture", "renamed"]);
+ assert_eq!(
+ home.json(&[
+ "photos",
+ "file",
+ "list",
+ "--account",
+ "renamed",
+ "--offline"
+ ]),
+ files
+ );
+ let mut two_accounts = home.read_vault();
+ let mut other = two_accounts["accounts"][0].clone();
+ other["storage_id"] = json!("45b3495e-92de-43a1-a58b-e0b1e57d8c68");
+ other["name"] = json!("other");
+ other["user_id"] = json!(123);
+ other["db_key"] = json!(vec![43u8; 32]);
+ two_accounts["accounts"].as_array_mut().unwrap().push(other);
+ home.write_vault(&two_accounts);
+ assert!(
+ failure(&home.run(&["photos", "album", "list", "--account", "other", "--offline"]))
+ .contains("no local data")
+ );
+ assert!(
+ !home
+ .dir
+ .path()
+ .join("accounts")
+ .join("45b3495e-92de-43a1-a58b-e0b1e57d8c68.lock")
+ .exists()
+ );
+ let other_directory = home
+ .dir
+ .path()
+ .join("accounts/45b3495e-92de-43a1-a58b-e0b1e57d8c68");
+ fs::create_dir(&other_directory).unwrap();
+ assert!(
+ failure(&home.run(&["photos", "album", "list", "--account", "other", "--offline"]))
+ .contains("unable to open database file")
+ );
+ assert!(!other_directory.join("data.db").exists());
+ home.json(&["accounts", "logout", "renamed", "--local", "--offline"]);
+ assert!(!directory.exists());
+ assert!(!lock_path.exists());
+}
+
+#[test]
+fn photos_lists_limit_output_and_stream_complete_results() {
+ let mut server = mockito::Server::new();
+ let home = TestHome::new();
+ home.seed(&server.url());
+ let key = Key::generate();
+ let albums = (1..=150)
+ .map(|id| collection(id, "Same album", &key))
+ .collect::<Vec<_>>();
+ let albums = server
+ .mock("GET", "/collections/v2")
+ .match_query(mockito::Matcher::Any)
+ .with_body(json!({"collections": albums}).to_string())
+ .expect(2)
+ .create();
+ let (file, _) = remote_file(1, &key, b"original");
+ let files = (1..=150)
+ .map(|id| {
+ let mut file = file.clone();
+ file["id"] = json!(id);
+ file
+ })
+ .collect::<Vec<_>>();
+ let files = page(&mut server, 1, 0, json!(files), false)
+ .expect(1)
+ .create();
+ let output = success(home.run(&["photos", "album", "list", "--json"]));
+ let listed: Vec<Value> = serde_json::from_slice(&output.stdout).unwrap();
+ assert_eq!(listed.len(), 100);
+ assert!(String::from_utf8_lossy(&output.stderr).contains("Showing first 100"));
+ assert_eq!(
+ home.json(&["photos", "file", "list", "--album", "1", "--limit", "1"])
+ .as_array()
+ .unwrap()
+ .len(),
+ 1
+ );
+ albums.assert();
+ files.assert();
+ drop(server);
+
+ for resource in ["album", "file"] {
+ let mut base = vec!["--offline", "photos", resource, "list"];
+ if resource == "file" {
+ base.extend(["--album", "1"]);
+ }
+ for (extra, count, truncated) in [
+ (vec![], 100, true),
+ (vec!["--limit", "1"], 1, true),
+ (vec!["--limit", "150"], 150, false),
+ (vec!["--limit", "4294967295"], 150, false),
+ (vec!["--all"], 150, false),
+ ] {
+ for as_json in [false, true] {
+ let mut args = base.clone();
+ args.extend(&extra);
+ if as_json {
+ args.push("--json");
+ }
+ let output = success(home.run(&args));
+ if as_json {
+ let values: Vec<Value> = serde_json::from_slice(&output.stdout).unwrap();
+ assert_eq!(values.len(), count);
+ assert_eq!(values[0]["id"], "1");
+ assert_eq!(values[count - 1]["id"], count.to_string());
+ } else {
+ assert_eq!(
+ String::from_utf8_lossy(&output.stdout).lines().count(),
+ count + 1
+ );
+ }
+ assert_eq!(!output.stderr.is_empty(), truncated);
+ }
+ }
+ for extra in [
+ vec!["--limit", "0"],
+ vec!["--limit", "-1"],
+ vec!["--all", "--limit", "1"],
+ ] {
+ let mut args = base.clone();
+ args.extend(extra);
+ failure(&home.run(&args));
+ }
+ }
+ for args in [
+ vec!["--offline", "photos", "album", "view", "Same album"],
+ vec![
+ "--offline",
+ "photos",
+ "file",
+ "view",
+ "Edited 🌧.jpg",
+ "--album",
+ "1",
+ ],
+ ] {
+ let output = home.run(&args);
+ let error = failure(&output);
+ assert!(output.stdout.is_empty());
+ assert_eq!(
+ error.lines().filter(|line| line.starts_with(" ")).count(),
+ 10
+ );
+ assert!(error.contains("More matches omitted"));
+ }
+
+ let state = home.read_vault();
+ let database = home
+ .dir
+ .path()
+ .join("accounts")
+ .join(state["accounts"][0]["storage_id"].as_str().unwrap())
+ .join("data.db");
+ let connection = rusqlite::Connection::open(database).unwrap();
+ connection
+ .pragma_update(None, "key", format!("x'{}'", "2a".repeat(32)))
+ .unwrap();
+ connection
+ .execute(
+ "UPDATE photos_files SET created_at=?1 WHERE id=150",
+ [i64::MAX],
+ )
+ .unwrap();
+ drop(connection);
+ for as_json in [false, true] {
+ let mut args = vec![
+ "--offline",
+ "photos",
+ "file",
+ "list",
+ "--album",
+ "1",
+ "--all",
+ ];
+ if as_json {
+ args.push("--json");
+ }
+ let output = home.run(&args);
+ assert!(failure(&output).contains("timestamp is outside the supported range"));
+ assert!(String::from_utf8_lossy(&output.stdout).contains("Edited 🌧.jpg"));
+ }
+}
+
fn collection(id: i64, name: &str, key: &Key) -> Value {
let wrapped = secretbox::encrypt(key.as_bytes(), &Key::from_bytes([0; 32]));
json!({
@@ -205,6 +614,4 @@ fn page(
mockito::Matcher::UrlEncoded("sinceTime".into(), since.to_string()),
]))
.with_body(json!({"diff": diff, "hasMore": more}).to_string())
- .expect_at_least(1)
- .create()
}
diff --git a/rust/apps/cli-next/tests/support/museum.rs b/rust/apps/cli-next/tests/support/museum.rs
index 04ff51b1f7..73ba8ae9ae 100644
--- a/rust/apps/cli-next/tests/support/museum.rs
+++ b/rust/apps/cli-next/tests/support/museum.rs
@@ -2,7 +2,10 @@ use ente_accounts::{
AccountsClient, AccountsClientConfig, AuthFlow, AuthFlowUi, AuthenticatedAccount,
CreateAccountParams, OtpPurpose, SecondFactorMethod, TotpPurpose,
};
-use ente_core::crypto::{PublicKey, blob, sealed, secretbox, stream};
+use ente_core::{
+ crypto::{PublicKey, blob, sealed, secretbox, stream},
+ io::Md5Writer,
+};
use ente_test_support::{HARDCODED_OTT, Museum, TestResult};
use uuid::Uuid;
use zeroize::Zeroizing;
@@ -366,6 +369,28 @@ async fn exercise(origin: String) -> TestResult {
assert_eq!(fs::read(output_path).unwrap(), original);
login(&home, "photos", &alice_email, &["--host", &origin]);
+ let added_file = upload_file(&origin, &alice, album, &key, b"added after the first sync")
+ .await
+ .to_string();
+ assert_eq!(home.json(&["photos", "file", "list", "--offline"]), files);
+ let refreshed = home.json(&["photos", "file", "list", "--all"]);
+ assert_eq!(refreshed.as_array().unwrap().len(), 2);
+ assert!(
+ refreshed
+ .as_array()
+ .unwrap()
+ .iter()
+ .any(|file| file["id"] == added_file)
+ );
+ assert_eq!(
+ home.json(&["photos", "file", "list", "--offline", "--all"]),
+ refreshed
+ );
+ assert_eq!(
+ home.json(&["photos", "file", "list", "--offline", "--limit", "1"]),
+ json!([refreshed[0]])
+ );
+
for selector in ["Monsoon 🌧", &album.to_string()] {
let (conflict, _) = create_album(&origin, &alice, selector, "album").await;
let output = home.run(&["photos", "album", "view", selector, "--json"]);
@@ -414,9 +439,26 @@ async fn exercise(origin: String) -> TestResult {
home.json(&["account", "view", "work"])["products"],
json!(["locker"])
);
+ let account_dir = home
+ .dir
+ .path()
+ .join("accounts")
+ .join(first_id.as_str().unwrap());
+ assert!(account_dir.join("data.db").is_file());
+ success(home.run(&["account", "logout", "work", "--local"]));
+ assert!(!account_dir.exists());
+ assert!(!account_dir.with_extension("lock").exists());
for entry in fs::read_dir(home.dir.path()).unwrap() {
let entry = entry.unwrap();
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt;
+ assert_eq!(entry.metadata().unwrap().permissions().mode() & 0o077, 0);
+ }
+ if entry.file_type().unwrap().is_dir() {
+ continue;
+ }
let bytes = fs::read(entry.path()).unwrap();
assert!(
!bytes
@@ -428,11 +470,6 @@ async fn exercise(origin: String) -> TestResult {
.windows(PASSWORD.len())
.any(|b| b == PASSWORD.as_bytes())
);
- #[cfg(unix)]
- {
- use std::os::unix::fs::PermissionsExt;
- assert_eq!(entry.metadata().unwrap().permissions().mode() & 0o077, 0);
- }
}
Ok(())
}
@@ -516,37 +553,40 @@ async fn upload_file(
) -> i64 {
let client = reqwest::Client::new();
let token = b64::encode_url_safe(&owner.secrets.token);
- let urls: Value = client
- .get(format!("{origin}/files/upload-urls"))
- .header("x-auth-token", &token)
- .header("x-client-package", "io.ente.photos")
- .query(&[("count", 2)])
- .send()
- .await
- .unwrap()
- .error_for_status()
- .unwrap()
- .json()
- .await
- .unwrap();
let key = Key::generate();
let wrapped = secretbox::encrypt(key.as_bytes(), collection_key);
let mut encrypted = Vec::new();
let header =
stream::encrypt_file(&mut std::io::Cursor::new(original), &mut encrypted, &key).unwrap();
let thumbnail = blob::encrypt(b"thumbnail", &key).unwrap();
- for (index, bytes) in [&encrypted, &thumbnail.encrypted_data]
- .into_iter()
- .enumerate()
- {
+ let mut object_keys = Vec::new();
+ for bytes in [&encrypted, &thumbnail.encrypted_data] {
+ let mut checksum = Md5Writer::new(std::io::sink());
+ checksum.write_all(bytes).unwrap();
+ let checksum = b64::encode(&checksum.finalize().1);
+ let upload: Value = client
+ .post(format!("{origin}/files/upload-url"))
+ .header("x-auth-token", &token)
+ .header("x-client-package", "io.ente.photos")
+ .json(&json!({"contentLength": bytes.len(), "contentMD5": checksum}))
+ .send()
+ .await
+ .unwrap()
+ .error_for_status()
+ .unwrap()
+ .json()
+ .await
+ .unwrap();
client
- .put(urls["urls"][index]["url"].as_str().unwrap())
+ .put(upload["url"].as_str().unwrap())
+ .header("content-md5", checksum)
.body(bytes.clone())
.send()
.await
.unwrap()
.error_for_status()
.unwrap();
+ object_keys.push(upload["objectKey"].as_str().unwrap().to_owned());
}
let metadata = blob::encrypt_json(
&json!({
@@ -561,8 +601,8 @@ async fn upload_file(
.json(&json!({
"collectionID": album, "encryptedKey": b64::encode(&wrapped.encrypted_data),
"keyDecryptionNonce": b64::encode(wrapped.nonce.as_bytes()),
- "file": {"objectKey": urls["urls"][0]["objectKey"], "decryptionHeader": b64::encode(header.as_bytes())},
- "thumbnail": {"objectKey": urls["urls"][1]["objectKey"], "decryptionHeader": b64::encode(thumbnail.decryption_header.as_bytes())},
+ "file": {"objectKey": object_keys[0], "decryptionHeader": b64::encode(header.as_bytes())},
+ "thumbnail": {"objectKey": object_keys[1], "decryptionHeader": b64::encode(thumbnail.decryption_header.as_bytes())},
"metadata": {"encryptedData": b64::encode(&metadata.encrypted_data),
"decryptionHeader": b64::encode(metadata.decryption_header.as_bytes())}
})).send().await.unwrap().error_for_status().unwrap().json().await.unwrap();
diff --git a/rust/crates/collections/Cargo.toml b/rust/crates/collections/Cargo.toml
index cc24b50c20..71a7ba6428 100644
--- a/rust/crates/collections/Cargo.toml
+++ b/rust/crates/collections/Cargo.toml
@@ -4,6 +4,7 @@ edition.workspace = true
[dependencies]
ente-core.workspace = true
+serde.workspace = true
thiserror.workspace = true
[lints]
diff --git a/rust/crates/collections/src/client.rs b/rust/crates/collections/src/client.rs
new file mode 100644
index 0000000000..3ec6ee0c87
--- /dev/null
+++ b/rust/crates/collections/src/client.rs
@@ -0,0 +1,204 @@
+use ente_core::{
+ Session, b64,
+ crypto::{Header, Key, Nonce, blob, secretbox},
+ http,
+};
+use serde::{Deserialize, de::DeserializeOwned};
+
+use crate::{Error, Result, open_collection_key};
+
+pub struct CollectionPage {
+ pub collections: Vec<Collection>,
+ pub cursor: i64,
+}
+
+pub async fn diff(session: &Session, since: i64) -> Result<CollectionPage> {
+ #[derive(Deserialize)]
+ struct Response {
+ collections: Vec<Collection>,
+ }
+ let response: Response = http::retry(|| async {
+ session
+ .api
+ .get("/collections/v2")
+ .query(&[("sinceTime", since)])
+ .send()
+ .await?
+ .error_for_code()
+ .await?
+ .json()
+ .await
+ })
+ .await?;
+ let cursor = response
+ .collections
+ .iter()
+ .map(|c| c.updation_time)
+ .max()
+ .unwrap_or(since);
+ Ok(CollectionPage {
+ collections: response.collections,
+ cursor,
+ })
+}
+
+pub struct FilePage {
+ pub files: Vec<File>,
+ pub cursor: i64,
+ pub has_more: bool,
+}
+
+pub async fn files_diff(session: &Session, collection_id: i64, since: i64) -> Result<FilePage> {
+ #[derive(Deserialize)]
+ #[serde(rename_all = "camelCase")]
+ struct Response {
+ diff: Vec<File>,
+ has_more: bool,
+ }
+ let response: Response = http::retry(|| async {
+ session
+ .api
+ .get("/collections/v2/diff")
+ .query(&[("collectionID", collection_id), ("sinceTime", since)])
+ .send()
+ .await?
+ .error_for_code()
+ .await?
+ .json()
+ .await
+ })
+ .await?;
+ let cursor = response
+ .diff
+ .iter()
+ .map(|f| f.updation_time)
+ .max()
+ .unwrap_or(since);
+ if response.has_more && cursor <= since {
+ return Err(Error::CursorDidNotAdvance(collection_id));
+ }
+ Ok(FilePage {
+ files: response.diff,
+ cursor,
+ has_more: response.has_more,
+ })
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Collection {
+ pub id: i64,
+ pub owner: Owner,
+ pub encrypted_key: String,
+ pub key_decryption_nonce: Option<String>,
+ pub encrypted_name: Option<String>,
+ pub name_decryption_nonce: Option<String>,
+ pub name: Option<String>,
+ #[serde(rename = "type")]
+ pub kind: String,
+ pub updation_time: i64,
+ pub is_deleted: Option<bool>,
+ pub magic_metadata: Option<EncryptedMetadata>,
+ pub shared_magic_metadata: Option<EncryptedMetadata>,
+}
+
+impl Collection {
+ pub fn open_key(&self, session: &Session) -> Result<Key> {
+ open_collection_key(
+ session,
+ self.owner.id,
+ &self.encrypted_key,
+ self.key_decryption_nonce.as_deref(),
+ )
+ }
+
+ pub fn open_name(&self, key: &Key) -> Result<String> {
+ if let Some(name) = self.name.as_ref().filter(|name| !name.is_empty()) {
+ return Ok(name.clone());
+ }
+ let encrypted = self
+ .encrypted_name
+ .as_deref()
+ .ok_or(Error::InvalidCollection {
+ id: self.id,
+ reason: "missing name",
+ })?;
+ let nonce = self
+ .name_decryption_nonce
+ .as_deref()
+ .ok_or(Error::InvalidCollection {
+ id: self.id,
+ reason: "missing name nonce",
+ })?;
+ let bytes = secretbox::decrypt(
+ &b64::decode(encrypted)?,
+ &Nonce::try_from_slice(&b64::decode(nonce)?)?,
+ key,
+ )?;
+ String::from_utf8(bytes).map_err(|_| Error::InvalidCollection {
+ id: self.id,
+ reason: "name is not UTF-8",
+ })
+ }
+}
+
+#[derive(Deserialize)]
+pub struct Owner {
+ pub id: i64,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct File {
+ pub id: i64,
+ #[serde(rename = "ownerID")]
+ pub owner_id: i64,
+ pub encrypted_key: String,
+ pub key_decryption_nonce: String,
+ pub file: FileAttributes,
+ pub metadata: FileAttributes,
+ pub is_deleted: bool,
+ pub updation_time: i64,
+ pub magic_metadata: Option<EncryptedMetadata>,
+ pub pub_magic_metadata: Option<EncryptedMetadata>,
+}
+
+impl File {
+ pub fn is_deleted(&self) -> bool {
+ self.is_deleted || self.metadata.encrypted_data.as_deref() == Some("-")
+ }
+
+ pub fn open_key(&self, collection_key: &Key) -> Result<Key> {
+ let bytes = secretbox::decrypt(
+ &b64::decode(&self.encrypted_key)?,
+ &Nonce::try_from_slice(&b64::decode(&self.key_decryption_nonce)?)?,
+ collection_key,
+ )?;
+ Ok(Key::try_from_slice(&bytes)?)
+ }
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FileAttributes {
+ pub encrypted_data: Option<String>,
+ pub decryption_header: String,
+}
+
+#[derive(Deserialize)]
+pub struct EncryptedMetadata {
+ pub data: String,
+ pub header: String,
+}
+
+impl EncryptedMetadata {
+ pub fn open<T: DeserializeOwned>(&self, key: &Key) -> Result<T> {
+ Ok(blob::decrypt_json(
+ &blob::EncryptedBlob {
+ encrypted_data: b64::decode(&self.data)?,
+ decryption_header: Header::try_from_slice(&b64::decode(&self.header)?)?,
+ },
+ key,
+ )?)
+ }
+}
diff --git a/rust/crates/collections/src/lib.rs b/rust/crates/collections/src/lib.rs
index 06ee4aec41..f932f360dc 100644
--- a/rust/crates/collections/src/lib.rs
+++ b/rust/crates/collections/src/lib.rs
@@ -1,3 +1,5 @@
+pub mod client;
+
use ente_core::{
Session, b64,
crypto::{self, Key, Nonce, sealed, secretbox},
@@ -31,12 +33,18 @@ pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug, thiserror::Error)]
pub enum Error {
+ #[error(transparent)]
+ Http(#[from] ente_core::http::Error),
#[error(transparent)]
Decode(#[from] b64::DecodeError),
#[error(transparent)]
Crypto(#[from] crypto::Error),
#[error("Owned collection is missing its key decryption nonce")]
MissingKeyDecryptionNonce,
+ #[error("invalid collection {id}: {reason}")]
+ InvalidCollection { id: i64, reason: &'static str },
+ #[error("file changes did not advance the cursor for collection {0}")]
+ CursorDidNotAdvance(i64),
}
#[cfg(test)]
diff --git a/rust/crates/core/src/crypto/stream.rs b/rust/crates/core/src/crypto/stream.rs
index def2e15bdf..4a05459df9 100644
--- a/rust/crates/core/src/crypto/stream.rs
+++ b/rust/crates/core/src/crypto/stream.rs
@@ -238,6 +238,52 @@ pub struct StreamingDecryptor<R: Read> {
seen_final: bool,
}
+pub struct DecryptingWriter<W: Write> {
+ decryptor: Decryptor,
+ writer: W,
+ buffer: Vec<u8>,
+}
+
+impl<W: Write> DecryptingWriter<W> {
+ pub fn new(header: &Header, key: &Key, writer: W) -> Self {
+ Self {
+ decryptor: Decryptor::new(header, key),
+ writer,
+ buffer: Vec::with_capacity(DECRYPTION_CHUNK_SIZE),
+ }
+ }
+
+ pub fn write(&mut self, mut data: &[u8]) -> Result<()> {
+ while !data.is_empty() {
+ if self.decryptor.seen_final {
+ return Err(Error::StreamTrailingData);
+ }
+ let count = data.len().min(DECRYPTION_CHUNK_SIZE - self.buffer.len());
+ self.buffer.extend_from_slice(&data[..count]);
+ data = &data[count..];
+ if self.buffer.len() == DECRYPTION_CHUNK_SIZE {
+ self.flush_chunk()?;
+ }
+ }
+ Ok(())
+ }
+
+ fn flush_chunk(&mut self) -> Result<()> {
+ self.decryptor.pull_in_place(&mut self.buffer)?;
+ self.writer.write_all(&self.buffer)?;
+ self.buffer.clear();
+ Ok(())
+ }
+
+ pub fn finish(mut self) -> Result<W> {
+ if !self.buffer.is_empty() {
+ self.flush_chunk()?;
+ }
+ self.decryptor.finish()?;
+ Ok(self.writer)
+ }
+}
+
fn ensure_reader_exhausted<R: Read>(reader: &mut R) -> Result<()> {
let mut extra = [0u8; 1];
@@ -865,6 +911,22 @@ mod tests {
assert_eq!(ABYTES, 17);
}
+ #[test]
+ fn decrypting_writer_accepts_arbitrary_chunk_boundaries() {
+ let key = Key::generate();
+ for size in [0, 2 * ENCRYPTION_CHUNK_SIZE + 71] {
+ let plaintext = vec![37; size];
+ let mut ciphertext = Vec::new();
+ let header = encrypt_file(&mut plaintext.as_slice(), &mut ciphertext, &key).unwrap();
+ let mut writer = DecryptingWriter::new(&header, &key, Vec::new());
+ writer.write(&ciphertext[..13]).unwrap();
+ for chunk in ciphertext[13..].chunks(DECRYPTION_CHUNK_SIZE + 5) {
+ writer.write(chunk).unwrap();
+ }
+ assert_eq!(writer.finish().unwrap(), plaintext);
+ }
+ }
+
#[test]
fn test_estimate_encrypted_size() {
assert_eq!(estimate_encrypted_size(0), ABYTES);
diff --git a/rust/crates/core/src/http.rs b/rust/crates/core/src/http.rs
index 37a6f6c81c..b18f68aa58 100644
--- a/rust/crates/core/src/http.rs
+++ b/rust/crates/core/src/http.rs
@@ -77,7 +77,8 @@ impl Error {
pub fn is_retryable(&self) -> bool {
match self {
- Error::Network(e) => e.0.is_request() || e.0.is_body(),
+ // reqwest also reports interrupted bodies as decoding errors.
+ Error::Network(e) => e.0.is_request() || e.0.is_body() || e.0.is_decode(),
Error::Http { status, .. } | Error::Api { status, .. } => {
*status == 429 || *status >= 500
}
@@ -450,16 +451,38 @@ where
retry_with_delays(&profile.delays(), operation).await
}
+pub async fn retry_if<T, E, F, Fut>(operation: F, should_retry: impl Fn(&E) -> bool) -> Result<T, E>
+where
+ E: std::fmt::Display,
+ F: FnMut() -> Fut,
+ Fut: Future<Output = Result<T, E>>,
+{
+ retry_operation(&RetryProfile::Interactive.delays(), operation, should_retry).await
+}
+
async fn retry_with_delays<T, F, Fut>(delays: &[Duration], mut operation: F) -> Result<T, Error>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, Error>>,
+{
+ retry_operation(delays, &mut operation, Error::is_retryable).await
+}
+
+async fn retry_operation<T, E, F, Fut>(
+ delays: &[Duration],
+ mut operation: F,
+ should_retry: impl Fn(&E) -> bool,
+) -> Result<T, E>
+where
+ E: std::fmt::Display,
+ F: FnMut() -> Fut,
+ Fut: Future<Output = Result<T, E>>,
{
let mut delays = delays.iter();
loop {
match operation().await {
Ok(value) => return Ok(value),
- Err(error) if error.is_retryable() => match delays.next() {
+ Err(error) if should_retry(&error) => match delays.next() {
Some(delay) => {
log::warn!("retrying in {delay:?}: {error}");
sleep(*delay).await;
@@ -745,6 +768,7 @@ mod tests {
let api = api(&server, None);
let err = api.ping().await.unwrap_err();
assert!(matches!(err, Error::Parse(_)));
+ assert!(!err.is_retryable());
}
#[tokio::test]
diff --git a/rust/crates/photos/src/collections.rs b/rust/crates/photos/src/collections.rs
index 7623721ff5..3719a17a21 100644
--- a/rust/crates/photos/src/collections.rs
+++ b/rust/crates/photos/src/collections.rs
@@ -1,9 +1,4 @@
-use ente_collections::open_collection_key;
-use ente_core::{
- Session, b64,
- crypto::{self, Header, Key, Nonce, blob, secretbox},
- http,
-};
+use ente_core::{Session, crypto::Key};
use serde::Deserialize;
const DEFAULT_HIDDEN_SUBTYPE: u8 = 1;
@@ -55,151 +50,93 @@ impl Visibility {
}
}
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
- #[error(transparent)]
- Http(#[from] http::Error),
- #[error(transparent)]
- Crypto(#[from] crypto::Error),
- #[error(transparent)]
- Base64(#[from] b64::DecodeError),
- #[error(transparent)]
- Collections(#[from] ente_collections::Error),
- #[error("invalid collection {id}: {reason}")]
- InvalidCollection { id: i64, reason: &'static str },
+pub type Error = ente_collections::Error;
+
+pub struct Change {
+ pub id: i64,
+ pub collection: Option<Collection>,
}
-pub async fn list(session: &Session) -> Result<Vec<Collection>, Error> {
- let response: CollectionsResponse = session
- .api
- .get("/collections/v2")
- .query(&[("sinceTime", 0)])
- .send()
- .await?
- .error_for_code()
- .await?
- .json()
- .await?;
- response
+pub struct Page {
+ pub changes: Vec<Change>,
+ pub cursor: i64,
+}
+
+pub async fn diff(session: &Session, since: i64) -> Result<Page, Error> {
+ let page = ente_collections::client::diff(session, since).await?;
+ let changes = page
.collections
.into_iter()
- .filter(|c| c.is_deleted != Some(true))
- .map(|c| {
- let owned = c.owner.id == session.user_id;
- let key = open_collection_key(
- session,
- c.owner.id,
- &c.encrypted_key,
- c.key_decryption_nonce.as_deref(),
- )?;
- let name = match c.name.as_deref().filter(|name| !name.is_empty()) {
- Some(name) => name.to_owned(),
- None => {
- let encrypted_name = c.required(&c.encrypted_name, "missing name")?;
- let nonce = c.required(&c.name_decryption_nonce, "missing name nonce")?;
- let bytes = secretbox::decrypt(
- &b64::decode(encrypted_name)?,
- &Nonce::try_from_slice(&b64::decode(nonce)?)?,
- &key,
- )?;
- String::from_utf8(bytes).map_err(|_| Error::InvalidCollection {
- id: c.id,
- reason: "name is not UTF-8",
- })?
- }
- };
- let metadata = if owned {
- c.magic_metadata.as_ref()
- } else {
- c.shared_magic_metadata.as_ref()
- };
- let visibility = match metadata {
- Some(metadata) => {
- let metadata = blob::decrypt_json::<Metadata>(
- &blob::EncryptedBlob {
- encrypted_data: b64::decode(&metadata.data)?,
- decryption_header: Header::try_from_slice(&b64::decode(
- &metadata.header,
- )?)?,
- },
- &key,
- )?;
- if owned && metadata.sub_type == Some(DEFAULT_HIDDEN_SUBTYPE) {
- HIDDEN_VISIBILITY
- } else {
- metadata.visibility.unwrap_or(0)
- }
- }
- None => 0,
- };
- Ok(Collection {
- id: c.id,
- key,
- name,
- kind: match c.kind.as_str() {
- "favorites" => Kind::Favorites,
- "uncategorized" => Kind::Uncategorized,
- _ => Kind::Album,
- },
- visibility: match visibility {
- ARCHIVED_VISIBILITY => Visibility::Archived,
- HIDDEN_VISIBILITY => Visibility::Hidden,
- _ => Visibility::Visible,
+ .map(|remote| {
+ Ok(Change {
+ id: remote.id,
+ collection: if remote.is_deleted == Some(true) {
+ None
+ } else {
+ Some(Collection::open(remote, session)?)
},
- owner_id: c.owner.id,
- updated_at_micros: c.updation_time,
})
})
- .collect()
-}
-
-#[derive(Deserialize)]
-struct CollectionsResponse {
- collections: Vec<RemoteCollection>,
+ .collect::<Result<_, Error>>()?;
+ Ok(Page {
+ changes,
+ cursor: page.cursor,
+ })
}
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct RemoteCollection {
- id: i64,
- owner: Owner,
- encrypted_key: String,
- key_decryption_nonce: Option<String>,
- encrypted_name: Option<String>,
- name_decryption_nonce: Option<String>,
- name: Option<String>,
- #[serde(rename = "type")]
- kind: String,
- updation_time: i64,
- is_deleted: Option<bool>,
- magic_metadata: Option<EncryptedMetadata>,
- shared_magic_metadata: Option<EncryptedMetadata>,
+pub async fn list(session: &Session) -> Result<Vec<Collection>, Error> {
+ Ok(diff(session, 0)
+ .await?
+ .changes
+ .into_iter()
+ .filter_map(|change| change.collection)
+ .collect())
}
-impl RemoteCollection {
- fn required<'a>(
- &self,
- value: &'a Option<String>,
- reason: &'static str,
- ) -> Result<&'a str, Error> {
- value.as_deref().ok_or(Error::InvalidCollection {
- id: self.id,
- reason,
+impl Collection {
+ fn open(
+ remote: ente_collections::client::Collection,
+ session: &Session,
+ ) -> Result<Self, Error> {
+ let key = remote.open_key(session)?;
+ let name = remote.open_name(&key)?;
+ let owned = remote.owner.id == session.user_id;
+ let metadata = if owned {
+ remote.magic_metadata.as_ref()
+ } else {
+ remote.shared_magic_metadata.as_ref()
+ };
+ let visibility = match metadata {
+ Some(metadata) => {
+ let metadata: Metadata = metadata.open(&key)?;
+ if owned && metadata.sub_type == Some(DEFAULT_HIDDEN_SUBTYPE) {
+ HIDDEN_VISIBILITY
+ } else {
+ metadata.visibility.unwrap_or(0)
+ }
+ }
+ None => 0,
+ };
+ Ok(Self {
+ id: remote.id,
+ key,
+ name,
+ kind: match remote.kind.as_str() {
+ "favorites" => Kind::Favorites,
+ "uncategorized" => Kind::Uncategorized,
+ _ => Kind::Album,
+ },
+ visibility: match visibility {
+ ARCHIVED_VISIBILITY => Visibility::Archived,
+ HIDDEN_VISIBILITY => Visibility::Hidden,
+ _ => Visibility::Visible,
+ },
+ owner_id: remote.owner.id,
+ updated_at_micros: remote.updation_time,
})
}
}
-#[derive(Deserialize)]
-struct Owner {
- id: i64,
-}
-
-#[derive(Deserialize)]
-struct EncryptedMetadata {
- data: String,
- header: String,
-}
-
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct Metadata {
diff --git a/rust/crates/photos/src/files.rs b/rust/crates/photos/src/files.rs
index 1b73c05b0e..56b82b7c8e 100644
--- a/rust/crates/photos/src/files.rs
+++ b/rust/crates/photos/src/files.rs
@@ -1,8 +1,6 @@
-use std::collections::BTreeMap;
-
use ente_core::{
Session, b64,
- crypto::{self, Header, Key, Nonce, blob, secretbox},
+ crypto::{self, Header, Key, blob},
http,
};
use serde::Deserialize;
@@ -77,130 +75,75 @@ pub enum Error {
Base64(#[from] b64::DecodeError),
#[error(transparent)]
Io(#[from] std::io::Error),
- #[error("file changes did not advance the cursor for collection {0}")]
- CursorDidNotAdvance(i64),
+ #[error(transparent)]
+ Collections(#[from] ente_collections::Error),
}
pub async fn diff(session: &Session, collection: &Collection, since: i64) -> Result<Page, Error> {
- let response: DiffResponse = http::retry(|| async {
- session
- .api
- .get("/collections/v2/diff")
- .query(&[("collectionID", collection.id), ("sinceTime", since)])
- .send()
- .await?
- .error_for_code()
- .await?
- .json()
- .await
- })
- .await?;
- let mut cursor = since;
- let mut changes = Vec::with_capacity(response.diff.len());
- for remote in response.diff {
- cursor = cursor.max(remote.updation_time);
- changes.push(Change {
- id: remote.id,
- file: if remote.is_deleted || remote.file.encrypted_data.as_deref() == Some("-") {
- None
- } else {
- Some(remote.open(&collection.key)?)
- },
- });
- }
- if response.has_more && cursor <= since {
- return Err(Error::CursorDidNotAdvance(collection.id));
- }
+ let page = ente_collections::client::files_diff(session, collection.id, since).await?;
+ let changes = page
+ .files
+ .into_iter()
+ .map(|remote| {
+ Ok(Change {
+ id: remote.id,
+ file: if remote.is_deleted() {
+ None
+ } else {
+ Some(File::open(remote, &collection.key)?)
+ },
+ })
+ })
+ .collect::<Result<_, Error>>()?;
Ok(Page {
changes,
- cursor,
- has_more: response.has_more,
+ cursor: page.cursor,
+ has_more: page.has_more,
})
}
-pub async fn list(session: &Session, collection: &Collection) -> Result<Vec<File>, Error> {
- let mut files = BTreeMap::new();
- let mut cursor = 0;
- loop {
- let page = diff(session, collection, cursor).await?;
- for change in page.changes {
- if let Some(file) = change.file {
- files.insert(change.id, file);
- } else {
- files.remove(&change.id);
- }
- }
- cursor = page.cursor;
- if !page.has_more {
- return Ok(files.into_values().collect());
- }
- }
-}
-
#[cfg(not(target_arch = "wasm32"))]
-pub async fn download(
+pub async fn download<W: std::io::Write>(
session: &Session,
file: &File,
- output: &mut impl std::io::Write,
+ mut output: impl FnMut() -> std::io::Result<W>,
) -> Result<(), Error> {
- use ente_core::crypto::stream::{DECRYPTION_CHUNK_SIZE, Decryptor};
+ use ente_core::crypto::stream::DecryptingWriter;
use futures_util::StreamExt;
- let response = http::retry(|| async {
- let signed: DownloadUrl = session
- .api
- .get(&format!("/files/download/v3/{}", file.id))
- .send()
- .await?
- .error_for_code()
- .await?
- .json()
- .await?;
- session
- .api
- .http()
- .get(&signed.url)
- .send()
- .await?
- .error_for_status()
- })
- .await?;
- let body = response.bytes_stream();
- let mut body = std::pin::pin!(body);
- let mut decryptor = Decryptor::new(&file.header, &file.key);
- let mut buffer = Vec::with_capacity(DECRYPTION_CHUNK_SIZE);
- let mut seen_final = false;
- while let Some(chunk) = body.next().await {
- let chunk = chunk?;
- let mut remaining = chunk.as_ref();
- while !remaining.is_empty() {
- if seen_final {
- return Err(crypto::Error::StreamTrailingData.into());
- }
- let count = remaining.len().min(DECRYPTION_CHUNK_SIZE - buffer.len());
- buffer.extend_from_slice(&remaining[..count]);
- remaining = &remaining[count..];
- if buffer.len() == DECRYPTION_CHUNK_SIZE {
- let (plaintext, is_final) = decryptor.pull(&buffer)?;
- output.write_all(&plaintext)?;
- seen_final = is_final;
- buffer.clear();
+ http::retry_if(
+ || {
+ let output = output();
+ async move {
+ let output = output?;
+ let signed: DownloadUrl = session
+ .api
+ .get(&format!("/files/download/v3/{}", file.id))
+ .send()
+ .await?
+ .error_for_code()
+ .await?
+ .json()
+ .await?;
+ let response = session
+ .api
+ .http()
+ .get(&signed.url)
+ .send()
+ .await?
+ .error_for_status()?;
+ let mut body = std::pin::pin!(response.bytes_stream());
+ let mut decryptor = DecryptingWriter::new(&file.header, &file.key, output);
+ while let Some(chunk) = body.next().await {
+ decryptor.write(&chunk?)?;
+ }
+ decryptor.finish()?;
+ Ok(())
}
- }
- }
- if !buffer.is_empty() {
- let (plaintext, _) = decryptor.pull(&buffer)?;
- output.write_all(&plaintext)?;
- }
- decryptor.finish()?;
- Ok(())
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct DiffResponse {
- diff: Vec<RemoteFile>,
- has_more: bool,
+ },
+ |error| matches!(error, Error::Http(error) if error.is_retryable()),
+ )
+ .await
}
#[cfg(not(target_arch = "wasm32"))]
@@ -209,43 +152,25 @@ struct DownloadUrl {
url: String,
}
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct RemoteFile {
- id: i64,
- #[serde(rename = "ownerID")]
- owner_id: i64,
- encrypted_key: String,
- key_decryption_nonce: String,
- file: FileAttributes,
- metadata: FileAttributes,
- is_deleted: bool,
- updation_time: i64,
- magic_metadata: Option<EncryptedMetadata>,
- pub_magic_metadata: Option<EncryptedMetadata>,
-}
-
-impl RemoteFile {
- fn open(self, collection_key: &Key) -> Result<File, Error> {
- let key = Key::try_from_slice(&secretbox::decrypt(
- &b64::decode(&self.encrypted_key)?,
- &Nonce::try_from_slice(&b64::decode(&self.key_decryption_nonce)?)?,
- collection_key,
- )?)?;
+impl File {
+ fn open(remote: ente_collections::client::File, collection_key: &Key) -> Result<Self, Error> {
+ let key = remote.open_key(collection_key)?;
let metadata: Metadata = blob::decrypt_json(
&blob::EncryptedBlob {
- encrypted_data: b64::decode(self.metadata.encrypted_data.as_deref().unwrap_or(""))?,
+ encrypted_data: b64::decode(
+ remote.metadata.encrypted_data.as_deref().unwrap_or(""),
+ )?,
decryption_header: Header::try_from_slice(&b64::decode(
- &self.metadata.decryption_header,
+ &remote.metadata.decryption_header,
)?)?,
},
&key,
)?;
- let public: PublicMetadata = match self.pub_magic_metadata {
+ let public: PublicMetadata = match remote.pub_magic_metadata {
Some(encrypted) => encrypted.open(&key)?,
None => PublicMetadata::default(),
};
- let private: PrivateMetadata = match self.magic_metadata {
+ let private: PrivateMetadata = match remote.magic_metadata {
Some(encrypted) => encrypted.open(&key)?,
None => PrivateMetadata::default(),
};
@@ -275,9 +200,9 @@ impl RemoteFile {
longitude,
});
Ok(File {
- id: self.id,
- owner_id: self.owner_id,
- updated_at_micros: self.updation_time,
+ id: remote.id,
+ owner_id: remote.owner_id,
+ updated_at_micros: remote.updation_time,
name: public.edited_name.unwrap_or(metadata.title),
kind,
created_at_micros: public.edited_time.unwrap_or(metadata.creation_time),
@@ -299,36 +224,11 @@ impl RemoteFile {
_ => Visibility::Visible,
},
key,
- header: Header::try_from_slice(&b64::decode(&self.file.decryption_header)?)?,
+ header: Header::try_from_slice(&b64::decode(&remote.file.decryption_header)?)?,
})
}
}
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct FileAttributes {
- encrypted_data: Option<String>,
- decryption_header: String,
-}
-
-#[derive(Deserialize)]
-struct EncryptedMetadata {
- data: String,
- header: String,
-}
-
-impl EncryptedMetadata {
- fn open<T: serde::de::DeserializeOwned>(&self, key: &Key) -> Result<T, Error> {
- Ok(blob::decrypt_json(
- &blob::EncryptedBlob {
- encrypted_data: b64::decode(&self.data)?,
- decryption_header: Header::try_from_slice(&b64::decode(&self.header)?)?,
- },
- key,
- )?)
- }
-}
-
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct Metadata {