Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Add vault_sync Rust module (#2159)
Leendert de Borst3396cd3fcd53
diff --git a/core/rust/src/vault_sync/db.rs b/core/rust/src/vault_sync/db.rsnew file mode 100644index 000000000..eefcfd26f--- /dev/null+++ b/core/rust/src/vault_sync/db.rs@@ -0,0 +1,403 @@+//! The vault's SQLite through the host.++use std::collections::HashMap;++use base64::engine::general_purpose::STANDARD as BASE64;+use base64::Engine;+use serde_json::{json, Map, Value};++use super::errors::{SyncError, SyncResult};+use super::session::Host;+use super::types::{Ack, Command, Db, DbBytes, DbRows, LogLevel, SqlStatement};+use super::version;+use crate::vault_codec::{CodecRecord, CodecTableData, MaterializedTables};++pub(crate) type Row = Map<String, Value>;++/// Rows per `dbExec` batch when bulk-loading a materialized vault.+const INSERT_BATCH_ROWS: usize = 400;++/// The id a row carries while it belongs to no manifest yet.+pub const UNSTAMPED_MANIFEST_ID: &str = "00000000-0000-0000-0000-000000000000";++pub(crate) async fn query(host: &Host, db: Db, sql: &str, params: Vec<Value>) -> SyncResult<Vec<Row>> {+ let response: DbRows = host.call(Command::DbQuery { db, sql: sql.to_string(), params }).await?;+ Ok(response.rows)+}++pub(crate) async fn exec(host: &Host, db: Db, statements: Vec<SqlStatement>) -> SyncResult<()> {+ if statements.is_empty() {+ return Ok(());+ }+ host.call::<Ack>(Command::DbExec { db, statements }).await?;+ Ok(())+}++pub(crate) async fn exec_one(host: &Host, db: Db, sql: &str, params: Vec<Value>) -> SyncResult<()> {+ exec(host, db, vec![SqlStatement { sql: sql.to_string(), params }]).await+}++/// Open the staging database: fresh with the current schema, or from SQLite bytes.+pub(crate) async fn open_staging(host: &Host, bytes: Option<&[u8]>) -> SyncResult<()> {+ host.call::<Ack>(Command::DbOpen { db: Db::Staging, bytes: bytes.map(|b| BASE64.encode(b)) }).await?;+ Ok(())+}++/// Serialize a database to SQLite bytes.+pub(crate) async fn export(host: &Host, db: Db) -> SyncResult<Vec<u8>> {+ let response: DbBytes = host.call(Command::DbExport { db }).await?;+ BASE64.decode(response.bytes).map_err(|_| SyncError::Other("host returned invalid database bytes".to_string()))+}++pub(crate) fn b64_param(bytes: &[u8]) -> Value {+ json!({ "__b64": BASE64.encode(bytes) })+}++/// The lookup key of a manifest id: ids compare case-insensitively.+pub(crate) fn id_key(id: &str) -> String {+ id.to_lowercase()+}++pub(crate) fn ids_equal(a: &str, b: &str) -> bool {+ a.eq_ignore_ascii_case(b)+}++/// SQLite booleans arrive as 0/1 numbers or as JSON booleans.+pub(crate) fn truthy(value: Option<&Value>) -> bool {+ match value {+ Some(Value::Bool(b)) => *b,+ Some(Value::Number(n)) => n.as_i64().unwrap_or(0) != 0 || n.as_f64().unwrap_or(0.0) != 0.0,+ Some(Value::String(s)) => s == "1" || s.eq_ignore_ascii_case("true"),+ _ => false,+ }+}++/// A cell as text: strings as they are, null as empty, anything else in its JSON form.+pub(crate) fn value_string(value: &Value) -> String {+ match value {+ Value::String(s) => s.clone(),+ Value::Null => String::new(),+ other => other.to_string(),+ }+}++/// A row's cell as text, empty when absent.+pub(crate) fn cell_string(row: &Row, column: &str) -> String {+ row.get(column).map(value_string).unwrap_or_default()+}++/// The names of every user table.+pub(crate) async fn list_user_tables(host: &Host, db: Db) -> SyncResult<Vec<String>> {+ let rows = query(host, db, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name", vec![]).await?;+ Ok(rows.iter().filter_map(|row| row.get("name").and_then(Value::as_str).map(str::to_string)).collect())+}++/// Every user table with its rows, the shape canonicalize takes.+pub(crate) async fn read_tables(host: &Host, db: Db) -> SyncResult<Vec<CodecTableData>> {+ let mut tables = Vec::new();+ for name in list_user_tables(host, db).await? {+ let rows = query(host, db, &format!("SELECT * FROM \"{}\"", name), vec![]).await?;+ tables.push(CodecTableData { name, records: rows.into_iter().map(row_to_record).collect() });+ }+ Ok(tables)+}++/// The named tables that exist, keyed by name.+pub(crate) async fn read_named_tables(host: &Host, db: Db, names: &[String]) -> SyncResult<HashMap<String, Vec<CodecRecord>>> {+ let existing = list_user_tables(host, db).await?;+ let mut out = HashMap::new();+ for name in names {+ if !existing.contains(name) {+ continue;+ }+ let rows = query(host, db, &format!("SELECT * FROM \"{}\"", name), vec![]).await?;+ out.insert(name.clone(), rows.into_iter().map(row_to_record).collect());+ }+ Ok(out)+}++fn row_to_record(row: Row) -> CodecRecord {+ row.into_iter().collect()+}++/// The column set of every table in a database, as materialize needs it.+pub(crate) async fn schema_columns(host: &Host, db: Db) -> SyncResult<HashMap<String, Vec<String>>> {+ let mut out = HashMap::new();+ let tables = query(host, db, "SELECT name FROM sqlite_master WHERE type='table'", vec![]).await?;+ for table in tables.iter().filter_map(|row| row.get("name").and_then(Value::as_str)) {+ let columns = query(host, db, &format!("PRAGMA table_info(\"{}\")", table), vec![]).await?;+ out.insert(table.to_string(), columns.iter().filter_map(|row| row.get("name").and_then(Value::as_str).map(str::to_string)).collect());+ }+ Ok(out)+}++/// The latest EF migration id a database is stamped with, empty when unstamped.+pub(crate) async fn latest_migration_id(host: &Host, db: Db) -> SyncResult<String> {+ let has_table = !query(host, db, "SELECT name FROM sqlite_master WHERE type='table' AND name='__EFMigrationsHistory'", vec![]).await?.is_empty();+ if !has_table {+ return Ok(String::new());+ }+ let rows = query(host, db, "SELECT MigrationId FROM __EFMigrationsHistory ORDER BY MigrationId DESC LIMIT 1", vec![]).await?;+ Ok(rows.first().and_then(|row| row.get("MigrationId")).and_then(Value::as_str).unwrap_or("").to_string())+}++/// Whether the local vault still has to walk the frozen sqlite-blob upgrade chain (a pre-2.0.0 vault).+pub(crate) async fn requires_legacy_sqlite_blob_migration(host: &Host, db: Db) -> SyncResult<bool> {+ let migration_id = latest_migration_id(host, db).await?;+ if migration_id.is_empty() {+ return Err(SyncError::Other("No migrations found in the database.".to_string()));+ }+ let database_version = version::extract_version_from_migration_id(&migration_id).ok_or_else(|| SyncError::Other("Could not extract version from migration ID".to_string()))?;+ let revision = version::legacy_revision_for(&database_version).map_err(SyncError::VaultVersionIncompatible)?;+ Ok(revision < version::latest_legacy_revision())+}++/// Whether the local schema predates the current full schema (whose stamp the staging database carries).+pub(crate) async fn requires_schema_migration(host: &Host, schema_migration_id: &str) -> SyncResult<bool> {+ if requires_legacy_sqlite_blob_migration(host, Db::Local).await? {+ return Ok(false);+ }+ let local = latest_migration_id(host, Db::Local).await?;+ if local.is_empty() || schema_migration_id.is_empty() {+ return Ok(false);+ }+ Ok(local.as_str() < schema_migration_id)+}++pub(crate) async fn has_column(host: &Host, db: Db, table: &str, column: &str) -> SyncResult<bool> {+ Ok(!query(host, db, "SELECT name FROM pragma_table_info(?) WHERE name = ?", vec![json!(table), json!(column)]).await?.is_empty())+}++/// The tables carrying a manifest stamp.+pub(crate) async fn stamped_tables(host: &Host, db: Db) -> SyncResult<Vec<String>> {+ let mut out = Vec::new();+ for name in list_user_tables(host, db).await? {+ if has_column(host, db, &name, "ManifestId").await? {+ out.push(name);+ }+ }+ Ok(out)+}++/// Every manifest id this vault holds a row for.+pub(crate) async fn manifest_ids_in_vault(host: &Host, db: Db) -> SyncResult<Vec<String>> {+ let mut ids: Vec<String> = Vec::new();+ for table in stamped_tables(host, db).await? {+ let rows = query(host, db, &format!("SELECT DISTINCT ManifestId FROM \"{}\" WHERE ManifestId IS NOT NULL AND ManifestId != ''", table), vec![]).await?;+ for id in rows.iter().filter_map(|row| row.get("ManifestId").and_then(Value::as_str)) {+ if !id.eq_ignore_ascii_case(UNSTAMPED_MANIFEST_ID) && !ids.iter().any(|known| known == id) {+ ids.push(id.to_string());+ }+ }+ }+ Ok(ids)+}++/// Load materialized tables into the (fresh) staging database, re-embedding blob bytes, and verify its+/// referential integrity.+pub(crate) async fn insert_materialized(host: &Host, materialized: &MaterializedTables, blobs: &HashMap<String, Vec<u8>>) -> SyncResult<()> {+ let schema_tables: Vec<String> = query(host, Db::Staging, "SELECT name FROM sqlite_master WHERE type='table'", vec![])+ .await?+ .iter()+ .filter_map(|row| row.get("name").and_then(Value::as_str).map(str::to_string))+ .collect();++ for table in &materialized.tables {+ if table.records.is_empty() {+ continue;+ }+ if !schema_tables.contains(&table.name) {+ host.log(LogLevel::Warn, format!("[VaultCodec] Skipping table \"{}\" ({} rows), not present in the schema.", table.name, table.records.len())).await;+ continue;+ }++ let mut batch = Vec::with_capacity(INSERT_BATCH_ROWS);+ for row in &table.records {+ let mut columns: Vec<&String> = row.keys().collect();+ columns.sort();+ let params: Vec<Value> = columns.iter().map(|column| bind_value(&row[*column], blobs)).collect();+ let quoted: Vec<String> = columns.iter().map(|column| format!("\"{}\"", column)).collect();+ let placeholders = vec!["?"; columns.len()].join(", ");+ batch.push(SqlStatement { sql: format!("INSERT INTO \"{}\" ({}) VALUES ({})", table.name, quoted.join(", "), placeholders), params });+ if batch.len() >= INSERT_BATCH_ROWS {+ exec(host, Db::Staging, std::mem::take(&mut batch)).await?;+ }+ }+ exec(host, Db::Staging, batch).await?;+ }++ let violations = query(host, Db::Staging, "PRAGMA foreign_key_check", vec![]).await?;+ if !violations.is_empty() {+ let sample: Vec<String> = violations.iter().take(5).map(|v| format!("{} row {} > missing parent in {}", cell_string(v, "table"), cell_string(v, "rowid"), cell_string(v, "parent"))).collect();+ return Err(SyncError::Other(format!("VaultCodec: materialized database fails foreign key check ({} violations): {}", violations.len(), sample.join("; "))));+ }+ Ok(())+}++/// A materialized cell as a bind parameter: blob markers become bytes (or null when the bytes are missing),+/// inline `{ __b64 }` payloads bind as bytes, everything else binds as is.+fn bind_value(value: &Value, blobs: &HashMap<String, Vec<u8>>) -> Value {+ if let Some(reference) = value.get("__blobRef").and_then(Value::as_str) {+ return blobs.get(reference).map(|bytes| b64_param(bytes)).unwrap_or(Value::Null);+ }+ value.clone()+}++/// The current timestamp in the vault's row format (`yyyy-MM-dd HH:mm:ss.SSS`, UTC).+pub(crate) fn now() -> String {+ crate::timestamp::now_vault_datetime()+}++/// The current instant as ISO-8601 UTC with milliseconds.+pub(crate) fn now_iso() -> String {+ crate::timestamp::now_iso_utc()+}++/// A random lowercase UUID v4.+pub(crate) fn new_id() -> String {+ let mut bytes = [0u8; 16];+ crate::crypto::fill_random(&mut bytes);+ bytes[6] = (bytes[6] & 0x0f) | 0x40;+ bytes[8] = (bytes[8] & 0x3f) | 0x80;+ let hex: Vec<String> = bytes.iter().map(|b| format!("{:02x}", b)).collect();+ let hex = hex.join("");+ format!("{}-{}-{}-{}-{}", &hex[0..8], &hex[8..12], &hex[12..16], &hex[16..20], &hex[20..32])+}++pub(crate) const GET_ACTIVE_KEY_FOR_MANIFEST: &str = "SELECT x.Id, x.PublicKey, x.PrivateKey, x.IsPrimary FROM EncryptionKeys x WHERE x.ManifestId = ? AND x.IsPrimary = 1 AND x.IsDeleted = 0 LIMIT 1";+pub(crate) const GET_ACCOUNT_KEY_BY_PUBLIC_KEY: &str = "SELECT x.PublicKey, x.PrivateKey, x.IsPrimary FROM EncryptionKeys x WHERE x.ManifestId = ? AND x.PublicKey = ? AND x.IsDeleted = 0 LIMIT 1";+pub(crate) const DEMOTE_KEYS_FOR_MANIFEST: &str = "UPDATE EncryptionKeys SET IsPrimary = 0, UpdatedAt = ? WHERE ManifestId = ? AND IsPrimary = 1";+pub(crate) const INSERT_KEY_FOR_MANIFEST: &str = "INSERT INTO EncryptionKeys (Id, ManifestId, PublicKey, PrivateKey, IsPrimary, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, 1, ?, ?, 0)";+pub(crate) const INSERT_FOLDER: &str = "INSERT INTO Folders (Id, Name, ParentFolderId, ManifestId, Weight, IsDeleted, CreatedAt, UpdatedAt) VALUES (?, ?, ?, COALESCE((SELECT ManifestId FROM Folders WHERE Id = ?), ?), 0, 0, ?, ?)";+pub(crate) const RESTAMP_SUBTREE_FOLDERS: &str = "UPDATE Folders SET ManifestId = ? WHERE Id IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";+pub(crate) const RESTAMP_SUBTREE_ITEMS: &str = "UPDATE Items SET ManifestId = ? WHERE FolderId IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";+pub(crate) const FIND_ITEMS_WITH_FOREIGN_LOGO: &str = "SELECT i.Id, i.ManifestId, origin.Kind, origin.Source FROM Items i INNER JOIN Logos origin ON origin.Id = i.LogoId LEFT JOIN Logos own ON own.Id = i.LogoId AND own.ManifestId = i.ManifestId WHERE i.LogoId IS NOT NULL AND own.Id IS NULL AND i.IsDeleted = 0";+pub(crate) const GET_LOGO_ID_FOR_KEY: &str = "SELECT Id FROM Logos WHERE ManifestId = ? AND Kind = ? AND Source = ? AND IsDeleted = 0 LIMIT 1";+pub(crate) const GET_BEST_LOGO_FOR_KEY: &str = "SELECT FileData, MimeType, Name FROM Logos WHERE Kind = ? AND Source = ? AND IsDeleted = 0 ORDER BY (FileData IS NOT NULL AND LENGTH(FileData) > 0) DESC, UpdatedAt DESC LIMIT 1";+pub(crate) const UPSERT_LOGO: &str = "INSERT INTO Logos (Id, Kind, Source, ManifestId, FileData, MimeType, Name, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0) ON CONFLICT(ManifestId, Id) DO UPDATE SET FileData = excluded.FileData, MimeType = excluded.MimeType, Name = COALESCE(excluded.Name, Logos.Name), UpdatedAt = excluded.UpdatedAt, IsDeleted = 0";+pub(crate) const REPOINT_ITEM_LOGO: &str = "UPDATE Items SET LogoId = ? WHERE Id = ? AND ManifestId = ?";+pub(crate) const RENDERED_MANIFEST_FOLDER: &str = "SELECT Id FROM Folders WHERE ManifestId = ? AND IsDeleted = 0 AND ParentFolderId IS NULL";+pub(crate) const MANIFEST_ROOT_FOLDER_NAMES: &str = "SELECT ManifestId, Name FROM Folders WHERE IsDeleted = 0 AND ManifestId IS NOT NULL AND UPPER(Id) = UPPER(ManifestId)";++/// The active mail delivery keypair of a manifest, when it has one.+pub(crate) async fn active_key_for_manifest(host: &Host, manifest_id: &str) -> SyncResult<Option<Row>> {+ Ok(query(host, Db::Local, GET_ACTIVE_KEY_FOR_MANIFEST, vec![json!(manifest_id)]).await?.into_iter().next())+}++/// The private half of the account keypair with the given public half, held in the personal manifest.+pub(crate) async fn account_private_key_for(host: &Host, personal_manifest_id: &str, public_key: &str) -> SyncResult<Option<String>> {+ let rows = query(host, Db::Local, GET_ACCOUNT_KEY_BY_PUBLIC_KEY, vec![json!(personal_manifest_id), json!(public_key)]).await?;+ Ok(rows.first().and_then(|row| row.get("PrivateKey")).and_then(Value::as_str).map(str::to_string))+}++/// Make a keypair the manifest's active one, demoting (never deleting) whatever it supersedes.+pub(crate) async fn set_active_key_for_manifest(host: &Host, manifest_id: &str, public_key: &str, private_key: &str) -> SyncResult<()> {+ let now = now();+ exec(+ host,+ Db::Local,+ vec![+ SqlStatement { sql: DEMOTE_KEYS_FOR_MANIFEST.to_string(), params: vec![json!(now), json!(manifest_id)] },+ SqlStatement { sql: INSERT_KEY_FOR_MANIFEST.to_string(), params: vec![json!(new_id()), json!(manifest_id), json!(public_key), json!(private_key), json!(now), json!(now)] },+ ],+ )+ .await+}++/// What each shared manifest is called: the name of the top-level folder it is rendered as, keyed by lower-cased id.+pub(crate) async fn manifest_display_names(host: &Host) -> SyncResult<HashMap<String, String>> {+ // A vault whose schema predates the manifest stamp names none.+ if !has_column(host, Db::Local, "Folders", "ManifestId").await? {+ return Ok(HashMap::new());+ }+ let rows = query(host, Db::Local, MANIFEST_ROOT_FOLDER_NAMES, vec![]).await?;+ Ok(rows+ .iter()+ .filter_map(|row| Some((row.get("ManifestId")?.as_str()?.to_lowercase(), row.get("Name")?.as_str()?.to_string())))+ .collect())+}++/// Whether a shared manifest already has the top-level folder it is rendered as.+pub(crate) async fn has_rendered_manifest_folder(host: &Host, manifest_id: &str) -> SyncResult<bool> {+ Ok(!query(host, Db::Local, RENDERED_MANIFEST_FOLDER, vec![json!(manifest_id)]).await?.is_empty())+}++/// Create the folder a shared manifest is rendered as and pull everything under it into the manifest.+pub(crate) async fn render_manifest_folder(host: &Host, manifest_id: &str, name: &str, active_manifest_id: &str) -> SyncResult<()> {+ let folder_id = manifest_id.to_lowercase();+ let now = now();+ exec(+ host,+ Db::Local,+ vec![+ SqlStatement { sql: INSERT_FOLDER.to_string(), params: vec![json!(folder_id), json!(name), Value::Null, Value::Null, json!(active_manifest_id), json!(now), json!(now)] },+ SqlStatement { sql: RESTAMP_SUBTREE_FOLDERS.to_string(), params: vec![json!(manifest_id), json!(folder_id)] },+ SqlStatement { sql: RESTAMP_SUBTREE_ITEMS.to_string(), params: vec![json!(manifest_id), json!(folder_id)] },+ ],+ )+ .await?;+ reconcile_item_logo_scopes(host, &now).await+}++/// Point every item at a copy of its logo inside its own manifest, copying the image in when needed.+pub(crate) async fn reconcile_item_logo_scopes(host: &Host, now: &str) -> SyncResult<()> {+ let foreign = query(host, Db::Local, FIND_ITEMS_WITH_FOREIGN_LOGO, vec![]).await?;+ for item in foreign {+ let manifest_id = cell_string(&item, "ManifestId");+ let kind = cell_string(&item, "Kind");+ let source = cell_string(&item, "Source");+ let item_id = cell_string(&item, "Id");++ let in_scope = query(host, Db::Local, GET_LOGO_ID_FOR_KEY, vec![json!(manifest_id), json!(kind), json!(source)]).await?;+ let logo_id = match in_scope.first().and_then(|row| row.get("Id")).and_then(Value::as_str) {+ Some(id) => id.to_string(),+ None => {+ let origin = query(host, Db::Local, GET_BEST_LOGO_FOR_KEY, vec![json!(kind), json!(source)]).await?;+ let Some(origin) = origin.into_iter().next() else { continue };+ let logo_id = crate::vault_codec::logo_id_for(&manifest_id, &kind, &source);+ let file_data = origin.get("FileData").cloned().unwrap_or(Value::Null);+ let mime = origin.get("MimeType").cloned().unwrap_or(Value::Null);+ let name = origin.get("Name").cloned().unwrap_or(Value::Null);+ exec_one(host, Db::Local, UPSERT_LOGO, vec![json!(logo_id), json!(kind), json!(source), json!(manifest_id), file_data, mime, name, json!(now), json!(now)]).await?;+ logo_id+ }+ };+ exec_one(host, Db::Local, REPOINT_ITEM_LOGO, vec![json!(logo_id), json!(item_id), json!(manifest_id)]).await?;+ }+ Ok(())+}++/// Prune expired trash items in place. Returns the number of statements executed.+pub(crate) async fn prune_in_place(host: &Host, retention_days: u32) -> SyncResult<usize> {+ let mut tables = Vec::new();+ for table_query in crate::vault_pruner::get_prune_table_queries() {+ let rows = query(host, Db::Local, &table_query.query, vec![]).await?;+ tables.push(crate::vault_pruner::TableData { name: table_query.name, records: rows.into_iter().map(|row| row.into_iter().collect()).collect() });+ }+ let output = crate::vault_pruner::prune_vault(crate::vault_pruner::PruneInput { tables, current_time: now_iso(), retention_days })?;+ let count = output.statements.len();+ exec(host, Db::Local, output.statements.into_iter().map(|s| SqlStatement { sql: s.sql, params: s.params }).collect()).await?;+ Ok(count)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn ids_are_lowercase_uuid_v4() {+ let id = new_id();+ assert_eq!(id.len(), 36);+ assert_eq!(id, id.to_lowercase());+ assert_eq!(&id[14..15], "4");+ assert!(matches!(&id[19..20], "8" | "9" | "a" | "b"));+ }++ #[test]+ fn timestamps_use_the_vault_format() {+ let stamp = now();+ assert_eq!(stamp.len(), 23);+ assert_eq!(&stamp[10..11], " ");+ assert!(now_iso().ends_with('Z'));+ }+}diff --git a/core/rust/src/vault_sync/email_routing.rs b/core/rust/src/vault_sync/email_routing.rsnew file mode 100644index 000000000..e0c14a2a5--- /dev/null+++ b/core/rust/src/vault_sync/email_routing.rs@@ -0,0 +1,97 @@+//! The email routing set a push sends.++use std::collections::{HashMap, HashSet};++use serde_json::Value;++use super::types::{ClaimedEmailAddress, EmailRoutingPush};+use super::db::{truthy, value_string};+use crate::vault_codec::Manifest;++/// The field key of an item's login email.+const FIELD_KEY_LOGIN_EMAIL: &str = "login.email";++/// Build the routing set from the canonicalized manifests, the user's own included.+pub fn build_email_routing(manifests: &[Manifest], private_email_domains: &[String]) -> EmailRoutingPush {+ let mut by_pair: HashMap<String, ClaimedEmailAddress> = HashMap::new();+ let mut order: Vec<String> = Vec::new();++ for manifest in manifests {+ let live_item_ids: HashSet<String> = rows_of(manifest, "Items")+ .iter()+ .filter(|row| !truthy(row.get("IsDeleted")) && row.get("DeletedAt").map_or(true, Value::is_null))+ .filter_map(|row| row.get("Id").map(value_string))+ .collect();++ for field_value in rows_of(manifest, "FieldValues") {+ if field_value.get("FieldKey").and_then(Value::as_str) != Some(FIELD_KEY_LOGIN_EMAIL) || truthy(field_value.get("IsDeleted")) {+ continue;+ }+ let item_id = field_value.get("ItemId").map(value_string).unwrap_or_default();+ if !live_item_ids.contains(&item_id) {+ continue;+ }++ let address = field_value.get("Value").and_then(Value::as_str).map(|value| value.trim().to_lowercase()).unwrap_or_default();+ let domain = match address.split_once('@') {+ Some((_, domain)) if !domain.is_empty() => domain.to_string(),+ _ => continue,+ };+ if !private_email_domains.iter().any(|candidate| *candidate == domain) {+ continue;+ }++ let paused = truthy(field_value.get("IsDisabled"));+ let pair_key = format!("{}\0{}", address, manifest.manifest_id);+ match by_pair.get_mut(&pair_key) {+ // Several items in one manifest may carry the same address; one of them still wanting mail keeps it routed.+ Some(existing) => existing.paused = existing.paused && paused,+ None => {+ order.push(pair_key.clone());+ by_pair.insert(pair_key, ClaimedEmailAddress { address, manifest_id: manifest.manifest_id.clone(), paused });+ }+ }+ }+ }++ EmailRoutingPush {+ email_address_list: order.into_iter().filter_map(|key| by_pair.remove(&key)).collect(),+ covered_manifest_ids: manifests.iter().map(|manifest| manifest.manifest_id.clone()).collect(),+ }+}++fn rows_of<'a>(manifest: &'a Manifest, table: &str) -> &'a [crate::vault_codec::CodecRecord] {+ manifest.tables.get(table).map(Vec::as_slice).unwrap_or(&[])+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::collections::HashMap;++ fn manifest(id: &str, items: Vec<Value>, fields: Vec<Value>) -> Manifest {+ let mut tables = HashMap::new();+ tables.insert("Items".to_string(), items.into_iter().map(|v| serde_json::from_value(v).unwrap()).collect());+ tables.insert("FieldValues".to_string(), fields.into_iter().map(|v| serde_json::from_value(v).unwrap()).collect());+ Manifest { schema_version: 1, manifest_salt: String::new(), canonicalized_at: String::new(), manifest_id: id.to_string(), name: None, tables, extra: HashMap::new() }+ }++ #[test]+ fn claims_private_addresses_of_live_items_once_per_manifest() {+ let m = manifest(+ "m1",+ vec![serde_json::json!({"Id": "i1", "IsDeleted": 0, "DeletedAt": null}), serde_json::json!({"Id": "i2", "IsDeleted": 0, "DeletedAt": "2026-01-01"})],+ vec![+ serde_json::json!({"ItemId": "i1", "FieldKey": "login.email", "Value": " [email protected] ", "IsDeleted": 0, "IsDisabled": 1}),+ serde_json::json!({"ItemId": "i1", "FieldKey": "login.email", "Value": "[email protected]", "IsDeleted": 0, "IsDisabled": 0}),+ serde_json::json!({"ItemId": "i2", "FieldKey": "login.email", "Value": "[email protected]", "IsDeleted": 0}),+ serde_json::json!({"ItemId": "i1", "FieldKey": "login.email", "Value": "[email protected]", "IsDeleted": 0}),+ ],+ );+ let routing = build_email_routing(&[m], &["private.io".to_string()]);+ assert_eq!(routing.email_address_list.len(), 1);+ assert_eq!(routing.email_address_list[0].address, "[email protected]");+ assert!(!routing.email_address_list[0].paused);+ assert_eq!(routing.covered_manifest_ids, vec!["m1"]);+ }+}diff --git a/core/rust/src/vault_sync/engine.rs b/core/rust/src/vault_sync/engine.rsnew file mode 100644index 000000000..56a3a7659--- /dev/null+++ b/core/rust/src/vault_sync/engine.rs@@ -0,0 +1,628 @@+//! The sync flow: status check, server-directed changes, then pull (and merge) or push as the revisions decide.++use std::collections::HashMap;++use serde_json::Value;++use super::errors::{Failure, LogoutReason, SyncError, SyncResult};+use super::merge::{self, PullAndMergeOutcome};+use super::pull::{self, PullOutcome, SnapshotFacts};+use super::push::{self, CanonicalizedSet, PushStatus};+use super::session::Host;+use super::state::{self, Ctx};+use super::types::{Db, EmailRoutingDto, FullSyncResult, LogLevel, MigrateManifestResult, MigrationKind, MigrationStatusResult, OperationResult, SessionOutcome, StatusCheckResult, StatusResponse, SyncOperation, SyncRequest};+use super::{db, http, keys, legacy, version};+use crate::crypto;++/// How many times a chain of syncs may re-sync after an outdated push before giving up.+const MAX_OUTDATED_RESYNCS: u32 = 3;++/// The pending action type this build carries out.+const ACTION_ROTATE_MANIFEST_DELIVERY_KEY: &str = "RotateManifestDeliveryKey";++/// Run one operation to completion; the value is the `done` command's result.+pub(crate) async fn run(host: Host, request: SyncRequest) -> Value {+ let mut ctx = Ctx::new(host, request);+ match ctx.request.operation {+ SyncOperation::FullSync => {+ let result = full_sync_operation(&mut ctx).await;+ finish(&ctx, result)+ }+ SyncOperation::MigrationStatus => {+ let result = migration_status(&mut ctx).await;+ finish(&ctx, result)+ }+ SyncOperation::MigrateManifest => {+ let result = migrate_manifest(&mut ctx).await;+ finish(&ctx, result)+ }+ SyncOperation::StatusCheck => {+ let result = status_check(&mut ctx).await;+ finish(&ctx, result)+ }+ }+}++/// Attach the sync outcome onto an operation's result and serialize it.+fn finish<T: OperationResult>(ctx: &Ctx, mut result: T) -> Value {+ *result.session_mut() = SessionOutcome { session_updates: ctx.updates.clone(), vault_changed: ctx.vault_changed };+ serde_json::to_value(result).unwrap_or(Value::Null)+}++/// What one pass of the sync decided: a final outcome, or that the whole sync has to re-run.+enum Flow {+ Done(FullSyncResult),+ /// Re-run the sync; `outdated` counts against the resync limit.+ Resync { outdated: bool },+}++/// What the preflight decided: run the sync on this state, or stop with a final outcome.+enum Preflight {+ Proceed {+ status: StatusResponse,+ needs_pull: bool,+ canonicalize_cache: Option<(u64, CanonicalizedSet)>,+ },+ Finish(FullSyncResult),+}++fn success() -> FullSyncResult {+ FullSyncResult { success: true, ..Default::default() }+}++/// The outcome of a failed sync: a forced logout, or a coded error.+fn failure(error: &SyncError) -> FullSyncResult {+ match error.failure() {+ Failure::Logout(reason) => logout(reason),+ Failure::Coded(code) => FullSyncResult { success: false, error: Some(error.to_string()), error_code: Some(code), ..Default::default() },+ }+}++fn logout(reason: LogoutReason) -> FullSyncResult {+ FullSyncResult { success: false, requires_logout: true, error_key: Some(reason), ..Default::default() }+}++/// Turn an error thrown during a sync into the outcome the host acts on. A transport failure with a local vault+/// to fall back on is not a failure but puts the host in offline mode instead.+async fn map_sync_failure(ctx: &mut Ctx, error: SyncError) -> FullSyncResult {+ ctx.warn(format!("[VaultSync] Sync failed: {}", error)).await;+ if let SyncError::Network(_) = &error {+ if let Ok(true) = ctx.has_local_vault().await {+ return FullSyncResult { success: true, was_offline: true, is_offline_mode: true, ..Default::default() };+ }+ }+ failure(&error)+}++async fn full_sync_operation(ctx: &mut Ctx) -> FullSyncResult {+ match full_sync(ctx).await {+ Ok(result) => result,+ Err(error) => map_sync_failure(ctx, error).await,+ }+}++/// Full vault sync, re-running itself when a mutation raced a store or the server refused a push as outdated.+async fn full_sync(ctx: &mut Ctx) -> SyncResult<FullSyncResult> {+ let mut outdated_resyncs = 0u32;+ loop {+ match full_sync_once(ctx).await? {+ Flow::Done(result) => return Ok(result),+ Flow::Resync { outdated } => {+ if outdated {+ if outdated_resyncs >= MAX_OUTDATED_RESYNCS {+ ctx.warn(format!("[VaultSync] The server still refuses the write after {} re-syncs; giving up on this chain.", MAX_OUTDATED_RESYNCS)).await;+ return Ok(failure(&SyncError::ResyncLimitReached));+ }+ outdated_resyncs += 1;+ }+ ctx.log("[VaultSync] Re-running the sync.").await;+ }+ }+ }+}++async fn full_sync_once(ctx: &mut Ctx) -> SyncResult<Flow> {+ ctx.log("[VaultSync] Sync started").await;+ let (status, mut needs_pull, canonicalize_cache) = match run_sync_preflight(ctx).await? {+ Preflight::Proceed { status, needs_pull, canonicalize_cache } => (status, needs_pull, canonicalize_cache),+ Preflight::Finish(result) => return Ok(Flow::Done(result)),+ };+ announce_phase(ctx, needs_pull, ctx.is_dirty).await;++ let grant_sync_changed_vault = apply_server_directed_changes(ctx, &status).await?;+ // The rows just reconciled can turn a clean vault dirty after the preflight decided; a dirty vault was checked there.+ if !needs_pull && grant_sync_changed_vault && push::vault_holds_unwritable_manifests(ctx).await {+ needs_pull = true;+ announce_phase(ctx, true, true).await;+ }++ let flow = if needs_pull {+ pull_and_materialize_server_vault(ctx, grant_sync_changed_vault).await?+ } else if ctx.is_dirty && !vault_predates_current_schema(ctx).await? {+ push_pending_local_changes(ctx, canonicalize_cache, grant_sync_changed_vault).await?+ } else {+ // A vault the codec cannot canonicalize has no way to the server; the upgrade gate takes it from here.+ Flow::Done(pending_migration_result(ctx).await?.unwrap_or_else(success))+ };++ Ok(match flow {+ Flow::Done(mut result) => {+ result.server_version = Some(status.server_version.clone());+ result.capabilities = status.capabilities.clone();+ ctx.log("[VaultSync] Sync finished").await;+ Flow::Done(result)+ }+ other => other,+ })+}++/// Tell the host what the sync is about to do, so it can show a pull or push indicator.+async fn announce_phase(ctx: &Ctx, needs_pull: bool, is_dirty: bool) {+ if needs_pull {+ ctx.host.log(LogLevel::Phase, "pull").await;+ } else if is_dirty {+ ctx.host.log(LogLevel::Phase, "push").await;+ }+}++/// Sanity checks before the sync touches the vault.+async fn run_sync_preflight(ctx: &mut Ctx) -> SyncResult<Preflight> {+ if ctx.request.username.is_empty() {+ return Ok(Preflight::Finish(failure(&SyncError::Other("no username in the sync request".to_string()))));+ }+ if ctx.encryption_key.is_none() {+ return Ok(Preflight::Finish(failure(&SyncError::VaultLocked)));+ }++ let status = http::get_status(&ctx.host).await?;+ let mut needs_pull = ctx.request.force_pull || server_state_needs_pull(ctx, &status).await?;+ ctx.log(format!("[VaultSync] Status received (needsPull {}, isDirty {})", needs_pull, ctx.is_dirty)).await;++ if status.server_version == http::SERVER_UNREACHABLE_VERSION {+ return Ok(Preflight::Finish(enter_offline_mode(ctx).await?));+ }+ if !status.client_version_supported {+ return Ok(Preflight::Finish(logout(LogoutReason::ClientVersionNotSupported)));+ }+ if server_too_old(ctx, &status) {+ return Ok(Preflight::Finish(logout(LogoutReason::ServerVersionNotSupported)));+ }++ // A changed server salt means the password was changed elsewhere, which warrants a logout.+ assert_salt_unchanged(ctx, status.srp_salt.as_deref()).await?;++ if needs_pull && !keys::adopt_remote_vault_key_if_needed(ctx).await? {+ return Ok(Preflight::Finish(logout(LogoutReason::PasswordChanged)));+ }++ let mut canonicalize_cache = None;+ if ctx.is_dirty && !needs_pull && !vault_predates_current_schema(ctx).await? {+ match push::detect_no_op_mutation(ctx).await {+ Ok((true, _)) => {+ if state::mark_clean(&ctx.host, ctx.mutation_sequence).await? {+ ctx.is_dirty = false;+ }+ }+ Ok((false, set)) => canonicalize_cache = Some((ctx.mutation_sequence, set)),+ Err(error) => ctx.warn(format!("[VaultSync] No-op mutation pre-check failed, proceeding with a normal push: {}", error)).await,+ }+ }++ if ctx.is_dirty && !needs_pull && push::vault_holds_unwritable_manifests(ctx).await {+ needs_pull = true;+ }++ Ok(Preflight::Proceed { status, needs_pull, canonicalize_cache })+}++/// Whether the server is older than the oldest version this client supports.+fn server_too_old(ctx: &Ctx, status: &StatusResponse) -> bool {+ ctx.request.min_server_version.as_deref().map(|min| !version::version_gte(&status.server_version, min)).unwrap_or(false)+}++/// Whether the client has to pull and re-materialize: any manifest or data bucket changed on the server.+async fn server_state_needs_pull(ctx: &Ctx, status: &StatusResponse) -> SyncResult<bool> {+ let server_manifests: HashMap<String, i64> = status.manifest_revisions.iter().map(|m| (m.manifest_id.clone(), m.revision)).collect();+ let server_buckets: HashMap<String, i64> = status.bucket_revisions.iter().map(|b| (state::bucket_revision_key(&b.manifest_id, &b.category), b.revision)).collect();+ let local_manifests: HashMap<String, i64> = state::get(&ctx.host, state::SERVER_MANIFEST_REVISIONS).await?.unwrap_or_default();+ let local_buckets: HashMap<String, i64> = state::get(&ctx.host, state::VAULT_BUCKET_REVISIONS).await?.unwrap_or_default();++ let manifests_to_pull = requiring_pull(&server_manifests, &local_manifests);+ let buckets_to_pull = requiring_pull(&server_buckets, &local_buckets);+ if manifests_to_pull == 0 && buckets_to_pull == 0 {+ ctx.log("[VaultSync] No pull needed: every manifest and data bucket matches the local revisions.").await;+ return Ok(false);+ }+ ctx.log(format!("[VaultSync] Pull needed for {} manifest(s) and {} data bucket(s).", manifests_to_pull, buckets_to_pull)).await;+ Ok(true)+}++/// The entries whose local revision no longer matches the server's, plus the ones the server no longer lists.+fn requiring_pull(server: &HashMap<String, i64>, local: &HashMap<String, i64>) -> usize {+ server.iter().filter(|(id, revision)| local.get(*id) != Some(revision)).count() + local.keys().filter(|id| !server.contains_key(*id)).count()+}++async fn assert_salt_unchanged(ctx: &Ctx, server_salt: Option<&str>) -> SyncResult<()> {+ let stored: Option<Value> = state::get(&ctx.host, state::ENCRYPTION_KEY_DERIVATION_PARAMS).await?;+ let stored_salt = stored.as_ref().and_then(|params| params.get("salt")).and_then(Value::as_str).map(str::to_string);+ if let (Some(stored), Some(server)) = (stored_salt, server_salt) {+ if !server.is_empty() && server != stored {+ return Err(SyncError::PasswordChangedElsewhere);+ }+ }+ Ok(())+}++/// Fall back to offline mode when the server cannot be reached; without a local vault that is a failure.+async fn enter_offline_mode(ctx: &mut Ctx) -> SyncResult<FullSyncResult> {+ if !ctx.has_local_vault().await? {+ let unreachable = SyncError::Network("server unreachable and no local vault to fall back on".to_string());+ return Ok(FullSyncResult { was_offline: true, is_offline_mode: true, ..failure(&unreachable) });+ }+ Ok(FullSyncResult { success: true, was_offline: true, is_offline_mode: true, ..Default::default() })+}++/// Whether the vault still has to run the manifest migration: a stale schema, or a missing account key hierarchy.+async fn vault_requires_manifest_migration(ctx: &mut Ctx) -> SyncResult<bool> {+ let schema = ctx.schema().await?;+ Ok(db::requires_schema_migration(&ctx.host, &schema.migration_id).await? || !keys::has_local_vault_key(&ctx.host).await?)+}++/// Carry out the work the server has addressed to the current client (e.g. shared groups revocation/rotation actions).+async fn apply_server_directed_changes(ctx: &mut Ctx, status: &StatusResponse) -> SyncResult<bool> {+ let shared = keys::shared_manifest_records(ctx).await?;+ let can_reconcile = ctx.has_local_vault().await? && !vault_requires_manifest_migration(ctx).await?;+ let mut vault_changed = false;++ if can_reconcile {+ let personal = state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await?;+ for record in shared.values().filter(|r| r.can_administer) {+ // A member waiting for the administrator's first push has nothing to render yet.+ if db::has_rendered_manifest_folder(&ctx.host, &record.manifest_id).await? {+ continue;+ }+ let Some(active) = personal.as_deref() else { break };+ let name = record.name.clone().filter(|n| !n.is_empty()).or_else(|| ctx.request.unnamed_shared_vault_name.clone()).unwrap_or_else(|| "Shared vault".to_string());+ db::render_manifest_folder(&ctx.host, &record.manifest_id, &name, active).await?;+ ctx.warn(format!("[Sharing] Shared manifest {} had no folder; recreated it.", record.manifest_id)).await;+ vault_changed = true;+ }++ let mut completed = 0usize;+ for action in &status.pending_actions {+ let outcome: SyncResult<(bool, bool)> = async {+ if action.action_type != ACTION_ROTATE_MANIFEST_DELIVERY_KEY {+ ctx.log(format!("[PendingActions] Action type {} is not known to this client; leaving it for one that knows it.", action.action_type)).await;+ return Ok((false, false));+ }+ let record = shared.values().find(|r| action.manifest_id.as_deref().map(|id| id.eq_ignore_ascii_case(&r.manifest_id)).unwrap_or(false));+ let Some(record) = record.filter(|r| r.can_administer) else {+ ctx.log(format!("[PendingActions] Manifest {:?} is not open to this session as an administrator; leaving its delivery key rotation for another sync.", action.manifest_id)).await;+ return Ok((false, false));+ };+ let pair = crypto::generate_rsa_key_pair()?;+ db::set_active_key_for_manifest(&ctx.host, &record.manifest_id, &pair.public_key, &pair.private_key).await?;+ ctx.log(format!("[PendingActions] Rotated the mail delivery key of shared manifest {}.", record.manifest_id)).await;+ Ok((true, true))+ }+ .await;+ match outcome {+ Ok((changed, done)) => {+ vault_changed |= changed;+ if done {+ http::delete(&ctx.host, &format!("ClientActions/{}", action.id)).await?;+ completed += 1;+ }+ }+ Err(error) => ctx.warn(format!("[PendingActions] Could not carry out action {} ({}); retrying on the next sync. {}", action.action_type, action.id, error)).await,+ }+ }+ if !status.pending_actions.is_empty() {+ ctx.log(format!("[PendingActions] {} of {} action(s) carried out.", completed, status.pending_actions.len())).await;+ }+ }++ if !vault_changed {+ return Ok(false);+ }++ let key = ctx.encryption_key()?;+ let bytes = db::export(&ctx.host, Db::Local).await?;+ let stored = state::store_vault(&ctx.host, &state::encrypt_vault_blob(&bytes, &key)?, true, None, None).await?;+ let was_dirty = ctx.is_dirty;+ ctx.is_dirty = true;+ ctx.mutation_sequence = stored.mutation_sequence;+ ctx.vault_changed = true;+ if !was_dirty {+ // The reconciled rows ride out on this very sync, which the host had not announced as a push yet.+ announce_phase(ctx, false, true).await;+ }+ Ok(true)+}++/// Outcome for a sync that stored a freshly materialized vault.+async fn materialized_vault_result(ctx: &mut Ctx, revision: i64, email_routing: EmailRoutingDto) -> SyncResult<FullSyncResult> {+ let sqlite_blob_upgrade_required = db::requires_legacy_sqlite_blob_migration(&ctx.host, Db::Local).await?;+ let manifest_migration_required = vault_requires_manifest_migration(ctx).await?;+ Ok(FullSyncResult { success: true, has_new_vault: true, sqlite_blob_upgrade_required, manifest_migration_required, pulled_revision: Some(revision), email_routing: Some(email_routing), ..Default::default() })+}++/// Store a pulled vault as the local vault, refusing (and re-syncing) when a mutation raced the pull.+async fn store_pulled_vault(ctx: &mut Ctx, encrypted_vault: &str, revision: i64) -> SyncResult<bool> {+ let stored = state::store_vault(&ctx.host, encrypted_vault, false, Some(ctx.mutation_sequence), Some(revision)).await?;+ if !stored.success {+ ctx.log("[VaultSync] Mutation detected during sync, re-syncing...").await;+ return Ok(false);+ }+ ctx.vault_changed = true;+ Ok(true)+}++/// Store the server's vault as the local vault, replacing whatever was there.+async fn adopt_server_vault(ctx: &mut Ctx, encrypted_vault: &str, revision: i64, email_routing: EmailRoutingDto) -> SyncResult<Flow> {+ if !store_pulled_vault(ctx, encrypted_vault, revision).await? {+ return Ok(Flow::Resync { outdated: false });+ }+ Ok(Flow::Done(materialized_vault_result(ctx, revision, email_routing).await?))+}++/// Pull the server's latest vault and merge it with what is stored locally when needed.+async fn pull_and_materialize_server_vault(ctx: &mut Ctx, grant_sync_changed_vault: bool) -> SyncResult<Flow> {+ let mut facts = SnapshotFacts::new();++ if !ctx.is_dirty || !ctx.has_local_vault().await? {+ return adopt_pulled_vault(ctx, &mut facts).await;+ }++ // A dirty vault the codec cannot canonicalize is not merged.+ if vault_predates_current_schema(ctx).await? {+ ctx.warn("[VaultSync] The local vault predates the current storage model, so its pending changes can be neither merged nor uploaded; taking the server's vault.").await;+ return adopt_pulled_vault(ctx, &mut facts).await;+ }+ canonical_pull_and_merge(ctx, &mut facts, grant_sync_changed_vault).await+}++/// Pull the server's vault and store it as the local one.+async fn adopt_pulled_vault(ctx: &mut Ctx, facts: &mut SnapshotFacts) -> SyncResult<Flow> {+ match pull::pull(ctx, facts).await? {+ PullOutcome::Legacy(legacy, routing) => adopt_server_vault(ctx, &legacy.encrypted_blob, legacy.revision, routing).await,+ PullOutcome::Materialized { encrypted_vault, revision, email_routing } => adopt_server_vault(ctx, &encrypted_vault, revision, email_routing).await,+ }+}++/// Whether the local vault is still on a schema the codec cannot canonicalize.+async fn vault_predates_current_schema(ctx: &mut Ctx) -> SyncResult<bool> {+ if db::requires_legacy_sqlite_blob_migration(&ctx.host, Db::Local).await? {+ return Ok(true);+ }+ let schema = ctx.schema().await?;+ db::requires_schema_migration(&ctx.host, &schema.migration_id).await+}++/// Canonical-merge path of a dirty pull.+async fn canonical_pull_and_merge(ctx: &mut Ctx, facts: &mut SnapshotFacts, grant_sync_changed_vault: bool) -> SyncResult<Flow> {+ match merge::pull_and_merge(ctx, facts).await? {+ PullAndMergeOutcome::LegacyServer(legacy) => {+ ctx.warn(format!("[VaultSync] Server vault (revision {}) is still on the legacy storage format while the local vault is migrated; pushing the local vault.", legacy.revision)).await;+ match push_and_report(ctx, None, true, false).await? {+ Some(flow) => Ok(flow),+ None => Ok(Flow::Done(FullSyncResult { success: true, has_new_vault: grant_sync_changed_vault, ..Default::default() })),+ }+ }+ PullAndMergeOutcome::ServerOnly { encrypted_vault, revision, email_routing, manifest_revisions, bucket_revisions } => {+ if !store_pulled_vault(ctx, &encrypted_vault, revision).await? {+ return Ok(Flow::Resync { outdated: false });+ }+ pull::commit_revisions(ctx, &manifest_revisions, &bucket_revisions).await?;+ Ok(Flow::Done(materialized_vault_result(ctx, revision, email_routing).await?))+ }+ PullAndMergeOutcome::Merged { encrypted_vault, revision, email_routing, stats, fallback_manifest_ids, dropped_local_manifest_ids, manifest_revisions, bucket_revisions, push_canonical } => {+ if !store_pulled_vault(ctx, &encrypted_vault, revision).await? {+ return Ok(Flow::Resync { outdated: false });+ }+ // Only now do the pulled revisions become the local truth.+ pull::commit_revisions(ctx, &manifest_revisions, &bucket_revisions).await?;+ ctx.log(format!("[VaultSync] Canonical vault merge completed: {:?}; {} validation fallback(s), {} dropped local manifest(s).", stats, fallback_manifest_ids.len(), dropped_local_manifest_ids.len())).await;++ if let Some(flow) = push_and_report(ctx, push_canonical.map(|set| (ctx.mutation_sequence, set)), false, false).await? {+ return Ok(flow);+ }+ Ok(Flow::Done(materialized_vault_result(ctx, revision, email_routing).await?))+ }+ }+}++/// Push path: server and client agree on every revision, so the pending local changes upload as-is.+async fn push_pending_local_changes(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)>, grant_sync_changed_vault: bool) -> SyncResult<Flow> {+ let upload = push::upload_vault(ctx, cache, false, false).await?;+ match upload.status {+ PushStatus::Ok => {+ state::mark_clean(&ctx.host, upload.mutation_seq_at_start).await?;+ Ok(Flow::Done(FullSyncResult { success: true, has_new_vault: upload.vault_changed || grant_sync_changed_vault, ..Default::default() }))+ }+ PushStatus::Outdated => Ok(Flow::Resync { outdated: true }),+ }+}++/// Push the pending local changes. None on success, else the flow to return.+async fn push_and_report(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)>, force_full_write: bool, create_vault_key: bool) -> SyncResult<Option<Flow>> {+ let upload = push::upload_vault(ctx, cache, force_full_write, create_vault_key).await?;+ match upload.status {+ PushStatus::Ok => {+ state::mark_clean(&ctx.host, upload.mutation_seq_at_start).await?;+ Ok(None)+ }+ PushStatus::Outdated => Ok(Some(Flow::Resync { outdated: true })),+ }+}++/// Check for any pending migrations of the stored vault.+async fn pending_migration_result(ctx: &mut Ctx) -> SyncResult<Option<FullSyncResult>> {+ let checked: SyncResult<Option<FullSyncResult>> = async {+ if db::requires_legacy_sqlite_blob_migration(&ctx.host, Db::Local).await? {+ return Ok(Some(FullSyncResult { success: true, sqlite_blob_upgrade_required: true, ..Default::default() }));+ }+ if vault_requires_manifest_migration(ctx).await? {+ return Ok(Some(FullSyncResult { success: true, manifest_migration_required: true, ..Default::default() }));+ }+ Ok(None)+ }+ .await;+ match checked {+ Ok(result) => Ok(result),+ Err(error @ SyncError::VaultVersionIncompatible(_)) => Err(error),+ Err(error) => {+ ctx.warn(format!("[VaultSync] Ignoring failed pending migration check: {}", error)).await;+ Ok(None)+ }+ }+}++/// Classify the pending migration so the upgrade gate knows whether it may run on its own.+///+/// A vault still on the frozen sqlite-blob chain classifies as `None`: nothing here applies until the host has+/// walked that chain. `migrate_manifest` treats the same state as a hard error instead, because reaching the+/// executor with the chain outstanding is a caller bug, while asking the classifier about it is not.+async fn migration_status(ctx: &mut Ctx) -> MigrationStatusResult {+ let classified: SyncResult<MigrationKind> = async {+ if db::requires_legacy_sqlite_blob_migration(&ctx.host, Db::Local).await? {+ return Ok(MigrationKind::None);+ }+ match keys::fetch_vault_key(&ctx.host).await {+ Ok((_, vault_key)) => {+ if vault_key.is_some() {+ keys::adopt_remote_vault_key_if_needed(ctx).await?;+ }+ }+ Err(error) => ctx.warn(format!("[ManifestMigration] Vault key probe failed, classifying from local state: {}", error)).await,+ }+ if !keys::has_local_vault_key(&ctx.host).await? {+ return Ok(MigrationKind::StorageFormatUpgrade);+ }+ let schema = ctx.schema().await?;+ if db::requires_schema_migration(&ctx.host, &schema.migration_id).await? {+ return Ok(MigrationKind::SchemaRebuild);+ }+ Ok(MigrationKind::None)+ }+ .await;+ let kind = match classified {+ Ok(kind) => kind,+ Err(error) => {+ ctx.warn(format!("[ManifestMigration] Could not classify the pending migration, assuming it crosses the storage format: {}", error)).await;+ MigrationKind::StorageFormatUpgrade+ }+ };+ MigrationStatusResult { kind, session: SessionOutcome::default() }+}++/// Upgrade the local vault to the current storage model and push it.+async fn migrate_manifest(ctx: &mut Ctx) -> MigrateManifestResult {+ let migrated: SyncResult<bool> = async {+ let encryption_key = ctx.encryption_key()?;+ if db::requires_legacy_sqlite_blob_migration(&ctx.host, Db::Local).await? {+ return Err(SyncError::LegacyUpgradePending);+ }+ if !keys::adopt_remote_vault_key_if_needed(ctx).await? {+ return Err(SyncError::KeyOutOfSync);+ }+ let schema = ctx.schema().await?;+ let needs_schema_migration = db::requires_schema_migration(&ctx.host, &schema.migration_id).await?;+ let needs_vault_key = !keys::has_local_vault_key(&ctx.host).await?;+ if !needs_schema_migration && !needs_vault_key {+ return Ok(true);+ }+ if needs_schema_migration {+ let migrated_bytes = legacy::migrate_vault_to_current_schema(ctx).await?;+ let stored = state::store_vault(&ctx.host, &state::encrypt_vault_blob(&migrated_bytes, &encryption_key)?, true, None, None).await?;+ ctx.mutation_sequence = stored.mutation_sequence;+ ctx.is_dirty = true;+ ctx.vault_changed = true;+ }+ match push::upload_vault(ctx, None, false, needs_vault_key).await {+ Ok(upload) if upload.status == PushStatus::Ok => {+ state::mark_clean(&ctx.host, upload.mutation_seq_at_start).await?;+ Ok(true)+ }+ Ok(upload) => {+ ctx.warn(format!("[ManifestMigration] Migration push did not succeed ({:?}), vault stays dirty for the next sync.", upload.status)).await;+ Ok(false)+ }+ Err(error) => {+ ctx.warn(format!("[ManifestMigration] Migration push failed, vault stays dirty for the next sync: {}", error)).await;+ Ok(false)+ }+ }+ }+ .await;+ match migrated {+ Ok(pushed) => MigrateManifestResult { success: true, pushed, ..Default::default() },+ Err(error) => {+ ctx.warn(format!("[ManifestMigration] Migration failed: {}", error)).await;+ let (error_code, error_key) = match error.failure() {+ Failure::Coded(code) => (Some(code), None),+ Failure::Logout(reason) => (None, Some(reason)),+ };+ MigrateManifestResult { success: false, pushed: false, error: Some(error.to_string()), error_code, error_key, requires_logout: error_key.is_some(), ..Default::default() }+ }+ }+}++/// A lightweight status check: does the server hold newer state, and does the session still stand.+async fn status_check(ctx: &mut Ctx) -> StatusCheckResult {+ let checked: SyncResult<StatusCheckResult> = async {+ let status = http::get_status(&ctx.host).await?;+ if status.server_version == http::SERVER_UNREACHABLE_VERSION {+ return Ok(StatusCheckResult { success: true, has_dirty_changes: ctx.is_dirty, is_offline: true, ..Default::default() });+ }+ let logout_reason = if !status.client_version_supported {+ Some(LogoutReason::ClientVersionNotSupported)+ } else if server_too_old(ctx, &status) {+ Some(LogoutReason::ServerVersionNotSupported)+ } else {+ None+ };+ if let Some(reason) = logout_reason {+ return Ok(StatusCheckResult { requires_logout: true, error_key: Some(reason), server_version: Some(status.server_version), ..Default::default() });+ }+ assert_salt_unchanged(ctx, status.srp_salt.as_deref()).await?;+ let has_newer_vault = server_state_needs_pull(ctx, &status).await?;+ Ok(StatusCheckResult { success: true, has_newer_vault, has_dirty_changes: ctx.is_dirty, server_version: Some(status.server_version), capabilities: status.capabilities, ..Default::default() })+ }+ .await;+ match checked {+ Ok(result) => result,+ Err(error) => {+ let outcome = map_sync_failure(ctx, error).await;+ StatusCheckResult {+ success: outcome.success,+ has_dirty_changes: ctx.is_dirty,+ is_offline: outcome.was_offline,+ requires_logout: outcome.requires_logout,+ error_key: outcome.error_key,+ error: outcome.error,+ error_code: outcome.error_code,+ ..Default::default()+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use super::super::errors::ErrorCode;++ #[test]+ fn failures_carry_a_code_or_a_logout_reason_never_both() {+ let coded = failure(&SyncError::Timeout("slow".to_string()));+ assert_eq!(coded.error_code, Some(ErrorCode::UploadTimeout));+ assert_eq!(coded.error_key, None);+ assert!(!coded.requires_logout);++ let logout = failure(&SyncError::VaultVersionIncompatible("3.0.0".to_string()));+ assert_eq!(logout.error_key, Some(LogoutReason::VaultVersionIncompatible));+ assert_eq!(logout.error_code, None);+ assert!(logout.requires_logout);+ }+}diff --git a/core/rust/src/vault_sync/errors.rs b/core/rust/src/vault_sync/errors.rsnew file mode 100644index 000000000..4d8813d6a--- /dev/null+++ b/core/rust/src/vault_sync/errors.rs@@ -0,0 +1,225 @@+//! Sync related error definitions.++use serde::{Deserialize, Serialize};++use super::types::CommandKind;+use crate::error::VaultError;++macro_rules! error_codes {+ ($($(#[$doc:meta])* $variant:ident = $code:literal),* $(,)?) => {+ /// Client error codes, shared with the client apps (`AppErrorCodes` in the client core).+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]+ pub enum ErrorCode {+ $($(#[$doc])* #[serde(rename = $code)] $variant,)*+ }++ impl ErrorCode {+ /// The wire form of the code (`E-202`).+ pub const fn as_str(self) -> &'static str {+ match self {+ $(ErrorCode::$variant => $code,)*+ }+ }+ }+ };+}++error_codes! {+ UnknownError = "E-001",+ /// No encryption key in the session.+ VaultLocked = "E-202",+ /// The stored vault does not decrypt with the session key.+ VaultDecryptFailed = "E-203",+ /// The server's snapshot cannot be assembled into a vault.+ SyncVaultFetchFailed = "E-502",+ /// A server manifest or bucket fails its hash check or does not decrypt.+ SyncVaultDecryptFailed = "E-503",+ /// The server cannot be reached and there is no local vault to fall back on.+ SyncServerUnreachable = "E-505",+ /// The server answered a request with an unexpected HTTP failure.+ SyncServerError = "E-506",+ /// The host failed a read command (state, database, at-rest blob).+ StorageReadFailed = "E-601",+ /// The host failed a write command (state, database, at-rest blob, dirty flag).+ StorageWriteFailed = "E-602",+ /// The host could not open the staging database.+ DatabaseInitFailed = "E-603",+ MergeFailed = "E-701",+ /// The server kept refusing the write as outdated after the re-sync limit.+ MergeConflict = "E-702",+ /// The write was rejected: integrity check, or blobs the server asked for that this client cannot supply.+ UploadFailed = "E-801",+ UploadTooLarge = "E-804",+ UploadTimeout = "E-805",+ /// A migration cannot run yet (the vault still has to walk the legacy upgrade chain).+ MigrationCheckFailed = "E-901",+ ServerUpdateRequired = "E-903",+}++impl std::fmt::Display for ErrorCode {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str(self.as_str())+ }+}++/// Why the sync has to end and host needs to logout user session.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub enum LogoutReason {+ ClientVersionNotSupported,+ ServerVersionNotSupported,+ SessionExpired,+ PasswordChanged,+ VaultVersionIncompatible,+}++/// How the host reports a failure: a forced logout, or a coded error it translates.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Failure {+ Logout(LogoutReason),+ Coded(ErrorCode),+}++#[derive(Debug, thiserror::Error)]+pub enum SyncError {+ /// The server could not be reached (transport failure).+ #[error("Network error: {0}")]+ Network(String),+ /// A request exceeded its timeout.+ #[error("Request timed out: {0}")]+ Timeout(String),+ /// The session is no longer valid (401/403 after the host's token refresh).+ #[error("Session expired")]+ Auth,+ /// The server refuses this client version (HTTP 426).+ #[error("The server no longer supports this client version")]+ ClientUpgradeRequired,+ /// The server predates the v2 API.+ #[error("The server predates the v2 API and has to be updated")]+ ServerUpdateRequired,+ /// HTTP 413.+ #[error("Request rejected with HTTP 413: payload exceeds server limit")]+ PayloadTooLarge,+ /// The server's SRP salt no longer matches the one this device derived its keys from.+ #[error("Password was changed on another device")]+ PasswordChangedElsewhere,+ /// The local vault's data version cannot be read by this client.+ #[error("{0}")]+ VaultVersionIncompatible(String),+ /// Any other HTTP failure.+ #[error("HTTP {status}: {body}")]+ Http { status: u16, body: String },+ /// The session holds no encryption key.+ #[error("No encryption key available")]+ VaultLocked,+ /// The stored vault does not decrypt with the session key.+ #[error("Vault could not be decrypted: {0}")]+ VaultDecryptFailed(String),+ /// The session key matches neither the server's KEK nor its VEK; only a re-login recovers.+ #[error("Vault encryption key out of sync with the server; log in again")]+ KeyOutOfSync,+ /// A server manifest or bucket failed its hash check or did not decrypt.+ #[error("Server vault could not be opened: {0}")]+ ServerVaultUnreadable(String),+ /// The server's snapshot is inconsistent and cannot be assembled.+ #[error("Server snapshot cannot be assembled: {0}")]+ Snapshot(String),+ #[error("Merge failed: {0}")]+ MergeFailed(String),+ /// The server kept refusing the write as outdated after the re-sync limit.+ #[error("The server keeps refusing the write as outdated")]+ ResyncLimitReached,+ /// The write failed the vault integrity check; the failed rules.+ #[error("Upload rejected by the vault integrity check: {}", .0.join("; "))]+ UploadRejected(Vec<String>),+ /// The server asked for blobs this client cannot supply.+ #[error("Server reported blobs this client cannot supply: {}", .0.join(", "))]+ MissingBlobs(Vec<String>),+ /// The manifest migration cannot run before the legacy sqlite-blob upgrade chain.+ #[error("The vault has to walk the legacy upgrade chain first")]+ LegacyUpgradePending,+ /// The host reported a failure for a command.+ #[error("Host command {command} failed: {message}")]+ Host { command: CommandKind, message: String },+ #[error("JSON error: {0}")]+ Json(#[from] serde_json::Error),+ /// A codec, merge or crypto failure from the core library.+ #[error(transparent)]+ Core(#[from] VaultError),+ /// Anything else.+ #[error("{0}")]+ Other(String),+}++impl SyncError {+ /// How the host reports this failure.+ pub fn failure(&self) -> Failure {+ match self {+ SyncError::Auth => Failure::Logout(LogoutReason::SessionExpired),+ SyncError::ClientUpgradeRequired => Failure::Logout(LogoutReason::ClientVersionNotSupported),+ SyncError::PasswordChangedElsewhere => Failure::Logout(LogoutReason::PasswordChanged),+ SyncError::VaultVersionIncompatible(_) => Failure::Logout(LogoutReason::VaultVersionIncompatible),+ SyncError::Network(_) => Failure::Coded(ErrorCode::SyncServerUnreachable),+ SyncError::Timeout(_) => Failure::Coded(ErrorCode::UploadTimeout),+ SyncError::ServerUpdateRequired => Failure::Coded(ErrorCode::ServerUpdateRequired),+ SyncError::PayloadTooLarge => Failure::Coded(ErrorCode::UploadTooLarge),+ SyncError::Http { .. } => Failure::Coded(ErrorCode::SyncServerError),+ SyncError::VaultLocked => Failure::Coded(ErrorCode::VaultLocked),+ SyncError::VaultDecryptFailed(_) | SyncError::KeyOutOfSync => Failure::Coded(ErrorCode::VaultDecryptFailed),+ SyncError::ServerVaultUnreadable(_) => Failure::Coded(ErrorCode::SyncVaultDecryptFailed),+ SyncError::Snapshot(_) => Failure::Coded(ErrorCode::SyncVaultFetchFailed),+ SyncError::MergeFailed(_) => Failure::Coded(ErrorCode::MergeFailed),+ SyncError::ResyncLimitReached => Failure::Coded(ErrorCode::MergeConflict),+ SyncError::UploadRejected(_) | SyncError::MissingBlobs(_) => Failure::Coded(ErrorCode::UploadFailed),+ SyncError::LegacyUpgradePending => Failure::Coded(ErrorCode::MigrationCheckFailed),+ SyncError::Host { command, .. } => Failure::Coded(command.storage_error_code()),+ SyncError::Json(_) | SyncError::Core(_) | SyncError::Other(_) => Failure::Coded(ErrorCode::UnknownError),+ }+ }++ /// The client error code for this failure; `None` when it is a logout instead.+ pub fn code(&self) -> Option<ErrorCode> {+ match self.failure() {+ Failure::Coded(code) => Some(code),+ Failure::Logout(_) => None,+ }+ }++ /// The logout reason for this failure; `None` when it is a coded error instead.+ pub fn logout_reason(&self) -> Option<LogoutReason> {+ match self.failure() {+ Failure::Logout(reason) => Some(reason),+ Failure::Coded(_) => None,+ }+ }+}++impl CommandKind {+ /// The client error code a host failure of this command maps to.+ pub fn storage_error_code(self) -> ErrorCode {+ match self {+ CommandKind::DbOpen => ErrorCode::DatabaseInitFailed,+ CommandKind::StateGet | CommandKind::DbQuery | CommandKind::DbExport | CommandKind::VaultLoad => ErrorCode::StorageReadFailed,+ CommandKind::StateSet | CommandKind::StateRemove | CommandKind::DbExec | CommandKind::VaultStore | CommandKind::MarkClean => ErrorCode::StorageWriteFailed,+ CommandKind::Http | CommandKind::Log | CommandKind::Done => ErrorCode::UnknownError,+ }+ }+}++pub type SyncResult<T> = Result<T, SyncError>;++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn codes_and_reasons_serialize_to_the_client_vocabulary() {+ assert_eq!(serde_json::to_string(&ErrorCode::VaultLocked).unwrap(), "\"E-202\"");+ assert_eq!(ErrorCode::UploadTimeout.as_str(), "E-805");+ assert_eq!(serde_json::to_string(&LogoutReason::PasswordChanged).unwrap(), "\"passwordChanged\"");+ assert_eq!(SyncError::Auth.logout_reason(), Some(LogoutReason::SessionExpired));+ assert_eq!(SyncError::Auth.code(), None);+ assert_eq!(SyncError::Host { command: CommandKind::DbOpen, message: String::new() }.code(), Some(ErrorCode::DatabaseInitFailed));+ assert_eq!(SyncError::Host { command: CommandKind::DbExec, message: String::new() }.code(), Some(ErrorCode::StorageWriteFailed));+ }+}diff --git a/core/rust/src/vault_sync/http.rs b/core/rust/src/vault_sync/http.rsnew file mode 100644index 000000000..ac659812c--- /dev/null+++ b/core/rust/src/vault_sync/http.rs@@ -0,0 +1,86 @@+//! API calls through the host's transport.++use serde::de::DeserializeOwned;+use serde::Serialize;++use super::errors::{SyncError, SyncResult};+use super::session::Host;+use super::types::{Command, HttpMethod, HttpResponse, LogLevel, StatusResponse};++async fn send(host: &Host, method: HttpMethod, path: &str, body: Option<String>, auth: bool, large_transfer: bool) -> SyncResult<HttpResponse> {+ let response: HttpResponse = host.call(Command::Http { method, path: path.to_string(), body, auth, large_transfer }).await?;+ if response.status == 0 {+ let message = response.transport_error.unwrap_or_else(|| "request failed".to_string());+ return Err(if response.timed_out { SyncError::Timeout(message) } else { SyncError::Network(message) });+ }+ if response.status == 426 {+ return Err(SyncError::ClientUpgradeRequired);+ }+ Ok(response)+}++fn check(response: HttpResponse) -> SyncResult<String> {+ match response.status {+ 200..=299 => Ok(response.body),+ 401 | 403 => Err(SyncError::Auth),+ 413 => Err(SyncError::PayloadTooLarge),+ status => Err(SyncError::Http { status, body: response.body }),+ }+}++fn parse<T: DeserializeOwned>(body: &str) -> SyncResult<T> {+ serde_json::from_str(body).map_err(|e| SyncError::Other(format!("Unexpected API response: {}", e)))+}++fn v2(path: &str) -> String {+ format!("v2/{}", path)+}++/// Authenticated GET of a v2 endpoint.+pub(crate) async fn get<T: DeserializeOwned>(host: &Host, path: &str, large_transfer: bool) -> SyncResult<T> {+ let body = check(send(host, HttpMethod::Get, &v2(path), None, true, large_transfer).await?)?;+ parse(&body)+}++/// Authenticated POST of a v2 endpoint with a JSON body.+pub(crate) async fn post<B: Serialize, T: DeserializeOwned>(host: &Host, path: &str, body: &B, large_transfer: bool) -> SyncResult<T> {+ let body = check(send(host, HttpMethod::Post, &v2(path), Some(serde_json::to_string(body)?), true, large_transfer).await?)?;+ parse(&body)+}++/// Authenticated POST that returns no body.+pub(crate) async fn post_no_content<B: Serialize>(host: &Host, path: &str, body: &B) -> SyncResult<()> {+ check(send(host, HttpMethod::Post, &v2(path), Some(serde_json::to_string(body)?), true, false).await?)?;+ Ok(())+}++/// Authenticated DELETE of a v2 endpoint.+pub(crate) async fn delete(host: &Host, path: &str) -> SyncResult<()> {+ check(send(host, HttpMethod::Delete, &v2(path), None, true, false).await?)?;+ Ok(())+}++/// Turn a 404 from a v2 vault endpoint into "the server predates the v2 API".+pub(crate) fn with_outdated_server_guard<T>(result: SyncResult<T>) -> SyncResult<T> {+ match result {+ Err(SyncError::Http { status: 404, .. }) => Err(SyncError::ServerUpdateRequired),+ other => other,+ }+}++/// The server version a status call reports when the server cannot be reached.+pub(crate) const SERVER_UNREACHABLE_VERSION: &str = "0.0.0";++/// `GET v2/Status`. Anything but an auth failure or a client-version refusal reads as "server unreachable",+/// reported as server version [`SERVER_UNREACHABLE_VERSION`] so the caller can go offline.+pub(crate) async fn get_status(host: &Host) -> SyncResult<StatusResponse> {+ match get::<StatusResponse>(host, "Status", false).await {+ Ok(status) => Ok(status),+ Err(SyncError::Auth) => Err(SyncError::Auth),+ Err(SyncError::ClientUpgradeRequired) => Err(SyncError::ClientUpgradeRequired),+ Err(error) => {+ host.log(LogLevel::Warn, format!("[VaultSync] Status call failed, treating the server as unreachable: {}", error)).await;+ Ok(StatusResponse { client_version_supported: true, server_version: SERVER_UNREACHABLE_VERSION.to_string(), ..Default::default() })+ }+ }+}diff --git a/core/rust/src/vault_sync/keys.rs b/core/rust/src/vault_sync/keys.rsnew file mode 100644index 000000000..79ea9a4c8--- /dev/null+++ b/core/rust/src/vault_sync/keys.rs@@ -0,0 +1,162 @@+//! Key material during a sync.++use std::collections::HashMap;++use super::errors::{SyncError, SyncResult};+use super::session::Host;+use super::state::{self, Ctx};+use super::types::{SharedManifestRecord, VaultKeyGetResponse, VaultKeyResponse, ALGORITHM_RSA_OAEP_SHA256};+use super::{db, http};+use crate::crypto;++/// Whether this device holds a vault key.+pub(crate) async fn has_local_vault_key(host: &Host) -> SyncResult<bool> {+ Ok(state::get::<String>(host, state::ENCRYPTED_ACCOUNT_KEY).await?.is_some())+}++/// `GET v2/VaultKey/Password`.+pub(crate) async fn fetch_vault_key(host: &Host) -> SyncResult<(bool, Option<VaultKeyResponse>)> {+ match http::get::<VaultKeyGetResponse>(host, "VaultKey/Password", false).await {+ Ok(response) => Ok((true, response.vault_key)),+ Err(SyncError::Http { status: 404, .. }) => Ok((false, None)),+ Err(error) => Err(error),+ }+}++/// Persist a server vault-key response's encrypted blobs.+pub(crate) async fn cache_vault_key_blobs(host: &Host, vault_key: &VaultKeyResponse) -> SyncResult<()> {+ state::set(host, state::ENCRYPTED_ACCOUNT_KEY, &vault_key.encrypted_account_key).await?;+ state::set(host, state::ENCRYPTED_VEK, &vault_key.encrypted_vek).await?;+ match (&vault_key.account_public_key, &vault_key.encrypted_account_private_key) {+ (Some(public), Some(private)) => {+ state::set(host, state::ACCOUNT_PUBLIC_KEY, public).await?;+ state::set(host, state::ENCRYPTED_ACCOUNT_PRIVATE_KEY, private).await?;+ }+ _ => {+ state::remove(host, state::ACCOUNT_PUBLIC_KEY).await?;+ state::remove(host, state::ENCRYPTED_ACCOUNT_PRIVATE_KEY).await?;+ }+ }+ Ok(())+}++/// The shared-manifest key records, keyed by manifest id.+pub(crate) async fn shared_manifest_records(ctx: &Ctx) -> SyncResult<HashMap<String, SharedManifestRecord>> {+ let Some(ciphertext) = state::get::<String>(&ctx.host, state::SHARED_MANIFESTS).await? else { return Ok(HashMap::new()) };+ let Some(key) = &ctx.encryption_key else { return Ok(HashMap::new()) };+ match crypto::symmetric_decrypt(&ciphertext, key).and_then(|json| serde_json::from_str(&json).map_err(Into::into)) {+ Ok(records) => Ok(records),+ Err(error) => {+ ctx.warn(format!("[Sharing] The stored shared-manifest key records did not decrypt (re-keyed vault?); treating them as absent. {}", error)).await;+ Ok(HashMap::new())+ }+ }+}++/// Persist the shared-manifest key records.+pub(crate) async fn set_shared_manifest_records(host: &Host, records: &HashMap<String, SharedManifestRecord>, key: &str) -> SyncResult<()> {+ let ciphertext = crypto::symmetric_encrypt(&serde_json::to_string(records)?, key)?;+ state::set(host, state::SHARED_MANIFESTS, &ciphertext).await?;+ Ok(())+}++/// Re-encrypt the records under a new vault encryption key.+pub(crate) async fn re_encrypt_shared_manifest_records(ctx: &Ctx, new_key: &str) -> SyncResult<()> {+ let records = shared_manifest_records(ctx).await?;+ if records.is_empty() {+ return Ok(());+ }+ set_shared_manifest_records(&ctx.host, &records, new_key).await+}++/// The private key that opens a grant made out to `public_key`.+pub(crate) async fn resolve_grant_private_key(ctx: &Ctx, public_key: &str) -> SyncResult<Option<String>> {+ if ctx.account_public_key.as_deref() == Some(public_key) {+ if let Some(private) = &ctx.account_private_key {+ return Ok(Some(private.clone()));+ }+ }+ let Some(personal_manifest_id) = state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await? else { return Ok(None) };+ db::account_private_key_for(&ctx.host, &personal_manifest_id, public_key).await+}++/// Decrypt an RSA-OAEP encrypted manifest VEK.+pub(crate) fn decrypt_manifest_vek(encrypted_vek: &str, private_key_jwk: &str) -> SyncResult<String> {+ let plaintext = crypto::decrypt_with_private_key(encrypted_vek, private_key_jwk)?;+ String::from_utf8(plaintext).map_err(|_| SyncError::Other("Decrypted manifest key is not valid text".to_string()))+}++/// Unwrap one shared manifest's VEK from the grant this account holds on it.+pub(crate) async fn open_shared_manifest_vek(ctx: &Ctx, record: &SharedManifestRecord) -> SyncResult<Option<String>> {+ if record.algorithm != ALGORITHM_RSA_OAEP_SHA256 {+ ctx.warn(format!("[Sharing] Manifest {} grants its key under an unsupported algorithm \"{}\" (newer server?); leaving it closed.", record.manifest_id, record.algorithm)).await;+ return Ok(None);+ }+ let Some(private_key) = resolve_grant_private_key(ctx, &record.encryption_public_key).await? else {+ ctx.warn(format!("[Sharing] No account key in this vault opens the grant on manifest {}; leaving it closed.", record.manifest_id)).await;+ return Ok(None);+ };+ match decrypt_manifest_vek(&record.encrypted_vek, &private_key) {+ Ok(vek) => Ok(Some(vek)),+ Err(error) => {+ ctx.warn(format!("[Sharing] Failed to unwrap the key of manifest {}; leaving it closed. {}", record.manifest_id, error)).await;+ Ok(None)+ }+ }+}++/// The VEK of every shared manifest this account holds a grant on, keyed by manifest id.+pub(crate) async fn open_shared_manifest_veks(ctx: &Ctx) -> SyncResult<HashMap<String, String>> {+ let mut veks = HashMap::new();+ for record in shared_manifest_records(ctx).await?.values() {+ if let Some(vek) = open_shared_manifest_vek(ctx, record).await? {+ veks.insert(record.manifest_id.clone(), vek);+ }+ }+ Ok(veks)+}++/// Adopt a server-side vault key this device does not know about yet (another device performed the KEK/VEK+/// migration while this one held the old password-derived key). False only when this device's key matches+/// neither the KEK nor the VEK, which requires a re-login.+pub(crate) async fn adopt_remote_vault_key_if_needed(ctx: &mut Ctx) -> SyncResult<bool> {+ if has_local_vault_key(&ctx.host).await? {+ return Ok(true);+ }+ let Some(session_key) = ctx.encryption_key.clone() else { return Ok(true) };++ let (_, vault_key) = match fetch_vault_key(&ctx.host).await {+ Ok(result) => result,+ Err(error) => {+ ctx.warn(format!("[VaultSync] Vault key probe failed, deferring vault key adoption: {}", error)).await;+ return Ok(true);+ }+ };+ let Some(vault_key) = vault_key else { return Ok(true) };+ let Some(encrypted_vek) = vault_key.encrypted_vek.clone() else { return Ok(true) };++ let adopted: SyncResult<()> = async {+ let (vek, _account_key) = crypto::resolve_vault_encryption_key(&vault_key.encrypted_account_key, &encrypted_vek, &session_key)?;+ if let Some(encrypted_vault) = state::load_vault(&ctx.host).await? {+ let plaintext = state::decrypt_vault_blob(&encrypted_vault, &session_key)?;+ let re_encrypted = crypto::symmetric_encrypt_bytes(&plaintext, &vek)?;+ state::store_vault_with_key(&ctx.host, &re_encrypted, false, None, None, Some(vek.to_string())).await?;+ }+ re_encrypt_shared_manifest_records(ctx, &vek).await?;+ cache_vault_key_blobs(&ctx.host, &vault_key).await?;+ ctx.set_encryption_key(vek.to_string());+ Ok(())+ }+ .await;++ match adopted {+ Ok(()) => {+ ctx.log("[VaultSync] Adopted vault key created by another client; session key swapped to the VEK.").await;+ Ok(true)+ }+ Err(error) => {+ ctx.warn(format!("[VaultSync] Session key matches neither the KEK nor the VEK, forcing re-login: {}", error)).await;+ Ok(false)+ }+ }+}diff --git a/core/rust/src/vault_sync/legacy.rs b/core/rust/src/vault_sync/legacy.rsnew file mode 100644index 000000000..0a7132b70--- /dev/null+++ b/core/rust/src/vault_sync/legacy.rs@@ -0,0 +1,29 @@+//! LEGACY: what only a vault still on the sqlite-blob storage format needs, namely its local migration onto+//! the current schema. TODO: remove this module once all users have migrated to the manifest-v1 storage model.++use std::collections::HashMap;++use super::errors::SyncResult;+use super::pull::materialize_to_sqlite;+use super::push::{canonicalize_vault, resolve_personal_manifest_id};+use super::state::Ctx;+use crate::crypto;+use crate::vault_codec::Manifest;++/// Migrate the local vault onto the current schema, entirely locally.+pub(crate) async fn migrate_vault_to_current_schema(ctx: &mut Ctx) -> SyncResult<Vec<u8>> {+ ctx.log("[ManifestMigration] Migrating local vault onto the current schema (local round-trip, no server involved)...").await;+ // A sqlite-blob vault's rows carry no ManifestId yet, so this one canonicalize adopts them.+ let personal = resolve_personal_manifest_id(ctx).await?;+ let set = canonicalize_vault(ctx, Some(personal)).await?;+ let mut blob_map = HashMap::new();+ for entry in &set.canonicalized.manifests {+ for (hash, blob) in &entry.blobs {+ blob_map.insert(hash.clone(), crypto::aes_gcm::decode_base64(&blob.bytes_base64)?);+ }+ }+ let manifests: Vec<Manifest> = set.canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect();+ let bytes = materialize_to_sqlite(ctx, &manifests, &set.canonicalized.data_buckets, &blob_map).await?;+ ctx.log(format!("[ManifestMigration] Migration complete: {} blobs re-embedded, {} bytes.", blob_map.len(), bytes.len())).await;+ Ok(bytes)+}diff --git a/core/rust/src/vault_sync/merge.rs b/core/rust/src/vault_sync/merge.rsnew file mode 100644index 000000000..163dc7a95--- /dev/null+++ b/core/rust/src/vault_sync/merge.rs@@ -0,0 +1,269 @@+//! Merge: a dirty vault's pull, which merges the local vault onto the server snapshot at canonical level, one+//! manifest at a time, and materializes the result.++use std::collections::{HashMap, HashSet};++use base64::engine::general_purpose::STANDARD as BASE64;+use base64::Engine;+use serde::{Deserialize, Serialize};++use super::db::{id_key, ids_equal};+use super::errors::{SyncError, SyncResult};+use super::pull::{self, LegacyPassthrough, OpenedManifestSet, SnapshotFacts};+use super::push::{canonicalize_vault, CanonicalizedSet, ManifestRecord};+use super::state::{self, Ctx};+use super::types::EmailRoutingDto;+use crate::crypto;+use crate::vault_codec::{self, BlobEntry, CanonicalizedVault, DataBucket, Manifest};+use crate::vault_merge::{merge_canonical, CanonicalManifestMerge, CanonicalMergeInput, MergeStats};++/// Aggregate merge statistics across all manifests.+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct MergeSummary {+ pub tables_processed: u32,+ pub records_from_local: u32,+ pub records_from_server: u32,+ pub records_created_locally: u32,+ pub conflicts: u32,+ pub records_inserted: u32,+}++impl MergeSummary {+ fn add(&mut self, stats: &MergeStats) {+ self.tables_processed += stats.tables_processed;+ self.records_from_local += stats.records_from_local;+ self.records_from_server += stats.records_from_server;+ self.records_created_locally += stats.records_created_locally;+ self.conflicts += stats.conflicts;+ self.records_inserted += stats.records_inserted;+ }+}++/// What a dirty pull (pull and merge) produced.+pub(crate) enum PullAndMergeOutcome {+ LegacyServer(LegacyPassthrough),+ Merged {+ encrypted_vault: String,+ revision: i64,+ email_routing: EmailRoutingDto,+ stats: MergeSummary,+ fallback_manifest_ids: Vec<String>,+ dropped_local_manifest_ids: Vec<String>,+ manifest_revisions: HashMap<String, i64>,+ bucket_revisions: HashMap<String, i64>,+ push_canonical: Option<CanonicalizedSet>,+ },+ ServerOnly {+ encrypted_vault: String,+ revision: i64,+ email_routing: EmailRoutingDto,+ manifest_revisions: HashMap<String, i64>,+ bucket_revisions: HashMap<String, i64>,+ },+}++/// Pull the latest snapshot and merge the local vault onto it at canonical level, one manifest at a time.+pub(crate) async fn pull_and_merge(ctx: &mut Ctx, facts: &mut SnapshotFacts) -> SyncResult<PullAndMergeOutcome> {+ let vek = ctx.encryption_key()?;+ ctx.log("[V2Merge] Fetching vault snapshot for canonical merge (GET /v2/Vault)...").await;+ let snapshot = pull::fetch_snapshot(ctx, facts).await?;+ let email_routing = pull::email_routing_of(&snapshot);++ // LEGACY: a server still on the sqlite-blob format cannot merge with a manifest-v1 vault; the caller pushes over it.+ if facts.legacy {+ return Ok(PullAndMergeOutcome::LegacyServer(pull::open_legacy_snapshot(ctx, &snapshot).await?));+ }++ let opened = pull::open_manifests_and_record_sync_state(ctx, &snapshot, &vek).await?;+ let local_side = canonicalize_vault(ctx, None).await?;+ let local_side = match ground_moved_under_canonicalize(&local_side.manifest_records, &opened) {+ None => local_side,+ Some(moved) => {+ ctx.warn(format!("[V2Merge] This snapshot changed {}; canonicalizing the local vault again so its blob hashes match the server's.", moved)).await;+ canonicalize_vault(ctx, None).await?+ }+ };++ match merge_onto_opened_manifests(ctx, &opened, local_side, &vek, email_routing.clone()).await {+ Ok(outcome) => Ok(outcome),+ Err(merge_error) => {+ // The merge-failure fallback, from the same snapshot: the server vault stands, local changes are dropped.+ ctx.warn(format!("[V2Merge] Canonical merge failed, falling back to the server vault: {}", merge_error)).await;+ let sqlite_bytes = pull::materialize_to_sqlite(ctx, &opened.manifests(), &opened.data_buckets, &opened.blob_map).await?;+ Ok(PullAndMergeOutcome::ServerOnly {+ encrypted_vault: state::encrypt_vault_blob(&sqlite_bytes, &vek)?,+ revision: opened.personal_revision,+ email_routing,+ manifest_revisions: opened.manifest_revisions,+ bucket_revisions: opened.bucket_revisions,+ })+ }+ }+}++/// What a snapshot changed under a canonicalize that ran against the previous local vault.+fn ground_moved_under_canonicalize(records: &[ManifestRecord], opened: &OpenedManifestSet) -> Option<String> {+ let personal_record = records.iter().find(|r| r.is_personal);+ let server_personal = opened.resolved.iter().find(|m| m.is_personal);+ if let Some(server) = server_personal {+ if !personal_record.map(|r| ids_equal(&r.manifest_id, &server.manifest_id)).unwrap_or(false) {+ return Some(format!("the personal manifest id ({} became {})", personal_record.map(|r| r.manifest_id.as_str()).unwrap_or("none"), server.manifest_id));+ }+ }+ for record in records {+ if let Some(server) = opened.resolved.iter().find(|m| ids_equal(&m.manifest_id, &record.manifest_id)) {+ if server.manifest.manifest_salt != record.salt {+ return Some(format!("the blob salt of manifest {}", record.manifest_id));+ }+ }+ }+ None+}++/// Merge the local side against the opened manifests, validate per manifest, and materialize.+async fn merge_onto_opened_manifests(ctx: &mut Ctx, opened: &OpenedManifestSet, local_side: CanonicalizedSet, vek: &str, email_routing: EmailRoutingDto) -> SyncResult<PullAndMergeOutcome> {+ let schema = ctx.schema().await?;+ ctx.log(format!("[V2Merge] Merging {} local manifest(s) onto {} server manifest(s)...", local_side.canonicalized.manifests.len(), opened.resolved.len())).await;+ let merge_output = merge_canonical(CanonicalMergeInput {+ server_manifests: opened.manifests(),+ server_buckets: opened.data_buckets.clone(),+ contentless_server_manifest_ids: opened.contentless_manifest_ids.clone(),+ local_manifests: local_side.canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect(),+ local_buckets: local_side.canonicalized.data_buckets.clone(),+ schema_columns: schema.columns.clone(),+ })?;++ let server_manifest_by_id: HashMap<String, &Manifest> = opened.resolved.iter().map(|m| (id_key(&m.manifest_id), &m.manifest)).collect();+ let mut server_buckets_by_id: HashMap<String, Vec<DataBucket>> = HashMap::new();+ for bucket in &opened.data_buckets {+ server_buckets_by_id.entry(id_key(&bucket.manifest_id)).or_default().push(bucket.clone());+ }+ let contentless: HashSet<String> = opened.contentless_manifest_ids.iter().map(|id| id_key(id)).collect();++ let mut manifests: Vec<Manifest> = Vec::new();+ let mut data_buckets: Vec<DataBucket> = Vec::new();+ let mut fallback_manifest_ids = Vec::new();+ let mut stats = MergeSummary::default();+ for entry in merge_output.manifests {+ let failure = validate_merged_manifest(&entry);+ if let Some(failure) = &failure {+ if !contentless.contains(&id_key(&entry.manifest_id)) {+ ctx.warn(format!("[V2Merge] Merged manifest {} failed validation ({}); the server's version stands and local changes to it are dropped.", entry.manifest_id, failure)).await;+ fallback_manifest_ids.push(entry.manifest_id.clone());+ if let Some(server) = server_manifest_by_id.get(&id_key(&entry.manifest_id)) {+ manifests.push((*server).clone());+ data_buckets.extend(server_buckets_by_id.get(&id_key(&entry.manifest_id)).cloned().unwrap_or_default());+ }+ continue;+ }+ ctx.warn(format!("[V2Merge] Pass-through manifest {} failed validation ({}); keeping its local rows.", entry.manifest_id, failure)).await;+ }+ manifests.push(entry.manifest);+ data_buckets.extend(entry.buckets);+ stats.add(&entry.stats);+ }+ for dropped in &merge_output.dropped_local_manifest_ids {+ ctx.warn(format!("[V2Merge] Local manifest {} is no longer served; its rows are dropped from the merged vault.", dropped)).await;+ }++ // Blob bytes for materialize: the server download plus everything the local canonicalize extracted.+ let mut blob_map = opened.blob_map.clone();+ let mut local_blobs: HashMap<String, BlobEntry> = HashMap::new();+ for canonicalized in &local_side.canonicalized.manifests {+ for (hash, blob) in &canonicalized.blobs {+ local_blobs.insert(hash.clone(), blob.clone());+ if !blob_map.contains_key(hash) {+ blob_map.insert(hash.clone(), crypto::aes_gcm::decode_base64(&blob.bytes_base64)?);+ }+ }+ }+ let personal_id = opened.resolved.first().map(|m| m.manifest_id.clone()).unwrap_or_default();+ let merged_blobs = resolve_merged_blob_refs(ctx, &manifests, &blob_map, &personal_id, &local_blobs).await?;++ let sqlite_bytes = pull::materialize_to_sqlite(ctx, &manifests, &data_buckets, &blob_map).await?;+ let encrypted_vault = state::encrypt_vault_blob(&sqlite_bytes, vek)?;+ ctx.log(format!("[V2Merge] Canonical merge complete: {} conflict(s), {} offline row(s) kept, {} validation fallback(s), {} dropped local manifest(s).", stats.conflicts, stats.records_inserted, fallback_manifest_ids.len(), merge_output.dropped_local_manifest_ids.len())).await;++ let push_canonical = merge_output_for_push(&manifests, &data_buckets, merged_blobs, &local_side.manifest_records, !fallback_manifest_ids.is_empty());+ Ok(PullAndMergeOutcome::Merged {+ encrypted_vault,+ revision: opened.personal_revision,+ email_routing,+ stats,+ fallback_manifest_ids,+ dropped_local_manifest_ids: merge_output.dropped_local_manifest_ids,+ manifest_revisions: opened.manifest_revisions.clone(),+ bucket_revisions: opened.bucket_revisions.clone(),+ push_canonical,+ })+}++/// The merged vault in the shape the push writes from.+fn merge_output_for_push(manifests: &[Manifest], data_buckets: &[DataBucket], blobs_by_manifest: Option<HashMap<String, HashMap<String, BlobEntry>>>, manifest_records: &[ManifestRecord], had_fallbacks: bool) -> Option<CanonicalizedSet> {+ let blobs_by_manifest = blobs_by_manifest?;+ if had_fallbacks {+ return None;+ }+ let merged_ids: HashSet<String> = manifests.iter().map(|m| id_key(&m.manifest_id)).collect();+ let record_ids: Vec<String> = manifest_records.iter().map(|r| id_key(&r.manifest_id)).collect();+ if merged_ids.len() != record_ids.len() || record_ids.iter().any(|id| !merged_ids.contains(id)) {+ return None;+ }+ Some(CanonicalizedSet {+ canonicalized: CanonicalizedVault {+ manifests: manifests.iter().map(|m| vault_codec::CanonicalizedManifest { manifest: m.clone(), blobs: blobs_by_manifest.get(&id_key(&m.manifest_id)).cloned().unwrap_or_default() }).collect(),+ data_buckets: data_buckets.to_vec(),+ },+ manifest_records: manifest_records.to_vec(),+ })+}++/// Validate one merged manifest and its buckets.+fn validate_merged_manifest(entry: &CanonicalManifestMerge) -> Option<String> {+ let validation = vault_codec::validate_manifest(&entry.manifest);+ if !validation.ok {+ return Some(validation.failed_rules.join(", "));+ }+ for bucket in &entry.buckets {+ let bucket_validation = vault_codec::validate_data_bucket(bucket);+ if !bucket_validation.ok {+ return Some(format!("bucket \"{}\": {}", bucket.category, bucket_validation.failed_rules.join(", ")));+ }+ }+ None+}++/// Every blob marker in the merged manifests must resolve to bytes.+async fn resolve_merged_blob_refs(ctx: &Ctx, manifests: &[Manifest], blob_map: &HashMap<String, Vec<u8>>, personal_manifest_id: &str, local_blobs: &HashMap<String, BlobEntry>) -> SyncResult<Option<HashMap<String, HashMap<String, BlobEntry>>>> {+ let mut by_manifest: HashMap<String, HashMap<String, BlobEntry>> = HashMap::new();+ let mut complete = true;+ for manifest in manifests {+ let is_personal = ids_equal(&manifest.manifest_id, personal_manifest_id);+ let blobs = by_manifest.entry(id_key(&manifest.manifest_id)).or_default();+ for rows in manifest.tables.values() {+ for row in rows {+ for value in row.values() {+ let Some(reference) = value.get("__blobRef").and_then(serde_json::Value::as_str) else { continue };+ let kind = value.get("__blobKind").and_then(serde_json::Value::as_str);+ let Some(bytes) = blob_map.get(reference) else {+ if kind == Some("attachment") && is_personal {+ return Err(SyncError::MergeFailed(format!("merged vault references attachment blob {} with no bytes available, refusing to materialize an incomplete vault", reference)));+ }+ ctx.warn(format!("[V2Merge] Merged {} {} has no bytes available; it will materialize as empty.", kind.unwrap_or("blob"), reference)).await;+ complete = false;+ continue;+ };+ if let Some(local) = local_blobs.get(reference) {+ blobs.insert(reference.to_string(), local.clone());+ } else if let Some(kind) = kind {+ blobs.insert(reference.to_string(), BlobEntry { kind: kind.to_string(), bytes_base64: BASE64.encode(bytes) });+ } else {+ complete = false;+ }+ }+ }+ }+ }+ Ok(if complete { Some(by_manifest) } else { None })+}diff --git a/core/rust/src/vault_sync/mod.rs b/core/rust/src/vault_sync/mod.rsnew file mode 100644index 000000000..a4a55efc8--- /dev/null+++ b/core/rust/src/vault_sync/mod.rs@@ -0,0 +1,23 @@+//! The vault sync engine that all AliasVault clients use to synchronize their vaults with the server.++pub(crate) mod db;+pub(crate) mod email_routing;+mod engine;+pub mod errors;+pub(crate) mod http;+pub(crate) mod keys;+mod legacy;+mod merge;+mod pull;+mod push;+pub mod session;+pub(crate) mod state;+pub mod types;+pub mod version;++#[cfg(test)]+mod tests;++pub use errors::{ErrorCode, Failure, LogoutReason, SyncError};+pub use session::SyncSession;+pub use types::{Command, CommandKind, Db, FullSyncResult, HttpMethod, LogLevel, MigrateManifestResult, MigrationKind, MigrationStatusResult, SessionOutcome, SessionUpdates, SqlStatement, StatusCheckResult, SyncOperation, SyncRequest};diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rsnew file mode 100644index 000000000..791fd52cc--- /dev/null+++ b/core/rust/src/vault_sync/pull.rs@@ -0,0 +1,404 @@+//! Pull: fetch the server snapshot, open every manifest and bucket this session holds a key for, download the+//! referenced blobs and materialize the whole into a fresh SQLite database.++use std::collections::HashMap;++use super::db::ids_equal;+use super::errors::{SyncError, SyncResult};+use super::state::{self, Ctx};+use super::types::{self, BlobDto, BlobHashesRequest, Db, EmailRoutingDto, GetResponse, ManifestDto, SharedManifestRecord, StoredBlobRef};+use super::{db, http, keys};+use crate::crypto;+use crate::vault_codec::{self, DataBucket, Manifest, MaterializeInput};++const VAULT_ENDPOINT: &str = "Vault";+const BLOBS_DOWNLOAD_ENDPOINT: &str = "Vault/blobs/download";++/// Max base64 characters in one blob transfer request or response body.+pub(crate) const BLOB_TRANSFER_BATCH_MAX_CHARS: usize = 4 * 1024 * 1024;+/// Upper bound on the number of blobs in one transfer batch.+pub(crate) const BLOB_TRANSFER_BATCH_MAX_COUNT: usize = 100;++/// One manifest of a pull, opened.+pub(crate) struct ResolvedManifest {+ pub manifest_id: String,+ pub is_personal: bool,+ pub manifest: Manifest,+ pub vek: String,+ pub revision: i64,+ pub blob_references: Vec<StoredBlobRef>,+ pub content_fingerprint: String,+}++/// Every manifest of one snapshot, decrypted, with the sync state learned from it.+pub(crate) struct OpenedManifestSet {+ pub resolved: Vec<ResolvedManifest>,+ pub data_buckets: Vec<DataBucket>,+ pub blob_map: HashMap<String, Vec<u8>>,+ pub contentless_manifest_ids: Vec<String>,+ pub personal_revision: i64,+ pub manifest_revisions: HashMap<String, i64>,+ pub bucket_revisions: HashMap<String, i64>,+}++impl OpenedManifestSet {+ pub fn manifests(&self) -> Vec<Manifest> {+ self.resolved.iter().map(|m| m.manifest.clone()).collect()+ }+}++/// A not-yet-migrated server's vault, passed through as served.+/// TODO: remove all legacy related code once all users have migrated to the manifest-v1 storage model.+pub(crate) struct LegacyPassthrough {+ pub encrypted_blob: String,+ pub revision: i64,+}++/// What a pull produced.+pub(crate) enum PullOutcome {+ Legacy(LegacyPassthrough, EmailRoutingDto),+ Materialized { encrypted_vault: String, revision: i64, email_routing: EmailRoutingDto },+}++/// What the last snapshot this run fetched looked like.+pub(crate) struct SnapshotFacts {+ pub served_manifest_ids: Vec<String>,+ /// Whether the snapshot is still on the legacy sqlite-blob format.+ pub legacy: bool,+}++impl SnapshotFacts {+ pub fn new() -> Self {+ Self { served_manifest_ids: Vec::new(), legacy: false }+ }++ fn record(&mut self, snapshot: &GetResponse) {+ self.served_manifest_ids = snapshot.manifests.iter().map(|m| m.manifest_id.clone()).collect();+ self.legacy = is_legacy_sqlite_blob_snapshot(snapshot);+ }+}++/// `GET v2/Vault`.+pub(crate) async fn fetch_snapshot(ctx: &Ctx, facts: &mut SnapshotFacts) -> SyncResult<GetResponse> {+ let snapshot = http::with_outdated_server_guard(http::get::<GetResponse>(&ctx.host, VAULT_ENDPOINT, true).await)?;+ facts.record(&snapshot);+ Ok(snapshot)+}++pub(crate) fn is_legacy_sqlite_blob_snapshot(snapshot: &GetResponse) -> bool {+ snapshot.storage_format != Some(types::STORAGE_FORMAT_MANIFEST)+}++pub(crate) fn email_routing_of(snapshot: &GetResponse) -> EmailRoutingDto {+ snapshot.email_routing.clone().unwrap_or_default()+}++/// Split items into request batches bounded by both transfer limits.+pub(crate) fn batch_by_transfer_cost<T>(items: Vec<T>, cost_of: impl Fn(&T) -> usize) -> Vec<Vec<T>> {+ let mut batches = Vec::new();+ let mut batch = Vec::new();+ let mut chars = 0usize;+ for item in items {+ let cost = cost_of(&item);+ if !batch.is_empty() && (chars + cost > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.len() >= BLOB_TRANSFER_BATCH_MAX_COUNT) {+ batches.push(std::mem::take(&mut batch));+ chars = 0;+ }+ batch.push(item);+ chars += cost;+ }+ if !batch.is_empty() {+ batches.push(batch);+ }+ batches+}++fn base64_chars(size_bytes: i64) -> usize {+ ((size_bytes.max(0) as usize).div_ceil(3)) * 4+}++/// Verify a ciphertext against the server's hash, decrypt it and open it via the codec.+fn verify_decrypt_unpack(base64_ciphertext: &str, vek: &str, expected_ciphertext_hash: Option<&str>, label: &str) -> SyncResult<String> {+ if let Some(expected) = expected_ciphertext_hash.filter(|hash| !hash.is_empty()) {+ if vault_codec::compute_ciphertext_hash(base64_ciphertext) != expected {+ return Err(SyncError::ServerVaultUnreadable(format!("{} ciphertext hash mismatch, refusing to load (possible storage corruption)", label)));+ }+ }+ let unreadable = |e: crate::error::VaultError| SyncError::ServerVaultUnreadable(format!("{}: {}", label, e));+ let encrypted = crypto::aes_gcm::decode_base64(base64_ciphertext).map_err(unreadable)?;+ let plain = crypto::symmetric_decrypt_bytes(&encrypted, vek).map_err(unreadable)?;+ vault_codec::unpack_payload(&plain).map_err(unreadable)+}++/// The grant a shared manifest is remembered by, when the snapshot carries one.+fn grant_of(dto: &ManifestDto) -> Option<(String, String, String)> {+ match (&dto.encrypted_vek, &dto.encryption_public_key) {+ (Some(vek), Some(public)) if !vek.is_empty() && !public.is_empty() => Some((vek.clone(), public.clone(), dto.algorithm.clone().unwrap_or_else(|| types::ALGORITHM_RSA_OAEP_SHA256.to_string()))),+ _ => None,+ }+}++fn select_personal_manifest(snapshot: &GetResponse) -> Option<&ManifestDto> {+ let personal_id = snapshot.personal_manifest_id.as_deref()?;+ snapshot.manifests.iter().find(|m| m.manifest_id == personal_id)+}++/// Take a legacy snapshot apart for local storage: the blob passes through untouched, the manifest-v1 state+/// is reset, and the personal manifest id is recorded for the migration push.+pub(crate) async fn open_legacy_snapshot(ctx: &Ctx, snapshot: &GetResponse) -> SyncResult<LegacyPassthrough> {+ state::remove(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?;+ let revision = snapshot.legacy_revision.unwrap_or(0);+ if let Some(personal) = &snapshot.personal_manifest_id {+ state::set(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID, personal).await?;+ let mut revisions = HashMap::new();+ revisions.insert(personal.clone(), revision);+ state::set(&ctx.host, state::SERVER_MANIFEST_REVISIONS, &revisions).await?;+ }+ ctx.log("[V2Pull] Legacy sqlite-blob pass-through (user not yet migrated), returning the blob as-is.").await;+ Ok(LegacyPassthrough { encrypted_blob: snapshot.legacy_vault_blob.clone().unwrap_or_default(), revision })+}++/// Pull the latest vault: fetch, open, materialize and re-encrypt for local storage.+pub(crate) async fn pull(ctx: &mut Ctx, facts: &mut SnapshotFacts) -> SyncResult<PullOutcome> {+ let vek = ctx.encryption_key()?;+ ctx.log("[V2Pull] Step 1/4: fetching vault snapshot (GET /v2/Vault)...").await;+ let snapshot = fetch_snapshot(ctx, facts).await?;+ let email_routing = email_routing_of(&snapshot);++ if facts.legacy {+ return Ok(PullOutcome::Legacy(open_legacy_snapshot(ctx, &snapshot).await?, email_routing));+ }++ ctx.log("[V2Pull] Step 2/4: manifest format: decrypting and reassembling local SQLite...").await;+ let opened = open_manifests_and_record_sync_state(ctx, &snapshot, &vek).await?;+ commit_revisions(ctx, &opened.manifest_revisions, &opened.bucket_revisions).await?;+ let sqlite_bytes = materialize_to_sqlite(ctx, &opened.manifests(), &opened.data_buckets, &opened.blob_map).await?;+ ctx.log(format!("[V2Pull] Step 3/4: materialized SQLite ({} bytes); re-encrypting for local storage...", sqlite_bytes.len())).await;+ let encrypted_vault = state::encrypt_vault_blob(&sqlite_bytes, &vek)?;+ Ok(PullOutcome::Materialized { encrypted_vault, revision: opened.personal_revision, email_routing })+}++/// Open every manifest a snapshot carries, personal first, and record the snapshot as this device's sync state.+/// The revision maps are returned for the caller to commit once the pulled state is local truth.+pub(crate) async fn open_manifests_and_record_sync_state(ctx: &mut Ctx, snapshot: &GetResponse, vek: &str) -> SyncResult<OpenedManifestSet> {+ let personal_dto = select_personal_manifest(snapshot).ok_or_else(|| SyncError::Snapshot("server returned no personal manifest, refusing to assemble".to_string()))?;+ if personal_dto.blob.as_deref().unwrap_or("").is_empty() {+ return Err(SyncError::Snapshot("server returned no manifest blob, nothing to assemble".to_string()));+ }++ let mut pulled_fingerprints: HashMap<String, String> = HashMap::new();+ let mut resolved: Vec<ResolvedManifest> = Vec::new();+ let mut shared_records: HashMap<String, SharedManifestRecord> = HashMap::new();+ let mut contentless_revisions: HashMap<String, i64> = HashMap::new();++ let ordered: Vec<&ManifestDto> = std::iter::once(personal_dto).chain(snapshot.manifests.iter().filter(|m| m.manifest_id != personal_dto.manifest_id)).collect();+ for dto in ordered {+ let is_personal = dto.manifest_id == personal_dto.manifest_id;+ let manifest_key = resolve_manifest_vek(ctx, dto, &personal_dto.manifest_id, vek, is_personal, resolved.first().map(|m| &m.manifest)).await?;+ if dto.blob.as_deref().unwrap_or("").is_empty() {+ // A shared manifest served without content yet (created but never written); its grant and revision are still tracked.+ let (encrypted_vek, encryption_public_key, algorithm) = grant_of(dto).ok_or_else(|| SyncError::Snapshot(format!("shared manifest {} was served without content and without a grant, refusing to assemble", dto.manifest_id)))?;+ shared_records.insert(dto.manifest_id.clone(), SharedManifestRecord { manifest_id: dto.manifest_id.clone(), encrypted_vek, encryption_public_key, algorithm, salt: vault_codec::generate_manifest_salt(), name: None, can_administer: dto.can_administer });+ contentless_revisions.insert(dto.manifest_id.clone(), dto.revision);+ continue;+ }++ ctx.log(format!("[V2Pull] Verifying ciphertext hash; decrypting + opening {}...", if is_personal { "personal manifest".to_string() } else { format!("shared manifest {}", dto.manifest_id) })).await;+ let entry = open_manifest(dto, &manifest_key, is_personal)?;+ let table_summary: Vec<String> = entry.manifest.tables.iter().map(|(t, rows)| format!("{}={}", t, rows.len())).collect();+ ctx.log(format!("[V2Pull] Manifest {} opened (content hash verified): tables: {}", entry.manifest_id, table_summary.join(", "))).await;++ if is_personal {+ state::set(&ctx.host, state::VAULT_MANIFEST_SALT, &entry.manifest.manifest_salt).await?;+ state::set(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID, &entry.manifest_id).await?;+ resolved.push(entry);+ continue;+ }++ let (encrypted_vek, encryption_public_key, algorithm) = grant_of(dto).ok_or_else(|| SyncError::Snapshot(format!("shared manifest {} carries no grant this account can re-open, refusing to assemble", entry.manifest_id)))?;+ shared_records.insert(entry.manifest_id.clone(), SharedManifestRecord { manifest_id: entry.manifest_id.clone(), encrypted_vek, encryption_public_key, algorithm, salt: entry.manifest.manifest_salt.clone(), name: entry.manifest.name.clone(), can_administer: dto.can_administer });+ resolved.push(entry);+ }+ keys::set_shared_manifest_records(&ctx.host, &shared_records, vek).await?;++ let personal_revision = resolved[0].revision;++ let (data_buckets, bucket_fingerprints, bucket_revisions) = open_data_buckets(ctx, snapshot, &resolved).await?;+ pulled_fingerprints.extend(bucket_fingerprints);++ let mut manifest_revisions: HashMap<String, i64> = resolved.iter().map(|m| (m.manifest_id.clone(), m.revision)).collect();+ manifest_revisions.extend(contentless_revisions.iter().map(|(id, rev)| (id.clone(), *rev)));++ let blob_map = download_referenced_blobs(ctx, &resolved, vek).await?;++ for entry in &resolved {+ pulled_fingerprints.insert(state::fingerprint_manifest_key(&entry.manifest_id), entry.content_fingerprint.clone());+ }+ state::set(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS, &pulled_fingerprints).await?;+ ctx.log(format!("[V2Pull] Stored {} content fingerprint baseline(s) for push-side change detection.", pulled_fingerprints.len())).await;++ Ok(OpenedManifestSet {+ resolved,+ data_buckets,+ blob_map,+ contentless_manifest_ids: contentless_revisions.keys().cloned().collect(),+ personal_revision,+ manifest_revisions,+ bucket_revisions,+ })+}++/// Commit a snapshot's revision maps as the local believed-current revisions, manifests and buckets together.+pub(crate) async fn commit_revisions(ctx: &Ctx, manifest_revisions: &HashMap<String, i64>, bucket_revisions: &HashMap<String, i64>) -> SyncResult<()> {+ state::set(&ctx.host, state::SERVER_MANIFEST_REVISIONS, manifest_revisions).await?;+ state::set(&ctx.host, state::VAULT_BUCKET_REVISIONS, bucket_revisions).await?;+ ctx.log(format!("[V2Pull] Stored local manifest revisions from snapshot: {:?}; bucket revisions: {:?}.", manifest_revisions, bucket_revisions)).await;+ Ok(())+}++/// Open every data bucket a snapshot carries, record the snapshot's bucket revisions.+async fn open_data_buckets(ctx: &Ctx, snapshot: &GetResponse, resolved: &[ResolvedManifest]) -> SyncResult<(Vec<DataBucket>, HashMap<String, String>, HashMap<String, i64>)> {+ let key_by_manifest: HashMap<&str, &str> = resolved.iter().map(|m| (m.manifest_id.as_str(), m.vek.as_str())).collect();+ let mut buckets = Vec::new();+ let mut fingerprints = HashMap::new();+ let mut revisions = HashMap::new();+ for dto in &snapshot.buckets {+ let Some(blob) = dto.blob.as_deref().filter(|b| !b.is_empty()) else { continue };+ let key = key_by_manifest.get(dto.manifest_id.as_str()).ok_or_else(|| SyncError::Snapshot(format!("data bucket \"{}\" belongs to manifest {}, which this vault did not open, refusing to assemble", dto.category, dto.manifest_id)))?;+ let label = format!("\"{}\" bucket of manifest {}", dto.category, dto.manifest_id);+ let bucket_json = verify_decrypt_unpack(blob, key, dto.ciphertext_hash.as_deref(), &label)?;+ let bucket: DataBucket = serde_json::from_str(&bucket_json)?;+ if !ids_equal(&bucket.manifest_id, &dto.manifest_id) || bucket.category != dto.category {+ return Err(SyncError::Snapshot(format!("{} declares a different address (manifest {}, category \"{}\") inside its encrypted payload, refusing to assemble", label, bucket.manifest_id, bucket.category)));+ }+ let rows: usize = bucket.tables.values().map(Vec::len).sum();+ ctx.log(format!("[V2Pull] Data bucket {} opened: {} rows (revision {:?}).", label, rows, dto.revision)).await;+ fingerprints.insert(state::fingerprint_bucket_key(&dto.manifest_id, &dto.category), vault_codec::compute_content_fingerprint(&bucket_json));+ if let Some(revision) = dto.revision {+ revisions.insert(state::bucket_revision_key(&dto.manifest_id, &dto.category), revision);+ }+ buckets.push(bucket);+ }+ Ok((buckets, fingerprints, revisions))+}++/// Fetch every referenced blob not cached locally, decrypt it, and prune the cache to the referenced set.+async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], fallback_vek: &str) -> SyncResult<HashMap<String, Vec<u8>>> {+ let mut owners: HashMap<String, &ResolvedManifest> = HashMap::new();+ let mut refs: Vec<StoredBlobRef> = Vec::new();+ for entry in resolved {+ for reference in &entry.blob_references {+ if owners.contains_key(&reference.hash) {+ continue;+ }+ owners.insert(reference.hash.clone(), entry);+ refs.push(reference.clone());+ }+ }++ let mut cache: HashMap<String, String> = state::get(&ctx.host, state::VAULT_BLOB_CIPHER_CACHE).await?.unwrap_or_default();+ let missing: Vec<StoredBlobRef> = refs.iter().filter(|r| !cache.contains_key(&r.hash)).cloned().collect();+ ctx.log(format!("[V2Pull] Blob refs: {} referenced, {} cached locally, {} to download.", refs.len(), refs.len() - missing.len(), missing.len())).await;++ let batches = batch_by_transfer_cost(missing, |r| base64_chars(r.size_bytes));+ let batch_count = batches.len();+ for (index, chunk) in batches.into_iter().enumerate() {+ let blobs: Vec<BlobDto> = http::post(&ctx.host, BLOBS_DOWNLOAD_ENDPOINT, &BlobHashesRequest { hashes: chunk.iter().map(|r| r.hash.clone()).collect() }, true).await?;+ ctx.log(format!("[V2Pull] Downloaded blob batch {}/{}: requested {}, received {}.", index + 1, batch_count, chunk.len(), blobs.len())).await;+ for dto in blobs {+ cache.insert(dto.hash, dto.encrypted_data_base64);+ }+ }++ let mut pruned_cache: HashMap<String, String> = HashMap::new();+ let mut blob_map = HashMap::new();+ for reference in &refs {+ let Some(ciphertext) = cache.get(&reference.hash) else {+ ctx.warn(format!("[V2Sync] Referenced {} blob {} missing on server, continuing without it.", reference.category, reference.hash)).await;+ continue;+ };+ let key = owners.get(&reference.hash).map(|o| o.vek.as_str()).unwrap_or(fallback_vek);+ match crypto::aes_gcm::decode_base64(ciphertext).and_then(|bytes| crypto::symmetric_decrypt_bytes(&bytes, key)) {+ Ok(bytes) => {+ blob_map.insert(reference.hash.clone(), bytes);+ pruned_cache.insert(reference.hash.clone(), ciphertext.clone());+ }+ Err(_) => ctx.warn(format!("[V2Sync] Referenced {} blob {} failed to decrypt with the current key, continuing without it.", reference.category, reference.hash)).await,+ }+ }+ state::set(&ctx.host, state::VAULT_BLOB_CIPHER_CACHE, &pruned_cache).await?;+ state::set(&ctx.host, state::VAULT_SERVER_BLOB_HASHES, &refs.iter().map(|r| r.hash.clone()).collect::<Vec<_>>()).await?;+ Ok(blob_map)+}++/// Materialize manifests and buckets into a fresh database and return its bytes.+pub(crate) async fn materialize_to_sqlite(ctx: &mut Ctx, manifests: &[Manifest], data_buckets: &[DataBucket], blob_map: &HashMap<String, Vec<u8>>) -> SyncResult<Vec<u8>> {+ ctx.log(format!("[V2Pull] {} blobs decrypted; running codec reassembly into a fresh SQLite ({} manifest(s) combined)...", blob_map.len(), manifests.len())).await;+ let schema = ctx.schema().await?;+ let materialized = vault_codec::materialize_as_sqlite(MaterializeInput { manifests: manifests.to_vec(), data_buckets: data_buckets.to_vec(), schema_columns: schema.columns })?;++ let overflow_tables = materialized.overflow.tables.len() + materialized.overflow.bucket_tables.values().map(HashMap::len).sum::<usize>();+ if overflow_tables > 0 || !materialized.overflow.columns.is_empty() {+ ctx.warn(format!("[V2Pull] Newer-schema data preserved as overflow: {} unknown table(s), unknown columns on [{}].", overflow_tables, materialized.overflow.columns.keys().cloned().collect::<Vec<_>>().join(", "))).await;+ }++ // Use a fresh staging database for every materialize.+ db::open_staging(&ctx.host, None).await?;+ db::insert_materialized(&ctx.host, &materialized, blob_map).await?;+ let bytes = db::export(&ctx.host, Db::Staging).await?;+ ctx.log(format!("[V2Pull] Codec reassembly complete: {} bytes.", bytes.len())).await;+ Ok(bytes)+}++fn open_manifest(dto: &ManifestDto, vek: &str, is_personal: bool) -> SyncResult<ResolvedManifest> {+ let label = if is_personal { "manifest".to_string() } else { format!("shared manifest {}", dto.manifest_id) };+ let manifest_json = verify_decrypt_unpack(dto.blob.as_deref().unwrap_or(""), vek, dto.ciphertext_hash.as_deref(), &label)?;+ let manifest: Manifest = serde_json::from_str(&manifest_json)?;+ if !ids_equal(&manifest.manifest_id, &dto.manifest_id) {+ return Err(SyncError::Snapshot(format!("manifest {} declares a different id ({}) inside its encrypted payload, refusing to open it", dto.manifest_id, manifest.manifest_id)));+ }+ Ok(ResolvedManifest {+ manifest_id: dto.manifest_id.clone(),+ is_personal,+ manifest,+ vek: vek.to_string(),+ revision: dto.revision,+ blob_references: dto.blob_references.clone(),+ content_fingerprint: vault_codec::compute_content_fingerprint(&manifest_json),+ })+}++/// The key that opens one snapshot manifest.+async fn resolve_manifest_vek(ctx: &Ctx, dto: &ManifestDto, personal_manifest_id: &str, personal_vek: &str, is_personal: bool, personal_manifest: Option<&Manifest>) -> SyncResult<String> {+ let key_type = dto.key_type.clone().unwrap_or_else(|| if is_personal { types::KEY_TYPE_ACCOUNT_KEY.to_string() } else { types::KEY_TYPE_GRANT_KEY.to_string() });+ if key_type == types::KEY_TYPE_ACCOUNT_KEY {+ if dto.manifest_id != personal_manifest_id {+ return Err(SyncError::Snapshot(format!("manifest {} is unlocked by the account key hierarchy, but this session holds no key for it, refusing to assemble", dto.manifest_id)));+ }+ return Ok(personal_vek.to_string());+ }+ if key_type != types::KEY_TYPE_GRANT_KEY {+ return Err(SyncError::Snapshot(format!("manifest {} states an unknown key type \"{}\" (newer server?), refusing to assemble", dto.manifest_id, key_type)));+ }+ let personal_manifest = personal_manifest.ok_or_else(|| SyncError::Snapshot(format!("manifest {} is opened through a grant, but no personal manifest is open to resolve the private key from, refusing to assemble", dto.manifest_id)))?;+ resolve_granted_vek(ctx, personal_manifest, dto).await+}++async fn resolve_granted_vek(ctx: &Ctx, personal_manifest: &Manifest, dto: &ManifestDto) -> SyncResult<String> {+ let (encrypted_vek, public_key, algorithm) = grant_of(dto).ok_or_else(|| SyncError::Snapshot(format!("shared manifest {} carries no grant to open it with, refusing to assemble", dto.manifest_id)))?;+ if algorithm != types::ALGORITHM_RSA_OAEP_SHA256 {+ return Err(SyncError::Snapshot(format!("shared manifest {} grants its VEK under an unsupported algorithm \"{}\" (newer server?), refusing to assemble", dto.manifest_id, algorithm)));+ }+ let private_key = resolve_private_key_jwk(ctx, personal_manifest, &public_key).await?.ok_or_else(|| SyncError::Snapshot(format!("no private key in this vault opens the grant on shared manifest {}, refusing to assemble", dto.manifest_id)))?;+ keys::decrypt_manifest_vek(&encrypted_vek, &private_key).map_err(|e| SyncError::ServerVaultUnreadable(format!("failed to decrypt the VEK of shared manifest {}, refusing to assemble: {}", dto.manifest_id, e)))+}++/// The private key (JWK) matching a grant's public key: the session's, else one the personal manifest keeps.+async fn resolve_private_key_jwk(ctx: &Ctx, personal_manifest: &Manifest, encryption_public_key: &str) -> SyncResult<Option<String>> {+ if ctx.account_public_key.as_deref() == Some(encryption_public_key) {+ if let Some(private) = &ctx.account_private_key {+ return Ok(Some(private.clone()));+ }+ }+ Ok(vault_codec::extract_encryption_key_for_public_key(personal_manifest, encryption_public_key).and_then(|row| row.get("PrivateKey").and_then(serde_json::Value::as_str).map(str::to_string)))+}diff --git a/core/rust/src/vault_sync/push.rs b/core/rust/src/vault_sync/push.rsnew file mode 100644index 000000000..ef5b95640--- /dev/null+++ b/core/rust/src/vault_sync/push.rs@@ -0,0 +1,737 @@+//! Push: canonicalize the local vault, gate every manifest and bucket by its content fingerprint, encrypt what+//! changed under the key of the manifest that owns it, upload the blobs the server lacks, and `POST v2/Vault`.++use std::collections::{HashMap, HashSet};++use super::db::{id_key, ids_equal};+use super::email_routing::build_email_routing;+use super::errors::{SyncError, SyncResult};+use super::pull::{BLOB_TRANSFER_BATCH_MAX_CHARS, BLOB_TRANSFER_BATCH_MAX_COUNT};+use super::state::{self, Ctx};+use super::types::{BlobDto, BlobHashesRequest, BlobRef, BlobUploadRequest, BucketWrite, Db, ManifestWrite, MissingBlobsResponse, VaultWriteRequest, VaultWriteResponse};+use super::{db, http, keys};+use crate::crypto;+use crate::vault_codec::{self, BlobEntry, CanonicalizeInput, CanonicalizedVault, DataBucket, Manifest, ManifestSpec};+use crate::vault_sharing::{self, ManifestAccessRequest, ManifestWriteSetRequest, SharedManifestRecord as SharingRecord};++const VAULT_ENDPOINT: &str = "Vault";+const BLOBS_ENDPOINT: &str = "Vault/blobs";+const BLOBS_MISSING_ENDPOINT: &str = "Vault/blobs/missing";++/// Days an item stays in the trash before a push prunes it.+const TRASH_RETENTION_DAYS: u32 = 30;+/// The mutation scope that requires a full manifest push.+const MANIFEST_SCOPE: &str = "Main";++/// One manifest this vault can write, resolved from local state before canonicalizing.+#[derive(Debug, Clone)]+pub(crate) struct ManifestRecord {+ pub manifest_id: String,+ pub is_personal: bool,+ pub salt: String,+ /// The key this manifest encrypts with; None for the personal manifest, whose content key the push supplies.+ pub vek: Option<String>,+ pub name: Option<String>,+ pub can_administer: bool,+}++/// The canonicalized vault plus the records it was split against, personal manifest first.+#[derive(Clone)]+pub(crate) struct CanonicalizedSet {+ pub canonicalized: CanonicalizedVault,+ pub manifest_records: Vec<ManifestRecord>,+}++/// Outcome of a push that reached the server. Rejections travel as errors.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(crate) enum PushStatus {+ Ok,+ /// The server holds a newer revision; the caller pulls, merges and retries.+ Outdated,+}++/// What a completed upload reports to the flow.+pub(crate) struct UploadOutcome {+ pub status: PushStatus,+ pub mutation_seq_at_start: u64,+ /// Whether the stored vault changed under the upload (pruned or re-keyed).+ pub vault_changed: bool,+}++/*+ * The local write set.+ */++/// Canonicalize the local vault against every manifest this vault writes.+pub(crate) async fn canonicalize_vault(ctx: &Ctx, adopt_unstamped_into: Option<String>) -> SyncResult<CanonicalizedSet> {+ let tables = db::read_tables(&ctx.host, Db::Local).await?;+ let manifest_records = resolve_manifest_records(ctx).await?;+ let manifests: Vec<ManifestSpec> = manifest_records.iter().map(|r| ManifestSpec { manifest_id: r.manifest_id.clone(), manifest_salt: r.salt.clone(), name: r.name.clone() }).collect();+ let canonicalized = vault_codec::canonicalize_from_sqlite(CanonicalizeInput { tables, canonicalized_at: db::now_iso(), manifests, adopt_unstamped_into })?;+ Ok(CanonicalizedSet { canonicalized, manifest_records })+}++/// Every manifest this vault can write, personal manifest first. The core decides which.+async fn resolve_manifest_records(ctx: &Ctx) -> SyncResult<Vec<ManifestRecord>> {+ let manifest_salt = match state::get::<String>(&ctx.host, state::VAULT_MANIFEST_SALT).await? {+ Some(salt) => salt,+ None => {+ let salt = vault_codec::generate_manifest_salt();+ state::set(&ctx.host, state::VAULT_MANIFEST_SALT, &salt).await?;+ salt+ }+ };+ let shared_veks = keys::open_shared_manifest_veks(ctx).await?;+ let held = keys::shared_manifest_records(ctx).await?;+ let write_set = vault_sharing::resolve_manifest_write_set(ManifestWriteSetRequest {+ personal_manifest_id: resolve_personal_manifest_id(ctx).await?,+ personal_manifest_salt: manifest_salt,+ stamped_manifest_ids: db::manifest_ids_in_vault(&ctx.host, Db::Local).await?,+ opened_manifest_ids: shared_veks.keys().cloned().collect(),+ held_records: held.values().map(|r| SharingRecord { manifest_id: r.manifest_id.clone(), salt: r.salt.clone(), name: r.name.clone(), can_administer: r.can_administer }).collect(),+ display_names: db::manifest_display_names(&ctx.host).await?,+ });+ for skipped in &write_set.skipped {+ let why = match skipped.reason {+ vault_sharing::WriteSkipReason::NoRowsInVault => "has no rows in this vault; leaving it out of the write rather than emptying it server-side",+ vault_sharing::WriteSkipReason::KeyDidNotOpen => "did not open; leaving it out of the write",+ };+ ctx.log(format!("[V2Push] Shared manifest {} {}.", skipped.manifest_id, why)).await;+ }++ let vek_by_id: HashMap<String, String> = shared_veks.iter().map(|(id, vek)| (id_key(id), vek.clone())).collect();+ let mut records = Vec::new();+ for record in write_set.records {+ let vek = if record.is_personal { None } else { vek_by_id.get(&id_key(&record.manifest_id)).cloned() };+ if !record.is_personal && vek.is_none() {+ return Err(SyncError::Other(format!("manifest {} is in the write set without a key, refusing to write it", record.manifest_id)));+ }+ records.push(ManifestRecord { manifest_id: record.manifest_id, is_personal: record.is_personal, salt: record.salt, vek, name: record.name, can_administer: record.can_administer });+ }+ Ok(records)+}++/// The personal manifest id as the last pull recorded it. Pushing without one is impossible.+pub(crate) async fn resolve_personal_manifest_id(ctx: &Ctx) -> SyncResult<String> {+ state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await?.ok_or_else(|| SyncError::Other("no personal manifest id available (no snapshot baseline recorded); pull once before pushing".to_string()))+}++/// Whether the local vault is canonically identical to the last-known server state. Returns the canonicalize+/// result too, so a push right after does not canonicalize a second time.+pub(crate) async fn detect_no_op_mutation(ctx: &Ctx) -> SyncResult<(bool, CanonicalizedSet)> {+ let set = canonicalize_vault(ctx, None).await?;+ let fingerprints: HashMap<String, String> = state::get(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?.unwrap_or_default();+ let writable: HashSet<String> = set.manifest_records.iter().map(|r| id_key(&r.manifest_id)).collect();+ for entry in set.canonicalized.manifests.iter().filter(|m| writable.contains(&id_key(&m.manifest.manifest_id))) {+ let fingerprint = vault_codec::compute_content_fingerprint(&serde_json::to_string(&entry.manifest)?);+ if fingerprints.get(&state::fingerprint_manifest_key(&entry.manifest.manifest_id)) != Some(&fingerprint) {+ return Ok((false, set));+ }+ }+ for bucket in &set.canonicalized.data_buckets {+ let fingerprint = vault_codec::compute_content_fingerprint(&serde_json::to_string(bucket)?);+ if fingerprints.get(&state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category)) != Some(&fingerprint) {+ return Ok((false, set));+ }+ }+ Ok((true, set))+}++/// The manifests the local vault holds rows for that this session cannot write.+pub(crate) async fn find_unwritable_manifests(ctx: &Ctx) -> SyncResult<Vec<String>> {+ let mut writable: Vec<String> = keys::open_shared_manifest_veks(ctx).await?.keys().cloned().collect();+ if let Some(personal) = state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await? {+ writable.push(personal);+ }+ let partition = vault_sharing::partition_manifest_access(ManifestAccessRequest { manifest_ids_in_vault: db::manifest_ids_in_vault(&ctx.host, Db::Local).await?, writable_manifest_ids: writable, granted_manifest_ids: Vec::new() });+ Ok(partition.unwritable)+}++/// Whether the vault holds rows this session cannot write, logging why; a failed check reads as "no".+pub(crate) async fn vault_holds_unwritable_manifests(ctx: &Ctx) -> bool {+ match find_unwritable_manifests(ctx).await {+ Ok(unwritable) if unwritable.is_empty() => false,+ Ok(unwritable) => {+ ctx.warn(format!("[VaultSync] Vault holds rows for manifest(s) this session cannot write ({}); pulling before the push.", unwritable.join(", "))).await;+ true+ }+ Err(error) => {+ ctx.warn(format!("[VaultSync] Could not check which manifests this session can write; leaving the pull decision to the revisions. {}", error)).await;+ false+ }+ }+}++/// The key each manifest's data buckets are encrypted with.+async fn resolve_bucket_write_keys(ctx: &Ctx, personal_vek: &str) -> SyncResult<HashMap<String, String>> {+ let mut keys_by_manifest = HashMap::new();+ keys_by_manifest.insert(resolve_personal_manifest_id(ctx).await?, personal_vek.to_string());+ keys_by_manifest.extend(keys::open_shared_manifest_veks(ctx).await?);+ Ok(keys_by_manifest)+}++/*+ * The upload as the flow drives it.+ */++/// Upload the stored vault: bucket-only when every pending mutation is bucket-scoped, full otherwise. Fails+/// with `KeyOutOfSync` when the session key matches neither the server's KEK nor its VEK.+pub(crate) async fn upload_vault(ctx: &mut Ctx, cache: Option<(u64, CanonicalizedSet)>, force_full_write: bool, create_vault_key: bool) -> SyncResult<UploadOutcome> {+ let mutation_seq_at_start = ctx.mutation_sequence;+ if !keys::adopt_remote_vault_key_if_needed(ctx).await? {+ return Err(SyncError::KeyOutOfSync);+ }++ let create_vault_key = create_vault_key || !keys::has_local_vault_key(&ctx.host).await?;+ let scopes = ctx.request.dirty_scopes.clone();+ let bucket_only = !force_full_write && !create_vault_key && !scopes.is_empty() && !scopes.iter().any(|s| s == MANIFEST_SCOPE);++ let (status, vault_changed) = if bucket_only {+ (upload_dirty_buckets_only(ctx, &scopes).await?, false)+ } else {+ let cached = cache.filter(|(seq, _)| *seq == ctx.mutation_sequence).map(|(_, set)| set);+ upload_new_vault_to_server(ctx, cached, force_full_write, create_vault_key).await?+ };+ Ok(UploadOutcome { status, mutation_seq_at_start, vault_changed })+}++/// Push only the data buckets named by the dirty scopes, no manifest upload.+async fn upload_dirty_buckets_only(ctx: &mut Ctx, scopes: &[String]) -> SyncResult<PushStatus> {+ let key = ctx.encryption_key()?;+ if state::get::<String>(&ctx.host, state::VAULT_PERSONAL_MANIFEST_ID).await?.is_none() {+ ctx.warn("[V2Push] No manifest to address the bucket write to, falling back to a full vault upload.").await;+ return Ok(upload_new_vault_to_server(ctx, None, false, false).await?.0);+ }+ if vault_holds_unwritable_manifests(ctx).await {+ return Ok(PushStatus::Outdated);+ }++ let layout = vault_codec::bucket_layout();+ let write_keys = resolve_bucket_write_keys(ctx, &key).await?;+ let mut seen = HashSet::new();+ for category in scopes.iter().filter(|s| seen.insert((*s).clone())) {+ let Some(spec) = layout.iter().find(|entry| &entry.category == category) else {+ ctx.warn(format!("[V2Push] Unknown bucket scope \"{}\", falling back to full vault upload.", category)).await;+ return Ok(upload_new_vault_to_server(ctx, None, false, false).await?.0);+ };+ let tables = read_bucket_tables(ctx, &spec.tables).await?;+ let buckets = vault_codec::extract_buckets(category.clone(), write_keys.keys().cloned().collect(), tables)?;+ for bucket in buckets {+ let Some(vek) = write_keys.get(&bucket.manifest_id) else { continue };+ let (status, revision) = push_data_bucket_only(ctx, &bucket, vek).await?;+ if status != PushStatus::Ok {+ return Ok(PushStatus::Outdated);+ }+ ctx.log(format!("[V2Push] Bucket-only push for \"{}\" of manifest {} done (bucket revision {}).", category, bucket.manifest_id, revision)).await;+ }+ }+ Ok(PushStatus::Ok)+}++/// Upload a new version of the vault, pruning expired trash items first. Returns the status and whether the+/// stored vault changed (pruned or re-keyed).+async fn upload_new_vault_to_server(ctx: &mut Ctx, cached: Option<CanonicalizedSet>, force_full_write: bool, create_vault_key: bool) -> SyncResult<(PushStatus, bool)> {+ let key_before = ctx.encryption_key()?;+ let mut cached = cached;+ let mut vault_pruned = false;+ match db::prune_in_place(&ctx.host, TRASH_RETENTION_DAYS).await {+ Ok(0) => {}+ Ok(count) => {+ ctx.log(format!("[VaultMerge] Pruned expired items from trash ({} SQL statements executed)", count)).await;+ vault_pruned = true;+ cached = None;+ }+ Err(error) => ctx.warn(format!("[VaultSync] Failed to prune vault, continuing with upload: {}", error)).await,+ }++ let (status, new_key) = push(ctx, cached, create_vault_key, force_full_write).await?;+ if let Some(new_key) = new_key {+ // Migration succeeded: from now on the session key is the VEK, not the password-derived key.+ keys::re_encrypt_shared_manifest_records(ctx, &new_key).await?;+ ctx.set_encryption_key(new_key);+ }++ // Re-encrypt and persist locally only when the stored blob went stale or the encryption key changed.+ let key_after = ctx.encryption_key()?;+ if vault_pruned || key_after != key_before {+ let bytes = db::export(&ctx.host, Db::Local).await?;+ let new_key = if key_after != key_before { Some(key_after.clone()) } else { None };+ state::store_vault_with_key(&ctx.host, &state::encrypt_vault_blob(&bytes, &key_after)?, false, None, None, new_key).await?;+ ctx.vault_changed = true;+ }+ Ok((status, vault_pruned))+}++/// The tables of one bucket category read from the local vault, as extract_buckets takes them.+async fn read_bucket_tables(ctx: &Ctx, category_tables: &[String]) -> SyncResult<HashMap<String, Vec<vault_codec::CodecRecord>>> {+ let mut names: Vec<String> = category_tables.to_vec();+ names.push(vault_codec::OVERFLOW_TABLE.to_string());+ db::read_named_tables(&ctx.host, Db::Local, &names).await+}++/*+ * The full write.+ */++/// The local baselines a write gates against and, on success, advances.+struct PushBaselines {+ fingerprints: HashMap<String, String>,+ manifest_revisions: HashMap<String, i64>,+ bucket_revisions: HashMap<String, i64>,+ known_server_hashes: HashSet<String>,+}++impl PushBaselines {+ async fn load(ctx: &Ctx) -> SyncResult<Self> {+ Ok(Self {+ fingerprints: state::get(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?.unwrap_or_default(),+ manifest_revisions: state::get(&ctx.host, state::SERVER_MANIFEST_REVISIONS).await?.unwrap_or_default(),+ bucket_revisions: state::get(&ctx.host, state::VAULT_BUCKET_REVISIONS).await?.unwrap_or_default(),+ known_server_hashes: state::get::<Vec<String>>(&ctx.host, state::VAULT_SERVER_BLOB_HASHES).await?.unwrap_or_default().into_iter().collect(),+ })+ }+}++/// What decides whether an unchanged element still goes into the write.+#[derive(Clone, Copy)]+struct WriteGate {+ force_full_write: bool,+ /// A KEK/VEK migration push: the personal manifest and its blobs are re-encrypted whatever their content.+ migrating: bool,+}++/// One manifest of the write: its record, canonical content and the key it encrypts under.+struct Candidate<'a> {+ record: &'a ManifestRecord,+ manifest: &'a Manifest,+ vek: String,+ blobs: &'a HashMap<String, BlobEntry>,+ current_revision: i64,+}++impl Candidate<'_> {+ fn label(&self) -> String {+ if self.record.is_personal {+ "Personal manifest".to_string()+ } else {+ format!("Shared manifest \"{}\"", self.manifest.name.clone().unwrap_or_else(|| self.record.manifest_id.clone()))+ }+ }+}++/// A plaintext blob staged for upload.+struct UploadBlobEntry {+ bytes: Vec<u8>,+ kind: String,+ vek: String,+ from_personal: bool,+}++/// The plaintext blobs the candidates carry, in first-seen order.+struct UploadBlobs {+ entries: HashMap<String, UploadBlobEntry>,+ order: Vec<String>,+}++impl UploadBlobs {+ fn hashes(&self, from_personal: bool) -> Vec<String> {+ self.order.iter().filter(|h| self.entries[*h].from_personal == from_personal).cloned().collect()+ }+}++/// The freshly generated key hierarchy for one migration push.+#[derive(zeroize::Zeroize, zeroize::ZeroizeOnDrop)]+struct LegacyAccountKeyMigration {+ content_key: String,++ /// Ciphertext only, so there is nothing here to wipe.+ #[zeroize(skip)]+ account_keys: crypto::AccountKeyBlobs,+ account_private_key: String,+}++/// The change fingerprints of everything one write carried, to become the new baselines on success.+#[derive(Default)]+struct WrittenFingerprints {+ manifests: HashMap<String, String>,+ buckets: HashMap<String, String>,+}++/// A byte or character count for log lines.+fn format_kb(chars: usize) -> String {+ if chars < 1024 {+ format!("{} B", chars)+ } else {+ format!("{:.1} KB", chars as f64 / 1024.0)+ }+}++/// Pack-then-encrypt a JSON payload. Returns the base64 ciphertext plus the packed size, for logging.+fn pack_encrypt(payload_json: &str, vek: &str) -> SyncResult<(String, usize)> {+ let packed = vault_codec::pack_payload(payload_json)?;+ Ok((crypto::symmetric_encrypt_bytes(&packed, vek)?, packed.len()))+}++/// Canonicalize, gate by content fingerprint, encrypt and `POST v2/Vault`. Returns the outcome and, on a KEK/VEK+/// migration push, the new content key the session adopts.+pub(crate) async fn push(ctx: &mut Ctx, cached: Option<CanonicalizedSet>, create_vault_key: bool, force_full_write: bool) -> SyncResult<(PushStatus, Option<String>)> {+ http::with_outdated_server_guard(push_internal(ctx, cached, create_vault_key, force_full_write).await)+}++async fn push_internal(ctx: &mut Ctx, cached: Option<CanonicalizedSet>, create_vault_key: bool, force_full_write: bool) -> SyncResult<(PushStatus, Option<String>)> {+ let vek = ctx.encryption_key()?;++ let unwritable = find_unwritable_manifests(ctx).await?;+ if !unwritable.is_empty() {+ ctx.warn(format!("[V2Push] Vault holds rows for manifest(s) this session cannot write ({}); refusing the write until a pull restores them.", unwritable.join(", "))).await;+ return Ok((PushStatus::Outdated, None));+ }++ let migration = start_account_key_migration(ctx, &vek, create_vault_key).await?;+ let content_key = migration.as_ref().map(|m| m.content_key.clone()).unwrap_or_else(|| vek.clone());+ let gate = WriteGate { force_full_write, migrating: migration.is_some() };++ let set = match cached {+ Some(set) => {+ ctx.log("[V2Push] Reusing the canonicalize result already produced for this vault.").await;+ set+ }+ None => canonicalize_vault(ctx, None).await?,+ };+ let CanonicalizedSet { canonicalized, manifest_records } = set;+ if manifest_records.is_empty() {+ return Err(SyncError::Other("no manifest records to push".to_string()));+ }+ ctx.log(format!("[V2Push] Canonicalize produced {} manifest(s) + {} data bucket(s).", canonicalized.manifests.len(), canonicalized.data_buckets.len())).await;++ let baselines = PushBaselines::load(ctx).await?;+ let candidates = collect_candidates(&canonicalized, &manifest_records, &content_key, &baselines);+ let blobs = collect_upload_blobs(&candidates)?;+ let mut written = WrittenFingerprints::default();+ let bucket_writes = encrypt_changed_buckets(ctx, &canonicalized.data_buckets, &candidates, &baselines, gate, &mut written).await?;+ let manifest_writes = encrypt_changed_manifests(ctx, &candidates, &baselines, gate, &mut written).await?;++ if manifest_writes.is_empty() && bucket_writes.is_empty() {+ ctx.log("[V2Push] No content changes detected (every manifest and data bucket matches the server baselines); skipping upload.").await;+ return Ok((PushStatus::Ok, None));+ }++ let mut uploaded = upload_missing_blobs(ctx, &blobs, &baselines, gate).await?;+ let email_routing = build_email_routing(&canonicalized.manifests.iter().map(|m| m.manifest.clone()).collect::<Vec<_>>(), &ctx.request.private_email_domains);+ let payload = VaultWriteRequest { username: ctx.request.username.clone(), manifests: manifest_writes, buckets: bucket_writes, new_blobs: Vec::new(), email_routing: Some(email_routing), account_keys: migration.as_ref().map(|m| m.account_keys.clone()) };+ let response = write_vault(ctx, &payload, &blobs, gate, &mut uploaded).await?;++ if response.status != 0 {+ // All-or-nothing: a single stale manifest or bucket rejected the whole write; the caller pulls, merges and retries.+ return Ok((PushStatus::Outdated, None));+ }++ commit_push_baselines(ctx, baselines, &response, written, &blobs, &uploaded).await?;+ if let Some(migration) = &migration {+ complete_account_key_migration(ctx, migration).await?;+ ctx.log("[V2Push] Account-key migration complete: hierarchy created server-side, blob chain cached locally.").await;+ }+ Ok((PushStatus::Ok, migration.map(|m| m.content_key.clone())))+}++/// Generate the account key hierarchy for a migration push, when one is due.+async fn start_account_key_migration(ctx: &Ctx, kek: &str, create_vault_key: bool) -> SyncResult<Option<LegacyAccountKeyMigration>> {+ if !create_vault_key {+ return Ok(None);+ }+ let hierarchy = crypto::create_account_key_hierarchy(kek)?;+ ctx.log("[V2Push] Account-key migration: generated new VEK, AK and account keypair; vault content and all blobs will be re-encrypted and re-uploaded.").await;+ Ok(Some(LegacyAccountKeyMigration {+ content_key: hierarchy.vault_encryption_key.clone(),+ account_keys: hierarchy.account_keys.clone(),+ account_private_key: hierarchy.account_private_key.clone(),+ }))+}++/// Every canonicalized manifest a record exists for, with the key it encrypts under.+fn collect_candidates<'a>(canonicalized: &'a CanonicalizedVault, records: &'a [ManifestRecord], content_key: &str, baselines: &PushBaselines) -> Vec<Candidate<'a>> {+ let record_by_id: HashMap<&str, &ManifestRecord> = records.iter().map(|r| (r.manifest_id.as_str(), r)).collect();+ canonicalized+ .manifests+ .iter()+ .filter_map(|entry| {+ let record = record_by_id.get(entry.manifest.manifest_id.as_str())?;+ Some(Candidate { record, manifest: &entry.manifest, vek: record.vek.clone().unwrap_or_else(|| content_key.to_string()), blobs: &entry.blobs, current_revision: baselines.manifest_revisions.get(&record.manifest_id).copied().unwrap_or(0) })+ })+ .collect()+}++/// The plaintext blobs the candidates carry; a blob two manifests share is staged once, under the first owner's key.+fn collect_upload_blobs(candidates: &[Candidate]) -> SyncResult<UploadBlobs> {+ let mut blobs = UploadBlobs { entries: HashMap::new(), order: Vec::new() };+ for candidate in candidates {+ for (hash, blob) in candidate.blobs {+ if blobs.entries.contains_key(hash) {+ continue;+ }+ blobs.order.push(hash.clone());+ blobs.entries.insert(hash.clone(), UploadBlobEntry { bytes: crypto::aes_gcm::decode_base64(&blob.bytes_base64)?, kind: blob.kind.clone(), vek: candidate.vek.clone(), from_personal: candidate.record.is_personal });+ }+ }+ Ok(blobs)+}++/// Gate, validate, pack and encrypt each changed bucket under the key of the manifest that owns it.+async fn encrypt_changed_buckets(ctx: &Ctx, buckets: &[DataBucket], candidates: &[Candidate<'_>], baselines: &PushBaselines, gate: WriteGate, written: &mut WrittenFingerprints) -> SyncResult<Vec<BucketWrite>> {+ let key_by_manifest: HashMap<&str, &str> = candidates.iter().map(|c| (c.record.manifest_id.as_str(), c.vek.as_str())).collect();+ let mut writes = Vec::new();+ for bucket in buckets {+ let label = format!("Data bucket \"{}\" of manifest {}", bucket.category, bucket.manifest_id);+ let fingerprint_key = state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category);+ let plaintext = serde_json::to_string(bucket)?;+ let fingerprint = vault_codec::compute_content_fingerprint(&plaintext);+ if !gate.force_full_write && !gate.migrating && baselines.fingerprints.get(&fingerprint_key) == Some(&fingerprint) {+ ctx.log(format!("[V2Push] {} unchanged versus server baseline, leaving it out of this write.", label)).await;+ continue;+ }+ let Some(bucket_key) = key_by_manifest.get(bucket.manifest_id.as_str()) else {+ ctx.warn(format!("[V2Push] {} names a manifest this vault cannot write; leaving it out of this write.", label)).await;+ continue;+ };+ let validation = vault_codec::validate_data_bucket(bucket);+ if !validation.ok {+ return Err(SyncError::UploadRejected(vec![format!("{} validation failed: {}. {}", label, validation.failed_rules.join(", "), validation.message).trim().to_string()]));+ }+ let (ciphertext, compressed) = pack_encrypt(&plaintext, bucket_key)?;+ let ciphertext_hash = vault_codec::compute_ciphertext_hash(&ciphertext);+ let current_revision = baselines.bucket_revisions.get(&state::bucket_revision_key(&bucket.manifest_id, &bucket.category)).copied().unwrap_or(0);+ ctx.log(format!("[V2Push] {}: raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(compressed), format_kb(ciphertext.len()))).await;+ writes.push(BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: ciphertext, ciphertext_hash, current_revision });+ written.buckets.insert(fingerprint_key, fingerprint);+ }+ Ok(writes)+}++/// Gate, validate, pack and encrypt every changed candidate into the write batch, each with its own key.+async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], baselines: &PushBaselines, gate: WriteGate, written: &mut WrittenFingerprints) -> SyncResult<Vec<ManifestWrite>> {+ let mut writes = Vec::new();+ for candidate in candidates {+ let label = candidate.label();+ let plaintext = serde_json::to_string(candidate.manifest)?;+ let fingerprint = vault_codec::compute_content_fingerprint(&plaintext);+ let rekeyed = gate.migrating && candidate.record.is_personal;+ if !gate.force_full_write && !rekeyed && baselines.fingerprints.get(&state::fingerprint_manifest_key(&candidate.record.manifest_id)) == Some(&fingerprint) {+ ctx.log(format!("[V2Push] {} unchanged versus server baseline, leaving it out of this write.", label)).await;+ continue;+ }+ let validation = vault_codec::validate_manifest(candidate.manifest);+ if !validation.ok {+ if candidate.record.is_personal {+ return Err(SyncError::UploadRejected(vec![format!("Manifest validation failed: {}. {}", validation.failed_rules.join(", "), validation.message).trim().to_string()]));+ }+ ctx.warn(format!("[V2Push] {} failed validation ({}), dropping it from this write.", label, validation.failed_rules.join(", "))).await;+ continue;+ }+ let (ciphertext, compressed) = pack_encrypt(&plaintext, &candidate.vek)?;+ let ciphertext_hash = vault_codec::compute_ciphertext_hash(&ciphertext);+ ctx.log(format!("[V2Push] {}: raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(compressed), format_kb(ciphertext.len()))).await;++ // Publish the public half of this manifest's mail delivery keypair; only admins may publish a shared one.+ let may_publish = candidate.record.is_personal || candidate.record.can_administer;+ let manifest_key = if may_publish { db::active_key_for_manifest(&ctx.host, &candidate.record.manifest_id).await? } else { None };+ if may_publish && manifest_key.is_none() && !candidate.record.is_personal {+ ctx.warn(format!("[V2Push] {} is missing its email keypair; its aliases stay personal until sharing is re-enabled.", label)).await;+ }++ let mut blob_refs: Vec<BlobRef> = candidate.blobs.iter().map(|(hash, blob)| BlobRef { hash: hash.clone(), category: blob.kind.clone() }).collect();+ blob_refs.sort_by(|a, b| a.hash.cmp(&b.hash));+ writes.push(ManifestWrite {+ manifest_id: candidate.record.manifest_id.clone(),+ manifest_blob: ciphertext,+ manifest_ciphertext_hash: ciphertext_hash,+ current_revision: candidate.current_revision,+ credentials_count: candidate.manifest.tables.get("Items").map(Vec::len).unwrap_or(0),+ blob_references: blob_refs,+ encryption_public_key: manifest_key.and_then(|row| row.get("PublicKey").and_then(serde_json::Value::as_str).map(str::to_string)),+ });+ written.manifests.insert(candidate.record.manifest_id.clone(), fingerprint);+ }+ Ok(writes)+}++/// Encrypt and upload only the blobs the server does not already have. On a migration push every personal+/// blob is re-encrypted under the new key and overwritten. Returns hash to ciphertext for the local cache.+async fn upload_missing_blobs(ctx: &Ctx, blobs: &UploadBlobs, baselines: &PushBaselines, gate: WriteGate) -> SyncResult<HashMap<String, String>> {+ let personal_hashes = blobs.hashes(true);+ let shared_hashes = blobs.hashes(false);+ let unknown_to_server = |hashes: &[String]| -> Vec<String> { hashes.iter().filter(|h| !baselines.known_server_hashes.contains(*h)).cloned().collect() };++ let (personal_to_upload, shared_to_upload) = if gate.migrating {+ (personal_hashes.clone(), missing_on_server(ctx, unknown_to_server(&shared_hashes)).await?)+ } else {+ let missing: HashSet<String> = missing_on_server(ctx, unknown_to_server(&blobs.order)).await?.into_iter().collect();+ (personal_hashes.iter().filter(|h| missing.contains(*h)).cloned().collect(), shared_hashes.iter().filter(|h| missing.contains(*h)).cloned().collect())+ };+ ctx.log(format!("[V2Push] Blob diff: {} blobs, uploading {} personal + {} shared{}.", blobs.order.len(), personal_to_upload.len(), shared_to_upload.len(), if gate.migrating { " (personal manifest re-encrypted, VEK migration)" } else { "" })).await;++ let mut uploaded = upload_blobs(ctx, &blobs.entries, &personal_to_upload, gate.migrating).await?;+ uploaded.extend(upload_blobs(ctx, &blobs.entries, &shared_to_upload, false).await?);+ Ok(uploaded)+}++/// Ask the server which of the given hashes it lacks.+async fn missing_on_server(ctx: &Ctx, hashes: Vec<String>) -> SyncResult<Vec<String>> {+ if hashes.is_empty() {+ return Ok(Vec::new());+ }+ Ok(http::post::<_, MissingBlobsResponse>(&ctx.host, BLOBS_MISSING_ENDPOINT, &BlobHashesRequest { hashes }, false).await?.missing)+}++/// `POST v2/Vault`. When the server reports blobs it lacks (stale local knowledge of its blob set), upload+/// them and retry the identical write once; blobs this client cannot supply fail the push.+async fn write_vault(ctx: &Ctx, payload: &VaultWriteRequest, blobs: &UploadBlobs, gate: WriteGate, uploaded: &mut HashMap<String, String>) -> SyncResult<VaultWriteResponse> {+ let response: VaultWriteResponse = http::post(&ctx.host, VAULT_ENDPOINT, payload, true).await?;+ if response.missing_blob_hashes.is_empty() {+ return Ok(response);+ }+ let unsatisfiable: Vec<String> = response.missing_blob_hashes.iter().filter(|h| !blobs.entries.contains_key(*h)).cloned().collect();+ if !unsatisfiable.is_empty() {+ return Err(SyncError::MissingBlobs(unsatisfiable));+ }+ ctx.warn(format!("[V2Sync] Server reported {} missing blob(s); uploading and retrying once.", response.missing_blob_hashes.len())).await;+ let (missing_personal, missing_shared): (Vec<String>, Vec<String>) = response.missing_blob_hashes.iter().cloned().partition(|h| blobs.entries[h].from_personal);+ uploaded.extend(upload_blobs(ctx, &blobs.entries, &missing_personal, gate.migrating).await?);+ uploaded.extend(upload_blobs(ctx, &blobs.entries, &missing_shared, false).await?);+ let response: VaultWriteResponse = http::post(&ctx.host, VAULT_ENDPOINT, payload, true).await?;+ if !response.missing_blob_hashes.is_empty() {+ return Err(SyncError::MissingBlobs(response.missing_blob_hashes));+ }+ Ok(response)+}++/// Advance the local baselines of everything a successful write carried, and refresh the encrypted blob cache.+async fn commit_push_baselines(ctx: &Ctx, baselines: PushBaselines, response: &VaultWriteResponse, written: WrittenFingerprints, blobs: &UploadBlobs, uploaded: &HashMap<String, String>) -> SyncResult<()> {+ let PushBaselines { mut fingerprints, mut manifest_revisions, mut bucket_revisions, .. } = baselines;+ if !response.bucket_revisions.is_empty() {+ for br in &response.bucket_revisions {+ bucket_revisions.insert(state::bucket_revision_key(&br.manifest_id, &br.category), br.revision);+ }+ state::set(&ctx.host, state::VAULT_BUCKET_REVISIONS, &bucket_revisions).await?;+ }+ for mr in &response.manifest_revisions {+ manifest_revisions.insert(mr.manifest_id.clone(), mr.revision);+ }+ state::set(&ctx.host, state::SERVER_MANIFEST_REVISIONS, &manifest_revisions).await?;++ fingerprints.extend(written.buckets);+ for (manifest_id, fingerprint) in written.manifests {+ fingerprints.insert(state::fingerprint_manifest_key(&manifest_id), fingerprint);+ }+ state::set(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS, &fingerprints).await?;+ state::set(&ctx.host, state::VAULT_SERVER_BLOB_HASHES, &blobs.order).await?;++ // The encrypted blob cache: entries still referenced, plus the ciphertexts just uploaded.+ let cache: HashMap<String, String> = state::get(&ctx.host, state::VAULT_BLOB_CIPHER_CACHE).await?.unwrap_or_default();+ let mut new_cache: HashMap<String, String> = HashMap::new();+ for hash in &blobs.order {+ if let Some(ciphertext) = uploaded.get(hash).or_else(|| cache.get(hash)) {+ new_cache.insert(hash.clone(), ciphertext.clone());+ }+ }+ state::set(&ctx.host, state::VAULT_BLOB_CIPHER_CACHE, &new_cache).await+}++/// Adopt the hierarchy a migration push just committed: cache the encrypted chain and stage the private key.+async fn complete_account_key_migration(ctx: &mut Ctx, migration: &LegacyAccountKeyMigration) -> SyncResult<()> {+ let blobs = &migration.account_keys;+ state::set(&ctx.host, state::ENCRYPTED_ACCOUNT_KEY, &blobs.encrypted_account_key).await?;+ state::set(&ctx.host, state::ENCRYPTED_VEK, &blobs.encrypted_vek).await?;+ state::set(&ctx.host, state::ACCOUNT_PUBLIC_KEY, &blobs.account_public_key).await?;+ state::set(&ctx.host, state::ENCRYPTED_ACCOUNT_PRIVATE_KEY, &blobs.encrypted_account_private_key).await?;+ ctx.account_public_key = Some(blobs.account_public_key.clone());+ ctx.set_account_private_key(Some(migration.account_private_key.clone()));+ Ok(())+}++/// Encrypt the given blobs (each under its own key) and upload them in size-capped batches. Returns hash to+/// ciphertext for the local encrypted blob cache.+async fn upload_blobs(ctx: &Ctx, entries: &HashMap<String, UploadBlobEntry>, hashes: &[String], overwrite: bool) -> SyncResult<HashMap<String, String>> {+ let mut ciphertexts = HashMap::new();+ if hashes.is_empty() {+ return Ok(ciphertexts);+ }+ let mut batch: Vec<BlobDto> = Vec::new();+ let mut batch_chars = 0usize;+ for hash in hashes {+ let Some(entry) = entries.get(hash) else { continue };+ let ciphertext = crypto::symmetric_encrypt_bytes(&entry.bytes, &entry.vek)?;+ ciphertexts.insert(hash.clone(), ciphertext.clone());+ if !batch.is_empty() && (batch_chars + ciphertext.len() > BLOB_TRANSFER_BATCH_MAX_CHARS || batch.len() >= BLOB_TRANSFER_BATCH_MAX_COUNT) {+ ctx.log(format!("[V2Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(batch_chars))).await;+ http::post_no_content(&ctx.host, BLOBS_ENDPOINT, &BlobUploadRequest { blobs: std::mem::take(&mut batch), overwrite }).await?;+ batch_chars = 0;+ }+ batch_chars += ciphertext.len();+ batch.push(BlobDto { hash: hash.clone(), category: entry.kind.clone(), encrypted_data_base64: ciphertext });+ }+ if !batch.is_empty() {+ ctx.log(format!("[V2Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(batch_chars))).await;+ http::post_no_content(&ctx.host, BLOBS_ENDPOINT, &BlobUploadRequest { blobs: batch, overwrite }).await?;+ }+ Ok(ciphertexts)+}++/*+ * The bucket-only write.+ */++/// Single-data-bucket upload through the unified write, rebasing onto the server's revision and retrying once.+async fn push_data_bucket_only(ctx: &Ctx, bucket: &DataBucket, vek: &str) -> SyncResult<(PushStatus, i64)> {+ http::with_outdated_server_guard(push_data_bucket_only_internal(ctx, bucket, vek).await)+}++async fn push_data_bucket_only_internal(ctx: &Ctx, bucket: &DataBucket, vek: &str) -> SyncResult<(PushStatus, i64)> {+ let label = format!("Bucket \"{}\" of manifest {}", bucket.category, bucket.manifest_id);+ let revision_key = state::bucket_revision_key(&bucket.manifest_id, &bucket.category);+ let plaintext = serde_json::to_string(bucket)?;++ let mut fingerprints: HashMap<String, String> = state::get(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS).await?.unwrap_or_default();+ let mut bucket_revisions: HashMap<String, i64> = state::get(&ctx.host, state::VAULT_BUCKET_REVISIONS).await?.unwrap_or_default();+ let fingerprint = vault_codec::compute_content_fingerprint(&plaintext);+ let fingerprint_key = state::fingerprint_bucket_key(&bucket.manifest_id, &bucket.category);+ if fingerprints.get(&fingerprint_key) == Some(&fingerprint) {+ ctx.log(format!("[V2Push] {} (bucket-only) unchanged versus server baseline, skipping upload.", label)).await;+ return Ok((PushStatus::Ok, bucket_revisions.get(&revision_key).copied().unwrap_or(0)));+ }++ let (ciphertext, compressed) = pack_encrypt(&plaintext, vek)?;+ let ciphertext_hash = vault_codec::compute_ciphertext_hash(&ciphertext);+ ctx.log(format!("[V2Push] {} (bucket-only): raw {} > compressed {} > encrypted {}.", label, format_kb(plaintext.len()), format_kb(compressed), format_kb(ciphertext.len()))).await;++ let post = |current_revision: i64| {+ let payload = VaultWriteRequest {+ username: ctx.request.username.clone(),+ manifests: Vec::new(),+ buckets: vec![BucketWrite { manifest_id: bucket.manifest_id.clone(), category: bucket.category.clone(), blob: ciphertext.clone(), ciphertext_hash: ciphertext_hash.clone(), current_revision }],+ new_blobs: Vec::new(),+ email_routing: None,+ account_keys: None,+ };+ async move { http::post::<_, VaultWriteResponse>(&ctx.host, VAULT_ENDPOINT, &payload, true).await }+ };+ let reported = |response: &VaultWriteResponse| response.bucket_revisions.iter().find(|b| ids_equal(&b.manifest_id, &bucket.manifest_id) && b.category == bucket.category).map(|b| b.revision);++ let mut current_revision = bucket_revisions.get(&revision_key).copied().unwrap_or(0);+ let mut response = post(current_revision).await?;+ if response.status != 0 {+ let server_revision = reported(&response).unwrap_or(current_revision);+ ctx.warn(format!("[V2Push] {} outdated (server at revision {}, we assumed {}); rebasing and retrying once.", label, server_revision, current_revision)).await;+ current_revision = server_revision;+ response = post(current_revision).await?;+ }+ if response.status != 0 {+ return Ok((PushStatus::Outdated, reported(&response).unwrap_or(current_revision)));+ }++ let new_revision = reported(&response).unwrap_or(current_revision + 1);+ bucket_revisions.insert(revision_key, new_revision);+ state::set(&ctx.host, state::VAULT_BUCKET_REVISIONS, &bucket_revisions).await?;+ fingerprints.insert(fingerprint_key, fingerprint);+ state::set(&ctx.host, state::VAULT_CONTENT_FINGERPRINTS, &fingerprints).await?;+ Ok((PushStatus::Ok, new_revision))+}diff --git a/core/rust/src/vault_sync/session.rs b/core/rust/src/vault_sync/session.rsnew file mode 100644index 000000000..0fbd16279--- /dev/null+++ b/core/rust/src/vault_sync/session.rs@@ -0,0 +1,131 @@+//! The command loop.++use std::future::Future;+use std::pin::Pin;+use std::sync::{Arc, Mutex};+use std::task::{Context, Poll, Waker};++use serde::de::DeserializeOwned;+use serde_json::Value;++use super::engine;+use super::errors::{SyncError, SyncResult};+use super::types::{Ack, Command, LogLevel, SyncRequest};+use crate::error::{VaultError, VaultResult};++/// The exchange point between the engine's future and the host.+#[derive(Default)]+pub(crate) struct Slot {+ pub pending: Option<Command>,+ pub response: Option<Value>,+}++/// The engine's handle to the host.+#[derive(Clone)]+pub(crate) struct Host {+ slot: Arc<Mutex<Slot>>,+}++impl Host {+ pub fn new(slot: Arc<Mutex<Slot>>) -> Self {+ Self { slot }+ }++ /// Send a command and wait for the host's typed response. A `{ "error": ... }` response becomes an error.+ pub async fn call<R: DeserializeOwned>(&self, command: Command) -> SyncResult<R> {+ let kind = command.kind();+ let response = CommandFuture { slot: self.slot.clone(), command: Some(command), sent: false }.await;+ if let Some(error) = response.get("error").and_then(Value::as_str) {+ return Err(SyncError::Host { command: kind, message: error.to_string() });+ }+ serde_json::from_value(response).map_err(|e| SyncError::Host { command: kind, message: format!("unexpected response shape: {}", e) })+ }++ /// Emit a log line.+ pub async fn log(&self, level: LogLevel, message: impl Into<String>) {+ let _ = self.call::<Ack>(Command::Log { level, message: message.into() }).await;+ }+}++/// Parks the engine until the host has fed the response to one command.+struct CommandFuture {+ slot: Arc<Mutex<Slot>>,+ command: Option<Command>,+ sent: bool,+}++impl Future for CommandFuture {+ type Output = Value;++ fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Value> {+ let this = &mut *self;+ if !this.sent {+ this.slot.lock().expect("sync host slot poisoned").pending = this.command.take();+ this.sent = true;+ return Poll::Pending;+ }+ match this.slot.lock().expect("sync host slot poisoned").response.take() {+ Some(response) => Poll::Ready(response),+ None => Poll::Pending,+ }+ }+}++/// A running engine operation.+pub struct SyncSession {+ inner: Mutex<SessionInner>,+}++struct SessionInner {+ slot: Arc<Mutex<Slot>>,+ future: Option<Pin<Box<dyn Future<Output = Value> + Send>>>,+ finished: Option<Value>,+}++impl SyncSession {+ /// Start an operation.+ pub fn new(request_json: &str) -> VaultResult<Self> {+ let request: SyncRequest = serde_json::from_str(request_json).map_err(|e| VaultError::General(format!("Invalid sync request: {}", e)))?;+ let slot = Arc::new(Mutex::new(Slot::default()));+ let host = Host::new(slot.clone());+ let future: Pin<Box<dyn Future<Output = Value> + Send>> = Box::pin(engine::run(host, request));+ Ok(Self { inner: Mutex::new(SessionInner { slot, future: Some(future), finished: None }) })+ }++ /// The next command for the host, as JSON.+ pub fn next_command(&self) -> VaultResult<String> {+ let mut inner = self.inner.lock().map_err(|_| VaultError::General("sync session poisoned".to_string()))?;+ if let Some(result) = &inner.finished {+ return Ok(serde_json::to_string(&Command::Done { result: result.clone() })?);+ }++ let mut context = Context::from_waker(Waker::noop());+ let future = inner.future.as_mut().ok_or_else(|| VaultError::General("sync session has no operation".to_string()))?;+ match future.as_mut().poll(&mut context) {+ Poll::Ready(result) => {+ inner.future = None;+ inner.finished = Some(result.clone());+ Ok(serde_json::to_string(&Command::Done { result })?)+ }+ Poll::Pending => {+ let pending = inner.slot.lock().map_err(|_| VaultError::General("sync host slot poisoned".to_string()))?.pending.take();+ match pending {+ Some(command) => Ok(serde_json::to_string(&command)?),+ None => Err(VaultError::General("sync session is waiting for a response to its last command".to_string())),+ }+ }+ }+ }++ /// Hand the host's response to the last command back.+ pub fn resume(&self, response_json: &str) -> VaultResult<()> {+ let response: Value = if response_json.trim().is_empty() {+ Value::Object(Default::default())+ } else {+ serde_json::from_str(response_json).map_err(|e| VaultError::General(format!("Invalid host response: {}", e)))?+ };+ let inner = self.inner.lock().map_err(|_| VaultError::General("sync session poisoned".to_string()))?;+ inner.slot.lock().map_err(|_| VaultError::General("sync host slot poisoned".to_string()))?.response = Some(response);+ Ok(())+ }+}diff --git a/core/rust/src/vault_sync/state.rs b/core/rust/src/vault_sync/state.rsnew file mode 100644index 000000000..7ae649dac--- /dev/null+++ b/core/rust/src/vault_sync/state.rs@@ -0,0 +1,193 @@+//! Engine state: the run context.++use std::collections::HashMap;++use serde::de::DeserializeOwned;+use serde::Serialize;+use serde_json::Value;++use super::db;+use super::errors::{SyncError, SyncResult};+use super::session::Host;+use super::types::{Ack, CleanOutcome, Command, Db, LogLevel, SessionUpdates, StateValue, StoreOutcome, StoredVault, SyncRequest};+use crate::crypto;++/// The mutable state of one engine run.+pub(crate) struct Ctx {+ pub host: Host,+ pub request: SyncRequest,+ pub encryption_key: Option<String>,+ pub account_public_key: Option<String>,+ pub account_private_key: Option<String>,+ pub is_dirty: bool,+ pub mutation_sequence: u64,+ pub updates: SessionUpdates,+ pub vault_changed: bool,+ pub schema: Option<SchemaInfo>,+ has_local_vault: bool,+}++/// What a fresh staging database tells about the current client schema.+#[derive(Debug, Clone)]+pub(crate) struct SchemaInfo {+ pub columns: HashMap<String, Vec<String>>,+ pub migration_id: String,+}++impl Ctx {+ pub fn new(host: Host, request: SyncRequest) -> Self {+ Self {+ encryption_key: request.encryption_key.clone(),+ account_public_key: request.account_public_key.clone(),+ account_private_key: request.account_private_key.clone(),+ is_dirty: request.is_dirty,+ mutation_sequence: request.mutation_sequence,+ updates: SessionUpdates::default(),+ vault_changed: false,+ schema: None,+ has_local_vault: false,+ host,+ request,+ }+ }++ /// Whether the host holds an at-rest vault.+ pub async fn has_local_vault(&mut self) -> SyncResult<bool> {+ if !self.has_local_vault {+ self.has_local_vault = load_vault(&self.host).await?.is_some();+ }+ Ok(self.has_local_vault)+ }++ /// The session's vault encryption key, or the locked-vault error.+ pub fn encryption_key(&self) -> SyncResult<String> {+ self.encryption_key.clone().ok_or(SyncError::VaultLocked)+ }++ /// Adopt a new session encryption key, reporting it to the host.+ pub fn set_encryption_key(&mut self, key: String) {+ self.updates.encryption_key = Some(key.clone());+ self.encryption_key = Some(key);+ }++ pub fn set_account_private_key(&mut self, key: Option<String>) {+ match &key {+ Some(value) => self.updates.account_private_key = Some(value.clone()),+ None => self.updates.clear_account_private_key = true,+ }+ self.account_private_key = key;+ }++ /// The current schema.+ pub async fn schema(&mut self) -> SyncResult<SchemaInfo> {+ if let Some(schema) = &self.schema {+ return Ok(schema.clone());+ }+ db::open_staging(&self.host, None).await?;+ let schema = SchemaInfo { columns: db::schema_columns(&self.host, Db::Staging).await?, migration_id: db::latest_migration_id(&self.host, Db::Staging).await? };+ self.schema = Some(schema.clone());+ Ok(schema)+ }++ pub async fn log(&self, message: impl Into<String>) {+ self.host.log(LogLevel::Log, message).await;+ }++ pub async fn warn(&self, message: impl Into<String>) {+ self.host.log(LogLevel::Warn, message).await;+ }+}++/*+ * Persisted state.+ */++pub const SERVER_MANIFEST_REVISIONS: &str = "serverManifestRevisions";+pub const VAULT_BUCKET_REVISIONS: &str = "vaultBucketRevisions";+pub const VAULT_MANIFEST_SALT: &str = "vaultManifestSalt";+pub const VAULT_PERSONAL_MANIFEST_ID: &str = "vaultPersonalManifestId";+pub const VAULT_CONTENT_FINGERPRINTS: &str = "vaultContentFingerprints";+pub const VAULT_BLOB_CIPHER_CACHE: &str = "vaultBlobCipherCache";+pub const VAULT_SERVER_BLOB_HASHES: &str = "vaultServerBlobHashes";+pub const SHARED_MANIFESTS: &str = "sharedManifests";+pub const ENCRYPTED_VEK: &str = "encryptedVek";+pub const ENCRYPTED_ACCOUNT_KEY: &str = "encryptedAccountKey";+pub const ACCOUNT_PUBLIC_KEY: &str = "accountPublicKey";+pub const ENCRYPTED_ACCOUNT_PRIVATE_KEY: &str = "encryptedAccountPrivateKey";+pub const ENCRYPTION_KEY_DERIVATION_PARAMS: &str = "encryptionKeyDerivationParams";++/// Read a value, `None` when absent or null.+pub(crate) async fn get<T: DeserializeOwned>(host: &Host, key: &str) -> SyncResult<Option<T>> {+ let response: StateValue = host.call(Command::StateGet { key: key.to_string() }).await?;+ match response.value {+ Value::Null => Ok(None),+ value => Ok(Some(serde_json::from_value(value)?)),+ }+}++/// Write a value.+pub(crate) async fn set<T: Serialize>(host: &Host, key: &str, value: &T) -> SyncResult<()> {+ host.call::<Ack>(Command::StateSet { key: key.to_string(), value: serde_json::to_value(value)? }).await?;+ Ok(())+}++/// Delete a value.+pub(crate) async fn remove(host: &Host, key: &str) -> SyncResult<()> {+ host.call::<Ack>(Command::StateRemove { key: key.to_string() }).await?;+ Ok(())+}++/// The record key of one data bucket's revision and fingerprint.+pub fn bucket_revision_key(manifest_id: &str, category: &str) -> String {+ format!("{}:{}", manifest_id, category)+}++pub fn fingerprint_manifest_key(manifest_id: &str) -> String {+ format!("manifest:{}", manifest_id)+}++pub fn fingerprint_bucket_key(manifest_id: &str, category: &str) -> String {+ format!("bucket:{}:{}", manifest_id, category)+}++/// The at-rest vault blob.+pub(crate) async fn load_vault(host: &Host) -> SyncResult<Option<String>> {+ let stored: StoredVault = host.call(Command::VaultLoad).await?;+ Ok(stored.encrypted_blob.filter(|blob| !blob.is_empty()))+}++/// Persist the at-rest vault blob through the host.+pub(crate) async fn store_vault(host: &Host, encrypted_blob: &str, mark_dirty: bool, expected_mutation_seq: Option<u64>, revision: Option<i64>) -> SyncResult<StoreOutcome> {+ store_vault_with_key(host, encrypted_blob, mark_dirty, expected_mutation_seq, revision, None).await+}++/// Persist the at-rest vault blob, telling the host the (new) key it is encrypted under.+pub(crate) async fn store_vault_with_key(host: &Host, encrypted_blob: &str, mark_dirty: bool, expected_mutation_seq: Option<u64>, revision: Option<i64>, encryption_key: Option<String>) -> SyncResult<StoreOutcome> {+ host.call(Command::VaultStore { encrypted_blob: encrypted_blob.to_string(), mark_dirty, encryption_key, expected_mutation_seq, revision }).await+}++/// Clear the dirty flag unless a mutation happened since the given sequence.+pub(crate) async fn mark_clean(host: &Host, mutation_seq_at_start: u64) -> SyncResult<bool> {+ let outcome: CleanOutcome = host.call(Command::MarkClean { mutation_seq_at_start }).await?;+ Ok(outcome.cleared)+}++/// The bytes every SQLite database file begins with (the file header).+const SQLITE_HEADER: &[u8] = b"SQLite format 3\0";++/// Decrypt a stored vault blob into the plaintext SQLite database.+pub(crate) fn decrypt_vault_blob(encrypted_blob: &str, key: &str) -> SyncResult<Vec<u8>> {+ let ciphertext = crypto::aes_gcm::decode_base64(encrypted_blob)?;+ let plaintext = crypto::symmetric_decrypt_bytes(&ciphertext, key).map_err(|e| SyncError::VaultDecryptFailed(e.to_string()))?;+ if plaintext.starts_with(SQLITE_HEADER) {+ return Ok(plaintext);+ }+ let not_a_database = || SyncError::VaultDecryptFailed("plaintext is neither a database nor base64 text".to_string());+ let text = String::from_utf8(plaintext).map_err(|_| not_a_database())?;+ crypto::aes_gcm::decode_base64(text.trim()).map_err(|_| not_a_database())+}++/// Encrypt a plaintext SQLite database for local storage.+pub(crate) fn encrypt_vault_blob(sqlite_bytes: &[u8], key: &str) -> SyncResult<String> {+ Ok(crypto::symmetric_encrypt_bytes(sqlite_bytes, key)?)+}diff --git a/core/rust/src/vault_sync/types.rs b/core/rust/src/vault_sync/types.rsnew file mode 100644index 000000000..82c5a8acc--- /dev/null+++ b/core/rust/src/vault_sync/types.rs@@ -0,0 +1,718 @@+//! Every JSON shape the engine exchanges.++use std::collections::HashMap;+use std::fmt;++use serde::{Deserialize, Serialize};+use serde_json::{Map, Value};++use super::errors::{ErrorCode, LogoutReason};++/// The operation a session runs.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub enum SyncOperation {+ /// Status check, server-directed changes, then pull (and merge) or push based on the revision counters.+ FullSync,+ /// Classify the pending local migration.+ MigrationStatus,+ /// Run the local storage-model migration and push it.+ MigrateManifest,+ /// Run a status check.+ StatusCheck,+}++/// Sync request.+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct SyncRequest {+ pub operation: SyncOperation,+ pub username: String,+ #[serde(default)]+ pub encryption_key: Option<String>,+ #[serde(default)]+ pub account_public_key: Option<String>,+ #[serde(default)]+ pub account_private_key: Option<String>,+ #[serde(default)]+ pub is_dirty: bool,+ #[serde(default)]+ pub mutation_sequence: u64,+ #[serde(default)]+ pub dirty_scopes: Vec<String>,+ #[serde(default)]+ pub private_email_domains: Vec<String>,+ #[serde(default)]+ pub force_pull: bool,+ #[serde(default)]+ pub min_server_version: Option<String>,+ #[serde(default)]+ pub is_offline_mode: bool,+ #[serde(default)]+ pub unnamed_shared_vault_name: Option<String>,+}++/// Session values the engine changed and the host has to adopt.+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct SessionUpdates {+ #[serde(skip_serializing_if = "Option::is_none")]+ pub encryption_key: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub account_private_key: Option<String>,+ /// Set when the account private key must be dropped from the session.+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]+ pub clear_account_private_key: bool,+}++/// What every operation reports on top of its own outcome.+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct SessionOutcome {+ #[serde(default)]+ pub session_updates: SessionUpdates,+ /// Whether the stored vault changed; hosts reload what reads it (autofill stores, open UI).+ #[serde(default)]+ pub vault_changed: bool,+}++/// The result of one operation; the engine fills in the session outcome once the run is over.+pub(crate) trait OperationResult: Serialize {+ fn session_mut(&mut self) -> &mut SessionOutcome;+}++/// Outcome of a full sync.+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct FullSyncResult {+ pub success: bool,+ pub has_new_vault: bool,+ pub was_offline: bool,+ pub sqlite_blob_upgrade_required: bool,+ pub manifest_migration_required: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error_code: Option<ErrorCode>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error_key: Option<LogoutReason>,+ pub requires_logout: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub server_version: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub capabilities: Option<HashMap<String, String>>,+ pub is_offline_mode: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub pulled_revision: Option<i64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub email_routing: Option<EmailRoutingDto>,+ #[serde(flatten)]+ pub session: SessionOutcome,+}++impl OperationResult for FullSyncResult {+ fn session_mut(&mut self) -> &mut SessionOutcome {+ &mut self.session+ }+}++/// The pending local migration, as the upgrade gate classifies it.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "kebab-case")]+pub enum MigrationKind {+ /// Nothing to migrate, or nothing that may be migrated yet: a vault still on the frozen sqlite-blob chain+ /// classifies as `None` because that chain has to bring it to 2.0.0 before either kind below can apply.+ None,+ /// The local schema predates the current full schema; rebuilt locally, no server involved.+ SchemaRebuild,+ /// The account still lacks its key hierarchy; the migration push creates it.+ StorageFormatUpgrade,+}++/// Outcome of the migration classification.+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct MigrationStatusResult {+ pub kind: MigrationKind,+ #[serde(flatten)]+ pub session: SessionOutcome,+}++impl OperationResult for MigrationStatusResult {+ fn session_mut(&mut self) -> &mut SessionOutcome {+ &mut self.session+ }+}++/// Outcome of the manifest migration.+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct MigrateManifestResult {+ pub success: bool,+ /// Whether the migrated vault reached the server; false leaves it dirty for the next sync.+ pub pushed: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error_code: Option<ErrorCode>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error_key: Option<LogoutReason>,+ pub requires_logout: bool,+ #[serde(flatten)]+ pub session: SessionOutcome,+}++impl OperationResult for MigrateManifestResult {+ fn session_mut(&mut self) -> &mut SessionOutcome {+ &mut self.session+ }+}++/// Outcome of the lightweight status check.+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct StatusCheckResult {+ pub success: bool,+ pub has_newer_vault: bool,+ pub has_dirty_changes: bool,+ pub is_offline: bool,+ pub requires_logout: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error_key: Option<LogoutReason>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub error_code: Option<ErrorCode>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub server_version: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub capabilities: Option<HashMap<String, String>>,+ #[serde(flatten)]+ pub session: SessionOutcome,+}++impl OperationResult for StatusCheckResult {+ fn session_mut(&mut self) -> &mut SessionOutcome {+ &mut self.session+ }+}++/*+ * Host commands. Variant names and their fields both travel in camelCase (`vaultStore`, `encryptedBlob`). Every field a+ * host may leave out of a response has a default, so `{}` answers any command that returns nothing.+ */++/// The HTTP methods the engine uses.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "UPPERCASE")]+pub enum HttpMethod {+ Get,+ Post,+ Delete,+}++impl fmt::Display for HttpMethod {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.write_str(match self {+ HttpMethod::Get => "GET",+ HttpMethod::Post => "POST",+ HttpMethod::Delete => "DELETE",+ })+ }+}++/// The databases a command may address.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "lowercase")]+pub enum Db {+ /// The vault the host has open.+ Local,+ /// A throwaway database the engine opens for one step and discards: it materializes a pulled vault into it+ /// before that becomes the new local vault, and probes the current client schema by opening a fresh one.+ Staging,+}++/// The levels of a `log` command. `Phase` carries a sync phase (`pull` or `push`) for a host that shows what+/// the sync is doing; the others are development logging a host may ignore.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "lowercase")]+pub enum LogLevel {+ Log,+ Warn,+ Phase,+}++/// One parameterized SQL statement for the host to run.+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct SqlStatement {+ pub sql: String,+ #[serde(default)]+ pub params: Vec<Value>,+}++/// What the engine can ask the host to do.+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(tag = "kind", rename_all = "camelCase", rename_all_fields = "camelCase")]+pub enum Command {+ /// An API request; response [`HttpResponse`].+ Http {+ method: HttpMethod,+ path: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ body: Option<String>,+ auth: bool,+ /// A vault transfer, which the host may give a longer timeout than a status call (which uses default).+ large_transfer: bool,+ },+ /// Read an engine-owned persisted value; response [`StateValue`].+ StateGet { key: String },+ /// Write an engine-owned persisted value; response [`Ack`].+ StateSet { key: String, value: Value },+ /// Delete an engine-owned persisted value; response [`Ack`].+ StateRemove { key: String },+ /// Open the staging database, response [`Ack`]: from the given SQLite bytes (base64), or fresh with the+ /// current client schema applied when `bytes` is null.+ DbOpen {+ db: Db,+ #[serde(skip_serializing_if = "Option::is_none")]+ bytes: Option<String>,+ },+ /// Run a SELECT; response [`DbRows`].+ DbQuery { db: Db, sql: String, params: Vec<Value> },+ /// Run statements inside one transaction; response [`Ack`]. A `{ "__b64": ... }` parameter binds a BLOB.+ DbExec { db: Db, statements: Vec<SqlStatement> },+ /// Serialize a database; response [`DbBytes`].+ DbExport { db: Db },+ /// Persist the at-rest vault blob; response [`StoreOutcome`].+ VaultStore {+ encrypted_blob: String,+ mark_dirty: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ encryption_key: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ expected_mutation_seq: Option<u64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ revision: Option<i64>,+ },+ /// Read the at-rest vault blob; response [`StoredVault`].+ VaultLoad,+ /// Clear the dirty flag when no mutation happened since; response [`CleanOutcome`].+ MarkClean { mutation_seq_at_start: u64 },+ /// A log line; response [`Ack`].+ Log { level: LogLevel, message: String },+ /// The operation finished; `result` is the operation's outcome.+ Done { result: Value },+}++impl Command {+ pub fn kind(&self) -> CommandKind {+ match self {+ Command::Http { .. } => CommandKind::Http,+ Command::StateGet { .. } => CommandKind::StateGet,+ Command::StateSet { .. } => CommandKind::StateSet,+ Command::StateRemove { .. } => CommandKind::StateRemove,+ Command::DbOpen { .. } => CommandKind::DbOpen,+ Command::DbQuery { .. } => CommandKind::DbQuery,+ Command::DbExec { .. } => CommandKind::DbExec,+ Command::DbExport { .. } => CommandKind::DbExport,+ Command::VaultStore { .. } => CommandKind::VaultStore,+ Command::VaultLoad => CommandKind::VaultLoad,+ Command::MarkClean { .. } => CommandKind::MarkClean,+ Command::Log { .. } => CommandKind::Log,+ Command::Done { .. } => CommandKind::Done,+ }+ }+}++/// The kind of a command, for error reports.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum CommandKind {+ Http,+ StateGet,+ StateSet,+ StateRemove,+ DbOpen,+ DbQuery,+ DbExec,+ DbExport,+ VaultStore,+ VaultLoad,+ MarkClean,+ Log,+ Done,+}++impl fmt::Display for CommandKind {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.write_str(match self {+ CommandKind::Http => "http",+ CommandKind::StateGet => "stateGet",+ CommandKind::StateSet => "stateSet",+ CommandKind::StateRemove => "stateRemove",+ CommandKind::DbOpen => "dbOpen",+ CommandKind::DbQuery => "dbQuery",+ CommandKind::DbExec => "dbExec",+ CommandKind::DbExport => "dbExport",+ CommandKind::VaultStore => "vaultStore",+ CommandKind::VaultLoad => "vaultLoad",+ CommandKind::MarkClean => "markClean",+ CommandKind::Log => "log",+ CommandKind::Done => "done",+ })+ }+}++/*+ * Responses.+ */++/// An empty acknowledgement (`{}`).+#[derive(Debug, Default, Deserialize)]+pub struct Ack {}++/// Response of `http`. `status` 0 means the request never reached the server; `transport_error` says why.+#[derive(Debug, Default, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct HttpResponse {+ #[serde(default)]+ pub status: u16,+ #[serde(default)]+ pub body: String,+ #[serde(default)]+ pub transport_error: Option<String>,+ #[serde(default)]+ pub timed_out: bool,+}++/// Response of `stateGet`: the stored JSON value, null when absent.+#[derive(Debug, Default, Deserialize)]+pub struct StateValue {+ #[serde(default)]+ pub value: Value,+}++/// Response of `dbQuery`: one object per row, BLOB columns as `{ "__b64": ... }`.+#[derive(Debug, Default, Deserialize)]+pub struct DbRows {+ #[serde(default)]+ pub rows: Vec<Map<String, Value>>,+}++/// Response of `dbExport`: the SQLite file, base64.+#[derive(Debug, Deserialize)]+pub struct DbBytes {+ pub bytes: String,+}++/// Response of `vaultStore`: whether the store went through, and the host's mutation sequence after it.+#[derive(Debug, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct StoreOutcome {+ #[serde(default = "default_true")]+ pub success: bool,+ #[serde(default)]+ pub mutation_sequence: u64,+}++/// Response of `vaultLoad`: the at-rest blob, null when none is stored.+#[derive(Debug, Default, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct StoredVault {+ #[serde(default)]+ pub encrypted_blob: Option<String>,+}++/// Response of `markClean`: whether the dirty flag was cleared.+#[derive(Debug, Deserialize)]+pub struct CleanOutcome {+ #[serde(default = "default_true")]+ pub cleared: bool,+}++fn default_true() -> bool {+ true+}++/*+ * Server API: the v2 endpoints.+ */++/// `GET v2/Status`.+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct StatusResponse {+ #[serde(default)]+ pub client_version_supported: bool,+ #[serde(default)]+ pub server_version: String,+ #[serde(default)]+ pub manifest_revisions: Vec<ManifestRevision>,+ #[serde(default)]+ pub bucket_revisions: Vec<BucketRevision>,+ #[serde(default)]+ pub personal_manifest_id: Option<String>,+ #[serde(default)]+ pub srp_salt: Option<String>,+ #[serde(default)]+ pub capabilities: Option<HashMap<String, String>>,+ #[serde(default)]+ pub pending_actions: Vec<PendingAction>,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct ManifestRevision {+ pub manifest_id: String,+ pub revision: i64,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct BucketRevision {+ pub manifest_id: String,+ pub category: String,+ pub revision: i64,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct PendingAction {+ pub id: String,+ #[serde(rename = "type")]+ pub action_type: String,+ #[serde(default)]+ pub manifest_id: Option<String>,+ #[serde(default)]+ pub payload: Option<String>,+}++/// `GET v2/Vault`.+#[derive(Debug, Clone, Default, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct GetResponse {+ #[serde(default)]+ pub status: i32,+ /// 0 = sqlite-blob, 1 = manifest-v1; absent on servers predating the field.+ #[serde(default)]+ pub storage_format: Option<i32>,+ #[serde(default)]+ pub legacy_vault_blob: Option<String>,+ #[serde(default)]+ pub version: Option<String>,+ #[serde(default)]+ pub legacy_revision: Option<i64>,+ #[serde(default)]+ pub personal_manifest_id: Option<String>,+ #[serde(default)]+ pub manifests: Vec<ManifestDto>,+ #[serde(default)]+ pub buckets: Vec<BucketDto>,+ #[serde(default)]+ pub email_routing: Option<EmailRoutingDto>,+}++pub const STORAGE_FORMAT_MANIFEST: i32 = 1;++#[derive(Debug, Clone, Default, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct ManifestDto {+ pub manifest_id: String,+ #[serde(default)]+ pub blob: Option<String>,+ #[serde(default)]+ pub ciphertext_hash: Option<String>,+ #[serde(default)]+ pub revision: i64,+ #[serde(default)]+ pub blob_references: Vec<StoredBlobRef>,+ #[serde(default)]+ pub can_administer: bool,+ #[serde(default)]+ pub key_type: Option<String>,+ #[serde(default)]+ pub encrypted_vek: Option<String>,+ #[serde(default)]+ pub algorithm: Option<String>,+ #[serde(default)]+ pub encryption_public_key: Option<String>,+}++#[derive(Debug, Clone, Default, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct BucketDto {+ pub manifest_id: String,+ pub category: String,+ #[serde(default)]+ pub blob: Option<String>,+ #[serde(default)]+ pub ciphertext_hash: Option<String>,+ #[serde(default)]+ pub revision: Option<i64>,+}++#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct StoredBlobRef {+ pub hash: String,+ pub category: String,+ #[serde(default)]+ pub size_bytes: i64,+}++#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct EmailRoutingDto {+ #[serde(default)]+ pub email_address_list: Vec<String>,+ #[serde(default)]+ pub private_email_domain_list: Vec<String>,+ #[serde(default)]+ pub hidden_private_email_domain_list: Vec<String>,+ #[serde(default)]+ pub public_email_domain_list: Vec<String>,+}++pub const KEY_TYPE_ACCOUNT_KEY: &str = "accountkey";+pub const KEY_TYPE_GRANT_KEY: &str = "grantkey";+pub const ALGORITHM_RSA_OAEP_SHA256: &str = "rsa-oaep-sha256";++/// One manifest element of `POST v2/Vault`.+#[derive(Debug, Clone, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct ManifestWrite {+ pub manifest_id: String,+ pub manifest_blob: String,+ pub manifest_ciphertext_hash: String,+ pub current_revision: i64,+ pub credentials_count: usize,+ pub blob_references: Vec<BlobRef>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub encryption_public_key: Option<String>,+}++#[derive(Debug, Clone, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct BlobRef {+ pub hash: String,+ pub category: String,+}++#[derive(Debug, Clone, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct BucketWrite {+ pub manifest_id: String,+ pub category: String,+ pub blob: String,+ pub ciphertext_hash: String,+ pub current_revision: i64,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct BlobDto {+ pub hash: String,+ pub category: String,+ pub encrypted_data_base64: String,+}++#[derive(Debug, Clone, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct ClaimedEmailAddress {+ pub address: String,+ pub manifest_id: String,+ pub paused: bool,+}++#[derive(Debug, Clone, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct EmailRoutingPush {+ pub email_address_list: Vec<ClaimedEmailAddress>,+ pub covered_manifest_ids: Vec<String>,+}++#[derive(Debug, Clone, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct VaultWriteRequest {+ pub username: String,+ pub manifests: Vec<ManifestWrite>,+ pub buckets: Vec<BucketWrite>,+ pub new_blobs: Vec<BlobDto>,+ pub email_routing: Option<EmailRoutingPush>,+ pub account_keys: Option<crate::crypto::AccountKeyBlobs>,+}++#[derive(Debug, Clone, Default, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct VaultWriteResponse {+ #[serde(default)]+ pub status: i32,+ #[serde(default)]+ pub manifest_revisions: Vec<ManifestRevision>,+ #[serde(default)]+ pub bucket_revisions: Vec<BucketRevision>,+ #[serde(default)]+ pub missing_blob_hashes: Vec<String>,+}++#[derive(Debug, Clone, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct BlobUploadRequest {+ pub blobs: Vec<BlobDto>,+ pub overwrite: bool,+}++#[derive(Debug, Clone, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct BlobHashesRequest {+ pub hashes: Vec<String>,+}++#[derive(Debug, Clone, Default, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct MissingBlobsResponse {+ #[serde(default)]+ pub missing: Vec<String>,+}++/// `GET v2/VaultKey/Password`.+#[derive(Debug, Clone, Default, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct VaultKeyGetResponse {+ #[serde(default)]+ pub vault_key: Option<VaultKeyResponse>,+}++#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct VaultKeyResponse {+ #[serde(default, rename = "type")]+ pub key_type: String,+ #[serde(default)]+ pub encrypted_account_key: String,+ #[serde(default)]+ pub encrypted_account_private_key: Option<String>,+ #[serde(default)]+ pub account_public_key: Option<String>,+ #[serde(default)]+ pub encrypted_vek: Option<String>,+ #[serde(default)]+ pub salt: String,+ #[serde(default)]+ pub encryption_type: String,+ #[serde(default)]+ pub encryption_settings: String,+}+/// A shared manifest as this account holds it: the grant on its key plus what the last pull learned about it.+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct SharedManifestRecord {+ pub manifest_id: String,+ pub encrypted_vek: String,+ pub encryption_public_key: String,+ pub algorithm: String,+ pub salt: String,+ #[serde(default)]+ pub name: Option<String>,+ #[serde(default)]+ pub can_administer: bool,+}diff --git a/core/rust/src/vault_sync/version.rs b/core/rust/src/vault_sync/version.rsnew file mode 100644index 000000000..15e4c67f8--- /dev/null+++ b/core/rust/src/vault_sync/version.rs@@ -0,0 +1,112 @@+//! Version rules: server support, and the frozen legacy sqlite-blob vault version chain.++/// Whether `version1 >= version2` under the client's SemVer rules: a pre-release sorts below its release, and+/// two pre-releases compare lexically.+pub fn version_gte(version1: &str, version2: &str) -> bool {+ let (core1, pre1) = split(version1);+ let (core2, pre2) = split(version2);+ let parts1: Vec<u64> = core1.split('.').map(|part| part.parse().unwrap_or(0)).collect();+ let parts2: Vec<u64> = core2.split('.').map(|part| part.parse().unwrap_or(0)).collect();++ for index in 0..parts1.len().max(parts2.len()) {+ let part1 = parts1.get(index).copied().unwrap_or(0);+ let part2 = parts2.get(index).copied().unwrap_or(0);+ if part1 > part2 {+ return true;+ }+ if part1 < part2 {+ return false;+ }+ }++ match (pre1, pre2) {+ (None, Some(_)) => true,+ (Some(_), None) => false,+ (None, None) => true,+ (Some(a), Some(b)) => a >= b,+ }+}++fn split(version: &str) -> (&str, Option<&str>) {+ match version.split_once('-') {+ Some((core, pre)) if !pre.is_empty() => (core, Some(pre)),+ Some((core, _)) => (core, None),+ None => (version, None),+ }+}++/// The frozen sqlite-blob upgrade chain: (revision, data version). It ends at 2.0.0, the first schema compatible+/// with manifest-v1; later schema changes ship through the full schema only which is used by every materilization directly.+const LEGACY_VAULT_VERSIONS: &[(u32, &str)] = &[+ (1, "1.0.0"),+ (2, "1.0.1"),+ (3, "1.0.2"),+ (4, "1.1.0"),+ (5, "1.2.0"),+ (6, "1.3.0"),+ (7, "1.3.1"),+ (8, "1.4.0"),+ (9, "1.4.1"),+ (10, "1.5.0"),+ (11, "1.6.0"),+ (12, "1.7.0"),+ (13, "2.0.0"),+];++/// The revision of the last entry in the legacy chain.+pub fn latest_legacy_revision() -> u32 {+ LEGACY_VAULT_VERSIONS.last().map(|(revision, _)| *revision).unwrap_or(0)+}++/// The data version embedded in an EF migration id (`20250101000000_2.0.0-Name`).+pub fn extract_version_from_migration_id(migration_id: &str) -> Option<String> {+ let start = migration_id.find('_')? + 1;+ let rest = &migration_id[start..];+ let end = rest.find('-')?;+ let candidate = &rest[..end];+ let parts: Vec<&str> = candidate.split('.').collect();+ if parts.len() == 3 && parts.iter().all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit())) {+ Some(candidate.to_string())+ } else {+ None+ }+}++/// The legacy-chain revision a database version maps to. A known version maps to its entry; an unknown but+/// well-formed version is treated as the latest (backwards compatible); a malformed version is incompatible.+pub fn legacy_revision_for(database_version: &str) -> Result<u32, String> {+ if let Some((revision, _)) = LEGACY_VAULT_VERSIONS.iter().find(|(_, version)| *version == database_version) {+ return Ok(*revision);+ }+ let well_formed = database_version.split('.').count() == 3 && database_version.split('.').all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit()));+ if well_formed {+ Ok(latest_legacy_revision())+ } else {+ Err(format!("Vault version {} is not compatible with this client", database_version))+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn semver_rules_match_the_client() {+ assert!(version_gte("0.12.0", "0.12.0-dev"));+ assert!(!version_gte("0.12.0-dev", "0.12.0"));+ assert!(version_gte("0.13.1", "0.12.0-dev"));+ assert!(!version_gte("0.11.9", "0.12.0-dev"));+ assert!(version_gte("1.0", "1.0.0"));+ assert!(version_gte("0.12.0-beta", "0.12.0-alpha"));+ }++ #[test]+ fn migration_ids_yield_their_version() {+ assert_eq!(extract_version_from_migration_id("20250101000000_2.0.0-Squashed").as_deref(), Some("2.0.0"));+ assert_eq!(extract_version_from_migration_id("20250101000000_Initial"), None);+ assert_eq!(legacy_revision_for("1.7.0"), Ok(12));+ assert_eq!(legacy_revision_for("2.0.0"), Ok(13));+ assert_eq!(legacy_revision_for("2.1.0"), Ok(13));+ assert!(legacy_revision_for("nope").is_err());+ }+}