Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update vault_sync queries (#2159)

Leendert de Borst54ecec7bf3e6
diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rs
index 793dce5d3..4f785b98f 100644
--- a/core/rust/src/vault_codec/mod.rs
+++ b/core/rust/src/vault_codec/mod.rs
@@ -36,7 +36,7 @@ pub use manifest::{
pub use materialize::materialize_as_sqlite;
pub use scoped_assets::logo_id_for;
pub use sharing::extract_encryption_key_for_public_key;
-pub use types::{bucket_categories, identity_part, is_bucketed_table, tables_for_category};
+pub use types::{bucket_categories, identity_part, is_bucketed_table, is_skip_table, manifest_scoped_tables, tables_for_category};
pub use validate::{validate_data_bucket, validate_manifest, ValidationResult};
/// The bucket layout: every category and the tables it owns, in declaration order.
diff --git a/core/rust/src/vault_codec/sharing_tests.rs b/core/rust/src/vault_codec/sharing_tests.rs
index e4fe6aacc..df835f5fb 100644
--- a/core/rust/src/vault_codec/sharing_tests.rs
+++ b/core/rust/src/vault_codec/sharing_tests.rs
@@ -1243,6 +1243,40 @@ fn combine_drops_child_rows_naming_an_item_the_manifest_does_not_hold() {
assert_eq!(values(&map["FieldValues"]), vec!["me"], "the foreign row never reaches the reader's item");
}
+#[test]
+fn combine_restamps_rows_a_manifest_wrote_for_another_manifest() {
+ // A member of one shared manifest writes rows stamped for a manifest it cannot write. The stamp an
+ // author wrote is never trusted: every row claims the manifest it arrived in, so nothing crosses over.
+ let mut personal = canonicalize_from_sqlite(input_with_shares(vec![table("Items", vec![row(&[("Id", json!("i-mine")), ("Name", json!("mine")), ("FolderId", serde_json::Value::Null)])])], vec![])).unwrap().first().manifest.clone();
+ personal.tables.insert("Folders".to_string(), vec![]);
+
+ let mut victim = personal.clone();
+ victim.manifest_id = "m-victim".to_string();
+ victim.tables.insert("Items".to_string(), vec![row(&[("Id", json!("i-victim")), ("Name", json!("theirs")), ("FolderId", serde_json::Value::Null)])]);
+
+ let mut hostile = personal.clone();
+ hostile.manifest_id = "m-hostile".to_string();
+ hostile.tables.insert(
+ "Items".to_string(),
+ vec![
+ row(&[("Id", json!("i-planted")), ("ManifestId", json!("m-victim")), ("Name", json!("planted")), ("FolderId", serde_json::Value::Null)]),
+ row(&[("Id", json!("i-victim")), ("ManifestId", json!("m-victim")), ("Name", json!("overwritten")), ("FolderId", serde_json::Value::Null)]),
+ ],
+ );
+ hostile.tables.insert("Tags".to_string(), vec![row(&[("Id", json!("t-planted")), ("ManifestId", json!(PERSONAL_M)), ("Name", json!("planted"))])]);
+
+ let re = materialize_as_sqlite(materialize_input(personal, vec![victim, hostile], vec![])).unwrap();
+ let map = materialized_map(&re);
+
+ let in_victim: Vec<&CodecRecord> = map["Items"].iter().filter(|r| r["ManifestId"] == json!("m-victim")).collect();
+ assert_eq!(in_victim.len(), 1, "the victim manifest holds only its own item");
+ assert_eq!(in_victim[0]["Name"], json!("theirs"), "the victim's item is not overwritten");
+ for id in ["i-planted", "i-victim"] {
+ assert!(map["Items"].iter().any(|r| r["Id"] == json!(id) && r["ManifestId"] == json!("m-hostile")), "{id} stays in the manifest it arrived in");
+ }
+ assert!(map["Tags"].iter().all(|r| r["ManifestId"] == json!("m-hostile")), "a row stamped for the personal manifest stays out of it");
+}
+
#[test]
fn combine_nulls_an_item_folder_that_only_resolves_in_another_manifest() {
// Folder ids are per-manifest now: a folder with the right Id in the wrong namespace is not this
diff --git a/core/rust/src/vault_sync/db.rs b/core/rust/src/vault_sync/db.rs
index 9a5a3ece6..7512bfc53 100644
--- a/core/rust/src/vault_sync/db.rs
+++ b/core/rust/src/vault_sync/db.rs
@@ -12,14 +12,16 @@ use crate::encoding::{base64_decode, base64_encode, uuid_from_bytes};
use crate::sqlite_host::SqlStatement;
use crate::timestamp::{now_iso_utc, now_vault_datetime};
use crate::vault_codec::row::{blob_ref_of, inline_bytes};
-use crate::vault_codec::{CodecRecord, CodecTableData, MaterializedTables};
-use crate::vault_model::{id_key, ids_equal, UNSTAMPED_SCOPE_SENTINEL};
+use crate::vault_codec::{is_skip_table, manifest_scoped_tables, CodecRecord, CodecTableData, MaterializedTables};
+use crate::vault_model::{id_key, ids_equal, MANIFEST_ID_COL, UNSTAMPED_SCOPE_SENTINEL};
pub(crate) type Row = Map<String, Value>;
/// Rows per `dbExec` batch when bulk-loading a materialized vault.
const INSERT_BATCH_ROWS: usize = 400;
+const TABLE_COLUMNS: &str = "SELECT m.name AS TableName, p.name AS ColumnName FROM sqlite_master m JOIN pragma_table_info(m.name) p WHERE m.type = 'table' ORDER BY m.name, p.cid";
+
pub(crate) async fn query(host: &Host, db: Db, sql: &str, params: Vec<Value>) -> SyncResult<Vec<Row>> {
let response: DbRows = host.call(Command::DbQuery { db, sql: sql.to_string(), params }).await?;
Ok(response.rows)
@@ -68,7 +70,8 @@ pub(crate) async fn list_user_tables(host: &Host, db: Db) -> SyncResult<Vec<Stri
/// Every user table with its rows, the shape canonicalize takes.
pub(crate) async fn read_tables(host: &Host, db: Db) -> SyncResult<Vec<CodecTableData>> {
let mut tables = Vec::new();
- for name in list_user_tables(host, db).await? {
+ // Bookkeeping tables never reach a manifest, so their rows are not read either.
+ for name in list_user_tables(host, db).await?.into_iter().filter(|name| !is_skip_table(name)) {
let rows = query(host, db, &format!("SELECT * FROM \"{}\"", name), vec![]).await?;
tables.push(CodecTableData { name, records: rows.into_iter().map(row_to_record).collect() });
}
@@ -95,11 +98,10 @@ fn row_to_record(row: Row) -> CodecRecord {
/// The column set of every table in a database, as materialize needs it.
pub(crate) async fn schema_columns(host: &Host, db: Db) -> SyncResult<HashMap<String, Vec<String>>> {
- let mut out = HashMap::new();
- let tables = query(host, db, "SELECT name FROM sqlite_master WHERE type='table'", vec![]).await?;
- for table in tables.iter().filter_map(|row| row.get("name").and_then(Value::as_str)) {
- let columns = query(host, db, &format!("PRAGMA table_info(\"{}\")", table), vec![]).await?;
- out.insert(table.to_string(), columns.iter().filter_map(|row| row.get("name").and_then(Value::as_str).map(str::to_string)).collect());
+ let mut out: HashMap<String, Vec<String>> = HashMap::new();
+ for row in query(host, db, TABLE_COLUMNS, vec![]).await? {
+ let (Some(table), Some(column)) = (row.get("TableName").and_then(Value::as_str), row.get("ColumnName").and_then(Value::as_str)) else { continue };
+ out.entry(table.to_string()).or_default().push(column.to_string());
}
Ok(out)
}
@@ -140,26 +142,28 @@ pub(crate) async fn has_column(host: &Host, db: Db, table: &str, column: &str) -
Ok(!query(host, db, "SELECT name FROM pragma_table_info(?) WHERE name = ?", vec![json!(table), json!(column)]).await?.is_empty())
}
-/// The tables carrying a manifest stamp.
-pub(crate) async fn stamped_tables(host: &Host, db: Db) -> SyncResult<Vec<String>> {
- let mut out = Vec::new();
- for name in list_user_tables(host, db).await? {
- if has_column(host, db, &name, "ManifestId").await? {
- out.push(name);
- }
+/*
+ * Every manifest id the rows of this vault name. This reads what the vault holds and nothing else: which
+ * manifests may be written is decided by the grants the server served and the keys that opened, and the
+ * two are compared so rows naming a manifest outside that set stop the push instead of being left out of it.
+ */
+pub(crate) async fn manifest_ids_in_vault(host: &Host, db: Db) -> SyncResult<Vec<String>> {
+ // Table names come from the registry, never from the database; a vault on an older schema lacks some.
+ let columns = schema_columns(host, db).await?;
+ let selects: Vec<String> = manifest_scoped_tables()
+ .into_iter()
+ .filter(|table| columns.get(*table).is_some_and(|names| names.iter().any(|name| name == MANIFEST_ID_COL)))
+ .map(|table| format!("SELECT {} FROM \"{}\"", MANIFEST_ID_COL, table))
+ .collect();
+ if selects.is_empty() {
+ return Ok(Vec::new());
}
- Ok(out)
-}
-/// Every manifest id this vault holds a row for.
-pub(crate) async fn manifest_ids_in_vault(host: &Host, db: Db) -> SyncResult<Vec<String>> {
+ let rows = query(host, db, &format!("{} ORDER BY 1", selects.join(" UNION ")), vec![]).await?;
let mut ids: Vec<String> = Vec::new();
- for table in stamped_tables(host, db).await? {
- let rows = query(host, db, &format!("SELECT DISTINCT ManifestId FROM \"{}\" WHERE ManifestId IS NOT NULL AND ManifestId != ''", table), vec![]).await?;
- for id in rows.iter().filter_map(|row| row.get("ManifestId").and_then(Value::as_str)) {
- if !ids_equal(id, UNSTAMPED_SCOPE_SENTINEL) && !ids.iter().any(|known| known == id) {
- ids.push(id.to_string());
- }
+ for id in rows.iter().filter_map(|row| row.get(MANIFEST_ID_COL).and_then(Value::as_str)) {
+ if !id.is_empty() && !ids_equal(id, UNSTAMPED_SCOPE_SENTINEL) && !ids.iter().any(|known| known == id) {
+ ids.push(id.to_string());
}
}
Ok(ids)
@@ -167,18 +171,12 @@ pub(crate) async fn manifest_ids_in_vault(host: &Host, db: Db) -> SyncResult<Vec
/// Load materialized tables into the (fresh) staging database, re-embedding blob bytes, and verify its
/// referential integrity.
-pub(crate) async fn insert_materialized(host: &Host, materialized: &MaterializedTables, blobs: &HashMap<String, Vec<u8>>) -> SyncResult<()> {
- let schema_tables: Vec<String> = query(host, Db::Staging, "SELECT name FROM sqlite_master WHERE type='table'", vec![])
- .await?
- .iter()
- .filter_map(|row| row.get("name").and_then(Value::as_str).map(str::to_string))
- .collect();
-
+pub(crate) async fn insert_materialized(host: &Host, materialized: &MaterializedTables, schema_columns: &HashMap<String, Vec<String>>, blobs: &HashMap<String, Vec<u8>>) -> SyncResult<()> {
for table in &materialized.tables {
if table.records.is_empty() {
continue;
}
- if !schema_tables.contains(&table.name) {
+ if !schema_columns.contains_key(&table.name) {
host.log(LogLevel::Warn, format!("[VaultCodec] Skipping table \"{}\" ({} rows), not present in the schema.", table.name, table.records.len())).await;
continue;
}
@@ -222,17 +220,18 @@ pub(crate) fn new_id() -> String {
uuid_from_bytes(bytes, 4)
}
-const GET_ACTIVE_KEY_FOR_MANIFEST: &str = "SELECT x.Id, x.PublicKey, x.PrivateKey, x.IsPrimary FROM EncryptionKeys x WHERE x.ManifestId = ? AND x.IsPrimary = 1 AND x.IsDeleted = 0 LIMIT 1";
-const GET_ACCOUNT_KEY_BY_PUBLIC_KEY: &str = "SELECT x.PublicKey, x.PrivateKey, x.IsPrimary FROM EncryptionKeys x WHERE x.ManifestId = ? AND x.PublicKey = ? AND x.IsDeleted = 0 LIMIT 1";
+const GET_ACTIVE_PUBLIC_KEY_FOR_MANIFEST: &str = "SELECT x.PublicKey FROM EncryptionKeys x WHERE x.ManifestId = ? AND x.IsPrimary = 1 AND x.IsDeleted = 0 LIMIT 1";
+const GET_ACCOUNT_KEY_BY_PUBLIC_KEY: &str = "SELECT x.PrivateKey FROM EncryptionKeys x WHERE x.ManifestId = ? AND x.PublicKey = ? AND x.IsDeleted = 0 LIMIT 1";
const DEMOTE_KEYS_FOR_MANIFEST: &str = "UPDATE EncryptionKeys SET IsPrimary = 0, UpdatedAt = ? WHERE ManifestId = ? AND IsPrimary = 1";
const INSERT_KEY_FOR_MANIFEST: &str = "INSERT INTO EncryptionKeys (Id, ManifestId, PublicKey, PrivateKey, IsPrimary, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, 1, ?, ?, 0)";
const MANIFEST_NAMES: &str = "SELECT Id, Name FROM Manifests WHERE Name IS NOT NULL";
const UPDATE_MANIFEST_NAME: &str = "UPDATE Manifests SET Name = ? WHERE Id = ?";
const UPSERT_MANIFEST_NAME: &str = "INSERT INTO Manifests (Id, Name) VALUES (?, ?) ON CONFLICT(Id) DO UPDATE SET Name = excluded.Name";
-/// The active mail delivery keypair of a manifest, when it has one.
-pub(crate) async fn active_key_for_manifest(host: &Host, manifest_id: &str) -> SyncResult<Option<Row>> {
- Ok(query(host, Db::Local, GET_ACTIVE_KEY_FOR_MANIFEST, vec![json!(manifest_id)]).await?.into_iter().next())
+/// The public half of a manifest's active mail delivery keypair, when it has one.
+pub(crate) async fn active_public_key_for_manifest(host: &Host, manifest_id: &str) -> SyncResult<Option<String>> {
+ let rows = query(host, Db::Local, GET_ACTIVE_PUBLIC_KEY_FOR_MANIFEST, vec![json!(manifest_id)]).await?;
+ Ok(rows.first().and_then(|row| row.get("PublicKey")).and_then(Value::as_str).map(str::to_string))
}
/// The private half of the account keypair with the given public half, held in the personal manifest.
@@ -277,7 +276,7 @@ pub(crate) async fn set_manifest_name(host: &Host, manifest_id: &str, name: &str
/// Name the manifests a database already holds; a manifest it does not hold stays out of it.
pub(crate) async fn apply_manifest_names(host: &Host, db: Db, names: &HashMap<String, String>) -> SyncResult<()> {
- let statements: Vec<SqlStatement> = names.iter().map(|(manifest_id, name)| SqlStatement { sql: UPDATE_MANIFEST_NAME.to_string(), params: vec![json!(name), json!(manifest_id)] }).collect();
+ let statements: Vec<SqlStatement> = names.iter().map(|(manifest_id, name)| SqlStatement { sql: UPDATE_MANIFEST_NAME.to_string(), params: vec![json!(name), json!(id_key(manifest_id))] }).collect();
exec(host, db, statements).await
}
diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rs
index 802c565d7..bdf5f9656 100644
--- a/core/rust/src/vault_sync/pull.rs
+++ b/core/rust/src/vault_sync/pull.rs
@@ -297,7 +297,7 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], fal
pub(crate) async fn materialize_to_sqlite(ctx: &mut Ctx, manifests: &[Manifest], data_buckets: &[DataBucket], blob_map: &HashMap<String, Vec<u8>>, opened_names: &HashMap<String, String>) -> SyncResult<Vec<u8>> {
ctx.log(format!("[V2Pull] {} blobs decrypted; running codec reassembly into a fresh SQLite ({} manifest(s) combined)...", blob_map.len(), manifests.len())).await;
let schema = ctx.schema().await?;
- let materialized = vault_codec::materialize_as_sqlite(MaterializeInput { manifests: manifests.to_vec(), data_buckets: data_buckets.to_vec(), schema_columns: schema.columns })?;
+ let materialized = vault_codec::materialize_as_sqlite(MaterializeInput { manifests: manifests.to_vec(), data_buckets: data_buckets.to_vec(), schema_columns: schema.columns.clone() })?;
let overflow_tables = materialized.overflow.tables.len() + materialized.overflow.bucket_tables.values().map(HashMap::len).sum::<usize>();
if overflow_tables > 0 || !materialized.overflow.columns.is_empty() {
@@ -306,7 +306,7 @@ pub(crate) async fn materialize_to_sqlite(ctx: &mut Ctx, manifests: &[Manifest],
// Use a fresh staging database for every materialize.
db::open_staging(&ctx.host, None).await?;
- db::insert_materialized(&ctx.host, &materialized, blob_map).await?;
+ db::insert_materialized(&ctx.host, &materialized, &schema.columns, blob_map).await?;
// Names are not part of a manifest: the ones the vault already shows are kept, the ones this run opened win.
let mut names = if ctx.has_local_vault().await? { db::manifest_display_names(&ctx.host).await? } else { HashMap::new() };
diff --git a/core/rust/src/vault_sync/push.rs b/core/rust/src/vault_sync/push.rs
index 56febfd14..d935ecfbc 100644
--- a/core/rust/src/vault_sync/push.rs
+++ b/core/rust/src/vault_sync/push.rs
@@ -563,7 +563,7 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], base
// Publish the public half of this manifest's mail delivery keypair; only admins may publish a shared one.
let may_publish = candidate.record.is_personal || candidate.record.can_administer;
- let manifest_key = if may_publish { db::active_key_for_manifest(&ctx.host, &candidate.record.manifest_id).await? } else { None };
+ let manifest_key = if may_publish { db::active_public_key_for_manifest(&ctx.host, &candidate.record.manifest_id).await? } else { None };
if may_publish && manifest_key.is_none() && !candidate.record.is_personal {
ctx.warn(format!("[V2Push] {} is missing its email keypair; its aliases stay personal until sharing is re-enabled.", label)).await;
}
@@ -577,7 +577,7 @@ async fn encrypt_changed_manifests(ctx: &Ctx, candidates: &[Candidate<'_>], base
current_revision: candidate.current_revision,
credentials_count: candidate.manifest.tables.get("Items").map(Vec::len).unwrap_or(0),
blob_references: blob_refs,
- encryption_public_key: manifest_key.and_then(|row| row.get("PublicKey").and_then(serde_json::Value::as_str).map(str::to_string)),
+ encryption_public_key: manifest_key,
});
written.insert(fingerprint_key, fingerprint);
}