Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Refactor personalManifest tracking (#2159)
Leendert de Borst4904f5282526
diff --git a/apps/browser-extension/src/utils/BucketRowGrouping.ts b/apps/browser-extension/src/utils/BucketRowGrouping.tsindex dc9108a57..597e9e388 100644--- a/apps/browser-extension/src/utils/BucketRowGrouping.ts+++ b/apps/browser-extension/src/utils/BucketRowGrouping.ts@@ -46,25 +46,30 @@ export function findByManifestId<T>(map: Map<string, T>, manifestId: string): T}/**- * Split a bucket category's rows into one table set per owning manifest.+ * Split a bucket category's rows into one table set per owning manifest. Ownership is each row's own+ * `ManifestId` stamp — the writing manifest is no fallback: a row that names no manifest is a bug in+ * whatever wrote it, refused loudly here rather than silently homed somewhere plausible.* @param tableNames - The tables making up the category* @param tables - The rows read for those tables- * @param personalManifestId - The manifest rows belong to+ * @param writingManifestId - The manifest this vault writes from; its bucket is always declared, even empty* @param alsoInclude - Manifests that must be written even when they hold no rows* @returns Manifest id → that manifest's rows, per table*/export function groupBucketRowsByManifest(tableNames: string[],tables: BucketTables,- personalManifestId: string,+ writingManifestId: string,alsoInclude: string[] = []): Map<string, BucketTables> {- const grouped = new Map<string, BucketTables>([[personalManifestId, {}]]);+ const grouped = new Map<string, BucketTables>([[writingManifestId, {}]]);for (const tableName of tableNames) {for (const row of tables[tableName] ?? []) {const stamp = typeof row.ManifestId === 'string' ? row.ManifestId : '';- const owner = isUnstampedManifestId(stamp) ? personalManifestId : (matchManifestId(grouped.keys(), stamp) ?? stamp);+ if (isUnstampedManifestId(stamp)) {+ throw new Error(`Bucket table ${tableName} holds a row that names no manifest; every row must carry the manifest it belongs to`);+ }+ const owner = matchManifestId(grouped.keys(), stamp) ?? stamp;const manifestTables = grouped.get(owner) ?? {};manifestTables[tableName] = [...(manifestTables[tableName] ?? []), row];diff --git a/apps/browser-extension/src/utils/VaultSyncService.ts b/apps/browser-extension/src/utils/VaultSyncService.tsindex 5a2c52639..7244c18f8 100644--- a/apps/browser-extension/src/utils/VaultSyncService.ts+++ b/apps/browser-extension/src/utils/VaultSyncService.ts@@ -842,12 +842,7 @@ export class VaultSyncService {* run earlier (the pre-push no-op check caches its result) and a write in between advances them.*/const storedRevisions = await getManifestRevisions();- const currentManifestRevision = storedRevisions[await this.resolvePersonalManifestId()] ?? 0;- /**- * The revision one manifest's write rebases on, falling back to the revision its grant was resolved at.- * @param record - the manifest record- */- const revisionOf = (record: ManifestRecord): number => record.isPersonal ? currentManifestRevision : storedRevisions[record.manifestId] ?? record.revision;+ const revisionOf = (record: ManifestRecord): number => storedRevisions[record.manifestId] ?? record.revision;const recordByManifestId = new Map(manifestRecords.map(r => [r.manifestId, r]));const candidates: PushManifest[] = canonicalized.manifests.flatMap(({ manifest, blobs }): PushManifest[] => {diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rsindex cc701c94b..982ba7660 100644--- a/core/rust/src/vault_codec/canonicalize.rs+++ b/core/rust/src/vault_codec/canonicalize.rs@@ -21,7 +21,7 @@ use super::scoped_assets::{normalize_logo_scope, reconcile_logo_references};use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedManifest, CanonicalizedVault, CodecOverflow, DataBucket, Manifest, ManifestSpec, CodecRecord};use super::sharing::{clone_referenced_rows, partition_by_manifest, prune_unreferenced_logos, referenced_tables};use super::types::{- blob_spec_for, bucket_categories, bucket_category_for, is_bucketed_table, is_manifest_scoped, is_skip_table, is_unstamped_scope,+ blob_spec_for, bucket_categories, bucket_category_for, is_bucketed_table, is_skip_table, is_unstamped_scope,manifest_scoped_tables, row_identity, tables_for_category, MANIFEST_ID_COL, OVERFLOW_TABLE, SCHEMA_VERSION,};use crate::error::VaultResult;@@ -68,6 +68,9 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonicadopt_unstamped_rows(&mut all_tables, adopt_into);}reject_unstamped_rows(&all_tables)?;+ for bucket_tables in overflow.bucket_tables.values() {+ reject_unstamped_rows(bucket_tables)?;+ }let bucketed_names: Vec<String> = all_tables.keys().filter(|name| is_bucketed_table(name)).cloned().collect();let bucketed_rows: HashMap<String, Vec<CodecRecord>> = bucketed_names.into_iter().filter_map(|name| all_tables.remove_entry(&name)).collect();@@ -79,11 +82,6 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canoniclet no_rows: Vec<CodecRecord> = Vec::new();let all_logos = snapshots.get("Logos").unwrap_or(&no_rows);- /*- * Route every row to the manifest that owns it; `all_tables` keeps the writing manifest's rows plus- * every table that carries no stamp column. The remaining specs are passed in their original order- * so the emitted list mirrors the input.- */let other_specs: Vec<ManifestSpec> = input.manifests.iter().skip(1).cloned().collect();let partitions = partition_by_manifest(&mut all_tables, &other_specs, &snapshots, &writing_manifest_id)?;@@ -166,18 +164,19 @@ fn build_data_buckets(}}- /// The manifest a bucketed row belongs to.- fn owner<'a>(row: &'a CodecRecord, writing_manifest_id: &'a str, known: &HashSet<&str>) -> Option<&'a str> {+ /// The manifest a bucketed row belongs to: the one its stamp names. A stamp naming a manifest this+ /// vault no longer carries drops the row with that manifest (the partition split's `Gone` rule).+ fn owner<'a>(row: &'a CodecRecord, known: &HashSet<&str>) -> Option<&'a str> {match row.get(MANIFEST_ID_COL).and_then(|value| value.as_str()) {Some(id) if !is_unstamped_scope(Some(id)) => known.contains(id).then_some(id),- _ => Some(writing_manifest_id),+ _ => None,}}for (name, rows) in &bucketed_rows {let category = bucket_category_for(name).expect("only bucketed tables are collected into bucketed_rows");for row in rows {- let Some(manifest_id) = owner(row, writing_manifest_id, &known) else { continue };+ let Some(manifest_id) = owner(row, &known) else { continue };buckets.entry((manifest_id.to_string(), category.to_string())).or_default()@@ -194,7 +193,7 @@ fn build_data_buckets(continue;}for row in rows {- let Some(manifest_id) = owner(row, writing_manifest_id, &known) else { continue };+ let Some(manifest_id) = owner(row, &known) else { continue };buckets.entry((manifest_id.to_string(), category.clone())).or_default()@@ -228,10 +227,12 @@ fn adopt_unstamped_rows(tables: &mut HashMap<String, Vec<CodecRecord>>, manifest}}-/// Reject the whole push when any row of a manifest-scoped table names no manifest.+/// Reject the whole push when any row names no manifest.fn reject_unstamped_rows(tables: &HashMap<String, Vec<CodecRecord>>) -> VaultResult<()> {- for name in manifest_scoped_tables() {- let Some(rows) = tables.get(name) else { continue };+ let mut names: Vec<&String> = tables.keys().collect();+ names.sort();+ for name in names {+ let rows = &tables[name];let unstamped = rows.iter().filter(|row| is_unstamped(row)).count();if unstamped > 0 {let first = rows.iter().find(|row| is_unstamped(row)).and_then(|row| row.get("Id")).cloned().unwrap_or(Value::Null);@@ -328,7 +329,11 @@ fn primary_key_of(identity: &str) -> &str {/// Build a single data bucket for `(manifest_id, category)` from its already-normalized tables/// (name > rows). The bucket-only push path (a bucket changed but the manifest didn't).-pub fn extract_bucket(manifest_id: String, category: String, mut tables: HashMap<String, Vec<CodecRecord>>) -> DataBucket {+///+/// Every row must be stamped for the bucket's own manifest: the caller groups rows by their stamps+/// before calling, so a row naming no manifest or another one is a grouping bug, refused loudly+/// rather than repaired by re-homing the row into a scope it never claimed.+pub fn extract_bucket(manifest_id: String, category: String, mut tables: HashMap<String, Vec<CodecRecord>>) -> VaultResult<DataBucket> {let overflow = tables.remove(OVERFLOW_TABLE).map(|records| CodecOverflow::from_table_records(&records)).unwrap_or_default();remerge_overflow_columns(&mut tables, &overflow);if let Some(ov_tables) = overflow.bucket_tables.get(&category) {@@ -338,16 +343,27 @@ pub fn extract_bucket(manifest_id: String, category: String, mut tables: HashMap}for (name, rows) in tables.iter_mut() {- rows.retain(|row| match row.get(MANIFEST_ID_COL).and_then(|value| value.as_str()) {- Some(id) if !is_unstamped_scope(Some(id)) => id == manifest_id,- _ => true,- });- if is_manifest_scoped(name) {- for row in rows.iter_mut() {- row.insert(MANIFEST_ID_COL.to_string(), json!(manifest_id));+ for row in rows.iter_mut() {+ match row.get(MANIFEST_ID_COL).and_then(|value| value.as_str()) {+ Some(id) if !is_unstamped_scope(Some(id)) => {+ if !id.eq_ignore_ascii_case(&manifest_id) {+ return Err(crate::error::VaultError::General(format!(+ "extract_bucket for manifest {} got a {} row stamped for {}; the caller must group rows by the manifest they name",+ manifest_id, name, id+ )));+ }+ // Normalize the stamp to the bucket's declared spelling of the id.+ row.insert(MANIFEST_ID_COL.to_string(), json!(manifest_id));+ }+ _ => {+ return Err(crate::error::VaultError::General(format!(+ "extract_bucket refuses a {} row that names no manifest; every row must carry the manifest it belongs to",+ name+ )));+ }}}}- DataBucket::new(manifest_id, category, tables)+ Ok(DataBucket::new(manifest_id, category, tables))}diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rsindex a9f72aba6..50aabbda1 100644--- a/core/rust/src/vault_codec/materialize.rs+++ b/core/rust/src/vault_codec/materialize.rs@@ -20,7 +20,7 @@ use std::collections::{HashMap, HashSet};use serde_json::json;use super::manifest::{CodecOverflow, CodecRecord, CodecTableData, Manifest, MaterializeInput, MaterializedTables};-use super::types::{is_manifest_scoped, is_skip_table, row_identity, MANIFEST_ID_COL, MANIFESTS_TABLE, OVERFLOW_TABLE};+use super::types::{is_skip_table, row_identity, MANIFEST_ID_COL, MANIFESTS_TABLE, OVERFLOW_TABLE};use crate::error::{VaultError, VaultResult};/// Materialize the vault's manifests into the table set the platform inserts. Every manifest arrives@@ -70,10 +70,8 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materializeif is_skip_table(&name) || name == OVERFLOW_TABLE {continue;}- if is_manifest_scoped(&name) {- for row in records.iter_mut() {- row.insert(MANIFEST_ID_COL.to_string(), json!(bucket.manifest_id));- }+ for row in records.iter_mut() {+ row.insert(MANIFEST_ID_COL.to_string(), json!(bucket.manifest_id));}match split_for_schema(&name, records, &schema_columns, &mut overflow.columns) {SplitResult::Fits(records) => bucket_tables.entry(name).or_default().extend(records),diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rsindex 0b78f6018..6577d5be3 100644--- a/core/rust/src/vault_codec/mod.rs+++ b/core/rust/src/vault_codec/mod.rs@@ -52,7 +52,7 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize/// Build the `(manifest_id, category)` data bucket from its tables (bucket-only push path)./// Include the [`OVERFLOW_TABLE`] row in `tables` (read it alongside the bucket's tables) so a newer/// writer's columns/tables re-merge and survive; it is consumed and never emitted into the bucket.-pub fn extract_bucket(manifest_id: String, category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>) -> DataBucket {+pub fn extract_bucket(manifest_id: String, category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>) -> VaultResult<DataBucket> {canonicalize::extract_bucket(manifest_id, category, tables)}@@ -216,7 +216,7 @@ pub fn extract_bucket_json(input_json: &str) -> VaultResult<String> {tables: std::collections::HashMap<String, Vec<CodecRecord>>,}let input: Input = serde_json::from_str(input_json)?;- Ok(serde_json::to_string(&extract_bucket(input.manifest_id, input.category, input.tables))?)+ Ok(serde_json::to_string(&extract_bucket(input.manifest_id, input.category, input.tables)?)?)}/// JSON-string sibling of [`bucket_layout`]. Output: `[{ "category": <str>, "tables": [<str>] }]`.diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rsindex 33b8f5c0f..b61d41d5b 100644--- a/core/rust/src/vault_codec/types.rs+++ b/core/rust/src/vault_codec/types.rs@@ -143,7 +143,10 @@ pub fn identity_columns_for(table_name: &str) -> Vec<&'static str> {/// Stable string key identifying `row` within `table_name`.pub fn row_identity(table_name: &str, row: &super::manifest::CodecRecord) -> Option<String> {- let columns = identity_columns_for(table_name);+ let mut columns = identity_columns_for(table_name);+ if !columns.contains(&MANIFEST_ID_COL) && row.get(MANIFEST_ID_COL).filter(|value| !value.is_null()).is_some() {+ columns.insert(0, MANIFEST_ID_COL);+ }// A row with no primary key cannot be addressed at all.row.get(primary_key_for(table_name))?;Some(diff --git a/core/rust/src/wasm.rs b/core/rust/src/wasm.rsindex df1414776..4eaeef80a 100644--- a/core/rust/src/wasm.rs+++ b/core/rust/src/wasm.rs@@ -176,7 +176,9 @@ pub fn vault_codec_extract_bucket_js(input: JsValue) -> Result<JsValue, JsValue>}let input: Input = serde_wasm_bindgen::from_value(input).map_err(|e| JsValue::from_str(&format!("Failed to parse extract-bucket input: {}", e)))?;- codec_to_js(&vault_codec::extract_bucket(input.manifest_id, input.category, input.tables))+ let bucket = vault_codec::extract_bucket(input.manifest_id, input.category, input.tables)+ .map_err(|e| JsValue::from_str(&format!("extract_bucket failed: {}", e)))?;+ codec_to_js(&bucket)}/// The bucket layout: `[{ category, tables: [<name>] }]`. Source of truth for platform bucket-only sync.