Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Refactor personalManifest tracking (#2159)

Leendert de Borst4904f5282526
diff --git a/apps/browser-extension/src/utils/BucketRowGrouping.ts b/apps/browser-extension/src/utils/BucketRowGrouping.ts
index dc9108a57..597e9e388 100644
--- a/apps/browser-extension/src/utils/BucketRowGrouping.ts
+++ b/apps/browser-extension/src/utils/BucketRowGrouping.ts
@@ -46,25 +46,30 @@ export function findByManifestId<T>(map: Map<string, T>, manifestId: string): T
}
/**
- * Split a bucket category's rows into one table set per owning manifest.
+ * Split a bucket category's rows into one table set per owning manifest. Ownership is each row's own
+ * `ManifestId` stamp — the writing manifest is no fallback: a row that names no manifest is a bug in
+ * whatever wrote it, refused loudly here rather than silently homed somewhere plausible.
* @param tableNames - The tables making up the category
* @param tables - The rows read for those tables
- * @param personalManifestId - The manifest rows belong to
+ * @param writingManifestId - The manifest this vault writes from; its bucket is always declared, even empty
* @param alsoInclude - Manifests that must be written even when they hold no rows
* @returns Manifest id → that manifest's rows, per table
*/
export function groupBucketRowsByManifest(
tableNames: string[],
tables: BucketTables,
- personalManifestId: string,
+ writingManifestId: string,
alsoInclude: string[] = []
): Map<string, BucketTables> {
- const grouped = new Map<string, BucketTables>([[personalManifestId, {}]]);
+ const grouped = new Map<string, BucketTables>([[writingManifestId, {}]]);
for (const tableName of tableNames) {
for (const row of tables[tableName] ?? []) {
const stamp = typeof row.ManifestId === 'string' ? row.ManifestId : '';
- const owner = isUnstampedManifestId(stamp) ? personalManifestId : (matchManifestId(grouped.keys(), stamp) ?? stamp);
+ if (isUnstampedManifestId(stamp)) {
+ throw new Error(`Bucket table ${tableName} holds a row that names no manifest; every row must carry the manifest it belongs to`);
+ }
+ const owner = matchManifestId(grouped.keys(), stamp) ?? stamp;
const manifestTables = grouped.get(owner) ?? {};
manifestTables[tableName] = [...(manifestTables[tableName] ?? []), row];
diff --git a/apps/browser-extension/src/utils/VaultSyncService.ts b/apps/browser-extension/src/utils/VaultSyncService.ts
index 5a2c52639..7244c18f8 100644
--- a/apps/browser-extension/src/utils/VaultSyncService.ts
+++ b/apps/browser-extension/src/utils/VaultSyncService.ts
@@ -842,12 +842,7 @@ export class VaultSyncService {
* run earlier (the pre-push no-op check caches its result) and a write in between advances them.
*/
const storedRevisions = await getManifestRevisions();
- const currentManifestRevision = storedRevisions[await this.resolvePersonalManifestId()] ?? 0;
- /**
- * The revision one manifest's write rebases on, falling back to the revision its grant was resolved at.
- * @param record - the manifest record
- */
- const revisionOf = (record: ManifestRecord): number => record.isPersonal ? currentManifestRevision : storedRevisions[record.manifestId] ?? record.revision;
+ const revisionOf = (record: ManifestRecord): number => storedRevisions[record.manifestId] ?? record.revision;
const recordByManifestId = new Map(manifestRecords.map(r => [r.manifestId, r]));
const candidates: PushManifest[] = canonicalized.manifests.flatMap(({ manifest, blobs }): PushManifest[] => {
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rs
index cc701c94b..982ba7660 100644
--- a/core/rust/src/vault_codec/canonicalize.rs
+++ b/core/rust/src/vault_codec/canonicalize.rs
@@ -21,7 +21,7 @@ use super::scoped_assets::{normalize_logo_scope, reconcile_logo_references};
use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedManifest, CanonicalizedVault, CodecOverflow, DataBucket, Manifest, ManifestSpec, CodecRecord};
use super::sharing::{clone_referenced_rows, partition_by_manifest, prune_unreferenced_logos, referenced_tables};
use super::types::{
- blob_spec_for, bucket_categories, bucket_category_for, is_bucketed_table, is_manifest_scoped, is_skip_table, is_unstamped_scope,
+ blob_spec_for, bucket_categories, bucket_category_for, is_bucketed_table, is_skip_table, is_unstamped_scope,
manifest_scoped_tables, row_identity, tables_for_category, MANIFEST_ID_COL, OVERFLOW_TABLE, SCHEMA_VERSION,
};
use crate::error::VaultResult;
@@ -68,6 +68,9 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
adopt_unstamped_rows(&mut all_tables, adopt_into);
}
reject_unstamped_rows(&all_tables)?;
+ for bucket_tables in overflow.bucket_tables.values() {
+ reject_unstamped_rows(bucket_tables)?;
+ }
let bucketed_names: Vec<String> = all_tables.keys().filter(|name| is_bucketed_table(name)).cloned().collect();
let bucketed_rows: HashMap<String, Vec<CodecRecord>> = bucketed_names.into_iter().filter_map(|name| all_tables.remove_entry(&name)).collect();
@@ -79,11 +82,6 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
let no_rows: Vec<CodecRecord> = Vec::new();
let all_logos = snapshots.get("Logos").unwrap_or(&no_rows);
- /*
- * Route every row to the manifest that owns it; `all_tables` keeps the writing manifest's rows plus
- * every table that carries no stamp column. The remaining specs are passed in their original order
- * so the emitted list mirrors the input.
- */
let other_specs: Vec<ManifestSpec> = input.manifests.iter().skip(1).cloned().collect();
let partitions = partition_by_manifest(&mut all_tables, &other_specs, &snapshots, &writing_manifest_id)?;
@@ -166,18 +164,19 @@ fn build_data_buckets(
}
}
- /// The manifest a bucketed row belongs to.
- fn owner<'a>(row: &'a CodecRecord, writing_manifest_id: &'a str, known: &HashSet<&str>) -> Option<&'a str> {
+ /// The manifest a bucketed row belongs to: the one its stamp names. A stamp naming a manifest this
+ /// vault no longer carries drops the row with that manifest (the partition split's `Gone` rule).
+ fn owner<'a>(row: &'a CodecRecord, known: &HashSet<&str>) -> Option<&'a str> {
match row.get(MANIFEST_ID_COL).and_then(|value| value.as_str()) {
Some(id) if !is_unstamped_scope(Some(id)) => known.contains(id).then_some(id),
- _ => Some(writing_manifest_id),
+ _ => None,
}
}
for (name, rows) in &bucketed_rows {
let category = bucket_category_for(name).expect("only bucketed tables are collected into bucketed_rows");
for row in rows {
- let Some(manifest_id) = owner(row, writing_manifest_id, &known) else { continue };
+ let Some(manifest_id) = owner(row, &known) else { continue };
buckets
.entry((manifest_id.to_string(), category.to_string()))
.or_default()
@@ -194,7 +193,7 @@ fn build_data_buckets(
continue;
}
for row in rows {
- let Some(manifest_id) = owner(row, writing_manifest_id, &known) else { continue };
+ let Some(manifest_id) = owner(row, &known) else { continue };
buckets
.entry((manifest_id.to_string(), category.clone()))
.or_default()
@@ -228,10 +227,12 @@ fn adopt_unstamped_rows(tables: &mut HashMap<String, Vec<CodecRecord>>, manifest
}
}
-/// Reject the whole push when any row of a manifest-scoped table names no manifest.
+/// Reject the whole push when any row names no manifest.
fn reject_unstamped_rows(tables: &HashMap<String, Vec<CodecRecord>>) -> VaultResult<()> {
- for name in manifest_scoped_tables() {
- let Some(rows) = tables.get(name) else { continue };
+ let mut names: Vec<&String> = tables.keys().collect();
+ names.sort();
+ for name in names {
+ let rows = &tables[name];
let unstamped = rows.iter().filter(|row| is_unstamped(row)).count();
if unstamped > 0 {
let first = rows.iter().find(|row| is_unstamped(row)).and_then(|row| row.get("Id")).cloned().unwrap_or(Value::Null);
@@ -328,7 +329,11 @@ fn primary_key_of(identity: &str) -> &str {
/// Build a single data bucket for `(manifest_id, category)` from its already-normalized tables
/// (name > rows). The bucket-only push path (a bucket changed but the manifest didn't).
-pub fn extract_bucket(manifest_id: String, category: String, mut tables: HashMap<String, Vec<CodecRecord>>) -> DataBucket {
+///
+/// Every row must be stamped for the bucket's own manifest: the caller groups rows by their stamps
+/// before calling, so a row naming no manifest or another one is a grouping bug, refused loudly
+/// rather than repaired by re-homing the row into a scope it never claimed.
+pub fn extract_bucket(manifest_id: String, category: String, mut tables: HashMap<String, Vec<CodecRecord>>) -> VaultResult<DataBucket> {
let overflow = tables.remove(OVERFLOW_TABLE).map(|records| CodecOverflow::from_table_records(&records)).unwrap_or_default();
remerge_overflow_columns(&mut tables, &overflow);
if let Some(ov_tables) = overflow.bucket_tables.get(&category) {
@@ -338,16 +343,27 @@ pub fn extract_bucket(manifest_id: String, category: String, mut tables: HashMap
}
for (name, rows) in tables.iter_mut() {
- rows.retain(|row| match row.get(MANIFEST_ID_COL).and_then(|value| value.as_str()) {
- Some(id) if !is_unstamped_scope(Some(id)) => id == manifest_id,
- _ => true,
- });
- if is_manifest_scoped(name) {
- for row in rows.iter_mut() {
- row.insert(MANIFEST_ID_COL.to_string(), json!(manifest_id));
+ for row in rows.iter_mut() {
+ match row.get(MANIFEST_ID_COL).and_then(|value| value.as_str()) {
+ Some(id) if !is_unstamped_scope(Some(id)) => {
+ if !id.eq_ignore_ascii_case(&manifest_id) {
+ return Err(crate::error::VaultError::General(format!(
+ "extract_bucket for manifest {} got a {} row stamped for {}; the caller must group rows by the manifest they name",
+ manifest_id, name, id
+ )));
+ }
+ // Normalize the stamp to the bucket's declared spelling of the id.
+ row.insert(MANIFEST_ID_COL.to_string(), json!(manifest_id));
+ }
+ _ => {
+ return Err(crate::error::VaultError::General(format!(
+ "extract_bucket refuses a {} row that names no manifest; every row must carry the manifest it belongs to",
+ name
+ )));
+ }
}
}
}
- DataBucket::new(manifest_id, category, tables)
+ Ok(DataBucket::new(manifest_id, category, tables))
}
diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rs
index a9f72aba6..50aabbda1 100644
--- a/core/rust/src/vault_codec/materialize.rs
+++ b/core/rust/src/vault_codec/materialize.rs
@@ -20,7 +20,7 @@ use std::collections::{HashMap, HashSet};
use serde_json::json;
use super::manifest::{CodecOverflow, CodecRecord, CodecTableData, Manifest, MaterializeInput, MaterializedTables};
-use super::types::{is_manifest_scoped, is_skip_table, row_identity, MANIFEST_ID_COL, MANIFESTS_TABLE, OVERFLOW_TABLE};
+use super::types::{is_skip_table, row_identity, MANIFEST_ID_COL, MANIFESTS_TABLE, OVERFLOW_TABLE};
use crate::error::{VaultError, VaultResult};
/// Materialize the vault's manifests into the table set the platform inserts. Every manifest arrives
@@ -70,10 +70,8 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
if is_skip_table(&name) || name == OVERFLOW_TABLE {
continue;
}
- if is_manifest_scoped(&name) {
- for row in records.iter_mut() {
- row.insert(MANIFEST_ID_COL.to_string(), json!(bucket.manifest_id));
- }
+ for row in records.iter_mut() {
+ row.insert(MANIFEST_ID_COL.to_string(), json!(bucket.manifest_id));
}
match split_for_schema(&name, records, &schema_columns, &mut overflow.columns) {
SplitResult::Fits(records) => bucket_tables.entry(name).or_default().extend(records),
diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rs
index 0b78f6018..6577d5be3 100644
--- a/core/rust/src/vault_codec/mod.rs
+++ b/core/rust/src/vault_codec/mod.rs
@@ -52,7 +52,7 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materialize
/// Build the `(manifest_id, category)` data bucket from its tables (bucket-only push path).
/// Include the [`OVERFLOW_TABLE`] row in `tables` (read it alongside the bucket's tables) so a newer
/// writer's columns/tables re-merge and survive; it is consumed and never emitted into the bucket.
-pub fn extract_bucket(manifest_id: String, category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>) -> DataBucket {
+pub fn extract_bucket(manifest_id: String, category: String, tables: std::collections::HashMap<String, Vec<CodecRecord>>) -> VaultResult<DataBucket> {
canonicalize::extract_bucket(manifest_id, category, tables)
}
@@ -216,7 +216,7 @@ pub fn extract_bucket_json(input_json: &str) -> VaultResult<String> {
tables: std::collections::HashMap<String, Vec<CodecRecord>>,
}
let input: Input = serde_json::from_str(input_json)?;
- Ok(serde_json::to_string(&extract_bucket(input.manifest_id, input.category, input.tables))?)
+ Ok(serde_json::to_string(&extract_bucket(input.manifest_id, input.category, input.tables)?)?)
}
/// JSON-string sibling of [`bucket_layout`]. Output: `[{ "category": <str>, "tables": [<str>] }]`.
diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rs
index 33b8f5c0f..b61d41d5b 100644
--- a/core/rust/src/vault_codec/types.rs
+++ b/core/rust/src/vault_codec/types.rs
@@ -143,7 +143,10 @@ pub fn identity_columns_for(table_name: &str) -> Vec<&'static str> {
/// Stable string key identifying `row` within `table_name`.
pub fn row_identity(table_name: &str, row: &super::manifest::CodecRecord) -> Option<String> {
- let columns = identity_columns_for(table_name);
+ let mut columns = identity_columns_for(table_name);
+ if !columns.contains(&MANIFEST_ID_COL) && row.get(MANIFEST_ID_COL).filter(|value| !value.is_null()).is_some() {
+ columns.insert(0, MANIFEST_ID_COL);
+ }
// A row with no primary key cannot be addressed at all.
row.get(primary_key_for(table_name))?;
Some(
diff --git a/core/rust/src/wasm.rs b/core/rust/src/wasm.rs
index df1414776..4eaeef80a 100644
--- a/core/rust/src/wasm.rs
+++ b/core/rust/src/wasm.rs
@@ -176,7 +176,9 @@ pub fn vault_codec_extract_bucket_js(input: JsValue) -> Result<JsValue, JsValue>
}
let input: Input = serde_wasm_bindgen::from_value(input)
.map_err(|e| JsValue::from_str(&format!("Failed to parse extract-bucket input: {}", e)))?;
- codec_to_js(&vault_codec::extract_bucket(input.manifest_id, input.category, input.tables))
+ let bucket = vault_codec::extract_bucket(input.manifest_id, input.category, input.tables)
+ .map_err(|e| JsValue::from_str(&format!("extract_bucket failed: {}", e)))?;
+ codec_to_js(&bucket)
}
/// The bucket layout: `[{ category, tables: [<name>] }]`. Source of truth for platform bucket-only sync.