Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Add explicit logo dedupe to Rust logic (#2159)
Leendert de Borst66b79d1bee8a
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rsindex 839a23d98..93930a316 100644--- a/core/rust/src/vault_codec/canonicalize.rs+++ b/core/rust/src/vault_codec/canonicalize.rs@@ -17,6 +17,7 @@ use base64::Engine;use serde_json::json;use super::hash::salted_blob_hash;+use super::logos::dedupe_logos_by_source;use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedVault, DataBucket, Manifest, CodecRecord};use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_skip_table, SCHEMA_VERSION};use crate::error::VaultResult;@@ -24,6 +25,21 @@ use crate::error::VaultResult;/// Canonicalize normalized tables into the split resources: the manifest, one data bucket per declared/// category (see [`BUCKET_TABLES`](super::types::BUCKET_TABLES)), and the content-addressed blob map.pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<CanonicalizedVault> {+ // Collect every non-skip table into a name > rows map (row order preserved per table). Blob+ // extraction and bucket-splitting happen below; deduping Logos first (before extraction) means a+ // dropped duplicate never registers an orphan favicon blob.+ let mut all_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();+ for table in &input.tables {+ if is_skip_table(&table.name) {+ continue;+ }+ all_tables.entry(table.name.clone()).or_default().extend(table.records.iter().cloned());+ }++ // Collapse duplicate Logos rows sharing a Source (see `logos` module) and remap Items.LogoId so+ // the manifest we write is free of any UNIQUE(Source) collision that would break materialize.+ dedupe_logos_by_source(&mut all_tables);+let mut blobs: HashMap<String, BlobEntry> = HashMap::new();let mut manifest_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();@@ -34,33 +50,27 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonicbucketed.insert(category.to_string(), HashMap::new());}- for table in &input.tables {- let name = table.name.as_str();- if is_skip_table(name) {- continue;- }-+ for (name, records) in all_tables {// Bucketed table: carried verbatim in its bucket (bucketed tables own no blob columns today).- if let Some(category) = bucket_category_for(name) {+ if let Some(category) = bucket_category_for(&name) {bucketed.get_mut(category).expect("declared bucket category is seeded above")- .insert(name.to_string(), table.records.clone());+ .insert(name, records);continue;}// Manifest table: extract any blob column into the content-addressed map.- let blob_spec = blob_spec_for(name);- let mut out_rows: Vec<CodecRecord> = Vec::with_capacity(table.records.len());- for row in &table.records {- let mut row = row.clone();+ let blob_spec = blob_spec_for(&name);+ let mut out_rows: Vec<CodecRecord> = Vec::with_capacity(records.len());+ for mut row in records {if let Some((_, blob_col, kind)) = blob_spec {let extracted = extract_blob_cell(row.get(*blob_col), &input.user_salt, kind, &mut blobs);row.insert((*blob_col).to_string(), extracted);}out_rows.push(row);}- manifest_tables.insert(name.to_string(), out_rows);+ manifest_tables.insert(name, out_rows);}let manifest = Manifest {diff --git a/core/rust/src/vault_codec/logos.rs b/core/rust/src/vault_codec/logos.rsnew file mode 100644index 000000000..7c4def4fe--- /dev/null+++ b/core/rust/src/vault_codec/logos.rs@@ -0,0 +1,135 @@+//! Logos deduplication: collapse duplicate `Logos` rows that share a `Source`.+//!+//! The client SQLite schema enforces `UNIQUE(Logos.Source)`. This module collapses duplicate `Logos`+//! rows sharing a `Source` down to one survivor each, then reconciles every `Items.LogoId`: repoint a+//! reference to a dropped duplicate at its survivor, and null a reference that resolves to no surviving+//! logo (dangling → `ON DELETE SET NULL`). Mutates the map in place.++use std::collections::{HashMap, HashSet};++use serde_json::{json, Value};++use super::manifest::CodecRecord;++const LOGOS_TABLE: &str = "Logos";+const ITEMS_TABLE: &str = "Items";+const SOURCE_COL: &str = "Source";+const ID_COL: &str = "Id";+const FILE_DATA_COL: &str = "FileData";+const LOGO_ID_COL: &str = "LogoId";++/// Collapse duplicate `Logos` rows sharing a `Source` down to one survivor each, then reconcile every+/// `Items.LogoId`: repoint a reference to a dropped duplicate at its survivor, and null a reference+/// that resolves to no surviving logo (dangling → `ON DELETE SET NULL`). Mutates the map in place.+pub(super) fn dedupe_logos_by_source(tables: &mut HashMap<String, Vec<CodecRecord>>) {+ let remap = collapse_logos(tables);++ // Ids of the logos that exist after the collapse — the set every Items.LogoId must resolve into.+ let valid_ids: HashSet<String> = tables+ .get(LOGOS_TABLE)+ .map(|rows| rows.iter().filter_map(|r| r.get(ID_COL).and_then(|v| v.as_str()).map(String::from)).collect())+ .unwrap_or_default();++ let items = match tables.get_mut(ITEMS_TABLE) {+ Some(rows) => rows,+ None => return,+ };+ for item in items.iter_mut() {+ let current = match item.get(LOGO_ID_COL).and_then(|v| v.as_str()) {+ Some(id) => id.to_string(),+ None => continue, // null/absent LogoId: nothing to reconcile.+ };+ // A dropped-duplicate reference resolves to its survivor; anything else stays as-is for the check.+ let resolved = remap.get(¤t).cloned().unwrap_or(current);+ let repaired = if valid_ids.contains(&resolved) { json!(resolved) } else { Value::Null };+ item.insert(LOGO_ID_COL.to_string(), repaired);+ }+}++/// Reduce the `Logos` table to one row per `Source`, returning the `dropped Id -> survivor Id` map.+/// Rows without a `Source` are left untouched (nothing to collide on).+fn collapse_logos(tables: &mut HashMap<String, Vec<CodecRecord>>) -> HashMap<String, String> {+ let mut remap: HashMap<String, String> = HashMap::new();+ let logos = match tables.get_mut(LOGOS_TABLE) {+ Some(rows) if rows.len() > 1 => rows,+ _ => return remap,+ };++ // Pick the surviving row index per Source (deterministic — see `is_better_logo`).+ let mut survivor_idx: HashMap<String, usize> = HashMap::new();+ for (idx, row) in logos.iter().enumerate() {+ let source = match row.get(SOURCE_COL).and_then(|v| v.as_str()) {+ Some(s) => s.to_string(),+ None => continue,+ };+ match survivor_idx.get(&source) {+ Some(&cur) if !is_better_logo(row, &logos[cur]) => {}+ _ => {+ survivor_idx.insert(source, idx);+ }+ }+ }++ let survivors: HashSet<usize> = survivor_idx.values().copied().collect();+ if survivors.len() == logos.len() {+ // Every row is a survivor: no Source collisions.+ return remap;+ }++ // Map each surviving Source to its Id so dropped rows can be repointed.+ let mut survivor_id_by_source: HashMap<String, String> = HashMap::new();+ for (source, &idx) in &survivor_idx {+ if let Some(id) = logos[idx].get(ID_COL).and_then(|v| v.as_str()) {+ survivor_id_by_source.insert(source.clone(), id.to_string());+ }+ }++ for (idx, row) in logos.iter().enumerate() {+ if survivors.contains(&idx) {+ continue;+ }+ let source = match row.get(SOURCE_COL).and_then(|v| v.as_str()) {+ Some(s) => s,+ None => continue,+ };+ let dropped_id = match row.get(ID_COL).and_then(|v| v.as_str()) {+ Some(s) => s,+ None => continue,+ };+ if let Some(survivor_id) = survivor_id_by_source.get(source) {+ if survivor_id != dropped_id {+ remap.insert(dropped_id.to_string(), survivor_id.clone());+ }+ }+ }++ // Keep only survivors, preserving original row order.+ let kept: Vec<CodecRecord> = logos+ .iter()+ .enumerate()+ .filter(|(idx, _)| survivors.contains(idx))+ .map(|(_, row)| row.clone())+ .collect();+ *logos = kept;++ remap+}++/// Total order used to choose which row survives a `Source` collision: prefer a row that actually+/// carries favicon bytes, then the lexicographically-highest (newest) `Id`.+fn is_better_logo(candidate: &CodecRecord, incumbent: &CodecRecord) -> bool {+ let cand_has_data = has_file_data(candidate);+ let inc_has_data = has_file_data(incumbent);+ if cand_has_data != inc_has_data {+ return cand_has_data;+ }+ let cand_id = candidate.get(ID_COL).and_then(|v| v.as_str()).unwrap_or("");+ let inc_id = incumbent.get(ID_COL).and_then(|v| v.as_str()).unwrap_or("");+ cand_id > inc_id+}++/// True when `FileData` holds actual bytes (an inline `{ __b64 }` or extracted `{ __blobRef }`),+/// i.e. not absent and not JSON null.+fn has_file_data(row: &CodecRecord) -> bool {+ matches!(row.get(FILE_DATA_COL), Some(v) if !v.is_null())+}diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rsindex e606055fa..f6d661ed4 100644--- a/core/rust/src/vault_codec/mod.rs+++ b/core/rust/src/vault_codec/mod.rs@@ -10,6 +10,7 @@mod compress;mod canonicalize;mod hash;+mod logos;mod manifest;mod materialize;mod types;diff --git a/core/rust/src/vault_codec/tests.rs b/core/rust/src/vault_codec/tests.rsindex 44722ea2e..1285b63c7 100644--- a/core/rust/src/vault_codec/tests.rs+++ b/core/rust/src/vault_codec/tests.rs@@ -255,6 +255,129 @@ fn forward_compat_unknown_manifest_fields_preserved() {assert_eq!(reser["futureField"], json!({ "nested": true }));}+/// Rows of `table` in the materialized output (empty if absent).+fn materialized_rows<'a>(m: &'a MaterializedTables, table: &str) -> &'a [CodecRecord] {+ m.tables.iter().find(|t| t.name == table).map(|t| t.records.as_slice()).unwrap_or(&[])+}++#[test]+fn canonicalize_dedupes_duplicate_logo_sources_and_remaps_items() {+ // Two clients minted distinct Ids for the same domain; Items point at each. Canonicalize must+ // collapse to one Logos row and repoint the orphaned Item at the survivor.+ let favicon = vec![0x01, 0x02, 0x03];+ let out = canonicalize_from_sqlite(basic_input(vec![+ CodecTableData {+ name: "Logos".to_string(),+ records: vec![+ row(&[("Id", json!("logo-a")), ("Source", json!("github.com")), ("FileData", json!({ "__b64": b64(&favicon) }))]),+ row(&[("Id", json!("logo-b")), ("Source", json!("github.com")), ("FileData", serde_json::Value::Null)]),+ ],+ },+ CodecTableData {+ name: "Items".to_string(),+ records: vec![+ row(&[("Id", json!("i1")), ("LogoId", json!("logo-a"))]),+ row(&[("Id", json!("i2")), ("LogoId", json!("logo-b"))]),+ ],+ },+ ]))+ .unwrap();++ let logos = &out.manifest.tables["Logos"];+ assert_eq!(logos.len(), 1, "duplicate Source collapsed to one row");+ // Survivor is the row that actually carries favicon bytes (logo-a), not the empty one.+ assert_eq!(logos[0]["Id"], json!("logo-a"));+ assert!(logos[0]["FileData"].get("__blobRef").is_some());++ let items = &out.manifest.tables["Items"];+ let i1 = items.iter().find(|r| r["Id"] == json!("i1")).unwrap();+ let i2 = items.iter().find(|r| r["Id"] == json!("i2")).unwrap();+ assert_eq!(i1["LogoId"], json!("logo-a"));+ assert_eq!(i2["LogoId"], json!("logo-a"), "Item pointing at the dropped duplicate is remapped");++ // No orphan blob: exactly the survivor's favicon is registered.+ assert_eq!(out.blobs.len(), 1);+}++#[test]+fn materialize_dedupes_duplicate_logo_sources_from_legacy_manifest() {+ // A manifest already on the server carrying duplicate Source rows must materialize cleanly+ // (self-heal on pull) rather than exploding on the UNIQUE(Source) index.+ let mut manifest = canonicalize_from_sqlite(basic_input(vec![+ CodecTableData { name: "Items".to_string(), records: vec![] },+ ]))+ .unwrap()+ .manifest;+ manifest.tables.insert(+ "Logos".to_string(),+ vec![+ row(&[("Id", json!("logo-a")), ("Source", json!("github.com"))]),+ row(&[("Id", json!("logo-b")), ("Source", json!("github.com"))]),+ ],+ );+ manifest.tables.insert(+ "Items".to_string(),+ vec![row(&[("Id", json!("i1")), ("LogoId", json!("logo-b"))])],+ );++ let re = materialize_as_sqlite(MaterializeInput { manifest, data_buckets: vec![] }).unwrap();+ let logos = materialized_rows(&re, "Logos");+ assert_eq!(logos.len(), 1, "duplicate Source collapsed on materialize");+ let survivor = logos[0]["Id"].as_str().unwrap().to_string();+ // The Item's LogoId now points at whichever row survived (deterministically the smaller Id, logo-a).+ assert_eq!(survivor, "logo-a");+ assert_eq!(materialized_rows(&re, "Items")[0]["LogoId"], json!("logo-a"));+}++#[test]+fn materialize_nulls_dangling_logo_reference() {+ // An Item pointing at a logo Id that no longer exists (e.g. collapsed away by a cross-client merge)+ // is nulled — matching FK_Items_Logos_LogoId ON DELETE SET NULL — so foreign_key_check passes.+ let mut manifest = canonicalize_from_sqlite(basic_input(vec![+ CodecTableData { name: "Items".to_string(), records: vec![] },+ ]))+ .unwrap()+ .manifest;+ manifest.tables.insert(+ "Logos".to_string(),+ vec![row(&[("Id", json!("logo-a")), ("Source", json!("github.com"))])],+ );+ manifest.tables.insert(+ "Items".to_string(),+ vec![+ row(&[("Id", json!("i1")), ("LogoId", json!("logo-a"))]),+ row(&[("Id", json!("i2")), ("LogoId", json!("ghost"))]),+ ],+ );++ let re = materialize_as_sqlite(MaterializeInput { manifest, data_buckets: vec![] }).unwrap();+ let items = materialized_rows(&re, "Items");+ let i1 = items.iter().find(|r| r["Id"] == json!("i1")).unwrap();+ let i2 = items.iter().find(|r| r["Id"] == json!("i2")).unwrap();+ assert_eq!(i1["LogoId"], json!("logo-a"), "valid reference untouched");+ assert_eq!(i2["LogoId"], serde_json::Value::Null, "dangling reference nulled");+}++#[test]+fn validate_manifest_rejects_duplicate_logo_sources() {+ // Guard: if a duplicate-Source manifest ever reaches validation (dedup bypassed), reject it.+ let mut manifest = canonicalize_from_sqlite(basic_input(vec![+ CodecTableData { name: "Items".to_string(), records: vec![] },+ ]))+ .unwrap()+ .manifest;+ manifest.tables.insert(+ "Logos".to_string(),+ vec![+ row(&[("Id", json!("logo-a")), ("Source", json!("github.com"))]),+ row(&[("Id", json!("logo-b")), ("Source", json!("github.com"))]),+ ],+ );+ let result = validate_manifest(&manifest);+ assert!(!result.ok);+ assert!(result.failed_rules.iter().any(|r| r == "logo-sources-not-unique"));+}+#[test]fn json_siblings_roundtrip() {let input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }]);diff --git a/core/rust/src/vault_codec/validate.rs b/core/rust/src/vault_codec/validate.rsindex e01b5d49c..e7263e378 100644--- a/core/rust/src/vault_codec/validate.rs+++ b/core/rust/src/vault_codec/validate.rs@@ -120,6 +120,15 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {failed.push("folder-ids-not-unique".to_string());}+ // Logos.Source is UNIQUE in the client schema (IX_Logos_Source); a duplicate would pass through+ // canonicalize but fail on materialize. Refuse the upload rather than write a manifest no client can load.+ let logos = table(manifest, "Logos");+ let logo_sources: std::collections::HashSet<&str> = logos.iter().filter_map(|l| str_field(l, "Source")).collect();+ let logos_with_source = logos.iter().filter(|l| str_field(l, "Source").is_some()).count();+ if logo_sources.len() != logos_with_source {+ failed.push("logo-sources-not-unique".to_string());+ }+ValidationResult {ok: failed.is_empty(),failed_rules: failed,diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rsindex 7d10c48f4..698dfe2b3 100644--- a/core/rust/src/vault_merge/mod.rs+++ b/core/rust/src/vault_merge/mod.rs@@ -472,6 +472,41 @@ mod tests {assert!(output.statements[0].sql.starts_with("UPDATE Items SET"));}+ fn logo(id: &str, source: &str, updated_at: &str) -> Record {+ let mut r = HashMap::new();+ r.insert("Id".to_string(), serde_json::json!(id));+ r.insert("Source".to_string(), serde_json::json!(source));+ r.insert("UpdatedAt".to_string(), serde_json::json!(updated_at));+ r+ }++ #[test]+ fn logos_merge_by_source_collapses_distinct_ids_and_keeps_local_id() {+ // Two clients minted different Ids for github.com. Merging by Source (server newer) must UPDATE+ // the local row in place — keeping the local Id so Items.LogoId stays valid — not INSERT a+ // second same-Source row (which would violate UNIQUE(Source) / break materialize).+ let local = vec![logo("local-id", "github.com", "2024-01-01T00:00:00Z")];+ let server = vec![logo("server-id", "github.com", "2024-01-02T00:00:00Z")];+ let mut stats = MergeStats::default();++ let statements = merge_table_by_composite_key("Logos", &local, &server, &["Source"], &mut stats);++ assert_eq!(stats.conflicts, 1);+ assert_eq!(stats.records_inserted, 0, "no second same-Source row inserted");+ assert_eq!(statements.len(), 1);+ assert!(statements[0].sql.starts_with("UPDATE Logos SET"));+ assert!(statements[0].sql.ends_with("WHERE Id = ?"));+ // WHERE-clause Id is the LOCAL id (last param), so the local row is updated in place.+ assert_eq!(statements[0].params.last().unwrap(), &serde_json::json!("local-id"));+ }++ #[test]+ fn logos_config_uses_source_composite_key() {+ let cfg = SYNCABLE_TABLES.iter().find(|t| t.name == "Logos").unwrap();+ assert!(cfg.uses_composite_key());+ assert_eq!(cfg.composite_key_columns, &["Source"]);+ }+#[test]fn test_generate_insert_sql() {let record = make_record("test-id", "2024-01-01T00:00:00Z");diff --git a/core/rust/src/vault_merge/types.rs b/core/rust/src/vault_merge/types.rsindex dfffb82fe..ed27cd799 100644--- a/core/rust/src/vault_merge/types.rs+++ b/core/rust/src/vault_merge/types.rs@@ -32,6 +32,10 @@ impl TableConfig {/// All tables that need LWW merge./// FieldValues uses composite key (ItemId + FieldKey) for merging.+/// Logos uses composite key (Source): Id is a per-client random GUID but Source is the natural unique+/// key (schema enforces UNIQUE(Source)). Merging by Source collapses two clients' rows for the same+/// domain into one — and the composite-key path keeps the local Id on conflict, so Items.LogoId stays+/// valid — instead of the by-Id path minting a second same-Source row that breaks materialize.pub static SYNCABLE_TABLES: &[TableConfig] = &[TableConfig::new("Items"),TableConfig::new("FieldValues").with_composite_key(&["ItemId", "FieldKey"]),@@ -43,7 +47,7 @@ pub static SYNCABLE_TABLES: &[TableConfig] = &[TableConfig::new("Passkeys"),TableConfig::new("FieldDefinitions"),TableConfig::new("FieldHistories"),- TableConfig::new("Logos"),+ TableConfig::new("Logos").with_composite_key(&["Source"]),];/// List of syncable table names (for clients to know which tables to read).