Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Add explicit logo dedupe to Rust logic (#2159)

Leendert de Borst66b79d1bee8a
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rs
index 839a23d98..93930a316 100644
--- a/core/rust/src/vault_codec/canonicalize.rs
+++ b/core/rust/src/vault_codec/canonicalize.rs
@@ -17,6 +17,7 @@ use base64::Engine;
use serde_json::json;
use super::hash::salted_blob_hash;
+use super::logos::dedupe_logos_by_source;
use super::manifest::{BlobEntry, CanonicalizeInput, CanonicalizedVault, DataBucket, Manifest, CodecRecord};
use super::types::{blob_spec_for, bucket_categories, bucket_category_for, is_skip_table, SCHEMA_VERSION};
use crate::error::VaultResult;
@@ -24,6 +25,21 @@ use crate::error::VaultResult;
/// Canonicalize normalized tables into the split resources: the manifest, one data bucket per declared
/// category (see [`BUCKET_TABLES`](super::types::BUCKET_TABLES)), and the content-addressed blob map.
pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<CanonicalizedVault> {
+ // Collect every non-skip table into a name > rows map (row order preserved per table). Blob
+ // extraction and bucket-splitting happen below; deduping Logos first (before extraction) means a
+ // dropped duplicate never registers an orphan favicon blob.
+ let mut all_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();
+ for table in &input.tables {
+ if is_skip_table(&table.name) {
+ continue;
+ }
+ all_tables.entry(table.name.clone()).or_default().extend(table.records.iter().cloned());
+ }
+
+ // Collapse duplicate Logos rows sharing a Source (see `logos` module) and remap Items.LogoId so
+ // the manifest we write is free of any UNIQUE(Source) collision that would break materialize.
+ dedupe_logos_by_source(&mut all_tables);
+
let mut blobs: HashMap<String, BlobEntry> = HashMap::new();
let mut manifest_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();
@@ -34,33 +50,27 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
bucketed.insert(category.to_string(), HashMap::new());
}
- for table in &input.tables {
- let name = table.name.as_str();
- if is_skip_table(name) {
- continue;
- }
-
+ for (name, records) in all_tables {
// Bucketed table: carried verbatim in its bucket (bucketed tables own no blob columns today).
- if let Some(category) = bucket_category_for(name) {
+ if let Some(category) = bucket_category_for(&name) {
bucketed
.get_mut(category)
.expect("declared bucket category is seeded above")
- .insert(name.to_string(), table.records.clone());
+ .insert(name, records);
continue;
}
// Manifest table: extract any blob column into the content-addressed map.
- let blob_spec = blob_spec_for(name);
- let mut out_rows: Vec<CodecRecord> = Vec::with_capacity(table.records.len());
- for row in &table.records {
- let mut row = row.clone();
+ let blob_spec = blob_spec_for(&name);
+ let mut out_rows: Vec<CodecRecord> = Vec::with_capacity(records.len());
+ for mut row in records {
if let Some((_, blob_col, kind)) = blob_spec {
let extracted = extract_blob_cell(row.get(*blob_col), &input.user_salt, kind, &mut blobs);
row.insert((*blob_col).to_string(), extracted);
}
out_rows.push(row);
}
- manifest_tables.insert(name.to_string(), out_rows);
+ manifest_tables.insert(name, out_rows);
}
let manifest = Manifest {
diff --git a/core/rust/src/vault_codec/logos.rs b/core/rust/src/vault_codec/logos.rs
new file mode 100644
index 000000000..7c4def4fe
--- /dev/null
+++ b/core/rust/src/vault_codec/logos.rs
@@ -0,0 +1,135 @@
+//! Logos deduplication: collapse duplicate `Logos` rows that share a `Source`.
+//!
+//! The client SQLite schema enforces `UNIQUE(Logos.Source)`. This module collapses duplicate `Logos`
+//! rows sharing a `Source` down to one survivor each, then reconciles every `Items.LogoId`: repoint a
+//! reference to a dropped duplicate at its survivor, and null a reference that resolves to no surviving
+//! logo (dangling → `ON DELETE SET NULL`). Mutates the map in place.
+
+use std::collections::{HashMap, HashSet};
+
+use serde_json::{json, Value};
+
+use super::manifest::CodecRecord;
+
+const LOGOS_TABLE: &str = "Logos";
+const ITEMS_TABLE: &str = "Items";
+const SOURCE_COL: &str = "Source";
+const ID_COL: &str = "Id";
+const FILE_DATA_COL: &str = "FileData";
+const LOGO_ID_COL: &str = "LogoId";
+
+/// Collapse duplicate `Logos` rows sharing a `Source` down to one survivor each, then reconcile every
+/// `Items.LogoId`: repoint a reference to a dropped duplicate at its survivor, and null a reference
+/// that resolves to no surviving logo (dangling → `ON DELETE SET NULL`). Mutates the map in place.
+pub(super) fn dedupe_logos_by_source(tables: &mut HashMap<String, Vec<CodecRecord>>) {
+ let remap = collapse_logos(tables);
+
+ // Ids of the logos that exist after the collapse — the set every Items.LogoId must resolve into.
+ let valid_ids: HashSet<String> = tables
+ .get(LOGOS_TABLE)
+ .map(|rows| rows.iter().filter_map(|r| r.get(ID_COL).and_then(|v| v.as_str()).map(String::from)).collect())
+ .unwrap_or_default();
+
+ let items = match tables.get_mut(ITEMS_TABLE) {
+ Some(rows) => rows,
+ None => return,
+ };
+ for item in items.iter_mut() {
+ let current = match item.get(LOGO_ID_COL).and_then(|v| v.as_str()) {
+ Some(id) => id.to_string(),
+ None => continue, // null/absent LogoId: nothing to reconcile.
+ };
+ // A dropped-duplicate reference resolves to its survivor; anything else stays as-is for the check.
+ let resolved = remap.get(&current).cloned().unwrap_or(current);
+ let repaired = if valid_ids.contains(&resolved) { json!(resolved) } else { Value::Null };
+ item.insert(LOGO_ID_COL.to_string(), repaired);
+ }
+}
+
+/// Reduce the `Logos` table to one row per `Source`, returning the `dropped Id -> survivor Id` map.
+/// Rows without a `Source` are left untouched (nothing to collide on).
+fn collapse_logos(tables: &mut HashMap<String, Vec<CodecRecord>>) -> HashMap<String, String> {
+ let mut remap: HashMap<String, String> = HashMap::new();
+ let logos = match tables.get_mut(LOGOS_TABLE) {
+ Some(rows) if rows.len() > 1 => rows,
+ _ => return remap,
+ };
+
+ // Pick the surviving row index per Source (deterministic — see `is_better_logo`).
+ let mut survivor_idx: HashMap<String, usize> = HashMap::new();
+ for (idx, row) in logos.iter().enumerate() {
+ let source = match row.get(SOURCE_COL).and_then(|v| v.as_str()) {
+ Some(s) => s.to_string(),
+ None => continue,
+ };
+ match survivor_idx.get(&source) {
+ Some(&cur) if !is_better_logo(row, &logos[cur]) => {}
+ _ => {
+ survivor_idx.insert(source, idx);
+ }
+ }
+ }
+
+ let survivors: HashSet<usize> = survivor_idx.values().copied().collect();
+ if survivors.len() == logos.len() {
+ // Every row is a survivor: no Source collisions.
+ return remap;
+ }
+
+ // Map each surviving Source to its Id so dropped rows can be repointed.
+ let mut survivor_id_by_source: HashMap<String, String> = HashMap::new();
+ for (source, &idx) in &survivor_idx {
+ if let Some(id) = logos[idx].get(ID_COL).and_then(|v| v.as_str()) {
+ survivor_id_by_source.insert(source.clone(), id.to_string());
+ }
+ }
+
+ for (idx, row) in logos.iter().enumerate() {
+ if survivors.contains(&idx) {
+ continue;
+ }
+ let source = match row.get(SOURCE_COL).and_then(|v| v.as_str()) {
+ Some(s) => s,
+ None => continue,
+ };
+ let dropped_id = match row.get(ID_COL).and_then(|v| v.as_str()) {
+ Some(s) => s,
+ None => continue,
+ };
+ if let Some(survivor_id) = survivor_id_by_source.get(source) {
+ if survivor_id != dropped_id {
+ remap.insert(dropped_id.to_string(), survivor_id.clone());
+ }
+ }
+ }
+
+ // Keep only survivors, preserving original row order.
+ let kept: Vec<CodecRecord> = logos
+ .iter()
+ .enumerate()
+ .filter(|(idx, _)| survivors.contains(idx))
+ .map(|(_, row)| row.clone())
+ .collect();
+ *logos = kept;
+
+ remap
+}
+
+/// Total order used to choose which row survives a `Source` collision: prefer a row that actually
+/// carries favicon bytes, then the lexicographically-highest (newest) `Id`.
+fn is_better_logo(candidate: &CodecRecord, incumbent: &CodecRecord) -> bool {
+ let cand_has_data = has_file_data(candidate);
+ let inc_has_data = has_file_data(incumbent);
+ if cand_has_data != inc_has_data {
+ return cand_has_data;
+ }
+ let cand_id = candidate.get(ID_COL).and_then(|v| v.as_str()).unwrap_or("");
+ let inc_id = incumbent.get(ID_COL).and_then(|v| v.as_str()).unwrap_or("");
+ cand_id > inc_id
+}
+
+/// True when `FileData` holds actual bytes (an inline `{ __b64 }` or extracted `{ __blobRef }`),
+/// i.e. not absent and not JSON null.
+fn has_file_data(row: &CodecRecord) -> bool {
+ matches!(row.get(FILE_DATA_COL), Some(v) if !v.is_null())
+}
diff --git a/core/rust/src/vault_codec/mod.rs b/core/rust/src/vault_codec/mod.rs
index e606055fa..f6d661ed4 100644
--- a/core/rust/src/vault_codec/mod.rs
+++ b/core/rust/src/vault_codec/mod.rs
@@ -10,6 +10,7 @@
mod compress;
mod canonicalize;
mod hash;
+mod logos;
mod manifest;
mod materialize;
mod types;
diff --git a/core/rust/src/vault_codec/tests.rs b/core/rust/src/vault_codec/tests.rs
index 44722ea2e..1285b63c7 100644
--- a/core/rust/src/vault_codec/tests.rs
+++ b/core/rust/src/vault_codec/tests.rs
@@ -255,6 +255,129 @@ fn forward_compat_unknown_manifest_fields_preserved() {
assert_eq!(reser["futureField"], json!({ "nested": true }));
}
+/// Rows of `table` in the materialized output (empty if absent).
+fn materialized_rows<'a>(m: &'a MaterializedTables, table: &str) -> &'a [CodecRecord] {
+ m.tables.iter().find(|t| t.name == table).map(|t| t.records.as_slice()).unwrap_or(&[])
+}
+
+#[test]
+fn canonicalize_dedupes_duplicate_logo_sources_and_remaps_items() {
+ // Two clients minted distinct Ids for the same domain; Items point at each. Canonicalize must
+ // collapse to one Logos row and repoint the orphaned Item at the survivor.
+ let favicon = vec![0x01, 0x02, 0x03];
+ let out = canonicalize_from_sqlite(basic_input(vec![
+ CodecTableData {
+ name: "Logos".to_string(),
+ records: vec![
+ row(&[("Id", json!("logo-a")), ("Source", json!("github.com")), ("FileData", json!({ "__b64": b64(&favicon) }))]),
+ row(&[("Id", json!("logo-b")), ("Source", json!("github.com")), ("FileData", serde_json::Value::Null)]),
+ ],
+ },
+ CodecTableData {
+ name: "Items".to_string(),
+ records: vec![
+ row(&[("Id", json!("i1")), ("LogoId", json!("logo-a"))]),
+ row(&[("Id", json!("i2")), ("LogoId", json!("logo-b"))]),
+ ],
+ },
+ ]))
+ .unwrap();
+
+ let logos = &out.manifest.tables["Logos"];
+ assert_eq!(logos.len(), 1, "duplicate Source collapsed to one row");
+ // Survivor is the row that actually carries favicon bytes (logo-a), not the empty one.
+ assert_eq!(logos[0]["Id"], json!("logo-a"));
+ assert!(logos[0]["FileData"].get("__blobRef").is_some());
+
+ let items = &out.manifest.tables["Items"];
+ let i1 = items.iter().find(|r| r["Id"] == json!("i1")).unwrap();
+ let i2 = items.iter().find(|r| r["Id"] == json!("i2")).unwrap();
+ assert_eq!(i1["LogoId"], json!("logo-a"));
+ assert_eq!(i2["LogoId"], json!("logo-a"), "Item pointing at the dropped duplicate is remapped");
+
+ // No orphan blob: exactly the survivor's favicon is registered.
+ assert_eq!(out.blobs.len(), 1);
+}
+
+#[test]
+fn materialize_dedupes_duplicate_logo_sources_from_legacy_manifest() {
+ // A manifest already on the server carrying duplicate Source rows must materialize cleanly
+ // (self-heal on pull) rather than exploding on the UNIQUE(Source) index.
+ let mut manifest = canonicalize_from_sqlite(basic_input(vec![
+ CodecTableData { name: "Items".to_string(), records: vec![] },
+ ]))
+ .unwrap()
+ .manifest;
+ manifest.tables.insert(
+ "Logos".to_string(),
+ vec![
+ row(&[("Id", json!("logo-a")), ("Source", json!("github.com"))]),
+ row(&[("Id", json!("logo-b")), ("Source", json!("github.com"))]),
+ ],
+ );
+ manifest.tables.insert(
+ "Items".to_string(),
+ vec![row(&[("Id", json!("i1")), ("LogoId", json!("logo-b"))])],
+ );
+
+ let re = materialize_as_sqlite(MaterializeInput { manifest, data_buckets: vec![] }).unwrap();
+ let logos = materialized_rows(&re, "Logos");
+ assert_eq!(logos.len(), 1, "duplicate Source collapsed on materialize");
+ let survivor = logos[0]["Id"].as_str().unwrap().to_string();
+ // The Item's LogoId now points at whichever row survived (deterministically the smaller Id, logo-a).
+ assert_eq!(survivor, "logo-a");
+ assert_eq!(materialized_rows(&re, "Items")[0]["LogoId"], json!("logo-a"));
+}
+
+#[test]
+fn materialize_nulls_dangling_logo_reference() {
+ // An Item pointing at a logo Id that no longer exists (e.g. collapsed away by a cross-client merge)
+ // is nulled — matching FK_Items_Logos_LogoId ON DELETE SET NULL — so foreign_key_check passes.
+ let mut manifest = canonicalize_from_sqlite(basic_input(vec![
+ CodecTableData { name: "Items".to_string(), records: vec![] },
+ ]))
+ .unwrap()
+ .manifest;
+ manifest.tables.insert(
+ "Logos".to_string(),
+ vec![row(&[("Id", json!("logo-a")), ("Source", json!("github.com"))])],
+ );
+ manifest.tables.insert(
+ "Items".to_string(),
+ vec![
+ row(&[("Id", json!("i1")), ("LogoId", json!("logo-a"))]),
+ row(&[("Id", json!("i2")), ("LogoId", json!("ghost"))]),
+ ],
+ );
+
+ let re = materialize_as_sqlite(MaterializeInput { manifest, data_buckets: vec![] }).unwrap();
+ let items = materialized_rows(&re, "Items");
+ let i1 = items.iter().find(|r| r["Id"] == json!("i1")).unwrap();
+ let i2 = items.iter().find(|r| r["Id"] == json!("i2")).unwrap();
+ assert_eq!(i1["LogoId"], json!("logo-a"), "valid reference untouched");
+ assert_eq!(i2["LogoId"], serde_json::Value::Null, "dangling reference nulled");
+}
+
+#[test]
+fn validate_manifest_rejects_duplicate_logo_sources() {
+ // Guard: if a duplicate-Source manifest ever reaches validation (dedup bypassed), reject it.
+ let mut manifest = canonicalize_from_sqlite(basic_input(vec![
+ CodecTableData { name: "Items".to_string(), records: vec![] },
+ ]))
+ .unwrap()
+ .manifest;
+ manifest.tables.insert(
+ "Logos".to_string(),
+ vec![
+ row(&[("Id", json!("logo-a")), ("Source", json!("github.com"))]),
+ row(&[("Id", json!("logo-b")), ("Source", json!("github.com"))]),
+ ],
+ );
+ let result = validate_manifest(&manifest);
+ assert!(!result.ok);
+ assert!(result.failed_rules.iter().any(|r| r == "logo-sources-not-unique"));
+}
+
#[test]
fn json_siblings_roundtrip() {
let input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }]);
diff --git a/core/rust/src/vault_codec/validate.rs b/core/rust/src/vault_codec/validate.rs
index e01b5d49c..e7263e378 100644
--- a/core/rust/src/vault_codec/validate.rs
+++ b/core/rust/src/vault_codec/validate.rs
@@ -120,6 +120,15 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {
failed.push("folder-ids-not-unique".to_string());
}
+ // Logos.Source is UNIQUE in the client schema (IX_Logos_Source); a duplicate would pass through
+ // canonicalize but fail on materialize. Refuse the upload rather than write a manifest no client can load.
+ let logos = table(manifest, "Logos");
+ let logo_sources: std::collections::HashSet<&str> = logos.iter().filter_map(|l| str_field(l, "Source")).collect();
+ let logos_with_source = logos.iter().filter(|l| str_field(l, "Source").is_some()).count();
+ if logo_sources.len() != logos_with_source {
+ failed.push("logo-sources-not-unique".to_string());
+ }
+
ValidationResult {
ok: failed.is_empty(),
failed_rules: failed,
diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rs
index 7d10c48f4..698dfe2b3 100644
--- a/core/rust/src/vault_merge/mod.rs
+++ b/core/rust/src/vault_merge/mod.rs
@@ -472,6 +472,41 @@ mod tests {
assert!(output.statements[0].sql.starts_with("UPDATE Items SET"));
}
+ fn logo(id: &str, source: &str, updated_at: &str) -> Record {
+ let mut r = HashMap::new();
+ r.insert("Id".to_string(), serde_json::json!(id));
+ r.insert("Source".to_string(), serde_json::json!(source));
+ r.insert("UpdatedAt".to_string(), serde_json::json!(updated_at));
+ r
+ }
+
+ #[test]
+ fn logos_merge_by_source_collapses_distinct_ids_and_keeps_local_id() {
+ // Two clients minted different Ids for github.com. Merging by Source (server newer) must UPDATE
+ // the local row in place — keeping the local Id so Items.LogoId stays valid — not INSERT a
+ // second same-Source row (which would violate UNIQUE(Source) / break materialize).
+ let local = vec![logo("local-id", "github.com", "2024-01-01T00:00:00Z")];
+ let server = vec![logo("server-id", "github.com", "2024-01-02T00:00:00Z")];
+ let mut stats = MergeStats::default();
+
+ let statements = merge_table_by_composite_key("Logos", &local, &server, &["Source"], &mut stats);
+
+ assert_eq!(stats.conflicts, 1);
+ assert_eq!(stats.records_inserted, 0, "no second same-Source row inserted");
+ assert_eq!(statements.len(), 1);
+ assert!(statements[0].sql.starts_with("UPDATE Logos SET"));
+ assert!(statements[0].sql.ends_with("WHERE Id = ?"));
+ // WHERE-clause Id is the LOCAL id (last param), so the local row is updated in place.
+ assert_eq!(statements[0].params.last().unwrap(), &serde_json::json!("local-id"));
+ }
+
+ #[test]
+ fn logos_config_uses_source_composite_key() {
+ let cfg = SYNCABLE_TABLES.iter().find(|t| t.name == "Logos").unwrap();
+ assert!(cfg.uses_composite_key());
+ assert_eq!(cfg.composite_key_columns, &["Source"]);
+ }
+
#[test]
fn test_generate_insert_sql() {
let record = make_record("test-id", "2024-01-01T00:00:00Z");
diff --git a/core/rust/src/vault_merge/types.rs b/core/rust/src/vault_merge/types.rs
index dfffb82fe..ed27cd799 100644
--- a/core/rust/src/vault_merge/types.rs
+++ b/core/rust/src/vault_merge/types.rs
@@ -32,6 +32,10 @@ impl TableConfig {
/// All tables that need LWW merge.
/// FieldValues uses composite key (ItemId + FieldKey) for merging.
+/// Logos uses composite key (Source): Id is a per-client random GUID but Source is the natural unique
+/// key (schema enforces UNIQUE(Source)). Merging by Source collapses two clients' rows for the same
+/// domain into one — and the composite-key path keeps the local Id on conflict, so Items.LogoId stays
+/// valid — instead of the by-Id path minting a second same-Source row that breaks materialize.
pub static SYNCABLE_TABLES: &[TableConfig] = &[
TableConfig::new("Items"),
TableConfig::new("FieldValues").with_composite_key(&["ItemId", "FieldKey"]),
@@ -43,7 +47,7 @@ pub static SYNCABLE_TABLES: &[TableConfig] = &[
TableConfig::new("Passkeys"),
TableConfig::new("FieldDefinitions"),
TableConfig::new("FieldHistories"),
- TableConfig::new("Logos"),
+ TableConfig::new("Logos").with_composite_key(&["Source"]),
];
/// List of syncable table names (for clients to know which tables to read).