Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Refactor vault_codec (#2159)
Leendert de Borst05479e1cb23e
diff --git a/core/rust/src/vault_codec/manifest.rs b/core/rust/src/vault_codec/manifest.rsindex 9d340f364..7e6573729 100644--- a/core/rust/src/vault_codec/manifest.rs+++ b/core/rust/src/vault_codec/manifest.rs@@ -105,6 +105,7 @@ pub struct CanonicalizedVault {pub data_buckets: Vec<DataBucket>,}+#[cfg(test)]impl CanonicalizedVault {/// The first manifest, which is the one the caller wrote this vault from. Canonicalize refuses empty input, so it always exists.pub fn first(&self) -> &CanonicalizedManifest {@@ -216,18 +217,9 @@ pub struct ExtractBucketsInput {#[derive(Debug, Clone, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]pub struct MaterializeInput {+ /// Every manifest to combine, the caller's own manifest first: the others are combined into it.pub manifests: Vec<Manifest>,pub data_buckets: Vec<DataBucket>,+ /// The caller's local schema, table name > column names; what it cannot hold goes to [`CodecOverflow`].pub schema_columns: HashMap<String, Vec<String>>,}--impl MaterializeInput {- /// Build an input from the caller's own manifest, the other manifests combined into it, and the- /// vault's data buckets.- pub fn new(own: Manifest, others: Vec<Manifest>, data_buckets: Vec<DataBucket>, schema_columns: HashMap<String, Vec<String>>) -> Self {- let mut manifests = Vec::with_capacity(1 + others.len());- manifests.push(own);- manifests.extend(others);- Self { manifests, data_buckets, schema_columns }- }-}diff --git a/core/rust/src/vault_codec/materialize.rs b/core/rust/src/vault_codec/materialize.rsindex 7443afb55..c3dc6bd8c 100644--- a/core/rust/src/vault_codec/materialize.rs+++ b/core/rust/src/vault_codec/materialize.rs@@ -20,7 +20,7 @@ use std::collections::{HashMap, HashSet};use serde_json::json;use super::manifest::{CodecOverflow, CodecRecord, CodecTableData, Manifest, MaterializeInput, MaterializedTables};-use super::types::{is_skip_table, row_identity, MANIFEST_ID_COL, MANIFESTS_TABLE, OVERFLOW_TABLE};+use super::types::{is_local_only_table, row_identity, MANIFEST_ID_COL, MANIFESTS_TABLE, OVERFLOW_TABLE};use crate::error::{VaultError, VaultResult};/// Materialize the vault's manifests into the table set the platform inserts. Every manifest arrives@@ -39,7 +39,7 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materializelet manifest_records = manifest_bookkeeping_records(&manifests);- // The first manifest is the caller's own (see `MaterializeInput::new`).+ // The first manifest is the caller's own (see `MaterializeInput::manifests`).let base = manifests.remove(0);let others: Vec<Manifest> = manifests;@@ -57,9 +57,9 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materializelet mut tables: Vec<CodecTableData> = Vec::with_capacity(combined.len() + data_buckets.len());for (name, records) in combined {- // OVERFLOW_TABLE is local-only bookkeeping: it must never occur in a manifest, and passing- // one through would collide with the row this function emits below.- if is_skip_table(&name) || name == OVERFLOW_TABLE {+ // A local-only table must never occur in a manifest, and passing an OVERFLOW_TABLE row+ // through would collide with the row this function emits below.+ if is_local_only_table(&name) {continue;}match split_for_schema(&name, records, &schema_columns, &mut overflow.columns) {@@ -74,7 +74,7 @@ pub fn materialize_as_sqlite(input: MaterializeInput) -> VaultResult<Materializelet mut bucket_tables: HashMap<String, Vec<CodecRecord>> = HashMap::new();for bucket in data_buckets {for (name, mut records) in bucket.tables {- if is_skip_table(&name) || name == OVERFLOW_TABLE {+ if is_local_only_table(&name) {continue;}for row in records.iter_mut() {diff --git a/core/rust/src/vault_codec/normalize.rs b/core/rust/src/vault_codec/normalize.rsindex e5f83e346..42d1907b9 100644--- a/core/rust/src/vault_codec/normalize.rs+++ b/core/rust/src/vault_codec/normalize.rs@@ -5,7 +5,10 @@ use std::collections::{HashMap, HashSet};use serde_json::json;use super::manifest::CodecRecord;+use super::row::{str_col, truthy};use super::types::{is_guid, is_id_column, MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS};+use crate::timestamp::updated_at;+use crate::vault_model::id_key;use crate::vault_model::names::{CHANGED_AT_COL, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_KEY_COL,FIELD_VALUES_TABLE, ID_COL, IS_MULTI_VALUE_COL, ITEM_ID_COL, ITEM_TAGS_TABLE, TAG_ID_COL, VALUE_INDEX_COL,@@ -20,8 +23,7 @@ const FIELD_HISTORY_ID_NAMESPACE: &str = "aliasvault:fieldhistory:v1";/// The `FieldValues.Id` of the single-value row `(manifest, item, field, position)`: a UUIDv8 whose/// bytes come from `sha256(namespace | manifest | item | field | position)`.pub fn field_value_id_for(manifest_id: &str, item_id: &str, field_key: &str, field_definition_id: &str, value_index: i64) -> String {- let field = if field_key.is_empty() { format!("fd:{}", field_definition_id.to_lowercase()) } else { format!("fk:{}", field_key.to_lowercase()) };- super::hash::derived_uuid(&format!("{}\n{}\n{}\n{}\n{}", FIELD_VALUE_ID_NAMESPACE, manifest_id.to_lowercase(), item_id.to_lowercase(), field, value_index))+ derived_field_id(FIELD_VALUE_ID_NAMESPACE, manifest_id, item_id, field_key, field_definition_id, &value_index.to_string())}/// The `FieldHistories.Id` of the history row `(manifest, item, field, changed at)`: every row derives@@ -29,8 +31,13 @@ pub fn field_value_id_for(manifest_id: &str, item_id: &str, field_key: &str, fie/// same field concurrently snapshot at different times and union. Two snapshots of one field in the very/// same millisecond collapse to one, which history can afford. `changed_at` is used verbatim.pub fn field_history_id_for(manifest_id: &str, item_id: &str, field_key: &str, field_definition_id: &str, changed_at: &str) -> String {- let field = if field_key.is_empty() { format!("fd:{}", field_definition_id.to_lowercase()) } else { format!("fk:{}", field_key.to_lowercase()) };- super::hash::derived_uuid(&format!("{}\n{}\n{}\n{}\n{}", FIELD_HISTORY_ID_NAMESPACE, manifest_id.to_lowercase(), item_id.to_lowercase(), field, changed_at))+ derived_field_id(FIELD_HISTORY_ID_NAMESPACE, manifest_id, item_id, field_key, field_definition_id, changed_at)+}++/// A UUIDv8 from `sha256(namespace | manifest | item | field | tail)`, ids lowercased.+fn derived_field_id(namespace: &str, manifest_id: &str, item_id: &str, field_key: &str, field_definition_id: &str, tail: &str) -> String {+ let field = field_discriminator_of(field_key, field_definition_id);+ super::hash::derived_uuid(&format!("{}\n{}\n{}\n{}\n{}", namespace, manifest_id.to_lowercase(), item_id.to_lowercase(), field, tail))}/// Lowercase every id in `tables` which is the normalized spelling expected by all AliasVault clients.@@ -93,7 +100,7 @@ pub(crate) fn derive_missing_ids(tables: &mut HashMap<String, Vec<CodecRecord>>)fn multi_value_definition_ids(tables: &HashMap<String, Vec<CodecRecord>>) -> HashSet<(String, String)> {let Some(rows) = tables.get(FIELD_DEFINITIONS_TABLE) else { return HashSet::new() };rows.iter()- .filter(|row| is_truthy(row.get(IS_MULTI_VALUE_COL)))+ .filter(|row| truthy(row.get(IS_MULTI_VALUE_COL))).filter_map(|row| Some((lower_str(row, MANIFEST_ID_COL)?, lower_str(row, ID_COL)?))).collect()}@@ -127,7 +134,7 @@ fn normalize_field_values(rows: &mut Vec<CodecRecord>, multi_value_defs: &HashSe} else {// A single-value field is one row; duplicates collapse to the newest UpdatedAt instead of// materializing into a primary-key violation (or a field the UI renders as an array).- let winner = *positions.iter().max_by_key(|p| (crate::vault_merge::get_updated_at(&rows[**p]), std::cmp::Reverse(**p))).unwrap();+ let winner = *positions.iter().max_by_key(|p| (updated_at(&rows[**p]), std::cmp::Reverse(**p))).unwrap();removed.extend(positions.iter().filter(|p| **p != winner));let row = &mut rows[winner];row.insert(VALUE_INDEX_COL.to_string(), json!(0));@@ -148,54 +155,41 @@ fn normalize_field_values(rows: &mut Vec<CodecRecord>, multi_value_defs: &HashSe/// Strip every history row's id (all of them derive it) and collapse rows sharing a natural key,/// which takes two same-millisecond snapshots of one field, to the newest `UpdatedAt`.fn normalize_field_histories(rows: &mut Vec<CodecRecord>) {- for row in rows.iter_mut() {- row.remove(ID_COL);- }-- let mut winners: HashMap<(String, String, String, String), usize> = HashMap::new();- for (position, row) in rows.iter().enumerate() {- let changed_at = row.get(CHANGED_AT_COL).and_then(|v| v.as_str()).unwrap_or_default().to_string();- let key = (lower_str(row, MANIFEST_ID_COL).unwrap_or_default(), lower_str(row, ITEM_ID_COL).unwrap_or_default(), field_discriminator(row), changed_at);- match winners.get(&key) {- // Ties keep the earlier row, matching the single-value collapse rule.- Some(current) if crate::vault_merge::get_updated_at(row) <= crate::vault_merge::get_updated_at(&rows[*current]) => {}- _ => {- winners.insert(key, position);- }- }- }-- retain_positions(rows, &winners.into_values().collect());-}--/// The derived id for a normalized FieldHistories row.-fn derive_history_row_id(row: &CodecRecord) -> String {- let manifest = row.get(MANIFEST_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();- let item = row.get(ITEM_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();- let field_key = row.get(FIELD_KEY_COL).and_then(|v| v.as_str()).unwrap_or_default();- let field_def = row.get(FIELD_DEFINITION_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();- let changed_at = row.get(CHANGED_AT_COL).and_then(|v| v.as_str()).unwrap_or_default();- field_history_id_for(manifest, item, field_key, field_def, changed_at)+ strip_ids(rows);+ collapse_newest_by_key(rows, |row| {+ let changed_at = str_col(row, CHANGED_AT_COL).unwrap_or_default().to_string();+ (lower_str(row, MANIFEST_ID_COL).unwrap_or_default(), lower_str(row, ITEM_ID_COL).unwrap_or_default(), field_discriminator(row), changed_at)+ });}/// Drop the legacy surrogate `Id` and collapse duplicates to the newest `UpdatedAt` per natural key.fn normalize_item_tags(rows: &mut Vec<CodecRecord>) {+ strip_ids(rows);+ collapse_newest_by_key(rows, |row| {+ (lower_str(row, MANIFEST_ID_COL).unwrap_or_default(), lower_str(row, ITEM_ID_COL).unwrap_or_default(), lower_str(row, TAG_ID_COL).unwrap_or_default())+ });+}++/// Remove the `Id` column from every row.+fn strip_ids(rows: &mut [CodecRecord]) {for row in rows.iter_mut() {row.remove(ID_COL);}+}- let mut winners: HashMap<(String, String, String), usize> = HashMap::new();+/// Keep one row per `key_of` value: the newest `UpdatedAt`, ties keeping the earlier row (matching+/// the single-value collapse rule). Row order is preserved.+fn collapse_newest_by_key<K: std::hash::Hash + Eq>(rows: &mut Vec<CodecRecord>, key_of: impl Fn(&CodecRecord) -> K) {+ let mut winners: HashMap<K, usize> = HashMap::new();for (position, row) in rows.iter().enumerate() {- let key = (lower_str(row, MANIFEST_ID_COL).unwrap_or_default(), lower_str(row, ITEM_ID_COL).unwrap_or_default(), lower_str(row, TAG_ID_COL).unwrap_or_default());+ let key = key_of(row);match winners.get(&key) {- // Ties keep the earlier row, matching the single-value collapse rule.- Some(current) if crate::vault_merge::get_updated_at(row) <= crate::vault_merge::get_updated_at(&rows[*current]) => {}+ Some(current) if updated_at(row) <= updated_at(&rows[*current]) => {}_ => {winners.insert(key, position);}}}-retain_positions(rows, &winners.into_values().collect());}@@ -211,23 +205,30 @@ fn retain_positions(rows: &mut Vec<CodecRecord>, keep: &HashSet<usize>) {/// The derived id for a normalized FieldValues row at `value_index`.fn derive_row_id(row: &CodecRecord, value_index: i64) -> String {- let manifest = row.get(MANIFEST_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();- let item = row.get(ITEM_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();- let field_key = row.get(FIELD_KEY_COL).and_then(|v| v.as_str()).unwrap_or_default();- let field_def = row.get(FIELD_DEFINITION_ID_COL).and_then(|v| v.as_str()).unwrap_or_default();+ let [manifest, item, field_key, field_def] = field_key_parts(row);field_value_id_for(manifest, item, field_key, field_def, value_index)}-/// The field half of a FieldValues row's natural key: `fk:<key>` for a system field, `fd:<id>` for a-/// custom field (exactly one is set; a row carrying neither groups under `fk:` and collapses).+/// The derived id for a normalized FieldHistories row.+fn derive_history_row_id(row: &CodecRecord) -> String {+ let [manifest, item, field_key, field_def] = field_key_parts(row);+ field_history_id_for(manifest, item, field_key, field_def, str_col(row, CHANGED_AT_COL).unwrap_or_default())+}++/// The `(ManifestId, ItemId, FieldKey, FieldDefinitionId)` of a row, absent columns as empty strings.+fn field_key_parts(row: &CodecRecord) -> [&str; 4] {+ [MANIFEST_ID_COL, ITEM_ID_COL, FIELD_KEY_COL, FIELD_DEFINITION_ID_COL].map(|column| str_col(row, column).unwrap_or_default())+}++/// The field half of a FieldValues row's natural key, see [`field_discriminator_of`].fn field_discriminator(row: &CodecRecord) -> String {- match row.get(FIELD_KEY_COL).and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {- Some(key) => format!("fk:{}", key.to_lowercase()),- None => match row.get(FIELD_DEFINITION_ID_COL).and_then(|v| v.as_str()).filter(|s| !s.is_empty()) {- Some(def) => format!("fd:{}", def.to_lowercase()),- None => "fk:".to_string(),- },- }+ let [_, _, field_key, field_def] = field_key_parts(row);+ field_discriminator_of(field_key, field_def)+}++/// The field half of a natural key: `fk:<key>` for a system field, `fd:<id>` for a custom field.+fn field_discriminator_of(field_key: &str, field_definition_id: &str) -> String {+ if field_key.is_empty() { format!("fd:{}", field_definition_id.to_lowercase()) } else { format!("fk:{}", field_key.to_lowercase()) }}fn has_id(row: &CodecRecord) -> bool {@@ -242,17 +243,9 @@ fn value_index_of(row: &CodecRecord) -> Option<i64> {}}+/// An id column as its comparison key, `None` when absent or empty.fn lower_str(row: &CodecRecord, column: &str) -> Option<String> {- row.get(column).and_then(|v| v.as_str()).filter(|s| !s.is_empty()).map(str::to_lowercase)-}--fn is_truthy(value: Option<&serde_json::Value>) -> bool {- match value {- Some(serde_json::Value::Bool(b)) => *b,- Some(serde_json::Value::Number(n)) => n.as_i64().unwrap_or(0) != 0,- Some(serde_json::Value::String(s)) => s == "1" || s.eq_ignore_ascii_case("true"),- _ => false,- }+ str_col(row, column).filter(|s| !s.is_empty()).map(id_key)}#[cfg(test)]diff --git a/core/rust/src/vault_codec/row.rs b/core/rust/src/vault_codec/row.rsnew file mode 100644index 000000000..100f48529--- /dev/null+++ b/core/rust/src/vault_codec/row.rs@@ -0,0 +1,51 @@+//! Reading cells out of a row as the platforms hand them over: SQLite booleans arrive as 0/1+//! numbers, JSON booleans or "1"/"true" strings, and a missing table reads as no rows.++use std::collections::HashMap;++use serde_json::Value;++use super::manifest::CodecRecord;+use crate::vault_model::names::IS_DELETED_COL;++/// A cell as a string slice, `None` when absent or not a string.+pub(crate) fn str_col<'a>(row: &'a CodecRecord, column: &str) -> Option<&'a str> {+ row.get(column).and_then(Value::as_str)+}++/// SQLite-tolerant truthiness: boolean true, a non-zero number, or a "1"/"true" string.+pub(crate) fn truthy(value: Option<&Value>) -> bool {+ match value {+ Some(Value::Bool(b)) => *b,+ Some(Value::Number(n)) => n.as_f64().is_some_and(|f| f != 0.0),+ Some(Value::String(s)) => s == "1" || s.eq_ignore_ascii_case("true"),+ _ => false,+ }+}++/// True when the row's `IsDeleted` flag is set.+pub(crate) fn is_deleted(row: &CodecRecord) -> bool {+ truthy(row.get(IS_DELETED_COL))+}++/// The rows of `table`, empty when the table set does not carry it.+pub(crate) fn rows_of<'a>(tables: &'a HashMap<String, Vec<CodecRecord>>, table: &str) -> &'a [CodecRecord] {+ tables.get(table).map(Vec::as_slice).unwrap_or(&[])+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn truthy_accepts_every_spelling_sqlite_hands_over() {+ for value in [json!(true), json!(1), json!(1.0), json!(2), json!("1"), json!("true"), json!("TRUE")] {+ assert!(truthy(Some(&value)), "{value} should be truthy");+ }+ for value in [json!(false), json!(0), json!(0.0), json!("0"), json!(""), json!("false"), json!(null)] {+ assert!(!truthy(Some(&value)), "{value} should be falsy");+ }+ assert!(!truthy(None));+ }+}diff --git a/core/rust/src/vault_codec/scoped_assets.rs b/core/rust/src/vault_codec/scoped_assets.rsindex c2825a34a..3bbd69347 100644--- a/core/rust/src/vault_codec/scoped_assets.rs+++ b/core/rust/src/vault_codec/scoped_assets.rs@@ -20,18 +20,10 @@ use std::collections::{HashMap, HashSet};use serde_json::{json, Value};use super::manifest::CodecRecord;+use super::row::{is_deleted, str_col};use super::types::MANIFEST_ID_COL;-use crate::vault_model::names::{FILE_DATA_COL, ID_COL, ITEMS_TABLE, KIND_COL, LOGOS_TABLE, LOGO_ID_COL, SOURCE_COL, UPDATED_AT_COL};--/// The kind of a logo fetched automatically from an item's URL. Also what a row that carries no-/// `Kind` at all means: it was written before the column existed, when every logo was a favicon.-pub use crate::vault_model::names::LOGO_KIND_FAVICON as KIND_FAVICON;--/// The kind of a logo picked from the built-in catalog.-pub use crate::vault_model::names::LOGO_KIND_BUILTIN as KIND_BUILTIN;--/// The kind of a logo the user uploaded.-pub use crate::vault_model::names::LOGO_KIND_CUSTOM as KIND_CUSTOM;+use crate::vault_model::id_key;+use crate::vault_model::names::{FILE_DATA_COL, ID_COL, ITEMS_TABLE, KIND_COL, LOGOS_TABLE, LOGO_ID_COL, LOGO_KIND_CUSTOM, LOGO_KIND_FAVICON, SOURCE_COL, UPDATED_AT_COL};/// Domain-separation prefix for favicon ids. It predates the `Kind` column and is kept verbatim so/// every favicon row that already exists keeps its id: changing it would re-derive the logo id of@@ -59,26 +51,27 @@ pub fn logo_id_for(manifest_id: &str, kind: &str, source: &str) -> String {/// way, and the rows simply round-trip until this client learns to render them.fn namespace_for_kind(kind: &str) -> String {match normalize_kind(kind) {- k if k == KIND_FAVICON => FAVICON_ID_NAMESPACE.to_string(),+ k if k == LOGO_KIND_FAVICON => FAVICON_ID_NAMESPACE.to_string(),k => format!("aliasvault:logo:{}:v1", k),}}-/// A row's kind, defaulting to [`KIND_FAVICON`] when absent or empty (pre-`Kind` writers) and-/// lowercased so `Kind` matching is case-insensitive like the rest of the natural key.+/// A row's kind, lowercased so `Kind` matching is case-insensitive like the rest of the natural key.+/// Absent or empty means [`LOGO_KIND_FAVICON`]: the row was written before the column existed, when+/// every logo was a favicon.fn normalize_kind(kind: &str) -> String {let trimmed = kind.trim();if trimmed.is_empty() {- return KIND_FAVICON.to_string();+ return LOGO_KIND_FAVICON.to_string();}trimmed.to_lowercase()}-/// True when a row holds an image the user supplied themselves ([`KIND_CUSTOM`]) rather than one the+/// True when a row holds an image the user supplied themselves ([`LOGO_KIND_CUSTOM`]) rather than one the/// client can produce again on its own: a favicon it can refetch from the domain, a built-in logo it/// draws from the catalog. Only that first group is worth keeping around once nothing references it.pub(super) fn is_custom_logo(row: &CodecRecord) -> bool {- normalize_kind(str_col(row, KIND_COL).unwrap_or("")) == KIND_CUSTOM+ normalize_kind(str_col(row, KIND_COL).unwrap_or("")) == LOGO_KIND_CUSTOM}/// The `(kind, source)` natural key of a row, or `None` when it carries no `Source` to key on.@@ -127,8 +120,7 @@ fn rewrite_logo_rows(tables: &mut HashMap<String, Vec<CodecRecord>>, scope: &strlet (Some((kind, source)), Some(old_id)) = (natural_key(row), str_col(row, ID_COL)) else { continue };let new_id = logo_id_for(scope, &kind, &source);if old_id != new_id || !survivors.contains(&idx) {- // GUIDs are case-insensitive and should be stored/compared as such.- remap.insert(old_id.to_lowercase(), new_id);+ remap.insert(id_key(old_id), new_id);}}@@ -162,8 +154,8 @@ fn repoint_items(tables: &mut HashMap<String, Vec<CodecRecord>>, remap: &HashMaplet Some(items) = tables.get_mut(ITEMS_TABLE) else { return };for item in items.iter_mut() {let Some(current) = str_col(item, LOGO_ID_COL).map(str::to_string) else { continue };- let resolved = remap.get(¤t.to_lowercase()).cloned().unwrap_or(current);- let repaired = if valid_ids.contains(&resolved.to_lowercase()) { json!(resolved) } else { Value::Null };+ let resolved = remap.get(&id_key(¤t)).cloned().unwrap_or(current);+ let repaired = if valid_ids.contains(&id_key(&resolved)) { json!(resolved) } else { Value::Null };item.insert(LOGO_ID_COL.to_string(), repaired);}}@@ -181,7 +173,7 @@ pub(super) fn reconcile_logo_references(tables: &mut HashMap<String, Vec<CodecRelet mut ids: Vec<String> = tables[ITEMS_TABLE].iter().filter_map(|item| str_col(item, LOGO_ID_COL))- .filter(|id| !present.contains(&id.to_lowercase()))+ .filter(|id| !present.contains(&id_key(id))).map(str::to_string).collect();ids.sort();@@ -209,7 +201,7 @@ pub(super) fn reconcile_logo_references(tables: &mut HashMap<String, Vec<CodecReif let Some(existing_id) = id_by_key.get(&(kind.clone(), source.clone())).cloned() {refill_empty_scope_row(tables, &existing_id, origin);- remap.insert(missing_id.to_lowercase(), existing_id);+ remap.insert(id_key(&missing_id), existing_id);continue;}@@ -220,7 +212,7 @@ pub(super) fn reconcile_logo_references(tables: &mut HashMap<String, Vec<CodecReclone.insert(MANIFEST_ID_COL.to_string(), scope_value.clone());clones.push(clone);id_by_key.insert((kind, source), scoped_id.clone());- remap.insert(missing_id.to_lowercase(), scoped_id);+ remap.insert(id_key(&missing_id), scoped_id);}if !clones.is_empty() {@@ -234,7 +226,7 @@ pub(super) fn reconcile_logo_references(tables: &mut HashMap<String, Vec<CodecRefn refill_empty_scope_row(tables: &mut HashMap<String, Vec<CodecRecord>>, existing_id: &str, origin: &CodecRecord) {let Some(logos) = tables.get_mut(LOGOS_TABLE) else { return };let Some(existing) = logos.iter_mut().find(|r| str_col(r, ID_COL) == Some(existing_id)) else { return };- let upgrades = (!has_file_data(existing) && has_file_data(origin)) || (is_tombstoned(existing) && !is_tombstoned(origin));+ let upgrades = (!has_file_data(existing) && has_file_data(origin)) || (is_deleted(existing) && !is_deleted(origin));if !upgrades {return;}@@ -259,7 +251,7 @@ fn refill_empty_scope_row(tables: &mut HashMap<String, Vec<CodecRecord>>, existifn logo_ids(tables: &HashMap<String, Vec<CodecRecord>>) -> HashSet<String> {tables.get(LOGOS_TABLE)- .map(|rows| rows.iter().filter_map(|r| str_col(r, ID_COL).map(str::to_lowercase)).collect())+ .map(|rows| rows.iter().filter_map(|r| str_col(r, ID_COL).map(id_key)).collect()).unwrap_or_default()}@@ -267,8 +259,8 @@ fn logo_ids(tables: &HashMap<String, Vec<CodecRecord>>) -> HashSet<String> {/// (the id is derived either way): a live row beats a tombstoned one, a row with image bytes beats an/// empty one, and the lexicographically-highest (newest) `Id` breaks the remaining ties.fn is_better_logo(candidate: &CodecRecord, incumbent: &CodecRecord) -> bool {- let cand_live = !is_tombstoned(candidate);- let inc_live = !is_tombstoned(incumbent);+ let cand_live = !is_deleted(candidate);+ let inc_live = !is_deleted(incumbent);if cand_live != inc_live {return cand_live;}@@ -293,16 +285,3 @@ fn has_file_data(row: &CodecRecord) -> bool {}}-/// SQLite-tolerant `IsDeleted` truthiness (boolean, 0/1 number, or "1"/"true" string).-fn is_tombstoned(row: &CodecRecord) -> bool {- match row.get("IsDeleted") {- Some(Value::Bool(b)) => *b,- Some(Value::Number(n)) => n.as_f64().map(|f| f != 0.0).unwrap_or(false),- Some(Value::String(s)) => s == "1" || s.eq_ignore_ascii_case("true"),- _ => false,- }-}--fn str_col<'a>(row: &'a CodecRecord, column: &str) -> Option<&'a str> {- row.get(column).and_then(|v| v.as_str())-}diff --git a/core/rust/src/vault_codec/sharing.rs b/core/rust/src/vault_codec/sharing.rsindex c1c546d51..8481ea85b 100644--- a/core/rust/src/vault_codec/sharing.rs+++ b/core/rust/src/vault_codec/sharing.rs@@ -34,15 +34,16 @@ use std::collections::{HashMap, HashSet};use serde_json::{json, Value};use super::manifest::{CodecRecord, Manifest, ManifestSpec};+use super::row::{str_col, truthy};use super::scoped_assets::{is_custom_logo, normalize_logo_scope, reconcile_logo_references};use super::types::{- is_bucketed_table, is_manifest_scoped, is_personal_table, is_skip_table, manifest_scoped_tables, row_identity, ENCRYPTION_KEYS_TABLE,- MANIFEST_ID_COL, OVERFLOW_TABLE,+ is_bucketed_table, is_local_only_table, is_manifest_scoped, is_personal_table, manifest_scoped_tables, row_identity, ENCRYPTION_KEYS_TABLE,+ MANIFEST_ID_COL,};use crate::error::{VaultError, VaultResult};use crate::vault_model::names::{FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_VALUES_TABLE, FOLDERS_TABLE,- FOLDER_ID_COL, ID_COL, IS_DELETED_COL, IS_PRIMARY_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, LOGOS_TABLE,+ FOLDER_ID_COL, ID_COL, IS_DELETED_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, LOGOS_TABLE,LOGO_ID_COL, PARENT_FOLDER_ID_COL, PUBLIC_KEY_COL, TAGS_TABLE, TAG_ID_COL,};@@ -433,7 +434,7 @@ pub(super) fn combine_manifest_tables(* skip-tables never travel inside one. Dropping them here also means a manifest authored by* another user cannot inject rows into this vault's personal scope.*/- if is_skip_table(&name) || name == OVERFLOW_TABLE || is_bucketed_table(&name) || is_personal_table(&name) {+ if is_local_only_table(&name) || is_bucketed_table(&name) || is_personal_table(&name) {continue;}@@ -612,33 +613,7 @@ pub fn extract_encryption_key_for_public_key(manifest: &Manifest, public_key: &s.tables.get(ENCRYPTION_KEYS_TABLE)?.iter()- .find(|row| str_col(row, PUBLIC_KEY_COL) == Some(public_key) && str_col(row, MANIFEST_ID_COL) == scope && !is_truthy(row.get(IS_DELETED_COL)))+ .find(|row| str_col(row, PUBLIC_KEY_COL) == Some(public_key) && str_col(row, MANIFEST_ID_COL) == scope && !truthy(row.get(IS_DELETED_COL))).cloned()}-/// The manifest's *active* keypair: the live `IsPrimary` row stamped with the manifest's own id, whose-/// public half is published to the server for SMTP delivery. Superseded rows stay in the manifest, so-/// mail received before a rotation remains decryptable, but are never returned here.-pub fn active_encryption_key(manifest: &Manifest) -> Option<CodecRecord> {- let scope = Some(manifest.manifest_id.as_str());- manifest- .tables- .get(ENCRYPTION_KEYS_TABLE)?- .iter()- .find(|row| is_truthy(row.get(IS_PRIMARY_COL)) && str_col(row, MANIFEST_ID_COL) == scope && !is_truthy(row.get(IS_DELETED_COL)))- .cloned()-}--fn str_col<'a>(row: &'a CodecRecord, column: &str) -> Option<&'a str> {- row.get(column).and_then(|v| v.as_str())-}--/// SQLite-tolerant truthiness: boolean true, non-zero number, or "1"/"true" strings.-fn is_truthy(value: Option<&Value>) -> bool {- match value {- Some(Value::Bool(b)) => *b,- Some(Value::Number(n)) => n.as_f64().map(|f| f != 0.0).unwrap_or(false),- Some(Value::String(s)) => s == "1" || s.eq_ignore_ascii_case("true"),- _ => false,- }-}diff --git a/core/rust/src/vault_codec/sharing_tests.rs b/core/rust/src/vault_codec/sharing_tests.rsindex 8067de90b..12bfeb30b 100644--- a/core/rust/src/vault_codec/sharing_tests.rs+++ b/core/rust/src/vault_codec/sharing_tests.rs@@ -7,7 +7,8 @@use super::*;use super::tests::{fitting_schema, materialize_manifests, materialize_input, stamp_unstamped};-use super::types::manifest_scoped_tables;+use super::types::{is_bucketed_table, is_personal_table, manifest_scoped_tables, SCHEMA_VERSION};+use crate::vault_model::names::LOGO_KIND_FAVICON;use base64::engine::general_purpose::STANDARD as BASE64;use base64::Engine;use serde_json::json;@@ -16,7 +17,7 @@ use std::collections::{HashMap, HashSet};const SALT_PERSONAL: &str = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff";const SALT_SHARED: &str = "ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766554433221100";-/// The personal manifest id every test canonicalizes against. Personal rows are stamped with it — there is+/// The personal manifest id every test canonicalizes against. Personal rows are stamped with it, there is/// no NULL-scope convention anywhere in the format.const PERSONAL_M: &str = "m-personal";@@ -91,7 +92,7 @@ fn tag_links(records: &[CodecRecord]) -> Vec<(String, String)> {/// The scoped logo id the codec derives for `(manifest id, source)`, what tests assert against, since/// a logo's identity is a function of its manifest and domain rather than whatever id the writer generated.fn logo_id(scope: &str, source: &str) -> String {- scoped_assets::logo_id_for(scope, scoped_assets::KIND_FAVICON, source)+ scoped_assets::logo_id_for(scope, LOGO_KIND_FAVICON, source)}/// The `Logos` row for `source` in a manifest, resolved through its manifest-id-derived id.@@ -110,7 +111,7 @@ fn logo_sources(m: &Manifest) -> Vec<&str> {out}-/// Stamp a folder's whole subtree — its folders and the items in them — into `manifest_id`. Mirrors+/// Stamp a folder's whole subtree (its folders and the items in them) into `manifest_id`. Mirrors/// exactly what the client's `FolderRepository.restampSubtree` writes when a folder starts being/// shared, so fixtures carry the membership the codec now routes on.///@@ -308,7 +309,7 @@ fn split_gives_each_manifest_its_own_copy_of_the_tags_it_uses() {/** A tag is manifest-scoped like everything else, so the shared manifest gets its own stamped copy- * of the two its items carry — the composite `(ManifestId, TagId)` foreign key cannot reach into+ * of the two its items carry, the composite `(ManifestId, TagId)` foreign key cannot reach into* another namespace. The copies are copies: the base keeps every tag it holds, including the one* only shared items use now and the one nothing uses at all. An unused tag is still the user's.*/@@ -408,8 +409,8 @@ fn sqlite_blob_migration_stamps_every_row_of_a_vault_that_has_no_manifest_id_col/** The sqlite-blob > manifest-v1 migration, end to end: a vault whose schema stops short of the* `ManifestId` column (so no row carries the key at all) is canonicalized and materialized straight- * back out, with no share in sight. Nothing stamps those rows beforehand — the column they would be- * stamped in does not exist yet — so canonicalize adopting them into the manifest being written from+ * back out, with no share in sight. Nothing stamps those rows beforehand, the column they would be+ * stamped in does not exist yet, so canonicalize adopting them into the manifest being written from* IS the conversion. Every row must come out carrying a real manifest id: the materialized schema* declares the column NOT NULL, so a single unstamped row fails the whole migration.*/@@ -486,7 +487,7 @@ fn manifests_drop_logos_no_item_references() {fn orphan_favicons_are_pruned_everywhere_and_uploads_survive_everywhere() {/** A favicon nothing references is refetchable, so it goes, in every manifest alike. An image the- * user uploaded is not reproducible, so it stays — also in every manifest alike: each one carries+ * user uploaded is not reproducible, so it stays, also in every manifest alike: each one carries* its own library, and which of them a client offers to pick from is a runtime choice, not* something the format decides by anointing one manifest.*/@@ -556,7 +557,7 @@ fn split_keeps_bucketed_and_foreign_key_tables_out_of_shared_manifests() {assert!(!out.data_buckets.iter().any(|b| b.category == "EncryptionKeys"), "EncryptionKeys is no longer a bucket category");/*- * Every manifest gets its own Settings bucket, holding exactly its own rows — the shared one empty+ * Every manifest gets its own Settings bucket, holding exactly its own rows, the shared one empty* here, since the single Settings row belongs to the personal manifest. Leaving a manifest's bucket* out would read as "unchanged" rather than "emptied", so its last row could never be deleted.*/@@ -645,7 +646,7 @@ fn extract_buckets_splits_a_category_the_way_canonicalize_does() {* the caller hands over the whole category and gets one bucket per manifest it can write. Every* manifest asked for gets a bucket even when it holds nothing (otherwise deleting its last row* would never reach the server), a row naming a manifest this vault does not carry is dropped with- * that manifest, and a row that names no manifest has no scope the codec may invent — it refuses.+ * that manifest, and a row that names no manifest has no scope the codec may invent, it refuses.*/let settings = |rows: Vec<CodecRecord>| -> HashMap<String, Vec<CodecRecord>> { [("Settings".to_string(), rows)].into_iter().collect() };@@ -687,8 +688,8 @@ fn extract_buckets_splits_a_category_the_way_canonicalize_does() {#[test]fn split_routes_unregistered_tables_by_their_own_stamps() {/*- * A table this build's registry does not know — a newer writer's table carried through the codec- * overflow — routes by its rows' own stamps like every registered table. The writing (personal)+ * A table this build's registry does not know, a newer writer's table carried through the codec+ * overflow, routes by its rows' own stamps like every registered table. The writing (personal)* manifest is not special: it gets exactly its own rows, a shared manifest gets exactly its own,* and a row stamped for a manifest this vault no longer carries is dropped with it.*/@@ -866,7 +867,7 @@ fn split_regrafts_overflow_columns_onto_shared_rows() {// A newer writer added a column to a shared item; this client's schema couldn't hold it, so it// rode in the overflow carrier. On push it must re-graft and travel with the row into the// shared manifest, not the personal one.- // Recorded under the identity the row had when the overflow was written — before the folder was+ // Recorded under the identity the row had when the overflow was written, before the folder was// shared, so under the personal scope. The re-graft has to survive the row changing manifest.let overflow = CodecOverflow {columns: [(@@ -1111,7 +1112,7 @@ fn item_moved_into_shared_manifest_adopts_its_existing_logo() {fn combine_scopes_a_legacy_shared_manifest_before_it_can_collide() {// A shared manifest whose ROWS predate manifest-id scoping: its logos were generated at random and// carry no scope stamp. Materializing it next to the recipient's own rows must not collide on- // UNIQUE(ManifestId, Kind, Source) — combine normalizes every row to the manifest's own id.+ // UNIQUE(ManifestId, Kind, Source), combine normalizes every row to the manifest's own id.let (recipient_manifest, buckets) = recipient_personal_manifest();let legacy_shared = Manifest {schema_version: SCHEMA_VERSION,@@ -1146,8 +1147,8 @@ fn combine_scopes_a_legacy_shared_manifest_before_it_can_collide() {#[test]fn combine_keeps_both_rows_when_two_manifests_share_a_primary_key() {// Ids are client-generated, so a shared manifest may legitimately carry a row whose Id equals one of- // the reader's own. Because rows are keyed by (ManifestId, Id) — the primary key the local schema- // declares — both survive in their own namespace and neither shadows the other.+ // the reader's own. Because rows are keyed by (ManifestId, Id), the primary key the local schema+ // declares, both survive in their own namespace and neither shadows the other.let mut personal = canonicalize_from_sqlite(input_with_shares(vec![table("Items", vec![row(&[("Id", json!("i-dup")), ("Name", json!("personal-version")), ("FolderId", serde_json::Value::Null)])])],vec![],@@ -1211,7 +1212,7 @@ fn combine_keeps_each_manifests_own_copy_of_a_shared_tag() {#[test]fn split_stamps_item_scoped_rows_with_the_manifest_their_item_joined() {// A child row's foreign key is (ManifestId, ItemId), so it has to carry the same stamp as the item- // it followed — otherwise it points at a row in a namespace it isn't in.+ // it followed, otherwise it points at a row in a namespace it isn't in.let tables = stamp_subtree(owner_tables(), "f-shared", "m-f-shared");let out = canonicalize_from_sqlite(input_with_shares(tables, vec![spec("f-shared")])).unwrap();@@ -1336,7 +1337,7 @@ fn combine_applies_schema_overflow_to_shared_rows_too() {.collect();let (recipient_manifest, buckets) = recipient_personal_manifest();- let re = materialize_as_sqlite(MaterializeInput::new(recipient_manifest, vec![shared], buckets, schema)).unwrap();+ let re = materialize_as_sqlite(MaterializeInput { manifests: vec![recipient_manifest, shared], data_buckets: buckets, schema_columns: schema }).unwrap();let map = materialized_map(&re);assert!(map["Items"].iter().all(|r| !r.contains_key("FutureCol")));let overflowed: Vec<&String> = re.overflow.columns.get("Items").map(|m| m.keys().collect()).unwrap_or_default();@@ -1375,8 +1376,17 @@ fn extract_encryption_key_for_public_key_picks_matching_row_over_primary() {let cur = extract_encryption_key_for_public_key(&out.first().manifest, "pub-cur").expect("current key present");assert_eq!(cur["Id"], json!("ek-cur"));- // The manifest's active key is the primary row of its own scope.- assert_eq!(active_encryption_key(&out.first().manifest).expect("primary key present")["Id"], json!("ek-cur"));+ // The manifest's live primary row, the one whose public half is published for delivery, is the current key.+ assert_eq!(ids(&primary_keys(&out.first().manifest)), vec!["ek-cur"]);+}++/// The live `IsPrimary` rows of a manifest's `EncryptionKeys` table stamped with the manifest's own id.+fn primary_keys(m: &Manifest) -> Vec<CodecRecord> {+ rows(m, "EncryptionKeys")+ .iter()+ .filter(|r| r["IsPrimary"] == json!(1) && r["ManifestId"] == json!(m.manifest_id) && r.get("IsDeleted").is_none_or(|d| *d == json!(0)))+ .cloned()+ .collect()}#[test]@@ -1398,11 +1408,11 @@ fn extract_encryption_key_for_public_key_skips_deleted_and_returns_none_on_miss(fn extract_encryption_key_for_public_key_json_sibling_roundtrips() {let out = canonicalize_from_sqlite(input_with_shares(owner_tables(), vec![])).unwrap();let manifest_json = serde_json::to_string(&out.first().manifest).unwrap();- let key_json = extract_encryption_key_for_public_key_json(&manifest_json, "pub").unwrap();+ let key_json = crate::error::json_call(&manifest_json, |m: Manifest| Ok(extract_encryption_key_for_public_key(&m, "pub"))).unwrap();let key: serde_json::Value = serde_json::from_str(&key_json).unwrap();assert_eq!(key["PrivateKey"], json!("priv"));- let miss = extract_encryption_key_for_public_key_json(&manifest_json, "nope").unwrap();+ let miss = crate::error::json_call(&manifest_json, |m: Manifest| Ok(extract_encryption_key_for_public_key(&m, "nope"))).unwrap();assert_eq!(miss, "null");}@@ -1412,7 +1422,7 @@ fn extract_encryption_key_for_public_key_json_sibling_roundtrips() {#[test]fn manifest_specs_deserialize_from_camel_case_json() {- // Every manifest is described the same way on the wire — id, salt, optional name — with nothing+ // Every manifest is described the same way on the wire (id, salt, optional name) with nothing// marking one of them as special. A manifest carries a name, never a folder anchor.let input_json = json!({"tables": [{ "name": "Items", "records": [] }],@@ -1423,7 +1433,7 @@ fn manifest_specs_deserialize_from_camel_case_json() {]}).to_string();- let out_json = canonicalize_from_sqlite_json(&input_json).unwrap();+ let out_json = crate::error::json_call(&input_json, |input: CanonicalizeInput| canonicalize_from_sqlite(input)).unwrap();let out: CanonicalizedVault = serde_json::from_str(&out_json).unwrap();assert_eq!(out.rest().len(), 1);assert_eq!(out.rest()[0].manifest.manifest_id, "m-1");@@ -1450,7 +1460,7 @@ fn materialize_input_accepts_every_manifest_in_one_list_from_json() {"schemaColumns": schema,}).to_string();- let out_json = materialize_as_sqlite_json(&input_json).unwrap();+ let out_json = crate::error::json_call(&input_json, |input: MaterializeInput| materialize_as_sqlite(input)).unwrap();let out: MaterializedTables = serde_json::from_str(&out_json).unwrap();let items = out.tables.iter().find(|t| t.name == "Items").unwrap();assert_eq!(items.records.len(), 4);@@ -1465,7 +1475,7 @@ fn materialize_rejects_an_empty_manifest_list() {}/// Nothing marks a manifest as the caller's own: the list order says it. The first entry is the base-/// the others combine into, and the bookkeeping rows record every manifest alike — id and name, no+/// the others combine into, and the bookkeeping rows record every manifest alike: id and name, no/// claim about which one is whose. Nothing about the base becomes a vault-wide fact either: the base/// contributes no schema/version claim the other manifests are then held to.#[test]@@ -1722,7 +1732,7 @@ fn a_builtin_logo_survives_without_any_image_bytes() {// Folder keypairs (manifest-stamped `EncryptionKeys` rows)//// One table serves every manifest: personal-stamped rows are the personal manifest's own delivery keys,-// shared-manifest-stamped rows are that folder's delivery keypair. The guards point both ways — a+// shared-manifest-stamped rows are that folder's delivery keypair. The guards point both ways, a// personal key must never travel INTO a shared manifest, and a folder key must never travel OUT of// its own. Both directions are attacks a folder co-owner can attempt by writing rows into a manifest// the victim materializes.@@ -1777,7 +1787,7 @@ fn split_drops_a_keypair_stamped_for_a_manifest_that_is_not_in_this_push() {#[test]fn split_drops_folder_keypair_whose_scope_is_not_shared() {// A key row stamped for a manifest that is not part of this push (revoked, deleted, or fabricated- // locally) has nowhere to go: it must be dropped, NOT fall back into the personal manifest — a stale+ // locally) has nowhere to go: it must be dropped, NOT fall back into the personal manifest, a stale// copy demoted into the personal manifest would resurrect the old keypair on a future re-share, and would// leave a private delivery key in a namespace revocation cannot reach.let out = canonicalize_from_sqlite(input_with_shares(@@ -1882,7 +1892,7 @@ fn encryption_keys_is_scoped_rather_than_personal_or_bucketed() {/** And pins the two axes apart. Bucketed says where a table syncs; personal says which manifests may- * hold it. Settings is bucketed and NOT personal — its bucket belongs to a manifest, so a shared+ * hold it. Settings is bucketed and NOT personal, its bucket belongs to a manifest, so a shared* manifest carrying its own settings is a routing question, never a rule violation.*/assert!(is_bucketed_table("Settings"));@@ -1915,7 +1925,7 @@ fn active_folder_key_is_the_primary_row_and_rotated_keys_stay_resolvable() {.unwrap();let shared = &out.rest()[0].manifest;- assert_eq!(active_encryption_key(shared).expect("primary key present")["Id"], json!("sfk-cur"));+ assert_eq!(ids(&primary_keys(shared)), vec!["sfk-cur"]);assert_eq!(extract_encryption_key_for_public_key(shared, "pub-old").expect("rotated key still resolvable")["PrivateKey"],json!("priv-pub-old")@@ -1975,11 +1985,11 @@ fn validate_rejects_misplaced_folder_keypairs_before_upload() {// Cross-manifest routing: the same id in two manifests// ─────────────────────────────────────────────────────────────────────────────//-// A manifest is a namespace, so two of them may each hold a row with the same `Id` — and they do, for+// A manifest is a namespace, so two of them may each hold a row with the same `Id`, and they do, for// ordinary reasons: a member moves a shared item into their own vault (the client re-stamps the row and// keeps its id), or an item that was moved out comes back through another member's push. Every rule// that resolves a reference therefore has to resolve it by `(ManifestId, Id)`, both halves. Anything-// that keys on the bare id sends rows across a manifest boundary — which for a shared manifest means+// that keys on the bare id sends rows across a manifest boundary, which for a shared manifest means// handing them to other people./// Two manifests each holding an item with the same id, each with its own child rows. `personal_secret`@@ -2086,7 +2096,7 @@ fn split_never_pulls_a_shared_items_child_into_the_personal_manifest() {#[test]fn split_keeps_child_rows_when_a_manifest_this_vault_lost_shares_their_item_id() {/*- * A row stamped for a manifest this vault no longer carries is dropped — but only its own rows. An+ * A row stamped for a manifest this vault no longer carries is dropped, but only its own rows. An* item left behind by a revoked share must not take the child rows of a live item that happens to* carry the same id down with it.*/@@ -2130,7 +2140,7 @@ fn split_still_follows_an_item_whose_children_were_left_unstamped() {fn split_falls_back_to_a_rows_own_stamp_when_two_items_could_claim_it() {/** The fallback above is only safe while it is unambiguous. With two same-id items in different- * manifests, a child naming neither (its own manifest holds no such item) has no item to follow —+ * manifests, a child naming neither (its own manifest holds no such item) has no item to follow,* so it routes by its own stamp rather than being handed to whichever manifest happens to be found* first.*/@@ -2149,7 +2159,7 @@ fn split_falls_back_to_a_rows_own_stamp_when_two_items_could_claim_it() {fn split_keeps_a_parent_folder_link_that_resolves_in_the_folders_own_manifest() {/** The same-id rule for folders. A personal folder carrying the id of the share's subfolder must not- * make the share's own parent link look foreign — nulling it would silently flatten the shared tree.+ * make the share's own parent link look foreign, nulling it would silently flatten the shared tree.*/let tables = vec![table("Folders", vec![diff --git a/core/rust/src/vault_codec/tests.rs b/core/rust/src/vault_codec/tests.rsindex 3a5380623..c82408464 100644--- a/core/rust/src/vault_codec/tests.rs+++ b/core/rust/src/vault_codec/tests.rs@@ -1,6 +1,7 @@//! Unit tests for vault_codec, covering the round-trip contract.use super::*;-use super::types::OVERFLOW_ROW_ID;+use super::types::{bucket_category_for, MANIFESTS_TABLE, MANIFEST_ID_COL, OVERFLOW_ROW_ID, SCHEMA_VERSION};+use crate::vault_model::names::LOGO_KIND_FAVICON;use base64::engine::general_purpose::STANDARD as BASE64;use base64::Engine;use serde_json::json;@@ -81,7 +82,7 @@ fn bucket_layout_matches_bucket_tables_source_of_truth() {}}assert_eq!(layout.iter().map(|e| e.category.as_str()).collect::<Vec<_>>(), vec!["Settings", "Stats"]);- assert_eq!(bucket_layout_json().unwrap(), serde_json::to_string(&layout).unwrap());+ assert_eq!(serde_json::to_string(&bucket_layout()).unwrap(), serde_json::to_string(&layout).unwrap());}#[test]@@ -89,7 +90,7 @@ fn every_bucketed_table_is_manifest_scoped_with_a_composite_identity() {// The invariant that makes per-manifest buckets work at all: a bucket is addressed by `(manifest_id, category)`, so a row that carries no manifest cannot be routed into one.for (table, category) in super::types::BUCKET_TABLES {assert!(- crate::vault_merge::SYNCABLE_TABLES.iter().any(|t| t.name == *table),+ crate::vault_model::SYNCABLE_TABLES.iter().any(|t| t.name == *table),"bucketed table {} ({}) is not registered in SYNCABLE_TABLES, so it would never row-merge",table,category@@ -339,7 +340,7 @@ fn canonicalize_rederives_legacy_logo_ids_and_collapses_duplicate_sources() {])).unwrap();- let expected_id = json!(scoped_assets::logo_id_for(PERSONAL_MANIFEST, scoped_assets::KIND_FAVICON, "github.com"));+ let expected_id = json!(scoped_assets::logo_id_for(PERSONAL_MANIFEST, LOGO_KIND_FAVICON, "github.com"));let logos = &out.first().manifest.tables["Logos"];assert_eq!(logos.len(), 1, "duplicate Source collapsed to one row");assert_eq!(logos[0]["Id"], expected_id, "id derived from (personal manifest id, source)");@@ -379,7 +380,7 @@ fn canonicalize_dedup_tiebreak_prefers_the_row_with_favicon_bytes() {let logos = &out.first().manifest.tables["Logos"];assert_eq!(logos.len(), 1, "duplicate Source collapsed to one row");assert_eq!(logos[0]["MimeType"], json!("image/png"), "the row with bytes supplies the surviving content");- assert_eq!(out.first().manifest.tables["Items"][0]["LogoId"], json!(scoped_assets::logo_id_for(PERSONAL_MANIFEST, scoped_assets::KIND_FAVICON, "github.com")));+ assert_eq!(out.first().manifest.tables["Items"][0]["LogoId"], json!(scoped_assets::logo_id_for(PERSONAL_MANIFEST, LOGO_KIND_FAVICON, "github.com")));}#[test]@@ -404,7 +405,7 @@ fn canonicalize_nulls_dangling_logo_reference() {let items = &out.first().manifest.tables["Items"];let i1 = items.iter().find(|r| r["Id"] == json!("i1")).unwrap();let i2 = items.iter().find(|r| r["Id"] == json!("i2")).unwrap();- assert_eq!(i1["LogoId"], json!(scoped_assets::logo_id_for(PERSONAL_MANIFEST, scoped_assets::KIND_FAVICON, "github.com")), "valid reference follows the re-derive");+ assert_eq!(i1["LogoId"], json!(scoped_assets::logo_id_for(PERSONAL_MANIFEST, LOGO_KIND_FAVICON, "github.com")), "valid reference follows the re-derive");assert_eq!(i2["LogoId"], serde_json::Value::Null, "dangling reference nulled");}@@ -488,7 +489,7 @@ fn materialize_splits_unknown_columns_into_overflow_table_and_canonicalize_remer}])).unwrap();- let re = materialize_as_sqlite(MaterializeInput::new(out.first().manifest.clone(), vec![], out.data_buckets.clone(), narrow_client_schema())).unwrap();+ let re = materialize_as_sqlite(MaterializeInput { manifests: vec![out.first().manifest.clone()], data_buckets: out.data_buckets.clone(), schema_columns: narrow_client_schema() }).unwrap();let items = re.tables.iter().find(|t| t.name == "Items").unwrap();assert!(!items.records[0].contains_key("AliasEnabled"), "unknown column filtered out of the insert set");assert_eq!(items.records[0]["Name"], json!("GitHub"));@@ -537,11 +538,11 @@ fn materialize_splits_unknown_tables_into_overflow_and_canonicalize_reemits() {let settings_bucket = out.data_buckets.iter_mut().find(|b| b.category == "Settings").expect("Settings bucket");settings_bucket.tables.insert("Preferences".to_string(), vec![row(&[("Key", json!("p1"))])]);- let re = materialize_as_sqlite(MaterializeInput::new(out.first().manifest.clone(), vec![], out.data_buckets.clone(), narrow_client_schema())).unwrap();+ let re = materialize_as_sqlite(MaterializeInput { manifests: vec![out.first().manifest.clone()], data_buckets: out.data_buckets.clone(), schema_columns: narrow_client_schema() }).unwrap();assert!(!re.tables.iter().any(|t| t.name == "NewTable" || t.name == "Preferences"), "unknown tables never reach the insert set");assert_eq!(re.overflow.tables["NewTable"].len(), 1);assert_eq!(re.overflow.bucket_tables["Settings"]["Preferences"].len(), 1);- // Materialize stamps every row with the manifest it arrived in — unknown tables included. The stamp is what routes the row back to its own manifest on the next canonicalize.+ // Materialize stamps every row with the manifest it arrived in, unknown tables included. The stamp is what routes the row back to its own manifest on the next canonicalize.assert_eq!(re.overflow.tables["NewTable"][0]["ManifestId"], json!(PERSONAL_MANIFEST));assert_eq!(re.overflow.bucket_tables["Settings"]["Preferences"][0]["ManifestId"], json!(PERSONAL_MANIFEST));let overflow_table = overflow_table_of(&re).expect("overflow emitted as a regular table row").clone();@@ -686,7 +687,7 @@ fn materialize_drops_overflow_table_smuggled_into_a_manifest() {let mut out = canonicalize_from_sqlite(basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1")), ("AliasEnabled", json!(true))])] }])).unwrap();out.manifests[0].manifest.tables.insert(OVERFLOW_TABLE.to_string(), vec![row(&[("Id", json!("smuggled")), ("Data", json!("{}"))])]);- let re = materialize_as_sqlite(MaterializeInput::new(out.first().manifest.clone(), vec![], out.data_buckets.clone(), narrow_client_schema())).unwrap();+ let re = materialize_as_sqlite(MaterializeInput { manifests: vec![out.first().manifest.clone()], data_buckets: out.data_buckets.clone(), schema_columns: narrow_client_schema() }).unwrap();let overflow_table = overflow_table_of(&re).expect("legitimate overflow row still emitted");assert_eq!(overflow_table.records.len(), 1);assert_eq!(overflow_table.records[0]["Id"], json!(OVERFLOW_ROW_ID), "smuggled row dropped, only the codec's own row remains");@@ -696,13 +697,13 @@ fn materialize_drops_overflow_table_smuggled_into_a_manifest() {fn json_siblings_roundtrip() {let input = basic_input(vec![CodecTableData { name: "Items".to_string(), records: vec![row(&[("Id", json!("i1"))])] }]);let input_json = serde_json::to_string(&input).unwrap();- let canonicalized_json = canonicalize_from_sqlite_json(&input_json).unwrap();+ let canonicalized_json = crate::error::json_call(&input_json, |input: CanonicalizeInput| canonicalize_from_sqlite(input)).unwrap();let canonicalized: CanonicalizedVault = serde_json::from_str(&canonicalized_json).unwrap();assert!(canonicalized.first().manifest.tables.contains_key("Items"));let schema = fitting_schema(std::iter::once(&canonicalized.first().manifest), &canonicalized.data_buckets.clone());let input_value = json!({ "manifests": [canonicalized.first().manifest], "dataBuckets": canonicalized.data_buckets.clone(), "schemaColumns": schema });- let materialized_json = materialize_as_sqlite_json(&input_value.to_string()).unwrap();+ let materialized_json = crate::error::json_call(&input_value.to_string(), |input: MaterializeInput| materialize_as_sqlite(input)).unwrap();let materialized: MaterializedTables = serde_json::from_str(&materialized_json).unwrap();assert!(materialized.tables.iter().any(|t| t.name == "Items"));}@@ -830,7 +831,7 @@ pub(super) fn fitting_schema<'a>(manifests: impl IntoIterator<Item = &'a Manifes/// buckets hung off it and a schema fitted to everything they carry (see [`fitting_schema`]).pub(super) fn materialize_input(own: Manifest, others: Vec<Manifest>, data_buckets: Vec<DataBucket>) -> MaterializeInput {let schema = fitting_schema(std::iter::once(&own).chain(others.iter()), &data_buckets);- MaterializeInput::new(own, others, data_buckets, schema)+ MaterializeInput { manifests: std::iter::once(own).chain(others).collect(), data_buckets, schema_columns: schema }}/// A `MaterializeInput` from an explicit manifest list, for the tests that assert on inputsdiff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rsindex 61e524bf3..545fb2688 100644--- a/core/rust/src/vault_codec/types.rs+++ b/core/rust/src/vault_codec/types.rs@@ -54,6 +54,11 @@ pub fn is_skip_table(table_name: &str) -> bool {SKIP_TABLES.contains(&table_name)}+/// True when a table lives only in the local vault DB: a skip-table or the codec overflow carrier.+pub fn is_local_only_table(table_name: &str) -> bool {+ is_skip_table(table_name) || table_name == OVERFLOW_TABLE+}+/// The data-bucket category a table belongs to, if it is bucketed out of the manifest.pub fn bucket_category_for(table_name: &str) -> Option<&'static str> {BUCKET_TABLES.iter().find(|(t, _)| *t == table_name).map(|(_, c)| *c)@@ -96,22 +101,22 @@ pub fn is_bucketed_table(table_name: &str) -> bool {/// Get the primary key columns for a table.pub fn primary_key_columns_for(table_name: &str) -> &'static [&'static str] {- crate::vault_merge::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key_columns).unwrap_or(&["Id"])+ crate::vault_model::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.primary_key_columns).unwrap_or(&["Id"])}/// True when `table_name`'s rows are namespaced per manifest.pub fn is_manifest_scoped(table_name: &str) -> bool {- crate::vault_merge::SYNCABLE_TABLES.iter().any(|t| t.name == table_name && t.manifest_scoped)+ crate::vault_model::SYNCABLE_TABLES.iter().any(|t| t.name == table_name && t.manifest_scoped)}/// Every manifest-scoped table, in registry order.pub fn manifest_scoped_tables() -> Vec<&'static str> {- crate::vault_merge::SYNCABLE_TABLES.iter().filter(|t| t.manifest_scoped).map(|t| t.name).collect()+ crate::vault_model::SYNCABLE_TABLES.iter().filter(|t| t.manifest_scoped).map(|t| t.name).collect()}/// The columns that together identify one row of `table_name`.pub fn identity_columns_for(table_name: &str) -> Vec<&'static str> {- crate::vault_merge::SYNCABLE_TABLES+ crate::vault_model::SYNCABLE_TABLES.iter().find(|t| t.name == table_name).map(|t| t.identity_columns())diff --git a/core/rust/src/vault_codec/validate.rs b/core/rust/src/vault_codec/validate.rsindex eea8b8c3d..4bd930509 100644--- a/core/rust/src/vault_codec/validate.rs+++ b/core/rust/src/vault_codec/validate.rs@@ -4,7 +4,8 @@use serde::{Deserialize, Serialize};-use super::manifest::{DataBucket, Manifest, CodecRecord};+use super::manifest::{DataBucket, Manifest};+use super::row::{rows_of, str_col};use crate::vault_model::names::{FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_VALUES_TABLE, FOLDERS_TABLE, FOLDER_ID_COL, ID_COL,ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, KIND_COL, LOGOS_TABLE, LOGO_KIND_FAVICON, SOURCE_COL, TAGS_TABLE, TAG_ID_COL,@@ -21,14 +22,6 @@ pub struct ValidationResult {pub message: String,}-fn table<'a>(m: &'a Manifest, name: &str) -> &'a [CodecRecord] {- m.tables.get(name).map(|v| v.as_slice()).unwrap_or(&[])-}--fn str_field<'a>(r: &'a CodecRecord, key: &str) -> Option<&'a str> {- r.get(key).and_then(|v| v.as_str())-}-/// Structurally validate a fresh manifest before it is encrypted/uploaded.pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {let mut failed: Vec<String> = Vec::new();@@ -50,7 +43,7 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {return ValidationResult {ok: false,failed_rules: failed,- message: "Manifest has no tables — refusing upload.".to_string(),+ message: "Manifest has no tables, refusing upload.".to_string(),};}@@ -78,73 +71,73 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {* Every EncryptionKeys row must be stamped with the manifest's own id.*/let expected_scope = Some(manifest.manifest_id.as_str());- if table(manifest, super::types::ENCRYPTION_KEYS_TABLE).iter().any(|r| str_field(r, super::types::MANIFEST_ID_COL) != expected_scope) {+ if rows_of(&manifest.tables, super::types::ENCRYPTION_KEYS_TABLE).iter().any(|r| str_col(r, super::types::MANIFEST_ID_COL) != expected_scope) {failed.push("encryption-keys-scope-mismatch".to_string());explain.push("EncryptionKeys carries rows stamped for another manifest".to_string());}// Items and Folders are restamped by canonicalize: a mismatched stamp here means a codec bug.for name in [ITEMS_TABLE, FOLDERS_TABLE] {- if table(manifest, name).iter().any(|r| str_field(r, super::types::MANIFEST_ID_COL) != expected_scope) {+ if rows_of(&manifest.tables, name).iter().any(|r| str_col(r, super::types::MANIFEST_ID_COL) != expected_scope) {failed.push("content-scope-mismatch".to_string());explain.push(format!("{} carries rows stamped for another manifest", name));break;}}- let items = table(manifest, ITEMS_TABLE);- let folders = table(manifest, FOLDERS_TABLE);- let tags = table(manifest, TAGS_TABLE);- let item_tags = table(manifest, ITEM_TAGS_TABLE);- let field_values = table(manifest, FIELD_VALUES_TABLE);- let field_defs = table(manifest, FIELD_DEFINITIONS_TABLE);+ let items = rows_of(&manifest.tables, ITEMS_TABLE);+ let folders = rows_of(&manifest.tables, FOLDERS_TABLE);+ let tags = rows_of(&manifest.tables, TAGS_TABLE);+ let item_tags = rows_of(&manifest.tables, ITEM_TAGS_TABLE);+ let field_values = rows_of(&manifest.tables, FIELD_VALUES_TABLE);+ let field_defs = rows_of(&manifest.tables, FIELD_DEFINITIONS_TABLE);- let item_ids: std::collections::HashSet<&str> = items.iter().filter_map(|i| str_field(i, ID_COL)).collect();- let folder_ids: std::collections::HashSet<&str> = folders.iter().filter_map(|f| str_field(f, ID_COL)).collect();- let tag_ids: std::collections::HashSet<&str> = tags.iter().filter_map(|t| str_field(t, ID_COL)).collect();- let field_def_ids: std::collections::HashSet<&str> = field_defs.iter().filter_map(|f| str_field(f, ID_COL)).collect();+ let item_ids: std::collections::HashSet<&str> = items.iter().filter_map(|i| str_col(i, ID_COL)).collect();+ let folder_ids: std::collections::HashSet<&str> = folders.iter().filter_map(|f| str_col(f, ID_COL)).collect();+ let tag_ids: std::collections::HashSet<&str> = tags.iter().filter_map(|t| str_col(t, ID_COL)).collect();+ let field_def_ids: std::collections::HashSet<&str> = field_defs.iter().filter_map(|f| str_col(f, ID_COL)).collect();// Referential integrity.for item in items {- if let Some(folder_id) = str_field(item, FOLDER_ID_COL) {+ if let Some(folder_id) = str_col(item, FOLDER_ID_COL) {if !folder_ids.contains(folder_id) {failed.push("item-folder-fk-broken".to_string());- explain.push(format!("Item {} references missing folder {}", str_field(item, ID_COL).unwrap_or(""), folder_id));+ explain.push(format!("Item {} references missing folder {}", str_col(item, ID_COL).unwrap_or(""), folder_id));break;}}}for it in item_tags {- if let Some(item_id) = str_field(it, ITEM_ID_COL) {+ if let Some(item_id) = str_col(it, ITEM_ID_COL) {if !item_ids.contains(item_id) {failed.push("itemtag-item-fk-broken".to_string());- explain.push(format!("ItemTag {} references missing item {}", str_field(it, ID_COL).unwrap_or(""), item_id));+ explain.push(format!("ItemTag {} references missing item {}", str_col(it, ID_COL).unwrap_or(""), item_id));break;}}}for it in item_tags {- if let Some(tag_id) = str_field(it, TAG_ID_COL) {+ if let Some(tag_id) = str_col(it, TAG_ID_COL) {if !tag_ids.contains(tag_id) {failed.push("itemtag-tag-fk-broken".to_string());- explain.push(format!("ItemTag {} references missing tag {}", str_field(it, ID_COL).unwrap_or(""), tag_id));+ explain.push(format!("ItemTag {} references missing tag {}", str_col(it, ID_COL).unwrap_or(""), tag_id));break;}}}for fv in field_values {- if let Some(item_id) = str_field(fv, ITEM_ID_COL) {+ if let Some(item_id) = str_col(fv, ITEM_ID_COL) {if !item_ids.contains(item_id) {failed.push("fieldvalue-item-fk-broken".to_string());- explain.push(format!("FieldValue {} references missing item {}", str_field(fv, ID_COL).unwrap_or(""), item_id));+ explain.push(format!("FieldValue {} references missing item {}", str_col(fv, ID_COL).unwrap_or(""), item_id));break;}}}for fv in field_values {- if let Some(field_def_id) = str_field(fv, FIELD_DEFINITION_ID_COL) {+ if let Some(field_def_id) = str_col(fv, FIELD_DEFINITION_ID_COL) {if !field_def_ids.contains(field_def_id) {failed.push("fieldvalue-fielddef-fk-broken".to_string());break;@@ -164,19 +157,19 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {* A logo belongs to exactly one manifest: (ManifestId, Kind, Source) must be UNIQUE in the* client schema.*/- let logos = table(manifest, LOGOS_TABLE);+ let logos = rows_of(&manifest.tables, LOGOS_TABLE);let logo_keys: std::collections::HashSet<(String, String)> = logos.iter()- .filter_map(|l| Some((str_field(l, KIND_COL).unwrap_or(LOGO_KIND_FAVICON).to_lowercase(), str_field(l, SOURCE_COL)?.to_lowercase())))+ .filter_map(|l| Some((str_col(l, KIND_COL).unwrap_or(LOGO_KIND_FAVICON).to_lowercase(), str_col(l, SOURCE_COL)?.to_lowercase()))).collect();- let logos_with_source = logos.iter().filter(|l| str_field(l, SOURCE_COL).is_some()).count();+ let logos_with_source = logos.iter().filter(|l| str_col(l, SOURCE_COL).is_some()).count();if logo_keys.len() != logos_with_source {failed.push("logo-sources-not-unique".to_string());}// Every logo in a manifest must claim that manifest's scope, otherwise the row would materialize// into the wrong uniqueness bucket and could collide with the reader's own rows.- if logos.iter().any(|l| str_field(l, super::types::MANIFEST_ID_COL) != expected_scope) {+ if logos.iter().any(|l| str_col(l, super::types::MANIFEST_ID_COL) != expected_scope) {failed.push("logo-scope-mismatch".to_string());explain.push("Logos carry a ManifestId that is not this manifest's own id".to_string());}@@ -212,7 +205,7 @@ pub fn validate_data_bucket(bucket: &DataBucket) -> ValidationResult {if !super::types::is_manifest_scoped(name) {continue;}- if rows.iter().any(|row| str_field(row, super::types::MANIFEST_ID_COL) != expected_scope) {+ if rows.iter().any(|row| str_col(row, super::types::MANIFEST_ID_COL) != expected_scope) {failed.push("dataBucket-scope-mismatch".to_string());explain.push(format!("{} carries rows stamped for another manifest", name));break;