Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update Rust core to use transpiled Typescript model source files (#2159)

Leendert de Borst1a2c14937826
diff --git a/core/rust/README.md b/core/rust/README.md
index 87e7e14b0..ee9a2008c 100644
--- a/core/rust/README.md
+++ b/core/rust/README.md
@@ -366,7 +366,9 @@ pub fn my_function() -> VaultResult<String> {
| `src/lib.rs` | Entry point, exports all modules |
| `src/error.rs` | `VaultError` and `VaultResult` types |
| `src/vault_merge/mod.rs` | LWW merge implementation |
-| `src/vault_merge/types.rs` | Table configurations, composite keys |
+| `src/vault_merge/types.rs` | Re-exports the table registry for merge call sites |
+| `src/vault_model/mod.rs` | `TableConfig` type; owns the generated registry data |
+| `src/vault_model/generated.rs` | Registry data generated from `core/models/src/vault/VaultTableRegistry.ts` (do not edit) |
| `src/vault_pruner/mod.rs` | Trash cleanup implementation |
| `src/credential_matcher/mod.rs` | Autofill filtering algorithm |
| `src/credential_matcher/domain.rs` | URL/domain extraction utilities |
diff --git a/core/rust/src/lib.rs b/core/rust/src/lib.rs
index 9d33796db..b4d85a60e 100644
--- a/core/rust/src/lib.rs
+++ b/core/rust/src/lib.rs
@@ -17,6 +17,7 @@
pub mod error;
mod hex;
mod rng;
+pub mod vault_model;
pub mod vault_merge;
pub mod vault_codec;
pub mod vault_sharing;
diff --git a/core/rust/src/vault_codec/canonicalize.rs b/core/rust/src/vault_codec/canonicalize.rs
index 83c1b4965..f1b020048 100644
--- a/core/rust/src/vault_codec/canonicalize.rs
+++ b/core/rust/src/vault_codec/canonicalize.rs
@@ -26,6 +26,7 @@ use super::types::{
manifest_scoped_tables, row_identity, MANIFEST_ID_COL, OVERFLOW_TABLE, SCHEMA_VERSION,
};
use crate::error::VaultResult;
+use crate::vault_model::names::LOGOS_TABLE;
/// Canonicalize normalized tables into the split resources: the manifest, one data bucket per declared
/// category (see [`BUCKET_TABLES`](super::types::BUCKET_TABLES)), and the content-addressed blob map.
@@ -81,7 +82,7 @@ pub fn canonicalize_from_sqlite(input: CanonicalizeInput) -> VaultResult<Canonic
.filter_map(|name| all_tables.get(name).map(|rows| (name.to_string(), rows.clone())))
.collect();
let no_rows: Vec<CodecRecord> = Vec::new();
- let all_logos = snapshots.get("Logos").unwrap_or(&no_rows);
+ let all_logos = snapshots.get(LOGOS_TABLE).unwrap_or(&no_rows);
let other_specs: Vec<ManifestSpec> = input.manifests.iter().skip(1).cloned().collect();
let other_ids = other_specs.iter().map(|spec| spec.manifest_id.clone());
diff --git a/core/rust/src/vault_codec/normalize.rs b/core/rust/src/vault_codec/normalize.rs
index 73711010b..3073b4db9 100644
--- a/core/rust/src/vault_codec/normalize.rs
+++ b/core/rust/src/vault_codec/normalize.rs
@@ -6,19 +6,10 @@ use serde_json::json;
use super::manifest::CodecRecord;
use super::types::{MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS};
-
-const FIELD_VALUES_TABLE: &str = "FieldValues";
-const FIELD_HISTORIES_TABLE: &str = "FieldHistories";
-const FIELD_DEFINITIONS_TABLE: &str = "FieldDefinitions";
-const ITEM_TAGS_TABLE: &str = "ItemTags";
-const ID_COL: &str = "Id";
-const ITEM_ID_COL: &str = "ItemId";
-const TAG_ID_COL: &str = "TagId";
-const FIELD_KEY_COL: &str = "FieldKey";
-const FIELD_DEFINITION_ID_COL: &str = "FieldDefinitionId";
-const VALUE_INDEX_COL: &str = "ValueIndex";
-const CHANGED_AT_COL: &str = "ChangedAt";
-const IS_MULTI_VALUE_COL: &str = "IsMultiValue";
+use crate::vault_model::names::{
+ CHANGED_AT_COL, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_KEY_COL,
+ FIELD_VALUES_TABLE, ID_COL, IS_MULTI_VALUE_COL, ITEM_ID_COL, ITEM_TAGS_TABLE, TAG_ID_COL, VALUE_INDEX_COL,
+};
/// Domain-separation prefix for derived field value ids.
const FIELD_VALUE_ID_NAMESPACE: &str = "aliasvault:fieldvalue:v1";
diff --git a/core/rust/src/vault_codec/scoped_assets.rs b/core/rust/src/vault_codec/scoped_assets.rs
index 0554601ae..18a886f63 100644
--- a/core/rust/src/vault_codec/scoped_assets.rs
+++ b/core/rust/src/vault_codec/scoped_assets.rs
@@ -21,25 +21,17 @@ use serde_json::{json, Value};
use super::manifest::CodecRecord;
use super::types::MANIFEST_ID_COL;
-
-const LOGOS_TABLE: &str = "Logos";
-const ITEMS_TABLE: &str = "Items";
-const LOGO_ID_COL: &str = "LogoId";
-const KIND_COL: &str = "Kind";
-const SOURCE_COL: &str = "Source";
-const ID_COL: &str = "Id";
-const FILE_DATA_COL: &str = "FileData";
-const UPDATED_AT_COL: &str = "UpdatedAt";
+use crate::vault_model::names::{FILE_DATA_COL, ID_COL, ITEMS_TABLE, KIND_COL, LOGOS_TABLE, LOGO_ID_COL, SOURCE_COL, UPDATED_AT_COL};
/// The kind of a logo fetched automatically from an item's URL. Also what a row that carries no
/// `Kind` at all means: it was written before the column existed, when every logo was a favicon.
-pub const KIND_FAVICON: &str = "favicon";
+pub use crate::vault_model::names::LOGO_KIND_FAVICON as KIND_FAVICON;
/// The kind of a logo picked from the built-in catalog.
-pub const KIND_BUILTIN: &str = "builtin";
+pub use crate::vault_model::names::LOGO_KIND_BUILTIN as KIND_BUILTIN;
/// The kind of a logo the user uploaded.
-pub const KIND_CUSTOM: &str = "custom";
+pub use crate::vault_model::names::LOGO_KIND_CUSTOM as KIND_CUSTOM;
/// Domain-separation prefix for favicon ids. It predates the `Kind` column and is kept verbatim so
/// every favicon row that already exists keeps its id: changing it would re-mint the logo of every
diff --git a/core/rust/src/vault_codec/sharing.rs b/core/rust/src/vault_codec/sharing.rs
index 29a4062e6..4f2d388cf 100644
--- a/core/rust/src/vault_codec/sharing.rs
+++ b/core/rust/src/vault_codec/sharing.rs
@@ -40,24 +40,13 @@ use super::types::{
MANIFEST_ID_COL, OVERFLOW_TABLE,
};
use crate::error::{VaultError, VaultResult};
+use crate::vault_model::names::{
+ FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_VALUES_TABLE, FOLDERS_TABLE,
+ FOLDER_ID_COL, ID_COL, IS_DELETED_COL, IS_PRIMARY_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, LOGOS_TABLE,
+ LOGO_ID_COL, PARENT_FOLDER_ID_COL, PUBLIC_KEY_COL, TAGS_TABLE, TAG_ID_COL,
+};
-const FOLDERS_TABLE: &str = "Folders";
-const ITEMS_TABLE: &str = "Items";
-const LOGOS_TABLE: &str = "Logos";
-const PARENT_FOLDER_ID_COL: &str = "ParentFolderId";
-const ITEM_ID_COL: &str = "ItemId";
-const ID_COL: &str = "Id";
-const TAGS_TABLE: &str = "Tags";
-const ITEM_TAGS_TABLE: &str = "ItemTags";
-const TAG_ID_COL: &str = "TagId";
-const FIELD_DEFINITIONS_TABLE: &str = "FieldDefinitions";
-const FIELD_DEFINITION_ID_COL: &str = "FieldDefinitionId";
-
-/// Rows referenced from inside a manifest: `(target_table, [(referencing_table, column)])`.
-static REFERENCED_TABLES: &[(&str, &[(&str, &str)])] = &[
- ("Tags", &[("ItemTags", "TagId")]),
- ("FieldDefinitions", &[("FieldValues", "FieldDefinitionId"), ("FieldHistories", "FieldDefinitionId")]),
-];
+use crate::vault_model::REFERENCED_TABLES;
/// The tables a caller must snapshot before routing, so the referenced content can be copied out of
/// them afterwards: every [`REFERENCED_TABLES`] target plus `Logos`.
@@ -394,7 +383,7 @@ pub(super) fn clone_referenced_rows(tables: &mut HashMap<String, Vec<CodecRecord
pub(super) fn prune_unreferenced_logos(tables: &mut HashMap<String, Vec<CodecRecord>>) {
let referenced: HashSet<String> = tables
.get(ITEMS_TABLE)
- .map(|rows| rows.iter().filter_map(|r| str_col(r, "LogoId")).map(str::to_string).collect())
+ .map(|rows| rows.iter().filter_map(|r| str_col(r, LOGO_ID_COL)).map(str::to_string).collect())
.unwrap_or_default();
if let Some(logos) = tables.get_mut(LOGOS_TABLE) {
logos.retain(|row| is_custom_logo(row) || str_col(row, ID_COL).map(|id| referenced.contains(id)).unwrap_or(false));
@@ -522,7 +511,7 @@ fn null_dangling_parent_folders(tables: &mut HashMap<String, Vec<CodecRecord>>)
/// Null every `Items.FolderId` that doesn't resolve to a folder *in the same manifest*.
fn null_dangling_item_folders(tables: &mut HashMap<String, Vec<CodecRecord>>) {
- null_dangling_reference(tables, ITEMS_TABLE, FOLDERS_TABLE, "FolderId");
+ null_dangling_reference(tables, ITEMS_TABLE, FOLDERS_TABLE, FOLDER_ID_COL);
}
/// Null every `FieldDefinitionId` that names no definition in the referencing row's own manifest.
@@ -531,7 +520,7 @@ fn null_dangling_item_folders(tables: &mut HashMap<String, Vec<CodecRecord>>) {
/// keeps the user's value and loses only the metadata this vault does not hold anyway. Dropping the row
/// instead would throw away what the user actually typed.
fn null_dangling_field_definitions(tables: &mut HashMap<String, Vec<CodecRecord>>) {
- for table in ["FieldValues", "FieldHistories"] {
+ for table in [FIELD_VALUES_TABLE, FIELD_HISTORIES_TABLE] {
null_dangling_reference(tables, table, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL);
}
}
@@ -619,7 +608,7 @@ pub fn extract_encryption_key_for_public_key(manifest: &Manifest, public_key: &s
.tables
.get(ENCRYPTION_KEYS_TABLE)?
.iter()
- .find(|row| str_col(row, "PublicKey") == Some(public_key) && str_col(row, MANIFEST_ID_COL) == scope && !is_truthy(row.get("IsDeleted")))
+ .find(|row| str_col(row, PUBLIC_KEY_COL) == Some(public_key) && str_col(row, MANIFEST_ID_COL) == scope && !is_truthy(row.get(IS_DELETED_COL)))
.cloned()
}
@@ -632,7 +621,7 @@ pub fn active_encryption_key(manifest: &Manifest) -> Option<CodecRecord> {
.tables
.get(ENCRYPTION_KEYS_TABLE)?
.iter()
- .find(|row| is_truthy(row.get("IsPrimary")) && str_col(row, MANIFEST_ID_COL) == scope && !is_truthy(row.get("IsDeleted")))
+ .find(|row| is_truthy(row.get(IS_PRIMARY_COL)) && str_col(row, MANIFEST_ID_COL) == scope && !is_truthy(row.get(IS_DELETED_COL)))
.cloned()
}
diff --git a/core/rust/src/vault_codec/types.rs b/core/rust/src/vault_codec/types.rs
index 09a191347..98997b6d5 100644
--- a/core/rust/src/vault_codec/types.rs
+++ b/core/rust/src/vault_codec/types.rs
@@ -1,70 +1,19 @@
//! Format constants for the manifest-v1 storage layout.
//!
-//! These constants are defined here so every platform shares the exact same rules.
+//! The datamodel registry data (tables, keys, bucket layout, blob columns, sentinels) lives in
+//! [`crate::vault_model`], generated from the TypeScript source of truth in
+//! core/models/src/vault/VaultTableRegistry.ts; this module re-exports it for the codec and adds
+//! the codec-owned accessors on top.
-// ---------------------------------------------------------------------------
-// Static / const definitions
-// ---------------------------------------------------------------------------
-
-/// The SQLite columns whose contents are extracted into content-addressed blobs rather than
-/// kept inline in the manifest. Tuple form `(table_name, blob_column, kind_label)`. The kind label
-/// is reported to the server on upload (used for metrics / retention).
-pub static BLOB_COLUMNS: &[(&str, &str, &str)] = &[
- ("Logos", "FileData", "favicon"),
- ("Attachments", "Blob", "attachment"),
-];
-
-/// Tables never serialized into the server-stored manifest: internal SQLite, platform, or EF bookkeeping
-/// that is temporary and only used/required during runtime, and therefore should not become part of a persisted manifest.
-pub static SKIP_TABLES: &[&str] = &[
- "__EFMigrationsHistory",
- "__EFMigrationsLock",
- "sqlite_sequence",
- "android_metadata",
- "Manifests",
-];
-
-/// Tables split out of the manifest into a data bucket, keyed by category, so each bucket syncs on its
-/// own server revision without rewriting the manifest. Tuple form `(table_name, bucket_category)`;
-/// `category` mirrors the server `VaultDataBucketCategory`. Several tables may share a category to sync together.
-pub static BUCKET_TABLES: &[(&str, &str)] = &[
- ("Settings", "Settings"),
- ("ItemStats", "Stats"),
-];
-
-/// Tables that belong exclusively to the user's own (personal) vault, never to a shared manifest.
-/// Empty today, and deliberately independent of [`BUCKET_TABLES`]: a bucketed table is kept out of the
-/// manifest blob, which is a different question from which manifests may hold its rows at all. Kept as
-/// the declaration point for a future personal-only table.
-pub static PERSONAL_TABLES: &[&str] = &[];
-
-/// System field keys whose field holds multiple values. A value of such a field owns its row id
-/// (two devices each adding a value are adding two different things) and are not derived.
-pub static MULTI_VALUE_FIELD_KEYS: &[&str] = &["login.url"];
-
-/// The per-manifest delivery-keypair table. Every manifest carries its own asymmetric keypair(s),
-/// stamped with that manifest's id (`ManifestId`).
-pub const ENCRYPTION_KEYS_TABLE: &str = "EncryptionKeys";
-
-/// The scope column every stamped table carries: the id of the manifest that owns the row.
-pub const MANIFEST_ID_COL: &str = "ManifestId";
-
-/// Local bookkeeping table materialize writes into the vault DB: one row per manifest this
-/// vault is materialized from (`Id`, `Name`).
-pub const MANIFESTS_TABLE: &str = "Manifests";
+pub use crate::vault_model::{
+ BLOB_COLUMNS, BUCKET_TABLES, ENCRYPTION_KEYS_TABLE, MANIFESTS_TABLE, MANIFEST_ID_COL,
+ MULTI_VALUE_FIELD_KEYS, OVERFLOW_ROW_ID, OVERFLOW_TABLE, PERSONAL_TABLES, SKIP_TABLES,
+ UNSTAMPED_SCOPE_SENTINEL,
+};
/// Manifest / metadata schema version.
pub const SCHEMA_VERSION: u32 = 1;
-/// Client-local SQLite table that carries the codec overflow inside the vault database itself (see
-/// `CodecOverflow`): materialize writes a single row `{ Id: OVERFLOW_ROW_ID, Data: <json> }`, and
-/// canonicalize / extract_bucket consume it to build the manifest.
-pub const OVERFLOW_TABLE: &str = "CodecOverflows";
-
-/// Fixed sentinel primary key of the single `OVERFLOW_TABLE` row (deterministic on purpose:
-/// materialize output must not depend on a random source).
-pub const OVERFLOW_ROW_ID: &str = "00000000-0000-0000-0000-00000000c0de";
-
// ---------------------------------------------------------------------------
// Accessor methods
// ---------------------------------------------------------------------------
@@ -101,9 +50,6 @@ pub fn tables_for_category(category: &str) -> Vec<&'static str> {
BUCKET_TABLES.iter().filter(|(_, c)| *c == category).map(|(t, _)| *t).collect()
}
-/// All zero GUID used for default values which indicate unstamped rows.
-pub const UNSTAMPED_SCOPE_SENTINEL: &str = "00000000-0000-0000-0000-000000000000";
-
/// True when a `ManifestId` value names no manifest.
pub fn is_unstamped_scope(scope: Option<&str>) -> bool {
match scope {
diff --git a/core/rust/src/vault_codec/validate.rs b/core/rust/src/vault_codec/validate.rs
index e149feacc..eea8b8c3d 100644
--- a/core/rust/src/vault_codec/validate.rs
+++ b/core/rust/src/vault_codec/validate.rs
@@ -5,6 +5,10 @@
use serde::{Deserialize, Serialize};
use super::manifest::{DataBucket, Manifest, CodecRecord};
+use crate::vault_model::names::{
+ FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_VALUES_TABLE, FOLDERS_TABLE, FOLDER_ID_COL, ID_COL,
+ ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, KIND_COL, LOGOS_TABLE, LOGO_KIND_FAVICON, SOURCE_COL, TAGS_TABLE, TAG_ID_COL,
+};
/// Outcome of a structural validation run.
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -80,7 +84,7 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {
}
// Items and Folders are restamped by canonicalize: a mismatched stamp here means a codec bug.
- for name in ["Items", "Folders"] {
+ for name in [ITEMS_TABLE, FOLDERS_TABLE] {
if table(manifest, name).iter().any(|r| str_field(r, super::types::MANIFEST_ID_COL) != expected_scope) {
failed.push("content-scope-mismatch".to_string());
explain.push(format!("{} carries rows stamped for another manifest", name));
@@ -88,59 +92,59 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {
}
}
- let items = table(manifest, "Items");
- let folders = table(manifest, "Folders");
- let tags = table(manifest, "Tags");
- let item_tags = table(manifest, "ItemTags");
- let field_values = table(manifest, "FieldValues");
- let field_defs = table(manifest, "FieldDefinitions");
+ let items = table(manifest, ITEMS_TABLE);
+ let folders = table(manifest, FOLDERS_TABLE);
+ let tags = table(manifest, TAGS_TABLE);
+ let item_tags = table(manifest, ITEM_TAGS_TABLE);
+ let field_values = table(manifest, FIELD_VALUES_TABLE);
+ let field_defs = table(manifest, FIELD_DEFINITIONS_TABLE);
- let item_ids: std::collections::HashSet<&str> = items.iter().filter_map(|i| str_field(i, "Id")).collect();
- let folder_ids: std::collections::HashSet<&str> = folders.iter().filter_map(|f| str_field(f, "Id")).collect();
- let tag_ids: std::collections::HashSet<&str> = tags.iter().filter_map(|t| str_field(t, "Id")).collect();
- let field_def_ids: std::collections::HashSet<&str> = field_defs.iter().filter_map(|f| str_field(f, "Id")).collect();
+ let item_ids: std::collections::HashSet<&str> = items.iter().filter_map(|i| str_field(i, ID_COL)).collect();
+ let folder_ids: std::collections::HashSet<&str> = folders.iter().filter_map(|f| str_field(f, ID_COL)).collect();
+ let tag_ids: std::collections::HashSet<&str> = tags.iter().filter_map(|t| str_field(t, ID_COL)).collect();
+ let field_def_ids: std::collections::HashSet<&str> = field_defs.iter().filter_map(|f| str_field(f, ID_COL)).collect();
// Referential integrity.
for item in items {
- if let Some(folder_id) = str_field(item, "FolderId") {
+ if let Some(folder_id) = str_field(item, FOLDER_ID_COL) {
if !folder_ids.contains(folder_id) {
failed.push("item-folder-fk-broken".to_string());
- explain.push(format!("Item {} references missing folder {}", str_field(item, "Id").unwrap_or(""), folder_id));
+ explain.push(format!("Item {} references missing folder {}", str_field(item, ID_COL).unwrap_or(""), folder_id));
break;
}
}
}
for it in item_tags {
- if let Some(item_id) = str_field(it, "ItemId") {
+ if let Some(item_id) = str_field(it, ITEM_ID_COL) {
if !item_ids.contains(item_id) {
failed.push("itemtag-item-fk-broken".to_string());
- explain.push(format!("ItemTag {} references missing item {}", str_field(it, "Id").unwrap_or(""), item_id));
+ explain.push(format!("ItemTag {} references missing item {}", str_field(it, ID_COL).unwrap_or(""), item_id));
break;
}
}
}
for it in item_tags {
- if let Some(tag_id) = str_field(it, "TagId") {
+ if let Some(tag_id) = str_field(it, TAG_ID_COL) {
if !tag_ids.contains(tag_id) {
failed.push("itemtag-tag-fk-broken".to_string());
- explain.push(format!("ItemTag {} references missing tag {}", str_field(it, "Id").unwrap_or(""), tag_id));
+ explain.push(format!("ItemTag {} references missing tag {}", str_field(it, ID_COL).unwrap_or(""), tag_id));
break;
}
}
}
for fv in field_values {
- if let Some(item_id) = str_field(fv, "ItemId") {
+ if let Some(item_id) = str_field(fv, ITEM_ID_COL) {
if !item_ids.contains(item_id) {
failed.push("fieldvalue-item-fk-broken".to_string());
- explain.push(format!("FieldValue {} references missing item {}", str_field(fv, "Id").unwrap_or(""), item_id));
+ explain.push(format!("FieldValue {} references missing item {}", str_field(fv, ID_COL).unwrap_or(""), item_id));
break;
}
}
}
for fv in field_values {
- if let Some(field_def_id) = str_field(fv, "FieldDefinitionId") {
+ if let Some(field_def_id) = str_field(fv, FIELD_DEFINITION_ID_COL) {
if !field_def_ids.contains(field_def_id) {
failed.push("fieldvalue-fielddef-fk-broken".to_string());
break;
@@ -160,12 +164,12 @@ pub fn validate_manifest(manifest: &Manifest) -> ValidationResult {
* A logo belongs to exactly one manifest: (ManifestId, Kind, Source) must be UNIQUE in the
* client schema.
*/
- let logos = table(manifest, "Logos");
+ let logos = table(manifest, LOGOS_TABLE);
let logo_keys: std::collections::HashSet<(String, String)> = logos
.iter()
- .filter_map(|l| Some((str_field(l, "Kind").unwrap_or("favicon").to_lowercase(), str_field(l, "Source")?.to_lowercase())))
+ .filter_map(|l| Some((str_field(l, KIND_COL).unwrap_or(LOGO_KIND_FAVICON).to_lowercase(), str_field(l, SOURCE_COL)?.to_lowercase())))
.collect();
- let logos_with_source = logos.iter().filter(|l| str_field(l, "Source").is_some()).count();
+ let logos_with_source = logos.iter().filter(|l| str_field(l, SOURCE_COL).is_some()).count();
if logo_keys.len() != logos_with_source {
failed.push("logo-sources-not-unique".to_string());
}
diff --git a/core/rust/src/vault_merge/types.rs b/core/rust/src/vault_merge/types.rs
index 0c995cc6b..beb79e451 100644
--- a/core/rust/src/vault_merge/types.rs
+++ b/core/rust/src/vault_merge/types.rs
@@ -1,132 +1,12 @@
//! Common types used across the AliasVault core library.
+//!
+//! The table registry itself lives in [`crate::vault_model`], generated from the TypeScript source
+//! of truth in core/models/src/vault/VaultTableRegistry.ts; this module re-exports it for the
+//! merge code and its existing call sites.
-use crate::vault_codec::MANIFEST_ID_COL;
-
-/// Configuration for a syncable table.
-#[derive(Debug, Clone)]
-pub struct TableConfig {
- /// Table name in the database
- pub name: &'static str,
- /// Columns to use for composite key matching (if any).
- /// When empty, matching falls back to the table's [`identity_columns`](TableConfig::identity_columns).
- /// When set, these columns are concatenated to form the composite key.
- pub composite_key_columns: &'static [&'static str],
- /// Match columns for the canonical merge only, where both sides are first normalized to the manifest
- /// shape (see `vault_codec::normalize`). Empty means the canonical merge uses the same rule as
- /// the statement merge (composite key, else identity).
- pub canonical_key_columns: &'static [&'static str],
- /// The columns that name a row within its manifest. Defaults to `["Id"]`; tables keyed
- /// differently (Settings by "Key", ItemTags by its natural key) override this.
- pub primary_key_columns: &'static [&'static str],
- /// True when the table's rows are namespaced per manifest.
- pub manifest_scoped: bool,
-}
-
-impl TableConfig {
- pub const fn new(name: &'static str) -> Self {
- Self {
- name,
- composite_key_columns: &[],
- canonical_key_columns: &[],
- primary_key_columns: &["Id"],
- manifest_scoped: false,
- }
- }
-
- pub const fn with_composite_key(mut self, columns: &'static [&'static str]) -> Self {
- self.composite_key_columns = columns;
- self
- }
-
- pub const fn with_canonical_key(mut self, columns: &'static [&'static str]) -> Self {
- self.canonical_key_columns = columns;
- self
- }
-
- pub const fn with_primary_key(mut self, columns: &'static [&'static str]) -> Self {
- self.primary_key_columns = columns;
- self
- }
-
- pub const fn manifest_scoped(mut self) -> Self {
- self.manifest_scoped = true;
- self
- }
-
- /// Returns true if this table uses composite key matching.
- pub const fn uses_composite_key(&self) -> bool {
- !self.composite_key_columns.is_empty()
- }
-
- /// The columns that together identify one row of this table: `(ManifestId, primary key columns)`
- /// for a manifest-scoped table, the primary key columns alone otherwise.
- pub fn identity_columns(&self) -> Vec<&'static str> {
- if self.manifest_scoped {
- let mut columns = Vec::with_capacity(1 + self.primary_key_columns.len());
- columns.push(MANIFEST_ID_COL);
- columns.extend_from_slice(self.primary_key_columns);
- columns
- } else {
- self.primary_key_columns.to_vec()
- }
- }
-}
-
-/// All tables that need LWW merge.
-/// Allows for specifying manifest_scoped(), custom composite key columns and primary key columns for each table.
-pub static SYNCABLE_TABLES: &[TableConfig] = &[
- TableConfig::new("Items").manifest_scoped(),
- // Keyed by the item it describes: `Id` *is* the item's id, so recording a use is an upsert and two
- // devices never mint competing rows. Listed after Items so a merge inserts the item first.
- TableConfig::new("ItemStats").manifest_scoped(),
- // Legacy statement merge: a field value matches on the field it belongs to (FieldKey for system
- // fields, FieldDefinitionId for custom ones; exactly one is set), so independently minted rows of
- // the same field converge. Canonical merge: both sides are normalized first, which strips the
- // derived id of every single-value row, so adding `Id` to the key makes a single-value row match
- // by its field (id part empty on both sides) while a multi-value row matches by its OWNED id:
- // two devices each adding a `login.url` are two different rows that must both survive, and the id,
- // unlike ValueIndex, is stable under reordering. See vault_codec::normalize.
- TableConfig::new("FieldValues")
- .manifest_scoped()
- .with_composite_key(&[MANIFEST_ID_COL, "ItemId", "FieldKey", "FieldDefinitionId"])
- .with_canonical_key(&[MANIFEST_ID_COL, "ItemId", "FieldKey", "FieldDefinitionId", "Id"]),
- TableConfig::new("Folders").manifest_scoped(),
- TableConfig::new("Tags").manifest_scoped(),
- // A pure join table keyed by its natural key; it carries no surrogate id (since client schema 2.1.6),
- // so two devices tagging the same item converge on one row.
- TableConfig::new("ItemTags").manifest_scoped().with_primary_key(&["ItemId", "TagId"]),
- TableConfig::new("Attachments").manifest_scoped(),
- TableConfig::new("TotpCodes").manifest_scoped(),
- TableConfig::new("Passkeys").manifest_scoped(),
- TableConfig::new("FieldDefinitions").manifest_scoped(),
- // Canonical merge: every history row derives its id from `(item, field, ChangedAt)`, so after
- // normalization the natural key IS the identity: concurrent changes union (distinct ChangedAt),
- // same-millisecond snapshots converge. Legacy statement merge keeps plain (ManifestId, Id).
- TableConfig::new("FieldHistories").manifest_scoped().with_canonical_key(&[MANIFEST_ID_COL, "ItemId", "FieldKey", "FieldDefinitionId", "ChangedAt"]),
- TableConfig::new("Logos").manifest_scoped(),
- TableConfig::new("EncryptionKeys").manifest_scoped(),
- TableConfig::new("Settings").with_primary_key(&["Key"]).manifest_scoped(),
-];
+pub use crate::vault_model::{TableConfig, SYNCABLE_TABLES, SYNCABLE_TABLE_NAMES};
/// The tables a platform must read into the merge input: the syncable tables.
pub fn merge_table_names() -> Vec<&'static str> {
SYNCABLE_TABLE_NAMES.to_vec()
}
-
-/// List of syncable table names (for clients to know which tables to read).
-pub const SYNCABLE_TABLE_NAMES: &[&str] = &[
- "Items",
- "ItemStats",
- "FieldValues",
- "Folders",
- "Tags",
- "ItemTags",
- "Attachments",
- "TotpCodes",
- "Passkeys",
- "FieldDefinitions",
- "FieldHistories",
- "Logos",
- "EncryptionKeys",
- "Settings",
-];
diff --git a/core/rust/src/vault_model/generated.rs b/core/rust/src/vault_model/generated.rs
new file mode 100644
index 000000000..0426f8dfe
--- /dev/null
+++ b/core/rust/src/vault_model/generated.rs
@@ -0,0 +1,203 @@
+//! Generated client vault datamodel registry data.
+//!
+//! @generated: do NOT edit this file directly. It is generated from
+//! core/models/src/vault/VaultTableRegistry.ts by core/models/scripts/generate-vault-table-registry.cjs.
+//! Edit the TypeScript source and run 'core/models/build.sh' to regenerate.
+
+use super::TableConfig;
+
+/// All tables that need LWW merge, in registry order. Order is load-bearing: a merge inserts rows
+/// in this order, so child tables must be listed after the table they reference (Items first).
+/// Per-table rationale (merge key choices, natural keys) is documented in the TypeScript source.
+pub static SYNCABLE_TABLES: &[TableConfig] = &[
+ TableConfig::new("Items").manifest_scoped(),
+ TableConfig::new("ItemStats").manifest_scoped().item_child(),
+ TableConfig::new("FieldValues")
+ .manifest_scoped()
+ .item_child()
+ .with_composite_key(&["ManifestId", "ItemId", "FieldKey", "FieldDefinitionId"])
+ .with_canonical_key(&["ManifestId", "ItemId", "FieldKey", "FieldDefinitionId", "Id"]),
+ TableConfig::new("Folders").manifest_scoped(),
+ TableConfig::new("Tags").manifest_scoped(),
+ TableConfig::new("ItemTags").manifest_scoped().item_child().with_primary_key(&["ItemId", "TagId"]),
+ TableConfig::new("Attachments").manifest_scoped().item_child(),
+ TableConfig::new("TotpCodes").manifest_scoped().item_child(),
+ TableConfig::new("Passkeys").manifest_scoped().item_child(),
+ TableConfig::new("FieldDefinitions").manifest_scoped(),
+ TableConfig::new("FieldHistories")
+ .manifest_scoped()
+ .item_child()
+ .with_canonical_key(&["ManifestId", "ItemId", "FieldKey", "FieldDefinitionId", "ChangedAt"]),
+ TableConfig::new("Logos").manifest_scoped(),
+ TableConfig::new("EncryptionKeys").manifest_scoped(),
+ TableConfig::new("Settings").manifest_scoped().with_primary_key(&["Key"]),
+];
+
+/// List of syncable table names (for clients to know which tables to read).
+pub const SYNCABLE_TABLE_NAMES: &[&str] = &[
+ "Items",
+ "ItemStats",
+ "FieldValues",
+ "Folders",
+ "Tags",
+ "ItemTags",
+ "Attachments",
+ "TotpCodes",
+ "Passkeys",
+ "FieldDefinitions",
+ "FieldHistories",
+ "Logos",
+ "EncryptionKeys",
+ "Settings",
+];
+
+/// The SQLite columns whose contents are extracted into content-addressed blobs rather than
+/// kept inline in the manifest. Tuple form `(table_name, blob_column, kind_label)`. The kind label
+/// is reported to the server on upload (used for metrics / retention).
+pub static BLOB_COLUMNS: &[(&str, &str, &str)] = &[
+ ("Attachments", "Blob", "attachment"),
+ ("Logos", "FileData", "favicon"),
+];
+
+/// Tables never serialized into the server-stored manifest: internal SQLite, platform, or EF bookkeeping
+/// that is temporary and only used/required during runtime, and therefore should not become part of a persisted manifest.
+pub static SKIP_TABLES: &[&str] = &[
+ "__EFMigrationsHistory",
+ "__EFMigrationsLock",
+ "sqlite_sequence",
+ "android_metadata",
+ "Manifests",
+];
+
+/// Tables split out of the manifest into a data bucket, keyed by category, so each bucket syncs on its
+/// own server revision without rewriting the manifest. Tuple form `(table_name, bucket_category)`;
+/// `category` mirrors the server `VaultDataBucketCategory`. Several tables may share a category to sync together.
+pub static BUCKET_TABLES: &[(&str, &str)] = &[
+ ("Settings", "Settings"),
+ ("ItemStats", "Stats"),
+];
+
+/// Tables that belong exclusively to the user's own (personal) vault, never to a shared manifest.
+/// Deliberately independent of [`BUCKET_TABLES`]: a bucketed table is kept out of the manifest blob,
+/// which is a different question from which manifests may hold its rows at all.
+pub static PERSONAL_TABLES: &[&str] = &[];
+
+/// System field keys whose field holds multiple values, derived from SystemFieldRegistry
+/// (IsMultiValue). A value of such a field owns its row id (two devices each adding a value are
+/// adding two different things) and is not derived.
+pub static MULTI_VALUE_FIELD_KEYS: &[&str] = &["login.url"];
+
+/// The per-manifest delivery-keypair table. Every manifest carries its own asymmetric keypair(s),
+/// stamped with that manifest's id (`ManifestId`).
+pub const ENCRYPTION_KEYS_TABLE: &str = "EncryptionKeys";
+
+/// The scope column every stamped table carries: the id of the manifest that owns the row.
+pub const MANIFEST_ID_COL: &str = "ManifestId";
+
+/// Local bookkeeping table materialize writes into the vault DB: one row per manifest this
+/// vault is materialized from (`Id`, `Name`).
+pub const MANIFESTS_TABLE: &str = "Manifests";
+
+/// Client-local SQLite table that carries the codec overflow inside the vault database itself (see
+/// `CodecOverflow`): materialize writes a single row `{ Id: OVERFLOW_ROW_ID, Data: <json> }`, and
+/// canonicalize / extract_bucket consume it to build the manifest.
+pub const OVERFLOW_TABLE: &str = "CodecOverflows";
+
+/// Fixed sentinel primary key of the single `OVERFLOW_TABLE` row (deterministic on purpose:
+/// materialize output must not depend on a random source).
+pub const OVERFLOW_ROW_ID: &str = "00000000-0000-0000-0000-00000000c0de";
+
+/// All zero GUID used for default values which indicate unstamped rows.
+pub const UNSTAMPED_SCOPE_SENTINEL: &str = "00000000-0000-0000-0000-000000000000";
+
+/// Rows referenced from inside manifest content: `(target_table, [(referencing_table, column)])`.
+/// On a manifest split the referenced rows are reference-copied into the destination manifest.
+pub static REFERENCED_TABLES: &[(&str, &[(&str, &str)])] = &[
+ ("Tags", &[("ItemTags", "TagId")]),
+ ("FieldDefinitions", &[("FieldValues", "FieldDefinitionId"), ("FieldHistories", "FieldDefinitionId")]),
+];
+
+/// Default trash retention in days: how long a trashed item survives before the pruner deletes it.
+pub const TRASH_RETENTION_DEFAULT_DAYS: u32 = 30;
+
+/// Table, column, and logo-kind name vocabulary shared by the vault logic, so every module spells
+/// them identically. The full column set of each table is deliberately not modeled here: it lives
+/// in the SQL schema and is passed to the codec at runtime.
+pub mod names {
+ /// The `Items` table.
+ pub const ITEMS_TABLE: &str = "Items";
+ /// The `ItemStats` table.
+ pub const ITEM_STATS_TABLE: &str = "ItemStats";
+ /// The `FieldValues` table.
+ pub const FIELD_VALUES_TABLE: &str = "FieldValues";
+ /// The `Folders` table.
+ pub const FOLDERS_TABLE: &str = "Folders";
+ /// The `Tags` table.
+ pub const TAGS_TABLE: &str = "Tags";
+ /// The `ItemTags` table.
+ pub const ITEM_TAGS_TABLE: &str = "ItemTags";
+ /// The `Attachments` table.
+ pub const ATTACHMENTS_TABLE: &str = "Attachments";
+ /// The `TotpCodes` table.
+ pub const TOTP_CODES_TABLE: &str = "TotpCodes";
+ /// The `Passkeys` table.
+ pub const PASSKEYS_TABLE: &str = "Passkeys";
+ /// The `FieldDefinitions` table.
+ pub const FIELD_DEFINITIONS_TABLE: &str = "FieldDefinitions";
+ /// The `FieldHistories` table.
+ pub const FIELD_HISTORIES_TABLE: &str = "FieldHistories";
+ /// The `Logos` table.
+ pub const LOGOS_TABLE: &str = "Logos";
+ /// The `EncryptionKeys` table.
+ pub const ENCRYPTION_KEYS_TABLE: &str = "EncryptionKeys";
+ /// The `Settings` table.
+ pub const SETTINGS_TABLE: &str = "Settings";
+
+ /// The `Id` column.
+ pub const ID_COL: &str = "Id";
+ /// The `ItemId` column.
+ pub const ITEM_ID_COL: &str = "ItemId";
+ /// The `TagId` column.
+ pub const TAG_ID_COL: &str = "TagId";
+ /// The `FieldKey` column.
+ pub const FIELD_KEY_COL: &str = "FieldKey";
+ /// The `FieldDefinitionId` column.
+ pub const FIELD_DEFINITION_ID_COL: &str = "FieldDefinitionId";
+ /// The `ValueIndex` column.
+ pub const VALUE_INDEX_COL: &str = "ValueIndex";
+ /// The `ChangedAt` column.
+ pub const CHANGED_AT_COL: &str = "ChangedAt";
+ /// The `IsMultiValue` column.
+ pub const IS_MULTI_VALUE_COL: &str = "IsMultiValue";
+ /// The `ParentFolderId` column.
+ pub const PARENT_FOLDER_ID_COL: &str = "ParentFolderId";
+ /// The `FolderId` column.
+ pub const FOLDER_ID_COL: &str = "FolderId";
+ /// The `LogoId` column.
+ pub const LOGO_ID_COL: &str = "LogoId";
+ /// The `Kind` column.
+ pub const KIND_COL: &str = "Kind";
+ /// The `Source` column.
+ pub const SOURCE_COL: &str = "Source";
+ /// The `FileData` column.
+ pub const FILE_DATA_COL: &str = "FileData";
+ /// The `Blob` column.
+ pub const BLOB_COL: &str = "Blob";
+ /// The `UpdatedAt` column.
+ pub const UPDATED_AT_COL: &str = "UpdatedAt";
+ /// The `DeletedAt` column.
+ pub const DELETED_AT_COL: &str = "DeletedAt";
+ /// The `PublicKey` column.
+ pub const PUBLIC_KEY_COL: &str = "PublicKey";
+ /// The `IsPrimary` column.
+ pub const IS_PRIMARY_COL: &str = "IsPrimary";
+ /// The `IsDeleted` column.
+ pub const IS_DELETED_COL: &str = "IsDeleted";
+
+ /// The `favicon` logo kind.
+ pub const LOGO_KIND_FAVICON: &str = "favicon";
+ /// The `builtin` logo kind.
+ pub const LOGO_KIND_BUILTIN: &str = "builtin";
+ /// The `custom` logo kind.
+ pub const LOGO_KIND_CUSTOM: &str = "custom";
+}
diff --git a/core/rust/src/vault_model/mod.rs b/core/rust/src/vault_model/mod.rs
new file mode 100644
index 000000000..35d20d6d5
--- /dev/null
+++ b/core/rust/src/vault_model/mod.rs
@@ -0,0 +1,87 @@
+//! The client vault datamodel registry.
+//!
+//! The data (table list, keys, scoping, bucket layout, format sentinels) is generated from the
+//! TypeScript source of truth in core/models/src/vault/VaultTableRegistry.ts; see generated.rs.
+//! This module owns the [`TableConfig`] type the generated data instantiates.
+
+mod generated;
+
+pub use generated::*;
+
+/// Configuration for a syncable table.
+#[derive(Debug, Clone)]
+pub struct TableConfig {
+ /// Table name in the database
+ pub name: &'static str,
+ /// Columns to use for composite key matching (if any).
+ /// When empty, matching falls back to the table's [`identity_columns`](TableConfig::identity_columns).
+ /// When set, these columns are concatenated to form the composite key.
+ pub composite_key_columns: &'static [&'static str],
+ /// Match columns for the canonical merge only, where both sides are first normalized to the manifest
+ /// shape (see `vault_codec::normalize`). Empty means the canonical merge uses the same rule as
+ /// the statement merge (composite key, else identity).
+ pub canonical_key_columns: &'static [&'static str],
+ /// The columns that name a row within its manifest. Defaults to `["Id"]`; tables keyed
+ /// differently (Settings by "Key", ItemTags by its natural key) override this.
+ pub primary_key_columns: &'static [&'static str],
+ /// True when the table's rows are namespaced per manifest.
+ pub manifest_scoped: bool,
+ /// True when the table's rows hang off an Item row (re-stamped with their item, cascaded on delete).
+ pub item_child: bool,
+}
+
+impl TableConfig {
+ pub const fn new(name: &'static str) -> Self {
+ Self {
+ name,
+ composite_key_columns: &[],
+ canonical_key_columns: &[],
+ primary_key_columns: &["Id"],
+ manifest_scoped: false,
+ item_child: false,
+ }
+ }
+
+ pub const fn with_composite_key(mut self, columns: &'static [&'static str]) -> Self {
+ self.composite_key_columns = columns;
+ self
+ }
+
+ pub const fn with_canonical_key(mut self, columns: &'static [&'static str]) -> Self {
+ self.canonical_key_columns = columns;
+ self
+ }
+
+ pub const fn with_primary_key(mut self, columns: &'static [&'static str]) -> Self {
+ self.primary_key_columns = columns;
+ self
+ }
+
+ pub const fn manifest_scoped(mut self) -> Self {
+ self.manifest_scoped = true;
+ self
+ }
+
+ pub const fn item_child(mut self) -> Self {
+ self.item_child = true;
+ self
+ }
+
+ /// Returns true if this table uses composite key matching.
+ pub const fn uses_composite_key(&self) -> bool {
+ !self.composite_key_columns.is_empty()
+ }
+
+ /// The columns that together identify one row of this table: `(ManifestId, primary key columns)`
+ /// for a manifest-scoped table, the primary key columns alone otherwise.
+ pub fn identity_columns(&self) -> Vec<&'static str> {
+ if self.manifest_scoped {
+ let mut columns = Vec::with_capacity(1 + self.primary_key_columns.len());
+ columns.push(MANIFEST_ID_COL);
+ columns.extend_from_slice(self.primary_key_columns);
+ columns
+ } else {
+ self.primary_key_columns.to_vec()
+ }
+ }
+}
diff --git a/core/rust/src/vault_pruner/mod.rs b/core/rust/src/vault_pruner/mod.rs
index c17aac959..a39c391c1 100644
--- a/core/rust/src/vault_pruner/mod.rs
+++ b/core/rust/src/vault_pruner/mod.rs
@@ -11,6 +11,11 @@ use std::collections::HashMap;
use crate::error::VaultResult;
use crate::vault_merge::SqlStatement;
+use crate::vault_model::names::{
+ ATTACHMENTS_TABLE, DELETED_AT_COL, FIELD_HISTORIES_TABLE, FIELD_VALUES_TABLE, FILE_DATA_COL, ID_COL,
+ IS_DELETED_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_STATS_TABLE, ITEM_TAGS_TABLE, KIND_COL, LOGOS_TABLE,
+ LOGO_ID_COL, PASSKEYS_TABLE, TOTP_CODES_TABLE,
+};
/// A record is a map of column names to JSON values.
pub type Record = HashMap<String, serde_json::Value>;
@@ -49,7 +54,7 @@ pub struct PruneInput {
}
fn default_retention_days() -> u32 {
- 30
+ crate::vault_model::TRASH_RETENTION_DEFAULT_DAYS
}
/// Statistics about what was pruned.
@@ -69,6 +74,16 @@ pub struct PruneStats {
/// Number of per-item statistics rows soft-deleted along with their item.
#[serde(default)]
pub item_stats_pruned: u32,
+ /// Number of item-tag join rows permanently deleted.
+ #[serde(default)]
+ pub item_tags_pruned: u32,
+ /// Number of field history rows permanently deleted.
+ #[serde(default)]
+ pub field_histories_pruned: u32,
+ /// Rows of item-child tables that have no dedicated counter (a table added to the registry
+ /// after these stats were defined).
+ #[serde(default)]
+ pub other_children_pruned: u32,
/// Number of orpha logos soft-deleted (no remaining active item references them)
#[serde(default)]
pub logos_pruned: u32,
@@ -82,7 +97,7 @@ pub struct PruneStats {
/// The `Logos.Kind` of an automatically fetched favicon. A row without
/// a `Kind` predates the column and is a favicon by definition.
-const KIND_FAVICON: &str = "favicon";
+use crate::vault_model::names::LOGO_KIND_FAVICON as KIND_FAVICON;
/// Output of the prune operation.
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -111,23 +126,38 @@ pub struct PruneTableQuery {
/// to a 1-byte presence marker (via `substr`) to avoid serializing large binary
/// data to JSON on every prune.
pub fn get_prune_table_queries() -> Vec<PruneTableQuery> {
- [
- ("Items", "SELECT Id, IsDeleted, DeletedAt, LogoId FROM Items"),
- ("FieldValues", "SELECT ItemId, IsDeleted FROM FieldValues"),
- ("Attachments", "SELECT Id, ItemId, IsDeleted, substr(Blob, 1, 1) AS Blob FROM Attachments"),
- ("TotpCodes", "SELECT ItemId, IsDeleted FROM TotpCodes"),
- ("Passkeys", "SELECT ItemId, IsDeleted FROM Passkeys"),
- ("ItemStats", "SELECT Id, IsDeleted FROM ItemStats"),
- ("Logos", "SELECT Id, Kind, IsDeleted, substr(FileData, 1, 1) AS FileData FROM Logos"),
- ]
- .iter()
- .map(|(name, query)| PruneTableQuery {
- name: (*name).to_string(),
- query: (*query).to_string(),
- })
- .collect()
+ let mut queries = vec![PruneTableQuery {
+ name: ITEMS_TABLE.to_string(),
+ query: format!("SELECT {}, {}, {}, {} FROM {}", ID_COL, IS_DELETED_COL, DELETED_AT_COL, LOGO_ID_COL, ITEMS_TABLE),
+ }];
+ // One query per registered item-child table (TableConfig::item_child), so a table added to the
+ // registry is read by the pruner automatically.
+ for child in crate::vault_merge::SYNCABLE_TABLES.iter().filter(|t| t.item_child) {
+ let query = match blob_column_for(child.name) {
+ Some(blob_col) => format!("SELECT {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", ID_COL, ITEM_ID_COL, IS_DELETED_COL, blob_col, blob_col, child.name),
+ None => format!("SELECT {}, {} FROM {}", item_ref_column(child.name), IS_DELETED_COL, child.name),
+ };
+ queries.push(PruneTableQuery { name: child.name.to_string(), query });
+ }
+ queries.push(PruneTableQuery {
+ name: LOGOS_TABLE.to_string(),
+ query: format!("SELECT {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", ID_COL, KIND_COL, IS_DELETED_COL, FILE_DATA_COL, FILE_DATA_COL, LOGOS_TABLE),
+ });
+ queries
+}
+
+/// The column of an item-child table that carries the owning item's id: `Id` for ItemStats
+/// (its `Id` *is* the item's id), `ItemId` everywhere else.
+fn item_ref_column(table_name: &str) -> &'static str {
+ if table_name == ITEM_STATS_TABLE { ID_COL } else { ITEM_ID_COL }
+}
+
+/// The extracted blob column of a table, if it has one (see `vault_model::BLOB_COLUMNS`).
+fn blob_column_for(table_name: &str) -> Option<&'static str> {
+ crate::vault_model::BLOB_COLUMNS.iter().find(|(t, _, _)| *t == table_name).map(|(_, col, _)| *col)
}
+
/// Main entry point: prune expired items from trash.
///
/// This function finds all Items with DeletedAt set that are older than
@@ -154,7 +184,7 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
let cutoff_date = now - Duration::days(input.retention_days as i64);
// Items table is required for both passes (trash purge + logo orphan check).
- let items_table = match input.tables.iter().find(|t| t.name == "Items") {
+ let items_table = match input.tables.iter().find(|t| t.name == ITEMS_TABLE) {
Some(t) => t,
None => {
return Ok(PruneOutput {
@@ -207,79 +237,36 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
});
stats.items_pruned += 1;
- // Mark related FieldValues as deleted
- if let Some(field_values_table) = input.tables.iter().find(|t| t.name == "FieldValues") {
- let related_count = count_related_records(&field_values_table.records, "ItemId", item_id);
- if related_count > 0 {
- statements.push(SqlStatement {
- sql: "UPDATE FieldValues SET IsDeleted = 1, UpdatedAt = ? WHERE ItemId = ? AND IsDeleted = 0".to_string(),
- params: vec![
- serde_json::json!(now_str),
- serde_json::json!(item_id),
- ],
- });
- stats.field_values_pruned += related_count;
- }
- }
-
- // Mark related Attachments as deleted and drop their blob bytes. Leaves the
- // column non-null while reclaiming the storage on the next save.
- if let Some(attachments_table) = input.tables.iter().find(|t| t.name == "Attachments") {
- let related_count = count_related_records(&attachments_table.records, "ItemId", item_id);
- if related_count > 0 {
- statements.push(SqlStatement {
- sql: "UPDATE Attachments SET IsDeleted = 1, Blob = X'', UpdatedAt = ? WHERE ItemId = ? AND IsDeleted = 0".to_string(),
- params: vec![
- serde_json::json!(now_str),
- serde_json::json!(item_id),
- ],
- });
- stats.attachments_pruned += related_count;
- }
- }
-
- // Mark related TotpCodes as deleted
- if let Some(totp_table) = input.tables.iter().find(|t| t.name == "TotpCodes") {
- let related_count = count_related_records(&totp_table.records, "ItemId", item_id);
- if related_count > 0 {
- statements.push(SqlStatement {
- sql: "UPDATE TotpCodes SET IsDeleted = 1, UpdatedAt = ? WHERE ItemId = ? AND IsDeleted = 0".to_string(),
- params: vec![
- serde_json::json!(now_str),
- serde_json::json!(item_id),
- ],
- });
- stats.totp_codes_pruned += related_count;
- }
- }
-
- // Mark related Passkeys as deleted
- if let Some(passkeys_table) = input.tables.iter().find(|t| t.name == "Passkeys") {
- let related_count = count_related_records(&passkeys_table.records, "ItemId", item_id);
- if related_count > 0 {
- statements.push(SqlStatement {
- sql: "UPDATE Passkeys SET IsDeleted = 1, UpdatedAt = ? WHERE ItemId = ? AND IsDeleted = 0".to_string(),
- params: vec![
- serde_json::json!(now_str),
- serde_json::json!(item_id),
- ],
- });
- stats.passkeys_pruned += related_count;
+ /*
+ * Tombstone the rows of every registered item-child table (TableConfig::item_child) for
+ * this item, so a table added to the registry is swept automatically. A child with an
+ * extracted blob column has its bytes dropped in the same statement, leaving the column
+ * non-null while reclaiming the storage on the next save.
+ */
+ for child in crate::vault_merge::SYNCABLE_TABLES.iter().filter(|t| t.item_child) {
+ let Some(child_table) = input.tables.iter().find(|t| t.name == child.name) else { continue };
+ let match_col = item_ref_column(child.name);
+ let related_count = count_related_records(&child_table.records, match_col, item_id);
+ if related_count == 0 {
+ continue;
}
- }
-
- // Tombstone the item's statistics row.
- if let Some(item_stats_table) = input.tables.iter().find(|t| t.name == "ItemStats") {
- let related_count = count_related_records(&item_stats_table.records, "Id", item_id);
- if related_count > 0 {
- statements.push(SqlStatement {
- sql: "UPDATE ItemStats SET IsDeleted = 1, UpdatedAt = ? WHERE Id = ? AND IsDeleted = 0".to_string(),
- params: vec![
- serde_json::json!(now_str),
- serde_json::json!(item_id),
- ],
- });
- stats.item_stats_pruned += related_count;
+ let blob_clear = blob_column_for(child.name).map(|col| format!(", {} = X''", col)).unwrap_or_default();
+ statements.push(SqlStatement {
+ sql: format!("UPDATE {} SET IsDeleted = 1{}, UpdatedAt = ? WHERE {} = ? AND IsDeleted = 0", child.name, blob_clear, match_col),
+ params: vec![
+ serde_json::json!(now_str),
+ serde_json::json!(item_id),
+ ],
+ });
+ match child.name {
+ FIELD_VALUES_TABLE => stats.field_values_pruned += related_count,
+ ATTACHMENTS_TABLE => stats.attachments_pruned += related_count,
+ TOTP_CODES_TABLE => stats.totp_codes_pruned += related_count,
+ PASSKEYS_TABLE => stats.passkeys_pruned += related_count,
+ ITEM_STATS_TABLE => stats.item_stats_pruned += related_count,
+ ITEM_TAGS_TABLE => stats.item_tags_pruned += related_count,
+ FIELD_HISTORIES_TABLE => stats.field_histories_pruned += related_count,
+ _ => stats.other_children_pruned += related_count,
}
}
}
@@ -294,7 +281,7 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
* in their logo library even after the item that first used it is gone. Those are removed only
* when the user deletes them, and Pass 4 then reclaims the bytes.
*/
- if let Some(logos_table) = input.tables.iter().find(|t| t.name == "Logos") {
+ if let Some(logos_table) = input.tables.iter().find(|t| t.name == LOGOS_TABLE) {
let expired_set: std::collections::HashSet<&str> =
expired_item_ids.iter().map(|s| s.as_str()).collect();
@@ -350,7 +337,7 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
// This pass empties those blobs in place. The attachments tombstoned by
// Pass 1 in this same call are already cleared there, so this pass only
// catches historical leftovers.
- if let Some(attachments_table) = input.tables.iter().find(|t| t.name == "Attachments") {
+ if let Some(attachments_table) = input.tables.iter().find(|t| t.name == ATTACHMENTS_TABLE) {
for attachment in &attachments_table.records {
let is_deleted = attachment.get("IsDeleted")
.map(|v| v.as_i64() == Some(1) || v.as_bool() == Some(true))
@@ -380,7 +367,7 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
// soft-deleted but still have FileData bytes take up space for no reason. For favicons this
// could have occurred in older clients (before 0.29.x); for an uploaded logo it is the normal
// path, a user deleting one from their logo library tombstones the row here.
- if let Some(logos_table) = input.tables.iter().find(|t| t.name == "Logos") {
+ if let Some(logos_table) = input.tables.iter().find(|t| t.name == LOGOS_TABLE) {
for logo in &logos_table.records {
let is_deleted = logo.get("IsDeleted")
.map(|v| v.as_i64() == Some(1) || v.as_bool() == Some(true))
@@ -1121,4 +1108,47 @@ mod tests {
assert_eq!(output.stats.attachment_blobs_cleared, 0);
assert!(output.statements.is_empty());
}
+
+ #[test]
+ fn prune_queries_cover_every_item_child_table() {
+ let names: Vec<String> = get_prune_table_queries().iter().map(|q| q.name.clone()).collect();
+ for child in crate::vault_merge::SYNCABLE_TABLES.iter().filter(|t| t.item_child) {
+ assert!(names.contains(&child.name.to_string()), "pruner does not read item child table {}", child.name);
+ }
+ assert!(names.contains(&ITEMS_TABLE.to_string()));
+ assert!(names.contains(&LOGOS_TABLE.to_string()));
+ }
+
+ #[test]
+ fn test_prune_cascades_to_field_histories_and_item_tags() {
+ let now = Utc::now();
+ let now_str = now.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
+ let old_date = (now - Duration::days(60)).format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
+
+ let mut history = HashMap::new();
+ history.insert("ItemId".to_string(), serde_json::json!("item-1"));
+ history.insert("IsDeleted".to_string(), serde_json::json!(0));
+ let mut item_tag = HashMap::new();
+ item_tag.insert("ItemId".to_string(), serde_json::json!("item-1"));
+ item_tag.insert("IsDeleted".to_string(), serde_json::json!(0));
+
+ let input = PruneInput {
+ tables: vec![
+ TableData { name: "Items".to_string(), records: vec![make_item_record("item-1", Some(&old_date), false)] },
+ TableData { name: "FieldHistories".to_string(), records: vec![history] },
+ TableData { name: "ItemTags".to_string(), records: vec![item_tag] },
+ ],
+ retention_days: 30,
+ current_time: now_str,
+ };
+
+ let output = prune_vault(input).unwrap();
+
+ assert_eq!(output.stats.items_pruned, 1);
+ assert_eq!(output.stats.field_histories_pruned, 1);
+ assert_eq!(output.stats.item_tags_pruned, 1);
+ let sql: Vec<&str> = output.statements.iter().map(|s| s.sql.as_str()).collect();
+ assert!(sql.iter().any(|s| s.starts_with("UPDATE FieldHistories SET IsDeleted = 1")), "field history rows must die with their item: {:?}", sql);
+ assert!(sql.iter().any(|s| s.starts_with("UPDATE ItemTags SET IsDeleted = 1")), "item tag rows must die with their item: {:?}", sql);
+ }
}