Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.
Update LWW merge logic for delete/create usecase with item child content (#2159)
Leendert de Borst5088ae8ed388
diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/models/VaultSql.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/models/VaultSql.ktindex de4d29516..59e009b79 100644--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/models/VaultSql.kt+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/models/VaultSql.kt@@ -1786,6 +1786,21 @@ INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion")VALUES ('20260913090000_2.1.0-ManifestScopedStorage', '10.0.10');BEGIN TRANSACTION;+CREATE TRIGGER IF NOT EXISTS "TR_Items_ClearTombstoneBeforeReturn"+BEFORE UPDATE OF "ManifestId" ON "Items"+FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"+ AND EXISTS (SELECT 1 FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1)+BEGIN+ DELETE FROM "FieldValues" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "FieldHistories" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "ItemTags" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "Attachments" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "Passkeys" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "TotpCodes" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "ItemStats" WHERE "Id" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1;+END;+CREATE TRIGGER IF NOT EXISTS "TR_Items_ResyncChildManifestIds"AFTER UPDATE OF "ManifestId" ON "Items"FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"@@ -1797,6 +1812,8 @@ BEGINUPDATE "Passkeys" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";UPDATE "TotpCodes" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";UPDATE "ItemStats" SET "ManifestId" = NEW."ManifestId" WHERE "Id" = NEW."Id" AND "ManifestId" = OLD."ManifestId";+ INSERT OR IGNORE INTO "Items" ("ManifestId", "Id", "ItemType", "CreatedAt", "UpdatedAt", "IsDeleted")+ SELECT OLD."ManifestId", OLD."Id", OLD."ItemType", OLD."CreatedAt", NEW."UpdatedAt", 1 WHERE OLD."IsDeleted" = 0;END;COMMIT;diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/ItemQueries.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/ItemQueries.ktindex 94a4bad52..a0f648da0 100644--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/ItemQueries.kt+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/ItemQueries.kt@@ -65,7 +65,7 @@ object ItemQueries {LEFT JOIN FieldDefinitions fd ON fd.ManifestId = fv.ManifestId AND fd.Id = fv.FieldDefinitionIdWHERE (fv.ManifestId, fv.ItemId) IN (VALUES $placeholders)AND fv.IsDeleted = 0- ORDER BY fv.ItemId, fv.Weight+ ORDER BY fv.ItemId, fv.Weight, fv.ValueIndex""".trimIndent()}diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/PasskeyQueries.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/PasskeyQueries.ktindex 0210cea4a..b67ff0efe 100644--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/PasskeyQueries.kt+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/PasskeyQueries.kt@@ -133,8 +133,10 @@ object PasskeyQueries {* Insert a field value. Binds (id, itemId, fieldDefinitionId, fieldKey, value, weight, now, now, 0, manifestId).*/const val INSERT_FIELD_VALUE = """- INSERT INTO FieldValues (Id, ItemId, FieldDefinitionId, FieldKey, Value, Weight, CreatedAt, UpdatedAt, IsDeleted, ManifestId)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)+ INSERT INTO FieldValues (Id, ItemId, FieldDefinitionId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted, ManifestId)+ VALUES (?1, ?2, ?3, ?4, ?5, ?6,+ (SELECT COALESCE(MAX(ValueIndex), -1) + 1 FROM FieldValues WHERE ItemId = ?2 AND ManifestId = ?10 AND FieldKey = ?4 AND IsDeleted = 0),+ ?7, ?8, ?9, ?10)"""/**diff --git a/apps/mobile-app/ios/VaultModels/VaultSql.swift b/apps/mobile-app/ios/VaultModels/VaultSql.swiftindex 16bf25db6..22577a7c6 100644--- a/apps/mobile-app/ios/VaultModels/VaultSql.swift+++ b/apps/mobile-app/ios/VaultModels/VaultSql.swift@@ -1782,6 +1782,21 @@ public struct VaultSql {VALUES ('20260913090000_2.1.0-ManifestScopedStorage', '10.0.10');BEGIN TRANSACTION;+ CREATE TRIGGER IF NOT EXISTS \"TR_Items_ClearTombstoneBeforeReturn\"+ BEFORE UPDATE OF \"ManifestId\" ON \"Items\"+ FOR EACH ROW WHEN OLD.\"ManifestId\" <> NEW.\"ManifestId\"+ AND EXISTS (SELECT 1 FROM \"Items\" WHERE \"ManifestId\" = NEW.\"ManifestId\" AND \"Id\" = NEW.\"Id\" AND \"IsDeleted\" = 1)+ BEGIN+ DELETE FROM \"FieldValues\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";+ DELETE FROM \"FieldHistories\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";+ DELETE FROM \"ItemTags\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";+ DELETE FROM \"Attachments\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";+ DELETE FROM \"Passkeys\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";+ DELETE FROM \"TotpCodes\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";+ DELETE FROM \"ItemStats\" WHERE \"Id\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";+ DELETE FROM \"Items\" WHERE \"ManifestId\" = NEW.\"ManifestId\" AND \"Id\" = NEW.\"Id\" AND \"IsDeleted\" = 1;+ END;+CREATE TRIGGER IF NOT EXISTS \"TR_Items_ResyncChildManifestIds\"AFTER UPDATE OF \"ManifestId\" ON \"Items\"FOR EACH ROW WHEN OLD.\"ManifestId\" <> NEW.\"ManifestId\"@@ -1793,6 +1808,8 @@ public struct VaultSql {UPDATE \"Passkeys\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";UPDATE \"TotpCodes\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";UPDATE \"ItemStats\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"Id\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";+ INSERT OR IGNORE INTO \"Items\" (\"ManifestId\", \"Id\", \"ItemType\", \"CreatedAt\", \"UpdatedAt\", \"IsDeleted\")+ SELECT OLD.\"ManifestId\", OLD.\"Id\", OLD.\"ItemType\", OLD.\"CreatedAt\", NEW.\"UpdatedAt\", 1 WHERE OLD.\"IsDeleted\" = 0;END;COMMIT;diff --git a/apps/mobile-app/ios/VaultStoreKit/Database/Queries/ItemQueries.swift b/apps/mobile-app/ios/VaultStoreKit/Database/Queries/ItemQueries.swiftindex c1962e3ba..2035cc5ff 100644--- a/apps/mobile-app/ios/VaultStoreKit/Database/Queries/ItemQueries.swift+++ b/apps/mobile-app/ios/VaultStoreKit/Database/Queries/ItemQueries.swift@@ -50,7 +50,7 @@ public struct ItemQueries {LEFT JOIN FieldDefinitions fd ON fd.ManifestId = fv.ManifestId AND fd.Id = fv.FieldDefinitionIdWHERE (fv.ManifestId, fv.ItemId) IN (VALUES \(placeholders))AND fv.IsDeleted = 0- ORDER BY fv.ItemId, fv.Weight+ ORDER BY fv.ItemId, fv.Weight, fv.ValueIndex"""}@@ -75,8 +75,10 @@ public struct ItemQueries {public struct FieldValueQueries {/// Insert a new field value, bound as [id, itemId, fieldDefinitionId, fieldKey, value, weight, now, now, 0, manifestId].public static let insert = """- INSERT INTO FieldValues (Id, ItemId, FieldDefinitionId, FieldKey, Value, Weight, CreatedAt, UpdatedAt, IsDeleted, ManifestId)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)+ INSERT INTO FieldValues (Id, ItemId, FieldDefinitionId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted, ManifestId)+ VALUES (?1, ?2, ?3, ?4, ?5, ?6,+ (SELECT COALESCE(MAX(ValueIndex), -1) + 1 FROM FieldValues WHERE ItemId = ?2 AND ManifestId = ?10 AND FieldKey = ?4 AND IsDeleted = 0),+ ?7, ?8, ?9, ?10)"""/// The Weight a system field's values are written with: the field's DefaultDisplayOrder fromdiff --git a/apps/server/Databases/AliasClientDb/Migrations/ItemChildManifestTriggerSql.cs b/apps/server/Databases/AliasClientDb/Migrations/ItemChildManifestTriggerSql.csindex 59ae86ead..f349c22ed 100644--- a/apps/server/Databases/AliasClientDb/Migrations/ItemChildManifestTriggerSql.cs+++ b/apps/server/Databases/AliasClientDb/Migrations/ItemChildManifestTriggerSql.cs@@ -10,8 +10,8 @@ namespace AliasClientDb.Migrations;using System.Text;/// <summary>-/// DDL of the trigger that keeps item-scoped rows stamped with their item's manifest.-/// This trigger needs to be dropped and recreated after a table is rebuilt, hence the shared code.+/// DDL of the triggers that carry an item across manifests: its rows follow it and a tombstone stays behind.+/// The triggers need to be dropped and recreated after a table is rebuilt, hence the shared code./// </summary>internal static class ItemChildManifestTriggerSql{@@ -26,31 +26,54 @@ internal static class ItemChildManifestTriggerSqlpublic static string Name => "TR_Items_ResyncChildManifestIds";/// <summary>- /// Gets the statement that drops the trigger.+ /// Gets the name of the trigger that clears the way for an item moving back into a manifest it left before./// </summary>- public static string Drop => $"DROP TRIGGER IF EXISTS \"{Name}\";";+ public static string ReturnName => "TR_Items_ClearTombstoneBeforeReturn";/// <summary>- /// Gets the statement that creates the trigger.+ /// Gets the statement that drops the triggers.+ /// </summary>+ public static string Drop => $"DROP TRIGGER IF EXISTS \"{Name}\"; DROP TRIGGER IF EXISTS \"{ReturnName}\";";++ /// <summary>+ /// Gets the statement that creates the triggers./// </summary>public static string Create{get{- var body = new StringBuilder();+ var restamp = new StringBuilder();+ var clear = new StringBuilder();foreach (var table in ItemIdChildTables){- body.AppendLine($" UPDATE \"{table}\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";");+ restamp.AppendLine($" UPDATE \"{table}\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";");+ clear.AppendLine($" DELETE FROM \"{table}\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";");}- body.AppendLine($" UPDATE \"ItemStats\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"Id\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";");+ restamp.AppendLine($" UPDATE \"ItemStats\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"Id\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";");+ clear.AppendLine($" DELETE FROM \"ItemStats\" WHERE \"Id\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";");+ /*+ * When moving an item to a new manifest, a tombstone is left behind in the old manifest.+ * This trigger clears it before the item is moved back.+ */return $"""+ CREATE TRIGGER IF NOT EXISTS "{ReturnName}"+ BEFORE UPDATE OF "ManifestId" ON "Items"+ FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"+ AND EXISTS (SELECT 1 FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1)+ BEGIN+ {clear.ToString().TrimEnd()}+ DELETE FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1;+ END;+CREATE TRIGGER IF NOT EXISTS "{Name}"AFTER UPDATE OF "ManifestId" ON "Items"FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"BEGIN- {body.ToString().TrimEnd()}+ {restamp.ToString().TrimEnd()}+ INSERT OR IGNORE INTO "Items" ("ManifestId", "Id", "ItemType", "CreatedAt", "UpdatedAt", "IsDeleted")+ SELECT OLD."ManifestId", OLD."Id", OLD."ItemType", OLD."CreatedAt", NEW."UpdatedAt", 1 WHERE OLD."IsDeleted" = 0;END;""";}diff --git a/apps/server/Databases/AliasClientDb/Scripts/MigrationSql/000_FullSchema.sql b/apps/server/Databases/AliasClientDb/Scripts/MigrationSql/000_FullSchema.sqlindex bc36e3c5d..ad57ffe9f 100644--- a/apps/server/Databases/AliasClientDb/Scripts/MigrationSql/000_FullSchema.sql+++ b/apps/server/Databases/AliasClientDb/Scripts/MigrationSql/000_FullSchema.sql@@ -1771,6 +1771,21 @@ INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion")VALUES ('20260913090000_2.1.0-ManifestScopedStorage', '10.0.10');BEGIN TRANSACTION;+CREATE TRIGGER IF NOT EXISTS "TR_Items_ClearTombstoneBeforeReturn"+BEFORE UPDATE OF "ManifestId" ON "Items"+FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"+ AND EXISTS (SELECT 1 FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1)+BEGIN+ DELETE FROM "FieldValues" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "FieldHistories" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "ItemTags" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "Attachments" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "Passkeys" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "TotpCodes" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "ItemStats" WHERE "Id" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1;+END;+CREATE TRIGGER IF NOT EXISTS "TR_Items_ResyncChildManifestIds"AFTER UPDATE OF "ManifestId" ON "Items"FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"@@ -1782,6 +1797,8 @@ BEGINUPDATE "Passkeys" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";UPDATE "TotpCodes" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";UPDATE "ItemStats" SET "ManifestId" = NEW."ManifestId" WHERE "Id" = NEW."Id" AND "ManifestId" = OLD."ManifestId";+ INSERT OR IGNORE INTO "Items" ("ManifestId", "Id", "ItemType", "CreatedAt", "UpdatedAt", "IsDeleted")+ SELECT OLD."ManifestId", OLD."Id", OLD."ItemType", OLD."CreatedAt", NEW."UpdatedAt", 1 WHERE OLD."IsDeleted" = 0;END;COMMIT;diff --git a/core/client/src/database/__tests__/FieldValueRows.test.ts b/core/client/src/database/__tests__/FieldValueRows.test.tsnew file mode 100644index 000000000..eec22a094--- /dev/null+++ b/core/client/src/database/__tests__/FieldValueRows.test.ts@@ -0,0 +1,198 @@+import { FieldTypes, ItemTypes } from '@aliasvault/models/vault';+import { VaultSqlGenerator } from '@aliasvault/vault';+import { describe, expect, it } from 'vitest';++import { getPlatform } from '../../platform/ClientPlatform';+import { ItemRepository } from '../repositories/ItemRepository';+import { LogoRepository } from '../repositories/LogoRepository';++import type { ISqliteDatabase, SqliteValue } from '../../platform/SqliteEngine';+import type { ISyncDatabaseClient, SqliteBindValue } from '../BaseRepository';+import type { DraftItem } from '../ItemRef';+import type { ItemField } from '@aliasvault/models/vault';++const PERSONAL = '11111111-1111-4111-8111-111111111111';+const ITEM = '33333333-3333-4333-8333-333333333333';++type StoredRow = { Id: string; Value: string; ValueIndex: number; IsDeleted: number };++/**+ * A database client over an in-memory vault on the real schema.+ */+class TestDatabaseClient implements ISyncDatabaseClient {+ private inTransaction = false;++ /**+ * Constructor for the TestDatabaseClient class.+ * @param db - The opened database+ */+ public constructor(private readonly db: ISqliteDatabase) {}++ /**+ * Get the database.+ * @returns The database+ */+ public getDb(): ISqliteDatabase {+ return this.db;+ }++ /**+ * Run a SELECT.+ * @param query - The statement+ * @param params - The bound parameters+ * @returns The rows+ */+ public executeQuery<T>(query: string, params: SqliteBindValue[] = []): T[] {+ return this.db.query<T>(query, params as SqliteValue[]);+ }++ /**+ * Run a write.+ * @param query - The statement+ * @param params - The bound parameters+ * @returns The number of rows changed+ */+ public executeUpdate(query: string, params: SqliteBindValue[] = []): number {+ return this.db.run(query, params as SqliteValue[]);+ }++ /**+ * Begin a transaction.+ */+ public beginTransaction(): void {+ this.inTransaction = true;+ }++ /**+ * Commit the transaction.+ */+ public async commitTransaction(): Promise<void> {+ this.inTransaction = false;+ }++ /**+ * Roll the transaction back.+ */+ public rollbackTransaction(): void {+ this.inTransaction = false;+ }++ /**+ * Whether a transaction is open.+ * @returns True while one is+ */+ public isInTransaction(): boolean {+ return this.inTransaction;+ }++ /**+ * Get the manifest new rows are written into.+ * @returns The personal manifest id+ */+ public getActiveManifestId(): string {+ return PERSONAL;+ }++ /**+ * Get the personal manifest.+ * @returns The personal manifest id+ */+ public getPersonalManifestId(): string {+ return PERSONAL;+ }+}++/**+ * A system field as the item form hands it to the repository.+ * @param fieldKey - The system field key+ * @param value - The field's value or values+ * @returns The field+ */+function field(fieldKey: string, value: string | string[]): ItemField {+ return { FieldKey: fieldKey, Label: fieldKey, FieldType: FieldTypes.Text, Value: value, IsHidden: false, DisplayOrder: 0, IsCustomField: false, EnableHistory: false };+}++/**+ * Build a vault holding one item, saved with the given fields.+ * @param fields - The fields of the first save+ * @returns The database and a way to save the item again+ */+async function vaultWithItem(fields: ItemField[]): Promise<{ db: ISqliteDatabase; save: (fields: ItemField[]) => Promise<void> }> {+ const db = await getPlatform().sqlite.open();+ db.exec('PRAGMA foreign_keys = OFF;');+ db.exec(new VaultSqlGenerator().getCompleteSchemaSql());+ const client = new TestDatabaseClient(db);+ const repository = new ItemRepository(client, new LogoRepository(client));+ /**+ * The item as the form hands it over, holding the given fields.+ */+ const draft = (itemFields: ItemField[]): DraftItem => ({ Id: ITEM, Name: 'Note', ItemType: ItemTypes.Note, Fields: itemFields, CreatedAt: '', UpdatedAt: '' } as DraftItem);++ await repository.create(draft(fields));+ return {+ db,+ /**+ * Save the item again with the given fields.+ */+ save: async (next: ItemField[]): Promise<void> => {+ expect(await repository.update(draft(next))).toBe(1);+ },+ };+}++/**+ * The stored rows of one field, tombstones included.+ * @param db - The database+ * @param fieldKey - The system field key+ * @returns The rows, in value order+ */+function rowsOf(db: ISqliteDatabase, fieldKey: string): StoredRow[] {+ return db.query<StoredRow>('SELECT Id, Value, ValueIndex, IsDeleted FROM FieldValues WHERE ItemId = ? AND FieldKey = ? ORDER BY IsDeleted, ValueIndex', [ITEM, fieldKey]);+}++describe('field value rows', () => {+ it('brings a removed field back on its own row instead of adding a second one', async () => {+ /*+ * Two rows for one single-value field collapse to the newest on the next sync, and a tombstone stamped+ * by a device whose clock runs ahead would then beat the value the user just typed.+ */+ const { db, save } = await vaultWithItem([field('login.username', 'first')]);+ const original = rowsOf(db, 'login.username')[0];++ await save([]);+ expect(rowsOf(db, 'login.username')).toMatchObject([{ Id: original.Id, IsDeleted: 1 }]);++ await save([field('login.username', 'second')]);+ expect(rowsOf(db, 'login.username')).toMatchObject([{ Id: original.Id, Value: 'second', IsDeleted: 0 }]);+ });++ it('keeps every url on its own row when one before it is removed', async () => {+ // Shifting values across rows makes a concurrent edit of the last url come back as a near duplicate.+ const { db, save } = await vaultWithItem([field('login.url', ['https://a.example', 'https://b.example', 'https://c.example'])]);+ const [a, b, c] = rowsOf(db, 'login.url');+ expect([a.ValueIndex, b.ValueIndex, c.ValueIndex]).toEqual([0, 1, 2]);++ await save([field('login.url', ['https://b.example', 'https://c.example'])]);++ expect(rowsOf(db, 'login.url')).toMatchObject([+ { Id: b.Id, Value: 'https://b.example', ValueIndex: 0, IsDeleted: 0 },+ { Id: c.Id, Value: 'https://c.example', ValueIndex: 1, IsDeleted: 0 },+ { Id: a.Id, Value: 'https://a.example', IsDeleted: 1 },+ ]);+ });++ it('appends a new url after the existing ones and edits a url in place', async () => {+ const { db, save } = await vaultWithItem([field('login.url', ['https://a.example', 'https://b.example'])]);+ const [a, b] = rowsOf(db, 'login.url');++ await save([field('login.url', ['https://a.example', 'https://b-edited.example', 'https://c.example'])]);++ const rows = rowsOf(db, 'login.url');+ expect(rows).toMatchObject([+ { Id: a.Id, Value: 'https://a.example', ValueIndex: 0 },+ { Id: b.Id, Value: 'https://b-edited.example', ValueIndex: 1 },+ { Value: 'https://c.example', ValueIndex: 2 },+ ]);+ expect(rows).toHaveLength(3);+ });+});diff --git a/core/client/src/database/queries/ItemQueries.ts b/core/client/src/database/queries/ItemQueries.tsindex 8c995e0f9..08bef8932 100644--- a/core/client/src/database/queries/ItemQueries.ts+++ b/core/client/src/database/queries/ItemQueries.ts@@ -144,7 +144,7 @@ export class ItemQueries {LEFT JOIN FieldDefinitions fd ON fd.ManifestId = fv.ManifestId AND fd.Id = fv.FieldDefinitionIdWHERE (fv.ManifestId, fv.ItemId) IN (VALUES ${placeholders})AND fv.IsDeleted = 0- ORDER BY fv.ItemId, fv.Weight`;+ ORDER BY fv.ItemId, fv.Weight, fv.ValueIndex`;}/**@@ -163,7 +163,7 @@ export class ItemQueries {FROM FieldValues fvLEFT JOIN FieldDefinitions fd ON fd.ManifestId = fv.ManifestId AND fd.Id = fv.FieldDefinitionIdWHERE fv.ItemId = ? AND fv.ManifestId = ? AND fv.IsDeleted = 0- ORDER BY fv.Weight`;+ ORDER BY fv.Weight, fv.ValueIndex`;/*** Get tags for multiple items, matched on the whole item key (see {@link getFieldValuesForItems}).@@ -335,27 +335,31 @@ export class ItemQueries {*/export class FieldValueQueries {/**- * Get existing field values for an item.+ * Get every field value row of an item, tombstones included, so a write can bring a removed field's row back+ * instead of adding a second row for the same field.*/- public static readonly GET_EXISTING_FOR_ITEM = `- SELECT Id, FieldKey, FieldDefinitionId, Value, Weight+ public static readonly GET_ALL_FOR_ITEM = `+ SELECT Id, FieldKey, FieldDefinitionId, Value, Weight, ValueIndex, IsDeletedFROM FieldValues- WHERE ItemId = ? AND ManifestId = ? AND IsDeleted = 0`;+ WHERE ItemId = ? AND ManifestId = ?+ ORDER BY ValueIndex`;/*** Insert a new field value, stamped with the manifest of the item it hangs off.*/public static readonly INSERT = `- INSERT INTO FieldValues (Id, ItemId, ManifestId, FieldDefinitionId, FieldKey, Value, Weight, CreatedAt, UpdatedAt, IsDeleted)- VALUES (?, ?, ${BaseQueries.MANIFEST_OF_ITEM}, ?, ?, ?, ?, ?, ?, ?)`;+ INSERT INTO FieldValues (Id, ItemId, ManifestId, FieldDefinitionId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted)+ VALUES (?, ?, ${BaseQueries.MANIFEST_OF_ITEM}, ?, ?, ?, ?, ?, ?, ?, ?)`;/**- * Update an existing field value.+ * Update an existing field value, bringing it back when it was removed before.*/public static readonly UPDATE = `UPDATE FieldValuesSET Value = ?,Weight = ?,+ ValueIndex = ?,+ IsDeleted = 0,UpdatedAt = ?WHERE Id = ? AND ManifestId = ?`;@@ -374,7 +378,8 @@ export class FieldValueQueries {public static readonly GET_FOR_HISTORY = `SELECT FieldKey, ValueFROM FieldValues- WHERE ItemId = ? AND ManifestId = ? AND IsDeleted = 0 AND FieldKey IS NOT NULL`;+ WHERE ItemId = ? AND ManifestId = ? AND IsDeleted = 0 AND FieldKey IS NOT NULL+ ORDER BY ValueIndex`;}/**diff --git a/core/client/src/database/repositories/ItemRepository.ts b/core/client/src/database/repositories/ItemRepository.tsindex 713a26ca2..6e65237af 100644--- a/core/client/src/database/repositories/ItemRepository.ts+++ b/core/client/src/database/repositories/ItemRepository.ts@@ -1,4 +1,4 @@-import { FieldKey, LogoKinds, MAX_FIELD_HISTORY_RECORDS, normalizeTotpAlgorithm, normalizeTotpDigits, normalizeTotpPeriod } from '@aliasvault/models/vault';+import { FieldKey, LogoKinds, MAX_FIELD_HISTORY_RECORDS, getSystemField, normalizeTotpAlgorithm, normalizeTotpDigits, normalizeTotpPeriod } from '@aliasvault/models/vault';import { getFolderPath } from '../../items/FolderUtils';import { selectFaviconTarget, toUrlList } from '../../rust/RustCore';@@ -20,6 +20,19 @@ import type { Folder } from './FolderRepository';import type { LogoRepository } from './LogoRepository';import type { Item, ItemField, Attachment, TotpCode, FieldHistory, LogoSelection } from '@aliasvault/models/vault';+/**+ * A stored FieldValues row as the write path reads it, tombstones included.+ */+type StoredFieldValue = {+ Id: string;+ FieldKey: string | null;+ FieldDefinitionId: string | null;+ Value: string;+ Weight: number;+ ValueIndex: number;+ IsDeleted: number;+};+/*** Repository for Item CRUD operations.* Handles items, field values, field definitions, and field history.@@ -920,7 +933,7 @@ export class ItemRepository extends BaseRepository {? ['']: filteredValues;- for (const value of valuesToInsert) {+ for (const [valueIndex, value] of valuesToInsert.entries()) {yield* this.execute(FieldValueQueries.INSERT, [this.generateId(),itemId,@@ -930,6 +943,7 @@ export class ItemRepository extends BaseRepository {field.IsCustomField ? null : field.FieldKey,value,field.DisplayOrder ?? 0,+ valueIndex,currentDateTime,currentDateTime,0@@ -992,91 +1006,110 @@ export class ItemRepository extends BaseRepository {* Update field values for an existing item.*/private *updateFieldValues(item: DraftItem, manifestId: string, writeManifestId: string, currentDateTime: string): DbOp<void> {- // Get existing FieldValues- const existingFieldValues = yield* this.query<{- Id: string;- FieldKey: string | null;- FieldDefinitionId: string | null;- Value: string;- Weight: number;- }>(FieldValueQueries.GET_EXISTING_FOR_ITEM, [item.Id, manifestId]);-- // Build a map of existing FieldValues by key:index- const existingByKey = new Map<string, { Id: string; Value: string; Weight: number }>();- const fieldValueCounts = new Map<string, number>();-- for (const fv of existingFieldValues) {- const key = fv.FieldKey || fv.FieldDefinitionId || '';- const count = fieldValueCounts.get(key) || 0;- existingByKey.set(`${key}:${count}`, { Id: fv.Id, Value: fv.Value, Weight: fv.Weight });- fieldValueCounts.set(key, count + 1);+ const storedRows = yield* this.query<StoredFieldValue>(FieldValueQueries.GET_ALL_FOR_ITEM, [item.Id, manifestId]);++ const rowsByField = new Map<string, StoredFieldValue[]>();+ for (const row of storedRows) {+ const key = (row.FieldKey || row.FieldDefinitionId || '').toLowerCase();+ rowsByField.set(key, [...(rowsByField.get(key) ?? []), row]);}- const processedIds = new Set<string>();+ const keptIds = new Set<string>();- // Update existing or insert new FieldValues- if (item.Fields && item.Fields.length > 0) {- for (const field of item.Fields) {- // Skip empty system fields, but always persist custom fields (even if empty)- const isEmpty = !field.Value || (typeof field.Value === 'string' && field.Value.trim() === '');- if (isEmpty && !field.IsCustomField) {- continue;- }+ for (const field of item.Fields ?? []) {+ // Skip empty system fields, but always persist custom fields (even if empty)+ const isEmpty = !field.Value || (typeof field.Value === 'string' && field.Value.trim() === '');+ if (isEmpty && !field.IsCustomField) {+ continue;+ }++ let fieldDefinitionId = null;++ if (field.IsCustomField) {+ fieldDefinitionId = yield* this.ensureOrUpdateFieldDefinition(field, item.Id, item.ItemType, writeManifestId, currentDateTime);+ }- let fieldDefinitionId = null;+ const values = Array.isArray(field.Value) ? field.Value : [field.Value];- if (field.IsCustomField) {- fieldDefinitionId = yield* this.ensureOrUpdateFieldDefinition(field, item.Id, item.ItemType, writeManifestId, currentDateTime);+ // For custom fields with no values, use empty string to preserve the field+ const filteredValues = values.filter(v => v && (typeof v !== 'string' || v.trim() !== ''));+ const valuesToProcess = field.IsCustomField && filteredValues.length === 0+ ? ['']+ : filteredValues;++ const isMultiValue = !field.IsCustomField && (getSystemField(field.FieldKey)?.IsMultiValue ?? false);+ const rows = ItemRepository.assignFieldValueRows(valuesToProcess, rowsByField.get(field.FieldKey.toLowerCase()) ?? [], isMultiValue);+ const weight = field.DisplayOrder ?? 0;++ for (const [valueIndex, value] of valuesToProcess.entries()) {+ const row = rows[valueIndex];+ if (!row) {+ yield* this.execute(FieldValueQueries.INSERT, [+ this.generateId(),+ item.Id,+ item.Id,+ writeManifestId,+ fieldDefinitionId,+ field.IsCustomField ? null : field.FieldKey,+ value,+ weight,+ valueIndex,+ currentDateTime,+ currentDateTime,+ 0+ ]);+ continue;}- const values = Array.isArray(field.Value) ? field.Value : [field.Value];- const effectiveKey = field.FieldKey;-- // For custom fields with no values, use empty string to preserve the field- const filteredValues = values.filter(v => v && (typeof v !== 'string' || v.trim() !== ''));- const valuesToProcess = field.IsCustomField && filteredValues.length === 0- ? ['']- : filteredValues;-- for (let i = 0; i < valuesToProcess.length; i++) {- const value = valuesToProcess[i];-- const lookupKey = `${effectiveKey}:${i}`;- const existing = existingByKey.get(lookupKey);-- if (existing) {- processedIds.add(existing.Id);- const newWeight = field.DisplayOrder ?? 0;- if (existing.Value !== value || existing.Weight !== newWeight) {- yield* this.execute(FieldValueQueries.UPDATE, [value, newWeight, currentDateTime, existing.Id, manifestId]);- }- } else {- yield* this.execute(FieldValueQueries.INSERT, [- this.generateId(),- item.Id,- item.Id,- writeManifestId,- fieldDefinitionId,- field.IsCustomField ? null : field.FieldKey,- value,- field.DisplayOrder ?? 0,- currentDateTime,- currentDateTime,- 0- ]);- }+ keptIds.add(row.Id);+ if (row.IsDeleted || row.Value !== value || row.Weight !== weight || row.ValueIndex !== valueIndex) {+ yield* this.execute(FieldValueQueries.UPDATE, [value, weight, valueIndex, currentDateTime, row.Id, manifestId]);}}}- // Soft-delete any FieldValues that were not processed- for (const fv of existingFieldValues) {- if (!processedIds.has(fv.Id)) {- yield* this.execute(FieldValueQueries.SOFT_DELETE, [currentDateTime, fv.Id, manifestId]);+ // Soft-delete every live row no value was kept on+ for (const row of storedRows) {+ if (!row.IsDeleted && !keptIds.has(row.Id)) {+ yield* this.execute(FieldValueQueries.SOFT_DELETE, [currentDateTime, row.Id, manifestId]);}}}+ /**+ * Decide which stored row each value of one field is written to, `undefined` meaning a new row.+ * @param values - The field's values, in display order+ * @param fieldRows - The field's stored rows, tombstones included+ * @param isMultiValue - Whether the field holds several values, each owning its row+ * @returns One entry per value+ */+ private static assignFieldValueRows(values: string[], fieldRows: StoredFieldValue[], isMultiValue: boolean): (StoredFieldValue | undefined)[] {+ const assigned: (StoredFieldValue | undefined)[] = values.map(() => undefined);+ const free = new Set(fieldRows);++ /**+ * Give every still unassigned value the first free row that qualifies for it.+ */+ const assign = (qualifies: (row: StoredFieldValue, value: string) => boolean): void => {+ for (const [index, value] of values.entries()) {+ const row = assigned[index] ? undefined : [...free].find(candidate => qualifies(candidate, value));+ if (row) {+ assigned[index] = row;+ free.delete(row);+ }+ }+ };++ // 1. An unchanged value stays on its row.+ assign((row, value) => !row.IsDeleted && row.Value === value);+ // 2. An edited value keeps a row that is still live.+ assign(row => !row.IsDeleted);+ // 3. A removed value that returns gets its row back. A single-value field has one row, whatever it held.+ assign((row, value) => Boolean(row.IsDeleted) && (!isMultiValue || row.Value === value));++ return assigned;+ }+/*** Ensure a field definition exists and is up-to-date.*/diff --git a/core/models/scripts/lib/vault-table-registry-emit.cjs b/core/models/scripts/lib/vault-table-registry-emit.cjsindex 02af0cec4..404629218 100644--- a/core/models/scripts/lib/vault-table-registry-emit.cjs+++ b/core/models/scripts/lib/vault-table-registry-emit.cjs@@ -157,6 +157,10 @@ pub static PERSONAL_TABLES: &[&str] = &[${registry.VAULT_PERSONAL_TABLES.map((t)/// adding two different things) and is not derived.pub static MULTI_VALUE_FIELD_KEYS: &[&str] = ${rustStrSlice(registry.MULTI_VALUE_FIELD_KEYS)};+/// System field keys whose field holds one value. A key in neither list is a field this build does+/// not know (a newer writer's), whose rows the codec leaves as it found them.+pub static SINGLE_VALUE_FIELD_KEYS: &[&str] = ${rustStrSlice(registry.SINGLE_VALUE_FIELD_KEYS)};+/// The per-manifest delivery-keypair table. Every manifest carries its own asymmetric keypair(s),/// stamped with that manifest's id (\`ManifestId\`).pub const ENCRYPTION_KEYS_TABLE: &str = "${registry.ENCRYPTION_KEYS_TABLE}";diff --git a/core/models/src/vault/VaultTableRegistry.ts b/core/models/src/vault/VaultTableRegistry.tsindex 4c6648c15..88f84b3ec 100644--- a/core/models/src/vault/VaultTableRegistry.ts+++ b/core/models/src/vault/VaultTableRegistry.ts@@ -196,6 +196,14 @@ export const MULTI_VALUE_FIELD_KEYS: string[] = Object.values(SystemFieldRegistr.filter((field) => field.IsMultiValue).map((field) => field.FieldKey.toLowerCase());+/**+ * System field keys whose field holds one value. A key in neither list is a field this build does not+ * know (a newer writer's), whose rows the codec leaves as it found them.+ */+export const SINGLE_VALUE_FIELD_KEYS: string[] = Object.values(SystemFieldRegistry)+ .filter((field) => !field.IsMultiValue)+ .map((field) => field.FieldKey.toLowerCase());+/*** The per-manifest delivery-keypair table. Every manifest carries its own asymmetric keypair(s),* stamped with that manifest's id.diff --git a/core/rust/src/vault_codec/normalize.rs b/core/rust/src/vault_codec/normalize.rsindex fe8cd36f6..956c67548 100644--- a/core/rust/src/vault_codec/normalize.rs+++ b/core/rust/src/vault_codec/normalize.rs@@ -5,13 +5,13 @@ use std::collections::{HashMap, HashSet};use serde_json::json;use super::manifest::CodecRecord;-use super::row::{str_col, truthy};+use super::row::{is_deleted, str_col, truthy};use super::types::{is_guid, is_id_column};use crate::timestamp::updated_at;-use crate::vault_model::{id_key, MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS};+use crate::vault_model::{id_key, MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS, SINGLE_VALUE_FIELD_KEYS, SYNCABLE_TABLES};use crate::vault_model::names::{CHANGED_AT_COL, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_KEY_COL,- FIELD_VALUES_TABLE, ID_COL, IS_MULTI_VALUE_COL, ITEM_ID_COL, ITEM_TAGS_TABLE, TAG_ID_COL, VALUE_INDEX_COL,+ FIELD_VALUES_TABLE, ID_COL, IS_MULTI_VALUE_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, TAG_ID_COL, VALUE_INDEX_COL,};/// Domain-separation prefix for derived field value ids.@@ -66,6 +66,8 @@ pub(crate) fn normalize_row_id_spelling(row: &mut CodecRecord) {/// Normalize the shape of rows for converting from materialized SQLite to the manifest format to save on filesize.pub(crate) fn normalize_row_shapes(tables: &mut HashMap<String, Vec<CodecRecord>>) {+ // Drop rows of deleted items first (any leftover state after a potential delete/update scenario in LWW merge).+ drop_children_of_deleted_items(tables);let multi_value_defs = multi_value_definition_ids(tables);if let Some(rows) = tables.get_mut(FIELD_VALUES_TABLE) {normalize_field_values(rows, &multi_value_defs);@@ -78,6 +80,26 @@ pub(crate) fn normalize_row_shapes(tables: &mut HashMap<String, Vec<CodecRecord>}}+/*+ * Check for deleted items and remove their child rows in case they are still present after a LWW merge (leftover state).+ */+fn drop_children_of_deleted_items(tables: &mut HashMap<String, Vec<CodecRecord>>) {+ let deleted: HashSet<(String, String)> = tables+ .get(ITEMS_TABLE)+ .map(|items| items.iter().filter(|item| is_deleted(item)).filter_map(|item| Some((lower_str(item, MANIFEST_ID_COL)?, lower_str(item, ID_COL)?))).collect())+ .unwrap_or_default();+ if deleted.is_empty() {+ return;+ }+ for child in SYNCABLE_TABLES.iter().filter(|table| table.item_child) {+ let Some(rows) = tables.get_mut(child.name) else { continue };+ rows.retain(|row| match (lower_str(row, MANIFEST_ID_COL), lower_str(row, child.item_ref_column())) {+ (Some(manifest), Some(item)) => !deleted.contains(&(manifest, item)),+ _ => true,+ });+ }+}+/// The materialize direction: a FieldValues or FieldHistories row without an `Id` is a row whose/// derived id the wire omits; derive it from the row's own natural key so every device materializes/// the same SQLite row.@@ -105,6 +127,27 @@ fn multi_value_definition_ids(tables: &HashMap<String, Vec<CodecRecord>>) -> Has.collect()}+/// How many values a field holds, which decides the shape its rows take on the wire.+enum FieldShape {+ MultiValue,+ SingleValue,+ /*+ * A system field this build's registry does not list, so it is unknown to this build.+ */+ Unknown,+}++/// The shape of the field `row` belongs to: by registry for a system field, by its definition for a custom one.+fn field_shape(row: &CodecRecord, manifest: &str, multi_value_defs: &HashSet<(String, String)>) -> FieldShape {+ match str_col(row, FIELD_KEY_COL).filter(|key| !key.is_empty()).map(str::to_lowercase) {+ Some(key) if MULTI_VALUE_FIELD_KEYS.contains(&key.as_str()) => FieldShape::MultiValue,+ Some(key) if SINGLE_VALUE_FIELD_KEYS.contains(&key.as_str()) => FieldShape::SingleValue,+ Some(_) => FieldShape::Unknown,+ None if lower_str(row, FIELD_DEFINITION_ID_COL).is_some_and(|def| multi_value_defs.contains(&(manifest.to_string(), def))) => FieldShape::MultiValue,+ None => FieldShape::SingleValue,+ }+}+/// Renumber, collapse and re-id one FieldValues row set, per `(manifest, item, field)` group.fn normalize_field_values(rows: &mut Vec<CodecRecord>, multi_value_defs: &HashSet<(String, String)>) {// Group row positions by natural key, preserving read order within each group.@@ -117,28 +160,47 @@ fn normalize_field_values(rows: &mut Vec<CodecRecord>, multi_value_defs: &HashSelet mut removed: HashSet<usize> = HashSet::new();for ((manifest, _, _), mut positions) in groups {- let multi_value = str_col(&rows[positions[0]], FIELD_KEY_COL).is_some_and(|key| MULTI_VALUE_FIELD_KEYS.contains(&key.to_lowercase().as_str()))- || lower_str(&rows[positions[0]], FIELD_DEFINITION_ID_COL).is_some_and(|def| multi_value_defs.contains(&(manifest.clone(), def)));-- if multi_value {- // Stable order: declared position first (a row written before the column existed sorts last), read order breaks ties.- positions.sort_by_key(|p| value_index_of(&rows[*p]).unwrap_or(i64::MAX));- for (index, position) in positions.iter().enumerate() {- let row = &mut rows[*position];- row.insert(VALUE_INDEX_COL.to_string(), json!(index as i64));- if !has_id(row) {- let id = derive_row_id(row, index as i64);- row.insert(ID_COL.to_string(), json!(id));+ match field_shape(&rows[positions[0]], &manifest, multi_value_defs) {+ FieldShape::MultiValue => {+ // Stable order: declared position first (a row written before the column existed sorts last), read order breaks ties.+ positions.sort_by_key(|p| value_index_of(&rows[*p]).unwrap_or(i64::MAX));+ for (index, position) in positions.iter().enumerate() {+ let row = &mut rows[*position];+ row.insert(VALUE_INDEX_COL.to_string(), json!(index as i64));+ if !has_id(row) {+ let id = derive_row_id(row, index as i64);+ row.insert(ID_COL.to_string(), json!(id));+ }+ }+ }+ FieldShape::SingleValue => {+ // A single-value field is one row; duplicates collapse to the newest UpdatedAt instead of+ // materializing into a primary-key violation.+ let winner = *positions.iter().max_by_key(|p| (updated_at(&rows[**p]), std::cmp::Reverse(**p))).unwrap();+ removed.extend(positions.iter().filter(|p| **p != winner));+ let row = &mut rows[winner];+ row.insert(VALUE_INDEX_COL.to_string(), json!(0));+ row.remove(ID_COL);+ }+ FieldShape::Unknown => {+ // Only rows naming one row collapse (one carrying the id the other would derive), since they would materialize into one primary key.+ let mut newest: HashMap<String, usize> = HashMap::new();+ for position in positions {+ let id = materialized_id(&rows[position]);+ match newest.get(&id).copied() {+ Some(current) if updated_at(&rows[position]) <= updated_at(&rows[current]) => {+ removed.insert(position);+ }+ Some(current) => {+ removed.insert(current);+ newest.insert(id, position);+ }+ None => {+ newest.insert(id, position);+ }+ }}}- } else {- // A single-value field is one row; duplicates collapse to the newest UpdatedAt instead of- // materializing into a primary-key violation (or a field the UI renders as an array).- let winner = *positions.iter().max_by_key(|p| (updated_at(&rows[**p]), std::cmp::Reverse(**p))).unwrap();- removed.extend(positions.iter().filter(|p| **p != winner));- let row = &mut rows[winner];- row.insert(VALUE_INDEX_COL.to_string(), json!(0));- row.remove(ID_COL);}}@@ -209,6 +271,11 @@ fn derive_row_id(row: &CodecRecord, value_index: i64) -> String {field_value_id_for(manifest, item, field_key, field_def, value_index)}+/// The id a row materializes under: the one it carries, or the one [`derive_missing_ids`] would give it.+fn materialized_id(row: &CodecRecord) -> String {+ lower_str(row, ID_COL).unwrap_or_else(|| derive_row_id(row, value_index_of(row).unwrap_or(0)))+}+/// The derived id for a normalized FieldHistories row.fn derive_history_row_id(row: &CodecRecord) -> String {let [manifest, item, field_key, field_def] = field_key_parts(row);@@ -279,6 +346,83 @@ mod tests {assert_ne!(field_value_id_for("m", "i", "login.url", "", 0), field_value_id_for("m", "i", "login.url", "", 1));}+ fn field_value(id: Option<&str>, field_key: &str, value: &str, updated_at: &str) -> CodecRecord {+ let mut row: CodecRecord = [+ (MANIFEST_ID_COL.to_string(), json!("m-1")),+ (ITEM_ID_COL.to_string(), json!("item-1")),+ (FIELD_KEY_COL.to_string(), json!(field_key)),+ ("Value".to_string(), json!(value)),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect();+ if let Some(id) = id {+ row.insert(ID_COL.to_string(), json!(id));+ }+ row+ }++ fn normalized(rows: Vec<CodecRecord>) -> Vec<CodecRecord> {+ let mut tables: HashMap<String, Vec<CodecRecord>> = [(FIELD_VALUES_TABLE.to_string(), rows)].into_iter().collect();+ normalize_row_shapes(&mut tables);+ tables.remove(FIELD_VALUES_TABLE).unwrap()+ }++ #[test]+ fn a_field_this_build_does_not_know_keeps_every_value() {+ // A newer writer's multi-value system field: collapsing it as single-value would destroy all but+ // one value on this build's next push.+ let rows = normalized(vec![+ field_value(Some("a"), "login.future_multi", "one", "2024-01-01T00:00:00Z"),+ field_value(Some("b"), "login.future_multi", "two", "2024-01-02T00:00:00Z"),+ ]);+ assert_eq!(rows.len(), 2);+ assert!(rows.iter().all(|row| has_id(row)), "and the rows keep the ids they own");+ }++ #[test]+ fn an_unknown_field_row_and_its_id_less_twin_are_one_row() {+ // A newer writer's single-value field arrives id-less; this build materialized it under the derived+ // id and writes it back with that id. Both name one SQLite row, so they collapse to the newest.+ let derived = field_value_id_for("m-1", "item-1", "login.future_single", "", 0);+ let rows = normalized(vec![+ field_value(None, "login.future_single", "server", "2024-01-01T00:00:00Z"),+ field_value(Some(&derived), "login.future_single", "local-newer", "2024-01-09T00:00:00Z"),+ ]);+ assert_eq!(rows.len(), 1);+ assert_eq!(rows[0]["Value"], json!("local-newer"));+ }++ #[test]+ fn a_known_single_value_field_still_collapses_to_the_newest() {+ let rows = normalized(vec![+ field_value(Some("a"), "login.username", "old", "2024-01-01T00:00:00Z"),+ field_value(Some("b"), "login.username", "new", "2024-01-09T00:00:00Z"),+ ]);+ assert_eq!(rows.len(), 1);+ assert_eq!(rows[0]["Value"], json!("new"));+ assert!(!has_id(&rows[0]), "a single-value row travels without its derived id");+ }++ #[test]+ fn a_tombstoned_item_keeps_no_rows() {+ let item = |id: &str, deleted: i64| -> CodecRecord {+ [(MANIFEST_ID_COL.to_string(), json!("m-1")), (ID_COL.to_string(), json!(id)), ("IsDeleted".to_string(), json!(deleted))].into_iter().collect()+ };+ let child = |item_id: &str| -> CodecRecord {+ [(MANIFEST_ID_COL.to_string(), json!("m-1")), (ID_COL.to_string(), json!(format!("totp-{}", item_id))), (ITEM_ID_COL.to_string(), json!(item_id))].into_iter().collect()+ };+ let mut tables: HashMap<String, Vec<CodecRecord>> = [+ (ITEMS_TABLE.to_string(), vec![item("gone", 1), item("kept", 0)]),+ ("TotpCodes".to_string(), vec![child("gone"), child("kept")]),+ ]+ .into_iter()+ .collect();+ normalize_row_shapes(&mut tables);+ let left: Vec<&str> = tables["TotpCodes"].iter().map(|row| row[ITEM_ID_COL].as_str().unwrap()).collect();+ assert_eq!(left, vec!["kept"], "a pruned or permanently deleted item's secrets do not travel on");+ }+#[test]fn history_derivation_is_stable_and_disjoint_from_field_values() {let id = field_history_id_for("m-1", "item-1", "login.password", "", "2026-01-01 10:00:00.000");diff --git a/core/rust/src/vault_merge/item_deletes.rs b/core/rust/src/vault_merge/item_deletes.rsnew file mode 100644index 000000000..c5c0a7548--- /dev/null+++ b/core/rust/src/vault_merge/item_deletes.rs@@ -0,0 +1,78 @@+//! Item deletes in the canonical merge, decided once per item instead of row by row.+//!+//! A permanent delete tombstones the item and takes the rows hanging off it along, so the item and+//! its children are one unit. Row-level LWW breaks that unit both ways: children the deleting side+//! removed come back from the other side under a tombstoned item, and child tombstones stamped at+//! delete time wipe the untouched fields of an item whose delete lost. So a delete only one side made+//! is settled here first, against everything the other side did to that item (a field edit does not+//! touch the item row), and the losing side's rows of the item leave the merge. A delete that stands+//! needs nothing more: the merged output drops a tombstoned item's children when it is normalized.++use std::collections::{HashMap, HashSet};++use chrono::{DateTime, Utc};++use super::get_key;+use crate::timestamp::updated_at;+use crate::vault_codec::row::is_deleted;+use crate::vault_codec::{is_bucketed_table, CodecRecord};+use crate::vault_model::names::{ID_COL, ITEMS_TABLE};+use crate::vault_model::{MANIFEST_ID_COL, SYNCABLE_TABLES};++type Tables = HashMap<String, Vec<CodecRecord>>;++/// Settle every delete only one side made: a delete the other side outlived is taken out of the deleting+/// side's input together with that side's rows of the item, so the surviving side's rows pass through whole.+pub(super) fn resolve_item_deletes(base: &mut Tables, incoming: &mut Tables) {+ // The base keeps ties, exactly as it does for a single row.+ let lost_in_base = lost_deletes(base, incoming, |delete, activity| activity > delete);+ let lost_in_incoming = lost_deletes(incoming, base, |delete, activity| activity >= delete);+ remove_items(base, &lost_in_base);+ remove_items(incoming, &lost_in_incoming);+}++/// The items `deleting` tombstoned that `other` still holds live and touched late enough for `outlives` to hold.+fn lost_deletes(deleting: &Tables, other: &Tables, outlives: impl Fn(Option<DateTime<Utc>>, Option<DateTime<Utc>>) -> bool) -> HashSet<String> {+ let items = |tables: &Tables, deleted: bool| -> HashMap<String, Option<DateTime<Utc>>> {+ let rows = tables.get(ITEMS_TABLE).map(Vec::as_slice).unwrap_or(&[]);+ rows.iter().filter(|row| is_deleted(row) == deleted).map(|row| (item_key(row, ID_COL), updated_at(row))).collect()+ };+ let tombstones = items(deleting, true);+ let mut live = items(other, false);+ live.retain(|key, _| tombstones.contains_key(key));+ if live.is_empty() {+ return HashSet::new();+ }++ // What the surviving side last did to the item: its own row, or any row hanging off it. Bucketed+ // tables stay out, since a usage counter ticking is not an edit.+ for child in SYNCABLE_TABLES.iter().filter(|table| table.item_child && !is_bucketed_table(table.name)) {+ for row in other.get(child.name).map(Vec::as_slice).unwrap_or(&[]) {+ if let Some(activity) = live.get_mut(&item_key(row, child.item_ref_column())) {+ *activity = (*activity).max(updated_at(row));+ }+ }+ }++ live.into_iter().filter(|(key, activity)| outlives(tombstones[key], *activity)).map(|(key, _)| key).collect()+}++/// Remove the item rows named by `keys` and every row hanging off them.+fn remove_items(tables: &mut Tables, keys: &HashSet<String>) {+ if keys.is_empty() {+ return;+ }+ if let Some(rows) = tables.get_mut(ITEMS_TABLE) {+ rows.retain(|row| !keys.contains(&item_key(row, ID_COL)));+ }+ for child in SYNCABLE_TABLES.iter().filter(|table| table.item_child) {+ if let Some(rows) = tables.get_mut(child.name) {+ rows.retain(|row| !keys.contains(&item_key(row, child.item_ref_column())));+ }+ }+}++/// The `(ManifestId, item id)` key of an item row, or of a child row through `item_column`.+fn item_key(row: &CodecRecord, item_column: &str) -> String {+ get_key(row, &[MANIFEST_ID_COL, item_column])+}diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rsindex 4accd1160..f08bf461f 100644--- a/core/rust/src/vault_merge/mod.rs+++ b/core/rust/src/vault_merge/mod.rs@@ -17,6 +17,7 @@ use crate::timestamp::updated_at;use crate::vault_model::{id_key, TableConfig, SYNCABLE_TABLES};use crate::vault_codec::{bucket_categories, identity_part, is_bucketed_table, tables_for_category, CodecRecord, DataBucket, Manifest};+mod item_deletes;#[cfg(test)]mod tests;@@ -140,6 +141,9 @@ fn merge_manifest_pair(let mut incoming_tables = local.map(|m| m.tables).unwrap_or_default();incoming_tables.extend(local_bucket_tables);+ // An item is deleted as a unit including any child rows.+ item_deletes::resolve_item_deletes(&mut base_tables, &mut incoming_tables);+let table_names: BTreeSet<String> = base_tables.keys().chain(incoming_tables.keys()).cloned().collect();let mut merged: HashMap<String, Vec<CodecRecord>> = HashMap::new();@@ -165,8 +169,9 @@ fn merge_manifest_pair(}}- // A union of concurrently added multi-value rows can leave two rows at the same ValueIndex;- // re-normalizing the output renumbers them, so a merged manifest is normalized like any other.+ // A union of concurrently added multi-value rows can leave two rows at the same ValueIndex, and a+ // delete that won can leave the other side's children under a tombstoned item; re-normalizing the+ // output renumbers the first and drops the second, so a merged manifest is normalized like any other.crate::vault_codec::normalize::normalize_row_shapes(&mut merged);let mut buckets = unknown_server_buckets;diff --git a/core/rust/src/vault_merge/tests.rs b/core/rust/src/vault_merge/tests.rsindex f4bb3d567..31f17adb7 100644--- a/core/rust/src/vault_merge/tests.rs+++ b/core/rust/src/vault_merge/tests.rs@@ -36,10 +36,10 @@ fn items_manifest(manifest_id: &str, rows: Vec<CodecRecord>) -> Manifest {fn schema() -> HashMap<String, Vec<String>> {[- ("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Name".to_string(), "UpdatedAt".to_string()]),+ ("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Name".to_string(), "IsDeleted".to_string(), "UpdatedAt".to_string()]),("Settings".to_string(), vec!["ManifestId".to_string(), "Key".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),("ItemStats".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "LastUsedAt".to_string(), "UpdatedAt".to_string()]),- ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),+ ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string(), "IsDeleted".to_string(), "UpdatedAt".to_string()]),("ItemTags".to_string(), vec!["ManifestId".to_string(), "ItemId".to_string(), "TagId".to_string(), "UpdatedAt".to_string()]),("Attachments".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Blob".to_string(), "UpdatedAt".to_string()]),]@@ -582,3 +582,140 @@ fn keys_compare_guids_case_insensitively_and_survive_a_missing_column() {assert_ne!(get_key(&row(MANIFEST, ROW), &columns), get_key(&row(OTHER_MANIFEST, ROW), &columns));assert_eq!(get_key(&HashMap::from([("Id".to_string(), json!(ROW))]), &columns), format!(":{}", ROW));}++/// An `Items` row, live or tombstoned.+fn item_row(manifest_id: &str, id: &str, deleted: bool, updated_at: &str) -> CodecRecord {+ let mut row = item(manifest_id, id, "item", updated_at);+ row.insert("IsDeleted".to_string(), json!(if deleted { 1 } else { 0 }));+ row+}++/// A single-value `FieldValues` row of `item-1` in wire shape, live or tombstoned.+fn field_of_item(manifest_id: &str, field_key: &str, value: &str, deleted: bool, updated_at: &str) -> CodecRecord {+ [+ ("ManifestId".to_string(), json!(manifest_id)),+ ("ItemId".to_string(), json!("item-1")),+ ("FieldKey".to_string(), json!(field_key)),+ ("ValueIndex".to_string(), json!(0)),+ ("Value".to_string(), json!(value)),+ ("IsDeleted".to_string(), json!(if deleted { 1 } else { 0 })),+ ("UpdatedAt".to_string(), json!(updated_at)),+ ]+ .into_iter()+ .collect()+}++fn item_manifest(manifest_id: &str, items: Vec<CodecRecord>, field_values: Vec<CodecRecord>) -> Manifest {+ manifest(manifest_id, [("Items".to_string(), items), ("FieldValues".to_string(), field_values)].into_iter().collect())+}++fn merge_manifests(server: Vec<Manifest>, local: Vec<Manifest>) -> Vec<CanonicalManifestMerge> {+ merge_canonical(CanonicalMergeInput { server_manifests: server, server_buckets: vec![], contentless_server_manifest_ids: vec![], local_manifests: local, local_buckets: vec![], schema_columns: schema() }).unwrap().manifests+}++fn live_values(merged: &CanonicalManifestMerge) -> Vec<String> {+ let mut values: Vec<String> = merged.manifest.tables["FieldValues"].iter().filter(|r| r["IsDeleted"] == json!(0)).map(|r| r["Value"].as_str().unwrap().to_string()).collect();+ values.sort();+ values+}++const T_OLD: &str = "2024-01-01T00:00:00Z";+const T_DELETE: &str = "2024-01-05T05:00:00Z";+const T_EDIT: &str = "2024-01-05T05:05:00Z";++#[test]+fn a_permanent_delete_does_not_get_its_children_back_from_the_other_side() {+ // The deleting side removed the item's rows; the server still holds them and must not resurrect them.+ let server = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "secret", false, T_OLD)]);+ let local = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_DELETE)], vec![]);+ for (base, incoming) in [(server.clone(), local.clone()), (local, server)] {+ let merged = &merge_manifests(vec![base], vec![incoming])[0];+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1), "the delete stands");+ assert!(merged.manifest.tables["FieldValues"].is_empty(), "and the deleted item keeps no rows");+ }+}++#[test]+fn an_item_edited_after_the_other_side_deleted_it_survives_whole() {+ // One side deleted the item at 05:00, tombstoning its rows as the pruner does. The other edited one+ // field at 05:05, which never touches the item row. The edit is the last write, so the item survives,+ // and with every field: the delete's child tombstones must not wipe the fields the edit left alone.+ let deleting = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_DELETE)], vec![+ field_of_item(PERSONAL, "login.username", "user", true, T_DELETE),+ field_of_item(PERSONAL, "login.password", "old-secret", true, T_DELETE),+ ]);+ let editing = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![+ field_of_item(PERSONAL, "login.username", "user", false, T_OLD),+ field_of_item(PERSONAL, "login.password", "new-secret", false, T_EDIT),+ ]);+ for (base, incoming) in [(deleting.clone(), editing.clone()), (editing, deleting)] {+ let merged = &merge_manifests(vec![base], vec![incoming])[0];+ assert_eq!(merged.manifest.tables["Items"].len(), 1);+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(0), "the later edit outlives the delete");+ assert_eq!(live_values(merged), vec!["new-secret", "user"], "with the untouched field intact");+ }+}++#[test]+fn a_delete_newer_than_everything_the_other_side_did_stands() {+ let editing = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "edited", false, T_DELETE)]);+ let deleting = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_EDIT)], vec![]);+ for (base, incoming) in [(deleting.clone(), editing.clone()), (editing, deleting)] {+ let merged = &merge_manifests(vec![base], vec![incoming])[0];+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1));+ assert!(merged.manifest.tables["FieldValues"].is_empty());+ }+}++#[test]+fn a_delete_and_an_edit_at_the_same_instant_keep_the_server_side() {+ let deleting = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_EDIT)], vec![]);+ let editing = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "edited", false, T_EDIT)]);++ let merged = &merge_manifests(vec![deleting.clone()], vec![editing.clone()])[0];+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1), "the server's delete keeps the tie");++ let merged = &merge_manifests(vec![editing], vec![deleting])[0];+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(0), "the server's edit keeps the tie");+ assert_eq!(live_values(merged), vec!["edited"]);+}++#[test]+fn a_usage_counter_does_not_outlive_a_delete() {+ // ItemStats ticks on every autofill; that is not an edit and must not undo a delete.+ let stats = |updated_at: &str| -> CodecRecord {+ [("ManifestId".to_string(), json!(PERSONAL)), ("Id".to_string(), json!("item-1")), ("LastUsedAt".to_string(), json!(updated_at)), ("UpdatedAt".to_string(), json!(updated_at))].into_iter().collect()+ };+ let output = merge_canonical(CanonicalMergeInput {+ server_manifests: vec![item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_DELETE)], vec![])],+ server_buckets: vec![],+ contentless_server_manifest_ids: vec![],+ local_manifests: vec![item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![])],+ local_buckets: vec![DataBucket::new(PERSONAL, "Stats", [("ItemStats".to_string(), vec![stats(T_EDIT)])].into_iter().collect())],+ schema_columns: schema(),+ })+ .unwrap();+ let merged = &output.manifests[0];+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1));+ assert!(merged.buckets.iter().all(|bucket| bucket.tables.get("ItemStats").is_none_or(Vec::is_empty)), "the deleted item's stats row goes with it");+}++#[test]+fn an_item_moved_to_another_manifest_does_not_stay_behind_in_the_one_it_left() {+ // The move leaves a tombstone in the source manifest, so the server's copy there loses instead of+ // surviving the union next to the moved one.+ let server = vec![+ item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "secret", false, T_OLD)]),+ item_manifest(SHARED, vec![], vec![]),+ ];+ let local = vec![+ item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_EDIT)], vec![]),+ item_manifest(SHARED, vec![item_row(SHARED, "item-1", false, T_EDIT)], vec![field_of_item(SHARED, "login.password", "secret", false, T_OLD)]),+ ];+ let merged = merge_manifests(server, local);+ let by_id: HashMap<&str, &CanonicalManifestMerge> = merged.iter().map(|m| (m.manifest_id.as_str(), m)).collect();+ assert_eq!(by_id[PERSONAL].manifest.tables["Items"][0]["IsDeleted"], json!(1), "the source manifest keeps only the tombstone");+ assert!(by_id[PERSONAL].manifest.tables["FieldValues"].is_empty());+ assert_eq!(by_id[SHARED].manifest.tables["Items"][0]["IsDeleted"], json!(0));+ assert_eq!(live_values(by_id[SHARED]), vec!["secret"], "and the moved item arrives whole");+}diff --git a/core/rust/src/vault_model/generated.rs b/core/rust/src/vault_model/generated.rsindex aa7e52225..9abffe5df 100644--- a/core/rust/src/vault_model/generated.rs+++ b/core/rust/src/vault_model/generated.rs@@ -86,6 +86,10 @@ pub static PERSONAL_TABLES: &[&str] = &[];/// adding two different things) and is not derived.pub static MULTI_VALUE_FIELD_KEYS: &[&str] = &["login.url"];+/// System field keys whose field holds one value. A key in neither list is a field this build does+/// not know (a newer writer's), whose rows the codec leaves as it found them.+pub static SINGLE_VALUE_FIELD_KEYS: &[&str] = &["login.email", "login.username", "login.password", "alias.first_name", "alias.last_name", "alias.gender", "alias.birthdate", "card.cardholder_name", "card.number", "card.expiry_month", "card.expiry_year", "card.cvv", "card.pin", "notes.content"];+/// The per-manifest delivery-keypair table. Every manifest carries its own asymmetric keypair(s),/// stamped with that manifest's id (`ManifestId`).pub const ENCRYPTION_KEYS_TABLE: &str = "EncryptionKeys";diff --git a/core/rust/src/vault_model/mod.rs b/core/rust/src/vault_model/mod.rsindex 95f54699b..ff81ce41f 100644--- a/core/rust/src/vault_model/mod.rs+++ b/core/rust/src/vault_model/mod.rs@@ -66,6 +66,11 @@ impl TableConfig {self}+ /// The column of an item-child table that carries the owning item's id: `Id` for ItemStats (its `Id` is the item's id), `ItemId` everywhere else.+ pub fn item_ref_column(&self) -> &'static str {+ if self.name == names::ITEM_STATS_TABLE { names::ID_COL } else { names::ITEM_ID_COL }+ }+/// The columns that together identify one row of this table: `(ManifestId, primary key columns)`/// for a manifest-scoped table, the primary key columns alone otherwise.pub fn identity_columns(&self) -> Vec<&'static str> {diff --git a/core/rust/src/vault_pruner/mod.rs b/core/rust/src/vault_pruner/mod.rsindex b7378c769..e02bea9d6 100644--- a/core/rust/src/vault_pruner/mod.rs+++ b/core/rust/src/vault_pruner/mod.rs@@ -14,7 +14,7 @@ use crate::sqlite_host::SqlStatement;use crate::vault_codec::row::{has_bytes, is_deleted, logo_kind, str_col};use crate::vault_codec::{CodecRecord, CodecTableData};use crate::vault_model::names::{- DELETED_AT_COL, FILE_DATA_COL, ID_COL, IS_DELETED_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_STATS_TABLE, KIND_COL, LOGOS_TABLE,+ DELETED_AT_COL, FILE_DATA_COL, ID_COL, IS_DELETED_COL, ITEMS_TABLE, ITEM_ID_COL, KIND_COL, LOGOS_TABLE,LOGO_ID_COL, LOGO_KIND_FAVICON, UPDATED_AT_COL,};use crate::vault_model::{BLOB_COLUMNS, SYNCABLE_TABLES, TRASH_RETENTION_DEFAULT_DAYS};@@ -92,7 +92,7 @@ pub fn get_prune_table_queries() -> Vec<PruneTableQuery> {for child in item_child_tables() {let query = match blob_column_for(child.name) {Some(blob_col) => format!("SELECT {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", ID_COL, ITEM_ID_COL, IS_DELETED_COL, blob_col, blob_col, child.name),- None => format!("SELECT {}, {} FROM {}", item_ref_column(child.name), IS_DELETED_COL, child.name),+ None => format!("SELECT {}, {} FROM {}", child.item_ref_column(), IS_DELETED_COL, child.name),};queries.push(PruneTableQuery { name: child.name.to_string(), query });}@@ -108,12 +108,6 @@ fn item_child_tables() -> impl Iterator<Item = &'static crate::vault_model::TablSYNCABLE_TABLES.iter().filter(|t| t.item_child)}-/// The column of an item-child table that carries the owning item's id: `Id` for ItemStats-/// (its `Id` *is* the item's id), `ItemId` everywhere else.-fn item_ref_column(table_name: &str) -> &'static str {- if table_name == ITEM_STATS_TABLE { ID_COL } else { ITEM_ID_COL }-}-/// The extracted blob column of a table, if it has one (see `vault_model::BLOB_COLUMNS`).fn blob_column_for(table_name: &str) -> Option<&'static str> {BLOB_COLUMNS.iter().find(|(t, _, _)| *t == table_name).map(|(_, col, _)| *col)@@ -188,7 +182,7 @@ fn expired_item_ids(items: &[CodecRecord], cutoff_date: DateTime<Utc>) -> Vec<Stfn tombstone_item_children(tables: &[CodecTableData], item_id: &str, now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {for child in item_child_tables() {let Some(records) = records_of(tables, child.name) else { continue };- let match_col = item_ref_column(child.name);+ let match_col = child.item_ref_column();let related_count = count_related_records(records, match_col, item_id);if related_count == 0 {continue;diff --git a/core/rust/src/vault_sync/db.rs b/core/rust/src/vault_sync/db.rsindex 018263b76..551f1140c 100644--- a/core/rust/src/vault_sync/db.rs+++ b/core/rust/src/vault_sync/db.rs@@ -232,7 +232,7 @@ const DEMOTE_KEYS_FOR_MANIFEST: &str = "UPDATE EncryptionKeys SET IsPrimary = 0,const INSERT_KEY_FOR_MANIFEST: &str = "INSERT INTO EncryptionKeys (Id, ManifestId, PublicKey, PrivateKey, IsPrimary, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, 1, ?, ?, 0)";const INSERT_FOLDER: &str = "INSERT INTO Folders (Id, Name, ParentFolderId, ManifestId, Weight, IsDeleted, CreatedAt, UpdatedAt) VALUES (?, ?, ?, COALESCE((SELECT ManifestId FROM Folders WHERE Id = ?), ?), 0, 0, ?, ?)";const RESTAMP_SUBTREE_FOLDERS: &str = "UPDATE Folders SET ManifestId = ? WHERE Id IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";-const RESTAMP_SUBTREE_ITEMS: &str = "UPDATE Items SET ManifestId = ? WHERE FolderId IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";+const RESTAMP_SUBTREE_ITEMS: &str = "UPDATE Items SET ManifestId = ?, UpdatedAt = ? WHERE ManifestId <> ? AND FolderId IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";const FIND_ITEMS_WITH_FOREIGN_LOGO: &str = "SELECT i.Id, i.ManifestId, origin.Kind, origin.Source FROM Items i INNER JOIN Logos origin ON origin.Id = i.LogoId LEFT JOIN Logos own ON own.Id = i.LogoId AND own.ManifestId = i.ManifestId WHERE i.LogoId IS NOT NULL AND own.Id IS NULL AND i.IsDeleted = 0";const GET_LOGO_ID_FOR_KEY: &str = "SELECT Id FROM Logos WHERE ManifestId = ? AND Kind = ? AND Source = ? AND IsDeleted = 0 LIMIT 1";const GET_BEST_LOGO_FOR_KEY: &str = "SELECT FileData, MimeType, Name FROM Logos WHERE Kind = ? AND Source = ? AND IsDeleted = 0 ORDER BY (FileData IS NOT NULL AND LENGTH(FileData) > 0) DESC, UpdatedAt DESC LIMIT 1";@@ -294,7 +294,7 @@ pub(crate) async fn render_manifest_folder(host: &Host, manifest_id: &str, name:vec![SqlStatement { sql: INSERT_FOLDER.to_string(), params: vec![json!(folder_id), json!(name), Value::Null, Value::Null, json!(active_manifest_id), json!(now), json!(now)] },SqlStatement { sql: RESTAMP_SUBTREE_FOLDERS.to_string(), params: vec![json!(manifest_id), json!(folder_id)] },- SqlStatement { sql: RESTAMP_SUBTREE_ITEMS.to_string(), params: vec![json!(manifest_id), json!(folder_id)] },+ SqlStatement { sql: RESTAMP_SUBTREE_ITEMS.to_string(), params: vec![json!(manifest_id), json!(now), json!(manifest_id), json!(folder_id)] },],).await?;diff --git a/core/rust/src/vault_sync/tests/item_move.rs b/core/rust/src/vault_sync/tests/item_move.rsnew file mode 100644index 000000000..168e2941c--- /dev/null+++ b/core/rust/src/vault_sync/tests/item_move.rs@@ -0,0 +1,67 @@+//! The schema triggers that carry an item across manifests, run against the real client schema.++use rusqlite::Connection;++use super::test_host::{complete_schema_sql, open_schema_db};++const PERSONAL: &str = "11111111-1111-4111-8111-111111111111";+const SHARED: &str = "22222222-2222-4222-8222-222222222222";+const ITEM: &str = "33333333-3333-4333-8333-333333333333";+const T_CREATED: &str = "2024-01-01 00:00:00.000";++/// A vault holding one personal item with a password and a TOTP code.+fn vault_with_item() -> Connection {+ let db = open_schema_db(&complete_schema_sql());+ db.execute("INSERT INTO Items (ManifestId, Id, Name, ItemType, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, ?2, 'Bank', 'Login', ?3, ?3, 0)", (PERSONAL, ITEM, T_CREATED)).unwrap();+ db.execute("INSERT INTO FieldValues (ManifestId, Id, ItemId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, 'fv-1', ?2, 'login.password', 'secret', 0, 0, ?3, ?3, 0)", (PERSONAL, ITEM, T_CREATED)).unwrap();+ db.execute("INSERT INTO TotpCodes (ManifestId, Id, ItemId, Name, SecretKey, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, 'totp-1', ?2, 'code', 'ABC', ?3, ?3, 0)", (PERSONAL, ITEM, T_CREATED)).unwrap();+ db+}++fn move_item(db: &Connection, from: &str, to: &str, at: &str) {+ assert_eq!(db.execute("UPDATE Items SET ManifestId = ?1, UpdatedAt = ?2 WHERE Id = ?3 AND ManifestId = ?4", (to, at, ITEM, from)).unwrap(), 1);+}++/// `(IsDeleted, UpdatedAt)` of the item row in `manifest`, when it has one.+fn item_in(db: &Connection, manifest: &str) -> Option<(i64, String)> {+ db.query_row("SELECT IsDeleted, UpdatedAt FROM Items WHERE ManifestId = ?1 AND Id = ?2", (manifest, ITEM), |row| Ok((row.get(0)?, row.get(1)?))).ok()+}++fn child_rows_in(db: &Connection, manifest: &str) -> i64 {+ let count = |table: &str| -> i64 { db.query_row(&format!("SELECT COUNT(*) FROM {} WHERE ManifestId = ?1 AND ItemId = ?2", table), (manifest, ITEM), |row| row.get(0)).unwrap() };+ count("FieldValues") + count("TotpCodes")+}++#[test]+fn moving_an_item_leaves_a_tombstone_in_the_manifest_it_left() {+ // Without it the merge, a union per manifest, keeps the server's copy in the source manifest and the item ends up in both.+ let db = vault_with_item();+ move_item(&db, PERSONAL, SHARED, "2024-02-01 00:00:00.000");++ assert_eq!(item_in(&db, SHARED), Some((0, "2024-02-01 00:00:00.000".to_string())));+ assert_eq!(child_rows_in(&db, SHARED), 2, "the item's rows follow it");+ assert_eq!(item_in(&db, PERSONAL), Some((1, "2024-02-01 00:00:00.000".to_string())), "a tombstone stamped with the move stays behind");+ assert_eq!(child_rows_in(&db, PERSONAL), 0);+}++#[test]+fn an_item_can_move_back_into_the_manifest_it_left() {+ // The tombstone holds the primary key the item returns to, so the move back clears it first.+ let db = vault_with_item();+ move_item(&db, PERSONAL, SHARED, "2024-02-01 00:00:00.000");+ move_item(&db, SHARED, PERSONAL, "2024-03-01 00:00:00.000");++ assert_eq!(item_in(&db, PERSONAL), Some((0, "2024-03-01 00:00:00.000".to_string())));+ assert_eq!(child_rows_in(&db, PERSONAL), 2);+ assert_eq!(item_in(&db, SHARED), Some((1, "2024-03-01 00:00:00.000".to_string())), "and the shared manifest now holds the tombstone");+}++#[test]+fn a_live_item_with_the_same_id_in_the_destination_is_never_cleared() {+ // Two manifests holding the same item id is a supported state; only a tombstone gives way to a move.+ let db = vault_with_item();+ db.execute("INSERT INTO Items (ManifestId, Id, Name, ItemType, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, ?2, 'Theirs', 'Login', ?3, ?3, 0)", (SHARED, ITEM, T_CREATED)).unwrap();+ let moved = db.execute("UPDATE Items SET ManifestId = ?1 WHERE Id = ?2 AND ManifestId = ?3", (SHARED, ITEM, PERSONAL));+ assert!(moved.is_err(), "the move is refused rather than overwriting the other item");+ assert_eq!(item_in(&db, SHARED).map(|(deleted, _)| deleted), Some(0));+}diff --git a/core/rust/src/vault_sync/tests/mod.rs b/core/rust/src/vault_sync/tests/mod.rsindex 100e7ed10..ce3ec5d01 100644--- a/core/rust/src/vault_sync/tests/mod.rs+++ b/core/rust/src/vault_sync/tests/mod.rs@@ -1,6 +1,6 @@//! Engine tests: the whole sync driven through the command loop against a real SQLite host.-mod sharing;+mod item_move;mod test_host;use std::collections::HashMap;diff --git a/core/vault/src/sql/SqlConstants.ts b/core/vault/src/sql/SqlConstants.tsindex f7ceb050c..7e4543cb3 100644--- a/core/vault/src/sql/SqlConstants.ts+++ b/core/vault/src/sql/SqlConstants.ts@@ -1778,6 +1778,21 @@ INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion")VALUES ('20260913090000_2.1.0-ManifestScopedStorage', '10.0.10');BEGIN TRANSACTION;+CREATE TRIGGER IF NOT EXISTS "TR_Items_ClearTombstoneBeforeReturn"+BEFORE UPDATE OF "ManifestId" ON "Items"+FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"+ AND EXISTS (SELECT 1 FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1)+BEGIN+ DELETE FROM "FieldValues" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "FieldHistories" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "ItemTags" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "Attachments" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "Passkeys" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "TotpCodes" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "ItemStats" WHERE "Id" = NEW."Id" AND "ManifestId" = NEW."ManifestId";+ DELETE FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1;+END;+CREATE TRIGGER IF NOT EXISTS "TR_Items_ResyncChildManifestIds"AFTER UPDATE OF "ManifestId" ON "Items"FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"@@ -1789,6 +1804,8 @@ BEGINUPDATE "Passkeys" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";UPDATE "TotpCodes" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";UPDATE "ItemStats" SET "ManifestId" = NEW."ManifestId" WHERE "Id" = NEW."Id" AND "ManifestId" = OLD."ManifestId";+ INSERT OR IGNORE INTO "Items" ("ManifestId", "Id", "ItemType", "CreatedAt", "UpdatedAt", "IsDeleted")+ SELECT OLD."ManifestId", OLD."Id", OLD."ItemType", OLD."CreatedAt", NEW."UpdatedAt", 1 WHERE OLD."IsDeleted" = 0;END;COMMIT;