Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update LWW merge logic for delete/create usecase with item child content (#2159)

Leendert de Borst5088ae8ed388
diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/models/VaultSql.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/models/VaultSql.kt
index de4d29516..59e009b79 100644
--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/models/VaultSql.kt
+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/models/VaultSql.kt
@@ -1786,6 +1786,21 @@ INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion")
VALUES ('20260913090000_2.1.0-ManifestScopedStorage', '10.0.10');
BEGIN TRANSACTION;
+CREATE TRIGGER IF NOT EXISTS "TR_Items_ClearTombstoneBeforeReturn"
+BEFORE UPDATE OF "ManifestId" ON "Items"
+FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"
+ AND EXISTS (SELECT 1 FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1)
+BEGIN
+ DELETE FROM "FieldValues" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "FieldHistories" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "ItemTags" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "Attachments" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "Passkeys" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "TotpCodes" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "ItemStats" WHERE "Id" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1;
+END;
+
CREATE TRIGGER IF NOT EXISTS "TR_Items_ResyncChildManifestIds"
AFTER UPDATE OF "ManifestId" ON "Items"
FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"
@@ -1797,6 +1812,8 @@ BEGIN
UPDATE "Passkeys" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";
UPDATE "TotpCodes" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";
UPDATE "ItemStats" SET "ManifestId" = NEW."ManifestId" WHERE "Id" = NEW."Id" AND "ManifestId" = OLD."ManifestId";
+ INSERT OR IGNORE INTO "Items" ("ManifestId", "Id", "ItemType", "CreatedAt", "UpdatedAt", "IsDeleted")
+ SELECT OLD."ManifestId", OLD."Id", OLD."ItemType", OLD."CreatedAt", NEW."UpdatedAt", 1 WHERE OLD."IsDeleted" = 0;
END;
COMMIT;
diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/ItemQueries.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/ItemQueries.kt
index 94a4bad52..a0f648da0 100644
--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/ItemQueries.kt
+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/ItemQueries.kt
@@ -65,7 +65,7 @@ object ItemQueries {
LEFT JOIN FieldDefinitions fd ON fd.ManifestId = fv.ManifestId AND fd.Id = fv.FieldDefinitionId
WHERE (fv.ManifestId, fv.ItemId) IN (VALUES $placeholders)
AND fv.IsDeleted = 0
- ORDER BY fv.ItemId, fv.Weight
+ ORDER BY fv.ItemId, fv.Weight, fv.ValueIndex
""".trimIndent()
}
diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/PasskeyQueries.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/PasskeyQueries.kt
index 0210cea4a..b67ff0efe 100644
--- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/PasskeyQueries.kt
+++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/queries/PasskeyQueries.kt
@@ -133,8 +133,10 @@ object PasskeyQueries {
* Insert a field value. Binds (id, itemId, fieldDefinitionId, fieldKey, value, weight, now, now, 0, manifestId).
*/
const val INSERT_FIELD_VALUE = """
- INSERT INTO FieldValues (Id, ItemId, FieldDefinitionId, FieldKey, Value, Weight, CreatedAt, UpdatedAt, IsDeleted, ManifestId)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ INSERT INTO FieldValues (Id, ItemId, FieldDefinitionId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted, ManifestId)
+ VALUES (?1, ?2, ?3, ?4, ?5, ?6,
+ (SELECT COALESCE(MAX(ValueIndex), -1) + 1 FROM FieldValues WHERE ItemId = ?2 AND ManifestId = ?10 AND FieldKey = ?4 AND IsDeleted = 0),
+ ?7, ?8, ?9, ?10)
"""
/**
diff --git a/apps/mobile-app/ios/VaultModels/VaultSql.swift b/apps/mobile-app/ios/VaultModels/VaultSql.swift
index 16bf25db6..22577a7c6 100644
--- a/apps/mobile-app/ios/VaultModels/VaultSql.swift
+++ b/apps/mobile-app/ios/VaultModels/VaultSql.swift
@@ -1782,6 +1782,21 @@ public struct VaultSql {
VALUES ('20260913090000_2.1.0-ManifestScopedStorage', '10.0.10');
BEGIN TRANSACTION;
+ CREATE TRIGGER IF NOT EXISTS \"TR_Items_ClearTombstoneBeforeReturn\"
+ BEFORE UPDATE OF \"ManifestId\" ON \"Items\"
+ FOR EACH ROW WHEN OLD.\"ManifestId\" <> NEW.\"ManifestId\"
+ AND EXISTS (SELECT 1 FROM \"Items\" WHERE \"ManifestId\" = NEW.\"ManifestId\" AND \"Id\" = NEW.\"Id\" AND \"IsDeleted\" = 1)
+ BEGIN
+ DELETE FROM \"FieldValues\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";
+ DELETE FROM \"FieldHistories\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";
+ DELETE FROM \"ItemTags\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";
+ DELETE FROM \"Attachments\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";
+ DELETE FROM \"Passkeys\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";
+ DELETE FROM \"TotpCodes\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";
+ DELETE FROM \"ItemStats\" WHERE \"Id\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";
+ DELETE FROM \"Items\" WHERE \"ManifestId\" = NEW.\"ManifestId\" AND \"Id\" = NEW.\"Id\" AND \"IsDeleted\" = 1;
+ END;
+
CREATE TRIGGER IF NOT EXISTS \"TR_Items_ResyncChildManifestIds\"
AFTER UPDATE OF \"ManifestId\" ON \"Items\"
FOR EACH ROW WHEN OLD.\"ManifestId\" <> NEW.\"ManifestId\"
@@ -1793,6 +1808,8 @@ public struct VaultSql {
UPDATE \"Passkeys\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";
UPDATE \"TotpCodes\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";
UPDATE \"ItemStats\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"Id\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";
+ INSERT OR IGNORE INTO \"Items\" (\"ManifestId\", \"Id\", \"ItemType\", \"CreatedAt\", \"UpdatedAt\", \"IsDeleted\")
+ SELECT OLD.\"ManifestId\", OLD.\"Id\", OLD.\"ItemType\", OLD.\"CreatedAt\", NEW.\"UpdatedAt\", 1 WHERE OLD.\"IsDeleted\" = 0;
END;
COMMIT;
diff --git a/apps/mobile-app/ios/VaultStoreKit/Database/Queries/ItemQueries.swift b/apps/mobile-app/ios/VaultStoreKit/Database/Queries/ItemQueries.swift
index c1962e3ba..2035cc5ff 100644
--- a/apps/mobile-app/ios/VaultStoreKit/Database/Queries/ItemQueries.swift
+++ b/apps/mobile-app/ios/VaultStoreKit/Database/Queries/ItemQueries.swift
@@ -50,7 +50,7 @@ public struct ItemQueries {
LEFT JOIN FieldDefinitions fd ON fd.ManifestId = fv.ManifestId AND fd.Id = fv.FieldDefinitionId
WHERE (fv.ManifestId, fv.ItemId) IN (VALUES \(placeholders))
AND fv.IsDeleted = 0
- ORDER BY fv.ItemId, fv.Weight
+ ORDER BY fv.ItemId, fv.Weight, fv.ValueIndex
"""
}
@@ -75,8 +75,10 @@ public struct ItemQueries {
public struct FieldValueQueries {
/// Insert a new field value, bound as [id, itemId, fieldDefinitionId, fieldKey, value, weight, now, now, 0, manifestId].
public static let insert = """
- INSERT INTO FieldValues (Id, ItemId, FieldDefinitionId, FieldKey, Value, Weight, CreatedAt, UpdatedAt, IsDeleted, ManifestId)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ INSERT INTO FieldValues (Id, ItemId, FieldDefinitionId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted, ManifestId)
+ VALUES (?1, ?2, ?3, ?4, ?5, ?6,
+ (SELECT COALESCE(MAX(ValueIndex), -1) + 1 FROM FieldValues WHERE ItemId = ?2 AND ManifestId = ?10 AND FieldKey = ?4 AND IsDeleted = 0),
+ ?7, ?8, ?9, ?10)
"""
/// The Weight a system field's values are written with: the field's DefaultDisplayOrder from
diff --git a/apps/server/Databases/AliasClientDb/Migrations/ItemChildManifestTriggerSql.cs b/apps/server/Databases/AliasClientDb/Migrations/ItemChildManifestTriggerSql.cs
index 59ae86ead..f349c22ed 100644
--- a/apps/server/Databases/AliasClientDb/Migrations/ItemChildManifestTriggerSql.cs
+++ b/apps/server/Databases/AliasClientDb/Migrations/ItemChildManifestTriggerSql.cs
@@ -10,8 +10,8 @@ namespace AliasClientDb.Migrations;
using System.Text;
/// <summary>
-/// DDL of the trigger that keeps item-scoped rows stamped with their item's manifest.
-/// This trigger needs to be dropped and recreated after a table is rebuilt, hence the shared code.
+/// DDL of the triggers that carry an item across manifests: its rows follow it and a tombstone stays behind.
+/// The triggers need to be dropped and recreated after a table is rebuilt, hence the shared code.
/// </summary>
internal static class ItemChildManifestTriggerSql
{
@@ -26,31 +26,54 @@ internal static class ItemChildManifestTriggerSql
public static string Name => "TR_Items_ResyncChildManifestIds";
/// <summary>
- /// Gets the statement that drops the trigger.
+ /// Gets the name of the trigger that clears the way for an item moving back into a manifest it left before.
/// </summary>
- public static string Drop => $"DROP TRIGGER IF EXISTS \"{Name}\";";
+ public static string ReturnName => "TR_Items_ClearTombstoneBeforeReturn";
/// <summary>
- /// Gets the statement that creates the trigger.
+ /// Gets the statement that drops the triggers.
+ /// </summary>
+ public static string Drop => $"DROP TRIGGER IF EXISTS \"{Name}\"; DROP TRIGGER IF EXISTS \"{ReturnName}\";";
+
+ /// <summary>
+ /// Gets the statement that creates the triggers.
/// </summary>
public static string Create
{
get
{
- var body = new StringBuilder();
+ var restamp = new StringBuilder();
+ var clear = new StringBuilder();
foreach (var table in ItemIdChildTables)
{
- body.AppendLine($" UPDATE \"{table}\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";");
+ restamp.AppendLine($" UPDATE \"{table}\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";");
+ clear.AppendLine($" DELETE FROM \"{table}\" WHERE \"ItemId\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";");
}
- body.AppendLine($" UPDATE \"ItemStats\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"Id\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";");
+ restamp.AppendLine($" UPDATE \"ItemStats\" SET \"ManifestId\" = NEW.\"ManifestId\" WHERE \"Id\" = NEW.\"Id\" AND \"ManifestId\" = OLD.\"ManifestId\";");
+ clear.AppendLine($" DELETE FROM \"ItemStats\" WHERE \"Id\" = NEW.\"Id\" AND \"ManifestId\" = NEW.\"ManifestId\";");
+ /*
+ * When moving an item to a new manifest, a tombstone is left behind in the old manifest.
+ * This trigger clears it before the item is moved back.
+ */
return $"""
+ CREATE TRIGGER IF NOT EXISTS "{ReturnName}"
+ BEFORE UPDATE OF "ManifestId" ON "Items"
+ FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"
+ AND EXISTS (SELECT 1 FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1)
+ BEGIN
+ {clear.ToString().TrimEnd()}
+ DELETE FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1;
+ END;
+
CREATE TRIGGER IF NOT EXISTS "{Name}"
AFTER UPDATE OF "ManifestId" ON "Items"
FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"
BEGIN
- {body.ToString().TrimEnd()}
+ {restamp.ToString().TrimEnd()}
+ INSERT OR IGNORE INTO "Items" ("ManifestId", "Id", "ItemType", "CreatedAt", "UpdatedAt", "IsDeleted")
+ SELECT OLD."ManifestId", OLD."Id", OLD."ItemType", OLD."CreatedAt", NEW."UpdatedAt", 1 WHERE OLD."IsDeleted" = 0;
END;
""";
}
diff --git a/apps/server/Databases/AliasClientDb/Scripts/MigrationSql/000_FullSchema.sql b/apps/server/Databases/AliasClientDb/Scripts/MigrationSql/000_FullSchema.sql
index bc36e3c5d..ad57ffe9f 100644
--- a/apps/server/Databases/AliasClientDb/Scripts/MigrationSql/000_FullSchema.sql
+++ b/apps/server/Databases/AliasClientDb/Scripts/MigrationSql/000_FullSchema.sql
@@ -1771,6 +1771,21 @@ INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion")
VALUES ('20260913090000_2.1.0-ManifestScopedStorage', '10.0.10');
BEGIN TRANSACTION;
+CREATE TRIGGER IF NOT EXISTS "TR_Items_ClearTombstoneBeforeReturn"
+BEFORE UPDATE OF "ManifestId" ON "Items"
+FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"
+ AND EXISTS (SELECT 1 FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1)
+BEGIN
+ DELETE FROM "FieldValues" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "FieldHistories" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "ItemTags" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "Attachments" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "Passkeys" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "TotpCodes" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "ItemStats" WHERE "Id" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1;
+END;
+
CREATE TRIGGER IF NOT EXISTS "TR_Items_ResyncChildManifestIds"
AFTER UPDATE OF "ManifestId" ON "Items"
FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"
@@ -1782,6 +1797,8 @@ BEGIN
UPDATE "Passkeys" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";
UPDATE "TotpCodes" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";
UPDATE "ItemStats" SET "ManifestId" = NEW."ManifestId" WHERE "Id" = NEW."Id" AND "ManifestId" = OLD."ManifestId";
+ INSERT OR IGNORE INTO "Items" ("ManifestId", "Id", "ItemType", "CreatedAt", "UpdatedAt", "IsDeleted")
+ SELECT OLD."ManifestId", OLD."Id", OLD."ItemType", OLD."CreatedAt", NEW."UpdatedAt", 1 WHERE OLD."IsDeleted" = 0;
END;
COMMIT;
diff --git a/core/client/src/database/__tests__/FieldValueRows.test.ts b/core/client/src/database/__tests__/FieldValueRows.test.ts
new file mode 100644
index 000000000..eec22a094
--- /dev/null
+++ b/core/client/src/database/__tests__/FieldValueRows.test.ts
@@ -0,0 +1,198 @@
+import { FieldTypes, ItemTypes } from '@aliasvault/models/vault';
+import { VaultSqlGenerator } from '@aliasvault/vault';
+import { describe, expect, it } from 'vitest';
+
+import { getPlatform } from '../../platform/ClientPlatform';
+import { ItemRepository } from '../repositories/ItemRepository';
+import { LogoRepository } from '../repositories/LogoRepository';
+
+import type { ISqliteDatabase, SqliteValue } from '../../platform/SqliteEngine';
+import type { ISyncDatabaseClient, SqliteBindValue } from '../BaseRepository';
+import type { DraftItem } from '../ItemRef';
+import type { ItemField } from '@aliasvault/models/vault';
+
+const PERSONAL = '11111111-1111-4111-8111-111111111111';
+const ITEM = '33333333-3333-4333-8333-333333333333';
+
+type StoredRow = { Id: string; Value: string; ValueIndex: number; IsDeleted: number };
+
+/**
+ * A database client over an in-memory vault on the real schema.
+ */
+class TestDatabaseClient implements ISyncDatabaseClient {
+ private inTransaction = false;
+
+ /**
+ * Constructor for the TestDatabaseClient class.
+ * @param db - The opened database
+ */
+ public constructor(private readonly db: ISqliteDatabase) {}
+
+ /**
+ * Get the database.
+ * @returns The database
+ */
+ public getDb(): ISqliteDatabase {
+ return this.db;
+ }
+
+ /**
+ * Run a SELECT.
+ * @param query - The statement
+ * @param params - The bound parameters
+ * @returns The rows
+ */
+ public executeQuery<T>(query: string, params: SqliteBindValue[] = []): T[] {
+ return this.db.query<T>(query, params as SqliteValue[]);
+ }
+
+ /**
+ * Run a write.
+ * @param query - The statement
+ * @param params - The bound parameters
+ * @returns The number of rows changed
+ */
+ public executeUpdate(query: string, params: SqliteBindValue[] = []): number {
+ return this.db.run(query, params as SqliteValue[]);
+ }
+
+ /**
+ * Begin a transaction.
+ */
+ public beginTransaction(): void {
+ this.inTransaction = true;
+ }
+
+ /**
+ * Commit the transaction.
+ */
+ public async commitTransaction(): Promise<void> {
+ this.inTransaction = false;
+ }
+
+ /**
+ * Roll the transaction back.
+ */
+ public rollbackTransaction(): void {
+ this.inTransaction = false;
+ }
+
+ /**
+ * Whether a transaction is open.
+ * @returns True while one is
+ */
+ public isInTransaction(): boolean {
+ return this.inTransaction;
+ }
+
+ /**
+ * Get the manifest new rows are written into.
+ * @returns The personal manifest id
+ */
+ public getActiveManifestId(): string {
+ return PERSONAL;
+ }
+
+ /**
+ * Get the personal manifest.
+ * @returns The personal manifest id
+ */
+ public getPersonalManifestId(): string {
+ return PERSONAL;
+ }
+}
+
+/**
+ * A system field as the item form hands it to the repository.
+ * @param fieldKey - The system field key
+ * @param value - The field's value or values
+ * @returns The field
+ */
+function field(fieldKey: string, value: string | string[]): ItemField {
+ return { FieldKey: fieldKey, Label: fieldKey, FieldType: FieldTypes.Text, Value: value, IsHidden: false, DisplayOrder: 0, IsCustomField: false, EnableHistory: false };
+}
+
+/**
+ * Build a vault holding one item, saved with the given fields.
+ * @param fields - The fields of the first save
+ * @returns The database and a way to save the item again
+ */
+async function vaultWithItem(fields: ItemField[]): Promise<{ db: ISqliteDatabase; save: (fields: ItemField[]) => Promise<void> }> {
+ const db = await getPlatform().sqlite.open();
+ db.exec('PRAGMA foreign_keys = OFF;');
+ db.exec(new VaultSqlGenerator().getCompleteSchemaSql());
+ const client = new TestDatabaseClient(db);
+ const repository = new ItemRepository(client, new LogoRepository(client));
+ /**
+ * The item as the form hands it over, holding the given fields.
+ */
+ const draft = (itemFields: ItemField[]): DraftItem => ({ Id: ITEM, Name: 'Note', ItemType: ItemTypes.Note, Fields: itemFields, CreatedAt: '', UpdatedAt: '' } as DraftItem);
+
+ await repository.create(draft(fields));
+ return {
+ db,
+ /**
+ * Save the item again with the given fields.
+ */
+ save: async (next: ItemField[]): Promise<void> => {
+ expect(await repository.update(draft(next))).toBe(1);
+ },
+ };
+}
+
+/**
+ * The stored rows of one field, tombstones included.
+ * @param db - The database
+ * @param fieldKey - The system field key
+ * @returns The rows, in value order
+ */
+function rowsOf(db: ISqliteDatabase, fieldKey: string): StoredRow[] {
+ return db.query<StoredRow>('SELECT Id, Value, ValueIndex, IsDeleted FROM FieldValues WHERE ItemId = ? AND FieldKey = ? ORDER BY IsDeleted, ValueIndex', [ITEM, fieldKey]);
+}
+
+describe('field value rows', () => {
+ it('brings a removed field back on its own row instead of adding a second one', async () => {
+ /*
+ * Two rows for one single-value field collapse to the newest on the next sync, and a tombstone stamped
+ * by a device whose clock runs ahead would then beat the value the user just typed.
+ */
+ const { db, save } = await vaultWithItem([field('login.username', 'first')]);
+ const original = rowsOf(db, 'login.username')[0];
+
+ await save([]);
+ expect(rowsOf(db, 'login.username')).toMatchObject([{ Id: original.Id, IsDeleted: 1 }]);
+
+ await save([field('login.username', 'second')]);
+ expect(rowsOf(db, 'login.username')).toMatchObject([{ Id: original.Id, Value: 'second', IsDeleted: 0 }]);
+ });
+
+ it('keeps every url on its own row when one before it is removed', async () => {
+ // Shifting values across rows makes a concurrent edit of the last url come back as a near duplicate.
+ const { db, save } = await vaultWithItem([field('login.url', ['https://a.example', 'https://b.example', 'https://c.example'])]);
+ const [a, b, c] = rowsOf(db, 'login.url');
+ expect([a.ValueIndex, b.ValueIndex, c.ValueIndex]).toEqual([0, 1, 2]);
+
+ await save([field('login.url', ['https://b.example', 'https://c.example'])]);
+
+ expect(rowsOf(db, 'login.url')).toMatchObject([
+ { Id: b.Id, Value: 'https://b.example', ValueIndex: 0, IsDeleted: 0 },
+ { Id: c.Id, Value: 'https://c.example', ValueIndex: 1, IsDeleted: 0 },
+ { Id: a.Id, Value: 'https://a.example', IsDeleted: 1 },
+ ]);
+ });
+
+ it('appends a new url after the existing ones and edits a url in place', async () => {
+ const { db, save } = await vaultWithItem([field('login.url', ['https://a.example', 'https://b.example'])]);
+ const [a, b] = rowsOf(db, 'login.url');
+
+ await save([field('login.url', ['https://a.example', 'https://b-edited.example', 'https://c.example'])]);
+
+ const rows = rowsOf(db, 'login.url');
+ expect(rows).toMatchObject([
+ { Id: a.Id, Value: 'https://a.example', ValueIndex: 0 },
+ { Id: b.Id, Value: 'https://b-edited.example', ValueIndex: 1 },
+ { Value: 'https://c.example', ValueIndex: 2 },
+ ]);
+ expect(rows).toHaveLength(3);
+ });
+});
diff --git a/core/client/src/database/queries/ItemQueries.ts b/core/client/src/database/queries/ItemQueries.ts
index 8c995e0f9..08bef8932 100644
--- a/core/client/src/database/queries/ItemQueries.ts
+++ b/core/client/src/database/queries/ItemQueries.ts
@@ -144,7 +144,7 @@ export class ItemQueries {
LEFT JOIN FieldDefinitions fd ON fd.ManifestId = fv.ManifestId AND fd.Id = fv.FieldDefinitionId
WHERE (fv.ManifestId, fv.ItemId) IN (VALUES ${placeholders})
AND fv.IsDeleted = 0
- ORDER BY fv.ItemId, fv.Weight`;
+ ORDER BY fv.ItemId, fv.Weight, fv.ValueIndex`;
}
/**
@@ -163,7 +163,7 @@ export class ItemQueries {
FROM FieldValues fv
LEFT JOIN FieldDefinitions fd ON fd.ManifestId = fv.ManifestId AND fd.Id = fv.FieldDefinitionId
WHERE fv.ItemId = ? AND fv.ManifestId = ? AND fv.IsDeleted = 0
- ORDER BY fv.Weight`;
+ ORDER BY fv.Weight, fv.ValueIndex`;
/**
* Get tags for multiple items, matched on the whole item key (see {@link getFieldValuesForItems}).
@@ -335,27 +335,31 @@ export class ItemQueries {
*/
export class FieldValueQueries {
/**
- * Get existing field values for an item.
+ * Get every field value row of an item, tombstones included, so a write can bring a removed field's row back
+ * instead of adding a second row for the same field.
*/
- public static readonly GET_EXISTING_FOR_ITEM = `
- SELECT Id, FieldKey, FieldDefinitionId, Value, Weight
+ public static readonly GET_ALL_FOR_ITEM = `
+ SELECT Id, FieldKey, FieldDefinitionId, Value, Weight, ValueIndex, IsDeleted
FROM FieldValues
- WHERE ItemId = ? AND ManifestId = ? AND IsDeleted = 0`;
+ WHERE ItemId = ? AND ManifestId = ?
+ ORDER BY ValueIndex`;
/**
* Insert a new field value, stamped with the manifest of the item it hangs off.
*/
public static readonly INSERT = `
- INSERT INTO FieldValues (Id, ItemId, ManifestId, FieldDefinitionId, FieldKey, Value, Weight, CreatedAt, UpdatedAt, IsDeleted)
- VALUES (?, ?, ${BaseQueries.MANIFEST_OF_ITEM}, ?, ?, ?, ?, ?, ?, ?)`;
+ INSERT INTO FieldValues (Id, ItemId, ManifestId, FieldDefinitionId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted)
+ VALUES (?, ?, ${BaseQueries.MANIFEST_OF_ITEM}, ?, ?, ?, ?, ?, ?, ?, ?)`;
/**
- * Update an existing field value.
+ * Update an existing field value, bringing it back when it was removed before.
*/
public static readonly UPDATE = `
UPDATE FieldValues
SET Value = ?,
Weight = ?,
+ ValueIndex = ?,
+ IsDeleted = 0,
UpdatedAt = ?
WHERE Id = ? AND ManifestId = ?`;
@@ -374,7 +378,8 @@ export class FieldValueQueries {
public static readonly GET_FOR_HISTORY = `
SELECT FieldKey, Value
FROM FieldValues
- WHERE ItemId = ? AND ManifestId = ? AND IsDeleted = 0 AND FieldKey IS NOT NULL`;
+ WHERE ItemId = ? AND ManifestId = ? AND IsDeleted = 0 AND FieldKey IS NOT NULL
+ ORDER BY ValueIndex`;
}
/**
diff --git a/core/client/src/database/repositories/ItemRepository.ts b/core/client/src/database/repositories/ItemRepository.ts
index 713a26ca2..6e65237af 100644
--- a/core/client/src/database/repositories/ItemRepository.ts
+++ b/core/client/src/database/repositories/ItemRepository.ts
@@ -1,4 +1,4 @@
-import { FieldKey, LogoKinds, MAX_FIELD_HISTORY_RECORDS, normalizeTotpAlgorithm, normalizeTotpDigits, normalizeTotpPeriod } from '@aliasvault/models/vault';
+import { FieldKey, LogoKinds, MAX_FIELD_HISTORY_RECORDS, getSystemField, normalizeTotpAlgorithm, normalizeTotpDigits, normalizeTotpPeriod } from '@aliasvault/models/vault';
import { getFolderPath } from '../../items/FolderUtils';
import { selectFaviconTarget, toUrlList } from '../../rust/RustCore';
@@ -20,6 +20,19 @@ import type { Folder } from './FolderRepository';
import type { LogoRepository } from './LogoRepository';
import type { Item, ItemField, Attachment, TotpCode, FieldHistory, LogoSelection } from '@aliasvault/models/vault';
+/**
+ * A stored FieldValues row as the write path reads it, tombstones included.
+ */
+type StoredFieldValue = {
+ Id: string;
+ FieldKey: string | null;
+ FieldDefinitionId: string | null;
+ Value: string;
+ Weight: number;
+ ValueIndex: number;
+ IsDeleted: number;
+};
+
/**
* Repository for Item CRUD operations.
* Handles items, field values, field definitions, and field history.
@@ -920,7 +933,7 @@ export class ItemRepository extends BaseRepository {
? ['']
: filteredValues;
- for (const value of valuesToInsert) {
+ for (const [valueIndex, value] of valuesToInsert.entries()) {
yield* this.execute(FieldValueQueries.INSERT, [
this.generateId(),
itemId,
@@ -930,6 +943,7 @@ export class ItemRepository extends BaseRepository {
field.IsCustomField ? null : field.FieldKey,
value,
field.DisplayOrder ?? 0,
+ valueIndex,
currentDateTime,
currentDateTime,
0
@@ -992,91 +1006,110 @@ export class ItemRepository extends BaseRepository {
* Update field values for an existing item.
*/
private *updateFieldValues(item: DraftItem, manifestId: string, writeManifestId: string, currentDateTime: string): DbOp<void> {
- // Get existing FieldValues
- const existingFieldValues = yield* this.query<{
- Id: string;
- FieldKey: string | null;
- FieldDefinitionId: string | null;
- Value: string;
- Weight: number;
- }>(FieldValueQueries.GET_EXISTING_FOR_ITEM, [item.Id, manifestId]);
-
- // Build a map of existing FieldValues by key:index
- const existingByKey = new Map<string, { Id: string; Value: string; Weight: number }>();
- const fieldValueCounts = new Map<string, number>();
-
- for (const fv of existingFieldValues) {
- const key = fv.FieldKey || fv.FieldDefinitionId || '';
- const count = fieldValueCounts.get(key) || 0;
- existingByKey.set(`${key}:${count}`, { Id: fv.Id, Value: fv.Value, Weight: fv.Weight });
- fieldValueCounts.set(key, count + 1);
+ const storedRows = yield* this.query<StoredFieldValue>(FieldValueQueries.GET_ALL_FOR_ITEM, [item.Id, manifestId]);
+
+ const rowsByField = new Map<string, StoredFieldValue[]>();
+ for (const row of storedRows) {
+ const key = (row.FieldKey || row.FieldDefinitionId || '').toLowerCase();
+ rowsByField.set(key, [...(rowsByField.get(key) ?? []), row]);
}
- const processedIds = new Set<string>();
+ const keptIds = new Set<string>();
- // Update existing or insert new FieldValues
- if (item.Fields && item.Fields.length > 0) {
- for (const field of item.Fields) {
- // Skip empty system fields, but always persist custom fields (even if empty)
- const isEmpty = !field.Value || (typeof field.Value === 'string' && field.Value.trim() === '');
- if (isEmpty && !field.IsCustomField) {
- continue;
- }
+ for (const field of item.Fields ?? []) {
+ // Skip empty system fields, but always persist custom fields (even if empty)
+ const isEmpty = !field.Value || (typeof field.Value === 'string' && field.Value.trim() === '');
+ if (isEmpty && !field.IsCustomField) {
+ continue;
+ }
+
+ let fieldDefinitionId = null;
+
+ if (field.IsCustomField) {
+ fieldDefinitionId = yield* this.ensureOrUpdateFieldDefinition(field, item.Id, item.ItemType, writeManifestId, currentDateTime);
+ }
- let fieldDefinitionId = null;
+ const values = Array.isArray(field.Value) ? field.Value : [field.Value];
- if (field.IsCustomField) {
- fieldDefinitionId = yield* this.ensureOrUpdateFieldDefinition(field, item.Id, item.ItemType, writeManifestId, currentDateTime);
+ // For custom fields with no values, use empty string to preserve the field
+ const filteredValues = values.filter(v => v && (typeof v !== 'string' || v.trim() !== ''));
+ const valuesToProcess = field.IsCustomField && filteredValues.length === 0
+ ? ['']
+ : filteredValues;
+
+ const isMultiValue = !field.IsCustomField && (getSystemField(field.FieldKey)?.IsMultiValue ?? false);
+ const rows = ItemRepository.assignFieldValueRows(valuesToProcess, rowsByField.get(field.FieldKey.toLowerCase()) ?? [], isMultiValue);
+ const weight = field.DisplayOrder ?? 0;
+
+ for (const [valueIndex, value] of valuesToProcess.entries()) {
+ const row = rows[valueIndex];
+ if (!row) {
+ yield* this.execute(FieldValueQueries.INSERT, [
+ this.generateId(),
+ item.Id,
+ item.Id,
+ writeManifestId,
+ fieldDefinitionId,
+ field.IsCustomField ? null : field.FieldKey,
+ value,
+ weight,
+ valueIndex,
+ currentDateTime,
+ currentDateTime,
+ 0
+ ]);
+ continue;
}
- const values = Array.isArray(field.Value) ? field.Value : [field.Value];
- const effectiveKey = field.FieldKey;
-
- // For custom fields with no values, use empty string to preserve the field
- const filteredValues = values.filter(v => v && (typeof v !== 'string' || v.trim() !== ''));
- const valuesToProcess = field.IsCustomField && filteredValues.length === 0
- ? ['']
- : filteredValues;
-
- for (let i = 0; i < valuesToProcess.length; i++) {
- const value = valuesToProcess[i];
-
- const lookupKey = `${effectiveKey}:${i}`;
- const existing = existingByKey.get(lookupKey);
-
- if (existing) {
- processedIds.add(existing.Id);
- const newWeight = field.DisplayOrder ?? 0;
- if (existing.Value !== value || existing.Weight !== newWeight) {
- yield* this.execute(FieldValueQueries.UPDATE, [value, newWeight, currentDateTime, existing.Id, manifestId]);
- }
- } else {
- yield* this.execute(FieldValueQueries.INSERT, [
- this.generateId(),
- item.Id,
- item.Id,
- writeManifestId,
- fieldDefinitionId,
- field.IsCustomField ? null : field.FieldKey,
- value,
- field.DisplayOrder ?? 0,
- currentDateTime,
- currentDateTime,
- 0
- ]);
- }
+ keptIds.add(row.Id);
+ if (row.IsDeleted || row.Value !== value || row.Weight !== weight || row.ValueIndex !== valueIndex) {
+ yield* this.execute(FieldValueQueries.UPDATE, [value, weight, valueIndex, currentDateTime, row.Id, manifestId]);
}
}
}
- // Soft-delete any FieldValues that were not processed
- for (const fv of existingFieldValues) {
- if (!processedIds.has(fv.Id)) {
- yield* this.execute(FieldValueQueries.SOFT_DELETE, [currentDateTime, fv.Id, manifestId]);
+ // Soft-delete every live row no value was kept on
+ for (const row of storedRows) {
+ if (!row.IsDeleted && !keptIds.has(row.Id)) {
+ yield* this.execute(FieldValueQueries.SOFT_DELETE, [currentDateTime, row.Id, manifestId]);
}
}
}
+ /**
+ * Decide which stored row each value of one field is written to, `undefined` meaning a new row.
+ * @param values - The field's values, in display order
+ * @param fieldRows - The field's stored rows, tombstones included
+ * @param isMultiValue - Whether the field holds several values, each owning its row
+ * @returns One entry per value
+ */
+ private static assignFieldValueRows(values: string[], fieldRows: StoredFieldValue[], isMultiValue: boolean): (StoredFieldValue | undefined)[] {
+ const assigned: (StoredFieldValue | undefined)[] = values.map(() => undefined);
+ const free = new Set(fieldRows);
+
+ /**
+ * Give every still unassigned value the first free row that qualifies for it.
+ */
+ const assign = (qualifies: (row: StoredFieldValue, value: string) => boolean): void => {
+ for (const [index, value] of values.entries()) {
+ const row = assigned[index] ? undefined : [...free].find(candidate => qualifies(candidate, value));
+ if (row) {
+ assigned[index] = row;
+ free.delete(row);
+ }
+ }
+ };
+
+ // 1. An unchanged value stays on its row.
+ assign((row, value) => !row.IsDeleted && row.Value === value);
+ // 2. An edited value keeps a row that is still live.
+ assign(row => !row.IsDeleted);
+ // 3. A removed value that returns gets its row back. A single-value field has one row, whatever it held.
+ assign((row, value) => Boolean(row.IsDeleted) && (!isMultiValue || row.Value === value));
+
+ return assigned;
+ }
+
/**
* Ensure a field definition exists and is up-to-date.
*/
diff --git a/core/models/scripts/lib/vault-table-registry-emit.cjs b/core/models/scripts/lib/vault-table-registry-emit.cjs
index 02af0cec4..404629218 100644
--- a/core/models/scripts/lib/vault-table-registry-emit.cjs
+++ b/core/models/scripts/lib/vault-table-registry-emit.cjs
@@ -157,6 +157,10 @@ pub static PERSONAL_TABLES: &[&str] = &[${registry.VAULT_PERSONAL_TABLES.map((t)
/// adding two different things) and is not derived.
pub static MULTI_VALUE_FIELD_KEYS: &[&str] = ${rustStrSlice(registry.MULTI_VALUE_FIELD_KEYS)};
+/// System field keys whose field holds one value. A key in neither list is a field this build does
+/// not know (a newer writer's), whose rows the codec leaves as it found them.
+pub static SINGLE_VALUE_FIELD_KEYS: &[&str] = ${rustStrSlice(registry.SINGLE_VALUE_FIELD_KEYS)};
+
/// The per-manifest delivery-keypair table. Every manifest carries its own asymmetric keypair(s),
/// stamped with that manifest's id (\`ManifestId\`).
pub const ENCRYPTION_KEYS_TABLE: &str = "${registry.ENCRYPTION_KEYS_TABLE}";
diff --git a/core/models/src/vault/VaultTableRegistry.ts b/core/models/src/vault/VaultTableRegistry.ts
index 4c6648c15..88f84b3ec 100644
--- a/core/models/src/vault/VaultTableRegistry.ts
+++ b/core/models/src/vault/VaultTableRegistry.ts
@@ -196,6 +196,14 @@ export const MULTI_VALUE_FIELD_KEYS: string[] = Object.values(SystemFieldRegistr
.filter((field) => field.IsMultiValue)
.map((field) => field.FieldKey.toLowerCase());
+/**
+ * System field keys whose field holds one value. A key in neither list is a field this build does not
+ * know (a newer writer's), whose rows the codec leaves as it found them.
+ */
+export const SINGLE_VALUE_FIELD_KEYS: string[] = Object.values(SystemFieldRegistry)
+ .filter((field) => !field.IsMultiValue)
+ .map((field) => field.FieldKey.toLowerCase());
+
/**
* The per-manifest delivery-keypair table. Every manifest carries its own asymmetric keypair(s),
* stamped with that manifest's id.
diff --git a/core/rust/src/vault_codec/normalize.rs b/core/rust/src/vault_codec/normalize.rs
index fe8cd36f6..956c67548 100644
--- a/core/rust/src/vault_codec/normalize.rs
+++ b/core/rust/src/vault_codec/normalize.rs
@@ -5,13 +5,13 @@ use std::collections::{HashMap, HashSet};
use serde_json::json;
use super::manifest::CodecRecord;
-use super::row::{str_col, truthy};
+use super::row::{is_deleted, str_col, truthy};
use super::types::{is_guid, is_id_column};
use crate::timestamp::updated_at;
-use crate::vault_model::{id_key, MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS};
+use crate::vault_model::{id_key, MANIFEST_ID_COL, MULTI_VALUE_FIELD_KEYS, SINGLE_VALUE_FIELD_KEYS, SYNCABLE_TABLES};
use crate::vault_model::names::{
CHANGED_AT_COL, FIELD_DEFINITIONS_TABLE, FIELD_DEFINITION_ID_COL, FIELD_HISTORIES_TABLE, FIELD_KEY_COL,
- FIELD_VALUES_TABLE, ID_COL, IS_MULTI_VALUE_COL, ITEM_ID_COL, ITEM_TAGS_TABLE, TAG_ID_COL, VALUE_INDEX_COL,
+ FIELD_VALUES_TABLE, ID_COL, IS_MULTI_VALUE_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_TAGS_TABLE, TAG_ID_COL, VALUE_INDEX_COL,
};
/// Domain-separation prefix for derived field value ids.
@@ -66,6 +66,8 @@ pub(crate) fn normalize_row_id_spelling(row: &mut CodecRecord) {
/// Normalize the shape of rows for converting from materialized SQLite to the manifest format to save on filesize.
pub(crate) fn normalize_row_shapes(tables: &mut HashMap<String, Vec<CodecRecord>>) {
+ // Drop rows of deleted items first (any leftover state after a potential delete/update scenario in LWW merge).
+ drop_children_of_deleted_items(tables);
let multi_value_defs = multi_value_definition_ids(tables);
if let Some(rows) = tables.get_mut(FIELD_VALUES_TABLE) {
normalize_field_values(rows, &multi_value_defs);
@@ -78,6 +80,26 @@ pub(crate) fn normalize_row_shapes(tables: &mut HashMap<String, Vec<CodecRecord>
}
}
+/*
+ * Check for deleted items and remove their child rows in case they are still present after a LWW merge (leftover state).
+ */
+fn drop_children_of_deleted_items(tables: &mut HashMap<String, Vec<CodecRecord>>) {
+ let deleted: HashSet<(String, String)> = tables
+ .get(ITEMS_TABLE)
+ .map(|items| items.iter().filter(|item| is_deleted(item)).filter_map(|item| Some((lower_str(item, MANIFEST_ID_COL)?, lower_str(item, ID_COL)?))).collect())
+ .unwrap_or_default();
+ if deleted.is_empty() {
+ return;
+ }
+ for child in SYNCABLE_TABLES.iter().filter(|table| table.item_child) {
+ let Some(rows) = tables.get_mut(child.name) else { continue };
+ rows.retain(|row| match (lower_str(row, MANIFEST_ID_COL), lower_str(row, child.item_ref_column())) {
+ (Some(manifest), Some(item)) => !deleted.contains(&(manifest, item)),
+ _ => true,
+ });
+ }
+}
+
/// The materialize direction: a FieldValues or FieldHistories row without an `Id` is a row whose
/// derived id the wire omits; derive it from the row's own natural key so every device materializes
/// the same SQLite row.
@@ -105,6 +127,27 @@ fn multi_value_definition_ids(tables: &HashMap<String, Vec<CodecRecord>>) -> Has
.collect()
}
+/// How many values a field holds, which decides the shape its rows take on the wire.
+enum FieldShape {
+ MultiValue,
+ SingleValue,
+ /*
+ * A system field this build's registry does not list, so it is unknown to this build.
+ */
+ Unknown,
+}
+
+/// The shape of the field `row` belongs to: by registry for a system field, by its definition for a custom one.
+fn field_shape(row: &CodecRecord, manifest: &str, multi_value_defs: &HashSet<(String, String)>) -> FieldShape {
+ match str_col(row, FIELD_KEY_COL).filter(|key| !key.is_empty()).map(str::to_lowercase) {
+ Some(key) if MULTI_VALUE_FIELD_KEYS.contains(&key.as_str()) => FieldShape::MultiValue,
+ Some(key) if SINGLE_VALUE_FIELD_KEYS.contains(&key.as_str()) => FieldShape::SingleValue,
+ Some(_) => FieldShape::Unknown,
+ None if lower_str(row, FIELD_DEFINITION_ID_COL).is_some_and(|def| multi_value_defs.contains(&(manifest.to_string(), def))) => FieldShape::MultiValue,
+ None => FieldShape::SingleValue,
+ }
+}
+
/// Renumber, collapse and re-id one FieldValues row set, per `(manifest, item, field)` group.
fn normalize_field_values(rows: &mut Vec<CodecRecord>, multi_value_defs: &HashSet<(String, String)>) {
// Group row positions by natural key, preserving read order within each group.
@@ -117,28 +160,47 @@ fn normalize_field_values(rows: &mut Vec<CodecRecord>, multi_value_defs: &HashSe
let mut removed: HashSet<usize> = HashSet::new();
for ((manifest, _, _), mut positions) in groups {
- let multi_value = str_col(&rows[positions[0]], FIELD_KEY_COL).is_some_and(|key| MULTI_VALUE_FIELD_KEYS.contains(&key.to_lowercase().as_str()))
- || lower_str(&rows[positions[0]], FIELD_DEFINITION_ID_COL).is_some_and(|def| multi_value_defs.contains(&(manifest.clone(), def)));
-
- if multi_value {
- // Stable order: declared position first (a row written before the column existed sorts last), read order breaks ties.
- positions.sort_by_key(|p| value_index_of(&rows[*p]).unwrap_or(i64::MAX));
- for (index, position) in positions.iter().enumerate() {
- let row = &mut rows[*position];
- row.insert(VALUE_INDEX_COL.to_string(), json!(index as i64));
- if !has_id(row) {
- let id = derive_row_id(row, index as i64);
- row.insert(ID_COL.to_string(), json!(id));
+ match field_shape(&rows[positions[0]], &manifest, multi_value_defs) {
+ FieldShape::MultiValue => {
+ // Stable order: declared position first (a row written before the column existed sorts last), read order breaks ties.
+ positions.sort_by_key(|p| value_index_of(&rows[*p]).unwrap_or(i64::MAX));
+ for (index, position) in positions.iter().enumerate() {
+ let row = &mut rows[*position];
+ row.insert(VALUE_INDEX_COL.to_string(), json!(index as i64));
+ if !has_id(row) {
+ let id = derive_row_id(row, index as i64);
+ row.insert(ID_COL.to_string(), json!(id));
+ }
+ }
+ }
+ FieldShape::SingleValue => {
+ // A single-value field is one row; duplicates collapse to the newest UpdatedAt instead of
+ // materializing into a primary-key violation.
+ let winner = *positions.iter().max_by_key(|p| (updated_at(&rows[**p]), std::cmp::Reverse(**p))).unwrap();
+ removed.extend(positions.iter().filter(|p| **p != winner));
+ let row = &mut rows[winner];
+ row.insert(VALUE_INDEX_COL.to_string(), json!(0));
+ row.remove(ID_COL);
+ }
+ FieldShape::Unknown => {
+ // Only rows naming one row collapse (one carrying the id the other would derive), since they would materialize into one primary key.
+ let mut newest: HashMap<String, usize> = HashMap::new();
+ for position in positions {
+ let id = materialized_id(&rows[position]);
+ match newest.get(&id).copied() {
+ Some(current) if updated_at(&rows[position]) <= updated_at(&rows[current]) => {
+ removed.insert(position);
+ }
+ Some(current) => {
+ removed.insert(current);
+ newest.insert(id, position);
+ }
+ None => {
+ newest.insert(id, position);
+ }
+ }
}
}
- } else {
- // A single-value field is one row; duplicates collapse to the newest UpdatedAt instead of
- // materializing into a primary-key violation (or a field the UI renders as an array).
- let winner = *positions.iter().max_by_key(|p| (updated_at(&rows[**p]), std::cmp::Reverse(**p))).unwrap();
- removed.extend(positions.iter().filter(|p| **p != winner));
- let row = &mut rows[winner];
- row.insert(VALUE_INDEX_COL.to_string(), json!(0));
- row.remove(ID_COL);
}
}
@@ -209,6 +271,11 @@ fn derive_row_id(row: &CodecRecord, value_index: i64) -> String {
field_value_id_for(manifest, item, field_key, field_def, value_index)
}
+/// The id a row materializes under: the one it carries, or the one [`derive_missing_ids`] would give it.
+fn materialized_id(row: &CodecRecord) -> String {
+ lower_str(row, ID_COL).unwrap_or_else(|| derive_row_id(row, value_index_of(row).unwrap_or(0)))
+}
+
/// The derived id for a normalized FieldHistories row.
fn derive_history_row_id(row: &CodecRecord) -> String {
let [manifest, item, field_key, field_def] = field_key_parts(row);
@@ -279,6 +346,83 @@ mod tests {
assert_ne!(field_value_id_for("m", "i", "login.url", "", 0), field_value_id_for("m", "i", "login.url", "", 1));
}
+ fn field_value(id: Option<&str>, field_key: &str, value: &str, updated_at: &str) -> CodecRecord {
+ let mut row: CodecRecord = [
+ (MANIFEST_ID_COL.to_string(), json!("m-1")),
+ (ITEM_ID_COL.to_string(), json!("item-1")),
+ (FIELD_KEY_COL.to_string(), json!(field_key)),
+ ("Value".to_string(), json!(value)),
+ ("UpdatedAt".to_string(), json!(updated_at)),
+ ]
+ .into_iter()
+ .collect();
+ if let Some(id) = id {
+ row.insert(ID_COL.to_string(), json!(id));
+ }
+ row
+ }
+
+ fn normalized(rows: Vec<CodecRecord>) -> Vec<CodecRecord> {
+ let mut tables: HashMap<String, Vec<CodecRecord>> = [(FIELD_VALUES_TABLE.to_string(), rows)].into_iter().collect();
+ normalize_row_shapes(&mut tables);
+ tables.remove(FIELD_VALUES_TABLE).unwrap()
+ }
+
+ #[test]
+ fn a_field_this_build_does_not_know_keeps_every_value() {
+ // A newer writer's multi-value system field: collapsing it as single-value would destroy all but
+ // one value on this build's next push.
+ let rows = normalized(vec![
+ field_value(Some("a"), "login.future_multi", "one", "2024-01-01T00:00:00Z"),
+ field_value(Some("b"), "login.future_multi", "two", "2024-01-02T00:00:00Z"),
+ ]);
+ assert_eq!(rows.len(), 2);
+ assert!(rows.iter().all(|row| has_id(row)), "and the rows keep the ids they own");
+ }
+
+ #[test]
+ fn an_unknown_field_row_and_its_id_less_twin_are_one_row() {
+ // A newer writer's single-value field arrives id-less; this build materialized it under the derived
+ // id and writes it back with that id. Both name one SQLite row, so they collapse to the newest.
+ let derived = field_value_id_for("m-1", "item-1", "login.future_single", "", 0);
+ let rows = normalized(vec![
+ field_value(None, "login.future_single", "server", "2024-01-01T00:00:00Z"),
+ field_value(Some(&derived), "login.future_single", "local-newer", "2024-01-09T00:00:00Z"),
+ ]);
+ assert_eq!(rows.len(), 1);
+ assert_eq!(rows[0]["Value"], json!("local-newer"));
+ }
+
+ #[test]
+ fn a_known_single_value_field_still_collapses_to_the_newest() {
+ let rows = normalized(vec![
+ field_value(Some("a"), "login.username", "old", "2024-01-01T00:00:00Z"),
+ field_value(Some("b"), "login.username", "new", "2024-01-09T00:00:00Z"),
+ ]);
+ assert_eq!(rows.len(), 1);
+ assert_eq!(rows[0]["Value"], json!("new"));
+ assert!(!has_id(&rows[0]), "a single-value row travels without its derived id");
+ }
+
+ #[test]
+ fn a_tombstoned_item_keeps_no_rows() {
+ let item = |id: &str, deleted: i64| -> CodecRecord {
+ [(MANIFEST_ID_COL.to_string(), json!("m-1")), (ID_COL.to_string(), json!(id)), ("IsDeleted".to_string(), json!(deleted))].into_iter().collect()
+ };
+ let child = |item_id: &str| -> CodecRecord {
+ [(MANIFEST_ID_COL.to_string(), json!("m-1")), (ID_COL.to_string(), json!(format!("totp-{}", item_id))), (ITEM_ID_COL.to_string(), json!(item_id))].into_iter().collect()
+ };
+ let mut tables: HashMap<String, Vec<CodecRecord>> = [
+ (ITEMS_TABLE.to_string(), vec![item("gone", 1), item("kept", 0)]),
+ ("TotpCodes".to_string(), vec![child("gone"), child("kept")]),
+ ]
+ .into_iter()
+ .collect();
+ normalize_row_shapes(&mut tables);
+ let left: Vec<&str> = tables["TotpCodes"].iter().map(|row| row[ITEM_ID_COL].as_str().unwrap()).collect();
+ assert_eq!(left, vec!["kept"], "a pruned or permanently deleted item's secrets do not travel on");
+ }
+
#[test]
fn history_derivation_is_stable_and_disjoint_from_field_values() {
let id = field_history_id_for("m-1", "item-1", "login.password", "", "2026-01-01 10:00:00.000");
diff --git a/core/rust/src/vault_merge/item_deletes.rs b/core/rust/src/vault_merge/item_deletes.rs
new file mode 100644
index 000000000..c5c0a7548
--- /dev/null
+++ b/core/rust/src/vault_merge/item_deletes.rs
@@ -0,0 +1,78 @@
+//! Item deletes in the canonical merge, decided once per item instead of row by row.
+//!
+//! A permanent delete tombstones the item and takes the rows hanging off it along, so the item and
+//! its children are one unit. Row-level LWW breaks that unit both ways: children the deleting side
+//! removed come back from the other side under a tombstoned item, and child tombstones stamped at
+//! delete time wipe the untouched fields of an item whose delete lost. So a delete only one side made
+//! is settled here first, against everything the other side did to that item (a field edit does not
+//! touch the item row), and the losing side's rows of the item leave the merge. A delete that stands
+//! needs nothing more: the merged output drops a tombstoned item's children when it is normalized.
+
+use std::collections::{HashMap, HashSet};
+
+use chrono::{DateTime, Utc};
+
+use super::get_key;
+use crate::timestamp::updated_at;
+use crate::vault_codec::row::is_deleted;
+use crate::vault_codec::{is_bucketed_table, CodecRecord};
+use crate::vault_model::names::{ID_COL, ITEMS_TABLE};
+use crate::vault_model::{MANIFEST_ID_COL, SYNCABLE_TABLES};
+
+type Tables = HashMap<String, Vec<CodecRecord>>;
+
+/// Settle every delete only one side made: a delete the other side outlived is taken out of the deleting
+/// side's input together with that side's rows of the item, so the surviving side's rows pass through whole.
+pub(super) fn resolve_item_deletes(base: &mut Tables, incoming: &mut Tables) {
+ // The base keeps ties, exactly as it does for a single row.
+ let lost_in_base = lost_deletes(base, incoming, |delete, activity| activity > delete);
+ let lost_in_incoming = lost_deletes(incoming, base, |delete, activity| activity >= delete);
+ remove_items(base, &lost_in_base);
+ remove_items(incoming, &lost_in_incoming);
+}
+
+/// The items `deleting` tombstoned that `other` still holds live and touched late enough for `outlives` to hold.
+fn lost_deletes(deleting: &Tables, other: &Tables, outlives: impl Fn(Option<DateTime<Utc>>, Option<DateTime<Utc>>) -> bool) -> HashSet<String> {
+ let items = |tables: &Tables, deleted: bool| -> HashMap<String, Option<DateTime<Utc>>> {
+ let rows = tables.get(ITEMS_TABLE).map(Vec::as_slice).unwrap_or(&[]);
+ rows.iter().filter(|row| is_deleted(row) == deleted).map(|row| (item_key(row, ID_COL), updated_at(row))).collect()
+ };
+ let tombstones = items(deleting, true);
+ let mut live = items(other, false);
+ live.retain(|key, _| tombstones.contains_key(key));
+ if live.is_empty() {
+ return HashSet::new();
+ }
+
+ // What the surviving side last did to the item: its own row, or any row hanging off it. Bucketed
+ // tables stay out, since a usage counter ticking is not an edit.
+ for child in SYNCABLE_TABLES.iter().filter(|table| table.item_child && !is_bucketed_table(table.name)) {
+ for row in other.get(child.name).map(Vec::as_slice).unwrap_or(&[]) {
+ if let Some(activity) = live.get_mut(&item_key(row, child.item_ref_column())) {
+ *activity = (*activity).max(updated_at(row));
+ }
+ }
+ }
+
+ live.into_iter().filter(|(key, activity)| outlives(tombstones[key], *activity)).map(|(key, _)| key).collect()
+}
+
+/// Remove the item rows named by `keys` and every row hanging off them.
+fn remove_items(tables: &mut Tables, keys: &HashSet<String>) {
+ if keys.is_empty() {
+ return;
+ }
+ if let Some(rows) = tables.get_mut(ITEMS_TABLE) {
+ rows.retain(|row| !keys.contains(&item_key(row, ID_COL)));
+ }
+ for child in SYNCABLE_TABLES.iter().filter(|table| table.item_child) {
+ if let Some(rows) = tables.get_mut(child.name) {
+ rows.retain(|row| !keys.contains(&item_key(row, child.item_ref_column())));
+ }
+ }
+}
+
+/// The `(ManifestId, item id)` key of an item row, or of a child row through `item_column`.
+fn item_key(row: &CodecRecord, item_column: &str) -> String {
+ get_key(row, &[MANIFEST_ID_COL, item_column])
+}
diff --git a/core/rust/src/vault_merge/mod.rs b/core/rust/src/vault_merge/mod.rs
index 4accd1160..f08bf461f 100644
--- a/core/rust/src/vault_merge/mod.rs
+++ b/core/rust/src/vault_merge/mod.rs
@@ -17,6 +17,7 @@ use crate::timestamp::updated_at;
use crate::vault_model::{id_key, TableConfig, SYNCABLE_TABLES};
use crate::vault_codec::{bucket_categories, identity_part, is_bucketed_table, tables_for_category, CodecRecord, DataBucket, Manifest};
+mod item_deletes;
#[cfg(test)]
mod tests;
@@ -140,6 +141,9 @@ fn merge_manifest_pair(
let mut incoming_tables = local.map(|m| m.tables).unwrap_or_default();
incoming_tables.extend(local_bucket_tables);
+ // An item is deleted as a unit including any child rows.
+ item_deletes::resolve_item_deletes(&mut base_tables, &mut incoming_tables);
+
let table_names: BTreeSet<String> = base_tables.keys().chain(incoming_tables.keys()).cloned().collect();
let mut merged: HashMap<String, Vec<CodecRecord>> = HashMap::new();
@@ -165,8 +169,9 @@ fn merge_manifest_pair(
}
}
- // A union of concurrently added multi-value rows can leave two rows at the same ValueIndex;
- // re-normalizing the output renumbers them, so a merged manifest is normalized like any other.
+ // A union of concurrently added multi-value rows can leave two rows at the same ValueIndex, and a
+ // delete that won can leave the other side's children under a tombstoned item; re-normalizing the
+ // output renumbers the first and drops the second, so a merged manifest is normalized like any other.
crate::vault_codec::normalize::normalize_row_shapes(&mut merged);
let mut buckets = unknown_server_buckets;
diff --git a/core/rust/src/vault_merge/tests.rs b/core/rust/src/vault_merge/tests.rs
index f4bb3d567..31f17adb7 100644
--- a/core/rust/src/vault_merge/tests.rs
+++ b/core/rust/src/vault_merge/tests.rs
@@ -36,10 +36,10 @@ fn items_manifest(manifest_id: &str, rows: Vec<CodecRecord>) -> Manifest {
fn schema() -> HashMap<String, Vec<String>> {
[
- ("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Name".to_string(), "UpdatedAt".to_string()]),
+ ("Items".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Name".to_string(), "IsDeleted".to_string(), "UpdatedAt".to_string()]),
("Settings".to_string(), vec!["ManifestId".to_string(), "Key".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),
("ItemStats".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "LastUsedAt".to_string(), "UpdatedAt".to_string()]),
- ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string(), "UpdatedAt".to_string()]),
+ ("FieldValues".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "ItemId".to_string(), "FieldKey".to_string(), "FieldDefinitionId".to_string(), "ValueIndex".to_string(), "Value".to_string(), "IsDeleted".to_string(), "UpdatedAt".to_string()]),
("ItemTags".to_string(), vec!["ManifestId".to_string(), "ItemId".to_string(), "TagId".to_string(), "UpdatedAt".to_string()]),
("Attachments".to_string(), vec!["ManifestId".to_string(), "Id".to_string(), "Blob".to_string(), "UpdatedAt".to_string()]),
]
@@ -582,3 +582,140 @@ fn keys_compare_guids_case_insensitively_and_survive_a_missing_column() {
assert_ne!(get_key(&row(MANIFEST, ROW), &columns), get_key(&row(OTHER_MANIFEST, ROW), &columns));
assert_eq!(get_key(&HashMap::from([("Id".to_string(), json!(ROW))]), &columns), format!(":{}", ROW));
}
+
+/// An `Items` row, live or tombstoned.
+fn item_row(manifest_id: &str, id: &str, deleted: bool, updated_at: &str) -> CodecRecord {
+ let mut row = item(manifest_id, id, "item", updated_at);
+ row.insert("IsDeleted".to_string(), json!(if deleted { 1 } else { 0 }));
+ row
+}
+
+/// A single-value `FieldValues` row of `item-1` in wire shape, live or tombstoned.
+fn field_of_item(manifest_id: &str, field_key: &str, value: &str, deleted: bool, updated_at: &str) -> CodecRecord {
+ [
+ ("ManifestId".to_string(), json!(manifest_id)),
+ ("ItemId".to_string(), json!("item-1")),
+ ("FieldKey".to_string(), json!(field_key)),
+ ("ValueIndex".to_string(), json!(0)),
+ ("Value".to_string(), json!(value)),
+ ("IsDeleted".to_string(), json!(if deleted { 1 } else { 0 })),
+ ("UpdatedAt".to_string(), json!(updated_at)),
+ ]
+ .into_iter()
+ .collect()
+}
+
+fn item_manifest(manifest_id: &str, items: Vec<CodecRecord>, field_values: Vec<CodecRecord>) -> Manifest {
+ manifest(manifest_id, [("Items".to_string(), items), ("FieldValues".to_string(), field_values)].into_iter().collect())
+}
+
+fn merge_manifests(server: Vec<Manifest>, local: Vec<Manifest>) -> Vec<CanonicalManifestMerge> {
+ merge_canonical(CanonicalMergeInput { server_manifests: server, server_buckets: vec![], contentless_server_manifest_ids: vec![], local_manifests: local, local_buckets: vec![], schema_columns: schema() }).unwrap().manifests
+}
+
+fn live_values(merged: &CanonicalManifestMerge) -> Vec<String> {
+ let mut values: Vec<String> = merged.manifest.tables["FieldValues"].iter().filter(|r| r["IsDeleted"] == json!(0)).map(|r| r["Value"].as_str().unwrap().to_string()).collect();
+ values.sort();
+ values
+}
+
+const T_OLD: &str = "2024-01-01T00:00:00Z";
+const T_DELETE: &str = "2024-01-05T05:00:00Z";
+const T_EDIT: &str = "2024-01-05T05:05:00Z";
+
+#[test]
+fn a_permanent_delete_does_not_get_its_children_back_from_the_other_side() {
+ // The deleting side removed the item's rows; the server still holds them and must not resurrect them.
+ let server = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "secret", false, T_OLD)]);
+ let local = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_DELETE)], vec![]);
+ for (base, incoming) in [(server.clone(), local.clone()), (local, server)] {
+ let merged = &merge_manifests(vec![base], vec![incoming])[0];
+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1), "the delete stands");
+ assert!(merged.manifest.tables["FieldValues"].is_empty(), "and the deleted item keeps no rows");
+ }
+}
+
+#[test]
+fn an_item_edited_after_the_other_side_deleted_it_survives_whole() {
+ // One side deleted the item at 05:00, tombstoning its rows as the pruner does. The other edited one
+ // field at 05:05, which never touches the item row. The edit is the last write, so the item survives,
+ // and with every field: the delete's child tombstones must not wipe the fields the edit left alone.
+ let deleting = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_DELETE)], vec![
+ field_of_item(PERSONAL, "login.username", "user", true, T_DELETE),
+ field_of_item(PERSONAL, "login.password", "old-secret", true, T_DELETE),
+ ]);
+ let editing = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![
+ field_of_item(PERSONAL, "login.username", "user", false, T_OLD),
+ field_of_item(PERSONAL, "login.password", "new-secret", false, T_EDIT),
+ ]);
+ for (base, incoming) in [(deleting.clone(), editing.clone()), (editing, deleting)] {
+ let merged = &merge_manifests(vec![base], vec![incoming])[0];
+ assert_eq!(merged.manifest.tables["Items"].len(), 1);
+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(0), "the later edit outlives the delete");
+ assert_eq!(live_values(merged), vec!["new-secret", "user"], "with the untouched field intact");
+ }
+}
+
+#[test]
+fn a_delete_newer_than_everything_the_other_side_did_stands() {
+ let editing = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "edited", false, T_DELETE)]);
+ let deleting = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_EDIT)], vec![]);
+ for (base, incoming) in [(deleting.clone(), editing.clone()), (editing, deleting)] {
+ let merged = &merge_manifests(vec![base], vec![incoming])[0];
+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1));
+ assert!(merged.manifest.tables["FieldValues"].is_empty());
+ }
+}
+
+#[test]
+fn a_delete_and_an_edit_at_the_same_instant_keep_the_server_side() {
+ let deleting = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_EDIT)], vec![]);
+ let editing = item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "edited", false, T_EDIT)]);
+
+ let merged = &merge_manifests(vec![deleting.clone()], vec![editing.clone()])[0];
+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1), "the server's delete keeps the tie");
+
+ let merged = &merge_manifests(vec![editing], vec![deleting])[0];
+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(0), "the server's edit keeps the tie");
+ assert_eq!(live_values(merged), vec!["edited"]);
+}
+
+#[test]
+fn a_usage_counter_does_not_outlive_a_delete() {
+ // ItemStats ticks on every autofill; that is not an edit and must not undo a delete.
+ let stats = |updated_at: &str| -> CodecRecord {
+ [("ManifestId".to_string(), json!(PERSONAL)), ("Id".to_string(), json!("item-1")), ("LastUsedAt".to_string(), json!(updated_at)), ("UpdatedAt".to_string(), json!(updated_at))].into_iter().collect()
+ };
+ let output = merge_canonical(CanonicalMergeInput {
+ server_manifests: vec![item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_DELETE)], vec![])],
+ server_buckets: vec![],
+ contentless_server_manifest_ids: vec![],
+ local_manifests: vec![item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![])],
+ local_buckets: vec![DataBucket::new(PERSONAL, "Stats", [("ItemStats".to_string(), vec![stats(T_EDIT)])].into_iter().collect())],
+ schema_columns: schema(),
+ })
+ .unwrap();
+ let merged = &output.manifests[0];
+ assert_eq!(merged.manifest.tables["Items"][0]["IsDeleted"], json!(1));
+ assert!(merged.buckets.iter().all(|bucket| bucket.tables.get("ItemStats").is_none_or(Vec::is_empty)), "the deleted item's stats row goes with it");
+}
+
+#[test]
+fn an_item_moved_to_another_manifest_does_not_stay_behind_in_the_one_it_left() {
+ // The move leaves a tombstone in the source manifest, so the server's copy there loses instead of
+ // surviving the union next to the moved one.
+ let server = vec![
+ item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", false, T_OLD)], vec![field_of_item(PERSONAL, "login.password", "secret", false, T_OLD)]),
+ item_manifest(SHARED, vec![], vec![]),
+ ];
+ let local = vec![
+ item_manifest(PERSONAL, vec![item_row(PERSONAL, "item-1", true, T_EDIT)], vec![]),
+ item_manifest(SHARED, vec![item_row(SHARED, "item-1", false, T_EDIT)], vec![field_of_item(SHARED, "login.password", "secret", false, T_OLD)]),
+ ];
+ let merged = merge_manifests(server, local);
+ let by_id: HashMap<&str, &CanonicalManifestMerge> = merged.iter().map(|m| (m.manifest_id.as_str(), m)).collect();
+ assert_eq!(by_id[PERSONAL].manifest.tables["Items"][0]["IsDeleted"], json!(1), "the source manifest keeps only the tombstone");
+ assert!(by_id[PERSONAL].manifest.tables["FieldValues"].is_empty());
+ assert_eq!(by_id[SHARED].manifest.tables["Items"][0]["IsDeleted"], json!(0));
+ assert_eq!(live_values(by_id[SHARED]), vec!["secret"], "and the moved item arrives whole");
+}
diff --git a/core/rust/src/vault_model/generated.rs b/core/rust/src/vault_model/generated.rs
index aa7e52225..9abffe5df 100644
--- a/core/rust/src/vault_model/generated.rs
+++ b/core/rust/src/vault_model/generated.rs
@@ -86,6 +86,10 @@ pub static PERSONAL_TABLES: &[&str] = &[];
/// adding two different things) and is not derived.
pub static MULTI_VALUE_FIELD_KEYS: &[&str] = &["login.url"];
+/// System field keys whose field holds one value. A key in neither list is a field this build does
+/// not know (a newer writer's), whose rows the codec leaves as it found them.
+pub static SINGLE_VALUE_FIELD_KEYS: &[&str] = &["login.email", "login.username", "login.password", "alias.first_name", "alias.last_name", "alias.gender", "alias.birthdate", "card.cardholder_name", "card.number", "card.expiry_month", "card.expiry_year", "card.cvv", "card.pin", "notes.content"];
+
/// The per-manifest delivery-keypair table. Every manifest carries its own asymmetric keypair(s),
/// stamped with that manifest's id (`ManifestId`).
pub const ENCRYPTION_KEYS_TABLE: &str = "EncryptionKeys";
diff --git a/core/rust/src/vault_model/mod.rs b/core/rust/src/vault_model/mod.rs
index 95f54699b..ff81ce41f 100644
--- a/core/rust/src/vault_model/mod.rs
+++ b/core/rust/src/vault_model/mod.rs
@@ -66,6 +66,11 @@ impl TableConfig {
self
}
+ /// The column of an item-child table that carries the owning item's id: `Id` for ItemStats (its `Id` is the item's id), `ItemId` everywhere else.
+ pub fn item_ref_column(&self) -> &'static str {
+ if self.name == names::ITEM_STATS_TABLE { names::ID_COL } else { names::ITEM_ID_COL }
+ }
+
/// The columns that together identify one row of this table: `(ManifestId, primary key columns)`
/// for a manifest-scoped table, the primary key columns alone otherwise.
pub fn identity_columns(&self) -> Vec<&'static str> {
diff --git a/core/rust/src/vault_pruner/mod.rs b/core/rust/src/vault_pruner/mod.rs
index b7378c769..e02bea9d6 100644
--- a/core/rust/src/vault_pruner/mod.rs
+++ b/core/rust/src/vault_pruner/mod.rs
@@ -14,7 +14,7 @@ use crate::sqlite_host::SqlStatement;
use crate::vault_codec::row::{has_bytes, is_deleted, logo_kind, str_col};
use crate::vault_codec::{CodecRecord, CodecTableData};
use crate::vault_model::names::{
- DELETED_AT_COL, FILE_DATA_COL, ID_COL, IS_DELETED_COL, ITEMS_TABLE, ITEM_ID_COL, ITEM_STATS_TABLE, KIND_COL, LOGOS_TABLE,
+ DELETED_AT_COL, FILE_DATA_COL, ID_COL, IS_DELETED_COL, ITEMS_TABLE, ITEM_ID_COL, KIND_COL, LOGOS_TABLE,
LOGO_ID_COL, LOGO_KIND_FAVICON, UPDATED_AT_COL,
};
use crate::vault_model::{BLOB_COLUMNS, SYNCABLE_TABLES, TRASH_RETENTION_DEFAULT_DAYS};
@@ -92,7 +92,7 @@ pub fn get_prune_table_queries() -> Vec<PruneTableQuery> {
for child in item_child_tables() {
let query = match blob_column_for(child.name) {
Some(blob_col) => format!("SELECT {}, {}, {}, substr({}, 1, 1) AS {} FROM {}", ID_COL, ITEM_ID_COL, IS_DELETED_COL, blob_col, blob_col, child.name),
- None => format!("SELECT {}, {} FROM {}", item_ref_column(child.name), IS_DELETED_COL, child.name),
+ None => format!("SELECT {}, {} FROM {}", child.item_ref_column(), IS_DELETED_COL, child.name),
};
queries.push(PruneTableQuery { name: child.name.to_string(), query });
}
@@ -108,12 +108,6 @@ fn item_child_tables() -> impl Iterator<Item = &'static crate::vault_model::Tabl
SYNCABLE_TABLES.iter().filter(|t| t.item_child)
}
-/// The column of an item-child table that carries the owning item's id: `Id` for ItemStats
-/// (its `Id` *is* the item's id), `ItemId` everywhere else.
-fn item_ref_column(table_name: &str) -> &'static str {
- if table_name == ITEM_STATS_TABLE { ID_COL } else { ITEM_ID_COL }
-}
-
/// The extracted blob column of a table, if it has one (see `vault_model::BLOB_COLUMNS`).
fn blob_column_for(table_name: &str) -> Option<&'static str> {
BLOB_COLUMNS.iter().find(|(t, _, _)| *t == table_name).map(|(_, col, _)| *col)
@@ -188,7 +182,7 @@ fn expired_item_ids(items: &[CodecRecord], cutoff_date: DateTime<Utc>) -> Vec<St
fn tombstone_item_children(tables: &[CodecTableData], item_id: &str, now_str: &str, statements: &mut Vec<SqlStatement>, stats: &mut PruneStats) {
for child in item_child_tables() {
let Some(records) = records_of(tables, child.name) else { continue };
- let match_col = item_ref_column(child.name);
+ let match_col = child.item_ref_column();
let related_count = count_related_records(records, match_col, item_id);
if related_count == 0 {
continue;
diff --git a/core/rust/src/vault_sync/db.rs b/core/rust/src/vault_sync/db.rs
index 018263b76..551f1140c 100644
--- a/core/rust/src/vault_sync/db.rs
+++ b/core/rust/src/vault_sync/db.rs
@@ -232,7 +232,7 @@ const DEMOTE_KEYS_FOR_MANIFEST: &str = "UPDATE EncryptionKeys SET IsPrimary = 0,
const INSERT_KEY_FOR_MANIFEST: &str = "INSERT INTO EncryptionKeys (Id, ManifestId, PublicKey, PrivateKey, IsPrimary, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, 1, ?, ?, 0)";
const INSERT_FOLDER: &str = "INSERT INTO Folders (Id, Name, ParentFolderId, ManifestId, Weight, IsDeleted, CreatedAt, UpdatedAt) VALUES (?, ?, ?, COALESCE((SELECT ManifestId FROM Folders WHERE Id = ?), ?), 0, 0, ?, ?)";
const RESTAMP_SUBTREE_FOLDERS: &str = "UPDATE Folders SET ManifestId = ? WHERE Id IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";
-const RESTAMP_SUBTREE_ITEMS: &str = "UPDATE Items SET ManifestId = ? WHERE FolderId IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";
+const RESTAMP_SUBTREE_ITEMS: &str = "UPDATE Items SET ManifestId = ?, UpdatedAt = ? WHERE ManifestId <> ? AND FolderId IN (WITH RECURSIVE subtree(Id) AS (SELECT Id FROM Folders WHERE Id = ? UNION ALL SELECT f.Id FROM Folders f INNER JOIN subtree s ON f.ParentFolderId = s.Id) SELECT Id FROM subtree)";
const FIND_ITEMS_WITH_FOREIGN_LOGO: &str = "SELECT i.Id, i.ManifestId, origin.Kind, origin.Source FROM Items i INNER JOIN Logos origin ON origin.Id = i.LogoId LEFT JOIN Logos own ON own.Id = i.LogoId AND own.ManifestId = i.ManifestId WHERE i.LogoId IS NOT NULL AND own.Id IS NULL AND i.IsDeleted = 0";
const GET_LOGO_ID_FOR_KEY: &str = "SELECT Id FROM Logos WHERE ManifestId = ? AND Kind = ? AND Source = ? AND IsDeleted = 0 LIMIT 1";
const GET_BEST_LOGO_FOR_KEY: &str = "SELECT FileData, MimeType, Name FROM Logos WHERE Kind = ? AND Source = ? AND IsDeleted = 0 ORDER BY (FileData IS NOT NULL AND LENGTH(FileData) > 0) DESC, UpdatedAt DESC LIMIT 1";
@@ -294,7 +294,7 @@ pub(crate) async fn render_manifest_folder(host: &Host, manifest_id: &str, name:
vec![
SqlStatement { sql: INSERT_FOLDER.to_string(), params: vec![json!(folder_id), json!(name), Value::Null, Value::Null, json!(active_manifest_id), json!(now), json!(now)] },
SqlStatement { sql: RESTAMP_SUBTREE_FOLDERS.to_string(), params: vec![json!(manifest_id), json!(folder_id)] },
- SqlStatement { sql: RESTAMP_SUBTREE_ITEMS.to_string(), params: vec![json!(manifest_id), json!(folder_id)] },
+ SqlStatement { sql: RESTAMP_SUBTREE_ITEMS.to_string(), params: vec![json!(manifest_id), json!(now), json!(manifest_id), json!(folder_id)] },
],
)
.await?;
diff --git a/core/rust/src/vault_sync/tests/item_move.rs b/core/rust/src/vault_sync/tests/item_move.rs
new file mode 100644
index 000000000..168e2941c
--- /dev/null
+++ b/core/rust/src/vault_sync/tests/item_move.rs
@@ -0,0 +1,67 @@
+//! The schema triggers that carry an item across manifests, run against the real client schema.
+
+use rusqlite::Connection;
+
+use super::test_host::{complete_schema_sql, open_schema_db};
+
+const PERSONAL: &str = "11111111-1111-4111-8111-111111111111";
+const SHARED: &str = "22222222-2222-4222-8222-222222222222";
+const ITEM: &str = "33333333-3333-4333-8333-333333333333";
+const T_CREATED: &str = "2024-01-01 00:00:00.000";
+
+/// A vault holding one personal item with a password and a TOTP code.
+fn vault_with_item() -> Connection {
+ let db = open_schema_db(&complete_schema_sql());
+ db.execute("INSERT INTO Items (ManifestId, Id, Name, ItemType, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, ?2, 'Bank', 'Login', ?3, ?3, 0)", (PERSONAL, ITEM, T_CREATED)).unwrap();
+ db.execute("INSERT INTO FieldValues (ManifestId, Id, ItemId, FieldKey, Value, Weight, ValueIndex, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, 'fv-1', ?2, 'login.password', 'secret', 0, 0, ?3, ?3, 0)", (PERSONAL, ITEM, T_CREATED)).unwrap();
+ db.execute("INSERT INTO TotpCodes (ManifestId, Id, ItemId, Name, SecretKey, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, 'totp-1', ?2, 'code', 'ABC', ?3, ?3, 0)", (PERSONAL, ITEM, T_CREATED)).unwrap();
+ db
+}
+
+fn move_item(db: &Connection, from: &str, to: &str, at: &str) {
+ assert_eq!(db.execute("UPDATE Items SET ManifestId = ?1, UpdatedAt = ?2 WHERE Id = ?3 AND ManifestId = ?4", (to, at, ITEM, from)).unwrap(), 1);
+}
+
+/// `(IsDeleted, UpdatedAt)` of the item row in `manifest`, when it has one.
+fn item_in(db: &Connection, manifest: &str) -> Option<(i64, String)> {
+ db.query_row("SELECT IsDeleted, UpdatedAt FROM Items WHERE ManifestId = ?1 AND Id = ?2", (manifest, ITEM), |row| Ok((row.get(0)?, row.get(1)?))).ok()
+}
+
+fn child_rows_in(db: &Connection, manifest: &str) -> i64 {
+ let count = |table: &str| -> i64 { db.query_row(&format!("SELECT COUNT(*) FROM {} WHERE ManifestId = ?1 AND ItemId = ?2", table), (manifest, ITEM), |row| row.get(0)).unwrap() };
+ count("FieldValues") + count("TotpCodes")
+}
+
+#[test]
+fn moving_an_item_leaves_a_tombstone_in_the_manifest_it_left() {
+ // Without it the merge, a union per manifest, keeps the server's copy in the source manifest and the item ends up in both.
+ let db = vault_with_item();
+ move_item(&db, PERSONAL, SHARED, "2024-02-01 00:00:00.000");
+
+ assert_eq!(item_in(&db, SHARED), Some((0, "2024-02-01 00:00:00.000".to_string())));
+ assert_eq!(child_rows_in(&db, SHARED), 2, "the item's rows follow it");
+ assert_eq!(item_in(&db, PERSONAL), Some((1, "2024-02-01 00:00:00.000".to_string())), "a tombstone stamped with the move stays behind");
+ assert_eq!(child_rows_in(&db, PERSONAL), 0);
+}
+
+#[test]
+fn an_item_can_move_back_into_the_manifest_it_left() {
+ // The tombstone holds the primary key the item returns to, so the move back clears it first.
+ let db = vault_with_item();
+ move_item(&db, PERSONAL, SHARED, "2024-02-01 00:00:00.000");
+ move_item(&db, SHARED, PERSONAL, "2024-03-01 00:00:00.000");
+
+ assert_eq!(item_in(&db, PERSONAL), Some((0, "2024-03-01 00:00:00.000".to_string())));
+ assert_eq!(child_rows_in(&db, PERSONAL), 2);
+ assert_eq!(item_in(&db, SHARED), Some((1, "2024-03-01 00:00:00.000".to_string())), "and the shared manifest now holds the tombstone");
+}
+
+#[test]
+fn a_live_item_with_the_same_id_in_the_destination_is_never_cleared() {
+ // Two manifests holding the same item id is a supported state; only a tombstone gives way to a move.
+ let db = vault_with_item();
+ db.execute("INSERT INTO Items (ManifestId, Id, Name, ItemType, CreatedAt, UpdatedAt, IsDeleted) VALUES (?1, ?2, 'Theirs', 'Login', ?3, ?3, 0)", (SHARED, ITEM, T_CREATED)).unwrap();
+ let moved = db.execute("UPDATE Items SET ManifestId = ?1 WHERE Id = ?2 AND ManifestId = ?3", (SHARED, ITEM, PERSONAL));
+ assert!(moved.is_err(), "the move is refused rather than overwriting the other item");
+ assert_eq!(item_in(&db, SHARED).map(|(deleted, _)| deleted), Some(0));
+}
diff --git a/core/rust/src/vault_sync/tests/mod.rs b/core/rust/src/vault_sync/tests/mod.rs
index 100e7ed10..ce3ec5d01 100644
--- a/core/rust/src/vault_sync/tests/mod.rs
+++ b/core/rust/src/vault_sync/tests/mod.rs
@@ -1,6 +1,6 @@
//! Engine tests: the whole sync driven through the command loop against a real SQLite host.
-mod sharing;
+mod item_move;
mod test_host;
use std::collections::HashMap;
diff --git a/core/vault/src/sql/SqlConstants.ts b/core/vault/src/sql/SqlConstants.ts
index f7ceb050c..7e4543cb3 100644
--- a/core/vault/src/sql/SqlConstants.ts
+++ b/core/vault/src/sql/SqlConstants.ts
@@ -1778,6 +1778,21 @@ INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion")
VALUES ('20260913090000_2.1.0-ManifestScopedStorage', '10.0.10');
BEGIN TRANSACTION;
+CREATE TRIGGER IF NOT EXISTS "TR_Items_ClearTombstoneBeforeReturn"
+BEFORE UPDATE OF "ManifestId" ON "Items"
+FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"
+ AND EXISTS (SELECT 1 FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1)
+BEGIN
+ DELETE FROM "FieldValues" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "FieldHistories" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "ItemTags" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "Attachments" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "Passkeys" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "TotpCodes" WHERE "ItemId" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "ItemStats" WHERE "Id" = NEW."Id" AND "ManifestId" = NEW."ManifestId";
+ DELETE FROM "Items" WHERE "ManifestId" = NEW."ManifestId" AND "Id" = NEW."Id" AND "IsDeleted" = 1;
+END;
+
CREATE TRIGGER IF NOT EXISTS "TR_Items_ResyncChildManifestIds"
AFTER UPDATE OF "ManifestId" ON "Items"
FOR EACH ROW WHEN OLD."ManifestId" <> NEW."ManifestId"
@@ -1789,6 +1804,8 @@ BEGIN
UPDATE "Passkeys" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";
UPDATE "TotpCodes" SET "ManifestId" = NEW."ManifestId" WHERE "ItemId" = NEW."Id" AND "ManifestId" = OLD."ManifestId";
UPDATE "ItemStats" SET "ManifestId" = NEW."ManifestId" WHERE "Id" = NEW."Id" AND "ManifestId" = OLD."ManifestId";
+ INSERT OR IGNORE INTO "Items" ("ManifestId", "Id", "ItemType", "CreatedAt", "UpdatedAt", "IsDeleted")
+ SELECT OLD."ManifestId", OLD."Id", OLD."ItemType", OLD."CreatedAt", NEW."UpdatedAt", 1 WHERE OLD."IsDeleted" = 0;
END;
COMMIT;