Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Add import export repository (#2413)

Leendert de Borst8df40d7460ba
diff --git a/core/client/src/database/SqliteClient.ts b/core/client/src/database/SqliteClient.ts
index e6a992fe3..6a1695700 100644
--- a/core/client/src/database/SqliteClient.ts
+++ b/core/client/src/database/SqliteClient.ts
@@ -19,6 +19,7 @@ import {
SettingsRepository,
EncryptionKeyRepository,
LogoRepository,
+ ImportExportRepository
} from './index';
import type { ISyncDatabaseClient, SqliteBindValue } from './BaseRepository';
@@ -60,6 +61,7 @@ export class SqliteClient implements ISyncDatabaseClient {
private _settings: SyncRepository<SettingsRepository> | null = null;
private _encryptionKeys: SyncRepository<EncryptionKeyRepository> | null = null;
private _logos: SyncRepository<LogoRepository> | null = null;
+ private _importExport: SyncRepository<ImportExportRepository> | null = null;
private _logoRepository: LogoRepository | null = null;
/**
@@ -141,7 +143,17 @@ export class SqliteClient implements ISyncDatabaseClient {
}
/**
- * The logo repository itself, which the item repository calls into.
+ * Repository for the import/export logic.
+ */
+ public get importExport(): SyncRepository<ImportExportRepository> {
+ if (!this._importExport) {
+ this._importExport = syncRepository(new ImportExportRepository(this, this.logoRepository), this);
+ }
+ return this._importExport;
+ }
+
+ /**
+ * The logo repository itself, which the item and import/export repositories call into.
*/
private get logoRepository(): LogoRepository {
if (!this._logoRepository) {
@@ -185,6 +197,7 @@ export class SqliteClient implements ISyncDatabaseClient {
this._folders = null;
this._settings = null;
this._logos = null;
+ this._importExport = null;
} catch (error) {
logDefect('[Sqlite] Initializing the database failed', error);
throw error;
diff --git a/core/client/src/database/index.ts b/core/client/src/database/index.ts
index 0e5d5eb23..14c775526 100644
--- a/core/client/src/database/index.ts
+++ b/core/client/src/database/index.ts
@@ -23,3 +23,4 @@ export { SettingsRepository } from './repositories/SettingsRepository';
export { EncryptionKeyRepository } from './repositories/EncryptionKeyRepository';
export { LogoRepository } from './repositories/LogoRepository';
export { ItemStatsRepository, type ItemUsageAction } from './repositories/ItemStatsRepository';
+export { ImportExportRepository, type VaultExportData } from './repositories/ImportExportRepository';
diff --git a/core/client/src/database/repositories/ImportExportRepository.ts b/core/client/src/database/repositories/ImportExportRepository.ts
new file mode 100644
index 000000000..bb3599e74
--- /dev/null
+++ b/core/client/src/database/repositories/ImportExportRepository.ts
@@ -0,0 +1,386 @@
+import { LogoKinds, type ItemType } from '@aliasvault/models/vault';
+
+import { fromStandardFormat, toStandardFormat } from '../../utilities/DateFormatter';
+import { BaseRepository } from '../BaseRepository';
+import { scopedKey } from '../ItemRef';
+import { AttachmentQueries, FieldDefinitionQueries, FieldValueQueries, ItemQueries, TotpCodeQueries } from '../queries/ItemQueries';
+import { PasskeyQueries } from '../queries/PasskeyQueries';
+
+import type { AttachmentEntity, FieldDefinitionEntity, FieldValueEntity, FolderEntity, ItemEntity, ItemTagEntity, LogoEntity, PasskeyEntity, TagEntity, TotpCodeEntity } from '../../transfer/shared/VaultEntities';
+import type { IDatabaseClient } from '../BaseRepository';
+import type { DbOp } from '../DbOp';
+import type { LogoRepository } from './LogoRepository';
+
+/**
+ * Everything a full vault export carries.
+ */
+export type VaultExportData = {
+ items: ItemEntity[];
+ folders: FolderEntity[];
+ tags: TagEntity[];
+ itemTags: ItemTagEntity[];
+ fieldDefinitions: FieldDefinitionEntity[];
+ logos: LogoEntity[];
+};
+
+/** A row of a manifest-scoped child table, with the key of the item it hangs off. */
+type ChildRow = { ItemId: string; ManifestId: string };
+
+/** The tables a vault reset empties, dependents first. */
+const RESET_TABLES = ['Attachments', 'FieldValues', 'FieldHistories', 'TotpCodes', 'Passkeys', 'ItemTags', 'FieldDefinitions', 'Tags', 'Items', 'Logos', 'Folders'];
+
+/**
+ * Repository for the import/export feature: reads whole item graphs for an export and writes imported item graphs
+ * with their original timestamps, which the regular item repository does not allow.
+ */
+export class ImportExportRepository extends BaseRepository {
+ /**
+ * Constructor for the ImportExportRepository class.
+ * @param client - The database client to use for the repository
+ * @param logoRepository - The logo repository, to resolve the logo of an imported item
+ */
+ public constructor(client: IDatabaseClient, private logoRepository: LogoRepository) {
+ super(client);
+ }
+
+ /**
+ * Read every live item with its child rows, plus the folders, tags, custom field definitions and the logos the
+ * items use. Trashed items are left out, archived items are included.
+ * @returns The vault data
+ */
+ public *getExportData(): DbOp<VaultExportData> {
+ const itemRows = yield* this.query<{ Id: string; ManifestId: string; Name: string | null; ItemType: string; FolderId: string | null; LogoId: string | null; CreatedAt: string; UpdatedAt: string }>(
+ 'SELECT Id, ManifestId, Name, ItemType, FolderId, LogoId, CreatedAt, UpdatedAt FROM Items WHERE IsDeleted = 0 AND DeletedAt IS NULL ORDER BY CreatedAt'
+ );
+
+ const fieldDefinitionRows = yield* this.query<{ Id: string; ManifestId: string; FieldType: string; Label: string; IsMultiValue: number; IsHidden: number; EnableHistory: number; Weight: number; ApplicableToTypes: string | null; CreatedAt: string; UpdatedAt: string }>(
+ 'SELECT Id, ManifestId, FieldType, Label, IsMultiValue, IsHidden, EnableHistory, Weight, ApplicableToTypes, CreatedAt, UpdatedAt FROM FieldDefinitions WHERE IsDeleted = 0'
+ );
+ const fieldDefinitions = fieldDefinitionRows.map((row): FieldDefinitionEntity & { ManifestId: string } => ({
+ Id: row.Id,
+ ManifestId: row.ManifestId,
+ FieldType: row.FieldType,
+ Label: row.Label,
+ IsMultiValue: row.IsMultiValue === 1,
+ IsHidden: row.IsHidden === 1,
+ EnableHistory: row.EnableHistory === 1,
+ Weight: row.Weight,
+ ApplicableToTypes: row.ApplicableToTypes,
+ CreatedAt: fromStandardFormat(row.CreatedAt),
+ UpdatedAt: fromStandardFormat(row.UpdatedAt),
+ IsDeleted: false,
+ }));
+ const definitionsByKey = new Map(fieldDefinitions.map(definition => [scopedKey(definition.ManifestId, definition.Id), definition]));
+
+ const fieldValueRows = yield* this.query<ChildRow & { Id: string; FieldKey: string | null; FieldDefinitionId: string | null; Value: string | null; Weight: number; CreatedAt: string; UpdatedAt: string }>(
+ 'SELECT fv.Id, fv.ItemId, fv.ManifestId, fv.FieldKey, fv.FieldDefinitionId, fv.Value, fv.Weight, fv.CreatedAt, fv.UpdatedAt FROM FieldValues fv INNER JOIN Items i ON i.Id = fv.ItemId AND i.ManifestId = fv.ManifestId WHERE fv.IsDeleted = 0 AND i.IsDeleted = 0 AND i.DeletedAt IS NULL ORDER BY fv.Weight'
+ );
+ const fieldValuesByItem = ImportExportRepository.groupByItem(fieldValueRows, (row): FieldValueEntity => ({
+ Id: row.Id,
+ ItemId: row.ItemId,
+ FieldKey: row.FieldKey,
+ FieldDefinitionId: row.FieldDefinitionId,
+ FieldDefinition: row.FieldDefinitionId ? definitionsByKey.get(scopedKey(row.ManifestId, row.FieldDefinitionId)) ?? null : null,
+ Value: row.Value,
+ Weight: row.Weight,
+ CreatedAt: fromStandardFormat(row.CreatedAt),
+ UpdatedAt: fromStandardFormat(row.UpdatedAt),
+ IsDeleted: false,
+ }));
+
+ const attachmentRows = yield* this.query<ChildRow & { Id: string; Filename: string; Blob: Uint8Array | null; CreatedAt: string; UpdatedAt: string }>(
+ 'SELECT a.Id, a.ItemId, a.ManifestId, a.Filename, a.Blob, a.CreatedAt, a.UpdatedAt FROM Attachments a INNER JOIN Items i ON i.Id = a.ItemId AND i.ManifestId = a.ManifestId WHERE a.IsDeleted = 0 AND i.IsDeleted = 0 AND i.DeletedAt IS NULL'
+ );
+ const attachmentsByItem = ImportExportRepository.groupByItem(attachmentRows, (row): AttachmentEntity => ({
+ Id: row.Id,
+ ItemId: row.ItemId,
+ Filename: row.Filename,
+ Blob: row.Blob ? new Uint8Array(row.Blob) : new Uint8Array(0),
+ CreatedAt: fromStandardFormat(row.CreatedAt),
+ UpdatedAt: fromStandardFormat(row.UpdatedAt),
+ IsDeleted: false,
+ }));
+
+ const totpRows = yield* this.query<ChildRow & { Id: string; Name: string; SecretKey: string; Algorithm: string | null; Digits: number | null; Period: number | null; CreatedAt: string; UpdatedAt: string }>(
+ 'SELECT t.Id, t.ItemId, t.ManifestId, t.Name, t.SecretKey, t.Algorithm, t.Digits, t.Period, t.CreatedAt, t.UpdatedAt FROM TotpCodes t INNER JOIN Items i ON i.Id = t.ItemId AND i.ManifestId = t.ManifestId WHERE t.IsDeleted = 0 AND i.IsDeleted = 0 AND i.DeletedAt IS NULL'
+ );
+ const totpCodesByItem = ImportExportRepository.groupByItem(totpRows, (row): TotpCodeEntity => ({
+ Id: row.Id,
+ ItemId: row.ItemId,
+ Name: row.Name ?? '',
+ SecretKey: row.SecretKey,
+ Algorithm: row.Algorithm ?? 'SHA1',
+ Digits: row.Digits ?? 6,
+ Period: row.Period ?? 30,
+ CreatedAt: fromStandardFormat(row.CreatedAt),
+ UpdatedAt: fromStandardFormat(row.UpdatedAt),
+ IsDeleted: false,
+ }));
+
+ const passkeyRows = yield* this.query<ChildRow & { Id: string; RpId: string; UserHandle: Uint8Array | null; PublicKey: string; PrivateKey: string; PrfKey: Uint8Array | null; DisplayName: string; CreatedAt: string; UpdatedAt: string }>(
+ 'SELECT p.Id, p.ItemId, p.ManifestId, p.RpId, p.UserHandle, p.PublicKey, p.PrivateKey, p.PrfKey, p.DisplayName, p.CreatedAt, p.UpdatedAt FROM Passkeys p INNER JOIN Items i ON i.Id = p.ItemId AND i.ManifestId = p.ManifestId WHERE p.IsDeleted = 0 AND i.IsDeleted = 0 AND i.DeletedAt IS NULL'
+ );
+ const passkeysByItem = ImportExportRepository.groupByItem(passkeyRows, (row): PasskeyEntity => ({
+ Id: row.Id,
+ ItemId: row.ItemId,
+ RpId: row.RpId,
+ UserHandle: row.UserHandle ? new Uint8Array(row.UserHandle) : null,
+ PublicKey: row.PublicKey,
+ PrivateKey: row.PrivateKey,
+ PrfKey: row.PrfKey ? new Uint8Array(row.PrfKey) : null,
+ DisplayName: row.DisplayName ?? '',
+ CreatedAt: fromStandardFormat(row.CreatedAt),
+ UpdatedAt: fromStandardFormat(row.UpdatedAt),
+ IsDeleted: false,
+ }));
+
+ const items = itemRows.map((row): ItemEntity => {
+ const key = scopedKey(row.ManifestId, row.Id);
+ return {
+ Id: row.Id,
+ Name: row.Name,
+ ItemType: row.ItemType as ItemType,
+ FolderId: row.FolderId,
+ LogoId: row.LogoId,
+ CreatedAt: fromStandardFormat(row.CreatedAt),
+ UpdatedAt: fromStandardFormat(row.UpdatedAt),
+ IsDeleted: false,
+ FieldValues: fieldValuesByItem.get(key) ?? [],
+ Attachments: attachmentsByItem.get(key) ?? [],
+ TotpCodes: totpCodesByItem.get(key) ?? [],
+ Passkeys: passkeysByItem.get(key) ?? [],
+ };
+ });
+
+ const folderRows = yield* this.query<{ Id: string; Name: string; ParentFolderId: string | null; Weight: number; CreatedAt: string; UpdatedAt: string }>(
+ 'SELECT Id, Name, ParentFolderId, Weight, CreatedAt, UpdatedAt FROM Folders WHERE IsDeleted = 0'
+ );
+ const folders = folderRows.map((row): FolderEntity => ({
+ Id: row.Id,
+ Name: row.Name,
+ ParentFolderId: row.ParentFolderId,
+ Weight: row.Weight,
+ CreatedAt: fromStandardFormat(row.CreatedAt),
+ UpdatedAt: fromStandardFormat(row.UpdatedAt),
+ IsDeleted: false,
+ }));
+
+ const tagRows = yield* this.query<{ Id: string; Name: string; Color: string | null; DisplayOrder: number; CreatedAt: string; UpdatedAt: string }>(
+ 'SELECT Id, Name, Color, DisplayOrder, CreatedAt, UpdatedAt FROM Tags WHERE IsDeleted = 0'
+ );
+ const tags = tagRows.map((row): TagEntity => ({
+ Id: row.Id,
+ Name: row.Name,
+ Color: row.Color,
+ DisplayOrder: row.DisplayOrder,
+ CreatedAt: fromStandardFormat(row.CreatedAt),
+ UpdatedAt: fromStandardFormat(row.UpdatedAt),
+ IsDeleted: false,
+ }));
+
+ const itemTagRows = yield* this.query<{ ItemId: string; TagId: string }>('SELECT ItemId, TagId FROM ItemTags WHERE IsDeleted = 0');
+ const itemTags = itemTagRows.map((row): ItemTagEntity => ({ ItemId: row.ItemId, TagId: row.TagId, IsDeleted: false }));
+
+ // Only the logos the exported items point at.
+ const logoIds = new Set(items.map(item => item.LogoId).filter((id): id is string => id !== null));
+ const logoRows = yield* this.query<{ Id: string; Source: string; FileData: Uint8Array | null; MimeType: string | null; UpdatedAt: string }>(
+ 'SELECT Id, Source, FileData, MimeType, UpdatedAt FROM Logos WHERE IsDeleted = 0'
+ );
+ const logos = logoRows.filter(row => logoIds.has(row.Id)).map((row): LogoEntity => ({
+ Id: row.Id,
+ Source: row.Source,
+ FileData: row.FileData ? new Uint8Array(row.FileData) : null,
+ MimeType: row.MimeType,
+ FetchedAt: fromStandardFormat(row.UpdatedAt),
+ IsDeleted: false,
+ }));
+
+ return { items, folders, tags, itemTags, fieldDefinitions, logos };
+ }
+
+ /**
+ * Group child rows by the key of their item.
+ * @param rows - The rows
+ * @param map - Maps a row to its entity
+ * @returns The entities per item key (see {@link scopedKey})
+ */
+ private static groupByItem<TRow extends ChildRow, TEntity>(rows: TRow[], map: (row: TRow) => TEntity): Map<string, TEntity[]> {
+ const grouped = new Map<string, TEntity[]>();
+ for (const row of rows) {
+ const key = scopedKey(row.ManifestId, row.ItemId);
+ const list = grouped.get(key) ?? [];
+ list.push(map(row));
+ grouped.set(key, list);
+ }
+ return grouped;
+ }
+
+ /**
+ * Resolve the logo an imported item gets for a source domain: the favicon the item's manifest already holds when
+ * it has image data, else a row filled (or refilled) with the given bytes, else the favicon another manifest holds.
+ * @param source - The Logos.Source key (the domain)
+ * @param faviconBytes - The favicon bytes to store, or null when none are available
+ * @returns The logo id, or null when the item gets no logo
+ */
+ public async resolveImportLogo(source: string, faviconBytes: Uint8Array | null): Promise<string | null> {
+ const scope = await this.run(this.writeManifestId());
+ const currentDateTime = this.now();
+
+ const existing = (await this.run(this.query<{ Id: string; FileData: Uint8Array | null }>(
+ 'SELECT Id, FileData FROM Logos WHERE ManifestId = ? AND Kind = ? AND Source = ? AND IsDeleted = 0 LIMIT 1',
+ [scope, LogoKinds.Favicon, source]
+ )))[0];
+ if (existing && existing.FileData && existing.FileData.length > 0) {
+ return existing.Id;
+ }
+
+ if (faviconBytes && faviconBytes.length > 0) {
+ return this.logoRepository.getOrCreate(scope, LogoKinds.Favicon, source, faviconBytes, currentDateTime, { mimeType: 'image/x-icon' });
+ }
+
+ return this.logoRepository.ensureInScope(scope, LogoKinds.Favicon, source, currentDateTime);
+ }
+
+ /**
+ * Insert an imported item with its field values, custom field definitions, TOTP codes, passkeys and
+ * attachments, keeping the timestamps the source export carried.
+ * @param item - The item graph to insert
+ */
+ public async insertImportedItem(item: ItemEntity): Promise<void> {
+ return this.withTransaction(() => this.run(this.insertItemGraph(item)));
+ }
+
+ /**
+ * Write one item graph.
+ * @param item - The item graph
+ */
+ private *insertItemGraph(item: ItemEntity): DbOp<void> {
+ const manifestId = yield* this.writeManifestId();
+ const createdAt = toStandardFormat(item.CreatedAt);
+ const updatedAt = toStandardFormat(item.UpdatedAt);
+
+ yield* this.execute(ItemQueries.INSERT_ITEM, [
+ item.Id,
+ item.Name,
+ item.ItemType,
+ item.LogoId,
+ item.FolderId,
+ // Second bind of the folder id, then the manifest an item outside any folder joins (see INSERT_ITEM).
+ item.FolderId,
+ manifestId,
+ createdAt,
+ updatedAt,
+ 0,
+ ]);
+
+ // Custom field definitions come before the values that reference them; each is written once.
+ const writtenDefinitions = new Set<string>();
+ for (const fieldValue of item.FieldValues) {
+ const definition = fieldValue.FieldDefinition;
+ if (definition && !writtenDefinitions.has(definition.Id)) {
+ writtenDefinitions.add(definition.Id);
+ yield* this.execute(FieldDefinitionQueries.INSERT, [
+ definition.Id,
+ item.Id,
+ manifestId,
+ definition.FieldType,
+ definition.Label,
+ definition.IsMultiValue ? 1 : 0,
+ definition.IsHidden ? 1 : 0,
+ definition.EnableHistory ? 1 : 0,
+ definition.Weight,
+ definition.ApplicableToTypes,
+ toStandardFormat(definition.CreatedAt),
+ toStandardFormat(definition.UpdatedAt),
+ 0,
+ ]);
+ }
+
+ yield* this.execute(FieldValueQueries.INSERT, [
+ fieldValue.Id,
+ item.Id,
+ item.Id,
+ manifestId,
+ fieldValue.FieldDefinitionId,
+ fieldValue.FieldKey,
+ fieldValue.Value ?? '',
+ fieldValue.Weight,
+ toStandardFormat(fieldValue.CreatedAt),
+ toStandardFormat(fieldValue.UpdatedAt),
+ 0,
+ ]);
+ }
+
+ for (const totpCode of item.TotpCodes) {
+ yield* this.execute(TotpCodeQueries.INSERT, [
+ totpCode.Id,
+ totpCode.Name,
+ totpCode.SecretKey,
+ totpCode.Algorithm,
+ totpCode.Digits,
+ totpCode.Period,
+ item.Id,
+ item.Id,
+ manifestId,
+ toStandardFormat(totpCode.CreatedAt),
+ toStandardFormat(totpCode.UpdatedAt),
+ 0,
+ ]);
+ }
+
+ for (const passkey of item.Passkeys) {
+ yield* this.execute(PasskeyQueries.INSERT, [
+ passkey.Id,
+ item.Id,
+ item.Id,
+ manifestId,
+ passkey.RpId,
+ passkey.UserHandle,
+ passkey.PublicKey,
+ passkey.PrivateKey,
+ passkey.PrfKey,
+ passkey.DisplayName,
+ null,
+ toStandardFormat(passkey.CreatedAt),
+ toStandardFormat(passkey.UpdatedAt),
+ 0,
+ ]);
+ }
+
+ for (const attachment of item.Attachments) {
+ yield* this.execute(AttachmentQueries.INSERT, [
+ attachment.Id,
+ attachment.Filename,
+ attachment.Blob,
+ item.Id,
+ item.Id,
+ manifestId,
+ toStandardFormat(attachment.CreatedAt),
+ toStandardFormat(attachment.UpdatedAt),
+ 0,
+ ]);
+ }
+ }
+
+ /**
+ * Empty the user's own vault: every row of the personal manifest in every item-related table is hard deleted,
+ * dependents first. Rows of shared manifests are left alone. Settings and encryption keys are kept.
+ */
+ public async hardDeleteAllVaultData(): Promise<void> {
+ return this.withTransaction(() => this.run(this.deleteAllRows()));
+ }
+
+ /**
+ * Delete the rows of the reset.
+ */
+ private *deleteAllRows(): DbOp<void> {
+ const personalManifestId = yield* this.personalManifestId();
+ for (const table of RESET_TABLES) {
+ if (personalManifestId) {
+ yield* this.execute(`DELETE FROM ${table} WHERE ManifestId = ?`, [personalManifestId]);
+ } else {
+ yield* this.execute(`DELETE FROM ${table}`);
+ }
+ }
+ }
+}