A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: skip attachments with undecryptable keys when restoring backups (#10627)
Abdullah Attae25b1c8f94dc
diff --git a/packages/core/__tests__/backup.test.ts b/packages/core/__tests__/backup.test.tsindex e1a1e2f20..71489ea19 100644--- a/packages/core/__tests__/backup.test.ts+++ b/packages/core/__tests__/backup.test.ts@@ -18,9 +18,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { expect, test } from "vitest";-import { databaseTest, loginFakeUser } from "./utils";+import { expect, test, vi } from "vitest";+import { databaseTest, loginFakeUser, TEST_NOTE } from "./utils";import Database from "../src/api/index.js";+import { FileStorage } from "../src/database/fs.js";async function restoreWithDifferentKey(db: Database,@@ -96,3 +97,94 @@ test("restoring a backup attachment with a different key should adopt the backupexpect(restored?.chunkSize).toBe(4096);expect(restored?.dateUploaded).toBeFalsy();}));++const IMAGE =+ "iVBORw0KGgoAAAANSUhEUgAAAAEEAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";++async function exportBackup(db: Database, mode: "full" | "partial") {+ const files: any[] = [];+ for await (const file of db.backup.export({+ type: "node",+ encrypt: false,+ mode+ }))+ files.push(file);+ const keyFile = files.find((f) => f.path === "attachments/.attachments_key");+ return {+ attachmentsKey: keyFile ? JSON.parse(keyFile.data) : undefined,+ backups: files+ .filter(+ (f) =>+ f.type === "file" &&+ f.path !== ".nnbackup" &&+ !f.path.startsWith("attachments/")+ )+ .map((f) => JSON.parse(f.data))+ };+}++async function importBackup(+ db: Database,+ { attachmentsKey, backups }: Awaited<ReturnType<typeof exportBackup>>+) {+ const skipped: string[] = [];+ for (const backup of backups)+ skipped.push(...((await db.backup.import(backup, { attachmentsKey })) || []));+ return skipped;+}++async function createAccountWithAttachment() {+ const db = await databaseTest();+ await loginFakeUser(db);+ const noteId = await db.notes.add(TEST_NOTE);+ const hash = await db.attachments.save(IMAGE, "image/png", "test.png");+ if (!hash) throw new Error("Failed to create attachment");+ return { db, noteId, hash };+}++test("restoring a partial backup from another account should skip its attachments", async () => {+ const { db: other, noteId, hash } = await createAccountWithAttachment();+ const db = await databaseTest();+ await loginFakeUser(db);++ const skipped = await importBackup(db, await exportBackup(other, "partial"));++ expect(skipped).toEqual([hash]);+ expect(await db.notes.note(noteId)).toBeDefined();+ expect(await db.attachments.attachment(hash)).toBeUndefined();+});++test("restoring a full backup with an undecryptable attachment key should restore everything else", async () => {+ const { db: other, hash } = await createAccountWithAttachment();+ const db = await databaseTest();+ await loginFakeUser(db);+ await db.attachments.collection.upsert(+ (await other.attachments.attachment(hash))!+ );+ const noteId = await db.notes.add({ ...TEST_NOTE, title: "local note" });++ const target = await databaseTest();+ await loginFakeUser(target);+ const skipped = await importBackup(target, await exportBackup(db, "full"));++ expect(skipped).toEqual([hash]);+ expect(await target.notes.note(noteId)).toBeDefined();+ expect(await target.attachments.attachment(hash)).toBeUndefined();+});++test("restoring a full backup should re-upload an attachment whose file is lost everywhere", async () => {+ const { db, hash } = await createAccountWithAttachment();+ const attachment = (await db.attachments.attachment(hash))!;+ await db.attachments.collection.update([attachment.id], {+ dateUploaded: Date.now()+ });+ const backup = await exportBackup(db, "full");+ await db.fs().deleteFile(hash, true);+ vi.spyOn(FileStorage.prototype, "getUploadedFileSize").mockResolvedValue(0);++ const skipped = await importBackup(db, backup);++ expect(skipped).toEqual([]);+ expect((await db.attachments.attachment(hash))?.dateUploaded).toBeFalsy();+ vi.restoreAllMocks();+});diff --git a/packages/core/src/database/backup.ts b/packages/core/src/database/backup.tsindex ac613dfc3..6ef2eecca 100644--- a/packages/core/src/database/backup.ts+++ b/packages/core/src/database/backup.ts@@ -613,88 +613,11 @@ export default class Backup {if (!itemType || itemType === "topic" || itemType === "settings")continue;- if (item.type === "attachment" && (item.metadata?.hash || item.hash)) {- const attachment = await this.db.attachments.attachment(- item.metadata?.hash || item.hash- );- const isSameKey =- !!attachment &&- attachment.key.iv === item.key.iv &&- attachment.key.cipher === item.key.cipher &&- attachment.key.salt === item.key.salt;-- if (- attachmentsKey &&- !isSameKey &&- attachment &&- (attachment.dateUploaded ||- (await this.db.fs().exists(attachment.hash)))- ) {- // the server won't overwrite an uploaded file & other devices may have- // it cached, so we must keep the local key and file.- skippedAttachments.push(attachment.hash);- } else if (attachmentsKey && !isSameKey) {- const newKey = await this.db.attachments.encryptKey(- JSON.parse(- await this.db.storage().decrypt(attachmentsKey, item.key)- )- );- if (attachment) {- attachment.key = newKey;- attachment.iv = item.iv;- attachment.salt = item.salt;- attachment.size = item.size;- attachment.chunkSize = item.chunkSize;- attachment.alg = item.alg;- delete attachment.dateUploaded;- delete attachment.failed;- } else item.key = newKey;- delete item.dateUploaded;- delete item.failed;- }-- if (attachment) {- if (- attachmentsKey &&- isSameKey &&- !(await this.db.fs().exists(attachment.hash)) &&- (await this.db.fs().getUploadedFileSize(attachment.hash)) <= 0- ) {- delete item.dateUploaded;- delete item.failed;- }-- const isNewGeneric =- item.metadata?.type === "application/octet-stream" ||- item.mimeType === "application/octet-stream";- const isOldGeneric =- attachment.mimeType === "application/octet-stream";- item = {- ...attachment,- mimeType:- // we keep whichever mime type is more specific- isNewGeneric && !isOldGeneric- ? attachment.mimeType- : item.metadata?.type || item.mimeType,- filename:- // we keep the filename based on which item's mime type we kept- isNewGeneric && !isOldGeneric- ? attachment.filename- : item.metadata?.filename || item.filename- };- for (const noteId of item.noteIds || []) {- await this.db.relations.add(- {- id: noteId,- type: "note"- },- attachment- );- }- } else {- delete item.dateUploaded;- delete item.failed;- }+ if (item.type === "attachment") {+ const result = await this.restoreAttachment(item, attachmentsKey);+ if (result.skipFile) skippedAttachments.push(item.hash);+ if (!result.attachment) continue;+ item = result.attachment;}const collectionKey: CollectionName =@@ -732,6 +655,80 @@ export default class Backup {return skippedAttachments;}+ private async restoreAttachment(+ item: Attachment,+ attachmentsKey?: SerializedKey+ ): Promise<{ attachment?: Attachment; skipFile?: boolean }> {+ const local = await this.db.attachments.attachment(item.hash);++ if (!attachmentsKey) {+ if (local) return { attachment: mergeAttachmentInfo(local, item) };+ if (await this.isOwnAttachmentKey(item.key))+ return { attachment: withoutUploadState(item) };+ return skipAttachment(item);+ }++ if (local && isSameKey(local, item)) {+ const attachment = mergeAttachmentInfo(local, item);+ return {+ attachment: (await this.hasFile(local.hash))+ ? attachment+ : withoutUploadState(attachment)+ };+ }++ const key = await this.reencryptBackupKey(item.key, attachmentsKey);+ if (!local)+ return key+ ? { attachment: withoutUploadState({ ...item, key }) }+ : skipAttachment(item);++ // the server won't overwrite an uploaded file & other devices may have+ // it cached, so we must keep the local key and file.+ if (!key || local.dateUploaded || (await this.db.fs().exists(local.hash)))+ return { attachment: mergeAttachmentInfo(local, item), skipFile: true };++ return {+ attachment: withoutUploadState({+ ...mergeAttachmentInfo(local, item),+ key,+ iv: item.iv,+ salt: item.salt,+ size: item.size,+ chunkSize: item.chunkSize,+ alg: item.alg+ })+ };+ }++ private async reencryptBackupKey(+ key: Cipher<"base64">,+ attachmentsKey: SerializedKey+ ) {+ const decrypted = await this.db+ .storage()+ .decrypt(attachmentsKey, key)+ .catch(() => undefined);+ if (!decrypted) return;+ return await this.db.attachments.encryptKey(JSON.parse(decrypted));+ }++ private async isOwnAttachmentKey(key: Cipher<"base64">) {+ const attachmentsKey = await this.db.user.getAttachmentsKey();+ if (!attachmentsKey) return true;+ return !!(await this.db+ .storage()+ .decrypt(attachmentsKey, key)+ .catch(() => undefined));+ }++ private async hasFile(hash: string) {+ return (+ (await this.db.fs().exists(hash)) ||+ (await this.db.fs().getUploadedFileSize(hash)) > 0+ );+ }+private validate(backup: LegacyBackupFile | BackupFile) {return (!!backup.date &&@@ -759,3 +756,32 @@ export default class Backup {}}}++function isSameKey(a: Attachment, b: Attachment) {+ return (+ a.key.iv === b.key.iv &&+ a.key.cipher === b.key.cipher &&+ a.key.salt === b.key.salt+ );+}++function withoutUploadState(attachment: Attachment): Attachment {+ const { dateUploaded: _dateUploaded, failed: _failed, ...rest } = attachment;+ return rest;+}++function skipAttachment(item: Attachment) {+ logger.warn("Skipping attachment with an undecryptable key", {+ hash: item.hash+ });+ return { skipFile: true };+}++function mergeAttachmentInfo(local: Attachment, item: Attachment): Attachment {+ const keepLocal =+ item.mimeType === "application/octet-stream" &&+ local.mimeType !== "application/octet-stream";+ return keepLocal+ ? local+ : { ...local, mimeType: item.mimeType, filename: item.filename };+}