A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: fix backup restore not re-uploading re-keyed attachments (#10524)
Abdullah Attab1477075b104
diff --git a/apps/mobile/app/screens/settings/restore-backup/index.tsx b/apps/mobile/app/screens/settings/restore-backup/index.tsxindex ef2b077f5..22fa910bb 100644--- a/apps/mobile/app/screens/settings/restore-backup/index.tsx+++ b/apps/mobile/app/screens/settings/restore-backup/index.tsx@@ -156,6 +156,7 @@ const restoreBackup = async (options: {: undefined;let count = 0;+ const skippedAttachments = new Set<string>();await db.transaction(async () => {let passwordOrKey: PasswordOrKey | undefined = undefined;for (const path of extractedBackupFiles) {@@ -186,10 +187,11 @@ const restoreBackup = async (options: {throw new Error(strings.failedToDecryptBackup());}- await db.backup.import(backup, {+ const skipped = await db.backup.import(backup, {...passwordOrKey,attachmentsKey: attachmentsKey});+ skipped?.forEach((hash) => skippedAttachments.add(hash));}});@@ -205,7 +207,7 @@ const restoreBackup = async (options: {});const hash = path;const attachment = await db.attachments.attachment(hash as string);- if (!attachment) continue;+ if (!attachment || skippedAttachments.has(hash)) continue;await deleteCacheFileByName(hash);await RNFetchBlob.fs.cp(diff --git a/apps/web/src/common/index.ts b/apps/web/src/common/index.tsindex 34cd9a513..3e9e6e8ca 100644--- a/apps/web/src/common/index.ts+++ b/apps/web/src/common/index.ts@@ -276,6 +276,7 @@ export async function restoreBackupFile(backupFile: File) {const attachments: ZipEntry[] = [];let attachmentsKey: SerializedKey | Cipher<"base64"> | undefined;let filesProcessed = 0;+ const skippedAttachments = new Set<string>();let isValid = false;for await (const entry of createUnzipIterator(backupFile)) {@@ -304,11 +305,12 @@ export async function restoreBackupFile(backupFile: File) {const result = await BackupPasswordDialog.show({validate: async ({ password, key: encryptionKey }) => {if (!password && !encryptionKey) return false;- await db.backup?.import(backup, {+ const skipped = await db.backup?.import(backup, {password,encryptionKey,attachmentsKey});+ skipped?.forEach((hash) => skippedAttachments.add(hash));cachedPassword = password;cachedKey = encryptionKey;return true;@@ -316,13 +318,17 @@ export async function restoreBackupFile(backupFile: File) {});if (!result) break;} else- await db.backup?.import(backup, {- password: cachedPassword,- encryptionKey: cachedKey,- attachmentsKey- });+ (+ await db.backup?.import(backup, {+ password: cachedPassword,+ encryptionKey: cachedKey,+ attachmentsKey+ })+ )?.forEach((hash) => skippedAttachments.add(hash));} else {- await db.backup?.import(backup, { attachmentsKey });+ (await db.backup?.import(backup, { attachmentsKey }))?.forEach(+ (hash) => skippedAttachments.add(hash)+ );}report({@@ -345,7 +351,7 @@ export async function restoreBackupFile(backupFile: File) {});const attachment = await db.attachments.attachment(hash);- if (!attachment) continue;+ if (!attachment || skippedAttachments.has(hash)) continue;await streamablefs.deleteFile(attachment.hash);const handle = await streamablefs.createFile(diff --git a/packages/core/__tests__/backup.test.ts b/packages/core/__tests__/backup.test.tsnew file mode 100644index 000000000..e1a1e2f20--- /dev/null+++ b/packages/core/__tests__/backup.test.ts@@ -0,0 +1,98 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/+++import { expect, test } from "vitest";+import { databaseTest, loginFakeUser } from "./utils";+import Database from "../src/api/index.js";++async function restoreWithDifferentKey(+ db: Database,+ local: { dateUploaded?: number; fileExists: boolean }+) {+ await loginFakeUser(db);++ const hash = await db.attachments.save(+ "iVBORw0KGgoAAAANSUhEUgAAAAEEAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ "image/png",+ "test.png"+ );+ if (!hash) throw new Error("Failed to create attachment");+ const attachment = await db.attachments.attachment(hash);+ if (!attachment) throw new Error("Failed to find attachment");+ await db.attachments.collection.update([attachment.id], {+ dateUploaded: local.dateUploaded,+ chunkSize: 1+ });+ if (!local.fileExists) await db.fs().deleteFile(hash, true);++ const attachmentsKey = await db.user?.getAttachmentsKey();+ if (!attachmentsKey) throw new Error("Missing attachments key");+ const backupKey = await db+ .storage()+ .encrypt(attachmentsKey, JSON.stringify({ iv: "a", salt: "b", key: "c" }));++ const skipped = await (db.backup as any).migrateData(+ [+ {+ ...attachment,+ key: backupKey,+ iv: "new-iv",+ salt: "new-salt",+ chunkSize: 4096,+ dateUploaded: undefined+ }+ ],+ 6.0,+ attachmentsKey+ );++ return {+ skipped,+ original: attachment,+ restored: await db.attachments.attachment(hash)+ };+}++test("restoring a backup attachment with a different key should keep the uploaded local attachment", () =>+ databaseTest().then(async (db) => {+ const dateUploaded = Date.now();+ const { skipped, original, restored } = await restoreWithDifferentKey(db, {+ dateUploaded,+ fileExists: true+ });++ expect(skipped).toEqual([original.hash]);+ expect(restored?.key).toEqual(original.key);+ expect(restored?.iv).toBe(original.iv);+ expect(restored?.chunkSize).toBe(1);+ expect(restored?.dateUploaded).toBe(dateUploaded);+ }));++test("restoring a backup attachment with a different key should adopt the backup key if local file is unavailable", () =>+ databaseTest().then(async (db) => {+ const { skipped, restored } = await restoreWithDifferentKey(db, {+ fileExists: false+ });++ expect(skipped).toEqual([]);+ expect(restored?.iv).toBe("new-iv");+ expect(restored?.chunkSize).toBe(4096);+ expect(restored?.dateUploaded).toBeFalsy();+ }));diff --git a/packages/core/src/database/backup.ts b/packages/core/src/database/backup.tsindex 17c5ef062..ac613dfc3 100644--- a/packages/core/src/database/backup.ts+++ b/packages/core/src/database/backup.ts@@ -519,7 +519,7 @@ export default class Backup {: typeof data === "object"? Object.values(data): [];- await this.migrateData(+ return await this.migrateData(normalizedData,backup.version === 6.1 && isLegacyBackup(normalizedData)? 5.9@@ -561,6 +561,7 @@ export default class Backup {attachmentsKey?: SerializedKey) {const queue: Partial<Record<CollectionName, MaybeDeletedItem<Item>[]>> = {};+ const skippedAttachments: string[] = [];for (let item of data) {// we do not want to restore deleted itemsif (@@ -622,7 +623,17 @@ export default class Backup {attachment.key.cipher === item.key.cipher &&attachment.key.salt === item.key.salt;- if (attachmentsKey && !isSameKey) {+ if (+ attachmentsKey &&+ !isSameKey &&+ attachment &&+ (attachment.dateUploaded ||+ (await this.db.fs().exists(attachment.hash)))+ ) {+ // the server won't overwrite an uploaded file & other devices may have+ // it cached, so we must keep the local key and file.+ skippedAttachments.push(attachment.hash);+ } else if (attachmentsKey && !isSameKey) {const newKey = await this.db.attachments.encryptKey(JSON.parse(await this.db.storage().decrypt(attachmentsKey, item.key)@@ -633,6 +644,10 @@ export default class Backup {attachment.iv = item.iv;attachment.salt = item.salt;attachment.size = item.size;+ attachment.chunkSize = item.chunkSize;+ attachment.alg = item.alg;+ delete attachment.dateUploaded;+ delete attachment.failed;} else item.key = newKey;delete item.dateUploaded;delete item.failed;@@ -714,6 +729,7 @@ export default class Backup {await collection.put(items as any[]);}+ return skippedAttachments;}private validate(backup: LegacyBackupFile | BackupFile) {