Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: skip attachments with undecryptable keys when restoring backups (#10627)

Abdullah Attae25b1c8f94dc
diff --git a/packages/core/__tests__/backup.test.ts b/packages/core/__tests__/backup.test.ts
index e1a1e2f20..71489ea19 100644
--- a/packages/core/__tests__/backup.test.ts
+++ b/packages/core/__tests__/backup.test.ts
@@ -18,9 +18,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import { expect, test } from "vitest";
-import { databaseTest, loginFakeUser } from "./utils";
+import { expect, test, vi } from "vitest";
+import { databaseTest, loginFakeUser, TEST_NOTE } from "./utils";
import Database from "../src/api/index.js";
+import { FileStorage } from "../src/database/fs.js";
async function restoreWithDifferentKey(
db: Database,
@@ -96,3 +97,94 @@ test("restoring a backup attachment with a different key should adopt the backup
expect(restored?.chunkSize).toBe(4096);
expect(restored?.dateUploaded).toBeFalsy();
}));
+
+const IMAGE =
+ "iVBORw0KGgoAAAANSUhEUgAAAAEEAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
+
+async function exportBackup(db: Database, mode: "full" | "partial") {
+ const files: any[] = [];
+ for await (const file of db.backup.export({
+ type: "node",
+ encrypt: false,
+ mode
+ }))
+ files.push(file);
+ const keyFile = files.find((f) => f.path === "attachments/.attachments_key");
+ return {
+ attachmentsKey: keyFile ? JSON.parse(keyFile.data) : undefined,
+ backups: files
+ .filter(
+ (f) =>
+ f.type === "file" &&
+ f.path !== ".nnbackup" &&
+ !f.path.startsWith("attachments/")
+ )
+ .map((f) => JSON.parse(f.data))
+ };
+}
+
+async function importBackup(
+ db: Database,
+ { attachmentsKey, backups }: Awaited<ReturnType<typeof exportBackup>>
+) {
+ const skipped: string[] = [];
+ for (const backup of backups)
+ skipped.push(...((await db.backup.import(backup, { attachmentsKey })) || []));
+ return skipped;
+}
+
+async function createAccountWithAttachment() {
+ const db = await databaseTest();
+ await loginFakeUser(db);
+ const noteId = await db.notes.add(TEST_NOTE);
+ const hash = await db.attachments.save(IMAGE, "image/png", "test.png");
+ if (!hash) throw new Error("Failed to create attachment");
+ return { db, noteId, hash };
+}
+
+test("restoring a partial backup from another account should skip its attachments", async () => {
+ const { db: other, noteId, hash } = await createAccountWithAttachment();
+ const db = await databaseTest();
+ await loginFakeUser(db);
+
+ const skipped = await importBackup(db, await exportBackup(other, "partial"));
+
+ expect(skipped).toEqual([hash]);
+ expect(await db.notes.note(noteId)).toBeDefined();
+ expect(await db.attachments.attachment(hash)).toBeUndefined();
+});
+
+test("restoring a full backup with an undecryptable attachment key should restore everything else", async () => {
+ const { db: other, hash } = await createAccountWithAttachment();
+ const db = await databaseTest();
+ await loginFakeUser(db);
+ await db.attachments.collection.upsert(
+ (await other.attachments.attachment(hash))!
+ );
+ const noteId = await db.notes.add({ ...TEST_NOTE, title: "local note" });
+
+ const target = await databaseTest();
+ await loginFakeUser(target);
+ const skipped = await importBackup(target, await exportBackup(db, "full"));
+
+ expect(skipped).toEqual([hash]);
+ expect(await target.notes.note(noteId)).toBeDefined();
+ expect(await target.attachments.attachment(hash)).toBeUndefined();
+});
+
+test("restoring a full backup should re-upload an attachment whose file is lost everywhere", async () => {
+ const { db, hash } = await createAccountWithAttachment();
+ const attachment = (await db.attachments.attachment(hash))!;
+ await db.attachments.collection.update([attachment.id], {
+ dateUploaded: Date.now()
+ });
+ const backup = await exportBackup(db, "full");
+ await db.fs().deleteFile(hash, true);
+ vi.spyOn(FileStorage.prototype, "getUploadedFileSize").mockResolvedValue(0);
+
+ const skipped = await importBackup(db, backup);
+
+ expect(skipped).toEqual([]);
+ expect((await db.attachments.attachment(hash))?.dateUploaded).toBeFalsy();
+ vi.restoreAllMocks();
+});
diff --git a/packages/core/src/database/backup.ts b/packages/core/src/database/backup.ts
index ac613dfc3..6ef2eecca 100644
--- a/packages/core/src/database/backup.ts
+++ b/packages/core/src/database/backup.ts
@@ -613,88 +613,11 @@ export default class Backup {
if (!itemType || itemType === "topic" || itemType === "settings")
continue;
- if (item.type === "attachment" && (item.metadata?.hash || item.hash)) {
- const attachment = await this.db.attachments.attachment(
- item.metadata?.hash || item.hash
- );
- const isSameKey =
- !!attachment &&
- attachment.key.iv === item.key.iv &&
- attachment.key.cipher === item.key.cipher &&
- attachment.key.salt === item.key.salt;
-
- if (
- attachmentsKey &&
- !isSameKey &&
- attachment &&
- (attachment.dateUploaded ||
- (await this.db.fs().exists(attachment.hash)))
- ) {
- // the server won't overwrite an uploaded file & other devices may have
- // it cached, so we must keep the local key and file.
- skippedAttachments.push(attachment.hash);
- } else if (attachmentsKey && !isSameKey) {
- const newKey = await this.db.attachments.encryptKey(
- JSON.parse(
- await this.db.storage().decrypt(attachmentsKey, item.key)
- )
- );
- if (attachment) {
- attachment.key = newKey;
- attachment.iv = item.iv;
- attachment.salt = item.salt;
- attachment.size = item.size;
- attachment.chunkSize = item.chunkSize;
- attachment.alg = item.alg;
- delete attachment.dateUploaded;
- delete attachment.failed;
- } else item.key = newKey;
- delete item.dateUploaded;
- delete item.failed;
- }
-
- if (attachment) {
- if (
- attachmentsKey &&
- isSameKey &&
- !(await this.db.fs().exists(attachment.hash)) &&
- (await this.db.fs().getUploadedFileSize(attachment.hash)) <= 0
- ) {
- delete item.dateUploaded;
- delete item.failed;
- }
-
- const isNewGeneric =
- item.metadata?.type === "application/octet-stream" ||
- item.mimeType === "application/octet-stream";
- const isOldGeneric =
- attachment.mimeType === "application/octet-stream";
- item = {
- ...attachment,
- mimeType:
- // we keep whichever mime type is more specific
- isNewGeneric && !isOldGeneric
- ? attachment.mimeType
- : item.metadata?.type || item.mimeType,
- filename:
- // we keep the filename based on which item's mime type we kept
- isNewGeneric && !isOldGeneric
- ? attachment.filename
- : item.metadata?.filename || item.filename
- };
- for (const noteId of item.noteIds || []) {
- await this.db.relations.add(
- {
- id: noteId,
- type: "note"
- },
- attachment
- );
- }
- } else {
- delete item.dateUploaded;
- delete item.failed;
- }
+ if (item.type === "attachment") {
+ const result = await this.restoreAttachment(item, attachmentsKey);
+ if (result.skipFile) skippedAttachments.push(item.hash);
+ if (!result.attachment) continue;
+ item = result.attachment;
}
const collectionKey: CollectionName =
@@ -732,6 +655,80 @@ export default class Backup {
return skippedAttachments;
}
+ private async restoreAttachment(
+ item: Attachment,
+ attachmentsKey?: SerializedKey
+ ): Promise<{ attachment?: Attachment; skipFile?: boolean }> {
+ const local = await this.db.attachments.attachment(item.hash);
+
+ if (!attachmentsKey) {
+ if (local) return { attachment: mergeAttachmentInfo(local, item) };
+ if (await this.isOwnAttachmentKey(item.key))
+ return { attachment: withoutUploadState(item) };
+ return skipAttachment(item);
+ }
+
+ if (local && isSameKey(local, item)) {
+ const attachment = mergeAttachmentInfo(local, item);
+ return {
+ attachment: (await this.hasFile(local.hash))
+ ? attachment
+ : withoutUploadState(attachment)
+ };
+ }
+
+ const key = await this.reencryptBackupKey(item.key, attachmentsKey);
+ if (!local)
+ return key
+ ? { attachment: withoutUploadState({ ...item, key }) }
+ : skipAttachment(item);
+
+ // the server won't overwrite an uploaded file & other devices may have
+ // it cached, so we must keep the local key and file.
+ if (!key || local.dateUploaded || (await this.db.fs().exists(local.hash)))
+ return { attachment: mergeAttachmentInfo(local, item), skipFile: true };
+
+ return {
+ attachment: withoutUploadState({
+ ...mergeAttachmentInfo(local, item),
+ key,
+ iv: item.iv,
+ salt: item.salt,
+ size: item.size,
+ chunkSize: item.chunkSize,
+ alg: item.alg
+ })
+ };
+ }
+
+ private async reencryptBackupKey(
+ key: Cipher<"base64">,
+ attachmentsKey: SerializedKey
+ ) {
+ const decrypted = await this.db
+ .storage()
+ .decrypt(attachmentsKey, key)
+ .catch(() => undefined);
+ if (!decrypted) return;
+ return await this.db.attachments.encryptKey(JSON.parse(decrypted));
+ }
+
+ private async isOwnAttachmentKey(key: Cipher<"base64">) {
+ const attachmentsKey = await this.db.user.getAttachmentsKey();
+ if (!attachmentsKey) return true;
+ return !!(await this.db
+ .storage()
+ .decrypt(attachmentsKey, key)
+ .catch(() => undefined));
+ }
+
+ private async hasFile(hash: string) {
+ return (
+ (await this.db.fs().exists(hash)) ||
+ (await this.db.fs().getUploadedFileSize(hash)) > 0
+ );
+ }
+
private validate(backup: LegacyBackupFile | BackupFile) {
return (
!!backup.date &&
@@ -759,3 +756,32 @@ export default class Backup {
}
}
}
+
+function isSameKey(a: Attachment, b: Attachment) {
+ return (
+ a.key.iv === b.key.iv &&
+ a.key.cipher === b.key.cipher &&
+ a.key.salt === b.key.salt
+ );
+}
+
+function withoutUploadState(attachment: Attachment): Attachment {
+ const { dateUploaded: _dateUploaded, failed: _failed, ...rest } = attachment;
+ return rest;
+}
+
+function skipAttachment(item: Attachment) {
+ logger.warn("Skipping attachment with an undecryptable key", {
+ hash: item.hash
+ });
+ return { skipFile: true };
+}
+
+function mergeAttachmentInfo(local: Attachment, item: Attachment): Attachment {
+ const keepLocal =
+ item.mimeType === "application/octet-stream" &&
+ local.mimeType !== "application/octet-stream";
+ return keepLocal
+ ? local
+ : { ...local, mimeType: item.mimeType, filename: item.filename };
+}