Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

web: fix full backup running out of memory with many attachments (#10612)

Abdullah Atta21deef745beb
diff --git a/apps/web/src/common/index.ts b/apps/web/src/common/index.ts
index 3e9e6e8ca..2f1612fc6 100644
--- a/apps/web/src/common/index.ts
+++ b/apps/web/src/common/index.ts
@@ -37,7 +37,7 @@ import {
import { showToast } from "../utils/toast";
import { readFile, showFilePicker } from "../utils/file-picker";
import { logger } from "../utils/logger";
-import { TaskManager } from "./task-manager";
+import { ProgressReportCallback, TaskManager } from "./task-manager";
import { EVENTS, parseInternalLink } from "@notesnook/core";
import { createWritableStream } from "./desktop-bridge";
import { FeatureDialog, FeatureKeys } from "../dialogs/feature-dialog";
@@ -130,7 +130,6 @@ export async function createBackup(
const ext = "nnbackupz";
const filePath = `${filename}.${ext}`;
- const encoder = new TextEncoder();
const error = await TaskManager.startTask<Error | void>({
type: background ? "status" : "modal",
id: "creating-backup",
@@ -139,44 +138,23 @@ export async function createBackup(
action: async (report) => {
const { createZipStream } = await import("../utils/streams/zip-stream");
const writeStream = await createWritableStream(filePath);
- await new ReadableStream<ZipFile>({
- start() {},
- async pull(controller) {
- for await (const output of db.backup!.export({
- type: "web",
- encrypt: encryptedBackups,
- mode
- })) {
- if (output.type === "file") {
- const file = output;
- report({
- text: background
- ? strings.creatingBackupNamed(file.path)
- : strings.savingFile(file.path)
- });
- controller.enqueue({
- path: file.path,
- data: encoder.encode(file.data)
- });
- } else if (output.type === "attachment") {
- report({
- text: background
- ? strings.creatingBackupNamed(output.hash)
- : strings.savingAttachment(output.hash),
- total: output.total,
- current: output.current
- });
- const handle = await streamablefs.readFile(output.hash);
- if (!handle) continue;
- controller.enqueue({
- path: output.path,
- data: handle.readable
- });
- }
+ const files = backupToZipFiles(
+ { encrypt: encryptedBackups, mode, background },
+ report
+ );
+ await new ReadableStream<ZipFile>(
+ {
+ async pull(controller) {
+ const { value, done } = await files.next();
+ if (done) controller.close();
+ else controller.enqueue(value);
+ },
+ async cancel() {
+ await files.return(undefined);
}
- controller.close();
- }
- })
+ },
+ { highWaterMark: 0 }
+ )
.pipeThrough(createZipStream())
.pipeTo(writeStream);
}
@@ -201,6 +179,50 @@ export async function createBackup(
return false;
}
+async function* backupToZipFiles(
+ options: {
+ encrypt?: boolean;
+ mode?: "full" | "partial";
+ background?: boolean;
+ },
+ report: ProgressReportCallback
+): AsyncGenerator<ZipFile> {
+ const { encrypt, mode, background } = options;
+ const encoder = new TextEncoder();
+ for await (const output of db.backup.export({
+ type: "web",
+ encrypt,
+ mode
+ })) {
+ if (output.type === "file") {
+ const file = output;
+ report({
+ text: background
+ ? strings.creatingBackupNamed(file.path)
+ : strings.savingFile(file.path)
+ });
+ yield {
+ path: file.path,
+ data: encoder.encode(file.data)
+ };
+ } else if (output.type === "attachment") {
+ report({
+ text: background
+ ? strings.creatingBackupNamed(output.hash)
+ : strings.savingAttachment(output.hash),
+ total: output.total,
+ current: output.current
+ });
+ const handle = await streamablefs.readFile(output.hash);
+ if (!handle) continue;
+ yield {
+ path: output.path,
+ data: handle.readable
+ };
+ }
+ }
+}
+
export async function selectBackupFile() {
const [file] = await showFilePicker({
acceptedFileTypes: ".nnbackup,.nnbackupz"
diff --git a/apps/web/src/common/task-manager.ts b/apps/web/src/common/task-manager.ts
index 97d504a8d..e34a8af5c 100644
--- a/apps/web/src/common/task-manager.ts
+++ b/apps/web/src/common/task-manager.ts
@@ -53,7 +53,7 @@ type TaskProgress = {
text: string;
};
-type ProgressReportCallback = (progress: TaskProgress) => void;
+export type ProgressReportCallback = (progress: TaskProgress) => void;
export class TaskManager {
static async startTask<T>(task: TaskDefinition<T>): Promise<T | Error> {
diff --git a/packages/streamable-fs/src/filehandle.ts b/packages/streamable-fs/src/filehandle.ts
index a7a835aab..fff91fdf9 100644
--- a/packages/streamable-fs/src/filehandle.ts
+++ b/packages/streamable-fs/src/filehandle.ts
@@ -31,7 +31,8 @@ export default class FileHandle {
get readable() {
return new ReadableStream(
- new FileStreamSource(this.storage, this.file, this.chunks)
+ new FileStreamSource(this.storage, this.file, this.chunks),
+ { highWaterMark: 0 }
);
}