Skip to content

AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.

fix(web): import backups that contain Office attachments

tianma-if12af0581d238
diff --git a/apps/web/src/lib/json-backup.ts b/apps/web/src/lib/json-backup.ts
index ed7bf725..8455a771 100644
--- a/apps/web/src/lib/json-backup.ts
+++ b/apps/web/src/lib/json-backup.ts
@@ -16,14 +16,12 @@ import {
type Resource,
} from "@edgeever/shared";
import {
+ Inflate,
strFromU8,
strToU8,
- Unzip,
- UnzipInflate,
Zip,
ZipDeflate,
ZipPassThrough,
- type UnzipFile,
} from "fflate";
import {
buildMarkdownFrontMatter,
@@ -411,44 +409,191 @@ type ZipEntryConsumer = {
close?: () => void | Promise<void>;
};
-const streamZipEntries = async (
+type ZipDirectoryEntry = {
+ name: string;
+ method: number;
+ compressedSize: number;
+ localHeaderOffset: number;
+};
+
+const ZIP_LOCAL_FILE_HEADER = 0x04034b50;
+const ZIP_CENTRAL_DIRECTORY_HEADER = 0x02014b50;
+const ZIP_END_OF_CENTRAL_DIRECTORY = 0x06054b50;
+const ZIP_LOCAL_HEADER_BYTES = 30;
+const ZIP_CENTRAL_HEADER_BYTES = 46;
+const ZIP64_MAGIC_32 = 0xffffffff;
+const ZIP64_ENTRY_COUNT = 0xffff;
+const MAX_ZIP_COMMENT_BYTES = 65535;
+const MAX_CENTRAL_DIRECTORY_BYTES = 32 * 1024 * 1024;
+const ZIP_ENTRY_CHUNK_BYTES = 1024 * 1024;
+
+const readU16 = (bytes: Uint8Array, offset: number) => bytes[offset] | (bytes[offset + 1] << 8);
+
+const readU32 = (bytes: Uint8Array, offset: number) => (
+ bytes[offset] | (bytes[offset + 1] << 8) | (bytes[offset + 2] << 16) | (bytes[offset + 3] << 24)
+) >>> 0;
+
+const rejectInvalidZip = (error?: unknown): never => {
+ if (error instanceof EdgeEverZipImportError) throw error;
+ throw new EdgeEverZipImportError("invalidZip", error);
+};
+
+const readBlobRange = async (blob: Blob, start: number, length: number) => {
+ if (!Number.isSafeInteger(start) || !Number.isSafeInteger(length) || start < 0 || length < 0 || start + length > blob.size) {
+ rejectInvalidZip();
+ }
+ const bytes = new Uint8Array(await blob.slice(start, start + length).arrayBuffer());
+ if (bytes.byteLength !== length) rejectInvalidZip();
+ return bytes;
+};
+
+// Stored attachments are written with a data descriptor, so their local headers
+// have no size. Office files are ZIP archives and contain the same signature
+// inside those bytes. The central directory already records each entry's offset
+// and compressed size, which keeps nested ZIP contents inside their attachment.
+const readZipCentralDirectory = async (blob: Blob) => {
+ if (blob.size < 22) rejectInvalidZip();
+ const tailLength = Math.min(blob.size, 22 + MAX_ZIP_COMMENT_BYTES);
+ const tail = await readBlobRange(blob, blob.size - tailLength, tailLength);
+ let endOffset = -1;
+ for (let offset = tail.length - 22; offset >= 0; offset -= 1) {
+ if (readU32(tail, offset) !== ZIP_END_OF_CENTRAL_DIRECTORY) continue;
+ const commentLength = readU16(tail, offset + 20);
+ if (offset + 22 + commentLength === tail.length) {
+ endOffset = offset;
+ break;
+ }
+ }
+ if (endOffset < 0) rejectInvalidZip();
+ const entryCount = readU16(tail, endOffset + 10);
+ const directorySize = readU32(tail, endOffset + 12);
+ const directoryOffset = readU32(tail, endOffset + 16);
+ if (directorySize > MAX_CENTRAL_DIRECTORY_BYTES || directoryOffset === ZIP64_MAGIC_32 || directorySize === ZIP64_MAGIC_32) {
+ rejectInvalidZip();
+ }
+ const endOfDirectory = blob.size - tailLength + endOffset;
+ if (directoryOffset > endOfDirectory || directorySize > endOfDirectory - directoryOffset) rejectInvalidZip();
+ const directory = directorySize === 0
+ ? new Uint8Array()
+ : await readBlobRange(blob, directoryOffset, directorySize);
+ const entries: ZipDirectoryEntry[] = [];
+ let offset = 0;
+ while (offset < directory.length) {
+ if (offset + ZIP_CENTRAL_HEADER_BYTES > directory.length || readU32(directory, offset) !== ZIP_CENTRAL_DIRECTORY_HEADER) {
+ rejectInvalidZip();
+ }
+ const flags = readU16(directory, offset + 8);
+ const method = readU16(directory, offset + 10);
+ const compressedSize = readU32(directory, offset + 20);
+ const localHeaderOffset = readU32(directory, offset + 42);
+ const nameLength = readU16(directory, offset + 28);
+ const extraLength = readU16(directory, offset + 30);
+ const commentLength = readU16(directory, offset + 32);
+ const nameStart = offset + ZIP_CENTRAL_HEADER_BYTES;
+ const nextOffset = nameStart + nameLength + extraLength + commentLength;
+ if (
+ nextOffset > directory.length
+ || (method !== 0 && method !== 8)
+ || compressedSize === ZIP64_MAGIC_32
+ || localHeaderOffset === ZIP64_MAGIC_32
+ ) {
+ rejectInvalidZip();
+ }
+ let name = "";
+ try {
+ name = strFromU8(directory.subarray(nameStart, nameStart + nameLength), (flags & 0x800) === 0);
+ } catch (error) {
+ rejectInvalidZip(error);
+ }
+ if (!name) rejectInvalidZip();
+ entries.push({ name, method, compressedSize, localHeaderOffset });
+ offset = nextOffset;
+ }
+ if (entryCount !== ZIP64_ENTRY_COUNT && entries.length !== entryCount) rejectInvalidZip();
+ return entries;
+};
+
+const locateZipEntryData = async (blob: Blob, entry: ZipDirectoryEntry) => {
+ const header = await readBlobRange(blob, entry.localHeaderOffset, ZIP_LOCAL_HEADER_BYTES);
+ if (readU32(header, 0) !== ZIP_LOCAL_FILE_HEADER) rejectInvalidZip();
+ const flags = readU16(header, 6);
+ if (readU16(header, 8) !== entry.method) rejectInvalidZip();
+ if ((flags & 0x08) === 0 && readU32(header, 18) !== entry.compressedSize) rejectInvalidZip();
+ const dataOffset = entry.localHeaderOffset + ZIP_LOCAL_HEADER_BYTES + readU16(header, 26) + readU16(header, 28);
+ if (dataOffset < entry.localHeaderOffset || dataOffset + entry.compressedSize > blob.size) rejectInvalidZip();
+ return dataOffset;
+};
+
+const streamZipEntry = async (
blob: Blob,
- onEntry: (file: UnzipFile) => ZipEntryConsumer,
- onInputProgress?: (completedBytes: number) => void,
+ entry: ZipDirectoryEntry,
+ consumer: ZipEntryConsumer,
+ report: (bytes: number) => void,
) => {
- let processing = Promise.resolve();
- const unzipper = new Unzip((file) => {
- const consumer = onEntry(file);
- file.ondata = (error, data, final) => {
- processing = processing.then(async () => {
- if (error) throw new EdgeEverZipImportError("invalidZip", error);
- if (data?.byteLength) await consumer.write?.(data);
- if (final) await consumer.close?.();
- });
- };
- file.start();
- });
- unzipper.register(UnzipInflate);
+ const dataOffset = await locateZipEntryData(blob, entry);
+ if (entry.method === 0) {
+ let remaining = entry.compressedSize;
+ let offset = dataOffset;
+ while (remaining > 0) {
+ const length = Math.min(ZIP_ENTRY_CHUNK_BYTES, remaining);
+ const chunk = await readBlobRange(blob, offset, length);
+ offset += length;
+ remaining -= length;
+ report(length);
+ await consumer.write?.(chunk);
+ }
+ await consumer.close?.();
+ return;
+ }
- const reader = blob.stream().getReader();
- let completedBytes = 0;
- try {
- while (true) {
- const next = await reader.read();
- completedBytes += next.value?.byteLength ?? 0;
- onInputProgress?.(completedBytes);
- try {
- unzipper.push(next.value ?? new Uint8Array(), next.done);
- } catch (error) {
- if (error instanceof EdgeEverZipImportError) throw error;
- throw new EdgeEverZipImportError("invalidZip", error);
+ let deliveredFinal = false;
+ let delivering = Promise.resolve();
+ const inflate = new Inflate((data, final) => {
+ const copy = data?.byteLength ? new Uint8Array(data) : undefined;
+ delivering = delivering.then(async () => {
+ if (copy) await consumer.write?.(copy);
+ if (final) {
+ deliveredFinal = true;
+ await consumer.close?.();
}
- await processing;
- if (next.done) break;
+ });
+ });
+ let remaining = entry.compressedSize;
+ let offset = dataOffset;
+ do {
+ const length = Math.min(ZIP_ENTRY_CHUNK_BYTES, remaining);
+ const chunk = length === 0 ? new Uint8Array() : await readBlobRange(blob, offset, length);
+ offset += length;
+ remaining -= length;
+ if (length > 0) report(length);
+ try {
+ inflate.push(chunk, remaining === 0);
+ } catch (error) {
+ await delivering.catch((deliveryError: unknown) => rejectInvalidZip(deliveryError));
+ rejectInvalidZip(error);
}
- } finally {
- reader.releaseLock();
+ await delivering;
+ } while (remaining > 0);
+ if (!deliveredFinal) rejectInvalidZip();
+};
+
+const streamZipEntries = async (
+ blob: Blob,
+ onEntry: (file: { name: string }) => ZipEntryConsumer,
+ onInputProgress?: (completedBytes: number) => void,
+) => {
+ const entries = await readZipCentralDirectory(blob);
+ const totalCompressed = entries.reduce((total, entry) => total + entry.compressedSize, 0);
+ let consumed = 0;
+ const report = (bytes: number) => {
+ consumed += bytes;
+ if (!onInputProgress || blob.size <= 0 || totalCompressed <= 0) return;
+ onInputProgress(Math.min(blob.size, Math.round((consumed / totalCompressed) * blob.size)));
+ };
+ for (const entry of entries) {
+ await streamZipEntry(blob, entry, onEntry({ name: entry.name }), report);
}
+ onInputProgress?.(blob.size);
};
const MAX_ZIP_METADATA_ENTRY_BYTES = 32 * 1024 * 1024;
diff --git a/tests/json-backup.test.ts b/tests/json-backup.test.ts
index 075a9e53..869cca27 100644
--- a/tests/json-backup.test.ts
+++ b/tests/json-backup.test.ts
@@ -82,6 +82,11 @@ describe("EdgeEver ZIP", () => {
await expect(parseEdgeEverZip(new Blob(["not a zip"]))).rejects.toEqual(
expect.objectContaining({ code: "invalidZip" })
);
+ const localHeaderOnly = new Uint8Array(64);
+ localHeaderOnly.set([0x50, 0x4b, 0x03, 0x04]);
+ await expect(parseEdgeEverZip(new Blob([localHeaderOnly]))).rejects.toEqual(
+ expect.objectContaining({ code: "invalidZip" })
+ );
await expect(parseEdgeEverZip(new Blob([zipSync({ "notebooks.json": strToU8("[]") })]))).rejects.toEqual(
expect.objectContaining({ code: "missingManifest" })
@@ -181,7 +186,7 @@ describe("EdgeEver ZIP", () => {
expect(backup.prompts).toEqual([]);
});
- test("inspects multi-gigabyte archives through their stream instead of buffering the ZIP", async () => {
+ test("reads large archives by bounded ranges instead of buffering the ZIP", async () => {
const manifest = {
format: "edgeever-zip",
formatVersion: 1,
@@ -192,16 +197,28 @@ describe("EdgeEver ZIP", () => {
includesTrash: false,
counts: { notebooks: 0, memos: 0, revisions: 0, resources: 0 },
};
+ const storedPayload = new Uint8Array(1024 * 1024 + 10);
const actualArchive = new Blob([zipSync({
"manifest.json": strToU8(JSON.stringify(manifest)),
"notebooks.json": strToU8("[]"),
- })]);
+ "pad.bin": storedPayload,
+ }, { level: 0 })]);
+ const rangeLengths: number[] = [];
let wholeArchiveBuffered = false;
const largeStreamingArchive = {
- size: 2 * 1024 * 1024 * 1024,
+ size: actualArchive.size,
type: "application/zip",
- slice: actualArchive.slice.bind(actualArchive),
- stream: actualArchive.stream.bind(actualArchive),
+ slice(start?: number, end?: number) {
+ const from = start ?? 0;
+ const to = end ?? actualArchive.size;
+ const length = to - from;
+ rangeLengths.push(length);
+ if (length > 1024 * 1024) throw new Error(`Buffered ${length} bytes from the archive`);
+ return actualArchive.slice(from, to);
+ },
+ stream() {
+ throw new Error("The whole archive must not be streamed");
+ },
arrayBuffer: async () => {
wholeArchiveBuffered = true;
throw new Error("The whole archive must not be buffered");
@@ -213,6 +230,107 @@ describe("EdgeEver ZIP", () => {
expect(backup.manifest.buildId).toBe("large-stream-test");
expect(backup.archive).toBe(largeStreamingArchive);
expect(wholeArchiveBuffered).toBe(false);
+ expect(rangeLengths).toContain(1024 * 1024);
+ expect(Math.max(...rangeLengths)).toBeLessThanOrEqual(1024 * 1024);
+ });
+
+ test("restores stored Office attachments that contain their own ZIP entries", async () => {
+ const docx = zipSync({
+ "[Content_Types].xml": strToU8("<Types></Types>"),
+ "word/document.xml": strToU8("<document>hello</document>"),
+ });
+ const xlsx = zipSync({
+ "[Content_Types].xml": strToU8("<Types></Types>"),
+ "xl/workbook.xml": strToU8("<workbook/>"),
+ });
+ const officeNotebook = notebook("nb_office", "资料");
+ const officeMemo: MemoDetail = {
+ ...memo,
+ id: "memo_office",
+ notebookId: officeNotebook.id,
+ title: "Office",
+ contentMarkdown: "office",
+ };
+ const officeResource = (
+ id: string,
+ filename: string,
+ mimeType: string,
+ bytes: Uint8Array,
+ ): Resource => ({
+ ...resource,
+ id,
+ memoId: officeMemo.id,
+ kind: "attachment",
+ mimeType,
+ filename,
+ byteSize: bytes.byteLength,
+ sha256: null,
+ width: null,
+ height: null,
+ url: `/api/v1/resources/${id}/blob`,
+ });
+ const docxResource = officeResource(
+ "res_docx",
+ "brief.docx",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ docx,
+ );
+ const xlsxResource = officeResource(
+ "res_xlsx",
+ "sheet.xlsx",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ xlsx,
+ );
+ const files = new Map<string, Uint8Array>([
+ [docxResource.url, docx],
+ [xlsxResource.url, xlsx],
+ ]);
+ const blob = await createEdgeEverZip({
+ listNotebooks: async () => ({ notebooks: [officeNotebook] }),
+ listPrompts: async () => ({ prompts: [] }),
+ getPage: async () => ({
+ memos: [officeMemo],
+ resources: [docxResource, xlsxResource],
+ revisions: [],
+ totalCount: 1,
+ nextOffset: null,
+ }),
+ getResourceResponse: async (url) => new Response(files.get(url)),
+ }, { edgeeverVersion: "1.95.0", buildId: "office-zip" });
+
+ const backup = await parseEdgeEverZip(blob);
+ const restored = new Map<string, Uint8Array>();
+ await restoreEdgeEverZip(backup, {
+ restoreNotebooks: async () => {},
+ restoreMemos: async () => {},
+ restorePrompts: async () => {},
+ createResourceRestoreSink: async (resourceId) => {
+ const chunks: Uint8Array[] = [];
+ return {
+ write: async (chunk) => {
+ chunks.push(new Uint8Array(chunk));
+ },
+ close: async () => {
+ const bytes = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.byteLength, 0));
+ let offset = 0;
+ for (const chunk of chunks) {
+ bytes.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ restored.set(resourceId, bytes);
+ },
+ abort: async () => {},
+ };
+ },
+ });
+
+ expect(backup.manifest.counts.resources).toBe(2);
+ expect(backup.memos[0].resources.map((item) => item.archivePath)).toEqual([
+ "notes/资料/Office.assets/brief.docx",
+ "notes/资料/Office.assets/sheet.xlsx",
+ ]);
+ expect(restored.get("res_docx")).toEqual(docx);
+ expect(restored.get("res_xlsx")).toEqual(xlsx);
});
test("restores notebooks before memos and binary resources", async () => {