AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.
HTML ai-agentcloudflarecloudflare-d1cloudflare-r2cloudflare-workersdockerelectronevernoteevernote-alternativeknowledge-basemcpnote-takingobsidian-alternativeoffline-firstpersonal-knowledge-basepwaself-hostedsqlitetiptapweb-clipper
fix(web): import backups that contain Office attachments
tianma-if12af0581d238
diff --git a/apps/web/src/lib/json-backup.ts b/apps/web/src/lib/json-backup.tsindex ed7bf725..8455a771 100644--- a/apps/web/src/lib/json-backup.ts+++ b/apps/web/src/lib/json-backup.ts@@ -16,14 +16,12 @@ import {type Resource,} from "@edgeever/shared";import {+ Inflate,strFromU8,strToU8,- Unzip,- UnzipInflate,Zip,ZipDeflate,ZipPassThrough,- type UnzipFile,} from "fflate";import {buildMarkdownFrontMatter,@@ -411,44 +409,191 @@ type ZipEntryConsumer = {close?: () => void | Promise<void>;};-const streamZipEntries = async (+type ZipDirectoryEntry = {+ name: string;+ method: number;+ compressedSize: number;+ localHeaderOffset: number;+};++const ZIP_LOCAL_FILE_HEADER = 0x04034b50;+const ZIP_CENTRAL_DIRECTORY_HEADER = 0x02014b50;+const ZIP_END_OF_CENTRAL_DIRECTORY = 0x06054b50;+const ZIP_LOCAL_HEADER_BYTES = 30;+const ZIP_CENTRAL_HEADER_BYTES = 46;+const ZIP64_MAGIC_32 = 0xffffffff;+const ZIP64_ENTRY_COUNT = 0xffff;+const MAX_ZIP_COMMENT_BYTES = 65535;+const MAX_CENTRAL_DIRECTORY_BYTES = 32 * 1024 * 1024;+const ZIP_ENTRY_CHUNK_BYTES = 1024 * 1024;++const readU16 = (bytes: Uint8Array, offset: number) => bytes[offset] | (bytes[offset + 1] << 8);++const readU32 = (bytes: Uint8Array, offset: number) => (+ bytes[offset] | (bytes[offset + 1] << 8) | (bytes[offset + 2] << 16) | (bytes[offset + 3] << 24)+) >>> 0;++const rejectInvalidZip = (error?: unknown): never => {+ if (error instanceof EdgeEverZipImportError) throw error;+ throw new EdgeEverZipImportError("invalidZip", error);+};++const readBlobRange = async (blob: Blob, start: number, length: number) => {+ if (!Number.isSafeInteger(start) || !Number.isSafeInteger(length) || start < 0 || length < 0 || start + length > blob.size) {+ rejectInvalidZip();+ }+ const bytes = new Uint8Array(await blob.slice(start, start + length).arrayBuffer());+ if (bytes.byteLength !== length) rejectInvalidZip();+ return bytes;+};++// Stored attachments are written with a data descriptor, so their local headers+// have no size. Office files are ZIP archives and contain the same signature+// inside those bytes. The central directory already records each entry's offset+// and compressed size, which keeps nested ZIP contents inside their attachment.+const readZipCentralDirectory = async (blob: Blob) => {+ if (blob.size < 22) rejectInvalidZip();+ const tailLength = Math.min(blob.size, 22 + MAX_ZIP_COMMENT_BYTES);+ const tail = await readBlobRange(blob, blob.size - tailLength, tailLength);+ let endOffset = -1;+ for (let offset = tail.length - 22; offset >= 0; offset -= 1) {+ if (readU32(tail, offset) !== ZIP_END_OF_CENTRAL_DIRECTORY) continue;+ const commentLength = readU16(tail, offset + 20);+ if (offset + 22 + commentLength === tail.length) {+ endOffset = offset;+ break;+ }+ }+ if (endOffset < 0) rejectInvalidZip();+ const entryCount = readU16(tail, endOffset + 10);+ const directorySize = readU32(tail, endOffset + 12);+ const directoryOffset = readU32(tail, endOffset + 16);+ if (directorySize > MAX_CENTRAL_DIRECTORY_BYTES || directoryOffset === ZIP64_MAGIC_32 || directorySize === ZIP64_MAGIC_32) {+ rejectInvalidZip();+ }+ const endOfDirectory = blob.size - tailLength + endOffset;+ if (directoryOffset > endOfDirectory || directorySize > endOfDirectory - directoryOffset) rejectInvalidZip();+ const directory = directorySize === 0+ ? new Uint8Array()+ : await readBlobRange(blob, directoryOffset, directorySize);+ const entries: ZipDirectoryEntry[] = [];+ let offset = 0;+ while (offset < directory.length) {+ if (offset + ZIP_CENTRAL_HEADER_BYTES > directory.length || readU32(directory, offset) !== ZIP_CENTRAL_DIRECTORY_HEADER) {+ rejectInvalidZip();+ }+ const flags = readU16(directory, offset + 8);+ const method = readU16(directory, offset + 10);+ const compressedSize = readU32(directory, offset + 20);+ const localHeaderOffset = readU32(directory, offset + 42);+ const nameLength = readU16(directory, offset + 28);+ const extraLength = readU16(directory, offset + 30);+ const commentLength = readU16(directory, offset + 32);+ const nameStart = offset + ZIP_CENTRAL_HEADER_BYTES;+ const nextOffset = nameStart + nameLength + extraLength + commentLength;+ if (+ nextOffset > directory.length+ || (method !== 0 && method !== 8)+ || compressedSize === ZIP64_MAGIC_32+ || localHeaderOffset === ZIP64_MAGIC_32+ ) {+ rejectInvalidZip();+ }+ let name = "";+ try {+ name = strFromU8(directory.subarray(nameStart, nameStart + nameLength), (flags & 0x800) === 0);+ } catch (error) {+ rejectInvalidZip(error);+ }+ if (!name) rejectInvalidZip();+ entries.push({ name, method, compressedSize, localHeaderOffset });+ offset = nextOffset;+ }+ if (entryCount !== ZIP64_ENTRY_COUNT && entries.length !== entryCount) rejectInvalidZip();+ return entries;+};++const locateZipEntryData = async (blob: Blob, entry: ZipDirectoryEntry) => {+ const header = await readBlobRange(blob, entry.localHeaderOffset, ZIP_LOCAL_HEADER_BYTES);+ if (readU32(header, 0) !== ZIP_LOCAL_FILE_HEADER) rejectInvalidZip();+ const flags = readU16(header, 6);+ if (readU16(header, 8) !== entry.method) rejectInvalidZip();+ if ((flags & 0x08) === 0 && readU32(header, 18) !== entry.compressedSize) rejectInvalidZip();+ const dataOffset = entry.localHeaderOffset + ZIP_LOCAL_HEADER_BYTES + readU16(header, 26) + readU16(header, 28);+ if (dataOffset < entry.localHeaderOffset || dataOffset + entry.compressedSize > blob.size) rejectInvalidZip();+ return dataOffset;+};++const streamZipEntry = async (blob: Blob,- onEntry: (file: UnzipFile) => ZipEntryConsumer,- onInputProgress?: (completedBytes: number) => void,+ entry: ZipDirectoryEntry,+ consumer: ZipEntryConsumer,+ report: (bytes: number) => void,) => {- let processing = Promise.resolve();- const unzipper = new Unzip((file) => {- const consumer = onEntry(file);- file.ondata = (error, data, final) => {- processing = processing.then(async () => {- if (error) throw new EdgeEverZipImportError("invalidZip", error);- if (data?.byteLength) await consumer.write?.(data);- if (final) await consumer.close?.();- });- };- file.start();- });- unzipper.register(UnzipInflate);+ const dataOffset = await locateZipEntryData(blob, entry);+ if (entry.method === 0) {+ let remaining = entry.compressedSize;+ let offset = dataOffset;+ while (remaining > 0) {+ const length = Math.min(ZIP_ENTRY_CHUNK_BYTES, remaining);+ const chunk = await readBlobRange(blob, offset, length);+ offset += length;+ remaining -= length;+ report(length);+ await consumer.write?.(chunk);+ }+ await consumer.close?.();+ return;+ }- const reader = blob.stream().getReader();- let completedBytes = 0;- try {- while (true) {- const next = await reader.read();- completedBytes += next.value?.byteLength ?? 0;- onInputProgress?.(completedBytes);- try {- unzipper.push(next.value ?? new Uint8Array(), next.done);- } catch (error) {- if (error instanceof EdgeEverZipImportError) throw error;- throw new EdgeEverZipImportError("invalidZip", error);+ let deliveredFinal = false;+ let delivering = Promise.resolve();+ const inflate = new Inflate((data, final) => {+ const copy = data?.byteLength ? new Uint8Array(data) : undefined;+ delivering = delivering.then(async () => {+ if (copy) await consumer.write?.(copy);+ if (final) {+ deliveredFinal = true;+ await consumer.close?.();}- await processing;- if (next.done) break;+ });+ });+ let remaining = entry.compressedSize;+ let offset = dataOffset;+ do {+ const length = Math.min(ZIP_ENTRY_CHUNK_BYTES, remaining);+ const chunk = length === 0 ? new Uint8Array() : await readBlobRange(blob, offset, length);+ offset += length;+ remaining -= length;+ if (length > 0) report(length);+ try {+ inflate.push(chunk, remaining === 0);+ } catch (error) {+ await delivering.catch((deliveryError: unknown) => rejectInvalidZip(deliveryError));+ rejectInvalidZip(error);}- } finally {- reader.releaseLock();+ await delivering;+ } while (remaining > 0);+ if (!deliveredFinal) rejectInvalidZip();+};++const streamZipEntries = async (+ blob: Blob,+ onEntry: (file: { name: string }) => ZipEntryConsumer,+ onInputProgress?: (completedBytes: number) => void,+) => {+ const entries = await readZipCentralDirectory(blob);+ const totalCompressed = entries.reduce((total, entry) => total + entry.compressedSize, 0);+ let consumed = 0;+ const report = (bytes: number) => {+ consumed += bytes;+ if (!onInputProgress || blob.size <= 0 || totalCompressed <= 0) return;+ onInputProgress(Math.min(blob.size, Math.round((consumed / totalCompressed) * blob.size)));+ };+ for (const entry of entries) {+ await streamZipEntry(blob, entry, onEntry({ name: entry.name }), report);}+ onInputProgress?.(blob.size);};const MAX_ZIP_METADATA_ENTRY_BYTES = 32 * 1024 * 1024;diff --git a/tests/json-backup.test.ts b/tests/json-backup.test.tsindex 075a9e53..869cca27 100644--- a/tests/json-backup.test.ts+++ b/tests/json-backup.test.ts@@ -82,6 +82,11 @@ describe("EdgeEver ZIP", () => {await expect(parseEdgeEverZip(new Blob(["not a zip"]))).rejects.toEqual(expect.objectContaining({ code: "invalidZip" }));+ const localHeaderOnly = new Uint8Array(64);+ localHeaderOnly.set([0x50, 0x4b, 0x03, 0x04]);+ await expect(parseEdgeEverZip(new Blob([localHeaderOnly]))).rejects.toEqual(+ expect.objectContaining({ code: "invalidZip" })+ );await expect(parseEdgeEverZip(new Blob([zipSync({ "notebooks.json": strToU8("[]") })]))).rejects.toEqual(expect.objectContaining({ code: "missingManifest" })@@ -181,7 +186,7 @@ describe("EdgeEver ZIP", () => {expect(backup.prompts).toEqual([]);});- test("inspects multi-gigabyte archives through their stream instead of buffering the ZIP", async () => {+ test("reads large archives by bounded ranges instead of buffering the ZIP", async () => {const manifest = {format: "edgeever-zip",formatVersion: 1,@@ -192,16 +197,28 @@ describe("EdgeEver ZIP", () => {includesTrash: false,counts: { notebooks: 0, memos: 0, revisions: 0, resources: 0 },};+ const storedPayload = new Uint8Array(1024 * 1024 + 10);const actualArchive = new Blob([zipSync({"manifest.json": strToU8(JSON.stringify(manifest)),"notebooks.json": strToU8("[]"),- })]);+ "pad.bin": storedPayload,+ }, { level: 0 })]);+ const rangeLengths: number[] = [];let wholeArchiveBuffered = false;const largeStreamingArchive = {- size: 2 * 1024 * 1024 * 1024,+ size: actualArchive.size,type: "application/zip",- slice: actualArchive.slice.bind(actualArchive),- stream: actualArchive.stream.bind(actualArchive),+ slice(start?: number, end?: number) {+ const from = start ?? 0;+ const to = end ?? actualArchive.size;+ const length = to - from;+ rangeLengths.push(length);+ if (length > 1024 * 1024) throw new Error(`Buffered ${length} bytes from the archive`);+ return actualArchive.slice(from, to);+ },+ stream() {+ throw new Error("The whole archive must not be streamed");+ },arrayBuffer: async () => {wholeArchiveBuffered = true;throw new Error("The whole archive must not be buffered");@@ -213,6 +230,107 @@ describe("EdgeEver ZIP", () => {expect(backup.manifest.buildId).toBe("large-stream-test");expect(backup.archive).toBe(largeStreamingArchive);expect(wholeArchiveBuffered).toBe(false);+ expect(rangeLengths).toContain(1024 * 1024);+ expect(Math.max(...rangeLengths)).toBeLessThanOrEqual(1024 * 1024);+ });++ test("restores stored Office attachments that contain their own ZIP entries", async () => {+ const docx = zipSync({+ "[Content_Types].xml": strToU8("<Types></Types>"),+ "word/document.xml": strToU8("<document>hello</document>"),+ });+ const xlsx = zipSync({+ "[Content_Types].xml": strToU8("<Types></Types>"),+ "xl/workbook.xml": strToU8("<workbook/>"),+ });+ const officeNotebook = notebook("nb_office", "资料");+ const officeMemo: MemoDetail = {+ ...memo,+ id: "memo_office",+ notebookId: officeNotebook.id,+ title: "Office",+ contentMarkdown: "office",+ };+ const officeResource = (+ id: string,+ filename: string,+ mimeType: string,+ bytes: Uint8Array,+ ): Resource => ({+ ...resource,+ id,+ memoId: officeMemo.id,+ kind: "attachment",+ mimeType,+ filename,+ byteSize: bytes.byteLength,+ sha256: null,+ width: null,+ height: null,+ url: `/api/v1/resources/${id}/blob`,+ });+ const docxResource = officeResource(+ "res_docx",+ "brief.docx",+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ docx,+ );+ const xlsxResource = officeResource(+ "res_xlsx",+ "sheet.xlsx",+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",+ xlsx,+ );+ const files = new Map<string, Uint8Array>([+ [docxResource.url, docx],+ [xlsxResource.url, xlsx],+ ]);+ const blob = await createEdgeEverZip({+ listNotebooks: async () => ({ notebooks: [officeNotebook] }),+ listPrompts: async () => ({ prompts: [] }),+ getPage: async () => ({+ memos: [officeMemo],+ resources: [docxResource, xlsxResource],+ revisions: [],+ totalCount: 1,+ nextOffset: null,+ }),+ getResourceResponse: async (url) => new Response(files.get(url)),+ }, { edgeeverVersion: "1.95.0", buildId: "office-zip" });++ const backup = await parseEdgeEverZip(blob);+ const restored = new Map<string, Uint8Array>();+ await restoreEdgeEverZip(backup, {+ restoreNotebooks: async () => {},+ restoreMemos: async () => {},+ restorePrompts: async () => {},+ createResourceRestoreSink: async (resourceId) => {+ const chunks: Uint8Array[] = [];+ return {+ write: async (chunk) => {+ chunks.push(new Uint8Array(chunk));+ },+ close: async () => {+ const bytes = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.byteLength, 0));+ let offset = 0;+ for (const chunk of chunks) {+ bytes.set(chunk, offset);+ offset += chunk.byteLength;+ }+ restored.set(resourceId, bytes);+ },+ abort: async () => {},+ };+ },+ });++ expect(backup.manifest.counts.resources).toBe(2);+ expect(backup.memos[0].resources.map((item) => item.archivePath)).toEqual([+ "notes/资料/Office.assets/brief.docx",+ "notes/资料/Office.assets/sheet.xlsx",+ ]);+ expect(restored.get("res_docx")).toEqual(docx);+ expect(restored.get("res_xlsx")).toEqual(xlsx);});test("restores notebooks before memos and binary resources", async () => {