Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
fix(import): read zips padded past the central directory
Elian Doran16d407da353d
diff --git a/apps/server/src/zip_provider.spec.ts b/apps/server/src/zip_provider.spec.tsindex 8e7a708c71..2c0157fd1b 100644--- a/apps/server/src/zip_provider.spec.ts+++ b/apps/server/src/zip_provider.spec.ts@@ -163,6 +163,66 @@ describe("NodejsZipProvider", () => {});});+ describe("bytes after the central directory", () => {+ const dir = mkdtempSync(join(tmpdir(), "trilium-zip-provider-trailing-spec-"));+ afterAll(() => rmSync(dir, { recursive: true, force: true }));++ let zipCounter = 0;+ function writeTo(buffer: Buffer): string {+ const path = join(dir, `trailing-${zipCounter++}.zip`);+ writeFileSync(path, buffer);+ return path;+ }++ /** Rewrites the comment length in a zip's end-of-central-directory record. */+ function setCommentLength(buffer: Buffer, length: number) {+ buffer.writeUInt16LE(length, buffer.lastIndexOf("PK\x05\x06", buffer.length, "binary") + 20);+ }++ it("reads a zip padded with trailing bytes, from its bytes and from a path alike", async () => {+ // Padding after the record is what several font distributions and download proxies emit; yauzl+ // reads it as the archive comment and refuses the file unless the size it sees is trimmed.+ // Three entries, so the reader has to stay usable across successive entry reads.+ const padded = Buffer.concat([+ await buildZip([+ { name: "a.txt", content: "alpha" },+ { name: "b.txt", content: "beta" },+ { name: "dir/c.txt", content: "gamma" }+ ]),+ Buffer.alloc(24)+ ]);++ const fromBytes = await readZip(padded);+ expect(Object.keys(fromBytes).sort()).toEqual(["a.txt", "b.txt", "dir/c.txt"]);++ const path = writeTo(padded);+ const fromPath: Record<string, Buffer> = {};+ await provider.readZipFile({ path }, async (entry: ZipEntry, readContent) => {+ fromPath[entry.fileName] = Buffer.from(await readContent());+ });+ expect(Object.keys(fromPath).sort()).toEqual(["a.txt", "b.txt", "dir/c.txt"]);+ expect(fromPath["dir/c.txt"].toString("utf-8")).toBe("gamma");+ expect(await provider.detectFilenameEncoding({ path })).toBe(await provider.detectFilenameEncoding(padded));+ });++ it("keeps a genuine archive comment, and trims only the padding that follows it", async () => {+ const comment = Buffer.from("built by trilium");+ const base = await buildZip([{ name: "a.txt", content: "alpha" }]);+ const commented = Buffer.concat([base, comment]);+ setCommentLength(commented, comment.length);+ const padded = Buffer.concat([commented, Buffer.alloc(8)]);+ setCommentLength(padded, comment.length);++ expect((await readZip(commented))["a.txt"].toString("utf-8")).toBe("alpha");+ expect((await readZip(padded))["a.txt"].toString("utf-8")).toBe("alpha");+ });++ it("leaves a zip with no end-of-central-directory record to yauzl to reject", async () => {+ const truncated = (await buildZip([{ name: "a.txt", content: "alpha" }])).subarray(0, 40);+ await expect(readZip(truncated)).rejects.toBeTruthy();+ });+ });+describe("reading from a path (in place, no full-buffer load)", () => {const dir = mkdtempSync(join(tmpdir(), "trilium-zip-provider-spec-"));afterAll(() => rmSync(dir, { recursive: true, force: true }));diff --git a/apps/server/src/zip_provider.ts b/apps/server/src/zip_provider.tsindex f7c9d269b7..83c7a592dc 100644--- a/apps/server/src/zip_provider.ts+++ b/apps/server/src/zip_provider.ts@@ -1,6 +1,7 @@import type { FileStream, ZipArchive, ZipArchiveEntryOptions, ZipEntry, ZipProvider, ZipSource } from "@triliumnext/core/src/services/zip_provider.js";import { ZipArchive as ArchiverZip } from "archiver";import fs from "fs";+import type { FileHandle } from "fs/promises";import type { Stream } from "stream";import * as yauzl from "yauzl";@@ -102,9 +103,31 @@ async function openZip(source: ZipSource) {// Wrap the bytes in a Buffer *view* (no copy); fall through to fromBuffer. A `path` source is// opened straight from disk so a multi-GB zip is never held in memory (and dodges fs.readFile's// ~2 GiB ceiling) — yauzl reads the central directory and each entry on demand from the fd.- return yauzl.fromBufferPromise(asBuffer(source), options);+ const buffer = asBuffer(source);+ const tail = buffer.subarray(Math.max(0, buffer.length - EOCD_MAX_SIZE));+ return yauzl.fromBufferPromise(buffer.subarray(0, findArchiveEnd(tail, buffer.length)), options);}- return yauzl.openPromise(source.path, options);++ const handle = await fs.promises.open(source.path, "r");+ let size: number;+ let end: number;+ try {+ size = (await handle.stat()).size;+ const tail = Buffer.alloc(Math.min(size, EOCD_MAX_SIZE));+ if (tail.length >= EOCD_MIN_SIZE) {+ await handle.read(tail, 0, tail.length, size - tail.length);+ }+ end = findArchiveEnd(tail, size);+ } catch (e) {+ await handle.close();+ throw e;+ }++ if (end === size) {+ await handle.close();+ return yauzl.openPromise(source.path, options);+ }+ return yauzl.fromRandomAccessReaderPromise(new FileRangeReader(source.path, handle), end, options);}export default class NodejsZipProvider implements ZipProvider {@@ -186,6 +209,71 @@ export default class NodejsZipProvider implements ZipProvider {}}+/** End-of-central-directory record signature (`PK\x05\x06`). */+const EOCD_SIGNATURE = 0x06054b50;+/** Size of the end-of-central-directory record itself, before its variable-length comment. */+const EOCD_MIN_SIZE = 22;+/** How far back from the end of a zip the record can start: the record plus a maximum-length comment. */+const EOCD_MAX_SIZE = EOCD_MIN_SIZE + 0xffff;++/**+ * Returns the offset one past the archive's last byte — the end-of-central-directory record plus its+ * comment — given the last {@link EOCD_MAX_SIZE} bytes of a zip and its total size, or `totalSize` when+ * no record is found, which leaves yauzl to report the malformed archive itself.+ *+ * Zips are often padded or concatenated with trailing bytes and every mainstream extractor ignores them.+ * yauzl instead takes whatever follows the record to be its comment and rejects the archive with+ * "Invalid comment length", so the size it is given has to stop where the archive does.+ */+function findArchiveEnd(tail: Buffer, totalSize: number): number {+ // Scan backwards like yauzl does, so the record nearest the end wins.+ for (let i = tail.length - EOCD_MIN_SIZE; i >= 0; i--) {+ if (tail.readUInt32LE(i) !== EOCD_SIGNATURE) {+ continue;+ }+ const end = totalSize - tail.length + i + EOCD_MIN_SIZE + tail.readUInt16LE(i + 20);+ return end < totalSize ? end : totalSize;+ }+ return totalSize;+}++/**+ * Reads a zip for {@link yauzl.fromRandomAccessReaderPromise}, which — unlike `yauzl.open()`, whose size+ * comes from fstat — accepts an archive size smaller than the file it is read from.+ */+class FileRangeReader extends yauzl.RandomAccessReader {+ readonly #path: string;+ readonly #handle: FileHandle;++ constructor(path: string, handle: FileHandle) {+ super();+ this.#path = path;+ this.#handle = handle;+ }++ _readStreamForRange(start: number, end: number) {+ // A stream over its own descriptor rather than over #handle: yauzl destroys the stream once the+ // entry has been read, and Node closes the descriptor behind it whatever `autoClose` says. yauzl's+ // range end is exclusive, createReadStream's is inclusive.+ return fs.createReadStream(this.#path, { start, end: end - 1 });+ }++ read(+ buffer: Buffer,+ offset: number,+ length: number,+ position: number,+ callback: (err: Error | null, bytesRead?: number) => void+ ) {+ this.#handle.read(buffer, offset, length, position)+ .then(({ bytesRead }) => callback(null, bytesRead), callback);+ }++ close(callback: (err: Error | null) => void) {+ this.#handle.close().then(() => callback(null), callback);+ }+}+function decodeBuffer(buf: Buffer, encoding: string): string {try {return new TextDecoder(encoding).decode(buf);