Skip to content

Build your personal knowledge base with Trilium Notes

fix(import): read zips padded past the central directory

Elian Doran16d407da353d
diff --git a/apps/server/src/zip_provider.spec.ts b/apps/server/src/zip_provider.spec.ts
index 8e7a708c71..2c0157fd1b 100644
--- a/apps/server/src/zip_provider.spec.ts
+++ b/apps/server/src/zip_provider.spec.ts
@@ -163,6 +163,66 @@ describe("NodejsZipProvider", () => {
});
});
+ describe("bytes after the central directory", () => {
+ const dir = mkdtempSync(join(tmpdir(), "trilium-zip-provider-trailing-spec-"));
+ afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+ let zipCounter = 0;
+ function writeTo(buffer: Buffer): string {
+ const path = join(dir, `trailing-${zipCounter++}.zip`);
+ writeFileSync(path, buffer);
+ return path;
+ }
+
+ /** Rewrites the comment length in a zip's end-of-central-directory record. */
+ function setCommentLength(buffer: Buffer, length: number) {
+ buffer.writeUInt16LE(length, buffer.lastIndexOf("PK\x05\x06", buffer.length, "binary") + 20);
+ }
+
+ it("reads a zip padded with trailing bytes, from its bytes and from a path alike", async () => {
+ // Padding after the record is what several font distributions and download proxies emit; yauzl
+ // reads it as the archive comment and refuses the file unless the size it sees is trimmed.
+ // Three entries, so the reader has to stay usable across successive entry reads.
+ const padded = Buffer.concat([
+ await buildZip([
+ { name: "a.txt", content: "alpha" },
+ { name: "b.txt", content: "beta" },
+ { name: "dir/c.txt", content: "gamma" }
+ ]),
+ Buffer.alloc(24)
+ ]);
+
+ const fromBytes = await readZip(padded);
+ expect(Object.keys(fromBytes).sort()).toEqual(["a.txt", "b.txt", "dir/c.txt"]);
+
+ const path = writeTo(padded);
+ const fromPath: Record<string, Buffer> = {};
+ await provider.readZipFile({ path }, async (entry: ZipEntry, readContent) => {
+ fromPath[entry.fileName] = Buffer.from(await readContent());
+ });
+ expect(Object.keys(fromPath).sort()).toEqual(["a.txt", "b.txt", "dir/c.txt"]);
+ expect(fromPath["dir/c.txt"].toString("utf-8")).toBe("gamma");
+ expect(await provider.detectFilenameEncoding({ path })).toBe(await provider.detectFilenameEncoding(padded));
+ });
+
+ it("keeps a genuine archive comment, and trims only the padding that follows it", async () => {
+ const comment = Buffer.from("built by trilium");
+ const base = await buildZip([{ name: "a.txt", content: "alpha" }]);
+ const commented = Buffer.concat([base, comment]);
+ setCommentLength(commented, comment.length);
+ const padded = Buffer.concat([commented, Buffer.alloc(8)]);
+ setCommentLength(padded, comment.length);
+
+ expect((await readZip(commented))["a.txt"].toString("utf-8")).toBe("alpha");
+ expect((await readZip(padded))["a.txt"].toString("utf-8")).toBe("alpha");
+ });
+
+ it("leaves a zip with no end-of-central-directory record to yauzl to reject", async () => {
+ const truncated = (await buildZip([{ name: "a.txt", content: "alpha" }])).subarray(0, 40);
+ await expect(readZip(truncated)).rejects.toBeTruthy();
+ });
+ });
+
describe("reading from a path (in place, no full-buffer load)", () => {
const dir = mkdtempSync(join(tmpdir(), "trilium-zip-provider-spec-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
diff --git a/apps/server/src/zip_provider.ts b/apps/server/src/zip_provider.ts
index f7c9d269b7..83c7a592dc 100644
--- a/apps/server/src/zip_provider.ts
+++ b/apps/server/src/zip_provider.ts
@@ -1,6 +1,7 @@
import type { FileStream, ZipArchive, ZipArchiveEntryOptions, ZipEntry, ZipProvider, ZipSource } from "@triliumnext/core/src/services/zip_provider.js";
import { ZipArchive as ArchiverZip } from "archiver";
import fs from "fs";
+import type { FileHandle } from "fs/promises";
import type { Stream } from "stream";
import * as yauzl from "yauzl";
@@ -102,9 +103,31 @@ async function openZip(source: ZipSource) {
// Wrap the bytes in a Buffer *view* (no copy); fall through to fromBuffer. A `path` source is
// opened straight from disk so a multi-GB zip is never held in memory (and dodges fs.readFile's
// ~2 GiB ceiling) — yauzl reads the central directory and each entry on demand from the fd.
- return yauzl.fromBufferPromise(asBuffer(source), options);
+ const buffer = asBuffer(source);
+ const tail = buffer.subarray(Math.max(0, buffer.length - EOCD_MAX_SIZE));
+ return yauzl.fromBufferPromise(buffer.subarray(0, findArchiveEnd(tail, buffer.length)), options);
}
- return yauzl.openPromise(source.path, options);
+
+ const handle = await fs.promises.open(source.path, "r");
+ let size: number;
+ let end: number;
+ try {
+ size = (await handle.stat()).size;
+ const tail = Buffer.alloc(Math.min(size, EOCD_MAX_SIZE));
+ if (tail.length >= EOCD_MIN_SIZE) {
+ await handle.read(tail, 0, tail.length, size - tail.length);
+ }
+ end = findArchiveEnd(tail, size);
+ } catch (e) {
+ await handle.close();
+ throw e;
+ }
+
+ if (end === size) {
+ await handle.close();
+ return yauzl.openPromise(source.path, options);
+ }
+ return yauzl.fromRandomAccessReaderPromise(new FileRangeReader(source.path, handle), end, options);
}
export default class NodejsZipProvider implements ZipProvider {
@@ -186,6 +209,71 @@ export default class NodejsZipProvider implements ZipProvider {
}
}
+/** End-of-central-directory record signature (`PK\x05\x06`). */
+const EOCD_SIGNATURE = 0x06054b50;
+/** Size of the end-of-central-directory record itself, before its variable-length comment. */
+const EOCD_MIN_SIZE = 22;
+/** How far back from the end of a zip the record can start: the record plus a maximum-length comment. */
+const EOCD_MAX_SIZE = EOCD_MIN_SIZE + 0xffff;
+
+/**
+ * Returns the offset one past the archive's last byte — the end-of-central-directory record plus its
+ * comment — given the last {@link EOCD_MAX_SIZE} bytes of a zip and its total size, or `totalSize` when
+ * no record is found, which leaves yauzl to report the malformed archive itself.
+ *
+ * Zips are often padded or concatenated with trailing bytes and every mainstream extractor ignores them.
+ * yauzl instead takes whatever follows the record to be its comment and rejects the archive with
+ * "Invalid comment length", so the size it is given has to stop where the archive does.
+ */
+function findArchiveEnd(tail: Buffer, totalSize: number): number {
+ // Scan backwards like yauzl does, so the record nearest the end wins.
+ for (let i = tail.length - EOCD_MIN_SIZE; i >= 0; i--) {
+ if (tail.readUInt32LE(i) !== EOCD_SIGNATURE) {
+ continue;
+ }
+ const end = totalSize - tail.length + i + EOCD_MIN_SIZE + tail.readUInt16LE(i + 20);
+ return end < totalSize ? end : totalSize;
+ }
+ return totalSize;
+}
+
+/**
+ * Reads a zip for {@link yauzl.fromRandomAccessReaderPromise}, which — unlike `yauzl.open()`, whose size
+ * comes from fstat — accepts an archive size smaller than the file it is read from.
+ */
+class FileRangeReader extends yauzl.RandomAccessReader {
+ readonly #path: string;
+ readonly #handle: FileHandle;
+
+ constructor(path: string, handle: FileHandle) {
+ super();
+ this.#path = path;
+ this.#handle = handle;
+ }
+
+ _readStreamForRange(start: number, end: number) {
+ // A stream over its own descriptor rather than over #handle: yauzl destroys the stream once the
+ // entry has been read, and Node closes the descriptor behind it whatever `autoClose` says. yauzl's
+ // range end is exclusive, createReadStream's is inclusive.
+ return fs.createReadStream(this.#path, { start, end: end - 1 });
+ }
+
+ read(
+ buffer: Buffer,
+ offset: number,
+ length: number,
+ position: number,
+ callback: (err: Error | null, bytesRead?: number) => void
+ ) {
+ this.#handle.read(buffer, offset, length, position)
+ .then(({ bytesRead }) => callback(null, bytesRead), callback);
+ }
+
+ close(callback: (err: Error | null) => void) {
+ this.#handle.close().then(() => callback(null), callback);
+ }
+}
+
function decodeBuffer(buf: Buffer, encoding: string): string {
try {
return new TextDecoder(encoding).decode(buf);