Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
fix(import): reject oversized ZIP entries while reading
Elian Doranc029630ce351
diff --git a/apps/server/src/zip_provider.spec.ts b/apps/server/src/zip_provider.spec.tsindex 594df964cc..da2470b38a 100644--- a/apps/server/src/zip_provider.spec.ts+++ b/apps/server/src/zip_provider.spec.ts@@ -223,6 +223,41 @@ describe("NodejsZipProvider", () => {});});+ describe("per-entry size limit", () => {+ const limited = new NodejsZipProvider(64 * 1024);+ const tooLarge = /"zeros\.bin" decompresses to more than 65536 bytes/;++ async function readWithLimit(buffer: Buffer) {+ const out: Record<string, number> = {};+ await limited.readZipFile(buffer, async (entry, readContent) => {+ out[entry.fileName] = (await readContent()).length;+ });+ return out;+ }++ /** Rewrites every central directory record's uncompressed size, as a decompression bomb can. */+ function setDeclaredSizes(buffer: Buffer, size: number) {+ let offset = buffer.indexOf("PK\x01\x02", 0, "binary");+ while (offset !== -1) {+ buffer.writeUInt32LE(size, offset + 24);+ offset = buffer.indexOf("PK\x01\x02", offset + 4, "binary");+ }+ }++ it("reads entries up to the limit and rejects one declared past it", async () => {+ expect(await readWithLimit(await buildZip([{ name: "small.bin", content: new Uint8Array(64 * 1024) }])))+ .toEqual({ "small.bin": 64 * 1024 });+ await expect(readWithLimit(await buildZip([{ name: "zeros.bin", content: new Uint8Array(1024 * 1024) }])))+ .rejects.toThrow(tooLarge);+ });++ it("stops inflating an entry whose declared size understates its content", async () => {+ const bomb = await buildZip([{ name: "zeros.bin", content: new Uint8Array(1024 * 1024) }]);+ setDeclaredSizes(bomb, 16);+ await expect(readWithLimit(bomb)).rejects.toThrow(tooLarge);+ });+ });+it("rejects a Blob source, which only the browser provider reads", async () => {await expect(provider.readZipFile({ blob: new Blob([]) }, async () => {})).rejects.toThrow("A Blob zip source is only readable in the browser.");diff --git a/apps/server/src/zip_provider.ts b/apps/server/src/zip_provider.tsindex 92ce05e9c8..4e7bf9b092 100644--- a/apps/server/src/zip_provider.ts+++ b/apps/server/src/zip_provider.ts@@ -1,8 +1,9 @@-import type { FileStream, ZipArchive, ZipArchiveEntryOptions, ZipEntry, ZipProvider, ZipSource } from "@triliumnext/core/src/services/zip_provider.js";+import { MAX_BLOB_CONTENT_LENGTH } from "@triliumnext/core";+import { entryTooLargeError, type FileStream, type ZipArchive, type ZipArchiveEntryOptions, type ZipEntry, type ZipProvider, type ZipSource } from "@triliumnext/core/src/services/zip_provider.js";import { ZipArchive as ArchiverZip } from "archiver";import fs from "fs";import type { FileHandle } from "fs/promises";-import type { Stream } from "stream";+import type { Readable } from "stream";import * as yauzl from "yauzl";import { asBuffer } from "./services/binary.js";@@ -88,9 +89,21 @@ class NodejsZipArchive implements ZipArchive {}}-function streamToBuffer(stream: Stream): Promise<Buffer> {+/**+ * Collects an entry's decompressed bytes, destroying the stream as soon as they pass `maxLength`: a small+ * deflated entry can expand to gigabytes, and the entry's declared size is not trusted.+ */+function streamToBuffer(stream: Readable, fileName: string, maxLength: number): Promise<Buffer> {const chunks: Uint8Array[] = [];- stream.on("data", (chunk: Uint8Array) => chunks.push(chunk));+ let length = 0;+ stream.on("data", (chunk: Uint8Array) => {+ length += chunk.length;+ if (length > maxLength) {+ stream.destroy(entryTooLargeError(fileName, maxLength));+ return;+ }+ chunks.push(chunk);+ });return new Promise((res, rej) => {stream.on("end", () => res(Buffer.concat(chunks)));stream.on("error", rej);@@ -135,6 +148,13 @@ async function openZip(source: ZipSource) {}export default class NodejsZipProvider implements ZipProvider {+ /** The most bytes `readContent()` decompresses from one entry; nothing larger can be stored as a blob. */+ readonly #maxEntryLength: number;++ constructor(maxEntryLength = MAX_BLOB_CONTENT_LENGTH) {+ this.#maxEntryLength = maxEntryLength;+ }+async detectFilenameEncoding(source: ZipSource): Promise<string> {const zipfile = await openZip(source);@@ -198,8 +218,11 @@ export default class NodejsZipProvider implements ZipProvider {const fileName = decodeBuffer(entry.fileNameRaw, encoding);const readContent = async () => {+ if (entry.uncompressedSize > this.#maxEntryLength) {+ throw entryTooLargeError(fileName, this.#maxEntryLength);+ }const readStream = await zipfile.openReadStreamPromise(entry);- return await streamToBuffer(readStream);+ return await streamToBuffer(readStream, fileName, this.#maxEntryLength);};await processEntry({ fileName, lastModified: entry.getLastModDate() }, readContent);diff --git a/apps/standalone/src/lightweight/zip_provider.spec.ts b/apps/standalone/src/lightweight/zip_provider.spec.tsindex 5e2216f8ce..5626fedba4 100644--- a/apps/standalone/src/lightweight/zip_provider.spec.ts+++ b/apps/standalone/src/lightweight/zip_provider.spec.ts@@ -180,6 +180,19 @@ describe("BrowserZipProvider.readZipFile", () => {await expect(provider.readZipFile(garbage, async () => {})).rejects.toBeDefined();});+ it("reads entries up to the size limit and rejects one declared past it", async () => {+ const limited = new BrowserZipProvider(64 * 1024);+ const sizes: Record<string, number> = {};+ const read = (zip: Uint8Array) => limited.readZipFile(zip, async (entry, readContent) => {+ sizes[entry.fileName] = (await readContent()).length;+ });++ await read(zipSync({ "small.bin": new Uint8Array(64 * 1024) }));+ expect(sizes).toEqual({ "small.bin": 64 * 1024 });+ await expect(read(zipSync({ "zeros.bin": new Uint8Array(1024 * 1024) })))+ .rejects.toThrow(/"zeros\.bin" decompresses to more than 65536 bytes/);+ });+it("propagates errors thrown while processing an entry", async () => {const zip = makeZip({ "one.txt": "first" });await expect(diff --git a/apps/standalone/src/lightweight/zip_provider.ts b/apps/standalone/src/lightweight/zip_provider.tsindex b49697385b..3bedb9960b 100644--- a/apps/standalone/src/lightweight/zip_provider.ts+++ b/apps/standalone/src/lightweight/zip_provider.ts@@ -1,4 +1,5 @@-import type { FileStream, ZipArchive, ZipEntry, ZipProvider, ZipSource } from "@triliumnext/core/src/services/zip_provider.js";+import { MAX_BLOB_CONTENT_LENGTH } from "@triliumnext/core";+import { entryTooLargeError, type FileStream, type ZipArchive, type ZipEntry, type ZipProvider, type ZipSource } from "@triliumnext/core/src/services/zip_provider.js";import { inflateSync, strToU8, zipSync } from "fflate";type ZipOutput = {@@ -54,6 +55,13 @@ class BrowserZipArchive implements ZipArchive {}export default class BrowserZipProvider implements ZipProvider {+ /** The most bytes `readContent()` inflates from one entry; nothing larger can be stored as a blob. */+ readonly #maxEntryLength: number;++ constructor(maxEntryLength = MAX_BLOB_CONTENT_LENGTH) {+ this.#maxEntryLength = maxEntryLength;+ }+async detectFilenameEncoding(source: ZipSource): Promise<string> {const entries = await readCentralDirectory(toByteSource(source));const rawSamples = collectRawFilenameSamples(entries);@@ -74,10 +82,15 @@ export default class BrowserZipProvider implements ZipProvider {): Promise<void> {const byteSource = toByteSource(source);for (const record of await readCentralDirectory(byteSource)) {- await processEntry(- { fileName: decodeZipFileName(record, filenameEncoding) },- () => readEntryContent(byteSource, record)- );+ const fileName = decodeZipFileName(record, filenameEncoding);+ await processEntry({ fileName }, async () => {+ // `inflateSync()` writes into a buffer of the declared size and drops anything past it,+ // so checking the declared size bounds the allocation.+ if (record.uncompressedSize > this.#maxEntryLength) {+ throw entryTooLargeError(fileName, this.#maxEntryLength);+ }+ return await readEntryContent(byteSource, record);+ });}}diff --git a/packages/trilium-core/src/services/zip_provider.ts b/packages/trilium-core/src/services/zip_provider.tsindex 46210dd669..0995a73351 100644--- a/packages/trilium-core/src/services/zip_provider.ts+++ b/packages/trilium-core/src/services/zip_provider.ts@@ -57,7 +57,8 @@ export interface ZipProvider {/*** Iterates over every entry in a ZIP, calling `processEntry` for each one. `readContent()` inside the- * callback reads the raw bytes of that entry on demand. If `filenameEncoding` is provided,+ * callback reads the raw bytes of that entry on demand, and rejects with {@link entryTooLargeError}+ * before buffering more than the provider's per-entry limit. If `filenameEncoding` is provided,* non-UTF-8-flagged filenames are decoded using it. Accepts raw bytes or — server/desktop only — a* {@link ZipSource} path, in which case the zip is read straight from disk per entry.*/@@ -73,6 +74,13 @@ export interface ZipProvider {createFileStream(filePath: string): FileStream;}+/** The error `readContent()` rejects with when an entry decompresses to more than `maxLength` bytes. */+export function entryTooLargeError(fileName: string, maxLength: number): Error {+ const mib = Math.round(maxLength / 1024 / 1024);+ return new Error(`"${fileName}" decompresses to more than ${maxLength} bytes (~${mib} MiB), `+ + "the most a note or attachment can hold.");+}+let zipProvider: ZipProvider | null = null;export function initZipProvider(provider: ZipProvider) {