Skip to content

Build your personal knowledge base with Trilium Notes

12.4 KB
Raw Blame
import { MAX_BLOB_CONTENT_LENGTH } from "@triliumnext/core";
import { entryTooLargeError, type FileStream, type ZipArchive, type ZipArchiveEntryOptions, type ZipEntry, type ZipProvider, type ZipSource } from "@triliumnext/core/src/services/zip_provider.js";
import { ZipArchive as ArchiverZip } from "archiver";
import fs from "fs";
import type { FileHandle } from "fs/promises";
import type { Readable } from "stream";
import * as yauzl from "yauzl";

import { asBuffer } from "./services/binary.js";

class NodejsZipArchive implements ZipArchive {
    readonly #archive: ArchiverZip;
    // Byte sizes of appended entries not yet written out, in FIFO order.
    // archiver processes its queue strictly in order at concurrency 1 and emits
    // exactly one "entry" event per append, so this stays aligned with it.
    readonly #pendingSizes: number[] = [];
    #queuedBytes = 0;
    #capacityWaiter: { resolve: () => void; reject: (err: Error) => void } | null = null;
    // First error emitted by archiver (disk full, zlib failure, …); surfaced through append/waitForCapacity.
    #error: Error | null = null;

    // Cap how much appended-but-unwritten data archiver holds in its internal
    // queue. The export walk appends the whole note tree synchronously; without
    // this the entire archive (multi-GB) is read into memory before draining.
    static readonly #HIGH_WATER_MARK = 64 * 1024 * 1024; // 64 MiB

    constructor() {
        this.#archive = new ArchiverZip({
            // Level 6 (zlib default) is the speed/ratio sweet spot; level 9 costs
            // ~2x the CPU for <2% smaller output. Already-compressed entries skip
            // deflate entirely via the per-entry `store` flag below.
            zlib: { level: 6 }
        });

        // An EventEmitter with no "error" listener throws as an *unhandled* exception when it errors, which
        // crashes the process. Capture archiver's error and re-surface it through append/waitForCapacity/
        // finalize so the export fails cleanly instead.
        this.#archive.on("error", (err: Error) => {
            this.#error = err;
            if (this.#capacityWaiter) {
                const waiter = this.#capacityWaiter;
                this.#capacityWaiter = null;
                waiter.reject(err);
            }
        });

        // Fires as each queued entry finishes being written to the output.
        this.#archive.on("entry", () => {
            this.#queuedBytes -= this.#pendingSizes.shift() ?? 0;
            if (this.#capacityWaiter && this.#queuedBytes < NodejsZipArchive.#HIGH_WATER_MARK) {
                const waiter = this.#capacityWaiter;
                this.#capacityWaiter = null;
                waiter.resolve();
            }
        });
    }

    append(content: string | Uint8Array, options: ZipArchiveEntryOptions) {
        if (this.#error) {
            throw this.#error;
        }
        // A Buffer view over the same memory rather than a copy; archiver only reads, so sharing is safe.
        const payload = typeof content === "string" ? content : asBuffer(content);
        const size = typeof content === "string" ? Buffer.byteLength(content) : content.byteLength;
        this.#pendingSizes.push(size);
        this.#queuedBytes += size;
        // `store` (and `date`) pass straight through: archiver preserves extra
        // entry-data fields and zip-stream switches to the STORE method when set.
        this.#archive.append(payload, options);
    }

    waitForCapacity(): Promise<void> {
        if (this.#error) {
            return Promise.reject(this.#error);
        }
        if (this.#queuedBytes < NodejsZipArchive.#HIGH_WATER_MARK) {
            return Promise.resolve();
        }
        // At most one waiter: the export loop awaits this before the next append.
        return new Promise((resolve, reject) => { this.#capacityWaiter = { resolve, reject }; });
    }

    pipe(destination: unknown) {
        this.#archive.pipe(destination as NodeJS.WritableStream);
    }

    finalize(): Promise<void> {
        return this.#archive.finalize();
    }
}

/**
 * Collects an entry's decompressed bytes, destroying the stream as soon as they pass `maxLength`: a small
 * deflated entry can expand to gigabytes, and the entry's declared size is not trusted.
 */
function streamToBuffer(stream: Readable, fileName: string, maxLength: number): Promise<Buffer> {
    const chunks: Uint8Array[] = [];
    let length = 0;
    stream.on("data", (chunk: Uint8Array) => {
        length += chunk.length;
        if (length > maxLength) {
            stream.destroy(entryTooLargeError(fileName, maxLength));
            return;
        }
        chunks.push(chunk);
    });
    return new Promise((res, rej) => {
        stream.on("end", () => res(Buffer.concat(chunks)));
        stream.on("error", rej);
    });
}

async function openZip(source: ZipSource) {
    const options = { validateEntrySizes: false, decodeStrings: false };
    if (source instanceof Uint8Array) {
        // Wrap the bytes in a Buffer *view* (no copy); fall through to fromBuffer. A `path` source is
        // opened straight from disk so a multi-GB zip is never held in memory (and dodges fs.readFile's
        // ~2 GiB ceiling) — yauzl reads the central directory and each entry on demand from the fd.
        const buffer = asBuffer(source);
        const tail = buffer.subarray(Math.max(0, buffer.length - EOCD_MAX_SIZE));
        return yauzl.fromBufferPromise(buffer.subarray(0, findArchiveEnd(tail, buffer.length)), options);
    }

    if ("blob" in source) {
        throw new Error("A Blob zip source is only readable in the browser.");
    }

    const handle = await fs.promises.open(source.path, "r");
    let size: number;
    let end: number;
    try {
        size = (await handle.stat()).size;
        const tail = Buffer.alloc(Math.min(size, EOCD_MAX_SIZE));
        if (tail.length >= EOCD_MIN_SIZE) {
            await handle.read(tail, 0, tail.length, size - tail.length);
        }
        end = findArchiveEnd(tail, size);
    } catch (e) {
        await handle.close();
        throw e;
    }

    if (end === size) {
        await handle.close();
        return yauzl.openPromise(source.path, options);
    }
    return yauzl.fromRandomAccessReaderPromise(new FileRangeReader(source.path, handle), end, options);
}

export default class NodejsZipProvider implements ZipProvider {
    /** The most bytes `readContent()` decompresses from one entry; nothing larger can be stored as a blob. */
    readonly #maxEntryLength: number;

    constructor(maxEntryLength = MAX_BLOB_CONTENT_LENGTH) {
        this.#maxEntryLength = maxEntryLength;
    }

    async detectFilenameEncoding(source: ZipSource): Promise<string> {
        const zipfile = await openZip(source);

        const samples: Buffer[] = [];
        try {
            for await (const entry of zipfile.eachEntry()) {
                const isUtf8Flagged = !!(entry.generalPurposeBitFlag & 0x800);
                if (!isUtf8Flagged) {
                    samples.push(entry.fileNameRaw);
                }
            }
        } finally {
            // Release the file descriptor for a path source (no-op for an already-closed buffer reader).
            if (zipfile.isOpen) {
                zipfile.close();
            }
        }

        if (samples.length === 0) {
            return "utf-8";
        }

        const combined = Buffer.concat(samples);
        try {
            const chardet = await import("chardet");
            return chardet.default.detect(combined) || "utf-8";
        } catch {
            return "utf-8";
        }
    }

    createZipArchive(): ZipArchive {
        return new NodejsZipArchive();
    }

    createFileStream(filePath: string): FileStream {
        const stream = fs.createWriteStream(filePath);
        return {
            destination: stream,
            waitForFinish: () => new Promise((resolve, reject) => {
                stream.on("finish", resolve);
                stream.on("error", reject);
            })
        };
    }

    async readZipFile(
        source: ZipSource,
        processEntry: (entry: ZipEntry, readContent: () => Promise<Uint8Array>) => Promise<void>,
        filenameEncoding?: string
    ): Promise<void> {
        const zipfile = await openZip(source);

        try {
            for await (const entry of zipfile.eachEntry()) {
                // yauzl with decodeStrings: false leaves file names undecoded.
                // Use the detected encoding for non-UTF-8-flagged entries,
                // falling back to UTF-8.
                const isUtf8Flagged = !!(entry.generalPurposeBitFlag & 0x800);
                const encoding = isUtf8Flagged ? "utf-8" : (filenameEncoding || "utf-8");
                const fileName = decodeBuffer(entry.fileNameRaw, encoding);

                const readContent = async () => {
                    if (entry.uncompressedSize > this.#maxEntryLength) {
                        throw entryTooLargeError(fileName, this.#maxEntryLength);
                    }
                    const readStream = await zipfile.openReadStreamPromise(entry);
                    return await streamToBuffer(readStream, fileName, this.#maxEntryLength);
                };

                await processEntry({ fileName, lastModified: entry.getLastModDate() }, readContent);
            }
        } finally {
            // Release the file descriptor for a path source (no-op for an already-closed buffer reader).
            if (zipfile.isOpen) {
                zipfile.close();
            }
        }
    }
}

/** End-of-central-directory record signature (`PK\x05\x06`). */
const EOCD_SIGNATURE = 0x06054b50;
/** Size of the end-of-central-directory record itself, before its variable-length comment. */
const EOCD_MIN_SIZE = 22;
/** How far back from the end of a zip the record can start: the record plus a maximum-length comment. */
const EOCD_MAX_SIZE = EOCD_MIN_SIZE + 0xffff;

/**
 * Returns the offset one past the archive's last byte — the end-of-central-directory record plus its
 * comment — given the last {@link EOCD_MAX_SIZE} bytes of a zip and its total size, or `totalSize` when
 * no record is found, which leaves yauzl to report the malformed archive itself.
 *
 * Zips are often padded or concatenated with trailing bytes and every mainstream extractor ignores them.
 * yauzl instead takes whatever follows the record to be its comment and rejects the archive with
 * "Invalid comment length", so the size it is given has to stop where the archive does.
 */
function findArchiveEnd(tail: Buffer, totalSize: number): number {
    // Scan backwards like yauzl does, so the record nearest the end wins.
    for (let i = tail.length - EOCD_MIN_SIZE; i >= 0; i--) {
        if (tail.readUInt32LE(i) !== EOCD_SIGNATURE) {
            continue;
        }
        const end = totalSize - tail.length + i + EOCD_MIN_SIZE + tail.readUInt16LE(i + 20);
        return end < totalSize ? end : totalSize;
    }
    return totalSize;
}

/**
 * Reads a zip for {@link yauzl.fromRandomAccessReaderPromise}, which — unlike `yauzl.open()`, whose size
 * comes from fstat — accepts an archive size smaller than the file it is read from.
 */
class FileRangeReader extends yauzl.RandomAccessReader {
    readonly #path: string;
    readonly #handle: FileHandle;

    constructor(path: string, handle: FileHandle) {
        super();
        this.#path = path;
        this.#handle = handle;
    }

    _readStreamForRange(start: number, end: number) {
        // A stream over its own descriptor rather than over #handle: yauzl destroys the stream once the
        // entry has been read, and Node closes the descriptor behind it whatever `autoClose` says. yauzl's
        // range end is exclusive, createReadStream's is inclusive.
        return fs.createReadStream(this.#path, { start, end: end - 1 });
    }

    read(
        buffer: Buffer,
        offset: number,
        length: number,
        position: number,
        callback: (err: Error | null, bytesRead?: number) => void
    ) {
        this.#handle.read(buffer, offset, length, position)
            .then(({ bytesRead }) => callback(null, bytesRead), callback);
    }

    close(callback: (err: Error | null) => void) {
        this.#handle.close().then(() => callback(null), callback);
    }
}

function decodeBuffer(buf: Buffer, encoding: string): string {
    try {
        return new TextDecoder(encoding).decode(buf);
    } catch {
        // Fallback if the encoding label isn't supported by TextDecoder
        return buf.toString("utf-8");
    }
}