Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
import { MAX_BLOB_CONTENT_LENGTH } from "@triliumnext/core";
import { entryTooLargeError, type FileStream, type ZipArchive, type ZipArchiveEntryOptions, type ZipEntry, type ZipProvider, type ZipSource } from "@triliumnext/core/src/services/zip_provider.js";
import { ZipArchive as ArchiverZip } from "archiver";
import fs from "fs";
import type { FileHandle } from "fs/promises";
import type { Readable } from "stream";
import * as yauzl from "yauzl";
import { asBuffer } from "./services/binary.js";
class NodejsZipArchive implements ZipArchive {
readonly #archive: ArchiverZip;
// Byte sizes of appended entries not yet written out, in FIFO order.
// archiver processes its queue strictly in order at concurrency 1 and emits
// exactly one "entry" event per append, so this stays aligned with it.
readonly #pendingSizes: number[] = [];
#queuedBytes = 0;
#capacityWaiter: { resolve: () => void; reject: (err: Error) => void } | null = null;
// First error emitted by archiver (disk full, zlib failure, …); surfaced through append/waitForCapacity.
#error: Error | null = null;
// Cap how much appended-but-unwritten data archiver holds in its internal
// queue. The export walk appends the whole note tree synchronously; without
// this the entire archive (multi-GB) is read into memory before draining.
static readonly #HIGH_WATER_MARK = 64 * 1024 * 1024; // 64 MiB
constructor() {
this.#archive = new ArchiverZip({
// Level 6 (zlib default) is the speed/ratio sweet spot; level 9 costs
// ~2x the CPU for <2% smaller output. Already-compressed entries skip
// deflate entirely via the per-entry `store` flag below.
zlib: { level: 6 }
});
// An EventEmitter with no "error" listener throws as an *unhandled* exception when it errors, which
// crashes the process. Capture archiver's error and re-surface it through append/waitForCapacity/
// finalize so the export fails cleanly instead.
this.#archive.on("error", (err: Error) => {
this.#error = err;
if (this.#capacityWaiter) {
const waiter = this.#capacityWaiter;
this.#capacityWaiter = null;
waiter.reject(err);
}
});
// Fires as each queued entry finishes being written to the output.
this.#archive.on("entry", () => {
this.#queuedBytes -= this.#pendingSizes.shift() ?? 0;
if (this.#capacityWaiter && this.#queuedBytes < NodejsZipArchive.#HIGH_WATER_MARK) {
const waiter = this.#capacityWaiter;
this.#capacityWaiter = null;
waiter.resolve();
}
});
}
append(content: string | Uint8Array, options: ZipArchiveEntryOptions) {
if (this.#error) {
throw this.#error;
}
// A Buffer view over the same memory rather than a copy; archiver only reads, so sharing is safe.
const payload = typeof content === "string" ? content : asBuffer(content);
const size = typeof content === "string" ? Buffer.byteLength(content) : content.byteLength;
this.#pendingSizes.push(size);
this.#queuedBytes += size;
// `store` (and `date`) pass straight through: archiver preserves extra
// entry-data fields and zip-stream switches to the STORE method when set.
this.#archive.append(payload, options);
}
waitForCapacity(): Promise<void> {
if (this.#error) {
return Promise.reject(this.#error);
}
if (this.#queuedBytes < NodejsZipArchive.#HIGH_WATER_MARK) {
return Promise.resolve();
}
// At most one waiter: the export loop awaits this before the next append.
return new Promise((resolve, reject) => { this.#capacityWaiter = { resolve, reject }; });
}
pipe(destination: unknown) {
this.#archive.pipe(destination as NodeJS.WritableStream);
}
finalize(): Promise<void> {
return this.#archive.finalize();
}
}
/**
* Collects an entry's decompressed bytes, destroying the stream as soon as they pass `maxLength`: a small
* deflated entry can expand to gigabytes, and the entry's declared size is not trusted.
*/
function streamToBuffer(stream: Readable, fileName: string, maxLength: number): Promise<Buffer> {
const chunks: Uint8Array[] = [];
let length = 0;
stream.on("data", (chunk: Uint8Array) => {
length += chunk.length;
if (length > maxLength) {
stream.destroy(entryTooLargeError(fileName, maxLength));
return;
}
chunks.push(chunk);
});
return new Promise((res, rej) => {
stream.on("end", () => res(Buffer.concat(chunks)));
stream.on("error", rej);
});
}
async function openZip(source: ZipSource) {
const options = { validateEntrySizes: false, decodeStrings: false };
if (source instanceof Uint8Array) {
// Wrap the bytes in a Buffer *view* (no copy); fall through to fromBuffer. A `path` source is
// opened straight from disk so a multi-GB zip is never held in memory (and dodges fs.readFile's
// ~2 GiB ceiling) — yauzl reads the central directory and each entry on demand from the fd.
const buffer = asBuffer(source);
const tail = buffer.subarray(Math.max(0, buffer.length - EOCD_MAX_SIZE));
return yauzl.fromBufferPromise(buffer.subarray(0, findArchiveEnd(tail, buffer.length)), options);
}
if ("blob" in source) {
throw new Error("A Blob zip source is only readable in the browser.");
}
const handle = await fs.promises.open(source.path, "r");
let size: number;
let end: number;
try {
size = (await handle.stat()).size;
const tail = Buffer.alloc(Math.min(size, EOCD_MAX_SIZE));
if (tail.length >= EOCD_MIN_SIZE) {
await handle.read(tail, 0, tail.length, size - tail.length);
}
end = findArchiveEnd(tail, size);
} catch (e) {
await handle.close();
throw e;
}
if (end === size) {
await handle.close();
return yauzl.openPromise(source.path, options);
}
return yauzl.fromRandomAccessReaderPromise(new FileRangeReader(source.path, handle), end, options);
}
export default class NodejsZipProvider implements ZipProvider {
/** The most bytes `readContent()` decompresses from one entry; nothing larger can be stored as a blob. */
readonly #maxEntryLength: number;
constructor(maxEntryLength = MAX_BLOB_CONTENT_LENGTH) {
this.#maxEntryLength = maxEntryLength;
}
async detectFilenameEncoding(source: ZipSource): Promise<string> {
const zipfile = await openZip(source);
const samples: Buffer[] = [];
try {
for await (const entry of zipfile.eachEntry()) {
const isUtf8Flagged = !!(entry.generalPurposeBitFlag & 0x800);
if (!isUtf8Flagged) {
samples.push(entry.fileNameRaw);
}
}
} finally {
// Release the file descriptor for a path source (no-op for an already-closed buffer reader).
if (zipfile.isOpen) {
zipfile.close();
}
}
if (samples.length === 0) {
return "utf-8";
}
const combined = Buffer.concat(samples);
try {
const chardet = await import("chardet");
return chardet.default.detect(combined) || "utf-8";
} catch {
return "utf-8";
}
}
createZipArchive(): ZipArchive {
return new NodejsZipArchive();
}
createFileStream(filePath: string): FileStream {
const stream = fs.createWriteStream(filePath);
return {
destination: stream,
waitForFinish: () => new Promise((resolve, reject) => {
stream.on("finish", resolve);
stream.on("error", reject);
})
};
}
async readZipFile(
source: ZipSource,
processEntry: (entry: ZipEntry, readContent: () => Promise<Uint8Array>) => Promise<void>,
filenameEncoding?: string
): Promise<void> {
const zipfile = await openZip(source);
try {
for await (const entry of zipfile.eachEntry()) {
// yauzl with decodeStrings: false leaves file names undecoded.
// Use the detected encoding for non-UTF-8-flagged entries,
// falling back to UTF-8.
const isUtf8Flagged = !!(entry.generalPurposeBitFlag & 0x800);
const encoding = isUtf8Flagged ? "utf-8" : (filenameEncoding || "utf-8");
const fileName = decodeBuffer(entry.fileNameRaw, encoding);
const readContent = async () => {
if (entry.uncompressedSize > this.#maxEntryLength) {
throw entryTooLargeError(fileName, this.#maxEntryLength);
}
const readStream = await zipfile.openReadStreamPromise(entry);
return await streamToBuffer(readStream, fileName, this.#maxEntryLength);
};
await processEntry({ fileName, lastModified: entry.getLastModDate() }, readContent);
}
} finally {
// Release the file descriptor for a path source (no-op for an already-closed buffer reader).
if (zipfile.isOpen) {
zipfile.close();
}
}
}
}
/** End-of-central-directory record signature (`PK\x05\x06`). */
const EOCD_SIGNATURE = 0x06054b50;
/** Size of the end-of-central-directory record itself, before its variable-length comment. */
const EOCD_MIN_SIZE = 22;
/** How far back from the end of a zip the record can start: the record plus a maximum-length comment. */
const EOCD_MAX_SIZE = EOCD_MIN_SIZE + 0xffff;
/**
* Returns the offset one past the archive's last byte — the end-of-central-directory record plus its
* comment — given the last {@link EOCD_MAX_SIZE} bytes of a zip and its total size, or `totalSize` when
* no record is found, which leaves yauzl to report the malformed archive itself.
*
* Zips are often padded or concatenated with trailing bytes and every mainstream extractor ignores them.
* yauzl instead takes whatever follows the record to be its comment and rejects the archive with
* "Invalid comment length", so the size it is given has to stop where the archive does.
*/
function findArchiveEnd(tail: Buffer, totalSize: number): number {
// Scan backwards like yauzl does, so the record nearest the end wins.
for (let i = tail.length - EOCD_MIN_SIZE; i >= 0; i--) {
if (tail.readUInt32LE(i) !== EOCD_SIGNATURE) {
continue;
}
const end = totalSize - tail.length + i + EOCD_MIN_SIZE + tail.readUInt16LE(i + 20);
return end < totalSize ? end : totalSize;
}
return totalSize;
}
/**
* Reads a zip for {@link yauzl.fromRandomAccessReaderPromise}, which — unlike `yauzl.open()`, whose size
* comes from fstat — accepts an archive size smaller than the file it is read from.
*/
class FileRangeReader extends yauzl.RandomAccessReader {
readonly #path: string;
readonly #handle: FileHandle;
constructor(path: string, handle: FileHandle) {
super();
this.#path = path;
this.#handle = handle;
}
_readStreamForRange(start: number, end: number) {
// A stream over its own descriptor rather than over #handle: yauzl destroys the stream once the
// entry has been read, and Node closes the descriptor behind it whatever `autoClose` says. yauzl's
// range end is exclusive, createReadStream's is inclusive.
return fs.createReadStream(this.#path, { start, end: end - 1 });
}
read(
buffer: Buffer,
offset: number,
length: number,
position: number,
callback: (err: Error | null, bytesRead?: number) => void
) {
this.#handle.read(buffer, offset, length, position)
.then(({ bytesRead }) => callback(null, bytesRead), callback);
}
close(callback: (err: Error | null) => void) {
this.#handle.close().then(() => callback(null), callback);
}
}
function decodeBuffer(buf: Buffer, encoding: string): string {
try {
return new TextDecoder(encoding).decode(buf);
} catch {
// Fallback if the encoding label isn't supported by TextDecoder
return buf.toString("utf-8");
}
}