Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: bound ZIP import expansion and respect slow consumers (#12565)
[._.]/ Adam Eivy6b2c497c9645
diff --git a/server/lib/zipStream.js b/server/lib/zipStream.jsindex a0d7ca60b0..8001b57f16 100644--- a/server/lib/zipStream.js+++ b/server/lib/zipStream.js@@ -13,8 +13,8 @@import { createInflateRaw } from 'zlib';import { createReadStream } from 'fs';-import { EventEmitter } from 'events';-import { PassThrough, Writable } from 'stream';+import { PassThrough, Writable, Transform, pipeline } from 'stream';+import { ServerError } from './errorHandler.js';// Local file header signature: PK\x03\x04const LOCAL_SIG = 0x04034b50;@@ -28,19 +28,12 @@ const EOCD_SIG = 0x06054b50;const LOCAL_HEADER_SIZE = 30; // fixed portion (before variable-length name + extra)export function parseZip() {- const emitter = new EventEmitter();let buf = Buffer.alloc(0);- let closed = false;const sink = new Writable({write(chunk, _, cb) {buf = buf.length ? Buffer.concat([buf, chunk]) : chunk;- processBuffer();- cb();- },- final(cb) {- if (!closed) { closed = true; emitter.emit('close'); }- cb();+ processBuffer().then(() => cb(), cb);},destroy(err, cb) {// On abort (a consumer calls `parser.destroy()` to stop ingestion mid-ZIP),@@ -57,15 +50,13 @@ export function parseZip() {}});- sink.on('error', err => emitter.emit('error', err));-let state = 'HEADER'; // HEADER | ENTRY | SKIP_CENTRALlet currentEntry = null;let entryBytesRemaining = 0; // compressed size remaining; only used when !dataDescriptor- function processBuffer() {- while (true) {- if (state === 'SKIP_CENTRAL') return; // done with entries+ async function processBuffer() {+ while (!sink.destroyed) {+ if (state === 'SKIP_CENTRAL') { buf = Buffer.alloc(0); return; } // done with entriesif (state === 'HEADER') {if (buf.length < 4) return;@@ -73,6 +64,7 @@ export function parseZip() {if (sig === CENTRAL_SIG || sig === EOCD_SIG) {state = 'SKIP_CENTRAL';+ buf = Buffer.alloc(0);return;}@@ -102,24 +94,23 @@ export function parseZip() {const passThrough = new PassThrough();let piped = false;+ let complete;+ const completion = new Promise(resolve => { complete = resolve; });const entry = {path: name,pipe(dest) {piped = true;- if (method === 8) {- const inflate = createInflateRaw();- // Forward inflate failures (corrupt deflate stream) to dest so a- // consumer awaiting completion rejects/errors instead of hanging.- inflate.on('error', (err) => dest.destroy(err));- passThrough.pipe(inflate).pipe(dest);- } else {- passThrough.pipe(dest);- }+ const streams = method === 8 ? [passThrough, createInflateRaw(), dest] : [passThrough, dest];+ pipeline(...streams, (err) => {+ complete();+ if (err && !sink.destroyed) sink.destroy(err);+ });return dest;},autodrain() {piped = true;+ passThrough.once('end', complete);passThrough.resume(); // discard}};@@ -129,15 +120,15 @@ export function parseZip() {if (!piped) entry.autodrain();});- emitter.emit('entry', entry);-if (dataDescriptor) {- currentEntry = { passThrough, method, name, dataDescriptor: true };+ currentEntry = { passThrough, completion, method, name, dataDescriptor: true };} else {- currentEntry = { passThrough, method, name, dataDescriptor: false };+ currentEntry = { passThrough, completion, method, name, dataDescriptor: false };entryBytesRemaining = compSize;}state = 'ENTRY';+ sink.emit('entry', entry);+ if (sink.destroyed) return;}if (state === 'ENTRY') {@@ -161,27 +152,31 @@ export function parseZip() {// Flush safe bytes (keep last 16 for boundary overlap)const safe = buf.length - 16;if (safe > 0) {- currentEntry.passThrough.write(buf.slice(0, safe));+ await writeEntry(buf.slice(0, safe));buf = buf.slice(safe);}return;}- currentEntry.passThrough.write(buf.slice(0, found));+ // A descriptor may itself span input chunks.+ if (buf.length < found + descLen) return;+ await writeEntry(buf.slice(0, found));currentEntry.passThrough.end();+ await currentEntry.completion;buf = buf.slice(found + descLen);currentEntry = null;state = 'HEADER';} else {- if (buf.length === 0) return;+ if (buf.length === 0 && entryBytesRemaining > 0) return;const take = Math.min(entryBytesRemaining, buf.length);- currentEntry.passThrough.write(buf.slice(0, take));+ await writeEntry(buf.slice(0, take));buf = buf.slice(take);entryBytesRemaining -= take;if (entryBytesRemaining === 0) {currentEntry.passThrough.end();+ await currentEntry.completion;currentEntry = null;state = 'HEADER';} else {@@ -192,9 +187,21 @@ export function parseZip() {}}- // Delegate EventEmitter interface to emitter so .pipe() syntax works- for (const m of ['on', 'once', 'off', 'emit', 'addListener', 'removeListener', 'removeAllListeners', 'listenerCount']) {- sink[m] = emitter[m].bind(emitter);+ // Hold the input write callback until downstream capacity returns. Waiting+ // for each entry's consumer at EOF also bounds simultaneous inflate pipelines.+ async function writeEntry(chunk) {+ const stream = currentEntry.passThrough;+ if (!chunk.length || stream.destroyed || stream.writableEnded) return;+ if (stream.write(chunk)) return;+ await new Promise(resolve => {+ const done = () => {+ stream.off('drain', done);+ stream.off('close', done);+ resolve();+ };+ stream.once('drain', done);+ stream.once('close', done);+ });}return sink;@@ -205,6 +212,40 @@ export function parseZip() {// archive members (chatgptZipImport, appleHealth clinical records) share it.export const MAX_ZIP_MEMBER_BYTES = 100 * 1024 * 1024;+// Buffered JSON expands further when parsed; keep the archive budget modest.+export const MAX_ZIP_TOTAL_BYTES = 256 * 1024 * 1024;+export const MAX_ZIP_ENTRIES = 10000;++/** Share one budget across every selected member of an import. */+export function createZipBudget({ maxTotalBytes = MAX_ZIP_TOTAL_BYTES, maxEntries = MAX_ZIP_ENTRIES } = {}) {+ let bytes = 0;+ let entries = 0;+ return {+ addEntry() {+ if (++entries > maxEntries) throw new ServerError(`ZIP exceeds ${maxEntries} entry limit`, { status: 413 });+ },+ addBytes(count) {+ bytes += count;+ if (bytes > maxTotalBytes) throw new ServerError(`ZIP exceeds ${maxTotalBytes} total byte limit`, { status: 413 });+ }+ };+}++/** Count decompressed bytes before forwarding them to disk or another consumer. */+export function createZipByteLimit(maxBytes, budget) {+ let size = 0;+ return new Transform({+ transform(chunk, _enc, cb) {+ try {+ size += chunk.length;+ if (size > maxBytes) throw new ServerError(`ZIP member exceeds ${maxBytes} byte limit`, { status: 413 });+ budget?.addBytes(chunk.length);+ cb(null, chunk);+ } catch (err) { cb(err); }+ }+ });+}+/*** Collect a single `parseZip()` entry's decompressed bytes into one Buffer.*@@ -219,21 +260,23 @@ export const MAX_ZIP_MEMBER_BYTES = 100 * 1024 * 1024;* next tick if nothing attaches a pipe/drain), and gather the returned promises* so the parser's `close` handler can `Promise.all` them before settling.*/-export function collectZipEntry(entry, maxBytes = MAX_ZIP_MEMBER_BYTES) {+export function collectZipEntry(entry, options = {}) {+ const { maxBytes = MAX_ZIP_MEMBER_BYTES, budget = createZipBudget(options) } =+ typeof options === 'number' ? { maxBytes: options } : options;return new Promise((resolve, reject) => {const chunks = [];- let size = 0;+ budget.addEntry();const sink = new Writable({write(chunk, _enc, cb) {- size += chunk.length;- if (size > maxBytes) { cb(new Error(`ZIP member exceeds ${maxBytes} byte limit`)); return; }chunks.push(chunk);cb();}});sink.on('finish', () => resolve(Buffer.concat(chunks)));sink.on('error', reject);- entry.pipe(sink);+ const counter = createZipByteLimit(maxBytes, budget);+ pipeline(counter, sink, err => { if (err) reject(err); });+ entry.pipe(counter);});}@@ -274,7 +317,10 @@ export function isZipUpload(file) {* pathological member can't exhaust RAM. Leaves the caller with ONLY its domain* match/parse logic.*/-export function collectZipEntries(zipPath, { match, onMatch, maxBytes = MAX_ZIP_MEMBER_BYTES } = {}) {+export function collectZipEntries(zipPath, { match, onMatch, maxBytes = MAX_ZIP_MEMBER_BYTES,+ maxTotalBytes = MAX_ZIP_TOTAL_BYTES, maxEntries = MAX_ZIP_ENTRIES } = {}) {+ const budget = createZipBudget({ maxTotalBytes, maxEntries });+ const archiveBudget = createZipBudget({ maxEntries });const test = typeof match === 'function' ? match : (name) => name.includes(match);const reads = [];return new Promise((resolve, reject) => {@@ -293,10 +339,11 @@ export function collectZipEntries(zipPath, { match, onMatch, maxBytes = MAX_ZIP_src.pipe(parser).on('entry', (entry) => {+ try { archiveBudget.addEntry(); } catch (err) { settle(reject)(err); return; }if (test(entry.path)) {const entryPath = entry.path;reads.push(- collectZipEntry(entry, maxBytes)+ collectZipEntry(entry, { maxBytes, budget }).then((buf) => onMatch(buf, entryPath)).catch(settle(reject)),);diff --git a/server/lib/zipStream.test.js b/server/lib/zipStream.test.jsindex 2f6289c980..3bd89d7ae3 100644--- a/server/lib/zipStream.test.js+++ b/server/lib/zipStream.test.js@@ -54,6 +54,51 @@ function collectEntries(zipBuf) {}describe('parseZip', () => {+ it.each([false, true])('bounds upstream reads with a blocked consumer (descriptor=%s)', async (descriptor) => {+ const zip = buildEntry('slow.bin', Buffer.alloc(2 * 1024 * 1024, 65));+ let archive = zip;+ if (descriptor) {+ zip.writeUInt16LE(8, 6);+ const footer = Buffer.alloc(16);+ footer.writeUInt32LE(0x08074b50);+ archive = Buffer.concat([zip, footer, buildEocd()]);+ }+ let produced = 0;+ let release;+ let blocked;+ const firstWrite = new Promise(resolve => { blocked = resolve; });+ const destination = new Writable({+ highWaterMark: 1024,+ write(_chunk, _enc, cb) {+ if (!release) { release = cb; blocked(); } else cb();+ }+ });+ const parser = parseZip();+ parser.on('entry', entry => entry.pipe(destination));+ const done = new Promise((resolve, reject) => {+ parser.on('close', resolve);+ parser.on('error', reject);+ });+ const source = new Readable({+ highWaterMark: 16384,+ read() {+ if (produced === archive.length) { this.push(null); return; }+ const end = Math.min(produced + 16384, archive.length);+ const chunk = archive.subarray(produced, end);+ produced = end;+ this.push(chunk);+ }+ });+ source.pipe(parser);+ await firstWrite;+ // Flush scheduled stream work without relying on elapsed-time sleeps.+ await new Promise(resolve => setImmediate(resolve));+ expect(produced).toBeLessThan(512 * 1024);+ release();+ await done;+ expect(produced).toBe(archive.length);+ });+it('parses a single stored entry', async () => {const data = Buffer.from('hello world');const zip = Buffer.concat([buildEntry('hello.txt', data), buildEocd()]);@@ -402,6 +447,19 @@ describe('collectZipEntries', () => {).rejects.toThrow();});+ it.each([+ { maxTotalBytes: 5 },+ { maxEntries: 1 }+ ])('rejects many individually valid members over the archive budget: %j', async (limits) => {+ const zipPath = await writeZip('aggregate.zip', Buffer.concat([+ buildEntry('a.json', Buffer.from('{} ')),+ buildEntry('b.json', Buffer.from('{} ')), buildEocd()+ ]));+ await expect(collectZipEntries(zipPath, {+ match: '.json', onMatch: () => {}, ...limits+ })).rejects.toMatchObject({ status: 413 });+ });+it('resolves with no matches on a truncated archive whose central directory is cut off', async () => {// A truncated ZIP (upload interrupted): a complete stored entry followed by a// sliced-off second header and no EOCD. The parser reads the whole first entrydiff --git a/server/routes/appleHealth.js b/server/routes/appleHealth.jsindex 014ee89387..f1996ef76b 100644--- a/server/routes/appleHealth.js+++ b/server/routes/appleHealth.js@@ -1,9 +1,10 @@+import { pipeline } from 'stream';import { Router } from 'express';import { tmpdir } from 'os';import { join } from 'path';import { createReadStream, createWriteStream, promises as fs } from 'fs';import { uploadSingle } from '../lib/multipart.js';-import { parseZip, collectZipEntry } from '../lib/zipStream.js';+import { parseZip, collectZipEntry, createZipBudget, createZipByteLimit } from '../lib/zipStream.js';import { asyncHandler, ServerError } from '../lib/errorHandler.js';import { validateRequest } from '../lib/validation.js';import { healthIngestSchema, appleHealthLatestMetricsQuerySchema } from '../lib/appleHealthValidation.js';@@ -19,9 +20,6 @@ import {getLatestMetricValues} from '../services/appleHealthQuery.js';-// Polling interval and timeout for XML write-stream completion (ms)-const XML_WRITE_POLL_INTERVAL_MS = 50;-const XML_WRITE_TIMEOUT_MS = 5000;const isZip = (file) =>file.mimetype === 'application/zip' ||@@ -125,7 +123,9 @@ router.post('/import/xml', uploadXml, asyncHandler(async (req, res) => {extractDir = await fs.mkdtemp(join(tmpdir(), 'portos-apple-health-'));const xmlPath = join(extractDir, 'export.xml');let foundXml = false;- let xmlWriteFinished = false;+ const clinicalBudget = createZipBudget();+ const archiveBudget = createZipBudget();+ const maxXmlBytes = Math.min(20 * 1024 ** 3, (await fs.stat(filePath)).size * 200);// Each clinical-record collect is async (entries stream through an inflate// pipeline), so track them and await all before resolving — otherwise// 'close' can fire and drop records whose buffers haven't finished.@@ -160,19 +160,23 @@ router.post('/import/xml', uploadXml, asyncHandler(async (req, res) => {src.pipe(parser).on('entry', (entry) => {+ try { archiveBudget.addEntry(); } catch (err) { settle(reject)(err); return; }if (entry.path === 'apple_health_export/export.xml' || entry.path === 'export.xml') {+ if (foundXml) { settle(reject)(new ServerError('ZIP contains duplicate export.xml', { status: 400 })); return; }foundXml = true;xmlWriteStream = createWriteStream(xmlPath);- entry.pipe(xmlWriteStream)- .on('finish', () => { xmlWriteFinished = true; })- .on('error', settle(reject));+ const counter = createZipByteLimit(maxXmlBytes);+ clinicalReads.push(new Promise((res, rej) => {+ pipeline(counter, xmlWriteStream, err => err ? rej(err) : res());+ }).catch(settle(reject)));+ entry.pipe(counter);} else if (entry.path.includes('clinical_records/') && entry.path.endsWith('.json')) {// Buffer clinical record JSON files (~1-5KB each). parseZip() entries// aren't EventEmitters (no entry.on('data'/'end') — that throws), so// pipe into a collecting Writable via the shared helper (which applies// MAX_ZIP_MEMBER_BYTES) and track the read so 'close' can await it.clinicalReads.push(- collectZipEntry(entry)+ collectZipEntry(entry, { budget: clinicalBudget }).then((buf) => { clinicalJsons.push(buf.toString('utf-8')); }).catch(settle(reject)));@@ -182,13 +186,8 @@ router.post('/import/xml', uploadXml, asyncHandler(async (req, res) => {}).on('close', () => {if (!foundXml) return settle(reject)(new ServerError('ZIP does not contain export.xml', { status: 400, code: 'BAD_REQUEST' }));- // Resolve once the XML write stream has flushed AND every clinical- // record collect has finished. XML completion is signaled via the- // polled flag; the clinical collects are awaited as promises.- const finalize = () => Promise.all(clinicalReads).then(settle(resolve));- if (xmlWriteFinished) return finalize();- const check = setInterval(() => { if (xmlWriteFinished) { clearInterval(check); clearTimeout(timer); finalize(); } }, XML_WRITE_POLL_INTERVAL_MS);- const timer = setTimeout(() => { clearInterval(check); finalize(); }, XML_WRITE_TIMEOUT_MS);+ // parseZip waits for each consumer to flush before closing.+ Promise.all(clinicalReads).then(settle(resolve), settle(reject));}).on('error', settle(reject));}).catch(async (err) => {diff --git a/server/routes/appleHealth.test.js b/server/routes/appleHealth.test.jsindex 07cbe56c0c..0ddf7ec6a6 100644--- a/server/routes/appleHealth.test.js+++ b/server/routes/appleHealth.test.js@@ -15,7 +15,11 @@ import { request } from '../lib/testHelper.js';import { errorMiddleware } from '../lib/errorHandler.js';import { createZip } from '../lib/zipWriter.js';-const imports = vi.hoisted(() => ({ paths: [], impl: null }));+const imports = vi.hoisted(() => ({ paths: [], impl: null, limits: undefined }));+vi.mock('../lib/zipStream.js', async (importOriginal) => {+ const actual = await importOriginal();+ return { ...actual, createZipBudget: options => actual.createZipBudget(imports.limits ?? options) };+});vi.mock('../services/appleHealthXml.js', () => ({importAppleHealthXml: vi.fn(async (filePath) => {imports.paths.push(filePath);@@ -63,6 +67,7 @@ describe('POST /api/health/import/xml — ZIP extraction temp dir', () => {beforeEach(async () => {imports.paths = [];imports.impl = null;+ imports.limits = undefined;preexisting = new Set(await extractDirs());});@@ -71,6 +76,36 @@ describe('POST /api/health/import/xml — ZIP extraction temp dir', () => {expect(leaked).toEqual([]);});+ it('rejects a high-ratio XML member and removes the partial extraction', async () => {+ const zip = createZip([{ name: 'export.xml', data: 'A'.repeat(1024 * 1024) }], { compress: true });+ const res = await request(buildApp()).post('/api/health/import/xml')+ .set('content-type', 'multipart/form-data; boundary=----portoshealth')+ .send(multipart('----portoshealth', zip));+ expect(res.status).toBe(413);+ expect(imports.paths).toHaveLength(0);+ // afterEach checks that the partial XML and extraction directory are gone.+ });++ it.each([{ maxTotalBytes: 5 }, { maxEntries: 2 }])('rejects aggregate clinical-record limits and cleans extraction: %j', async (limits) => {+ imports.limits = limits;+ const res = await postZip([+ { name: 'export.xml', data: XML },+ { name: 'clinical_records/a.json', data: '{} ' },+ { name: 'clinical_records/b.json', data: '{} ' }+ ]);+ expect(res.status).toBe(413);+ expect(imports.paths).toHaveLength(0);+ });++ it('rejects duplicate XML destinations without leaving an extracted file', async () => {+ const res = await postZip([+ { name: 'export.xml', data: XML },+ { name: 'apple_health_export/export.xml', data: XML }+ ]);+ expect(res.status).toBe(400);+ expect(imports.paths).toHaveLength(0);+ });+it('gives back-to-back imports distinct extraction directories and cleans both up', async () => {const entries = [{ name: 'apple_health_export/export.xml', data: XML }];diff --git a/server/services/chatgptZipImport.js b/server/services/chatgptZipImport.jsindex e8d1fffd59..af4abcdd74 100644--- a/server/services/chatgptZipImport.js+++ b/server/services/chatgptZipImport.js@@ -35,7 +35,7 @@ import { join } from 'path';import { Writable } from 'stream';import { PATHS, getMimeType, ensureDir, createWriteStreamGuarded, unlinkGuarded } from '../lib/fileUtils.js';import { isTopLevelEntryName } from '../lib/pathSafety.js';-import { parseZip, collectZipEntry, MAX_ZIP_MEMBER_BYTES } from '../lib/zipStream.js';+import { parseZip, collectZipEntry, createZipBudget, MAX_ZIP_MEMBER_BYTES } from '../lib/zipStream.js';import { parseExport, importConversations, assetPointerId } from './chatgptImport.js';import { ServerError } from '../lib/errorHandler.js';@@ -216,9 +216,11 @@ const streamAssetToFile = (entry, filePath, max) => new Promise((resolve, reject* Assets are written to `data/brain/imports/assets/<assetId><ext>` and surfaced* at `/data/brain-imports/<assetId><ext>`.*/-export async function extractChatgptZip(zipPath, { assetDir = PATHS.brainImportAssets } = {}) {+export async function extractChatgptZip(zipPath, { assetDir = PATHS.brainImportAssets, maxTotalBytes, maxEntries } = {}) {await ensureDir(assetDir);+ const jsonBudget = createZipBudget({ maxTotalBytes, maxEntries });+ const archiveBudget = createZipBudget({ maxEntries });const convoBuffers = []; // { path, buffer }let assetNameMap = {};// assetId -> { datPath, tempPath, sniffedExt } recorded as each `.dat` streams@@ -256,13 +258,14 @@ export async function extractChatgptZip(zipPath, { assetDir = PATHS.brainImportA.on('error', onErr).pipe(parser).on('entry', (entry) => {+ try { archiveBudget.addEntry(); } catch (err) { onErr(err); return; }const { path } = entry;if (IS_ASSET_NAME_MAP(path)) {- inFlight.push(collectZipEntry(entry, MAX_MEMBER_BYTES)+ inFlight.push(collectZipEntry(entry, { maxBytes: MAX_MEMBER_BYTES, budget: jsonBudget }).then((buf) => { assetNameMap = JSON.parse(buf.toString('utf8')); }).catch(onErr));} else if (IS_CONVO_JSON(path)) {- inFlight.push(collectZipEntry(entry, MAX_MEMBER_BYTES)+ inFlight.push(collectZipEntry(entry, { maxBytes: MAX_MEMBER_BYTES, budget: jsonBudget }).then((buf) => { convoBuffers.push({ path, buffer: buf }); }).catch(onErr));} else if (IS_DAT(path)) {diff --git a/server/services/chatgptZipImport.test.js b/server/services/chatgptZipImport.test.jsindex 4487425fa1..7f4ad902e0 100644--- a/server/services/chatgptZipImport.test.js+++ b/server/services/chatgptZipImport.test.js@@ -55,6 +55,15 @@ describe('chatgptZipImport service', () => {return p;};+ it.each([{ maxTotalBytes: 3 }, { maxEntries: 1 }])('rejects archive budget overflow and removes temporary assets: %j', async (limits) => {+ const zipPath = await writeZip([+ ['file-asset.dat', PNG], ['conversations.json', '[]'], ['conversations-1.json', '[]']+ ]);+ const assetDir = join(TMP, 'assets');+ await expect(extractChatgptZip(zipPath, { assetDir, ...limits })).rejects.toMatchObject({ status: 413 });+ expect(await readdir(assetDir)).toEqual([]);+ });+describe('sniffExtension', () => {it('detects png/jpeg/wav/pdf from magic bytes', () => {expect(sniffExtension(PNG)).toBe('.png');