Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: cap aggregate ChatGPT asset extraction (#12565)
[._.]/ Adam Eivy388e22b0ffd3
diff --git a/server/routes/appleHealth.js b/server/routes/appleHealth.jsindex f1996ef76b..0596cbf6a3 100644--- a/server/routes/appleHealth.js+++ b/server/routes/appleHealth.js@@ -20,7 +20,6 @@ import {getLatestMetricValues} from '../services/appleHealthQuery.js';-const isZip = (file) =>file.mimetype === 'application/zip' ||file.mimetype === 'application/x-zip-compressed' ||diff --git a/server/services/chatgptZipImport.js b/server/services/chatgptZipImport.jsindex af4abcdd74..137cfe833f 100644--- a/server/services/chatgptZipImport.js+++ b/server/services/chatgptZipImport.js@@ -152,7 +152,7 @@ const UNSAFE_MEMBER_WARNING = '⚠️ ChatGPT import: skipped a ZIP member with// the whole file (peak RAM is one chunk). Enforces the per-member size cap.// Resolves with the sniffed extension (or null when the magic bytes aren't// recognized — the caller then falls back to the friendly name / `.bin`).-const streamAssetToFile = (entry, filePath, max) => new Promise((resolve, reject) => {+const streamAssetToFile = (entry, filePath, max, budget) => new Promise((resolve, reject) => {const outPromise = createWriteStreamGuarded(filePath);let out = null;const head = [];@@ -173,7 +173,8 @@ const streamAssetToFile = (entry, filePath, max) => new Promise((resolve, rejectconst sink = new Writable({async write(chunk, _enc, cb) {size += chunk.length;- if (size > max) { cb(new Error(`ZIP member exceeds ${max} byte limit`)); return; }+ if (size > max) { cb(new ServerError(`ZIP member exceeds ${max} byte limit`, { status: 413 })); return; }+ try { budget.addBytes(chunk.length); } catch (err) { cb(err); return; }if (headLen < SNIFF_BYTES) {// Keep only the bytes still needed to reach SNIFF_BYTES, so `head` holds// at most SNIFF_BYTES regardless of how the producer chunks the asset.@@ -216,9 +217,10 @@ const streamAssetToFile = (entry, filePath, max) => new Promise((resolve, reject* Assets are written to `data/brain/imports/assets/<assetId><ext>` and surfaced* at `/data/brain-imports/<assetId><ext>`.*/-export async function extractChatgptZip(zipPath, { assetDir = PATHS.brainImportAssets, maxTotalBytes, maxEntries } = {}) {+export async function extractChatgptZip(zipPath, { assetDir = PATHS.brainImportAssets, maxTotalBytes, maxEntries, maxAssetBytes = 20 * 1024 ** 3 } = {}) {await ensureDir(assetDir);+ const assetBudget = createZipBudget({ maxTotalBytes: maxAssetBytes });const jsonBudget = createZipBudget({ maxTotalBytes, maxEntries });const archiveBudget = createZipBudget({ maxEntries });const convoBuffers = []; // { path, buffer }@@ -271,8 +273,8 @@ export async function extractChatgptZip(zipPath, { assetDir = PATHS.brainImportA} else if (IS_DAT(path)) {// Stream the asset straight to a temp file, holding only its leading// bytes to sniff the extension. Peak RAM is one chunk regardless of- // asset size, so there is no aggregate ceiling and a multi-GB export- // imports without OOM risk. Renamed to its final served name below.+ // asset size. A separate aggregate disk ceiling permits multi-GB+ // exports while bounding expansion. Renamed to its served name below.// Refuse any member whose name isn't a plain filename BEFORE the temp// file is opened. Skip the member rather than failing the whole// import: one malformed member in a multi-GB export shouldn't cost the@@ -286,7 +288,7 @@ export async function extractChatgptZip(zipPath, { assetDir = PATHS.brainImportAconst assetId = datAssetId(path);const tempPath = join(assetDir, `${assetId}.part`);tempPaths.add(tempPath);- inFlight.push(streamAssetToFile(entry, tempPath, MAX_MEMBER_BYTES)+ inFlight.push(streamAssetToFile(entry, tempPath, MAX_MEMBER_BYTES, assetBudget).then((sniffedExt) => { pendingAssets.set(assetId, { datPath: path, tempPath, sniffedExt }); }).catch(onErr));} else {diff --git a/server/services/chatgptZipImport.test.js b/server/services/chatgptZipImport.test.jsindex 7f4ad902e0..16b2c5ddf8 100644--- a/server/services/chatgptZipImport.test.js+++ b/server/services/chatgptZipImport.test.js@@ -64,6 +64,14 @@ describe('chatgptZipImport service', () => {expect(await readdir(assetDir)).toEqual([]);});+ it('bounds aggregate asset expansion and removes every partial file', async () => {+ const zipPath = await writeZip([['file-first.dat', PNG], ['file-second.dat', PNG]]);+ const assetDir = join(TMP, 'assets');+ await expect(extractChatgptZip(zipPath, { assetDir, maxAssetBytes: PNG.length + 1 }))+ .rejects.toMatchObject({ status: 413 });+ expect(await readdir(assetDir)).toEqual([]);+ });+describe('sniffExtension', () => {it('detects png/jpeg/wav/pdf from magic bytes', () => {expect(sniffExtension(PNG)).toBe('.png');