Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: exclude ignored ZIP members from import record budgets (#12565)

[._.]/ Adam Eivyd2e7625f273a
diff --git a/server/lib/zipStream.js b/server/lib/zipStream.js
index 8001b57f16..8e64de8418 100644
--- a/server/lib/zipStream.js
+++ b/server/lib/zipStream.js
@@ -314,13 +314,14 @@ export function isZipUpload(file) {
* is honored) or a substring shorthand. `onMatch` may be sync or async and
* accumulates into the caller's own closure (an array, a `Map`, a string). Each
* matching member is buffered under `maxBytes` (see `collectZipEntry`), so a
- * pathological member can't exhaust RAM. Leaves the caller with ONLY its domain
- * match/parse logic.
+ * pathological member can't exhaust RAM. maxTotalBytes and maxEntries bound
+ * selected members together; ignored members drain without inflation or retained
+ * tasks, so large exports full of unrelated media do not spend this budget.
+ * Leaves the caller with ONLY its domain match/parse logic.
*/
export function collectZipEntries(zipPath, { match, onMatch, maxBytes = MAX_ZIP_MEMBER_BYTES,
maxTotalBytes = MAX_ZIP_TOTAL_BYTES, maxEntries = MAX_ZIP_ENTRIES } = {}) {
const budget = createZipBudget({ maxTotalBytes, maxEntries });
- const archiveBudget = createZipBudget({ maxEntries });
const test = typeof match === 'function' ? match : (name) => name.includes(match);
const reads = [];
return new Promise((resolve, reject) => {
@@ -339,7 +340,6 @@ export function collectZipEntries(zipPath, { match, onMatch, maxBytes = MAX_ZIP_
src
.pipe(parser)
.on('entry', (entry) => {
- try { archiveBudget.addEntry(); } catch (err) { settle(reject)(err); return; }
if (test(entry.path)) {
const entryPath = entry.path;
reads.push(
diff --git a/server/lib/zipStream.test.js b/server/lib/zipStream.test.js
index 3bd89d7ae3..3b727c2c0b 100644
--- a/server/lib/zipStream.test.js
+++ b/server/lib/zipStream.test.js
@@ -460,6 +460,16 @@ describe('collectZipEntries', () => {
})).rejects.toMatchObject({ status: 413 });
});
+ it('allows more than 10000 ignored members without spending the selected-entry budget', async () => {
+ const entries = Array.from({ length: 10001 }, (_, i) => buildEntry(`skip/${i}`, Buffer.alloc(0)));
+ const zipPath = await writeZip('skipped.zip', Buffer.concat([
+ ...entries, buildEntry('keep.json', Buffer.from('{}')), buildEocd()
+ ]));
+ const found = [];
+ await collectZipEntries(zipPath, { match: '.json', maxEntries: 1, onMatch: buf => found.push(buf.toString()) });
+ expect(found).toEqual(['{}']);
+ });
+
it('resolves with no matches on a truncated archive whose central directory is cut off', async () => {
// A truncated ZIP (upload interrupted): a complete stored entry followed by a
// sliced-off second header and no EOCD. The parser reads the whole first entry
diff --git a/server/routes/appleHealth.js b/server/routes/appleHealth.js
index 0596cbf6a3..5aaf9564d9 100644
--- a/server/routes/appleHealth.js
+++ b/server/routes/appleHealth.js
@@ -123,7 +123,6 @@ router.post('/import/xml', uploadXml, asyncHandler(async (req, res) => {
const xmlPath = join(extractDir, 'export.xml');
let foundXml = false;
const clinicalBudget = createZipBudget();
- const archiveBudget = createZipBudget();
const maxXmlBytes = Math.min(20 * 1024 ** 3, (await fs.stat(filePath)).size * 200);
// Each clinical-record collect is async (entries stream through an inflate
// pipeline), so track them and await all before resolving — otherwise
@@ -159,7 +158,6 @@ router.post('/import/xml', uploadXml, asyncHandler(async (req, res) => {
src
.pipe(parser)
.on('entry', (entry) => {
- try { archiveBudget.addEntry(); } catch (err) { settle(reject)(err); return; }
if (entry.path === 'apple_health_export/export.xml' || entry.path === 'export.xml') {
if (foundXml) { settle(reject)(new ServerError('ZIP contains duplicate export.xml', { status: 400 })); return; }
foundXml = true;
diff --git a/server/routes/appleHealth.test.js b/server/routes/appleHealth.test.js
index 0ddf7ec6a6..eb2582975e 100644
--- a/server/routes/appleHealth.test.js
+++ b/server/routes/appleHealth.test.js
@@ -86,7 +86,7 @@ describe('POST /api/health/import/xml — ZIP extraction temp dir', () => {
// afterEach checks that the partial XML and extraction directory are gone.
});
- it.each([{ maxTotalBytes: 5 }, { maxEntries: 2 }])('rejects aggregate clinical-record limits and cleans extraction: %j', async (limits) => {
+ it.each([{ maxTotalBytes: 5 }, { maxEntries: 1 }])('rejects aggregate clinical-record limits and cleans extraction: %j', async (limits) => {
imports.limits = limits;
const res = await postZip([
{ name: 'export.xml', data: XML },
diff --git a/server/services/chatgptZipImport.js b/server/services/chatgptZipImport.js
index 137cfe833f..47641680b1 100644
--- a/server/services/chatgptZipImport.js
+++ b/server/services/chatgptZipImport.js
@@ -260,8 +260,10 @@ export async function extractChatgptZip(zipPath, { assetDir = PATHS.brainImportA
.on('error', onErr)
.pipe(parser)
.on('entry', (entry) => {
- try { archiveBudget.addEntry(); } catch (err) { onErr(err); return; }
const { path } = entry;
+ if (IS_ASSET_NAME_MAP(path) || IS_CONVO_JSON(path) || (IS_DAT(path) && isSafeDatMember(path))) {
+ try { archiveBudget.addEntry(); } catch (err) { onErr(err); return; }
+ }
if (IS_ASSET_NAME_MAP(path)) {
inFlight.push(collectZipEntry(entry, { maxBytes: MAX_MEMBER_BYTES, budget: jsonBudget })
.then((buf) => { assetNameMap = JSON.parse(buf.toString('utf8')); })