Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Collect garbage before the render guard judges the browser heap (#1061)

Nguyen Xuan An431e01e4adb3
diff --git a/changes/1061.md b/changes/1061.md
new file mode 100644
index 00000000..26cd66e7
--- /dev/null
+++ b/changes/1061.md
@@ -0,0 +1 @@
+- Fix render jobs of a fractured wall tripping the browser heap guard: the guard collects garbage before it judges the heap, so only live memory counts.
diff --git a/docs/agent-render-jobs.md b/docs/agent-render-jobs.md
index a7c5d736..133b2e94 100644
--- a/docs/agent-render-jobs.md
+++ b/docs/agent-render-jobs.md
@@ -324,6 +324,8 @@ The controller never removes a dead process's lock automatically.
After a crash, confirm that no process owns the dedicated private store before manually removing `store.lock`.
This conservative repair step prevents two recovery processes from deleting a newly acquired lock.
+A render or review job fails when its Node RSS passes 512 MiB or its browser heap passes 256 MiB. After each frame the browser heap is read, and when it is over the limit it is garbage collected and read again, so only memory the page still holds counts. A physics bake leaves tens of MiB of garbage after its first frame; a real leak survives the collection and still stops the job.
+
| Budget | Default |
| --- | --- |
| Retained records | 32 |
diff --git a/scripts/agent-memory-guard.mjs b/scripts/agent-memory-guard.mjs
new file mode 100644
index 00000000..4b7f7a72
--- /dev/null
+++ b/scripts/agent-memory-guard.mjs
@@ -0,0 +1,23 @@
+// The memory guard of render and review jobs: a job fails when the Node process or the live browser heap stays above its limit.
+
+export const NODE_RSS_LIMIT_BYTES = 512 * 1024 ** 2;
+export const BROWSER_HEAP_LIMIT_BYTES = 256 * 1024 ** 2;
+
+/**
+ * The browser heap in use, counting only memory the page still holds.
+ * `Runtime.getHeapUsage` also counts garbage that was not collected yet, and a scene with a physics bake
+ * leaves tens of MiB of it behind on its first frame. Above the limit the heap is collected and read again,
+ * so only memory that survives a collection (a real leak or a scene that really needs more) trips the guard.
+ */
+export async function sampleBrowserHeap(session, limitBytes = BROWSER_HEAP_LIMIT_BYTES) {
+ let usage = await session.send('Runtime.getHeapUsage');
+ if (usage.usedSize > limitBytes) {
+ await session.send('HeapProfiler.collectGarbage');
+ usage = await session.send('Runtime.getHeapUsage');
+ }
+ return usage.usedSize;
+}
+
+export function exceedsMemoryGuard({ nodeBytes, browserHeapBytes }) {
+ return nodeBytes > NODE_RSS_LIMIT_BYTES || browserHeapBytes > BROWSER_HEAP_LIMIT_BYTES;
+}
diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjs
index 0669abd8..08c45bfb 100644
--- a/scripts/agent-runtime.mjs
+++ b/scripts/agent-runtime.mjs
@@ -1,6 +1,7 @@
import { createServer } from 'node:http';
import { createHash, randomUUID } from 'node:crypto';
import { existsSync } from 'node:fs';
+import { sampleBrowserHeap, exceedsMemoryGuard } from './agent-memory-guard.mjs';
import { lstat, mkdir, readFile, rm, unlink, writeFile } from 'node:fs/promises';
import { dirname, isAbsolute, join } from 'node:path';
import { createRequire } from 'node:module';
@@ -720,9 +721,9 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
}
}
metrics.peakNodeBytes = Math.max(metrics.peakNodeBytes, process.memoryUsage().rss);
- const heap = await session.send('Runtime.getHeapUsage');
- metrics.peakBrowserHeap = Math.max(metrics.peakBrowserHeap, heap.usedSize);
- if (metrics.peakNodeBytes > 512 * 1024 ** 2 || metrics.peakBrowserHeap > 256 * 1024 ** 2) {
+ const heapBytes = await sampleBrowserHeap(session);
+ metrics.peakBrowserHeap = Math.max(metrics.peakBrowserHeap, heapBytes);
+ if (exceedsMemoryGuard({ nodeBytes: metrics.peakNodeBytes, browserHeapBytes: metrics.peakBrowserHeap })) {
throw new Error('Render job exceeded its Node RSS or sampled browser heap guard.');
}
await progress({ frames: frame + 1, maximumPendingFrames: 1 });
@@ -1031,8 +1032,8 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
width: settings.width, height: settings.height, columns: settings.columns, labels: settings.labels });
if (backend === 'cpu' && !/swiftshader/i.test(rendered.graphics)) throw new Error('CPU review did not select SwiftShader.');
peakNodeBytes = Math.max(peakNodeBytes, process.memoryUsage().rss);
- peakBrowserHeap = Math.max(peakBrowserHeap, (await session.send('Runtime.getHeapUsage')).usedSize);
- if (peakNodeBytes > 512 * 1024 ** 2 || peakBrowserHeap > 256 * 1024 ** 2) {
+ peakBrowserHeap = Math.max(peakBrowserHeap, await sampleBrowserHeap(session));
+ if (exceedsMemoryGuard({ nodeBytes: peakNodeBytes, browserHeapBytes: peakBrowserHeap })) {
throw new Error('Review exceeded its Node RSS or sampled browser heap guard.');
}
shots[index] = { ...rendered, ...(group.branch === undefined ? {} : { branch: group.branch, branchStale: branchRecipe.stale }) };
diff --git a/tests/agent-memory-guard.test.js b/tests/agent-memory-guard.test.js
new file mode 100644
index 00000000..87023465
--- /dev/null
+++ b/tests/agent-memory-guard.test.js
@@ -0,0 +1,50 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { BROWSER_HEAP_LIMIT_BYTES, NODE_RSS_LIMIT_BYTES, exceedsMemoryGuard, sampleBrowserHeap } from '../scripts/agent-memory-guard.mjs';
+
+const MEBIBYTE = 1024 ** 2;
+
+/** A browser heap whose uncollected garbage disappears on `HeapProfiler.collectGarbage`, as a page after a physics bake does. */
+function fakeSession({ garbageBytes, liveBytes }) {
+ const calls = [];
+ let collected = false;
+ return {
+ calls,
+ async send(method) {
+ calls.push(method);
+ if (method === 'HeapProfiler.collectGarbage') {
+ collected = true;
+ return {};
+ }
+ return { usedSize: collected ? liveBytes : liveBytes + garbageBytes };
+ },
+ };
+}
+
+test('uncollected garbage from the first fractured-wall frame does not trip the heap guard', async () => {
+ // Measured on a 28-cell fractured wall render: 311 MiB sampled on frame 0, 11 MiB once collected.
+ const session = fakeSession({ garbageBytes: 300 * MEBIBYTE, liveBytes: 11 * MEBIBYTE });
+ const heapBytes = await sampleBrowserHeap(session);
+ assert.equal(heapBytes, 11 * MEBIBYTE);
+ assert.deepEqual(session.calls, ['Runtime.getHeapUsage', 'HeapProfiler.collectGarbage', 'Runtime.getHeapUsage']);
+ assert.equal(exceedsMemoryGuard({ nodeBytes: 200 * MEBIBYTE, browserHeapBytes: heapBytes }), false);
+});
+
+test('a heap under the limit is read once and never collected', async () => {
+ const session = fakeSession({ garbageBytes: 0, liveBytes: 90 * MEBIBYTE });
+ assert.equal(await sampleBrowserHeap(session), 90 * MEBIBYTE);
+ assert.deepEqual(session.calls, ['Runtime.getHeapUsage']);
+});
+
+test('memory that survives a collection still trips the guard', async () => {
+ const session = fakeSession({ garbageBytes: 50 * MEBIBYTE, liveBytes: BROWSER_HEAP_LIMIT_BYTES + MEBIBYTE });
+ const heapBytes = await sampleBrowserHeap(session);
+ assert.equal(heapBytes, BROWSER_HEAP_LIMIT_BYTES + MEBIBYTE);
+ assert.equal(exceedsMemoryGuard({ nodeBytes: 0, browserHeapBytes: heapBytes }), true);
+});
+
+test('the Node limit is unchanged and still enforced', () => {
+ assert.equal(NODE_RSS_LIMIT_BYTES, 512 * MEBIBYTE);
+ assert.equal(exceedsMemoryGuard({ nodeBytes: NODE_RSS_LIMIT_BYTES + 1, browserHeapBytes: 0 }), true);
+ assert.equal(exceedsMemoryGuard({ nodeBytes: NODE_RSS_LIMIT_BYTES, browserHeapBytes: BROWSER_HEAP_LIMIT_BYTES }), false);
+});
diff --git a/tests/agent-runtime.test.js b/tests/agent-runtime.test.js
index 9445052d..3cc67486 100644
--- a/tests/agent-runtime.test.js
+++ b/tests/agent-runtime.test.js
@@ -2,6 +2,7 @@ import test, { after } from 'node:test';
import assert from 'node:assert/strict';
import { createAgentRuntime } from '../scripts/agent-runtime.mjs';
import { isPreviewHost } from '../scripts/preview-host.mjs';
+import { sampleBrowserHeap, exceedsMemoryGuard } from '../scripts/agent-memory-guard.mjs';
import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { isAbsolute, join } from 'node:path';
@@ -126,6 +127,7 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = fa
recipeAssetNeeds, gameAssetIds, sequenceFrameIds, catalogLookReferences, BUNDLED_ENVIRONMENT_NAMES, BUNDLED_CUBE_LOOK_NAMES, TEXTURE_SEQUENCE_CAPABILITIES, AGENT_ASSET_LIMITS, assertCombinedAssetBytes,
audioInputArguments, createAudioLibrary, measureStreamSeconds, filmTimelineDuration, filmTimelineShotStarts, sampleFilmTimeline, normalizeReview, REVIEW_LIMITS, isPreviewHost, BRANCH_LIMITS, validateBranchName, sessionProblem, SCENE_STORE_LIMITS,
runAnnotateSession: async () => ({ outcome: 'timeout' }), openInDefaultBrowser: async () => {}, ANNOTATE_LIMITS,
+ sampleBrowserHeap, exceedsMemoryGuard,
chromium: { launch: async () => browser }, URL, Buffer, process, performance, clearTimeout,
runtimeOptions, mkdir, rm, join, isAbsolute, fileURLToPath, writeWebBuild, EXPORT_DIRECTORY_NAME, createAgentRenderJobs: async () => ({ list: () => [], close: async () => {} }), ...contextOverrides,
setTimeout: (callback, delay) => { deadlines.push(delay); return setTimeout(callback, delay); },