Skip to content

3D engine that AI agents drive

Recover the preview runtime after a lost page and name the operation that timed out (#2198, #2200)

Nguyen Xuan Ancec835bebb4d
diff --git a/changes/2198.md b/changes/2198.md
new file mode 100644
index 00000000..4d10a84d
--- /dev/null
+++ b/changes/2198.md
@@ -0,0 +1 @@
+- A render job whose page navigates or reloads now fails with that cause, the frame, the memory budget and the estimate instead of a stale `reading 'dispose'` error, a preview timeout names the operation that ran long, and a hung browser no longer delays the next `scene_create` (#2198, #2200)
diff --git a/scripts/agent-browser-loss.mjs b/scripts/agent-browser-loss.mjs
index 76692695..15883da0 100644
--- a/scripts/agent-browser-loss.mjs
+++ b/scripts/agent-browser-loss.mjs
@@ -1,16 +1,26 @@
-// What a render job reports when the preview browser dies under it (#2018). Playwright words a lost page or browser
-// in several ways; the job failure names the frame, the size and the passes so the caller knows which limit to lower.
+// What a render job reports when the preview browser dies under it (#2018) or its page navigates away (#2198). Playwright words a
+// lost page or browser in several ways; the job failure names the frame, the size and the passes so the caller knows which limit to lower.
const BROWSER_LOSS_PATTERN = /target (page, context or browser )?(has been )?closed|(page|target|browser) (has )?crashed|browser has been closed|browser (has )?disconnected|connection closed|protocol error|session closed/i;
+// A page that reloaded or navigated under a call: its context is gone, and the new document has no `window.__agentPreview` yet.
+const PAGE_LOST_PATTERN = /execution context was destroyed|cannot read properties of (undefined|null) \(reading '(dispose|create|frame|resume|suspend)'\)|__agentPreview is (undefined|not defined)/i;
export function isBrowserLoss(error) {
- return BROWSER_LOSS_PATTERN.test(String(error?.message ?? error));
+ const message = String(error?.message ?? error);
+ return BROWSER_LOSS_PATTERN.test(message) || PAGE_LOST_PATTERN.test(message);
}
-export function describeBrowserLoss({ error, frameIndex, frameCount, width, height, passes = [], memoryMiB }) {
+/** True when the page navigated or reloaded, so its preview bridge is gone, rather than the browser dying. */
+export function isPageLost(error) {
+ return PAGE_LOST_PATTERN.test(String(error?.message ?? error));
+}
+
+export function describeBrowserLoss({ error, frameIndex, frameCount, width, height, passes = [], memoryMiB, estimatedPeakMemoryMiB }) {
const where = frameIndex < 0
? 'before the first frame'
: `while rendering ${frameCount > 1 ? 'still' : 'frame'} ${frameIndex + 1} of ${frameCount}`;
const passNote = passes.length ? ` with ${passes.join(', ')} ${passes.length === 1 ? 'pass' : 'passes'}` : '';
const cause = String(error?.message ?? error).split('\n')[0];
- return new Error(`The preview browser crashed ${where} at ${width}x${height}${passNote} (${cause}). Lower the size or the frame count, or raise --memory (now ${memoryMiB} MiB). The server is still running.`, { cause: error });
+ const what = isPageLost(error) ? 'The preview page navigated or reloaded' : 'The preview browser crashed';
+ const estimateNote = estimatedPeakMemoryMiB > memoryMiB ? ` The estimate was ${estimatedPeakMemoryMiB} MiB against the budget, so memory is the likely cause.` : '';
+ return new Error(`${what} ${where} at ${width}x${height}${passNote} (${cause}).${estimateNote} Lower the size or the frame count, or raise --memory (now ${memoryMiB} MiB). The server is still running.`, { cause: error });
}
diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjs
index 98d3f737..1caa139f 100644
--- a/scripts/agent-runtime.mjs
+++ b/scripts/agent-runtime.mjs
@@ -83,6 +83,8 @@ const MAXIMUM_KEPT_WEB_BUILDS = 4;
const EXPORTED_GLB_NAME = /^[0-9a-f-]{36}\.glb$/;
const EXPORTED_WEB_BUILD_NAME = /^web-[0-9a-f-]{36}$/;
const EVALUATION_TIMEOUT_MS = 30000;
+// A browser stuck in a long shader compile may never answer the close request; teardown waits this long, then goes on (#2200).
+const BROWSER_CLOSE_TIMEOUT_MS = 5000;
const RECOVERY_HINT = 'The scenes in this runtime are lost. Call scene_create (or scene_create_asset) to start a fresh runtime; no server restart is needed.';
const FAILED_RUNTIME_MESSAGE = `The agent runtime is failed. ${RECOVERY_HINT}`;
// Handing the scene page to a render job or review and bringing it back gets longer than an ordinary call. A job cut short by its deadline usually ran
@@ -560,9 +562,19 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
return result;
}
+ // Closing the browser that just timed out can hang as long as its stuck operation does. The scenes are already forgotten, so the
+ // teardown does not wait for a browser that will not answer, and the next call launches a fresh one instead of queueing behind it (#2200).
+ async function closeBrowserWithinLimit(browser) {
+ if (!browser) return;
+ let timer;
+ const closed = browser.close().catch(() => {});
+ await Promise.race([closed, new Promise(resolve => { timer = setTimeout(resolve, BROWSER_CLOSE_TIMEOUT_MS); })]);
+ clearTimeout(timer);
+ }
+
async function shutdownResources() {
try {
- await browserInstance?.close();
+ await closeBrowserWithinLimit(browserInstance);
} finally {
browserInstance = undefined;
page = undefined;
@@ -637,10 +649,21 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
});
}
+ // A start that fails on its page load is tried once more on a fresh browser: right after a timeout the old browser may still be
+ // dying and hold the card, and the caller would otherwise need several retries before the runtime came back (#2200).
async function boot() {
if (page) {
return;
}
+ try {
+ await bootOnce();
+ } catch (error) {
+ if (closing || !/page.goto|Timeout d+ms exceeded/i.test(String(error?.message))) throw error;
+ await bootOnce();
+ }
+ }
+
+ async function bootOnce() {
phase = 'starting';
try {
const [{ chromium }, code] = await Promise.all([
@@ -798,6 +821,12 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
return evaluate(method, input, page, timeoutMs);
}
+ // Names the call that ran long: its method, and for an edit list the kinds of operation it carried, so the caller knows what to split (#2200).
+ function describeOperation(method, input) {
+ const kinds = Array.isArray(input?.operations) ? [...new Set(input.operations.map(operation => operation?.type).filter(Boolean))] : [];
+ return kinds.length ? `"${method}" (${input.operations.length} operations: ${kinds.join(', ')})` : `"${method}"`;
+ }
+
async function evaluate(method, input, target = page, timeoutMs = EVALUATION_TIMEOUT_MS) {
assertCombinedAssetBytes(input);
let timer;
@@ -810,8 +839,8 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
timedOut = true;
if (target === page) phase = 'failed';
reject(new Error(target === page
- ? `The preview operation exceeded ${timeoutMs / 1000} seconds. ${RECOVERY_HINT}`
- : `The private render operation exceeded ${timeoutMs / 1000} seconds.`));
+ ? `The preview operation ${describeOperation(method, input)} exceeded ${timeoutMs / 1000} seconds. ${RECOVERY_HINT}`
+ : `The private render operation ${describeOperation(method, input)} exceeded ${timeoutMs / 1000} seconds.`));
}, timeoutMs);
}),
]);
@@ -1173,7 +1202,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
return { artifacts, metrics };
} catch (error) {
if (signal.aborted || !isBrowserLoss(error)) throw error;
- browserLossFailure = describeBrowserLoss({ error, frameIndex, frameCount: settings.frameCount, width: settings.width, height: settings.height, passes: settings.passes, memoryMiB: memory });
+ browserLossFailure = describeBrowserLoss({ error, frameIndex, frameCount: settings.frameCount, width: settings.width, height: settings.height, passes: settings.passes, memoryMiB: memory, estimatedPeakMemoryMiB: estimate?.peakMemoryMiB });
throw browserLossFailure;
} finally {
signal.removeEventListener('abort', abort);
@@ -1184,9 +1213,9 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
finally {
try {
if (jobPage && !jobPage.isClosed()) {
- // A crashed page cannot dispose its scene; the close below frees it, and the failure that matters is the job's own.
- try { if (settings.kind !== 'icons') await jobPage.evaluate(sessionId => window.__agentPreview.dispose({ sessionId }), jobId).catch(error => { if (!browserLossFailure) throw error; }); }
- finally { await jobPage.close().catch(error => { if (!browserLossFailure) throw error; }); }
+ // A crashed or navigated page cannot dispose its scene (its bridge is gone: "reading 'dispose'"); the close below frees it, and the failure that matters is the job's own (#2198).
+ try { if (settings.kind !== 'icons') await jobPage.evaluate(sessionId => window.__agentPreview.dispose({ sessionId }), jobId).catch(error => { if (!browserLossFailure && !isBrowserLoss(error)) throw error; }); }
+ finally { await jobPage.close().catch(error => { if (!browserLossFailure && !isBrowserLoss(error)) throw error; }); }
}
}
finally {
diff --git a/tests/agent-runtime.test.js b/tests/agent-runtime.test.js
index 7763615c..940939fa 100644
--- a/tests/agent-runtime.test.js
+++ b/tests/agent-runtime.test.js
@@ -121,7 +121,7 @@ function hitNearestLamp(ray) {
}
async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = false, deadlineMs,
- failDispose = false, heapSize = 1024, stallPrivatePage = false, blockPrivateCreate = false, resumeDelayMs = 0, suspendDelayMs = 0, failResume = false, hangMethod, failPrivateCreate = false, failPrivateCreateMessage, disposeGate, jobRender, sceneBake, exportedRecipe, runtimeOptions = {}, contextOverrides = {}, lookBytesPerPixel = 0, inspection } = {}) {
+ failDispose = false, heapSize = 1024, stallPrivatePage = false, blockPrivateCreate = false, resumeDelayMs = 0, suspendDelayMs = 0, failResume = false, hangMethod, failPrivateCreate = false, failPrivateCreateMessage, disposeGate, disposeLostBridge = false, hangBrowserClose = false, closeTimeoutMs, failFirstGoto = false, jobRender, sceneBake, exportedRecipe, runtimeOptions = {}, contextOverrides = {}, lookBytesPerPixel = 0, inspection } = {}) {
const events = [];
const sourceCalls = [];
const sandboxTimers = new Set();
@@ -141,7 +141,12 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = fa
route() {}
setDefaultTimeout() {}
setViewportSize(size) { this.viewport = size; viewports.push(size); }
- goto() {}
+ goto() {
+ if (!this.privateTarget && failFirstGoto) {
+ failFirstGoto = false;
+ throw new Error('page.goto: Timeout 15000ms exceeded.');
+ }
+ }
waitForFunction() {}
isClosed() { return this.closed; }
context() { return { newCDPSession: async () => ({
@@ -157,6 +162,7 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = fa
failPrivateCreate = false;
throw new Error(typeof failPrivateCreateMessage === 'string' ? failPrivateCreateMessage : 'Injected private create failure');
}
+ if (method === 'dispose' && this.privateTarget && disposeLostBridge) throw new TypeError("page.evaluate: TypeError: Cannot read properties of undefined (reading 'dispose')");
if (method === 'dispose' && disposeGate) await disposeGate;
if (method === 'resume' && failResume) throw new Error('Injected resume failure');
if (method === 'create' && this.privateTarget && blockPrivateCreate) return new Promise((resolve, reject) => { this.rejectShot = reject; });
@@ -205,7 +211,7 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = fa
resolveAcquisitionStarted();
return stallPrivatePage ? acquiredPage : (privatePage = new FakePage(true));
},
- close: async () => { await privatePage?.close(); await sourcePage.close(); events.push('browser:close'); },
+ close: async () => { if (hangBrowserClose) { hangBrowserClose = false; return new Promise(() => {}); } await privatePage?.close(); await sourcePage.close(); events.push('browser:close'); },
};
const originalSource = await readFile(new URL('../scripts/agent-runtime.mjs', import.meta.url), 'utf8');
let source = originalSource
@@ -214,6 +220,7 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = fa
.replace('export function createAgentRuntime', 'function createAgentRuntime')
.replaceAll('import.meta.url', JSON.stringify(import.meta.url))
.replace("import('playwright-core')", 'Promise.resolve({ chromium: globalThis.chromium })');
+ if (closeTimeoutMs) source = source.replace('const BROWSER_CLOSE_TIMEOUT_MS = 5000;', `const BROWSER_CLOSE_TIMEOUT_MS = ${closeTimeoutMs};`);
if (deadlineMs) source = source.replace('const EVALUATION_TIMEOUT_MS = 30000;', `const EVALUATION_TIMEOUT_MS = ${deadlineMs};`);
const context = createContext({ ...(await importedBindings(originalSource)),
packageMetadata: { version: 'test' }, existsSync: () => true, readFile: async () => '',
@@ -559,6 +566,65 @@ runTest('a scene page that times out fails the runtime with the recovery hint, a
} finally { await runtime.close(); }
});
+runTest('a timed-out preview call names the operation that ran long (#2200)', async () => {
+ const { runtime, sessionId } = await reviewRuntimeHarness({ hangMethod: 'apply', deadlineMs: 40 });
+ try {
+ await assert.rejects(runtime.apply({ sessionId, operations: [{ type: 'add-material', material: { id: 'stone', pattern: 'stone' } }, { type: 'add-material', material: { id: 'tiles', pattern: 'tiles' } }] }),
+ /operation "apply" \(2 operations: add-material\) exceeded 0\.04 seconds/);
+ } finally { await runtime.close(); }
+});
+
+runTest('the next scene_create after a timeout succeeds first try even when the old browser never finishes closing (#2200)', async () => {
+ const { runtime, sessionId } = await reviewRuntimeHarness({ hangMethod: 'inspect', deadlineMs: 40, hangBrowserClose: true, closeTimeoutMs: 30 });
+ try {
+ await assert.rejects(runtime.inspect({ sessionId }), /operation "inspect" exceeded.*scene_create/s);
+ const fresh = await runtime.create({ recipe: FRESH_RECIPE });
+ assert.equal((await runtime.apply({ sessionId: fresh.sessionId })).revision, 1);
+ } finally { await runtime.close(); }
+});
+
+runTest('a browser start whose page load times out is retried once on a fresh browser (#2200)', async () => {
+ const harness = await reviewRuntimeHarness({ failFirstGoto: true });
+ try {
+ assert.equal(typeof harness.sessionId, 'string');
+ } finally { await harness.runtime.close(); }
+});
+
+runTest('a job whose page navigates under it fails with the page loss and the budget, then the next scene works (#2198)', async () => {
+ let runJob;
+ const controller = { list: () => [], close: async () => {}, submit: async () => ({ jobId: 'job' }) };
+ const submitted = { kind: 'video', width: 1280, height: 720, fps: 24, duration: 12 };
+ const settings = { ...submitted, frameCount: 288 };
+ const { runtime, sessionId } = await reviewRuntimeHarness({ failPrivateCreate: true, disposeLostBridge: true,
+ failPrivateCreateMessage: 'page.evaluate: Execution context was destroyed, most likely because of a navigation',
+ runtimeOptions: { jobs: join(tmpdir(), 'crucible-job-root-test'), memory: 192 },
+ contextOverrides: { createAgentRenderJobs: async ({ run }) => { runJob = run; return controller; } } });
+ try {
+ await runtime.submitJob({ sessionId, expectedRevision: 0, settings: submitted });
+ const failure = await runJob({ jobId: randomUUID(), sessionId, directory: tmpdir(), snapshot: { recipe: {} }, settings,
+ signal: new AbortController().signal, progress: async () => {} }).then(() => assert.fail('the job must fail'), error => error);
+ assert.match(failure.message, /preview page navigated or reloaded before the first frame at 1280x720/);
+ assert.match(failure.message, /raise --memory \(now 192 MiB\)/);
+ assert.doesNotMatch(failure.message, /reading 'dispose'/);
+ const fresh = await runtime.create({ recipe: FRESH_RECIPE });
+ assert.equal((await runtime.apply({ sessionId: fresh.sessionId })).revision, 1);
+ } finally { await runtime.close(); }
+});
+
+runTest('a job that fails for its own reason is not masked by a lost page bridge on dispose (#2198)', async () => {
+ let runJob;
+ const controller = { list: () => [], close: async () => {}, submit: async () => ({ jobId: 'job' }) };
+ const { runtime, sessionId } = await reviewRuntimeHarness({ failPrivateCreate: true, disposeLostBridge: true, failPrivateCreateMessage: 'Injected scene problem',
+ runtimeOptions: { jobs: join(tmpdir(), 'crucible-job-root-test') },
+ contextOverrides: { createAgentRenderJobs: async ({ run }) => { runJob = run; return controller; } } });
+ try {
+ await runtime.submitJob({ sessionId, expectedRevision: 0, settings: { kind: 'image', width: 64, height: 64 } });
+ const failure = await runJob({ jobId: randomUUID(), sessionId, directory: tmpdir(), snapshot: { recipe: {} },
+ settings: { kind: 'image', width: 64, height: 64 }, signal: new AbortController().signal, progress: async () => {} }).then(() => assert.fail('the job must fail'), error => error);
+ assert.match(failure.message, /Injected scene problem/);
+ } finally { await runtime.close(); }
+});
+
runTest('a failed resume after a render job fails the runtime, and the next scene starts fresh (#1256)', async () => {
let runJob;
const controller = { list: () => [], close: async () => {}, submit: async () => ({ jobId: 'job' }) };