3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
Recover the preview runtime after a lost page and name the operation that timed out (#2198, #2200)
Nguyen Xuan Ancec835bebb4d
diff --git a/changes/2198.md b/changes/2198.mdnew file mode 100644index 00000000..4d10a84d--- /dev/null+++ b/changes/2198.md@@ -0,0 +1 @@+- A render job whose page navigates or reloads now fails with that cause, the frame, the memory budget and the estimate instead of a stale `reading 'dispose'` error, a preview timeout names the operation that ran long, and a hung browser no longer delays the next `scene_create` (#2198, #2200)diff --git a/scripts/agent-browser-loss.mjs b/scripts/agent-browser-loss.mjsindex 76692695..15883da0 100644--- a/scripts/agent-browser-loss.mjs+++ b/scripts/agent-browser-loss.mjs@@ -1,16 +1,26 @@-// What a render job reports when the preview browser dies under it (#2018). Playwright words a lost page or browser-// in several ways; the job failure names the frame, the size and the passes so the caller knows which limit to lower.+// What a render job reports when the preview browser dies under it (#2018) or its page navigates away (#2198). Playwright words a+// lost page or browser in several ways; the job failure names the frame, the size and the passes so the caller knows which limit to lower.const BROWSER_LOSS_PATTERN = /target (page, context or browser )?(has been )?closed|(page|target|browser) (has )?crashed|browser has been closed|browser (has )?disconnected|connection closed|protocol error|session closed/i;+// A page that reloaded or navigated under a call: its context is gone, and the new document has no `window.__agentPreview` yet.+const PAGE_LOST_PATTERN = /execution context was destroyed|cannot read properties of (undefined|null) \(reading '(dispose|create|frame|resume|suspend)'\)|__agentPreview is (undefined|not defined)/i;export function isBrowserLoss(error) {- return BROWSER_LOSS_PATTERN.test(String(error?.message ?? error));+ const message = String(error?.message ?? error);+ return BROWSER_LOSS_PATTERN.test(message) || PAGE_LOST_PATTERN.test(message);}-export function describeBrowserLoss({ error, frameIndex, frameCount, width, height, passes = [], memoryMiB }) {+/** True when the page navigated or reloaded, so its preview bridge is gone, rather than the browser dying. */+export function isPageLost(error) {+ return PAGE_LOST_PATTERN.test(String(error?.message ?? error));+}++export function describeBrowserLoss({ error, frameIndex, frameCount, width, height, passes = [], memoryMiB, estimatedPeakMemoryMiB }) {const where = frameIndex < 0? 'before the first frame': `while rendering ${frameCount > 1 ? 'still' : 'frame'} ${frameIndex + 1} of ${frameCount}`;const passNote = passes.length ? ` with ${passes.join(', ')} ${passes.length === 1 ? 'pass' : 'passes'}` : '';const cause = String(error?.message ?? error).split('\n')[0];- return new Error(`The preview browser crashed ${where} at ${width}x${height}${passNote} (${cause}). Lower the size or the frame count, or raise --memory (now ${memoryMiB} MiB). The server is still running.`, { cause: error });+ const what = isPageLost(error) ? 'The preview page navigated or reloaded' : 'The preview browser crashed';+ const estimateNote = estimatedPeakMemoryMiB > memoryMiB ? ` The estimate was ${estimatedPeakMemoryMiB} MiB against the budget, so memory is the likely cause.` : '';+ return new Error(`${what} ${where} at ${width}x${height}${passNote} (${cause}).${estimateNote} Lower the size or the frame count, or raise --memory (now ${memoryMiB} MiB). The server is still running.`, { cause: error });}diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjsindex 98d3f737..1caa139f 100644--- a/scripts/agent-runtime.mjs+++ b/scripts/agent-runtime.mjs@@ -83,6 +83,8 @@ const MAXIMUM_KEPT_WEB_BUILDS = 4;const EXPORTED_GLB_NAME = /^[0-9a-f-]{36}\.glb$/;const EXPORTED_WEB_BUILD_NAME = /^web-[0-9a-f-]{36}$/;const EVALUATION_TIMEOUT_MS = 30000;+// A browser stuck in a long shader compile may never answer the close request; teardown waits this long, then goes on (#2200).+const BROWSER_CLOSE_TIMEOUT_MS = 5000;const RECOVERY_HINT = 'The scenes in this runtime are lost. Call scene_create (or scene_create_asset) to start a fresh runtime; no server restart is needed.';const FAILED_RUNTIME_MESSAGE = `The agent runtime is failed. ${RECOVERY_HINT}`;// Handing the scene page to a render job or review and bringing it back gets longer than an ordinary call. A job cut short by its deadline usually ran@@ -560,9 +562,19 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browreturn result;}+ // Closing the browser that just timed out can hang as long as its stuck operation does. The scenes are already forgotten, so the+ // teardown does not wait for a browser that will not answer, and the next call launches a fresh one instead of queueing behind it (#2200).+ async function closeBrowserWithinLimit(browser) {+ if (!browser) return;+ let timer;+ const closed = browser.close().catch(() => {});+ await Promise.race([closed, new Promise(resolve => { timer = setTimeout(resolve, BROWSER_CLOSE_TIMEOUT_MS); })]);+ clearTimeout(timer);+ }+async function shutdownResources() {try {- await browserInstance?.close();+ await closeBrowserWithinLimit(browserInstance);} finally {browserInstance = undefined;page = undefined;@@ -637,10 +649,21 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow});}+ // A start that fails on its page load is tried once more on a fresh browser: right after a timeout the old browser may still be+ // dying and hold the card, and the caller would otherwise need several retries before the runtime came back (#2200).async function boot() {if (page) {return;}+ try {+ await bootOnce();+ } catch (error) {+ if (closing || !/page.goto|Timeout d+ms exceeded/i.test(String(error?.message))) throw error;+ await bootOnce();+ }+ }++ async function bootOnce() {phase = 'starting';try {const [{ chromium }, code] = await Promise.all([@@ -798,6 +821,12 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browreturn evaluate(method, input, page, timeoutMs);}+ // Names the call that ran long: its method, and for an edit list the kinds of operation it carried, so the caller knows what to split (#2200).+ function describeOperation(method, input) {+ const kinds = Array.isArray(input?.operations) ? [...new Set(input.operations.map(operation => operation?.type).filter(Boolean))] : [];+ return kinds.length ? `"${method}" (${input.operations.length} operations: ${kinds.join(', ')})` : `"${method}"`;+ }+async function evaluate(method, input, target = page, timeoutMs = EVALUATION_TIMEOUT_MS) {assertCombinedAssetBytes(input);let timer;@@ -810,8 +839,8 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browtimedOut = true;if (target === page) phase = 'failed';reject(new Error(target === page- ? `The preview operation exceeded ${timeoutMs / 1000} seconds. ${RECOVERY_HINT}`- : `The private render operation exceeded ${timeoutMs / 1000} seconds.`));+ ? `The preview operation ${describeOperation(method, input)} exceeded ${timeoutMs / 1000} seconds. ${RECOVERY_HINT}`+ : `The private render operation ${describeOperation(method, input)} exceeded ${timeoutMs / 1000} seconds.`));}, timeoutMs);}),]);@@ -1173,7 +1202,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browreturn { artifacts, metrics };} catch (error) {if (signal.aborted || !isBrowserLoss(error)) throw error;- browserLossFailure = describeBrowserLoss({ error, frameIndex, frameCount: settings.frameCount, width: settings.width, height: settings.height, passes: settings.passes, memoryMiB: memory });+ browserLossFailure = describeBrowserLoss({ error, frameIndex, frameCount: settings.frameCount, width: settings.width, height: settings.height, passes: settings.passes, memoryMiB: memory, estimatedPeakMemoryMiB: estimate?.peakMemoryMiB });throw browserLossFailure;} finally {signal.removeEventListener('abort', abort);@@ -1184,9 +1213,9 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browfinally {try {if (jobPage && !jobPage.isClosed()) {- // A crashed page cannot dispose its scene; the close below frees it, and the failure that matters is the job's own.- try { if (settings.kind !== 'icons') await jobPage.evaluate(sessionId => window.__agentPreview.dispose({ sessionId }), jobId).catch(error => { if (!browserLossFailure) throw error; }); }- finally { await jobPage.close().catch(error => { if (!browserLossFailure) throw error; }); }+ // A crashed or navigated page cannot dispose its scene (its bridge is gone: "reading 'dispose'"); the close below frees it, and the failure that matters is the job's own (#2198).+ try { if (settings.kind !== 'icons') await jobPage.evaluate(sessionId => window.__agentPreview.dispose({ sessionId }), jobId).catch(error => { if (!browserLossFailure && !isBrowserLoss(error)) throw error; }); }+ finally { await jobPage.close().catch(error => { if (!browserLossFailure && !isBrowserLoss(error)) throw error; }); }}}finally {diff --git a/tests/agent-runtime.test.js b/tests/agent-runtime.test.jsindex 7763615c..940939fa 100644--- a/tests/agent-runtime.test.js+++ b/tests/agent-runtime.test.js@@ -121,7 +121,7 @@ function hitNearestLamp(ray) {}async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = false, deadlineMs,- failDispose = false, heapSize = 1024, stallPrivatePage = false, blockPrivateCreate = false, resumeDelayMs = 0, suspendDelayMs = 0, failResume = false, hangMethod, failPrivateCreate = false, failPrivateCreateMessage, disposeGate, jobRender, sceneBake, exportedRecipe, runtimeOptions = {}, contextOverrides = {}, lookBytesPerPixel = 0, inspection } = {}) {+ failDispose = false, heapSize = 1024, stallPrivatePage = false, blockPrivateCreate = false, resumeDelayMs = 0, suspendDelayMs = 0, failResume = false, hangMethod, failPrivateCreate = false, failPrivateCreateMessage, disposeGate, disposeLostBridge = false, hangBrowserClose = false, closeTimeoutMs, failFirstGoto = false, jobRender, sceneBake, exportedRecipe, runtimeOptions = {}, contextOverrides = {}, lookBytesPerPixel = 0, inspection } = {}) {const events = [];const sourceCalls = [];const sandboxTimers = new Set();@@ -141,7 +141,12 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = faroute() {}setDefaultTimeout() {}setViewportSize(size) { this.viewport = size; viewports.push(size); }- goto() {}+ goto() {+ if (!this.privateTarget && failFirstGoto) {+ failFirstGoto = false;+ throw new Error('page.goto: Timeout 15000ms exceeded.');+ }+ }waitForFunction() {}isClosed() { return this.closed; }context() { return { newCDPSession: async () => ({@@ -157,6 +162,7 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = fafailPrivateCreate = false;throw new Error(typeof failPrivateCreateMessage === 'string' ? failPrivateCreateMessage : 'Injected private create failure');}+ if (method === 'dispose' && this.privateTarget && disposeLostBridge) throw new TypeError("page.evaluate: TypeError: Cannot read properties of undefined (reading 'dispose')");if (method === 'dispose' && disposeGate) await disposeGate;if (method === 'resume' && failResume) throw new Error('Injected resume failure');if (method === 'create' && this.privateTarget && blockPrivateCreate) return new Promise((resolve, reject) => { this.rejectShot = reject; });@@ -205,7 +211,7 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = faresolveAcquisitionStarted();return stallPrivatePage ? acquiredPage : (privatePage = new FakePage(true));},- close: async () => { await privatePage?.close(); await sourcePage.close(); events.push('browser:close'); },+ close: async () => { if (hangBrowserClose) { hangBrowserClose = false; return new Promise(() => {}); } await privatePage?.close(); await sourcePage.close(); events.push('browser:close'); },};const originalSource = await readFile(new URL('../scripts/agent-runtime.mjs', import.meta.url), 'utf8');let source = originalSource@@ -214,6 +220,7 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = fa.replace('export function createAgentRuntime', 'function createAgentRuntime').replaceAll('import.meta.url', JSON.stringify(import.meta.url)).replace("import('playwright-core')", 'Promise.resolve({ chromium: globalThis.chromium })');+ if (closeTimeoutMs) source = source.replace('const BROWSER_CLOSE_TIMEOUT_MS = 5000;', `const BROWSER_CLOSE_TIMEOUT_MS = ${closeTimeoutMs};`);if (deadlineMs) source = source.replace('const EVALUATION_TIMEOUT_MS = 30000;', `const EVALUATION_TIMEOUT_MS = ${deadlineMs};`);const context = createContext({ ...(await importedBindings(originalSource)),packageMetadata: { version: 'test' }, existsSync: () => true, readFile: async () => '',@@ -559,6 +566,65 @@ runTest('a scene page that times out fails the runtime with the recovery hint, a} finally { await runtime.close(); }});+runTest('a timed-out preview call names the operation that ran long (#2200)', async () => {+ const { runtime, sessionId } = await reviewRuntimeHarness({ hangMethod: 'apply', deadlineMs: 40 });+ try {+ await assert.rejects(runtime.apply({ sessionId, operations: [{ type: 'add-material', material: { id: 'stone', pattern: 'stone' } }, { type: 'add-material', material: { id: 'tiles', pattern: 'tiles' } }] }),+ /operation "apply" \(2 operations: add-material\) exceeded 0\.04 seconds/);+ } finally { await runtime.close(); }+});++runTest('the next scene_create after a timeout succeeds first try even when the old browser never finishes closing (#2200)', async () => {+ const { runtime, sessionId } = await reviewRuntimeHarness({ hangMethod: 'inspect', deadlineMs: 40, hangBrowserClose: true, closeTimeoutMs: 30 });+ try {+ await assert.rejects(runtime.inspect({ sessionId }), /operation "inspect" exceeded.*scene_create/s);+ const fresh = await runtime.create({ recipe: FRESH_RECIPE });+ assert.equal((await runtime.apply({ sessionId: fresh.sessionId })).revision, 1);+ } finally { await runtime.close(); }+});++runTest('a browser start whose page load times out is retried once on a fresh browser (#2200)', async () => {+ const harness = await reviewRuntimeHarness({ failFirstGoto: true });+ try {+ assert.equal(typeof harness.sessionId, 'string');+ } finally { await harness.runtime.close(); }+});++runTest('a job whose page navigates under it fails with the page loss and the budget, then the next scene works (#2198)', async () => {+ let runJob;+ const controller = { list: () => [], close: async () => {}, submit: async () => ({ jobId: 'job' }) };+ const submitted = { kind: 'video', width: 1280, height: 720, fps: 24, duration: 12 };+ const settings = { ...submitted, frameCount: 288 };+ const { runtime, sessionId } = await reviewRuntimeHarness({ failPrivateCreate: true, disposeLostBridge: true,+ failPrivateCreateMessage: 'page.evaluate: Execution context was destroyed, most likely because of a navigation',+ runtimeOptions: { jobs: join(tmpdir(), 'crucible-job-root-test'), memory: 192 },+ contextOverrides: { createAgentRenderJobs: async ({ run }) => { runJob = run; return controller; } } });+ try {+ await runtime.submitJob({ sessionId, expectedRevision: 0, settings: submitted });+ const failure = await runJob({ jobId: randomUUID(), sessionId, directory: tmpdir(), snapshot: { recipe: {} }, settings,+ signal: new AbortController().signal, progress: async () => {} }).then(() => assert.fail('the job must fail'), error => error);+ assert.match(failure.message, /preview page navigated or reloaded before the first frame at 1280x720/);+ assert.match(failure.message, /raise --memory \(now 192 MiB\)/);+ assert.doesNotMatch(failure.message, /reading 'dispose'/);+ const fresh = await runtime.create({ recipe: FRESH_RECIPE });+ assert.equal((await runtime.apply({ sessionId: fresh.sessionId })).revision, 1);+ } finally { await runtime.close(); }+});++runTest('a job that fails for its own reason is not masked by a lost page bridge on dispose (#2198)', async () => {+ let runJob;+ const controller = { list: () => [], close: async () => {}, submit: async () => ({ jobId: 'job' }) };+ const { runtime, sessionId } = await reviewRuntimeHarness({ failPrivateCreate: true, disposeLostBridge: true, failPrivateCreateMessage: 'Injected scene problem',+ runtimeOptions: { jobs: join(tmpdir(), 'crucible-job-root-test') },+ contextOverrides: { createAgentRenderJobs: async ({ run }) => { runJob = run; return controller; } } });+ try {+ await runtime.submitJob({ sessionId, expectedRevision: 0, settings: { kind: 'image', width: 64, height: 64 } });+ const failure = await runJob({ jobId: randomUUID(), sessionId, directory: tmpdir(), snapshot: { recipe: {} },+ settings: { kind: 'image', width: 64, height: 64 }, signal: new AbortController().signal, progress: async () => {} }).then(() => assert.fail('the job must fail'), error => error);+ assert.match(failure.message, /Injected scene problem/);+ } finally { await runtime.close(); }+});+runTest('a failed resume after a render job fails the runtime, and the next scene starts fresh (#1256)', async () => {let runJob;const controller = { list: () => [], close: async () => {}, submit: async () => ({ jobId: 'job' }) };