3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
Name the cause when a render job loses its preview browser (#2018)
Nguyen Xuan An9ef05eccd7a6
diff --git a/changes/2018.md b/changes/2018.mdnew file mode 100644index 00000000..9992c365--- /dev/null+++ b/changes/2018.md@@ -0,0 +1 @@+- A render job that loses its preview browser now fails with a message that names the frame, the size, the passes and `--memory` instead of "Preview recovery failed after render job", and the server keeps answering (#2018).diff --git a/scripts/agent-browser-loss.mjs b/scripts/agent-browser-loss.mjsnew file mode 100644index 00000000..76692695--- /dev/null+++ b/scripts/agent-browser-loss.mjs@@ -0,0 +1,16 @@+// What a render job reports when the preview browser dies under it (#2018). Playwright words a lost page or browser+// in several ways; the job failure names the frame, the size and the passes so the caller knows which limit to lower.+const BROWSER_LOSS_PATTERN = /target (page, context or browser )?(has been )?closed|(page|target|browser) (has )?crashed|browser has been closed|browser (has )?disconnected|connection closed|protocol error|session closed/i;++export function isBrowserLoss(error) {+ return BROWSER_LOSS_PATTERN.test(String(error?.message ?? error));+}++export function describeBrowserLoss({ error, frameIndex, frameCount, width, height, passes = [], memoryMiB }) {+ const where = frameIndex < 0+ ? 'before the first frame'+ : `while rendering ${frameCount > 1 ? 'still' : 'frame'} ${frameIndex + 1} of ${frameCount}`;+ const passNote = passes.length ? ` with ${passes.join(', ')} ${passes.length === 1 ? 'pass' : 'passes'}` : '';+ const cause = String(error?.message ?? error).split('\n')[0];+ return new Error(`The preview browser crashed ${where} at ${width}x${height}${passNote} (${cause}). Lower the size or the frame count, or raise --memory (now ${memoryMiB} MiB). The server is still running.`, { cause: error });+}diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjsindex bf9ea5f4..98d3f737 100644--- a/scripts/agent-runtime.mjs+++ b/scripts/agent-runtime.mjs@@ -62,6 +62,7 @@ import { normalizeReview, reviewDeadlineMessage, REVIEW_LIMITS } from './agent-rimport { mergeFrameDiagnostics, pathTracerFindingsLimit } from '../src/frame-checks.js';import { BRANCH_LIMITS, validateBranchName } from '../src/agent-branches.js';import { createSharedGpuCard } from './agent-gpu-queue.mjs';+import { isBrowserLoss, describeBrowserLoss } from './agent-browser-loss.mjs';import { sessionProblem, SCENE_STORE_LIMITS } from '../src/agent-errors.js';const SCENE_LIMITS = AGENT_SCENE_DEFAULT_LIMITS;@@ -843,6 +844,9 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browlet nativeTemporalShotIndex;let pageEncodes = false;let suspended = false;+ // The frame being rendered, so a browser that dies mid-job is reported with where it died (#2018).+ let frameIndex = -1;+ let browserLossFailure;const generationAtStart = runtimeGeneration;const started = performance.now();// One job runs at a time, so the estimate made at submission belongs to the job that starts next.@@ -949,6 +953,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow}for (let frame = 0; frame < settings.frameCount; frame++) {signal.throwIfAborted();+ frameIndex = frame;const elapsed = settings.kind === 'video' ? frame / settings.fps: settings.kind === 'stills' ? (settings.stillCount ? frame / settings.fps : settings.duration * frame / (settings.frameCount - 1)) : 0;// A timeline job reads one clock for the shot, the motion and the renderer, so a seek matches the frame in a video.@@ -1166,6 +1171,10 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browif (settings.renderer?.backend === 'gpu') gpuTracerWarm = true;await rememberRenderJob({ settings, estimate, report: metrics.report });return { artifacts, metrics };+ } catch (error) {+ if (signal.aborted || !isBrowserLoss(error)) throw error;+ browserLossFailure = describeBrowserLoss({ error, frameIndex, frameCount: settings.frameCount, width: settings.width, height: settings.height, passes: settings.passes, memoryMiB: memory });+ throw browserLossFailure;} finally {signal.removeEventListener('abort', abort);nativeSequence?.close();@@ -1175,8 +1184,9 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browfinally {try {if (jobPage && !jobPage.isClosed()) {- try { if (settings.kind !== 'icons') await jobPage.evaluate(sessionId => window.__agentPreview.dispose({ sessionId }), jobId); }- finally { await jobPage.close(); }+ // A crashed page cannot dispose its scene; the close below frees it, and the failure that matters is the job's own.+ try { if (settings.kind !== 'icons') await jobPage.evaluate(sessionId => window.__agentPreview.dispose({ sessionId }), jobId).catch(error => { if (!browserLossFailure) throw error; }); }+ finally { await jobPage.close().catch(error => { if (!browserLossFailure) throw error; }); }}}finally {@@ -1188,7 +1198,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browphase = 'ready';} catch (error) {phase = 'failed';- throw new Error(`Preview recovery failed after render job: ${error.message}`);+ throw browserLossFailure ?? new Error(`Preview recovery failed after render job: ${error.message}`);}} else if (!closing && phase !== 'failed') phase = 'ready';}diff --git a/tests/agent-runtime.test.js b/tests/agent-runtime.test.jsindex 9be9c088..7763615c 100644--- a/tests/agent-runtime.test.js+++ b/tests/agent-runtime.test.js@@ -121,7 +121,7 @@ function hitNearestLamp(ray) {}async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = false, deadlineMs,- failDispose = false, heapSize = 1024, stallPrivatePage = false, blockPrivateCreate = false, resumeDelayMs = 0, suspendDelayMs = 0, failResume = false, hangMethod, failPrivateCreate = false, disposeGate, jobRender, sceneBake, exportedRecipe, runtimeOptions = {}, contextOverrides = {}, lookBytesPerPixel = 0, inspection } = {}) {+ failDispose = false, heapSize = 1024, stallPrivatePage = false, blockPrivateCreate = false, resumeDelayMs = 0, suspendDelayMs = 0, failResume = false, hangMethod, failPrivateCreate = false, failPrivateCreateMessage, disposeGate, jobRender, sceneBake, exportedRecipe, runtimeOptions = {}, contextOverrides = {}, lookBytesPerPixel = 0, inspection } = {}) {const events = [];const sourceCalls = [];const sandboxTimers = new Set();@@ -155,7 +155,7 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = faif (method === hangMethod && !this.privateTarget) return new Promise(() => {});if (method === 'create' && this.privateTarget && failPrivateCreate) {failPrivateCreate = false;- throw new Error('Injected private create failure');+ throw new Error(typeof failPrivateCreateMessage === 'string' ? failPrivateCreateMessage : 'Injected private create failure');}if (method === 'dispose' && disposeGate) await disposeGate;if (method === 'resume' && failResume) throw new Error('Injected resume failure');@@ -576,6 +576,41 @@ runTest('a failed resume after a render job fails the runtime, and the next scen} finally { await runtime.close(); }});+runTest('a browser lost under a stills job fails the job with its size and limit, and the runtime recovers (#2018)', async () => {+ let runJob;+ const controller = { list: () => [], close: async () => {}, submit: async () => ({ jobId: 'job' }) };+ const submitted = { kind: 'stills', width: 1920, height: 1080, stillCount: 12, fps: 24, passes: ['depth'] };+ const settings = { ...submitted, frameCount: 12 };+ const { runtime, sessionId } = await reviewRuntimeHarness({ failPrivateCreate: true, failPrivateCreateMessage: 'Target page, context or browser has been closed', failResume: true,+ runtimeOptions: { jobs: join(tmpdir(), 'crucible-job-root-test'), memory: 1024 },+ contextOverrides: { createAgentRenderJobs: async ({ run }) => { runJob = run; return controller; } } });+ try {+ await runtime.submitJob({ sessionId, expectedRevision: 0, settings: submitted });+ const failure = await runJob({ jobId: randomUUID(), sessionId, directory: tmpdir(), snapshot: { recipe: {} }, settings,+ signal: new AbortController().signal, progress: async () => {} }).then(() => assert.fail('the job must fail'), error => error);+ assert.match(failure.message, /preview browser crashed before the first frame at 1920x1080 with depth pass/);+ assert.match(failure.message, /Lower the size or the frame count, or raise --memory \(now \d+ MiB\)/);+ assert.doesNotMatch(failure.message, /Preview recovery failed/);+ const fresh = await runtime.create({ recipe: FRESH_RECIPE });+ assert.equal((await runtime.apply({ sessionId: fresh.sessionId })).revision, 1);+ } finally { await runtime.close(); }+});++runTest('a stills job that fails for another reason keeps its own message (#2018)', async () => {+ let runJob;+ const controller = { list: () => [], close: async () => {}, submit: async () => ({ jobId: 'job' }) };+ const submitted = { kind: 'stills', width: 64, height: 64, stillCount: 2, fps: 24 };+ const settings = { ...submitted, frameCount: 2 };+ const { runtime, sessionId } = await reviewRuntimeHarness({ failPrivateCreate: true, failPrivateCreateMessage: 'Injected scene problem',+ runtimeOptions: { jobs: join(tmpdir(), 'crucible-job-root-test') },+ contextOverrides: { createAgentRenderJobs: async ({ run }) => { runJob = run; return controller; } } });+ try {+ await runtime.submitJob({ sessionId, expectedRevision: 0, settings: submitted });+ await assert.rejects(runJob({ jobId: randomUUID(), sessionId, directory: tmpdir(), snapshot: { recipe: {} }, settings,+ signal: new AbortController().signal, progress: async () => {} }), /^Error: Injected scene problem$/);+ } finally { await runtime.close(); }+});+runTest('a job still unwinding when the runtime restarts does not fail the new runtime (#1256)', async () => {let runJob;let openGate;