Skip to content

3D engine that AI agents drive

Name the cause when a render job loses its preview browser (#2018)

Nguyen Xuan An9ef05eccd7a6
diff --git a/changes/2018.md b/changes/2018.md
new file mode 100644
index 00000000..9992c365
--- /dev/null
+++ b/changes/2018.md
@@ -0,0 +1 @@
+- A render job that loses its preview browser now fails with a message that names the frame, the size, the passes and `--memory` instead of "Preview recovery failed after render job", and the server keeps answering (#2018).
diff --git a/scripts/agent-browser-loss.mjs b/scripts/agent-browser-loss.mjs
new file mode 100644
index 00000000..76692695
--- /dev/null
+++ b/scripts/agent-browser-loss.mjs
@@ -0,0 +1,16 @@
+// What a render job reports when the preview browser dies under it (#2018). Playwright words a lost page or browser
+// in several ways; the job failure names the frame, the size and the passes so the caller knows which limit to lower.
+const BROWSER_LOSS_PATTERN = /target (page, context or browser )?(has been )?closed|(page|target|browser) (has )?crashed|browser has been closed|browser (has )?disconnected|connection closed|protocol error|session closed/i;
+
+export function isBrowserLoss(error) {
+ return BROWSER_LOSS_PATTERN.test(String(error?.message ?? error));
+}
+
+export function describeBrowserLoss({ error, frameIndex, frameCount, width, height, passes = [], memoryMiB }) {
+ const where = frameIndex < 0
+ ? 'before the first frame'
+ : `while rendering ${frameCount > 1 ? 'still' : 'frame'} ${frameIndex + 1} of ${frameCount}`;
+ const passNote = passes.length ? ` with ${passes.join(', ')} ${passes.length === 1 ? 'pass' : 'passes'}` : '';
+ const cause = String(error?.message ?? error).split('\n')[0];
+ return new Error(`The preview browser crashed ${where} at ${width}x${height}${passNote} (${cause}). Lower the size or the frame count, or raise --memory (now ${memoryMiB} MiB). The server is still running.`, { cause: error });
+}
diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjs
index bf9ea5f4..98d3f737 100644
--- a/scripts/agent-runtime.mjs
+++ b/scripts/agent-runtime.mjs
@@ -62,6 +62,7 @@ import { normalizeReview, reviewDeadlineMessage, REVIEW_LIMITS } from './agent-r
import { mergeFrameDiagnostics, pathTracerFindingsLimit } from '../src/frame-checks.js';
import { BRANCH_LIMITS, validateBranchName } from '../src/agent-branches.js';
import { createSharedGpuCard } from './agent-gpu-queue.mjs';
+import { isBrowserLoss, describeBrowserLoss } from './agent-browser-loss.mjs';
import { sessionProblem, SCENE_STORE_LIMITS } from '../src/agent-errors.js';
const SCENE_LIMITS = AGENT_SCENE_DEFAULT_LIMITS;
@@ -843,6 +844,9 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
let nativeTemporalShotIndex;
let pageEncodes = false;
let suspended = false;
+ // The frame being rendered, so a browser that dies mid-job is reported with where it died (#2018).
+ let frameIndex = -1;
+ let browserLossFailure;
const generationAtStart = runtimeGeneration;
const started = performance.now();
// One job runs at a time, so the estimate made at submission belongs to the job that starts next.
@@ -949,6 +953,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
}
for (let frame = 0; frame < settings.frameCount; frame++) {
signal.throwIfAborted();
+ frameIndex = frame;
const elapsed = settings.kind === 'video' ? frame / settings.fps
: settings.kind === 'stills' ? (settings.stillCount ? frame / settings.fps : settings.duration * frame / (settings.frameCount - 1)) : 0;
// A timeline job reads one clock for the shot, the motion and the renderer, so a seek matches the frame in a video.
@@ -1166,6 +1171,10 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
if (settings.renderer?.backend === 'gpu') gpuTracerWarm = true;
await rememberRenderJob({ settings, estimate, report: metrics.report });
return { artifacts, metrics };
+ } catch (error) {
+ if (signal.aborted || !isBrowserLoss(error)) throw error;
+ browserLossFailure = describeBrowserLoss({ error, frameIndex, frameCount: settings.frameCount, width: settings.width, height: settings.height, passes: settings.passes, memoryMiB: memory });
+ throw browserLossFailure;
} finally {
signal.removeEventListener('abort', abort);
nativeSequence?.close();
@@ -1175,8 +1184,9 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
finally {
try {
if (jobPage && !jobPage.isClosed()) {
- try { if (settings.kind !== 'icons') await jobPage.evaluate(sessionId => window.__agentPreview.dispose({ sessionId }), jobId); }
- finally { await jobPage.close(); }
+ // A crashed page cannot dispose its scene; the close below frees it, and the failure that matters is the job's own.
+ try { if (settings.kind !== 'icons') await jobPage.evaluate(sessionId => window.__agentPreview.dispose({ sessionId }), jobId).catch(error => { if (!browserLossFailure) throw error; }); }
+ finally { await jobPage.close().catch(error => { if (!browserLossFailure) throw error; }); }
}
}
finally {
@@ -1188,7 +1198,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
phase = 'ready';
} catch (error) {
phase = 'failed';
- throw new Error(`Preview recovery failed after render job: ${error.message}`);
+ throw browserLossFailure ?? new Error(`Preview recovery failed after render job: ${error.message}`);
}
} else if (!closing && phase !== 'failed') phase = 'ready';
}
diff --git a/tests/agent-runtime.test.js b/tests/agent-runtime.test.js
index 9be9c088..7763615c 100644
--- a/tests/agent-runtime.test.js
+++ b/tests/agent-runtime.test.js
@@ -121,7 +121,7 @@ function hitNearestLamp(ray) {
}
async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = false, deadlineMs,
- failDispose = false, heapSize = 1024, stallPrivatePage = false, blockPrivateCreate = false, resumeDelayMs = 0, suspendDelayMs = 0, failResume = false, hangMethod, failPrivateCreate = false, disposeGate, jobRender, sceneBake, exportedRecipe, runtimeOptions = {}, contextOverrides = {}, lookBytesPerPixel = 0, inspection } = {}) {
+ failDispose = false, heapSize = 1024, stallPrivatePage = false, blockPrivateCreate = false, resumeDelayMs = 0, suspendDelayMs = 0, failResume = false, hangMethod, failPrivateCreate = false, failPrivateCreateMessage, disposeGate, jobRender, sceneBake, exportedRecipe, runtimeOptions = {}, contextOverrides = {}, lookBytesPerPixel = 0, inspection } = {}) {
const events = [];
const sourceCalls = [];
const sandboxTimers = new Set();
@@ -155,7 +155,7 @@ async function reviewRuntimeHarness({ failShot, blockShot = false, oversize = fa
if (method === hangMethod && !this.privateTarget) return new Promise(() => {});
if (method === 'create' && this.privateTarget && failPrivateCreate) {
failPrivateCreate = false;
- throw new Error('Injected private create failure');
+ throw new Error(typeof failPrivateCreateMessage === 'string' ? failPrivateCreateMessage : 'Injected private create failure');
}
if (method === 'dispose' && disposeGate) await disposeGate;
if (method === 'resume' && failResume) throw new Error('Injected resume failure');
@@ -576,6 +576,41 @@ runTest('a failed resume after a render job fails the runtime, and the next scen
} finally { await runtime.close(); }
});
+runTest('a browser lost under a stills job fails the job with its size and limit, and the runtime recovers (#2018)', async () => {
+ let runJob;
+ const controller = { list: () => [], close: async () => {}, submit: async () => ({ jobId: 'job' }) };
+ const submitted = { kind: 'stills', width: 1920, height: 1080, stillCount: 12, fps: 24, passes: ['depth'] };
+ const settings = { ...submitted, frameCount: 12 };
+ const { runtime, sessionId } = await reviewRuntimeHarness({ failPrivateCreate: true, failPrivateCreateMessage: 'Target page, context or browser has been closed', failResume: true,
+ runtimeOptions: { jobs: join(tmpdir(), 'crucible-job-root-test'), memory: 1024 },
+ contextOverrides: { createAgentRenderJobs: async ({ run }) => { runJob = run; return controller; } } });
+ try {
+ await runtime.submitJob({ sessionId, expectedRevision: 0, settings: submitted });
+ const failure = await runJob({ jobId: randomUUID(), sessionId, directory: tmpdir(), snapshot: { recipe: {} }, settings,
+ signal: new AbortController().signal, progress: async () => {} }).then(() => assert.fail('the job must fail'), error => error);
+ assert.match(failure.message, /preview browser crashed before the first frame at 1920x1080 with depth pass/);
+ assert.match(failure.message, /Lower the size or the frame count, or raise --memory \(now \d+ MiB\)/);
+ assert.doesNotMatch(failure.message, /Preview recovery failed/);
+ const fresh = await runtime.create({ recipe: FRESH_RECIPE });
+ assert.equal((await runtime.apply({ sessionId: fresh.sessionId })).revision, 1);
+ } finally { await runtime.close(); }
+});
+
+runTest('a stills job that fails for another reason keeps its own message (#2018)', async () => {
+ let runJob;
+ const controller = { list: () => [], close: async () => {}, submit: async () => ({ jobId: 'job' }) };
+ const submitted = { kind: 'stills', width: 64, height: 64, stillCount: 2, fps: 24 };
+ const settings = { ...submitted, frameCount: 2 };
+ const { runtime, sessionId } = await reviewRuntimeHarness({ failPrivateCreate: true, failPrivateCreateMessage: 'Injected scene problem',
+ runtimeOptions: { jobs: join(tmpdir(), 'crucible-job-root-test') },
+ contextOverrides: { createAgentRenderJobs: async ({ run }) => { runJob = run; return controller; } } });
+ try {
+ await runtime.submitJob({ sessionId, expectedRevision: 0, settings: submitted });
+ await assert.rejects(runJob({ jobId: randomUUID(), sessionId, directory: tmpdir(), snapshot: { recipe: {} }, settings,
+ signal: new AbortController().signal, progress: async () => {} }), /^Error: Injected scene problem$/);
+ } finally { await runtime.close(); }
+});
+
runTest('a job still unwinding when the runtime restarts does not fail the new runtime (#1256)', async () => {
let runJob;
let openGate;