3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
// What a render job will cost on this machine, and what it did cost. The estimate comes from the scene's size and the job's
// settings alone, so the same inputs give the same answer. A correction learned from the last few jobs on this machine
// (kept in the jobs folder, never sent anywhere) can move it, within a fixed bound.
import { MAXIMUM_MEMORY_MIB } from '../src/agent-scene-capabilities.js';
import { MAX_TRACE_TRIANGLES } from '../src/path-tracer-scene.js';
const MIB = 1024 ** 2;
/** What scene_capabilities topic jobs says about the estimate and the report. The details are in docs/mcp.md. */
export const RENDER_ESTIMATE_NOTE = 'scene_render_submit returns estimate { seconds, peakMemoryMiB, basis } for this device before the job runs (from the scene size, the settings and the last 16 jobs here; about 2x either way). scene_render_result metrics.report holds measured time and peak memory, the ratio to the estimate, samplesDrawn for the path tracer, and fallbacks (what ran differently and why).';
/** Jobs kept in the history, newest last. Older ones are dropped. */
export const HISTORY_LIMIT = 16;
/** The history may scale the estimate by at most this factor either way, so one odd run cannot make it useless. */
export const CORRECTION_BOUND = 4;
/** The history needs this many runs of the same kind before it changes anything. */
const MINIMUM_HISTORY_RUNS = 2;
/**
* Seconds and bytes per unit of work, fitted to jobs measured with scripts/measure-render-estimate.mjs. A frame costs a fixed
* part, a part per pixel and a part per triangle; the scene is built once per job, which dominates large scenes. The path
* tracer's trace cost is per pixel, sample and bounce, and its acceleration structure costs per triangle.
*/
export const COST_MODEL = Object.freeze({
startupSeconds: 0.25,
setupSecondsPerTriangle: 3.3e-6,
// A placed asset is rebuilt from its glTF parts (skeleton, materials, textures) however few triangles it has.
setupSecondsPerAssetPart: 1.2e-3,
drawSecondsPerAssetPart: 3.5e-5,
drawFrameSeconds: 0.004,
drawSecondsPerPixel: 5e-8,
drawSecondsPerTriangle: 1e-8,
drawSecondsPerLight: 5e-4,
passFactor: 0.6,
imageCaptureSeconds: 0.02,
videoEncodeSeconds: 0.005,
pathTracerSecondsPerSampleBounce: Object.freeze({ cpu: 5e-7, gpu: 1e-7, native: 1e-7 }),
// The first GPU path trace in a runtime compiles its kernel; measured at about 10 s, and later traces do not pay it.
gpuTracerColdStartSeconds: 9.5,
pathTracerBuildSecondsPerTriangle: 3.7e-6,
pathTracerOutputSeconds: 0.03,
nodeBaseMiB: 185,
extraMiBPerPart: 1,
extraMiBPerThousandTriangles: 0.3,
maximumSceneExtraMiB: 250,
pathTracerExtraMiB: 30,
miBPerTexture: 4,
nodeBytesPerPixel: 24,
});
function textureCount(recipe) {
const text = JSON.stringify(recipe?.materials ?? []);
return (text.match(/"(?:map|texture|image|[a-zA-Z]+Map)"s*:/g) ?? []).length;
}
/**
* The numbers about the scene that the estimate uses, from the scene's inspection result (listing every node) and its recipe.
* Triangles count each placed node, so ten spheres cost ten spheres even when they share one geometry.
*/
export function sceneSizeFrom(inspection, recipe) {
const look = inspection?.look ?? {};
const listed = inspection?.nodes ?? [];
const listedTriangles = listed.reduce((total, node) => total + (node.triangles ?? 0), 0);
const scale = inspection?.truncated && listed.length ? (inspection.counts?.parts ?? listed.length) / listed.length : 1;
// The node list leaves out the parts of placed assets, and generated geometry (vegetation, crowds, arrays) has no node of its own;
// the resident vertex count covers both, so the larger of the two counts stands.
// Copies of one placed asset share its geometry in that count, so the part of it that no node lists is counted once per copy.
const residentTriangles = Math.round((inspection?.counts?.vertices ?? 0) / 2);
const listedScaled = Math.round(listedTriangles * scale);
const placements = (inspection?.assets ?? []).filter(placement => typeof placement.assetId === 'string');
const copiesPerAsset = placements.length / Math.max(1, new Set(placements.map(placement => placement.assetId)).size);
const generatedTriangles = Math.max(0, residentTriangles - listedScaled) * Math.max(1, copiesPerAsset);
return {
parts: inspection?.counts?.parts ?? 0,
assetParts: placements.reduce((total, placement) => total + (placement.parts ?? 0), 0),
triangles: listedScaled + generatedTriangles,
// The triangles the node list itself reports, never scaled or guessed: a floor of the real count.
listedTriangles,
materials: inspection?.counts?.materials ?? 0,
textures: textureCount(recipe),
lights: (look.pointLights ?? 0) + (look.spotLights ?? 0) + (look.areaLights ?? 0) + 1,
};
}
function round(value, digits) {
const scale = 10 ** digits;
return Math.round(value * scale) / scale;
}
// A kernel compile shorter than this is lost beside a trace; the native tracer's per-frame note uses the same line.
export const NOTEWORTHY_COMPILE_SECONDS = 1;
/**
* Adds one traced frame's kernel compile to the job's total. The path tracer report in the metrics describes only the last
* frame, so without this a film whose first frame compiled for seconds ended with no trace of it (#1949).
*/
export function addKernelCompile(total, frameReport) {
const seconds = frameReport?.compileSeconds;
if (!(seconds > 0)) return total;
return { seconds: (total?.seconds ?? 0) + seconds, frames: (total?.frames ?? 0) + 1 };
}
function kernelCompileFrom(metrics) {
const compile = metrics.kernelCompile;
if (!(compile?.seconds >= NOTEWORTHY_COMPILE_SECONDS)) return undefined;
const seconds = round(compile.seconds, 1);
const note = `First render of this scene variant: ${seconds.toFixed(1)} s of the job compiled kernels, which is not trace time and is counted in secondsRatio. A later job of the same variant in this server reuses them.`;
return { seconds, frames: compile.frames, note };
}
function pathTracerFrameSeconds({ size, settings, backend }) {
const renderer = settings.renderer;
const secondsPerSampleBounce = COST_MODEL.pathTracerSecondsPerSampleBounce[renderer.backend ?? backend] ?? COST_MODEL.pathTracerSecondsPerSampleBounce.cpu;
const traceSeconds = settings.width * settings.height * renderer.samples * renderer.bounces * secondsPerSampleBounce;
return Math.min(renderer.maxSeconds, traceSeconds) + size.triangles * COST_MODEL.pathTracerBuildSecondsPerTriangle + COST_MODEL.pathTracerOutputSeconds;
}
function rasterFrameSeconds({ size, settings }) {
const drawSeconds = COST_MODEL.drawFrameSeconds + settings.width * settings.height * COST_MODEL.drawSecondsPerPixel
+ size.triangles * COST_MODEL.drawSecondsPerTriangle + size.assetParts * COST_MODEL.drawSecondsPerAssetPart + size.lights * COST_MODEL.drawSecondsPerLight;
const outputSeconds = settings.kind === 'video' ? COST_MODEL.videoEncodeSeconds : COST_MODEL.imageCaptureSeconds;
return drawSeconds * (1 + ((settings.passes?.length ?? 0) + (settings.mattes?.length ?? 0)) * COST_MODEL.passFactor) + outputSeconds;
}
function peakMemoryMiB({ size, settings }) {
const sceneMiB = Math.min(COST_MODEL.maximumSceneExtraMiB, size.parts * COST_MODEL.extraMiBPerPart + (size.triangles / 1000) * COST_MODEL.extraMiBPerThousandTriangles);
return COST_MODEL.nodeBaseMiB + sceneMiB + size.textures * COST_MODEL.miBPerTexture
+ (settings.renderer ? COST_MODEL.pathTracerExtraMiB : 0) + (settings.width * settings.height * COST_MODEL.nodeBytesPerPixel) / MIB;
}
/** The kind of job a history entry and a new job share: same output kind and same way of rendering. */
export function jobKind(settings, backend) {
return `${settings.renderer ? `pathTracer-${settings.renderer.backend ?? backend}` : 'raster'}-${settings.kind === 'video' ? 'video' : 'image'}-${backend}`;
}
function medianOf(values) {
const sorted = [...values].sort((first, second) => first - second);
const middle = Math.floor(sorted.length / 2);
return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
}
function boundedRatio(ratio) {
return Math.min(CORRECTION_BOUND, Math.max(1 / CORRECTION_BOUND, ratio));
}
/** Median of measured over estimated, from the last runs of the same kind; 1 when there are too few. */
export function correctionFromHistory(history, kind) {
const runs = (history ?? []).filter(entry => entry.kind === kind && entry.estimatedSeconds > 0 && entry.measuredSeconds > 0);
if (runs.length < MINIMUM_HISTORY_RUNS) return { seconds: 1, memory: 1, runs: runs.length, secondsClamped: false, memoryFromHistory: false };
const rawSeconds = medianOf(runs.map(entry => entry.measuredSeconds / entry.estimatedSeconds));
const seconds = boundedRatio(rawSeconds);
const memoryRuns = runs.filter(entry => entry.estimatedPeakMemoryMiB > 0 && entry.measuredPeakMemoryMiB > 0);
const memory = memoryRuns.length >= MINIMUM_HISTORY_RUNS ? boundedRatio(medianOf(memoryRuns.map(entry => entry.measuredPeakMemoryMiB / entry.estimatedPeakMemoryMiB))) : 1;
return { seconds, memory, runs: runs.length, secondsClamped: seconds !== rawSeconds, memoryFromHistory: memoryRuns.length >= MINIMUM_HISTORY_RUNS };
}
/** Says where the numbers came from: the history is whatever sits in this jobs folder, not a named machine. */
function basisOf(correction, kind) {
if (correction.runs < MINIMUM_HISTORY_RUNS) return 'scene size';
const notes = [];
if (correction.secondsClamped) notes.push(`time factor limited to ${CORRECTION_BOUND}x`);
if (!correction.memoryFromHistory) notes.push('memory from scene size only');
return `scene size and the last ${correction.runs} ${kind} jobs in this jobs folder${notes.length ? ` (${notes.join('; ')})` : ''}`;
}
/**
* Expected time and peak memory of a job. `settings` is the normalized job settings (width, height, kind, frameCount, passes,
* renderer). The raw size-only numbers are `modelSeconds` and `modelPeakMemoryMiB`; the headline numbers apply the history.
*/
export function estimateRenderJob({ size, settings, backend, memoryBudgetMiB, history = [], gpuTracerWarm = true }) {
const frameSeconds = settings.renderer ? pathTracerFrameSeconds({ size, settings, backend }) : rasterFrameSeconds({ size, settings });
const coldStartSeconds = settings.renderer?.backend === 'gpu' && !gpuTracerWarm ? COST_MODEL.gpuTracerColdStartSeconds : 0;
const modelSeconds = COST_MODEL.startupSeconds + coldStartSeconds + size.triangles * COST_MODEL.setupSecondsPerTriangle + size.assetParts * COST_MODEL.setupSecondsPerAssetPart + frameSeconds * settings.frameCount;
const modelPeakMemoryMiB = peakMemoryMiB({ size, settings });
const correction = correctionFromHistory(history, jobKind(settings, backend));
const estimate = {
seconds: round(modelSeconds * correction.seconds, 1),
peakMemoryMiB: Math.round(modelPeakMemoryMiB * correction.memory),
framesPerSecond: round(settings.frameCount / Math.max(0.001, modelSeconds * correction.seconds), 2),
frameSeconds: round(frameSeconds * correction.seconds, 4),
fixedSeconds: round((modelSeconds - frameSeconds * settings.frameCount) * correction.seconds, 2),
basis: basisOf(correction, jobKind(settings, backend)),
modelSeconds: round(modelSeconds, 1),
modelPeakMemoryMiB: Math.round(modelPeakMemoryMiB),
scene: size,
};
if (memoryBudgetMiB !== undefined) estimate.renderTargetBudgetMiB = memoryBudgetMiB;
return estimate;
}
/** Facts about what did not run as asked: a budget that shrank the frame, a backend that fell back, a denoiser that was swapped. */
function frameWasReduced(metrics) {
return Boolean(metrics.width && metrics.requestedWidth && (metrics.width < metrics.requestedWidth || metrics.height < metrics.requestedHeight));
}
/** What to do about a reduced frame: raise --memory when the full frame fits under the server ceiling, else say what to shrink. */
function memoryAdvice(metrics) {
const running = metrics.memoryBudgetMiB ? `; this server runs with --memory ${metrics.memoryBudgetMiB}` : '';
const needs = ` The full frame needs a render budget of ${metrics.requiredMemoryMiB} MiB${running}.`;
const maximumMemoryMiB = metrics.maximumMemoryMiB ?? MAXIMUM_MEMORY_MIB;
if (metrics.requiredMemoryMiB > maximumMemoryMiB) {
return `${needs} --memory accepts at most ${maximumMemoryMiB} on this machine, so a restart cannot fit it. Reduce the textures, the geometry, the frame size, or turn off HDR or bloom to render at ${metrics.requestedWidth}x${metrics.requestedHeight}.`;
}
return `${needs} Restart it with --memory ${metrics.requiredMemoryMiB} or more to render at ${metrics.requestedWidth}x${metrics.requestedHeight}.`;
}
function fallbacksFrom(settings, metrics) {
const fallbacks = [];
if (frameWasReduced(metrics)) {
const neededBudget = metrics.requiredMemoryMiB ? memoryAdvice(metrics) : '';
fallbacks.push(`The memory budget reduced the frame from ${metrics.requestedWidth}x${metrics.requestedHeight} to ${metrics.width}x${metrics.height}.${neededBudget}`);
}
if (metrics.pathTracer?.backendNote) fallbacks.push(metrics.pathTracer.backendNote);
if (metrics.pathTracer?.denoiserNote) fallbacks.push(metrics.pathTracer.denoiserNote);
return fallbacks;
}
/** The one record of a finished job: estimate beside what was measured, the samples drawn and what fell back. */
export function renderReport({ estimate, settings, metrics }) {
const measuredSeconds = metrics.elapsedSeconds;
const measuredPeakMemoryMiB = Math.round(((metrics.peakNodeBytes ?? 0) + (metrics.peakBrowserHeap ?? 0)) / MIB);
const report = {
measured: { seconds: round(measuredSeconds, 2), peakMemoryMiB: measuredPeakMemoryMiB, frames: settings.frameCount, secondsPerFrame: round(measuredSeconds / settings.frameCount, 3) },
fallbacks: fallbacksFrom(settings, metrics),
};
if (estimate) {
report.estimate = { seconds: estimate.seconds, peakMemoryMiB: estimate.peakMemoryMiB, basis: estimate.basis };
report.secondsRatio = round(measuredSeconds / estimate.seconds, 2);
}
if (frameWasReduced(metrics)) report.frameReduced = true;
const kernelCompile = kernelCompileFrom(metrics);
if (kernelCompile) report.kernelCompile = kernelCompile;
const traced = metrics.pathTracer;
if (traced) {
// The path tracer's report describes the last frame; the samples asked for are in the job's settings.
if (traced.averageSamples !== undefined) report.samplesDrawn = traced.averageSamples;
if (settings.renderer) report.samplesRequested = settings.renderer.samples;
report.noise = noiseFrom(traced);
}
return report;
}
/**
* Path traced frames only. `relative` is the root mean square noise of a pixel over the frame's mean luminance, measured before
* any denoiser ran (see estimateRelativeNoise). The 0.03 line sits between a 4 sample and a 16 sample trace of the same lit test
* scene (0.038 and 0.020), where the grain stops being visible at full size.
*/
export const NOISY_RELATIVE_NOISE = 0.03;
function noiseFrom(traced) {
const relative = traced.estimatedNoise;
if (!Number.isFinite(relative)) return { relative: null, verdict: 'not measured: it needs at least 2 samples per pixel and a frame that is not black.' };
const rounded = round(relative, 4);
if (relative < NOISY_RELATIVE_NOISE) return { relative: rounded, verdict: 'clean enough: no change needed.' };
const advice = traced.denoised ? 'raise renderer.samples' : 'raise renderer.samples or set renderer.denoise';
return { relative: rounded, verdict: `noisy: ${advice}.` };
}
/** The history with this job added, trimmed to the newest HISTORY_LIMIT entries. */
export function historyWithJob(history, { settings, backend, estimate, report }) {
// The estimate was for the requested size, so a job that ran smaller says nothing about it.
if (!estimate || !report.measured.seconds || report.frameReduced) return history;
const entry = {
kind: jobKind(settings, backend),
estimatedSeconds: estimate.modelSeconds,
// A compile is paid once per scene variant and process; the estimate is for the render, so the history learns without it.
measuredSeconds: Math.max(0.01, round(report.measured.seconds - (report.kernelCompile?.seconds ?? 0), 2)),
estimatedPeakMemoryMiB: estimate.modelPeakMemoryMiB,
measuredPeakMemoryMiB: report.measured.peakMemoryMiB,
};
return [...history, entry].slice(-HISTORY_LIMIT);
}
/** Accepts only a plain list of well-formed entries from the history file; anything else is treated as no history. */
export function parseHistory(text) {
try {
const parsed = JSON.parse(text);
if (!Array.isArray(parsed)) return [];
const numeric = ['estimatedSeconds', 'measuredSeconds', 'estimatedPeakMemoryMiB', 'measuredPeakMemoryMiB'];
return parsed.filter(entry => entry && typeof entry.kind === 'string' && numeric.every(field => Number.isFinite(entry[field]))).slice(-HISTORY_LIMIT);
} catch {
return [];
}
}
/**
* The estimate of `sceneSizeFrom` counts triangles as vertices / 2, which is up to 1.5 times the real count of non-indexed geometry
* (vertices / 3), and it does not subtract the parts the tracer skips. So the submit check refuses only above this multiple of the
* cap; a scene between the cap and the margin goes through to the exact check made when the trace starts.
*/
export const TRIANGLE_ESTIMATE_SAFETY_FACTOR = 2;
/**
* The message of a path traced job the CPU tracer would refuse after it started, or undefined. The CPU tracer (the default
* backend) flattens the scene and takes at most MAX_TRACE_TRIANGLES triangles in total; the count here is the estimate of
* `sceneSizeFrom`. Only its generated and truncated part is approximate, so the safety margin applies to that part alone: a scene
* whose listed nodes (exact counts) are over the cap is refused at once, and otherwise only an estimate clearly over the margin
* is, with "about" in the message. The gpu and native backends have
* other bounds, checked when the trace starts. `canChooseBackend` is false when the caller cannot set renderer.backend (a runtime on
* SwiftShader, or a budget or quality job that picks the renderer), so the advice then names only ways that work.
*/
export function pathTraceTriangleProblem(size, renderer, { canChooseBackend = true, picksRenderer = false } = {}) {
if (!renderer || (renderer.backend !== undefined && renderer.backend !== 'cpu')) return undefined;
const listedTriangles = size.listedTriangles ?? 0;
const exactlyOverCap = listedTriangles > MAX_TRACE_TRIANGLES;
if (!exactlyOverCap && size.triangles <= MAX_TRACE_TRIANGLES * TRIANGLE_ESTIMATE_SAFETY_FACTOR) return undefined;
const count = exactlyOverCap ? `${listedTriangles} triangles` : `about ${size.triangles} triangles`;
const base = `settings.renderer: this scene has ${count} and the cpu path tracer takes at most ${MAX_TRACE_TRIANGLES} in total.`;
if (picksRenderer) return `${base} Simplify or drop meshes, or leave budget and quality out and pick the renderer yourself.`;
if (!canChooseBackend) return `${base} Simplify or drop meshes; this runtime has no GPU backend.`;
return `${base} Simplify or drop meshes, or set renderer.backend to "gpu" or "native", which hold more.`;
}