3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
import { readFileSync, unlinkSync, constants as fsConstants } from 'node:fs';
import { mkdir, open, readFile, writeFile, rename, link, unlink, rmdir, readdir, lstat, realpath } from 'node:fs/promises';
import { randomUUID, createHash } from 'node:crypto';
import { basename, isAbsolute, join, resolve } from 'node:path';
import { RENDER_PASS_NAMES, RENDER_OUTPUT_FORMATS, MAXIMUM_JOB_FPS, MAXIMUM_JOB_FRAMES, VIDEO_ENCODERS, AUDIO_FILE_PATTERN, MAXIMUM_VIDEO_SECONDS, MAXIMUM_JOB_SIDE, MAXIMUM_JOB_PIXELS, MAXIMUM_SEQUENCE_JOB_SIDE, MAXIMUM_SEQUENCE_JOB_PIXELS, MAXIMUM_STILLS_FRAMES, MAXIMUM_SQUARE_SEQUENCE_JOB_SIDE } from '../src/agent-scene-capabilities.js';
import { parseQuality } from './agent-render-quality.mjs';
import { estimateFrameBytes } from '../src/budget.js';
import { recipeAssetNeeds } from '../src/agent-scene-assets.js';
import { recipeMoves } from '../src/motion-hint.js';
import { normalizeFilmTimeline, filmTimelineDuration, videoFrameCount } from '../src/film-timeline.js';
import { OIDN_DEFAULT_SAMPLES, PATH_TRACER_DENOISERS, PATH_TRACER_JOB_LIMITS } from '../src/path-tracer-capabilities.js';
import { AGENT_ASSET_LIMITS, describeAssetBytes } from '../src/agent-asset-limits.js';
import { iconArtifactNames, normalizeIconSettings } from '../src/icon-sheet.js';
const UUID =/^[a-f0-9]{8}-[a-f0-9]{4}-[1-8][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/;
const STATES = ['queued', 'running', 'finalizing', 'completed', 'cancelling', 'cancelled', 'failed', 'interrupted'];
const TERMINAL = new Set(['completed', 'cancelled', 'failed', 'interrupted']);
/** Hard ceiling on one job's elapsed time. The deadline below it grows with the job's frames and pixels. */
export const MAXIMUM_JOB_DEADLINE_MS = 30 * 60 * 1000;
/** How long a job may wait behind other calls and for the scene page handoff before its render clock starts. */
export const MAXIMUM_QUEUE_WAIT_MS = 5 * 60 * 1000;
const BASE_DEADLINE_MS = 60 * 1000;
const FRAME_ALLOWANCE_MS = 2000;
const REFERENCE_FRAME_PIXELS = 1024 * 576;
/** A 1920x1080 PNG of a busy scene runs to several MiB, so a frame may be up to 12 MiB. */
const DEFAULT_LIMITS = Object.freeze({ maxRecords: 32, maxStoreBytes: 512 * 1024 ** 2, maxOutputBytes: 128 * 1024 ** 2, maxFrameBytes: 12 * 1024 ** 2, deadlineMs: MAXIMUM_JOB_DEADLINE_MS, queueWaitMs: MAXIMUM_QUEUE_WAIT_MS, cleanupMs: 5000 });
/** Time the page needs to start, draw the scene and encode the PNG around a path trace that runs for its whole maxSeconds. */
const PATH_TRACE_OVERHEAD_MS = 60 * 1000;
const ICON_FRAME_ALLOWANCE_MS = 4000;
const ICON_BREAKDOWN_ALLOWANCE_MS = 6000;
/**
* An icon is its own scene: the job loads the GLB and draws it up to four times to frame it, and draws it all again when the
* set shares one scale (relative, or several angles). A breakdown adds three draws of each item. Measured on the cpu backend
* at about 5 s per icon with a breakdown and 3 s without.
*/
function iconDeadlineMs({ frameCount, scale, angles, breakdown, items }) {
const passes = scale === 'relative' || angles > 1 ? 2 : 1;
return BASE_DEADLINE_MS + frameCount * ICON_FRAME_ALLOWANCE_MS * passes + (breakdown ? items.length * ICON_BREAKDOWN_ALLOWANCE_MS : 0);
}
/**
* 60 s plus 2 s per frame, scaled up for frames larger than 1024x576, never above the ceiling. A path trace may run for
* its own maxSeconds on every frame, so its deadline also reaches that plus the overhead and the trace stops itself first.
*/
export function jobDeadlineMs({ kind, frameCount, width, height, renderer, scale, angles, breakdown, items }, maximumMs = MAXIMUM_JOB_DEADLINE_MS) {
if (kind === 'icons') return Math.min(maximumMs, iconDeadlineMs({ frameCount, scale, angles, breakdown, items }));
const sizeScale = Math.max(1, (width * height) / REFERENCE_FRAME_PIXELS);
const frameDeadline = Math.ceil(BASE_DEADLINE_MS + frameCount * FRAME_ALLOWANCE_MS * sizeScale);
const traceDeadline = renderer?.type === 'pathTracer' ? Math.ceil(renderer.maxSeconds * 1000 * frameCount) + PATH_TRACE_OVERHEAD_MS : 0;
return Math.min(maximumMs, Math.max(frameDeadline, traceDeadline));
}
const MINIMUM_REGION_SIDE = 16;
/** A 16-bit PNG or an EXR of a busy 1920x1080 frame is bigger than the 8-bit PNG the frame limit was sized for. */
export const HIGH_PRECISION_FRAME_FACTOR = 4;
const MAXIMUM_JSON_BYTES = 1024 ** 2;
const MAXIMUM_ASSET_BYTES = AGENT_ASSET_LIMITS.maxBytes;
/** Sibling folder of the job directories where the runtime keeps GLB exports; the store never reads or counts it. */
export const EXPORT_DIRECTORY_NAME = 'exports';
/** Recent estimate-versus-measured pairs the runtime keeps beside the jobs; the store counts its bytes and never reads it. */
export const RENDER_HISTORY_FILE = 'render-history.json';
const STORE_FILES = ['store.json', 'store.lock', EXPORT_DIRECTORY_NAME, RENDER_HISTORY_FILE];
// A starter taking over a dead lock holds a claim file, and a starter publishing a lock or claim holds a temporary file, for a
// moment (a crash can leave one). A controller scanning the store must not call these foreign.
const LOCK_CLAIM = /^store\.lock\.(claim-[0-9a-f-]{36}|(claim-[0-9a-f-]{36}\.)?tmp-[0-9a-f-]{36})$/i;
const INTERNAL_FILES =new Set(['owner.json', 'snapshot.json', 'asset.glb', 'status.json', 'status.json.pending']);
function objectKeys(value, allowed, label) {
if (!value || Object.getPrototypeOf(value) !== Object.prototype) throw new TypeError(`${label} must be a plain object.`);
for (const key of Object.keys(value)) {
if (!allowed.includes(key)) throw new TypeError(`Unknown ${label} field: ${key}. Allowed fields: ${allowed.join(", ")}.`);
}
}
function boundedNumber(value, minimum, maximum, label, integer = false) {
if (!Number.isFinite(value) || value < minimum || value > maximum || (integer && !Number.isInteger(value))) {
throw new RangeError(`${label} must be within ${minimum} and ${maximum}${integer ? ' as an integer' : ''}.`);
}
return value;
}
function jsonCopy(value, maximum = MAXIMUM_JSON_BYTES) {
const ancestors = new Set();
let estimatedBytes = 0;
function validate(input, depth) {
if (depth > 64) throw new RangeError('JSON nesting exceeds 64 levels.');
estimatedBytes += typeof input === 'string' ? Buffer.byteLength(input) : 1;
if (estimatedBytes > maximum) throw new RangeError('Snapshot JSON exceeds its byte budget.');
if (input === null || typeof input === 'boolean' || typeof input === 'string') return;
if (typeof input === 'number' && Number.isFinite(input)) return;
if (!input || typeof input !== 'object' || ancestors.has(input)) throw new TypeError('Snapshot must contain finite, acyclic JSON.');
if (!Array.isArray(input) && Object.getPrototypeOf(input) !== Object.prototype) throw new TypeError('Snapshot must contain plain JSON.');
ancestors.add(input);
if (Array.isArray(input) && (input.length > maximum || Object.keys(input).length !== input.length || Object.keys(input).some((key, index) => key !== String(index)))) throw new TypeError('Snapshot arrays must be bounded and dense.');
for (const key of Reflect.ownKeys(input)) {
if (Array.isArray(input) && key === 'length') continue;
const descriptor = Object.getOwnPropertyDescriptor(input, key);
if (typeof key !== 'string' || key === '__proto__' || !descriptor.enumerable || !Object.hasOwn(descriptor, 'value')) throw new TypeError('Snapshot accessors and hidden fields are unsupported.');
validate(descriptor.value, depth + 1);
}
ancestors.delete(input);
}
validate(value, 0);
const text = JSON.stringify(value);
if (Buffer.byteLength(text) > maximum) throw new RangeError('Snapshot JSON exceeds its byte budget.');
return JSON.parse(text);
}
/** Seconds the recipe's physics or cloth bake covers, or undefined when it has neither. Both play on the motion clock. */
function bakedPhysicsDuration(recipe) {
const durations = [recipe?.physics?.duration, recipe?.cloth?.duration].filter(Number.isFinite);
return durations.length ? Math.max(...durations) : undefined;
}
/** A scene that places clips plays them on scene time, so a film of one runs as long as its longest clip, counted from the start. */
function placedClipDefaultDuration(input, clips, recipe, bakeSeconds) {
const asked = input.motion ?? input.animation;
if (!placesSceneClips(recipe, clips) || recipe.motion || bakeSeconds || input.kind === 'image' || !asked) return undefined;
const longest = Math.max(...clips.map(clip => clip.duration));
return Math.min(MAXIMUM_VIDEO_SECONDS, Math.max(0.1, longest - (Number(asked.start) || 0)));
}
/** In a scene recipe the snapshot clips are the placed asset clips, one per placement, each as long as it needs to play once. */
function placesSceneClips(recipe, clips) {
return recipe?.format === 'crucible.scene' && clips.length > 0;
}
function playsPhysics(recipe) {
return Boolean(recipe?.physics || recipe?.cloth || recipe?.particles);
}
/**
* Whether a job with no `motion` setting should still play the recipe's motion tracks, particles, physics or cloth from second 0.
* Without a motion clock every frame is frame 0, so such a stills or video job would come out frozen with no warning.
*/
function playsSimulationFromStart(input, recipe, timelineStart) {
return input.kind !== 'image' && timelineStart === undefined && input.animation === undefined
&& recipe?.format === 'crucible.scene' && recipeMoves(recipe);
}
/** The settings of a job as the store keeps them: every default filled in, every limit checked. */
export function normalizeJobSettings(input, clips, recipe) {
return settingsFrom(input, clips, recipe);
}
function settingsFrom(input, clips = [], recipe) {
if (input?.kind === 'icons') return normalizeIconSettings(input);
objectKeys(input, ['kind', 'width', 'height', 'fps', 'duration', 'animation', 'motion', 'rendererTime', 'passes', 'renderer', 'timeline', 'audio', 'encoder', 'transparent', 'format', 'multilayer', 'region', 'quality', 'stillCount', 'mattes'], 'settings');
if (!['image', 'stills', 'video'].includes(input.kind)) throw new TypeError('kind must be image, stills, video or icons.');
const largeFrame = input.kind === 'image';
const maximumSide = largeFrame ? MAXIMUM_JOB_SIDE : MAXIMUM_SEQUENCE_JOB_SIDE;
const maximumPixels = largeFrame ? MAXIMUM_JOB_PIXELS : MAXIMUM_SEQUENCE_JOB_PIXELS;
const width = boundedNumber(input.width, 64, maximumSide, 'width', true);
const height = boundedNumber(input.height, 64, maximumSide, 'height', true);
if (width * height > maximumPixels) {
const limit = largeFrame ? '3840x2160' : `1920x1080 (only an image job goes up to 3840x2160; a square frame fits up to ${MAXIMUM_SQUARE_SEQUENCE_JOB_SIDE}x${MAXIMUM_SQUARE_SEQUENCE_JOB_SIDE})`;
throw new RangeError(`width x height must not exceed ${limit} pixels.`);
}
if (input.kind === 'video' && (width % 2 || height % 2)) throw new RangeError('Video dimensions must be even.');
const fps = boundedNumber(input.fps ?? 24, 1, MAXIMUM_JOB_FPS, 'fps', true);
const timelineStart = timelineStartFrom(input, recipe);
// A physics-only film runs as long as its bake, counted from the motion start.
const bakeSeconds = recipe?.format === 'crucible.scene' && !recipe.motion ? bakedPhysicsDuration(recipe) : undefined;
const bakeDefault = bakeSeconds && input.kind !== 'image' && input.motion ? Math.max(0.1, bakeSeconds - (Number(input.motion.start) || 0)) : undefined;
const clipDefault = placedClipDefaultDuration(input, clips, recipe, bakeSeconds);
const defaultDuration = timelineStart === undefined || input.kind === 'image' ? bakeDefault ?? clipDefault ?? 1 : filmTimelineDuration(recipe.timeline) - timelineStart;
const framesAsked = stillsFrameCountFrom(input);
// A stills job that sets frames samples the motion clock every 1 / fps seconds, like a video; its duration is the span of those samples.
const duration = framesAsked === undefined
? boundedNumber(input.duration ?? defaultDuration, 0.1, MAXIMUM_VIDEO_SECONDS, 'duration')
: stillsSequenceDuration(input, framesAsked, fps);
const frameCount = input.kind === 'image' ? 1 : input.kind === 'stills' ? framesAsked ?? 8 : videoFrameCount(fps, duration);
if (frameCount > MAXIMUM_JOB_FRAMES) throw new RangeError(`Job exceeds ${MAXIMUM_JOB_FRAMES} frames. Lower fps or duration.`);
const result = { kind: input.kind, width, height, fps, duration, frameCount };
if (framesAsked !== undefined) result.stillCount = framesAsked;
if (input.passes !== undefined) result.passes = passesFrom(input.passes, input.kind);
if (input.renderer !== undefined) result.renderer = rendererFrom(input.renderer, input.passes);
if (input.mattes !== undefined) result.mattes = mattesFrom(input);
Object.assign(result, outputOptionsFrom(input, result));
if (input.quality !== undefined) result.quality = qualityFrom(input.quality, result.renderer);
const lastSampleTime = input.kind === 'image' ? 0 : input.kind === 'stills' ? duration : (frameCount - 1) / fps;
if (timelineStart !== undefined) {
if (input.motion !== undefined || input.animation !== undefined) throw new TypeError('A timeline owns the motion clock; leave motion and animation out.');
if (timelineStart + lastSampleTime > filmTimelineDuration(recipe.timeline)) throw new RangeError('Timeline range exceeds the timeline duration.');
result.timeline = { start: timelineStart };
}
if (input.encoder !== undefined) {
if (input.kind !== 'video') throw new TypeError('encoder needs kind video.');
if (!VIDEO_ENCODERS.includes(input.encoder)) throw new TypeError(`encoder must be one of ${VIDEO_ENCODERS.join(', ')}.`);
result.encoder = input.encoder;
}
if (input.audio !== undefined) result.audio = audioFrom(input.audio, input.kind);
const impliedMotion = input.motion === undefined && playsSimulationFromStart(input, recipe, timelineStart);
if (input.motion !== undefined || impliedMotion) {
const motionInput = input.motion ?? { start: 0 };
objectKeys(motionInput, ['start'], 'motion');
// Clips of placed assets play on scene time, so a scene that places one needs no authored motion.
const placesClips = recipe?.format === 'crucible.scene' && recipeAssetNeeds(recipe).some(need => need.copies > 1);
// A map sequence and a grid volume sequence read the same clock.
const sequenced = JSON.stringify(recipe?.materials ?? []).includes('"sequence":{')
|| (Array.isArray(recipe?.environment?.localVolumes) && recipe.environment.localVolumes.some(volume => Array.isArray(volume?.frames)));
if (!['crucible.scene', 'crucible.asset'].includes(recipe?.format) || (!recipe.motion && !placesClips && !sequenced && !(recipe.format === 'crucible.scene' && playsPhysics(recipe)))) {
throw new RangeError('Motion settings require a recipe with authored motion, physics (or cloth, particles), a placed clip, a map sequence or a grid volume sequence.');
}
const motionDuration = recipe.motion ? boundedNumber(recipe.motion.duration, 0.1, 30, 'motion duration') : undefined;
if (recipe.motion?.loop !== undefined && typeof recipe.motion.loop !== 'boolean') {
throw new TypeError('Motion loop must be boolean.');
}
const start = boundedNumber(motionInput.start === undefined ? 0 : motionInput.start, 0, 3600, 'motion start');
boundedNumber(start + lastSampleTime, 0, 3600, 'motion end');
const exceedsAuthoredMotion = recipe.motion && !recipe.motion.loop && start + lastSampleTime > motionDuration;
const exceedsBake = !recipe.motion && bakeSeconds && start + lastSampleTime > bakeSeconds;
if (exceedsAuthoredMotion && !impliedMotion) throw new RangeError('Motion range exceeds the authored duration.');
if (exceedsBake && !impliedMotion) {
throw new RangeError(`Motion range exceeds the physics bake duration of ${bakeSeconds} s; shorten the job or lengthen physics.duration.`);
}
// An implied start of 0 is only kept while the job fits the motion; a longer job keeps the old held pose rather than failing.
if (!(impliedMotion && (exceedsAuthoredMotion || exceedsBake))) result.motion = { start };
}
if (input.animation !== undefined) {
objectKeys(input.animation, ['clip', 'start'], 'animation');
if (clips.length === 0) throw new RangeError('animation needs an animation clip, and this scene has none. Drop animation from the settings, or add a clip to the scene first.');
const clip = boundedNumber(input.animation.clip, 0, clips.length - 1, 'clip', true);
const start = boundedNumber(input.animation.start ?? 0, 0, clips[clip].duration, 'animation start');
if (placesSceneClips(recipe, clips)) {
// Placed clips play on scene time, the clock of motion, and loop or hold on their own settings: animation.start is the motion start.
if (input.motion !== undefined) throw new TypeError('Send either motion or animation for a scene that places clips: both set the scene time the clips play on. Keep motion.start.');
result.motion = { start };
} else {
if (input.kind !== 'image' && start + duration > clips[clip].duration) throw new RangeError('Animation range exceeds the selected clip duration.');
result.animation = { clip, start };
}
}
if (input.rendererTime !== undefined) {
objectKeys(input.rendererTime, ['start', 'rate'], 'rendererTime');
result.rendererTime = { start: boundedNumber(input.rendererTime.start ?? 0, 0, 3600, 'renderer time start'), rate: boundedNumber(input.rendererTime.rate ?? 1, -16, 16, 'renderer time rate') };
} else {
result.rendererTime = { start: 0, rate: 1 };
}
boundedNumber(result.rendererTime.start + lastSampleTime * result.rendererTime.rate, 0, 3600, 'renderer time end');
return result;
}
function stillsFrameCountFrom(input) {
if (input.stillCount === undefined) return undefined;
if (input.kind !== 'stills') throw new TypeError('stillCount needs kind stills; a video sets its length with duration and fps.');
return boundedNumber(input.stillCount, 2, MAXIMUM_STILLS_FRAMES, 'stillCount (the most one stills job writes; render a longer clip in parts, each from its own motion.start)', true);
}
function stillsSequenceDuration(input, frames, fps) {
const span = (frames - 1) / fps;
if (input.duration !== undefined && Math.abs(input.duration - span) > 1e-9) {
throw new RangeError(`Leave duration out when stillCount is set: it is (stillCount - 1) / fps = ${Number(span.toFixed(4))} s here, and you sent ${input.duration}.`);
}
return span;
}
/** The optional transparent, format and region settings, kept only when they differ from the defaults so old records read the same. */
function outputOptionsFrom(input, { kind, width, height, renderer, passes }) {
const options = {};
const tracerNote = 'The path tracer writes an opaque 8-bit PNG of the whole frame.';
if (input.transparent !== undefined) {
if (typeof input.transparent !== 'boolean') throw new TypeError('transparent must be boolean.');
if (input.transparent && kind === 'video') throw new RangeError('H.264 video has no alpha: transparent needs kind image or stills.');
if (input.transparent && renderer) throw new RangeError(`transparent cannot combine with renderer. ${tracerNote}`);
if (input.transparent) options.transparent = true;
}
if (input.format !== undefined) {
if (!RENDER_OUTPUT_FORMATS.includes(input.format)) throw new TypeError(`format must be one of ${RENDER_OUTPUT_FORMATS.join(', ')}.`);
if (input.format !== 'png' && kind === 'video') throw new RangeError('A video is H.264: format needs kind image or stills.');
if (input.format !== 'png' && renderer) throw new RangeError(`format cannot combine with renderer. ${tracerNote}`);
if (input.format !== 'png') options.format = input.format;
}
if (input.multilayer !== undefined) {
if (typeof input.multilayer !== 'boolean') throw new TypeError('multilayer must be boolean.');
if (input.multilayer && (input.format !== 'exr' || !passes)) throw new RangeError('multilayer requires format exr and passes.');
if (input.multilayer) options.multilayer = true;
}
if (input.region !== undefined) {
objectKeys(input.region, ['x', 'y', 'width', 'height'], 'region');
if (renderer) throw new RangeError(`region cannot combine with renderer. ${tracerNote}`);
const region = {
x: boundedNumber(input.region.x, 0, width - 1, 'region x', true),
y: boundedNumber(input.region.y, 0, height - 1, 'region y', true),
width: boundedNumber(input.region.width, MINIMUM_REGION_SIDE, width, 'region width', true),
height: boundedNumber(input.region.height, MINIMUM_REGION_SIDE, height, 'region height', true),
};
if (region.x + region.width > width || region.y + region.height > height) throw new RangeError('region must lie inside the frame.');
if (kind === 'video' && (region.width % 2 || region.height % 2)) throw new RangeError('A video region must have even width and height.');
options.region = region;
}
return options;
}
const PATH_TRACER_BACKENDS = ['cpu', 'gpu', 'native'];
const FRACTIONAL_RENDERER_FIELDS = new Set(['maxSeconds', 'noiseThreshold']);
// quality final is the path tracer and the other tiers are the raster renderer; scene_render_submit fills in the renderer for final.
function qualityFrom(quality, renderer) {
parseQuality(quality);
if (quality === 'final' && !renderer) throw new TypeError('quality final needs the path tracer: submit it with scene_render_submit, which adds renderer, or give settings.renderer.');
if (quality !== 'final' && renderer) throw new TypeError(`quality ${quality} is the raster renderer and cannot combine with renderer.`);
return quality;
}
function rendererFrom(renderer, passes) {
objectKeys(renderer, ['type', 'backend', 'samples', 'bounces', 'denoise', 'seed', 'maxSeconds', 'noiseThreshold'], 'renderer');
if (renderer.type !== 'pathTracer') throw new TypeError('renderer.type must be pathTracer.');
if (passes !== undefined) throw new RangeError('The path tracer writes the lit image only and has no object ID, depth or normal pass: leave passes out, or leave renderer out to render the passes on the raster renderer.');
const denoise = renderer.denoise ?? true;
if (typeof denoise !== 'boolean' && !PATH_TRACER_DENOISERS.includes(denoise)) throw new TypeError(`renderer.denoise must be a boolean or one of ${PATH_TRACER_DENOISERS.join(', ')}.`);
const backend = renderer.backend ?? 'cpu';
if (!PATH_TRACER_BACKENDS.includes(backend)) throw new TypeError('renderer.backend must be cpu, gpu or native.');
const result = { type: 'pathTracer', backend, denoise };
for (const name of Object.keys(PATH_TRACER_JOB_LIMITS)) {
const { minimum, maximum, default: fallback } = PATH_TRACER_JOB_LIMITS[name];
const defaultValue = name === 'samples' && denoise === 'oidn' ? OIDN_DEFAULT_SAMPLES : fallback;
result[name] = boundedNumber(renderer[name] ?? defaultValue, minimum, maximum, `renderer ${name}`, !FRACTIONAL_RENDERER_FIELDS.has(name));
}
return result;
}
function timelineStartFrom(input, recipe) {
if (input.timeline === undefined) return undefined;
objectKeys(input.timeline, ['start'], 'timeline');
if (recipe?.format !== 'crucible.scene' || !recipe.timeline) throw new RangeError('Timeline settings require a recipe with a timeline.');
normalizeFilmTimeline(recipe.timeline, recipe);
return boundedNumber(input.timeline.start ?? 0, 0, MAXIMUM_VIDEO_SECONDS, 'timeline start');
}
function audioFrom(audio, kind) {
if (kind !== 'video') throw new TypeError('audio needs kind video.');
objectKeys(audio, ['file', 'offset', 'track'], 'audio');
if (audio.track !== undefined && !['aac', 'pcm'].includes(audio.track)) throw new TypeError('audio.track must be aac or pcm.');
if (typeof audio.file !== 'string' || !AUDIO_FILE_PATTERN.test(audio.file)) {
throw new TypeError('audio.file must be a file name in the configured audio folder ending in .wav, .mp3, .ogg or .flac.');
}
return { file: audio.file, offset: boundedNumber(audio.offset ?? 0, 0, 3600, 'audio offset'), track: audio.track ?? 'aac' };
}
function passesFrom(passes, kind) {
if (kind === 'video') throw new RangeError('Render passes need kind image or stills; a video holds the lit image only.');
if (!Array.isArray(passes) || passes.length < 1 || passes.length > RENDER_PASS_NAMES.length) throw new TypeError(`passes must list 1 to ${RENDER_PASS_NAMES.length} pass names.`);
for (const name of passes) {
if (!RENDER_PASS_NAMES.includes(name)) throw new RangeError(`Unknown render pass: ${name}. Use ${RENDER_PASS_NAMES.join(', ')}.`);
}
if (new Set(passes).size !== passes.length) throw new RangeError('passes must not repeat a name.');
return [...passes];
}
const MAXIMUM_MATTES = 8;
const MAXIMUM_MATTE_NAME_LENGTH = 120;
/** Refuses a path tracer backend this runtime cannot provide. The runner calls it for each frame and submit calls it first, so a doomed job is never queued. */
export function requireTracerBackendAvailable(renderer, runtimeBackend) {
if (renderer?.backend === 'gpu' && runtimeBackend === 'cpu') throw new Error('renderer.backend "gpu" needs a runtime on the GPU backend, and this one runs on SwiftShader.');
}
/** The ID mattes of a job: each names one object (part) or one material, and writes one gray PNG per frame. */
function mattesFrom(input) {
const { mattes, kind, renderer, format, multilayer } = input;
if (kind === 'video') throw new RangeError('mattes need kind image or stills; a video holds the lit image only.');
if (renderer !== undefined) throw new RangeError('mattes cannot combine with renderer: the path tracer has no object ID or Cryptomatte pass. Leave renderer out.');
if (format !== undefined && format !== 'png') throw new RangeError('mattes are 8-bit PNG files: leave format out or use png.');
if (multilayer) throw new RangeError('mattes cannot combine with multilayer. Leave multilayer out.');
if (!Array.isArray(mattes) || mattes.length < 1 || mattes.length > MAXIMUM_MATTES) throw new TypeError(`mattes must list 1 to ${MAXIMUM_MATTES} entries, each { object: "<part name>" } or { material: "<material name>" }.`);
return mattes.map((matte, index) => {
objectKeys(matte, ['object', 'material'], `mattes[${index}]`);
const kindsGiven = ['object', 'material'].filter(key => matte[key] !== undefined);
if (kindsGiven.length !== 1) throw new TypeError(`mattes[${index}] needs exactly one of object or material. Pass { object: "<part name>" } or { material: "<material name>" }.`);
const name = matte[kindsGiven[0]];
if (typeof name !== 'string' || name.length < 1 || name.length > MAXIMUM_MATTE_NAME_LENGTH) throw new TypeError(`mattes[${index}].${kindsGiven[0]} must be a name of 1 to ${MAXIMUM_MATTE_NAME_LENGTH} characters. scene_render_result metrics.objectNames and metrics.cryptomatte list the names of a render.`);
return { [kindsGiven[0]]: name };
});
}
/** Every artifact filename of a job, lit image first and then each requested pass and matte, frame by frame. */
export function artifactNames(kind, passes = [], format = 'png', multilayer = false, stillCount = 8, mattes = [], iconSettings = {}) {
if (kind === 'video') return ['render.mp4'];
// An icons job has one frame per icon (an item at one angle); a breakdown adds one strip per item.
if (kind === 'icons') return iconArtifactNames(stillCount, { itemCount: iconSettings.breakdown ? iconSettings.items.length : 0 });
const extension = format === 'exr' ? 'exr' : 'png';
const stems = kind === 'image' ? ['render'] : Array.from({ length: stillCount }, (_, index) => `frame-${index}`);
if (multilayer) return stems.map(stem => `${stem}.${extension}`);
return stems.flatMap(stem => [`${stem}.${extension}`, ...passes.map(pass => `${stem}-${pass}.${extension}`), ...mattes.map((_, index) => `${stem}-matte-${index}.png`)]);
}
function artifactMimeType(settings, filename = '') {
if (settings.kind === 'video') return 'video/mp4';
if (settings.kind === 'icons' && filename.endsWith('.json')) return 'application/json';
return settings.format === 'exr' ? 'image/x-exr' : 'image/png';
}
/**
* A frame larger than 1920x1080 is never silently shrunk: when the memory budget reduced it, the error names the --memory
* the size needs (frame targets at the size asked for, plus the geometry and textures the budget already held), or says the
* size cannot fit under the ceiling. Returns undefined when the frame was rendered at the size asked for, or a 1080p-class size.
*/
export function oversizeFrameBudgetError({ requested, rendered, plan, memoryMiB, maximumMemoryMiB }) {
const exceedsStandardFrame = requested.width > MAXIMUM_SEQUENCE_JOB_SIDE || requested.height > MAXIMUM_SEQUENCE_JOB_SIDE
|| requested.width * requested.height > MAXIMUM_SEQUENCE_JOB_PIXELS;
if (!exceedsStandardFrame || (rendered.width >= requested.width && rendered.height >= requested.height)) return undefined;
const frameOptions = { hdr: plan.hdr, depth: plan.depth, bloom: plan.bloom ?? true };
const renderedFrameBytes = estimateFrameBytes(rendered.width, rendered.height, plan.samples, 2048, frameOptions);
const heldBytes = Math.max(0, plan.estimatedBytes - renderedFrameBytes);
const neededMiB = Math.ceil((heldBytes + estimateFrameBytes(requested.width, requested.height, plan.samples, 2048, frameOptions)) / 1024 ** 2);
const sizes = `${requested.width}x${requested.height} was reduced to ${rendered.width}x${rendered.height} by --memory ${memoryMiB}`;
if (neededMiB > maximumMemoryMiB) {
return new RangeError(`${sizes}. That size needs about ${neededMiB} MiB, over the ${maximumMemoryMiB} MiB ceiling. Ask for a smaller size or shrink the scene's textures.`);
}
return new RangeError(`${sizes}. That size needs about ${neededMiB} MiB. Fix: start the server with --memory ${neededMiB} or more.`);
}
/** The frame byte limit grows with the pixels beyond 1920x1080, so a 3840x2160 PNG may be up to four times the 1080p limit. */
export function frameByteLimitFor(width, height, baseBytes) {
return baseBytes * Math.max(1, Math.ceil((width * height) / MAXIMUM_SEQUENCE_JOB_PIXELS));
}
function artifactByteLimit(settings, limits) {
// The atlas of a full icon set is the largest file; the combined output limit still caps the whole job.
if (settings.kind === 'video' || settings.kind === 'icons') return limits.maxOutputBytes;
const frameBytes = frameByteLimitFor(settings.width, settings.height, limits.maxFrameBytes);
return settings.format === undefined ? frameBytes : frameBytes * HIGH_PRECISION_FRAME_FACTOR;
}
function allowedFile(name) {
const passNames = RENDER_PASS_NAMES.join('|');
const artifact = new RegExp(`^(?:render\\.(?:png|mp4|exr)|(?:icon-\\d{1,3}|breakdown-\\d{1,3}|atlas)\\.png|icons\\.json|(?:render|frame-\\d{1,3})(?:-(?:${passNames}|matte-[0-7]))?\\.(?:png|exr))(?:\\.partial)?$`);
return INTERNAL_FILES.has(name) || artifact.test(name);
}
const FILE_IN_USE_CODES = new Set(['EBUSY', 'EPERM', 'EACCES']);
// 7 attempts with 10 ms doubling waits is 630 ms in the worst case.
const STATUS_RENAME_ATTEMPTS = 7;
const STATUS_RENAME_BACKOFF_MS = 10;
async function regularFile(path) {
const details = await lstat(path);
if (!details.isFile() || details.isSymbolicLink() || details.nlink !== 1) throw new Error('Job files must be regular files without links or aliases.');
return details;
}
async function readJson(path, maximum = MAXIMUM_JSON_BYTES) {
const details = await regularFile(path);
if (details.size > maximum) throw new RangeError('Persisted job JSON exceeds its budget.');
return jsonCopy(JSON.parse(await readFile(path, 'utf8')), maximum);
}
async function hashFile(path, maximum) {
const before = await regularFile(path);
if (!before.size || before.size > maximum) throw new RangeError('Artifact exceeds its output byte budget or is empty.');
const handle = await open(path, 'r');
try {
const opened = await handle.stat();
if (opened.dev !== before.dev || opened.ino !== before.ino || opened.nlink !== 1) throw new Error('Artifact changed while opening.');
const hash = createHash('sha256');
let bytes = 0;
for await (const chunk of handle.createReadStream({ autoClose: false })) {
bytes += chunk.length;
if (bytes > maximum) throw new RangeError('Artifact exceeds its output byte budget.');
hash.update(chunk);
}
const after = await handle.stat();
if (bytes !== before.size || after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs) throw new Error('Artifact changed while hashing.');
return { bytes, sha256: hash.digest('hex') };
} finally {
await handle.close();
}
}
/** Persistent controller. The trusted runner must stop its owned browser and encoder when aborted. */
// `lockTakeoverHooks` is a seam for tests that need to pause a process at a chosen point of the lock takeover, or to replace how a
// job file is deleted (`removeJobFile`).
export async function createAgentRenderJobs({ root, run, acquireGpu, limits: inputLimits = {}, lockTakeoverHooks = {} }) {
if (typeof root !== 'string' || !isAbsolute(root)) throw new TypeError('Job root must be an explicit absolute directory.');
if (typeof run !== 'function') throw new TypeError('run must be an async render callback.');
if (acquireGpu !== undefined && typeof acquireGpu !== 'function') throw new TypeError('acquireGpu must be an async function that takes the GPU and returns its release function.');
objectKeys(inputLimits, Object.keys(DEFAULT_LIMITS), 'limits');
const limits = { ...DEFAULT_LIMITS, ...inputLimits };
for (const [key, value] of Object.entries(limits)) boundedNumber(value, 1, DEFAULT_LIMITS[key], key, true);
root = resolve(root);
await mkdir(root, { recursive: true, mode: 0o700 });
const rootDetails = await lstat(root);
if (!rootDetails.isDirectory() || rootDetails.isSymbolicLink()) throw new Error('Job root must be a real directory without links.');
root = await realpath(root);
const lockPath = join(root, 'store.lock');
const lockToken = randomUUID();
const jobs = new Map();
let active;
let closed = false;
let admission;
let storeId;
let lockHeld = false;
// Writes the whole text to a private temporary file, then hard-links it to `targetPath` (atomic, EEXIST when the target is
// taken) and removes the temporary name. The target therefore never exists empty or half written, even if the process is killed
// or the disk fills. Where hard links are not supported it falls back to creating the target with `wx` and writing into it.
// The file operations used here can be replaced through `lockTakeoverHooks.fileOperations`, so tests can make one of them fail.
const removeJobFile = lockTakeoverHooks.removeJobFile ?? unlink;
const publishOperations = { writeFile, link, open, unlink, rename, ...lockTakeoverHooks.fileOperations };
// Removing the temporary name can fail for a moment (a virus scanner holding the file on Windows), so it is retried with a
// short backoff. The target must never be left with two names: that makes it fail the single-link check of the store.
async function removeTemporaryName(temporaryPath) {
let lastError;
for (let attempt = 0; attempt < 5; attempt += 1) {
try {
await publishOperations.unlink(temporaryPath);
return;
} catch (error) {
if (error.code === 'ENOENT') return;
lastError = error;
await new Promise(resolveWait => setTimeout(resolveWait, 20 * 2 ** attempt));
}
}
throw lastError;
}
async function publishFile(targetPath, text) {
const temporaryPath = `${targetPath}.tmp-${randomUUID()}`;
let linked = false;
try {
await publishOperations.writeFile(temporaryPath, text, { flag: 'wx', mode: 0o600 });
await lockTakeoverHooks.beforeLockLink?.();
try {
await publishOperations.link(temporaryPath, targetPath);
linked = true;
} catch (error) {
if (!['EPERM', 'ENOTSUP', 'ENOSYS', 'EOPNOTSUPP'].includes(error.code)) throw error;
const target = await publishOperations.open(targetPath, 'wx', 0o600);
try {
await target.writeFile(text);
} catch (writeError) {
await target.close().catch(() => {});
await unlink(targetPath).catch(() => {});
throw writeError;
}
await target.close();
}
await lockTakeoverHooks.afterLockLink?.();
if (linked) {
try {
await removeTemporaryName(temporaryPath);
} catch (error) {
// Give the target back (only if it is still the file this call wrote) rather than keep a file with two names.
if (await readFile(targetPath, 'utf8').catch(() => null) === text) await unlink(targetPath).catch(() => {});
throw new Error(`Could not finish writing ${basename(targetPath)} (${error.code ?? error.message}); it was not kept. Close any program that holds files in the jobs folder and try again.`);
}
}
} finally {
await unlink(temporaryPath).catch(() => {});
}
}
async function createLockFile() {
await publishFile(lockPath, JSON.stringify({ schemaVersion: 1, pid: process.pid, token: lockToken }));
lockHeld = true;
process.once('exit', releaseLockOnExit);
}
// Opens a small coordination file once and checks and reads it through the same handle, so the size check and the read cannot
// see two different files (CodeQL js/file-system-race). A path that is not a plain file, or is larger than `maximum`, gives null.
async function readSmallFile(path, maximum) {
const handle = await open(path, fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0));
try {
const details = await handle.stat();
if (!details.isFile() || details.size > maximum) return null;
return await handle.readFile('utf8');
} finally {
await handle.close();
}
}
const LIVE_OWNER_MESSAGE = 'The job store is already owned by a live process. If another Crucible server is running, start this one with its own --jobs folder.';
// Reads a lock another process may be writing. A file that does not parse is a starter that has not finished (or a lock that was
// cut short), so it is treated as owned by a live process and never surfaces a raw SyntaxError. The hard link a starter makes
// for a moment is allowed here, unlike in the job files.
async function readLockFile(path) {
const text = await readSmallFile(path, 4096);
if (text === null) throw new Error('Unrecognized job store lock.');
try {
return JSON.parse(text);
} catch (error) {
if (error instanceof SyntaxError) throw new Error(`The job store lock ${path} is held by a live process or is incomplete. If no other Crucible server uses this folder, delete ${path}.`);
throw error;
}
}
// Throws unless the file at `path` is a well-formed lock whose owner process no longer exists.
// Limit: the operating system can give a dead owner's PID to an unrelated process, which then looks live and is refused until
// a person removes the file.
async function assertDeadOwner(path) {
const owner = await readLockFile(path);
objectKeys(owner, ['schemaVersion', 'pid', 'token'], 'store lock');
if (owner.schemaVersion !== 1 || !UUID.test(owner.token)) throw new Error('Unrecognized job store lock.');
boundedNumber(owner.pid, 1, 2147483647, 'lock process ID', true);
let ownerAlive = true;
try {
process.kill(owner.pid, 0);
} catch (failure) {
if (failure.code === 'ESRCH') ownerAlive = false;
else if (failure.code !== 'EPERM') throw failure;
}
if (ownerAlive) throw new Error(LIVE_OWNER_MESSAGE);
return owner.token;
}
// A lock left by a killed server (the client's SIGTERM is TerminateProcess on Windows and runs no exit handler) names a dead
// PID, and is taken over only when the file is a well-formed lock; a live PID or an unrecognized file is still refused.
//
// Why two processes cannot both end up owning the store: the shared lock path is never renamed or moved aside. A taker first
// reads the dead lock (token T0), then creates the claim file `store.lock.claim-<T0>` with `wx`, so only one process can hold
// the claim for T0. T0 can never become live again (its owner is gone), and the only process that replaces T0 with a new lock
// is the claim holder. A slow reader that validated T0 earlier and gets the claim later re-reads the lock under the claim, sees
// a different token (or no file), and takes over nothing. The holder unlinks the lock only while the file still holds T0.
// The claim is published whole (never empty), so a process killed while holding it is the only way one stays behind: that
// blocks the takeover of that one token, which is rare but real. The claim records its owner; one whose process is gone, or
// that cannot be read, is reported with the exact file to delete and is not recovered by itself.
// Returns 'lock-changed' when the lock under the claim was no longer the one this call validated (another process replaced
// it), and otherwise when the lock is gone or taken over; the caller then tries to create its own or starts again.
async function takeOverDeadOwnerLock(isRetry = false) {
let deadOwnerToken;
try {
deadOwnerToken = await assertDeadOwner(lockPath);
} catch (error) {
if (error.code === 'ENOENT') return 'gone';
throw error;
}
await lockTakeoverHooks.beforeLockClaim?.();
const claimPath = `${lockPath}.claim-${deadOwnerToken}`;
try {
await publishFile(claimPath, JSON.stringify({ pid: process.pid, token: lockToken }));
} catch (error) {
if (error.code !== 'EEXIST') throw error;
await refuseDeadClaim(claimPath);
if (isRetry) throw new Error(LIVE_OWNER_MESSAGE);
await waitForClaimToEnd(claimPath);
return takeOverDeadOwnerLock(true);
}
try {
let current;
try {
current = await readLockFile(lockPath);
} catch (error) {
if (error.code === 'ENOENT') return 'gone';
throw error;
}
if (current.token !== deadOwnerToken) return 'lock-changed';
await unlink(lockPath);
await createLockFile();
return 'taken-over';
} catch (error) {
if (error.code === 'EEXIST') throw new Error(LIVE_OWNER_MESSAGE);
throw error;
} finally {
await unlink(claimPath).catch(() => {});
}
}
// A claim whose process is gone, or that cannot be read, is not recovered here: someone confirms nothing owns the store and
// deletes the named file.
async function refuseDeadClaim(claimPath) {
const interrupted = new Error(`A job store takeover was interrupted or left an unreadable claim. Confirm that no process owns the store, then delete ${basename(claimPath)} in the jobs folder.`);
let claim;
try {
const text = await readSmallFile(claimPath, 4096);
if (text === null) throw interrupted;
claim = JSON.parse(text);
} catch (error) {
if (error.code === 'ENOENT') return; // The holder finished.
throw interrupted;
}
if (!Number.isInteger(claim?.pid) || claim.pid < 1) throw interrupted;
try {
process.kill(claim.pid, 0);
} catch (failure) {
if (failure.code === 'ESRCH') throw interrupted;
}
}
async function waitForClaimToEnd(claimPath) {
const deadline = Date.now() + 2000;
while (Date.now() < deadline) {
try {
await lstat(claimPath);
} catch (error) {
if (error.code === 'ENOENT') return;
throw error;
}
await new Promise(resolveWait => setTimeout(resolveWait, 20));
}
}
// When the lock changed between this call's first look and its claim (a starter took the dead lock over and then died too),
// the whole acquire starts again once, so the new dead lock can be taken over instead of being reported as live.
async function acquireLock(isSecondAttempt = false) {
try {
await createLockFile();
return;
} catch (error) {
if (error.code !== 'EEXIST') throw error;
}
const outcome = await takeOverDeadOwnerLock();
if (lockHeld) return;
if (outcome === 'lock-changed' && !isSecondAttempt) return acquireLock(true);
try {
await createLockFile();
} catch (retryError) {
if (retryError.code === 'EEXIST') throw new Error(LIVE_OWNER_MESSAGE);
throw retryError;
}
}
// A process that ends without a clean close (an exit call, an uncaught error, a signal Node turns into an exit) still gives the
// lock back, so the next run does not meet a "stale" lock. Only the owner's own token is removed.
function releaseLockOnExit() {
if (!lockHeld) return;
try {
if (JSON.parse(readFileSync(lockPath, 'utf8')).token === lockToken) unlinkSync(lockPath);
} catch {
// The lock is already gone or no longer ours: nothing to give back.
}
lockHeld = false;
}
async function unlock() {
if (!lockHeld) return;
if ((await readJson(lockPath, 4096)).token !== lockToken) throw new Error('Job store lock ownership changed.');
await unlink(lockPath);
lockHeld = false;
process.removeListener('exit', releaseLockOnExit);
}
async function directoryFor(jobId) {
if (!UUID.test(jobId)) throw new Error('Unknown job ID.');
const directory = join(root, jobId);
const details = await lstat(directory);
if (!details.isDirectory() || details.isSymbolicLink() || await realpath(directory) !== directory) throw new Error('Job directory must remain inside its owned store.');
const owner = await readJson(join(directory, 'owner.json'), 4096);
objectKeys(owner, ['schemaVersion', 'storeId', 'jobId'], 'job owner');
if (owner.schemaVersion !== 1 || owner.storeId !== storeId || owner.jobId !== jobId) throw new Error('Foreign job directory ownership.');
for (const filename of await readdir(directory)) {
if (!allowedFile(filename)) throw new Error(`Foreign job entry: ${filename}.`);
await regularFile(join(directory, filename));
}
return directory;
}
// status.json goes first and owner.json last. A release that fails midway then leaves a folder that has no status (so it never
// reads as a job with missing artifacts) and still proves it is ours, and a retry or the next start can finish it.
async function removeJobDirectory(directory) {
const filenames = await readdir(directory);
const rank = filename => filename === 'status.json' ? 0 : filename === 'owner.json' ? 2 : 1;
const ordered = [...filenames].sort((left, right) => rank(left) - rank(right));
// Refuse an alias or a foreign entry before anything is deleted.
for (const filename of ordered) await regularFile(join(directory, filename));
for (const filename of ordered) await removeJobFile(join(directory, filename));
await rmdir(directory);
}
// A folder that lost owner.json (what the 0.20.0 release left) is ours when its status names the same job and it holds only job files.
async function statusNamesJob(directory, filenames, jobId) {
if (!filenames.includes('status.json') || !filenames.every(allowedFile)) return false;
try {
return (await readJson(join(directory, 'status.json'))).jobId === jobId;
} catch {
return false;
}
}
function describeUnprovenFolder(directory) {
return `Job folder ${directory} has no owner file and cannot be proven to belong to this store. Delete that folder by hand and try again.`;
}
/**
* Finishes a release that stopped halfway. A job folder with no files, one that still has its owner file but lost its status or
* snapshot, or one that lost only its owner file is what such a release leaves. Returns true when the folder is gone (or was never
* there), false for a whole job.
*/
async function discardRemnant(jobId) {
const directory = join(root, jobId);
let details;
try {
details = await lstat(directory);
} catch (error) {
if (error.code === 'ENOENT') return true;
throw error;
}
if (!details.isDirectory() || details.isSymbolicLink() || await realpath(directory) !== directory) return false;
const filenames = await readdir(directory);
if (filenames.length) {
if (!filenames.includes('owner.json')) {
if (!await statusNamesJob(directory, filenames, jobId)) throw new Error(describeUnprovenFolder(directory));
} else {
if (filenames.includes('status.json') && filenames.includes('snapshot.json')) return false;
const owner = await readJson(join(directory, 'owner.json'), 4096);
if (owner.storeId !== storeId || owner.jobId !== jobId) return false;
}
}
await removeJobDirectory(directory);
return true;
}
// A file another program holds open (a viewer, a sync client) cannot be deleted on Windows yet. That is not a reason to take the
// store down: the next submit, release or start finishes the job.
async function tryDiscardRemnant(jobId) {
try {
return await discardRemnant(jobId) ? 'gone' : 'whole';
} catch (error) {
if (!FILE_IN_USE_CODES.has(error.code)) throw error;
return 'in-use';
}
}
async function directoryBytes(directory) {
let bytes = 0;
for (const filename of await readdir(directory)) bytes += (await regularFile(join(directory, filename))).size;
return bytes;
}
async function releaseFinishedJob(jobId, job) {
await job.persistence.catch(() => {});
// A retry after a failed release finds a half-deleted folder: finish it here instead of failing on the missing owner file.
try {
if (!await discardRemnant(jobId)) await removeJobDirectory(await directoryFor(jobId));
} catch (error) {
if (!FILE_IN_USE_CODES.has(error.code)) throw error;
throw new Error(`Job ${jobId} is only partly released: a file in ${join(root, jobId)} is open in another program. Close it and call release again.`, { cause: error });
}
jobs.delete(jobId);
}
/** Completed jobs whose result was read, oldest first: the only jobs the store may release on its own. */
function fetchedJobsOldestFirst() {
return [...jobs.entries()].filter(([, job]) => job.fetched && job.record.state === 'completed' && active !== job);
}
/**
* Frees `neededBytes` for a new job. When the store is too full, it releases jobs whose output was already read through
* result(), oldest first, and stops as soon as the job fits. A job whose output was never read stays.
*/
async function makeRoomInStore(neededBytes) {
let existingBytes = await storeBytes();
const releasable = fetchedJobsOldestFirst();
while (existingBytes + neededBytes > limits.maxStoreBytes && releasable.length) {
const [jobId, job] = releasable.shift();
await releaseFinishedJob(jobId, job);
existingBytes = await storeBytes();
}
if (existingBytes + neededBytes <= limits.maxStoreBytes) return;
const freeMiB = Math.max(0, Math.floor((limits.maxStoreBytes - existingBytes) / 1024 ** 2));
const neededMiB = Math.ceil(neededBytes / 1024 ** 2);
throw new RangeError(`Job store lacks space for the snapshot and output reservation: it needs ${neededMiB} MiB free and has ${freeMiB} MiB. Call scene_render_release on finished jobs you no longer need (jobs whose result you already read are released automatically, oldest first).`);
}
async function storeBytes() {
let bytes = 0;
for (const name of await readdir(root)) {
if (name === EXPORT_DIRECTORY_NAME || LOCK_CLAIM.test(name)) continue;
if (['store.json', 'store.lock', RENDER_HISTORY_FILE].includes(name)) {
bytes += (await regularFile(join(root, name))).size;
} else {
if (isPending(name)) {
const directory = await pendingDirectory(name);
for (const filename of await readdir(directory)) bytes += (await regularFile(join(directory, filename))).size;
continue;
}
if (!UUID.test(name)) throw new Error(`Foreign store entry: ${name}.`);
const outcome = await tryDiscardRemnant(name);
if (outcome === 'gone') continue;
bytes += await directoryBytes(outcome === 'in-use' ? join(root, name) : await directoryFor(name));
}
}
if (bytes > limits.maxStoreBytes) throw new RangeError('Job store exceeds its total byte budget. Release completed jobs.');
return bytes;
}
function isPending(name) {
return name.startsWith(`.pending-${storeId}-`) && UUID.test(name.slice(`.pending-${storeId}-`.length));
}
async function pendingDirectory(name) {
if (!isPending(name)) throw new Error('Unrecognized pending admission.');
const directory = join(root, name);
const details = await lstat(directory);
if (!details.isDirectory() || details.isSymbolicLink() || await realpath(directory) !== directory) throw new Error('Pending admission must remain inside its owned store.');
// Quarantine incomplete writes and foreign files; never interpret their contents.
for (const filename of await readdir(directory)) await regularFile(join(directory, filename));
return directory;
}
function getJob(jobId) {
if (typeof jobId !== 'string' || !UUID.test(jobId) || !jobs.has(jobId)) throw new Error('Unknown job ID.');
return jobs.get(jobId);
}
// A program that polls status.json (an agent's shell on Windows) can hold it open for a moment, and the rename onto it then fails
// with EPERM, EBUSY or EACCES. Retry for under a second. If it stays blocked, the last good status.json stays on disk, the job
// keeps running from its in-memory record, and a warning says so: a status write must never fail a render.
async function replaceStatusFile(job, pending, target) {
for (let attempt = 0; attempt < STATUS_RENAME_ATTEMPTS; attempt += 1) {
try {
await publishOperations.rename(pending, target);
return;
} catch (error) {
if (!FILE_IN_USE_CODES.has(error.code)) throw error;
if (attempt === STATUS_RENAME_ATTEMPTS - 1) {
job.statusWarning = `Could not update status.json (${error.code}); another program holds it open. The previous status stays on disk until it is released.`;
process.emitWarning(job.statusWarning, { code: 'CRUCIBLE_STATUS_BLOCKED' });
await publishOperations.unlink(pending).catch(() => {});
return;
}
await new Promise(resolveWait => setTimeout(resolveWait, STATUS_RENAME_BACKOFF_MS * 2 ** attempt));
}
}
}
function persist(job, record = job.record) {
const text = JSON.stringify(record);
// A write that failed already told its own caller; it must not make every later write reject with the same old error.
job.persistence = job.persistence.catch(() => {}).then(async () => {
const directory = await directoryFor(job.record.jobId);
const pending = join(directory, 'status.json.pending');
await writeFile(pending, text, { mode: 0o600 });
await replaceStatusFile(job, pending, join(directory, 'status.json'));
});
return job.persistence;
}
async function recover() {
const names = await readdir(root);
if (!names.includes('store.json')) {
if (names.some(name => name !== 'store.lock' && name !== EXPORT_DIRECTORY_NAME && !LOCK_CLAIM.test(name))) throw new Error('Job root contains foreign entries without store ownership.');
storeId = randomUUID();
await writeFile(join(root, 'store.json'), JSON.stringify({ schemaVersion: 1, storeId }), { flag: 'wx', mode: 0o600 });
} else {
const metadata = await readJson(join(root, 'store.json'), 4096);
objectKeys(metadata, ['schemaVersion', 'storeId'], 'store metadata');
if (metadata.schemaVersion !== 1 || !UUID.test(metadata.storeId)) throw new Error('Unsupported job store metadata.');
storeId = metadata.storeId;
}
for (const name of await readdir(root)) {
if (STORE_FILES.includes(name) || LOCK_CLAIM.test(name)) continue;
if (isPending(name)) {
const directory = await pendingDirectory(name);
if (!(await readdir(directory)).length) await rmdir(directory);
continue;
}
if (!UUID.test(name)) throw new Error(`Foreign store entry: ${name}.`);
if (await tryDiscardRemnant(name) !== 'whole') continue;
const directory = await directoryFor(name);
const record = await readJson(join(directory, 'status.json'));
objectKeys(record, ['schemaVersion', 'jobId', 'sourceRevision', 'state', 'settings', 'progress', 'artifacts', 'metrics', 'reason'], 'job status');
if (record.schemaVersion !== 1 || record.jobId !== name || !STATES.includes(record.state)) throw new Error('Invalid persisted job status.');
boundedNumber(record.sourceRevision, 0, Number.MAX_SAFE_INTEGER, 'source revision', true);
const snapshot = await readJson(join(directory, 'snapshot.json'));
objectKeys(snapshot, ['recipe', 'clips'], 'snapshot');
validateClips(snapshot.clips ?? []);
if (!snapshot.recipe || typeof snapshot.recipe !== 'object' || Array.isArray(snapshot.recipe)) throw new Error('Persisted snapshot recipe must be an object.');
if ((await readdir(directory)).includes('asset.glb') && (await regularFile(join(directory, 'asset.glb'))).size > MAXIMUM_ASSET_BYTES) throw new Error('Persisted asset exceeds its byte budget.');
if (snapshot.recipe.format === 'crucible.asset') {
const asset = await hashFile(join(directory, 'asset.glb'), MAXIMUM_ASSET_BYTES);
if (asset.sha256 !== snapshot.recipe.assetId) throw new Error('Persisted asset does not match its recipe digest.');
}
const { frameCount, ...inputSettings } = record.settings;
const settings = settingsFrom(inputSettings, snapshot.clips ?? [], snapshot.recipe);
if (frameCount !== settings.frameCount) throw new Error('Invalid persisted frame count.');
record.settings = settings;
objectKeys(record.progress, ['frames', 'maximumPendingFrames'], 'persisted progress');
boundedNumber(record.progress.frames, 0, settings.frameCount, 'persisted frames', true);
boundedNumber(record.progress.maximumPendingFrames, 0, 1, 'persisted pending frames', true);
if (record.metrics !== undefined) record.metrics = jsonCopy(record.metrics, 65536);
if (record.reason !== undefined && (typeof record.reason !== 'string' || record.reason.length > 2048)) throw new Error('Invalid persisted job reason.');
const namesAllowed = artifactNames(settings.kind, settings.passes, settings.format, settings.multilayer, settings.frameCount, settings.mattes, settings);
if (!Array.isArray(record.artifacts) || record.artifacts.length > namesAllowed.length) throw new Error('Invalid persisted artifacts.');
const seen = new Set();
for (const artifact of record.artifacts) {
objectKeys(artifact, ['filename', 'mimeType', 'bytes', 'sha256'], 'artifact');
if (!namesAllowed.includes(artifact.filename) || seen.has(artifact.filename)) throw new Error('Invalid persisted artifact name.');
seen.add(artifact.filename);
const actual = await hashFile(join(directory, artifact.filename), artifactByteLimit(settings, limits));
if (artifact.mimeType !== artifactMimeType(settings, artifact.filename) || artifact.bytes !== actual.bytes || artifact.sha256 !== actual.sha256) throw new Error('Persisted artifact integrity failed.');
}
if (record.state === 'completed' && record.artifacts.length !== namesAllowed.length) throw new Error('Completed job is missing its artifacts.');
const job = { record, persistence: Promise.resolve(), settled: Promise.resolve() };
jobs.set(name, job);
if (!TERMINAL.has(record.state)) {
record.state = 'interrupted';
record.reason = 'The previous process stopped before the job finished.';
record.artifacts = [];
await persist(job);
}
}
if (jobs.size > limits.maxRecords) throw new RangeError('Job store exceeds its record budget.');
await storeBytes();
}
function validateClips(clips) {
if (!Array.isArray(clips) || clips.length > 256) throw new TypeError('Snapshot clips must contain at most 256 durations.');
for (const clip of clips) {
objectKeys(clip, ['duration'], 'clip');
boundedNumber(clip.duration, 0, 3600, 'clip duration');
}
}
async function finish(job, snapshot) {
const { record, controller } = job;
let timer;
const expire = message => {
job.failure = message;
record.state = 'cancelling';
record.reason = job.failure;
controller.abort(new Error(job.failure));
persist(job).catch(() => {});
};
// Two clocks: the wait behind other calls is bounded on its own, and the elapsed deadline starts when the runner says rendering began.
// The wait for the person's GPU queue is bounded by --gpu-queue-wait instead, so the call-queue clock starts once the card is taken.
const { queueWaitMs } = limits;
function startQueueWaitClock() {
timer = setTimeout(() => expire(`Render job exceeded its queue wait deadline of ${Math.round(queueWaitMs / 1000)} seconds before rendering started. Submit the job again.`), queueWaitMs);
}
let renderClockStarted = false;
const startRenderClock = () => {
if (renderClockStarted) return;
renderClockStarted = true;
clearTimeout(timer);
timer = setTimeout(() => expire('Render job exceeded its elapsed deadline. If another process was using the GPU, submit the job again.'), jobDeadlineMs(record.settings, limits.deadlineMs));
};
let releaseGpu;
const renamed = [];
try {
if (controller.signal.aborted) throw controller.signal.reason;
// The state stays queued while the job waits for the GPU queue.
if (acquireGpu) releaseGpu = await acquireGpu(controller.signal);
startQueueWaitClock();
if (controller.signal.aborted) throw controller.signal.reason;
record.state = 'running';
await persist(job);
const runnerDirectory = await directoryFor(record.jobId);
if (controller.signal.aborted) throw controller.signal.reason;
const completed = await run({
jobId: record.jobId, sessionId: job.sessionId, directory: runnerDirectory, snapshot, settings: structuredClone(record.settings), signal: controller.signal, startRenderClock,
progress(update) {
objectKeys(update, ['frames', 'maximumPendingFrames'], 'progress');
if (controller.signal.aborted) throw controller.signal.reason;
if (record.state !== 'running') throw new Error('Progress is only accepted while the callback is running.');
const frames = boundedNumber(update.frames ?? record.progress.frames, record.progress.frames, record.settings.frameCount, 'captured frames', true);
const maximumPendingFrames = boundedNumber(update.maximumPendingFrames ?? record.progress.maximumPendingFrames, 0, 1, 'pending frames', true);
record.progress = { frames, maximumPendingFrames: Math.max(record.progress.maximumPendingFrames, maximumPendingFrames) };
return persist(job);
},
});
if (controller.signal.aborted) throw controller.signal.reason;
record.state = 'finalizing';
await persist(job);
objectKeys(completed, ['artifacts', 'metrics'], 'runner result');
const expected = artifactNames(record.settings.kind, record.settings.passes, record.settings.format, record.settings.multilayer, record.settings.frameCount, record.settings.mattes, record.settings);
if (!Array.isArray(completed.artifacts) || completed.artifacts.length !== expected.length) throw new Error('Runner did not return the required fixed artifacts.');
const directory = await directoryFor(record.jobId);
const metrics = jsonCopy(completed.metrics ?? {}, 65536);
const artifacts = [];
const seen = new Set();
let outputBytes = 0;
for (const output of completed.artifacts) {
objectKeys(output, ['temporary', 'filename', 'mimeType'], 'runner artifact');
if (!expected.includes(output.filename) || seen.has(output.filename) || output.temporary !== `${output.filename}.partial` || output.mimeType !== artifactMimeType(record.settings, output.filename)) throw new Error('Runner artifact must use the fixed filename and MIME type.');
seen.add(output.filename);
const artifact = { filename: output.filename, mimeType: output.mimeType, ...await hashFile(join(directory, output.temporary), artifactByteLimit(record.settings, limits)) };
outputBytes += artifact.bytes;
if (outputBytes > limits.maxOutputBytes) throw new RangeError('Job exceeds its combined output byte budget.');
artifacts.push(artifact);
}
await storeBytes();
if (controller.signal.aborted) throw controller.signal.reason;
for (const artifact of artifacts) {
if (controller.signal.aborted) throw controller.signal.reason;
const target = join(directory, artifact.filename);
try { await lstat(target); throw new Error('Final artifact already exists.'); }
catch (error) { if (error.code !== 'ENOENT') throw error; }
await rename(join(directory, `${artifact.filename}.partial`), target);
renamed.push(artifact.filename);
}
if (controller.signal.aborted) throw controller.signal.reason;
const completedRecord = { ...record, artifacts, metrics, state: 'completed' };
await persist(job, completedRecord);
if (controller.signal.aborted) throw controller.signal.reason;
Object.assign(record, completedRecord);
} catch (error) {
const failedRecord = { ...record, state: controller.signal.aborted && !job.failure ? 'cancelled' : 'failed', reason: String(job.failure ?? error?.message ?? error).slice(0, 2048), artifacts: [] };
try {
const directory = await directoryFor(record.jobId);
for (const filename of renamed) {
await regularFile(join(directory, filename));
await rename(join(directory, filename), join(directory, `${filename}.partial`));
}
await persist(job, failedRecord);
} catch (failure) {
failedRecord.reason = `${failedRecord.reason} Store cleanup or persistence failed: ${failure.message}`.slice(0, 2048);
}
Object.assign(record, failedRecord);
} finally {
clearTimeout(timer);
releaseGpu?.();
if (active === job) active = undefined;
}
}
await acquireLock();
try { await recover(); }
catch (error) { await unlock(); throw error; }
return Object.freeze({
limits: Object.freeze(limits),
async submit(input) {
if (closed) throw new Error('Render job controller is closed.');
if (active || admission) throw new Error('A render job is already active. No additional jobs are queued.');
objectKeys(input, ['sessionId', 'sourceRevision', 'snapshot', 'settings'], 'submission');
const sourceRevision = boundedNumber(input.sourceRevision, 0, Number.MAX_SAFE_INTEGER, 'source revision', true);
objectKeys(input.snapshot, ['recipe', 'assetBytes', 'clips'], 'snapshot');
const clips = jsonCopy(input.snapshot.clips ?? []);
validateClips(clips);
const recipe = jsonCopy(input.snapshot.recipe);
if (!recipe || typeof recipe !== 'object' || Array.isArray(recipe)) throw new TypeError('Snapshot recipe must be a JSON object.');
const snapshotMetadata = jsonCopy({ recipe, clips });
const settings = settingsFrom(input.settings, clips, recipe);
if (input.snapshot.assetBytes !== undefined && (!Buffer.isBuffer(input.snapshot.assetBytes) || input.snapshot.assetBytes.length > MAXIMUM_ASSET_BYTES || !input.snapshot.assetBytes.length)) throw new RangeError(`Asset snapshot must be a nonempty Buffer of at most ${MAXIMUM_ASSET_BYTES} bytes (${describeAssetBytes(MAXIMUM_ASSET_BYTES)}).`);
const snapshot = { recipe, clips, ...(input.snapshot.assetBytes ? { assetBytes: Buffer.from(input.snapshot.assetBytes) } : {}) };
if (recipe.format === 'crucible.asset' && (!snapshot.assetBytes || createHash('sha256').update(snapshot.assetBytes).digest('hex') !== recipe.assetId)) throw new Error('Asset snapshot bytes must match the recipe SHA256 assetId.');
const operation = (async () => {
if (jobs.size >= limits.maxRecords) throw new RangeError('Job record budget exhausted. Release a finished job.');
const snapshotText = JSON.stringify(snapshotMetadata);
const neededBytes = Buffer.byteLength(snapshotText) + (snapshot.assetBytes?.length ?? 0) + limits.maxOutputBytes + 131072;
await makeRoomInStore(neededBytes);
const jobId = randomUUID();
const directory = join(root, jobId);
const staging = join(root, `.pending-${storeId}-${jobId}`);
await mkdir(staging, { mode: 0o700 });
await writeFile(join(staging, 'owner.json'), JSON.stringify({ schemaVersion: 1, storeId, jobId }), { flag: 'wx', mode: 0o600 });
await writeFile(join(staging, 'snapshot.json'), snapshotText, { flag: 'wx', mode: 0o600 });
if (snapshot.assetBytes) await writeFile(join(staging, 'asset.glb'), snapshot.assetBytes, { flag: 'wx', mode: 0o600 });
const record = { schemaVersion: 1, jobId, sourceRevision, state: 'queued', settings, progress: { frames: 0, maximumPendingFrames: 0 }, artifacts: [] };
await writeFile(join(staging, 'status.json'), JSON.stringify(record), { flag: 'wx', mode: 0o600 });
await rename(staging, directory);
const job = { record, sessionId: input.sessionId, controller: new AbortController(), persistence: Promise.resolve() };
jobs.set(jobId, job);
active = job;
const result = structuredClone(record);
job.settled = Promise.resolve().then(() => finish(job, snapshot));
job.settled.catch(() => {});
return result;
})();
admission = operation;
try { return await operation; }
finally { admission = undefined; }
},
status(jobId) { return structuredClone(getJob(jobId).record); },
list() { return [...jobs.values()].map(job => structuredClone(job.record)); },
cancel(jobId) {
if (closed) throw new Error('Render job controller is closed.');
const job = getJob(jobId);
if (!TERMINAL.has(job.record.state)) {
job.record.state = 'cancelling';
job.controller.abort(new Error('Render job cancelled.'));
persist(job).catch(() => {});
}
return structuredClone(job.record);
},
result(jobId) {
const job = getJob(jobId);
const record = job.record;
if (record.state !== 'completed') throw new Error(`Render job is ${record.state}. No completed result is available.`);
job.fetched = true;
return structuredClone({ jobId, sourceRevision: record.sourceRevision, artifacts: record.artifacts, metrics: record.metrics });
},
async release(jobId) {
if (closed) throw new Error('Render job controller is closed.');
const job = getJob(jobId);
if (active === job || !TERMINAL.has(job.record.state)) throw new Error('Cancel and settle the active job before release.');
await releaseFinishedJob(jobId, job);
return { jobId, released: true };
},
async close() {
closed = true;
await admission?.catch(() => {});
if (active) {
const job = active;
job.record.state = 'cancelling';
job.controller.abort(new Error('Render job controller closed.'));
// A failed write must not stop close() from settling the job and giving the store lock back.
await persist(job).catch(() => {});
let timer;
try {
await Promise.race([job.settled, new Promise((resolveWait, reject) => { timer = setTimeout(() => reject(new Error('Render callback did not settle after cancellation. Store ownership remains locked.')), limits.cleanupMs); })]);
} finally { clearTimeout(timer); }
}
for (const job of jobs.values()) await job.persistence.catch(() => {});
await unlock();
},
});
}