3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
// Temporal accumulation for the frames of a native video job (issue 1120). Every path traced frame carries fresh noise, and
// a denoiser turns what is left of it into soft blotches that change from frame to frame, so a film shimmers even when each
// still looks clean. More samples do not cure it (a frozen camera still shimmers at 1024 samples), so the blend runs on the
// filtered picture: each pixel of a new frame finds where its surface point was in the frame before (the first-hit distance
// and the two cameras give that), and its filtered light is blended into the light gathered there so far. A pixel whose
// surface was hidden, off screen, another surface or seen from another side in the frame before starts again, and so does
// the whole image when anything but the camera changed.
//
// It assumes a still scene under a moving camera. Light that depends on the view (a sharp reflection, a highlight, what is
// seen through glass) is carried along with the surface, so it trails the camera by a few frames.
// KernelParamsV2 of Cook's pack revision 2 (FORMAT-v2.md): 336 bytes, with the anamorphic aperture ratio at byte 320.
const KERNEL_PARAMS_BYTES = 336;
const SEED_BYTES = [28, 32];
const KERNEL_FLAGS_BYTE = 32;
const ORTHOGRAPHIC_CAMERA_FLAG = 8;
const CAMERA_BYTES = [112, 192];
// packCameraParams rewrites the aperture ratio with the other camera words, so it belongs to the camera too.
const APERTURE_RATIO_BYTES = [320, 324];
const CAMERA_RANGES = [CAMERA_BYTES, APERTURE_RATIO_BYTES];
const TILE_ORIGIN_BYTES = [312, 320];
const CAMERA_POSITION_BYTE = 112;
const CAMERA_FORWARD_BYTE = 128;
const CAMERA_RIGHT_BYTE = 144;
const CAMERA_UP_BYTE = 160;
const LENS_SHIFT_BYTE = 176;
export const TEMPORAL_DEFAULTS = Object.freeze({
// The share of a new frame in a pixel with a long history while the camera moves: a pixel then never averages more than
// 1 / minimumWeight frames, so light that depends on the view catches up. Under a camera that stands still every frame
// counts the same and the picture converges.
minimumWeight: 0.1,
// A history texel belongs to the same surface when its distance is within this share of the expected one.
depthTolerance: 0.04,
// ... and when its normal is within about 26 degrees of the pixel's.
normalAgreement: 0.9,
// Below this much bilinear weight from texels of the same surface, the pixel starts again.
minimumFetchWeight: 0.3,
// The cosine between the two directions a surface point was seen from in the frame before and in this one. Below it (about
// 6 degrees apart) the pixel starts again: a cut to another view of the same room would otherwise carry the old view's
// reflections and highlights along.
viewAgreement: 0.995,
});
function requirePlane(plane, name, floatsPerPixel, pixelCount) {
if (!(plane instanceof Float32Array) || plane.length !== pixelCount * floatsPerPixel) {
const got = plane instanceof Float32Array ? `a Float32Array of ${plane.length}` : typeof plane;
throw new TypeError(`temporal accumulate: \`${name}\` must be a Float32Array of ${pixelCount * floatsPerPixel} floats (${floatsPerPixel} per pixel), got ${got}.`);
}
}
function requireKernelParams(kernelParams) {
if (!(kernelParams instanceof Uint8Array) || kernelParams.length !== KERNEL_PARAMS_BYTES) {
throw new TypeError(`temporal accumulate: \`kernelParams\` must be the ${KERNEL_PARAMS_BYTES} byte FORMAT-v2 block of the frame (a Uint8Array).`);
}
}
function readVector(view, byteOffset) {
return [view.getFloat32(byteOffset, true), view.getFloat32(byteOffset + 4, true), view.getFloat32(byteOffset + 8, true)];
}
function squaredLength(vector) {
return vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2];
}
/** The camera of a FORMAT-v2 kernel parameter block, as the kernel's camera ray reads it. */
export function readCameraView(kernelParams) {
requireKernelParams(kernelParams);
const view = new DataView(kernelParams.buffer, kernelParams.byteOffset, kernelParams.byteLength);
const right = readVector(view, CAMERA_RIGHT_BYTE);
const up = readVector(view, CAMERA_UP_BYTE);
return {
position: readVector(view, CAMERA_POSITION_BYTE),
forward: readVector(view, CAMERA_FORWARD_BYTE),
right,
up,
rightSquaredLength: squaredLength(right),
upSquaredLength: squaredLength(up),
lensShift: [view.getFloat32(LENS_SHIFT_BYTE, true), view.getFloat32(LENS_SHIFT_BYTE + 4, true)],
orthographic: (view.getUint32(KERNEL_FLAGS_BYTE, true) & ORTHOGRAPHIC_CAMERA_FLAG) !== 0,
};
}
function sameCameraBytes(firstParams, secondParams) {
for (const [start, end] of CAMERA_RANGES) {
for (let byteOffset = start; byteOffset < end; byteOffset++) {
if (firstParams[byteOffset] !== secondParams[byteOffset]) {
return false;
}
}
}
return true;
}
function insideRange(byteOffset, range) {
return byteOffset >= range[0] && byteOffset < range[1];
}
/**
* True when two kernel parameter blocks differ in nothing but the camera, the seed and the tile origin the host writes: the
* frames then show the same scene under the same light and render settings.
*/
export function onlyCameraDiffers(firstParams, secondParams) {
requireKernelParams(firstParams);
requireKernelParams(secondParams);
for (let byteOffset = 0; byteOffset < KERNEL_PARAMS_BYTES; byteOffset++) {
if (insideRange(byteOffset, SEED_BYTES) || CAMERA_RANGES.some(range => insideRange(byteOffset, range)) || insideRange(byteOffset, TILE_ORIGIN_BYTES)) {
continue;
}
const mask = byteOffset === KERNEL_FLAGS_BYTE ? 0xff & ~ORTHOGRAPHIC_CAMERA_FLAG : 0xff;
if ((firstParams[byteOffset] & mask) !== (secondParams[byteOffset] & mask)) {
return false;
}
}
return true;
}
function validateShare(value, name) {
if (typeof value !== 'number' || !(value > 0 && value <= 1)) {
throw new RangeError(`createTemporalAccumulator: \`${name}\` must be a number above 0 and at most 1, got ${JSON.stringify(value)}.`);
}
}
/**
* Holds the history of one job. `accumulate` takes the planes of a frame and returns the colour to show.
*
* @param {{ width: number, height: number, minimumWeight?: number, depthTolerance?: number, normalAgreement?: number, minimumFetchWeight?: number, viewAgreement?: number }} options
*/
export function createTemporalAccumulator(options = {}) {
const { width, height } = options;
if (!Number.isInteger(width) || !Number.isInteger(height) || width < 1 || height < 1) {
throw new RangeError(`createTemporalAccumulator: \`width\` and \`height\` must be whole numbers of at least 1, got ${JSON.stringify(width)} x ${JSON.stringify(height)}.`);
}
const settings = { ...TEMPORAL_DEFAULTS };
for (const name of Object.keys(TEMPORAL_DEFAULTS)) {
if (options[name] !== undefined) {
validateShare(options[name], name);
settings[name] = options[name];
}
}
const pixelCount = width * height;
let history = null;
let pendingRestartReason = 'the first frame of the job';
function restart(reason) {
history = null;
pendingRestartReason = reason;
}
function reasonToRestart(kernelParams, sceneChanged, camera) {
if (history === null) {
return pendingRestartReason;
}
if (sceneChanged) {
return 'the scene changed since the frame before';
}
if (camera.orthographic || history.camera.orthographic) {
return 'an orthographic camera is not reprojected';
}
if (!onlyCameraDiffers(history.kernelParams, kernelParams)) {
return 'the light or the render settings changed since the frame before';
}
return null;
}
/**
* @param {{ color: Float32Array, depth: Float32Array, normal: Float32Array, kernelParams: Uint8Array, sceneChanged?: boolean }} frame
* `color` and `normal` hold three floats per pixel (the normal is unit length, or zero where no surface was hit),
* `depth` one: the distance along the camera ray to the first surface, 0 for a miss. `kernelParams` is the block the
* frame was traced with. `sceneChanged` says a table other than the kernel parameters differs from the frame before.
* The accumulator keeps `depth` and `normal` (and the colour it returns) until the next frame: do not write to them.
* @returns {{ color: Float32Array, historyShare: number, restarted: string | null }} `color` is a new buffer;
* `historyShare` is the share of pixels that found their surface in the frame before; `restarted` says why the whole
* image started again, or null.
*/
function accumulate({ color, depth, normal, kernelParams, sceneChanged = false }) {
requirePlane(color, 'color', 3, pixelCount);
requirePlane(depth, 'depth', 1, pixelCount);
requirePlane(normal, 'normal', 3, pixelCount);
const camera = readCameraView(kernelParams);
const restarted = reasonToRestart(kernelParams, sceneChanged, camera);
const blended = new Float32Array(pixelCount * 3);
const counts = new Float32Array(pixelCount);
let pixelsWithHistory = 0;
if (restarted !== null) {
blended.set(color);
counts.fill(1);
} else {
const minimumShare = sameCameraBytes(history.kernelParams, kernelParams) ? 0 : settings.minimumWeight;
pixelsWithHistory = blendWithHistory({ color, depth, normal, camera, blended, counts, minimumShare });
}
history = { color: blended, depth, normal, counts, camera, kernelParams: Uint8Array.from(kernelParams) };
pendingRestartReason = null;
return { color: blended, historyShare: pixelsWithHistory / pixelCount, restarted };
}
// Writes `blended` and `counts`, and returns how many pixels found their surface in the history.
function blendWithHistory({ color, depth, normal, camera, blended, counts, minimumShare }) {
const past = history;
const pastCamera = past.camera;
const { position, forward, right, up, lensShift } = camera;
let pixelsWithHistory = 0;
for (let y = 0; y < height; y++) {
const sensorY = 1 - ((y + 0.5) / height) * 2 + 2 * lensShift[1];
for (let x = 0; x < width; x++) {
const pixel = y * width + x;
const colorIndex = pixel * 3;
const distance = depth[pixel];
let pastRed = 0;
let pastGreen = 0;
let pastBlue = 0;
let pastCount = 0;
let fetchWeight = 0;
if (distance > 0) {
const sensorX = ((x + 0.5) / width) * 2 - 1 + 2 * lensShift[0];
let directionX = forward[0] + right[0] * sensorX + up[0] * sensorY;
let directionY = forward[1] + right[1] * sensorX + up[1] * sensorY;
let directionZ = forward[2] + right[2] * sensorX + up[2] * sensorY;
const directionScale = distance / Math.hypot(directionX, directionY, directionZ);
directionX *= directionScale;
directionY *= directionScale;
directionZ *= directionScale;
const offsetX = position[0] + directionX - pastCamera.position[0];
const offsetY = position[1] + directionY - pastCamera.position[1];
const offsetZ = position[2] + directionZ - pastCamera.position[2];
const along = offsetX * pastCamera.forward[0] + offsetY * pastCamera.forward[1] + offsetZ * pastCamera.forward[2];
if (along > 1e-4) {
const pastSensorX = (offsetX * pastCamera.right[0] + offsetY * pastCamera.right[1] + offsetZ * pastCamera.right[2]) / (along * pastCamera.rightSquaredLength);
const pastSensorY = (offsetX * pastCamera.up[0] + offsetY * pastCamera.up[1] + offsetZ * pastCamera.up[2]) / (along * pastCamera.upSquaredLength);
const sourceX = ((pastSensorX - 2 * pastCamera.lensShift[0] + 1) / 2) * width - 0.5;
const sourceY = ((1 - pastSensorY + 2 * pastCamera.lensShift[1]) / 2) * height - 0.5;
const left = Math.floor(sourceX);
const top = Math.floor(sourceY);
// On screen up to the outer edge of the border pixels; a corner past the border reads the border texel.
const expectedDistance = Math.hypot(offsetX, offsetY, offsetZ);
const viewCosine = (offsetX * directionX + offsetY * directionY + offsetZ * directionZ) / (expectedDistance * distance);
const onScreen = sourceX >= -0.5 && sourceY >= -0.5 && sourceX <= width - 0.5 && sourceY <= height - 0.5;
if (onScreen && viewCosine >= settings.viewAgreement) {
const allowedDifference = settings.depthTolerance * expectedDistance;
const fractionX = sourceX - left;
const fractionY = sourceY - top;
const normalX = normal[colorIndex];
const normalY = normal[colorIndex + 1];
const normalZ = normal[colorIndex + 2];
// A bilinear fetch that leaves out the texels of another surface.
for (let corner = 0; corner < 4; corner++) {
const cornerX = corner & 1;
const cornerY = corner >> 1;
const weight = (cornerX === 1 ? fractionX : 1 - fractionX) * (cornerY === 1 ? fractionY : 1 - fractionY);
const sourceColumn = Math.min(width - 1, Math.max(0, left + cornerX));
const sourceRow = Math.min(height - 1, Math.max(0, top + cornerY));
const source = sourceRow * width + sourceColumn;
if (Math.abs(past.depth[source] - expectedDistance) > allowedDifference) {
continue;
}
const sourceIndex = source * 3;
const agreement = normalX * past.normal[sourceIndex] + normalY * past.normal[sourceIndex + 1] + normalZ * past.normal[sourceIndex + 2];
if (agreement < settings.normalAgreement) {
continue;
}
pastRed += past.color[sourceIndex] * weight;
pastGreen += past.color[sourceIndex + 1] * weight;
pastBlue += past.color[sourceIndex + 2] * weight;
pastCount += past.counts[source] * weight;
fetchWeight += weight;
}
}
}
}
if (fetchWeight > settings.minimumFetchWeight) {
const count = pastCount / fetchWeight + 1;
const share = Math.max(1 / count, minimumShare);
const kept = (1 - share) / fetchWeight;
blended[colorIndex] = pastRed * kept + color[colorIndex] * share;
blended[colorIndex + 1] = pastGreen * kept + color[colorIndex + 1] * share;
blended[colorIndex + 2] = pastBlue * kept + color[colorIndex + 2] * share;
counts[pixel] = count;
pixelsWithHistory += 1;
} else {
blended[colorIndex] = color[colorIndex];
blended[colorIndex + 1] = color[colorIndex + 1];
blended[colorIndex + 2] = color[colorIndex + 2];
counts[pixel] = 1;
}
}
}
return pixelsWithHistory;
}
return { accumulate, restart };
}
/** Whether a native video job blends its frames over time: on unless JS67_NATIVE_TEMPORAL=off. */
export function chooseNativeTemporal(environment = process.env) {
return environment.JS67_NATIVE_TEMPORAL !== 'off';
}
/** The seed of one frame of a job that accumulates over time: every frame needs its own noise for the blend to remove any. */
export function temporalFrameSeed(seed, frame) {
return ((seed ?? 0) + frame) >>> 0;
}