3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
import { mat4 } from 'gl-matrix';
import { applyProjection, requireNonDegenerateView, requirePositiveFinite } from './camera.js';
import { compileProgram, fullscreenVertex } from './gl-program.js';
// Depth of field and camera motion blur as optional full-frame stages. The core renderer carries no shader text for
// them: installCameraEffects adds the stages, and each one compiles and allocates only when a frame turns it on.
//
// Both are deterministic. They read only the resolved frame, its depth and the camera of this frame (plus the camera
// of an earlier instant that the caller samples from time, never from render history). Sample patterns are fixed and
// the per-pixel rotation comes from the pixel position, so a seek to a frame repeats the same pixels.
const DEPTH_OF_FIELD_SAMPLES = 32;
export const MOTION_BLUR_SAMPLES = 16;
// Largest blur circle as a fraction of image height at full blur; the blur setting scales it from zero.
const MAXIMUM_BLUR_HEIGHT_FRACTION = 0.05;
// Longest motion streak as a fraction of the frame, so a camera cut cannot smear the whole image.
export const MAXIMUM_STREAK = 0.1;
export const sharedFragmentHeader = `#version 300 es
precision highp float;
in vec2 textureCoordinate;
uniform sampler2D sourceTexture;
uniform sampler2D sceneDepth;
uniform mat4 inverseViewProjection;
uniform vec2 resolution;
out vec4 outputColor;
float pixelRotation() {
return 6.2831853 * fract(52.9829189 * fract(dot(gl_FragCoord.xy, vec2(0.06711056, 0.00583715))));
}
vec3 worldPointAt(vec2 coordinate) {
float depth = texture(sceneDepth, coordinate).r;
vec4 point = inverseViewProjection * vec4(coordinate * 2.0 - 1.0, depth * 2.0 - 1.0, 1.0);
return point.xyz / point.w;
}
`;
export const depthOfFieldFragment = `${sharedFragmentHeader}
uniform vec3 eye;
uniform vec3 forward;
uniform float focusDistance;
uniform float blurPixels;
float blurRadius(vec2 coordinate) {
float distanceAlongView = max(dot(worldPointAt(coordinate) - eye, forward), 0.001);
// Thin-lens shape: zero at the focus plane, rising toward one for very near and very far surfaces.
return blurPixels * clamp(abs(distanceAlongView - focusDistance) / distanceAlongView, 0.0, 1.0);
}
void main() {
float centerRadius = blurRadius(textureCoordinate);
vec4 center = texture(sourceTexture, textureCoordinate);
if (centerRadius < 0.5) {
outputColor = center;
return;
}
float rotation = pixelRotation();
vec4 sum = center;
float weightSum = 1.0;
for (int index = 0; index < ${DEPTH_OF_FIELD_SAMPLES}; index++) {
float ring = sqrt((float(index) + 0.5) / ${DEPTH_OF_FIELD_SAMPLES}.0);
float angle = float(index) * 2.3999632 + rotation;
vec2 offsetPixels = vec2(cos(angle), sin(angle)) * ring * centerRadius;
vec2 coordinate = textureCoordinate + offsetPixels / resolution;
// A sample only spreads onto this pixel when its own circle reaches it, so sharp foreground does not bleed.
float reach = clamp(blurRadius(coordinate) - length(offsetPixels) + 1.0, 0.0, 1.0);
sum += texture(sourceTexture, coordinate) * reach;
weightSum += reach;
}
outputColor = sum / weightSum;
}`;
export const motionBlurFragment = `${sharedFragmentHeader}
uniform mat4 previousViewProjection;
void main() {
vec3 world = worldPointAt(textureCoordinate);
vec4 previousClip = previousViewProjection * vec4(world, 1.0);
vec2 velocity = vec2(0.0);
if (previousClip.w > 0.0) velocity = (previousClip.xy / previousClip.w * 0.5 + 0.5) - textureCoordinate;
float streak = length(velocity);
if (streak > ${MAXIMUM_STREAK}) velocity *= ${MAXIMUM_STREAK} / streak;
if (length(velocity * resolution) < 0.5) {
outputColor = texture(sourceTexture, textureCoordinate);
return;
}
float jitter = pixelRotation() / 6.2831853;
vec4 sum = vec4(0.0);
for (int index = 0; index < ${MOTION_BLUR_SAMPLES}; index++) {
float along = (float(index) + jitter) / ${MOTION_BLUR_SAMPLES}.0;
sum += texture(sourceTexture, textureCoordinate + velocity * along);
}
outputColor = sum / ${MOTION_BLUR_SAMPLES}.0;
}`;
function createTarget(renderer) {
const context = renderer.context;
const { width, height } = renderer.canvas;
const texture = context.createTexture();
context.bindTexture(context.TEXTURE_2D, texture);
context.texImage2D(context.TEXTURE_2D, 0, renderer.hdr ? context.RGBA16F : context.RGBA8, width, height, 0, context.RGBA,
renderer.hdr ? context.HALF_FLOAT : context.UNSIGNED_BYTE, null);
context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MIN_FILTER, context.LINEAR);
context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MAG_FILTER, context.LINEAR);
context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_S, context.CLAMP_TO_EDGE);
context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_T, context.CLAMP_TO_EDGE);
const framebuffer = context.createFramebuffer();
context.bindFramebuffer(context.FRAMEBUFFER, framebuffer);
context.framebufferTexture2D(context.FRAMEBUFFER, context.COLOR_ATTACHMENT0, context.TEXTURE_2D, texture, 0);
if (context.checkFramebufferStatus(context.FRAMEBUFFER) !== context.FRAMEBUFFER_COMPLETE) {
throw new Error('Camera effect framebuffer allocation failed. Choose a lower quality preset.');
}
return { texture, framebuffer };
}
/**
* One full-frame stage. `isActive` decides from the resolved effects whether it runs, and `setUniforms` fills the
* values that are specific to the effect.
*/
export function createStage({ name, fragment, uniformNames, isActive, setUniforms }) {
let program = null;
let uniforms = null;
let target = null;
function releaseTargets(context) {
if (!target) return;
context.deleteTexture(target.texture);
context.deleteFramebuffer(target.framebuffer);
target = null;
}
function apply(renderer, frame, effects, sourceTexture) {
if (!isActive(effects, frame)) return null;
if (!renderer.depthTexture) throw new Error(`${name} requires depth:true on the renderer.`);
const context = renderer.context;
if (!program) {
program = compileProgram(context, fullscreenVertex, fragment);
const names = ['sourceTexture', 'sceneDepth', 'inverseViewProjection', 'resolution', ...uniformNames];
uniforms = Object.fromEntries(names.map(uniformName => [uniformName, context.getUniformLocation(program, uniformName)]));
}
target ??= createTarget(renderer);
context.bindFramebuffer(context.FRAMEBUFFER, target.framebuffer);
context.viewport(0, 0, renderer.canvas.width, renderer.canvas.height);
context.disable(context.DEPTH_TEST);
context.disable(context.CULL_FACE);
context.bindVertexArray(null);
context.useProgram(program);
context.activeTexture(context.TEXTURE0);
context.bindTexture(context.TEXTURE_2D, sourceTexture);
context.uniform1i(uniforms.sourceTexture, 0);
context.activeTexture(context.TEXTURE1);
context.bindTexture(context.TEXTURE_2D, renderer.depthTexture);
context.uniform1i(uniforms.sceneDepth, 1);
context.uniformMatrix4fv(uniforms.inverseViewProjection, false, renderer.inverseViewProjection);
context.uniform2f(uniforms.resolution, renderer.canvas.width, renderer.canvas.height);
setUniforms(context, uniforms, renderer, frame, effects);
context.drawArrays(context.TRIANGLES, 0, 3);
return target.texture;
}
function dispose(context) {
releaseTargets(context);
if (program) context.deleteProgram(program);
program = null;
}
return { name, apply, releaseTargets, dispose };
}
function createDepthOfFieldStage() {
return createStage({
name: 'Depth of field',
fragment: depthOfFieldFragment,
uniformNames: ['eye', 'forward', 'focusDistance', 'blurPixels'],
isActive: effects => effects.depthOfField.blur > 0,
setUniforms(context, uniforms, renderer, frame, effects) {
context.uniform3fv(uniforms.eye, frame.eye);
context.uniform3fv(uniforms.forward, renderer.viewForward);
context.uniform1f(uniforms.focusDistance, effects.depthOfField.focusDistance);
context.uniform1f(uniforms.blurPixels, effects.depthOfField.blur * MAXIMUM_BLUR_HEIGHT_FRACTION * renderer.canvas.height);
},
});
}
function createMotionBlurStage() {
return createStage({
name: 'Motion blur',
fragment: motionBlurFragment,
uniformNames: ['previousViewProjection'],
// Without a previous camera there is no known motion, so the frame passes through untouched.
// A frame that carries object motion is blurred by the object stage, which also covers the camera, so the two never stack.
isActive: (effects, frame) => effects.motionBlur.shutter > 0 && Boolean(frame.previousCamera) && !frame.objectMotion,
setUniforms(context, uniforms, renderer, frame) {
context.uniformMatrix4fv(uniforms.previousViewProjection, false, previousViewProjectionOf(renderer, frame));
},
});
}
// The previous camera only replaces what it names (eye, target, up, lens), so a partial camera works.
export function previousViewProjectionOf(renderer, frame) {
const previous = { ...frame, ...frame.previousCamera };
const aspect = renderer.canvas.width / renderer.canvas.height;
const projection = applyProjection(mat4.create(), previous, aspect, { world: renderer.world });
const lookView = { eye: previous.eye, target: previous.target ?? [0, 1.2, 0], up: previous.up ?? [0, 1, 0] };
requireNonDegenerateView(lookView, 'previousCamera');
const view = mat4.lookAt(mat4.create(), lookView.eye, lookView.target, lookView.up);
return mat4.multiply(mat4.create(), projection, view);
}
/**
* Adds depth of field and camera motion blur to a renderer. Both need `depth: true`. Motion blur reads
* `frame.previousCamera`, the camera at the start of the shutter interval; derive it from the frame time.
*/
export function installCameraEffects(renderer) {
if (!renderer.postStages.some(stage => stage.name === 'Depth of field')) {
renderer.postStages.push(createDepthOfFieldStage(), createMotionBlurStage());
}
return renderer;
}
// A lens at this f-stop or wider (smaller number) gets the full blur setting; narrower stops scale it down in proportion.
const FULL_BLUR_F_STOP = 1.4;
// The renderer aims a frame without a target here, so the focus distance follows the same default.
const DEFAULT_TARGET = [0, 1.2, 0];
/**
* The effects of one frame when the camera carries an f-stop: depth of field with the blur that stop gives (1.4 / fStop, at most
* 1), focused at `camera.focusDistance` along the view axis (as the path tracer and Blender do), by default at the camera's
* target. Effects with their own blur (a timeline shot's) are left as they are. A non-positive `fStop` or `focusDistance` throws.
*/
export function effectsWithFStop(effects, camera) {
if (camera.fStop === undefined) return effects;
requirePositiveFinite(camera.fStop, 'fStop');
if (camera.focusDistance !== undefined) requirePositiveFinite(camera.focusDistance, 'focusDistance');
if (effects.depthOfField.blur > 0) return effects;
const focusDistance = camera.focusDistance ?? Math.max(0.01, Math.hypot(...(camera.target ?? DEFAULT_TARGET).map((component, axis) => component - camera.eye[axis])));
return { ...effects, depthOfField: { blur: Math.min(1, FULL_BLUR_F_STOP / camera.fStop), focusDistance } };
}