Skip to content

3D engine that AI agents drive

17.4 KB
Raw Blame
import { createStage, sharedFragmentHeader } from './camera-effects.js';
import { numberInRange, groupOf, onlyKeys, triple } from './post-fields.js';
import { invalidField } from './field-error.js';

// Two named post steps that agents place in the scene and time: `distortion` bends the picture behind it (a shock ring that
// expands, or heat haze in a box or an ellipsoid) and `flash` lifts light around a point for a moment. Both read the depth
// buffer, run on the HDR frame before glare and bloom, and take their time from the scene clock (or from their own `time`
// field for a still), so a seek repeats the same pixels. Every list and number is bounded.

export const MAXIMUM_AIR_SOURCES = 4;
const MAXIMUM_SCENE_TIME = 3600;
const MAXIMUM_COORDINATE = 100000;
// The largest picture shift at full strength, as a fraction of the image height.
const RING_MAXIMUM_SHIFT = 0.15;
const HAZE_MAXIMUM_SHIFT = 0.04;
const MAXIMUM_NODE_ID_LENGTH = 128;

const PLACEMENT_FIELDS = ['centre', 'node', 'time'];
const RING_FIELDS = ['startTime', 'speed', 'thickness', 'strength', 'falloff', 'maxRadius', 'edge'];
const HAZE_FIELDS = ['region', 'size', 'noiseScale', 'scroll', 'strength'];
const FLASH_FIELDS = ['startTime', 'attack', 'decay', 'intensity', 'color', 'radius', 'ambient'];
const HAZE_REGIONS = ['box', 'sphere'];

function sourceList(value, step) {
  const sources = Array.isArray(value) ? value : [value];
  if (sources.length > MAXIMUM_AIR_SOURCES) {
    throw new RangeError(`${step} takes at most ${MAXIMUM_AIR_SOURCES} sources, got ${sources.length}. Fix: keep the ${MAXIMUM_AIR_SOURCES} that matter, or split the shot.`);
  }
  return sources;
}

function resolvePlacement(group, name) {
  if (group.node !== undefined && (typeof group.node !== 'string' || group.node.length === 0 || group.node.length > MAXIMUM_NODE_ID_LENGTH)) {
    throw new TypeError(`${name}.node must be the ID of a node in the recipe, such as "bomb". Fix: pass a node ID, or centre as three numbers.`);
  }
  return {
    centre: triple(group.centre, [0, 0, 0], -MAXIMUM_COORDINATE, MAXIMUM_COORDINATE, `${name}.centre`),
    ...(group.node === undefined ? {} : { node: group.node }),
    time: numberInRange(group.time, 0, 0, MAXIMUM_SCENE_TIME, `${name}.time`),
  };
}

function resolveRing(group, name) {
  onlyKeys(group, ['shape', ...PLACEMENT_FIELDS, ...RING_FIELDS], name);
  return {
    shape: 'ring',
    ...resolvePlacement(group, name),
    startTime: numberInRange(group.startTime, 0, 0, MAXIMUM_SCENE_TIME, `${name}.startTime`),
    speed: numberInRange(group.speed, 40, 0, 10000, `${name}.speed`),
    thickness: numberInRange(group.thickness, 1.5, 0.01, 1000, `${name}.thickness`),
    strength: numberInRange(group.strength, 0.5, 0, 1, `${name}.strength`),
    falloff: numberInRange(group.falloff, 0.4, 0, 20, `${name}.falloff`),
    maxRadius: numberInRange(group.maxRadius, 200, 0.01, MAXIMUM_COORDINATE, `${name}.maxRadius`),
    edge: numberInRange(group.edge, 0, 0, 2, `${name}.edge`),
  };
}

function resolveHaze(group, name) {
  onlyKeys(group, ['shape', ...PLACEMENT_FIELDS, ...HAZE_FIELDS], name);
  const region = group.region ?? 'box';
  if (!HAZE_REGIONS.includes(region)) throw new TypeError(`${name}.region must be one of ${HAZE_REGIONS.join(', ')}. Fix: pass "box" or "sphere".`);
  return {
    shape: 'haze',
    ...resolvePlacement(group, name),
    region,
    size: triple(group.size, [2, 2, 2], 0.01, 1000, `${name}.size`),
    noiseScale: numberInRange(group.noiseScale, 1.5, 0.01, 100, `${name}.noiseScale`),
    scroll: numberInRange(group.scroll, 1, 0, 100, `${name}.scroll`),
    strength: numberInRange(group.strength, 0.3, 0, 1, `${name}.strength`),
  };
}

function resolveDistortionSource(input, index) {
  const name = `distortion[${index}]`;
  const group = groupOf(input, name);
  if (group.shape !== 'ring' && group.shape !== 'haze') {
    throw new TypeError(`${name}.shape must be "ring" or "haze". Fix: "ring" is an expanding shock ring, "haze" is heat shimmer in a box or sphere.`);
  }
  const wrongFields = (group.shape === 'ring' ? HAZE_FIELDS : RING_FIELDS).filter(field => group[field] !== undefined && !(group.shape === 'ring' ? RING_FIELDS : HAZE_FIELDS).includes(field));
  if (wrongFields.length > 0) {
    throw new TypeError(`${name}.${wrongFields[0]} does not apply to shape "${group.shape}". Fix: ${group.shape === 'ring' ? 'a ring takes ' + RING_FIELDS.join(', ') : 'a haze takes ' + HAZE_FIELDS.join(', ')}.`);
  }
  return group.shape === 'ring' ? resolveRing(group, name) : resolveHaze(group, name);
}

/** The `distortion` step: one source or a list of up to four, each resolved with its defaults. */
export function resolveDistortion(input) {
  return sourceList(input, 'distortion').map(resolveDistortionSource);
}

function resolveFlashSource(input, index) {
  const name = `flash[${index}]`;
  const group = groupOf(input, name);
  onlyKeys(group, [...PLACEMENT_FIELDS, ...FLASH_FIELDS], name);
  return {
    ...resolvePlacement(group, name),
    startTime: numberInRange(group.startTime, 0, 0, MAXIMUM_SCENE_TIME, `${name}.startTime`),
    attack: numberInRange(group.attack, 0.05, 0, 60, `${name}.attack`),
    decay: numberInRange(group.decay, 0.4, 0.01, 60, `${name}.decay`),
    intensity: numberInRange(group.intensity, 4, 0, 1000, `${name}.intensity`),
    color: triple(group.color, [1, 0.85, 0.6], 0, 1, `${name}.color`),
    radius: numberInRange(group.radius, 50, 0.01, MAXIMUM_COORDINATE, `${name}.radius`),
    ambient: numberInRange(group.ambient, 0, 0, 0.5, `${name}.ambient`),
  };
}

/** The `flash` step: one source or a list of up to four, each resolved with its defaults. */
export function resolveFlash(input) {
  return sourceList(input, 'flash').map(resolveFlashSource);
}

/** Gives each source the scene clock time, and the world position of the node it follows. */
export function placeAirSteps(effects, sceneTime, parts) {
  for (const source of [...effects.distortion ?? [], ...effects.flash ?? []]) {
    if (sceneTime !== null && sceneTime !== undefined) source.time = sceneTime;
    const part = source.node === undefined ? null : parts.find(candidate => candidate.name === source.node);
    if (part) source.centre = [part.matrix[12], part.matrix[13], part.matrix[14]];
  }
  return effects;
}

/** Refuses a source that follows a node the recipe does not have, naming the nodes it has. */
export function checkAirStepNodes(recipe) {
  const nodeIds = recipe.nodes.map(node => node.id);
  for (const step of ['distortion', 'flash']) {
    for (const source of recipe.effects?.[step] ?? []) {
      if (source.node !== undefined && !nodeIds.includes(source.node)) {
        throw invalidField(`effects.${step}.node`, source.node, `one of the nodes of this recipe: ${nodeIds.join(', ') || 'none yet'}; or pass centre as three numbers`);
      }
    }
  }
}

/** The ring at this source's time, or null before it starts and once it passes `maxRadius`. */
export function ringState(source) {
  const age = source.time - source.startTime;
  if (age < 0) return null;
  const radius = source.speed * age;
  if (radius > source.maxRadius) return null;
  return { radius, amplitude: source.strength * Math.exp(-source.falloff * age) };
}

/** Flash brightness at this source's time: rises over `attack` seconds, then falls to zero over `decay` seconds. */
export function flashLevel(source) {
  const age = source.time - source.startTime;
  if (age < 0) return 0;
  if (age < source.attack) return source.intensity * (age / source.attack);
  const remaining = 1 - (age - source.attack) / source.decay;
  return remaining > 0 ? source.intensity * remaining * remaining : 0;
}

const airHeader = `${sharedFragmentHeader}
vec3 rayPointAt(vec2 coordinate, float clipDepth) {
  vec4 point = inverseViewProjection * vec4(coordinate * 2.0 - 1.0, clipDepth, 1.0);
  return point.xyz / point.w;
}
vec3 rayDirectionAt(vec2 coordinate) {
  return normalize(rayPointAt(coordinate, 1.0) - rayPointAt(coordinate, -1.0));
}
// Metres from the camera to the surface under the pixel; the sky is far away.
float surfaceDistanceAt(vec2 coordinate) {
  if (texture(sceneDepth, coordinate).r >= 1.0) return 1e9;
  return distance(worldPointAt(coordinate), rayPointAt(coordinate, -1.0));
}
`;

export const distortionFragment = `${airHeader}
uniform int sourceCount;
uniform vec4 placement[${MAXIMUM_AIR_SOURCES}];
uniform vec4 shape[${MAXIMUM_AIR_SOURCES}];
uniform vec3 extent[${MAXIMUM_AIR_SOURCES}];
float hashAt(vec3 cell) {
  vec3 scaled = fract(cell * 0.3183099 + 0.1) * 17.0;
  return fract(scaled.x * scaled.y * scaled.z * (scaled.x + scaled.y + scaled.z));
}
float noiseAt(vec3 point) {
  vec3 cell = floor(point);
  vec3 blend = fract(point);
  blend = blend * blend * (3.0 - 2.0 * blend);
  float low = mix(mix(hashAt(cell), hashAt(cell + vec3(1, 0, 0)), blend.x), mix(hashAt(cell + vec3(0, 1, 0)), hashAt(cell + vec3(1, 1, 0)), blend.x), blend.y);
  float high = mix(mix(hashAt(cell + vec3(0, 0, 1)), hashAt(cell + vec3(1, 0, 1)), blend.x), mix(hashAt(cell + vec3(0, 1, 1)), hashAt(cell + vec3(1, 1, 1)), blend.x), blend.y);
  return mix(low, high, blend.z);
}
void main() {
  vec3 origin = rayPointAt(textureCoordinate, -1.0);
  vec3 direction = rayDirectionAt(textureCoordinate);
  float surface = surfaceDistanceAt(textureCoordinate);
  vec3 rightDelta = rayDirectionAt(textureCoordinate + vec2(1.0 / resolution.x, 0.0)) - direction;
  vec3 upDelta = rayDirectionAt(textureCoordinate + vec2(0.0, 1.0 / resolution.y)) - direction;
  vec2 shift = vec2(0.0);
  float edgeLift = 0.0;
  float guardDistance = 1e9;
  for (int index = 0; index < ${MAXIMUM_AIR_SOURCES}; index++) {
    if (index >= sourceCount) break;
    vec3 centre = placement[index].xyz;
    if (placement[index].w < 0.5) {
      // Ring: the pixel's ray passes the centre at some distance; the picture bends where that distance equals the ring radius.
      float along = dot(centre - origin, direction);
      if (along <= 0.0) continue;
      vec3 radial = origin + direction * along - centre;
      float axisDistance = length(radial);
      float band = (axisDistance - shape[index].x) / shape[index].y;
      vec3 toCentre = centre - origin;
      float axisRight = length(cross(normalize(direction + rightDelta), toCentre));
      float axisUp = length(cross(normalize(direction + upDelta), toCentre));
      float pixelWidth = max(max(abs(axisRight - axisDistance), abs(axisUp - axisDistance)) * 3.0, 0.0001);
      float edgeBand = (axisDistance - shape[index].x) / min(shape[index].y, pixelWidth);
      float shellDistance = along - sqrt(max(shape[index].x * shape[index].x - axisDistance * axisDistance, 0.0));
      float weight = exp(-band * band) * (1.0 - smoothstep(surface - shape[index].y, surface, shellDistance));
      if (weight < 0.002 || axisDistance < 0.0001) continue;
      vec2 onScreen = vec2(dot(radial, rightDelta), dot(radial, upDelta));
      if (length(onScreen) < 1e-9) continue;
      shift += normalize(onScreen) * weight * shape[index].z * ${RING_MAXIMUM_SHIFT.toFixed(3)} * resolution.y;
      edgeLift += exp(-edgeBand * edgeBand) * (1.0 - smoothstep(surface - shape[index].y, surface, shellDistance)) * shape[index].w;
      guardDistance = min(guardDistance, shellDistance);
    } else {
      // Haze: walk the ray through the box or ellipsoid and shift by noise taken at the middle of the stretch inside it.
      vec3 scaledOrigin = (origin - centre) / extent[index];
      vec3 scaledDirection = direction / extent[index];
      float enter;
      float leave;
      if (placement[index].w < 1.5) {
        vec3 first = (-1.0 - scaledOrigin) / scaledDirection;
        vec3 second = (1.0 - scaledOrigin) / scaledDirection;
        enter = max(max(min(first.x, second.x), min(first.y, second.y)), min(first.z, second.z));
        leave = min(min(max(first.x, second.x), max(first.y, second.y)), max(first.z, second.z));
      } else {
        float quadratic = dot(scaledDirection, scaledDirection);
        float alongHalf = dot(scaledOrigin, scaledDirection);
        float discriminant = alongHalf * alongHalf - quadratic * (dot(scaledOrigin, scaledOrigin) - 1.0);
        if (discriminant < 0.0) continue;
        enter = (-alongHalf - sqrt(discriminant)) / quadratic;
        leave = (-alongHalf + sqrt(discriminant)) / quadratic;
      }
      enter = max(enter, 0.0);
      leave = min(leave, surface);
      if (leave <= enter) continue;
      vec3 middle = origin + direction * (enter + leave) * 0.5;
      vec3 inside = abs((middle - centre) / extent[index]);
      float edge = placement[index].w < 1.5 ? max(inside.x, max(inside.y, inside.z)) : length(inside);
      float weight = clamp((leave - enter) / min(extent[index].x, min(extent[index].y, extent[index].z)), 0.0, 1.0) * (1.0 - smoothstep(0.5, 1.0, edge));
      vec3 sampleAt = (middle - vec3(0.0, shape[index].y, 0.0)) * shape[index].x;
      vec2 wobble = vec2(noiseAt(sampleAt), noiseAt(sampleAt + vec3(17.3, 5.1, 9.7))) * 2.0 - 1.0;
      shift += wobble * weight * shape[index].z * ${HAZE_MAXIMUM_SHIFT.toFixed(3)} * resolution.y;
      guardDistance = min(guardDistance, enter);
    }
  }
  vec2 warped = textureCoordinate - shift / resolution;
  // A surface in front of the bending air must not be dragged along with it: keep the pixel where it is.
  if (guardDistance < 1e8 && surfaceDistanceAt(warped) < guardDistance - 0.05) warped = textureCoordinate;
  // The shock front itself: a warm lift on the thin band at the ring radius, so the ring reads as an edge on flat walls too.
  outputColor = texture(sourceTexture, warped) + vec4(edgeLift * vec3(1.0, 0.9, 0.75), 0.0);
}`;

export const flashFragment = `${airHeader}
uniform int sourceCount;
uniform vec4 flashPlacement[${MAXIMUM_AIR_SOURCES}];
uniform vec4 flashLift[${MAXIMUM_AIR_SOURCES}];
void main() {
  vec4 source = texture(sourceTexture, textureCoordinate);
  bool isSky = texture(sceneDepth, textureCoordinate).r >= 1.0;
  vec3 origin = rayPointAt(textureCoordinate, -1.0);
  vec3 direction = rayDirectionAt(textureCoordinate);
  vec3 lift = vec3(0.0);
  float ambient = 0.0;
  for (int index = 0; index < ${MAXIMUM_AIR_SOURCES}; index++) {
    if (index >= sourceCount) break;
    vec3 centre = flashPlacement[index].xyz;
    vec3 toCentre = centre - origin;
    // Sky has no surface, so the light is measured from the pixel's ray; surfaces are measured from the point they sit at.
    float reach = isSky ? length(toCentre - direction * max(dot(toCentre, direction), 0.0)) : distance(worldPointAt(textureCoordinate), centre);
    float ratio = reach / flashPlacement[index].w;
    lift += flashLift[index].rgb / (1.0 + ratio * ratio);
    ambient += flashLift[index].w;
  }
  // Near the source the light falls off with distance; a small screen-wide gain keeps the rest of the frame from going dark by contrast.
  outputColor = vec4(source.rgb * (1.0 + ambient) + lift, source.a);
}`;

/** Packs the active sources of a distortion step into the uniform arrays of its shader. */
export function packDistortion(sources) {
  const placement = new Float32Array(MAXIMUM_AIR_SOURCES * 4);
  const shape = new Float32Array(MAXIMUM_AIR_SOURCES * 4);
  const extent = new Float32Array(MAXIMUM_AIR_SOURCES * 3).fill(1);
  const active = [];
  for (const source of sources) {
    if (source.shape === 'ring') {
      const state = ringState(source);
      if (state && state.amplitude > 0) active.push({ source, row: [state.radius, source.thickness, state.amplitude, source.edge * state.amplitude], kind: 0 });
    } else if (source.strength > 0) {
      active.push({ source, row: [source.noiseScale, source.scroll * source.time, source.strength, 0], kind: source.region === 'box' ? 1 : 2 });
    }
  }
  active.forEach(({ source, row, kind }, index) => {
    placement.set([...source.centre, kind], index * 4);
    shape.set(row, index * 4);
    if (source.shape === 'haze') extent.set(source.size, index * 3);
  });
  return { count: active.length, placement, shape, extent };
}

/** Packs the lit sources of a flash step into the uniform arrays of its shader. */
export function packFlash(sources) {
  const placement = new Float32Array(MAXIMUM_AIR_SOURCES * 4);
  const lift = new Float32Array(MAXIMUM_AIR_SOURCES * 4);
  let count = 0;
  for (const source of sources) {
    const level = flashLevel(source);
    if (level <= 0) continue;
    placement.set([...source.centre, source.radius], count * 4);
    lift.set([...source.color.map(component => component * level), source.ambient * level / source.intensity], count * 4);
    count++;
  }
  return { count, placement, lift };
}

export function createDistortionStage(stepsOf) {
  return createStage({
    name: 'Distortion',
    fragment: distortionFragment,
    uniformNames: ['sourceCount', 'placement', 'shape', 'extent'],
    isActive: (effects, frame) => packDistortion(stepsOf(frame).distortion ?? []).count > 0,
    setUniforms(context, uniforms, renderer, frame) {
      const packed = packDistortion(stepsOf(frame).distortion);
      context.uniform1i(uniforms.sourceCount, packed.count);
      context.uniform4fv(uniforms.placement, packed.placement);
      context.uniform4fv(uniforms.shape, packed.shape);
      context.uniform3fv(uniforms.extent, packed.extent);
    },
  });
}

export function createFlashStage(stepsOf) {
  return createStage({
    name: 'Flash',
    fragment: flashFragment,
    uniformNames: ['sourceCount', 'flashPlacement', 'flashLift'],
    isActive: (effects, frame) => packFlash(stepsOf(frame).flash ?? []).count > 0,
    setUniforms(context, uniforms, renderer, frame) {
      const packed = packFlash(stepsOf(frame).flash);
      context.uniform1i(uniforms.sourceCount, packed.count);
      context.uniform4fv(uniforms.flashPlacement, packed.placement);
      context.uniform4fv(uniforms.flashLift, packed.lift);
    },
  });
}