Skip to content

3D engine that AI agents drive

18.9 KB
Raw Blame
/**
 * Cheap checks on a frame that renders without error but shows nothing useful: an empty frame, a subject out of frame
 * or too small, and clipped or black exposure. They use the shape of `agent-diagnostics.js` (kind, severity, target,
 * message, fix); each fix names a field and a value to try. Framing is sphere-against-frustum math on part bounds and
 * exposure reads a 64 by 64 copy of the frame. The stale bake check lives in `agent-bake.js`, which loads with a bake.
 */
const TINY_SUBJECT_RATIO = 0.05;
const BLACK_LUMA = 5 / 255;
const WHITE_LUMA = 250 / 255;
const BLACK_FRAME_SHARE = 0.98;
const CLIPPED_FRAME_SHARE = 0.35;

const MAXIMUM_FINDING_NODES = 8;
const MAXIMUM_JOB_DIAGNOSTICS = 16;

/** A finding names the parts it concerns in `nodes` and carries its numbers as extra fields. */
function warning(kind, target, message, fix, details = {}) {
  return { kind, severity: 'warning', target, message, fix, ...details };
}

/** At most 8 part names for a finding, with `nodeCount` when more were left out. */
export function namedNodes(names) {
  const nodes = names.slice(0, MAXIMUM_FINDING_NODES);
  return names.length > nodes.length ? { nodes, nodeCount: names.length } : { nodes };
}

function subtract(first, second) {
  return [first[0] - second[0], first[1] - second[1], first[2] - second[2]];
}

function dot(first, second) {
  return first[0] * second[0] + first[1] * second[1] + first[2] * second[2];
}

function cross(first, second) {
  return [first[1] * second[2] - first[2] * second[1], first[2] * second[0] - first[0] * second[2], first[0] * second[1] - first[1] * second[0]];
}

function normalize(vector) {
  const length = Math.hypot(...vector) || 1;
  return vector.map(component => component / length);
}

function round(value) {
  return Math.round(value * 100) / 100;
}

/** Half the vertical field of view as a tangent, or null for a camera these checks do not model. */
function halfHeightTangent(camera, aspect) {
  if (camera.projection === 'orthographic' || camera.shift) return null;
  if (Number.isFinite(camera.fov)) return Math.tan(camera.fov * Math.PI / 360);
  if (Number.isFinite(camera.focalLength)) return (camera.sensorWidth ?? 36) / aspect / 2 / camera.focalLength;
  return null;
}

function sphereOf(box) {
  const center = [0, 1, 2].map(axis => (box.min[axis] + box.max[axis]) / 2);
  return { center, radius: Math.hypot(...subtract(box.max, center)) };
}

function unionOf(boxes) {
  const min = [Infinity, Infinity, Infinity];
  const max = [-Infinity, -Infinity, -Infinity];
  for (const box of boxes) {
    for (let axis = 0; axis < 3; axis++) {
      min[axis] = Math.min(min[axis], box.min[axis]);
      max[axis] = Math.max(max[axis], box.max[axis]);
    }
  }
  return { min, max };
}

/** A camera that keeps the viewing direction, aims at the middle of the boxes and backs off until they fit. */
function framingSuggestion(camera, boxes, tangent) {
  const { center, radius } = sphereOf(unionOf(boxes));
  const direction = normalize(subtract(camera.eye, camera.target));
  const distance = Math.max(radius / tangent * 1.15, camera.near * 2);
  const eye = center.map((value, axis) => value + direction[axis] * distance);
  return `Set camera.target to [${center.map(round)}] and camera.eye to [${eye.map(round)}]`;
}

/**
 * Warnings for where the camera points. `reportPartsOutside` adds `part-outside-frame`, which only scene_look reports. `boxes` are world bounds ({ min, max }) of the visible parts; a camera that is
 * orthographic or shifted gets none, because the frustum test below assumes a centered perspective view.
 */
export function framingProblems({ camera, aspect, boxes, reportPartsOutside = true }) {
  const tangent = halfHeightTangent(camera, aspect);
  if (!tangent) return [];
  if (boxes.length === 0) {
    return [warning('empty-frame', 'nodes', 'The scene has no visible parts, so the frame shows only the background.',
      'Add a mesh node with scene_apply add-node (and add-geometry, add-material), or set visible: true on a hidden node.')];
  }
  const tangentWide = tangent * aspect;
  const forward = normalize(subtract(camera.target, camera.eye));
  const right = normalize(cross(forward, camera.up));
  const upward = cross(right, forward);
  const sideScale = Math.sqrt(1 + tangent * tangent);
  const wideScale = Math.sqrt(1 + tangentWide * tangentWide);
  const outside = [];
  const visibleBoxes = boxes.filter(box => {
    const { center, radius } = sphereOf(box);
    const offset = subtract(center, camera.eye);
    const depth = dot(offset, forward);
    if (depth + radius < camera.near || depth - radius > camera.far) {
      outside.push(box.name);
      return false;
    }
    const sideways = (Math.abs(dot(offset, right)) - tangentWide * depth) / wideScale;
    const vertical = (Math.abs(dot(offset, upward)) - tangent * depth) / sideScale;
    const inView = sideways <= radius && vertical <= radius;
    if (!inView) outside.push(box.name);
    return inView;
  });
  const outsideDetails = { ...namedNodes(outside), outsideCount: outside.length, partCount: boxes.length };
  if (visibleBoxes.length * 2 < boxes.length) {
    return [warning('subject-out-of-frame', 'camera',
      `${boxes.length - visibleBoxes.length} of ${boxes.length} parts are outside the camera frustum.`,
      `${framingSuggestion(camera, boxes, tangent)} to put every part in view.`, outsideDetails)];
  }
  const { center, radius } = sphereOf(unionOf(visibleBoxes));
  const depth = dot(subtract(center, camera.eye), forward);
  if (depth > 0 && radius / (depth * tangent) < TINY_SUBJECT_RATIO) {
    return [warning('subject-too-small', 'camera',
      `The visible parts fill under ${Math.round(TINY_SUBJECT_RATIO * 100)} percent of the frame height.`,
      `${framingSuggestion(camera, visibleBoxes, tangent)} to fill the frame, or lower camera.fov.`,
      { heightPercent: Math.round(radius / (depth * tangent) * 100) })];
  }
  if (reportPartsOutside && outside.length > 0) {
    return [warning('part-outside-frame', 'camera',
      `${outside.length} of ${boxes.length} parts are outside the camera frustum, so the picture does not show them.`,
      `Move them into view with scene_apply set-node, or ${framingSuggestion(camera, boxes, tangent)} to put every part in view.`,
      outsideDetails)];
  }
  return [];
}

/** The share of pixels near black and near white in RGBA bytes, by Rec. 709 luma. */
export function lumaShares(rgba) {
  let black = 0;
  let white = 0;
  const pixels = rgba.length / 4;
  for (let offset = 0; offset < rgba.length; offset += 4) {
    const luma = (0.2126 * rgba[offset] + 0.7152 * rgba[offset + 1] + 0.0722 * rgba[offset + 2]) / 255;
    if (luma <= BLACK_LUMA) black++;
    else if (luma >= WHITE_LUMA) white++;
  }
  return { black: black / pixels, white: white / pixels };
}

function percents(shares) {
  return { blackPercent: Math.round(shares.black * 100), whitePercent: Math.round(shares.white * 100) };
}

const MAXIMUM_NAMED_LIGHTS = 3;

/**
 * The lights of an environment, strongest first, as `{ field, intensity }`: the key light and each point, spot and area
 * light by its place in its list. Intensity is the number the recipe states, so a finding can point at the field to change.
 */
export function strongestLights(environment) {
  if (!environment) return [];
  const lights = [{ field: 'environment.keyIntensity', intensity: environment.keyIntensity ?? 1 }];
  for (const listName of ['pointLights', 'spotLights', 'areaLights']) {
    (environment[listName] ?? []).forEach((light, index) => {
      if (light.portal === true) return;
      lights.push({ field: `environment.${listName}[${index}].intensity`, intensity: light.intensity ?? 1 });
    });
  }
  return lights.sort((first, second) => second.intensity - first.intensity).slice(0, MAXIMUM_NAMED_LIGHTS);
}

function clippedFix(exposure, lights) {
  const exposureFix = `Lower effects.exposure from ${exposure} to ${round(exposure / 2)}`;
  if (lights.length === 0) return `${exposureFix}, or lower environment.keyIntensity.`;
  return `${exposureFix}, or lower the strongest light, ${lights[0].field} (now ${round(lights[0].intensity)}).`;
}

/**
 * Warnings for a frame that is nearly all black or has a large clipped area; `exposure` is the current effects.exposure.
 * With `environment`, a clipped finding lists the strongest lights in `lights` and names the field of the first in its fix,
 * so an agent can fix an over-exposed light without opening the image.
 */
export function exposureProblems({ shares, exposure, environment }) {
  if (shares.black >= BLACK_FRAME_SHARE) {
    return [warning('exposure-black', 'effects.exposure',
      `${Math.round(shares.black * 100)} percent of the frame is black after tone mapping.`,
      `Raise effects.exposure from ${exposure} to ${round(Math.max(exposure, 0.25) * 4)}, raise environment.keyIntensity, or aim a light at the subject.`, percents(shares))];
  }
  if (shares.white >= CLIPPED_FRAME_SHARE) {
    const lights = strongestLights(environment);
    return [warning('exposure-clipped', 'effects.exposure',
      `${Math.round(shares.white * 100)} percent of the frame is clipped to white after tone mapping.`,
      clippedFix(exposure, lights), { ...percents(shares), ...(lights.length ? { lights } : {}) })];
  }
  return [];
}

const FLOATING_GAP = 0.02;
const MINIMUM_PENETRATION = 0.02;
const MAXIMUM_PAIR_PARTS = 256;
const ENCLOSURE_VOLUME_RATIO = 20;
const ENCLOSURE_SIDE_RATIO = 10;
const MAXIMUM_PLACEMENT_FINDINGS = 8;

/** Distance between two boxes along each axis; 0 where they touch or overlap on that axis. */
function axisGaps(first, second) {
  return [0, 1, 2].map(axis => Math.max(first.min[axis] - second.max[axis], second.min[axis] - first.max[axis], 0));
}

/** How far two boxes overlap along each axis, or null when they are apart on any axis. */
function axisPenetrations(first, second) {
  const depths = [0, 1, 2].map(axis => Math.min(first.max[axis], second.max[axis]) - Math.max(first.min[axis], second.min[axis]));
  return depths.every(depth => depth > 0) ? depths : null;
}

function volumeOf(box) {
  return [0, 1, 2].reduce((volume, axis) => volume * (box.max[axis] - box.min[axis]), 1);
}

function coversFootprint(outer, inner) {
  return [0, 2].every(axis => outer.min[axis] <= inner.min[axis] && outer.max[axis] >= inner.max[axis]);
}

function longestSide(box) {
  return Math.max(...[0, 1, 2].map(axis => box.max[axis] - box.min[axis]));
}

/**
 * One box covers the other's footprint and is 20 times its volume and 10 times its longest side or more: a terrain, a
 * hollow room or a bowl whose bounds enclose what rests in it, even where a tree pokes through the top. Two crates sunk into each other, or a cup half inside a table, are not enclosures.
 */
function isEnclosure(first, second) {
  const [larger, smaller] = volumeOf(first) >= volumeOf(second) ? [first, second] : [second, first];
  return coversFootprint(larger, smaller)
    && volumeOf(larger) >= ENCLOSURE_VOLUME_RATIO * volumeOf(smaller)
    && longestSide(larger) >= ENCLOSURE_SIDE_RATIO * longestSide(smaller);
}

function droppedCount(total) {
  return total > MAXIMUM_PLACEMENT_FINDINGS ? { droppedCount: total - MAXIMUM_PLACEMENT_FINDINGS } : {};
}

/**
 * Findings about where parts sit, from world bounds ({ name, min, max }, y up): a part with nothing within 2 cm of it
 * that is above the lowest point of the scene (`floating-node`, `gapMeters` to the nearest part or floor), and two
 * parts whose bounds overlap by 2 cm or more on every axis (`intersecting-nodes`, `penetrationMeters`). Bounds are
 * axis aligned, so a rotated part may report an overlap its surface does not have. Each kind lists at most 8 and says
 * how many more there were in `droppedCount`. Scenes of over 256 parts are not checked.
 */
export function placementProblems({ boxes }) {
  if (boxes.length < 1 || boxes.length > MAXIMUM_PAIR_PARTS) return [];
  const floorHeight = Math.min(...boxes.map(box => box.min[1]));
  const floating = [];
  const intersecting = [];
  boxes.forEach((box, index) => {
    let nearest = { gap: box.min[1] - floorHeight, name: 'floor' };
    for (let otherIndex = 0; otherIndex < boxes.length; otherIndex++) {
      if (otherIndex === index) continue;
      const other = boxes[otherIndex];
      const gap = Math.hypot(...axisGaps(box, other));
      if (gap < nearest.gap) nearest = { gap, name: other.name };
      const penetrations = otherIndex > index ? axisPenetrations(box, other) : null;
      if (penetrations && Math.min(...penetrations) >= MINIMUM_PENETRATION && !isEnclosure(box, other)) {
        intersecting.push({ nodes: [box.name, other.name], penetrationMeters: round(Math.min(...penetrations)) });
      }
    }
    if (nearest.gap > FLOATING_GAP) floating.push({ node: box.name, gapMeters: round(nearest.gap), nearest: nearest.name });
  });
  const problems = [];
  const floatingWorstFirst = floating.sort((first, second) => second.gapMeters - first.gapMeters);
  for (const entry of floatingWorstFirst.slice(0, MAXIMUM_PLACEMENT_FINDINGS)) {
    problems.push(warning('floating-node', entry.node,
      `${entry.node} touches nothing: it is ${entry.gapMeters} m from ${entry.nearest === 'floor' ? 'the floor level' : entry.nearest}.`,
      'Lower it with scene_apply set-node, or give it a relation that rests it on a part, unless it is meant to fly.',
      { nodes: [entry.node], gapMeters: entry.gapMeters, nearest: entry.nearest, ...droppedCount(floating.length) }));
  }
  const intersectingDeepestFirst = intersecting.sort((first, second) => second.penetrationMeters - first.penetrationMeters);
  for (const entry of intersectingDeepestFirst.slice(0, MAXIMUM_PLACEMENT_FINDINGS)) {
    problems.push(warning('intersecting-nodes', entry.nodes[0],
      `${entry.nodes[0]} and ${entry.nodes[1]} overlap by ${entry.penetrationMeters} m inside their bounds.`,
      'Move one of them apart with scene_apply set-node, or ignore it where the overlap is meant, such as a part sunk into the ground.',
      { nodes: entry.nodes, penetrationMeters: entry.penetrationMeters, ...droppedCount(intersecting.length) }));
  }
  return problems;
}

const TEXTURE_FIELDS = ['map', 'normalMap', 'metallicRoughnessMap', 'emissiveMap', 'occlusionMap', 'aovMap', 'transmissionMap'];

/** Why a texture descriptor cannot be drawn, or null when it can: no image, an image of no size, or fewer pixel bytes than its size needs. */
function textureFault(descriptor) {
  const image = descriptor.image;
  if (!image) return 'has no image';
  if (!(image.width > 0 && image.height > 0)) return 'has an image of no size';
  const hasPixelBytes = image.data !== undefined && image.data !== null && !image.compressed;
  if (hasPixelBytes && image.data.length < image.width * image.height * 4) return 'has fewer pixel bytes than its size needs';
  return null;
}

/**
 * Findings for materials that name a texture the renderer cannot draw (`texture-missing`): the field is set but its
 * image is absent, empty or truncated, so the part draws flat where the texture should be. `parts` are drawable
 * parts ({ name, material }); parts that share a material are one finding with every part in `nodes`.
 */
export function textureProblems({ parts }) {
  const byFault = new Map();
  for (const part of parts) {
    for (const field of TEXTURE_FIELDS) {
      const descriptor = part.material?.[field];
      if (!descriptor) continue;
      const fault = textureFault(descriptor);
      if (!fault) continue;
      const key = `${field}|${fault}|${part.material.id ?? part.name}`;
      const entry = byFault.get(key) ?? { field, fault, material: part.material.id, names: [] };
      entry.names.push(part.name);
      byFault.set(key, entry);
    }
  }
  return [...byFault.values()].slice(0, MAXIMUM_PLACEMENT_FINDINGS).map(({ field, fault, material, names }) => {
    const owner = names.length > 1 ? `${names.length} parts use` : `${names[0]} uses`;
    const effect = field === 'map' ? 'the surface draws as its plain colour' : `the ${field} has no effect`;
    return warning('texture-missing', names[0],
      `${owner} ${material ? `material ${material}` : 'a material'} whose ${field} ${fault}, so ${effect}.`,
      `Fix the image behind ${field}${material ? ` of material ${material}` : ''} (re-add the asset or catalog image), or remove the field with scene_apply set-material.`,
      { ...namedNodes(names), field, ...(material ? { material } : {}) });
  });
}

/**
 * What a path traced job cannot report. The tracer writes the lit image only: it has no object ID, depth or normal pass,
 * and the page never draws the frame it traces, so there is no frame to read exposure from. Part findings (framing,
 * floating, intersecting, textures, z-fighting) come from part bounds and materials and still run.
 */
export function pathTracerFindingsLimit() {
  return {
    kind: 'findings-unavailable',
    severity: 'info',
    target: 'renderer',
    message: 'This job is path traced. The tracer writes the lit image only, so exposure findings and the object ID, depth and normal passes are unavailable; the page draws no raster frame to read them from.',
    fix: 'Check exposure with scene_look or a raster job (leave renderer out), then trace. On a daylit interior the traced frame comes out about 1 to 2 stops brighter than the raster one at the same effects.exposure, because the tracer adds the light the room bounces around and the raster fill is lower (docs/path-tracer.md, Exposure): lower effects.exposure for the trace instead of raising it. Part findings (framing, floating, intersecting, textures, z-fighting) are reported as usual.',
  };
}

/**
 * The warning a path traced job carries when its scene has `postPasses`. The tracer writes the lit frame only and has no
 * first-hit depth, normal or object ID buffer for a pass to read, so the job returns the frame without the passes.
 * Returns undefined when the scene has none, so a job without `postPasses` is unchanged.
 */
export function pathTracerSkippedPassesWarning(postPasses) {
  if (!postPasses?.length) return undefined;
  const names = postPasses.map(pass => pass.name).join(', ');
  return {
    kind: 'post-passes-skipped',
    severity: 'warning',
    target: 'renderer',
    message: `This job is path traced, and the path tracer does not run postPasses: ${postPasses.length === 1 ? 'the pass' : 'the passes'} ${names} did not touch the image. The PNG is the plain traced frame.`,
    fix: 'Render the look with a raster job (leave renderer out), or trace for the lit frame and accept it without the passes (docs/user-passes.md, "Where a pass runs").',
  };
}

/** Adds a frame's diagnostics to `metrics.diagnostics`, keeping one entry per kind, target and message across a job's frames, up to 16 entries. */
export function mergeFrameDiagnostics(metrics, frameDiagnostics) {
  if (!frameDiagnostics?.length) return;
  metrics.diagnostics ??= [];
  for (const problem of frameDiagnostics) {
    const known = metrics.diagnostics.some(entry => entry.kind === problem.kind && entry.target === problem.target && (problem.outsideCount !== undefined || entry.message === problem.message));
    if (!known && metrics.diagnostics.length < MAXIMUM_JOB_DIAGNOSTICS) metrics.diagnostics.push(problem);
  }
}