3D engine that AI agents drive
Add custom properties, distance and rotation difference variables, and light and shape key driver targets
Nguyen Xuan Anecc6399db9a8
diff --git a/changes/694.md b/changes/694.mdnew file mode 100644index 00000000..c932a29c--- /dev/null+++ b/changes/694.md@@ -0,0 +1 @@+- Motion drivers read custom properties on nodes and bones (`motion.properties`), distance and rotation difference variables, and can drive lights and shape keys (#694)diff --git a/changes/694.size.md b/changes/694.size.mdnew file mode 100644index 00000000..92e9b801--- /dev/null+++ b/changes/694.size.md@@ -0,0 +1,3 @@+- scene:agent-scene: custom properties and the distance and rotation difference variables are the rig inputs agents need to drive a lid, a light and a shape key from one control; the core grew 65 bytes gzip, the rest sits in the lazy motion chunk.+- scene:agent-scene+lazy: the property normalizer, three new expression variables and light and morph driver targets add 1978 bytes gzip to the lazy chunk, which only loads for scenes with motion.+- goal:agent-scene: the 65 bytes gzip are the capabilities entry that lets an agent see the new driver limits; the code stays in the lazy chunk.diff --git a/docs/agent-motion.md b/docs/agent-motion.mdindex 8602fa71..a16a0d70 100644--- a/docs/agent-motion.md+++ b/docs/agent-motion.md@@ -308,7 +308,10 @@ Not covered yet: `use_*` switches on `child_of`, `damped_track`, bone sub-target`motion.drivers` sets one node component or material property from a bounded expression, at most 32. A target is`{ "kind": "node", "id", "property": "translation" | "rotation" | "scale", "component": "x" | "y" | "z" }` (rotation is EulerXYZ in degrees) or `{ "kind": "material", "id", "property" }` for `emission`, `metallic` and `roughness`, or with a-`component` for `color` and `emissiveColor` (`x`, `y`, `z` for red, green, blue). A driven property cannot also have a track.+`component` for `color` and `emissiveColor` (`x`, `y`, `z` for red, green, blue). Two more targets set a look value:+`{ "kind": "light", "id": "point:0", "property": "intensity" }` (or `color` with a `component`, where the other components keep the+recipe colour) and `{ "kind": "morph", "id": "<shape key>", "property": "weight" }`. A driven property cannot also have a track.+Camera fields are not driven yet.An expression is JSON data and is never run as code. It is the motion context of the one expression language described in[field expressions](mesh.md#field-expressions): the same operators with the same results, the same syntax, and limits of 64@@ -317,6 +320,33 @@ parts and 8 levels. Where a field reads mesh attributes, a driver reads `"time"`inputs and are rejected here with a message that says so. `sin` and `cos` take radians. One difference remains: `div` by zerogives 0 in a field but is a non-finite result in a driver, which is an error, so a mistaken driver is reported.+### Variables and custom properties++Besides `"time"` and a node read (a transform channel), an expression reads three more variables:++- `{ "custom": { "node", "name", "component" } }`: a custom property. `component` (`x`, `y`, `z`) is required for a vector or colour and+ not allowed for a number.+- `{ "distance": { "from", "to" } }`: the world distance between two nodes, in metres.+- `{ "rotationDifference": { "from", "to" } }`: the angle between their world rotations, 0 to 180 degrees.++`motion.properties` declares custom properties on nodes and bones (any node ID of the recipe or of an imported rig), at most 32:+`{ "node", "name", "kind": "number" | "vector" | "color", "value", "min", "max", "keys" }`. A number defaults to 0 within+-1e6 to 1e6, a colour to 0 to 1. Give either a constant `value` or up to 16 `keys` of `{ "time", "value" }` with rising times: the+property is linear between them and holds the first and last value outside. Every value must lie inside `min` and `max`. One+property can drive several parts:++```json+{ "duration": 2, "tracks": [],+ "properties": [{ "node": "lid", "name": "open", "min": 0, "max": 1, "keys": [{ "time": 0, "value": 0 }, { "time": 2, "value": 1 }] }],+ "drivers": [+ { "target": { "kind": "node", "id": "lid", "property": "rotation", "component": "x" }, "expression": { "mul": [{ "custom": { "node": "lid", "name": "open" } }, 90] } },+ { "target": { "kind": "light", "id": "point:0", "property": "intensity" }, "expression": { "sub": [1, { "custom": { "node": "lid", "name": "open" } }] } }+ ] }+```++Only driver expressions read properties; a constraint `influence` expression does not yet. A property is a pure function of the+time, so seeking gives the same value as playing to it.+### Evaluation orderFor any time, the pose is built in this order and keeps no state between samples, so seeking gives the same pose as playingdiff --git a/src/agent-motion.js b/src/agent-motion.jsindex f67c3fe6..1b432b7f 100644--- a/src/agent-motion.js+++ b/src/agent-motion.js@@ -5,6 +5,7 @@ import { createCameraPath } from './camera-path.js';import { CAMERA_LENS_FIELDS } from './camera.js';import { createSkeletonPose } from './skeleton-pose.js';import { createNodeConstraintEvaluator, normalizeConstraints, normalizeDrivers } from './node-constraints.js';+import { normalizeProperties, propertyKinds } from './motion-properties.js';import { normalizeClipLayers } from './agent-clip-layers.js';import { normalizeKeyframeEasing } from './easing.js';import { normalizeFootLock } from './foot-lock.js';@@ -474,7 +475,7 @@ export function normalizeAgentMotion(input, recipe) {if (input === null || input === undefined) {return null;}- knownKeys(input, ['duration', 'loop', 'tracks', 'ik', 'constraints', 'drivers', 'layers', 'footLock', 'drive'], 'motion');+ knownKeys(input, ['duration', 'loop', 'tracks', 'ik', 'constraints', 'drivers', 'properties', 'layers', 'footLock', 'drive'], 'motion');const drive = input.drive === undefined ? null : normalizedDrive(input.drive, recipe);// A drive knows how long its path takes, so the motion may leave the duration out.const duration = bounded(input.duration === undefined && drive ? Math.ceil(drive.built.duration * 1000) / 1000 : input.duration, 0.1, 30, 'motion.duration');@@ -566,8 +567,12 @@ export function normalizeAgentMotion(input, recipe) {if (input.constraints !== undefined) {result.constraints = normalizeConstraints(input.constraints, nodes, visibleIds);}+ if (input.properties !== undefined) {+ result.properties = normalizeProperties(input.properties, duration, visibleIds);+ }if (input.drivers !== undefined) {- result.drivers = normalizeDrivers(input.drivers, nodes, materials, visibleIds);+ const lights = new Set(recipe.environment ? lightIds(recipe.environment) : []);+ result.drivers = normalizeDrivers(input.drivers, { nodes, materials, lights, morphs: morphTargets, visible: visibleIds, properties: propertyKinds(result.properties) });for (const { target } of result.drivers) {if (identities.has(JSON.stringify([target.kind, target.id, target.property]))) {throw invalidField('motion.drivers', `${target.id} ${target.property}`, `a property no track animates, since a track on ${target.kind} ${target.id} already moves ${target.property}; drop the track or the driver`);@@ -721,6 +726,14 @@ function validateRadiance(material, materialId) {}}+const COMPONENT_INDEX = { x: 0, y: 1, z: 2 };++/** The colour a light has in the recipe: the components of a driven colour that no driver sets keep it. */+function drivenLightColor(recipe, id) {+ const [type, index] = id.split(':');+ return recipe.environment[LIGHT_LISTS[type]][Number(index)].color ?? [1, 1, 1];+}+/** IDs of the nodes a driver or constraint writes. */function constrainedNodeIds(motion) {const driven = (motion.drivers ?? []).filter(driver => driver.target.kind === 'node').map(driver => driver.target.id);@@ -822,7 +835,7 @@ export function compileAgentMotion(input, recipe) {const ikEvaluator = motion.ik?.length ? createIkEvaluator(motion, recipe, stagedNodeTracks) : null;const constraintEvaluator = createNodeConstraintEvaluator({hierarchy: ikHierarchy(recipe), rootMatrix: recipe.ikRootMatrix,- constraints: motion.constraints, drivers: motion.drivers, materials: recipe.materials ?? [],+ constraints: motion.constraints, drivers: motion.drivers, materials: recipe.materials ?? [], properties: motion.properties,});// An imported asset's base pose comes from its decoder, so it asks for IK and constraints itself with solveStagedPose.const ikUsesStagedPose = Boolean((ikEvaluator || constraintEvaluator) && recipe.ikHierarchy);@@ -852,6 +865,13 @@ export function compileAgentMotion(input, recipe) {if (!output.materials.has(target.id)) output.materials.set(target.id, {});output.materials.get(target.id)[target.property] = target.component === undefined ? 0 : new Float64Array(3);}+ for (const { target } of (motion.drivers ?? []).filter(driver => driver.target.kind === 'light' || driver.target.kind === 'morph')) {+ const patches = target.kind === 'light' ? output.lights : output.morphs;+ if (!patches.has(target.id)) patches.set(target.id, {});+ const patch = patches.get(target.id);+ if (target.component === undefined) patch[target.property] = 0;+ else patch.color ??= new Float64Array(drivenLightColor(recipe, target.id));+ }const driveSampler = motion.drive ? recipe.buildDrive(motion.drive) : null;if (driveSampler) {// The drive touches the same nodes and tail-light materials at every time, so its patches are allocated once.@@ -971,6 +991,7 @@ export function compileAgentMotion(input, recipe) {materialSamples.get(target.id)[target.property] = value;drivenValues.push({ target, value });}+ for (const { target, value } of evaluated?.looks ?? []) validateValue(target.kind, target.property, value, `driver ${target.kind} ${target.id} ${target.property}`);for (const [materialId, material] of materialSamples) {validateRadiance(material, materialId);}@@ -1016,6 +1037,11 @@ export function compileAgentMotion(input, recipe) {if (typeof value === 'number') materialPatch[target.property] = value;else materialPatch[target.property].set(value);}+ for (const { target, value } of evaluated?.looks ?? []) {+ const patch = (target.kind === 'light' ? output.lights : output.morphs).get(target.id);+ if (target.component === undefined) patch[target.property] = value;+ else patch[target.property][COMPONENT_INDEX[target.component]] = value;+ }if (cameraPath) {for (const field of ['eye', 'target', 'up']) {output.camera[field].set(stagedCamera[field]);diff --git a/src/agent-scene-capabilities.js b/src/agent-scene-capabilities.jsindex 646b005a..cc77e68d 100644--- a/src/agent-scene-capabilities.js+++ b/src/agent-scene-capabilities.js@@ -285,7 +285,7 @@ export const AGENT_SCENE_CAPABILITIES = Object.freeze({}),rigging: Object.freeze({ presets: Object.freeze(['humanoid', 'quadruped']), maxArmatures: 16, maxBones: 128, maxInfluences: 4, weights: 'bone-distance' }),motionConstraints: Object.freeze(['copy_location', 'copy_rotation', 'copy_scale', 'limit_location', 'limit_rotation', 'track_to', 'child_of', 'floor', 'copy_transforms', 'limit_distance', 'limit_scale', 'stretch_to', 'locked_track', 'transformation', 'action', 'maintain_volume', 'damped_track', 'pivot', 'follow_path', 'clamp_to', 'shrinkwrap', 'armature']),- motionDrivers: Object.freeze({ maxDrivers: 32, maxConstraints: 32, maxExpressionParts: 64, maxExpressionDepth: 8 }),+ motionDrivers: Object.freeze({ maxDrivers: 32, maxConstraints: 32, maxProperties: 32, maxPropertyKeys: 16, targets: Object.freeze(['node', 'material', 'light', 'morph']), variables: Object.freeze(['time', 'node', 'custom', 'distance', 'rotationDifference']), maxExpressionParts: 64, maxExpressionDepth: 8 }),// Recipe physics: Rapier (an optional peer) bakes bodies on nodes offline; playback is a seek into the bake.physics: Object.freeze({bodyTypes: Object.freeze(['dynamic', 'fixed', 'kinematic']),diff --git a/src/motion-expression.js b/src/motion-expression.jsindex 49040038..ecb43c9d 100644--- a/src/motion-expression.js+++ b/src/motion-expression.js@@ -7,6 +7,10 @@ import { COMPONENTS, compileNode, createOperators, isPlainObject, liftOperator,// { "mul": [{ "node": "wheel", "property": "rotation", "component": "x" }, 0.01] }// { "clamp": [{ "mul": ["time", 0.5] }, 0, 1] }//+// Variables, besides "time" (the motion time in seconds):+// { "custom": { "node": "lid", "name": "open", "component": "x" } } a custom property (component for vector and colour)+// { "distance": { "from": "a", "to": "b" } } world distance between two nodes+// { "rotationDifference": { "from": "a", "to": "b" } } angle between two world rotations, 0 to 180 degrees// The string "time" is the motion time in seconds. A node read takes a component (x, y or z) of a node's translation,// scale, or rotation (Euler XYZ in degrees), in "local" (default) or "world" space. Vectors, attributes and noise// belong to fields and are rejected here. sin and cos take radians.@@ -33,29 +37,66 @@ function compileRead(input, state) {return scalarValue(context => context.read(input.node, input.property, component, space));}+function compileCustom(input, state) {+ const { label, nodeIds, properties } = state;+ knownKeys(input, ['node', 'name', 'component'], `${label} custom property read`);+ if (!nodeIds.has(input.node)) throw new RangeError(`${label}: node ${JSON.stringify(input.node)} is not in this scene.`);+ const kind = properties.get(input.node)?.get(input.name);+ if (!kind) {+ const names = [...(properties.get(input.node)?.keys() ?? [])];+ throw new RangeError(`${label}: node ${JSON.stringify(input.node)} has no custom property ${JSON.stringify(input.name)}; add it to motion.properties first${names.length ? ` (it has ${names.join(', ')})` : ''}.`);+ }+ if (kind === 'number' && input.component !== undefined) throw new RangeError(`${label}: custom property ${JSON.stringify(input.name)} is a number and takes no component.`);+ if (kind !== 'number' && !Object.hasOwn(COMPONENTS, input.component)) throw new RangeError(`${label}: custom property ${JSON.stringify(input.name)} is a ${kind}; component must be x, y or z.`);+ state.reads.add(input.node);+ const componentIndex = COMPONENTS[input.component] ?? 0;+ return scalarValue(context => context.custom(input.node, input.name, componentIndex));+}++function compilePair(input, state, name, read) {+ const { label, nodeIds } = state;+ knownKeys(input, ['from', 'to'], `${label} ${name}`);+ for (const id of [input.from, input.to]) {+ if (!nodeIds.has(id)) throw new RangeError(`${label}: ${name} node ${JSON.stringify(id)} is not in this scene.`);+ state.reads.add(id);+ }+ return scalarValue(context => read(context, input.from, input.to));+}++const PAIR_READS = new Map([+ ['distance', (context, from, to) => context.distance(from, to)],+ ['rotationDifference', (context, from, to) => context.rotationDifference(from, to)],+]);+function compileInput(node, state) {if (node === 'time') return scalarValue(context => context.time);- if (isPlainObject(node) && Object.hasOwn(node, 'node')) return compileRead(node, state);+ if (!isPlainObject(node)) return undefined;+ if (Object.hasOwn(node, 'node')) return compileRead(node, state);+ const keys = Object.keys(node);+ if (keys.length !== 1) return undefined;+ if (keys[0] === 'custom') return compileCustom(node.custom, state);+ if (PAIR_READS.has(keys[0])) return compilePair(node[keys[0]], state, keys[0], PAIR_READS.get(keys[0]));return undefined;}const MOTION_DIALECT = Object.freeze({name: 'motion',- accepts: 'a number, "time", a node read or an operator object',+ accepts: 'a number, "time", a node read, a custom property, a distance, a rotationDifference or an operator object',maxNodes: MAX_EXPRESSION_NODES,maxDepth: MAX_EXPRESSION_DEPTH,operators: OPERATORS,- sourceNames: Object.freeze(['time', 'node']),+ sourceNames: Object.freeze(['time', 'node', 'custom', 'distance', 'rotationDifference']),reject: (message, state) => new TypeError(`${state.label}: ${message}`),compileInput,});/**- * Compile an expression. `evaluate({ time, read })` returns a finite number; `read(id, property, component, space)`- * supplies node values. `reads` lists the node IDs the expression looks at.+ * Compile an expression. `evaluate({ time, read, custom, distance, rotationDifference })` returns a finite number;+ * `read(id, property, component, space)` supplies node values, `custom(node, name, componentIndex)` a custom property+ * and the last two the world distance and rotation difference of two nodes. `properties` is node -> name -> kind. `reads` lists the node IDs the expression looks at.*/-export function compileMotionExpression(input, nodeIds, label) {- const state = { dialect: MOTION_DIALECT, nodeCount: 0, label, nodeIds, reads: new Set() };+export function compileMotionExpression(input, nodeIds, label, properties = new Map()) {+ const state = { dialect: MOTION_DIALECT, nodeCount: 0, label, nodeIds, properties, reads: new Set() };const { evaluate: evaluateTree } = compileNode(input, state, 0);return Object.freeze({reads: [...state.reads],diff --git a/src/motion-properties.js b/src/motion-properties.jsnew file mode 100644index 00000000..ad1b2094--- /dev/null+++ b/src/motion-properties.js@@ -0,0 +1,124 @@+import { bounded, denseArray, knownKeys, valueFrom } from './motion-validation.js';+import { invalidField } from './field-error.js';++// Custom properties of nodes and bones: named control values that driver expressions read, such as an `open` slider+// of 0 to 1 that moves a lid and dims a light. A property holds one number, a vector or a colour, stays inside its+// min and max, and is constant or animated by linear keys. Its value is a pure function of the time.++export const MAX_PROPERTIES = 32;+export const MAX_PROPERTY_KEYS = 16;+export const PROPERTY_KINDS = Object.freeze(['number', 'vector', 'color']);+const MAX_NAME_LENGTH = 64;+const MAX_MAGNITUDE = 1e6;+const DEFAULT_RANGES = { number: [-MAX_MAGNITUDE, MAX_MAGNITUDE], vector: [-MAX_MAGNITUDE, MAX_MAGNITUDE], color: [0, 1] };++function sizeOf(kind) {+ return kind === 'number' ? 1 : 3;+}++function checkedValue(raw, kind, minimum, maximum, label) {+ if (sizeOf(kind) === 1) return bounded(raw, minimum, maximum, label);+ const value = valueFrom(raw, 3, label);+ for (const component of value) bounded(component, minimum, maximum, label);+ return value;+}++function defaultValue(kind, minimum, maximum) {+ const zero = Math.min(maximum, Math.max(minimum, 0));+ return sizeOf(kind) === 1 ? zero : [zero, zero, zero];+}++function normalizeProperty(input, position, duration, nodeIds) {+ const label = `Motion property ${position}`;+ knownKeys(input, ['node', 'name', 'kind', 'value', 'min', 'max', 'keys'], label);+ if (!nodeIds.has(input.node)) {+ throw invalidField(`${label} node`, input.node, 'a node or bone ID in this recipe, such as "lid"');+ }+ if (typeof input.name !== 'string' || !input.name.trim() || input.name.length > MAX_NAME_LENGTH) {+ throw invalidField(`${label} name`, input.name, `a name of 1 to ${MAX_NAME_LENGTH} characters, such as "open"`);+ }+ const kind = input.kind ?? 'number';+ if (!PROPERTY_KINDS.includes(kind)) {+ throw invalidField(`${label} kind`, kind, `one of ${PROPERTY_KINDS.join(', ')}`);+ }+ const [defaultMinimum, defaultMaximum] = DEFAULT_RANGES[kind];+ const minimum = bounded(input.min ?? defaultMinimum, -MAX_MAGNITUDE, MAX_MAGNITUDE, `${label} min`);+ const maximum = bounded(input.max ?? defaultMaximum, -MAX_MAGNITUDE, MAX_MAGNITUDE, `${label} max`);+ if (!(minimum < maximum)) {+ throw invalidField(`${label} min`, minimum, `a number below max (${maximum})`);+ }+ const result = { node: input.node, name: input.name, kind, min: minimum, max: maximum };+ if (input.keys === undefined) {+ result.value = checkedValue(input.value ?? defaultValue(kind, minimum, maximum), kind, minimum, maximum, `${label} value`);+ return result;+ }+ if (input.value !== undefined) {+ throw invalidField(`${label} value`, input.value, 'a field left out when keys are given, since the keys set the value');+ }+ denseArray(input.keys, MAX_PROPERTY_KEYS, `${label} keys`);+ if (!input.keys.length) {+ throw invalidField(`${label} keys`, input.keys, 'at least one key, such as [{ "time": 0, "value": 0 }, { "time": 1, "value": 1 }]');+ }+ result.keys = input.keys.map((key, index) => {+ knownKeys(key, ['time', 'value'], `${label} key ${index}`);+ return {+ time: bounded(key.time, 0, duration, `${label} key ${index} time`),+ value: checkedValue(key.value, kind, minimum, maximum, `${label} key ${index} value`),+ };+ });+ if (result.keys.some((key, index) => index && key.time <= result.keys[index - 1].time)) {+ throw invalidField(`${label} keys`, input.keys.map(key => key.time), 'keys with rising times, one value per time');+ }+ return result;+}++/** Validate a property list against the node IDs a motion may read. Returns plain data. */+export function normalizeProperties(input, duration, nodeIds) {+ denseArray(input, MAX_PROPERTIES, 'Motion properties');+ const seen = new Set();+ return input.map((property, position) => {+ const normalized = normalizeProperty(property, position, duration, nodeIds);+ const identity = JSON.stringify([normalized.node, normalized.name]);+ if (seen.has(identity)) {+ throw invalidField(`Motion property ${position} name`, normalized.name, `a name no other property of node ${normalized.node} has`);+ }+ seen.add(identity);+ return normalized;+ });+}++/** node ID -> property name -> kind, for compiling expressions that read properties. */+export function propertyKinds(properties = []) {+ const kinds = new Map();+ for (const { node, name, kind } of properties) {+ if (!kinds.has(node)) kinds.set(node, new Map());+ kinds.get(node).set(name, kind);+ }+ return kinds;+}++function valueAt(property, time) {+ const { keys } = property;+ if (!keys) return property.value;+ if (time <= keys[0].time) return keys[0].value;+ const last = keys[keys.length - 1];+ if (time >= last.time) return last.value;+ const after = keys.findIndex(key => key.time > time);+ const before = keys[after - 1];+ const fraction = (time - before.time) / (keys[after].time - before.time);+ const mixed = (start, end) => start + (end - start) * fraction;+ return typeof before.value === 'number' ? mixed(before.value, keys[after].value) : before.value.map((start, index) => mixed(start, keys[after].value[index]));+}++/** `read(node, name, componentIndex, time)` gives a number property or one component of a vector or colour. */+export function createPropertyReader(properties = []) {+ const byNode = new Map();+ for (const property of properties) {+ if (!byNode.has(property.node)) byNode.set(property.node, new Map());+ byNode.get(property.node).set(property.name, property);+ }+ return (node, name, componentIndex, time) => {+ const value = valueAt(byNode.get(node).get(name), time);+ return typeof value === 'number' ? value : value[componentIndex];+ };+}diff --git a/src/node-constraints.js b/src/node-constraints.jsindex 7a4df530..252384a4 100644--- a/src/node-constraints.js+++ b/src/node-constraints.js@@ -1,5 +1,6 @@import { mat3, mat4, quat, vec3 } from 'gl-matrix';import { compileMotionExpression } from './motion-expression.js';+import { createPropertyReader, propertyKinds } from './motion-properties.js';import { bounded, denseArray, knownKeys, valueFrom } from './motion-validation.js';import { eulerFromQuaternion, quaternionFromEuler } from './rotation-euler.js';@@ -21,6 +22,12 @@ const MAX_COORDINATE = 1e6;const SPACES = ['world', 'local'];const MATERIAL_SCALARS = ['emission', 'metallic', 'roughness'];const MATERIAL_VECTORS = ['color', 'emissiveColor'];+// Look targets a driver may write. A light is addressed as point:N, spot:N or area:N; a morph by its target name.+const LIGHT_DRIVEN = { intensity: false, color: true };+const MORPH_DRIVEN = { weight: false };+const SCALAR_TARGET_KINDS = ['light', 'morph'];+const TARGET_PROPERTIES = { light: LIGHT_DRIVEN, morph: MORPH_DRIVEN };+const RADIANS_TO_DEGREES = 180 / Math.PI;const COMMON_FIELDS = ['type', 'node', 'influence'];const CONSTRAINT_FIELDS = {@@ -278,7 +285,8 @@ export function normalizeConstraints(input, movableIds, visibleIds) {return input.map((constraint, position) => normalizeConstraint(constraint, position, movableIds, visibleIds));}-function normalizeDriver(input, position, movableIds, materialIds, visibleIds) {+function normalizeDriver(input, position, ids) {+ const { nodes: movableIds, materials: materialIds, visible: visibleIds } = ids;const label = `Motion driver ${position}`;knownKeys(input, ['target', 'expression'], label);knownKeys(input.target, ['kind', 'id', 'property', 'component'], `${label} target`);@@ -293,18 +301,28 @@ function normalizeDriver(input, position, movableIds, materialIds, visibleIds) {const isVector = MATERIAL_VECTORS.includes(property);if (!isVector && !MATERIAL_SCALARS.includes(property)) throw new RangeError(`${label} property must be one of ${[...MATERIAL_VECTORS, ...MATERIAL_SCALARS].join(', ')}.`);if (isVector !== (component !== undefined)) throw new RangeError(`${label}: color properties need a component (x, y, z for red, green, blue); the others take none.`);+ } else if (SCALAR_TARGET_KINDS.includes(kind)) {+ const known = kind === 'light' ? ids.lights : ids.morphs;+ if (!known.has(id)) throw new RangeError(`${label} ${kind} was not found: ${JSON.stringify(id)}; this scene has ${[...known].slice(0, 24).join(', ') || 'none'}.`);+ const isVector = TARGET_PROPERTIES[kind][property];+ if (isVector === undefined) throw new RangeError(`${label} property must be one of ${Object.keys(TARGET_PROPERTIES[kind]).join(', ')} for a ${kind}.`);+ if (isVector !== (component !== undefined)) throw new RangeError(`${label}: ${isVector ? 'a light color needs a component (x, y, z for red, green, blue)' : `${property} takes no component`}.`);} else {- throw new RangeError(`${label} target kind must be node or material.`);+ throw new RangeError(`${label} target kind must be node, material, light or morph.`);}- compileMotionExpression(input.expression, visibleIds, `${label} expression`);+ compileMotionExpression(input.expression, visibleIds, `${label} expression`, ids.properties);return { target: { kind, id, property, ...(component === undefined ? {} : { component }) }, expression: structuredClone(input.expression) };}-export function normalizeDrivers(input, movableIds, materialIds, visibleIds) {+/**+ * Validate a driver list. `ids` holds the Sets of node, material, light and morph IDs a driver may write ({ nodes,+ * materials, lights, morphs }), `visible` (the nodes an expression may read) and `properties` (node -> name -> kind).+ */+export function normalizeDrivers(input, ids) {denseArray(input, MAX_DRIVERS, 'Motion drivers');const seen = new Set();return input.map((driver, position) => {- const normalized = normalizeDriver(driver, position, movableIds, materialIds, visibleIds);+ const normalized = normalizeDriver(driver, position, ids);const identity = JSON.stringify(Object.values(normalized.target));if (seen.has(identity)) throw new RangeError(`Motion driver ${position} drives the same property as an earlier driver.`);seen.add(identity);@@ -726,12 +744,12 @@ function wantsEvaluation(constraints, drivers) {}/**- * Build an evaluator over { hierarchy, rootMatrix, constraints, drivers, materials }. `hierarchy` lists every node as+ * Build an evaluator over { hierarchy, rootMatrix, constraints, drivers, materials, properties }. `hierarchy` lists every node as* { id, parent, translation, rotation, scale }. `evaluate(time, bases)` takes the base pose as a Map of node ID to* { translation, rotation, scale } (missing parts use the rest pose) and returns the final local frames of every node- * a driver or constraint writes, plus driven material values. The base arrays are not changed.+ * a driver or constraint writes, plus driven material values and `looks`, the { target, value } of each driven light or morph. The base arrays are not changed.*/-export function createNodeConstraintEvaluator({ hierarchy, rootMatrix, constraints = [], drivers = [], materials = [] }) {+export function createNodeConstraintEvaluator({ hierarchy, rootMatrix, constraints = [], drivers = [], materials = [], properties = [] }) {if (!wantsEvaluation(constraints, drivers)) return null;const indexById = new Map(hierarchy.map((node, index) => [node.id, index]));const order = topologicalOrder(hierarchy, indexById);@@ -739,10 +757,12 @@ export function createNodeConstraintEvaluator({ hierarchy, rootMatrix, constrainconst rootWorld = rootMatrix ? mat4.clone(rootMatrix) : mat4.create();const nodeIds = new Set(indexById.keys());const materialBaselines = new Map(materials.map(material => [material.id, material]));- const compiledDrivers = drivers.map(driver => ({ ...driver, compiled: compileMotionExpression(driver.expression, nodeIds, 'Motion driver expression') }));+ const kindsByNode = propertyKinds(properties);+ const readProperty = createPropertyReader(properties);+ const compiledDrivers = drivers.map(driver => ({ ...driver, compiled: compileMotionExpression(driver.expression, nodeIds, 'Motion driver expression', kindsByNode) }));const compiledConstraints = constraints.map(constraint => ({...constraint,- compiledInfluence: typeof constraint.influence === 'number' ? null : compileMotionExpression(constraint.influence, nodeIds, 'Motion constraint influence'),+ compiledInfluence: typeof constraint.influence === 'number' ? null : compileMotionExpression(constraint.influence, nodeIds, 'Motion constraint influence', kindsByNode),}));function computeWorlds(locals) {@@ -766,10 +786,32 @@ export function createNodeConstraintEvaluator({ hierarchy, rootMatrix, constrain};}- function applyDriver(driver, locals, context, touchedMaterials) {+ function worldFrame(worlds, id) {+ return decompose(worlds[indexById.get(id)]);+ }++ function readContext(time, locals) {+ const worlds = computeWorlds(locals);+ return {+ time,+ read: readFrom(locals, worlds),+ custom: (node, name, componentIndex) => readProperty(node, name, componentIndex, time),+ distance: (from, to) => vec3.distance(worldFrame(worlds, from).translation, worldFrame(worlds, to).translation),+ rotationDifference: (from, to) => {+ const overlap = Math.abs(quat.dot(worldFrame(worlds, from).rotation, worldFrame(worlds, to).rotation));+ return 2 * Math.acos(Math.min(1, overlap)) * RADIANS_TO_DEGREES;+ },+ };+ }++ function applyDriver(driver, locals, context, touchedMaterials, drivenLooks) {const value = driver.compiled.evaluate(context);const { kind, id, property, component } = driver.target;const index = AXES.indexOf(component);+ if (SCALAR_TARGET_KINDS.includes(kind)) {+ drivenLooks.push({ target: driver.target, value });+ return;+ }if (kind === 'material') {if (index < 0) {touchedMaterials.set(id, { ...touchedMaterials.get(id), [property]: value });@@ -852,10 +894,11 @@ export function createNodeConstraintEvaluator({ hierarchy, rootMatrix, constrainif (base?.scale) frame.scale.set(base.scale);return frame;});- const context = { time, read: readFrom(baseLocals, computeWorlds(baseLocals)) };+ const context = readContext(time, baseLocals);const locals = baseLocals.map(copyFrame);const touchedMaterials = new Map();- for (const driver of compiledDrivers) applyDriver(driver, locals, context, touchedMaterials);+ const drivenLooks = [];+ for (const driver of compiledDrivers) applyDriver(driver, locals, context, touchedMaterials, drivenLooks);for (const constraint of compiledConstraints) applyConstraint(constraint, locals, context);const written = new Set([...compiledDrivers.filter(driver => driver.target.kind === 'node').map(driver => driver.target.id), ...compiledConstraints.map(constraint => constraint.node)]);const nodes = new Map([...written].map(id => [id, locals[indexById.get(id)]]));@@ -863,7 +906,7 @@ export function createNodeConstraintEvaluator({ hierarchy, rootMatrix, constrainif (![...frame.translation, ...frame.rotation, ...frame.scale].every(Number.isFinite)) throw new RangeError(`Motion constraints produced an invalid pose for node ${id}.`);if (frame.scale.some(component => !(Math.fround(component) > 0))) throw new RangeError(`Motion constraints made the scale of node ${id} not positive.`);}- return { nodes, materials: touchedMaterials };+ return { nodes, materials: touchedMaterials, looks: drivenLooks };},});}diff --git a/tests/agent-motion-properties.test.js b/tests/agent-motion-properties.test.jsnew file mode 100644index 00000000..eacf7ea6--- /dev/null+++ b/tests/agent-motion-properties.test.js@@ -0,0 +1,163 @@+import './support/load-agent-scene-modules.mjs';+import assert from 'node:assert/strict';+import test from 'node:test';+import { mat4 } from 'gl-matrix';+import { createAgentScene } from '../src/agent-scene.js';+import { AGENT_SCENE_CAPABILITIES } from '../src/agent-scene-capabilities.js';+import { eulerFromQuaternion, quaternionFromEuler } from '../src/rotation-euler.js';++const DEGREES = Math.PI / 180;++function rigRecipe(motion, nodes = []) {+ return {+ format: '67js.scene', schemaVersion: 1,+ geometries: [+ { id: 'box', type: 'box', bevel: 0 },+ { id: 'face', type: 'mesh', base: { type: 'box', bevel: 0, subdivisions: 2 }, shapeKeys: [{ name: 'lift', offsets: [{ select: { tag: 'top' }, offset: [0, 0.5, 0] }] }] },+ ],+ materials: [{ id: 'paint', color: [0.6, 0.6, 0.6] }],+ nodes: [+ { id: 'lid', geometry: 'box', material: 'paint' },+ { id: 'head', geometry: 'face', material: 'paint' },+ ...nodes.map(node => ({ geometry: 'box', material: 'paint', ...node })),+ ],+ environment: { pointLights: [{ position: [0, 3, 0], color: [1, 0.5, 0.25], intensity: 4 }] },+ effects: {},+ motion,+ };+}++const OPEN = { node: 'lid', name: 'open', min: 0, max: 1, keys: [{ time: 0, value: 0 }, { time: 2, value: 1 }] };+const part = (scene, name) => scene.parts.find(candidate => candidate.name === name);+const worldEuler = (scene, name) => eulerFromQuaternion(mat4.getRotation(new Float64Array(4), part(scene, name).matrix)).map(radians => radians / DEGREES);+const worldPosition = (scene, name) => Array.from(mat4.getTranslation(new Float64Array(3), part(scene, name).matrix));+const close = (actual, expected, tolerance = 1e-4) => assert.ok(Math.abs(actual - expected) <= tolerance, `${actual} vs ${expected}`);++function sampledAt(recipe, time) {+ const scene = createAgentScene(recipe);+ scene.sampleMotion(time);+ return scene;+}++test('one custom property opens a lid, dims a light and lifts a shape key', () => {+ const recipe = rigRecipe({+ duration: 2, tracks: [], properties: [OPEN],+ drivers: [+ { target: { kind: 'node', id: 'lid', property: 'rotation', component: 'x' }, expression: { mul: [{ custom: { node: 'lid', name: 'open' } }, 90] } },+ { target: { kind: 'light', id: 'point:0', property: 'intensity' }, expression: { sub: [1, { custom: { node: 'lid', name: 'open' } }] } },+ { target: { kind: 'morph', id: 'lift', property: 'weight' }, expression: { custom: { node: 'lid', name: 'open' } } },+ ],+ });+ const half = sampledAt(recipe, 1);+ close(worldEuler(half, 'lid')[0], 45, 1e-3);+ close(half.environment.pointLights[0].intensity, 0.5);+ close(Array.from(part(half, 'head').morph.weights)[0], 0.5);+ const open = sampledAt(recipe, 2);+ close(worldEuler(open, 'lid')[0], 90, 1e-3);+ close(open.environment.pointLights[0].intensity, 0);+ close(Array.from(part(open, 'head').morph.weights)[0], 1);+ const closed = sampledAt(recipe, 0);+ close(worldEuler(closed, 'lid')[0], 0, 1e-3);+ close(closed.environment.pointLights[0].intensity, 1);+});++test('vector and colour properties are read one component at a time and stay inside their range', () => {+ const properties = [+ { node: 'lid', name: 'offset', kind: 'vector', value: [1, 2, 3], min: -10, max: 10 },+ { node: 'lid', name: 'tint', kind: 'color', keys: [{ time: 0, value: [0, 0, 0] }, { time: 2, value: [1, 0.8, 0.25] }] },+ ];+ const scene = sampledAt(rigRecipe({+ duration: 2, tracks: [], properties,+ drivers: [+ { target: { kind: 'node', id: 'lid', property: 'translation', component: 'y' }, expression: { custom: { node: 'lid', name: 'offset', component: 'z' } } },+ { target: { kind: 'light', id: 'point:0', property: 'color', component: 'y' }, expression: { custom: { node: 'lid', name: 'tint', component: 'y' } } },+ ],+ }), 1);+ close(worldPosition(scene, 'lid')[1], 3);+ close(scene.environment.pointLights[0].color[1], 0.4);+ assert.deepEqual([scene.environment.pointLights[0].color[0], scene.environment.pointLights[0].color[2]], [1, 0.25]);+ const outOfRange = properties.map(property => ({ ...property }));+ outOfRange[0].value = [11, 0, 0];+ assert.throws(() => createAgentScene(rigRecipe({ duration: 2, tracks: [], properties: outOfRange, drivers: [{ target: { kind: 'node', id: 'lid', property: 'translation', component: 'x' }, expression: 1 }] })), /Motion property 0 value/);+});++test('a property lives on a bone as well as on a node', () => {+ const recipe = rigRecipe({+ duration: 1, tracks: [],+ properties: [{ node: 'rig_arm', name: 'bend', min: 0, max: 1, value: 0.5 }],+ drivers: [{ target: { kind: 'node', id: 'lid', property: 'translation', component: 'x' }, expression: { custom: { node: 'rig_arm', name: 'bend' } } }],+ }, [{ id: 'rig_arm' }]);+ close(worldPosition(sampledAt(recipe, 0), 'lid')[0], 0.5);+});++test('distance and rotation difference variables read world poses', () => {+ const recipe = rigRecipe({+ duration: 2,+ tracks: [+ { kind: 'node', id: 'a', property: 'translation', keyframes: [{ time: 0, value: [0, 0, 0] }, { time: 2, value: [3, 0, 0] }] },+ { kind: 'node', id: 'b', property: 'rotation', keyframes: [{ time: 0, value: Array.from(quaternionFromEuler(new Float64Array(4), 0, 0, 0)) }, { time: 1, value: Array.from(quaternionFromEuler(new Float64Array(4), 0, 60 * DEGREES, 0)) }] },+ ],+ drivers: [+ { target: { kind: 'node', id: 'lid', property: 'translation', component: 'x' }, expression: { distance: { from: 'a', to: 'c' } } },+ { target: { kind: 'node', id: 'lid', property: 'translation', component: 'y' }, expression: { rotationDifference: { from: 'b', to: 'c' } } },+ ],+ }, [{ id: 'a' }, { id: 'b' }, { id: 'c', translation: [0, 4, 0] }]);+ const scene = sampledAt(recipe, 1);+ close(worldPosition(scene, 'lid')[0], Math.hypot(1.5, 4));+ close(worldPosition(scene, 'lid')[1], 60, 1e-2);+});++test('the same time gives the same pose, and a seek equals play', () => {+ const recipe = rigRecipe({+ duration: 2, tracks: [], properties: [OPEN],+ drivers: [{ target: { kind: 'node', id: 'lid', property: 'rotation', component: 'x' }, expression: { mul: [{ custom: { node: 'lid', name: 'open' } }, 90] } }],+ });+ const played = createAgentScene(recipe);+ [0, 1.5, 0.25, 2, 0.75].forEach(time => played.sampleMotion(time));+ const sought = sampledAt(recipe, 0.75);+ assert.deepEqual(Array.from(part(played, 'lid').matrix), Array.from(part(sought, 'lid').matrix));+});++test('property and driver errors name the field and what to pass', () => {+ const build = (properties, expression = 1) => createAgentScene(rigRecipe({+ duration: 1, tracks: [], properties,+ drivers: [{ target: { kind: 'node', id: 'lid', property: 'translation', component: 'x' }, expression }],+ }));+ assert.throws(() => build([{ node: 'missing', name: 'open' }]), /Motion property 0 node.*node or bone ID/);+ assert.throws(() => build([{ node: 'lid', name: '' }]), /Motion property 0 name/);+ assert.throws(() => build([{ node: 'lid', name: 'open', kind: 'matrix' }]), /Motion property 0 kind.*number, vector, color/);+ assert.throws(() => build([{ node: 'lid', name: 'open', min: 1, max: 0 }]), /below max/);+ assert.throws(() => build([{ node: 'lid', name: 'open', value: 0, keys: [{ time: 0, value: 0 }] }]), /left out when keys are given/);+ assert.throws(() => build([{ node: 'lid', name: 'open', keys: [{ time: 1, value: 0 }, { time: 1, value: 1 }] }]), /rising times/);+ assert.throws(() => build([{ node: 'lid', name: 'open' }, { node: 'lid', name: 'open' }]), /no other property/);+ assert.throws(() => build(Array.from({ length: 33 }, (_, index) => ({ node: 'lid', name: `p${index}` }))), /bounded dense array/);+ assert.throws(() => build([{ node: 'lid', name: 'open' }], { custom: { node: 'lid', name: 'shut' } }), /no custom property "shut".*open/);+ assert.throws(() => build([{ node: 'lid', name: 'open' }], { custom: { node: 'lid', name: 'open', component: 'x' } }), /takes no component/);+ assert.throws(() => build([{ node: 'lid', name: 'tint', kind: 'color' }], { custom: { node: 'lid', name: 'tint' } }), /component must be x, y or z/);+ assert.throws(() => build([], { distance: { from: 'lid', to: 'nowhere' } }), /not in this scene/);+ assert.throws(() => build([], { distance: [1, 2] }), /plain object/);+ assert.throws(() => build([{ node: 'lid', name: 'open', code: 'x' }]), /Unknown or unsupported/);+});++test('light and morph driver targets are checked against the scene', () => {+ const build = target => createAgentScene(rigRecipe({ duration: 1, tracks: [], drivers: [{ target, expression: 0.5 }] }));+ assert.throws(() => build({ kind: 'light', id: 'point:3', property: 'intensity' }), /light was not found.*point:0/);+ assert.throws(() => build({ kind: 'light', id: 'point:0', property: 'position' }), /property must be one of intensity, color/);+ assert.throws(() => build({ kind: 'light', id: 'point:0', property: 'color' }), /needs a component/);+ assert.throws(() => build({ kind: 'morph', id: 'nope', property: 'weight' }), /morph was not found.*lift/);+ assert.throws(() => build({ kind: 'morph', id: 'lift', property: 'weight', component: 'x' }), /takes no component/);+ assert.throws(() => build({ kind: 'sun', id: 'x', property: 'intensity' }), /node, material, light or morph/);+ const clash = rigRecipe({+ duration: 1, tracks: [{ kind: 'light', id: 'point:0', property: 'intensity', keyframes: [{ time: 0, value: 1 }] }],+ drivers: [{ target: { kind: 'light', id: 'point:0', property: 'intensity' }, expression: 1 }],+ });+ assert.throws(() => createAgentScene(clash), /a track on light point:0 already moves/);+ const tooBright = rigRecipe({ duration: 1, tracks: [], drivers: [{ target: { kind: 'morph', id: 'lift', property: 'weight' }, expression: 2 }] });+ assert.throws(() => createAgentScene(tooBright).sampleMotion(0), /morph weight from 0 to 1/);+});++test('the capabilities list the property limits', () => {+ assert.equal(AGENT_SCENE_CAPABILITIES.motionDrivers.maxProperties, 32);+ assert.deepEqual(AGENT_SCENE_CAPABILITIES.motionDrivers.targets, ['node', 'material', 'light', 'morph']);+ assert.deepEqual(AGENT_SCENE_CAPABILITIES.motionDrivers.variables, ['time', 'node', 'custom', 'distance', 'rotationDifference']);+});