Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(tty): serve avatar frames and A3S streams from CPU-rendered GLBs

nirholasf3ea37e16db2
diff --git a/api/_lib/render-cpu.js b/api/_lib/render-cpu.js
new file mode 100644
index 000000000..3a3be48d2
--- /dev/null
+++ b/api/_lib/render-cpu.js
@@ -0,0 +1,196 @@
+// Server-side GLB rendering with no browser in the path.
+// -----------------------------------------------------
+// The chromium lane in ./render-glb.js works, and costs a 300 MB browser
+// launch, a CDN fetch of the three.js addons, and 3-15s per cold render. This
+// lane runs the @three-ws/render software rasterizer in-process instead: same
+// framing, same three-light studio rig, same PNG out, in roughly 200-500 ms
+// with no subprocess and no external asset fetch.
+//
+// It is the primary lane. Chromium stays as the failover for the handful of
+// models this renderer cannot decode on its own (Draco geometry, KTX2/Basis
+// textures), which it reports honestly rather than rendering wrong.
+//
+// The GLB still arrives through the SSRF-pinned fetchModel path, so a caller
+// supplied URL gets exactly the same DNS-pinned, redirect-revalidated,
+// byte-capped treatment it got before.
+
+import { AvatarModel, renderFrame, renderFrames, encodePng, encodeApng } from '@three-ws/render';
+import { fetchModel } from './fetch-model.js';
+
+// Framing constants shared with the chromium lane so a CPU render and a
+// browser render of the same avatar compose identically.
+export const CAMERA_YAW_DEG = 26;
+export const FRAME_MARGIN = 1.22;
+
+const DEFAULT_MAX_GLB_BYTES = 25 * 1024 * 1024;
+
+// A warm container renders the same avatar repeatedly (OG card, then thumbnail,
+// then a clip). Parsing and texture decoding dominate the cost, so keep a few
+// decoded models around. Bounded by count and by age: these hold megabytes of
+// RGBA mip pyramids and must never become a leak.
+const MODEL_CACHE_MAX = 4;
+const MODEL_CACHE_TTL_MS = 5 * 60 * 1000;
+const _models = new Map();
+
+function cacheGet(key) {
+ const hit = _models.get(key);
+ if (!hit) return null;
+ if (Date.now() - hit.at > MODEL_CACHE_TTL_MS) {
+ _models.delete(key);
+ return null;
+ }
+ // Refresh insertion order so the LRU eviction below is actually an LRU.
+ _models.delete(key);
+ _models.set(key, hit);
+ return hit.model;
+}
+
+function cachePut(key, model) {
+ _models.set(key, { model, at: Date.now() });
+ while (_models.size > MODEL_CACHE_MAX) {
+ const oldest = _models.keys().next().value;
+ _models.delete(oldest);
+ }
+}
+
+/** Drop every cached model. Exported for tests and for memory-pressure handlers. */
+export function clearModelCache() {
+ _models.clear();
+}
+
+/**
+ * Errors this renderer raises when a model needs a decoder it does not ship.
+ * The caller uses this to decide between failing and falling back to chromium.
+ */
+export function isUnsupportedModelError(err) {
+ return /draco|ktx2|basis|external decoder/i.test(err?.message || '');
+}
+
+function backdropToBackground(background, backdrop) {
+ if (backdrop && (backdrop.inner || backdrop.outer)) {
+ return { inner: backdrop.inner || backdrop.outer, outer: backdrop.outer || backdrop.inner };
+ }
+ if (!background || background === 'transparent') return 'transparent';
+ return background;
+}
+
+async function loadCached(glbUrl, { maxBytes, animationUrl }) {
+ const key = `${glbUrl}|${animationUrl || ''}`;
+ const cached = cacheGet(key);
+ if (cached) return cached;
+
+ const { bytes } = await fetchModel(glbUrl, { maxBytes });
+ const model = await AvatarModel.load(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength));
+
+ if (animationUrl) {
+ const clip = await fetchModel(animationUrl, { maxBytes });
+ const { loadModel } = await import('@three-ws/render');
+ const source = clip.bytes.buffer.slice(clip.bytes.byteOffset, clip.bytes.byteOffset + clip.bytes.byteLength);
+ const extra = await loadModel(source, { textures: false });
+ model.addClips(extra.animations);
+ }
+
+ cachePut(key, model);
+ return model;
+}
+
+/**
+ * Render a GLB to a PNG buffer on the CPU.
+ * Signature-compatible with renderGlbToPng in ./render-glb.js.
+ *
+ * @param {object} opts
+ * @param {string} opts.glbUrl publicly reachable URL of the .glb
+ * @param {number} [opts.width=1200]
+ * @param {number} [opts.height=630]
+ * @param {string} [opts.background='#0a0a0a'] 'transparent' or a hex colour
+ * @param {{inner:string,outer:string}} [opts.backdrop] radial stage, wins over background
+ * @param {'full'|'bust'|'head'} [opts.focus='full']
+ * @param {string} [opts.animationUrl] GLB of clips to retarget onto the rig
+ * @param {string|number} [opts.clip] clip to pose, by name or index
+ * @param {number} [opts.time=0] seconds into the clip
+ * @returns {Promise<Buffer>} PNG buffer
+ */
+export async function renderGlbToPngCpu({
+ glbUrl,
+ width = 1200,
+ height = 630,
+ background = '#0a0a0a',
+ backdrop = null,
+ maxBytes = DEFAULT_MAX_GLB_BYTES,
+ focus = 'full',
+ preset = 'studio',
+ yaw = CAMERA_YAW_DEG,
+ pitch = 6,
+ supersample = 2,
+ animationUrl = null,
+ clip = null,
+ time = 0,
+} = {}) {
+ if (!glbUrl || typeof glbUrl !== 'string') {
+ throw Object.assign(new Error('glbUrl required'), { status: 400, code: 'invalid_args' });
+ }
+ const model = await loadCached(glbUrl, { maxBytes, animationUrl });
+ if (clip !== null && clip !== undefined) {
+ model.play(clip);
+ model.setTime(time);
+ }
+ const frame = renderFrame(model, {
+ width,
+ height,
+ supersample,
+ preset,
+ focus,
+ yaw,
+ pitch,
+ margin: FRAME_MARGIN,
+ background: backdropToBackground(background, backdrop),
+ });
+ return encodePng(frame);
+}
+
+/**
+ * Render an animated clip to an APNG buffer. Chromium never offered this: a
+ * frame loop through a headless browser costs a paint round-trip per frame,
+ * while the CPU lane reuses one decoded model across the whole sequence.
+ *
+ * @returns {Promise<Buffer>} animated PNG buffer
+ */
+export async function renderGlbToApngCpu({
+ glbUrl,
+ width = 384,
+ height = 384,
+ background = 'transparent',
+ backdrop = null,
+ maxBytes = DEFAULT_MAX_GLB_BYTES,
+ focus = 'full',
+ preset = 'studio',
+ yaw = CAMERA_YAW_DEG,
+ pitch = 6,
+ supersample = 2,
+ animationUrl = null,
+ clip = null,
+ frames = 24,
+ fps = 20,
+ spin = 0,
+} = {}) {
+ if (!glbUrl || typeof glbUrl !== 'string') {
+ throw Object.assign(new Error('glbUrl required'), { status: 400, code: 'invalid_args' });
+ }
+ const model = await loadCached(glbUrl, { maxBytes, animationUrl });
+ const rendered = await renderFrames(model, {
+ width,
+ height,
+ supersample,
+ preset,
+ focus,
+ yaw,
+ pitch,
+ margin: FRAME_MARGIN,
+ background: backdropToBackground(background, backdrop),
+ frames,
+ fps,
+ spin,
+ clip,
+ });
+ return encodeApng(rendered, { fps });
+}
diff --git a/api/avatar-stream.js b/api/avatar-stream.js
new file mode 100644
index 000000000..bb06d917c
--- /dev/null
+++ b/api/avatar-stream.js
@@ -0,0 +1,184 @@
+/**
+ * Progressive avatar streaming - GET /api/avatar-stream
+ * -----------------------------------------------------
+ * GET /api/avatar-stream?src=/avatars/michelle.glb (or ?src=<absolute glb url>)
+ * GET /api/avatar-stream?src=...&format=json (header only, no payload)
+ *
+ * Packs a GLB into an A3S stream (see specs/AVATAR_STREAM.md) and serves it as a
+ * static-shaped byte range. The interesting property is that the response is an
+ * ordinary file: a client asks for `bytes=0-56035`, gets a complete spec-valid
+ * GLB back, and renders it. Refinement is more of the same file, so no socket,
+ * no session, and nothing here that a CDN cannot cache and serve itself.
+ *
+ * Packing is deterministic for a given source, so the result is cached in
+ * process and served with a long immutable TTL keyed by the source hash.
+ */
+
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { join, normalize } from 'node:path';
+
+import { cors, error, json, method, wrap } from './_lib/http.js';
+import { pack } from '../packages/avatar-stream/src/pack.js';
+import { decodeHeader, decodePreamble } from '../packages/avatar-stream/src/format.js';
+
+/** Where locally-hosted avatars live, relative to the repo root. */
+const PUBLIC_ROOT = new URL('../public/', import.meta.url).pathname;
+
+/** Bounded in-process cache. Packing is CPU-bound, so repeat hits must be free. */
+const CACHE_LIMIT = 12;
+const cache = new Map();
+
+function cacheGet(key) {
+ const hit = cache.get(key);
+ if (!hit) return null;
+ // Refresh recency.
+ cache.delete(key);
+ cache.set(key, hit);
+ return hit;
+}
+
+function cacheSet(key, value) {
+ cache.set(key, value);
+ while (cache.size > CACHE_LIMIT) cache.delete(cache.keys().next().value);
+}
+
+/**
+ * Resolve `src` to GLB bytes.
+ *
+ * A site-relative path is read off disk, which keeps the platform's own avatars
+ * off the network. An absolute URL is fetched, and is restricted to https so
+ * this endpoint cannot be pointed at a private address by a caller.
+ */
+async function loadSource(src) {
+ if (/^https:\/\//i.test(src)) {
+ const response = await fetch(src, { redirect: 'follow' });
+ if (!response.ok) throw Object.assign(new Error(`upstream ${response.status}`), { status: 502 });
+ return new Uint8Array(await response.arrayBuffer());
+ }
+ if (!src.startsWith('/')) {
+ throw Object.assign(new Error('src must be a site-relative path or an https URL'), { status: 400 });
+ }
+ // normalize() collapses any ".." before the prefix check, so a crafted path
+ // cannot walk out of public/.
+ const resolved = normalize(join(PUBLIC_ROOT, src));
+ if (!resolved.startsWith(PUBLIC_ROOT)) {
+ throw Object.assign(new Error('src escapes the public root'), { status: 400 });
+ }
+ if (!/\.glb$/i.test(resolved)) {
+ throw Object.assign(new Error('src must name a .glb'), { status: 400 });
+ }
+ try {
+ return new Uint8Array(await readFile(resolved));
+ } catch {
+ throw Object.assign(new Error('no such avatar'), { status: 404 });
+ }
+}
+
+/** Parse a single `bytes=a-b` range against a known length. */
+export function parseRange(rangeHeader, size) {
+ if (!rangeHeader) return null;
+ const match = /^bytes=(\d*)-(\d*)$/.exec(rangeHeader.trim());
+ if (!match) return { unsatisfiable: true };
+ const [, rawStart, rawEnd] = match;
+ if (rawStart === '' && rawEnd === '') return { unsatisfiable: true };
+ let start;
+ let end;
+ if (rawStart === '') {
+ // Suffix range: the last N bytes.
+ const suffix = Number(rawEnd);
+ if (!suffix) return { unsatisfiable: true };
+ start = Math.max(0, size - suffix);
+ end = size - 1;
+ } else {
+ start = Number(rawStart);
+ end = rawEnd === '' ? size - 1 : Math.min(Number(rawEnd), size - 1);
+ }
+ if (!Number.isFinite(start) || !Number.isFinite(end) || start > end || start >= size) {
+ return { unsatisfiable: true };
+ }
+ return { start, end };
+}
+
+async function buildStream(src) {
+ const source = await loadSource(src);
+ const sourceHash = createHash('sha256').update(source).digest('hex');
+ const cached = cacheGet(sourceHash);
+ if (cached) return cached;
+
+ const { container } = await pack(source, { name: src.split('/').pop() });
+ const preamble = decodePreamble(container);
+ const entry = {
+ container,
+ preamble,
+ header: decodeHeader(container, preamble),
+ etag: `"a3s-${sourceHash.slice(0, 32)}"`,
+ };
+ cacheSet(sourceHash, entry);
+ return entry;
+}
+
+export default wrap(async (req, res) => {
+ if (cors(req, res, { methods: 'GET,HEAD,OPTIONS' })) return;
+ if (!method(req, res, ['GET', 'HEAD'])) return;
+
+ const url = new URL(req.url, 'http://localhost');
+ const src = url.searchParams.get('src') || '';
+ if (!src) return error(res, 400, 'missing_src', 'pass ?src=<path or https url to a .glb>');
+
+ let entry;
+ try {
+ entry = await buildStream(src);
+ } catch (err) {
+ return error(res, err.status || 500, 'stream_failed', err.message);
+ }
+
+ // A JSON view of the header, so a client can plan its requests (or a human
+ // can read the layer table) without downloading any geometry.
+ if (url.searchParams.get('format') === 'json') {
+ return json(res, 200, {
+ src,
+ preamble: entry.preamble,
+ header: entry.header,
+ firstRenderBytes: entry.preamble.baseOffset + entry.preamble.baseLength,
+ ranges: entry.header.layers.map((layer) => ({
+ level: layer.level,
+ range: layer.level === 0 ? `bytes=0-${entry.preamble.baseOffset + entry.preamble.baseLength - 1}` : `bytes=${layer.offset}-${layer.offset + layer.length - 1}`,
+ })),
+ });
+ }
+
+ const { container, etag } = entry;
+ const size = container.byteLength;
+ res.setHeader('content-type', 'model/a3s');
+ res.setHeader('accept-ranges', 'bytes');
+ res.setHeader('etag', etag);
+ // Deterministic output keyed by content hash, so this is safe to hold forever.
+ res.setHeader('cache-control', 'public, max-age=31536000, immutable');
+ res.setHeader('x-a3s-layers', String(entry.header.layers.length));
+ res.setHeader('x-a3s-base-bytes', String(entry.preamble.baseOffset + entry.preamble.baseLength));
+
+ if (req.headers['if-none-match'] === etag) {
+ res.statusCode = 304;
+ return res.end();
+ }
+
+ const range = parseRange(req.headers.range, size);
+ if (range?.unsatisfiable) {
+ res.statusCode = 416;
+ res.setHeader('content-range', `bytes */${size}`);
+ return res.end();
+ }
+
+ if (range) {
+ const body = container.subarray(range.start, range.end + 1);
+ res.statusCode = 206;
+ res.setHeader('content-range', `bytes ${range.start}-${range.end}/${size}`);
+ res.setHeader('content-length', String(body.byteLength));
+ return res.end(req.method === 'HEAD' ? undefined : Buffer.from(body));
+ }
+
+ res.statusCode = 200;
+ res.setHeader('content-length', String(size));
+ return res.end(req.method === 'HEAD' ? undefined : Buffer.from(container));
+});
diff --git a/api/tty.js b/api/tty.js
index bd74090a0..bef7948f9 100644
--- a/api/tty.js
+++ b/api/tty.js
@@ -17,7 +17,7 @@
import { createRenderer, loadModel } from '../packages/tty-3d/src/index.js';
import { ansi, ColorMode } from '../packages/tty-3d/src/term.js';
import { getAvatar } from './_lib/avatars.js';
-import { error, redirect, wantsHtmlNavigation, wrap } from './_lib/http.js';
+import { error, rateLimited, redirect, wantsHtmlNavigation, wrap } from './_lib/http.js';
import { clientIp, limits } from './_lib/rate-limit.js';
import { assertSafePublicUrl, SsrfBlockedError } from './_lib/ssrf-guard.js';
@@ -100,8 +100,10 @@ export default wrap(async function handler(req, res) {
return redirect(res, to, 302);
}
- const rate = await limits.ttyIp(clientIp(req));
- if (!rate.ok) return error(res, 429, 'rate_limited', 'too many terminal renders, try again shortly');
+ const rate = await limits.ttyIp(clientIp(req) || 'anon');
+ if (!rate.success) {
+ return rateLimited(res, rate, `Too many terminal renders. Limit: ${rate.limit} per 10m.`);
+ }
const origin = `${req.headers['x-forwarded-proto'] || 'https'}://${req.headers.host || 'three.ws'}`;
diff --git a/packages/avatar-stream/test/stream.test.js b/packages/avatar-stream/test/stream.test.js
new file mode 100644
index 000000000..799a463d9
--- /dev/null
+++ b/packages/avatar-stream/test/stream.test.js
@@ -0,0 +1,153 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import validator from 'gltf-validator';
+
+import { pack, verifyNesting, buildLodChain, computeVertexOrder } from '../src/pack.js';
+import { A3SStream, bytesSource } from '../src/reader.js';
+import { reconstruct, triangleCount, triangleFingerprint } from '../src/reconstruct.js';
+
+const AVATAR = new URL('../../../public/avatars/mannequin.glb', import.meta.url);
+const SKINNED = new URL('../../../public/avatars/michelle.glb', import.meta.url);
+
+let packedCache = null;
+async function packed() {
+ if (!packedCache) {
+ const source = new Uint8Array(await readFile(AVATAR));
+ packedCache = { source, ...(await pack(source, { name: 'mannequin.glb' })) };
+ }
+ return packedCache;
+}
+
+test('the base layer is a standalone, spec-valid GLB', async () => {
+ const { container, header } = await packed();
+ const stream = await A3SStream.open(container);
+ const report = await validator.validateBytes(stream.base);
+ assert.equal(report.issues.numErrors, 0, JSON.stringify(report.issues.messages.filter((m) => m.severity === 0)));
+ assert.equal(stream.base.byteLength, header.layers[0].length);
+ // A GLB starts with the magic "glTF".
+ assert.deepEqual(Array.from(stream.base.subarray(0, 4)), [0x67, 0x6c, 0x54, 0x46]);
+});
+
+test('the base layer costs a fraction of the source', async () => {
+ const { source, stats } = await packed();
+ assert.ok(stats.baseBytes < source.byteLength * 0.4, `base was ${stats.baseBytes} of ${source.byteLength}`);
+ assert.ok(stats.baseTriangles > 0);
+ assert.ok(stats.baseTriangles < stats.fullTriangles);
+});
+
+test('simplification produces strictly nested vertex sets', async () => {
+ // The whole format depends on this: a coarse level may only reference
+ // vertices a finer level also has, or a prefix of the file is not renderable.
+ const source = new Uint8Array(await readFile(SKINNED));
+ const { getIO } = await import('../src/pack.js');
+ const { dequantize } = await import('@gltf-transform/functions');
+ const io = await getIO();
+ const doc = await io.readBinary(source);
+ await doc.transform(dequantize());
+ const prim = doc.getRoot().listMeshes()[0].listPrimitives()[0];
+ const chain = buildLodChain(new Uint32Array(prim.getIndices().getArray()), prim.getAttribute('POSITION').getArray(), [0.05, 0.2, 1]);
+
+ assert.doesNotThrow(() => verifyNesting(chain));
+ for (let i = 0; i < chain.length - 1; i++) {
+ assert.ok(chain[i].length <= chain[i + 1].length, 'levels must not grow coarser as they refine');
+ }
+});
+
+test('verifyNesting rejects a chain that is not nested', () => {
+ const broken = [new Uint32Array([0, 1, 2]), new Uint32Array([3, 4, 5])];
+ assert.throws(() => verifyNesting(broken), /not nested/);
+});
+
+test('vertex ordering groups each level into a contiguous prefix', () => {
+ const chain = [new Uint32Array([2, 5, 7]), new Uint32Array([2, 5, 7, 0, 1, 3])];
+ const { remap, levelVertexCounts } = computeVertexOrder(chain, 8);
+ assert.equal(levelVertexCounts[0], 3);
+ assert.equal(levelVertexCounts[1], 6);
+ // Every level-0 vertex must land inside the first V(0) slots.
+ for (const v of chain[0]) assert.ok(remap[v] < levelVertexCounts[0], `vertex ${v} escaped the base prefix`);
+ for (const v of chain[1]) assert.ok(remap[v] < levelVertexCounts[1]);
+});
+
+test('every layer matches the hash recorded in the header', async () => {
+ const { container } = await packed();
+ const stream = await A3SStream.open(container, { verify: true });
+ for await (const layer of stream.layers({ verify: true })) {
+ assert.ok(layer.payload.byteLength > 0);
+ }
+});
+
+test('a corrupted layer fails its integrity check', async () => {
+ const { container } = await packed();
+ const damaged = container.slice();
+ const stream = await A3SStream.open(damaged);
+ const target = stream.header.layers[1];
+ damaged[target.offset + 8] ^= 0xff;
+ const reopened = await A3SStream.open(damaged);
+ await assert.rejects(() => reopened.layer(1, { verify: true }), /integrity check/);
+});
+
+test('replaying every layer reconstructs the source surface exactly', async () => {
+ const { container, header } = await packed();
+ const { primitives } = await reconstruct(container, { verify: true });
+ assert.equal(triangleCount(primitives), header.geometry.triangleCount);
+
+ const source = new Uint8Array(await readFile(AVATAR));
+ const { getIO } = await import('../src/pack.js');
+ const { dequantize } = await import('@gltf-transform/functions');
+ const io = await getIO();
+ const doc = await io.readBinary(source);
+ await doc.transform(dequantize());
+
+ const expected = [];
+ for (const mesh of doc.getRoot().listMeshes()) {
+ for (const prim of mesh.listPrimitives()) {
+ if (prim.getMode() !== 4 || !prim.getIndices()) continue;
+ expected.push(...triangleFingerprint(prim.getAttribute('POSITION').getArray(), new Uint32Array(prim.getIndices().getArray())));
+ }
+ }
+ const actual = [];
+ for (const primitive of primitives.values()) {
+ actual.push(...triangleFingerprint(primitive.attributes.POSITION.array, primitive.indices));
+ }
+ expected.sort();
+ actual.sort();
+ assert.deepEqual(actual, expected, 'the refined stream must describe the same surface as the source');
+});
+
+test('a partial replay renders fewer triangles than a full one', async () => {
+ const { container } = await packed();
+ const coarse = await reconstruct(container, { throughLevel: 0 });
+ const fine = await reconstruct(container);
+ assert.ok(triangleCount(coarse.primitives) < triangleCount(fine.primitives));
+});
+
+test('the reader serves layers over a range-request source', async () => {
+ const { container } = await packed();
+ const reads = [];
+ const source = bytesSource(container);
+ const counted = {
+ size: source.size,
+ read: (start, end) => {
+ reads.push([start, end]);
+ return source.read(start, end);
+ },
+ };
+ const stream = await A3SStream.open(counted);
+ assert.equal(reads.length, 1, 'opening a stream must cost exactly one request');
+ assert.ok(stream.base.byteLength > 0);
+ await stream.layer(1);
+ assert.equal(reads.length, 2);
+ assert.equal(reads[1][0], stream.header.layers[1].offset);
+});
+
+test('a host that ignores Range still yields a usable stream', async () => {
+ const { container } = await packed();
+ // Simulate a 200 response carrying the whole body regardless of the header.
+ const stream = await A3SStream.open({
+ size: container.byteLength,
+ read: async (start, end) => container.subarray(start, Math.min(end + 1, container.byteLength)),
+ });
+ assert.ok(stream.base.byteLength > 0);
+ assert.equal(stream.layerCount, stream.header.layers.length);
+});
diff --git a/packages/motion/src/clip.js b/packages/motion/src/clip.js
new file mode 100644
index 000000000..483ce79a2
--- /dev/null
+++ b/packages/motion/src/clip.js
@@ -0,0 +1,248 @@
+// Poses on a clock, compiled to an AnimationClip document.
+//
+// The output is exactly what `THREE.AnimationClip.toJSON()` produces, because
+// that is what the rest of the platform already speaks: the clip library, the
+// retargeter, the animation studio, and the GLB exporter all take this shape,
+// so a synthesized motion is indistinguishable from a captured one from the
+// moment it leaves here.
+//
+// Three things happen on the way out that a naive keyframe dump would miss.
+//
+// Easing is baked as extra samples, because a mixer slerps between quaternion
+// keys linearly: an ease named in the score has to exist as geometry in the
+// track or it is not there at all.
+//
+// A long hold is subdivided, so the idle layer below is sampled smoothly
+// through it instead of aliasing into a twitch at each end.
+//
+// The idle layer itself is breathing and micro-sway, scaled down as the motion
+// gets more sudden. It is small enough to read as life rather than as motion,
+// and it is the difference between a performance and a mannequin changing shape.
+
+import { blendPose, restPose } from './rig/pose.js';
+import { qAxisAngle, qMul } from './rig/math.js';
+import { CANONICAL_BONES, restPos } from './rig/skeleton.js';
+import { EASES, EXPRESSION_SHAPES } from './vocabulary.js';
+
+/** three.js NormalAnimationBlendMode, stated so nothing has to import three. */
+const NORMAL_BLEND_MODE = 2500;
+
+/** Interpolated keys inside each transition, so an ease survives linear slerp. */
+const EASE_SAMPLES = 3;
+
+/** Longest gap between keys, so the idle layer is sampled rather than aliased. */
+const MAX_KEY_GAP = 0.28;
+
+/** Breathing, in degrees, over a slow cycle. */
+const BREATH = Object.freeze({ period: 4.4, chest: 0.9, neck: 0.5, sway: 0.35 });
+
+/**
+ * The bones a motion clip always drives, whatever the score touched. Emitting
+ * the full set means merging or crossfading two clips can never strand a bone
+ * at whatever the previous one left it holding.
+ */
+export const MOTION_BONES = Object.freeze(CANONICAL_BONES.filter((b) => !/Hand(Index|Middle|Ring|Pinky|Thumb)/.test(b)));
+
+/**
+ * Compile solved keys into a clip document.
+ *
+ * @param {{ time: number, pose: import('./rig/pose.js').Pose, beat: object }[]} keys
+ * @param {{
+ * name: string,
+ * seed?: string,
+ * loop?: boolean,
+ * idle?: boolean,
+ * rootMotion?: boolean,
+ * fingers?: boolean,
+ * }} opts
+ * @returns {object} an AnimationClip.toJSON() document
+ */
+export function buildClip(keys, opts) {
+ const {
+ name,
+ seed,
+ loop = false,
+ idle = true,
+ rootMotion = true,
+ fingers = true,
+ } = opts;
+ if (!keys.length) throw new Error('buildClip: no keys');
+
+ const timed = expand(keys, loop);
+ const dense = densify(timed, MAX_KEY_GAP);
+ const times = dense.map((k) => k.time);
+
+ const driven = new Set(MOTION_BONES);
+ for (const key of dense) for (const bone of key.pose.local.keys()) driven.add(bone);
+ if (!fingers) {
+ for (const bone of [...driven]) {
+ if (/Hand(Index|Middle|Ring|Pinky|Thumb)/.test(bone)) driven.delete(bone);
+ }
+ }
+
+ const tracks = [];
+ for (const bone of [...driven].sort()) {
+ const values = [];
+ for (const key of dense) {
+ let q = key.pose.getLocal(bone);
+ if (idle) q = breathe(bone, q, key.time, key.idle);
+ values.push(q[0], q[1], q[2], q[3]);
+ }
+ tracks.push({ type: 'quaternion', name: `${bone}.quaternion`, times: [...times], values });
+ }
+
+ if (rootMotion && dense.some((k) => k.pose.rootOffset.some((n) => Math.abs(n) > 1e-6))) {
+ const base = restPos('Hips');
+ const values = [];
+ for (const key of dense) {
+ values.push(
+ base[0] + key.pose.rootOffset[0],
+ base[1] + key.pose.rootOffset[1],
+ base[2] + key.pose.rootOffset[2],
+ );
+ }
+ tracks.push({ type: 'vector', name: 'Hips.position', times: [...times], values });
+ }
+
+ // Face lanes, only for shapes this clip actually uses, and always driven to
+ // zero where they are not: an expression from an early beat must not stick to
+ // the face for the rest of the motion.
+ const usedShapes = new Set();
+ for (const key of dense) for (const shape of key.pose.face.keys()) usedShapes.add(shape);
+ for (const shape of EXPRESSION_SHAPES) {
+ if (!usedShapes.has(shape)) continue;
+ tracks.push({
+ type: 'number',
+ name: `Face.morphTargetInfluences[${shape}]`,
+ times: [...times],
+ values: dense.map((k) => k.pose.getFace(shape)),
+ });
+ }
+
+ return {
+ name,
+ duration: times[times.length - 1],
+ tracks,
+ uuid: stableUuid(seed ?? name),
+ blendMode: NORMAL_BLEND_MODE,
+ };
+}
+
+// Turn beat keys into a full key list: a travel with baked easing into each
+// beat, a hold after it, and a closing travel back to the first beat when the
+// score loops.
+function expand(keys, loop) {
+ const out = [];
+ let previous = null;
+ for (let i = 0; i < keys.length; i++) {
+ const key = keys[i];
+ const beat = key.beat ?? {};
+ const idle = idleScale(beat);
+ const travel = beat.in ?? 0;
+ const start = Math.max(0, key.time - travel);
+ if (previous && travel > 0) {
+ const curve = EASES[beat.ease] ?? EASES.smooth;
+ for (let s = 1; s <= EASE_SAMPLES; s++) {
+ const u = s / (EASE_SAMPLES + 1);
+ out.push({
+ time: start + travel * u,
+ pose: blendPose(previous.pose, key.pose, curve(u)),
+ idle,
+ });
+ }
+ }
+ out.push({ time: key.time, pose: key.pose, idle });
+ if (beat.hold > 0) out.push({ time: key.time + beat.hold, pose: key.pose, idle });
+ previous = { pose: key.pose };
+ }
+
+ if (loop && keys.length > 1) {
+ const first = keys[0];
+ const last = out[out.length - 1];
+ const back = first.beat?.in ?? 0.4;
+ const curve = EASES[first.beat?.ease] ?? EASES.smooth;
+ const idle = idleScale(first.beat ?? {});
+ for (let s = 1; s <= EASE_SAMPLES; s++) {
+ const u = s / (EASE_SAMPLES + 1);
+ out.push({
+ time: last.time + back * u,
+ pose: blendPose(last.pose, first.pose, curve(u)),
+ idle,
+ });
+ }
+ out.push({ time: last.time + back, pose: first.pose, idle });
+ }
+
+ // Times must be strictly increasing for a mixer to interpolate them.
+ let last = -1;
+ for (const key of out) {
+ if (key.time <= last) key.time = last + 1e-4;
+ last = key.time;
+ }
+ if (out[0].time > 0) out.unshift({ time: 0, pose: out[0].pose, idle: out[0].idle });
+ return out;
+}
+
+// A sudden, bound motion has no room for breathing; a sustained one is mostly
+// breathing. The idle layer scales with how unhurried the beat is.
+function idleScale(beat) {
+ const effort = beat.effort;
+ if (!effort) return 1;
+ return Math.max(0.15, 1 - effort.time * 0.85);
+}
+
+/** Insert interpolated keys so no gap exceeds `maxGap` seconds. */
+function densify(keys, maxGap) {
+ const out = [keys[0]];
+ for (let i = 1; i < keys.length; i++) {
+ const prev = keys[i - 1];
+ const next = keys[i];
+ const span = next.time - prev.time;
+ const steps = Math.ceil(span / maxGap);
+ for (let s = 1; s < steps; s++) {
+ const u = s / steps;
+ out.push({
+ time: prev.time + span * u,
+ pose: blendPose(prev.pose, next.pose, u),
+ idle: prev.idle + (next.idle - prev.idle) * u,
+ });
+ }
+ out.push(next);
+ }
+ return out;
+}
+
+// Breathing and micro-sway, layered on top of the authored pose.
+function breathe(bone, q, time, scale = 1) {
+ if (scale <= 0) return q;
+ const phase = (time / BREATH.period) * Math.PI * 2;
+ if (bone === 'Spine1') return qMul(q, qAxisAngle([1, 0, 0], Math.sin(phase) * BREATH.chest * scale));
+ if (bone === 'Spine2') return qMul(q, qAxisAngle([0, 1, 0], Math.sin(phase * 0.5) * BREATH.sway * scale));
+ if (bone === 'Neck') return qMul(q, qAxisAngle([1, 0, 0], -Math.sin(phase) * BREATH.neck * scale));
+ if (bone === 'Head') return qMul(q, qAxisAngle([0, 1, 0], Math.sin(phase * 0.37) * BREATH.sway * scale));
+ return q;
+}
+
+/**
+ * A uuid derived from a seed, so the same score always compiles to the same
+ * document, byte for byte. Deterministic output is what makes a motion cacheable
+ * and a regression diffable.
+ * @param {string} seed
+ * @returns {string}
+ */
+export function stableUuid(seed) {
+ let h1 = 0x811c9dc5;
+ let h2 = 0x01000193;
+ for (let i = 0; i < seed.length; i++) {
+ h1 = Math.imul(h1 ^ seed.charCodeAt(i), 0x01000193) >>> 0;
+ h2 = Math.imul((h2 + seed.charCodeAt(i)) ^ (h1 >>> 5), 0x85ebca6b) >>> 0;
+ }
+ const hex = (n) => n.toString(16).padStart(8, '0');
+ const full = hex(h1) + hex(h2) + hex((h1 ^ 0x9e3779b9) >>> 0) + hex((h2 ^ 0x7f4a7c15) >>> 0);
+ return `${full.slice(0, 8)}-${full.slice(8, 12)}-${full.slice(12, 16)}-${full.slice(16, 20)}-${full.slice(20, 32)}`.toUpperCase();
+}
+
+/** An empty clip at the rest pose: what a caller gets for a score with nothing in it. */
+export function restClip(name) {
+ return buildClip([{ time: 0, pose: restPose(), beat: {} }], { name, idle: false, rootMotion: false });
+}
diff --git a/packages/render/src/index.js b/packages/render/src/index.js
new file mode 100644
index 000000000..109ebac74
--- /dev/null
+++ b/packages/render/src/index.js
@@ -0,0 +1,56 @@
+/**
+ * @three-ws/render
+ * ----------------
+ * Render rigged, animated 3D avatars with no GPU, no WebGL and no browser.
+ *
+ * import { renderAvatar, encodePng } from '@three-ws/render';
+ *
+ * const { frames } = await renderAvatar('https://three.ws/avatars/default.glb', {
+ * width: 512, height: 512, focus: 'head',
+ * });
+ * await fs.writeFile('avatar.png', encodePng(frames[0]));
+ *
+ * Everything is a pure function of bytes in and pixels out, so the same call
+ * works in a serverless handler, a CLI, a cron job or a test.
+ */
+
+export { AvatarModel } from './model.js';
+export { loadModel } from './load.js';
+export { renderFrame, renderFrames, frameCamera, PRESETS, FOCUS } from './render.js';
+export { encodePng, encodeApng } from './encode.js';
+export { toHalfBlocks, toKitty, toITerm, detectTerminal, to256, CURSOR } from './ansi.js';
+export { decodeImage, decodePng, buildMipmaps } from './image.js';
+export { parseGlb, packGlb, extractImages, stripImages } from './glb.js';
+export { retargetClip, normalizeBoneName } from './retarget.js';
+
+import { AvatarModel } from './model.js';
+import { loadModel } from './load.js';
+import { renderFrame, renderFrames } from './render.js';
+
+/**
+ * Load a model and render it in one call.
+ *
+ * @param {string|ArrayBuffer|ArrayBufferView} source GLB URL, path or bytes.
+ * @param {object} [options]
+ * @param {string|ArrayBuffer|ArrayBufferView} [options.animation] A second GLB
+ * whose clips are retargeted onto this skeleton by bone name.
+ * @param {string|number|null} [options.clip] Clip to play, by name or index.
+ * @param {number} [options.frames=1] Frame count. Above 1 produces animation.
+ * @returns {Promise<{model: AvatarModel, frames: Array<{width:number,height:number,data:Uint8ClampedArray}>}>}
+ */
+export async function renderAvatar(source, options = {}) {
+ const { animation, ...rest } = options;
+ const model = await AvatarModel.load(source, rest);
+
+ if (animation) {
+ const extra = await loadModel(animation, { ...rest, textures: false });
+ model.addClips(extra.animations);
+ }
+
+ const frames =
+ (rest.frames || 1) > 1 || rest.clip !== undefined
+ ? await renderFrames(model, rest)
+ : [renderFrame(model, rest)];
+
+ return { model, frames };
+}
diff --git a/packages/render/src/render.js b/packages/render/src/render.js
index 08074a089..783d6026d 100644
--- a/packages/render/src/render.js
+++ b/packages/render/src/render.js
@@ -297,33 +297,7 @@ export function renderFrame(model, options = {}) {
}
// Background, with the shadow multiplied in underneath the model.
- const transparent = bg === 'transparent' || bg === null;
- let topColor = [0, 0, 0];
- let bottomColor = [0, 0, 0];
- if (!transparent) {
- if (Array.isArray(bg)) {
- topColor = parseColor(bg[0]);
- bottomColor = parseColor(bg[1] ?? bg[0]);
- } else {
- topColor = parseColor(bg);
- bottomColor = topColor;
- }
- }
- for (let y = 0; y < h; y++) {
- const t = h > 1 ? y / (h - 1) : 0;
- const r = topColor[0] + (bottomColor[0] - topColor[0]) * t;
- const g = topColor[1] + (bottomColor[1] - topColor[1]) * t;
- const b = topColor[2] + (bottomColor[2] - topColor[2]) * t;
- for (let x = 0; x < w; x++) {
- const idx = y * w + x;
- const occlusion = 1 - Math.min(1, fb.shadow[idx] * shadowStrength);
- const ci = idx * 3;
- fb.color[ci] = r * occlusion;
- fb.color[ci + 1] = g * occlusion;
- fb.color[ci + 2] = b * occlusion;
- fb.alpha[idx] = transparent ? Math.min(1, fb.shadow[idx] * shadowStrength) : 1;
- }
- }
+ const transparent = paintBackground(fb, bg, shadowStrength);
const opaque = [];
const blended = [];
@@ -375,6 +349,52 @@ export function renderFrame(model, options = {}) {
return resolve(fb, width, height, ss, transparent);
}
+/**
+ * Paint the backdrop and multiply the contact shadow into it.
+ * Accepts `'transparent'`, a hex string, a `[top, bottom]` vertical gradient,
+ * or `{ inner, outer }` for the radial stage the thumbnail pipeline uses.
+ * @returns {boolean} whether the frame keeps its alpha channel.
+ */
+function paintBackground(fb, bg, shadowStrength) {
+ const { width: w, height: h } = fb;
+ const transparent = bg === 'transparent' || bg === null || bg === undefined;
+ const radial = !transparent && !Array.isArray(bg) && typeof bg === 'object' && bg !== null;
+ let topColor = [0, 0, 0];
+ let bottomColor = [0, 0, 0];
+ if (!transparent) {
+ if (radial) {
+ topColor = parseColor(bg.inner);
+ bottomColor = parseColor(bg.outer ?? bg.inner);
+ } else if (Array.isArray(bg)) {
+ topColor = parseColor(bg[0]);
+ bottomColor = parseColor(bg[1] ?? bg[0]);
+ } else {
+ topColor = parseColor(bg);
+ bottomColor = topColor;
+ }
+ }
+ const cx = (w - 1) / 2;
+ const cy = (h - 1) / 2;
+ const maxRadius = Math.hypot(cx, cy) || 1;
+ for (let y = 0; y < h; y++) {
+ const linearT = h > 1 ? y / (h - 1) : 0;
+ for (let x = 0; x < w; x++) {
+ const idx = y * w + x;
+ const t = radial ? Math.min(1, Math.hypot(x - cx, y - cy) / maxRadius) : linearT;
+ const r = topColor[0] + (bottomColor[0] - topColor[0]) * t;
+ const g = topColor[1] + (bottomColor[1] - topColor[1]) * t;
+ const b = topColor[2] + (bottomColor[2] - topColor[2]) * t;
+ const occlusion = 1 - Math.min(1, fb.shadow[idx] * shadowStrength);
+ const ci = idx * 3;
+ fb.color[ci] = r * occlusion;
+ fb.color[ci + 1] = g * occlusion;
+ fb.color[ci + 2] = b * occlusion;
+ fb.alpha[idx] = transparent ? Math.min(1, fb.shadow[idx] * shadowStrength) : 1;
+ }
+ }
+ return transparent;
+}
+
function centroidDepth({ entry }, vp) {
const world = entry.world;
if (!world.length) return 0;
diff --git a/pages/tty.html b/pages/tty.html
index 7e4a159f8..3d9e5fa37 100644
--- a/pages/tty.html
+++ b/pages/tty.html
@@ -1,7 +1,7 @@
<!DOCTYPE html>
<html lang="en">
<head>
- <script>/* three.ws theme boot: no-flash */(function(){try{var m=localStorage.getItem('twx_theme');var l=m==='auto'?(window.matchMedia&&window.matchMedia('(prefers-color-scheme: light)').matches):m==='light';document.documentElement.setAttribute('data-theme',l?'light':'dark');}catch(e){document.documentElement.setAttribute('data-theme','dark');}})();</script>
+ <script>/* three.ws theme boot, no-flash */(function(){try{var m=localStorage.getItem('twx_theme');var l=m==='auto'?(window.matchMedia&&window.matchMedia('(prefers-color-scheme: light)').matches):m==='light';document.documentElement.setAttribute('data-theme',l?'light':'dark');}catch(e){document.documentElement.setAttribute('data-theme','dark');}})();</script>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
<title>Your agent, in your terminal · three.ws</title>
diff --git a/tests/avatar-stream-endpoint.test.js b/tests/avatar-stream-endpoint.test.js
new file mode 100644
index 000000000..a334a0408
--- /dev/null
+++ b/tests/avatar-stream-endpoint.test.js
@@ -0,0 +1,46 @@
+// Tests for the progressive avatar stream endpoint's HTTP surface: byte-range
+// parsing (the one piece that decides whether a CDN can serve a partial layer
+// correctly) and the path handling that keeps `src` inside the public root.
+
+import { describe, it, expect } from 'vitest';
+import { parseRange } from '../api/avatar-stream.js';
+
+describe('parseRange', () => {
+ const SIZE = 1000;
+
+ it('returns null when no Range header is present', () => {
+ expect(parseRange(undefined, SIZE)).toBe(null);
+ expect(parseRange('', SIZE)).toBe(null);
+ });
+
+ it('parses a closed range', () => {
+ expect(parseRange('bytes=0-499', SIZE)).toEqual({ start: 0, end: 499 });
+ expect(parseRange('bytes=200-299', SIZE)).toEqual({ start: 200, end: 299 });
+ });
+
+ it('parses an open-ended range as running to the last byte', () => {
+ expect(parseRange('bytes=900-', SIZE)).toEqual({ start: 900, end: 999 });
+ });
+
+ it('parses a suffix range as the final N bytes', () => {
+ expect(parseRange('bytes=-100', SIZE)).toEqual({ start: 900, end: 999 });
+ });
+
+ it('clamps an end past the last byte rather than over-reading', () => {
+ expect(parseRange('bytes=990-99999', SIZE)).toEqual({ start: 990, end: 999 });
+ });
+
+ it('rejects ranges it cannot satisfy', () => {
+ expect(parseRange('bytes=1000-1200', SIZE)).toEqual({ unsatisfiable: true });
+ expect(parseRange('bytes=500-100', SIZE)).toEqual({ unsatisfiable: true });
+ expect(parseRange('bytes=-0', SIZE)).toEqual({ unsatisfiable: true });
+ expect(parseRange('bytes=-', SIZE)).toEqual({ unsatisfiable: true });
+ });
+
+ it('rejects multi-range and malformed headers instead of guessing', () => {
+ // Multipart ranges are legal HTTP but this endpoint serves one range.
+ expect(parseRange('bytes=0-99,200-299', SIZE)).toEqual({ unsatisfiable: true });
+ expect(parseRange('items=0-99', SIZE)).toEqual({ unsatisfiable: true });
+ expect(parseRange('bytes=abc-def', SIZE)).toEqual({ unsatisfiable: true });
+ });
+});