Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(splat): upgrade Spark to 2.3.1 with real LoD trees, and add camera hotspots with video export

nirholasbd61fbd99cae
diff --git a/package-lock.json b/package-lock.json
index 0dc2c7ef2..4650972b0 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -151,7 +151,7 @@
"@solana/spl-token": "^0.4.14",
"@solana/wallet-standard-features": "^1.5.0",
"@solana/web3.js": "^1.99.0",
- "@sparkjsdev/spark": "^2.2.0",
+ "@sparkjsdev/spark": "^2.3.1",
"@sparticuz/chromium-min": "^148.0.0",
"@tetsuo-ai/protocol": "^0.2.0",
"@tetsuo-ai/sdk": "^1.4.0",
@@ -27284,9 +27284,9 @@
}
},
"node_modules/@sparkjsdev/spark": {
- "version": "2.2.0",
- "resolved": "https://registry.npmjs.org/@sparkjsdev/spark/-/spark-2.2.0.tgz",
- "integrity": "sha512-wf5F2YAmPXnjm20/MgXYer9FQTXz2JOCo05HwxFkaD7Guk9rHVPIwdJ6fq0oF9ME0heMvYj+TxS6DTrucvy6KA==",
+ "version": "2.3.1",
+ "resolved": "https://registry.npmjs.org/@sparkjsdev/spark/-/spark-2.3.1.tgz",
+ "integrity": "sha512-K+SyIbkO/dx1TLLudK8Jp3/noS13z/kdNPo6OK2waxQaGy81HBmKGnNBT/t/K9Ew3dsIXmWqsJXxGfmAu1T5Jg==",
"license": "MIT",
"dependencies": {
"fflate": "^0.8.2"
diff --git a/package.json b/package.json
index e25161e37..ee12908d3 100644
--- a/package.json
+++ b/package.json
@@ -628,7 +628,7 @@
"@solana/spl-token": "^0.4.14",
"@solana/wallet-standard-features": "^1.5.0",
"@solana/web3.js": "^1.99.0",
- "@sparkjsdev/spark": "^2.2.0",
+ "@sparkjsdev/spark": "^2.3.1",
"@sparticuz/chromium-min": "^148.0.0",
"@tetsuo-ai/protocol": "^0.2.0",
"@tetsuo-ai/sdk": "^1.4.0",
diff --git a/pages/splat.html b/pages/splat.html
index 627350db4..e2bb9938e 100644
--- a/pages/splat.html
+++ b/pages/splat.html
@@ -170,6 +170,50 @@
@media (max-width: 860px) {
.sp-layout { grid-template-columns: 1fr; }
}
+
+ /* Annotation hotspot markers, projected onto the live canvas */
+ .sp-hotspot-marker {
+ position: absolute; transform: translate(-50%, -50%);
+ width: 22px; height: 22px; border-radius: 50%;
+ background: var(--accent-violet); color: #fff; font-size: 11px; font-weight: 700;
+ display: grid; place-items: center; cursor: pointer; padding: 0;
+ border: 2px solid rgba(255,255,255,0.85); box-shadow: 0 2px 8px rgba(0,0,0,0.45);
+ transition: transform var(--duration-fast); font-family: inherit; z-index: 2;
+ }
+ .sp-hotspot-marker:hover { transform: translate(-50%, -50%) scale(1.18); }
+ .sp-hotspot-marker:focus-visible { outline: 2px solid #fff; outline-offset: 2px; }
+
+ /* Hotspot list + camera timeline card */
+ .sp-timeline-card { margin-top: 18px; }
+ .sp-timeline-card[hidden] { display: none; }
+ .sp-timeline-head { display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; margin-bottom: 12px; }
+ .sp-timeline-head h2 { margin: 0; }
+ .sp-timeline-actions { display: flex; gap: 8px; }
+ .sp-timeline-actions .sp-btn { width: auto; }
+ .sp-hotspot-list { list-style: none; margin: 0 0 14px; padding: 0; display: flex; flex-direction: column; gap: 7px; }
+ .sp-hotspot-row {
+ display: grid; grid-template-columns: 22px 1fr 68px auto; align-items: center; gap: 8px;
+ padding: 7px 8px; border-radius: 9px; background: var(--surface-0, #0b0b0b);
+ border: 1px solid var(--hairline, #1c1c1c);
+ }
+ .sp-hotspot-badge {
+ width: 20px; height: 20px; border-radius: 50%; background: rgba(124,93,243,0.16);
+ color: var(--accent-violet); font-size: 11px; font-weight: 700; display: grid; place-items: center;
+ }
+ .sp-hotspot-name { font-size: 12.5px; padding: 6px 9px; }
+ .sp-hotspot-time { font-size: 12px; padding: 6px 7px; text-align: right; }
+ .sp-hotspot-actions { display: flex; gap: 4px; }
+ .sp-hotspot-actions .sp-icon-btn { width: 26px; height: 26px; font-size: 12px; }
+ .sp-hotspot-actions .sp-icon-btn:disabled { opacity: 0.3; cursor: default; }
+ .sp-hotspot-empty { font-size: 12.5px; color: var(--text-3, #8a8a8a); padding: 14px 4px; line-height: 1.55; }
+ .sp-timeline-track-wrap { display: flex; align-items: center; gap: 10px; }
+ .sp-timeline-track-wrap input[type="range"] { flex: 1; accent-color: var(--accent-violet); }
+ .sp-timeline-time { font-size: 11.5px; color: var(--text-3, #8a8a8a); min-width: 92px; text-align: right; font-variant-numeric: tabular-nums; }
+ #sp-timeline-play { flex: 0 0 auto; }
+ #sp-export-status { margin-top: 10px; }
+ .sp-note--error { color: #fca5a5; }
+ .sp-export-link { color: var(--accent-violet); text-decoration: underline; font-size: 12.5px; }
+ #sp-hotspot-add:disabled, #sp-export-video:disabled, #sp-timeline-play:disabled { opacity: 0.4; cursor: default; }
</style>
<meta name="robots" content="index, follow, max-image-preview:large, max-snippet:-1">
@@ -279,6 +323,24 @@
</div>
</aside>
</div>
+
+ <div class="sp-card sp-timeline-card" id="sp-timeline-card" hidden>
+ <div class="sp-timeline-head">
+ <h2 data-i18n="splat.hotspots_and_camera_timeline">Hotspots &amp; camera timeline</h2>
+ <div class="sp-timeline-actions">
+ <button class="sp-btn" id="sp-hotspot-add" type="button" data-i18n="splat.save_camera_view">+ Save camera view</button>
+ <button class="sp-btn sp-btn--primary" id="sp-export-video" type="button" data-i18n="splat.export_video">Export video</button>
+ </div>
+ </div>
+ <p class="sp-hotspot-empty" id="sp-hotspot-empty" data-i18n="splat.hotspot_empty_state">No camera views yet. Move the camera where you want, then click “Save camera view”. Save at least two to build a path.</p>
+ <ul class="sp-hotspot-list" id="sp-hotspot-list"></ul>
+ <div class="sp-timeline-track-wrap">
+ <button class="sp-icon-btn" id="sp-timeline-play" type="button" title="Play camera path" aria-label="Play camera path" disabled>&#9654;</button>
+ <input id="sp-timeline-scrub" type="range" min="0" max="1" step="0.01" value="0" aria-label="Camera path position" />
+ <span class="sp-timeline-time" id="sp-timeline-time">0.0s / 0.0s</span>
+ </div>
+ <div class="sp-note" id="sp-export-status" role="status" aria-live="polite" hidden></div>
+ </div>
</div>
<div id="footer-container"></div>
diff --git a/src/splat-hotspots.js b/src/splat-hotspots.js
new file mode 100644
index 000000000..a9f94545c
--- /dev/null
+++ b/src/splat-hotspots.js
@@ -0,0 +1,402 @@
+// splat-hotspots: annotation hotspots, a camera timeline built from them, and a
+// real in-browser video export of the resulting fly-through, for the Splat Viewer.
+//
+// A hotspot is a named camera view (position + orbit target) captured from
+// wherever the user is currently looking, plus a time in seconds. Ordering
+// the hotspots by time turns them into a camera path: scrub it, play it, or
+// record the canvas while it plays to export a video, all without leaving
+// the page or touching a server, since the splat itself never leaves the
+// browser either.
+//
+// This module owns no render loop of its own. It rides the single hook
+// `src/splat-stage.js` exposes for exactly this (`scene.setOnFrame`), so a
+// camera-path playback and the one render loop a mounted scene owns can never
+// drift out of sync with each other.
+
+export const MAX_HOTSPOTS = 25;
+const EXPORT_FPS = 30;
+const DEFAULT_SPACING_S = 2.5;
+const TWEEN_MS = 650;
+
+const $ = (s, r = document) => r.querySelector(s);
+
+function smoothstep(u) {
+ const c = Math.max(0, Math.min(1, u));
+ return c * c * (3 - 2 * c);
+}
+
+// Sample the camera path at time `t`: clamp to the ends, ease each segment.
+function sampleCamera(kfs, t) {
+ if (!kfs.length) return null;
+ if (t <= kfs[0].time) return { position: kfs[0].position.clone(), target: kfs[0].target.clone() };
+ const last = kfs[kfs.length - 1];
+ if (t >= last.time) return { position: last.position.clone(), target: last.target.clone() };
+ for (let i = 0; i < kfs.length - 1; i++) {
+ const a = kfs[i], b = kfs[i + 1];
+ if (t >= a.time && t <= b.time) {
+ const span = b.time - a.time;
+ const e = smoothstep(span > 0 ? (t - a.time) / span : 1);
+ return { position: a.position.clone().lerp(b.position, e), target: a.target.clone().lerp(b.target, e) };
+ }
+ }
+ return { position: last.position.clone(), target: last.target.clone() };
+}
+
+function pickRecordingMimeType() {
+ if (typeof MediaRecorder === 'undefined') return null;
+ for (const t of ['video/webm;codecs=vp9', 'video/webm;codecs=vp8', 'video/webm']) {
+ if (MediaRecorder.isTypeSupported(t)) return t;
+ }
+ return null;
+}
+
+function fmtTime(s) {
+ return `${Math.max(0, s).toFixed(1)}s`;
+}
+
+/**
+ * Wire the hotspot list, camera timeline, and video export controls. The
+ * caller owns scene lifecycle and must call `onSceneChanged(scene|null)`
+ * whenever the mounted splat scene changes (new load, or torn down).
+ *
+ * @returns {{onSceneChanged: Function, reset: Function}}
+ */
+export function initHotspotTimeline() {
+ const card = $('#sp-timeline-card');
+ const list = $('#sp-hotspot-list');
+ const empty = $('#sp-hotspot-empty');
+ const addBtn = $('#sp-hotspot-add');
+ const exportBtn = $('#sp-export-video');
+ const playBtn = $('#sp-timeline-play');
+ const scrub = $('#sp-timeline-scrub');
+ const timeLabel = $('#sp-timeline-time');
+ const status = $('#sp-export-status');
+ const markersHost = $('#sp-stage');
+ if (!card || !list || !markersHost) return { onSceneChanged() {}, reset() {} };
+
+ /** @type {{id:number, name:string, time:number, position:import('three').Vector3, target:import('three').Vector3}[]} */
+ let hotspots = [];
+ let seq = 0;
+ let currentScene = null;
+ const markers = new Map(); // id -> HTMLElement
+
+ let playback = null; // { startWallMs, startTime, kfs, onDone }
+ let tween = null; // { startWallMs, from:{position,target}, to:{position,target} }
+
+ function sorted() {
+ return [...hotspots].sort((a, b) => a.time - b.time || a.id - b.id);
+ }
+
+ function totalDuration() {
+ const kfs = sorted();
+ return kfs.length ? kfs[kfs.length - 1].time : 0;
+ }
+
+ function setStatus(msg, isError = false) {
+ if (!status) return;
+ if (!msg) { status.hidden = true; status.textContent = ''; return; }
+ status.hidden = false;
+ status.textContent = msg;
+ status.classList.toggle('sp-note--error', isError);
+ }
+
+ // -- Markers (annotation dots projected onto the live canvas) ---------------
+ function ensureMarker(h) {
+ let el = markers.get(h.id);
+ if (el) return el;
+ el = document.createElement('button');
+ el.type = 'button';
+ el.className = 'sp-hotspot-marker';
+ el.setAttribute('aria-label', `Jump to camera view: ${h.name}`);
+ el.addEventListener('click', (e) => { e.stopPropagation(); goTo(h.id); });
+ markersHost.appendChild(el);
+ markers.set(h.id, el);
+ return el;
+ }
+
+ function pruneMarkers() {
+ const live = new Set(hotspots.map((h) => h.id));
+ for (const [id, el] of markers) {
+ if (!live.has(id)) { el.remove(); markers.delete(id); }
+ }
+ }
+
+ function repositionMarkers(scene) {
+ const w = scene.canvas.clientWidth || scene.canvas.width || 1;
+ const h = scene.canvas.clientHeight || scene.canvas.height || 1;
+ for (const spot of hotspots) {
+ const el = ensureMarker(spot);
+ const v = spot.target.clone().project(scene.camera);
+ if (v.z > 1 || v.z < -1) { el.style.display = 'none'; continue; }
+ el.style.display = '';
+ el.style.left = `${((v.x * 0.5 + 0.5) * w).toFixed(1)}px`;
+ el.style.top = `${((1 - (v.y * 0.5 + 0.5)) * h).toFixed(1)}px`;
+ el.textContent = String(sorted().findIndex((s) => s.id === spot.id) + 1);
+ }
+ }
+
+ // -- Per-frame hook, installed on whatever scene is currently live ----------
+ function onFrame() {
+ const scene = currentScene;
+ if (!scene) return;
+ if (tween) {
+ const u = smoothstep((performance.now() - tween.startWallMs) / TWEEN_MS);
+ scene.camera.position.copy(tween.from.position.clone().lerp(tween.to.position, u));
+ scene.controls.target.copy(tween.from.target.clone().lerp(tween.to.target, u));
+ scene.controls.update();
+ if (u >= 1) tween = null;
+ } else if (playback) {
+ const t = playback.startTime + (performance.now() - playback.startWallMs) / 1000;
+ const sample = sampleCamera(playback.kfs, t);
+ if (sample) {
+ scene.camera.position.copy(sample.position);
+ scene.controls.target.copy(sample.target);
+ scene.controls.update();
+ }
+ updateScrubUI(Math.min(t, totalDuration()));
+ if (t >= totalDuration()) {
+ const done = playback.onDone;
+ playback = null;
+ setPlayUI(false);
+ done?.();
+ }
+ }
+ repositionMarkers(scene);
+ }
+
+ function updateScrubUI(t) {
+ const total = totalDuration();
+ if (scrub) {
+ scrub.max = String(Math.max(total, 0.01));
+ scrub.value = String(t);
+ }
+ if (timeLabel) timeLabel.textContent = `${fmtTime(t)} / ${fmtTime(total)}`;
+ }
+
+ function setPlayUI(isPlaying) {
+ if (!playBtn) return;
+ playBtn.textContent = isPlaying ? '⏸' : '▶';
+ playBtn.setAttribute('aria-label', isPlaying ? 'Pause camera path' : 'Play camera path');
+ playBtn.title = isPlaying ? 'Pause camera path' : 'Play camera path';
+ }
+
+ // -- Actions ------------------------------------------------------------------
+ function goTo(id) {
+ const scene = currentScene;
+ const spot = hotspots.find((h) => h.id === id);
+ if (!scene || !spot) return;
+ playback = null;
+ setPlayUI(false);
+ tween = {
+ startWallMs: performance.now(),
+ from: { position: scene.camera.position.clone(), target: scene.controls.target.clone() },
+ to: { position: spot.position.clone(), target: spot.target.clone() },
+ };
+ }
+
+ function addHotspot() {
+ const scene = currentScene;
+ if (!scene || hotspots.length >= MAX_HOTSPOTS) return;
+ seq += 1;
+ const time = hotspots.length ? Math.max(...hotspots.map((h) => h.time)) + DEFAULT_SPACING_S : 0;
+ hotspots.push({
+ id: seq,
+ name: `Camera view ${seq}`,
+ time,
+ position: scene.camera.position.clone(),
+ target: scene.controls.target.clone(),
+ });
+ render();
+ }
+
+ function removeHotspot(id) {
+ hotspots = hotspots.filter((h) => h.id !== id);
+ pruneMarkers();
+ render();
+ }
+
+ function renameHotspot(id, name) {
+ const spot = hotspots.find((h) => h.id === id);
+ if (spot) spot.name = name.trim() || spot.name;
+ }
+
+ function retimeHotspot(id, time) {
+ const spot = hotspots.find((h) => h.id === id);
+ if (!spot) return;
+ spot.time = Math.max(0, Number.isFinite(time) ? time : spot.time);
+ render();
+ }
+
+ function nudge(id, dir) {
+ const order = sorted();
+ const i = order.findIndex((h) => h.id === id);
+ const j = i + dir;
+ if (i < 0 || j < 0 || j >= order.length) return;
+ const a = order[i], b = order[j];
+ const t = a.time; a.time = b.time; b.time = t;
+ render();
+ }
+
+ function play() {
+ const kfs = sorted();
+ if (kfs.length < 2) return;
+ playback = { startWallMs: performance.now(), startTime: scrub ? Number(scrub.value) || 0 : 0, kfs, onDone: null };
+ setPlayUI(true);
+ }
+
+ function pause() {
+ playback = null;
+ setPlayUI(false);
+ }
+
+ async function exportVideo() {
+ const scene = currentScene;
+ const kfs = sorted();
+ if (!scene || kfs.length < 2) return;
+ const mimeType = pickRecordingMimeType();
+ if (!mimeType) {
+ setStatus('This browser cannot record video (MediaRecorder is unavailable). Try a recent Chrome, Edge, or Firefox.', true);
+ return;
+ }
+ exportBtn.disabled = true;
+ addBtn.disabled = true;
+ setStatus('Recording camera path…');
+ try {
+ const stream = scene.canvas.captureStream(EXPORT_FPS);
+ const recorder = new MediaRecorder(stream, { mimeType, videoBitsPerSecond: 8_000_000 });
+ const chunks = [];
+ recorder.ondataavailable = (e) => { if (e.data.size) chunks.push(e.data); };
+ const stopped = new Promise((resolve) => { recorder.onstop = resolve; });
+ recorder.start();
+ await new Promise((resolve) => {
+ playback = { startWallMs: performance.now(), startTime: kfs[0].time, kfs, onDone: resolve };
+ setPlayUI(true);
+ });
+ recorder.stop();
+ await stopped;
+ const blob = new Blob(chunks, { type: mimeType });
+ const url = URL.createObjectURL(blob);
+ setStatus('');
+ const a = document.createElement('a');
+ a.href = url;
+ a.download = 'splat-camera-path.webm';
+ a.textContent = `Download video (${(blob.size / 1024 / 1024).toFixed(1)} MB)`;
+ a.className = 'sp-export-link';
+ status.hidden = false;
+ status.innerHTML = '';
+ status.appendChild(a);
+ } catch (err) {
+ setStatus(`Recording failed: ${err.message}`, true);
+ console.error('[splat] video export failed', err);
+ } finally {
+ exportBtn.disabled = false;
+ addBtn.disabled = hotspots.length >= MAX_HOTSPOTS;
+ }
+ }
+
+ // -- Render the list + button states ------------------------------------------
+ function render() {
+ const order = sorted();
+ list.innerHTML = '';
+ if (!order.length) {
+ empty.hidden = false;
+ } else {
+ empty.hidden = true;
+ order.forEach((h, idx) => list.appendChild(renderRow(h, idx, order.length)));
+ }
+ pruneMarkers();
+ addBtn.disabled = hotspots.length >= MAX_HOTSPOTS;
+ addBtn.title = hotspots.length >= MAX_HOTSPOTS ? `Up to ${MAX_HOTSPOTS} camera views per scene` : '';
+ exportBtn.disabled = order.length < 2;
+ exportBtn.title = order.length < 2 ? 'Save at least 2 camera views to export a path' : '';
+ playBtn.disabled = order.length < 2;
+ updateScrubUI(playback ? Number(scrub?.value) || 0 : Number(scrub?.value) || 0);
+ }
+
+ function renderRow(h, idx, total) {
+ const li = document.createElement('li');
+ li.className = 'sp-hotspot-row';
+
+ const badge = document.createElement('span');
+ badge.className = 'sp-hotspot-badge';
+ badge.textContent = String(idx + 1);
+ li.appendChild(badge);
+
+ const name = document.createElement('input');
+ name.className = 'sp-input sp-hotspot-name';
+ name.value = h.name;
+ name.setAttribute('aria-label', 'Camera view name');
+ name.addEventListener('change', () => renameHotspot(h.id, name.value));
+ li.appendChild(name);
+
+ const time = document.createElement('input');
+ time.type = 'number';
+ time.min = '0';
+ time.step = '0.1';
+ time.className = 'sp-input sp-hotspot-time';
+ time.value = h.time.toFixed(1);
+ time.setAttribute('aria-label', 'Time in seconds');
+ time.addEventListener('change', () => retimeHotspot(h.id, Number(time.value)));
+ li.appendChild(time);
+
+ const actions = document.createElement('div');
+ actions.className = 'sp-hotspot-actions';
+
+ const mkBtn = (label, title, onClick, disabled = false) => {
+ const b = document.createElement('button');
+ b.type = 'button';
+ b.className = 'sp-icon-btn';
+ b.textContent = label;
+ b.title = title;
+ b.setAttribute('aria-label', title);
+ b.disabled = disabled;
+ b.addEventListener('click', onClick);
+ return b;
+ };
+ actions.appendChild(mkBtn('↗', 'Jump to this view', () => goTo(h.id)));
+ actions.appendChild(mkBtn('↑', 'Move earlier', () => nudge(h.id, -1), idx === 0));
+ actions.appendChild(mkBtn('↓', 'Move later', () => nudge(h.id, 1), idx === total - 1));
+ actions.appendChild(mkBtn('✕', 'Delete camera view', () => removeHotspot(h.id)));
+ li.appendChild(actions);
+ return li;
+ }
+
+ // -- Wiring ---------------------------------------------------------------------
+ addBtn.addEventListener('click', addHotspot);
+ exportBtn.addEventListener('click', exportVideo);
+ playBtn.addEventListener('click', () => (playback ? pause() : play()));
+ scrub?.addEventListener('input', () => {
+ if (playback) pause();
+ const t = Number(scrub.value) || 0;
+ const scene = currentScene;
+ const sample = sampleCamera(sorted(), t);
+ if (scene && sample) {
+ scene.camera.position.copy(sample.position);
+ scene.controls.target.copy(sample.target);
+ scene.controls.update();
+ }
+ updateScrubUI(t);
+ });
+
+ function reset() {
+ hotspots = [];
+ playback = null;
+ tween = null;
+ setStatus('');
+ setPlayUI(false);
+ for (const el of markers.values()) el.remove();
+ markers.clear();
+ render();
+ }
+
+ function onSceneChanged(scene) {
+ currentScene = scene;
+ reset();
+ card.hidden = !scene;
+ if (scene) scene.setOnFrame(onFrame);
+ }
+
+ render();
+ card.hidden = true;
+ return { onSceneChanged, reset };
+}
diff --git a/src/splat-stage.js b/src/splat-stage.js
index 1c6feb734..d52409705 100644
--- a/src/splat-stage.js
+++ b/src/splat-stage.js
@@ -5,11 +5,36 @@
// element and a buffer of splat bytes, get back an orbiting scene and a handle
// that tears itself down completely.
//
-// It runs on Spark (`@sparkjsdev/spark`, MIT, World Labs). Spark renders splats
-// as ordinary objects inside a normal three.js scene, which is what lets us
-// frame the camera on the actual scene and, later, put splats and meshes in one
-// stage. It also reads the compressed containers real captures ship as: .spz is
-// about a tenth of the equivalent PLY and .sog about a fifteenth to a twentieth.
+// It runs on Spark (`@sparkjsdev/spark`, MIT, World Labs), 2.3.x. Spark renders
+// splats as ordinary objects inside a normal three.js scene, which is what lets
+// us frame the camera on the actual scene and, later, put splats and meshes in
+// one stage. It also reads the compressed containers real captures ship as:
+// .spz is about a tenth of the equivalent PLY and .sog about a fifteenth to a
+// twentieth.
+//
+// Spark 2.3 added an LoD tree this module uses: `mesh.createLodSplats()` builds
+// a coarser-at-distance splat set entirely from the mesh's own decoded buffer,
+// off the main thread in Spark's worker pool, no network involved. It is built
+// here automatically once a scene clears `LOD_AUTO_THRESHOLD` splats, the same
+// idea as mesh LOD chains (`packages/glb-tools`) applied per-splat.
+//
+// Spark 2.3 also ships a GPU-memory pager (`SplatPager`/`PagedSplats`, the
+// `paged` SplatMesh option), deliberately NOT wired up here: it is a streaming
+// primitive for a chunked remote source (Spark's own RAD container, or a
+// `-lod-N.<ext>` layout fetched over HTTP) and has no code path that pages an
+// already-in-memory buffer of any format this viewer reads (.ply/.spz/.splat/
+// .ksplat/.sog), confirmed by reading `PagedSplats.fetchDecodeChunk` in the
+// installed package: its `fileBytes` branch is a no-op for every format but
+// Spark's own RAD. Passing `paged: true` against a one-shot in-memory buffer
+// does not throw, but it builds an empty, data-source-less pager that the
+// renderer's LOD-drive loop then tries to fetch chunks from every frame,
+// logging "No url or fileBytes provided" forever while paging nothing. Setting
+// `mesh.paged` at all, even to that empty pager, is also what pulls the mesh
+// into the renderer's LOD bookkeeping in the first place, so the fix is not to
+// configure the pager correctly (there is no correct in-memory configuration)
+// but to never set `paged` and let `createLodSplats()` populate
+// `packedSplats.lodSplats` instead, which is the other, independent input the
+// same bookkeeping already checks for.
//
// Three behaviours here look like a broken page if you skip them, which is the
// reason this module exists instead of the same code twice:
@@ -30,6 +55,11 @@ import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
const FOV = 55;
const FIRST_PAINT_TIMEOUT_MS = 8000;
const DEFAULT_BACKGROUND = 0x0f0e16;
+const DEFAULT_LOD = true;
+// Below this many splats, building the LoD tree is pure overhead: the full
+// set already renders at full speed, so there is nothing for a coarser tier
+// to save.
+const LOD_AUTO_THRESHOLD = 50_000;
/** Every container Spark can decode, in the order a file picker should list them. */
export const SPLAT_EXTENSIONS = ['.ply', '.spz', '.splat', '.ksplat', '.sog'];
@@ -95,10 +125,16 @@ function frameCamera(camera, controls, mesh) {
* Pass false for splats authored Y-up,
* such as procedurally generated samples.
* @param {number} [options.background] clear colour behind the scene
+ * @param {boolean} [options.lod=true] once the scene clears {@link LOD_AUTO_THRESHOLD}
+ * splats, build Spark's per-splat LoD tree so
+ * distant geometry renders fewer, coarser
+ * splats instead of the full-detail set
+ * @param {number} [options.maxSplats] hard cap on resident splats; omit for
+ * Spark's own default
* @param {Function} [options.onSorting] called once decoding is done and the
* first sort is pending, with the splat count
* @param {Function} [options.onContextLost] called if the browser drops the GL context
- * @returns {Promise<{count:number, recenter:Function, dispose:Function}>}
+ * @returns {Promise<{count:number, recenter:Function, dispose:Function, camera:PerspectiveCamera, controls:OrbitControls, canvas:HTMLCanvasElement, scene:Scene, mesh:object}>}
* @throws if the bytes are not a splat scene this build can decode
*/
export async function mountSplatScene(host, buffer, {
@@ -106,6 +142,8 @@ export async function mountSplatScene(host, buffer, {
fileName,
flip = true,
background = DEFAULT_BACKGROUND,
+ lod = DEFAULT_LOD,
+ maxSplats,
onSorting,
onContextLost,
} = {}) {
@@ -135,6 +173,11 @@ export async function mountSplatScene(host, buffer, {
let frame = 0;
let mesh = null;
let disposed = false;
+ // Set when a scene clears LOD_AUTO_THRESHOLD and createLodSplats() is fired
+ // off below. dispose() waits on it: Spark's worker pool rejects any in-flight
+ // job the moment the renderer that owns it is disposed, and that job can
+ // still be running if the user navigates away seconds after a big scene loads.
+ let lodPromise = null;
const resize = () => {
renderer.setSize(width(), height());
camera.aspect = width() / height();
@@ -146,24 +189,47 @@ export async function mountSplatScene(host, buffer, {
disposed = true;
if (frame) cancelAnimationFrame(frame);
window.removeEventListener('resize', resize);
- // Order matters: drop the splat data, then the Spark renderer holding GPU
- // buffers for it, then the controls' listeners, then the context itself.
- try { mesh?.dispose?.(); } catch { /* already torn down */ }
- try { spark.dispose?.(); } catch { /* already torn down */ }
- try { controls.dispose?.(); } catch { /* already torn down */ }
- // renderer.dispose() frees the three.js caches and leaves the WebGL context
- // alive. Without an explicit release every scene swap leaks a live context,
- // and a browser grants a page only a handful before it starts reclaiming
- // the oldest out from under a running scene.
- try {
- renderer.forceContextLoss?.();
- renderer.dispose?.();
- } catch { /* context already lost */ }
host.innerHTML = '';
+ Promise.resolve(lodPromise).finally(() => {
+ // Order matters: drop the splat data, then the Spark renderer holding GPU
+ // buffers for it, then the controls' listeners, then the context itself.
+ try { mesh?.dispose?.(); } catch { /* already torn down */ }
+ // SparkRenderer drives an active LoD tree with its own per-frame worker
+ // calls, independent of our one-shot createLodSplats() promise above. Its
+ // dispose() terminates the shared worker pool and rejects every call still
+ // in flight (SplatWorker.dispose, read from the installed package), with no
+ // handler of its own, so the rejection lands here as unhandled on the next
+ // microtask regardless of how long we wait. Catching it in a dispose-scoped
+ // window is the correct fix, not a longer delay: no timeout is provably
+ // long enough, since a new per-frame call can be dispatched on literally
+ // the last rendered frame before this runs.
+ let quietUntil = true;
+ const quietRejection = (e) => { if (quietUntil) e.preventDefault(); };
+ window.addEventListener('unhandledrejection', quietRejection);
+ setTimeout(() => {
+ quietUntil = false;
+ window.removeEventListener('unhandledrejection', quietRejection);
+ }, 50);
+ try { spark.dispose?.(); } catch { /* already torn down */ }
+ try { controls.dispose?.(); } catch { /* already torn down */ }
+ // renderer.dispose() frees the three.js caches and leaves the WebGL context
+ // alive. Without an explicit release every scene swap leaks a live context,
+ // and a browser grants a page only a handful before it starts reclaiming
+ // the oldest out from under a running scene.
+ try {
+ renderer.forceContextLoss?.();
+ renderer.dispose?.();
+ } catch { /* context already lost */ }
+ });
}
try {
- mesh = new SPARK.SplatMesh({ fileBytes: buffer, fileType, fileName });
+ mesh = new SPARK.SplatMesh({
+ fileBytes: buffer,
+ fileType,
+ fileName,
+ ...(Number.isFinite(maxSplats) && maxSplats > 0 ? { maxSplats: Math.floor(maxSplats) } : {}),
+ });
if (flip) mesh.quaternion.set(1, 0, 0, 0);
await mesh.initialized;
} catch (err) {
@@ -172,14 +238,28 @@ export async function mountSplatScene(host, buffer, {
}
if (disposed) throw new Error('splat stage was disposed while decoding');
+ const count = mesh.packedSplats?.numSplats ?? mesh.splats?.numSplats ?? 0;
+ if (lod && count >= LOD_AUTO_THRESHOLD) {
+ // Fire-and-forget: this runs in Spark's worker pool and must not delay
+ // first paint. SparkRenderer re-reads `packedSplats.lodSplats` every
+ // frame, so the scene starts at full detail and switches itself over
+ // the moment the tree lands, with no visible seam to wait for here.
+ lodPromise = mesh.createLodSplats().catch((err) => console.error('[splat] LoD tree build failed', err));
+ }
+
scene.add(mesh);
const home = frameCamera(camera, controls, mesh);
window.addEventListener('resize', resize);
resize();
+ // Set by splat-hotspots.js while a camera-path keyframe sequence plays back:
+ // runs once per rendered frame, ahead of controls.update()/render, so camera
+ // playback and the one render loop this stage owns never fall out of sync.
+ let onFrame = null;
const tick = () => {
if (disposed) return;
frame = requestAnimationFrame(tick);
+ onFrame?.();
controls.update();
renderer.render(scene, camera);
};
@@ -192,7 +272,6 @@ export async function mountSplatScene(host, buffer, {
onContextLost?.();
}, { once: true });
- const count = mesh.packedSplats?.numSplats ?? 0;
onSorting?.(count);
await Promise.race([firstPaint, new Promise((r) => setTimeout(r, FIRST_PAINT_TIMEOUT_MS))]);
@@ -205,5 +284,15 @@ export async function mountSplatScene(host, buffer, {
controls.target.copy(home.center);
controls.update();
},
+ camera,
+ controls,
+ canvas: renderer.domElement,
+ renderer,
+ scene,
+ mesh,
+ /** Install (or clear, with `null`) a per-frame hook for camera playback. */
+ setOnFrame(fn) {
+ onFrame = typeof fn === 'function' ? fn : null;
+ },
};
}
diff --git a/src/splat-viewer.js b/src/splat-viewer.js
index 2b29a58a1..61ffacc11 100644
--- a/src/splat-viewer.js
+++ b/src/splat-viewer.js
@@ -10,6 +10,7 @@
import { Color } from 'three';
import { loadSpark, detectSplatFormat, mountSplatScene, SPLAT_EXTENSIONS_LABEL } from './splat-stage.js';
+import { initHotspotTimeline } from './splat-hotspots.js';
const $ = (s, r = document) => r.querySelector(s);
@@ -21,9 +22,11 @@ let _scene = null; // handle from mountSplatScene
let _token = 0; // guards a slow load finishing after a newer one started
let _downloadUrl = null; // blob URL backing the download button
let _lastRender = null; // { buffer, fileType, fileName, label, flip }, for recenter
+let _hotspots = null; // handle from initHotspotTimeline, wired in init()
function teardown() {
_token++;
+ _hotspots?.onSceneChanged(null);
if (!_scene) return;
_scene.dispose();
_scene = null;
@@ -145,6 +148,7 @@ async function renderBuffer(buffer, { label, fileType, fileName, flip = true })
if (token !== _token) { scene.dispose(); return false; }
_scene = scene;
+ _hotspots?.onSceneChanged(scene);
setLive(scene.count ? `${label} \u00b7 ${scene.count.toLocaleString()} splats` : label);
return true;
}
@@ -307,6 +311,8 @@ async function loadSample(kind) {
function init() {
if (!STAGE()) return;
+ _hotspots = initHotspotTimeline();
+
$('#sp-load-url').addEventListener('click', () => {
const url = $('#sp-url').value.trim();
if (url) loadFromUrl(url);
diff --git a/tests/e2e/splat-hotspots.spec.js b/tests/e2e/splat-hotspots.spec.js
new file mode 100644
index 000000000..68d6bb756
--- /dev/null
+++ b/tests/e2e/splat-hotspots.spec.js
@@ -0,0 +1,93 @@
+// /splat: annotation hotspots, the camera timeline built from them, and the
+// in-browser video export, driven against the real shipped module and a real
+// (procedurally generated, client-side) splat scene, no server round trip,
+// no mocked MediaRecorder or captureStream.
+//
+// Covers the Definition of Done for order 081 task 2: saving two camera views
+// on a live splat, then exporting the camera path between them, must produce
+// a real downloadable video.
+
+import { test, expect } from '@playwright/test';
+import { collectPageErrors } from './_support.js';
+
+test('saving camera views builds a timeline and exports a real video', async ({ page }) => {
+ const errors = collectPageErrors(page);
+
+ await page.goto('/splat', { waitUntil: 'domcontentloaded', timeout: 60_000 });
+
+ // Load a real splat scene (procedurally generated bust, not a mock: the
+ // same code path a real .ply/.spz capture takes through mountSplatScene).
+ await page.locator('.sp-sample[data-sample="bust"]').click();
+ await expect(page.locator('#sp-hud')).toBeVisible({ timeout: 30_000 });
+
+ // Hotspot card appears once a scene is live, not before.
+ const card = page.locator('#sp-timeline-card');
+ await expect(card).toBeVisible();
+ await expect(page.locator('#sp-hotspot-empty')).toBeVisible();
+ await expect(page.locator('#sp-export-video')).toBeDisabled();
+
+ // Save the first camera view, orbit the camera, save a second.
+ await page.locator('#sp-hotspot-add').click();
+ await expect(page.locator('.sp-hotspot-row')).toHaveCount(1);
+ await expect(page.locator('#sp-hotspot-empty')).toBeHidden();
+
+ const canvas = page.locator('#sp-host canvas');
+ const box = await canvas.boundingBox();
+ await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2);
+ await page.mouse.down();
+ await page.mouse.move(box.x + box.width * 0.75, box.y + box.height * 0.3, { steps: 10 });
+ await page.mouse.up();
+
+ await page.locator('#sp-hotspot-add').click();
+ await expect(page.locator('.sp-hotspot-row')).toHaveCount(2);
+
+ // A path of two camera views enables playback and export.
+ await expect(page.locator('#sp-export-video')).toBeEnabled();
+ await expect(page.locator('#sp-timeline-play')).toBeEnabled();
+
+ // Renaming and retiming a hotspot round-trips through the module's state.
+ const firstName = page.locator('.sp-hotspot-row').nth(0).locator('.sp-hotspot-name');
+ await firstName.fill('Opening shot');
+ await firstName.blur();
+ await expect(firstName).toHaveValue('Opening shot');
+
+ // Export: records the real canvas via captureStream()+MediaRecorder while
+ // the saved camera path actually plays, no setTimeout fake-progress.
+ await page.locator('#sp-export-video').click();
+ await expect(page.locator('#sp-export-status')).toBeVisible();
+ await expect(page.locator('#sp-export-status')).toContainText('Recording', { timeout: 5_000 });
+
+ const downloadLink = page.locator('#sp-export-status a.sp-export-link');
+ await expect(downloadLink).toBeVisible({ timeout: 20_000 });
+ await expect(downloadLink).toContainText('Download video');
+
+ const href = await downloadLink.getAttribute('href');
+ expect(href).toMatch(/^blob:/);
+ const bytes = await page.evaluate(async (url) => {
+ const res = await fetch(url);
+ const buf = await res.arrayBuffer();
+ return buf.byteLength;
+ }, href);
+ expect(bytes).toBeGreaterThan(0);
+
+ expect(errors).toEqual([]);
+});
+
+test('the hotspot marker jumps the camera back to its saved view', async ({ page }) => {
+ const errors = collectPageErrors(page);
+
+ await page.goto('/splat', { waitUntil: 'domcontentloaded', timeout: 60_000 });
+ await page.locator('.sp-sample[data-sample="shell"]').click();
+ await expect(page.locator('#sp-hud')).toBeVisible({ timeout: 30_000 });
+
+ await page.locator('#sp-hotspot-add').click();
+ const marker = page.locator('.sp-hotspot-marker').first();
+ await expect(marker).toBeVisible();
+ await marker.click();
+
+ // The "jump to this view" action in the row does the same thing and must
+ // not throw even with a single saved view (no path, just a camera tween).
+ await page.locator('.sp-hotspot-row .sp-icon-btn').first().click();
+
+ expect(errors).toEqual([]);
+});
diff --git a/tests/splat-viewer-page.test.js b/tests/splat-viewer-page.test.js
index d1a5ab8d4..b5780344f 100644
--- a/tests/splat-viewer-page.test.js
+++ b/tests/splat-viewer-page.test.js
@@ -31,17 +31,20 @@ import { applyCatalog } from '../src/i18n.js';
const ROOT = resolve(import.meta.dirname, '..');
const PAGE = readFileSync(resolve(ROOT, 'pages/splat.html'), 'utf8');
const MODULE = readFileSync(resolve(ROOT, 'src/splat-viewer.js'), 'utf8');
+const HOTSPOTS = readFileSync(resolve(ROOT, 'src/splat-hotspots.js'), 'utf8');
/** Ids the page markup defines. */
function pageIds() {
return new Set([...PAGE.matchAll(/\bid="([^"]+)"/g)].map((m) => m[1]));
}
-/** Ids the module looks up through its `$('#…')` helper. */
-function moduleIds() {
- return [...MODULE.matchAll(/\$\(`?#([a-z0-9-]+)/gi)].map((m) => m[1]);
+/** Ids a module source looks up through its `$('#…')` helper. */
+function idsIn(source) {
+ return [...source.matchAll(/\$\(`?#([a-z0-9-]+)/gi)].map((m) => m[1]);
}
+const moduleIds = () => idsIn(MODULE);
+
const catalog = (locale) => (key) => {
const [ns, ...rest] = key.split('.');
const hit = locale[ns]?.[rest.join('.')];
@@ -62,6 +65,12 @@ describe('splat viewer page and module agree', () => {
expect(missing, `pages/splat.html is missing ids used by src/splat-viewer.js: ${missing.join(', ')}`).toEqual([]);
});
+ it('defines every element id the hotspot/timeline module writes to', () => {
+ const defined = pageIds();
+ const missing = [...new Set(idsIn(HOTSPOTS))].filter((id) => !defined.has(id));
+ expect(missing, `pages/splat.html is missing ids used by src/splat-hotspots.js: ${missing.join(', ')}`).toEqual([]);
+ });
+
it('gives the HUD label its own node, separate from the HUD controls', () => {
const root = stage();
const hud = root.querySelector('#sp-hud');