Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.
JavaScript ai-agentsanimationcanvasclaude-codecodexexplainer-videoffmpegjavascriptkokoroproduct-demotext-to-speechvideo-generationwhisper
Show video clips in scenes with s.clip (#8)
Vincent Schmalbachf4db9059dead
diff --git a/AGENTS.md b/AGENTS.mdindex 8271283..b8b0bb0 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -347,6 +347,9 @@ A mouse pointer clicks through the screens and types into the fields, and thecamera zooms in on what the voice talks about. `examples/unspar-demo` is acomplete one.+You can also show a real screen recording with `s.clip` in a `browser`+frame. Use `s.camera` to zoom in on the detail the voice mentions.+**Keep it true.** A demo is advertising, so everything in it has to be right.- Take the wording from the app itself: its templates and language files@@ -788,7 +791,7 @@ stops at the edge of that element. Options: `bend` (about -1 to 1), `head``labelOffset`, `gap`, `dashed`, `width`, `headSize`. `s.connect` does thesame.-### Icons and images+### Icons, images and video clips```jss.icon('database', { x: 1500, y: 540, size: 140, color: 'purple', at: 'database' });@@ -807,6 +810,28 @@ Old Lucide names work too. Icon options: `size` (120), `color`, `weight` (2),(`none`, `card`, `browser`, `window`, `phone`), `url` (for the browser frame),`border`, `shadow: false`, `kenburns: true` for a slow zoom.+`s.clip(src, options)` draws a video from `assets/`. It takes the same+layout, frame and timing options as `s.image` and returns the same box.+Use `.mp4`, `.mov`, `.webm` or `.m4v` files.++```js+s.clip('assets/signup.mp4', { w: 1200, frame: 'browser', url: 'app.example.com', at: 'signup' });+s.clip('assets/demo.mov', { x: 1400, y: 540, h: 600, from: 12.5, speed: 1.5, at: '#demo', out: 9 });+```++- `from` (0) is the second in the clip where playback starts.+- `speed` (1) goes from 0.25 to 4. Playback starts at `at`.+- `loop` (false) repeats the whole clip. Otherwise it holds the last frame.+ `check` gives a hint when that freeze lasts more than a second.+- `volume` (0) goes from 0 to 1. Sound plays only at `speed: 1`.+ It stops at `out`, the clip's end or the scene's end. Loops repeat the+ sound too. Music ducks under clip sound as it does under the voice.++Frames are extracted once at the project's fps and saved in `build/clips/`.+They are at most 1920 pixels wide. Renders, stills and sheets use these+frames. The preview may show a nearby frame while the right one loads.+`s.video` still holds the whole video's duration, frames, fps and size.+### Pointing things out```js@@ -941,6 +966,11 @@ font roles are `display`, `body`, `hand` and `mono`.- Colored text in `paper` and `clean`. Green, orange and red are light enoughfor shapes but often too light for text. `check` warns about low contrast.Use `ink`, `accent` or a darker hex color for text.+- Long video clips fill `build/clips/` with frames. Trim footage to the part+ you show before you add it. Deleting `build/clips/` is safe.+- Screen recordings often have a variable frame rate. explainroo converts+ them to the project's fps, so a frame can repeat or drop. Record at a fixed+ frame rate when the motion matters.`check` looks at text only. It finds text off the frame, too close to theedge (3.5%), too small, with low contrast (against the page, or against thediff --git a/engine/audio/soundtrack.js b/engine/audio/soundtrack.jsindex e333bc1..c90ee43 100644--- a/engine/audio/soundtrack.js+++ b/engine/audio/soundtrack.js@@ -126,6 +126,7 @@ export async function renderSoundtrack(opts = {}) {sampleRate = 48000,voice = [],voiceSpans = [],+ clips = [],music = null,sfx = [],scenes = [],@@ -157,13 +158,32 @@ export async function renderSoundtrack(opts = {}) {}}+ const clipSpans = [];+ for (const clip of clips) {+ if (clip.speed !== 1 || clip.volume <= 0) continue;+ const data = await loadVoice(clip, sr, cache);+ const clipLength = Math.max(1, Math.round(clip.duration * sr));+ const start = Math.round(clip.start * sr);+ const end = Math.min(length, Math.round(clip.end * sr));+ const offset = Math.round(clip.from * sr);+ for (let i = Math.max(0, start); i < end; i++) {+ let source = offset + i - start;+ if (clip.loop) source %= clipLength;+ if (source >= clipLength) break;+ const sample = (data[source] || 0) * clip.volume;+ L[i] += sample;+ R[i] += sample;+ }+ if (end > Math.max(0, start)) clipSpans.push({ start: Math.max(0, clip.start), end: end / sr });+ }+// music: glue compressor, level with fades, ducked under narrationif (plan) {const { synth } = createMusic({ ...music, style: plan.style, seed: plan.seed, bpm: plan.bpm, key: plan.keyRoot, duration, scenes, sampleRate: sr });const volume = Math.max(0, Math.min(2, music.volume == null ? 0.5 : Number(music.volume)));renderInto(synth, L, R, {compressor: new Compressor(sr, { threshold: -18, ratio: 2, knee: 10, attack: 0.02, release: 0.3 }),- gains: [levelEnv(dbToGain(MUSIC_BASE_DB + plan.mixDb) * (volume / 0.5), duration), duckEnv(voiceSpans)],+ gains: [levelEnv(dbToGain(MUSIC_BASE_DB + plan.mixDb) * (volume / 0.5), duration), duckEnv([...voiceSpans, ...clipSpans])],});}diff --git a/engine/clips.js b/engine/clips.jsnew file mode 100644index 0000000..eec8db6--- /dev/null+++ b/engine/clips.js@@ -0,0 +1,84 @@+export const VIDEO_FILE = /\.(mp4|mov|webm|m4v)$/i;++export function clipFrame(info, t, { at = 0, from = 0, speed = 1, loop = false } = {}) {+ if (t < at) return null;+ let time = from + (t - at) * speed;+ if (loop) time = ((time % info.duration) + info.duration) % info.duration;+ return Math.max(0, Math.min(info.frames - 1, Math.floor(time * info.fps + 1e-8)));+}++export function clipURL(kind, src, frame) {+ return `/__clip/${kind}?path=${encodeURIComponent(src)}${frame === undefined ? '' : '&frame=' + frame}`;+}++export async function loadClips(assets) {+ const clips = new Map();+ await Promise.all(assets.filter((a) => VIDEO_FILE.test(a)).map(async (src) => {+ const res = await fetch(clipURL('info', src));+ if (!res.ok) throw new Error(`clip "${src}" could not be loaded. Put a video in the project's assets/ folder.`);+ clips.set(src, { src, ...await res.json() });+ }));+ return clips;+}++export class ClipFrames {+ constructor() {+ this.frames = new Map();+ this.pending = new Map();+ this.errors = new Map();+ this.closed = false;+ }++ key(clip, frame) { return `${clip.src}|${frame}`; }++ async load(clip, frame) {+ const key = this.key(clip, frame);+ if (this.frames.has(key)) return;+ if (!this.pending.has(key)) {+ const job = (async () => {+ const res = await fetch(clipURL('frame', clip.src, frame));+ if (!res.ok) throw new Error(`clip "${clip.src}" frame ${frame} could not be loaded`);+ const bitmap = await createImageBitmap(await res.blob());+ if (this.closed) { bitmap.close(); return; }+ this.frames.set(key, { clip, frame, bitmap });+ })();+ this.pending.set(key, job);+ job.finally(() => this.pending.delete(key)).catch(() => {});+ }+ return this.pending.get(key);+ }++ get(clip, frame) {+ const key = this.key(clip, frame);+ const entry = this.frames.get(key);+ if (entry) {+ this.frames.delete(key);+ this.frames.set(key, entry);+ return entry.bitmap;+ }+ return null;+ }++ nearest(clip, frame) {+ let best;+ for (const entry of this.frames.values()) {+ if (entry.clip.src === clip.src && (!best || Math.abs(entry.frame - frame) < Math.abs(best.frame - frame))) best = entry;+ }+ return best?.bitmap;+ }++ trim(keep = new Set()) {+ for (const [key, entry] of this.frames) {+ if (this.frames.size <= Math.max(24, keep.size)) break;+ if (keep.has(key)) continue;+ entry.bitmap.close();+ this.frames.delete(key);+ }+ }++ close() {+ this.closed = true;+ for (const entry of this.frames.values()) entry.bitmap.close();+ this.frames.clear();+ }+}diff --git a/engine/frame.html b/engine/frame.htmlindex 6b8968a..5079e2e 100644--- a/engine/frame.html+++ b/engine/frame.html@@ -26,10 +26,11 @@ window.explainrooReady = (async () => {info() {return { duration: state.timeline.duration, frames: state.timeline.frames, fps, width: canvas.width, height: canvas.height, scenes: state.timeline.scenes.map((s) => ({ id: s.id, start: s.start, dur: s.dur })) };},- frames(list, quality) {+ async frames(list, quality) {const out = [];for (const f of list) {try {+ if (engine.clips.size) await engine.prepareFrame(f / fps);engine.renderFrame(f / fps);} catch (e) {throw new Error(describe(e));@@ -41,6 +42,7 @@ window.explainrooReady = (async () => {async stills(list, { type = 'image/png', quality } = {}) {for (const { T, path } of list) {try {+ if (engine.clips.size) await engine.prepareFrame(T);engine.renderFrame(T);} catch (e) {throw new Error(describe(e));@@ -65,6 +67,7 @@ window.explainrooReady = (async () => {g.textBaseline = 'middle';for (let i = 0; i < times.length; i++) {try {+ if (engine.clips.size) await engine.prepareFrame(times[i]);engine.renderFrame(times[i]);} catch (e) {throw new Error(describe(e));diff --git a/engine/runtime.js b/engine/runtime.jsindex 7dae296..f2f2f10 100644--- a/engine/runtime.js+++ b/engine/runtime.js@@ -8,6 +8,7 @@ import { drawTransition } from './transitions.js';import { buildPhrases, drawCaptions } from './captions.js';import { hashStr, mulberry32, suggest, contrast, opaque, withAlpha } from './util.js';import { layoutAreas } from './layout.js';+import { loadClips, ClipFrames } from './clips.js';import { UI } from './ui.js';import { playbackTextIssue } from './readability.js';@@ -59,7 +60,7 @@ async function loadImages(assets) {}export class Engine {- constructor({ canvas, state, scenes, icons, images, scale = 1, sfxNames = [] }) {+ constructor({ canvas, state, scenes, icons, images, clips = new Map(), scale = 1, sfxNames = [] }) {const { config, timeline } = state;this.state = state;this.config = config;@@ -79,6 +80,9 @@ export class Engine {this.icons = icons;this.iconCache = new Map();this.images = images;+ this.clips = clips;+ this.clipFrames = new ClipFrames();+ this.clipProbe = null;this.sfxNames = sfxNames;this.sfxMode = config.sfx;this.recorder = null;@@ -114,6 +118,43 @@ export class Engine {return img;}+ clip(src) {+ const key = src.replace(/^\.?\//, '');+ const clip = this.clips.get(key) || this.clips.get('assets/' + key);+ if (!clip) throw new SceneError(`clip "${src}" not found. Put an MP4, MOV, WebM or M4V in the project's assets/ folder. Available: ${[...this.clips.keys()].join(', ') || 'none'}`);+ return clip;+ }++ clipImage(clip, frame) {+ if (this.recorder) return null;+ const key = this.clipFrames.key(clip, frame);+ if (this.clipProbe) {+ this.clipProbe.set(key, { clip, frame });+ return null;+ }+ const img = this.clipFrames.get(clip, frame);+ if (img) return img;+ if (this.clipFrames.errors.has(key)) throw this.clipFrames.errors.get(key);+ this.clipFrames.load(clip, frame).then(() => {+ this.clipFrames.trim();+ this.onClipFrame?.();+ }).catch((e) => {+ this.clipFrames.errors.set(key, e);+ this.onClipFrame?.();+ });+ return this.clipFrames.nearest(clip, frame);+ }++ async prepareFrame(T) {+ if (!this.clips.size) return;+ const needed = new Map();+ this.clipProbe = needed;+ try { this.renderFrame(T); }+ finally { this.clipProbe = null; }+ await Promise.all([...needed.values()].map(({ clip, frame }) => this.clipFrames.load(clip, frame)));+ this.clipFrames.trim(new Set(needed.keys()));+ }+sceneAt(T) {const list = this.timeline.scenes;let i = list.length - 1;@@ -245,8 +286,10 @@ export class Engine {// Sound cues for the soundtrack: element entrances, explicit s.sfx() calls// and scene transitions, thinned so effects never pile up.sfxCues() {- if (this.config.sfx === false) return { cues: [], errors: [] };+ if (this.config.sfx === false && !this.clips.size) return { cues: [], clips: [], errors: [] };const { events, errors } = this.collectEvents();+ const clips = events.filter((e) => e.kind === 'clip' && e.volume > 0 && e.speed === 1 && e.hasAudio).map((e) => ({ ...e, start: e.T, end: e.T + e.end - e.at }));+ if (this.config.sfx === false) return { cues: [], clips, errors };const cues = [];for (const e of events) {if (!e.sfx) continue;@@ -280,7 +323,7 @@ export class Engine {last.set(c.name, c.t);out.push(c);}- return { cues: out, errors };+ return { cues: out, clips, errors };}// Samples every scene and reports layout problems.@@ -374,6 +417,10 @@ export class Engine {this.textLog = null;const times = [...new Set(entrances.map((e) => Math.round(e.at * 100) / 100))].sort((a, b) => a - b);for (const e of entrances) {+ if (e.kind === 'clip') {+ if (e.volume > 0 && e.speed !== 1) once(`clip-sound:${sc.id}:${e.key}`, () => add('warn', sc.id, e.at, `clip "${e.src}" stays silent at speed ${e.speed}; use speed: 1 for its sound`));+ if (!e.loop && e.out - e.end > 1) once(`clip-freeze:${sc.id}:${e.key}`, () => add('hint', sc.id, e.end, `clip "${e.src}" freezes on its last frame from ${e.end.toFixed(2)}s until ${e.out.toFixed(2)}s; use loop: true or an earlier out`));+ }if (e.at > sc.dur) once(`late:${sc.id}:${e.key}`, () => add('error', sc.id, e.at, `a ${e.kind} is set to appear at ${e.at.toFixed(2)}s but the scene ends at ${sc.dur.toFixed(2)}s`));}if (!times.length) {@@ -400,7 +447,7 @@ export class Engine {export async function boot({ canvas, scale = 1, version = '' }) {const state = await fetchState();const theme = getTheme(state.config.theme, state.config.look);- const [icons, images, scenesMod, audio, iconTags] = await Promise.all([+ const [icons, images, scenesMod, audio, iconTags, clips] = await Promise.all([fetch('/engine/icons/lucide.json').then((r) => r.json()).then((j) => {// Old Lucide names keep working.for (const [alias, target] of Object.entries(j.aliases || {})) if (!j.icons[alias]) j.icons[alias] = j.icons[target];@@ -410,11 +457,12 @@ export async function boot({ canvas, scale = 1, version = '' }) {import(`/project/scenes.js?v=${encodeURIComponent(version || state.version)}`),import('/engine/audio/soundtrack.js').catch((e) => ({ SFX: {}, loadError: e })),fetch('/engine/icons/lucide-tags.json').then((r) => r.json()),+ loadClips(state.assets),loadFonts(theme, state.config),]);const scenes = scenesMod.default;if (!scenes || typeof scenes !== 'object') throw new Error('scenes.js must "export default { sceneId(s) { ... } }"');- const engine = new Engine({ canvas, state, scenes, icons, images, scale, sfxNames: Object.keys(audio.SFX) });+ const engine = new Engine({ canvas, state, scenes, icons, images, clips, scale, sfxNames: Object.keys(audio.SFX) });engine.iconTags = iconTags;return { engine, audio, state };}@@ -423,7 +471,7 @@ export async function boot({ canvas, scale = 1, version = '' }) {export async function renderAudio(engine, audio, { sampleRate = 48000 } = {}) {if (audio.loadError) throw new Error(`audio engine failed to load: ${audio.loadError.message}`);const { timeline, config } = engine.state;- const { cues, errors } = engine.sfxCues();+ const { cues, clips, errors } = engine.sfxCues();const buffer = await audio.renderSoundtrack({duration: timeline.duration,sampleRate,@@ -431,6 +479,7 @@ export async function renderAudio(engine, audio, { sampleRate = 48000 } = {}) {voiceSpans: timeline.voiceSpans,music: config.music,sfx: cues,+ clips,scenes: timeline.scenes.map((s) => ({ start: s.start, dur: s.dur })),});return { buffer, cues, errors };diff --git a/engine/stage.js b/engine/stage.jsindex 5b1e3e4..2fa9c07 100644--- a/engine/stage.js+++ b/engine/stage.js@@ -6,6 +6,7 @@ import { clamp, lerp, ease, easeFn, rand01, noise1, hashStr, mulberry32, suggestimport { colorOf, tintOf, fontString } from './themes.js';import { scalePath, quadD, polyD, pathLength, pointAt } from './pathdata.js';import { layoutText, lineStartX } from './text.js';+import { clipFrame, clipURL } from './clips.js';import { highlightLine } from './code.js';const DEFAULT_DUR = { draw: 0.9, write: 0.8, pop: 0.45, rise: 0.55, fade: 0.5, zoom: 0.6, drop: 0.55, grow: 1.0, none: 0.001, 'slide-left': 0.55, 'slide-right': 0.55, 'slide-up': 0.55, 'slide-down': 0.55 };@@ -852,13 +853,34 @@ export class Stage {}image(src, o = {}) {- const th = this.theme;const img = this.engine.image(src);- const ratio = img.naturalWidth / img.naturalHeight;+ return this._picture(src, img, img.naturalWidth, img.naturalHeight, o);+ }++ clip(src, o = {}) {+ const clip = this.engine.clip(src);+ const from = o.from ?? 0;+ const speed = o.speed ?? 1;+ const volume = o.volume ?? 0;+ for (const [name, value, lo, hi] of [['from', from, 0, Infinity], ['speed', speed, 0.25, 4], ['volume', volume, 0, 1]]) {+ if (!Number.isFinite(value) || value < lo || value > hi) throw new SceneError(`clip ${name} must be ${name === 'from' ? 'a non-negative number' : `from ${lo} to ${hi}`}`);+ }+ const at = this.time(o.at ?? 0);+ const out = o.out === undefined ? this.dur : Math.min(this.dur, this.time(o.out));+ const end = o.loop ? out : Math.min(out, at + Math.max(0, clip.duration - from) / speed);+ const n = this._clipSeq = (this._clipSeq || 0) + 1;+ if (this.engine.recorder) this.engine.recorder.push({ kind: 'clip', key: `clip|${at}|${o.id ?? n}|${src}`, at, end, out, from, speed, loop: !!o.loop, volume, hasAudio: clip.hasAudio, duration: clip.duration, src: clip.src, url: clipURL('audio', clip.src) });+ const frame = clipFrame(clip, this.t, { at, from, speed, loop: o.loop });+ return this._picture(src, () => frame === null ? null : this.engine.clipImage(clip, frame), clip.width, clip.height, o);+ }++ _picture(src, source, naturalWidth, naturalHeight, o) {+ const th = this.theme;+ const ratio = naturalWidth / naturalHeight;let w = o.w;let h = o.h;if (!w && !h) {- w = Math.min(this._w(0.6), img.naturalWidth);+ w = Math.min(this._w(0.6), naturalWidth);h = w / ratio;} else if (!h) h = w / ratio;else if (!w) w = h * ratio;@@ -924,7 +946,12 @@ export class Stage {}dw *= kb;dh *= kb;- ctx.drawImage(img, -dw / 2, iy - dh / 2, dw, dh);+ const img = typeof source === 'function' ? source() : source;+ if (img) ctx.drawImage(img, -dw / 2, iy - dh / 2, dw, dh);+ else {+ ctx.fillStyle = th.surface;+ ctx.fillRect(-w / 2, iy - h / 2, w, h);+ }ctx.restore();if (o.border) {this.pen.draw(ctx, { kind: 'rect', w, h, r }, { stroke: colorOf(th, o.border === true ? 'ink' : o.border), width: th.stroke.width, seed: this._seed('imgborder', o, src) }, { stroke: 1, fill: 1 });diff --git a/engine/studio.html b/engine/studio.htmlindex 3e2d5a1..6a2a8f3 100644--- a/engine/studio.html+++ b/engine/studio.html@@ -108,7 +108,12 @@ function buildUi() {async function load(version) {const scale = Math.min(1, 1280 / (await fetchState()).config.width);const r = await boot({ canvas, scale, version });+ if (engine) {+ engine.onClipFrame = null;+ engine.clipFrames.close();+ }engine = r.engine;+ engine.onClipFrame = draw;audioMod = r.audio;state = r.state;buildUi();diff --git a/src/clips.js b/src/clips.jsnew file mode 100644index 0000000..998c517--- /dev/null+++ b/src/clips.js@@ -0,0 +1,19 @@+import fs from 'node:fs';+import path from 'node:path';+import { ffmpeg, probe } from './ffmpeg.js';++export const VIDEO_FILE = /\.(mp4|mov|webm|m4v)$/i;++export async function extractClip(file, dir, fps) {+ const input = await probe(file);+ const video = input.streams.find((s) => s.codec_type === 'video');+ if (!video) throw new Error('this clip has no video. Use an MP4, MOV, WebM or M4V with video.');+ await ffmpeg(['-loglevel', 'error', '-i', file, '-map', '0:v:0', '-vf', `fps=${fps},scale=min(1920\\,iw):-1`, '-q:v', '2', '-threads', '1', '-start_number', '0', path.join(dir, '%08d.jpg')]);+ const frames = fs.readdirSync(dir).filter((n) => n.endsWith('.jpg')).length;+ if (!frames) throw new Error('this clip has no frames. Use a video with at least one frame.');+ const frame = await probe(path.join(dir, '00000000.jpg'));+ const { width, height } = frame.streams[0];+ const hasAudio = input.streams.some((s) => s.codec_type === 'audio');+ if (hasAudio) await ffmpeg(['-loglevel', 'error', '-i', file, '-map', '0:a:0', '-vn', '-ar', '48000', '-ac', '1', '-c:a', 'pcm_s16le', path.join(dir, 'audio.wav')]);+ return { frames, fps, width, height, duration: frames / fps, hasAudio };+}diff --git a/src/server.js b/src/server.jsindex d004d8d..12705a4 100644--- a/src/server.js+++ b/src/server.js@@ -3,7 +3,9 @@import http from 'node:http';import fs from 'node:fs';import path from 'node:path';+import crypto from 'node:crypto';import { fileURLToPath } from 'node:url';+import { extractClip, VIDEO_FILE } from './clips.js';export const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));@@ -26,6 +28,9 @@ const TYPES = {'.gif': 'image/gif','.svg': 'image/svg+xml','.mp4': 'video/mp4',+ '.mov': 'video/quicktime',+ '.webm': 'video/webm',+ '.m4v': 'video/mp4','.txt': 'text/plain; charset=utf-8',};@@ -42,7 +47,7 @@ function safeJoin(base, rel) {// Project files the engine page may read: scenes.js and the modules it// imports, images, fonts, audio and video. Never dotfiles such as .env, and// no JSON, Markdown or text files, which can hold keys or private notes.-const PROJECT_FILES = /\.(m?js|png|jpe?g|webp|gif|svg|wav|mp3|mp4|woff2?|ttf|otf)$/i;+const PROJECT_FILES = /\.(m?js|png|jpe?g|webp|gif|svg|wav|mp3|mp4|mov|webm|m4v|woff2?|ttf|otf)$/i;function allowedName(rel) {const parts = rel.split(/[\\/]+/);@@ -66,6 +71,53 @@ function projectFile(projectDir, rel) {return allowedName(realRel) ? real : null;}+function clipFile(projectDir, rel) {+ if (!rel.startsWith('assets/') || !VIDEO_FILE.test(rel)) return null;+ const file = projectFile(projectDir, rel);+ const assets = path.join(fs.realpathSync(projectDir), 'assets');+ if (!file || !VIDEO_FILE.test(file) || !file.startsWith(assets + path.sep) || !fs.statSync(file).isFile()) return null;+ return file;+}++function clipCacheFile(projectDir, rel) {+ const file = uploadFile(projectDir, rel);+ if (!file) return null;+ const realRel = path.relative(fs.realpathSync(projectDir), fs.realpathSync(path.dirname(file)));+ return realRel.split(path.sep).some((p) => p.startsWith('.')) ? null : file;+}++const clipJobs = new Map();++async function cachedClip(projectDir, file, fps) {+ const hash = crypto.createHash('sha256').update(JSON.stringify({ fps, width: 1920, version: 1 }));+ for await (const chunk of fs.createReadStream(file)) hash.update(chunk);+ const rel = `build/clips/${hash.digest('hex')}`;+ const infoFile = clipCacheFile(projectDir, `${rel}/info.json`);+ if (!infoFile) throw new Error('clip cache must stay in build/. Remove symlinks from build/clips/.');+ const key = infoFile;+ if (!clipJobs.has(key)) {+ const job = (async () => {+ if (fs.existsSync(infoFile)) return { rel, info: JSON.parse(fs.readFileSync(infoFile, 'utf8')) };+ const temp = fs.mkdtempSync(path.join(path.dirname(infoFile), 'extract-'));+ try {+ const info = await extractClip(file, temp, fps);+ for (const name of fs.readdirSync(temp)) {+ const dest = clipCacheFile(projectDir, `${rel}/${name}`);+ if (!dest) throw new Error('clip cache must stay in build/. Remove symlinks from build/clips/.');+ fs.renameSync(path.join(temp, name), dest);+ }+ fs.writeFileSync(infoFile, JSON.stringify(info));+ return { rel, info };+ } finally {+ fs.rmSync(temp, { recursive: true, force: true });+ }+ })();+ clipJobs.set(key, job);+ job.finally(() => clipJobs.delete(key)).catch(() => {});+ }+ return clipJobs.get(key);+}+// Uploads from the page may only land in build/ or out/: checked after the// path is normalized (so "build/../scenes.js" is refused), and again on the// real folder, and never onto a symlink.@@ -98,6 +150,7 @@ function isSymlink(p) {export function startServer({ projectDir, getState, port = 0 }) {const roughPath = resolveModule('roughjs/bundled/rough.esm.js');+ const clips = new Map();const server = http.createServer(async (req, res) => {try {// Only answer requests addressed to this machine. A web page on another@@ -156,6 +209,32 @@ export function startServer({ projectDir, getState, port = 0 }) {const state = await getState();return send(res, 200, JSON.stringify(state), TYPES['.json']);}+ if (['/__clip/info', '/__clip/frame', '/__clip/audio'].includes(p)) {+ const file = clipFile(projectDir, url.searchParams.get('path') || '');+ if (!file) return send(res, 404, 'clip not found. Put a video in assets/.');+ const state = await getState();+ const stat = fs.statSync(file);+ const stamp = `${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}:${state.config.fps}`;+ if (clips.get(file)?.stamp !== stamp) {+ const job = cachedClip(projectDir, file, state.config.fps);+ clips.set(file, { stamp, job });+ job.catch(() => { if (clips.get(file)?.job === job) clips.delete(file); });+ }+ const { rel, info } = await clips.get(file).job;+ if (p === '/__clip/info') return send(res, 200, JSON.stringify(info), TYPES['.json']);+ let name = 'audio.wav';+ if (p === '/__clip/frame') {+ const raw = url.searchParams.get('frame') || '0';+ if (!/^\d+$/.test(raw) || !Number.isSafeInteger(Number(raw))) return send(res, 400, 'frame must be a non-negative integer');+ name = `${String(Math.min(Number(raw), info.frames - 1)).padStart(8, '0')}.jpg`;+ } else if (!info.hasAudio) return send(res, 404, 'this clip has no sound');+ const cached = clipCacheFile(projectDir, `${rel}/${name}`);+ if (!cached || !fs.existsSync(cached) || !fs.statSync(cached).isFile()) return send(res, 404, 'clip frame or sound not found');+ res.writeHead(200, { 'Content-Type': TYPES[path.extname(name)], 'Cache-Control': 'no-store' });+ if (req.method === 'HEAD') return res.end();+ fs.createReadStream(cached).pipe(res);+ return;+ }let file = null;if (p === '/' || p === '/studio.html') file = path.join(ROOT, 'engine', 'studio.html');else if (p === '/frame.html') file = path.join(ROOT, 'engine', 'frame.html');diff --git a/test/clips-browser.test.js b/test/clips-browser.test.jsnew file mode 100644index 0000000..1b93b8a--- /dev/null+++ b/test/clips-browser.test.js@@ -0,0 +1,95 @@+import test from 'node:test';+import assert from 'node:assert/strict';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import { findChrome } from '../src/browser.js';+import { loadProject } from '../src/project.js';+import { withEngine } from '../src/pipeline.js';+import { readWav } from '../src/wav.js';+import { ffmpeg } from '../src/ffmpeg.js';++let chrome;+try { chrome = findChrome(); } catch {}+test('stills use exact clip frames in any order and reuse extracted frames', { skip: !chrome }, async () => {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'explainroo-clips-'));+ try {+ fs.mkdirSync(path.join(dir, 'assets'));+ await ffmpeg(['-loglevel', 'error', '-f', 'lavfi', '-i', 'color=red:s=160x90:r=30:d=1', '-f', 'lavfi', '-i', 'color=lime:s=160x90:r=30:d=1', '-f', 'lavfi', '-i', 'color=blue:s=160x90:r=30:d=1', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=3', '-filter_complex', '[0:v][1:v][2:v]concat=n=3:v=1:a=0[v]', '-map', '[v]', '-map', '3:a', '-c:a', 'aac', '-shortest', '-c:v', 'libx264', '-threads', '1', path.join(dir, 'assets/colors.mp4')]);+ fs.writeFileSync(path.join(dir, 'video.json'), JSON.stringify({ theme: 'clean', captions: false, watermark: false, music: false, sfx: false }));+ fs.writeFileSync(path.join(dir, 'script.md'), '## demo\n');+ fs.writeFileSync(path.join(dir, 'scenes.js'), `export default { demo(s) {+ s.clip('assets/colors.mp4', { w: 1200, at: 0.5, enter: 'none', shadow: false, radius: 0, volume: 0.6 });+ } };`);+ const project = loadProject(dir);+ const timeline = { pace: 1, duration: 6, frames: 180, fps: 30, width: 1920, height: 1080, transitionSeconds: 0,+ scenes: [{ id: 'demo', index: 0, start: 0, dur: 6, words: [], marks: {}, voice: null, lead: 0, transition: 'none' }] };+ let stamp;+ for (let run = 0; run < 2; run++) {+ await withEngine(project, timeline, { scale: 0.25 }, async (page) => {+ const samples = await page.evaluate(async () => {+ const times = [2.75, 0.75, 1.75, 5, 1.75, 0.25];+ await window.explainroo.stills(times.map((T, i) => ({ T, path: `out/stills/${i}.png` })));+ const out = [];+ for (let i = 0; i < times.length; i++) {+ const img = new Image();+ img.src = `/project/out/stills/${i}.png`;+ await img.decode();+ const canvas = document.createElement('canvas');+ canvas.width = img.width; canvas.height = img.height;+ const ctx = canvas.getContext('2d'); ctx.drawImage(img, 0, 0);+ out.push([...ctx.getImageData(img.width / 2, img.height / 2, 1, 1).data].slice(0, 3));+ }+ return out;+ });+ for (const [i, channel] of [[0, 2], [1, 0], [2, 1], [3, 2], [4, 1]]) {+ assert.ok(samples[i][channel] > 240, JSON.stringify(samples));+ assert.ok(samples[i].filter((_, c) => c !== channel).every((v) => v < 15), JSON.stringify(samples));+ }+ assert.ok(samples[5].every((v) => v > 220));+ const issues = await page.evaluate(() => window.explainroo.check());+ assert.ok(issues.some((i) => i.level === 'hint' && /freezes.*3.50s.*6.00s/.test(i.message)));+ if (run === 0) {+ await page.evaluate(() => window.explainroo.soundtrack('build/clip-mix.wav'));+ const { samples, sampleRate } = readWav(path.join(dir, 'build/clip-mix.wav'));+ const peak = (a, b) => samples.subarray(a * sampleRate, b * sampleRate).reduce((m, x) => Math.max(m, Math.abs(x)), 0);+ assert.equal(peak(0.1, 0.4), 0);+ assert.ok(peak(0.6, 0.8) > 0.03);+ assert.equal(peak(4, 5), 0);+ }+ const result = await page.evaluate(() => window.explainroo.frames([60, 30, 60], 0.9));+ assert.equal(result[0], result[2]);+ const diagnostics = await page.evaluate(async () => {+ const { boot } = await import('/engine/runtime.js');+ const { engine } = await boot({ canvas: document.createElement('canvas'), scale: 0.25 });+ engine.scenesModule = { demo(s) { s.clip('assets/colors.mp4', { speed: 2, volume: 0.6 }); } };+ const issues = engine.check();+ engine.clipFrames.close();+ return issues;+ });+ assert.ok(diagnostics.some((i) => i.level === 'warn' && /stays silent at speed 2/.test(i.message)));+ });+ const hashes = fs.readdirSync(path.join(dir, 'build/clips'));+ assert.equal(hashes.length, 1);+ const first = path.join(dir, 'build/clips', hashes[0], '00000000.jpg');+ if (run === 0) stamp = fs.statSync(first).mtimeMs;+ else assert.equal(fs.statSync(first).mtimeMs, stamp);+ }+ await withEngine(project, timeline, {}, async (page, { server }) => {+ await page.goto(server.url);+ await page.waitForFunction(() => document.querySelector('#c').width > 0 && document.querySelector('#meta').textContent);+ await page.keyboard.press('ArrowRight');+ await page.waitForFunction(() => {+ const c = document.querySelector('#c');+ const pixel = c.getContext('2d').getImageData(c.width / 2, c.height / 2, 1, 1).data;+ return pixel[0] > 240 && pixel[1] < 15;+ });+ await page.keyboard.press('ArrowRight');+ await page.waitForFunction(() => {+ const c = document.querySelector('#c');+ const pixel = c.getContext('2d').getImageData(c.width / 2, c.height / 2, 1, 1).data;+ return pixel[1] > 240 && pixel[0] < 15;+ });+ });+ } finally { fs.rmSync(dir, { recursive: true, force: true }); }+});diff --git a/test/clips.test.js b/test/clips.test.jsnew file mode 100644index 0000000..ed68ed5--- /dev/null+++ b/test/clips.test.js@@ -0,0 +1,67 @@+import test from 'node:test';+import assert from 'node:assert/strict';+import { clipFrame, ClipFrames } from '../engine/clips.js';+import { renderSoundtrack } from '../engine/audio/soundtrack.js';++const info = { fps: 30, frames: 90, duration: 3 };++test('clip frames follow at, from and speed, and hold the last frame', () => {+ assert.equal(clipFrame(info, 1, { at: 2 }), null);+ assert.equal(clipFrame(info, 2, { at: 2 }), 0);+ assert.equal(clipFrame(info, 2.5, { at: 2, from: 1, speed: 2 }), 60);+ assert.equal(clipFrame(info, 3, { at: 2, speed: 0.25 }), 7);+ assert.equal(clipFrame(info, 10), 89);+ assert.equal(clipFrame(info, 0, { from: 10 }), 89);+ assert.equal(clipFrame(info, 1 / 30), 1);+});++test('looping wraps through the whole clip, including from', () => {+ assert.equal(clipFrame(info, 3, { loop: true }), 0);+ assert.equal(clipFrame(info, 1, { from: 2.5, loop: true }), 15);+ assert.equal(clipFrame(info, 4, { at: 1, from: 1, speed: 2, loop: true }), 30);+});++test('clip audio starts at its offset, loops, stops at out, and is silent at other speeds', async () => {+ const sr = 1000;+ const data = Float32Array.from({ length: sr }, (_, i) => i < 500 ? 0.1 : 0.2);+ const clip = { data, sampleRate: sr, start: 0.5, end: 2.5, duration: 1, from: 0.5, loop: true, volume: 0.6, speed: 1 };+ const mix = await renderSoundtrack({ duration: 3, sampleRate: sr, clips: [clip] });+ const L = mix.getChannelData(0);+ assert.equal(L[400], 0);+ assert.ok(Math.abs(L[600] - 0.12) < 1e-6);+ assert.ok(Math.abs(L[1100] - 0.06) < 1e-6);+ assert.ok(Math.abs(L[1600] - 0.12) < 1e-6);+ assert.equal(L[2600], 0);+ const silent = await renderSoundtrack({ duration: 3, sampleRate: sr, clips: [{ ...clip, speed: 2 }] });+ assert.ok(silent.getChannelData(0).every((v) => v === 0));+});++test('decoded frame cache keeps requested frames and closes old bitmaps', () => {+ const cache = new ClipFrames();+ const closed = [];+ for (let i = 0; i < 50; i++) cache.frames.set(String(i), { bitmap: { close: () => closed.push(i) } });+ cache.trim(new Set(['0', '1']));+ assert.equal(cache.frames.size, 24);+ assert.ok(cache.frames.has('0') && cache.frames.has('1'));+ assert.equal(closed.length, 26);+ cache.close();+ assert.equal(cache.frames.size, 0);+ assert.equal(closed.length, 50);+});++test('clip audio stops at the clip end and handles an entrance before the scene', async () => {+ const sr = 1000;+ const data = new Float32Array(sr).fill(0.1);+ const clip = { data, sampleRate: sr, start: -0.25, end: 2, duration: 1, from: 0.25, loop: false, volume: 0.6, speed: 1 };+ const mix = await renderSoundtrack({ duration: 3, sampleRate: sr, clips: [clip] });+ assert.ok(Math.abs(mix.getChannelData(0)[100] - 0.06) < 1e-6);+ assert.equal(mix.getChannelData(0)[600], 0);+});++test('music ducks under clip sound with the same envelope as narration', async () => {+ const opts = { duration: 3, sampleRate: 8000, music: { style: 'calm', seed: 42 } };+ const voice = await renderSoundtrack({ ...opts, voiceSpans: [{ start: 1, end: 2 }] });+ const clip = await renderSoundtrack({ ...opts, clips: [{ data: new Float32Array(8000), sampleRate: 8000, start: 1, end: 2, duration: 1, from: 0, volume: 0.6, speed: 1 }] });+ assert.deepEqual(clip.getChannelData(0), voice.getChannelData(0));+ assert.deepEqual(clip.getChannelData(1), voice.getChannelData(1));+});diff --git a/test/server.test.js b/test/server.test.jsindex a168161..8723178 100644--- a/test/server.test.js+++ b/test/server.test.js@@ -147,3 +147,77 @@ test('the preview server keeps secrets and sources safe', async () => {fs.rmSync(dir, { recursive: true, force: true });}});++test('clip endpoints only read video assets and keep the cache inside build', async () => {+ const { ffmpeg } = await import('../src/ffmpeg.js');+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'explainroo-clip-server-'));+ const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'explainroo-clip-outside-'));+ fs.mkdirSync(path.join(dir, 'assets'));+ const video = path.join(dir, 'assets/ok.mp4');+ await ffmpeg(['-loglevel', 'error', '-f', 'lavfi', '-i', 'color=red:s=80x60:r=30:d=0.2', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=0.2', '-shortest', '-c:v', 'libx264', '-threads', '1', '-c:a', 'aac', video]);+ fs.copyFileSync(video, path.join(outside, 'away.mp4'));+ fs.copyFileSync(video, path.join(dir, 'root.mp4'));+ fs.copyFileSync(video, path.join(dir, 'assets/.private.mp4'));+ fs.mkdirSync(path.join(dir, 'assets/.hidden'));+ fs.copyFileSync(video, path.join(dir, 'assets/.hidden/ok.mp4'));+ fs.writeFileSync(path.join(dir, 'assets/notes.md'), 'secret');+ fs.symlinkSync(path.join(outside, 'away.mp4'), path.join(dir, 'assets/away.mp4'));+ fs.symlinkSync(path.join(dir, 'root.mp4'), path.join(dir, 'assets/root-link.mp4'));+ fs.symlinkSync(path.join(dir, 'assets/.private.mp4'), path.join(dir, 'assets/hidden-link.mp4'));+ fs.symlinkSync(path.join(dir, 'assets/notes.md'), path.join(dir, 'assets/notes.mp4'));+ const server = await startServer({ projectDir: dir, getState: () => ({ config: { fps: 30 } }) });+ try {+ for (const endpoint of ['info', 'frame', 'audio']) {+ const url = (p) => `/__clip/${endpoint}?path=${encodeURIComponent(p)}`;+ for (const p of ['root.mp4', 'assets/../root.mp4', 'assets/missing.mp4', 'assets/notes.md', 'assets/ok.avi', 'assets/.private.mp4', 'assets/.hidden/ok.mp4', 'assets/away.mp4', 'assets/root-link.mp4', 'assets/hidden-link.mp4', 'assets/notes.mp4', '../away.mp4', video]) {+ assert.equal((await request(server.port, { path: url(p) })).status, 404, `${endpoint}: ${p}`);+ }+ assert.equal((await request(server.port, { path: url('assets/ok.mp4'), host: 'evil.example' })).status, 403);+ assert.equal((await request(server.port, { path: url('assets/ok.mp4'), origin: 'http://evil.example' })).status, 403);+ assert.equal((await request(server.port, { path: url('assets/ok.mp4'), method: 'POST' })).status, 405);+ }+ for (const [ext, mime] of [['mp4', 'video/mp4'], ['mov', 'video/quicktime'], ['webm', 'video/webm'], ['m4v', 'video/mp4']]) {+ fs.copyFileSync(video, path.join(dir, `assets/type.${ext}`));+ const response = await fetch(`${server.url}/project/assets/type.${ext}`);+ assert.equal(response.status, 200);+ assert.equal(response.headers.get('content-type'), mime);+ await response.arrayBuffer();+ }+ const responses = await Promise.all(Array.from({ length: 4 }, () => request(server.port, { path: '/__clip/info?path=assets/ok.mp4' })));+ assert.ok(responses.every((r) => r.status === 200));+ const info = JSON.parse(responses[0].body);+ assert.deepEqual(info, { frames: 6, fps: 30, width: 80, height: 60, duration: 0.2, hasAudio: true });+ assert.equal(fs.readdirSync(path.join(dir, 'build/clips')).length, 1);+ const frame = await fetch(`${server.url}/__clip/frame?path=assets/ok.mp4&frame=999`);+ assert.equal(frame.headers.get('content-type'), 'image/jpeg');+ const last = await fetch(`${server.url}/__clip/frame?path=assets/ok.mp4&frame=5`);+ assert.deepEqual(Buffer.from(await frame.arrayBuffer()), Buffer.from(await last.arrayBuffer()));+ const audio = await fetch(`${server.url}/__clip/audio?path=assets/ok.mp4`);+ assert.equal(audio.headers.get('content-type'), 'audio/wav');+ assert.equal(Buffer.from(await audio.arrayBuffer()).toString('ascii', 0, 4), 'RIFF');+ for (const frame of ['-1', '1.2', 'foo', '9007199254740992']) {+ assert.equal((await request(server.port, { path: `/__clip/frame?path=assets/ok.mp4&frame=${frame}` })).status, 400);+ }+ const silent = path.join(dir, 'assets/silent.mp4');+ await ffmpeg(['-loglevel', 'error', '-i', video, '-an', '-c:v', 'copy', silent]);+ const silentInfo = await request(server.port, { path: '/__clip/info?path=assets/silent.mp4' });+ assert.equal(JSON.parse(silentInfo.body).hasAudio, false);+ assert.equal((await request(server.port, { path: '/__clip/audio?path=assets/silent.mp4' })).status, 404);+ const hash = fs.readdirSync(path.join(dir, 'build/clips')).find((hash) => fs.existsSync(path.join(dir, 'build/clips', hash, 'audio.wav')));+ const cached = path.join(dir, 'build/clips', hash, '00000000.jpg');+ fs.unlinkSync(cached);+ fs.symlinkSync(path.join(outside, 'away.mp4'), cached);+ assert.equal((await request(server.port, { path: '/__clip/frame?path=assets/ok.mp4' })).status, 404);+ fs.rmSync(path.join(dir, 'build/clips'), { recursive: true });+ fs.symlinkSync(outside, path.join(dir, 'build/clips'));+ const other = await startServer({ projectDir: dir, getState: () => ({ config: { fps: 30 } }) });+ try {+ assert.equal((await request(other.port, { path: '/__clip/info?path=assets/ok.mp4' })).status, 500);+ assert.deepEqual(fs.readdirSync(outside), ['away.mp4']);+ } finally { await other.close(); }+ } finally {+ await server.close();+ fs.rmSync(dir, { recursive: true, force: true });+ fs.rmSync(outside, { recursive: true, force: true });+ }+});