Skip to content

Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.

Show video clips in scenes with s.clip (#8)

Vincent Schmalbachf4db9059dead
diff --git a/AGENTS.md b/AGENTS.md
index 8271283..b8b0bb0 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -347,6 +347,9 @@ A mouse pointer clicks through the screens and types into the fields, and the
camera zooms in on what the voice talks about. `examples/unspar-demo` is a
complete one.
+You can also show a real screen recording with `s.clip` in a `browser`
+frame. Use `s.camera` to zoom in on the detail the voice mentions.
+
**Keep it true.** A demo is advertising, so everything in it has to be right.
- Take the wording from the app itself: its templates and language files
@@ -788,7 +791,7 @@ stops at the edge of that element. Options: `bend` (about -1 to 1), `head`
`labelOffset`, `gap`, `dashed`, `width`, `headSize`. `s.connect` does the
same.
-### Icons and images
+### Icons, images and video clips
```js
s.icon('database', { x: 1500, y: 540, size: 140, color: 'purple', at: 'database' });
@@ -807,6 +810,28 @@ Old Lucide names work too. Icon options: `size` (120), `color`, `weight` (2),
(`none`, `card`, `browser`, `window`, `phone`), `url` (for the browser frame),
`border`, `shadow: false`, `kenburns: true` for a slow zoom.
+`s.clip(src, options)` draws a video from `assets/`. It takes the same
+layout, frame and timing options as `s.image` and returns the same box.
+Use `.mp4`, `.mov`, `.webm` or `.m4v` files.
+
+```js
+s.clip('assets/signup.mp4', { w: 1200, frame: 'browser', url: 'app.example.com', at: 'signup' });
+s.clip('assets/demo.mov', { x: 1400, y: 540, h: 600, from: 12.5, speed: 1.5, at: '#demo', out: 9 });
+```
+
+- `from` (0) is the second in the clip where playback starts.
+- `speed` (1) goes from 0.25 to 4. Playback starts at `at`.
+- `loop` (false) repeats the whole clip. Otherwise it holds the last frame.
+ `check` gives a hint when that freeze lasts more than a second.
+- `volume` (0) goes from 0 to 1. Sound plays only at `speed: 1`.
+ It stops at `out`, the clip's end or the scene's end. Loops repeat the
+ sound too. Music ducks under clip sound as it does under the voice.
+
+Frames are extracted once at the project's fps and saved in `build/clips/`.
+They are at most 1920 pixels wide. Renders, stills and sheets use these
+frames. The preview may show a nearby frame while the right one loads.
+`s.video` still holds the whole video's duration, frames, fps and size.
+
### Pointing things out
```js
@@ -941,6 +966,11 @@ font roles are `display`, `body`, `hand` and `mono`.
- Colored text in `paper` and `clean`. Green, orange and red are light enough
for shapes but often too light for text. `check` warns about low contrast.
Use `ink`, `accent` or a darker hex color for text.
+- Long video clips fill `build/clips/` with frames. Trim footage to the part
+ you show before you add it. Deleting `build/clips/` is safe.
+- Screen recordings often have a variable frame rate. explainroo converts
+ them to the project's fps, so a frame can repeat or drop. Record at a fixed
+ frame rate when the motion matters.
`check` looks at text only. It finds text off the frame, too close to the
edge (3.5%), too small, with low contrast (against the page, or against the
diff --git a/engine/audio/soundtrack.js b/engine/audio/soundtrack.js
index e333bc1..c90ee43 100644
--- a/engine/audio/soundtrack.js
+++ b/engine/audio/soundtrack.js
@@ -126,6 +126,7 @@ export async function renderSoundtrack(opts = {}) {
sampleRate = 48000,
voice = [],
voiceSpans = [],
+ clips = [],
music = null,
sfx = [],
scenes = [],
@@ -157,13 +158,32 @@ export async function renderSoundtrack(opts = {}) {
}
}
+ const clipSpans = [];
+ for (const clip of clips) {
+ if (clip.speed !== 1 || clip.volume <= 0) continue;
+ const data = await loadVoice(clip, sr, cache);
+ const clipLength = Math.max(1, Math.round(clip.duration * sr));
+ const start = Math.round(clip.start * sr);
+ const end = Math.min(length, Math.round(clip.end * sr));
+ const offset = Math.round(clip.from * sr);
+ for (let i = Math.max(0, start); i < end; i++) {
+ let source = offset + i - start;
+ if (clip.loop) source %= clipLength;
+ if (source >= clipLength) break;
+ const sample = (data[source] || 0) * clip.volume;
+ L[i] += sample;
+ R[i] += sample;
+ }
+ if (end > Math.max(0, start)) clipSpans.push({ start: Math.max(0, clip.start), end: end / sr });
+ }
+
// music: glue compressor, level with fades, ducked under narration
if (plan) {
const { synth } = createMusic({ ...music, style: plan.style, seed: plan.seed, bpm: plan.bpm, key: plan.keyRoot, duration, scenes, sampleRate: sr });
const volume = Math.max(0, Math.min(2, music.volume == null ? 0.5 : Number(music.volume)));
renderInto(synth, L, R, {
compressor: new Compressor(sr, { threshold: -18, ratio: 2, knee: 10, attack: 0.02, release: 0.3 }),
- gains: [levelEnv(dbToGain(MUSIC_BASE_DB + plan.mixDb) * (volume / 0.5), duration), duckEnv(voiceSpans)],
+ gains: [levelEnv(dbToGain(MUSIC_BASE_DB + plan.mixDb) * (volume / 0.5), duration), duckEnv([...voiceSpans, ...clipSpans])],
});
}
diff --git a/engine/clips.js b/engine/clips.js
new file mode 100644
index 0000000..eec8db6
--- /dev/null
+++ b/engine/clips.js
@@ -0,0 +1,84 @@
+export const VIDEO_FILE = /\.(mp4|mov|webm|m4v)$/i;
+
+export function clipFrame(info, t, { at = 0, from = 0, speed = 1, loop = false } = {}) {
+ if (t < at) return null;
+ let time = from + (t - at) * speed;
+ if (loop) time = ((time % info.duration) + info.duration) % info.duration;
+ return Math.max(0, Math.min(info.frames - 1, Math.floor(time * info.fps + 1e-8)));
+}
+
+export function clipURL(kind, src, frame) {
+ return `/__clip/${kind}?path=${encodeURIComponent(src)}${frame === undefined ? '' : '&frame=' + frame}`;
+}
+
+export async function loadClips(assets) {
+ const clips = new Map();
+ await Promise.all(assets.filter((a) => VIDEO_FILE.test(a)).map(async (src) => {
+ const res = await fetch(clipURL('info', src));
+ if (!res.ok) throw new Error(`clip "${src}" could not be loaded. Put a video in the project's assets/ folder.`);
+ clips.set(src, { src, ...await res.json() });
+ }));
+ return clips;
+}
+
+export class ClipFrames {
+ constructor() {
+ this.frames = new Map();
+ this.pending = new Map();
+ this.errors = new Map();
+ this.closed = false;
+ }
+
+ key(clip, frame) { return `${clip.src}|${frame}`; }
+
+ async load(clip, frame) {
+ const key = this.key(clip, frame);
+ if (this.frames.has(key)) return;
+ if (!this.pending.has(key)) {
+ const job = (async () => {
+ const res = await fetch(clipURL('frame', clip.src, frame));
+ if (!res.ok) throw new Error(`clip "${clip.src}" frame ${frame} could not be loaded`);
+ const bitmap = await createImageBitmap(await res.blob());
+ if (this.closed) { bitmap.close(); return; }
+ this.frames.set(key, { clip, frame, bitmap });
+ })();
+ this.pending.set(key, job);
+ job.finally(() => this.pending.delete(key)).catch(() => {});
+ }
+ return this.pending.get(key);
+ }
+
+ get(clip, frame) {
+ const key = this.key(clip, frame);
+ const entry = this.frames.get(key);
+ if (entry) {
+ this.frames.delete(key);
+ this.frames.set(key, entry);
+ return entry.bitmap;
+ }
+ return null;
+ }
+
+ nearest(clip, frame) {
+ let best;
+ for (const entry of this.frames.values()) {
+ if (entry.clip.src === clip.src && (!best || Math.abs(entry.frame - frame) < Math.abs(best.frame - frame))) best = entry;
+ }
+ return best?.bitmap;
+ }
+
+ trim(keep = new Set()) {
+ for (const [key, entry] of this.frames) {
+ if (this.frames.size <= Math.max(24, keep.size)) break;
+ if (keep.has(key)) continue;
+ entry.bitmap.close();
+ this.frames.delete(key);
+ }
+ }
+
+ close() {
+ this.closed = true;
+ for (const entry of this.frames.values()) entry.bitmap.close();
+ this.frames.clear();
+ }
+}
diff --git a/engine/frame.html b/engine/frame.html
index 6b8968a..5079e2e 100644
--- a/engine/frame.html
+++ b/engine/frame.html
@@ -26,10 +26,11 @@ window.explainrooReady = (async () => {
info() {
return { duration: state.timeline.duration, frames: state.timeline.frames, fps, width: canvas.width, height: canvas.height, scenes: state.timeline.scenes.map((s) => ({ id: s.id, start: s.start, dur: s.dur })) };
},
- frames(list, quality) {
+ async frames(list, quality) {
const out = [];
for (const f of list) {
try {
+ if (engine.clips.size) await engine.prepareFrame(f / fps);
engine.renderFrame(f / fps);
} catch (e) {
throw new Error(describe(e));
@@ -41,6 +42,7 @@ window.explainrooReady = (async () => {
async stills(list, { type = 'image/png', quality } = {}) {
for (const { T, path } of list) {
try {
+ if (engine.clips.size) await engine.prepareFrame(T);
engine.renderFrame(T);
} catch (e) {
throw new Error(describe(e));
@@ -65,6 +67,7 @@ window.explainrooReady = (async () => {
g.textBaseline = 'middle';
for (let i = 0; i < times.length; i++) {
try {
+ if (engine.clips.size) await engine.prepareFrame(times[i]);
engine.renderFrame(times[i]);
} catch (e) {
throw new Error(describe(e));
diff --git a/engine/runtime.js b/engine/runtime.js
index 7dae296..f2f2f10 100644
--- a/engine/runtime.js
+++ b/engine/runtime.js
@@ -8,6 +8,7 @@ import { drawTransition } from './transitions.js';
import { buildPhrases, drawCaptions } from './captions.js';
import { hashStr, mulberry32, suggest, contrast, opaque, withAlpha } from './util.js';
import { layoutAreas } from './layout.js';
+import { loadClips, ClipFrames } from './clips.js';
import { UI } from './ui.js';
import { playbackTextIssue } from './readability.js';
@@ -59,7 +60,7 @@ async function loadImages(assets) {
}
export class Engine {
- constructor({ canvas, state, scenes, icons, images, scale = 1, sfxNames = [] }) {
+ constructor({ canvas, state, scenes, icons, images, clips = new Map(), scale = 1, sfxNames = [] }) {
const { config, timeline } = state;
this.state = state;
this.config = config;
@@ -79,6 +80,9 @@ export class Engine {
this.icons = icons;
this.iconCache = new Map();
this.images = images;
+ this.clips = clips;
+ this.clipFrames = new ClipFrames();
+ this.clipProbe = null;
this.sfxNames = sfxNames;
this.sfxMode = config.sfx;
this.recorder = null;
@@ -114,6 +118,43 @@ export class Engine {
return img;
}
+ clip(src) {
+ const key = src.replace(/^\.?\//, '');
+ const clip = this.clips.get(key) || this.clips.get('assets/' + key);
+ if (!clip) throw new SceneError(`clip "${src}" not found. Put an MP4, MOV, WebM or M4V in the project's assets/ folder. Available: ${[...this.clips.keys()].join(', ') || 'none'}`);
+ return clip;
+ }
+
+ clipImage(clip, frame) {
+ if (this.recorder) return null;
+ const key = this.clipFrames.key(clip, frame);
+ if (this.clipProbe) {
+ this.clipProbe.set(key, { clip, frame });
+ return null;
+ }
+ const img = this.clipFrames.get(clip, frame);
+ if (img) return img;
+ if (this.clipFrames.errors.has(key)) throw this.clipFrames.errors.get(key);
+ this.clipFrames.load(clip, frame).then(() => {
+ this.clipFrames.trim();
+ this.onClipFrame?.();
+ }).catch((e) => {
+ this.clipFrames.errors.set(key, e);
+ this.onClipFrame?.();
+ });
+ return this.clipFrames.nearest(clip, frame);
+ }
+
+ async prepareFrame(T) {
+ if (!this.clips.size) return;
+ const needed = new Map();
+ this.clipProbe = needed;
+ try { this.renderFrame(T); }
+ finally { this.clipProbe = null; }
+ await Promise.all([...needed.values()].map(({ clip, frame }) => this.clipFrames.load(clip, frame)));
+ this.clipFrames.trim(new Set(needed.keys()));
+ }
+
sceneAt(T) {
const list = this.timeline.scenes;
let i = list.length - 1;
@@ -245,8 +286,10 @@ export class Engine {
// Sound cues for the soundtrack: element entrances, explicit s.sfx() calls
// and scene transitions, thinned so effects never pile up.
sfxCues() {
- if (this.config.sfx === false) return { cues: [], errors: [] };
+ if (this.config.sfx === false && !this.clips.size) return { cues: [], clips: [], errors: [] };
const { events, errors } = this.collectEvents();
+ const clips = events.filter((e) => e.kind === 'clip' && e.volume > 0 && e.speed === 1 && e.hasAudio).map((e) => ({ ...e, start: e.T, end: e.T + e.end - e.at }));
+ if (this.config.sfx === false) return { cues: [], clips, errors };
const cues = [];
for (const e of events) {
if (!e.sfx) continue;
@@ -280,7 +323,7 @@ export class Engine {
last.set(c.name, c.t);
out.push(c);
}
- return { cues: out, errors };
+ return { cues: out, clips, errors };
}
// Samples every scene and reports layout problems.
@@ -374,6 +417,10 @@ export class Engine {
this.textLog = null;
const times = [...new Set(entrances.map((e) => Math.round(e.at * 100) / 100))].sort((a, b) => a - b);
for (const e of entrances) {
+ if (e.kind === 'clip') {
+ if (e.volume > 0 && e.speed !== 1) once(`clip-sound:${sc.id}:${e.key}`, () => add('warn', sc.id, e.at, `clip "${e.src}" stays silent at speed ${e.speed}; use speed: 1 for its sound`));
+ if (!e.loop && e.out - e.end > 1) once(`clip-freeze:${sc.id}:${e.key}`, () => add('hint', sc.id, e.end, `clip "${e.src}" freezes on its last frame from ${e.end.toFixed(2)}s until ${e.out.toFixed(2)}s; use loop: true or an earlier out`));
+ }
if (e.at > sc.dur) once(`late:${sc.id}:${e.key}`, () => add('error', sc.id, e.at, `a ${e.kind} is set to appear at ${e.at.toFixed(2)}s but the scene ends at ${sc.dur.toFixed(2)}s`));
}
if (!times.length) {
@@ -400,7 +447,7 @@ export class Engine {
export async function boot({ canvas, scale = 1, version = '' }) {
const state = await fetchState();
const theme = getTheme(state.config.theme, state.config.look);
- const [icons, images, scenesMod, audio, iconTags] = await Promise.all([
+ const [icons, images, scenesMod, audio, iconTags, clips] = await Promise.all([
fetch('/engine/icons/lucide.json').then((r) => r.json()).then((j) => {
// Old Lucide names keep working.
for (const [alias, target] of Object.entries(j.aliases || {})) if (!j.icons[alias]) j.icons[alias] = j.icons[target];
@@ -410,11 +457,12 @@ export async function boot({ canvas, scale = 1, version = '' }) {
import(`/project/scenes.js?v=${encodeURIComponent(version || state.version)}`),
import('/engine/audio/soundtrack.js').catch((e) => ({ SFX: {}, loadError: e })),
fetch('/engine/icons/lucide-tags.json').then((r) => r.json()),
+ loadClips(state.assets),
loadFonts(theme, state.config),
]);
const scenes = scenesMod.default;
if (!scenes || typeof scenes !== 'object') throw new Error('scenes.js must "export default { sceneId(s) { ... } }"');
- const engine = new Engine({ canvas, state, scenes, icons, images, scale, sfxNames: Object.keys(audio.SFX) });
+ const engine = new Engine({ canvas, state, scenes, icons, images, clips, scale, sfxNames: Object.keys(audio.SFX) });
engine.iconTags = iconTags;
return { engine, audio, state };
}
@@ -423,7 +471,7 @@ export async function boot({ canvas, scale = 1, version = '' }) {
export async function renderAudio(engine, audio, { sampleRate = 48000 } = {}) {
if (audio.loadError) throw new Error(`audio engine failed to load: ${audio.loadError.message}`);
const { timeline, config } = engine.state;
- const { cues, errors } = engine.sfxCues();
+ const { cues, clips, errors } = engine.sfxCues();
const buffer = await audio.renderSoundtrack({
duration: timeline.duration,
sampleRate,
@@ -431,6 +479,7 @@ export async function renderAudio(engine, audio, { sampleRate = 48000 } = {}) {
voiceSpans: timeline.voiceSpans,
music: config.music,
sfx: cues,
+ clips,
scenes: timeline.scenes.map((s) => ({ start: s.start, dur: s.dur })),
});
return { buffer, cues, errors };
diff --git a/engine/stage.js b/engine/stage.js
index 5b1e3e4..2fa9c07 100644
--- a/engine/stage.js
+++ b/engine/stage.js
@@ -6,6 +6,7 @@ import { clamp, lerp, ease, easeFn, rand01, noise1, hashStr, mulberry32, suggest
import { colorOf, tintOf, fontString } from './themes.js';
import { scalePath, quadD, polyD, pathLength, pointAt } from './pathdata.js';
import { layoutText, lineStartX } from './text.js';
+import { clipFrame, clipURL } from './clips.js';
import { highlightLine } from './code.js';
const DEFAULT_DUR = { draw: 0.9, write: 0.8, pop: 0.45, rise: 0.55, fade: 0.5, zoom: 0.6, drop: 0.55, grow: 1.0, none: 0.001, 'slide-left': 0.55, 'slide-right': 0.55, 'slide-up': 0.55, 'slide-down': 0.55 };
@@ -852,13 +853,34 @@ export class Stage {
}
image(src, o = {}) {
- const th = this.theme;
const img = this.engine.image(src);
- const ratio = img.naturalWidth / img.naturalHeight;
+ return this._picture(src, img, img.naturalWidth, img.naturalHeight, o);
+ }
+
+ clip(src, o = {}) {
+ const clip = this.engine.clip(src);
+ const from = o.from ?? 0;
+ const speed = o.speed ?? 1;
+ const volume = o.volume ?? 0;
+ for (const [name, value, lo, hi] of [['from', from, 0, Infinity], ['speed', speed, 0.25, 4], ['volume', volume, 0, 1]]) {
+ if (!Number.isFinite(value) || value < lo || value > hi) throw new SceneError(`clip ${name} must be ${name === 'from' ? 'a non-negative number' : `from ${lo} to ${hi}`}`);
+ }
+ const at = this.time(o.at ?? 0);
+ const out = o.out === undefined ? this.dur : Math.min(this.dur, this.time(o.out));
+ const end = o.loop ? out : Math.min(out, at + Math.max(0, clip.duration - from) / speed);
+ const n = this._clipSeq = (this._clipSeq || 0) + 1;
+ if (this.engine.recorder) this.engine.recorder.push({ kind: 'clip', key: `clip|${at}|${o.id ?? n}|${src}`, at, end, out, from, speed, loop: !!o.loop, volume, hasAudio: clip.hasAudio, duration: clip.duration, src: clip.src, url: clipURL('audio', clip.src) });
+ const frame = clipFrame(clip, this.t, { at, from, speed, loop: o.loop });
+ return this._picture(src, () => frame === null ? null : this.engine.clipImage(clip, frame), clip.width, clip.height, o);
+ }
+
+ _picture(src, source, naturalWidth, naturalHeight, o) {
+ const th = this.theme;
+ const ratio = naturalWidth / naturalHeight;
let w = o.w;
let h = o.h;
if (!w && !h) {
- w = Math.min(this._w(0.6), img.naturalWidth);
+ w = Math.min(this._w(0.6), naturalWidth);
h = w / ratio;
} else if (!h) h = w / ratio;
else if (!w) w = h * ratio;
@@ -924,7 +946,12 @@ export class Stage {
}
dw *= kb;
dh *= kb;
- ctx.drawImage(img, -dw / 2, iy - dh / 2, dw, dh);
+ const img = typeof source === 'function' ? source() : source;
+ if (img) ctx.drawImage(img, -dw / 2, iy - dh / 2, dw, dh);
+ else {
+ ctx.fillStyle = th.surface;
+ ctx.fillRect(-w / 2, iy - h / 2, w, h);
+ }
ctx.restore();
if (o.border) {
this.pen.draw(ctx, { kind: 'rect', w, h, r }, { stroke: colorOf(th, o.border === true ? 'ink' : o.border), width: th.stroke.width, seed: this._seed('imgborder', o, src) }, { stroke: 1, fill: 1 });
diff --git a/engine/studio.html b/engine/studio.html
index 3e2d5a1..6a2a8f3 100644
--- a/engine/studio.html
+++ b/engine/studio.html
@@ -108,7 +108,12 @@ function buildUi() {
async function load(version) {
const scale = Math.min(1, 1280 / (await fetchState()).config.width);
const r = await boot({ canvas, scale, version });
+ if (engine) {
+ engine.onClipFrame = null;
+ engine.clipFrames.close();
+ }
engine = r.engine;
+ engine.onClipFrame = draw;
audioMod = r.audio;
state = r.state;
buildUi();
diff --git a/src/clips.js b/src/clips.js
new file mode 100644
index 0000000..998c517
--- /dev/null
+++ b/src/clips.js
@@ -0,0 +1,19 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { ffmpeg, probe } from './ffmpeg.js';
+
+export const VIDEO_FILE = /\.(mp4|mov|webm|m4v)$/i;
+
+export async function extractClip(file, dir, fps) {
+ const input = await probe(file);
+ const video = input.streams.find((s) => s.codec_type === 'video');
+ if (!video) throw new Error('this clip has no video. Use an MP4, MOV, WebM or M4V with video.');
+ await ffmpeg(['-loglevel', 'error', '-i', file, '-map', '0:v:0', '-vf', `fps=${fps},scale=min(1920\\,iw):-1`, '-q:v', '2', '-threads', '1', '-start_number', '0', path.join(dir, '%08d.jpg')]);
+ const frames = fs.readdirSync(dir).filter((n) => n.endsWith('.jpg')).length;
+ if (!frames) throw new Error('this clip has no frames. Use a video with at least one frame.');
+ const frame = await probe(path.join(dir, '00000000.jpg'));
+ const { width, height } = frame.streams[0];
+ const hasAudio = input.streams.some((s) => s.codec_type === 'audio');
+ if (hasAudio) await ffmpeg(['-loglevel', 'error', '-i', file, '-map', '0:a:0', '-vn', '-ar', '48000', '-ac', '1', '-c:a', 'pcm_s16le', path.join(dir, 'audio.wav')]);
+ return { frames, fps, width, height, duration: frames / fps, hasAudio };
+}
diff --git a/src/server.js b/src/server.js
index d004d8d..12705a4 100644
--- a/src/server.js
+++ b/src/server.js
@@ -3,7 +3,9 @@
import http from 'node:http';
import fs from 'node:fs';
import path from 'node:path';
+import crypto from 'node:crypto';
import { fileURLToPath } from 'node:url';
+import { extractClip, VIDEO_FILE } from './clips.js';
export const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
@@ -26,6 +28,9 @@ const TYPES = {
'.gif': 'image/gif',
'.svg': 'image/svg+xml',
'.mp4': 'video/mp4',
+ '.mov': 'video/quicktime',
+ '.webm': 'video/webm',
+ '.m4v': 'video/mp4',
'.txt': 'text/plain; charset=utf-8',
};
@@ -42,7 +47,7 @@ function safeJoin(base, rel) {
// Project files the engine page may read: scenes.js and the modules it
// imports, images, fonts, audio and video. Never dotfiles such as .env, and
// no JSON, Markdown or text files, which can hold keys or private notes.
-const PROJECT_FILES = /\.(m?js|png|jpe?g|webp|gif|svg|wav|mp3|mp4|woff2?|ttf|otf)$/i;
+const PROJECT_FILES = /\.(m?js|png|jpe?g|webp|gif|svg|wav|mp3|mp4|mov|webm|m4v|woff2?|ttf|otf)$/i;
function allowedName(rel) {
const parts = rel.split(/[\\/]+/);
@@ -66,6 +71,53 @@ function projectFile(projectDir, rel) {
return allowedName(realRel) ? real : null;
}
+function clipFile(projectDir, rel) {
+ if (!rel.startsWith('assets/') || !VIDEO_FILE.test(rel)) return null;
+ const file = projectFile(projectDir, rel);
+ const assets = path.join(fs.realpathSync(projectDir), 'assets');
+ if (!file || !VIDEO_FILE.test(file) || !file.startsWith(assets + path.sep) || !fs.statSync(file).isFile()) return null;
+ return file;
+}
+
+function clipCacheFile(projectDir, rel) {
+ const file = uploadFile(projectDir, rel);
+ if (!file) return null;
+ const realRel = path.relative(fs.realpathSync(projectDir), fs.realpathSync(path.dirname(file)));
+ return realRel.split(path.sep).some((p) => p.startsWith('.')) ? null : file;
+}
+
+const clipJobs = new Map();
+
+async function cachedClip(projectDir, file, fps) {
+ const hash = crypto.createHash('sha256').update(JSON.stringify({ fps, width: 1920, version: 1 }));
+ for await (const chunk of fs.createReadStream(file)) hash.update(chunk);
+ const rel = `build/clips/${hash.digest('hex')}`;
+ const infoFile = clipCacheFile(projectDir, `${rel}/info.json`);
+ if (!infoFile) throw new Error('clip cache must stay in build/. Remove symlinks from build/clips/.');
+ const key = infoFile;
+ if (!clipJobs.has(key)) {
+ const job = (async () => {
+ if (fs.existsSync(infoFile)) return { rel, info: JSON.parse(fs.readFileSync(infoFile, 'utf8')) };
+ const temp = fs.mkdtempSync(path.join(path.dirname(infoFile), 'extract-'));
+ try {
+ const info = await extractClip(file, temp, fps);
+ for (const name of fs.readdirSync(temp)) {
+ const dest = clipCacheFile(projectDir, `${rel}/${name}`);
+ if (!dest) throw new Error('clip cache must stay in build/. Remove symlinks from build/clips/.');
+ fs.renameSync(path.join(temp, name), dest);
+ }
+ fs.writeFileSync(infoFile, JSON.stringify(info));
+ return { rel, info };
+ } finally {
+ fs.rmSync(temp, { recursive: true, force: true });
+ }
+ })();
+ clipJobs.set(key, job);
+ job.finally(() => clipJobs.delete(key)).catch(() => {});
+ }
+ return clipJobs.get(key);
+}
+
// Uploads from the page may only land in build/ or out/: checked after the
// path is normalized (so "build/../scenes.js" is refused), and again on the
// real folder, and never onto a symlink.
@@ -98,6 +150,7 @@ function isSymlink(p) {
export function startServer({ projectDir, getState, port = 0 }) {
const roughPath = resolveModule('roughjs/bundled/rough.esm.js');
+ const clips = new Map();
const server = http.createServer(async (req, res) => {
try {
// Only answer requests addressed to this machine. A web page on another
@@ -156,6 +209,32 @@ export function startServer({ projectDir, getState, port = 0 }) {
const state = await getState();
return send(res, 200, JSON.stringify(state), TYPES['.json']);
}
+ if (['/__clip/info', '/__clip/frame', '/__clip/audio'].includes(p)) {
+ const file = clipFile(projectDir, url.searchParams.get('path') || '');
+ if (!file) return send(res, 404, 'clip not found. Put a video in assets/.');
+ const state = await getState();
+ const stat = fs.statSync(file);
+ const stamp = `${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}:${state.config.fps}`;
+ if (clips.get(file)?.stamp !== stamp) {
+ const job = cachedClip(projectDir, file, state.config.fps);
+ clips.set(file, { stamp, job });
+ job.catch(() => { if (clips.get(file)?.job === job) clips.delete(file); });
+ }
+ const { rel, info } = await clips.get(file).job;
+ if (p === '/__clip/info') return send(res, 200, JSON.stringify(info), TYPES['.json']);
+ let name = 'audio.wav';
+ if (p === '/__clip/frame') {
+ const raw = url.searchParams.get('frame') || '0';
+ if (!/^\d+$/.test(raw) || !Number.isSafeInteger(Number(raw))) return send(res, 400, 'frame must be a non-negative integer');
+ name = `${String(Math.min(Number(raw), info.frames - 1)).padStart(8, '0')}.jpg`;
+ } else if (!info.hasAudio) return send(res, 404, 'this clip has no sound');
+ const cached = clipCacheFile(projectDir, `${rel}/${name}`);
+ if (!cached || !fs.existsSync(cached) || !fs.statSync(cached).isFile()) return send(res, 404, 'clip frame or sound not found');
+ res.writeHead(200, { 'Content-Type': TYPES[path.extname(name)], 'Cache-Control': 'no-store' });
+ if (req.method === 'HEAD') return res.end();
+ fs.createReadStream(cached).pipe(res);
+ return;
+ }
let file = null;
if (p === '/' || p === '/studio.html') file = path.join(ROOT, 'engine', 'studio.html');
else if (p === '/frame.html') file = path.join(ROOT, 'engine', 'frame.html');
diff --git a/test/clips-browser.test.js b/test/clips-browser.test.js
new file mode 100644
index 0000000..1b93b8a
--- /dev/null
+++ b/test/clips-browser.test.js
@@ -0,0 +1,95 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { findChrome } from '../src/browser.js';
+import { loadProject } from '../src/project.js';
+import { withEngine } from '../src/pipeline.js';
+import { readWav } from '../src/wav.js';
+import { ffmpeg } from '../src/ffmpeg.js';
+
+let chrome;
+try { chrome = findChrome(); } catch {}
+test('stills use exact clip frames in any order and reuse extracted frames', { skip: !chrome }, async () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'explainroo-clips-'));
+ try {
+ fs.mkdirSync(path.join(dir, 'assets'));
+ await ffmpeg(['-loglevel', 'error', '-f', 'lavfi', '-i', 'color=red:s=160x90:r=30:d=1', '-f', 'lavfi', '-i', 'color=lime:s=160x90:r=30:d=1', '-f', 'lavfi', '-i', 'color=blue:s=160x90:r=30:d=1', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=3', '-filter_complex', '[0:v][1:v][2:v]concat=n=3:v=1:a=0[v]', '-map', '[v]', '-map', '3:a', '-c:a', 'aac', '-shortest', '-c:v', 'libx264', '-threads', '1', path.join(dir, 'assets/colors.mp4')]);
+ fs.writeFileSync(path.join(dir, 'video.json'), JSON.stringify({ theme: 'clean', captions: false, watermark: false, music: false, sfx: false }));
+ fs.writeFileSync(path.join(dir, 'script.md'), '## demo\n');
+ fs.writeFileSync(path.join(dir, 'scenes.js'), `export default { demo(s) {
+ s.clip('assets/colors.mp4', { w: 1200, at: 0.5, enter: 'none', shadow: false, radius: 0, volume: 0.6 });
+ } };`);
+ const project = loadProject(dir);
+ const timeline = { pace: 1, duration: 6, frames: 180, fps: 30, width: 1920, height: 1080, transitionSeconds: 0,
+ scenes: [{ id: 'demo', index: 0, start: 0, dur: 6, words: [], marks: {}, voice: null, lead: 0, transition: 'none' }] };
+ let stamp;
+ for (let run = 0; run < 2; run++) {
+ await withEngine(project, timeline, { scale: 0.25 }, async (page) => {
+ const samples = await page.evaluate(async () => {
+ const times = [2.75, 0.75, 1.75, 5, 1.75, 0.25];
+ await window.explainroo.stills(times.map((T, i) => ({ T, path: `out/stills/${i}.png` })));
+ const out = [];
+ for (let i = 0; i < times.length; i++) {
+ const img = new Image();
+ img.src = `/project/out/stills/${i}.png`;
+ await img.decode();
+ const canvas = document.createElement('canvas');
+ canvas.width = img.width; canvas.height = img.height;
+ const ctx = canvas.getContext('2d'); ctx.drawImage(img, 0, 0);
+ out.push([...ctx.getImageData(img.width / 2, img.height / 2, 1, 1).data].slice(0, 3));
+ }
+ return out;
+ });
+ for (const [i, channel] of [[0, 2], [1, 0], [2, 1], [3, 2], [4, 1]]) {
+ assert.ok(samples[i][channel] > 240, JSON.stringify(samples));
+ assert.ok(samples[i].filter((_, c) => c !== channel).every((v) => v < 15), JSON.stringify(samples));
+ }
+ assert.ok(samples[5].every((v) => v > 220));
+ const issues = await page.evaluate(() => window.explainroo.check());
+ assert.ok(issues.some((i) => i.level === 'hint' && /freezes.*3.50s.*6.00s/.test(i.message)));
+ if (run === 0) {
+ await page.evaluate(() => window.explainroo.soundtrack('build/clip-mix.wav'));
+ const { samples, sampleRate } = readWav(path.join(dir, 'build/clip-mix.wav'));
+ const peak = (a, b) => samples.subarray(a * sampleRate, b * sampleRate).reduce((m, x) => Math.max(m, Math.abs(x)), 0);
+ assert.equal(peak(0.1, 0.4), 0);
+ assert.ok(peak(0.6, 0.8) > 0.03);
+ assert.equal(peak(4, 5), 0);
+ }
+ const result = await page.evaluate(() => window.explainroo.frames([60, 30, 60], 0.9));
+ assert.equal(result[0], result[2]);
+ const diagnostics = await page.evaluate(async () => {
+ const { boot } = await import('/engine/runtime.js');
+ const { engine } = await boot({ canvas: document.createElement('canvas'), scale: 0.25 });
+ engine.scenesModule = { demo(s) { s.clip('assets/colors.mp4', { speed: 2, volume: 0.6 }); } };
+ const issues = engine.check();
+ engine.clipFrames.close();
+ return issues;
+ });
+ assert.ok(diagnostics.some((i) => i.level === 'warn' && /stays silent at speed 2/.test(i.message)));
+ });
+ const hashes = fs.readdirSync(path.join(dir, 'build/clips'));
+ assert.equal(hashes.length, 1);
+ const first = path.join(dir, 'build/clips', hashes[0], '00000000.jpg');
+ if (run === 0) stamp = fs.statSync(first).mtimeMs;
+ else assert.equal(fs.statSync(first).mtimeMs, stamp);
+ }
+ await withEngine(project, timeline, {}, async (page, { server }) => {
+ await page.goto(server.url);
+ await page.waitForFunction(() => document.querySelector('#c').width > 0 && document.querySelector('#meta').textContent);
+ await page.keyboard.press('ArrowRight');
+ await page.waitForFunction(() => {
+ const c = document.querySelector('#c');
+ const pixel = c.getContext('2d').getImageData(c.width / 2, c.height / 2, 1, 1).data;
+ return pixel[0] > 240 && pixel[1] < 15;
+ });
+ await page.keyboard.press('ArrowRight');
+ await page.waitForFunction(() => {
+ const c = document.querySelector('#c');
+ const pixel = c.getContext('2d').getImageData(c.width / 2, c.height / 2, 1, 1).data;
+ return pixel[1] > 240 && pixel[0] < 15;
+ });
+ });
+ } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
diff --git a/test/clips.test.js b/test/clips.test.js
new file mode 100644
index 0000000..ed68ed5
--- /dev/null
+++ b/test/clips.test.js
@@ -0,0 +1,67 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { clipFrame, ClipFrames } from '../engine/clips.js';
+import { renderSoundtrack } from '../engine/audio/soundtrack.js';
+
+const info = { fps: 30, frames: 90, duration: 3 };
+
+test('clip frames follow at, from and speed, and hold the last frame', () => {
+ assert.equal(clipFrame(info, 1, { at: 2 }), null);
+ assert.equal(clipFrame(info, 2, { at: 2 }), 0);
+ assert.equal(clipFrame(info, 2.5, { at: 2, from: 1, speed: 2 }), 60);
+ assert.equal(clipFrame(info, 3, { at: 2, speed: 0.25 }), 7);
+ assert.equal(clipFrame(info, 10), 89);
+ assert.equal(clipFrame(info, 0, { from: 10 }), 89);
+ assert.equal(clipFrame(info, 1 / 30), 1);
+});
+
+test('looping wraps through the whole clip, including from', () => {
+ assert.equal(clipFrame(info, 3, { loop: true }), 0);
+ assert.equal(clipFrame(info, 1, { from: 2.5, loop: true }), 15);
+ assert.equal(clipFrame(info, 4, { at: 1, from: 1, speed: 2, loop: true }), 30);
+});
+
+test('clip audio starts at its offset, loops, stops at out, and is silent at other speeds', async () => {
+ const sr = 1000;
+ const data = Float32Array.from({ length: sr }, (_, i) => i < 500 ? 0.1 : 0.2);
+ const clip = { data, sampleRate: sr, start: 0.5, end: 2.5, duration: 1, from: 0.5, loop: true, volume: 0.6, speed: 1 };
+ const mix = await renderSoundtrack({ duration: 3, sampleRate: sr, clips: [clip] });
+ const L = mix.getChannelData(0);
+ assert.equal(L[400], 0);
+ assert.ok(Math.abs(L[600] - 0.12) < 1e-6);
+ assert.ok(Math.abs(L[1100] - 0.06) < 1e-6);
+ assert.ok(Math.abs(L[1600] - 0.12) < 1e-6);
+ assert.equal(L[2600], 0);
+ const silent = await renderSoundtrack({ duration: 3, sampleRate: sr, clips: [{ ...clip, speed: 2 }] });
+ assert.ok(silent.getChannelData(0).every((v) => v === 0));
+});
+
+test('decoded frame cache keeps requested frames and closes old bitmaps', () => {
+ const cache = new ClipFrames();
+ const closed = [];
+ for (let i = 0; i < 50; i++) cache.frames.set(String(i), { bitmap: { close: () => closed.push(i) } });
+ cache.trim(new Set(['0', '1']));
+ assert.equal(cache.frames.size, 24);
+ assert.ok(cache.frames.has('0') && cache.frames.has('1'));
+ assert.equal(closed.length, 26);
+ cache.close();
+ assert.equal(cache.frames.size, 0);
+ assert.equal(closed.length, 50);
+});
+
+test('clip audio stops at the clip end and handles an entrance before the scene', async () => {
+ const sr = 1000;
+ const data = new Float32Array(sr).fill(0.1);
+ const clip = { data, sampleRate: sr, start: -0.25, end: 2, duration: 1, from: 0.25, loop: false, volume: 0.6, speed: 1 };
+ const mix = await renderSoundtrack({ duration: 3, sampleRate: sr, clips: [clip] });
+ assert.ok(Math.abs(mix.getChannelData(0)[100] - 0.06) < 1e-6);
+ assert.equal(mix.getChannelData(0)[600], 0);
+});
+
+test('music ducks under clip sound with the same envelope as narration', async () => {
+ const opts = { duration: 3, sampleRate: 8000, music: { style: 'calm', seed: 42 } };
+ const voice = await renderSoundtrack({ ...opts, voiceSpans: [{ start: 1, end: 2 }] });
+ const clip = await renderSoundtrack({ ...opts, clips: [{ data: new Float32Array(8000), sampleRate: 8000, start: 1, end: 2, duration: 1, from: 0, volume: 0.6, speed: 1 }] });
+ assert.deepEqual(clip.getChannelData(0), voice.getChannelData(0));
+ assert.deepEqual(clip.getChannelData(1), voice.getChannelData(1));
+});
diff --git a/test/server.test.js b/test/server.test.js
index a168161..8723178 100644
--- a/test/server.test.js
+++ b/test/server.test.js
@@ -147,3 +147,77 @@ test('the preview server keeps secrets and sources safe', async () => {
fs.rmSync(dir, { recursive: true, force: true });
}
});
+
+test('clip endpoints only read video assets and keep the cache inside build', async () => {
+ const { ffmpeg } = await import('../src/ffmpeg.js');
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'explainroo-clip-server-'));
+ const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'explainroo-clip-outside-'));
+ fs.mkdirSync(path.join(dir, 'assets'));
+ const video = path.join(dir, 'assets/ok.mp4');
+ await ffmpeg(['-loglevel', 'error', '-f', 'lavfi', '-i', 'color=red:s=80x60:r=30:d=0.2', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=0.2', '-shortest', '-c:v', 'libx264', '-threads', '1', '-c:a', 'aac', video]);
+ fs.copyFileSync(video, path.join(outside, 'away.mp4'));
+ fs.copyFileSync(video, path.join(dir, 'root.mp4'));
+ fs.copyFileSync(video, path.join(dir, 'assets/.private.mp4'));
+ fs.mkdirSync(path.join(dir, 'assets/.hidden'));
+ fs.copyFileSync(video, path.join(dir, 'assets/.hidden/ok.mp4'));
+ fs.writeFileSync(path.join(dir, 'assets/notes.md'), 'secret');
+ fs.symlinkSync(path.join(outside, 'away.mp4'), path.join(dir, 'assets/away.mp4'));
+ fs.symlinkSync(path.join(dir, 'root.mp4'), path.join(dir, 'assets/root-link.mp4'));
+ fs.symlinkSync(path.join(dir, 'assets/.private.mp4'), path.join(dir, 'assets/hidden-link.mp4'));
+ fs.symlinkSync(path.join(dir, 'assets/notes.md'), path.join(dir, 'assets/notes.mp4'));
+ const server = await startServer({ projectDir: dir, getState: () => ({ config: { fps: 30 } }) });
+ try {
+ for (const endpoint of ['info', 'frame', 'audio']) {
+ const url = (p) => `/__clip/${endpoint}?path=${encodeURIComponent(p)}`;
+ for (const p of ['root.mp4', 'assets/../root.mp4', 'assets/missing.mp4', 'assets/notes.md', 'assets/ok.avi', 'assets/.private.mp4', 'assets/.hidden/ok.mp4', 'assets/away.mp4', 'assets/root-link.mp4', 'assets/hidden-link.mp4', 'assets/notes.mp4', '../away.mp4', video]) {
+ assert.equal((await request(server.port, { path: url(p) })).status, 404, `${endpoint}: ${p}`);
+ }
+ assert.equal((await request(server.port, { path: url('assets/ok.mp4'), host: 'evil.example' })).status, 403);
+ assert.equal((await request(server.port, { path: url('assets/ok.mp4'), origin: 'http://evil.example' })).status, 403);
+ assert.equal((await request(server.port, { path: url('assets/ok.mp4'), method: 'POST' })).status, 405);
+ }
+ for (const [ext, mime] of [['mp4', 'video/mp4'], ['mov', 'video/quicktime'], ['webm', 'video/webm'], ['m4v', 'video/mp4']]) {
+ fs.copyFileSync(video, path.join(dir, `assets/type.${ext}`));
+ const response = await fetch(`${server.url}/project/assets/type.${ext}`);
+ assert.equal(response.status, 200);
+ assert.equal(response.headers.get('content-type'), mime);
+ await response.arrayBuffer();
+ }
+ const responses = await Promise.all(Array.from({ length: 4 }, () => request(server.port, { path: '/__clip/info?path=assets/ok.mp4' })));
+ assert.ok(responses.every((r) => r.status === 200));
+ const info = JSON.parse(responses[0].body);
+ assert.deepEqual(info, { frames: 6, fps: 30, width: 80, height: 60, duration: 0.2, hasAudio: true });
+ assert.equal(fs.readdirSync(path.join(dir, 'build/clips')).length, 1);
+ const frame = await fetch(`${server.url}/__clip/frame?path=assets/ok.mp4&frame=999`);
+ assert.equal(frame.headers.get('content-type'), 'image/jpeg');
+ const last = await fetch(`${server.url}/__clip/frame?path=assets/ok.mp4&frame=5`);
+ assert.deepEqual(Buffer.from(await frame.arrayBuffer()), Buffer.from(await last.arrayBuffer()));
+ const audio = await fetch(`${server.url}/__clip/audio?path=assets/ok.mp4`);
+ assert.equal(audio.headers.get('content-type'), 'audio/wav');
+ assert.equal(Buffer.from(await audio.arrayBuffer()).toString('ascii', 0, 4), 'RIFF');
+ for (const frame of ['-1', '1.2', 'foo', '9007199254740992']) {
+ assert.equal((await request(server.port, { path: `/__clip/frame?path=assets/ok.mp4&frame=${frame}` })).status, 400);
+ }
+ const silent = path.join(dir, 'assets/silent.mp4');
+ await ffmpeg(['-loglevel', 'error', '-i', video, '-an', '-c:v', 'copy', silent]);
+ const silentInfo = await request(server.port, { path: '/__clip/info?path=assets/silent.mp4' });
+ assert.equal(JSON.parse(silentInfo.body).hasAudio, false);
+ assert.equal((await request(server.port, { path: '/__clip/audio?path=assets/silent.mp4' })).status, 404);
+ const hash = fs.readdirSync(path.join(dir, 'build/clips')).find((hash) => fs.existsSync(path.join(dir, 'build/clips', hash, 'audio.wav')));
+ const cached = path.join(dir, 'build/clips', hash, '00000000.jpg');
+ fs.unlinkSync(cached);
+ fs.symlinkSync(path.join(outside, 'away.mp4'), cached);
+ assert.equal((await request(server.port, { path: '/__clip/frame?path=assets/ok.mp4' })).status, 404);
+ fs.rmSync(path.join(dir, 'build/clips'), { recursive: true });
+ fs.symlinkSync(outside, path.join(dir, 'build/clips'));
+ const other = await startServer({ projectDir: dir, getState: () => ({ config: { fps: 30 } }) });
+ try {
+ assert.equal((await request(other.port, { path: '/__clip/info?path=assets/ok.mp4' })).status, 500);
+ assert.deepEqual(fs.readdirSync(outside), ['away.mp4']);
+ } finally { await other.close(); }
+ } finally {
+ await server.close();
+ fs.rmSync(dir, { recursive: true, force: true });
+ fs.rmSync(outside, { recursive: true, force: true });
+ }
+});