Art and animation, written as code. Illustrations, loops, interactive web art, launch-videos and scored films in dozens of styles, identical on every render.
TypeScript agent-skillsanimationcanvasclaude-codeclaude-pluginclaude-skillscodex-plugincreative-codecreative-codingcreative-toolsdeterministicgenerative-arthand-drawnillustrationlaunch-videomotion-designmotion-graphicspluginprocedural-generationskill
verify-export: audio-to-video sync on the delivered file (decoded vs master cross-correlation at each sync marker, sub-sample, 1 ms limit; the cue's attack vs the master's within 5 ms) and the true peak after the encode (ebur128, -1 dBTP under --delivery); captions.mjs writes .srt/.vtt from meta.captions; avsync tests
Alex Greenshpun3cb1ffab5bbf
diff --git a/skills/anidoodle/engine/test/avsync.test.ts b/skills/anidoodle/engine/test/avsync.test.tsnew file mode 100644index 0000000..05a12a1--- /dev/null+++ b/skills/anidoodle/engine/test/avsync.test.ts@@ -0,0 +1,33 @@+// AV SYNC TESTS. verify-export measures a delivered file's sound against its master mix and against+// the picture's own sync frames; these hold the measurements to known answers on synthetic signals.+// @ts-ignore: a plain ES module shared with the tools+import { xcorrOffset, onsetNear, toSrt, toVtt } from "../tools/avsync.mjs";++export const name = "avsync";+export const run = (ok: (cond: boolean, label: string) => void) => {+ const sr = 48000, n = sr * 3, ref = new Float32Array(n);+ let s = 12345; const rnd = () => ((s = (s * 1103515245 + 12345) >>> 0) / 4294967296) * 2 - 1;+ for (let i = 0; i < n; i++) ref[i] = 0.05 * rnd(); // a noise bed+ const click = (x: Float32Array, at: number) => { for (let k = 0; k < 480; k++) x[at + k] += 0.8 * Math.exp(-k / 60) * Math.sin(k * 0.9); };+ click(ref, sr * 1); click(ref, Math.round(sr * 2.2));+ for (const shift of [0, 7, -13, 96]) {+ const dec = new Float32Array(n); for (let i = 0; i < n; i++) { const j = i - shift; dec[i] = j >= 0 && j < n ? ref[j] : 0; }+ const r = xcorrOffset(ref, dec, 1.0, sr);+ ok(Math.abs(r.lag - shift) < 0.05, `a ${shift}-sample shift (${((shift / sr) * 1000).toFixed(2)} ms) is measured as ${r.lag.toFixed(2)} samples (corr ${r.corr.toFixed(3)})`);+ }+ // a half-sample shift (linear interpolation) is found to a fraction of a sample+ const half = new Float32Array(n); for (let i = 1; i < n; i++) half[i] = 0.5 * (ref[i] + ref[i - 1]);+ ok(Math.abs(xcorrOffset(ref, half, 1.0, sr).lag - 0.5) < 0.2, `a half-sample delay reads as ${xcorrOffset(ref, half, 1.0, sr).lag.toFixed(2)} samples (sub-sample)`);+ ok(Number.isNaN(xcorrOffset(new Float32Array(n), ref, 1.0, sr).ms), "silence has no offset to measure (NaN, never a fake zero)");+ // onsets: the click at 1.000 s is found within a millisecond, and on the decoded copy 7 samples late+ const t1 = onsetNear(ref, 1.0, sr), late = new Float32Array(n); for (let i = 480; i < n; i++) late[i] = ref[i - 480];+ ok(Math.abs(t1 - 1.0) <= 0.001, `the onset of a cue at 1.000 s is found at ${t1.toFixed(4)} s`);+ ok(Math.abs(onsetNear(late, 1.0, sr) - 1.01) <= 0.001, `a cue 10 ms late reads as ${((onsetNear(late, 1.0, sr) - 1) * 1000).toFixed(1)} ms late`);+ ok(Number.isNaN(onsetNear(ref, 1.6, sr)), "no cue near a frame: NaN, not the nearest noise");+ // captions+ const caps = [{ from: 0, to: 45, text: "a lighthouse at sunset, as a print" }, { from: 3600 * 30 + 15, to: 3600 * 30 + 90, text: "One sentence that says what it is, and then a second sentence that wraps" }];+ const srt = toSrt(caps, 30), vtt = toVtt(caps, 30);+ ok(srt.startsWith("1\n00:00:00,000 --> 00:00:01,500\na lighthouse at sunset, as a print\n"), "SubRip: numbered cues, comma milliseconds");+ ok(/01:00:00,500 --> 01:00:03,000\n.{1,42}\n.+\n/.test(srt), "an hour in, and a long line wraps to two");+ ok(vtt.startsWith("WEBVTT\n\n00:00:00.000 --> 00:00:01.500\n"), "WebVTT: the header and dot milliseconds");+};diff --git a/skills/anidoodle/engine/tools/avsync.mjs b/skills/anidoodle/engine/tools/avsync.mjsnew file mode 100644index 0000000..c4c3c71--- /dev/null+++ b/skills/anidoodle/engine/tools/avsync.mjs@@ -0,0 +1,39 @@+// AV SYNC. The two measurements verify-export makes on a delivered file's sound, as pure functions+// (no ffmpeg, no browser) so they are unit-tested on synthetic signals:+// xcorrOffset where the decoded audio sits against the master mix around one sync marker: the lag+// (samples, sub-sample by a parabola through the peak) that best lines the decoded window+// up with the master's. An encoder or muxer that shifts the sound shows up here, exactly.+// onsetNear when a cue actually starts in the delivered audio, near the frame it is meant for:+// the first 1 ms hop whose energy rises 12 dB over the quiet just before the window.+// The two answer different questions: the first is "did the file keep the mix's timing", the second+// "is the sound on its picture frame".+export const mono = (L, R) => { const m = new Float32Array(L.length); for (let i = 0; i < L.length; i++) m[i] = 0.5 * (L[i] + R[i]); return m; };++/** Lag (in samples; + = the decoded sound is late) of `dec` against `ref` around time `t` (s). */+export const xcorrOffset = (ref, dec, t, sr, { win = 0.3, maxLagMs = 40 } = {}) => {+ const W = Math.round(win * sr), a = Math.max(0, Math.round(t * sr - W / 3)), b = Math.min(ref.length, a + W), L = Math.round((maxLagMs / 1000) * sr);+ let e = 0; for (let i = a; i < b; i++) e += ref[i] * ref[i];+ if (e < 1e-9) return { lag: NaN, ms: NaN, corr: 0 };+ const at = (lag) => { let s = 0, d = 0; for (let i = a; i < b; i++) { const j = i + lag; const v = j >= 0 && j < dec.length ? dec[j] : 0; s += ref[i] * v; d += v * v; } return d > 0 ? s / Math.sqrt(e * d) : 0; };+ let best = 0, bc = -Infinity; const c = new Map();+ for (let lag = -L; lag <= L; lag++) { const v = at(lag); c.set(lag, v); if (v > bc) { bc = v; best = lag; } }+ const y0 = c.get(best - 1) ?? bc, y2 = c.get(best + 1) ?? bc, den = y0 - 2 * bc + y2, frac = den < 0 ? (0.5 * (y0 - y2)) / den : 0;+ const lag = best + Math.max(-0.5, Math.min(0.5, frac));+ return { lag, ms: (lag / sr) * 1000, corr: bc };+};++/** The onset time (s) of the sound that starts nearest `t` in [t - before, t + after], or NaN. */+export const onsetNear = (x, t, sr, { before = 0.05, after = 0.08, riseDb = 12, hopMs = 1 } = {}) => {+ const hop = Math.max(1, Math.round((hopMs / 1000) * sr)), e = (i) => { let s = 0; for (let k = i; k < Math.min(x.length, i + hop); k++) s += x[k] * x[k]; return s / hop; };+ const a = Math.max(0, Math.round((t - before) * sr)), b = Math.min(x.length - hop, Math.round((t + after) * sr));+ const pre = []; for (let i = Math.max(0, a - Math.round(0.06 * sr)); i < a; i += hop) pre.push(e(i));+ pre.sort((p, q) => p - q); const floor = Math.max(pre[pre.length >> 1] ?? 0, 1e-10), thr = floor * Math.pow(10, riseDb / 10);+ for (let i = a; i < b; i += hop) if (e(i) > thr) return i / sr;+ return NaN;+};++/** WebVTT / SubRip from caption cues in frames: [{ from, to, text }], lines wrapped at `width` characters, two lines at most. */+const wrap = (s, width = 42) => { const out = []; let line = ""; for (const w of s.split(/\s+/)) { if ((line + " " + w).trim().length > width && line) { out.push(line); line = w; } else line = (line + " " + w).trim(); } out.push(line); return out.length > 2 ? [out[0], out.slice(1).join(" ")] : out; };+const stamp = (sec, sep) => { const ms = Math.round(sec * 1000), h = Math.floor(ms / 3600000), m = Math.floor(ms / 60000) % 60, s = Math.floor(ms / 1000) % 60; return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}${sep}${String(ms % 1000).padStart(3, "0")}`; };+export const toSrt = (caps, fps) => caps.map((c, i) => `${i + 1}\n${stamp(c.from / fps, ",")} --> ${stamp(c.to / fps, ",")}\n${wrap(c.text).join("\n")}\n`).join("\n");+export const toVtt = (caps, fps) => "WEBVTT\n\n" + caps.map((c) => `${stamp(c.from / fps, ".")} --> ${stamp(c.to / fps, ".")}\n${wrap(c.text).join("\n")}\n`).join("\n");diff --git a/skills/anidoodle/engine/tools/captions.mjs b/skills/anidoodle/engine/tools/captions.mjsnew file mode 100644index 0000000..0af1eb0--- /dev/null+++ b/skills/anidoodle/engine/tools/captions.mjs@@ -0,0 +1,33 @@+// CAPTIONS. The words a viewer reads in a film, as sidecar caption files: SubRip (.srt) and WebVTT+// (.vtt), timed in the film's own frames (meta.captions; a launch template film writes them from+// its prompts, its word pages, its split beats and its end card). Feeds autoplay muted: the captions+// are the channel most viewers get. A platform that takes a sidecar (YouTube, LinkedIn, the site's+// <track>) gets these; a feed that does not gets them burned into their own band (captions: "burn").+// node tools/captions.mjs <film>[-<shape>] [--out out/<film>] writes <out>.srt and <out>.vtt+import { build } from "esbuild";+import { mkdirSync, writeFileSync } from "node:fs";+import { dirname, resolve } from "node:path";+import { overlay } from "./overlay.mjs";+import { requireFilm, splitShape } from "./names.mjs";+import { toSrt, toVtt } from "./avsync.mjs";++export const filmMeta = async (name) => {+ const { base, shape } = splitShape(name);+ const js = (await build({ stdin: { contents: `export { ${base} as film } from "./src/canvas-core/${base}";`, resolveDir: process.cwd(), loader: "ts" }, bundle: true, format: "esm", write: false, platform: "neutral", plugins: [overlay], logLevel: "error" })).outputFiles[0].text;+ const { film } = await import("data:text/javascript;base64," + Buffer.from(js).toString("base64"));+ return (shape ? film.reshape(shape) : film).meta;+};+export const writeCaptions = async (name, out = `out/${name}`) => {+ const meta = await filmMeta(name), caps = meta.captions ?? [];+ if (!caps.length) return { count: 0, files: [] };+ mkdirSync(dirname(resolve(out)), { recursive: true });+ const srt = resolve(`${out}.srt`), vtt = resolve(`${out}.vtt`);+ writeFileSync(srt, toSrt(caps, meta.fps)); writeFileSync(vtt, toVtt(caps, meta.fps));+ return { count: caps.length, files: [srt, vtt] };+};+if (import.meta.url === `file://${process.argv[1]}`) {+ const name = requireFilm(process.argv[2], "captions", "node tools/captions.mjs <film>[-<shape>] [--out out/<film>]");+ const i = process.argv.indexOf("--out"), r = await writeCaptions(name, i > 0 ? process.argv[i + 1] : undefined);+ if (!r.count) { console.error(`captions: ${name} has no meta.captions (a launch template film writes them; add your own to meta.captions)`); process.exit(1); }+ console.log(`captions: ${r.count} cues -> ${r.files.join(", ")}`);+}diff --git a/skills/anidoodle/engine/tools/verify-export.mjs b/skills/anidoodle/engine/tools/verify-export.mjsindex b9a77a4..1bf95dc 100644--- a/skills/anidoodle/engine/tools/verify-export.mjs+++ b/skills/anidoodle/engine/tools/verify-export.mjs@@ -15,7 +15,17 @@// [--fidelity-psnr N] a decoded frame must sit within N dB of the source frame (renders it once)// [--frame N] which frame --fidelity-psnr compares (default 0)// [--delivery] this file ships: a near-blank frame 0 (the platform thumbnail) FAILS-// instead of warning; fix with render --poster-frame N+// instead of warning; fix with render --poster-frame N; and the true+// peak AFTER the encode must be at or under -1 dBTP (else it is reported)+// [--sync-tol MS] audio-to-video: the decoded sound may sit this far from the master mix+// at each sync marker (default 1 ms)+//+// With a score, an MP4 is also checked for SYNC and PEAK: at each of the film's sync markers+// (meta.sync: the presses and the impact of a launch film; three evenly spaced points otherwise)+// the decoded audio is cross-correlated with the master mix the page makes (the offset in ms, sub-+// sample), and the cue's attack in the delivered file is measured against its frame and against the+// master's own attack (5 ms: AAC pre-echo); the true peak is measured on the DECODED file (ebur128, 4x oversampled), because an AAC+// encode can push a -1 dBTP master over the ceiling.//// Always checked: the file decodes, its pixel size, its exact frame count, its duration against// the film's meta, and for an MP4 the score: an audio stream exactly when the film has a score,@@ -25,8 +35,9 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs";import { basename, resolve } from "node:path";import { defaultOutput, requireFilm } from "./names.mjs";import { firstFrameBlank } from "./thumb.mjs";+import { mono, onsetNear, xcorrOffset } from "./avsync.mjs";-const VAL = new Set(["file", "adapter", "width", "first-frame", "tol", "fidelity-psnr", "scale", "frame"]);+const VAL = new Set(["file", "adapter", "width", "first-frame", "tol", "fidelity-psnr", "scale", "frame", "sync-tol"]);const pos = [], opt = {};for (let i = 2; i < process.argv.length; i++) { const a = process.argv[i]; if (a.startsWith("--")) opt[a.slice(2)] = VAL.has(a.slice(2)) ? process.argv[++i] : true; else pos.push(a); }// a file path names its film: out/<film>.mp4 (or a range render's out/<film>.<from>-<to>.mp4)@@ -98,6 +109,36 @@ if (isMp4) {const vd = spawnSync("ffmpeg", ["-v", "info", "-i", file, "-map", "0:a:0", "-af", "volumedetect", "-f", "null", "-"], { encoding: "utf8" });const mx = Number(((vd.stdout + vd.stderr).match(/max_volume:\s*(-?[0-9.]+|-inf) dB/) ?? [])[1]);say(Number.isFinite(mx) && mx > -40 && mx <= 0, `audio peak ${Number.isFinite(mx) ? mx.toFixed(1) : "-inf"} dBFS`, "audible (> -40 dBFS) and not clipped past 0");++ head("1c", "SYNC the delivered sound sits on the picture's frames");+ const SR = 48000, dec = spawnSync("ffmpeg", ["-v", "error", "-i", file, "-map", "0:a:0", "-f", "f32le", "-ac", "2", "-ar", String(SR), "-"], { maxBuffer: 1 << 30 });+ if (dec.status !== 0) die(`ffmpeg could not decode the audio: ${dec.stderr.toString().trim()}`);+ const di = new Float32Array(dec.stdout.buffer.slice(dec.stdout.byteOffset, dec.stdout.byteOffset + dec.stdout.byteLength)), dL = new Float32Array(di.length / 2), dR = new Float32Array(di.length / 2);+ for (let i = 0; i < dL.length; i++) { dL[i] = di[2 * i]; dR[i] = di[2 * i + 1]; }+ const master = await s.audio(SR), mb = Buffer.from(master.float32, "base64"), mi = new Float32Array(mb.buffer.slice(mb.byteOffset, mb.byteOffset + mb.byteLength)), mL = new Float32Array(master.frames), mR = new Float32Array(master.frames);+ for (let i = 0; i < master.frames; i++) { mL[i] = mi[2 * i]; mR[i] = mi[2 * i + 1]; }+ const ref = mono(mL, mR), got = mono(dL, dR), tol = Number(opt["sync-tol"] ?? 1);+ const marks = meta.sync?.length ? meta.sync : [0.25, 0.5, 0.75].map((q) => ({ frame: Math.round(q * N), label: `${Math.round(q * 100)} % in` }));+ if (!meta.sync?.length) note("no sync markers in meta.sync", "checking three points of the mix instead (a launch template film names its presses and its impact)");+ for (const mk of marks) {+ const t = mk.frame / meta.fps, x = xcorrOffset(ref, got, t, SR);+ if (!Number.isFinite(x.ms)) { note(`frame ${mk.frame} (${mk.label}): the master is silent there`, "nothing to line up"); continue; }+ say(Math.abs(x.ms) <= tol && x.corr > 0.5, `frame ${mk.frame} ${mk.label}: decoded vs master ${x.ms >= 0 ? "+" : ""}${x.ms.toFixed(2)} ms`, `limit ${tol} ms (correlation ${x.corr.toFixed(3)})`);+ // the cue's attack, heard in the file, against where the master put it: a sound may be designed to+ // lead its hit (a press's finger noise, an impact's breath), so the frame itself is the master's truth+ if (meta.sync?.length) {+ const on = onsetNear(got, t, SR), want = onsetNear(ref, t, SR);+ if (Number.isFinite(on) && Number.isFinite(want)) say(Math.abs(on - want) * 1000 <= 5, `frame ${mk.frame} ${mk.label}: the attack starts ${((on - t) * 1000).toFixed(1)} ms from its frame`, `the master's attack is at ${((want - t) * 1000).toFixed(1)} ms (the sound's own lead); delivered within 5 ms of it (AAC pre-echo can pull a sharp attack a few ms early)`);+ else note(`frame ${mk.frame} ${mk.label}: no attack found within 80 ms`, "a soft attack; the cross-correlation above is the sync measurement");+ }+ }++ head("1d", "PEAK the true peak of the file as delivered (after the encode)");+ const eb = spawnSync("ffmpeg", ["-v", "info", "-nostats", "-i", file, "-map", "0:a:0", "-af", "ebur128=peak=true", "-f", "null", "-"], { encoding: "utf8", maxBuffer: 1 << 26 });+ const txt = eb.stdout + eb.stderr, tp = Number((txt.match(/True peak:\s*\n\s*Peak:\s*(-?[0-9.]+|-inf) dBFS/) ?? [])[1]), il = Number((txt.match(/Integrated loudness:\s*\n\s*I:\s*(-?[0-9.]+) LUFS/) ?? [])[1]);+ if (!Number.isFinite(tp)) note("true peak unreadable from ffmpeg's ebur128", txt.split("\n").slice(-4).join(" "));+ else if (opt.delivery) say(tp <= -1.0, `true peak after the encode ${tp.toFixed(2)} dBTP`, `ceiling -1 dBTP; integrated ${Number.isFinite(il) ? il.toFixed(1) : "?"} LUFS`);+ else note(`true peak after the encode ${tp.toFixed(2)} dBTP${tp > -1 ? " (over -1: --delivery fails this)" : ""}`, `integrated ${Number.isFinite(il) ? il.toFixed(1) : "?"} LUFS`);}}