Skip to content

Art and animation, written as code. Illustrations, loops, interactive web art, launch-videos and scored films in dozens of styles, identical on every render.

verify-export: audio-to-video sync on the delivered file (decoded vs master cross-correlation at each sync marker, sub-sample, 1 ms limit; the cue's attack vs the master's within 5 ms) and the true peak after the encode (ebur128, -1 dBTP under --delivery); captions.mjs writes .srt/.vtt from meta.captions; avsync tests

Alex Greenshpun3cb1ffab5bbf
diff --git a/skills/anidoodle/engine/test/avsync.test.ts b/skills/anidoodle/engine/test/avsync.test.ts
new file mode 100644
index 0000000..05a12a1
--- /dev/null
+++ b/skills/anidoodle/engine/test/avsync.test.ts
@@ -0,0 +1,33 @@
+// AV SYNC TESTS. verify-export measures a delivered file's sound against its master mix and against
+// the picture's own sync frames; these hold the measurements to known answers on synthetic signals.
+// @ts-ignore: a plain ES module shared with the tools
+import { xcorrOffset, onsetNear, toSrt, toVtt } from "../tools/avsync.mjs";
+
+export const name = "avsync";
+export const run = (ok: (cond: boolean, label: string) => void) => {
+ const sr = 48000, n = sr * 3, ref = new Float32Array(n);
+ let s = 12345; const rnd = () => ((s = (s * 1103515245 + 12345) >>> 0) / 4294967296) * 2 - 1;
+ for (let i = 0; i < n; i++) ref[i] = 0.05 * rnd(); // a noise bed
+ const click = (x: Float32Array, at: number) => { for (let k = 0; k < 480; k++) x[at + k] += 0.8 * Math.exp(-k / 60) * Math.sin(k * 0.9); };
+ click(ref, sr * 1); click(ref, Math.round(sr * 2.2));
+ for (const shift of [0, 7, -13, 96]) {
+ const dec = new Float32Array(n); for (let i = 0; i < n; i++) { const j = i - shift; dec[i] = j >= 0 && j < n ? ref[j] : 0; }
+ const r = xcorrOffset(ref, dec, 1.0, sr);
+ ok(Math.abs(r.lag - shift) < 0.05, `a ${shift}-sample shift (${((shift / sr) * 1000).toFixed(2)} ms) is measured as ${r.lag.toFixed(2)} samples (corr ${r.corr.toFixed(3)})`);
+ }
+ // a half-sample shift (linear interpolation) is found to a fraction of a sample
+ const half = new Float32Array(n); for (let i = 1; i < n; i++) half[i] = 0.5 * (ref[i] + ref[i - 1]);
+ ok(Math.abs(xcorrOffset(ref, half, 1.0, sr).lag - 0.5) < 0.2, `a half-sample delay reads as ${xcorrOffset(ref, half, 1.0, sr).lag.toFixed(2)} samples (sub-sample)`);
+ ok(Number.isNaN(xcorrOffset(new Float32Array(n), ref, 1.0, sr).ms), "silence has no offset to measure (NaN, never a fake zero)");
+ // onsets: the click at 1.000 s is found within a millisecond, and on the decoded copy 7 samples late
+ const t1 = onsetNear(ref, 1.0, sr), late = new Float32Array(n); for (let i = 480; i < n; i++) late[i] = ref[i - 480];
+ ok(Math.abs(t1 - 1.0) <= 0.001, `the onset of a cue at 1.000 s is found at ${t1.toFixed(4)} s`);
+ ok(Math.abs(onsetNear(late, 1.0, sr) - 1.01) <= 0.001, `a cue 10 ms late reads as ${((onsetNear(late, 1.0, sr) - 1) * 1000).toFixed(1)} ms late`);
+ ok(Number.isNaN(onsetNear(ref, 1.6, sr)), "no cue near a frame: NaN, not the nearest noise");
+ // captions
+ const caps = [{ from: 0, to: 45, text: "a lighthouse at sunset, as a print" }, { from: 3600 * 30 + 15, to: 3600 * 30 + 90, text: "One sentence that says what it is, and then a second sentence that wraps" }];
+ const srt = toSrt(caps, 30), vtt = toVtt(caps, 30);
+ ok(srt.startsWith("1\n00:00:00,000 --> 00:00:01,500\na lighthouse at sunset, as a print\n"), "SubRip: numbered cues, comma milliseconds");
+ ok(/01:00:00,500 --> 01:00:03,000\n.{1,42}\n.+\n/.test(srt), "an hour in, and a long line wraps to two");
+ ok(vtt.startsWith("WEBVTT\n\n00:00:00.000 --> 00:00:01.500\n"), "WebVTT: the header and dot milliseconds");
+};
diff --git a/skills/anidoodle/engine/tools/avsync.mjs b/skills/anidoodle/engine/tools/avsync.mjs
new file mode 100644
index 0000000..c4c3c71
--- /dev/null
+++ b/skills/anidoodle/engine/tools/avsync.mjs
@@ -0,0 +1,39 @@
+// AV SYNC. The two measurements verify-export makes on a delivered file's sound, as pure functions
+// (no ffmpeg, no browser) so they are unit-tested on synthetic signals:
+// xcorrOffset where the decoded audio sits against the master mix around one sync marker: the lag
+// (samples, sub-sample by a parabola through the peak) that best lines the decoded window
+// up with the master's. An encoder or muxer that shifts the sound shows up here, exactly.
+// onsetNear when a cue actually starts in the delivered audio, near the frame it is meant for:
+// the first 1 ms hop whose energy rises 12 dB over the quiet just before the window.
+// The two answer different questions: the first is "did the file keep the mix's timing", the second
+// "is the sound on its picture frame".
+export const mono = (L, R) => { const m = new Float32Array(L.length); for (let i = 0; i < L.length; i++) m[i] = 0.5 * (L[i] + R[i]); return m; };
+
+/** Lag (in samples; + = the decoded sound is late) of `dec` against `ref` around time `t` (s). */
+export const xcorrOffset = (ref, dec, t, sr, { win = 0.3, maxLagMs = 40 } = {}) => {
+ const W = Math.round(win * sr), a = Math.max(0, Math.round(t * sr - W / 3)), b = Math.min(ref.length, a + W), L = Math.round((maxLagMs / 1000) * sr);
+ let e = 0; for (let i = a; i < b; i++) e += ref[i] * ref[i];
+ if (e < 1e-9) return { lag: NaN, ms: NaN, corr: 0 };
+ const at = (lag) => { let s = 0, d = 0; for (let i = a; i < b; i++) { const j = i + lag; const v = j >= 0 && j < dec.length ? dec[j] : 0; s += ref[i] * v; d += v * v; } return d > 0 ? s / Math.sqrt(e * d) : 0; };
+ let best = 0, bc = -Infinity; const c = new Map();
+ for (let lag = -L; lag <= L; lag++) { const v = at(lag); c.set(lag, v); if (v > bc) { bc = v; best = lag; } }
+ const y0 = c.get(best - 1) ?? bc, y2 = c.get(best + 1) ?? bc, den = y0 - 2 * bc + y2, frac = den < 0 ? (0.5 * (y0 - y2)) / den : 0;
+ const lag = best + Math.max(-0.5, Math.min(0.5, frac));
+ return { lag, ms: (lag / sr) * 1000, corr: bc };
+};
+
+/** The onset time (s) of the sound that starts nearest `t` in [t - before, t + after], or NaN. */
+export const onsetNear = (x, t, sr, { before = 0.05, after = 0.08, riseDb = 12, hopMs = 1 } = {}) => {
+ const hop = Math.max(1, Math.round((hopMs / 1000) * sr)), e = (i) => { let s = 0; for (let k = i; k < Math.min(x.length, i + hop); k++) s += x[k] * x[k]; return s / hop; };
+ const a = Math.max(0, Math.round((t - before) * sr)), b = Math.min(x.length - hop, Math.round((t + after) * sr));
+ const pre = []; for (let i = Math.max(0, a - Math.round(0.06 * sr)); i < a; i += hop) pre.push(e(i));
+ pre.sort((p, q) => p - q); const floor = Math.max(pre[pre.length >> 1] ?? 0, 1e-10), thr = floor * Math.pow(10, riseDb / 10);
+ for (let i = a; i < b; i += hop) if (e(i) > thr) return i / sr;
+ return NaN;
+};
+
+/** WebVTT / SubRip from caption cues in frames: [{ from, to, text }], lines wrapped at `width` characters, two lines at most. */
+const wrap = (s, width = 42) => { const out = []; let line = ""; for (const w of s.split(/\s+/)) { if ((line + " " + w).trim().length > width && line) { out.push(line); line = w; } else line = (line + " " + w).trim(); } out.push(line); return out.length > 2 ? [out[0], out.slice(1).join(" ")] : out; };
+const stamp = (sec, sep) => { const ms = Math.round(sec * 1000), h = Math.floor(ms / 3600000), m = Math.floor(ms / 60000) % 60, s = Math.floor(ms / 1000) % 60; return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}${sep}${String(ms % 1000).padStart(3, "0")}`; };
+export const toSrt = (caps, fps) => caps.map((c, i) => `${i + 1}\n${stamp(c.from / fps, ",")} --> ${stamp(c.to / fps, ",")}\n${wrap(c.text).join("\n")}\n`).join("\n");
+export const toVtt = (caps, fps) => "WEBVTT\n\n" + caps.map((c) => `${stamp(c.from / fps, ".")} --> ${stamp(c.to / fps, ".")}\n${wrap(c.text).join("\n")}\n`).join("\n");
diff --git a/skills/anidoodle/engine/tools/captions.mjs b/skills/anidoodle/engine/tools/captions.mjs
new file mode 100644
index 0000000..0af1eb0
--- /dev/null
+++ b/skills/anidoodle/engine/tools/captions.mjs
@@ -0,0 +1,33 @@
+// CAPTIONS. The words a viewer reads in a film, as sidecar caption files: SubRip (.srt) and WebVTT
+// (.vtt), timed in the film's own frames (meta.captions; a launch template film writes them from
+// its prompts, its word pages, its split beats and its end card). Feeds autoplay muted: the captions
+// are the channel most viewers get. A platform that takes a sidecar (YouTube, LinkedIn, the site's
+// <track>) gets these; a feed that does not gets them burned into their own band (captions: "burn").
+// node tools/captions.mjs <film>[-<shape>] [--out out/<film>] writes <out>.srt and <out>.vtt
+import { build } from "esbuild";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { overlay } from "./overlay.mjs";
+import { requireFilm, splitShape } from "./names.mjs";
+import { toSrt, toVtt } from "./avsync.mjs";
+
+export const filmMeta = async (name) => {
+ const { base, shape } = splitShape(name);
+ const js = (await build({ stdin: { contents: `export { ${base} as film } from "./src/canvas-core/${base}";`, resolveDir: process.cwd(), loader: "ts" }, bundle: true, format: "esm", write: false, platform: "neutral", plugins: [overlay], logLevel: "error" })).outputFiles[0].text;
+ const { film } = await import("data:text/javascript;base64," + Buffer.from(js).toString("base64"));
+ return (shape ? film.reshape(shape) : film).meta;
+};
+export const writeCaptions = async (name, out = `out/${name}`) => {
+ const meta = await filmMeta(name), caps = meta.captions ?? [];
+ if (!caps.length) return { count: 0, files: [] };
+ mkdirSync(dirname(resolve(out)), { recursive: true });
+ const srt = resolve(`${out}.srt`), vtt = resolve(`${out}.vtt`);
+ writeFileSync(srt, toSrt(caps, meta.fps)); writeFileSync(vtt, toVtt(caps, meta.fps));
+ return { count: caps.length, files: [srt, vtt] };
+};
+if (import.meta.url === `file://${process.argv[1]}`) {
+ const name = requireFilm(process.argv[2], "captions", "node tools/captions.mjs <film>[-<shape>] [--out out/<film>]");
+ const i = process.argv.indexOf("--out"), r = await writeCaptions(name, i > 0 ? process.argv[i + 1] : undefined);
+ if (!r.count) { console.error(`captions: ${name} has no meta.captions (a launch template film writes them; add your own to meta.captions)`); process.exit(1); }
+ console.log(`captions: ${r.count} cues -> ${r.files.join(", ")}`);
+}
diff --git a/skills/anidoodle/engine/tools/verify-export.mjs b/skills/anidoodle/engine/tools/verify-export.mjs
index b9a77a4..1bf95dc 100644
--- a/skills/anidoodle/engine/tools/verify-export.mjs
+++ b/skills/anidoodle/engine/tools/verify-export.mjs
@@ -15,7 +15,17 @@
// [--fidelity-psnr N] a decoded frame must sit within N dB of the source frame (renders it once)
// [--frame N] which frame --fidelity-psnr compares (default 0)
// [--delivery] this file ships: a near-blank frame 0 (the platform thumbnail) FAILS
-// instead of warning; fix with render --poster-frame N
+// instead of warning; fix with render --poster-frame N; and the true
+// peak AFTER the encode must be at or under -1 dBTP (else it is reported)
+// [--sync-tol MS] audio-to-video: the decoded sound may sit this far from the master mix
+// at each sync marker (default 1 ms)
+//
+// With a score, an MP4 is also checked for SYNC and PEAK: at each of the film's sync markers
+// (meta.sync: the presses and the impact of a launch film; three evenly spaced points otherwise)
+// the decoded audio is cross-correlated with the master mix the page makes (the offset in ms, sub-
+// sample), and the cue's attack in the delivered file is measured against its frame and against the
+// master's own attack (5 ms: AAC pre-echo); the true peak is measured on the DECODED file (ebur128, 4x oversampled), because an AAC
+// encode can push a -1 dBTP master over the ceiling.
//
// Always checked: the file decodes, its pixel size, its exact frame count, its duration against
// the film's meta, and for an MP4 the score: an audio stream exactly when the film has a score,
@@ -25,8 +35,9 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs";
import { basename, resolve } from "node:path";
import { defaultOutput, requireFilm } from "./names.mjs";
import { firstFrameBlank } from "./thumb.mjs";
+import { mono, onsetNear, xcorrOffset } from "./avsync.mjs";
-const VAL = new Set(["file", "adapter", "width", "first-frame", "tol", "fidelity-psnr", "scale", "frame"]);
+const VAL = new Set(["file", "adapter", "width", "first-frame", "tol", "fidelity-psnr", "scale", "frame", "sync-tol"]);
const pos = [], opt = {};
for (let i = 2; i < process.argv.length; i++) { const a = process.argv[i]; if (a.startsWith("--")) opt[a.slice(2)] = VAL.has(a.slice(2)) ? process.argv[++i] : true; else pos.push(a); }
// a file path names its film: out/<film>.mp4 (or a range render's out/<film>.<from>-<to>.mp4)
@@ -98,6 +109,36 @@ if (isMp4) {
const vd = spawnSync("ffmpeg", ["-v", "info", "-i", file, "-map", "0:a:0", "-af", "volumedetect", "-f", "null", "-"], { encoding: "utf8" });
const mx = Number(((vd.stdout + vd.stderr).match(/max_volume:\s*(-?[0-9.]+|-inf) dB/) ?? [])[1]);
say(Number.isFinite(mx) && mx > -40 && mx <= 0, `audio peak ${Number.isFinite(mx) ? mx.toFixed(1) : "-inf"} dBFS`, "audible (> -40 dBFS) and not clipped past 0");
+
+ head("1c", "SYNC the delivered sound sits on the picture's frames");
+ const SR = 48000, dec = spawnSync("ffmpeg", ["-v", "error", "-i", file, "-map", "0:a:0", "-f", "f32le", "-ac", "2", "-ar", String(SR), "-"], { maxBuffer: 1 << 30 });
+ if (dec.status !== 0) die(`ffmpeg could not decode the audio: ${dec.stderr.toString().trim()}`);
+ const di = new Float32Array(dec.stdout.buffer.slice(dec.stdout.byteOffset, dec.stdout.byteOffset + dec.stdout.byteLength)), dL = new Float32Array(di.length / 2), dR = new Float32Array(di.length / 2);
+ for (let i = 0; i < dL.length; i++) { dL[i] = di[2 * i]; dR[i] = di[2 * i + 1]; }
+ const master = await s.audio(SR), mb = Buffer.from(master.float32, "base64"), mi = new Float32Array(mb.buffer.slice(mb.byteOffset, mb.byteOffset + mb.byteLength)), mL = new Float32Array(master.frames), mR = new Float32Array(master.frames);
+ for (let i = 0; i < master.frames; i++) { mL[i] = mi[2 * i]; mR[i] = mi[2 * i + 1]; }
+ const ref = mono(mL, mR), got = mono(dL, dR), tol = Number(opt["sync-tol"] ?? 1);
+ const marks = meta.sync?.length ? meta.sync : [0.25, 0.5, 0.75].map((q) => ({ frame: Math.round(q * N), label: `${Math.round(q * 100)} % in` }));
+ if (!meta.sync?.length) note("no sync markers in meta.sync", "checking three points of the mix instead (a launch template film names its presses and its impact)");
+ for (const mk of marks) {
+ const t = mk.frame / meta.fps, x = xcorrOffset(ref, got, t, SR);
+ if (!Number.isFinite(x.ms)) { note(`frame ${mk.frame} (${mk.label}): the master is silent there`, "nothing to line up"); continue; }
+ say(Math.abs(x.ms) <= tol && x.corr > 0.5, `frame ${mk.frame} ${mk.label}: decoded vs master ${x.ms >= 0 ? "+" : ""}${x.ms.toFixed(2)} ms`, `limit ${tol} ms (correlation ${x.corr.toFixed(3)})`);
+ // the cue's attack, heard in the file, against where the master put it: a sound may be designed to
+ // lead its hit (a press's finger noise, an impact's breath), so the frame itself is the master's truth
+ if (meta.sync?.length) {
+ const on = onsetNear(got, t, SR), want = onsetNear(ref, t, SR);
+ if (Number.isFinite(on) && Number.isFinite(want)) say(Math.abs(on - want) * 1000 <= 5, `frame ${mk.frame} ${mk.label}: the attack starts ${((on - t) * 1000).toFixed(1)} ms from its frame`, `the master's attack is at ${((want - t) * 1000).toFixed(1)} ms (the sound's own lead); delivered within 5 ms of it (AAC pre-echo can pull a sharp attack a few ms early)`);
+ else note(`frame ${mk.frame} ${mk.label}: no attack found within 80 ms`, "a soft attack; the cross-correlation above is the sync measurement");
+ }
+ }
+
+ head("1d", "PEAK the true peak of the file as delivered (after the encode)");
+ const eb = spawnSync("ffmpeg", ["-v", "info", "-nostats", "-i", file, "-map", "0:a:0", "-af", "ebur128=peak=true", "-f", "null", "-"], { encoding: "utf8", maxBuffer: 1 << 26 });
+ const txt = eb.stdout + eb.stderr, tp = Number((txt.match(/True peak:\s*\n\s*Peak:\s*(-?[0-9.]+|-inf) dBFS/) ?? [])[1]), il = Number((txt.match(/Integrated loudness:\s*\n\s*I:\s*(-?[0-9.]+) LUFS/) ?? [])[1]);
+ if (!Number.isFinite(tp)) note("true peak unreadable from ffmpeg's ebur128", txt.split("\n").slice(-4).join(" "));
+ else if (opt.delivery) say(tp <= -1.0, `true peak after the encode ${tp.toFixed(2)} dBTP`, `ceiling -1 dBTP; integrated ${Number.isFinite(il) ? il.toFixed(1) : "?"} LUFS`);
+ else note(`true peak after the encode ${tp.toFixed(2)} dBTP${tp > -1 ? " (over -1: --delivery fails this)" : ""}`, `integrated ${Number.isFinite(il) ? il.toFixed(1) : "?"} LUFS`);
}
}