Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
fix(caption-fine): light whole CJK words in step karaoke
rpone5c11b499e84b
diff --git a/packages/caption-fine/src/karaoke.ts b/packages/caption-fine/src/karaoke.tsnew file mode 100644index 00000000..7b4c9323--- /dev/null+++ b/packages/caption-fine/src/karaoke.ts@@ -0,0 +1,70 @@+import type { CaptionAlignmentUnit } from "@hypit/narrative";++import { wordGaps } from "./spacing.js";++type ActivationFrames = ReadonlyMap<string, { readonly start: number; readonly end: number }>;++/**+ * Karaoke repaints one Alignment Unit at a time. A Unit is one lexical unit, and in Han, Hiragana+ * and Katakana that unit is a single character, so the highlight crawls character by character+ * across a word the reader takes in at once.+ *+ * Reading the Cue as prose recovers its words. `wordGaps` says which boundaries carry a space, so+ * the surfaces join back into exactly the prose the renderer draws, and the runtime's word+ * segmenter returns the words inside it. A word that covers several Units gives them one shared+ * activation window, so the whole word lights together.+ *+ * Only the window is shared. Layout, spacing, per-Unit motion and the CaptionDocument are the same+ * values they were, so speech tokens, authored markers and measured timing are untouched.+ *+ * Segmentation is advisory here: it decides which characters light together and nothing else. A+ * word the segmenter splits differently changes how the highlight steps, never the timing beneath+ * it. Scripts written without spaces that arrive as one Unit, such as Thai, Khmer and Lao, keep the+ * highlight they have, because a Unit is the smallest thing that carries a measured window and this+ * pass can join Units but cannot divide one.+ */+const segmenter = new Intl.Segmenter(undefined, { granularity: "word" });++export function karaokeWordActivation(+ atoms: readonly CaptionAlignmentUnit[],+ wordText: ReadonlyMap<string, string>,+ activation: ActivationFrames,+): ActivationFrames {+ if (atoms.length < 2) return activation;++ const surfaces = atoms.map((atom) => atom.wordIds.map((wordId) => wordText.get(wordId) ?? "").join(""));+ const gaps = wordGaps(surfaces);++ let prose = "";+ const spans = surfaces.map((surface, index) => {+ if (gaps[index]) prose += " ";+ const start = prose.length;+ prose += surface;+ return { start, end: prose.length };+ });++ const merged = new Map(activation);+ for (const word of segmenter.segment(prose)) {+ if (!word.isWordLike) continue;+ const wordEnd = word.index + word.segment.length;+ const members = spans+ .map((span, index) => ({ span, index }))+ .filter(({ span }) => span.start < wordEnd && span.end > word.index)+ .map(({ index }) => index);+ if (members.length < 2) continue;++ let start = Number.POSITIVE_INFINITY;+ let end = Number.NEGATIVE_INFINITY;+ for (const member of members) {+ // The exclusive window already ends each Unit where the next one starts, so the run's own+ // bounds are the shared window. Read them from the incoming map: a Unit that two words touch+ // must not compound the widening.+ const timing = activation.get(atoms[member]!.id);+ if (timing === undefined) return activation;+ start = Math.min(start, timing.start);+ end = Math.max(end, timing.end);+ }+ for (const member of members) merged.set(atoms[member]!.id, { start, end });+ }+ return merged;+}diff --git a/packages/caption-fine/src/render.ts b/packages/caption-fine/src/render.tsindex 6b330af8..b47b225a 100644--- a/packages/caption-fine/src/render.ts+++ b/packages/caption-fine/src/render.ts@@ -20,6 +20,7 @@ import type { CanvasSpace, SpatialFrame, SpatialRegionTimeline } from "@hypit/spimport { assertFineCaptionParameters, FINE_CAPTION_FAMILY } from "./style.js";import { assertFineCaptionSchedule } from "./schedule.js";+import { karaokeWordActivation } from "./karaoke.js";import { joinSurfaces, uniformGap, wordGapBetween, wordGaps } from "./spacing.js";import type {FineCaptionActiveUnderline,@@ -692,7 +693,8 @@ function cueElements(})();const cueMotion = cueAnimation(parameters, durationFrames);const cueLoop = parameters.motion.loopTarget === "cue" ? loopAnimation(parameters, durationFrames) : undefined;- const activationFrames = exclusiveActivationFrames(atoms, atomFrames);+ const exclusiveFrames = exclusiveActivationFrames(atoms, atomFrames);+ const activationFrames = karaokeWordActivation(atoms, wordText, exclusiveFrames);const gapPx = `${compactNumber(parameters.layout.wordGapPx)}px`;const atomSurfaces = atoms.map((atom) => atom.wordIds.map((wordId) => wordText.get(wordId) ?? ""));// A Cue whose boundaries all agree carries one `column-gap`, which is also what keeps a row that@@ -844,6 +846,8 @@ function cueElements(if (timing === undefined) throw new Error(`Fine Caption is missing timing for Atom ${atom.id}`);const activationTiming = activationFrames.get(atom.id);if (activationTiming === undefined) throw new Error(`Fine Caption is missing activation timing for Atom ${atom.id}`);+ const exclusiveTiming = exclusiveFrames.get(atom.id);+ if (exclusiveTiming === undefined) throw new Error(`Fine Caption is missing exclusive timing for Atom ${atom.id}`);const atomId = `atom-${atomIndex + 1}`;const surfaces = atomSurfaces[atomIndex] ?? [];// The base glyphs and the activated copy stacked over them are laid out from this one list, so@@ -945,8 +949,9 @@ function cueElements(],animation: activeBoxAnimation(parameters,- parameters.activeBox.mode === "current" ? activationTiming.start : timing.start,- parameters.activeBox.mode === "current" ? activationTiming.end : timing.end,+ // The capsule marks the word being read, so it follows the word-grouped window.+ activationTiming.start,+ activationTiming.end,durationFrames,),attributes: [{ name: "data-caption-active-box", value: "isolated" }],@@ -1017,7 +1022,11 @@ function cueElements(kind: "glyph" | "underline",): void => {const activeId = `${atomId}-${suffix}`;- const activeTiming = mode === "current" ? activationTiming : timing;+ // The step paints one whole word, so it follows the word-grouped window in both modes. The+ // wipe travels across the letterforms of one Unit and keeps that Unit's own interval, with+ // `current` still owning its span exclusively.+ const wipes = kind === "glyph" && parameters.karaoke.transition === "wipe";+ const activeTiming = wipes ? (mode === "current" ? exclusiveTiming : timing) : activationTiming;push({id: activeId,parent: atomId,@@ -1031,7 +1040,7 @@ function cueElements(// clip has opened and this was still cutting the outline off at the glyph advance.{ name: "position", value: "absolute" },],- animation: kind === "glyph" && parameters.karaoke.transition === "wipe"+ animation: wipes? karaokeWipeAnimation(parameters, mode, activeTiming.start, activeTiming.end, durationFrames): activationStepAnimation(mode, activeTiming.start, activeTiming.end, durationFrames),attributes: [{ name: kind === "glyph" ? "data-caption-karaoke" : "data-caption-active-underline", value: mode }],diff --git a/packages/caption-fine/test/caption-fine.test.ts b/packages/caption-fine/test/caption-fine.test.tsindex c2a331d1..697cdaf4 100644--- a/packages/caption-fine/test/caption-fine.test.ts+++ b/packages/caption-fine/test/caption-fine.test.ts@@ -333,12 +333,15 @@ function numericStyleAt(element: import("@hypit/composition").VisualElement, fra}for (const mode of ["current", "trail"] as const) {- test(`Chinese ${mode} karaoke, underline and boxes follow unequal character times and pauses`, () => {- const { track, windows } = unevenChineseCaption({+ test(`Chinese ${mode} karaoke, underline and boxes light whole words across unequal times and pauses`, () => {+ const { track } = unevenChineseCaption({karaoke: mode, "active-underline": mode, "active-box": mode,});const present = track.presents[0]!;- for (const [index, [start, end]] of windows.entries()) {+ // `你真好看` reads as 你 / 真好 / 看, so the two characters of 真好 share one window covering+ // both of their measured intervals and the pause between them.+ const activations = [[10, 13], [13, 42], [13, 42], [42, 55]] as const;+ for (const [index, [start, end]] of activations.entries()) {for (const suffix of ["active", "underline", "box"]) {const element = present.elements.find((item) => item.id === `atom-${index + 1}-${suffix}`)!;assert.ok(element);