Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

fix(caption-fine): light whole CJK words in step karaoke

rpone5c11b499e84b
diff --git a/packages/caption-fine/src/karaoke.ts b/packages/caption-fine/src/karaoke.ts
new file mode 100644
index 00000000..7b4c9323
--- /dev/null
+++ b/packages/caption-fine/src/karaoke.ts
@@ -0,0 +1,70 @@
+import type { CaptionAlignmentUnit } from "@hypit/narrative";
+
+import { wordGaps } from "./spacing.js";
+
+type ActivationFrames = ReadonlyMap<string, { readonly start: number; readonly end: number }>;
+
+/**
+ * Karaoke repaints one Alignment Unit at a time. A Unit is one lexical unit, and in Han, Hiragana
+ * and Katakana that unit is a single character, so the highlight crawls character by character
+ * across a word the reader takes in at once.
+ *
+ * Reading the Cue as prose recovers its words. `wordGaps` says which boundaries carry a space, so
+ * the surfaces join back into exactly the prose the renderer draws, and the runtime's word
+ * segmenter returns the words inside it. A word that covers several Units gives them one shared
+ * activation window, so the whole word lights together.
+ *
+ * Only the window is shared. Layout, spacing, per-Unit motion and the CaptionDocument are the same
+ * values they were, so speech tokens, authored markers and measured timing are untouched.
+ *
+ * Segmentation is advisory here: it decides which characters light together and nothing else. A
+ * word the segmenter splits differently changes how the highlight steps, never the timing beneath
+ * it. Scripts written without spaces that arrive as one Unit, such as Thai, Khmer and Lao, keep the
+ * highlight they have, because a Unit is the smallest thing that carries a measured window and this
+ * pass can join Units but cannot divide one.
+ */
+const segmenter = new Intl.Segmenter(undefined, { granularity: "word" });
+
+export function karaokeWordActivation(
+ atoms: readonly CaptionAlignmentUnit[],
+ wordText: ReadonlyMap<string, string>,
+ activation: ActivationFrames,
+): ActivationFrames {
+ if (atoms.length < 2) return activation;
+
+ const surfaces = atoms.map((atom) => atom.wordIds.map((wordId) => wordText.get(wordId) ?? "").join(""));
+ const gaps = wordGaps(surfaces);
+
+ let prose = "";
+ const spans = surfaces.map((surface, index) => {
+ if (gaps[index]) prose += " ";
+ const start = prose.length;
+ prose += surface;
+ return { start, end: prose.length };
+ });
+
+ const merged = new Map(activation);
+ for (const word of segmenter.segment(prose)) {
+ if (!word.isWordLike) continue;
+ const wordEnd = word.index + word.segment.length;
+ const members = spans
+ .map((span, index) => ({ span, index }))
+ .filter(({ span }) => span.start < wordEnd && span.end > word.index)
+ .map(({ index }) => index);
+ if (members.length < 2) continue;
+
+ let start = Number.POSITIVE_INFINITY;
+ let end = Number.NEGATIVE_INFINITY;
+ for (const member of members) {
+ // The exclusive window already ends each Unit where the next one starts, so the run's own
+ // bounds are the shared window. Read them from the incoming map: a Unit that two words touch
+ // must not compound the widening.
+ const timing = activation.get(atoms[member]!.id);
+ if (timing === undefined) return activation;
+ start = Math.min(start, timing.start);
+ end = Math.max(end, timing.end);
+ }
+ for (const member of members) merged.set(atoms[member]!.id, { start, end });
+ }
+ return merged;
+}
diff --git a/packages/caption-fine/src/render.ts b/packages/caption-fine/src/render.ts
index 6b330af8..b47b225a 100644
--- a/packages/caption-fine/src/render.ts
+++ b/packages/caption-fine/src/render.ts
@@ -20,6 +20,7 @@ import type { CanvasSpace, SpatialFrame, SpatialRegionTimeline } from "@hypit/sp
import { assertFineCaptionParameters, FINE_CAPTION_FAMILY } from "./style.js";
import { assertFineCaptionSchedule } from "./schedule.js";
+import { karaokeWordActivation } from "./karaoke.js";
import { joinSurfaces, uniformGap, wordGapBetween, wordGaps } from "./spacing.js";
import type {
FineCaptionActiveUnderline,
@@ -692,7 +693,8 @@ function cueElements(
})();
const cueMotion = cueAnimation(parameters, durationFrames);
const cueLoop = parameters.motion.loopTarget === "cue" ? loopAnimation(parameters, durationFrames) : undefined;
- const activationFrames = exclusiveActivationFrames(atoms, atomFrames);
+ const exclusiveFrames = exclusiveActivationFrames(atoms, atomFrames);
+ const activationFrames = karaokeWordActivation(atoms, wordText, exclusiveFrames);
const gapPx = `${compactNumber(parameters.layout.wordGapPx)}px`;
const atomSurfaces = atoms.map((atom) => atom.wordIds.map((wordId) => wordText.get(wordId) ?? ""));
// A Cue whose boundaries all agree carries one `column-gap`, which is also what keeps a row that
@@ -844,6 +846,8 @@ function cueElements(
if (timing === undefined) throw new Error(`Fine Caption is missing timing for Atom ${atom.id}`);
const activationTiming = activationFrames.get(atom.id);
if (activationTiming === undefined) throw new Error(`Fine Caption is missing activation timing for Atom ${atom.id}`);
+ const exclusiveTiming = exclusiveFrames.get(atom.id);
+ if (exclusiveTiming === undefined) throw new Error(`Fine Caption is missing exclusive timing for Atom ${atom.id}`);
const atomId = `atom-${atomIndex + 1}`;
const surfaces = atomSurfaces[atomIndex] ?? [];
// The base glyphs and the activated copy stacked over them are laid out from this one list, so
@@ -945,8 +949,9 @@ function cueElements(
],
animation: activeBoxAnimation(
parameters,
- parameters.activeBox.mode === "current" ? activationTiming.start : timing.start,
- parameters.activeBox.mode === "current" ? activationTiming.end : timing.end,
+ // The capsule marks the word being read, so it follows the word-grouped window.
+ activationTiming.start,
+ activationTiming.end,
durationFrames,
),
attributes: [{ name: "data-caption-active-box", value: "isolated" }],
@@ -1017,7 +1022,11 @@ function cueElements(
kind: "glyph" | "underline",
): void => {
const activeId = `${atomId}-${suffix}`;
- const activeTiming = mode === "current" ? activationTiming : timing;
+ // The step paints one whole word, so it follows the word-grouped window in both modes. The
+ // wipe travels across the letterforms of one Unit and keeps that Unit's own interval, with
+ // `current` still owning its span exclusively.
+ const wipes = kind === "glyph" && parameters.karaoke.transition === "wipe";
+ const activeTiming = wipes ? (mode === "current" ? exclusiveTiming : timing) : activationTiming;
push({
id: activeId,
parent: atomId,
@@ -1031,7 +1040,7 @@ function cueElements(
// clip has opened and this was still cutting the outline off at the glyph advance.
{ name: "position", value: "absolute" },
],
- animation: kind === "glyph" && parameters.karaoke.transition === "wipe"
+ animation: wipes
? karaokeWipeAnimation(parameters, mode, activeTiming.start, activeTiming.end, durationFrames)
: activationStepAnimation(mode, activeTiming.start, activeTiming.end, durationFrames),
attributes: [{ name: kind === "glyph" ? "data-caption-karaoke" : "data-caption-active-underline", value: mode }],
diff --git a/packages/caption-fine/test/caption-fine.test.ts b/packages/caption-fine/test/caption-fine.test.ts
index c2a331d1..697cdaf4 100644
--- a/packages/caption-fine/test/caption-fine.test.ts
+++ b/packages/caption-fine/test/caption-fine.test.ts
@@ -333,12 +333,15 @@ function numericStyleAt(element: import("@hypit/composition").VisualElement, fra
}
for (const mode of ["current", "trail"] as const) {
- test(`Chinese ${mode} karaoke, underline and boxes follow unequal character times and pauses`, () => {
- const { track, windows } = unevenChineseCaption({
+ test(`Chinese ${mode} karaoke, underline and boxes light whole words across unequal times and pauses`, () => {
+ const { track } = unevenChineseCaption({
karaoke: mode, "active-underline": mode, "active-box": mode,
});
const present = track.presents[0]!;
- for (const [index, [start, end]] of windows.entries()) {
+ // `你真好看` reads as 你 / 真好 / 看, so the two characters of 真好 share one window covering
+ // both of their measured intervals and the pause between them.
+ const activations = [[10, 13], [13, 42], [13, 42], [42, 55]] as const;
+ for (const [index, [start, end]] of activations.entries()) {
for (const suffix of ["active", "underline", "box"]) {
const element = present.elements.find((item) => item.id === `atom-${index + 1}-${suffix}`)!;
assert.ok(element);