Skip to content

Open source CapCut alternative.

TypeScript

feat(site): word effects reach captions, preview, and export [rebuild]

David Leab6c6ed61417
diff --git a/site/src/cut/components/OverlayLayer.tsx b/site/src/cut/components/OverlayLayer.tsx
index 7d2c2313..17577a26 100644
--- a/site/src/cut/components/OverlayLayer.tsx
+++ b/site/src/cut/components/OverlayLayer.tsx
@@ -10,14 +10,12 @@ import {
captionStyle,
cueAt,
cueOverlay,
- cueWordWindows,
- karaokeLook,
laneCues,
laneHidden,
subtitleLaneCount,
trackPos,
} from "@/cut/lib/subtitles";
-import { contrastText, evalOverlayFrame, glyphStateAt, hasGlyphMotion, hasMaskKeys, hasOverlayKeys, isOverlayAnimated, lineLikeShape, maskFrameAt, overlayWords, paintMaskCoverage, resolveShadow, shapeMetrics, shapePathD, textStretch, WORD_ACCENT_DEFAULT, wordAccent, wordAccentIndex, type LottieHandle, type Mask, type MaskKey, type OverlayFrameState } from "@donkeycut/effects-kit";
+import { evalOverlayFrame, glyphStateAt, hasGlyphMotion, hasMaskKeys, hasOverlayKeys, isOverlayAnimated, lineLikeShape, maskFrameAt, overlayWords, paintMaskCoverage, resolveShadow, shapeMetrics, shapePathD, textStretch, WORD_ACCENT_DEFAULT, wordDrawsAt, type LottieHandle, type Mask, type MaskKey, type OverlayFrameState, type WordDraw } from "@donkeycut/effects-kit";
import {
LINE_HEIGHT,
PLATE_PAD_X,
@@ -25,7 +23,7 @@ import {
PLATE_RADIUS,
plateFill,
SHADOW,
- wordAccentCss,
+ wordDrawCss,
} from "@/cut/lib/textRender";
import {
behindSubjectOverlay,
@@ -348,10 +346,19 @@ export function OverlayLayer({
);
}
-/** Text with one word emphasized. The words are walked exactly the way the
- * canvas painter walks them — line by line, single-spaced — so the index the
- * preview lights is the index the burn-in lights. */
-function AccentedText({ text, index, css }: { text: string; index: number; css: CSSProperties }) {
+/** Text drawn word by word, each word wearing what the word engine resolved
+ * for it. The words are walked exactly the way the canvas painter walks them
+ * — line by line, single-spaced — so the word the preview lights is the word
+ * the burn-in lights. */
+function WordText({
+ text,
+ draws,
+ lineHeight,
+}: {
+ text: string;
+ draws: WordDraw[];
+ lineHeight?: number;
+}) {
let k = 0;
return (
<>
@@ -361,12 +368,11 @@ function AccentedText({ text, index, css }: { text: string; index: number; css:
.split(" ")
.filter(Boolean)
.map((w, wi) => {
- const active = k === index;
- k++;
+ const d = draws[k++];
return (
<span key={wi}>
{wi > 0 && " "}
- <span style={active ? css : undefined}>{w}</span>
+ <span style={d ? wordDrawCss(d, lineHeight) : undefined}>{w}</span>
</span>
);
})}
@@ -427,24 +433,16 @@ function SubtitleCaption({
// Captions ride the same style/opener/anchor logic as the export burn-in,
// so the preview and the rendered file match exactly.
const style = captionStyle(subtitles.style);
+ // The track's word effect plays on the playhead: the cue resolves its words
+ // at this moment exactly as the export burn-in resolves them at that frame.
const ov = cueOverlay(
cue,
style,
cue.id === cues[0]?.id,
trackPos(subtitles, style, lane),
- undefined,
+ subtitles.wordHighlight ? t : undefined,
frame.w
);
- // Karaoke: the word under the playhead lights up as it is spoken.
- const wordIndex = subtitles.wordHighlight
- ? cueWordWindows(cue).findIndex((w) => t >= w.start && t < w.end)
- : -1;
- // The spoken word's treatment follows the style (with user overrides): a
- // swell in the accent color, an accent box (drawn with box-shadow spread so
- // the line never reflows), the accent color alone, or accent color +
- // underline. The swell is a transform, so it too leaves the line alone.
- const look = karaokeLook(style, subtitles);
- const activeStyle = wordAccentCss(look);
const scale = stageWidth / frame.w;
return (
<div
@@ -484,11 +482,7 @@ function SubtitleCaption({
borderRadius: ov.plate ? PLATE_RADIUS_EM : undefined,
}}
>
- {wordIndex < 0 ? (
- ov.text
- ) : (
- <AccentedText text={ov.text} index={wordIndex} css={activeStyle} />
- )}
+ {ov.wordDraw ? <WordText text={ov.text} draws={ov.wordDraw} /> : ov.text}
</div>
);
}
@@ -584,25 +578,22 @@ function OverlayItem({
// box from its left edge. Neither runs while the box is being edited.
const glyphs = isText && !editing && live && hasGlyphMotion(live) ? live : null;
const reveal = !editing ? live?.reveal : undefined;
- // Word emphasis: the word being said wears the accent while the rest of the
- // line stays as it is. Off while editing — the box is plain text then — and
- // a per-glyph ramp takes the frame for itself while it runs.
+ // Word effects: the line plays word by word on its own clock. Off while
+ // editing — the box is plain text then — and a per-glyph ramp takes the
+ // frame for itself while it runs.
const words = isText && !editing ? overlayWords(o) : undefined;
- // A rehearsal from the inspector sweeps the emphasis along the line on its
- // own clock; otherwise it follows the playhead, like everything else here.
+ // A rehearsal from the inspector sweeps the effect along the line on its own
+ // clock; otherwise it follows the playhead, like everything else here.
const wordLocal = running ? running.tLocal : Math.max(0, t - o.start);
- const wordIndex =
- words && !glyphs ? wordAccentIndex(o, wordLocal, Math.max(0.1, o.end - o.start)) : -1;
- const wordColor = wordAccent(words, isTextOverlay(o) ? o.color : WORD_ACCENT_DEFAULT);
- const wordCss =
- words && wordIndex >= 0
- ? wordAccentCss({
- mode: words.style,
- color: wordColor,
- text: contrastText(wordColor),
- scale: words.scale,
- lineHeight: (isTextOverlay(o) ? o.lineHeight : undefined) ?? LINE_HEIGHT,
- })
+ const wordDraw =
+ words && !glyphs && isTextOverlay(o)
+ ? wordDrawsAt(
+ o.text,
+ words,
+ o.color ?? WORD_ACCENT_DEFAULT,
+ Math.max(0.1, o.end - o.start),
+ wordLocal
+ )
: null;
// Typewriter: the visible slice of the text (display only, never while
// the box is being edited).
@@ -1053,8 +1044,12 @@ function OverlayItem({
</div>
) : glyphs ? (
<GlyphText text={shownText} phase={glyphs} scale={scale} />
- ) : wordCss ? (
- <AccentedText text={shownText} index={wordIndex} css={wordCss} />
+ ) : wordDraw ? (
+ <WordText
+ text={shownText}
+ draws={wordDraw}
+ lineHeight={(isTextOverlay(o) ? o.lineHeight : undefined) ?? LINE_HEIGHT}
+ />
) : (
<span>{shownText}</span>
)
diff --git a/site/src/cut/lib/behindPass.ts b/site/src/cut/lib/behindPass.ts
index af6750f6..61fb7bf4 100644
--- a/site/src/cut/lib/behindPass.ts
+++ b/site/src/cut/lib/behindPass.ts
@@ -10,14 +10,13 @@
*/
import {
- applyWordAccent,
+ applyWordDraw,
evalOverlayFrame,
glyphStateAt,
hasGlyphMotion,
maskComposite,
overlayWords,
- wordAccentIndex,
- wordWindows,
+ wordSampleWindows,
} from "@donkeycut/effects-kit";
import { personSegmenter, segmentSubjectAlpha } from "./cutout";
import { createRasterCanvas } from "./raster";
@@ -37,13 +36,33 @@ type Segmenter = import("@mediapipe/tasks-vision").ImageSegmenter;
/** An element's own length, the span its word emphasis is timed across. */
const spanOf = (o: Overlay): number => Math.max(0.1, o.end - o.start);
-/** The word indices an element needs a picture for: one per word while it
- * lights them as they are said, and the plain picture (-1) otherwise. */
-function rasterWords(o: Overlay): number[] {
+/** Pictures here are full-frame, so a word effect that ramps is sampled
+ * coarsely: enough slices that an arrival reads, few enough that a behind-
+ * subject line does not hold a hundred frames of pixels resident. */
+const BEHIND_WORD_FPS = 8;
+
+/** The spans an element's word effect holds still for — one picture each.
+ * Nothing here means the element draws as one piece. */
+function rasterSpans(o: Overlay): { start: number; end: number }[] | null {
const words = overlayWords(o);
- if (!words || !isTextOverlay(o)) return [-1];
- const n = wordWindows(o.text, spanOf(o), words.times).length;
- return n > 0 ? Array.from({ length: n }, (_, i) => i) : [-1];
+ if (!words || !isTextOverlay(o)) return null;
+ const spans = wordSampleWindows(o.text, words, spanOf(o), BEHIND_WORD_FPS);
+ return spans.length > 0 ? spans : null;
+}
+
+/** The picture indices an element needs: one per span while a word effect
+ * walks it, and the plain picture (-1) otherwise. */
+function rasterSlices(o: Overlay): number[] {
+ const spans = rasterSpans(o);
+ return spans ? spans.map((_, i) => i) : [-1];
+}
+
+/** Which picture stands for `tLocal`, or -1 for the plain one. */
+function rasterSliceAt(o: Overlay, tLocal: number): number {
+ const spans = rasterSpans(o);
+ if (!spans) return -1;
+ const i = spans.findIndex((s) => tLocal >= s.start && tLocal < s.end);
+ return i < 0 ? spans.length - 1 : i;
}
/** Segmentation input width — small on purpose; this runs per frame. */
@@ -144,8 +163,8 @@ export class SubjectMaskCompositor {
return null;
}
- /** One element picture, with the word it emphasizes at this moment baked
- * in, kept under the element it came from. */
+ /** One element picture, with its words drawn as they stand across the span
+ * it covers, kept under the element it came from. */
private async drawRaster(
o: Overlay,
w: number,
@@ -153,8 +172,11 @@ export class SubjectMaskCompositor {
assets: MediaAsset[],
word: number
): Promise<void> {
+ const spans = rasterSpans(o);
+ const span = word >= 0 ? spans?.[word] : undefined;
+ const at = span ? (span.start + span.end) / 2 : 0;
const png = await renderElementPng(
- applyWordAccent({ ...o, rotation: undefined, opacity: undefined }, word),
+ applyWordDraw({ ...o, rotation: undefined, opacity: undefined }, at, spanOf(o)),
w,
h,
assets
@@ -176,7 +198,7 @@ export class SubjectMaskCompositor {
behind.flatMap((o) =>
// An element that lights its words needs one picture per word, all of
// them resident before the first frame draws.
- rasterWords(o).map(async (word) => {
+ rasterSlices(o).map(async (word) => {
if (this.rasters.get(o)?.has(word)) return;
try {
await this.drawRaster(o, w, h, assets, word);
@@ -353,7 +375,7 @@ export class SubjectMaskCompositor {
const scale = Math.min(W, H) / 1080;
for (const o of active) {
const tLocal = Math.max(0, t - o.start);
- const bmp = this.rasterFor(o, W, H, assets, wordAccentIndex(o, tLocal, spanOf(o)));
+ const bmp = this.rasterFor(o, W, H, assets, rasterSliceAt(o, tLocal));
if (!bmp) continue;
const ev = evalOverlayFrame(o, tLocal);
// One cached picture per element here, so a per-glyph ramp or loop runs
diff --git a/site/src/cut/lib/captionWords.test.ts b/site/src/cut/lib/captionWords.test.ts
new file mode 100644
index 00000000..9dad9d4c
--- /dev/null
+++ b/site/src/cut/lib/captionWords.test.ts
@@ -0,0 +1,120 @@
+import { describe, expect, test } from "bun:test";
+import { CAPTION_STYLES, cueOverlay, cueWordFrames, cueWords, trackPos } from "./subtitles";
+import { emptySubtitles, type SubtitleCue, type SubtitlesBlock } from "./types";
+
+/**
+ * A caption's word effect against the speech it was measured from.
+ *
+ * The clock is the transcriber's own onsets, so a cue whose first word runs
+ * most of its length lights that word for most of it. What these check is that
+ * every side of the caption path reads that same clock: the picture the preview
+ * and the burn-in draw, and the spans the burn-in cuts to draw them at.
+ */
+
+const ACCENT = "#FF3D00";
+const STYLE = { ...CAPTION_STYLES.clean, accentMode: "color", accent: ACCENT };
+
+/** "one two three" with the first word holding four fifths of the cue. */
+const cue: SubtitleCue = {
+ id: "c1",
+ start: 10,
+ end: 12,
+ text: "one two three",
+ words: [
+ { t0: 10, t1: 11.6, w: "one" },
+ { t0: 11.6, t1: 11.8, w: "two" },
+ { t0: 11.8, t1: 12, w: "three" },
+ ],
+};
+
+/** Which display word wears the accent in a drawn cue, or -1. */
+const lit = (at: number, subs?: Partial<SubtitlesBlock>) =>
+ cueOverlay(
+ cue,
+ STYLE,
+ false,
+ trackPos({ ...emptySubtitles(), ...subs } as SubtitlesBlock, STYLE, 0),
+ at,
+ 1080
+ ).wordDraw?.findIndex((d) => d.color.toLowerCase() === ACCENT.toLowerCase()) ?? -1;
+
+describe("a caption's word clock", () => {
+ test("follows the transcriber's onsets, not an even share of the cue", () => {
+ // Halfway through the cue the measured speech is still on word 0; a split
+ // by word length would have moved on by then.
+ expect(lit(11)).toBe(0);
+ expect(lit(11.7)).toBe(1);
+ expect(lit(11.9)).toBe(2);
+ });
+
+ test("shares the span by word length when nothing measured the speech", () => {
+ const flat = { ...cue, words: undefined };
+ const draws = cueOverlay(flat, STYLE, false, undefined, 11, 1080).wordDraw;
+ expect(draws?.length).toBe(3);
+ expect(draws?.some((d) => d.color.toLowerCase() === ACCENT.toLowerCase())).toBe(true);
+ });
+
+ test("cue words carry the onsets the sampler cuts its spans from", () => {
+ const times = cueWords(cue, STYLE).times ?? [];
+ expect(times.length).toBe(3);
+ [0, 1.6, 1.8].forEach((t, i) => expect(times[i]).toBeCloseTo(t, 5));
+ });
+});
+
+describe("burn-in spans", () => {
+ const spans = cueWordFrames(cue, STYLE);
+
+ test("cover the cue end to end with no gaps", () => {
+ expect(spans[0].start).toBeCloseTo(cue.start, 5);
+ expect(spans[spans.length - 1].end).toBeCloseTo(cue.end, 5);
+ for (let i = 1; i < spans.length; i++)
+ expect(spans[i].start).toBeCloseTo(spans[i - 1].end, 5);
+ });
+
+ test("each stands for the word the picture drawn at its middle actually lights", () => {
+ // The span boundaries and the picture inside them have to come off one
+ // clock, or a span burns a word that is not the one it covers.
+ for (const s of spans) {
+ const mid = (s.start + s.end) / 2;
+ expect(lit(mid)).toBe(lit(s.start + (s.end - s.start) * 0.25));
+ }
+ });
+
+ test("an effect that lands on the beat stays one still per word", () => {
+ expect(spans.length).toBe(3);
+ });
+
+ test("a ramping effect is cut finer, and bounded however long its ramp runs", () => {
+ const built = cueWordFrames(cue, { ...STYLE, accentMode: "rise" });
+ expect(built.length).toBeGreaterThan(spans.length);
+ // Three words, at most two extra pictures each.
+ expect(built.length).toBeLessThanOrEqual(9);
+ });
+});
+
+describe("the caption's own settings", () => {
+ test("reach the drawn words", () => {
+ const faint = cueOverlay(
+ cue,
+ { ...STYLE, accentMode: "fill" },
+ false,
+ trackPos({ ...emptySubtitles(), accentDim: 0.6 } as SubtitlesBlock, STYLE, 0),
+ 10,
+ 1080
+ ).wordDraw;
+ expect(faint?.[2].opacity).toBeCloseTo(0.6, 5);
+ });
+
+ test("a swell set on the track lands on the word being said", () => {
+ const swollen = cueOverlay(
+ cue,
+ { ...STYLE, accentMode: "pop" },
+ false,
+ trackPos({ ...emptySubtitles(), accentScale: 1.5 } as SubtitlesBlock, STYLE, 0),
+ 10.5,
+ 1080
+ ).wordDraw;
+ expect(swollen?.[0].scale).toBeCloseTo(1.5, 5);
+ expect(swollen?.[1].scale).toBe(1);
+ });
+});
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index 771f8c3d..d58b95cc 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -14,7 +14,7 @@ import { renderProjectToMp4 } from "./exportRender";
import { putSigned } from "./media";
import { createRasterCanvas, rasterCanvasToPng } from "./raster";
import { clipSpeed, getClipSpans, overlayLayers, projectDuration, spanSequence, useEditor } from "./store";
-import { captionStyle, cueOverlay, cueWordWindows, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
+import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
import { isMaskAnimated, isOverlayAnimated, normalizeGrade, paintMaskLuma } from "@donkeycut/effects-kit";
import { renderElementFrames, renderElementPng } from "./textRender";
import { clipCovers, clipKeyed, clipPosed, clipPoseAt, clipZoom, contentRect, frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, regionPx, shadowInk, subjectMasked } from "./types";
@@ -838,10 +838,10 @@ export async function buildExportPayload(
for (let i = 0; i < cues.length; i++) {
const cue = cues[i];
if (cue.start >= duration || !cue.text.trim()) continue;
- // Karaoke burns one frame per word window (the spoken word accented);
+ // A word effect burns one frame per span its picture holds still for;
// otherwise the whole cue is a single still.
const windows = doc.subtitles.wordHighlight
- ? cueWordWindows(cue)
+ ? cueWordFrames(cue, capStyle, doc.subtitles)
: [{ start: cue.start, end: cue.end }];
for (let wi = 0; wi < windows.length; wi++) {
const win = windows[wi];
@@ -852,7 +852,7 @@ export async function buildExportPayload(
capStyle,
i === 0,
pos,
- doc.subtitles.wordHighlight ? wi : undefined,
+ doc.subtitles.wordHighlight ? (win.start + win.end) / 2 : undefined,
// Wrap in design space (1080 short side) from the project ratio —
// the same width the preview passes, whatever the render size.
frameOf(doc.aspect).w
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index cd277acf..a83e8d8d 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -33,7 +33,7 @@ import { FrameCompositor, MISSING_FRAME, type Frame } from "./composite";
import { overlayPlan, trackZeroPlan } from "./framePlan";
import { frameSink, openMedia, videoTrackOf } from "./mediaRead";
import { getClipSpans, overlayLayers, projectDuration, spanSequence } from "./store";
-import { captionStyle, cueOverlay, cueWordWindows, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
+import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
import { applyEffectToCanvas, evalOverlayFrame, grainTile, isAudioEffect, isMaskAnimated, isOverlayAnimated, maskFrameAt, planAnimatedLayers, type LottieHandle, type OverlayAnim, type PaintPhase } from "@donkeycut/effects-kit";
import { hasSubjectOverlays, SubjectMaskCompositor } from "./behindPass";
import { createRasterCanvas, type RasterSurface } from "./raster";
@@ -331,7 +331,7 @@ async function stampText(doc: ExportDoc): Promise<StampedLayer[]> {
const cue = cues[i];
if (cue.start >= duration || !cue.text.trim()) continue;
const windows = doc.subtitles.wordHighlight
- ? cueWordWindows(cue)
+ ? cueWordFrames(cue, capStyle, doc.subtitles)
: [{ start: cue.start, end: cue.end }];
for (let wi = 0; wi < windows.length; wi++) {
const win = windows[wi];
@@ -342,7 +342,7 @@ async function stampText(doc: ExportDoc): Promise<StampedLayer[]> {
capStyle,
i === 0,
pos,
- doc.subtitles.wordHighlight ? wi : undefined,
+ doc.subtitles.wordHighlight ? (win.start + win.end) / 2 : undefined,
designWidth
),
start: win.start,
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index 101835e2..e5969c7a 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -579,7 +579,7 @@ export interface EditorState {
* track's language. Timings copy over; word timings don't survive
* translation, so the new cues carry none. */
translateSubtitleTrack: (fromLane: number) => Promise<void>;
- setSubtitlesView: (patch: Partial<Pick<SubtitlesBlock, "showOnVideo" | "showOnTimeline" | "locale" | "style" | "size" | "font" | "x" | "y" | "wordHighlight" | "accentMode" | "accentColor">>) => void;
+ setSubtitlesView: (patch: Partial<Pick<SubtitlesBlock, "showOnVideo" | "showOnTimeline" | "locale" | "style" | "size" | "font" | "x" | "y" | "wordHighlight" | "accentMode" | "accentColor" | "accentScale" | "accentDim">>) => void;
/** The subtitle track (row) the panel edits and generation writes to. */
subtitleLane: number;
setSubtitleLane: (lane: number) => void;
diff --git a/site/src/cut/lib/subtitles.ts b/site/src/cut/lib/subtitles.ts
index 1db18275..e8ce7748 100644
--- a/site/src/cut/lib/subtitles.ts
+++ b/site/src/cut/lib/subtitles.ts
@@ -1,10 +1,10 @@
import {
contrastText as kitContrastText,
+ wordDrawsAt,
WORD_ACCENT_DEFAULT,
- WORD_ACCENT_LABELS,
- WORD_ACCENT_MODE_IDS,
- WORD_ACCENT_NOTES,
- wordWindows,
+ wordEffect,
+ wordSampleWindows,
+ type OverlayWords,
} from "@donkeycut/effects-kit";
import { formatTime } from "./time";
import type {
@@ -13,7 +13,7 @@ import type {
SubtitleCue,
SubtitlesBlock,
TextOverlay,
- WordAccentMode,
+ WordEffectId,
} from "./types";
export type { CaptionStyleId } from "./types";
@@ -46,16 +46,33 @@ export function trackLocale(subs: SubtitlesBlock, lane: number): string {
return subs.tracks?.[lane]?.locale ?? (lane === 0 ? subs.locale : undefined) ?? "en-US";
}
-/** A track's effective caption anchor plus the block's look overrides
- * (karaoke accent, font size) — the `pos` argument `cueOverlay` takes. The
- * track's own dragged spot wins, then the block-level legacy spot (first
- * track only), then the style's default stacked upward per track so
- * simultaneous languages never overlap. */
+/** The block-level word-effect overrides a caption track carries: which
+ * effect, and the two settings an effect may have. Absent = the caption
+ * style's own. */
+export interface CaptionWordOverrides {
+ accentMode?: WordEffectId;
+ accentColor?: string;
+ accentScale?: number;
+ accentDim?: number;
+}
+
+/** A track's effective caption anchor plus the block's look overrides — what
+ * `cueOverlay` reads to place and dress one cue. */
+export interface CaptionPos extends CaptionWordOverrides {
+ x: number;
+ y: number;
+ size?: number;
+ font?: FontId;
+}
+
+/** One track's anchor, filled in: the track's own dragged spot wins, then the
+ * block-level legacy spot (first track only), then the style's default
+ * stacked upward per track so simultaneous languages never overlap. */
export function trackPos(
subs: SubtitlesBlock,
style: CaptionStyle,
lane: number
-): { x: number; y: number; size?: number; font?: FontId; accentMode?: WordAccentMode; accentColor?: string } {
+): CaptionPos {
const meta = subs.tracks?.[lane];
return {
x: meta?.x ?? (lane === 0 ? subs.x : undefined) ?? style.x,
@@ -64,6 +81,8 @@ export function trackPos(
font: subs.font,
accentMode: subs.accentMode,
accentColor: subs.accentColor,
+ accentScale: subs.accentScale,
+ accentDim: subs.accentDim,
};
}
@@ -97,48 +116,35 @@ export interface CaptionStyle {
plateOpacity?: number;
/** Multiply the opening cue's size, so the hook lands bigger. */
openerScale?: number;
- /** Karaoke accent for the spoken word; absent = DEFAULT_ACCENT. */
+ /** Accent color for the word effect; absent = DEFAULT_ACCENT. */
accent?: string;
- /** How the spoken word lights up: an accent box behind it (plate styles) or
- * the accent color plus underline (open text). Absent = underline. */
- accentMode?: WordAccentMode;
- /** Word color on an accent box; absent = auto contrast. */
- accentText?: string;
+ /** Which word effect the style's captions run: an accent box behind the
+ * spoken word (plate styles), a line building itself up, and so on. Absent
+ * = underline. */
+ accentMode?: WordEffectId;
}
/** Spoken-word accent for styles that don't pick their own. */
export const DEFAULT_ACCENT = WORD_ACCENT_DEFAULT;
-/** The word emphasis treatments, in the order every picker offers them —
- * the caption panel, the element animation grid, and the assistant's tool
- * enums all read this one list, so a new treatment reaches all of them. */
-export const WORD_ACCENT_MODES: { id: WordAccentMode; label: string; hint: string }[] =
- WORD_ACCENT_MODE_IDS.map((id) => ({
- id,
- label: WORD_ACCENT_LABELS[id],
- hint: WORD_ACCENT_NOTES[id],
- }));
-
/** Black or white, whichever reads on the given hex fill. */
export const contrastText = kitContrastText;
-/** The effective karaoke treatment for the spoken word: user overrides on the
- * subtitles block win over the caption style's defaults, a swell with no
- * accent named keeps the caption's own color, and a custom color picks its
- * own contrast text for the box treatment. */
-export function karaokeLook(
- style: CaptionStyle,
- subs?: { accentMode?: WordAccentMode; accentColor?: string }
-): { mode: WordAccentMode; color: string; text: string } {
- const mode = subs?.accentMode ?? style.accentMode ?? "underline";
- // A swell says it with size, so with nobody naming an accent it keeps the
- // color the rest of the caption is written in.
+/** The word effect a caption track runs, as the element model carries it:
+ * user overrides on the subtitles block win over the caption style's
+ * defaults, and an effect that says everything with size keeps the color the
+ * rest of the caption is written in when nobody named an accent. */
+export function captionWords(style: CaptionStyle, subs?: CaptionWordOverrides): OverlayWords {
+ const id = subs?.accentMode ?? style.accentMode ?? "underline";
const color =
- subs?.accentColor ?? style.accent ?? (mode === "pop" ? style.color : DEFAULT_ACCENT);
+ subs?.accentColor ??
+ style.accent ??
+ (wordEffect(id).accentDefault === "text" ? style.color : DEFAULT_ACCENT);
return {
- mode,
+ style: id,
color,
- text: subs?.accentColor ? contrastText(color) : style.accentText ?? contrastText(color),
+ ...(subs?.accentScale !== undefined ? { scale: subs.accentScale } : {}),
+ ...(subs?.accentDim !== undefined ? { dim: subs.accentDim } : {}),
};
}
@@ -253,7 +259,6 @@ export const CAPTION_STYLES: Record<CaptionStyleId, CaptionStyle> = {
plateOpacity: 0.95,
accent: "#FF375F",
accentMode: "box",
- accentText: "#FFFFFF",
},
bubble: {
id: "bubble",
@@ -330,41 +335,27 @@ export function wrapCaptionForSize(text: string, size: number, frameW: number):
return lines.join("\n");
}
-/**
- * Per-display-word timeline windows for a cue, partitioning [start, end] with
- * no gaps so the karaoke highlight never blinks off between words. Uses the
- * transcriber's word timings while they still match the text one-to-one;
- * otherwise (hand edits, social rewrites) splits proportionally by word length.
- */
-export function cueWordWindows(cue: SubtitleCue): { start: number; end: number }[] {
- const dur = Math.max(0, cue.end - cue.start);
- return wordWindows(
- cue.text,
- dur,
- cue.words?.map((w) => w.t0 - cue.start)
- ).map((w) => ({ start: cue.start + w.start, end: cue.start + w.end }));
-}
-
/** A cue as a synthetic overlay, so captions ride the title pipeline. The style
* preset drives the look; a user font or font size overrides the preset's
* (the opener emphasis still multiplies the size); a dragged caption position
- * (frame fractions) overrides the preset's spot; a word index marks that word
- * for the karaoke accent. */
+ * (frame fractions) overrides the preset's spot; `at` resolves the track's
+ * word effect at that moment on the timeline. */
export function cueOverlay(
cue: SubtitleCue,
style: CaptionStyle = CAPTION_STYLES.clean,
isOpener = false,
- pos: { x?: number; y?: number; size?: number; font?: FontId; accentMode?: WordAccentMode; accentColor?: string } | undefined,
- wordIndex: number | undefined,
+ pos: Partial<CaptionPos> | undefined,
+ at: number | undefined,
frameW: number
): TextOverlay {
- const kl = wordIndex !== undefined ? karaokeLook(style, pos) : null;
+ const words = at !== undefined ? cueWords(cue, style, pos) : null;
const size = Math.round(
(pos?.size ?? style.size) * (isOpener && style.openerScale ? style.openerScale : 1)
);
+ const text = wrapCaptionForSize(cue.text, size, frameW);
return {
id: `sub-${cue.id}`,
- text: wrapCaptionForSize(cue.text, size, frameW),
+ text,
start: cue.start,
end: cue.end,
x: pos?.x ?? style.x,
@@ -377,17 +368,63 @@ export function cueOverlay(
plate: style.plate,
...(style.plateColor ? { plateColor: style.plateColor } : {}),
...(style.plateOpacity !== undefined ? { plateOpacity: style.plateOpacity } : {}),
- ...(kl && wordIndex !== undefined
+ ...(words && at !== undefined
? {
- highlightWord: wordIndex,
- highlightColor: kl.color,
- highlightMode: kl.mode,
- highlightText: kl.text,
+ wordDraw: wordDrawsAt(
+ text,
+ words,
+ style.color,
+ Math.max(0, cue.end - cue.start),
+ at - cue.start
+ ),
}
: {}),
};
}
+/** A caption's pictures are full frames, one per span, and a cut carries one
+ * of them for every word of speech in it. A ramp is sampled at this rate, and
+ * never at more than this many steps however long the ramp runs, which is
+ * enough to read an arrival while holding a subtitled export to a few
+ * pictures per word. */
+const CAPTION_WORD_FPS = 12;
+const CAPTION_WORD_SLICES = 2;
+
+/**
+ * One cue's word effect with the transcriber's own onsets on it. The clock is
+ * the measured speech wherever there is any, so the effect walks the line at
+ * the pace it is actually said; a cue with no timings shares its span by word
+ * length instead. Everything that draws or samples a cue reads this one
+ * function, so the picture a burn-in span stands for is resolved against the
+ * same windows the span was cut from.
+ */
+export function cueWords(
+ cue: SubtitleCue,
+ style: CaptionStyle,
+ subs?: CaptionWordOverrides
+): OverlayWords {
+ const times = cue.words?.map((w) => w.t0 - cue.start);
+ return { ...captionWords(style, subs), ...(times ? { times } : {}) };
+}
+
+/**
+ * The moments a cue has to be drawn at to carry its word effect: one span per
+ * stretch the picture holds still. An effect that lands on the beat comes
+ * back as one span per word, which is what the caption burn-in has always
+ * written; one that ramps is cut finer inside a word's window.
+ */
+export function cueWordFrames(
+ cue: SubtitleCue,
+ style: CaptionStyle,
+ subs?: CaptionWordOverrides
+): { start: number; end: number }[] {
+ const dur = Math.max(0, cue.end - cue.start);
+ return wordSampleWindows(cue.text, cueWords(cue, style, subs), dur, CAPTION_WORD_FPS, CAPTION_WORD_SLICES).map((w) => ({
+ start: cue.start + w.start,
+ end: cue.start + w.end,
+ }));
+}
+
/** The cue under a given time, if any. */
export function cueAt(cues: SubtitleCue[], t: number): SubtitleCue | undefined {
return cues.find((c) => t >= c.start && t < c.end);
diff --git a/site/src/cut/lib/textLooks.ts b/site/src/cut/lib/textLooks.ts
index 9bad0c38..976eca34 100644
--- a/site/src/cut/lib/textLooks.ts
+++ b/site/src/cut/lib/textLooks.ts
@@ -1,4 +1,4 @@
-import type { OverlayAnimStyle, OverlayLoopStyle, WordAccentMode } from "@donkeycut/effects-kit";
+import type { OverlayAnimStyle, OverlayLoopStyle, WordEffectId } from "@donkeycut/effects-kit";
import type { TextEnsemble, TextVariation } from "./textCompose";
import type { CaptionStyleId, FontId } from "./types";
@@ -78,7 +78,7 @@ export interface TextLook {
size?: number;
font?: FontId;
accentColor?: string;
- accentMode?: WordAccentMode;
+ accentMode?: WordEffectId;
};
}
diff --git a/site/src/cut/lib/textRender.ts b/site/src/cut/lib/textRender.ts
index 4184f441..f64909da 100644
--- a/site/src/cut/lib/textRender.ts
+++ b/site/src/cut/lib/textRender.ts
@@ -1,18 +1,20 @@
"use client";
import {
+ hexAlpha,
+ WORD_BASELINE_DROP,
renderElementPng as kitRenderElementPng,
renderOverlayFrames as kitRenderOverlayFrames,
LINE_HEIGHT,
- wordSwell,
type OverlayFrameSet,
type PaintPhase,
type RenderEnv,
type StickerImage,
+ type WordDraw,
} from "@donkeycut/effects-kit";
import type { CSSProperties } from "react";
import { createRasterCanvas, decodeRasterImage, rasterCanvasToPng } from "./raster";
-import { fontStack, type MediaAsset, type Overlay, type WordAccentMode } from "./types";
+import { fontStack, type MediaAsset, type Overlay } from "./types";
// The shared text metrics and painters live in the effects kit; these
// re-exports keep the app's preview components on the same constants.
@@ -26,49 +28,51 @@ export {
PLATE_RADIUS,
plateFill,
SHADOW,
- WORD_ACCENT_DEFAULT,
- WORD_ACCENT_LABELS,
- WORD_ACCENT_MODE_IDS,
- WORD_POP_SCALE,
} from "@donkeycut/effects-kit";
-/** The css one emphasized word wears — the DOM twin of what the canvas
- * painter draws for `highlightMode`. A swell is real type size, so the line
- * reflows around it: the words beside it move out of its way and settle back
- * as the emphasis travels on, which is what the burn-in lays out too. The
- * word keeps the line's own leading, so a bigger word never opens the line
- * box up. A box is box-shadow spread, which stays clear of the layout. */
-export function wordAccentCss(look: {
- mode: WordAccentMode;
- color: string;
- text: string;
- /** How far the word swells; absent = the mode's default. */
- scale?: number;
- /** The block's line height, so the swollen word can hold it. */
- lineHeight?: number;
-}): CSSProperties {
- const swell = wordSwell({ style: look.mode, scale: look.scale });
- const lh = look.lineHeight ?? LINE_HEIGHT;
- const size: CSSProperties =
- swell === 1 ? {} : { fontSize: `${swell}em`, lineHeight: lh / swell };
- const treatment: CSSProperties =
- look.mode === "box"
- ? {
- color: look.text,
- background: look.color,
- boxShadow: `0 0 0 0.12em ${look.color}`,
- borderRadius: "0.18em",
- textShadow: "none",
- }
- : look.mode === "color" || look.mode === "pop"
- ? { color: look.color }
- : {
- color: look.color,
- textDecoration: "underline",
- textDecorationThickness: "0.07em",
- textUnderlineOffset: "0.14em",
- };
- return { ...treatment, ...size };
+/** The css one word wears — the DOM twin of what the canvas painter draws for
+ * a resolved `WordDraw`. A resize is real type size, so the line reflows
+ * around it, which is what the burn-in lays out too; the word keeps the
+ * line's own leading, so a bigger word never opens the line box up. A box is
+ * box-shadow spread, which stays clear of the layout. */
+export function wordDrawCss(d: WordDraw, lineHeight = LINE_HEIGHT): CSSProperties {
+ const css: CSSProperties = { color: d.color };
+ if (d.opacity < 0.999) css.opacity = d.opacity;
+ // The word takes its room at the layout size and draws at its own, which
+ // differ only while it is growing into its place: real type size for the
+ // room, so the line reflows around a swollen word exactly as the burn-in
+ // lays it out, and a transform for the rest, which leaves the layout alone.
+ if (d.layoutScale !== 1) {
+ css.fontSize = `${d.layoutScale}em`;
+ css.lineHeight = lineHeight / d.layoutScale;
+ }
+ const shrink = d.scale / d.layoutScale;
+ // A word drawn smaller than the room it holds shares the line's baseline, so
+ // it hangs lower in its box than a scale about the box center would leave
+ // it; the drop makes up the difference. Offsets are em of the size the word
+ // draws at and the box is em of the size it takes room at, so they convert
+ // by the ratio between the two.
+ const sit = (WORD_BASELINE_DROP * (d.layoutScale - d.scale)) / d.layoutScale;
+ if (shrink !== 1 || d.dx || d.dy || d.rotate) {
+ css.display = "inline-block";
+ css.transform =
+ `translate(${d.dx * shrink}em, ${d.dy * shrink + sit}em) rotate(${d.rotate}deg)` +
+ (shrink !== 1 ? ` scale(${shrink})` : "");
+ }
+ const mark = d.markAlpha ?? 0;
+ if (d.mark === "box" && mark > 0.001) {
+ const fill = hexAlpha(d.markColor ?? "#000000", mark);
+ css.background = fill;
+ css.boxShadow = `0 0 0 0.12em ${fill}`;
+ css.borderRadius = "0.18em";
+ css.textShadow = "none";
+ } else if (d.mark === "underline" && mark > 0.001) {
+ css.textDecoration = "underline";
+ css.textDecorationColor = hexAlpha(d.color, mark);
+ css.textDecorationThickness = "0.07em";
+ css.textUnderlineOffset = "0.14em";
+ }
+ return css;
}
/** Decoded sticker images by asset id, shared across one page's renders. An
diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.ts
index 9ff489b6..fa25fc50 100644
--- a/site/src/cut/lib/types.ts
+++ b/site/src/cut/lib/types.ts
@@ -16,7 +16,7 @@ import {
type ShapeOverlay,
type StickerOverlay,
type TextOverlay as KitTextOverlay,
- type WordAccentMode,
+ type WordEffectId,
} from "@donkeycut/effects-kit";
import { getBackend, type CutBackend } from "./backend";
import type { VideoProject } from "./genvideo/types";
@@ -1142,7 +1142,7 @@ export type {
ShapeKind,
ShapeOverlay,
StickerOverlay,
- WordAccentMode,
+ WordEffectId,
};
/** One subtitle caption, timed against the timeline (not the source files). */
@@ -1217,12 +1217,17 @@ export interface SubtitlesBlock {
* per-track anchor in `tracks`. Read as the lane-0 fallback. */
x?: number;
y?: number;
- /** Karaoke mode: each word lights up as it is spoken, in the preview and
- * the export burn-in. */
+ /** Word effects on: the caption plays word by word as it is spoken, in the
+ * preview and the export burn-in. */
wordHighlight?: boolean;
- /** Word treatment overrides; absent = the caption style's defaults. */
- accentMode?: WordAccentMode;
+ /** Which word effect, overriding the caption style's own; absent = the
+ * style's default. */
+ accentMode?: WordEffectId;
accentColor?: string;
+ /** How far a word swells at its moment; absent = the effect's own. */
+ accentScale?: number;
+ /** The opacity a word wears off its moment; absent = the effect's own. */
+ accentDim?: number;
}
export const emptySubtitles = (): SubtitlesBlock => ({