Open source CapCut alternative.
TypeScript
feat(site): word effects reach captions, preview, and export [rebuild]
David Leab6c6ed61417
diff --git a/site/src/cut/components/OverlayLayer.tsx b/site/src/cut/components/OverlayLayer.tsxindex 7d2c2313..17577a26 100644--- a/site/src/cut/components/OverlayLayer.tsx+++ b/site/src/cut/components/OverlayLayer.tsx@@ -10,14 +10,12 @@ import {captionStyle,cueAt,cueOverlay,- cueWordWindows,- karaokeLook,laneCues,laneHidden,subtitleLaneCount,trackPos,} from "@/cut/lib/subtitles";-import { contrastText, evalOverlayFrame, glyphStateAt, hasGlyphMotion, hasMaskKeys, hasOverlayKeys, isOverlayAnimated, lineLikeShape, maskFrameAt, overlayWords, paintMaskCoverage, resolveShadow, shapeMetrics, shapePathD, textStretch, WORD_ACCENT_DEFAULT, wordAccent, wordAccentIndex, type LottieHandle, type Mask, type MaskKey, type OverlayFrameState } from "@donkeycut/effects-kit";+import { evalOverlayFrame, glyphStateAt, hasGlyphMotion, hasMaskKeys, hasOverlayKeys, isOverlayAnimated, lineLikeShape, maskFrameAt, overlayWords, paintMaskCoverage, resolveShadow, shapeMetrics, shapePathD, textStretch, WORD_ACCENT_DEFAULT, wordDrawsAt, type LottieHandle, type Mask, type MaskKey, type OverlayFrameState, type WordDraw } from "@donkeycut/effects-kit";import {LINE_HEIGHT,PLATE_PAD_X,@@ -25,7 +23,7 @@ import {PLATE_RADIUS,plateFill,SHADOW,- wordAccentCss,+ wordDrawCss,} from "@/cut/lib/textRender";import {behindSubjectOverlay,@@ -348,10 +346,19 @@ export function OverlayLayer({);}-/** Text with one word emphasized. The words are walked exactly the way the- * canvas painter walks them — line by line, single-spaced — so the index the- * preview lights is the index the burn-in lights. */-function AccentedText({ text, index, css }: { text: string; index: number; css: CSSProperties }) {+/** Text drawn word by word, each word wearing what the word engine resolved+ * for it. The words are walked exactly the way the canvas painter walks them+ * — line by line, single-spaced — so the word the preview lights is the word+ * the burn-in lights. */+function WordText({+ text,+ draws,+ lineHeight,+}: {+ text: string;+ draws: WordDraw[];+ lineHeight?: number;+}) {let k = 0;return (<>@@ -361,12 +368,11 @@ function AccentedText({ text, index, css }: { text: string; index: number; css:.split(" ").filter(Boolean).map((w, wi) => {- const active = k === index;- k++;+ const d = draws[k++];return (<span key={wi}>{wi > 0 && " "}- <span style={active ? css : undefined}>{w}</span>+ <span style={d ? wordDrawCss(d, lineHeight) : undefined}>{w}</span></span>);})}@@ -427,24 +433,16 @@ function SubtitleCaption({// Captions ride the same style/opener/anchor logic as the export burn-in,// so the preview and the rendered file match exactly.const style = captionStyle(subtitles.style);+ // The track's word effect plays on the playhead: the cue resolves its words+ // at this moment exactly as the export burn-in resolves them at that frame.const ov = cueOverlay(cue,style,cue.id === cues[0]?.id,trackPos(subtitles, style, lane),- undefined,+ subtitles.wordHighlight ? t : undefined,frame.w);- // Karaoke: the word under the playhead lights up as it is spoken.- const wordIndex = subtitles.wordHighlight- ? cueWordWindows(cue).findIndex((w) => t >= w.start && t < w.end)- : -1;- // The spoken word's treatment follows the style (with user overrides): a- // swell in the accent color, an accent box (drawn with box-shadow spread so- // the line never reflows), the accent color alone, or accent color +- // underline. The swell is a transform, so it too leaves the line alone.- const look = karaokeLook(style, subtitles);- const activeStyle = wordAccentCss(look);const scale = stageWidth / frame.w;return (<div@@ -484,11 +482,7 @@ function SubtitleCaption({borderRadius: ov.plate ? PLATE_RADIUS_EM : undefined,}}>- {wordIndex < 0 ? (- ov.text- ) : (- <AccentedText text={ov.text} index={wordIndex} css={activeStyle} />- )}+ {ov.wordDraw ? <WordText text={ov.text} draws={ov.wordDraw} /> : ov.text}</div>);}@@ -584,25 +578,22 @@ function OverlayItem({// box from its left edge. Neither runs while the box is being edited.const glyphs = isText && !editing && live && hasGlyphMotion(live) ? live : null;const reveal = !editing ? live?.reveal : undefined;- // Word emphasis: the word being said wears the accent while the rest of the- // line stays as it is. Off while editing — the box is plain text then — and- // a per-glyph ramp takes the frame for itself while it runs.+ // Word effects: the line plays word by word on its own clock. Off while+ // editing — the box is plain text then — and a per-glyph ramp takes the+ // frame for itself while it runs.const words = isText && !editing ? overlayWords(o) : undefined;- // A rehearsal from the inspector sweeps the emphasis along the line on its- // own clock; otherwise it follows the playhead, like everything else here.+ // A rehearsal from the inspector sweeps the effect along the line on its own+ // clock; otherwise it follows the playhead, like everything else here.const wordLocal = running ? running.tLocal : Math.max(0, t - o.start);- const wordIndex =- words && !glyphs ? wordAccentIndex(o, wordLocal, Math.max(0.1, o.end - o.start)) : -1;- const wordColor = wordAccent(words, isTextOverlay(o) ? o.color : WORD_ACCENT_DEFAULT);- const wordCss =- words && wordIndex >= 0- ? wordAccentCss({- mode: words.style,- color: wordColor,- text: contrastText(wordColor),- scale: words.scale,- lineHeight: (isTextOverlay(o) ? o.lineHeight : undefined) ?? LINE_HEIGHT,- })+ const wordDraw =+ words && !glyphs && isTextOverlay(o)+ ? wordDrawsAt(+ o.text,+ words,+ o.color ?? WORD_ACCENT_DEFAULT,+ Math.max(0.1, o.end - o.start),+ wordLocal+ ): null;// Typewriter: the visible slice of the text (display only, never while// the box is being edited).@@ -1053,8 +1044,12 @@ function OverlayItem({</div>) : glyphs ? (<GlyphText text={shownText} phase={glyphs} scale={scale} />- ) : wordCss ? (- <AccentedText text={shownText} index={wordIndex} css={wordCss} />+ ) : wordDraw ? (+ <WordText+ text={shownText}+ draws={wordDraw}+ lineHeight={(isTextOverlay(o) ? o.lineHeight : undefined) ?? LINE_HEIGHT}+ />) : (<span>{shownText}</span>)diff --git a/site/src/cut/lib/behindPass.ts b/site/src/cut/lib/behindPass.tsindex af6750f6..61fb7bf4 100644--- a/site/src/cut/lib/behindPass.ts+++ b/site/src/cut/lib/behindPass.ts@@ -10,14 +10,13 @@*/import {- applyWordAccent,+ applyWordDraw,evalOverlayFrame,glyphStateAt,hasGlyphMotion,maskComposite,overlayWords,- wordAccentIndex,- wordWindows,+ wordSampleWindows,} from "@donkeycut/effects-kit";import { personSegmenter, segmentSubjectAlpha } from "./cutout";import { createRasterCanvas } from "./raster";@@ -37,13 +36,33 @@ type Segmenter = import("@mediapipe/tasks-vision").ImageSegmenter;/** An element's own length, the span its word emphasis is timed across. */const spanOf = (o: Overlay): number => Math.max(0.1, o.end - o.start);-/** The word indices an element needs a picture for: one per word while it- * lights them as they are said, and the plain picture (-1) otherwise. */-function rasterWords(o: Overlay): number[] {+/** Pictures here are full-frame, so a word effect that ramps is sampled+ * coarsely: enough slices that an arrival reads, few enough that a behind-+ * subject line does not hold a hundred frames of pixels resident. */+const BEHIND_WORD_FPS = 8;++/** The spans an element's word effect holds still for — one picture each.+ * Nothing here means the element draws as one piece. */+function rasterSpans(o: Overlay): { start: number; end: number }[] | null {const words = overlayWords(o);- if (!words || !isTextOverlay(o)) return [-1];- const n = wordWindows(o.text, spanOf(o), words.times).length;- return n > 0 ? Array.from({ length: n }, (_, i) => i) : [-1];+ if (!words || !isTextOverlay(o)) return null;+ const spans = wordSampleWindows(o.text, words, spanOf(o), BEHIND_WORD_FPS);+ return spans.length > 0 ? spans : null;+}++/** The picture indices an element needs: one per span while a word effect+ * walks it, and the plain picture (-1) otherwise. */+function rasterSlices(o: Overlay): number[] {+ const spans = rasterSpans(o);+ return spans ? spans.map((_, i) => i) : [-1];+}++/** Which picture stands for `tLocal`, or -1 for the plain one. */+function rasterSliceAt(o: Overlay, tLocal: number): number {+ const spans = rasterSpans(o);+ if (!spans) return -1;+ const i = spans.findIndex((s) => tLocal >= s.start && tLocal < s.end);+ return i < 0 ? spans.length - 1 : i;}/** Segmentation input width — small on purpose; this runs per frame. */@@ -144,8 +163,8 @@ export class SubjectMaskCompositor {return null;}- /** One element picture, with the word it emphasizes at this moment baked- * in, kept under the element it came from. */+ /** One element picture, with its words drawn as they stand across the span+ * it covers, kept under the element it came from. */private async drawRaster(o: Overlay,w: number,@@ -153,8 +172,11 @@ export class SubjectMaskCompositor {assets: MediaAsset[],word: number): Promise<void> {+ const spans = rasterSpans(o);+ const span = word >= 0 ? spans?.[word] : undefined;+ const at = span ? (span.start + span.end) / 2 : 0;const png = await renderElementPng(- applyWordAccent({ ...o, rotation: undefined, opacity: undefined }, word),+ applyWordDraw({ ...o, rotation: undefined, opacity: undefined }, at, spanOf(o)),w,h,assets@@ -176,7 +198,7 @@ export class SubjectMaskCompositor {behind.flatMap((o) =>// An element that lights its words needs one picture per word, all of// them resident before the first frame draws.- rasterWords(o).map(async (word) => {+ rasterSlices(o).map(async (word) => {if (this.rasters.get(o)?.has(word)) return;try {await this.drawRaster(o, w, h, assets, word);@@ -353,7 +375,7 @@ export class SubjectMaskCompositor {const scale = Math.min(W, H) / 1080;for (const o of active) {const tLocal = Math.max(0, t - o.start);- const bmp = this.rasterFor(o, W, H, assets, wordAccentIndex(o, tLocal, spanOf(o)));+ const bmp = this.rasterFor(o, W, H, assets, rasterSliceAt(o, tLocal));if (!bmp) continue;const ev = evalOverlayFrame(o, tLocal);// One cached picture per element here, so a per-glyph ramp or loop runsdiff --git a/site/src/cut/lib/captionWords.test.ts b/site/src/cut/lib/captionWords.test.tsnew file mode 100644index 00000000..9dad9d4c--- /dev/null+++ b/site/src/cut/lib/captionWords.test.ts@@ -0,0 +1,120 @@+import { describe, expect, test } from "bun:test";+import { CAPTION_STYLES, cueOverlay, cueWordFrames, cueWords, trackPos } from "./subtitles";+import { emptySubtitles, type SubtitleCue, type SubtitlesBlock } from "./types";++/**+ * A caption's word effect against the speech it was measured from.+ *+ * The clock is the transcriber's own onsets, so a cue whose first word runs+ * most of its length lights that word for most of it. What these check is that+ * every side of the caption path reads that same clock: the picture the preview+ * and the burn-in draw, and the spans the burn-in cuts to draw them at.+ */++const ACCENT = "#FF3D00";+const STYLE = { ...CAPTION_STYLES.clean, accentMode: "color", accent: ACCENT };++/** "one two three" with the first word holding four fifths of the cue. */+const cue: SubtitleCue = {+ id: "c1",+ start: 10,+ end: 12,+ text: "one two three",+ words: [+ { t0: 10, t1: 11.6, w: "one" },+ { t0: 11.6, t1: 11.8, w: "two" },+ { t0: 11.8, t1: 12, w: "three" },+ ],+};++/** Which display word wears the accent in a drawn cue, or -1. */+const lit = (at: number, subs?: Partial<SubtitlesBlock>) =>+ cueOverlay(+ cue,+ STYLE,+ false,+ trackPos({ ...emptySubtitles(), ...subs } as SubtitlesBlock, STYLE, 0),+ at,+ 1080+ ).wordDraw?.findIndex((d) => d.color.toLowerCase() === ACCENT.toLowerCase()) ?? -1;++describe("a caption's word clock", () => {+ test("follows the transcriber's onsets, not an even share of the cue", () => {+ // Halfway through the cue the measured speech is still on word 0; a split+ // by word length would have moved on by then.+ expect(lit(11)).toBe(0);+ expect(lit(11.7)).toBe(1);+ expect(lit(11.9)).toBe(2);+ });++ test("shares the span by word length when nothing measured the speech", () => {+ const flat = { ...cue, words: undefined };+ const draws = cueOverlay(flat, STYLE, false, undefined, 11, 1080).wordDraw;+ expect(draws?.length).toBe(3);+ expect(draws?.some((d) => d.color.toLowerCase() === ACCENT.toLowerCase())).toBe(true);+ });++ test("cue words carry the onsets the sampler cuts its spans from", () => {+ const times = cueWords(cue, STYLE).times ?? [];+ expect(times.length).toBe(3);+ [0, 1.6, 1.8].forEach((t, i) => expect(times[i]).toBeCloseTo(t, 5));+ });+});++describe("burn-in spans", () => {+ const spans = cueWordFrames(cue, STYLE);++ test("cover the cue end to end with no gaps", () => {+ expect(spans[0].start).toBeCloseTo(cue.start, 5);+ expect(spans[spans.length - 1].end).toBeCloseTo(cue.end, 5);+ for (let i = 1; i < spans.length; i++)+ expect(spans[i].start).toBeCloseTo(spans[i - 1].end, 5);+ });++ test("each stands for the word the picture drawn at its middle actually lights", () => {+ // The span boundaries and the picture inside them have to come off one+ // clock, or a span burns a word that is not the one it covers.+ for (const s of spans) {+ const mid = (s.start + s.end) / 2;+ expect(lit(mid)).toBe(lit(s.start + (s.end - s.start) * 0.25));+ }+ });++ test("an effect that lands on the beat stays one still per word", () => {+ expect(spans.length).toBe(3);+ });++ test("a ramping effect is cut finer, and bounded however long its ramp runs", () => {+ const built = cueWordFrames(cue, { ...STYLE, accentMode: "rise" });+ expect(built.length).toBeGreaterThan(spans.length);+ // Three words, at most two extra pictures each.+ expect(built.length).toBeLessThanOrEqual(9);+ });+});++describe("the caption's own settings", () => {+ test("reach the drawn words", () => {+ const faint = cueOverlay(+ cue,+ { ...STYLE, accentMode: "fill" },+ false,+ trackPos({ ...emptySubtitles(), accentDim: 0.6 } as SubtitlesBlock, STYLE, 0),+ 10,+ 1080+ ).wordDraw;+ expect(faint?.[2].opacity).toBeCloseTo(0.6, 5);+ });++ test("a swell set on the track lands on the word being said", () => {+ const swollen = cueOverlay(+ cue,+ { ...STYLE, accentMode: "pop" },+ false,+ trackPos({ ...emptySubtitles(), accentScale: 1.5 } as SubtitlesBlock, STYLE, 0),+ 10.5,+ 1080+ ).wordDraw;+ expect(swollen?.[0].scale).toBeCloseTo(1.5, 5);+ expect(swollen?.[1].scale).toBe(1);+ });+});diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.tsindex 771f8c3d..d58b95cc 100644--- a/site/src/cut/lib/exportClient.ts+++ b/site/src/cut/lib/exportClient.ts@@ -14,7 +14,7 @@ import { renderProjectToMp4 } from "./exportRender";import { putSigned } from "./media";import { createRasterCanvas, rasterCanvasToPng } from "./raster";import { clipSpeed, getClipSpans, overlayLayers, projectDuration, spanSequence, useEditor } from "./store";-import { captionStyle, cueOverlay, cueWordWindows, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";+import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";import { isMaskAnimated, isOverlayAnimated, normalizeGrade, paintMaskLuma } from "@donkeycut/effects-kit";import { renderElementFrames, renderElementPng } from "./textRender";import { clipCovers, clipKeyed, clipPosed, clipPoseAt, clipZoom, contentRect, frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, regionPx, shadowInk, subjectMasked } from "./types";@@ -838,10 +838,10 @@ export async function buildExportPayload(for (let i = 0; i < cues.length; i++) {const cue = cues[i];if (cue.start >= duration || !cue.text.trim()) continue;- // Karaoke burns one frame per word window (the spoken word accented);+ // A word effect burns one frame per span its picture holds still for;// otherwise the whole cue is a single still.const windows = doc.subtitles.wordHighlight- ? cueWordWindows(cue)+ ? cueWordFrames(cue, capStyle, doc.subtitles): [{ start: cue.start, end: cue.end }];for (let wi = 0; wi < windows.length; wi++) {const win = windows[wi];@@ -852,7 +852,7 @@ export async function buildExportPayload(capStyle,i === 0,pos,- doc.subtitles.wordHighlight ? wi : undefined,+ doc.subtitles.wordHighlight ? (win.start + win.end) / 2 : undefined,// Wrap in design space (1080 short side) from the project ratio —// the same width the preview passes, whatever the render size.frameOf(doc.aspect).wdiff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.tsindex cd277acf..a83e8d8d 100644--- a/site/src/cut/lib/exportRender.ts+++ b/site/src/cut/lib/exportRender.ts@@ -33,7 +33,7 @@ import { FrameCompositor, MISSING_FRAME, type Frame } from "./composite";import { overlayPlan, trackZeroPlan } from "./framePlan";import { frameSink, openMedia, videoTrackOf } from "./mediaRead";import { getClipSpans, overlayLayers, projectDuration, spanSequence } from "./store";-import { captionStyle, cueOverlay, cueWordWindows, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";+import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";import { applyEffectToCanvas, evalOverlayFrame, grainTile, isAudioEffect, isMaskAnimated, isOverlayAnimated, maskFrameAt, planAnimatedLayers, type LottieHandle, type OverlayAnim, type PaintPhase } from "@donkeycut/effects-kit";import { hasSubjectOverlays, SubjectMaskCompositor } from "./behindPass";import { createRasterCanvas, type RasterSurface } from "./raster";@@ -331,7 +331,7 @@ async function stampText(doc: ExportDoc): Promise<StampedLayer[]> {const cue = cues[i];if (cue.start >= duration || !cue.text.trim()) continue;const windows = doc.subtitles.wordHighlight- ? cueWordWindows(cue)+ ? cueWordFrames(cue, capStyle, doc.subtitles): [{ start: cue.start, end: cue.end }];for (let wi = 0; wi < windows.length; wi++) {const win = windows[wi];@@ -342,7 +342,7 @@ async function stampText(doc: ExportDoc): Promise<StampedLayer[]> {capStyle,i === 0,pos,- doc.subtitles.wordHighlight ? wi : undefined,+ doc.subtitles.wordHighlight ? (win.start + win.end) / 2 : undefined,designWidth),start: win.start,diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex 101835e2..e5969c7a 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -579,7 +579,7 @@ export interface EditorState {* track's language. Timings copy over; word timings don't survive* translation, so the new cues carry none. */translateSubtitleTrack: (fromLane: number) => Promise<void>;- setSubtitlesView: (patch: Partial<Pick<SubtitlesBlock, "showOnVideo" | "showOnTimeline" | "locale" | "style" | "size" | "font" | "x" | "y" | "wordHighlight" | "accentMode" | "accentColor">>) => void;+ setSubtitlesView: (patch: Partial<Pick<SubtitlesBlock, "showOnVideo" | "showOnTimeline" | "locale" | "style" | "size" | "font" | "x" | "y" | "wordHighlight" | "accentMode" | "accentColor" | "accentScale" | "accentDim">>) => void;/** The subtitle track (row) the panel edits and generation writes to. */subtitleLane: number;setSubtitleLane: (lane: number) => void;diff --git a/site/src/cut/lib/subtitles.ts b/site/src/cut/lib/subtitles.tsindex 1db18275..e8ce7748 100644--- a/site/src/cut/lib/subtitles.ts+++ b/site/src/cut/lib/subtitles.ts@@ -1,10 +1,10 @@import {contrastText as kitContrastText,+ wordDrawsAt,WORD_ACCENT_DEFAULT,- WORD_ACCENT_LABELS,- WORD_ACCENT_MODE_IDS,- WORD_ACCENT_NOTES,- wordWindows,+ wordEffect,+ wordSampleWindows,+ type OverlayWords,} from "@donkeycut/effects-kit";import { formatTime } from "./time";import type {@@ -13,7 +13,7 @@ import type {SubtitleCue,SubtitlesBlock,TextOverlay,- WordAccentMode,+ WordEffectId,} from "./types";export type { CaptionStyleId } from "./types";@@ -46,16 +46,33 @@ export function trackLocale(subs: SubtitlesBlock, lane: number): string {return subs.tracks?.[lane]?.locale ?? (lane === 0 ? subs.locale : undefined) ?? "en-US";}-/** A track's effective caption anchor plus the block's look overrides- * (karaoke accent, font size) — the `pos` argument `cueOverlay` takes. The- * track's own dragged spot wins, then the block-level legacy spot (first- * track only), then the style's default stacked upward per track so- * simultaneous languages never overlap. */+/** The block-level word-effect overrides a caption track carries: which+ * effect, and the two settings an effect may have. Absent = the caption+ * style's own. */+export interface CaptionWordOverrides {+ accentMode?: WordEffectId;+ accentColor?: string;+ accentScale?: number;+ accentDim?: number;+}++/** A track's effective caption anchor plus the block's look overrides — what+ * `cueOverlay` reads to place and dress one cue. */+export interface CaptionPos extends CaptionWordOverrides {+ x: number;+ y: number;+ size?: number;+ font?: FontId;+}++/** One track's anchor, filled in: the track's own dragged spot wins, then the+ * block-level legacy spot (first track only), then the style's default+ * stacked upward per track so simultaneous languages never overlap. */export function trackPos(subs: SubtitlesBlock,style: CaptionStyle,lane: number-): { x: number; y: number; size?: number; font?: FontId; accentMode?: WordAccentMode; accentColor?: string } {+): CaptionPos {const meta = subs.tracks?.[lane];return {x: meta?.x ?? (lane === 0 ? subs.x : undefined) ?? style.x,@@ -64,6 +81,8 @@ export function trackPos(font: subs.font,accentMode: subs.accentMode,accentColor: subs.accentColor,+ accentScale: subs.accentScale,+ accentDim: subs.accentDim,};}@@ -97,48 +116,35 @@ export interface CaptionStyle {plateOpacity?: number;/** Multiply the opening cue's size, so the hook lands bigger. */openerScale?: number;- /** Karaoke accent for the spoken word; absent = DEFAULT_ACCENT. */+ /** Accent color for the word effect; absent = DEFAULT_ACCENT. */accent?: string;- /** How the spoken word lights up: an accent box behind it (plate styles) or- * the accent color plus underline (open text). Absent = underline. */- accentMode?: WordAccentMode;- /** Word color on an accent box; absent = auto contrast. */- accentText?: string;+ /** Which word effect the style's captions run: an accent box behind the+ * spoken word (plate styles), a line building itself up, and so on. Absent+ * = underline. */+ accentMode?: WordEffectId;}/** Spoken-word accent for styles that don't pick their own. */export const DEFAULT_ACCENT = WORD_ACCENT_DEFAULT;-/** The word emphasis treatments, in the order every picker offers them —- * the caption panel, the element animation grid, and the assistant's tool- * enums all read this one list, so a new treatment reaches all of them. */-export const WORD_ACCENT_MODES: { id: WordAccentMode; label: string; hint: string }[] =- WORD_ACCENT_MODE_IDS.map((id) => ({- id,- label: WORD_ACCENT_LABELS[id],- hint: WORD_ACCENT_NOTES[id],- }));-/** Black or white, whichever reads on the given hex fill. */export const contrastText = kitContrastText;-/** The effective karaoke treatment for the spoken word: user overrides on the- * subtitles block win over the caption style's defaults, a swell with no- * accent named keeps the caption's own color, and a custom color picks its- * own contrast text for the box treatment. */-export function karaokeLook(- style: CaptionStyle,- subs?: { accentMode?: WordAccentMode; accentColor?: string }-): { mode: WordAccentMode; color: string; text: string } {- const mode = subs?.accentMode ?? style.accentMode ?? "underline";- // A swell says it with size, so with nobody naming an accent it keeps the- // color the rest of the caption is written in.+/** The word effect a caption track runs, as the element model carries it:+ * user overrides on the subtitles block win over the caption style's+ * defaults, and an effect that says everything with size keeps the color the+ * rest of the caption is written in when nobody named an accent. */+export function captionWords(style: CaptionStyle, subs?: CaptionWordOverrides): OverlayWords {+ const id = subs?.accentMode ?? style.accentMode ?? "underline";const color =- subs?.accentColor ?? style.accent ?? (mode === "pop" ? style.color : DEFAULT_ACCENT);+ subs?.accentColor ??+ style.accent ??+ (wordEffect(id).accentDefault === "text" ? style.color : DEFAULT_ACCENT);return {- mode,+ style: id,color,- text: subs?.accentColor ? contrastText(color) : style.accentText ?? contrastText(color),+ ...(subs?.accentScale !== undefined ? { scale: subs.accentScale } : {}),+ ...(subs?.accentDim !== undefined ? { dim: subs.accentDim } : {}),};}@@ -253,7 +259,6 @@ export const CAPTION_STYLES: Record<CaptionStyleId, CaptionStyle> = {plateOpacity: 0.95,accent: "#FF375F",accentMode: "box",- accentText: "#FFFFFF",},bubble: {id: "bubble",@@ -330,41 +335,27 @@ export function wrapCaptionForSize(text: string, size: number, frameW: number):return lines.join("\n");}-/**- * Per-display-word timeline windows for a cue, partitioning [start, end] with- * no gaps so the karaoke highlight never blinks off between words. Uses the- * transcriber's word timings while they still match the text one-to-one;- * otherwise (hand edits, social rewrites) splits proportionally by word length.- */-export function cueWordWindows(cue: SubtitleCue): { start: number; end: number }[] {- const dur = Math.max(0, cue.end - cue.start);- return wordWindows(- cue.text,- dur,- cue.words?.map((w) => w.t0 - cue.start)- ).map((w) => ({ start: cue.start + w.start, end: cue.start + w.end }));-}-/** A cue as a synthetic overlay, so captions ride the title pipeline. The style* preset drives the look; a user font or font size overrides the preset's* (the opener emphasis still multiplies the size); a dragged caption position- * (frame fractions) overrides the preset's spot; a word index marks that word- * for the karaoke accent. */+ * (frame fractions) overrides the preset's spot; `at` resolves the track's+ * word effect at that moment on the timeline. */export function cueOverlay(cue: SubtitleCue,style: CaptionStyle = CAPTION_STYLES.clean,isOpener = false,- pos: { x?: number; y?: number; size?: number; font?: FontId; accentMode?: WordAccentMode; accentColor?: string } | undefined,- wordIndex: number | undefined,+ pos: Partial<CaptionPos> | undefined,+ at: number | undefined,frameW: number): TextOverlay {- const kl = wordIndex !== undefined ? karaokeLook(style, pos) : null;+ const words = at !== undefined ? cueWords(cue, style, pos) : null;const size = Math.round((pos?.size ?? style.size) * (isOpener && style.openerScale ? style.openerScale : 1));+ const text = wrapCaptionForSize(cue.text, size, frameW);return {id: `sub-${cue.id}`,- text: wrapCaptionForSize(cue.text, size, frameW),+ text,start: cue.start,end: cue.end,x: pos?.x ?? style.x,@@ -377,17 +368,63 @@ export function cueOverlay(plate: style.plate,...(style.plateColor ? { plateColor: style.plateColor } : {}),...(style.plateOpacity !== undefined ? { plateOpacity: style.plateOpacity } : {}),- ...(kl && wordIndex !== undefined+ ...(words && at !== undefined? {- highlightWord: wordIndex,- highlightColor: kl.color,- highlightMode: kl.mode,- highlightText: kl.text,+ wordDraw: wordDrawsAt(+ text,+ words,+ style.color,+ Math.max(0, cue.end - cue.start),+ at - cue.start+ ),}: {}),};}+/** A caption's pictures are full frames, one per span, and a cut carries one+ * of them for every word of speech in it. A ramp is sampled at this rate, and+ * never at more than this many steps however long the ramp runs, which is+ * enough to read an arrival while holding a subtitled export to a few+ * pictures per word. */+const CAPTION_WORD_FPS = 12;+const CAPTION_WORD_SLICES = 2;++/**+ * One cue's word effect with the transcriber's own onsets on it. The clock is+ * the measured speech wherever there is any, so the effect walks the line at+ * the pace it is actually said; a cue with no timings shares its span by word+ * length instead. Everything that draws or samples a cue reads this one+ * function, so the picture a burn-in span stands for is resolved against the+ * same windows the span was cut from.+ */+export function cueWords(+ cue: SubtitleCue,+ style: CaptionStyle,+ subs?: CaptionWordOverrides+): OverlayWords {+ const times = cue.words?.map((w) => w.t0 - cue.start);+ return { ...captionWords(style, subs), ...(times ? { times } : {}) };+}++/**+ * The moments a cue has to be drawn at to carry its word effect: one span per+ * stretch the picture holds still. An effect that lands on the beat comes+ * back as one span per word, which is what the caption burn-in has always+ * written; one that ramps is cut finer inside a word's window.+ */+export function cueWordFrames(+ cue: SubtitleCue,+ style: CaptionStyle,+ subs?: CaptionWordOverrides+): { start: number; end: number }[] {+ const dur = Math.max(0, cue.end - cue.start);+ return wordSampleWindows(cue.text, cueWords(cue, style, subs), dur, CAPTION_WORD_FPS, CAPTION_WORD_SLICES).map((w) => ({+ start: cue.start + w.start,+ end: cue.start + w.end,+ }));+}+/** The cue under a given time, if any. */export function cueAt(cues: SubtitleCue[], t: number): SubtitleCue | undefined {return cues.find((c) => t >= c.start && t < c.end);diff --git a/site/src/cut/lib/textLooks.ts b/site/src/cut/lib/textLooks.tsindex 9bad0c38..976eca34 100644--- a/site/src/cut/lib/textLooks.ts+++ b/site/src/cut/lib/textLooks.ts@@ -1,4 +1,4 @@-import type { OverlayAnimStyle, OverlayLoopStyle, WordAccentMode } from "@donkeycut/effects-kit";+import type { OverlayAnimStyle, OverlayLoopStyle, WordEffectId } from "@donkeycut/effects-kit";import type { TextEnsemble, TextVariation } from "./textCompose";import type { CaptionStyleId, FontId } from "./types";@@ -78,7 +78,7 @@ export interface TextLook {size?: number;font?: FontId;accentColor?: string;- accentMode?: WordAccentMode;+ accentMode?: WordEffectId;};}diff --git a/site/src/cut/lib/textRender.ts b/site/src/cut/lib/textRender.tsindex 4184f441..f64909da 100644--- a/site/src/cut/lib/textRender.ts+++ b/site/src/cut/lib/textRender.ts@@ -1,18 +1,20 @@"use client";import {+ hexAlpha,+ WORD_BASELINE_DROP,renderElementPng as kitRenderElementPng,renderOverlayFrames as kitRenderOverlayFrames,LINE_HEIGHT,- wordSwell,type OverlayFrameSet,type PaintPhase,type RenderEnv,type StickerImage,+ type WordDraw,} from "@donkeycut/effects-kit";import type { CSSProperties } from "react";import { createRasterCanvas, decodeRasterImage, rasterCanvasToPng } from "./raster";-import { fontStack, type MediaAsset, type Overlay, type WordAccentMode } from "./types";+import { fontStack, type MediaAsset, type Overlay } from "./types";// The shared text metrics and painters live in the effects kit; these// re-exports keep the app's preview components on the same constants.@@ -26,49 +28,51 @@ export {PLATE_RADIUS,plateFill,SHADOW,- WORD_ACCENT_DEFAULT,- WORD_ACCENT_LABELS,- WORD_ACCENT_MODE_IDS,- WORD_POP_SCALE,} from "@donkeycut/effects-kit";-/** The css one emphasized word wears — the DOM twin of what the canvas- * painter draws for `highlightMode`. A swell is real type size, so the line- * reflows around it: the words beside it move out of its way and settle back- * as the emphasis travels on, which is what the burn-in lays out too. The- * word keeps the line's own leading, so a bigger word never opens the line- * box up. A box is box-shadow spread, which stays clear of the layout. */-export function wordAccentCss(look: {- mode: WordAccentMode;- color: string;- text: string;- /** How far the word swells; absent = the mode's default. */- scale?: number;- /** The block's line height, so the swollen word can hold it. */- lineHeight?: number;-}): CSSProperties {- const swell = wordSwell({ style: look.mode, scale: look.scale });- const lh = look.lineHeight ?? LINE_HEIGHT;- const size: CSSProperties =- swell === 1 ? {} : { fontSize: `${swell}em`, lineHeight: lh / swell };- const treatment: CSSProperties =- look.mode === "box"- ? {- color: look.text,- background: look.color,- boxShadow: `0 0 0 0.12em ${look.color}`,- borderRadius: "0.18em",- textShadow: "none",- }- : look.mode === "color" || look.mode === "pop"- ? { color: look.color }- : {- color: look.color,- textDecoration: "underline",- textDecorationThickness: "0.07em",- textUnderlineOffset: "0.14em",- };- return { ...treatment, ...size };+/** The css one word wears — the DOM twin of what the canvas painter draws for+ * a resolved `WordDraw`. A resize is real type size, so the line reflows+ * around it, which is what the burn-in lays out too; the word keeps the+ * line's own leading, so a bigger word never opens the line box up. A box is+ * box-shadow spread, which stays clear of the layout. */+export function wordDrawCss(d: WordDraw, lineHeight = LINE_HEIGHT): CSSProperties {+ const css: CSSProperties = { color: d.color };+ if (d.opacity < 0.999) css.opacity = d.opacity;+ // The word takes its room at the layout size and draws at its own, which+ // differ only while it is growing into its place: real type size for the+ // room, so the line reflows around a swollen word exactly as the burn-in+ // lays it out, and a transform for the rest, which leaves the layout alone.+ if (d.layoutScale !== 1) {+ css.fontSize = `${d.layoutScale}em`;+ css.lineHeight = lineHeight / d.layoutScale;+ }+ const shrink = d.scale / d.layoutScale;+ // A word drawn smaller than the room it holds shares the line's baseline, so+ // it hangs lower in its box than a scale about the box center would leave+ // it; the drop makes up the difference. Offsets are em of the size the word+ // draws at and the box is em of the size it takes room at, so they convert+ // by the ratio between the two.+ const sit = (WORD_BASELINE_DROP * (d.layoutScale - d.scale)) / d.layoutScale;+ if (shrink !== 1 || d.dx || d.dy || d.rotate) {+ css.display = "inline-block";+ css.transform =+ `translate(${d.dx * shrink}em, ${d.dy * shrink + sit}em) rotate(${d.rotate}deg)` ++ (shrink !== 1 ? ` scale(${shrink})` : "");+ }+ const mark = d.markAlpha ?? 0;+ if (d.mark === "box" && mark > 0.001) {+ const fill = hexAlpha(d.markColor ?? "#000000", mark);+ css.background = fill;+ css.boxShadow = `0 0 0 0.12em ${fill}`;+ css.borderRadius = "0.18em";+ css.textShadow = "none";+ } else if (d.mark === "underline" && mark > 0.001) {+ css.textDecoration = "underline";+ css.textDecorationColor = hexAlpha(d.color, mark);+ css.textDecorationThickness = "0.07em";+ css.textUnderlineOffset = "0.14em";+ }+ return css;}/** Decoded sticker images by asset id, shared across one page's renders. Andiff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.tsindex 9ff489b6..fa25fc50 100644--- a/site/src/cut/lib/types.ts+++ b/site/src/cut/lib/types.ts@@ -16,7 +16,7 @@ import {type ShapeOverlay,type StickerOverlay,type TextOverlay as KitTextOverlay,- type WordAccentMode,+ type WordEffectId,} from "@donkeycut/effects-kit";import { getBackend, type CutBackend } from "./backend";import type { VideoProject } from "./genvideo/types";@@ -1142,7 +1142,7 @@ export type {ShapeKind,ShapeOverlay,StickerOverlay,- WordAccentMode,+ WordEffectId,};/** One subtitle caption, timed against the timeline (not the source files). */@@ -1217,12 +1217,17 @@ export interface SubtitlesBlock {* per-track anchor in `tracks`. Read as the lane-0 fallback. */x?: number;y?: number;- /** Karaoke mode: each word lights up as it is spoken, in the preview and- * the export burn-in. */+ /** Word effects on: the caption plays word by word as it is spoken, in the+ * preview and the export burn-in. */wordHighlight?: boolean;- /** Word treatment overrides; absent = the caption style's defaults. */- accentMode?: WordAccentMode;+ /** Which word effect, overriding the caption style's own; absent = the+ * style's default. */+ accentMode?: WordEffectId;accentColor?: string;+ /** How far a word swells at its moment; absent = the effect's own. */+ accentScale?: number;+ /** The opacity a word wears off its moment; absent = the effect's own. */+ accentDim?: number;}export const emptySubtitles = (): SubtitlesBlock => ({