Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
refactor: make caption timing a complete unit table
kashorin9559574f7407
diff --git a/packages/caption-fine/README.md b/packages/caption-fine/README.mdindex c2c2da96..4cba194b 100644--- a/packages/caption-fine/README.md+++ b/packages/caption-fine/README.md@@ -50,9 +50,10 @@ One SVS Recipe freezes three public dimensions:- **When**: the visible lead/tail envelope, handoff, Cue/Atom motion, reveal, Karaoke and loops.An upstream adapter supplies absolute Caption unit timing. Fine then produces an explicit visible-Schedule and renders that Schedule. Lead and tail never change the source timing-times used by Karaoke. Later Uses mask earlier presentation, including Hidden. The final visibility-is clipped to the winning Use Window while the original Cue envelope and animations are preserved.+Schedule: it forms Cues from document order, hard breaks and complete Unit timing, then renders that+Schedule. Lead and tail never change the source times used by Karaoke. Later Uses mask earlier+presentation, including Hidden. The final visibility is clipped to the winning Use Window while the+original Cue envelope and animations are preserved.The Fine Schedule preserves the CaptionTiming's Timeline and document identities. It contains noNarrative identity. The renderer rejects any mismatched Timeline or document. Studio may exposediff --git a/packages/caption-fine/src/schedule.ts b/packages/caption-fine/src/schedule.tsindex 833f85d8..d6431385 100644--- a/packages/caption-fine/src/schedule.ts+++ b/packages/caption-fine/src/schedule.ts@@ -1,4 +1,4 @@-import { assertCaptionProgramForDocument, assertCaptionTiming, captionUseVisibility } from "@hypit/hypit/caption";+import { assertCaptionProgramForDocument, assertCaptionTimingForDocument, captionUseVisibility } from "@hypit/hypit/caption";import type { CaptionProgram, CaptionTiming } from "@hypit/hypit/caption";import type { CaptionDocument } from "@hypit/hypit/caption";@@ -9,6 +9,41 @@ function assertIntegerFrame(value: number, label: string): void {if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${label} must be a non-negative integer Frame`);}+type FineCaptionContentCue = {+ readonly id: string;+ readonly startFrame: number;+ readonly endFrameExclusive: number;+ readonly units: CaptionTiming["units"];+};++/** Fine owns how complete timed Units become readable Cues. */+function fineCaptionContentCues(timing: CaptionTiming, document: CaptionDocument): readonly FineCaptionContentCue[] {+ const timingByUnit = new Map(timing.units.map((unit) => [unit.unitId, unit] as const));+ const breaks = new Set(document.cueBreaks.map((item) => item.afterUnitId));+ const cues: FineCaptionContentCue[] = [];+ let current: CaptionTiming["units"][number][] = [];+ let groupId: string | undefined;+ const flush = (): void => {+ if (current.length === 0) return;+ cues.push({+ id: `${document.id}:cue:${cues.length + 1}`,+ startFrame: Math.min(...current.map((unit) => unit.startFrame)),+ endFrameExclusive: Math.max(...current.map((unit) => unit.endFrameExclusive)),+ units: current,+ });+ current = [];+ groupId = undefined;+ };+ for (const [index, unit] of document.units.entries()) {+ const previous = index === 0 ? undefined : document.units[index - 1];+ if (current.length > 0 && (groupId !== unit.groupId || (previous !== undefined && breaks.has(previous.id)))) flush();+ if (current.length === 0) groupId = unit.groupId;+ current.push(timingByUnit.get(unit.id)!);+ }+ flush();+ return cues;+}+/*** Resolve visible Cue envelopes before the renderer runs.*@@ -21,9 +56,8 @@ export function scheduleFineCaption(program: CaptionProgram,document: CaptionDocument,): FineCaptionSchedule {- assertCaptionTiming(timing);assertCaptionProgramForDocument(program, document);- if (timing.documentId !== document.id) throw new Error("Fine Caption Schedule received another CaptionDocument");+ assertCaptionTimingForDocument(timing, document);const parameters = new Map<string, FineCaptionParameters>();const activeStyles = new Set(program.uses.map(use => use.styleId));for (const style of program.styles) {@@ -38,12 +72,13 @@ export function scheduleFineCaption(}const units = new Map(document.units.map(unit => [unit.id, unit]));+ const contentCues = fineCaptionContentCues(timing, document);const cues: FineCaptionScheduledCue[] = [];for (const [index, use] of program.uses.entries()) {if (use.window.start.timelineId !== timing.timelineId) throw new Error("Caption Use belongs to another Timeline");const style = parameters.get(use.styleId);if (style === undefined) continue; // Hidden still participates in coverage below.- const desired = timing.cues.filter(cue => use.role === undefined || units.get(cue.units[0]!.unitId)?.role === use.role).map((cue): FineCaptionScheduledCue => ({+ const desired = contentCues.filter(cue => use.role === undefined || units.get(cue.units[0]!.unitId)?.role === use.role).map((cue): FineCaptionScheduledCue => ({id: `${use.window.subjectId}:${cue.id}`, cueId: cue.id, styleId: use.styleId,timedStartFrame: cue.startFrame, timedEndFrameExclusive: cue.endFrameExclusive,visibleStartFrame: Math.max(0, cue.startFrame - style.timing.leadFrames),diff --git a/packages/caption-fine/src/studio.ts b/packages/caption-fine/src/studio.tsindex 1314277b..650bbfa7 100644--- a/packages/caption-fine/src/studio.ts+++ b/packages/caption-fine/src/studio.ts@@ -120,13 +120,15 @@ export function projectCaptionContents(context: StudioTrackCompanionContext): reconst schedule = requiredSurfaceValue(context, "schedule") as FineCaptionSchedule;const document = requiredReferencedValue(context, "document", captionTypes.document) as CaptionDocument;if (document.id !== content.documentId) throw new Error("Caption content belongs to another CaptionDocument.");- return content.cues.map((cue, index): StudioEntityDraft => ({- id: `${context.track.outputRef}:cue:${cue.id}`, authoredId: cue.id,+ const cues = new Map<string, FineCaptionSchedule["cues"][number]>();+ for (const cue of schedule.cues) if (!cues.has(cue.cueId)) cues.set(cue.cueId, cue);+ return [...cues.values()].map((cue, index): StudioEntityDraft => ({+ id: `${context.track.outputRef}:cue:${cue.cueId}`, authoredId: cue.cueId,display: { title: `#${index + 1}`, layers: [{ kind: "text", role: "content", text: cueText(document, cue.units.map(unit => unit.unitId)) }] },- startFrame: cue.startFrame, endFrameExclusive: cue.endFrameExclusive, stackOrder: 0,- renderIds: schedule.cues.filter(item => item.cueId === cue.id).map(item => item.id),+ startFrame: cue.timedStartFrame, endFrameExclusive: cue.timedEndFrameExclusive, stackOrder: 0,+ renderIds: schedule.cues.filter(item => item.cueId === cue.cueId).map(item => item.id),presentation: { entity: "caption-cue", chrome: "standard" },- inspector: [{ id: "range", label: "Range", domain: "when", section: { id: "cue", label: "Cue" }, value: `${cue.startFrame}–${cue.endFrameExclusive}`, unit: "f" }],+ inspector: [{ id: "range", label: "Range", domain: "when", section: { id: "cue", label: "Cue" }, value: `${cue.timedStartFrame}–${cue.timedEndFrameExclusive}`, unit: "f" }],}));}diff --git a/packages/caption-fine/test/caption-fine.test.ts b/packages/caption-fine/test/caption-fine.test.tsindex 3a8eb7c9..9a911f4d 100644--- a/packages/caption-fine/test/caption-fine.test.ts+++ b/packages/caption-fine/test/caption-fine.test.ts@@ -96,21 +96,11 @@ function fixture(script = "<line>one two || three four</line>") {const projection: CaptionTiming = {timelineId: "test-space",documentId: document.id,- cues: [- {- id: "captions:cue:1", startFrame: 10, endFrameExclusive: 30,- units: [- { unitId: first.id, startFrame: 10, endFrameExclusive: 20 },- { unitId: second.id, startFrame: 20, endFrameExclusive: 30 },- ],- },- {- id: "captions:cue:2", startFrame: 36, endFrameExclusive: 56,- units: [- { unitId: third.id, startFrame: 36, endFrameExclusive: 46 },- { unitId: fourth.id, startFrame: 46, endFrameExclusive: 56 },- ],- },+ units: [+ { unitId: first.id, startFrame: 10, endFrameExclusive: 20 },+ { unitId: second.id, startFrame: 20, endFrameExclusive: 30 },+ { unitId: third.id, startFrame: 36, endFrameExclusive: 46 },+ { unitId: fourth.id, startFrame: 46, endFrameExclusive: 56 },],};return { document, program, projection };@@ -274,14 +264,21 @@ test("Fine Caption gives a hidden Style no Cue, background or decoration", () =>const hidden = { id: "hidden", rendering: null };const hiddenProgram: CaptionProgram = { ...program, styles: [hidden],uses: program.uses.map(use => ({ ...use, styleId: hidden.id })) };- const hiddenProjection = { ...projection, cues: [] };- const schedule = scheduleFineCaption(hiddenProjection, hiddenProgram, document);+ const schedule = scheduleFineCaption(projection, hiddenProgram, document);const visual = renderCaption(schedule, hiddenProgram, document,sealTimeline({ id: "test-space", frameCount: 90, frameRate: { numerator: 30, denominator: 1 } }));assert.deepEqual(schedule.cues, []);assert.deepEqual(visual.presents, []);});+test("Fine Caption accepts one complete empty document as an ordinary no-content result", () => {+ const document = { id: "empty-caption", units: [], words: [], cueBreaks: [] };+ const program: CaptionProgram = { id: "empty", documentId: document.id, styles: [], uses: [] };+ assert.deepEqual(scheduleFineCaption({ timelineId: "test-space", documentId: document.id, units: [] }, program, document), {+ timelineId: "test-space", documentId: document.id, cues: [],+ });+});+test("Fine Caption gives overlapping acoustic Words one current Karaoke owner", () => {const { document, program, projection } = fixture();const style = fineCaptionStyle("plain", {@@ -289,16 +286,9 @@ test("Fine Caption gives overlapping acoustic Words one current Karaoke owner",properties: { ...recipe.properties, karaoke: "current", "karaoke-transition": "step" },}, [font]);const karaokeProgram: CaptionProgram = { ...program, styles: [style] };- const firstCue = projection.cues[0]!;const overlapped: CaptionTiming = {...projection,- cues: [{- ...firstCue,- units: [- { ...firstCue.units[0]!, endFrameExclusive: 21 },- firstCue.units[1]!,- ],- }, projection.cues[1]!],+ units: projection.units.map((unit, index) => index === 0 ? { ...unit, endFrameExclusive: 21 } : unit),};const track = renderCaption(scheduleFineCaption(overlapped, karaokeProgram, document),@@ -325,10 +315,8 @@ function unevenChineseCaption(properties: Recipe["properties"] = {}) {const windows = [[10, 13], [13, 32], [36, 42], [42, 55]] as const;const timed: CaptionTiming = {...projection,- cues: [{ ...projection.cues[0]!, endFrameExclusive: 55,- units: document.units.map((unit, index) => ({ unitId: unit.id,- startFrame: windows[index]![0], endFrameExclusive: windows[index]![1] })),- }],+ units: document.units.map((unit, index) => ({ unitId: unit.id,+ startFrame: windows[index]![0], endFrameExclusive: windows[index]![1] })),};const styled = { ...program, styles: [style] };const track = renderCaption(scheduleFineCaption(timed, styled, document), styled, document,@@ -439,13 +427,11 @@ test("Typewriter keeps combining graphemes whole and preserves unequal-width Lattest("Caption handoff follows placed time and retains simultaneous spoken envelopes", () => {const { document, program, projection } = fixture();- const first = projection.cues[0]!;- const second = projection.cues[1]!;- const overlapping = { ...second, startFrame: 20, endFrameExclusive: 40,- units: second.units.map(unit => ({ ...unit, startFrame: unit.startFrame - 16,- endFrameExclusive: unit.endFrameExclusive - 16 })) };- // Reverse declaration order: placement decides temporal neighbors, not Script order.- const schedule = scheduleFineCaption({ ...projection, cues: [overlapping, first] }, program, document);+ const first = projection.units.slice(0, 2);+ const overlapping = projection.units.slice(2).map(unit => ({ ...unit, startFrame: unit.startFrame - 16,+ endFrameExclusive: unit.endFrameExclusive - 16 }));+ // Reverse timing-table order: identities join content, while placed time decides temporal neighbors.+ const schedule = scheduleFineCaption({ ...projection, units: [...overlapping, ...first] }, program, document);assert.deepEqual(schedule.cues.map(cue => [cue.timedStartFrame, cue.timedEndFrameExclusive]), [[10, 30], [20, 40]]);assert.deepEqual(schedule.cues.map(cue => [cue.visibleStartFrame, cue.visibleEndFrameExclusive]), [[6, 36], [16, 46]]);const track = renderCaption(schedule, program, document,@@ -486,16 +472,14 @@ test("a Use beginning inside a Cue changes presentation without changing words oconst { document, program } = fixture("<line>test1 || test2 @{select} test3 || test4 @{/select}</line>");const timeline = sealTimeline({ id: "test-space", frameCount: 90, frameRate: { numerator: 30, denominator: 1 } });const unit = (index: number) => ({ unitId: document.units[index]!.id, startFrame: 10 + index * 10, endFrameExclusive: 20 + index * 10 });- const projection: CaptionTiming = { timelineId: timeline.id, documentId: document.id, cues: [- { id: "c1", startFrame: 10, endFrameExclusive: 20, units: [unit(0)] },- { id: "c2", startFrame: 20, endFrameExclusive: 40, units: [unit(1), unit(2)] },- { id: "c3", startFrame: 40, endFrameExclusive: 50, units: [unit(3)] },- ] };+ const projection: CaptionTiming = { timelineId: timeline.id, documentId: document.id,+ units: [unit(0), unit(1), unit(2), unit(3)] };const emphasis = fineCaptionStyle("emphasis", { ...recipe, properties: { ...recipe.properties, "active-fill": "#FF6347", karaoke: "trail" } }, [font]);const window = (id:string,start:number,end:number) => projectProgramWindow({itemId:id,semantic:timeline,projection:{start:{ref:"absolute",at:{unit:"frames",value:start}},end:{ref:"absolute",at:{unit:"frames",value:end}}}});const changed: CaptionProgram = { ...program, styles: [...program.styles, emphasis], uses: [...program.uses, { styleId: emphasis.id, window: window("emphasis",30,50) }] };const schedule = scheduleFineCaption(projection, changed, document);- const parts = schedule.cues.filter(cue=>cue.cueId==="c2");+ const cueId = `${document.id}:cue:2`;+ const parts = schedule.cues.filter(cue=>cue.cueId===cueId);assert.equal(parts.length,2);assert.deepEqual(parts.map(cue=>cue.units), [[unit(1),unit(2)],[unit(1),unit(2)]]);assert.deepEqual(parts.map(cue=>cue.visibility), [[{startFrame:20,endFrameExclusive:30}],[{startFrame:30,endFrameExclusive:40}]]);@@ -506,7 +490,7 @@ test("a Use beginning inside a Cue changes presentation without changing words oassert.deepEqual(entered.visibility,[{startFrame:30,endFrameExclusive:40}]);// Rendering the same Style over the whole timeline has identical animation data for this Cue.const whole: CaptionProgram = {...changed,uses:[{styleId:emphasis.id,window:window("whole",0,90)}]};- const original = renderCaption(scheduleFineCaption(projection,whole,document),whole,document,timeline).presents.find(p=>p.id.endsWith(":c2"))!;+ const original = renderCaption(scheduleFineCaption(projection,whole,document),whole,document,timeline).presents.find(p=>p.id.endsWith(`:${cueId}`))!;assert.deepEqual(entered.elements,original.elements);const hidden: CaptionProgram = {...changed,styles:[...changed.styles,{id:"hidden",rendering:null}],uses:[...changed.uses,{styleId:"hidden",window:window("hide",32,35)}]};@@ -519,18 +503,15 @@ test("a Use beginning inside a Cue changes presentation without changing words otest("Caption time windows select presentation, while Role filters preserve simultaneous speakers", () => {const { document, program, projection } = fixture("<line><A>one two || <B>three four</line>");const timeline = sealTimeline({id:"test-space",frameCount: 90, frameRate: { numerator: 30, denominator: 1 }});- const overlap: CaptionTiming = {...projection,cues:[projection.cues[0]!,{- ...projection.cues[1]!,startFrame:15,endFrameExclusive:35,- units:projection.cues[1]!.units.map((u,i)=>({...u,startFrame:15+i*10,endFrameExclusive:25+i*10})),- }]};+ const overlap: CaptionTiming = {...projection,units:projection.units.map((unit,index)=>index < 2 ? unit+ : {...unit,startFrame:15+(index-2)*10,endFrameExclusive:25+(index-2)*10})};const hidden: CaptionProgram = {...program,styles:[...program.styles,{id:"hidden",rendering:null}],uses:[...program.uses,{role:"A",styleId:"hidden",window:projectProgramWindow({itemId:"hideA",semantic:timeline,projection:{start:{ref:"timeline.start"},end:{ref:"timeline.end"}}}),}]};const schedule = scheduleFineCaption(overlap,hidden,document);assert.equal(schedule.cues.length,1);- assert.deepEqual(schedule.cues[0]!.units,overlap.cues[1]!.units);- const noContent = scheduleFineCaption({...projection,cues:[]},program,document);- assert.deepEqual(noContent.cues,[]);+ assert.deepEqual(schedule.cues[0]!.units,overlap.units.slice(2));+ assert.throws(() => scheduleFineCaption({...projection,units:[]},program,document), /must cover CaptionDocument units exactly/u);assert.deepEqual(scheduleFineCaption(projection,{...program,uses:[],styles:[]},document).cues,[]);});@@ -572,10 +553,8 @@ test("Fine uses author separators in ordinary text, shared groups, active copiesconst styled = { ...program, styles: [style] };const render = (source: string) => {const document = captionDocument(parseScript("spacing", `<line>${source}</line>`), "story.caption", "story");- const timed: CaptionTiming = { ...projection, cues: [{ ...projection.cues[0]!,- startFrame: 10, endFrameExclusive: 70,- units: document.units.map((unit, index) => ({ unitId: unit.id, startFrame: 10 + index * 5, endFrameExclusive: 15 + index * 5 })),- }] };+ const timed: CaptionTiming = { ...projection,+ units: document.units.map((unit, index) => ({ unitId: unit.id, startFrame: 10 + index * 5, endFrameExclusive: 15 + index * 5 })) };return renderCaption(scheduleFineCaption(timed, styled, document), styled, document,sealTimeline({ id: "test-space", frameCount: 90, frameRate: { numerator: 30, denominator: 1 } })).presents[0]!.elements;};diff --git a/packages/caption-fine/test/layout.browser.test.ts b/packages/caption-fine/test/layout.browser.test.tsindex 5a73d83a..a71fb563 100644--- a/packages/caption-fine/test/layout.browser.test.ts+++ b/packages/caption-fine/test/layout.browser.test.ts@@ -25,10 +25,8 @@ function cueHtml(source: string, width: number, extra: Recipe["properties"]): stconst program = { id: "p", documentId: document.id, styles: [style], uses: [{ styleId: "plain",window: projectProgramWindow({ itemId: "use", semantic: timeline, projection: { start: { ref: "timeline.start" }, end: { ref: "timeline.end" } } }),}] };- const timed = { timelineId: timeline.id, documentId: document.id, cues: [{ id: "cue", startFrame: 0,- endFrameExclusive: document.units.length * 5,- units: document.units.map((unit, index) => ({ unitId: unit.id, startFrame: index * 5, endFrameExclusive: index * 5 + 5 })),- }] };+ const timed = { timelineId: timeline.id, documentId: document.id,+ units: document.units.map((unit, index) => ({ unitId: unit.id, startFrame: index * 5, endFrameExclusive: index * 5 + 5 })) };const track = renderFineCaption(scheduleFineCaption(timed, program, document), program, document, timeline,{ xPx: 0, yPx: 0, widthPx: width, heightPx: 500 });const compiled = compileHtmlProgram(sealComposition({ id: "test",diff --git a/packages/caption-fine/test/studio-companion.test.ts b/packages/caption-fine/test/studio-companion.test.tsindex e91e05e4..e05dce1b 100644--- a/packages/caption-fine/test/studio-companion.test.ts+++ b/packages/caption-fine/test/studio-companion.test.ts@@ -64,7 +64,7 @@ test("Caption Companion projects Cue text and Style from public domain values",value: { visualIr: "hypit.visual-ir@1", id: "captions", presents: [] },},spans: [{ id: "cue-1", startFrame: 8, endFrameExclusive: 24, stackOrder: 70 }],- values: new Map<string, unknown>([["captions.schedule", schedule], ["captions.timing", {timelineId:"speech",documentId:document.id, cues:[{id:"cue-1",startFrame:10,endFrameExclusive:20,units:schedule.cues[0]!.units}]}], ["story.caption", document]]),+ values: new Map<string, unknown>([["captions.schedule", schedule], ["captions.timing", {timelineId:"speech",documentId:document.id,units:schedule.cues[0]!.units}], ["story.caption", document]]),temporalBindings: [],temporalDomains: [],generic: () => [base],@@ -104,7 +104,7 @@ test("Caption Uses keep authored windows and child ownership even with no renderconst context = {track:{outputRef:"captions.track",trace:{references:[{input:"document",typeRef:captionTypes.document,ref:"document"},{input:"timing",typeRef:captionTypes.timing,ref:"timing"}],outputPorts:[{name:"schedule",ref:"schedule"},{name:"program",ref:"program"}]}},placement:{children:uses.map((use,i)=>({range:{start:i*10,end:i*10+8},referenceAttributes:{style:use.styleId},values:[{type:temporalTypes.windowSpec,value:{id:use.window.subjectId}}]}))},- values:new Map<string,unknown>([["document",{id:"document",units:[],words:[]}],["timing",{timelineId:"film",documentId:"document",cues:[]}],["schedule",{cues:[]}],["program",{uses}]]),+ values:new Map<string,unknown>([["document",{id:"document",units:[],words:[]}],["timing",{timelineId:"film",documentId:"document",units:[]}],["schedule",{cues:[]}],["program",{uses}]]),spans:[],temporalBindings:[],} as unknown as StudioTrackCompanionContext;const entities=projectCaption(context);diff --git a/packages/caption/README.md b/packages/caption/README.mdindex 5d0f159d..fee66c1b 100644--- a/packages/caption/README.md+++ b/packages/caption/README.md@@ -4,14 +4,15 @@ External components import `@hypit/hypit/caption`. Source uses the `@hypit/captiidentity for shared declarations such as `Hidden`.Caption has independent content, timing and presentation structures: CaptionDocument organizes-displayed words into Cues, CaptionTiming locates those units, and timed Uses-choose how those Cues appear. A Use may begin inside a Cue. It changes presentation without changing-that Cue's text or restarting its word timing.+display units and hard content breaks, CaptionTiming locates every unit, and each rendering family+forms its own Cues before timed Uses choose how those Cues appear. A Use may begin inside a Cue. It+changes presentation without changing that Cue's text or restarting its word timing.`CaptionDocument` owns displayed words with authored separators, display units, word attributes and-Cue breaks. It contains no Narrative identities or time. `CaptionTiming` independently owns complete-Cues and absolute unit windows on one Timeline. A domain adapter can produce that timing from speech,-SRT/VTT can publish it directly, and authored absolute timing can use the same renderer.+hard breaks. It contains no Narrative identities or time. `CaptionTiming` independently owns one+complete, flat table of absolute unit boundaries on one Timeline. A domain adapter can produce that+timing from speech, SRT/VTT can publish it directly, and authored absolute timing can use the same+renderer. Cue grouping, visibility envelopes and handoff belong to the selected rendering family.A rendering family's Track accepts `document`, `timing`, `timeline`, the spatial inputs its rendereractually needs and ordered `Use` children. Fine takes one placement Frame:@@ -49,7 +50,7 @@ The public helpers and Types are in [index.ts](src/index.ts):typed Windows. The Track may export it for its Companion, like Visual and Audio Clips.- `captionUseVisibility(program, index, role, envelope)` intersects a Cue envelope with a Use andsubtracts later matching windows. Hidden participates even though it renders nothing.-- `CaptionTiming` retains only `timelineId`, `documentId`, Cue identities and absolute unit windows.+- `CaptionTiming` retains only `timelineId`, `documentId` and complete absolute unit boundaries.Narrative-specific binding and projection belong to `@hypit/narrative-caption`.Derive layout and animation from complete Cue content and original timing. Apply Use coverage as adiff --git a/packages/caption/src/index.ts b/packages/caption/src/index.tsindex 20afa094..04a08f7f 100644--- a/packages/caption/src/index.ts+++ b/packages/caption/src/index.ts@@ -28,7 +28,7 @@ export {sealCaptionProgram,sealCaptionStyle,} from "./style.js";-export { assertCaptionTiming } from "./temporalize.js";+export { assertCaptionTiming, assertCaptionTimingForDocument } from "./temporalize.js";export type * from "./types.js";export { captionUseVisibility } from "./visibility.js";diff --git a/packages/caption/src/manifest.ts b/packages/caption/src/manifest.tsindex cecc1a64..522262bb 100644--- a/packages/caption/src/manifest.ts+++ b/packages/caption/src/manifest.ts@@ -54,10 +54,7 @@ const timedUnit = object({ unitId: { schema: string }, startFrame: { schema: intexport const captionTimingSchema: ValueSchema = object({timelineId: { schema: string },documentId: { schema: string },- cues: { schema: { kind: "array", items: object({- id: { schema: string }, startFrame: { schema: integer }, endFrameExclusive: { schema: integer },- units: { schema: { kind: "array", minItems: 1, items: timedUnit } },- }) } },+ units: { schema: { kind: "array", items: timedUnit } },});export const captionMarkupSurfaces = [{diff --git a/packages/caption/src/style.ts b/packages/caption/src/style.tsindex 996c9058..f18ac344 100644--- a/packages/caption/src/style.ts+++ b/packages/caption/src/style.ts@@ -1,6 +1,6 @@import { canonicalStringify, canonicalize } from "@hypit/protocol";-import { assertCaptionDocument } from "./display.js";+import { assertCaptionDocumentIdentity } from "./identity.js";import { assertTemporalWindowFor } from "@hypit/temporal";import type { TemporalWindow } from "@hypit/temporal";import type { CaptionDocument, CaptionProgram, CaptionStyleIntent } from "./types.js";@@ -45,7 +45,7 @@ export function assertCaptionProgram(value: CaptionProgram): void {}export function assertCaptionProgramForDocument(value: CaptionProgram, document: CaptionDocument): void {assertCaptionProgram(value);- assertCaptionDocument(document);+ assertCaptionDocumentIdentity(document);assert(value.documentId === document.id, "Caption Uses belong to another CaptionDocument");}export function appendCaptionUse(program: CaptionProgram, window: TemporalWindow, style: CaptionStyleIntent, role?: string): CaptionProgram {diff --git a/packages/caption/src/temporalize.ts b/packages/caption/src/temporalize.tsindex 186bdc1b..b4b26e03 100644--- a/packages/caption/src/temporalize.ts+++ b/packages/caption/src/temporalize.ts@@ -1,22 +1,34 @@-import type { CaptionTiming } from "./types.js";+import { assertCaptionDocumentIdentity } from "./identity.js";+import type { CaptionDocument, CaptionTiming } from "./types.js";export function assertCaptionTiming(timing: CaptionTiming): void {if (timing.timelineId.length === 0 || timing.documentId.length === 0) {throw new Error("CaptionTiming provenance is invalid");}- const cueIds = new Set<string>();const unitIds = new Set<string>();- for (const cue of timing.cues) {- if (cue.id.length === 0 || cueIds.has(cue.id) || cue.units.length === 0- || !Number.isSafeInteger(cue.startFrame) || !Number.isSafeInteger(cue.endFrameExclusive)- || cue.startFrame < 0 || cue.endFrameExclusive <= cue.startFrame) throw new Error("CaptionTiming contains an invalid Cue");- cueIds.add(cue.id);- for (const unit of cue.units) {- if (unit.unitId.length === 0 || unitIds.has(unit.unitId) || !Number.isSafeInteger(unit.startFrame)- || !Number.isSafeInteger(unit.endFrameExclusive) || unit.startFrame < 0 || unit.endFrameExclusive <= unit.startFrame) {- throw new Error("CaptionTiming contains an invalid or repeated unit timing");- }- unitIds.add(unit.unitId);+ for (const unit of timing.units) {+ if (unit.unitId.length === 0 || unitIds.has(unit.unitId) || !Number.isSafeInteger(unit.startFrame)+ || !Number.isSafeInteger(unit.endFrameExclusive) || unit.startFrame < 0 || unit.endFrameExclusive <= unit.startFrame) {+ throw new Error("CaptionTiming contains an invalid or repeated unit timing");}+ unitIds.add(unit.unitId);+ }+}++/** Verify the named join without imposing array order or global temporal monotonicity. */+export function assertCaptionTimingForDocument(timing: CaptionTiming, document: CaptionDocument): void {+ assertCaptionTiming(timing);+ assertCaptionDocumentIdentity(document);+ if (timing.documentId !== document.id) throw new Error("CaptionTiming belongs to another CaptionDocument");+ const expected = new Set(document.units.map((unit) => unit.id));+ const actual = new Set(timing.units.map((unit) => unit.unitId));+ const missing = [...expected].filter((id) => !actual.has(id));+ const unknown = [...actual].filter((id) => !expected.has(id));+ if (missing.length > 0 || unknown.length > 0) {+ const details = [+ ...(missing.length === 0 ? [] : [`missing ${missing.join(", ")}`]),+ ...(unknown.length === 0 ? [] : [`unknown ${unknown.join(", ")}`]),+ ].join("; ");+ throw new Error(`CaptionTiming must cover CaptionDocument units exactly (${details})`);}}diff --git a/packages/caption/src/types.ts b/packages/caption/src/types.tsindex 2fd0c743..ac4d524b 100644--- a/packages/caption/src/types.ts+++ b/packages/caption/src/types.ts@@ -65,16 +65,10 @@ export type CaptionTimingUnit = {readonly endFrameExclusive: number;};-export type CaptionTimingCue = {- readonly id: string;- readonly startFrame: number;- readonly endFrameExclusive: number;- readonly units: readonly CaptionTimingUnit[];-};-export type CaptionTiming = {- /** Absolute Timeline on which every unit Window is resolved. */+ /** Absolute Timeline on which every unit boundary is resolved. */readonly timelineId: string;readonly documentId: string;- readonly cues: readonly CaptionTimingCue[];+ /** Complete, source-neutral timing for the selected CaptionDocument. */+ readonly units: readonly CaptionTimingUnit[];};diff --git a/packages/narrative-caption/README.md b/packages/narrative-caption/README.mdindex 602a4a0f..c1bc4555 100644--- a/packages/narrative-caption/README.md+++ b/packages/narrative-caption/README.md@@ -2,7 +2,9 @@`@hypit/narrative-caption` is the explicit adapter between source-neutral Caption content andNarrative time. It owns `NarrativeCaptionBinding` and projects that relation through one explicit-`NarrativeProjection` to ordinary absolute `CaptionTiming`.+`NarrativeProjection` to complete, flat, absolute `CaptionTiming`. Binding joins by Unit and Token+identities rather than array position. Missing Units, Tokens or Boundaries fail with their complete+identity path; the adapter never drops content, pads a zero interval, shortens overlaps or forms Cues.Script may emit a CaptionDocument and its binding from the same parse; the author does not repeatdisplay text or unit-to-Token correspondence. Caption renderers consume only CaptionDocument anddiff --git a/packages/narrative-caption/src/error.ts b/packages/narrative-caption/src/error.tsindex 87bd0d5a..3f7a37bc 100644--- a/packages/narrative-caption/src/error.ts+++ b/packages/narrative-caption/src/error.ts@@ -1,5 +1,5 @@export class NarrativeCaptionTimingError extends Error {- readonly code: "CAPTION_BINDING" | "CAPTION_TOKEN_ORDER";+ readonly code: "CAPTION_BINDING" | "CAPTION_PROJECTION" | "CAPTION_UNIT_WINDOW";constructor(code: NarrativeCaptionTimingError["code"], message: string) {super(message);diff --git a/packages/narrative-caption/src/identity.ts b/packages/narrative-caption/src/identity.tsindex 5f80c5c0..55452787 100644--- a/packages/narrative-caption/src/identity.ts+++ b/packages/narrative-caption/src/identity.ts@@ -38,10 +38,16 @@ export function assertNarrativeCaptionBindingFor(throw new Error("NarrativeCaptionBinding disagrees with its Narrative or CaptionDocument.");}const tokenIds = new Set(narrative.tokens.map((token) => token.id));- const documentUnitIds = document.units.map((unit) => unit.id);- if (value.units.length !== documentUnitIds.length- || value.units.some((unit, index) => unit.unitId !== documentUnitIds[index]- || unit.sourceTokenIds.some((tokenId) => !tokenIds.has(tokenId)))) {- throw new Error("NarrativeCaptionBinding must cover the document in order using known Narrative Tokens.");+ const documentUnitIds = new Set(document.units.map((unit) => unit.id));+ const bindingUnitIds = new Set(value.units.map((unit) => unit.unitId));+ if (documentUnitIds.size !== bindingUnitIds.size+ || [...documentUnitIds].some((unitId) => !bindingUnitIds.has(unitId))+ || [...bindingUnitIds].some((unitId) => !documentUnitIds.has(unitId))) {+ throw new Error("NarrativeCaptionBinding must cover CaptionDocument units exactly.");+ }+ const unknownToken = value.units.flatMap((unit) => unit.sourceTokenIds)+ .find((tokenId) => !tokenIds.has(tokenId));+ if (unknownToken !== undefined) {+ throw new Error(`NarrativeCaptionBinding references unknown Narrative Token ${unknownToken}.`);}}diff --git a/packages/narrative-caption/src/project.ts b/packages/narrative-caption/src/project.tsindex 05481dfa..0bba5d93 100644--- a/packages/narrative-caption/src/project.ts+++ b/packages/narrative-caption/src/project.ts@@ -1,68 +1,62 @@-import { assertCaptionDocument, assertCaptionTiming } from "@hypit/caption";-import type { CaptionDocument, CaptionTiming, CaptionTimingCue, CaptionTimingUnit } from "@hypit/caption";-import { tokenFrameSpan } from "@hypit/narrative-temporal";+import { assertCaptionDocumentIdentity, assertCaptionTimingForDocument } from "@hypit/caption";+import type { CaptionDocument, CaptionTiming, CaptionTimingUnit } from "@hypit/caption";import type { NarrativeProjection } from "@hypit/narrative-temporal";import type { NarrativeCaptionBinding } from "./types.js";import { NarrativeCaptionTimingError } from "./error.js";+import { assertNarrativeCaptionBindingIdentity } from "./identity.js";export function projectNarrativeCaptionTiming(document: CaptionDocument,binding: NarrativeCaptionBinding,projection: NarrativeProjection,): CaptionTiming {- assertCaptionDocument(document);+ assertCaptionDocumentIdentity(document);+ assertNarrativeCaptionBindingIdentity(binding);if (binding.documentId !== document.id || binding.narrativeId !== projection.narrativeId) {throw new NarrativeCaptionTimingError("CAPTION_BINDING", "Caption binding disagrees with its document or NarrativeProjection.");}- const sourceTokens = new Map(binding.units.map((unit) => [unit.unitId, unit.sourceTokenIds] as const));- const breaks = new Set(document.cueBreaks.map((cueBreak) => cueBreak.afterUnitId));- const timed: Array<{ unit: CaptionDocument["units"][number]; timing: CaptionTimingUnit }> = [];- const previousByRole = new Map<string | undefined, { timing: CaptionTimingUnit }>();+ const bindings = new Map(binding.units.map((unit) => [unit.unitId, unit.sourceTokenIds] as const));+ const documentIds = new Set(document.units.map((unit) => unit.id));+ const missingBinding = document.units.find((unit) => !bindings.has(unit.id));+ const unknownBinding = binding.units.find((unit) => !documentIds.has(unit.unitId));+ if (missingBinding !== undefined || unknownBinding !== undefined || bindings.size !== documentIds.size) {+ const path = missingBinding === undefined ? unknownBinding!.unitId : missingBinding.id;+ throw new NarrativeCaptionTimingError("CAPTION_BINDING",+ `CaptionDocument ${document.id} -> Binding ${binding.id} does not connect Unit ${path} exactly.`);+ }+ const tokens = new Map(projection.tokens.map((token) => [token.tokenId, token] as const));+ const boundaries = new Map(projection.boundaries.map((boundary) => [boundary.id, boundary.frame] as const));+ const timed: CaptionTimingUnit[] = [];for (const unit of document.units) {- const window = tokenFrameSpan(projection, sourceTokens.get(unit.id) ?? []);- if (window === undefined) continue;- if (window.endFrameExclusive < window.startFrame) {- throw new NarrativeCaptionTimingError("CAPTION_TOKEN_ORDER", `Caption unit ${unit.id} references Narrative Tokens in reverse order.`);+ const sourceTokenIds = bindings.get(unit.id)!;+ const sourceTokens = sourceTokenIds.map((tokenId) => {+ const token = tokens.get(tokenId);+ if (token === undefined) {+ throw new NarrativeCaptionTimingError("CAPTION_PROJECTION",+ `CaptionDocument ${document.id} -> Unit ${unit.id} -> Binding ${binding.id} -> Token ${tokenId} is absent from NarrativeProjection ${projection.id}.`);+ }+ return token;+ });+ const first = sourceTokens[0]!;+ const last = sourceTokens.at(-1)!;+ const startFrame = boundaries.get(first.startBoundaryId);+ if (startFrame === undefined) {+ throw new NarrativeCaptionTimingError("CAPTION_PROJECTION",+ `CaptionDocument ${document.id} -> Unit ${unit.id} -> Token ${first.tokenId} -> Boundary ${first.startBoundaryId} is absent from NarrativeProjection ${projection.id}.`);}- const startFrame = window.startFrame;- const endFrameExclusive = Math.max(startFrame + 1, window.endFrameExclusive);- const previous = previousByRole.get(unit.role);- if (previous !== undefined && startFrame > previous.timing.startFrame && startFrame < previous.timing.endFrameExclusive) {- previous.timing = { ...previous.timing, endFrameExclusive: startFrame };+ const endFrameExclusive = boundaries.get(last.endBoundaryId);+ if (endFrameExclusive === undefined) {+ throw new NarrativeCaptionTimingError("CAPTION_PROJECTION",+ `CaptionDocument ${document.id} -> Unit ${unit.id} -> Token ${last.tokenId} -> Boundary ${last.endBoundaryId} is absent from NarrativeProjection ${projection.id}.`);}- const entry = { unit, timing: { unitId: unit.id, startFrame, endFrameExclusive } };- previousByRole.set(unit.role, entry);- timed.push(entry);- }- const cues: CaptionTimingCue[] = [];- let current: { units: CaptionTimingUnit[]; groupId?: string } | undefined;- const flush = (): void => {- if (current === undefined || current.units.length === 0) return;- cues.push({- id: `${document.id}:cue:${cues.length + 1}`,- startFrame: current.units[0]!.startFrame,- endFrameExclusive: current.units.at(-1)!.endFrameExclusive,- units: current.units,- });- current = undefined;- };- const documentOrder = new Map(document.units.map((unit, index) => [unit.id, index]));- for (const [index, entry] of timed.entries()) {- const previousDocumentUnit = index === 0 ? undefined : timed[index - 1]!.unit;- const mustBreak = current !== undefined && (- current.groupId !== entry.unit.groupId- || (previousDocumentUnit !== undefined && (breaks.has(previousDocumentUnit.id)- || documentOrder.get(entry.unit.id)! !== documentOrder.get(previousDocumentUnit.id)! + 1))- );- if (mustBreak) flush();- if (current === undefined) {- current = { units: [], ...(entry.unit.groupId === undefined ? {} : { groupId: entry.unit.groupId }) };+ if (endFrameExclusive <= startFrame) {+ throw new NarrativeCaptionTimingError("CAPTION_UNIT_WINDOW",+ `CaptionDocument ${document.id} -> Unit ${unit.id} resolves to invalid absolute boundaries ${startFrame}..${endFrameExclusive} on Timeline ${projection.timelineId}.`);}- current.units.push(entry.timing);+ timed.push({ unitId: unit.id, startFrame, endFrameExclusive });}- flush();- const result: CaptionTiming = { timelineId: projection.timelineId, documentId: document.id, cues };- assertCaptionTiming(result);+ const result: CaptionTiming = { timelineId: projection.timelineId, documentId: document.id, units: timed };+ assertCaptionTimingForDocument(result, document);return result;}diff --git a/packages/provider-html-local/test/visual-readiness.test.ts b/packages/provider-html-local/test/visual-readiness.test.tsindex d8224a1e..cc9febfa 100644--- a/packages/provider-html-local/test/visual-readiness.test.ts+++ b/packages/provider-html-local/test/visual-readiness.test.ts@@ -294,9 +294,8 @@ test("Fine Caption Cues stay isolated over Picture during reverse seeks and inde{ styleId: "green", window: useWindow("green-use", 30, 60) },{ styleId: "blue", window: useWindow("blue-use", 60, 90) },] };- const timing = { timelineId: timeline.id, documentId: document.id, cues: document.units.map((unit, index) => ({- id: `cue-${index + 1}`, startFrame: index * 30, endFrameExclusive: index * 30 + 30,- units: [{ unitId: unit.id, startFrame: index * 30, endFrameExclusive: index * 30 + 30 }],+ const timing = { timelineId: timeline.id, documentId: document.id, units: document.units.map((unit, index) => ({+ unitId: unit.id, startFrame: index * 30, endFrameExclusive: index * 30 + 30,})) };const captions = renderFineCaption(scheduleFineCaption(timing, program, document), program, document,timeline, { xPx: 0, yPx: 0, widthPx: 320, heightPx: 180 });diff --git a/test/narrative-caption.test.ts b/test/narrative-caption.test.tsindex 6587d277..5d407e0a 100644--- a/test/narrative-caption.test.ts+++ b/test/narrative-caption.test.ts@@ -91,9 +91,8 @@ for (const grouped of [false, true]) test(`Chinese Script projects Caption timinconst timing = projectNarrativeCaptionTiming(document, binding, narrativeProjection);const wordText = new Map(document.words.map((word) => [word.id, word.text]));const unitText = new Map(document.units.map((unit) => [unit.id, unit.wordIds.map((id) => wordText.get(id)!).join("")]));- assert.deepEqual(timing.cues.map((cue) => cue.units.map((unit) => unitText.get(unit.unitId)).join("")),- ["用ElevenLabs做视频,", "真方便。", "今年2026年,這個很好。"]);- const units = timing.cues.flatMap((cue) => cue.units);+ assert.equal("cues" in timing, false);+ const units = timing.units;const windowsFor = (text: string) => units.filter((unit) => unitText.get(unit.unitId) === text).map((unit) => [unit.startFrame, unit.endFrameExclusive]);const first = evidenceWindows[0]!;@@ -113,3 +112,59 @@ for (const grouped of [false, true]) test(`Chinese Script projects Caption timinassert.deepEqual(windowsFor("個"), [[4000 + second[8]![0], 4000 + second[8]![1]]]);assert.equal(document.units[0]!.role, "HOST");});++test("Caption timing joins named units, preserves overlaps and never repairs projection evidence", () => {+ const document = {+ id: "caption",+ units: [+ { id: "unit-a", wordIds: ["word-a"] },+ { id: "unit-b", wordIds: ["word-b"] },+ ],+ words: [+ { id: "word-a", unitId: "unit-a", text: "A", separatorBefore: "" as const, attributes: [] },+ { id: "word-b", unitId: "unit-b", text: "B", separatorBefore: " " as const, attributes: [] },+ ],+ cueBreaks: [],+ };+ const binding = {+ id: "binding", narrativeId: "story", documentId: document.id,+ // Deliberately reversed: identity, not array position, owns the join.+ units: [+ { unitId: "unit-b", sourceTokenIds: ["token-b"] },+ { unitId: "unit-a", sourceTokenIds: ["token-a"] },+ ],+ };+ const projection = {+ id: "projection", narrativeId: "story", timelineId: "film",+ segments: [{ segmentId: "segment", startBoundaryId: "segment-start", endBoundaryId: "segment-end" }],+ tokens: [+ { tokenId: "token-a", segmentId: "segment", text: "A", startBoundaryId: "a-start", endBoundaryId: "a-end" },+ { tokenId: "token-b", segmentId: "segment", text: "B", startBoundaryId: "b-start", endBoundaryId: "b-end" },+ ],+ boundaries: [+ { id: "segment-start", frame: 0 }, { id: "a-start", frame: 0 },+ { id: "b-start", frame: 5 }, { id: "a-end", frame: 10 },+ { id: "b-end", frame: 15 }, { id: "segment-end", frame: 15 },+ ],+ };+ assert.deepEqual(projectNarrativeCaptionTiming(document, binding, projection).units, [+ { unitId: "unit-a", startFrame: 0, endFrameExclusive: 10 },+ { unitId: "unit-b", startFrame: 5, endFrameExclusive: 15 },+ ]);++ const missingToken = { ...projection, tokens: projection.tokens.slice(0, 1) };+ assert.throws(() => projectNarrativeCaptionTiming(document, binding, missingToken),+ /CaptionDocument caption -> Unit unit-b -> Binding binding -> Token token-b is absent/u);+ const zeroLength = { ...projection, boundaries: projection.boundaries.map((boundary) =>+ boundary.id === "a-end" ? { ...boundary, frame: 0 } : boundary) };+ assert.throws(() => projectNarrativeCaptionTiming(document, binding, zeroLength),+ /Unit unit-a resolves to invalid absolute boundaries 0\.\.0/u);+});++test("an empty CaptionDocument projects to one complete empty timing table", () => {+ assert.deepEqual(projectNarrativeCaptionTiming(+ { id: "empty", units: [], words: [], cueBreaks: [] },+ { id: "binding", narrativeId: "story", documentId: "empty", units: [] },+ { id: "projection", narrativeId: "story", timelineId: "film", segments: [], tokens: [], boundaries: [] },+ ), { timelineId: "film", documentId: "empty", units: [] });+});