Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

refactor: make caption cues authored content

kashorind7b38b709456
diff --git a/packages/caption-fine/README.md b/packages/caption-fine/README.md
index 4cba194b..e9ce0aa1 100644
--- a/packages/caption-fine/README.md
+++ b/packages/caption-fine/README.md
@@ -49,9 +49,9 @@ One SVS Recipe freezes three public dimensions:
- **How**: the exact font stack, typography, base/active glyph Paint, Cue box and decoration.
- **When**: the visible lead/tail envelope, handoff, Cue/Atom motion, reveal, Karaoke and loops.
-An upstream adapter supplies absolute Caption unit timing. Fine then produces an explicit visible
-Schedule: it forms Cues from document order, hard breaks and complete Unit timing, then renders that
-Schedule. Lead and tail never change the source times used by Karaoke. Later Uses mask earlier
+An upstream adapter supplies absolute Caption Unit timing. Fine then joins the document's authored
+Cues to that complete timing and produces an explicit visible Schedule. It never splits or merges a
+Cue. Lead and tail never change the source times used by Karaoke. Later Uses mask earlier
presentation, including Hidden. The final visibility is clipped to the winning Use Window while the
original Cue envelope and animations are preserved.
diff --git a/packages/caption-fine/src/manifest.ts b/packages/caption-fine/src/manifest.ts
index 4b2b616f..3a25b139 100644
--- a/packages/caption-fine/src/manifest.ts
+++ b/packages/caption-fine/src/manifest.ts
@@ -38,7 +38,7 @@ export const captionFineMarkupSurfaces = [
summary: "Resolves one SVS Recipe and one exact font stack into a complete fine-grained Caption Style.",
attributes: [
{ name: "id", kind: "identifier", required: true,
- summary: "Names this Style so a timed Use can select it." },
+ summary: "Names this Style so a Use can select it." },
{ name: "recipe", kind: "reference", required: true, accepts: [recipeType],
summary: "Chooses the Recipe that carries Cue geometry, Paint and local motion.",
recipe: [
@@ -316,7 +316,7 @@ export const captionFineMarkupSurfaces = [
{ name: "regions", kind: "reference", required: false, accepts: [regionEvidenceTypes.evidence],
summary: "Optionally follows external, frame-exact regions whose ids equal Script Roles, placing a Cue at its measured speaker region's top center and hiding it on null Frames." },
],
- children: [{ tag: "Use", cardinality: "many", summary: "Presents complete Cues inside a time window. Later matching Uses replace earlier ones.", attributes: [
+ children: [{ tag: "Use", cardinality: "many", summary: "Presents matching complete Cues. A Window can narrow its scope; later matching Uses replace earlier ones.", attributes: [
{ name: "id", kind: "identifier", required: false, summary: "Optional occurrence identity." },
{ name: "style", kind: "reference", required: true, accepts: [captionTypes.style], summary: "Complete Caption Style, including Hidden." },
{ name: "role", kind: "literal", required: false, summary: "Optional speaker filter, independent of the time window." },
@@ -333,8 +333,8 @@ export const captionFineMarkupSurfaces = [
<caption-fine:Use during={answer} style={answer-caption}/>
</caption-fine:Caption>`,
notes: [
- "Use without time attributes covers the whole Timeline. Role filters content without changing the window.",
- "Script Cue breaks organize text; Use windows change presentation without cutting Cues or restarting word timing. A hidden Use still replaces earlier presentation.",
+ "Use without time attributes applies to every matching Cue in the CaptionDocument. Role filters content without changing a Window.",
+ "Script authors complete Cues; Use windows change presentation without cutting those Cues or restarting Unit timing. A hidden Use still replaces earlier presentation.",
"Without regions, Recipe x/y placement is unchanged. With regions, a measured region overrides x/y for that Frame; a null Frame in that Role series hides the Cue, while a missing Role series retains authored x/y.",
"The tracked point replaces Recipe x/y while width and the Recipe anchor still decide the Caption box geometry; anchor-x=center and anchor-y=bottom place the box immediately above the measured region.",
],
diff --git a/packages/caption-fine/src/render.ts b/packages/caption-fine/src/render.ts
index 0abd7c2e..cac27fd9 100644
--- a/packages/caption-fine/src/render.ts
+++ b/packages/caption-fine/src/render.ts
@@ -1097,6 +1097,7 @@ export function renderFineCaption(
const styles = new Map(program.styles.map((style) => [style.id, style]));
const wordText = new Map(document.words.map((word) => [word.id, word]));
const atomById = new Map(document.units.map((atom) => [atom.id, atom]));
+ const cueById = new Map(document.cues.map((cue) => [cue.id, cue]));
for (const style of styles.values()) {
if (style.rendering === null) continue;
if (style.rendering.family !== FINE_CAPTION_FAMILY) {
@@ -1114,6 +1115,8 @@ export function renderFineCaption(
return new Map(regions.series.map((series) => [series.id, series]));
})();
const presents = schedule.cues.flatMap((cue, order) => {
+ const authoredCue = cueById.get(cue.cueId);
+ if (authoredCue === undefined) throw new Error(`Fine Caption Schedule references unknown authored Cue ${cue.cueId}`);
const atoms = cue.units.map((timing) => atomById.get(timing.unitId));
if (atoms.some((atom) => atom === undefined)) throw new Error(`Fine Caption Cue ${cue.id} references unknown Atom`);
const resolvedAtoms = atoms.map((atom) => atom!);
@@ -1127,10 +1130,8 @@ export function renderFineCaption(
if (startFrame >= totalFrames || endFrameExclusive <= startFrame) return [];
const durationFrames = endFrameExclusive - startFrame;
const trackedPlacement = regionSeriesById === undefined ? undefined : (() => {
- const role = resolvedAtoms[0]?.role;
- if (role === undefined || resolvedAtoms.some((atom) => atom.role !== role)) {
- throw new Error(`Fine Caption Cue ${cue.id} must contain exactly one Script Role to follow regions`);
- }
+ const role = authoredCue.role;
+ if (role === undefined) throw new Error(`Fine Caption Cue ${cue.id} must have one Script Role to follow regions`);
const series = regionSeriesById.get(role);
if (series === undefined) return undefined;
return {
diff --git a/packages/caption-fine/src/schedule.ts b/packages/caption-fine/src/schedule.ts
index 372b5ea0..43596a52 100644
--- a/packages/caption-fine/src/schedule.ts
+++ b/packages/caption-fine/src/schedule.ts
@@ -11,37 +11,25 @@ function assertIntegerFrame(value: number, label: string): void {
type FineCaptionContentCue = {
readonly id: string;
+ readonly role?: string;
readonly startFrame: number;
readonly endFrameExclusive: number;
readonly units: CaptionTiming["units"];
};
-/** Fine owns how complete timed Units become readable Cues. */
+/** Join authored Cues to their complete absolute Unit timing. */
function fineCaptionContentCues(timing: CaptionTiming, document: CaptionDocument): readonly FineCaptionContentCue[] {
const timingByUnit = new Map(timing.units.map((unit) => [unit.unitId, unit] as const));
- const breaks = new Set(document.cueBreaks.map((item) => item.afterUnitId));
- const cues: FineCaptionContentCue[] = [];
- let current: CaptionTiming["units"][number][] = [];
- let groupId: string | undefined;
- const flush = (): void => {
- if (current.length === 0) return;
- cues.push({
- id: `${document.id}:cue:${cues.length + 1}`,
- startFrame: Math.min(...current.map((unit) => unit.startFrame)),
- endFrameExclusive: Math.max(...current.map((unit) => unit.endFrameExclusive)),
- units: current,
- });
- current = [];
- groupId = undefined;
- };
- for (const [index, unit] of document.units.entries()) {
- const previous = index === 0 ? undefined : document.units[index - 1];
- if (current.length > 0 && (groupId !== unit.groupId || (previous !== undefined && breaks.has(previous.id)))) flush();
- if (current.length === 0) groupId = unit.groupId;
- current.push(timingByUnit.get(unit.id)!);
- }
- flush();
- return cues;
+ return document.cues.map((cue) => {
+ const units = cue.unitIds.map((unitId) => timingByUnit.get(unitId)!);
+ return {
+ id: cue.id,
+ ...(cue.role === undefined ? {} : { role: cue.role }),
+ startFrame: Math.min(...units.map((unit) => unit.startFrame)),
+ endFrameExclusive: Math.max(...units.map((unit) => unit.endFrameExclusive)),
+ units,
+ };
+ });
}
/**
@@ -71,14 +59,14 @@ export function scheduleFineCaption(
parameters.set(style.id, value);
}
- const units = new Map(document.units.map(unit => [unit.id, unit]));
const contentCues = fineCaptionContentCues(timing, document);
+ const roleByCue = new Map(contentCues.map((cue) => [cue.id, cue.role] as const));
const cues: FineCaptionScheduledCue[] = [];
for (const [index, use] of program.uses.entries()) {
if (use.window !== undefined && use.window.start.timelineId !== timing.timelineId) throw new Error("Caption Use belongs to another Timeline");
const style = parameters.get(use.styleId);
if (style === undefined) continue; // Hidden still participates in coverage below.
- const desired = contentCues.filter(cue => use.role === undefined || units.get(cue.units[0]!.unitId)?.role === use.role).map((cue): FineCaptionScheduledCue => ({
+ const desired = contentCues.filter(cue => use.role === undefined || cue.role === use.role).map((cue): FineCaptionScheduledCue => ({
id: `${use.id}:${cue.id}`, cueId: cue.id, styleId: use.styleId,
timedStartFrame: cue.startFrame, timedEndFrameExclusive: cue.endFrameExclusive,
visibleStartFrame: Math.max(0, cue.startFrame - style.timing.leadFrames),
@@ -91,7 +79,7 @@ export function scheduleFineCaption(
const envelopes: FineCaptionScheduledCue[] = [];
for (const wanted of desired) {
let cue = wanted;
- const role = units.get(cue.units[0]!.unitId)?.role;
+ const role = roleByCue.get(cue.cueId);
const previousIndex = previousByRole.get(role);
const previous = previousIndex === undefined ? undefined : envelopes[previousIndex];
if (previous !== undefined && style.timing.handoff === "cut" && previous.timedEndFrameExclusive <= cue.timedStartFrame) {
@@ -103,7 +91,7 @@ export function scheduleFineCaption(
envelopes.push(cue);
}
for (const cue of envelopes) {
- const role = units.get(cue.units[0]!.unitId)?.role;
+ const role = roleByCue.get(cue.cueId);
const visibility = captionUseVisibility(program, index, role, { startFrame: cue.visibleStartFrame, endFrameExclusive: cue.visibleEndFrameExclusive });
if (visibility.length) cues.push({ ...cue, visibility });
}
diff --git a/packages/caption-fine/test/caption-fine.test.ts b/packages/caption-fine/test/caption-fine.test.ts
index c8687d0b..b642aefe 100644
--- a/packages/caption-fine/test/caption-fine.test.ts
+++ b/packages/caption-fine/test/caption-fine.test.ts
@@ -272,7 +272,7 @@ test("Fine Caption gives a hidden Style no Cue, background or decoration", () =>
});
test("Fine Caption accepts one complete empty document as an ordinary no-content result", () => {
- const document = { id: "empty-caption", units: [], words: [], cueBreaks: [] };
+ const document = { id: "empty-caption", units: [], words: [], cues: [] };
const program: CaptionProgram = { id: "empty", documentId: document.id, styles: [], uses: [] };
assert.deepEqual(scheduleFineCaption({ timelineId: "test-space", documentId: document.id, units: [] }, program, document), {
timelineId: "test-space", documentId: document.id, cues: [],
diff --git a/packages/caption/README.md b/packages/caption/README.md
index 4495a853..6d9701c0 100644
--- a/packages/caption/README.md
+++ b/packages/caption/README.md
@@ -3,16 +3,19 @@
External components import `@hypit/hypit/caption`. Source uses the `@hypit/caption@1` Module
identity for shared declarations such as `Hidden`.
-Caption has independent content, timing and presentation structures: CaptionDocument organizes
-display units and hard content breaks, CaptionTiming locates every unit, and each rendering family
-forms its own Cues before timed Uses choose how those Cues appear. A Use may begin inside a Cue. It
-changes presentation without changing that Cue's text or restarting its word timing.
+Caption has independent content, timing and presentation structures: CaptionDocument owns display
+Words, correspondence Units and authored Cues; CaptionTiming locates every Unit; and each rendering
+family presents those Cues before timed Uses choose how they appear. A Use may begin inside a Cue. It
+changes presentation without changing that Cue's content or restarting its Unit timing.
`CaptionDocument` owns displayed words with authored separators, display units, word attributes and
-hard breaks. It contains no Narrative identities or time. `CaptionTiming` independently owns one
+complete Cue membership. Every Word belongs to exactly one Unit and every Unit belongs to exactly
+one Cue, in document order. A Cue may carry one Role. The document contains no Narrative token or
+Segment identities and no time. `CaptionTiming` independently owns one
complete, flat table of absolute unit boundaries on one Timeline. A domain adapter can produce that
timing from speech, SRT/VTT can publish it directly, and authored absolute timing can use the same
-renderer. Cue grouping, visibility envelopes and handoff belong to the selected rendering family.
+renderer. Cue membership belongs to the document; visibility envelopes, handoff, layout and motion
+belong to the selected rendering family.
A rendering family's Track accepts `document`, `timing`, `timeline`, the spatial inputs its renderer
actually needs and ordered `Use` children. Fine takes one placement Frame:
@@ -54,11 +57,12 @@ The public helpers and Types are in [index.ts](src/index.ts):
- `CaptionTiming` retains only `timelineId`, `documentId` and complete absolute unit boundaries.
Narrative-specific binding and projection belong to `@hypit/narrative-caption`.
-Derive layout and animation from complete Cue content and original timing. Apply Use coverage as a
+Derive layout and animation from authored Cue content and original timing. Apply Use coverage as a
visibility mask. Fine keeps the original Present span and element animations, with separate
`visibility` intervals; changing or briefly hiding a Style does not restart karaoke, typing or motion.
-A family may define lead/tail and handoff behavior, but its resulting visibility stays inside the
-winning Use window. Empty content produces no drawing.
+A family may define lead/tail, handoff, lines, pages and local states, but it does not split or merge
+authored Cues. Its resulting visibility stays inside the winning Use window. Empty content produces
+no drawing.
Word attributes remain on `CaptionDocument.words`. A structural family can interpret an explicit
attribute as a keyword role while retaining complete display/alignment units. Time selection does
diff --git a/packages/caption/src/display.ts b/packages/caption/src/display.ts
index 62883f70..977f0493 100644
--- a/packages/caption/src/display.ts
+++ b/packages/caption/src/display.ts
@@ -1,35 +1,14 @@
import type { CaptionDocument } from "./types.js";
+import { assertCaptionDocumentIdentity } from "./identity.js";
function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message);
}
export function assertCaptionDocument(value: CaptionDocument): void {
- assert(value.id.length > 0, "CaptionDocument identity is invalid");
- assert(value.units.length > 0 && value.words.length > 0, "CaptionDocument is empty");
+ assertCaptionDocumentIdentity(value);
+ assert(value.units.length > 0, "CaptionDocument is empty");
for (const word of value.words) assert(word.separatorBefore === "" || word.separatorBefore === " ", "Caption word must declare its authored separator");
- const words = new Map(value.words.map((word) => [word.id, word]));
- assert(words.size === value.words.length, "CaptionDocument word ids are repeated");
- const units = new Set<string>();
- const orderedWordIds: string[] = [];
- for (const unit of value.units) {
- assert(unit.id.length > 0 && !units.has(unit.id), "CaptionDocument unit ids are repeated");
- units.add(unit.id);
- assert(unit.wordIds.length > 0, `Caption unit ${unit.id} is empty`);
- for (const wordId of unit.wordIds) {
- const word = words.get(wordId);
- assert(word !== undefined && word.unitId === unit.id, `Caption unit ${unit.id} references a foreign word`);
- orderedWordIds.push(wordId);
- }
- }
- assert(orderedWordIds.join("\0") === value.words.map((word) => word.id).join("\0"),
- "CaptionDocument words must be partitioned by units in order");
- const breakIds = new Set<string>();
- for (const cueBreak of value.cueBreaks) {
- assert(units.has(cueBreak.afterUnitId) && !breakIds.has(cueBreak.afterUnitId),
- "CaptionDocument cue break names an unknown or repeated unit");
- breakIds.add(cueBreak.afterUnitId);
- }
}
export type CaptionUnitSubset = {
@@ -39,7 +18,8 @@ export type CaptionUnitSubset = {
export function captionUnitsForRole(document: CaptionDocument, role: string): CaptionUnitSubset {
assertCaptionDocument(document);
- const unitIds = document.units.filter((unit) => unit.role === role).map((unit) => unit.id);
+ const selected = new Set(document.cues.filter((cue) => cue.role === role).flatMap((cue) => cue.unitIds));
+ const unitIds = document.units.filter((unit) => selected.has(unit.id)).map((unit) => unit.id);
if (unitIds.length === 0) throw new Error(`Caption Role ${role} selects no display unit`);
return { documentId: document.id, unitIds };
}
diff --git a/packages/caption/src/identity.ts b/packages/caption/src/identity.ts
index 95262871..bb044044 100644
--- a/packages/caption/src/identity.ts
+++ b/packages/caption/src/identity.ts
@@ -17,10 +17,11 @@ function unique(values: readonly string[], subject: string): Set<string> {
/** Validate the source-neutral document identity, including the valid wordless document. */
export function assertCaptionDocumentIdentity(value: CaptionDocument): void {
nonempty(value.id, "CaptionDocument id");
- if ((value.units.length === 0) !== (value.words.length === 0)) {
- throw new Error("CaptionDocument units and words must be empty together.");
+ if (!([value.words.length, value.units.length, value.cues.length].every((length) => length === 0)
+ || [value.words.length, value.units.length, value.cues.length].every((length) => length > 0))) {
+ throw new Error("CaptionDocument words, units and Cues must be empty together.");
}
- const unitIds = unique(value.units.map((item) => item.id), "CaptionDocument unit id");
+ unique(value.units.map((item) => item.id), "CaptionDocument unit id");
const wordIds = unique(value.words.map((item) => item.id), "CaptionDocument word id");
const words = new Map(value.words.map((word) => [word.id, word] as const));
const orderedWords: string[] = [];
@@ -38,8 +39,15 @@ export function assertCaptionDocumentIdentity(value: CaptionDocument): void {
|| orderedWords.some((id, index) => id !== value.words[index]?.id)) {
throw new Error("CaptionDocument units must partition words in order.");
}
- unique(value.cueBreaks.map((item) => item.afterUnitId), "CaptionDocument Cue Break");
- if (value.cueBreaks.some((item) => !unitIds.has(item.afterUnitId))) {
- throw new Error("CaptionDocument Cue Break names an unknown unit.");
+ unique(value.cues.map((item) => item.id), "CaptionDocument Cue id");
+ const orderedUnits: string[] = [];
+ for (const cue of value.cues) {
+ if (cue.unitIds.length === 0) throw new Error(`Caption Cue ${cue.id} is empty.`);
+ if (cue.role !== undefined) nonempty(cue.role, `Caption Cue ${cue.id} Role`);
+ orderedUnits.push(...cue.unitIds);
+ }
+ if (orderedUnits.length !== value.units.length
+ || orderedUnits.some((id, index) => id !== value.units[index]?.id)) {
+ throw new Error("CaptionDocument Cues must partition units in order.");
}
}
diff --git a/packages/caption/src/manifest.ts b/packages/caption/src/manifest.ts
index 63c10421..c9e7bb1b 100644
--- a/packages/caption/src/manifest.ts
+++ b/packages/caption/src/manifest.ts
@@ -28,14 +28,18 @@ const captionDisplayWord = object({
}) } },
});
const captionUnit = object({
- id: { schema: string }, groupId: { schema: string, optional: true }, role: { schema: string, optional: true },
+ id: { schema: string },
wordIds: { schema: { kind: "array", minItems: 1, items: string } },
});
+const captionCue = object({
+ id: { schema: string }, unitIds: { schema: { kind: "array", minItems: 1, items: string } },
+ role: { schema: string, optional: true },
+});
export const captionDocumentSchema: ValueSchema = object({
id: { schema: string },
units: { schema: { kind: "array", items: captionUnit } },
words: { schema: { kind: "array", items: captionDisplayWord } },
- cueBreaks: { schema: { kind: "array", items: object({ afterUnitId: { schema: string } }) } },
+ cues: { schema: { kind: "array", items: captionCue } },
});
export const captionStyleSchema: ValueSchema = object({
id: { schema: string },
diff --git a/packages/caption/src/types.ts b/packages/caption/src/types.ts
index b66f9478..344a2e62 100644
--- a/packages/caption/src/types.ts
+++ b/packages/caption/src/types.ts
@@ -20,21 +20,22 @@ export type CaptionWordAttribute = {
/** The smallest author-declared N:M display unit. It owns no source-domain identity. */
export type CaptionUnit = {
readonly id: string;
- readonly groupId?: string;
- readonly role?: string;
readonly wordIds: readonly string[];
};
-export type CaptionCueBreak = {
- readonly afterUnitId: string;
+/** One authored reading group made from complete consecutive display units. */
+export type CaptionCue = {
+ readonly id: string;
+ readonly unitIds: readonly string[];
+ readonly role?: string;
};
-/** Complete display truth. It contains neither semantic identities nor measured time. */
+/** Complete display truth. It contains neither source token identities nor measured time. */
export type CaptionDocument = {
readonly id: string;
readonly units: readonly CaptionUnit[];
readonly words: readonly CaptionDisplayWord[];
- readonly cueBreaks: readonly CaptionCueBreak[];
+ readonly cues: readonly CaptionCue[];
};
export type CaptionStyleIntent = {
diff --git a/packages/caption/test/caption.test.ts b/packages/caption/test/caption.test.ts
new file mode 100644
index 00000000..d696db40
--- /dev/null
+++ b/packages/caption/test/caption.test.ts
@@ -0,0 +1,34 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { assertCaptionDocumentIdentity, captionUnitsForRole } from "@hypit/caption";
+import type { CaptionDocument } from "@hypit/caption";
+
+const document: CaptionDocument = {
+ id: "captions",
+ words: [
+ { id: "word-a", unitId: "unit-a", text: "Shown", separatorBefore: "", attributes: [] },
+ { id: "word-b", unitId: "unit-a", text: "words", separatorBefore: " ", attributes: [] },
+ { id: "word-c", unitId: "unit-b", text: "Next", separatorBefore: " ", attributes: [] },
+ ],
+ units: [
+ { id: "unit-a", wordIds: ["word-a", "word-b"] },
+ { id: "unit-b", wordIds: ["word-c"] },
+ ],
+ cues: [
+ { id: "cue-a", unitIds: ["unit-a"], role: "HOST" },
+ { id: "cue-b", unitIds: ["unit-b"], role: "GUEST" },
+ ],
+};
+
+test("CaptionDocument partitions Words into Units and Units into authored Cues", () => {
+ assert.doesNotThrow(() => assertCaptionDocumentIdentity(document));
+ assert.deepEqual(captionUnitsForRole(document, "HOST"), {
+ documentId: document.id,
+ unitIds: ["unit-a"],
+ });
+ assert.throws(() => assertCaptionDocumentIdentity({
+ ...document,
+ cues: [{ id: "cue", unitIds: ["unit-b", "unit-a"] }],
+ }), /Cues must partition units in order/u);
+});
diff --git a/packages/script/README.md b/packages/script/README.md
index 7aca8173..fe6dcafb 100644
--- a/packages/script/README.md
+++ b/packages/script/README.md
@@ -54,10 +54,10 @@ The Surface exports one full Narrative plus narrow, immutable views:
- `script.segment.<id>` is a narrow `NarrativeSegmentRef` used to align generated or supplied media with one Segment;
- `script.segment.<id>.dialogue` is ordinary `Text`: display-independent dialogue, including optional
- Role cues and right-side Dual Text pronunciation, for a speech-video model;
+ Role annotations and right-side Dual Text pronunciation, for a speech-video model;
- `script.segment.<id>.speech` is ordinary pronunciation-only `Text` for duration estimation or TTS;
- `script.caption` is one complete `CaptionDocument`: ordered display Words, N:M Alignment Units,
- and authored Cue Breaks, empty when the Script has no visible Caption words;
+ and authored Cues, empty when the Script has no visible Caption words;
- `script.selection.<id>` is a reusable explicit Selection;
- `script.moment.<id>` is a reusable explicit Moment.
@@ -77,14 +77,14 @@ CaptionDocument contracts.
- **Selection marker**: a named semantic range, written `@{name} ... @{/name}`.
- **Moment marker**: a named semantic point, written `@{name!}`.
- **CaptionDocument**: a Caption-owned value produced by Script; it contains **Display Words**,
- **Alignment Units** and **Cue Breaks**. It contains no seconds or frames.
+ **Alignment Units** and authored **Cues**. It contains no seconds or frames.
- **Token attribute**: a flat postfix display-word annotation such as `really{emphasis}` or
`really{emphasis,importance=2,tone=warm}`. Values may be strings, finite numbers or booleans. It
becomes `CaptionDisplayWord.attributes`; it is not a Selection and does not carry timing.
Within Dual Text, an unescaped `@` belongs to the source of spoken text: the right side when supplied,
or the shared left side in `<display text|>`. Write `\@` if an at-sign must be shown. An empty display side, such as `< | spoken words>`, keeps the speech
-tokens and omits them from Caption. `||` is an authored Caption Cue Break and must occur between
+tokens and omits them from Caption. `||` ends the current authored Caption Cue and must occur between
complete Alignment Units.
`<组件化|>` is shorthand for `<组件化|组件化>`, using the same exported Caption Alignment Unit.
@@ -117,7 +117,7 @@ Units; it does not split the Segment, cut the picture or end a Selection.
```
Dual Text preserves display spelling while supplying an explicit pronunciation. Its N:M Alignment
-Unit is indivisible for Caption timing and authored Cue Breaks. Roles are lexical speaking cues;
+Unit is indivisible for Caption timing and Cue membership. Roles are lexical speaking cues;
the Source's model references and action direction bind them to the intended performers.
English words and numbers normally form lexical units; Han characters form individual units, as do
diff --git a/packages/script/src/manifest.ts b/packages/script/src/manifest.ts
index a7f361d0..7f189374 100644
--- a/packages/script/src/manifest.ts
+++ b/packages/script/src/manifest.ts
@@ -44,7 +44,7 @@ export const scriptMarkupSurfaces = [
{ name: "segment.<id>.speech", type: textTypes.text,
summary: "One Segment as pronunciation only, with Role Cue labels dropped." },
{ name: "caption", type: captionDocumentType,
- summary: "A source-neutral CaptionDocument: display Words, grouping units and Cue breaks." },
+ summary: "A source-neutral CaptionDocument: display Words, correspondence Units and authored Cues." },
{ name: "caption-binding", type: narrativeCaptionBindingType,
summary: "The explicit relation from Caption units to this Narrative's speech Tokens." },
{ name: "selection.<id>", type: narrativeSelectionType,
diff --git a/packages/script/src/narrative.ts b/packages/script/src/narrative.ts
index 53b0df22..c710f1d1 100644
--- a/packages/script/src/narrative.ts
+++ b/packages/script/src/narrative.ts
@@ -1,6 +1,6 @@
import { canonicalize } from "@hypit/hypit/protocol";
import type { CanonicalValue } from "@hypit/hypit/protocol";
-import type { CaptionDocument, CaptionDisplayWord, CaptionUnit } from "@hypit/hypit/caption";
+import type { CaptionCue, CaptionDocument, CaptionDisplayWord, CaptionUnit } from "@hypit/hypit/caption";
import type { NarrativeCaptionBinding } from "@hypit/hypit/narrative-caption";
import { sealText } from "@hypit/hypit/text";
@@ -25,6 +25,7 @@ function projectCaption(
bindingId = `${documentId}.binding`,
): CaptionViews {
const units: CaptionUnit[] = [];
+ const unitContexts: Array<{ readonly groupId: string; readonly role?: string }> = [];
const bindings: Array<{ unitId: string; sourceTokenIds: readonly string[] }> = [];
const words: CaptionDisplayWord[] = [];
for (const region of parsed.captionProjection.regions) {
@@ -65,10 +66,10 @@ function projectCaption(
if (sourceTokenIds.length === 0) throw new Error(`Caption Unit ${unitId} has no authored speech correspondence`);
units.push({
id: unitId,
- groupId: `${region.segmentId}:${turn.id}`,
- ...(turn.role === undefined ? {} : { role: turn.role }),
wordIds: unitWordIds,
});
+ unitContexts.push({ groupId: `${region.segmentId}:${turn.id}`,
+ ...(turn.role === undefined ? {} : { role: turn.role }) });
bindings.push({ unitId, sourceTokenIds });
sourceCursor = sourceEnd;
}
@@ -102,13 +103,27 @@ function projectCaption(
// Keep explicit breaks, but de-duplicate the boundary when `||` was placed
// immediately before `</segment>`.
const breaks = new Set(cueBreaks.map((item) => item.afterUnitId));
- for (let index = 0; index < units.length - 1; index += 1) {
- if (units[index]!.groupId === units[index + 1]!.groupId) continue;
- breaks.add(units[index]!.id);
+ const cues: CaptionCue[] = [];
+ let cueUnits: string[] = [];
+ let cueContext: typeof unitContexts[number] | undefined;
+ const flushCue = (): void => {
+ if (cueUnits.length === 0) return;
+ cues.push({ id: `${documentId}:cue:${cues.length + 1}`, unitIds: cueUnits,
+ ...(cueContext?.role === undefined ? {} : { role: cueContext.role }) });
+ cueUnits = [];
+ cueContext = undefined;
+ };
+ for (const [index, unit] of units.entries()) {
+ const context = unitContexts[index]!;
+ const previous = units[index - 1];
+ if (cueUnits.length > 0 && (cueContext?.groupId !== context.groupId
+ || (previous !== undefined && breaks.has(previous.id)))) flushCue();
+ cueContext ??= context;
+ cueUnits.push(unit.id);
}
+ flushCue();
return {
- document: { id: documentId, units, words,
- cueBreaks: units.filter((unit) => breaks.has(unit.id)).map((unit) => ({ afterUnitId: unit.id })) },
+ document: { id: documentId, units, words, cues },
binding: { id: bindingId, narrativeId, documentId, units: bindings },
};
}
diff --git a/packages/script/test/markup-integration.test.ts b/packages/script/test/markup-integration.test.ts
index d8dcae0d..acb3146f 100644
--- a/packages/script/test/markup-integration.test.ts
+++ b/packages/script/test/markup-integration.test.ts
@@ -239,6 +239,6 @@ test("a wordless Script publishes its Segment and an empty CaptionDocument", asy
assert.ok(result.records.some((record) => record.id === "story.segment.empty"));
assert.deepEqual(result.records.find((record) => record.id === "story.caption")?.value, {
kind: "inline",
- value: { id: "story.caption", units: [], words: [], cueBreaks: [] },
+ value: { id: "story.caption", units: [], words: [], cues: [] },
});
});
diff --git a/packages/script/test/narrative-views.test.ts b/packages/script/test/narrative-views.test.ts
index c14ea5f8..175e20af 100644
--- a/packages/script/test/narrative-views.test.ts
+++ b/packages/script/test/narrative-views.test.ts
@@ -88,7 +88,7 @@ test("Script exports Narrative and CaptionDocument as independent views from one
assert.match(document.words.map((word) => word.text).join(""), /声工坊/);
assert.ok(!document.words.some((word) => word.text.includes("indeed")));
assert.ok(narrative.tokens.some((token) => token.text === "indeed"));
- assert.equal(document.cueBreaks.length, 1);
+ assert.equal(document.cues.length, 2);
});
test("Narrative selections query authored speech while Caption preserves display correspondence", () => {
diff --git a/packages/script/test/script.test.ts b/packages/script/test/script.test.ts
index 7f44f136..5a7757c5 100644
--- a/packages/script/test/script.test.ts
+++ b/packages/script/test/script.test.ts
@@ -80,16 +80,41 @@ test("Script keeps speech, dialogue and CaptionDocument as separate projections"
assert.equal(document.units.length, 3);
assert.equal(document.units[1]!.wordIds.length, 1);
assert.equal(narrativeCaptionBinding(parsed, "story.caption", "story").units[1]!.sourceTokenIds.length, 4);
- assert.deepEqual(document.cueBreaks, []);
+ assert.equal(document.cues.length, 1);
+ assert.deepEqual(document.cues[0]!.unitIds, document.units.map((unit) => unit.id));
assert.equal((narrativeValue(parsed, "story") as { anchors: unknown[] }).anchors.length,
2 * parsed.tokens.length + 2 * parsed.segments.length);
});
-test("Cue breaks are authored between complete units", () => {
+test("Dual Text binds several display Words through one Unit to several Narrative Tokens", () => {
+ const parsed = parseScript("dual-n-m.svml", "<line><test1 test2 | test3 test4 test5></line>");
+ const document = captionDocument(parsed, "story.caption", "story");
+ const binding = narrativeCaptionBinding(parsed, document.id, "story");
+ assert.deepEqual(document.words.map((word) => word.text), ["test1", "test2"]);
+ assert.equal(document.units.length, 1);
+ assert.deepEqual(document.units[0]!.wordIds, document.words.map((word) => word.id));
+ assert.deepEqual(parsed.tokens.map((token) => token.text), ["test3", "test4", "test5"]);
+ assert.deepEqual(binding.units[0]!.sourceTokenIds, parsed.tokens.map((token) => token.id));
+ assert.deepEqual(document.cues[0]!.unitIds, [document.units[0]!.id]);
+});
+
+test("Role turns and Segments author Cue boundaries without leaking Narrative groups into Units", () => {
+ const parsed = parseScript("cue-structure.svml", "<one><A>first <B>second</one><two>third</two>");
+ const document = captionDocument(parsed, "story.caption", "story");
+ assert.deepEqual(document.cues.map((cue) => cue.role), ["A", "B", undefined]);
+ assert.equal(document.cues.length, 3);
+ assert.deepEqual(document.cues.flatMap((cue) => cue.unitIds), document.units.map((unit) => unit.id));
+ assert.equal(document.units.some((unit) => "groupId" in unit || "role" in unit), false);
+});
+
+test("Cues partition complete units at authored breaks", () => {
const parsed = parseScript("break.svml", "<line>one two || three four</line>");
const document = captionDocument(parsed, "story.caption", "story");
- assert.equal(document.cueBreaks.length, 1);
- assert.equal(document.cueBreaks[0]!.afterUnitId, document.units[1]!.id);
+ assert.equal(document.cues.length, 2);
+ assert.deepEqual(document.cues.map((cue) => cue.unitIds), [
+ document.units.slice(0, 2).map((unit) => unit.id),
+ document.units.slice(2).map((unit) => unit.id),
+ ]);
});
test("Caption punctuation is display-only and CJK uses lexical character units", () => {
@@ -123,7 +148,7 @@ test("Mixed-script brand names preserve following character units and authored C
assert.deepEqual(document.words.slice(0, 6).map((word) => word.text), [
"用", "Hypit", "生", "成", "视", "频,",
]);
- assert.equal(document.cueBreaks[0]?.afterUnitId, document.units[5]?.id);
+ assert.deepEqual(document.cues[0]?.unitIds, document.units.slice(0, 6).map((unit) => unit.id));
});
test("Script keeps ordinary compounds and formatted numbers lexical", () => {
@@ -143,7 +168,7 @@ test("Script keeps ordinary compounds and formatted numbers lexical", () => {
test("A single pipe is literal and a double pipe is an authored Cue Break", () => {
const parsed = parseScript("pipes.svml", "<line>one | two || three \\|\\| four</line>");
const document = captionDocument(parsed, "story.caption", "story");
- assert.equal(document.cueBreaks.length, 1);
+ assert.equal(document.cues.length, 2);
assert.equal(serializeCaption(parsed), "one | two three || four");
assert.deepEqual(document.words.map((word) => word.text), ["one |", "two", "three ||", "four"]);
});
diff --git a/test/narrative-caption.test.ts b/test/narrative-caption.test.ts
index 5d407e0a..b4786072 100644
--- a/test/narrative-caption.test.ts
+++ b/test/narrative-caption.test.ts
@@ -12,7 +12,7 @@ import { interpretWhisperXTranscript } from "@hypit/whisperx";
import { fixtureResource } from "./fixture-resource.js";
-test("Caption uses complete Narrative selections and authored cue breaks", () => {
+test("Caption uses complete Narrative selections and authored Cues", () => {
const parsed = parseScript("caption.svml", "<line>one @{focus} two three @{/focus} || four</line>");
const narrative = narrativeValue(parsed, "story") as unknown as Narrative;
const document = captionDocument(parsed, "story.caption", "story");
@@ -20,7 +20,8 @@ test("Caption uses complete Narrative selections and authored cue breaks", () =>
const selection = narrative.selections[0]!;
const subset = captionUnitsForNarrativeSelection(document, binding, narrative, selection);
assert.equal(subset.unitIds.length, 2);
- assert.equal(document.cueBreaks.length, 1);
+ assert.equal(document.cues.length, 2);
+ assert.deepEqual(document.cues.map((cue) => cue.unitIds.length), [3, 1]);
});
test("Caption selects complete Segments and the program through structural anchors", () => {
@@ -36,7 +37,7 @@ test("Caption selects complete Segments and the program through structural ancho
assert.deepEqual(captionUnitsForNarrativeSelection(document, binding, narrative, whole).unitIds,
document.units.map((unit) => unit.id));
assert.deepEqual(captionUnitsForNarrativeSelection(document, binding, narrative, opening).unitIds,
- document.units.filter((unit) => unit.groupId?.startsWith("intro:")).map((unit) => unit.id));
+ document.cues[0]!.unitIds);
});
for (const grouped of [false, true]) test(`Chinese Script projects Caption timing through explicit alignments and domain Windows (shared groups: ${grouped})`, () => {
@@ -110,7 +111,7 @@ for (const grouped of [false, true]) test(`Chinese Script projects Caption timin
assert.deepEqual(windowsFor("2026"), [[4000 + second[2]![0], 4000 + second[5]![1]]]);
assert.deepEqual(windowsFor("這"), [[4000 + second[7]![0], 4000 + second[7]![1]]]);
assert.deepEqual(windowsFor("個"), [[4000 + second[8]![0], 4000 + second[8]![1]]]);
- assert.equal(document.units[0]!.role, "HOST");
+ assert.equal(document.cues[0]!.role, "HOST");
});
test("Caption timing joins named units, preserves overlaps and never repairs projection evidence", () => {
@@ -124,7 +125,7 @@ test("Caption timing joins named units, preserves overlaps and never repairs pro
{ id: "word-a", unitId: "unit-a", text: "A", separatorBefore: "" as const, attributes: [] },
{ id: "word-b", unitId: "unit-b", text: "B", separatorBefore: " " as const, attributes: [] },
],
- cueBreaks: [],
+ cues: [{ id: "cue", unitIds: ["unit-a", "unit-b"] }],
};
const binding = {
id: "binding", narrativeId: "story", documentId: document.id,
@@ -163,7 +164,7 @@ test("Caption timing joins named units, preserves overlaps and never repairs pro
test("an empty CaptionDocument projects to one complete empty timing table", () => {
assert.deepEqual(projectNarrativeCaptionTiming(
- { id: "empty", units: [], words: [], cueBreaks: [] },
+ { id: "empty", units: [], words: [], cues: [] },
{ id: "binding", narrativeId: "story", documentId: "empty", units: [] },
{ id: "projection", narrativeId: "story", timelineId: "film", segments: [], tokens: [], boundaries: [] },
), { timelineId: "film", documentId: "empty", units: [] });