Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
feat: align script timing from one WhisperX pass
kashorin0cb2de9544e8
diff --git a/README.md b/README.mdindex 894a5d59..f09ab868 100644--- a/README.md+++ b/README.md@@ -5,7 +5,9 @@> `packages/text` and `packages/script` now implement the first official authoring vertical:> a minimal Text Frontend that learns raw/structured Surfaces from locked module manifests, plus> an ordinary Script Surface package using named Segments such as `<opening>...</opening>`.-> Third-party parser sandboxing, source imports and `.svs` are not implemented yet. The existing+> `packages/whisperx` adds a pure, tested one-pass WhisperX timing locator; the Host adapter that+> invokes WhisperX is intentionally still outside this slice. Third-party parser sandboxing,+> source imports and `.svs` are not implemented yet. The existing> root compiler and standard library remain the executable v1 research oracle during migration.Semantic Video Markup Language is a semantic source language for information-flowdiff --git a/packages/whisperx/README.md b/packages/whisperx/README.mdnew file mode 100644index 00000000..beb2ccfe--- /dev/null+++ b/packages/whisperx/README.md@@ -0,0 +1,16 @@+# `@svml/whisperx`++Deterministic timing locator for the result of one ordinary WhisperX run. The package does not+invoke Python, read audio, rerun forced alignment against Script, use an LLM, infer speakers or+retain multiple candidate paths. A Host adapter preserves the raw WhisperX artifact and supplies+the normalized word, character, score, VAD and Segment fields used here.++The locator aligns each known Script Segment independently. A bounded monotonic M:N dynamic+program accepts exact, split, merge, replacement, source-omission and evidence-insertion groups.+Within a selected group, timed WhisperX characters determine Script token boundaries; word times,+VAD bounds and deterministic zero-width estimates are fallbacks in that order. The result contains+one timing for every Script token and all `2M + 2N` semantic anchors.++Dual Text is not located a second time. `deriveCaptionAtomTiming()` composes the speech map with+the authored `CaptionAtom -> speech token range` projection. The complete display atom inherits+the speech envelope; this package never invents linear timings for words inside its display side.diff --git a/packages/whisperx/package.json b/packages/whisperx/package.jsonnew file mode 100644index 00000000..849dc588--- /dev/null+++ b/packages/whisperx/package.json@@ -0,0 +1,13 @@+{+ "name": "@svml/whisperx",+ "version": "0.0.0-dev",+ "private": true,+ "type": "module",+ "exports": {+ ".": "./src/index.ts"+ },+ "dependencies": {+ "@svml/core": "workspace:*",+ "@svml/script": "workspace:*"+ }+}diff --git a/packages/whisperx/src/align.ts b/packages/whisperx/src/align.tsnew file mode 100644index 00000000..006bd26d--- /dev/null+++ b/packages/whisperx/src/align.ts@@ -0,0 +1,238 @@+import type { ParsedToken } from "@svml/script";++import { editDistance, normalizeForAlignment } from "./normalize.js";+import type {+ AlignmentGroup,+ AlignmentRelation,+ WhisperXWordEvidence,+} from "./types.js";++const EPSILON = 1e-9;+const DEFAULT_MAX_GROUP = 4;++type Cell = {+ readonly cost: number;+ readonly exactTokens: number;+ readonly omissions: number;+ readonly insertions: number;+ readonly complexity: number;+ readonly previousSource: number;+ readonly previousEvidence: number;+ readonly group: AlignmentGroup;+};++function comparisonText(values: readonly { readonly text: string }[]): string {+ return values.map((value) => normalizeForAlignment(value.text)).join("");+}++function evidenceReliability(words: readonly WhisperXWordEvidence[]): number {+ const scores = words+ .map((word) => word.score)+ .filter((score): score is number => score !== undefined && Number.isFinite(score));+ if (!scores.length) return 0.8;+ const average = scores.reduce((sum, score) => sum + Math.max(0, Math.min(1, score)), 0) / scores.length;+ return 0.5 + 0.5 * average;+}++function exactWordLcs(+ source: readonly ParsedToken[],+ evidence: readonly WhisperXWordEvidence[],+): number {+ const rows = Array.from({ length: source.length + 1 }, () =>+ Array<number>(evidence.length + 1).fill(0));+ for (let left = source.length - 1; left >= 0; left -= 1) {+ for (let right = evidence.length - 1; right >= 0; right -= 1) {+ rows[left]![right] = source[left]!.normalized === normalizeForAlignment(evidence[right]!.text)+ ? rows[left + 1]![right + 1]! + 1+ : Math.max(rows[left + 1]![right]!, rows[left]![right + 1]!);+ }+ }+ return rows[0]![0]!;+}++function relation(+ sourceCount: number,+ evidenceCount: number,+ exactOneToOne: boolean,+): AlignmentRelation {+ if (sourceCount === 0) return "evidence-insertion";+ if (evidenceCount === 0) return "source-omission";+ if (sourceCount === 1 && evidenceCount === 1 && exactOneToOne) return "exact";+ if (sourceCount > 1 && evidenceCount === 1) return "merge";+ if (sourceCount === 1 && evidenceCount > 1) return "split";+ return "replacement";+}++function pairedCost(+ source: readonly ParsedToken[],+ evidence: readonly WhisperXWordEvidence[],+): number {+ const sourceText = comparisonText(source);+ const evidenceText = comparisonText(evidence);+ const width = Math.max([...sourceText].length, [...evidenceText].length, 1);+ const distance = editDistance([...sourceText], [...evidenceText]) / width;+ const grouping = 0.055 * Math.max(0, source.length + evidence.length - 2);+ const exactWords = exactWordLcs(source, evidence);+ const unmatchedWordBoundaries = source.length + evidence.length - 2 * exactWords;+ return distance * evidenceReliability(evidence) + grouping + 0.06 * unmatchedWordBoundaries;+}++function better(candidate: Cell, current: Cell | undefined): boolean {+ if (!current) return true;+ if (candidate.cost < current.cost - EPSILON) return true;+ if (candidate.cost > current.cost + EPSILON) return false;+ if (candidate.exactTokens !== current.exactTokens) return candidate.exactTokens > current.exactTokens;+ if (candidate.omissions !== current.omissions) return candidate.omissions < current.omissions;+ if (candidate.insertions !== current.insertions) return candidate.insertions < current.insertions;+ return candidate.complexity < current.complexity;+}++function extend(+ previous: Cell | undefined,+ group: AlignmentGroup,+ sourceCount: number,+ evidenceCount: number,+ exactTokens: number,+): Cell {+ return {+ cost: (previous?.cost ?? 0) + group.cost,+ exactTokens: (previous?.exactTokens ?? 0) + exactTokens,+ omissions: (previous?.omissions ?? 0) + (evidenceCount === 0 ? sourceCount : 0),+ insertions: (previous?.insertions ?? 0) + (sourceCount === 0 ? evidenceCount : 0),+ complexity: (previous?.complexity ?? 0) + Math.max(0, sourceCount + evidenceCount - 2),+ previousSource: 0,+ previousEvidence: 0,+ group,+ };+}++export function alignWordGroups(+ sourceSegmentId: string,+ source: readonly ParsedToken[],+ evidence: readonly WhisperXWordEvidence[],+ maxGroupSize = DEFAULT_MAX_GROUP,+): AlignmentGroup[] {+ const rows = Array.from({ length: source.length + 1 }, () =>+ Array<Cell | undefined>(evidence.length + 1).fill(undefined));+ rows[0]![0] = {+ cost: 0,+ exactTokens: 0,+ omissions: 0,+ insertions: 0,+ complexity: 0,+ previousSource: -1,+ previousEvidence: -1,+ group: {+ sourceSegmentId,+ sourceTokenIds: [],+ evidenceWordStart: 0,+ evidenceWordEndExclusive: 0,+ relation: "exact",+ cost: 0,+ },+ };++ const update = (+ sourceIndex: number,+ evidenceIndex: number,+ nextSource: number,+ nextEvidence: number,+ group: AlignmentGroup,+ exactTokens: number,+ ): void => {+ const previous = rows[sourceIndex]![evidenceIndex];+ if (!previous) return;+ const candidate = {+ ...extend(+ sourceIndex === 0 && evidenceIndex === 0 ? undefined : previous,+ group,+ nextSource - sourceIndex,+ nextEvidence - evidenceIndex,+ exactTokens,+ ),+ previousSource: sourceIndex,+ previousEvidence: evidenceIndex,+ };+ if (better(candidate, rows[nextSource]![nextEvidence])) {+ rows[nextSource]![nextEvidence] = candidate;+ }+ };++ for (let sourceIndex = 0; sourceIndex <= source.length; sourceIndex += 1) {+ for (let evidenceIndex = 0; evidenceIndex <= evidence.length; evidenceIndex += 1) {+ if (!rows[sourceIndex]![evidenceIndex]) continue;+ for (let sourceCount = 1; sourceCount <= maxGroupSize && sourceIndex + sourceCount <= source.length; sourceCount += 1) {+ for (let evidenceCount = 1; evidenceCount <= maxGroupSize && evidenceIndex + evidenceCount <= evidence.length; evidenceCount += 1) {+ const sourceGroup = source.slice(sourceIndex, sourceIndex + sourceCount);+ const evidenceGroup = evidence.slice(evidenceIndex, evidenceIndex + evidenceCount);+ const exactOneToOne = sourceCount === 1+ && evidenceCount === 1+ && sourceGroup[0]!.normalized === normalizeForAlignment(evidenceGroup[0]!.text);+ const cost = pairedCost(sourceGroup, evidenceGroup);+ update(+ sourceIndex,+ evidenceIndex,+ sourceIndex + sourceCount,+ evidenceIndex + evidenceCount,+ {+ sourceSegmentId,+ sourceTokenIds: sourceGroup.map((token) => token.id),+ evidenceWordStart: evidenceIndex,+ evidenceWordEndExclusive: evidenceIndex + evidenceCount,+ relation: relation(sourceCount, evidenceCount, exactOneToOne),+ cost,+ },+ exactOneToOne ? 1 : 0,+ );+ }+ }+ if (sourceIndex < source.length) {+ const token = source[sourceIndex]!;+ update(+ sourceIndex,+ evidenceIndex,+ sourceIndex + 1,+ evidenceIndex,+ {+ sourceSegmentId,+ sourceTokenIds: [token.id],+ evidenceWordStart: evidenceIndex,+ evidenceWordEndExclusive: evidenceIndex,+ relation: "source-omission",+ cost: 0.5,+ },+ 0,+ );+ }+ if (evidenceIndex < evidence.length) {+ update(+ sourceIndex,+ evidenceIndex,+ sourceIndex,+ evidenceIndex + 1,+ {+ sourceSegmentId,+ sourceTokenIds: [],+ evidenceWordStart: evidenceIndex,+ evidenceWordEndExclusive: evidenceIndex + 1,+ relation: "evidence-insertion",+ cost: 0.35,+ },+ 0,+ );+ }+ }+ }++ const groups: AlignmentGroup[] = [];+ let sourceIndex = source.length;+ let evidenceIndex = evidence.length;+ while (sourceIndex > 0 || evidenceIndex > 0) {+ const cell = rows[sourceIndex]![evidenceIndex];+ if (!cell) throw new Error("Word alignment failed to produce a complete path.");+ groups.push(cell.group);+ sourceIndex = cell.previousSource;+ evidenceIndex = cell.previousEvidence;+ }+ return groups.reverse();+}diff --git a/packages/whisperx/src/caption.ts b/packages/whisperx/src/caption.tsnew file mode 100644index 00000000..4b406a83--- /dev/null+++ b/packages/whisperx/src/caption.ts@@ -0,0 +1,54 @@+import type { ParsedNarrative } from "@svml/script";++import { WhisperXAlignmentError } from "./error.js";+import type { CompleteSpeechTimeMap, TimedCaptionAtom, TimingQuality } from "./types.js";++const QUALITY_RANK: Readonly<Record<TimingQuality, number>> = {+ measured: 0,+ derived: 1,+ estimated: 2,+};++function worse(left: TimingQuality, right: TimingQuality): TimingQuality {+ return QUALITY_RANK[left] >= QUALITY_RANK[right] ? left : right;+}++/**+ * Derives only explicit Dual Text atoms. Ordinary caption text reuses its+ * corresponding speech token timing directly. A Dual Text display side stays+ * atomic and inherits the complete speech-side envelope; it is never linearly+ * split into invented display-word times.+ */+export function deriveCaptionAtomTiming(+ narrative: ParsedNarrative,+ map: CompleteSpeechTimeMap,+): TimedCaptionAtom[] {+ if (map.semanticIndexDigest !== narrative.semanticIndex.digest) {+ throw new WhisperXAlignmentError(+ "WHISPERX_CAPTION_MAP",+ "Caption projection and speech map use different Semantic Indexes.",+ );+ }+ return narrative.captionAtoms.map((atom) => {+ const source = narrative.tokens.slice(atom.startToken, atom.endTokenExclusive);+ const timed = source.map((token) => map.tokens.find((value) => value.tokenId === token.id));+ if (!timed.length || timed.some((token) => token === undefined)) {+ throw new WhisperXAlignmentError(+ "WHISPERX_CAPTION_COVERAGE",+ `Caption atom ${atom.id} is not covered by the complete speech map.`,+ );+ }+ const first = timed[0]!;+ const last = timed.at(-1)!;+ return {+ id: atom.id,+ display: atom.display,+ segmentId: atom.segmentId,+ sourceTokenIds: source.map((token) => token.id),+ startSec: first.startSec,+ endSec: last.endSec,+ startQuality: worse(first.startQuality, "derived"),+ endQuality: worse(last.endQuality, "derived"),+ };+ });+}diff --git a/packages/whisperx/src/error.ts b/packages/whisperx/src/error.tsnew file mode 100644index 00000000..3d7079ee--- /dev/null+++ b/packages/whisperx/src/error.ts@@ -0,0 +1,9 @@+export class WhisperXAlignmentError extends Error {+ readonly code: string;++ constructor(code: string, message: string) {+ super(message);+ this.name = "WhisperXAlignmentError";+ this.code = code;+ }+}diff --git a/packages/whisperx/src/index.ts b/packages/whisperx/src/index.tsnew file mode 100644index 00000000..b6ad78d4--- /dev/null+++ b/packages/whisperx/src/index.ts@@ -0,0 +1,6 @@+export { alignWordGroups } from "./align.js";+export { deriveCaptionAtomTiming } from "./caption.js";+export { WhisperXAlignmentError } from "./error.js";+export { locateWhisperXTiming } from "./locate.js";+export { alignCharacters, alignmentCharacters, normalizeForAlignment } from "./normalize.js";+export type * from "./types.js";diff --git a/packages/whisperx/src/locate.ts b/packages/whisperx/src/locate.tsnew file mode 100644index 00000000..d4137905--- /dev/null+++ b/packages/whisperx/src/locate.ts@@ -0,0 +1,390 @@+import { digestOf } from "@svml/core";+import type { ParsedNarrative, ParsedToken } from "@svml/script";++import { alignWordGroups } from "./align.js";+import { WhisperXAlignmentError } from "./error.js";+import { alignCharacters, alignmentCharacters } from "./normalize.js";+import type {+ AlignmentGroup,+ CompleteSpeechTimeMap,+ SemanticTimePoint,+ TimedSpeechSegment,+ TimedSpeechToken,+ TimingQuality,+ WhisperXCharEvidence,+ WhisperXEvidence,+ WhisperXSegmentEvidence,+ WhisperXWordEvidence,+} from "./types.js";++const EPSILON = 1e-6;++type MutableTiming = {+ startSec: number;+ endSec: number;+ startQuality: TimingQuality;+ endQuality: TimingQuality;+};++type TimedEvidenceChar = {+ readonly value: string;+ readonly wordIndex: number;+ readonly startSec?: number;+ readonly endSec?: number;+};++function fail(code: string, message: string): never {+ throw new WhisperXAlignmentError(code, message);+}++function finite(value: number | undefined): value is number {+ return value !== undefined && Number.isFinite(value);+}++function validateWindow(start: number, end: number, limit: number, label: string): void {+ if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end < start || end > limit + EPSILON) {+ fail("WHISPERX_WINDOW", `${label} has invalid time window ${start}..${end}.`);+ }+}++function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidence): void {+ if (evidence.contract !== "svml.whisperx-evidence@0") {+ fail("WHISPERX_CONTRACT", `Unsupported WhisperX evidence contract ${evidence.contract}.`);+ }+ if (!Number.isFinite(evidence.durationSec) || evidence.durationSec <= 0) {+ fail("WHISPERX_DURATION", "WhisperX evidence duration must be positive and finite.");+ }+ const expected = new Set(narrative.segments.map((segment) => segment.id));+ const seen = new Set<string>();+ for (const segment of evidence.segments) {+ if (!expected.has(segment.sourceSegmentId)) {+ fail("WHISPERX_SEGMENT_UNKNOWN", `WhisperX evidence references unknown Segment ${segment.sourceSegmentId}.`);+ }+ if (seen.has(segment.sourceSegmentId)) {+ fail("WHISPERX_SEGMENT_DUPLICATE", `WhisperX evidence repeats Segment ${segment.sourceSegmentId}.`);+ }+ seen.add(segment.sourceSegmentId);+ validateWindow(segment.startSec, segment.endSec, evidence.durationSec, `Segment ${segment.sourceSegmentId}`);+ let previousEnd = segment.startSec;+ for (const [index, word] of segment.words.entries()) {+ if (typeof word.text !== "string") fail("WHISPERX_WORD_TEXT", `Word ${index + 1} has no text.`);+ if (finite(word.startSec) !== finite(word.endSec)) {+ fail("WHISPERX_WORD_PARTIAL_TIME", `Word ${index + 1} must provide both start and end or neither.`);+ }+ if (finite(word.startSec) && finite(word.endSec)) {+ validateWindow(word.startSec, word.endSec, evidence.durationSec, `Word ${index + 1}`);+ if (+ word.startSec < segment.startSec - EPSILON+ || word.endSec > segment.endSec + EPSILON+ || word.startSec < previousEnd - EPSILON+ ) {+ fail("WHISPERX_WORD_ORDER", `Word ${index + 1} is outside or overlaps its Segment window.`);+ }+ previousEnd = word.endSec;+ }+ if (word.score !== undefined && (!Number.isFinite(word.score) || word.score < 0 || word.score > 1)) {+ fail("WHISPERX_SCORE", `Word ${index + 1} score must be between zero and one.`);+ }+ }+ for (const [index, char] of segment.chars.entries()) {+ if (!Number.isInteger(char.wordIndex) || char.wordIndex < 0 || char.wordIndex >= segment.words.length) {+ fail("WHISPERX_CHAR_WORD", `Character ${index + 1} has an invalid wordIndex.`);+ }+ if (finite(char.startSec) !== finite(char.endSec)) {+ fail("WHISPERX_CHAR_PARTIAL_TIME", `Character ${index + 1} must provide both start and end or neither.`);+ }+ if (finite(char.startSec) && finite(char.endSec)) {+ validateWindow(char.startSec, char.endSec, evidence.durationSec, `Character ${index + 1}`);+ const word = segment.words[char.wordIndex]!;+ if (+ char.startSec < segment.startSec - EPSILON+ || char.endSec > segment.endSec + EPSILON+ || (finite(word.startSec) && char.startSec < word.startSec - EPSILON)+ || (finite(word.endSec) && char.endSec > word.endSec + EPSILON)+ ) {+ fail("WHISPERX_CHAR_WINDOW", `Character ${index + 1} falls outside its Word or Segment window.`);+ }+ }+ if (char.score !== undefined && (!Number.isFinite(char.score) || char.score < 0 || char.score > 1)) {+ fail("WHISPERX_SCORE", `Character ${index + 1} score must be between zero and one.`);+ }+ }+ let previousVadEnd = segment.startSec;+ for (const [index, span] of (segment.vadSpans ?? []).entries()) {+ validateWindow(span.startSec, span.endSec, evidence.durationSec, `VAD span ${index + 1}`);+ if (+ span.startSec < segment.startSec - EPSILON+ || span.endSec > segment.endSec + EPSILON+ || span.startSec < previousVadEnd - EPSILON+ ) {+ fail("WHISPERX_VAD_WINDOW", `VAD span ${index + 1} falls outside Segment ${segment.sourceSegmentId}.`);+ }+ previousVadEnd = span.endSec;+ }+ }+ for (const id of expected) {+ if (!seen.has(id)) fail("WHISPERX_SEGMENT_MISSING", `WhisperX evidence is missing Segment ${id}.`);+ }+ const segmentsById = new Map(evidence.segments.map((segment) => [segment.sourceSegmentId, segment]));+ let previousSegmentStart = -Infinity;+ let previousSegmentEnd = -Infinity;+ for (const sourceSegment of narrative.segments) {+ const segment = segmentsById.get(sourceSegment.id)!;+ if (+ segment.startSec < previousSegmentStart - EPSILON+ || segment.endSec < previousSegmentEnd - EPSILON+ ) {+ fail("WHISPERX_SEGMENT_ORDER", `Segment ${segment.sourceSegmentId} violates Script source order.`);+ }+ previousSegmentStart = segment.startSec;+ previousSegmentEnd = segment.endSec;+ }+}++function syntheticCharTimes(+ characters: readonly string[],+ wordIndex: number,+ word: WhisperXWordEvidence,+): TimedEvidenceChar[] {+ const hasWindow = finite(word.startSec) && finite(word.endSec);+ return characters.map((value, index) => ({+ value,+ wordIndex,+ ...(hasWindow+ ? {+ startSec: word.startSec! + (word.endSec! - word.startSec!) * index / characters.length,+ endSec: word.startSec! + (word.endSec! - word.startSec!) * (index + 1) / characters.length,+ }+ : {}),+ }));+}++function timedCharsForWord(+ word: WhisperXWordEvidence,+ wordIndex: number,+ rawChars: readonly WhisperXCharEvidence[],+): TimedEvidenceChar[] {+ const expected = alignmentCharacters(word.text);+ if (!expected.length) return [];+ const supplied = rawChars+ .filter((char) => char.wordIndex === wordIndex)+ .flatMap((char) => alignmentCharacters(char.char).map((value) => ({+ value,+ wordIndex,+ ...(finite(char.startSec) && finite(char.endSec)+ ? { startSec: char.startSec, endSec: char.endSec }+ : {}),+ })));+ if (!supplied.length) return syntheticCharTimes(expected, wordIndex, word);++ const fallback = syntheticCharTimes(expected, wordIndex, word);+ const pairs = alignCharacters(expected, supplied.map((char) => char.value));+ for (const pair of pairs) {+ const suppliedChar = supplied[pair.evidenceIndex]!;+ if (!finite(suppliedChar.startSec) || !finite(suppliedChar.endSec)) continue;+ fallback[pair.sourceIndex] = {+ value: expected[pair.sourceIndex]!,+ wordIndex,+ startSec: suppliedChar.startSec,+ endSec: suppliedChar.endSec,+ };+ }+ return fallback;+}++function timedChars(segment: WhisperXSegmentEvidence): TimedEvidenceChar[] {+ return segment.words.flatMap((word, index) => timedCharsForWord(word, index, segment.chars));+}++function scoredWords(segment: WhisperXSegmentEvidence): WhisperXWordEvidence[] {+ return segment.words.map((word, wordIndex) => {+ const scores = [+ ...(word.score === undefined ? [] : [word.score]),+ ...segment.chars+ .filter((char) => char.wordIndex === wordIndex && char.score !== undefined)+ .map((char) => char.score!),+ ];+ return scores.length+ ? { ...word, score: scores.reduce((sum, score) => sum + score, 0) / scores.length }+ : word;+ });+}++function groupWindow(+ group: AlignmentGroup,+ words: readonly WhisperXWordEvidence[],+ chars: readonly TimedEvidenceChar[],+): { readonly startSec: number; readonly endSec: number } | undefined {+ const wordRun = words.slice(group.evidenceWordStart, group.evidenceWordEndExclusive);+ const charRun = chars.filter((char) =>+ char.wordIndex >= group.evidenceWordStart && char.wordIndex < group.evidenceWordEndExclusive);+ const starts = [+ ...wordRun.map((word) => word.startSec),+ ...charRun.map((char) => char.startSec),+ ].filter(finite);+ const ends = [+ ...wordRun.map((word) => word.endSec),+ ...charRun.map((char) => char.endSec),+ ].filter(finite);+ if (!starts.length || !ends.length) return undefined;+ return { startSec: Math.min(...starts), endSec: Math.max(...ends) };+}++function locatePairedGroup(+ group: AlignmentGroup,+ source: readonly ParsedToken[],+ words: readonly WhisperXWordEvidence[],+ chars: readonly TimedEvidenceChar[],+ output: Array<MutableTiming | undefined>,+): void {+ if (!group.sourceTokenIds.length || group.evidenceWordStart === group.evidenceWordEndExclusive) return;+ const sourceIndexById = new Map(source.map((token, index) => [token.id, index]));+ const sourceRun = group.sourceTokenIds.map((id) => source[sourceIndexById.get(id)!]!);+ const sourceCharacters = sourceRun.flatMap((token, tokenOffset) =>+ [...token.normalized].map((value) => ({ value, tokenOffset })));+ const evidenceCharacters = chars.filter((char) =>+ char.wordIndex >= group.evidenceWordStart && char.wordIndex < group.evidenceWordEndExclusive);+ const pairs = alignCharacters(+ sourceCharacters.map((char) => char.value),+ evidenceCharacters.map((char) => char.value),+ );+ const exactOneToOne = group.relation === "exact";+ const window = groupWindow(group, words, chars);++ for (let tokenOffset = 0; tokenOffset < sourceRun.length; tokenOffset += 1) {+ const evidenceForToken = pairs+ .filter((pair) => sourceCharacters[pair.sourceIndex]!.tokenOffset === tokenOffset)+ .map((pair) => evidenceCharacters[pair.evidenceIndex]!)+ .filter((char) => finite(char.startSec) && finite(char.endSec));+ let startSec = evidenceForToken[0]?.startSec;+ let endSec = evidenceForToken.at(-1)?.endSec;+ if (sourceRun.length === 1 && window) {+ startSec = window.startSec;+ endSec = window.endSec;+ } else if (window) {+ if (tokenOffset === 0 && finite(startSec)) startSec = window.startSec;+ if (tokenOffset === sourceRun.length - 1 && finite(endSec)) endSec = window.endSec;+ }+ if (!finite(startSec) || !finite(endSec)) continue;+ const sourceIndex = sourceIndexById.get(sourceRun[tokenOffset]!.id)!;+ output[sourceIndex] = {+ startSec,+ endSec: Math.max(startSec, endSec),+ startQuality: exactOneToOne ? "measured" : "derived",+ endQuality: exactOneToOne ? "measured" : "derived",+ };+ }+}++function speechBounds(segment: WhisperXSegmentEvidence): { readonly start: number; readonly end: number } {+ const spans = segment.vadSpans ?? [];+ return spans.length+ ? { start: spans[0]!.startSec, end: spans.at(-1)!.endSec }+ : { start: segment.startSec, end: segment.endSec };+}++function fillEstimated(+ values: Array<MutableTiming | undefined>,+ startBound: number,+ endBound: number,+): MutableTiming[] {+ let cursor = 0;+ while (cursor < values.length) {+ if (values[cursor]) {+ cursor += 1;+ continue;+ }+ const runStart = cursor;+ while (cursor < values.length && !values[cursor]) cursor += 1;+ const runEnd = cursor;+ const left = runStart > 0 ? values[runStart - 1]!.endSec : startBound;+ const right = runEnd < values.length ? values[runEnd]!.startSec : endBound;+ const usableRight = Math.max(left, right);+ for (let index = runStart; index < runEnd; index += 1) {+ const point = left + (usableRight - left) * (index - runStart + 1) / (runEnd - runStart + 1);+ values[index] = {+ startSec: point,+ endSec: point,+ startQuality: "estimated",+ endQuality: "estimated",+ };+ }+ }+ return values as MutableTiming[];+}++function assertMonotonic(tokens: readonly TimedSpeechToken[], segmentId: string): void {+ let previousEnd = -Infinity;+ for (const token of tokens) {+ if (token.startSec < previousEnd - EPSILON || token.endSec < token.startSec - EPSILON) {+ fail("WHISPERX_MAP_ORDER", `Located token ${token.tokenId} violates time order in Segment ${segmentId}.`);+ }+ previousEnd = token.endSec;+ }+}++function mapDigest(map: Omit<CompleteSpeechTimeMap, "mapDigest">): string {+ return digestOf(map);+}++export function locateWhisperXTiming(+ narrative: ParsedNarrative,+ evidence: WhisperXEvidence,+): CompleteSpeechTimeMap {+ validateEvidence(narrative, evidence);+ const evidenceBySegment = new Map(evidence.segments.map((segment) => [segment.sourceSegmentId, segment]));+ const timedSegments: TimedSpeechSegment[] = [];+ const timedTokens: TimedSpeechToken[] = [];+ const groups: AlignmentGroup[] = [];++ for (const segment of narrative.segments) {+ const aligned = evidenceBySegment.get(segment.id)!;+ const source = narrative.tokens.slice(segment.tokenStart, segment.tokenEndExclusive);+ const segmentGroups = alignWordGroups(segment.id, source, scoredWords(aligned));+ const chars = timedChars(aligned);+ const located: Array<MutableTiming | undefined> = Array(source.length).fill(undefined);+ for (const group of segmentGroups) locatePairedGroup(group, source, aligned.words, chars, located);+ const bounds = speechBounds(aligned);+ const complete = fillEstimated(located, bounds.start, bounds.end);+ const segmentTokens = source.map((token, index): TimedSpeechToken => ({+ tokenId: token.id,+ segmentId: segment.id,+ ...complete[index]!,+ }));+ assertMonotonic(segmentTokens, segment.id);+ timedTokens.push(...segmentTokens);+ timedSegments.push({+ segmentId: segment.id,+ startSec: aligned.startSec,+ endSec: aligned.endSec,+ startQuality: "measured",+ endQuality: "measured",+ });+ groups.push(...segmentGroups);+ }++ const tokensById = new Map(timedTokens.map((token) => [token.tokenId, token]));+ const segmentsById = new Map(timedSegments.map((segment) => [segment.segmentId, segment]));+ const anchors: SemanticTimePoint[] = narrative.semanticIndex.anchors.map((anchor) => {+ if (anchor.kind === "segment-start" || anchor.kind === "segment-end") {+ const segment = segmentsById.get(anchor.segmentId)!;+ return anchor.kind === "segment-start"+ ? { identity: anchor.id, timeSec: segment.startSec, quality: segment.startQuality }+ : { identity: anchor.id, timeSec: segment.endSec, quality: segment.endQuality };+ }+ const token = tokensById.get(anchor.tokenId!)!;+ return anchor.kind === "token-start"+ ? { identity: anchor.id, timeSec: token.startSec, quality: token.startQuality }+ : { identity: anchor.id, timeSec: token.endSec, quality: token.endQuality };+ });+ const payload = {+ contract: "svml.whisperx-speech-time-map@0" as const,+ semanticIndexDigest: narrative.semanticIndex.digest,+ durationSec: evidence.durationSec,+ segments: timedSegments,+ tokens: timedTokens,+ anchors,+ groups,+ };+ return { ...payload, mapDigest: mapDigest(payload) };+}diff --git a/packages/whisperx/src/normalize.ts b/packages/whisperx/src/normalize.tsnew file mode 100644index 00000000..b352f191--- /dev/null+++ b/packages/whisperx/src/normalize.ts@@ -0,0 +1,79 @@+export function normalizeForAlignment(value: string): string {+ return value+ .normalize("NFKC")+ .toLocaleLowerCase("en")+ .replace(/[^\p{L}\p{M}\p{N}]+/gu, "");+}++export function alignmentCharacters(value: string): string[] {+ return [...normalizeForAlignment(value)];+}++export function editDistance(left: readonly string[], right: readonly string[]): number {+ let previous = Array.from({ length: right.length + 1 }, (_, index) => index);+ for (let leftIndex = 0; leftIndex < left.length; leftIndex += 1) {+ const current = [leftIndex + 1];+ for (let rightIndex = 0; rightIndex < right.length; rightIndex += 1) {+ current.push(Math.min(+ previous[rightIndex + 1]! + 1,+ current[rightIndex]! + 1,+ previous[rightIndex]! + (left[leftIndex] === right[rightIndex] ? 0 : 1),+ ));+ }+ previous = current;+ }+ return previous[right.length]!;+}++export type CharacterPair = {+ readonly sourceIndex: number;+ readonly evidenceIndex: number;+ readonly exact: boolean;+};++/** A deterministic Levenshtein backtrace. Diagonal steps win exact ties. */+export function alignCharacters(+ source: readonly string[],+ evidence: readonly string[],+): CharacterPair[] {+ const rows = Array.from({ length: source.length + 1 }, () =>+ Array<number>(evidence.length + 1).fill(0));+ for (let index = 0; index <= source.length; index += 1) rows[index]![0] = index;+ for (let index = 0; index <= evidence.length; index += 1) rows[0]![index] = index;+ for (let left = 1; left <= source.length; left += 1) {+ for (let right = 1; right <= evidence.length; right += 1) {+ rows[left]![right] = Math.min(+ rows[left - 1]![right]! + 1,+ rows[left]![right - 1]! + 1,+ rows[left - 1]![right - 1]! + (source[left - 1] === evidence[right - 1] ? 0 : 1),+ );+ }+ }++ const pairs: CharacterPair[] = [];+ let left = source.length;+ let right = evidence.length;+ while (left > 0 || right > 0) {+ if (+ left > 0+ && right > 0+ && rows[left]![right]+ === rows[left - 1]![right - 1]! + (source[left - 1] === evidence[right - 1] ? 0 : 1)+ ) {+ pairs.push({+ sourceIndex: left - 1,+ evidenceIndex: right - 1,+ exact: source[left - 1] === evidence[right - 1],+ });+ left -= 1;+ right -= 1;+ continue;+ }+ if (left > 0 && rows[left]![right] === rows[left - 1]![right]! + 1) {+ left -= 1;+ continue;+ }+ right -= 1;+ }+ return pairs.reverse();+}diff --git a/packages/whisperx/src/types.ts b/packages/whisperx/src/types.tsnew file mode 100644index 00000000..11b42328--- /dev/null+++ b/packages/whisperx/src/types.ts@@ -0,0 +1,103 @@+export type TimingQuality = "measured" | "derived" | "estimated";++export type WhisperXWordEvidence = {+ readonly text: string;+ readonly startSec?: number;+ readonly endSec?: number;+ readonly score?: number;+};++export type WhisperXCharEvidence = {+ readonly char: string;+ readonly wordIndex: number;+ readonly startSec?: number;+ readonly endSec?: number;+ readonly score?: number;+};++export type WhisperXVadSpan = {+ readonly startSec: number;+ readonly endSec: number;+};++/**+ * Normalized evidence from one ordinary WhisperX transcription + alignment run.+ * A host adapter keeps the raw provider result as an artifact and lowers the+ * fields used by this deterministic locator into this value.+ */+export type WhisperXSegmentEvidence = {+ readonly sourceSegmentId: string;+ readonly startSec: number;+ readonly endSec: number;+ readonly words: readonly WhisperXWordEvidence[];+ readonly chars: readonly WhisperXCharEvidence[];+ readonly vadSpans?: readonly WhisperXVadSpan[];+};++export type WhisperXEvidence = {+ readonly contract: "svml.whisperx-evidence@0";+ readonly durationSec: number;+ readonly segments: readonly WhisperXSegmentEvidence[];+};++export type AlignmentRelation =+ | "exact"+ | "split"+ | "merge"+ | "replacement"+ | "source-omission"+ | "evidence-insertion";++export type AlignmentGroup = {+ readonly sourceSegmentId: string;+ readonly sourceTokenIds: readonly string[];+ readonly evidenceWordStart: number;+ readonly evidenceWordEndExclusive: number;+ readonly relation: AlignmentRelation;+ readonly cost: number;+};++export type TimedSpeechToken = {+ readonly tokenId: string;+ readonly segmentId: string;+ readonly startSec: number;+ readonly endSec: number;+ readonly startQuality: TimingQuality;+ readonly endQuality: TimingQuality;+};++export type TimedSpeechSegment = {+ readonly segmentId: string;+ readonly startSec: number;+ readonly endSec: number;+ readonly startQuality: TimingQuality;+ readonly endQuality: TimingQuality;+};++export type SemanticTimePoint = {+ readonly identity: string;+ readonly timeSec: number;+ readonly quality: TimingQuality;+};++export type CompleteSpeechTimeMap = {+ readonly contract: "svml.whisperx-speech-time-map@0";+ readonly semanticIndexDigest: string;+ readonly durationSec: number;+ readonly segments: readonly TimedSpeechSegment[];+ readonly tokens: readonly TimedSpeechToken[];+ readonly anchors: readonly SemanticTimePoint[];+ readonly groups: readonly AlignmentGroup[];+ readonly mapDigest: string;+};++export type TimedCaptionAtom = {+ readonly id: string;+ readonly display: string;+ readonly segmentId: string;+ readonly sourceTokenIds: readonly string[];+ readonly startSec: number;+ readonly endSec: number;+ readonly startQuality: TimingQuality;+ readonly endQuality: TimingQuality;+};diff --git a/packages/whisperx/test/whisperx.test.ts b/packages/whisperx/test/whisperx.test.tsnew file mode 100644index 00000000..70022338--- /dev/null+++ b/packages/whisperx/test/whisperx.test.ts@@ -0,0 +1,260 @@+import assert from "node:assert/strict";+import test from "node:test";++import { parseScript } from "@svml/script";+import {+ WhisperXAlignmentError,+ deriveCaptionAtomTiming,+ locateWhisperXTiming,+} from "@svml/whisperx";+import type {+ WhisperXCharEvidence,+ WhisperXEvidence,+ WhisperXWordEvidence,+} from "@svml/whisperx";++function evidence(args: {+ readonly segmentId?: string;+ readonly durationSec?: number;+ readonly startSec?: number;+ readonly endSec?: number;+ readonly words: readonly WhisperXWordEvidence[];+ readonly chars?: readonly WhisperXCharEvidence[];+ readonly vad?: readonly { readonly startSec: number; readonly endSec: number }[];+}): WhisperXEvidence {+ return {+ contract: "svml.whisperx-evidence@0",+ durationSec: args.durationSec ?? 2,+ segments: [+ {+ sourceSegmentId: args.segmentId ?? "line",+ startSec: args.startSec ?? 0,+ endSec: args.endSec ?? 2,+ words: args.words,+ chars: args.chars ?? [],+ ...(args.vad === undefined ? {} : { vadSpans: args.vad }),+ },+ ],+ };+}++function characters(+ text: string,+ starts: readonly number[],+ ends: readonly number[],+ wordIndex = 0,+): WhisperXCharEvidence[] {+ return [...text].map((char, index) => ({+ char,+ wordIndex,+ startSec: starts[index]!,+ endSec: ends[index]!,+ score: 0.95,+ }));+}++test("exact WhisperX words cover every Script and Segment anchor", () => {+ const narrative = parseScript("exact.svml", "<line>Hello world.</line>");+ const map = locateWhisperXTiming(narrative, evidence({+ words: [+ { text: "Hello", startSec: 0.1, endSec: 0.4, score: 0.97 },+ { text: "world", startSec: 0.5, endSec: 0.9, score: 0.96 },+ ],+ }));++ assert.equal(map.tokens.length, 2);+ assert.equal(map.anchors.length, 2 * narrative.tokens.length + 2 * narrative.segments.length);+ assert.deepEqual(+ map.tokens.map((token) => [token.startSec, token.endSec, token.startQuality, token.endQuality]),+ [+ [0.1, 0.4, "measured", "measured"],+ [0.5, 0.9, "measured", "measured"],+ ],+ );+ assert.equal(new Set(map.anchors.map((anchor) => anchor.identity)).size, map.anchors.length);+ assert.match(map.mapDigest, /^sha256:[a-f0-9]{64}$/u);+});++test("M:1 uses WhisperX character times instead of dividing a merged word by length", () => {+ const narrative = parseScript("merge.svml", "<line>can not</line>");+ const map = locateWhisperXTiming(narrative, evidence({+ endSec: 1,+ durationSec: 1,+ words: [{ text: "cannot", startSec: 0.1, endSec: 0.78, score: 0.93 }],+ chars: characters(+ "cannot",+ [0.1, 0.18, 0.27, 0.42, 0.51, 0.63],+ [0.17, 0.26, 0.36, 0.5, 0.62, 0.78],+ ),+ }));++ assert.equal(map.groups[0]?.relation, "merge");+ assert.deepEqual(+ map.tokens.map((token) => [token.startSec, token.endSec, token.startQuality]),+ [+ [0.1, 0.36, "derived"],+ [0.42, 0.78, "derived"],+ ],+ );+});++test("1:N wraps all WhisperX words in one Script token", () => {+ const narrative = parseScript("split.svml", "<line>website</line>");+ const map = locateWhisperXTiming(narrative, evidence({+ words: [+ { text: "web", startSec: 0.2, endSec: 0.45, score: 0.9 },+ { text: "site", startSec: 0.5, endSec: 0.82, score: 0.91 },+ ],+ }));++ assert.equal(map.groups[0]?.relation, "split");+ assert.deepEqual(+ [map.tokens[0]?.startSec, map.tokens[0]?.endSec, map.tokens[0]?.startQuality],+ [0.2, 0.82, "derived"],+ );+});++test("a recognized filler stays an insertion and does not absorb neighboring Script words", () => {+ const narrative = parseScript("insertion.svml", "<line>I really like it.</line>");+ const map = locateWhisperXTiming(narrative, evidence({+ words: [+ { text: "I", startSec: 0.1, endSec: 0.2, score: 0.98 },+ { text: "uh", startSec: 0.24, endSec: 0.34, score: 0.88 },+ { text: "really", startSec: 0.4, endSec: 0.62, score: 0.95 },+ { text: "like", startSec: 0.67, endSec: 0.82, score: 0.96 },+ { text: "it", startSec: 0.86, endSec: 0.96, score: 0.96 },+ ],+ }));++ assert.deepEqual(map.groups.map((group) => group.relation), [+ "exact",+ "evidence-insertion",+ "exact",+ "exact",+ "exact",+ ]);+ assert.deepEqual(+ map.tokens.map((token) => [token.startSec, token.endSec, token.startQuality]),+ [+ [0.1, 0.2, "measured"],+ [0.4, 0.62, "measured"],+ [0.67, 0.82, "measured"],+ [0.86, 0.96, "measured"],+ ],+ );+});++test("an omitted Script word gets a zero-width estimated point between measured neighbors", () => {+ const narrative = parseScript("omission.svml", "<line>This is very good.</line>");+ const map = locateWhisperXTiming(narrative, evidence({+ words: [+ { text: "This", startSec: 0.1, endSec: 0.25, score: 0.98 },+ { text: "is", startSec: 0.3, endSec: 0.4, score: 0.97 },+ { text: "good", startSec: 0.6, endSec: 0.82, score: 0.98 },+ ],+ }));++ assert.deepEqual(map.groups.map((group) => group.relation), [+ "exact",+ "exact",+ "source-omission",+ "exact",+ ]);+ assert.deepEqual(+ [map.tokens[2]?.startSec, map.tokens[2]?.endSec, map.tokens[2]?.startQuality],+ [0.5, 0.5, "estimated"],+ );+});++test("VAD bounds contain estimates when an entire Script Segment has no recognized words", () => {+ const narrative = parseScript("vad.svml", "<line>One two.</line>");+ const map = locateWhisperXTiming(narrative, evidence({+ words: [],+ vad: [{ startSec: 0.4, endSec: 1.2 }],+ }));++ assert.deepEqual(+ map.tokens.map((token) => [token.startSec, token.endSec, token.startQuality]),+ [+ [0.4 + 0.8 / 3, 0.4 + 0.8 / 3, "estimated"],+ [0.4 + 1.6 / 3, 0.4 + 1.6 / 3, "estimated"],+ ],+ );+});++test("Dual Text display inherits one speech envelope without linear display-word timing", () => {+ const narrative = parseScript(+ "dual.svml",+ "<line>That was <really funny | absolutely hilarious>.</line>",+ );+ const map = locateWhisperXTiming(narrative, evidence({+ words: [+ { text: "That", startSec: 0.1, endSec: 0.22 },+ { text: "was", startSec: 0.25, endSec: 0.34 },+ { text: "absolutely", startSec: 0.4, endSec: 0.72 },+ { text: "hilarious", startSec: 0.75, endSec: 1.1 },+ ],+ }));+ const captions = deriveCaptionAtomTiming(narrative, map);++ assert.equal(captions.length, 1);+ assert.deepEqual(+ {+ display: captions[0]?.display,+ sourceTokenIds: captions[0]?.sourceTokenIds,+ startSec: captions[0]?.startSec,+ endSec: captions[0]?.endSec,+ startQuality: captions[0]?.startQuality,+ },+ {+ display: "really funny",+ sourceTokenIds: narrative.tokens.slice(2, 4).map((token) => token.id),+ startSec: 0.4,+ endSec: 1.1,+ startQuality: "derived",+ },+ );+});++test("multiple Script Segments stay independent even when evidence records arrive out of order", () => {+ const narrative = parseScript("segments.svml", "<one>Hello.</one><two>Goodbye.</two>");+ const map = locateWhisperXTiming(narrative, {+ contract: "svml.whisperx-evidence@0",+ durationSec: 2,+ segments: [+ {+ sourceSegmentId: "two",+ startSec: 1,+ endSec: 2,+ words: [{ text: "Goodbye", startSec: 1.2, endSec: 1.6 }],+ chars: [],+ },+ {+ sourceSegmentId: "one",+ startSec: 0,+ endSec: 1,+ words: [{ text: "Hello", startSec: 0.2, endSec: 0.55 }],+ chars: [],+ },+ ],+ });++ assert.deepEqual(map.tokens.map((token) => [token.segmentId, token.startSec, token.endSec]), [+ ["one", 0.2, 0.55],+ ["two", 1.2, 1.6],+ ]);+ assert.deepEqual(map.groups.map((group) => group.sourceSegmentId), ["one", "two"]);+});++test("invalid overlapping WhisperX word windows fail instead of producing a reversed map", () => {+ const narrative = parseScript("bad.svml", "<line>one two</line>");+ assert.throws(+ () => locateWhisperXTiming(narrative, evidence({+ words: [+ { text: "one", startSec: 0.1, endSec: 0.5 },+ { text: "two", startSec: 0.4, endSec: 0.8 },+ ],+ })),+ (error: unknown) => error instanceof WhisperXAlignmentError && error.code === "WHISPERX_WORD_ORDER",+ );+});diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 8cd1e969..d0f61f46 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -61,6 +61,15 @@ importers:specifier: workspace:*version: link:../script+ packages/whisperx:+ dependencies:+ '@svml/core':+ specifier: workspace:*+ version: link:../core+ '@svml/script':+ specifier: workspace:*+ version: link:../script+packages:'@emnapi/[email protected]':diff --git a/tsconfig.v2.json b/tsconfig.v2.jsonindex 3368a676..a5348d84 100644--- a/tsconfig.v2.json+++ b/tsconfig.v2.json@@ -9,7 +9,8 @@"@svml/core": ["packages/core/src/index.ts"],"@svml/driver-node": ["packages/driver-node/src/index.ts"],"@svml/script": ["packages/script/src/index.ts"],- "@svml/text": ["packages/text/src/index.ts"]+ "@svml/text": ["packages/text/src/index.ts"],+ "@svml/whisperx": ["packages/whisperx/src/index.ts"]}},"include": ["packages/**/*.ts"],