Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat: align script timing from one WhisperX pass

kashorin0cb2de9544e8
diff --git a/README.md b/README.md
index 894a5d59..f09ab868 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,9 @@
> `packages/text` and `packages/script` now implement the first official authoring vertical:
> a minimal Text Frontend that learns raw/structured Surfaces from locked module manifests, plus
> an ordinary Script Surface package using named Segments such as `<opening>...</opening>`.
-> Third-party parser sandboxing, source imports and `.svs` are not implemented yet. The existing
+> `packages/whisperx` adds a pure, tested one-pass WhisperX timing locator; the Host adapter that
+> invokes WhisperX is intentionally still outside this slice. Third-party parser sandboxing,
+> source imports and `.svs` are not implemented yet. The existing
> root compiler and standard library remain the executable v1 research oracle during migration.
Semantic Video Markup Language is a semantic source language for information-flow
diff --git a/packages/whisperx/README.md b/packages/whisperx/README.md
new file mode 100644
index 00000000..beb2ccfe
--- /dev/null
+++ b/packages/whisperx/README.md
@@ -0,0 +1,16 @@
+# `@svml/whisperx`
+
+Deterministic timing locator for the result of one ordinary WhisperX run. The package does not
+invoke Python, read audio, rerun forced alignment against Script, use an LLM, infer speakers or
+retain multiple candidate paths. A Host adapter preserves the raw WhisperX artifact and supplies
+the normalized word, character, score, VAD and Segment fields used here.
+
+The locator aligns each known Script Segment independently. A bounded monotonic M:N dynamic
+program accepts exact, split, merge, replacement, source-omission and evidence-insertion groups.
+Within a selected group, timed WhisperX characters determine Script token boundaries; word times,
+VAD bounds and deterministic zero-width estimates are fallbacks in that order. The result contains
+one timing for every Script token and all `2M + 2N` semantic anchors.
+
+Dual Text is not located a second time. `deriveCaptionAtomTiming()` composes the speech map with
+the authored `CaptionAtom -> speech token range` projection. The complete display atom inherits
+the speech envelope; this package never invents linear timings for words inside its display side.
diff --git a/packages/whisperx/package.json b/packages/whisperx/package.json
new file mode 100644
index 00000000..849dc588
--- /dev/null
+++ b/packages/whisperx/package.json
@@ -0,0 +1,13 @@
+{
+ "name": "@svml/whisperx",
+ "version": "0.0.0-dev",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "dependencies": {
+ "@svml/core": "workspace:*",
+ "@svml/script": "workspace:*"
+ }
+}
diff --git a/packages/whisperx/src/align.ts b/packages/whisperx/src/align.ts
new file mode 100644
index 00000000..006bd26d
--- /dev/null
+++ b/packages/whisperx/src/align.ts
@@ -0,0 +1,238 @@
+import type { ParsedToken } from "@svml/script";
+
+import { editDistance, normalizeForAlignment } from "./normalize.js";
+import type {
+ AlignmentGroup,
+ AlignmentRelation,
+ WhisperXWordEvidence,
+} from "./types.js";
+
+const EPSILON = 1e-9;
+const DEFAULT_MAX_GROUP = 4;
+
+type Cell = {
+ readonly cost: number;
+ readonly exactTokens: number;
+ readonly omissions: number;
+ readonly insertions: number;
+ readonly complexity: number;
+ readonly previousSource: number;
+ readonly previousEvidence: number;
+ readonly group: AlignmentGroup;
+};
+
+function comparisonText(values: readonly { readonly text: string }[]): string {
+ return values.map((value) => normalizeForAlignment(value.text)).join("");
+}
+
+function evidenceReliability(words: readonly WhisperXWordEvidence[]): number {
+ const scores = words
+ .map((word) => word.score)
+ .filter((score): score is number => score !== undefined && Number.isFinite(score));
+ if (!scores.length) return 0.8;
+ const average = scores.reduce((sum, score) => sum + Math.max(0, Math.min(1, score)), 0) / scores.length;
+ return 0.5 + 0.5 * average;
+}
+
+function exactWordLcs(
+ source: readonly ParsedToken[],
+ evidence: readonly WhisperXWordEvidence[],
+): number {
+ const rows = Array.from({ length: source.length + 1 }, () =>
+ Array<number>(evidence.length + 1).fill(0));
+ for (let left = source.length - 1; left >= 0; left -= 1) {
+ for (let right = evidence.length - 1; right >= 0; right -= 1) {
+ rows[left]![right] = source[left]!.normalized === normalizeForAlignment(evidence[right]!.text)
+ ? rows[left + 1]![right + 1]! + 1
+ : Math.max(rows[left + 1]![right]!, rows[left]![right + 1]!);
+ }
+ }
+ return rows[0]![0]!;
+}
+
+function relation(
+ sourceCount: number,
+ evidenceCount: number,
+ exactOneToOne: boolean,
+): AlignmentRelation {
+ if (sourceCount === 0) return "evidence-insertion";
+ if (evidenceCount === 0) return "source-omission";
+ if (sourceCount === 1 && evidenceCount === 1 && exactOneToOne) return "exact";
+ if (sourceCount > 1 && evidenceCount === 1) return "merge";
+ if (sourceCount === 1 && evidenceCount > 1) return "split";
+ return "replacement";
+}
+
+function pairedCost(
+ source: readonly ParsedToken[],
+ evidence: readonly WhisperXWordEvidence[],
+): number {
+ const sourceText = comparisonText(source);
+ const evidenceText = comparisonText(evidence);
+ const width = Math.max([...sourceText].length, [...evidenceText].length, 1);
+ const distance = editDistance([...sourceText], [...evidenceText]) / width;
+ const grouping = 0.055 * Math.max(0, source.length + evidence.length - 2);
+ const exactWords = exactWordLcs(source, evidence);
+ const unmatchedWordBoundaries = source.length + evidence.length - 2 * exactWords;
+ return distance * evidenceReliability(evidence) + grouping + 0.06 * unmatchedWordBoundaries;
+}
+
+function better(candidate: Cell, current: Cell | undefined): boolean {
+ if (!current) return true;
+ if (candidate.cost < current.cost - EPSILON) return true;
+ if (candidate.cost > current.cost + EPSILON) return false;
+ if (candidate.exactTokens !== current.exactTokens) return candidate.exactTokens > current.exactTokens;
+ if (candidate.omissions !== current.omissions) return candidate.omissions < current.omissions;
+ if (candidate.insertions !== current.insertions) return candidate.insertions < current.insertions;
+ return candidate.complexity < current.complexity;
+}
+
+function extend(
+ previous: Cell | undefined,
+ group: AlignmentGroup,
+ sourceCount: number,
+ evidenceCount: number,
+ exactTokens: number,
+): Cell {
+ return {
+ cost: (previous?.cost ?? 0) + group.cost,
+ exactTokens: (previous?.exactTokens ?? 0) + exactTokens,
+ omissions: (previous?.omissions ?? 0) + (evidenceCount === 0 ? sourceCount : 0),
+ insertions: (previous?.insertions ?? 0) + (sourceCount === 0 ? evidenceCount : 0),
+ complexity: (previous?.complexity ?? 0) + Math.max(0, sourceCount + evidenceCount - 2),
+ previousSource: 0,
+ previousEvidence: 0,
+ group,
+ };
+}
+
+export function alignWordGroups(
+ sourceSegmentId: string,
+ source: readonly ParsedToken[],
+ evidence: readonly WhisperXWordEvidence[],
+ maxGroupSize = DEFAULT_MAX_GROUP,
+): AlignmentGroup[] {
+ const rows = Array.from({ length: source.length + 1 }, () =>
+ Array<Cell | undefined>(evidence.length + 1).fill(undefined));
+ rows[0]![0] = {
+ cost: 0,
+ exactTokens: 0,
+ omissions: 0,
+ insertions: 0,
+ complexity: 0,
+ previousSource: -1,
+ previousEvidence: -1,
+ group: {
+ sourceSegmentId,
+ sourceTokenIds: [],
+ evidenceWordStart: 0,
+ evidenceWordEndExclusive: 0,
+ relation: "exact",
+ cost: 0,
+ },
+ };
+
+ const update = (
+ sourceIndex: number,
+ evidenceIndex: number,
+ nextSource: number,
+ nextEvidence: number,
+ group: AlignmentGroup,
+ exactTokens: number,
+ ): void => {
+ const previous = rows[sourceIndex]![evidenceIndex];
+ if (!previous) return;
+ const candidate = {
+ ...extend(
+ sourceIndex === 0 && evidenceIndex === 0 ? undefined : previous,
+ group,
+ nextSource - sourceIndex,
+ nextEvidence - evidenceIndex,
+ exactTokens,
+ ),
+ previousSource: sourceIndex,
+ previousEvidence: evidenceIndex,
+ };
+ if (better(candidate, rows[nextSource]![nextEvidence])) {
+ rows[nextSource]![nextEvidence] = candidate;
+ }
+ };
+
+ for (let sourceIndex = 0; sourceIndex <= source.length; sourceIndex += 1) {
+ for (let evidenceIndex = 0; evidenceIndex <= evidence.length; evidenceIndex += 1) {
+ if (!rows[sourceIndex]![evidenceIndex]) continue;
+ for (let sourceCount = 1; sourceCount <= maxGroupSize && sourceIndex + sourceCount <= source.length; sourceCount += 1) {
+ for (let evidenceCount = 1; evidenceCount <= maxGroupSize && evidenceIndex + evidenceCount <= evidence.length; evidenceCount += 1) {
+ const sourceGroup = source.slice(sourceIndex, sourceIndex + sourceCount);
+ const evidenceGroup = evidence.slice(evidenceIndex, evidenceIndex + evidenceCount);
+ const exactOneToOne = sourceCount === 1
+ && evidenceCount === 1
+ && sourceGroup[0]!.normalized === normalizeForAlignment(evidenceGroup[0]!.text);
+ const cost = pairedCost(sourceGroup, evidenceGroup);
+ update(
+ sourceIndex,
+ evidenceIndex,
+ sourceIndex + sourceCount,
+ evidenceIndex + evidenceCount,
+ {
+ sourceSegmentId,
+ sourceTokenIds: sourceGroup.map((token) => token.id),
+ evidenceWordStart: evidenceIndex,
+ evidenceWordEndExclusive: evidenceIndex + evidenceCount,
+ relation: relation(sourceCount, evidenceCount, exactOneToOne),
+ cost,
+ },
+ exactOneToOne ? 1 : 0,
+ );
+ }
+ }
+ if (sourceIndex < source.length) {
+ const token = source[sourceIndex]!;
+ update(
+ sourceIndex,
+ evidenceIndex,
+ sourceIndex + 1,
+ evidenceIndex,
+ {
+ sourceSegmentId,
+ sourceTokenIds: [token.id],
+ evidenceWordStart: evidenceIndex,
+ evidenceWordEndExclusive: evidenceIndex,
+ relation: "source-omission",
+ cost: 0.5,
+ },
+ 0,
+ );
+ }
+ if (evidenceIndex < evidence.length) {
+ update(
+ sourceIndex,
+ evidenceIndex,
+ sourceIndex,
+ evidenceIndex + 1,
+ {
+ sourceSegmentId,
+ sourceTokenIds: [],
+ evidenceWordStart: evidenceIndex,
+ evidenceWordEndExclusive: evidenceIndex + 1,
+ relation: "evidence-insertion",
+ cost: 0.35,
+ },
+ 0,
+ );
+ }
+ }
+ }
+
+ const groups: AlignmentGroup[] = [];
+ let sourceIndex = source.length;
+ let evidenceIndex = evidence.length;
+ while (sourceIndex > 0 || evidenceIndex > 0) {
+ const cell = rows[sourceIndex]![evidenceIndex];
+ if (!cell) throw new Error("Word alignment failed to produce a complete path.");
+ groups.push(cell.group);
+ sourceIndex = cell.previousSource;
+ evidenceIndex = cell.previousEvidence;
+ }
+ return groups.reverse();
+}
diff --git a/packages/whisperx/src/caption.ts b/packages/whisperx/src/caption.ts
new file mode 100644
index 00000000..4b406a83
--- /dev/null
+++ b/packages/whisperx/src/caption.ts
@@ -0,0 +1,54 @@
+import type { ParsedNarrative } from "@svml/script";
+
+import { WhisperXAlignmentError } from "./error.js";
+import type { CompleteSpeechTimeMap, TimedCaptionAtom, TimingQuality } from "./types.js";
+
+const QUALITY_RANK: Readonly<Record<TimingQuality, number>> = {
+ measured: 0,
+ derived: 1,
+ estimated: 2,
+};
+
+function worse(left: TimingQuality, right: TimingQuality): TimingQuality {
+ return QUALITY_RANK[left] >= QUALITY_RANK[right] ? left : right;
+}
+
+/**
+ * Derives only explicit Dual Text atoms. Ordinary caption text reuses its
+ * corresponding speech token timing directly. A Dual Text display side stays
+ * atomic and inherits the complete speech-side envelope; it is never linearly
+ * split into invented display-word times.
+ */
+export function deriveCaptionAtomTiming(
+ narrative: ParsedNarrative,
+ map: CompleteSpeechTimeMap,
+): TimedCaptionAtom[] {
+ if (map.semanticIndexDigest !== narrative.semanticIndex.digest) {
+ throw new WhisperXAlignmentError(
+ "WHISPERX_CAPTION_MAP",
+ "Caption projection and speech map use different Semantic Indexes.",
+ );
+ }
+ return narrative.captionAtoms.map((atom) => {
+ const source = narrative.tokens.slice(atom.startToken, atom.endTokenExclusive);
+ const timed = source.map((token) => map.tokens.find((value) => value.tokenId === token.id));
+ if (!timed.length || timed.some((token) => token === undefined)) {
+ throw new WhisperXAlignmentError(
+ "WHISPERX_CAPTION_COVERAGE",
+ `Caption atom ${atom.id} is not covered by the complete speech map.`,
+ );
+ }
+ const first = timed[0]!;
+ const last = timed.at(-1)!;
+ return {
+ id: atom.id,
+ display: atom.display,
+ segmentId: atom.segmentId,
+ sourceTokenIds: source.map((token) => token.id),
+ startSec: first.startSec,
+ endSec: last.endSec,
+ startQuality: worse(first.startQuality, "derived"),
+ endQuality: worse(last.endQuality, "derived"),
+ };
+ });
+}
diff --git a/packages/whisperx/src/error.ts b/packages/whisperx/src/error.ts
new file mode 100644
index 00000000..3d7079ee
--- /dev/null
+++ b/packages/whisperx/src/error.ts
@@ -0,0 +1,9 @@
+export class WhisperXAlignmentError extends Error {
+ readonly code: string;
+
+ constructor(code: string, message: string) {
+ super(message);
+ this.name = "WhisperXAlignmentError";
+ this.code = code;
+ }
+}
diff --git a/packages/whisperx/src/index.ts b/packages/whisperx/src/index.ts
new file mode 100644
index 00000000..b6ad78d4
--- /dev/null
+++ b/packages/whisperx/src/index.ts
@@ -0,0 +1,6 @@
+export { alignWordGroups } from "./align.js";
+export { deriveCaptionAtomTiming } from "./caption.js";
+export { WhisperXAlignmentError } from "./error.js";
+export { locateWhisperXTiming } from "./locate.js";
+export { alignCharacters, alignmentCharacters, normalizeForAlignment } from "./normalize.js";
+export type * from "./types.js";
diff --git a/packages/whisperx/src/locate.ts b/packages/whisperx/src/locate.ts
new file mode 100644
index 00000000..d4137905
--- /dev/null
+++ b/packages/whisperx/src/locate.ts
@@ -0,0 +1,390 @@
+import { digestOf } from "@svml/core";
+import type { ParsedNarrative, ParsedToken } from "@svml/script";
+
+import { alignWordGroups } from "./align.js";
+import { WhisperXAlignmentError } from "./error.js";
+import { alignCharacters, alignmentCharacters } from "./normalize.js";
+import type {
+ AlignmentGroup,
+ CompleteSpeechTimeMap,
+ SemanticTimePoint,
+ TimedSpeechSegment,
+ TimedSpeechToken,
+ TimingQuality,
+ WhisperXCharEvidence,
+ WhisperXEvidence,
+ WhisperXSegmentEvidence,
+ WhisperXWordEvidence,
+} from "./types.js";
+
+const EPSILON = 1e-6;
+
+type MutableTiming = {
+ startSec: number;
+ endSec: number;
+ startQuality: TimingQuality;
+ endQuality: TimingQuality;
+};
+
+type TimedEvidenceChar = {
+ readonly value: string;
+ readonly wordIndex: number;
+ readonly startSec?: number;
+ readonly endSec?: number;
+};
+
+function fail(code: string, message: string): never {
+ throw new WhisperXAlignmentError(code, message);
+}
+
+function finite(value: number | undefined): value is number {
+ return value !== undefined && Number.isFinite(value);
+}
+
+function validateWindow(start: number, end: number, limit: number, label: string): void {
+ if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end < start || end > limit + EPSILON) {
+ fail("WHISPERX_WINDOW", `${label} has invalid time window ${start}..${end}.`);
+ }
+}
+
+function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidence): void {
+ if (evidence.contract !== "svml.whisperx-evidence@0") {
+ fail("WHISPERX_CONTRACT", `Unsupported WhisperX evidence contract ${evidence.contract}.`);
+ }
+ if (!Number.isFinite(evidence.durationSec) || evidence.durationSec <= 0) {
+ fail("WHISPERX_DURATION", "WhisperX evidence duration must be positive and finite.");
+ }
+ const expected = new Set(narrative.segments.map((segment) => segment.id));
+ const seen = new Set<string>();
+ for (const segment of evidence.segments) {
+ if (!expected.has(segment.sourceSegmentId)) {
+ fail("WHISPERX_SEGMENT_UNKNOWN", `WhisperX evidence references unknown Segment ${segment.sourceSegmentId}.`);
+ }
+ if (seen.has(segment.sourceSegmentId)) {
+ fail("WHISPERX_SEGMENT_DUPLICATE", `WhisperX evidence repeats Segment ${segment.sourceSegmentId}.`);
+ }
+ seen.add(segment.sourceSegmentId);
+ validateWindow(segment.startSec, segment.endSec, evidence.durationSec, `Segment ${segment.sourceSegmentId}`);
+ let previousEnd = segment.startSec;
+ for (const [index, word] of segment.words.entries()) {
+ if (typeof word.text !== "string") fail("WHISPERX_WORD_TEXT", `Word ${index + 1} has no text.`);
+ if (finite(word.startSec) !== finite(word.endSec)) {
+ fail("WHISPERX_WORD_PARTIAL_TIME", `Word ${index + 1} must provide both start and end or neither.`);
+ }
+ if (finite(word.startSec) && finite(word.endSec)) {
+ validateWindow(word.startSec, word.endSec, evidence.durationSec, `Word ${index + 1}`);
+ if (
+ word.startSec < segment.startSec - EPSILON
+ || word.endSec > segment.endSec + EPSILON
+ || word.startSec < previousEnd - EPSILON
+ ) {
+ fail("WHISPERX_WORD_ORDER", `Word ${index + 1} is outside or overlaps its Segment window.`);
+ }
+ previousEnd = word.endSec;
+ }
+ if (word.score !== undefined && (!Number.isFinite(word.score) || word.score < 0 || word.score > 1)) {
+ fail("WHISPERX_SCORE", `Word ${index + 1} score must be between zero and one.`);
+ }
+ }
+ for (const [index, char] of segment.chars.entries()) {
+ if (!Number.isInteger(char.wordIndex) || char.wordIndex < 0 || char.wordIndex >= segment.words.length) {
+ fail("WHISPERX_CHAR_WORD", `Character ${index + 1} has an invalid wordIndex.`);
+ }
+ if (finite(char.startSec) !== finite(char.endSec)) {
+ fail("WHISPERX_CHAR_PARTIAL_TIME", `Character ${index + 1} must provide both start and end or neither.`);
+ }
+ if (finite(char.startSec) && finite(char.endSec)) {
+ validateWindow(char.startSec, char.endSec, evidence.durationSec, `Character ${index + 1}`);
+ const word = segment.words[char.wordIndex]!;
+ if (
+ char.startSec < segment.startSec - EPSILON
+ || char.endSec > segment.endSec + EPSILON
+ || (finite(word.startSec) && char.startSec < word.startSec - EPSILON)
+ || (finite(word.endSec) && char.endSec > word.endSec + EPSILON)
+ ) {
+ fail("WHISPERX_CHAR_WINDOW", `Character ${index + 1} falls outside its Word or Segment window.`);
+ }
+ }
+ if (char.score !== undefined && (!Number.isFinite(char.score) || char.score < 0 || char.score > 1)) {
+ fail("WHISPERX_SCORE", `Character ${index + 1} score must be between zero and one.`);
+ }
+ }
+ let previousVadEnd = segment.startSec;
+ for (const [index, span] of (segment.vadSpans ?? []).entries()) {
+ validateWindow(span.startSec, span.endSec, evidence.durationSec, `VAD span ${index + 1}`);
+ if (
+ span.startSec < segment.startSec - EPSILON
+ || span.endSec > segment.endSec + EPSILON
+ || span.startSec < previousVadEnd - EPSILON
+ ) {
+ fail("WHISPERX_VAD_WINDOW", `VAD span ${index + 1} falls outside Segment ${segment.sourceSegmentId}.`);
+ }
+ previousVadEnd = span.endSec;
+ }
+ }
+ for (const id of expected) {
+ if (!seen.has(id)) fail("WHISPERX_SEGMENT_MISSING", `WhisperX evidence is missing Segment ${id}.`);
+ }
+ const segmentsById = new Map(evidence.segments.map((segment) => [segment.sourceSegmentId, segment]));
+ let previousSegmentStart = -Infinity;
+ let previousSegmentEnd = -Infinity;
+ for (const sourceSegment of narrative.segments) {
+ const segment = segmentsById.get(sourceSegment.id)!;
+ if (
+ segment.startSec < previousSegmentStart - EPSILON
+ || segment.endSec < previousSegmentEnd - EPSILON
+ ) {
+ fail("WHISPERX_SEGMENT_ORDER", `Segment ${segment.sourceSegmentId} violates Script source order.`);
+ }
+ previousSegmentStart = segment.startSec;
+ previousSegmentEnd = segment.endSec;
+ }
+}
+
+function syntheticCharTimes(
+ characters: readonly string[],
+ wordIndex: number,
+ word: WhisperXWordEvidence,
+): TimedEvidenceChar[] {
+ const hasWindow = finite(word.startSec) && finite(word.endSec);
+ return characters.map((value, index) => ({
+ value,
+ wordIndex,
+ ...(hasWindow
+ ? {
+ startSec: word.startSec! + (word.endSec! - word.startSec!) * index / characters.length,
+ endSec: word.startSec! + (word.endSec! - word.startSec!) * (index + 1) / characters.length,
+ }
+ : {}),
+ }));
+}
+
+function timedCharsForWord(
+ word: WhisperXWordEvidence,
+ wordIndex: number,
+ rawChars: readonly WhisperXCharEvidence[],
+): TimedEvidenceChar[] {
+ const expected = alignmentCharacters(word.text);
+ if (!expected.length) return [];
+ const supplied = rawChars
+ .filter((char) => char.wordIndex === wordIndex)
+ .flatMap((char) => alignmentCharacters(char.char).map((value) => ({
+ value,
+ wordIndex,
+ ...(finite(char.startSec) && finite(char.endSec)
+ ? { startSec: char.startSec, endSec: char.endSec }
+ : {}),
+ })));
+ if (!supplied.length) return syntheticCharTimes(expected, wordIndex, word);
+
+ const fallback = syntheticCharTimes(expected, wordIndex, word);
+ const pairs = alignCharacters(expected, supplied.map((char) => char.value));
+ for (const pair of pairs) {
+ const suppliedChar = supplied[pair.evidenceIndex]!;
+ if (!finite(suppliedChar.startSec) || !finite(suppliedChar.endSec)) continue;
+ fallback[pair.sourceIndex] = {
+ value: expected[pair.sourceIndex]!,
+ wordIndex,
+ startSec: suppliedChar.startSec,
+ endSec: suppliedChar.endSec,
+ };
+ }
+ return fallback;
+}
+
+function timedChars(segment: WhisperXSegmentEvidence): TimedEvidenceChar[] {
+ return segment.words.flatMap((word, index) => timedCharsForWord(word, index, segment.chars));
+}
+
+function scoredWords(segment: WhisperXSegmentEvidence): WhisperXWordEvidence[] {
+ return segment.words.map((word, wordIndex) => {
+ const scores = [
+ ...(word.score === undefined ? [] : [word.score]),
+ ...segment.chars
+ .filter((char) => char.wordIndex === wordIndex && char.score !== undefined)
+ .map((char) => char.score!),
+ ];
+ return scores.length
+ ? { ...word, score: scores.reduce((sum, score) => sum + score, 0) / scores.length }
+ : word;
+ });
+}
+
+function groupWindow(
+ group: AlignmentGroup,
+ words: readonly WhisperXWordEvidence[],
+ chars: readonly TimedEvidenceChar[],
+): { readonly startSec: number; readonly endSec: number } | undefined {
+ const wordRun = words.slice(group.evidenceWordStart, group.evidenceWordEndExclusive);
+ const charRun = chars.filter((char) =>
+ char.wordIndex >= group.evidenceWordStart && char.wordIndex < group.evidenceWordEndExclusive);
+ const starts = [
+ ...wordRun.map((word) => word.startSec),
+ ...charRun.map((char) => char.startSec),
+ ].filter(finite);
+ const ends = [
+ ...wordRun.map((word) => word.endSec),
+ ...charRun.map((char) => char.endSec),
+ ].filter(finite);
+ if (!starts.length || !ends.length) return undefined;
+ return { startSec: Math.min(...starts), endSec: Math.max(...ends) };
+}
+
+function locatePairedGroup(
+ group: AlignmentGroup,
+ source: readonly ParsedToken[],
+ words: readonly WhisperXWordEvidence[],
+ chars: readonly TimedEvidenceChar[],
+ output: Array<MutableTiming | undefined>,
+): void {
+ if (!group.sourceTokenIds.length || group.evidenceWordStart === group.evidenceWordEndExclusive) return;
+ const sourceIndexById = new Map(source.map((token, index) => [token.id, index]));
+ const sourceRun = group.sourceTokenIds.map((id) => source[sourceIndexById.get(id)!]!);
+ const sourceCharacters = sourceRun.flatMap((token, tokenOffset) =>
+ [...token.normalized].map((value) => ({ value, tokenOffset })));
+ const evidenceCharacters = chars.filter((char) =>
+ char.wordIndex >= group.evidenceWordStart && char.wordIndex < group.evidenceWordEndExclusive);
+ const pairs = alignCharacters(
+ sourceCharacters.map((char) => char.value),
+ evidenceCharacters.map((char) => char.value),
+ );
+ const exactOneToOne = group.relation === "exact";
+ const window = groupWindow(group, words, chars);
+
+ for (let tokenOffset = 0; tokenOffset < sourceRun.length; tokenOffset += 1) {
+ const evidenceForToken = pairs
+ .filter((pair) => sourceCharacters[pair.sourceIndex]!.tokenOffset === tokenOffset)
+ .map((pair) => evidenceCharacters[pair.evidenceIndex]!)
+ .filter((char) => finite(char.startSec) && finite(char.endSec));
+ let startSec = evidenceForToken[0]?.startSec;
+ let endSec = evidenceForToken.at(-1)?.endSec;
+ if (sourceRun.length === 1 && window) {
+ startSec = window.startSec;
+ endSec = window.endSec;
+ } else if (window) {
+ if (tokenOffset === 0 && finite(startSec)) startSec = window.startSec;
+ if (tokenOffset === sourceRun.length - 1 && finite(endSec)) endSec = window.endSec;
+ }
+ if (!finite(startSec) || !finite(endSec)) continue;
+ const sourceIndex = sourceIndexById.get(sourceRun[tokenOffset]!.id)!;
+ output[sourceIndex] = {
+ startSec,
+ endSec: Math.max(startSec, endSec),
+ startQuality: exactOneToOne ? "measured" : "derived",
+ endQuality: exactOneToOne ? "measured" : "derived",
+ };
+ }
+}
+
+function speechBounds(segment: WhisperXSegmentEvidence): { readonly start: number; readonly end: number } {
+ const spans = segment.vadSpans ?? [];
+ return spans.length
+ ? { start: spans[0]!.startSec, end: spans.at(-1)!.endSec }
+ : { start: segment.startSec, end: segment.endSec };
+}
+
+function fillEstimated(
+ values: Array<MutableTiming | undefined>,
+ startBound: number,
+ endBound: number,
+): MutableTiming[] {
+ let cursor = 0;
+ while (cursor < values.length) {
+ if (values[cursor]) {
+ cursor += 1;
+ continue;
+ }
+ const runStart = cursor;
+ while (cursor < values.length && !values[cursor]) cursor += 1;
+ const runEnd = cursor;
+ const left = runStart > 0 ? values[runStart - 1]!.endSec : startBound;
+ const right = runEnd < values.length ? values[runEnd]!.startSec : endBound;
+ const usableRight = Math.max(left, right);
+ for (let index = runStart; index < runEnd; index += 1) {
+ const point = left + (usableRight - left) * (index - runStart + 1) / (runEnd - runStart + 1);
+ values[index] = {
+ startSec: point,
+ endSec: point,
+ startQuality: "estimated",
+ endQuality: "estimated",
+ };
+ }
+ }
+ return values as MutableTiming[];
+}
+
+function assertMonotonic(tokens: readonly TimedSpeechToken[], segmentId: string): void {
+ let previousEnd = -Infinity;
+ for (const token of tokens) {
+ if (token.startSec < previousEnd - EPSILON || token.endSec < token.startSec - EPSILON) {
+ fail("WHISPERX_MAP_ORDER", `Located token ${token.tokenId} violates time order in Segment ${segmentId}.`);
+ }
+ previousEnd = token.endSec;
+ }
+}
+
+function mapDigest(map: Omit<CompleteSpeechTimeMap, "mapDigest">): string {
+ return digestOf(map);
+}
+
+export function locateWhisperXTiming(
+ narrative: ParsedNarrative,
+ evidence: WhisperXEvidence,
+): CompleteSpeechTimeMap {
+ validateEvidence(narrative, evidence);
+ const evidenceBySegment = new Map(evidence.segments.map((segment) => [segment.sourceSegmentId, segment]));
+ const timedSegments: TimedSpeechSegment[] = [];
+ const timedTokens: TimedSpeechToken[] = [];
+ const groups: AlignmentGroup[] = [];
+
+ for (const segment of narrative.segments) {
+ const aligned = evidenceBySegment.get(segment.id)!;
+ const source = narrative.tokens.slice(segment.tokenStart, segment.tokenEndExclusive);
+ const segmentGroups = alignWordGroups(segment.id, source, scoredWords(aligned));
+ const chars = timedChars(aligned);
+ const located: Array<MutableTiming | undefined> = Array(source.length).fill(undefined);
+ for (const group of segmentGroups) locatePairedGroup(group, source, aligned.words, chars, located);
+ const bounds = speechBounds(aligned);
+ const complete = fillEstimated(located, bounds.start, bounds.end);
+ const segmentTokens = source.map((token, index): TimedSpeechToken => ({
+ tokenId: token.id,
+ segmentId: segment.id,
+ ...complete[index]!,
+ }));
+ assertMonotonic(segmentTokens, segment.id);
+ timedTokens.push(...segmentTokens);
+ timedSegments.push({
+ segmentId: segment.id,
+ startSec: aligned.startSec,
+ endSec: aligned.endSec,
+ startQuality: "measured",
+ endQuality: "measured",
+ });
+ groups.push(...segmentGroups);
+ }
+
+ const tokensById = new Map(timedTokens.map((token) => [token.tokenId, token]));
+ const segmentsById = new Map(timedSegments.map((segment) => [segment.segmentId, segment]));
+ const anchors: SemanticTimePoint[] = narrative.semanticIndex.anchors.map((anchor) => {
+ if (anchor.kind === "segment-start" || anchor.kind === "segment-end") {
+ const segment = segmentsById.get(anchor.segmentId)!;
+ return anchor.kind === "segment-start"
+ ? { identity: anchor.id, timeSec: segment.startSec, quality: segment.startQuality }
+ : { identity: anchor.id, timeSec: segment.endSec, quality: segment.endQuality };
+ }
+ const token = tokensById.get(anchor.tokenId!)!;
+ return anchor.kind === "token-start"
+ ? { identity: anchor.id, timeSec: token.startSec, quality: token.startQuality }
+ : { identity: anchor.id, timeSec: token.endSec, quality: token.endQuality };
+ });
+ const payload = {
+ contract: "svml.whisperx-speech-time-map@0" as const,
+ semanticIndexDigest: narrative.semanticIndex.digest,
+ durationSec: evidence.durationSec,
+ segments: timedSegments,
+ tokens: timedTokens,
+ anchors,
+ groups,
+ };
+ return { ...payload, mapDigest: mapDigest(payload) };
+}
diff --git a/packages/whisperx/src/normalize.ts b/packages/whisperx/src/normalize.ts
new file mode 100644
index 00000000..b352f191
--- /dev/null
+++ b/packages/whisperx/src/normalize.ts
@@ -0,0 +1,79 @@
+export function normalizeForAlignment(value: string): string {
+ return value
+ .normalize("NFKC")
+ .toLocaleLowerCase("en")
+ .replace(/[^\p{L}\p{M}\p{N}]+/gu, "");
+}
+
+export function alignmentCharacters(value: string): string[] {
+ return [...normalizeForAlignment(value)];
+}
+
+export function editDistance(left: readonly string[], right: readonly string[]): number {
+ let previous = Array.from({ length: right.length + 1 }, (_, index) => index);
+ for (let leftIndex = 0; leftIndex < left.length; leftIndex += 1) {
+ const current = [leftIndex + 1];
+ for (let rightIndex = 0; rightIndex < right.length; rightIndex += 1) {
+ current.push(Math.min(
+ previous[rightIndex + 1]! + 1,
+ current[rightIndex]! + 1,
+ previous[rightIndex]! + (left[leftIndex] === right[rightIndex] ? 0 : 1),
+ ));
+ }
+ previous = current;
+ }
+ return previous[right.length]!;
+}
+
+export type CharacterPair = {
+ readonly sourceIndex: number;
+ readonly evidenceIndex: number;
+ readonly exact: boolean;
+};
+
+/** A deterministic Levenshtein backtrace. Diagonal steps win exact ties. */
+export function alignCharacters(
+ source: readonly string[],
+ evidence: readonly string[],
+): CharacterPair[] {
+ const rows = Array.from({ length: source.length + 1 }, () =>
+ Array<number>(evidence.length + 1).fill(0));
+ for (let index = 0; index <= source.length; index += 1) rows[index]![0] = index;
+ for (let index = 0; index <= evidence.length; index += 1) rows[0]![index] = index;
+ for (let left = 1; left <= source.length; left += 1) {
+ for (let right = 1; right <= evidence.length; right += 1) {
+ rows[left]![right] = Math.min(
+ rows[left - 1]![right]! + 1,
+ rows[left]![right - 1]! + 1,
+ rows[left - 1]![right - 1]! + (source[left - 1] === evidence[right - 1] ? 0 : 1),
+ );
+ }
+ }
+
+ const pairs: CharacterPair[] = [];
+ let left = source.length;
+ let right = evidence.length;
+ while (left > 0 || right > 0) {
+ if (
+ left > 0
+ && right > 0
+ && rows[left]![right]
+ === rows[left - 1]![right - 1]! + (source[left - 1] === evidence[right - 1] ? 0 : 1)
+ ) {
+ pairs.push({
+ sourceIndex: left - 1,
+ evidenceIndex: right - 1,
+ exact: source[left - 1] === evidence[right - 1],
+ });
+ left -= 1;
+ right -= 1;
+ continue;
+ }
+ if (left > 0 && rows[left]![right] === rows[left - 1]![right]! + 1) {
+ left -= 1;
+ continue;
+ }
+ right -= 1;
+ }
+ return pairs.reverse();
+}
diff --git a/packages/whisperx/src/types.ts b/packages/whisperx/src/types.ts
new file mode 100644
index 00000000..11b42328
--- /dev/null
+++ b/packages/whisperx/src/types.ts
@@ -0,0 +1,103 @@
+export type TimingQuality = "measured" | "derived" | "estimated";
+
+export type WhisperXWordEvidence = {
+ readonly text: string;
+ readonly startSec?: number;
+ readonly endSec?: number;
+ readonly score?: number;
+};
+
+export type WhisperXCharEvidence = {
+ readonly char: string;
+ readonly wordIndex: number;
+ readonly startSec?: number;
+ readonly endSec?: number;
+ readonly score?: number;
+};
+
+export type WhisperXVadSpan = {
+ readonly startSec: number;
+ readonly endSec: number;
+};
+
+/**
+ * Normalized evidence from one ordinary WhisperX transcription + alignment run.
+ * A host adapter keeps the raw provider result as an artifact and lowers the
+ * fields used by this deterministic locator into this value.
+ */
+export type WhisperXSegmentEvidence = {
+ readonly sourceSegmentId: string;
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly words: readonly WhisperXWordEvidence[];
+ readonly chars: readonly WhisperXCharEvidence[];
+ readonly vadSpans?: readonly WhisperXVadSpan[];
+};
+
+export type WhisperXEvidence = {
+ readonly contract: "svml.whisperx-evidence@0";
+ readonly durationSec: number;
+ readonly segments: readonly WhisperXSegmentEvidence[];
+};
+
+export type AlignmentRelation =
+ | "exact"
+ | "split"
+ | "merge"
+ | "replacement"
+ | "source-omission"
+ | "evidence-insertion";
+
+export type AlignmentGroup = {
+ readonly sourceSegmentId: string;
+ readonly sourceTokenIds: readonly string[];
+ readonly evidenceWordStart: number;
+ readonly evidenceWordEndExclusive: number;
+ readonly relation: AlignmentRelation;
+ readonly cost: number;
+};
+
+export type TimedSpeechToken = {
+ readonly tokenId: string;
+ readonly segmentId: string;
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly startQuality: TimingQuality;
+ readonly endQuality: TimingQuality;
+};
+
+export type TimedSpeechSegment = {
+ readonly segmentId: string;
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly startQuality: TimingQuality;
+ readonly endQuality: TimingQuality;
+};
+
+export type SemanticTimePoint = {
+ readonly identity: string;
+ readonly timeSec: number;
+ readonly quality: TimingQuality;
+};
+
+export type CompleteSpeechTimeMap = {
+ readonly contract: "svml.whisperx-speech-time-map@0";
+ readonly semanticIndexDigest: string;
+ readonly durationSec: number;
+ readonly segments: readonly TimedSpeechSegment[];
+ readonly tokens: readonly TimedSpeechToken[];
+ readonly anchors: readonly SemanticTimePoint[];
+ readonly groups: readonly AlignmentGroup[];
+ readonly mapDigest: string;
+};
+
+export type TimedCaptionAtom = {
+ readonly id: string;
+ readonly display: string;
+ readonly segmentId: string;
+ readonly sourceTokenIds: readonly string[];
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly startQuality: TimingQuality;
+ readonly endQuality: TimingQuality;
+};
diff --git a/packages/whisperx/test/whisperx.test.ts b/packages/whisperx/test/whisperx.test.ts
new file mode 100644
index 00000000..70022338
--- /dev/null
+++ b/packages/whisperx/test/whisperx.test.ts
@@ -0,0 +1,260 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { parseScript } from "@svml/script";
+import {
+ WhisperXAlignmentError,
+ deriveCaptionAtomTiming,
+ locateWhisperXTiming,
+} from "@svml/whisperx";
+import type {
+ WhisperXCharEvidence,
+ WhisperXEvidence,
+ WhisperXWordEvidence,
+} from "@svml/whisperx";
+
+function evidence(args: {
+ readonly segmentId?: string;
+ readonly durationSec?: number;
+ readonly startSec?: number;
+ readonly endSec?: number;
+ readonly words: readonly WhisperXWordEvidence[];
+ readonly chars?: readonly WhisperXCharEvidence[];
+ readonly vad?: readonly { readonly startSec: number; readonly endSec: number }[];
+}): WhisperXEvidence {
+ return {
+ contract: "svml.whisperx-evidence@0",
+ durationSec: args.durationSec ?? 2,
+ segments: [
+ {
+ sourceSegmentId: args.segmentId ?? "line",
+ startSec: args.startSec ?? 0,
+ endSec: args.endSec ?? 2,
+ words: args.words,
+ chars: args.chars ?? [],
+ ...(args.vad === undefined ? {} : { vadSpans: args.vad }),
+ },
+ ],
+ };
+}
+
+function characters(
+ text: string,
+ starts: readonly number[],
+ ends: readonly number[],
+ wordIndex = 0,
+): WhisperXCharEvidence[] {
+ return [...text].map((char, index) => ({
+ char,
+ wordIndex,
+ startSec: starts[index]!,
+ endSec: ends[index]!,
+ score: 0.95,
+ }));
+}
+
+test("exact WhisperX words cover every Script and Segment anchor", () => {
+ const narrative = parseScript("exact.svml", "<line>Hello world.</line>");
+ const map = locateWhisperXTiming(narrative, evidence({
+ words: [
+ { text: "Hello", startSec: 0.1, endSec: 0.4, score: 0.97 },
+ { text: "world", startSec: 0.5, endSec: 0.9, score: 0.96 },
+ ],
+ }));
+
+ assert.equal(map.tokens.length, 2);
+ assert.equal(map.anchors.length, 2 * narrative.tokens.length + 2 * narrative.segments.length);
+ assert.deepEqual(
+ map.tokens.map((token) => [token.startSec, token.endSec, token.startQuality, token.endQuality]),
+ [
+ [0.1, 0.4, "measured", "measured"],
+ [0.5, 0.9, "measured", "measured"],
+ ],
+ );
+ assert.equal(new Set(map.anchors.map((anchor) => anchor.identity)).size, map.anchors.length);
+ assert.match(map.mapDigest, /^sha256:[a-f0-9]{64}$/u);
+});
+
+test("M:1 uses WhisperX character times instead of dividing a merged word by length", () => {
+ const narrative = parseScript("merge.svml", "<line>can not</line>");
+ const map = locateWhisperXTiming(narrative, evidence({
+ endSec: 1,
+ durationSec: 1,
+ words: [{ text: "cannot", startSec: 0.1, endSec: 0.78, score: 0.93 }],
+ chars: characters(
+ "cannot",
+ [0.1, 0.18, 0.27, 0.42, 0.51, 0.63],
+ [0.17, 0.26, 0.36, 0.5, 0.62, 0.78],
+ ),
+ }));
+
+ assert.equal(map.groups[0]?.relation, "merge");
+ assert.deepEqual(
+ map.tokens.map((token) => [token.startSec, token.endSec, token.startQuality]),
+ [
+ [0.1, 0.36, "derived"],
+ [0.42, 0.78, "derived"],
+ ],
+ );
+});
+
+test("1:N wraps all WhisperX words in one Script token", () => {
+ const narrative = parseScript("split.svml", "<line>website</line>");
+ const map = locateWhisperXTiming(narrative, evidence({
+ words: [
+ { text: "web", startSec: 0.2, endSec: 0.45, score: 0.9 },
+ { text: "site", startSec: 0.5, endSec: 0.82, score: 0.91 },
+ ],
+ }));
+
+ assert.equal(map.groups[0]?.relation, "split");
+ assert.deepEqual(
+ [map.tokens[0]?.startSec, map.tokens[0]?.endSec, map.tokens[0]?.startQuality],
+ [0.2, 0.82, "derived"],
+ );
+});
+
+test("a recognized filler stays an insertion and does not absorb neighboring Script words", () => {
+ const narrative = parseScript("insertion.svml", "<line>I really like it.</line>");
+ const map = locateWhisperXTiming(narrative, evidence({
+ words: [
+ { text: "I", startSec: 0.1, endSec: 0.2, score: 0.98 },
+ { text: "uh", startSec: 0.24, endSec: 0.34, score: 0.88 },
+ { text: "really", startSec: 0.4, endSec: 0.62, score: 0.95 },
+ { text: "like", startSec: 0.67, endSec: 0.82, score: 0.96 },
+ { text: "it", startSec: 0.86, endSec: 0.96, score: 0.96 },
+ ],
+ }));
+
+ assert.deepEqual(map.groups.map((group) => group.relation), [
+ "exact",
+ "evidence-insertion",
+ "exact",
+ "exact",
+ "exact",
+ ]);
+ assert.deepEqual(
+ map.tokens.map((token) => [token.startSec, token.endSec, token.startQuality]),
+ [
+ [0.1, 0.2, "measured"],
+ [0.4, 0.62, "measured"],
+ [0.67, 0.82, "measured"],
+ [0.86, 0.96, "measured"],
+ ],
+ );
+});
+
+test("an omitted Script word gets a zero-width estimated point between measured neighbors", () => {
+ const narrative = parseScript("omission.svml", "<line>This is very good.</line>");
+ const map = locateWhisperXTiming(narrative, evidence({
+ words: [
+ { text: "This", startSec: 0.1, endSec: 0.25, score: 0.98 },
+ { text: "is", startSec: 0.3, endSec: 0.4, score: 0.97 },
+ { text: "good", startSec: 0.6, endSec: 0.82, score: 0.98 },
+ ],
+ }));
+
+ assert.deepEqual(map.groups.map((group) => group.relation), [
+ "exact",
+ "exact",
+ "source-omission",
+ "exact",
+ ]);
+ assert.deepEqual(
+ [map.tokens[2]?.startSec, map.tokens[2]?.endSec, map.tokens[2]?.startQuality],
+ [0.5, 0.5, "estimated"],
+ );
+});
+
+test("VAD bounds contain estimates when an entire Script Segment has no recognized words", () => {
+ const narrative = parseScript("vad.svml", "<line>One two.</line>");
+ const map = locateWhisperXTiming(narrative, evidence({
+ words: [],
+ vad: [{ startSec: 0.4, endSec: 1.2 }],
+ }));
+
+ assert.deepEqual(
+ map.tokens.map((token) => [token.startSec, token.endSec, token.startQuality]),
+ [
+ [0.4 + 0.8 / 3, 0.4 + 0.8 / 3, "estimated"],
+ [0.4 + 1.6 / 3, 0.4 + 1.6 / 3, "estimated"],
+ ],
+ );
+});
+
+test("Dual Text display inherits one speech envelope without linear display-word timing", () => {
+ const narrative = parseScript(
+ "dual.svml",
+ "<line>That was <really funny | absolutely hilarious>.</line>",
+ );
+ const map = locateWhisperXTiming(narrative, evidence({
+ words: [
+ { text: "That", startSec: 0.1, endSec: 0.22 },
+ { text: "was", startSec: 0.25, endSec: 0.34 },
+ { text: "absolutely", startSec: 0.4, endSec: 0.72 },
+ { text: "hilarious", startSec: 0.75, endSec: 1.1 },
+ ],
+ }));
+ const captions = deriveCaptionAtomTiming(narrative, map);
+
+ assert.equal(captions.length, 1);
+ assert.deepEqual(
+ {
+ display: captions[0]?.display,
+ sourceTokenIds: captions[0]?.sourceTokenIds,
+ startSec: captions[0]?.startSec,
+ endSec: captions[0]?.endSec,
+ startQuality: captions[0]?.startQuality,
+ },
+ {
+ display: "really funny",
+ sourceTokenIds: narrative.tokens.slice(2, 4).map((token) => token.id),
+ startSec: 0.4,
+ endSec: 1.1,
+ startQuality: "derived",
+ },
+ );
+});
+
+test("multiple Script Segments stay independent even when evidence records arrive out of order", () => {
+ const narrative = parseScript("segments.svml", "<one>Hello.</one><two>Goodbye.</two>");
+ const map = locateWhisperXTiming(narrative, {
+ contract: "svml.whisperx-evidence@0",
+ durationSec: 2,
+ segments: [
+ {
+ sourceSegmentId: "two",
+ startSec: 1,
+ endSec: 2,
+ words: [{ text: "Goodbye", startSec: 1.2, endSec: 1.6 }],
+ chars: [],
+ },
+ {
+ sourceSegmentId: "one",
+ startSec: 0,
+ endSec: 1,
+ words: [{ text: "Hello", startSec: 0.2, endSec: 0.55 }],
+ chars: [],
+ },
+ ],
+ });
+
+ assert.deepEqual(map.tokens.map((token) => [token.segmentId, token.startSec, token.endSec]), [
+ ["one", 0.2, 0.55],
+ ["two", 1.2, 1.6],
+ ]);
+ assert.deepEqual(map.groups.map((group) => group.sourceSegmentId), ["one", "two"]);
+});
+
+test("invalid overlapping WhisperX word windows fail instead of producing a reversed map", () => {
+ const narrative = parseScript("bad.svml", "<line>one two</line>");
+ assert.throws(
+ () => locateWhisperXTiming(narrative, evidence({
+ words: [
+ { text: "one", startSec: 0.1, endSec: 0.5 },
+ { text: "two", startSec: 0.4, endSec: 0.8 },
+ ],
+ })),
+ (error: unknown) => error instanceof WhisperXAlignmentError && error.code === "WHISPERX_WORD_ORDER",
+ );
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 8cd1e969..d0f61f46 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -61,6 +61,15 @@ importers:
specifier: workspace:*
version: link:../script
+ packages/whisperx:
+ dependencies:
+ '@svml/core':
+ specifier: workspace:*
+ version: link:../core
+ '@svml/script':
+ specifier: workspace:*
+ version: link:../script
+
packages:
'@emnapi/[email protected]':
diff --git a/tsconfig.v2.json b/tsconfig.v2.json
index 3368a676..a5348d84 100644
--- a/tsconfig.v2.json
+++ b/tsconfig.v2.json
@@ -9,7 +9,8 @@
"@svml/core": ["packages/core/src/index.ts"],
"@svml/driver-node": ["packages/driver-node/src/index.ts"],
"@svml/script": ["packages/script/src/index.ts"],
- "@svml/text": ["packages/text/src/index.ts"]
+ "@svml/text": ["packages/text/src/index.ts"],
+ "@svml/whisperx": ["packages/whisperx/src/index.ts"]
}
},
"include": ["packages/**/*.ts"],