Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat: separate caption evidence and runtime providers

kashorina04dcae52cca
diff --git a/packages/caption/README.md b/packages/caption/README.md
new file mode 100644
index 00000000..e7cfd0a6
--- /dev/null
+++ b/packages/caption/README.md
@@ -0,0 +1,10 @@
+# `@svml/caption`
+
+Pure composition between Script's untimed Caption Projection and a complete speech time map.
+The authored display region receives the evidence-backed envelope of the speech tokens it owns.
+Exact internal correspondences may reuse evidence timing; unresolved display words remain untimed
+until a track explicitly chooses a local presentation policy.
+
+`planCaptionPresentation()` is that optional track-local policy. `whole` preserves each authored
+region; `proportional-word` and `character-flow` create visibly labelled estimates. They never
+write those estimates back into the global speech map or Caption Projection.
diff --git a/packages/caption/package.json b/packages/caption/package.json
new file mode 100644
index 00000000..f70c6f28
--- /dev/null
+++ b/packages/caption/package.json
@@ -0,0 +1,14 @@
+{
+ "name": "@svml/caption",
+ "version": "0.0.0-dev",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "dependencies": {
+ "@svml/core": "workspace:*",
+ "@svml/script": "workspace:*",
+ "@svml/speech-align": "workspace:*"
+ }
+}
diff --git a/packages/whisperx/src/error.ts b/packages/caption/src/error.ts
similarity index 56%
rename from packages/whisperx/src/error.ts
rename to packages/caption/src/error.ts
index 3d7079ee..1dacbfec 100644
--- a/packages/whisperx/src/error.ts
+++ b/packages/caption/src/error.ts
@@ -1,9 +1,9 @@
-export class WhisperXAlignmentError extends Error {
+export class CaptionProjectionError extends Error {
readonly code: string;
constructor(code: string, message: string) {
super(message);
- this.name = "WhisperXAlignmentError";
+ this.name = "CaptionProjectionError";
this.code = code;
}
}
diff --git a/packages/caption/src/index.ts b/packages/caption/src/index.ts
new file mode 100644
index 00000000..5fcdea10
--- /dev/null
+++ b/packages/caption/src/index.ts
@@ -0,0 +1,4 @@
+export { CaptionProjectionError } from "./error.js";
+export { planCaptionPresentation } from "./presentation.js";
+export { temporalizeCaption } from "./temporalize.js";
+export type * from "./types.js";
diff --git a/packages/caption/src/presentation.ts b/packages/caption/src/presentation.ts
new file mode 100644
index 00000000..c6ac0c76
--- /dev/null
+++ b/packages/caption/src/presentation.ts
@@ -0,0 +1,140 @@
+import { digestOf } from "@svml/core";
+
+import type {
+ CaptionPresentationMode,
+ CaptionPresentationPlan,
+ CaptionPresentationUnit,
+ TimedCaptionRegion,
+} from "./types.js";
+
+const DISPLAY_WORD = /[\p{L}\p{M}\p{N}]+(?:['’.-][\p{L}\p{M}\p{N}]+)*/gu;
+
+type DisplayPart = {
+ readonly display: string;
+ readonly start: number;
+ readonly end: number;
+};
+
+function displayWords(value: string): DisplayPart[] {
+ const words: DisplayPart[] = [];
+ DISPLAY_WORD.lastIndex = 0;
+ let match: RegExpExecArray | null;
+ while ((match = DISPLAY_WORD.exec(value))) {
+ words.push({ display: match[0], start: match.index, end: match.index + match[0].length });
+ }
+ return words;
+}
+
+function wholeRegion(region: TimedCaptionRegion): CaptionPresentationUnit[] {
+ if (!region.display) return [];
+ return [{
+ id: `${region.id}:whole`,
+ regionId: region.id,
+ display: region.display,
+ displayStart: 0,
+ displayEnd: region.display.length,
+ startSec: region.startSec,
+ endSec: region.endSec,
+ basis: "region-envelope",
+ timingQuality: region.startQuality === "estimated" || region.endQuality === "estimated" ? "estimated" : "derived",
+ }];
+}
+
+function proportionalWords(region: TimedCaptionRegion): CaptionPresentationUnit[] {
+ const words = displayWords(region.display);
+ if (!words.length) return [];
+ const exact = new Map(region.refinements.map((item) => [`${item.displayStart}:${item.displayEnd}`, item]));
+ const units: Array<CaptionPresentationUnit | undefined> = words.map((word, index) => {
+ const refinement = exact.get(`${word.start}:${word.end}`);
+ return refinement ? {
+ id: `${region.id}:word:${index + 1}`,
+ regionId: region.id,
+ display: word.display,
+ displayStart: word.start,
+ displayEnd: word.end,
+ startSec: refinement.startSec,
+ endSec: refinement.endSec,
+ basis: "exact-correspondence",
+ timingQuality: refinement.startQuality === "estimated" || refinement.endQuality === "estimated" ? "estimated" : "derived",
+ } : undefined;
+ });
+
+ let cursor = 0;
+ while (cursor < words.length) {
+ if (units[cursor]) {
+ cursor += 1;
+ continue;
+ }
+ const runStart = cursor;
+ while (cursor < words.length && !units[cursor]) cursor += 1;
+ const runEnd = cursor;
+ const left = runStart === 0 ? region.startSec : units[runStart - 1]!.endSec;
+ const right = runEnd === words.length ? region.endSec : Math.max(left, units[runEnd]!.startSec);
+ const weights = words.slice(runStart, runEnd).map((word) => Math.max(1, [...word.display].length));
+ const total = weights.reduce((sum, weight) => sum + weight, 0);
+ let used = 0;
+ for (let index = runStart; index < runEnd; index += 1) {
+ const weight = weights[index - runStart]!;
+ const startSec = left + (right - left) * used / total;
+ used += weight;
+ const endSec = left + (right - left) * used / total;
+ const word = words[index]!;
+ units[index] = {
+ id: `${region.id}:word:${index + 1}`,
+ regionId: region.id,
+ display: word.display,
+ displayStart: word.start,
+ displayEnd: word.end,
+ startSec,
+ endSec,
+ basis: "presentation-estimate",
+ timingQuality: "estimated",
+ };
+ }
+ }
+ return units as CaptionPresentationUnit[];
+}
+
+function characterFlow(region: TimedCaptionRegion): CaptionPresentationUnit[] {
+ const characters = [...region.display];
+ if (!characters.length) return [];
+ const duration = region.endSec - region.startSec;
+ const units: CaptionPresentationUnit[] = [];
+ let sourceOffset = 0;
+ for (const [index, character] of characters.entries()) {
+ const start = sourceOffset;
+ sourceOffset += character.length;
+ if (!character.trim()) continue;
+ units.push({
+ id: `${region.id}:character:${index + 1}`,
+ regionId: region.id,
+ display: character,
+ displayStart: start,
+ displayEnd: sourceOffset,
+ startSec: region.startSec + duration * index / characters.length,
+ endSec: region.startSec + duration * (index + 1) / characters.length,
+ basis: "presentation-estimate",
+ timingQuality: "estimated",
+ });
+ }
+ return units;
+}
+
+export function planCaptionPresentation(
+ projection: { readonly projectionDigest: string; readonly regions: readonly TimedCaptionRegion[] },
+ mode: CaptionPresentationMode = "whole",
+): CaptionPresentationPlan {
+ const units = projection.regions.flatMap((region) =>
+ mode === "whole"
+ ? wholeRegion(region)
+ : mode === "proportional-word"
+ ? proportionalWords(region)
+ : characterFlow(region));
+ const payload = {
+ contract: "svml.caption-presentation-plan@0" as const,
+ sourceProjectionDigest: projection.projectionDigest,
+ mode,
+ units,
+ };
+ return { ...payload, planDigest: digestOf(payload) };
+}
diff --git a/packages/caption/src/temporalize.ts b/packages/caption/src/temporalize.ts
new file mode 100644
index 00000000..f198c806
--- /dev/null
+++ b/packages/caption/src/temporalize.ts
@@ -0,0 +1,79 @@
+import { digestOf } from "@svml/core";
+import type { ParsedNarrative } from "@svml/script";
+import type { CompleteSpeechTimeMap, TimingQuality } from "@svml/speech-align";
+
+import { CaptionProjectionError } from "./error.js";
+import type {
+ TimedCaptionProjection,
+ TimedCaptionRefinement,
+ TimedCaptionRegion,
+} from "./types.js";
+
+const QUALITY_RANK: Readonly<Record<TimingQuality, number>> = {
+ measured: 0,
+ derived: 1,
+ estimated: 2,
+};
+
+function composedQuality(value: TimingQuality): TimingQuality {
+ return QUALITY_RANK[value] >= QUALITY_RANK.derived ? value : "derived";
+}
+
+export function temporalizeCaption(
+ narrative: ParsedNarrative,
+ map: CompleteSpeechTimeMap,
+): TimedCaptionProjection {
+ if (map.semanticIndexDigest !== narrative.semanticIndex.digest) {
+ throw new CaptionProjectionError(
+ "CAPTION_SEMANTIC_INDEX",
+ "Caption Projection and speech time map use different Semantic Indexes.",
+ );
+ }
+ const timingByToken = new Map(map.tokens.map((token) => [token.tokenId, token]));
+ const sourceTiming = (startToken: number, endTokenExclusive: number, owner: string) => {
+ const source = narrative.tokens.slice(startToken, endTokenExclusive);
+ const timed = source.map((token) => timingByToken.get(token.id));
+ if (!timed.length || timed.some((token) => token === undefined)) {
+ throw new CaptionProjectionError(
+ "CAPTION_SPEECH_COVERAGE",
+ `${owner} is not covered by the complete speech time map.`,
+ );
+ }
+ const first = timed[0]!;
+ const last = timed.at(-1)!;
+ return {
+ sourceTokenIds: source.map((token) => token.id),
+ startSec: first.startSec,
+ endSec: last.endSec,
+ startQuality: composedQuality(first.startQuality),
+ endQuality: composedQuality(last.endQuality),
+ };
+ };
+
+ const regions: TimedCaptionRegion[] = narrative.captionProjection.regions.map((region) => {
+ const refinements: TimedCaptionRefinement[] = region.refinements.map((refinement) => ({
+ id: refinement.id,
+ display: refinement.display,
+ displayStart: refinement.displayStart,
+ displayEnd: refinement.displayEnd,
+ ...sourceTiming(refinement.startToken, refinement.endTokenExclusive, refinement.id),
+ relation: refinement.relation,
+ }));
+ return {
+ id: region.id,
+ display: region.display,
+ segmentId: region.segmentId,
+ kind: region.kind,
+ ...sourceTiming(region.startToken, region.endTokenExclusive, region.id),
+ refinements,
+ };
+ });
+ const payload = {
+ contract: "svml.timed-caption-projection@0" as const,
+ semanticIndexDigest: narrative.semanticIndex.digest,
+ speechTimeMapDigest: map.mapDigest,
+ text: narrative.captionProjection.text,
+ regions,
+ };
+ return { ...payload, projectionDigest: digestOf(payload) };
+}
diff --git a/packages/caption/src/types.ts b/packages/caption/src/types.ts
new file mode 100644
index 00000000..9a698181
--- /dev/null
+++ b/packages/caption/src/types.ts
@@ -0,0 +1,58 @@
+import type { TimingQuality } from "@svml/speech-align";
+
+export type TimedCaptionRefinement = {
+ readonly id: string;
+ readonly display: string;
+ readonly displayStart: number;
+ readonly displayEnd: number;
+ readonly sourceTokenIds: readonly string[];
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly startQuality: TimingQuality;
+ readonly endQuality: TimingQuality;
+ readonly relation: "exact";
+};
+
+export type TimedCaptionRegion = {
+ readonly id: string;
+ readonly display: string;
+ readonly segmentId: string;
+ readonly kind: "identity" | "alias" | "hidden";
+ readonly sourceTokenIds: readonly string[];
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly startQuality: TimingQuality;
+ readonly endQuality: TimingQuality;
+ readonly refinements: readonly TimedCaptionRefinement[];
+};
+
+export type TimedCaptionProjection = {
+ readonly contract: "svml.timed-caption-projection@0";
+ readonly semanticIndexDigest: string;
+ readonly speechTimeMapDigest: string;
+ readonly text: string;
+ readonly regions: readonly TimedCaptionRegion[];
+ readonly projectionDigest: string;
+};
+
+export type CaptionPresentationMode = "whole" | "proportional-word" | "character-flow";
+
+export type CaptionPresentationUnit = {
+ readonly id: string;
+ readonly regionId: string;
+ readonly display: string;
+ readonly displayStart: number;
+ readonly displayEnd: number;
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly basis: "region-envelope" | "exact-correspondence" | "presentation-estimate";
+ readonly timingQuality: TimingQuality;
+};
+
+export type CaptionPresentationPlan = {
+ readonly contract: "svml.caption-presentation-plan@0";
+ readonly sourceProjectionDigest: string;
+ readonly mode: CaptionPresentationMode;
+ readonly units: readonly CaptionPresentationUnit[];
+ readonly planDigest: string;
+};
diff --git a/packages/caption/test/caption.test.ts b/packages/caption/test/caption.test.ts
new file mode 100644
index 00000000..516e2394
--- /dev/null
+++ b/packages/caption/test/caption.test.ts
@@ -0,0 +1,79 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { planCaptionPresentation, temporalizeCaption } from "@svml/caption";
+import { parseScript } from "@svml/script";
+import { locateSpeechTiming } from "@svml/speech-align";
+
+test("Caption temporalization preserves evidence envelopes and labels local estimates", () => {
+ const narrative = parseScript(
+ "caption.svml",
+ "<line><that was insane | what the fuck> <15% off | fifteen percent off></line>",
+ );
+ const map = locateSpeechTiming(narrative, {
+ contract: "svml.aligned-transcript-evidence@0",
+ durationSec: 2,
+ segments: [{
+ sourceSegmentId: "line",
+ startSec: 0,
+ endSec: 2,
+ words: [
+ { text: "what", startSec: 0.1, endSec: 0.25 },
+ { text: "the", startSec: 0.3, endSec: 0.42 },
+ { text: "fuck", startSec: 0.48, endSec: 0.72 },
+ { text: "fifteen", startSec: 0.9, endSec: 1.15 },
+ { text: "percent", startSec: 1.2, endSec: 1.42 },
+ { text: "off", startSec: 1.48, endSec: 1.65 },
+ ],
+ chars: [],
+ }],
+ });
+ const originalMap = structuredClone(map);
+ const timed = temporalizeCaption(narrative, map);
+
+ assert.equal(timed.regions[0]?.display, "that was insane");
+ assert.deepEqual(
+ [timed.regions[0]?.startSec, timed.regions[0]?.endSec, timed.regions[0]?.refinements.length],
+ [0.1, 0.72, 0],
+ );
+ assert.deepEqual(timed.regions[1]?.refinements.map((item) => item.display), ["off"]);
+
+ const whole = planCaptionPresentation(timed, "whole");
+ assert.deepEqual(
+ [whole.units[0]?.display, whole.units[0]?.startSec, whole.units[0]?.endSec, whole.units[0]?.basis],
+ ["that was insane", 0.1, 0.72, "region-envelope"],
+ );
+
+ const words = planCaptionPresentation(timed, "proportional-word");
+ assert.deepEqual(words.units.slice(0, 3).map((unit) => unit.basis), [
+ "presentation-estimate",
+ "presentation-estimate",
+ "presentation-estimate",
+ ]);
+ assert.equal(words.units.find((unit) => unit.display === "off")?.basis, "exact-correspondence");
+ assert.equal(words.units.find((unit) => unit.display === "15")?.timingQuality, "estimated");
+ assert.deepEqual(map, originalMap, "caption presentation must not modify the global speech map");
+});
+
+test("hidden speech owns time but emits no visible presentation unit", () => {
+ const narrative = parseScript("hidden.svml", "<line>Hello < | um> world.</line>");
+ const map = locateSpeechTiming(narrative, {
+ contract: "svml.aligned-transcript-evidence@0",
+ durationSec: 1,
+ segments: [{
+ sourceSegmentId: "line",
+ startSec: 0,
+ endSec: 1,
+ words: [
+ { text: "Hello", startSec: 0.1, endSec: 0.25 },
+ { text: "um", startSec: 0.3, endSec: 0.4 },
+ { text: "world", startSec: 0.45, endSec: 0.7 },
+ ],
+ chars: [],
+ }],
+ });
+ const timed = temporalizeCaption(narrative, map);
+ const hidden = timed.regions.find((region) => region.kind === "hidden");
+ assert.deepEqual([hidden?.startSec, hidden?.endSec], [0.3, 0.4]);
+ assert.equal(planCaptionPresentation(timed).units.some((unit) => unit.regionId === hidden?.id), false);
+});
diff --git a/packages/driver-node/README.md b/packages/driver-node/README.md
index 5f4b8672..e3d75615 100644
--- a/packages/driver-node/README.md
+++ b/packages/driver-node/README.md
@@ -1,11 +1,16 @@
# `@svml/driver-node`
-Experimental Node Host for the v2 Core.
+Node runtime for executing Core commands without adding workflow semantics to Core.
-Driver v0 reads explicitly supplied static `svml.module.json` files, runs Host-registered producer
-functions, dispatches typed Needs to registered handlers, stores content-addressed bytes and can
-serialize or resume Core state.
+- `HostRegistry` registers deterministic Producer implementations locked by implementation digest.
+- `ProviderRegistry` registers external capabilities that may use APIs, credentials, queues or local runtimes.
+- A Producer emits a typed `Need`; it never reads provider configuration directly.
+- One matching Provider may fulfill a Need. Multiple matches require an explicit runtime `bind()`.
+- Provider identity becomes the Receipt fulfiller; handlers return only value, conformance, delivery and metadata.
-It deliberately does not load implementation code from a package locator. A future sandboxed
-executor must be designed and audited before third-party implementations can run. Frontend and
-Import Prologue behavior are also outside this package version.
+Build state is serializable. Missing providers, generation latency and transient provider errors pause a
+build without replaying completed Producers.
+
+The Driver still does not load implementation code from a package locator. A future sandboxed executor
+must be designed and audited before third-party implementations can run. Frontend and Import Prologue
+behavior remain outside this package.
diff --git a/packages/driver-node/src/driver.ts b/packages/driver-node/src/driver.ts
index dd876bf5..1266a32c 100644
--- a/packages/driver-node/src/driver.ts
+++ b/packages/driver-node/src/driver.ts
@@ -9,18 +9,19 @@ import type {
} from "@svml/protocol";
import { MemoryArtifactStore } from "./artifacts.js";
-import { HostRegistry, producerRegistryKey, requirementRegistryKey } from "./registry.js";
+import { HostRegistry, ProviderRegistry, producerRegistryKey, providerCapabilityKey } from "./registry.js";
import type {
ArtifactStore,
BlockedCommand,
DriverJournalEntry,
DriverRunResult,
ProducerHandlerResult,
- RequirementHandlerResult,
+ ProviderHandlerResult,
} from "./types.js";
export type NodeDriverOptions = {
readonly registry?: HostRegistry;
+ readonly providers?: ProviderRegistry;
readonly artifacts?: ArtifactStore;
readonly maxEvents?: number;
};
@@ -32,16 +33,19 @@ type Executable =
}
| {
readonly command: FulfillNeedCommand;
- readonly run: () => Promise<RequirementHandlerResult>;
+ readonly providerId: string;
+ readonly run: () => Promise<ProviderHandlerResult>;
};
export class NodeDriver {
readonly registry: HostRegistry;
+ readonly providers: ProviderRegistry;
readonly artifacts: ArtifactStore;
readonly maxEvents: number;
constructor(options: NodeDriverOptions = {}) {
this.registry = options.registry ?? new HostRegistry();
+ this.providers = options.providers ?? new ProviderRegistry();
this.artifacts = options.artifacts ?? new MemoryArtifactStore();
this.maxEvents = options.maxEvents ?? 1_000;
}
@@ -101,19 +105,32 @@ export class NodeDriver {
};
}
- const registration = this.registry.requirement(command.need.wants);
- if (registration === undefined) {
+ const resolution = this.providers.resolve(command.need);
+ if (resolution.status === "missing") {
return {
blocked: {
command: command.id,
- reason: "missing-handler",
- subject: requirementRegistryKey(command.need.wants),
+ reason: "missing-provider",
+ subject: resolution.providerId === undefined
+ ? providerCapabilityKey(command.need.wants)
+ : `${providerCapabilityKey(command.need.wants)} -> ${resolution.providerId}`,
},
};
}
+ if (resolution.status === "ambiguous") {
+ return {
+ blocked: {
+ command: command.id,
+ reason: "ambiguous-provider",
+ subject: `${providerCapabilityKey(command.need.wants)} -> ${resolution.providerIds.join(", ")}`,
+ },
+ };
+ }
+ const registration = resolution.registration;
return {
executable: {
command,
+ providerId: registration.id,
run: async () =>
registration.handler({
command: structuredClone(command),
@@ -125,7 +142,7 @@ export class NodeDriver {
}
async #execute(executable: Executable): Promise<BuildEvent> {
- if (executable.command.kind === "invoke-producer") {
+ if (!("providerId" in executable)) {
const result = (await executable.run()) as ProducerHandlerResult;
return {
kind: "producer-completed",
@@ -140,7 +157,7 @@ export class NodeDriver {
};
}
- const result = (await executable.run()) as RequirementHandlerResult;
+ const result = (await executable.run()) as ProviderHandlerResult;
return {
kind: "need-fulfilled",
id: `event:${digestOf({
@@ -151,7 +168,7 @@ export class NodeDriver {
command: executable.command.id,
value: result.value,
requestDigest: executable.command.need.requestDigest,
- fulfiller: result.fulfiller,
+ fulfiller: executable.providerId,
conformance: result.conformance,
delivery: result.delivery,
metadata: result.metadata,
diff --git a/packages/driver-node/src/index.ts b/packages/driver-node/src/index.ts
index b97338e2..859833ae 100644
--- a/packages/driver-node/src/index.ts
+++ b/packages/driver-node/src/index.ts
@@ -9,7 +9,8 @@ export {
export { parseBuildState, serializeBuildState } from "./persistence.js";
export {
HostRegistry,
+ ProviderRegistry,
producerRegistryKey,
- requirementRegistryKey,
+ providerCapabilityKey,
} from "./registry.js";
export type * from "./types.js";
diff --git a/packages/driver-node/src/registry.ts b/packages/driver-node/src/registry.ts
index da418a10..78025211 100644
--- a/packages/driver-node/src/registry.ts
+++ b/packages/driver-node/src/registry.ts
@@ -1,10 +1,11 @@
-import type { Digest, ProducerRef, TypeRef } from "@svml/protocol";
+import type { Digest, Need, ProducerRef, TypeRef } from "@svml/protocol";
import type {
ProducerHandler,
ProducerRegistration,
- RequirementHandler,
- RequirementRegistration,
+ ProviderHandler,
+ ProviderRegistration,
+ ProviderResolution,
} from "./types.js";
function moduleKey(ref: { readonly name: string; readonly version: string }): string {
@@ -15,13 +16,12 @@ export function producerRegistryKey(ref: ProducerRef): string {
return `${moduleKey(ref.module)}#${ref.name}`;
}
-export function requirementRegistryKey(ref: TypeRef): string {
+export function providerCapabilityKey(ref: TypeRef): string {
return `${moduleKey(ref.module)}#${ref.name}`;
}
export class HostRegistry {
readonly #producers = new Map<string, ProducerRegistration>();
- readonly #requirements = new Map<string, RequirementRegistration>();
registerProducer(
producer: ProducerRef,
@@ -33,17 +33,61 @@ export class HostRegistry {
this.#producers.set(key, { producer, implementationDigest, handler });
}
- registerRequirement(type: TypeRef, handler: RequirementHandler): void {
- const key = requirementRegistryKey(type);
- if (this.#requirements.has(key)) throw new Error(`requirement ${key} is already registered`);
- this.#requirements.set(key, { type, handler });
- }
-
producer(ref: ProducerRef): ProducerRegistration | undefined {
return this.#producers.get(producerRegistryKey(ref));
}
+}
+
+export class ProviderRegistry {
+ readonly #capabilities = new Map<string, ProviderRegistration[]>();
+ readonly #bindings = new Map<string, string>();
+
+ registerProvider(
+ id: string,
+ wants: TypeRef,
+ handler: ProviderHandler,
+ options: { readonly supports?: (need: Need) => boolean } = {},
+ ): void {
+ if (!id.trim()) throw new Error("provider id must not be empty");
+ const key = providerCapabilityKey(wants);
+ const registrations = this.#capabilities.get(key) ?? [];
+ if (registrations.some((registration) => registration.id === id)) {
+ throw new Error(`provider ${id} already registers capability ${key}`);
+ }
+ registrations.push({ id, wants, handler, ...options });
+ this.#capabilities.set(key, registrations);
+ }
+
+ bind(wants: TypeRef, providerId: string): void {
+ const key = providerCapabilityKey(wants);
+ if (!(this.#capabilities.get(key) ?? []).some((registration) => registration.id === providerId)) {
+ throw new Error(`provider ${providerId} does not register capability ${key}`);
+ }
+ this.#bindings.set(key, providerId);
+ }
+
+ resolve(need: Need): ProviderResolution {
+ const key = providerCapabilityKey(need.wants);
+ const bound = this.#bindings.get(key);
+ const registrations = (this.#capabilities.get(key) ?? [])
+ .filter((registration) => registration.supports?.(need) ?? true);
+ if (bound !== undefined) {
+ const registration = registrations.find((candidate) => candidate.id === bound);
+ return registration === undefined
+ ? { status: "missing", providerId: bound }
+ : { status: "resolved", registration };
+ }
+ if (registrations.length === 0) return { status: "missing" };
+ if (registrations.length > 1) {
+ return {
+ status: "ambiguous",
+ providerIds: registrations.map((registration) => registration.id).sort(),
+ };
+ }
+ return { status: "resolved", registration: registrations[0]! };
+ }
- requirement(ref: TypeRef): RequirementRegistration | undefined {
- return this.#requirements.get(requirementRegistryKey(ref));
+ providers(wants: TypeRef): readonly ProviderRegistration[] {
+ return [...(this.#capabilities.get(providerCapabilityKey(wants)) ?? [])];
}
}
diff --git a/packages/driver-node/src/types.ts b/packages/driver-node/src/types.ts
index eee874c6..63d89bfe 100644
--- a/packages/driver-node/src/types.ts
+++ b/packages/driver-node/src/types.ts
@@ -31,23 +31,22 @@ export type ProducerHandler = (
context: ProducerHandlerContext,
) => ProducerHandlerResult | Promise<ProducerHandlerResult>;
-export type RequirementHandlerResult = {
+export type ProviderHandlerResult = {
readonly value: StoredValue;
- readonly fulfiller: string;
readonly conformance: Conformance;
readonly delivery: Delivery;
readonly metadata: CanonicalValue;
};
-export type RequirementHandlerContext = {
+export type ProviderHandlerContext = {
readonly command: FulfillNeedCommand;
readonly need: Need;
readonly artifacts: ArtifactStore;
};
-export type RequirementHandler = (
- context: RequirementHandlerContext,
-) => RequirementHandlerResult | Promise<RequirementHandlerResult>;
+export type ProviderHandler = (
+ context: ProviderHandlerContext,
+) => ProviderHandlerResult | Promise<ProviderHandlerResult>;
export type ArtifactStore = {
put(bytes: Uint8Array, mediaType: string): Promise<BlobRef>;
@@ -65,7 +64,11 @@ export type DriverJournalEntry = {
export type BlockedCommand = {
readonly command: string;
- readonly reason: "missing-producer" | "implementation-mismatch" | "missing-handler";
+ readonly reason:
+ | "missing-producer"
+ | "implementation-mismatch"
+ | "missing-provider"
+ | "ambiguous-provider";
readonly subject: string;
};
@@ -82,7 +85,14 @@ export type ProducerRegistration = {
readonly handler: ProducerHandler;
};
-export type RequirementRegistration = {
- readonly type: TypeRef;
- readonly handler: RequirementHandler;
+export type ProviderRegistration = {
+ readonly id: string;
+ readonly wants: TypeRef;
+ readonly handler: ProviderHandler;
+ readonly supports?: (need: Need) => boolean;
};
+
+export type ProviderResolution =
+ | { readonly status: "resolved"; readonly registration: ProviderRegistration }
+ | { readonly status: "missing"; readonly providerId?: string }
+ | { readonly status: "ambiguous"; readonly providerIds: readonly string[] };
diff --git a/packages/driver-node/test/driver.test.ts b/packages/driver-node/test/driver.test.ts
index 8c66c4ab..4c3f7888 100644
--- a/packages/driver-node/test/driver.test.ts
+++ b/packages/driver-node/test/driver.test.ts
@@ -9,6 +9,7 @@ import {
HostRegistry,
MemoryArtifactStore,
NodeDriver,
+ ProviderRegistry,
loadResolvedClosure,
parseBuildState,
serializeBuildState,
@@ -36,9 +37,11 @@ type GreetingCalls = {
function configuredRegistry(): {
registry: HostRegistry;
+ providers: ProviderRegistry;
calls: GreetingCalls;
} {
const registry = new HostRegistry();
+ const providers = new ProviderRegistry();
const calls = { prompt: 0, request: 0, assemble: 0, fulfill: 0 };
registry.registerProducer(producers.makePrompt, implementationDigests.makePrompt, ({ inputs }) => {
@@ -81,25 +84,24 @@ function configuredRegistry(): {
};
});
- return { registry, calls };
+ return { registry, providers, calls };
}
test("Driver pauses at an unbound Need, serializes, then resumes without rerunning producers", async () => {
- const { registry, calls } = configuredRegistry();
- const driver = new NodeDriver({ registry });
+ const { registry, providers, calls } = configuredRegistry();
+ const driver = new NodeDriver({ registry, providers });
const paused = await driver.run(createGreetingBuild());
assert.equal(paused.status, "paused");
- assert.equal(paused.blocked[0]?.reason, "missing-handler");
+ assert.equal(paused.blocked[0]?.reason, "missing-provider");
assert.deepEqual(calls, { prompt: 1, request: 1, assemble: 0, fulfill: 0 });
const restored = parseBuildState(serializeBuildState(paused.state));
- registry.registerRequirement(types.generated, ({ need }) => {
+ providers.registerProvider("example:generation", types.generated, ({ need }) => {
calls.fulfill += 1;
assert.deepEqual(need.constraints, { prompt: "Greet Ada" });
return {
value: { kind: "inline", value: "Hello, Ada!" },
- fulfiller: "example:generation",
conformance: "exact",
delivery: "executed",
metadata: { provider: "fixture" },
@@ -110,19 +112,72 @@ test("Driver pauses at an unbound Need, serializes, then resumes without rerunni
assert.equal(completed.status, "complete");
assert.deepEqual(calls, { prompt: 1, request: 1, assemble: 1, fulfill: 1 });
assert.equal(completed.state.receipts[0]?.delivery, "executed");
+ assert.equal(completed.state.receipts[0]?.fulfiller, "example:generation");
assert.equal(completed.state.records.find((record) => record.id === "document:root")?.conformance, "exact");
});
+test("Provider Registry rejects ambiguity until the Runtime binds one provider", async () => {
+ const { registry, providers } = configuredRegistry();
+ providers.registerProvider("example:alpha", types.generated, () => ({
+ value: { kind: "inline", value: "Alpha" },
+ conformance: "exact",
+ delivery: "executed",
+ metadata: {},
+ }));
+ providers.registerProvider("example:beta", types.generated, () => ({
+ value: { kind: "inline", value: "Beta" },
+ conformance: "exact",
+ delivery: "executed",
+ metadata: {},
+ }));
+
+ const driver = new NodeDriver({ registry, providers });
+ const ambiguous = await driver.run(createGreetingBuild());
+ assert.equal(ambiguous.status, "paused");
+ assert.equal(ambiguous.blocked[0]?.reason, "ambiguous-provider");
+ assert.match(ambiguous.blocked[0]?.subject ?? "", /example:alpha, example:beta/u);
+
+ providers.bind(types.generated, "example:beta");
+ const completed = await driver.run(ambiguous.state);
+ assert.equal(completed.status, "complete");
+ assert.equal(completed.state.receipts[0]?.fulfiller, "example:beta");
+ assert.deepEqual(
+ completed.state.records.find((record) => record.id === "document:root")?.value,
+ { kind: "inline", value: { text: "Beta" } },
+ );
+});
+
+test("Provider capabilities may narrow themselves with typed Need constraints", async () => {
+ const { registry, providers } = configuredRegistry();
+ providers.registerProvider("example:wrong-model", types.generated, () => {
+ throw new Error("unsupported provider must never run");
+ }, { supports: () => false });
+ providers.registerProvider("example:compatible", types.generated, () => ({
+ value: { kind: "inline", value: "Compatible" },
+ conformance: "exact",
+ delivery: "executed",
+ metadata: {},
+ }), {
+ supports: (need) => {
+ const constraints = need.constraints as Readonly<Record<string, unknown>>;
+ return constraints.prompt === "Greet Ada";
+ },
+ });
+
+ const completed = await new NodeDriver({ registry, providers }).run(createGreetingBuild());
+ assert.equal(completed.status, "complete");
+ assert.equal(completed.state.receipts[0]?.fulfiller, "example:compatible");
+});
+
test("substitute conformance propagates through later producer outputs", async () => {
- const { registry } = configuredRegistry();
- registry.registerRequirement(types.generated, () => ({
+ const { registry, providers } = configuredRegistry();
+ providers.registerProvider("example:placeholder", types.generated, () => ({
value: { kind: "inline", value: "Placeholder" },
- fulfiller: "example:placeholder",
conformance: "substitute",
delivery: "provided",
metadata: { reason: "preview" },
}));
- const result = await new NodeDriver({ registry }).run(
+ const result = await new NodeDriver({ registry, providers }).run(
createGreetingBuild({ needAccepts: "substitute", goalAccepts: "substitute" }),
);
assert.equal(result.status, "complete");
@@ -186,31 +241,29 @@ test("Driver reads static manifests without executing package code", async () =>
});
test("Core still owns scheduling when Driver has every implementation", async () => {
- const { registry } = configuredRegistry();
- registry.registerRequirement(types.generated, () => ({
+ const { registry, providers } = configuredRegistry();
+ providers.registerProvider("example:cache", types.generated, () => ({
value: { kind: "inline", value: "Hello, Ada!" },
- fulfiller: "example:cache",
conformance: "exact",
delivery: "cache",
metadata: {},
}));
const start = createGreetingBuild();
assert.equal(reduce(start).commands[0]?.kind, "invoke-producer");
- const result = await new NodeDriver({ registry }).run(start);
+ const result = await new NodeDriver({ registry, providers }).run(start);
assert.equal(result.status, "complete");
assert.equal(result.state.receipts[0]?.delivery, "cache");
});
test("receipt metadata is covered by its identity during resume validation", async () => {
- const { registry } = configuredRegistry();
- registry.registerRequirement(types.generated, () => ({
+ const { registry, providers } = configuredRegistry();
+ providers.registerProvider("example:cache", types.generated, () => ({
value: { kind: "inline", value: "Hello, Ada!" },
- fulfiller: "example:cache",
conformance: "exact",
delivery: "cache",
metadata: { cacheKey: "stable" },
}));
- const result = await new NodeDriver({ registry }).run(createGreetingBuild());
+ const result = await new NodeDriver({ registry, providers }).run(createGreetingBuild());
assert.equal(result.status, "complete");
const tampered = structuredClone(result.state);
const receipt = tampered.receipts[0];
@@ -220,22 +273,21 @@ test("receipt metadata is covered by its identity during resume validation", asy
});
test("a transient Handler failure pauses and can resume without replaying completed producers", async () => {
- const { registry, calls } = configuredRegistry();
+ const { registry, providers, calls } = configuredRegistry();
let attempts = 0;
- registry.registerRequirement(types.generated, () => {
+ providers.registerProvider("example:unstable", types.generated, () => {
calls.fulfill += 1;
attempts += 1;
if (attempts === 1) throw new Error("temporary outage");
return {
value: { kind: "inline", value: "Hello after retry" },
- fulfiller: "example:unstable",
conformance: "exact",
delivery: "executed",
metadata: { attempt: attempts },
};
});
- const driver = new NodeDriver({ registry });
+ const driver = new NodeDriver({ registry, providers });
const paused = await driver.run(createGreetingBuild());
assert.equal(paused.status, "paused");
assert.match(paused.journal.at(-1)?.message ?? "", /temporary outage/u);
diff --git a/packages/driver-node/test/video-pipeline-fixture.ts b/packages/driver-node/test/video-pipeline-fixture.ts
new file mode 100644
index 00000000..78015284
--- /dev/null
+++ b/packages/driver-node/test/video-pipeline-fixture.ts
@@ -0,0 +1,195 @@
+import {
+ createResolvedClosure,
+ digestOf,
+ link,
+ sealRecord,
+ sealTypedModule,
+ start,
+} from "@svml/core";
+import type {
+ BuildPlan,
+ BuildState,
+ ModuleManifest,
+ ProducerRef,
+ TypeRef,
+ ValueSchema,
+} from "@svml/protocol";
+
+export const videoModule = { name: "example.video-pipeline", version: "0.0.0" } as const;
+
+export const videoTypes = {
+ narrative: { module: videoModule, name: "Narrative" },
+ schedule: { module: videoModule, name: "SpeechSchedule" },
+ basis: { module: videoModule, name: "SpeechBasis" },
+ evidence: { module: videoModule, name: "AlignedTranscriptEvidence" },
+ speechMap: { module: videoModule, name: "CompleteSpeechTimeMap" },
+ caption: { module: videoModule, name: "TimedCaption" },
+} satisfies Record<string, TypeRef>;
+
+export const videoProducers = {
+ requestEstimate: { module: videoModule, name: "request-estimate" },
+ requestGeneration: { module: videoModule, name: "request-generation" },
+ requestRecognition: { module: videoModule, name: "request-recognition" },
+ locateSpeech: { module: videoModule, name: "locate-speech" },
+ temporalizeCaption: { module: videoModule, name: "temporalize-caption" },
+} satisfies Record<string, ProducerRef>;
+
+export const videoImplementations = Object.fromEntries(
+ Object.entries(videoProducers).map(([name]) => [name, digestOf(`example.video-pipeline/${name}@0`)]),
+) as Readonly<Record<keyof typeof videoProducers, ReturnType<typeof digestOf>>>;
+
+const string = { kind: "string", minLength: 1 } as const;
+const number = { kind: "number", minimum: 0 } as const;
+const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema }>>): ValueSchema => ({
+ kind: "object",
+ fields,
+});
+
+export const videoManifest: ModuleManifest = {
+ format: "svml.module@0",
+ name: videoModule.name,
+ version: videoModule.version,
+ dependencies: [],
+ types: [
+ { name: videoTypes.narrative.name, schema: object({ speech: { schema: string }, display: { schema: string } }) },
+ { name: videoTypes.schedule.name, schema: object({ durationSec: { schema: number } }) },
+ { name: videoTypes.basis.name, schema: object({ audio: { schema: string }, visual: { schema: string } }) },
+ { name: videoTypes.evidence.name, schema: object({ transcript: { schema: string } }) },
+ { name: videoTypes.speechMap.name, schema: object({ startSec: { schema: number }, endSec: { schema: number } }) },
+ {
+ name: videoTypes.caption.name,
+ schema: object({ text: { schema: string }, startSec: { schema: number }, endSec: { schema: number } }),
+ },
+ ],
+ surfaces: [],
+ producers: [
+ {
+ name: videoProducers.requestEstimate.name,
+ inputs: [{ name: "narrative", type: videoTypes.narrative }],
+ outputs: [],
+ needs: [{ name: "schedule", wants: videoTypes.schedule }],
+ implementation: {
+ kind: "registered",
+ locator: "example.video-pipeline/request-estimate",
+ digest: videoImplementations.requestEstimate,
+ },
+ },
+ {
+ name: videoProducers.requestGeneration.name,
+ inputs: [
+ { name: "narrative", type: videoTypes.narrative },
+ { name: "schedule", type: videoTypes.schedule },
+ ],
+ outputs: [],
+ needs: [{ name: "basis", wants: videoTypes.basis }],
+ implementation: {
+ kind: "registered",
+ locator: "example.video-pipeline/request-generation",
+ digest: videoImplementations.requestGeneration,
+ },
+ },
+ {
+ name: videoProducers.requestRecognition.name,
+ inputs: [{ name: "basis", type: videoTypes.basis }],
+ outputs: [],
+ needs: [{ name: "evidence", wants: videoTypes.evidence }],
+ implementation: {
+ kind: "registered",
+ locator: "example.video-pipeline/request-recognition",
+ digest: videoImplementations.requestRecognition,
+ },
+ },
+ {
+ name: videoProducers.locateSpeech.name,
+ inputs: [
+ { name: "narrative", type: videoTypes.narrative },
+ { name: "evidence", type: videoTypes.evidence },
+ ],
+ outputs: [{ name: "map", type: videoTypes.speechMap }],
+ needs: [],
+ implementation: {
+ kind: "registered",
+ locator: "example.video-pipeline/locate-speech",
+ digest: videoImplementations.locateSpeech,
+ },
+ },
+ {
+ name: videoProducers.temporalizeCaption.name,
+ inputs: [
+ { name: "narrative", type: videoTypes.narrative },
+ { name: "map", type: videoTypes.speechMap },
+ ],
+ outputs: [{ name: "caption", type: videoTypes.caption }],
+ needs: [],
+ implementation: {
+ kind: "registered",
+ locator: "example.video-pipeline/temporalize-caption",
+ digest: videoImplementations.temporalizeCaption,
+ },
+ },
+ ],
+};
+
+export const videoPlan: BuildPlan = {
+ format: "svml.plan@0",
+ id: "video-pipeline",
+ steps: [
+ {
+ id: "request-estimate",
+ producer: videoProducers.requestEstimate,
+ inputs: { narrative: "narrative:root" },
+ outputs: {},
+ needs: { schedule: { id: "need:schedule", result: "schedule:root", accepts: "exact" } },
+ },
+ {
+ id: "request-generation",
+ producer: videoProducers.requestGeneration,
+ inputs: { narrative: "narrative:root", schedule: "schedule:root" },
+ outputs: {},
+ needs: { basis: { id: "need:basis", result: "basis:root", accepts: "exact" } },
+ },
+ {
+ id: "request-recognition",
+ producer: videoProducers.requestRecognition,
+ inputs: { basis: "basis:root" },
+ outputs: {},
+ needs: { evidence: { id: "need:evidence", result: "evidence:root", accepts: "exact" } },
+ },
+ {
+ id: "locate-speech",
+ producer: videoProducers.locateSpeech,
+ inputs: { narrative: "narrative:root", evidence: "evidence:root" },
+ outputs: { map: "speech-map:root" },
+ needs: {},
+ },
+ {
+ id: "temporalize-caption",
+ producer: videoProducers.temporalizeCaption,
+ inputs: { narrative: "narrative:root", map: "speech-map:root" },
+ outputs: { caption: "caption:root" },
+ needs: {},
+ },
+ ],
+ goals: [{ record: "caption:root", type: videoTypes.caption, accepts: "exact" }],
+};
+
+export function createVideoBuild(): BuildState {
+ const closure = createResolvedClosure([videoManifest]);
+ const narrative = sealRecord({
+ id: "narrative:root",
+ type: videoTypes.narrative,
+ value: { kind: "inline", value: { speech: "what the fuck", display: "that was insane" } },
+ conformance: "exact",
+ origin: {
+ kind: "authored",
+ sourceDigest: digestOf("source:video-pipeline"),
+ frontendClosureDigest: digestOf("frontend:script"),
+ sourceName: "pipeline.svml",
+ },
+ });
+ return start(link(closure, [sealTypedModule({
+ id: "author:video-pipeline",
+ closureDigest: closure.digest,
+ records: [narrative],
+ })]), videoPlan);
+}
diff --git a/packages/driver-node/test/video-pipeline.test.ts b/packages/driver-node/test/video-pipeline.test.ts
new file mode 100644
index 00000000..85144a68
--- /dev/null
+++ b/packages/driver-node/test/video-pipeline.test.ts
@@ -0,0 +1,128 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ HostRegistry,
+ NodeDriver,
+ ProviderRegistry,
+ parseBuildState,
+ serializeBuildState,
+} from "@svml/driver-node";
+
+import {
+ createVideoBuild,
+ videoImplementations,
+ videoProducers,
+ videoTypes,
+} from "./video-pipeline-fixture.js";
+
+function inlineObject(value: unknown): Readonly<Record<string, unknown>> {
+ if (value === null || Array.isArray(value) || typeof value !== "object") throw new Error("expected object");
+ return value as Readonly<Record<string, unknown>>;
+}
+
+function inlineString(value: unknown): string {
+ if (typeof value !== "string") throw new Error("expected string");
+ return value;
+}
+
+function inlineNumber(value: unknown): number {
+ if (typeof value !== "number" || !Number.isFinite(value)) throw new Error("expected number");
+ return value;
+}
+
+test("estimate, generation and recognition pause independently without replaying the immutable DAG", async () => {
+ const host = new HostRegistry();
+ const providers = new ProviderRegistry();
+ const calls = { estimate: 0, generate: 0, recognize: 0, locate: 0, caption: 0 };
+
+ host.registerProducer(videoProducers.requestEstimate, videoImplementations.requestEstimate, ({ inputs }) => {
+ calls.estimate += 1;
+ const narrative = inputs.narrative;
+ assert.equal(narrative?.value.kind, "inline");
+ return { outputs: {}, needs: { schedule: { speech: inlineString(inlineObject(narrative.value.value).speech) } } };
+ });
+ host.registerProducer(videoProducers.requestGeneration, videoImplementations.requestGeneration, ({ inputs }) => {
+ calls.generate += 1;
+ assert.equal(inputs.schedule?.value.kind, "inline");
+ return { outputs: {}, needs: { basis: { durationSec: inlineNumber(inlineObject(inputs.schedule.value.value).durationSec) } } };
+ });
+ host.registerProducer(videoProducers.requestRecognition, videoImplementations.requestRecognition, ({ inputs }) => {
+ calls.recognize += 1;
+ assert.equal(inputs.basis?.value.kind, "inline");
+ return { outputs: {}, needs: { evidence: { audio: inlineString(inlineObject(inputs.basis.value.value).audio) } } };
+ });
+ host.registerProducer(videoProducers.locateSpeech, videoImplementations.locateSpeech, ({ inputs }) => {
+ calls.locate += 1;
+ assert.equal(inputs.evidence?.value.kind, "inline");
+ return { outputs: { map: { kind: "inline", value: { startSec: 0.1, endSec: 0.72 } } }, needs: {} };
+ });
+ host.registerProducer(videoProducers.temporalizeCaption, videoImplementations.temporalizeCaption, ({ inputs }) => {
+ calls.caption += 1;
+ assert.equal(inputs.narrative?.value.kind, "inline");
+ assert.equal(inputs.map?.value.kind, "inline");
+ const narrative = inlineObject(inputs.narrative.value.value);
+ const map = inlineObject(inputs.map.value.value);
+ return {
+ outputs: {
+ caption: {
+ kind: "inline",
+ value: {
+ text: inlineString(narrative.display),
+ startSec: inlineNumber(map.startSec),
+ endSec: inlineNumber(map.endSec),
+ },
+ },
+ },
+ needs: {},
+ };
+ });
+
+ const driver = new NodeDriver({ registry: host, providers });
+ const atEstimate = await driver.run(createVideoBuild());
+ assert.equal(atEstimate.blocked[0]?.reason, "missing-provider");
+ assert.deepEqual(calls, { estimate: 1, generate: 0, recognize: 0, locate: 0, caption: 0 });
+
+ providers.registerProvider("runtime:estimate", videoTypes.schedule, () => ({
+ value: { kind: "inline", value: { durationSec: 1 } },
+ conformance: "exact",
+ delivery: "executed",
+ metadata: {},
+ }));
+ const atGeneration = await driver.run(parseBuildState(serializeBuildState(atEstimate.state)));
+ assert.equal(atGeneration.blocked[0]?.reason, "missing-provider");
+ assert.deepEqual(calls, { estimate: 1, generate: 1, recognize: 0, locate: 0, caption: 0 });
+
+ providers.registerProvider("runtime:seedance-mini", videoTypes.basis, ({ need }) => {
+ assert.deepEqual(need.constraints, { durationSec: 1 });
+ return {
+ value: { kind: "inline", value: { audio: "artifact:audio", visual: "artifact:visual" } },
+ conformance: "exact",
+ delivery: "executed",
+ metadata: { model: "mini" },
+ };
+ });
+ const atRecognition = await driver.run(parseBuildState(serializeBuildState(atGeneration.state)));
+ assert.equal(atRecognition.blocked[0]?.reason, "missing-provider");
+ assert.deepEqual(calls, { estimate: 1, generate: 1, recognize: 1, locate: 0, caption: 0 });
+
+ providers.registerProvider("runtime:whisperx", videoTypes.evidence, () => ({
+ value: { kind: "inline", value: { transcript: "what the fuck" } },
+ conformance: "exact",
+ delivery: "executed",
+ metadata: { adapter: "whisperx" },
+ }));
+ const completed = await driver.run(parseBuildState(serializeBuildState(atRecognition.state)));
+
+ assert.equal(completed.status, "complete");
+ assert.deepEqual(calls, { estimate: 1, generate: 1, recognize: 1, locate: 1, caption: 1 });
+ assert.deepEqual(completed.state.receipts.map((receipt) => receipt.fulfiller), [
+ "runtime:estimate",
+ "runtime:seedance-mini",
+ "runtime:whisperx",
+ ]);
+ assert.deepEqual(
+ completed.state.records.find((record) => record.id === "caption:root")?.value,
+ { kind: "inline", value: { text: "that was insane", startSec: 0.1, endSec: 0.72 } },
+ );
+});
diff --git a/packages/script/src/index.ts b/packages/script/src/index.ts
index 66af6162..39a567de 100644
--- a/packages/script/src/index.ts
+++ b/packages/script/src/index.ts
@@ -7,7 +7,13 @@ export {
scriptModuleRef,
scriptSurfaceImplementationDigest,
} from "./manifest.js";
-export { narrativeSourceMap, narrativeValue } from "./narrative.js";
+export {
+ narrativeSourceMap,
+ narrativeValue,
+ serializeCaption,
+ serializeDialogue,
+ serializeSpeech,
+} from "./narrative.js";
export { parseScript } from "./parser.js";
export { decodeScriptSurface } from "./surface.js";
export type * from "./types.js";
diff --git a/packages/script/src/manifest.ts b/packages/script/src/manifest.ts
index cdb9564a..bf84d853 100644
--- a/packages/script/src/manifest.ts
+++ b/packages/script/src/manifest.ts
@@ -132,17 +132,44 @@ export const narrativeSchema: ValueSchema = {
},
},
},
- captionAtoms: {
+ captionProjection: {
schema: {
- kind: "array",
- items: {
- kind: "object",
- fields: {
- id: { schema: stringSchema },
- display: { schema: { kind: "string" } },
- segmentId: { schema: stringSchema },
- startToken: { schema: integerSchema },
- endTokenExclusive: { schema: integerSchema },
+ kind: "object",
+ fields: {
+ contract: { schema: { kind: "literal", value: "svml.caption-projection@0" } },
+ text: { schema: { kind: "string" } },
+ regions: {
+ schema: {
+ kind: "array",
+ items: {
+ kind: "object",
+ fields: {
+ id: { schema: stringSchema },
+ display: { schema: { kind: "string" } },
+ segmentId: { schema: stringSchema },
+ startToken: { schema: integerSchema },
+ endTokenExclusive: { schema: integerSchema },
+ kind: { schema: { kind: "string", enum: ["identity", "alias", "hidden"] } },
+ refinements: {
+ schema: {
+ kind: "array",
+ items: {
+ kind: "object",
+ fields: {
+ id: { schema: stringSchema },
+ display: { schema: stringSchema },
+ displayStart: { schema: integerSchema },
+ displayEnd: { schema: integerSchema },
+ startToken: { schema: integerSchema },
+ endTokenExclusive: { schema: integerSchema },
+ relation: { schema: { kind: "literal", value: "exact" } },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
},
},
},
@@ -177,13 +204,12 @@ export const narrativeSchema: ValueSchema = {
},
},
},
- projections: {
+ serializations: {
schema: {
kind: "object",
fields: {
dialogue: { schema: { kind: "string" } },
speech: { schema: { kind: "string" } },
- caption: { schema: { kind: "string" } },
},
},
},
@@ -199,7 +225,7 @@ export const scriptManifest: ModuleManifest = {
{
name: narrativeType.name,
schema: narrativeSchema,
- description: "Authoritative narrative, projections and 2M + 2N semantic identities.",
+ description: "Authoritative narrative, pure serializations, Caption Projection and 2M + 2N identities.",
},
],
surfaces: [
diff --git a/packages/script/src/narrative.ts b/packages/script/src/narrative.ts
index 3491171b..de465549 100644
--- a/packages/script/src/narrative.ts
+++ b/packages/script/src/narrative.ts
@@ -61,18 +61,44 @@ export function narrativeValue(parsed: ParsedNarrative): CanonicalValue {
boundary: semanticBoundary(occurrence.boundary),
})),
})),
- captionAtoms: parsed.captionAtoms.map((atom) => ({
- id: atom.id,
- display: atom.display,
- segmentId: atom.segmentId,
- startToken: atom.startToken,
- endTokenExclusive: atom.endTokenExclusive,
- })),
+ captionProjection: {
+ contract: parsed.captionProjection.contract,
+ text: parsed.captionProjection.text,
+ regions: parsed.captionProjection.regions.map((region) => ({
+ id: region.id,
+ display: region.display,
+ segmentId: region.segmentId,
+ startToken: region.startToken,
+ endTokenExclusive: region.endTokenExclusive,
+ kind: region.kind,
+ refinements: region.refinements.map((refinement) => ({
+ id: refinement.id,
+ display: refinement.display,
+ displayStart: refinement.displayStart,
+ displayEnd: refinement.displayEnd,
+ startToken: refinement.startToken,
+ endTokenExclusive: refinement.endTokenExclusive,
+ relation: refinement.relation,
+ })),
+ })),
+ },
semanticIndex: parsed.semanticIndex,
- projections: parsed.projections,
+ serializations: parsed.serializations,
});
}
+export function serializeSpeech(parsed: ParsedNarrative): string {
+ return parsed.serializations.speech;
+}
+
+export function serializeDialogue(parsed: ParsedNarrative): string {
+ return parsed.serializations.dialogue;
+}
+
+export function serializeCaption(parsed: ParsedNarrative): string {
+ return parsed.captionProjection.text;
+}
+
export function narrativeSourceMap(recordId: string, parsed: ParsedNarrative): CanonicalValue {
return canonicalize({
format: "svml.script-source-map@0",
diff --git a/packages/script/src/parser.ts b/packages/script/src/parser.ts
index 7b385e7b..43b747fe 100644
--- a/packages/script/src/parser.ts
+++ b/packages/script/src/parser.ts
@@ -5,7 +5,8 @@ import type {
Affinity,
MarkerBoundary,
ParsedAtom,
- ParsedCaptionAtom,
+ ParsedCaptionRefinement,
+ ParsedCaptionRegion,
ParsedMoment,
ParsedNarrative,
ParsedSegment,
@@ -46,6 +47,89 @@ function normalizeWord(value: string): string {
.replace(/[^\p{L}\p{M}\p{N}]+/gu, "");
}
+type WordLexeme = {
+ readonly text: string;
+ readonly normalized: string;
+ readonly start: number;
+ readonly end: number;
+};
+
+function wordLexemes(value: string): WordLexeme[] {
+ const lexemes: WordLexeme[] = [];
+ WORD.lastIndex = 0;
+ let match: RegExpExecArray | null;
+ while ((match = WORD.exec(value))) {
+ const normalized = normalizeWord(match[0]);
+ if (!normalized) continue;
+ lexemes.push({
+ text: match[0],
+ normalized,
+ start: match.index,
+ end: match.index + match[0].length,
+ });
+ }
+ return lexemes;
+}
+
+function exactCaptionRefinements(
+ regionId: string,
+ display: string,
+ speechTokens: readonly ParsedToken[],
+): ParsedCaptionRefinement[] {
+ const displayWords = wordLexemes(display);
+ if (!displayWords.length || !speechTokens.length) return [];
+
+ let matches: Array<{ readonly displayIndex: number; readonly speechIndex: number }>;
+ if (
+ displayWords.length === speechTokens.length
+ && displayWords.every((word, index) => word.normalized === speechTokens[index]!.normalized)
+ ) {
+ matches = displayWords.map((_, index) => ({ displayIndex: index, speechIndex: index }));
+ } else {
+ const displayCounts = new Map<string, number>();
+ const speechCounts = new Map<string, number>();
+ for (const word of displayWords) displayCounts.set(word.normalized, (displayCounts.get(word.normalized) ?? 0) + 1);
+ for (const token of speechTokens) speechCounts.set(token.normalized, (speechCounts.get(token.normalized) ?? 0) + 1);
+ const candidates = displayWords.flatMap((word, displayIndex) => {
+ if (displayCounts.get(word.normalized) !== 1 || speechCounts.get(word.normalized) !== 1) return [];
+ const speechIndex = speechTokens.findIndex((token) => token.normalized === word.normalized);
+ return speechIndex < 0 ? [] : [{ displayIndex, speechIndex }];
+ });
+
+ // Keep a longest monotonic chain. Exact words that cross after an alias
+ // rewrite are individually plausible, but cannot form a temporal caption
+ // refinement without reversing display order.
+ const chains: Array<Array<{ readonly displayIndex: number; readonly speechIndex: number }>> = [];
+ for (let index = 0; index < candidates.length; index += 1) {
+ let best: Array<{ readonly displayIndex: number; readonly speechIndex: number }> = [];
+ for (let before = 0; before < index; before += 1) {
+ if (candidates[before]!.speechIndex < candidates[index]!.speechIndex && chains[before]!.length > best.length) {
+ best = chains[before]!;
+ }
+ }
+ chains.push([...best, candidates[index]!]);
+ }
+ matches = chains.reduce<typeof candidates>(
+ (best, chain) => chain.length > best.length ? chain : best,
+ [],
+ );
+ }
+
+ return matches.map(({ displayIndex, speechIndex }, index) => {
+ const word = displayWords[displayIndex]!;
+ const token = speechTokens[speechIndex]!;
+ return {
+ id: `${regionId}:exact:${index + 1}`,
+ display: word.text,
+ displayStart: word.start,
+ displayEnd: word.end,
+ startToken: token.index,
+ endTokenExclusive: token.index + 1,
+ relation: "exact",
+ };
+ });
+}
+
function cleanProjection(value: string): string {
return value
.replace(/\s+/gu, " ")
@@ -124,7 +208,7 @@ export function parseScript(
const segments: ParsedSegment[] = [];
const tokens: ParsedToken[] = [];
- const captionAtoms: ParsedCaptionAtom[] = [];
+ const captionRegions: ParsedCaptionRegion[] = [];
const selections = new Map<string, ParsedSelection["occurrences"] extends readonly (infer T)[] ? T[] : never>();
const moments = new Map<string, ParsedMoment["occurrences"] extends readonly (infer T)[] ? T[] : never>();
const openSelections = new Map<string, {
@@ -267,7 +351,13 @@ export function parseScript(
const literalString = (raw: string, start: number, dual = false): string =>
literalPieces(raw, start, dual).map((piece) => piece.value).join("");
- const addText = (speech: string, caption: string, start: number, end: number): void => {
+ const addText = (
+ speech: string,
+ caption: string,
+ start: number,
+ end: number,
+ captureCaptionRegion = true,
+ ): void => {
if (!current) {
if (speech.trim() || caption.trim()) {
fail("SCRIPT_TEXT_OUTSIDE_SEGMENT", "Natural-language text is only allowed inside a named Segment.", start);
@@ -307,6 +397,20 @@ export function parseScript(
tokenEndExclusive: tokens.length,
range: { start: sourceOffset + start, end: sourceOffset + end },
});
+ const display = cleanProjection(caption);
+ if (captureCaptionRegion && display && tokens.length > tokenStart) {
+ const id = `caption-region:${captionRegions.length + 1}`;
+ captionRegions.push({
+ id,
+ display,
+ segmentId: current.id,
+ startToken: tokenStart,
+ endTokenExclusive: tokens.length,
+ kind: "identity",
+ refinements: exactCaptionRefinements(id, display, tokens.slice(tokenStart)),
+ range: { start: sourceOffset + start, end: sourceOffset + end },
+ });
+ }
};
const consumeSpeechSide = (raw: string, absoluteStart: number, caption: string): void => {
@@ -316,7 +420,7 @@ export function parseScript(
let emittedCaption = false;
const addLiteral = (part: string, start: number): void => {
for (const piece of literalPieces(part, start, true)) {
- addText(piece.value, emittedCaption ? "" : caption, piece.start, piece.end);
+ addText(piece.value, emittedCaption ? "" : caption, piece.start, piece.end, false);
emittedCaption = true;
}
};
@@ -344,12 +448,17 @@ export function parseScript(
}
addLiteral(raw.slice(partStart), absoluteStart + partStart);
if (tokens.length > startToken) {
- captionAtoms.push({
- id: `caption-atom:${captionAtoms.length + 1}`,
- display: cleanProjection(caption),
+ const display = cleanProjection(caption);
+ const id = `caption-region:${captionRegions.length + 1}`;
+ const speechTokens = tokens.slice(startToken);
+ captionRegions.push({
+ id,
+ display,
segmentId: current!.id,
startToken,
endTokenExclusive: tokens.length,
+ kind: display ? (cleanProjection(speechTokens.map((token) => token.text).join(" ")) === display ? "identity" : "alias") : "hidden",
+ refinements: exactCaptionRefinements(id, display, speechTokens),
range: { start: sourceOffset + absoluteStart, end: sourceOffset + absoluteStart + raw.length },
});
}
@@ -590,16 +699,19 @@ export function parseScript(
turns,
selections: [...selections].sort(([left], [right]) => left.localeCompare(right)).map(([id, occurrences]) => ({ id, occurrences })),
moments: [...moments].sort(([left], [right]) => left.localeCompare(right)).map(([id, occurrences]) => ({ id, occurrences })),
- captionAtoms,
+ captionProjection: {
+ contract: "svml.caption-projection@0",
+ text: captionSegments.join("\n"),
+ regions: captionRegions,
+ },
semanticIndex: {
contract: "svml.semantic-index@0",
anchors,
digest: digestOf(semanticPayload),
},
- projections: {
+ serializations: {
dialogue: dialogueTurns.join("\n"),
speech: speechSegments.join("\n"),
- caption: captionSegments.join("\n"),
},
};
}
diff --git a/packages/script/src/types.ts b/packages/script/src/types.ts
index 3bbd10ff..f96ae2d5 100644
--- a/packages/script/src/types.ts
+++ b/packages/script/src/types.ts
@@ -89,15 +89,33 @@ export type ParsedMoment = {
readonly occurrences: readonly ParsedMomentOccurrence[];
};
-export type ParsedCaptionAtom = {
+export type ParsedCaptionRefinement = {
+ readonly id: string;
+ readonly display: string;
+ readonly displayStart: number;
+ readonly displayEnd: number;
+ readonly startToken: number;
+ readonly endTokenExclusive: number;
+ readonly relation: "exact";
+};
+
+export type ParsedCaptionRegion = {
readonly id: string;
readonly display: string;
readonly segmentId: string;
readonly startToken: number;
readonly endTokenExclusive: number;
+ readonly kind: "identity" | "alias" | "hidden";
+ readonly refinements: readonly ParsedCaptionRefinement[];
readonly range: SourceRange;
};
+export type CaptionProjection = {
+ readonly contract: "svml.caption-projection@0";
+ readonly text: string;
+ readonly regions: readonly ParsedCaptionRegion[];
+};
+
export type SemanticAnchor = {
readonly id: string;
readonly kind: "segment-start" | "token-start" | "token-end" | "segment-end";
@@ -112,16 +130,15 @@ export type ParsedNarrative = {
readonly turns: readonly ParsedTurn[];
readonly selections: readonly ParsedSelection[];
readonly moments: readonly ParsedMoment[];
- readonly captionAtoms: readonly ParsedCaptionAtom[];
+ readonly captionProjection: CaptionProjection;
readonly semanticIndex: {
readonly contract: "svml.semantic-index@0";
readonly anchors: readonly SemanticAnchor[];
readonly digest: string;
};
- readonly projections: {
+ readonly serializations: {
readonly dialogue: string;
readonly speech: string;
- readonly caption: string;
};
};
diff --git a/packages/script/test/script.test.ts b/packages/script/test/script.test.ts
index b5151f68..82d354aa 100644
--- a/packages/script/test/script.test.ts
+++ b/packages/script/test/script.test.ts
@@ -7,6 +7,9 @@ import {
formatScript,
narrativeValue,
parseScript,
+ serializeCaption,
+ serializeDialogue,
+ serializeSpeech,
} from "@svml/script";
test("named blocks are Segments and Role Cues do not depend on line breaks", () => {
@@ -29,7 +32,7 @@ test("named blocks are Segments and Role Cues do not depend on line breaks", ()
assert.equal(digestOf(narrativeValue(compact)), digestOf(narrativeValue(multiline)));
assert.deepEqual(compact.segments.map((segment) => segment.id), ["opening", "pause"]);
assert.deepEqual(compact.turns.map((turn) => turn.role), ["ALICE", "BOB"]);
- assert.equal(compact.projections.dialogue, "ALICE: Hello there.\nBOB: Good morning.");
+ assert.equal(serializeDialogue(compact), "ALICE: Hello there.\nBOB: Good morning.");
});
test("Script produces exactly 2M + 2N independent semantic anchors", () => {
@@ -58,12 +61,37 @@ test("selections, moments and Dual Text preserve separate semantic projections",
@/whole~`,
);
- assert.equal(parsed.projections.speech, "I really laughed my ass off there.");
- assert.equal(parsed.projections.caption, "I really laughed there.");
- assert.equal(parsed.projections.dialogue, "BOB: I really laughed my ass off there.");
+ assert.equal(serializeSpeech(parsed), "I really laughed my ass off there.");
+ assert.equal(serializeCaption(parsed), "I really laughed there.");
+ assert.equal(serializeDialogue(parsed), "BOB: I really laughed my ass off there.");
assert.deepEqual(parsed.selections.map((selection) => selection.id), ["whole"]);
assert.deepEqual(parsed.moments.map((moment) => moment.id), ["beat"]);
- assert.equal(parsed.captionAtoms[0]?.display, "laughed");
+ assert.equal(parsed.captionProjection.regions.find((region) => region.kind === "alias")?.display, "laughed");
+});
+
+test("Caption Projection owns speech ranges without inventing alias word timing", () => {
+ const parsed = parseScript(
+ "caption.svml",
+ `<line>
+ <test this | test this>
+ <15% off | fifteen percent off>
+ <that was insane | what the fuck>
+ <what the— | what the fuck>
+ < | um>
+ </line>`,
+ );
+ const [identity, discount, replacement, interrupted, hidden] = parsed.captionProjection.regions;
+
+ assert.equal(parsed.captionProjection.contract, "svml.caption-projection@0");
+ assert.equal(identity?.kind, "identity");
+ assert.deepEqual(identity?.refinements.map((item) => item.display), ["test", "this"]);
+ assert.equal(discount?.kind, "alias");
+ assert.deepEqual(discount?.refinements.map((item) => item.display), ["off"]);
+ assert.deepEqual(replacement?.refinements, []);
+ assert.deepEqual(interrupted?.refinements.map((item) => item.display), ["what", "the"]);
+ assert.equal(hidden?.kind, "hidden");
+ assert.equal(hidden?.display, "");
+ assert.equal(hidden?.endTokenExclusive! - hidden?.startToken!, 1);
});
test("mismatched named Segment closes are rejected", () => {
diff --git a/packages/speech-align/README.md b/packages/speech-align/README.md
new file mode 100644
index 00000000..0051fbad
--- /dev/null
+++ b/packages/speech-align/README.md
@@ -0,0 +1,15 @@
+# `@svml/speech-align`
+
+Provider-neutral deterministic timing locator for one aligned transcript. The package does not
+invoke Python, read audio, call a speech provider, use an LLM, infer speakers or retain multiple
+candidate paths. A runtime adapter preserves its raw provider artifact and supplies the normalized
+word, character, score, speech-activity and Segment evidence used here.
+
+The locator aligns each known Script Segment independently. A bounded monotonic M:N dynamic
+program accepts exact, split, merge, replacement, source-omission and evidence-insertion groups.
+Within a selected group, timed evidence characters determine Script token boundaries; word times,
+speech-activity bounds and deterministic zero-width estimates are fallbacks in that order. The result contains
+one timing for every Script token and all `2M + 2N` semantic anchors.
+
+Caption display is outside this package. `@svml/caption` composes the resulting complete speech
+map with Script's authored Caption Projection and keeps presentation estimates local to the track.
diff --git a/packages/whisperx/package.json b/packages/speech-align/package.json
similarity index 86%
rename from packages/whisperx/package.json
rename to packages/speech-align/package.json
index 849dc588..3466e5b1 100644
--- a/packages/whisperx/package.json
+++ b/packages/speech-align/package.json
@@ -1,5 +1,5 @@
{
- "name": "@svml/whisperx",
+ "name": "@svml/speech-align",
"version": "0.0.0-dev",
"private": true,
"type": "module",
diff --git a/packages/whisperx/src/align.ts b/packages/speech-align/src/align.ts
similarity index 97%
rename from packages/whisperx/src/align.ts
rename to packages/speech-align/src/align.ts
index 006bd26d..ba7f9563 100644
--- a/packages/whisperx/src/align.ts
+++ b/packages/speech-align/src/align.ts
@@ -4,7 +4,7 @@ import { editDistance, normalizeForAlignment } from "./normalize.js";
import type {
AlignmentGroup,
AlignmentRelation,
- WhisperXWordEvidence,
+ SpeechWordEvidence,
} from "./types.js";
const EPSILON = 1e-9;
@@ -25,7 +25,7 @@ function comparisonText(values: readonly { readonly text: string }[]): string {
return values.map((value) => normalizeForAlignment(value.text)).join("");
}
-function evidenceReliability(words: readonly WhisperXWordEvidence[]): number {
+function evidenceReliability(words: readonly SpeechWordEvidence[]): number {
const scores = words
.map((word) => word.score)
.filter((score): score is number => score !== undefined && Number.isFinite(score));
@@ -36,7 +36,7 @@ function evidenceReliability(words: readonly WhisperXWordEvidence[]): number {
function exactWordLcs(
source: readonly ParsedToken[],
- evidence: readonly WhisperXWordEvidence[],
+ evidence: readonly SpeechWordEvidence[],
): number {
const rows = Array.from({ length: source.length + 1 }, () =>
Array<number>(evidence.length + 1).fill(0));
@@ -65,7 +65,7 @@ function relation(
function pairedCost(
source: readonly ParsedToken[],
- evidence: readonly WhisperXWordEvidence[],
+ evidence: readonly SpeechWordEvidence[],
): number {
const sourceText = comparisonText(source);
const evidenceText = comparisonText(evidence);
@@ -109,7 +109,7 @@ function extend(
export function alignWordGroups(
sourceSegmentId: string,
source: readonly ParsedToken[],
- evidence: readonly WhisperXWordEvidence[],
+ evidence: readonly SpeechWordEvidence[],
maxGroupSize = DEFAULT_MAX_GROUP,
): AlignmentGroup[] {
const rows = Array.from({ length: source.length + 1 }, () =>
diff --git a/packages/speech-align/src/error.ts b/packages/speech-align/src/error.ts
new file mode 100644
index 00000000..19d966ca
--- /dev/null
+++ b/packages/speech-align/src/error.ts
@@ -0,0 +1,9 @@
+export class SpeechAlignmentError extends Error {
+ readonly code: string;
+
+ constructor(code: string, message: string) {
+ super(message);
+ this.name = "SpeechAlignmentError";
+ this.code = code;
+ }
+}
diff --git a/packages/whisperx/src/index.ts b/packages/speech-align/src/index.ts
similarity index 51%
rename from packages/whisperx/src/index.ts
rename to packages/speech-align/src/index.ts
index b6ad78d4..d13913e8 100644
--- a/packages/whisperx/src/index.ts
+++ b/packages/speech-align/src/index.ts
@@ -1,6 +1,5 @@
export { alignWordGroups } from "./align.js";
-export { deriveCaptionAtomTiming } from "./caption.js";
-export { WhisperXAlignmentError } from "./error.js";
-export { locateWhisperXTiming } from "./locate.js";
+export { SpeechAlignmentError } from "./error.js";
+export { locateSpeechTiming } from "./locate.js";
export { alignCharacters, alignmentCharacters, normalizeForAlignment } from "./normalize.js";
export type * from "./types.js";
diff --git a/packages/whisperx/src/locate.ts b/packages/speech-align/src/locate.ts
similarity index 82%
rename from packages/whisperx/src/locate.ts
rename to packages/speech-align/src/locate.ts
index d4137905..030aab6b 100644
--- a/packages/whisperx/src/locate.ts
+++ b/packages/speech-align/src/locate.ts
@@ -2,7 +2,7 @@ import { digestOf } from "@svml/core";
import type { ParsedNarrative, ParsedToken } from "@svml/script";
import { alignWordGroups } from "./align.js";
-import { WhisperXAlignmentError } from "./error.js";
+import { SpeechAlignmentError } from "./error.js";
import { alignCharacters, alignmentCharacters } from "./normalize.js";
import type {
AlignmentGroup,
@@ -11,10 +11,10 @@ import type {
TimedSpeechSegment,
TimedSpeechToken,
TimingQuality,
- WhisperXCharEvidence,
- WhisperXEvidence,
- WhisperXSegmentEvidence,
- WhisperXWordEvidence,
+ AlignedTranscriptEvidence,
+ AlignedTranscriptSegment,
+ SpeechCharacterEvidence,
+ SpeechWordEvidence,
} from "./types.js";
const EPSILON = 1e-6;
@@ -34,7 +34,7 @@ type TimedEvidenceChar = {
};
function fail(code: string, message: string): never {
- throw new WhisperXAlignmentError(code, message);
+ throw new SpeechAlignmentError(code, message);
}
function finite(value: number | undefined): value is number {
@@ -43,33 +43,33 @@ function finite(value: number | undefined): value is number {
function validateWindow(start: number, end: number, limit: number, label: string): void {
if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end < start || end > limit + EPSILON) {
- fail("WHISPERX_WINDOW", `${label} has invalid time window ${start}..${end}.`);
+ fail("SPEECH_WINDOW", `${label} has invalid time window ${start}..${end}.`);
}
}
-function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidence): void {
- if (evidence.contract !== "svml.whisperx-evidence@0") {
- fail("WHISPERX_CONTRACT", `Unsupported WhisperX evidence contract ${evidence.contract}.`);
+function validateEvidence(narrative: ParsedNarrative, evidence: AlignedTranscriptEvidence): void {
+ if (evidence.contract !== "svml.aligned-transcript-evidence@0") {
+ fail("SPEECH_CONTRACT", `Unsupported aligned-transcript contract ${evidence.contract}.`);
}
if (!Number.isFinite(evidence.durationSec) || evidence.durationSec <= 0) {
- fail("WHISPERX_DURATION", "WhisperX evidence duration must be positive and finite.");
+ fail("SPEECH_DURATION", "Aligned-transcript duration must be positive and finite.");
}
const expected = new Set(narrative.segments.map((segment) => segment.id));
const seen = new Set<string>();
for (const segment of evidence.segments) {
if (!expected.has(segment.sourceSegmentId)) {
- fail("WHISPERX_SEGMENT_UNKNOWN", `WhisperX evidence references unknown Segment ${segment.sourceSegmentId}.`);
+ fail("SPEECH_SEGMENT_UNKNOWN", `Aligned transcript references unknown Segment ${segment.sourceSegmentId}.`);
}
if (seen.has(segment.sourceSegmentId)) {
- fail("WHISPERX_SEGMENT_DUPLICATE", `WhisperX evidence repeats Segment ${segment.sourceSegmentId}.`);
+ fail("SPEECH_SEGMENT_DUPLICATE", `Aligned transcript repeats Segment ${segment.sourceSegmentId}.`);
}
seen.add(segment.sourceSegmentId);
validateWindow(segment.startSec, segment.endSec, evidence.durationSec, `Segment ${segment.sourceSegmentId}`);
let previousEnd = segment.startSec;
for (const [index, word] of segment.words.entries()) {
- if (typeof word.text !== "string") fail("WHISPERX_WORD_TEXT", `Word ${index + 1} has no text.`);
+ if (typeof word.text !== "string") fail("SPEECH_WORD_TEXT", `Word ${index + 1} has no text.`);
if (finite(word.startSec) !== finite(word.endSec)) {
- fail("WHISPERX_WORD_PARTIAL_TIME", `Word ${index + 1} must provide both start and end or neither.`);
+ fail("SPEECH_WORD_PARTIAL_TIME", `Word ${index + 1} must provide both start and end or neither.`);
}
if (finite(word.startSec) && finite(word.endSec)) {
validateWindow(word.startSec, word.endSec, evidence.durationSec, `Word ${index + 1}`);
@@ -78,20 +78,20 @@ function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidence
|| word.endSec > segment.endSec + EPSILON
|| word.startSec < previousEnd - EPSILON
) {
- fail("WHISPERX_WORD_ORDER", `Word ${index + 1} is outside or overlaps its Segment window.`);
+ fail("SPEECH_WORD_ORDER", `Word ${index + 1} is outside or overlaps its Segment window.`);
}
previousEnd = word.endSec;
}
if (word.score !== undefined && (!Number.isFinite(word.score) || word.score < 0 || word.score > 1)) {
- fail("WHISPERX_SCORE", `Word ${index + 1} score must be between zero and one.`);
+ fail("SPEECH_SCORE", `Word ${index + 1} score must be between zero and one.`);
}
}
for (const [index, char] of segment.chars.entries()) {
if (!Number.isInteger(char.wordIndex) || char.wordIndex < 0 || char.wordIndex >= segment.words.length) {
- fail("WHISPERX_CHAR_WORD", `Character ${index + 1} has an invalid wordIndex.`);
+ fail("SPEECH_CHAR_WORD", `Character ${index + 1} has an invalid wordIndex.`);
}
if (finite(char.startSec) !== finite(char.endSec)) {
- fail("WHISPERX_CHAR_PARTIAL_TIME", `Character ${index + 1} must provide both start and end or neither.`);
+ fail("SPEECH_CHAR_PARTIAL_TIME", `Character ${index + 1} must provide both start and end or neither.`);
}
if (finite(char.startSec) && finite(char.endSec)) {
validateWindow(char.startSec, char.endSec, evidence.durationSec, `Character ${index + 1}`);
@@ -102,28 +102,28 @@ function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidence
|| (finite(word.startSec) && char.startSec < word.startSec - EPSILON)
|| (finite(word.endSec) && char.endSec > word.endSec + EPSILON)
) {
- fail("WHISPERX_CHAR_WINDOW", `Character ${index + 1} falls outside its Word or Segment window.`);
+ fail("SPEECH_CHAR_WINDOW", `Character ${index + 1} falls outside its Word or Segment window.`);
}
}
if (char.score !== undefined && (!Number.isFinite(char.score) || char.score < 0 || char.score > 1)) {
- fail("WHISPERX_SCORE", `Character ${index + 1} score must be between zero and one.`);
+ fail("SPEECH_SCORE", `Character ${index + 1} score must be between zero and one.`);
}
}
let previousVadEnd = segment.startSec;
- for (const [index, span] of (segment.vadSpans ?? []).entries()) {
+ for (const [index, span] of (segment.speechActivity ?? []).entries()) {
validateWindow(span.startSec, span.endSec, evidence.durationSec, `VAD span ${index + 1}`);
if (
span.startSec < segment.startSec - EPSILON
|| span.endSec > segment.endSec + EPSILON
|| span.startSec < previousVadEnd - EPSILON
) {
- fail("WHISPERX_VAD_WINDOW", `VAD span ${index + 1} falls outside Segment ${segment.sourceSegmentId}.`);
+ fail("SPEECH_ACTIVITY_WINDOW", `Speech activity ${index + 1} falls outside Segment ${segment.sourceSegmentId}.`);
}
previousVadEnd = span.endSec;
}
}
for (const id of expected) {
- if (!seen.has(id)) fail("WHISPERX_SEGMENT_MISSING", `WhisperX evidence is missing Segment ${id}.`);
+ if (!seen.has(id)) fail("SPEECH_SEGMENT_MISSING", `Aligned transcript is missing Segment ${id}.`);
}
const segmentsById = new Map(evidence.segments.map((segment) => [segment.sourceSegmentId, segment]));
let previousSegmentStart = -Infinity;
@@ -134,7 +134,7 @@ function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidence
segment.startSec < previousSegmentStart - EPSILON
|| segment.endSec < previousSegmentEnd - EPSILON
) {
- fail("WHISPERX_SEGMENT_ORDER", `Segment ${segment.sourceSegmentId} violates Script source order.`);
+ fail("SPEECH_SEGMENT_ORDER", `Segment ${segment.sourceSegmentId} violates Script source order.`);
}
previousSegmentStart = segment.startSec;
previousSegmentEnd = segment.endSec;
@@ -144,7 +144,7 @@ function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidence
function syntheticCharTimes(
characters: readonly string[],
wordIndex: number,
- word: WhisperXWordEvidence,
+ word: SpeechWordEvidence,
): TimedEvidenceChar[] {
const hasWindow = finite(word.startSec) && finite(word.endSec);
return characters.map((value, index) => ({
@@ -160,9 +160,9 @@ function syntheticCharTimes(
}
function timedCharsForWord(
- word: WhisperXWordEvidence,
+ word: SpeechWordEvidence,
wordIndex: number,
- rawChars: readonly WhisperXCharEvidence[],
+ rawChars: readonly SpeechCharacterEvidence[],
): TimedEvidenceChar[] {
const expected = alignmentCharacters(word.text);
if (!expected.length) return [];
@@ -192,11 +192,11 @@ function timedCharsForWord(
return fallback;
}
-function timedChars(segment: WhisperXSegmentEvidence): TimedEvidenceChar[] {
+function timedChars(segment: AlignedTranscriptSegment): TimedEvidenceChar[] {
return segment.words.flatMap((word, index) => timedCharsForWord(word, index, segment.chars));
}
-function scoredWords(segment: WhisperXSegmentEvidence): WhisperXWordEvidence[] {
+function scoredWords(segment: AlignedTranscriptSegment): SpeechWordEvidence[] {
return segment.words.map((word, wordIndex) => {
const scores = [
...(word.score === undefined ? [] : [word.score]),
@@ -212,7 +212,7 @@ function scoredWords(segment: WhisperXSegmentEvidence): WhisperXWordEvidence[] {
function groupWindow(
group: AlignmentGroup,
- words: readonly WhisperXWordEvidence[],
+ words: readonly SpeechWordEvidence[],
chars: readonly TimedEvidenceChar[],
): { readonly startSec: number; readonly endSec: number } | undefined {
const wordRun = words.slice(group.evidenceWordStart, group.evidenceWordEndExclusive);
@@ -233,7 +233,7 @@ function groupWindow(
function locatePairedGroup(
group: AlignmentGroup,
source: readonly ParsedToken[],
- words: readonly WhisperXWordEvidence[],
+ words: readonly SpeechWordEvidence[],
chars: readonly TimedEvidenceChar[],
output: Array<MutableTiming | undefined>,
): void {
@@ -276,8 +276,8 @@ function locatePairedGroup(
}
}
-function speechBounds(segment: WhisperXSegmentEvidence): { readonly start: number; readonly end: number } {
- const spans = segment.vadSpans ?? [];
+function speechBounds(segment: AlignedTranscriptSegment): { readonly start: number; readonly end: number } {
+ const spans = segment.speechActivity ?? [];
return spans.length
? { start: spans[0]!.startSec, end: spans.at(-1)!.endSec }
: { start: segment.startSec, end: segment.endSec };
@@ -317,7 +317,7 @@ function assertMonotonic(tokens: readonly TimedSpeechToken[], segmentId: string)
let previousEnd = -Infinity;
for (const token of tokens) {
if (token.startSec < previousEnd - EPSILON || token.endSec < token.startSec - EPSILON) {
- fail("WHISPERX_MAP_ORDER", `Located token ${token.tokenId} violates time order in Segment ${segmentId}.`);
+ fail("SPEECH_MAP_ORDER", `Located token ${token.tokenId} violates time order in Segment ${segmentId}.`);
}
previousEnd = token.endSec;
}
@@ -327,9 +327,9 @@ function mapDigest(map: Omit<CompleteSpeechTimeMap, "mapDigest">): string {
return digestOf(map);
}
-export function locateWhisperXTiming(
+export function locateSpeechTiming(
narrative: ParsedNarrative,
- evidence: WhisperXEvidence,
+ evidence: AlignedTranscriptEvidence,
): CompleteSpeechTimeMap {
validateEvidence(narrative, evidence);
const evidenceBySegment = new Map(evidence.segments.map((segment) => [segment.sourceSegmentId, segment]));
@@ -378,7 +378,7 @@ export function locateWhisperXTiming(
: { identity: anchor.id, timeSec: token.endSec, quality: token.endQuality };
});
const payload = {
- contract: "svml.whisperx-speech-time-map@0" as const,
+ contract: "svml.speech-time-map@0" as const,
semanticIndexDigest: narrative.semanticIndex.digest,
durationSec: evidence.durationSec,
segments: timedSegments,
diff --git a/packages/whisperx/src/normalize.ts b/packages/speech-align/src/normalize.ts
similarity index 100%
rename from packages/whisperx/src/normalize.ts
rename to packages/speech-align/src/normalize.ts
diff --git a/packages/whisperx/src/types.ts b/packages/speech-align/src/types.ts
similarity index 66%
rename from packages/whisperx/src/types.ts
rename to packages/speech-align/src/types.ts
index 11b42328..d312b99a 100644
--- a/packages/whisperx/src/types.ts
+++ b/packages/speech-align/src/types.ts
@@ -1,13 +1,13 @@
export type TimingQuality = "measured" | "derived" | "estimated";
-export type WhisperXWordEvidence = {
+export type SpeechWordEvidence = {
readonly text: string;
readonly startSec?: number;
readonly endSec?: number;
readonly score?: number;
};
-export type WhisperXCharEvidence = {
+export type SpeechCharacterEvidence = {
readonly char: string;
readonly wordIndex: number;
readonly startSec?: number;
@@ -15,29 +15,29 @@ export type WhisperXCharEvidence = {
readonly score?: number;
};
-export type WhisperXVadSpan = {
+export type SpeechActivitySpan = {
readonly startSec: number;
readonly endSec: number;
};
/**
- * Normalized evidence from one ordinary WhisperX transcription + alignment run.
- * A host adapter keeps the raw provider result as an artifact and lowers the
+ * Provider-neutral evidence produced by a speech-recognition adapter. The
+ * adapter keeps its raw provider result as an artifact and lowers only the
* fields used by this deterministic locator into this value.
*/
-export type WhisperXSegmentEvidence = {
+export type AlignedTranscriptSegment = {
readonly sourceSegmentId: string;
readonly startSec: number;
readonly endSec: number;
- readonly words: readonly WhisperXWordEvidence[];
- readonly chars: readonly WhisperXCharEvidence[];
- readonly vadSpans?: readonly WhisperXVadSpan[];
+ readonly words: readonly SpeechWordEvidence[];
+ readonly chars: readonly SpeechCharacterEvidence[];
+ readonly speechActivity?: readonly SpeechActivitySpan[];
};
-export type WhisperXEvidence = {
- readonly contract: "svml.whisperx-evidence@0";
+export type AlignedTranscriptEvidence = {
+ readonly contract: "svml.aligned-transcript-evidence@0";
readonly durationSec: number;
- readonly segments: readonly WhisperXSegmentEvidence[];
+ readonly segments: readonly AlignedTranscriptSegment[];
};
export type AlignmentRelation =
@@ -81,7 +81,7 @@ export type SemanticTimePoint = {
};
export type CompleteSpeechTimeMap = {
- readonly contract: "svml.whisperx-speech-time-map@0";
+ readonly contract: "svml.speech-time-map@0";
readonly semanticIndexDigest: string;
readonly durationSec: number;
readonly segments: readonly TimedSpeechSegment[];
@@ -90,14 +90,3 @@ export type CompleteSpeechTimeMap = {
readonly groups: readonly AlignmentGroup[];
readonly mapDigest: string;
};
-
-export type TimedCaptionAtom = {
- readonly id: string;
- readonly display: string;
- readonly segmentId: string;
- readonly sourceTokenIds: readonly string[];
- readonly startSec: number;
- readonly endSec: number;
- readonly startQuality: TimingQuality;
- readonly endQuality: TimingQuality;
-};
diff --git a/packages/whisperx/test/whisperx.test.ts b/packages/speech-align/test/speech-align.test.ts
similarity index 71%
rename from packages/whisperx/test/whisperx.test.ts
rename to packages/speech-align/test/speech-align.test.ts
index 70022338..6d4f21a6 100644
--- a/packages/whisperx/test/whisperx.test.ts
+++ b/packages/speech-align/test/speech-align.test.ts
@@ -3,27 +3,26 @@ import test from "node:test";
import { parseScript } from "@svml/script";
import {
- WhisperXAlignmentError,
- deriveCaptionAtomTiming,
- locateWhisperXTiming,
-} from "@svml/whisperx";
+ SpeechAlignmentError,
+ locateSpeechTiming,
+} from "@svml/speech-align";
import type {
- WhisperXCharEvidence,
- WhisperXEvidence,
- WhisperXWordEvidence,
-} from "@svml/whisperx";
+ AlignedTranscriptEvidence,
+ SpeechCharacterEvidence,
+ SpeechWordEvidence,
+} from "@svml/speech-align";
function evidence(args: {
readonly segmentId?: string;
readonly durationSec?: number;
readonly startSec?: number;
readonly endSec?: number;
- readonly words: readonly WhisperXWordEvidence[];
- readonly chars?: readonly WhisperXCharEvidence[];
+ readonly words: readonly SpeechWordEvidence[];
+ readonly chars?: readonly SpeechCharacterEvidence[];
readonly vad?: readonly { readonly startSec: number; readonly endSec: number }[];
-}): WhisperXEvidence {
+}): AlignedTranscriptEvidence {
return {
- contract: "svml.whisperx-evidence@0",
+ contract: "svml.aligned-transcript-evidence@0",
durationSec: args.durationSec ?? 2,
segments: [
{
@@ -32,7 +31,7 @@ function evidence(args: {
endSec: args.endSec ?? 2,
words: args.words,
chars: args.chars ?? [],
- ...(args.vad === undefined ? {} : { vadSpans: args.vad }),
+ ...(args.vad === undefined ? {} : { speechActivity: args.vad }),
},
],
};
@@ -43,7 +42,7 @@ function characters(
starts: readonly number[],
ends: readonly number[],
wordIndex = 0,
-): WhisperXCharEvidence[] {
+): SpeechCharacterEvidence[] {
return [...text].map((char, index) => ({
char,
wordIndex,
@@ -53,9 +52,9 @@ function characters(
}));
}
-test("exact WhisperX words cover every Script and Segment anchor", () => {
+test("exact transcript words cover every Script and Segment anchor", () => {
const narrative = parseScript("exact.svml", "<line>Hello world.</line>");
- const map = locateWhisperXTiming(narrative, evidence({
+ const map = locateSpeechTiming(narrative, evidence({
words: [
{ text: "Hello", startSec: 0.1, endSec: 0.4, score: 0.97 },
{ text: "world", startSec: 0.5, endSec: 0.9, score: 0.96 },
@@ -75,9 +74,9 @@ test("exact WhisperX words cover every Script and Segment anchor", () => {
assert.match(map.mapDigest, /^sha256:[a-f0-9]{64}$/u);
});
-test("M:1 uses WhisperX character times instead of dividing a merged word by length", () => {
+test("M:1 uses evidence character times instead of dividing a merged word by length", () => {
const narrative = parseScript("merge.svml", "<line>can not</line>");
- const map = locateWhisperXTiming(narrative, evidence({
+ const map = locateSpeechTiming(narrative, evidence({
endSec: 1,
durationSec: 1,
words: [{ text: "cannot", startSec: 0.1, endSec: 0.78, score: 0.93 }],
@@ -98,9 +97,9 @@ test("M:1 uses WhisperX character times instead of dividing a merged word by len
);
});
-test("1:N wraps all WhisperX words in one Script token", () => {
+test("1:N wraps all evidence words in one Script token", () => {
const narrative = parseScript("split.svml", "<line>website</line>");
- const map = locateWhisperXTiming(narrative, evidence({
+ const map = locateSpeechTiming(narrative, evidence({
words: [
{ text: "web", startSec: 0.2, endSec: 0.45, score: 0.9 },
{ text: "site", startSec: 0.5, endSec: 0.82, score: 0.91 },
@@ -116,7 +115,7 @@ test("1:N wraps all WhisperX words in one Script token", () => {
test("a recognized filler stays an insertion and does not absorb neighboring Script words", () => {
const narrative = parseScript("insertion.svml", "<line>I really like it.</line>");
- const map = locateWhisperXTiming(narrative, evidence({
+ const map = locateSpeechTiming(narrative, evidence({
words: [
{ text: "I", startSec: 0.1, endSec: 0.2, score: 0.98 },
{ text: "uh", startSec: 0.24, endSec: 0.34, score: 0.88 },
@@ -146,7 +145,7 @@ test("a recognized filler stays an insertion and does not absorb neighboring Scr
test("an omitted Script word gets a zero-width estimated point between measured neighbors", () => {
const narrative = parseScript("omission.svml", "<line>This is very good.</line>");
- const map = locateWhisperXTiming(narrative, evidence({
+ const map = locateSpeechTiming(narrative, evidence({
words: [
{ text: "This", startSec: 0.1, endSec: 0.25, score: 0.98 },
{ text: "is", startSec: 0.3, endSec: 0.4, score: 0.97 },
@@ -168,7 +167,7 @@ test("an omitted Script word gets a zero-width estimated point between measured
test("VAD bounds contain estimates when an entire Script Segment has no recognized words", () => {
const narrative = parseScript("vad.svml", "<line>One two.</line>");
- const map = locateWhisperXTiming(narrative, evidence({
+ const map = locateSpeechTiming(narrative, evidence({
words: [],
vad: [{ startSec: 0.4, endSec: 1.2 }],
}));
@@ -182,44 +181,10 @@ test("VAD bounds contain estimates when an entire Script Segment has no recogniz
);
});
-test("Dual Text display inherits one speech envelope without linear display-word timing", () => {
- const narrative = parseScript(
- "dual.svml",
- "<line>That was <really funny | absolutely hilarious>.</line>",
- );
- const map = locateWhisperXTiming(narrative, evidence({
- words: [
- { text: "That", startSec: 0.1, endSec: 0.22 },
- { text: "was", startSec: 0.25, endSec: 0.34 },
- { text: "absolutely", startSec: 0.4, endSec: 0.72 },
- { text: "hilarious", startSec: 0.75, endSec: 1.1 },
- ],
- }));
- const captions = deriveCaptionAtomTiming(narrative, map);
-
- assert.equal(captions.length, 1);
- assert.deepEqual(
- {
- display: captions[0]?.display,
- sourceTokenIds: captions[0]?.sourceTokenIds,
- startSec: captions[0]?.startSec,
- endSec: captions[0]?.endSec,
- startQuality: captions[0]?.startQuality,
- },
- {
- display: "really funny",
- sourceTokenIds: narrative.tokens.slice(2, 4).map((token) => token.id),
- startSec: 0.4,
- endSec: 1.1,
- startQuality: "derived",
- },
- );
-});
-
test("multiple Script Segments stay independent even when evidence records arrive out of order", () => {
const narrative = parseScript("segments.svml", "<one>Hello.</one><two>Goodbye.</two>");
- const map = locateWhisperXTiming(narrative, {
- contract: "svml.whisperx-evidence@0",
+ const map = locateSpeechTiming(narrative, {
+ contract: "svml.aligned-transcript-evidence@0",
durationSec: 2,
segments: [
{
@@ -246,15 +211,15 @@ test("multiple Script Segments stay independent even when evidence records arriv
assert.deepEqual(map.groups.map((group) => group.sourceSegmentId), ["one", "two"]);
});
-test("invalid overlapping WhisperX word windows fail instead of producing a reversed map", () => {
+test("invalid overlapping evidence word windows fail instead of producing a reversed map", () => {
const narrative = parseScript("bad.svml", "<line>one two</line>");
assert.throws(
- () => locateWhisperXTiming(narrative, evidence({
+ () => locateSpeechTiming(narrative, evidence({
words: [
{ text: "one", startSec: 0.1, endSec: 0.5 },
{ text: "two", startSec: 0.4, endSec: 0.8 },
],
})),
- (error: unknown) => error instanceof WhisperXAlignmentError && error.code === "WHISPERX_WORD_ORDER",
+ (error: unknown) => error instanceof SpeechAlignmentError && error.code === "SPEECH_WORD_ORDER",
);
});
diff --git a/packages/whisperx/README.md b/packages/whisperx/README.md
deleted file mode 100644
index beb2ccfe..00000000
--- a/packages/whisperx/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-# `@svml/whisperx`
-
-Deterministic timing locator for the result of one ordinary WhisperX run. The package does not
-invoke Python, read audio, rerun forced alignment against Script, use an LLM, infer speakers or
-retain multiple candidate paths. A Host adapter preserves the raw WhisperX artifact and supplies
-the normalized word, character, score, VAD and Segment fields used here.
-
-The locator aligns each known Script Segment independently. A bounded monotonic M:N dynamic
-program accepts exact, split, merge, replacement, source-omission and evidence-insertion groups.
-Within a selected group, timed WhisperX characters determine Script token boundaries; word times,
-VAD bounds and deterministic zero-width estimates are fallbacks in that order. The result contains
-one timing for every Script token and all `2M + 2N` semantic anchors.
-
-Dual Text is not located a second time. `deriveCaptionAtomTiming()` composes the speech map with
-the authored `CaptionAtom -> speech token range` projection. The complete display atom inherits
-the speech envelope; this package never invents linear timings for words inside its display side.
diff --git a/packages/whisperx/src/caption.ts b/packages/whisperx/src/caption.ts
deleted file mode 100644
index 4b406a83..00000000
--- a/packages/whisperx/src/caption.ts
+++ /dev/null
@@ -1,54 +0,0 @@
-import type { ParsedNarrative } from "@svml/script";
-
-import { WhisperXAlignmentError } from "./error.js";
-import type { CompleteSpeechTimeMap, TimedCaptionAtom, TimingQuality } from "./types.js";
-
-const QUALITY_RANK: Readonly<Record<TimingQuality, number>> = {
- measured: 0,
- derived: 1,
- estimated: 2,
-};
-
-function worse(left: TimingQuality, right: TimingQuality): TimingQuality {
- return QUALITY_RANK[left] >= QUALITY_RANK[right] ? left : right;
-}
-
-/**
- * Derives only explicit Dual Text atoms. Ordinary caption text reuses its
- * corresponding speech token timing directly. A Dual Text display side stays
- * atomic and inherits the complete speech-side envelope; it is never linearly
- * split into invented display-word times.
- */
-export function deriveCaptionAtomTiming(
- narrative: ParsedNarrative,
- map: CompleteSpeechTimeMap,
-): TimedCaptionAtom[] {
- if (map.semanticIndexDigest !== narrative.semanticIndex.digest) {
- throw new WhisperXAlignmentError(
- "WHISPERX_CAPTION_MAP",
- "Caption projection and speech map use different Semantic Indexes.",
- );
- }
- return narrative.captionAtoms.map((atom) => {
- const source = narrative.tokens.slice(atom.startToken, atom.endTokenExclusive);
- const timed = source.map((token) => map.tokens.find((value) => value.tokenId === token.id));
- if (!timed.length || timed.some((token) => token === undefined)) {
- throw new WhisperXAlignmentError(
- "WHISPERX_CAPTION_COVERAGE",
- `Caption atom ${atom.id} is not covered by the complete speech map.`,
- );
- }
- const first = timed[0]!;
- const last = timed.at(-1)!;
- return {
- id: atom.id,
- display: atom.display,
- segmentId: atom.segmentId,
- sourceTokenIds: source.map((token) => token.id),
- startSec: first.startSec,
- endSec: last.endSec,
- startQuality: worse(first.startQuality, "derived"),
- endQuality: worse(last.endQuality, "derived"),
- };
- });
-}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d0f61f46..f0e1de69 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -22,6 +22,18 @@ importers:
specifier: 5.9.3
version: 5.9.3
+ packages/caption:
+ dependencies:
+ '@svml/core':
+ specifier: workspace:*
+ version: link:../core
+ '@svml/script':
+ specifier: workspace:*
+ version: link:../script
+ '@svml/speech-align':
+ specifier: workspace:*
+ version: link:../speech-align
+
packages/core:
dependencies:
'@svml/protocol':
@@ -48,24 +60,24 @@ importers:
specifier: workspace:*
version: link:../protocol
- packages/text:
+ packages/speech-align:
dependencies:
'@svml/core':
specifier: workspace:*
version: link:../core
- '@svml/protocol':
- specifier: workspace:*
- version: link:../protocol
- devDependencies:
'@svml/script':
specifier: workspace:*
version: link:../script
- packages/whisperx:
+ packages/text:
dependencies:
'@svml/core':
specifier: workspace:*
version: link:../core
+ '@svml/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+ devDependencies:
'@svml/script':
specifier: workspace:*
version: link:../script
diff --git a/tsconfig.v2.json b/tsconfig.v2.json
index a5348d84..7c9cc4f4 100644
--- a/tsconfig.v2.json
+++ b/tsconfig.v2.json
@@ -10,7 +10,8 @@
"@svml/driver-node": ["packages/driver-node/src/index.ts"],
"@svml/script": ["packages/script/src/index.ts"],
"@svml/text": ["packages/text/src/index.ts"],
- "@svml/whisperx": ["packages/whisperx/src/index.ts"]
+ "@svml/speech-align": ["packages/speech-align/src/index.ts"],
+ "@svml/caption": ["packages/caption/src/index.ts"]
}
},
"include": ["packages/**/*.ts"],