Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
feat: separate caption evidence and runtime providers
kashorina04dcae52cca
diff --git a/packages/caption/README.md b/packages/caption/README.mdnew file mode 100644index 00000000..e7cfd0a6--- /dev/null+++ b/packages/caption/README.md@@ -0,0 +1,10 @@+# `@svml/caption`++Pure composition between Script's untimed Caption Projection and a complete speech time map.+The authored display region receives the evidence-backed envelope of the speech tokens it owns.+Exact internal correspondences may reuse evidence timing; unresolved display words remain untimed+until a track explicitly chooses a local presentation policy.++`planCaptionPresentation()` is that optional track-local policy. `whole` preserves each authored+region; `proportional-word` and `character-flow` create visibly labelled estimates. They never+write those estimates back into the global speech map or Caption Projection.diff --git a/packages/caption/package.json b/packages/caption/package.jsonnew file mode 100644index 00000000..f70c6f28--- /dev/null+++ b/packages/caption/package.json@@ -0,0 +1,14 @@+{+ "name": "@svml/caption",+ "version": "0.0.0-dev",+ "private": true,+ "type": "module",+ "exports": {+ ".": "./src/index.ts"+ },+ "dependencies": {+ "@svml/core": "workspace:*",+ "@svml/script": "workspace:*",+ "@svml/speech-align": "workspace:*"+ }+}diff --git a/packages/whisperx/src/error.ts b/packages/caption/src/error.tssimilarity index 56%rename from packages/whisperx/src/error.tsrename to packages/caption/src/error.tsindex 3d7079ee..1dacbfec 100644--- a/packages/whisperx/src/error.ts+++ b/packages/caption/src/error.ts@@ -1,9 +1,9 @@-export class WhisperXAlignmentError extends Error {+export class CaptionProjectionError extends Error {readonly code: string;constructor(code: string, message: string) {super(message);- this.name = "WhisperXAlignmentError";+ this.name = "CaptionProjectionError";this.code = code;}}diff --git a/packages/caption/src/index.ts b/packages/caption/src/index.tsnew file mode 100644index 00000000..5fcdea10--- /dev/null+++ b/packages/caption/src/index.ts@@ -0,0 +1,4 @@+export { CaptionProjectionError } from "./error.js";+export { planCaptionPresentation } from "./presentation.js";+export { temporalizeCaption } from "./temporalize.js";+export type * from "./types.js";diff --git a/packages/caption/src/presentation.ts b/packages/caption/src/presentation.tsnew file mode 100644index 00000000..c6ac0c76--- /dev/null+++ b/packages/caption/src/presentation.ts@@ -0,0 +1,140 @@+import { digestOf } from "@svml/core";++import type {+ CaptionPresentationMode,+ CaptionPresentationPlan,+ CaptionPresentationUnit,+ TimedCaptionRegion,+} from "./types.js";++const DISPLAY_WORD = /[\p{L}\p{M}\p{N}]+(?:['’.-][\p{L}\p{M}\p{N}]+)*/gu;++type DisplayPart = {+ readonly display: string;+ readonly start: number;+ readonly end: number;+};++function displayWords(value: string): DisplayPart[] {+ const words: DisplayPart[] = [];+ DISPLAY_WORD.lastIndex = 0;+ let match: RegExpExecArray | null;+ while ((match = DISPLAY_WORD.exec(value))) {+ words.push({ display: match[0], start: match.index, end: match.index + match[0].length });+ }+ return words;+}++function wholeRegion(region: TimedCaptionRegion): CaptionPresentationUnit[] {+ if (!region.display) return [];+ return [{+ id: `${region.id}:whole`,+ regionId: region.id,+ display: region.display,+ displayStart: 0,+ displayEnd: region.display.length,+ startSec: region.startSec,+ endSec: region.endSec,+ basis: "region-envelope",+ timingQuality: region.startQuality === "estimated" || region.endQuality === "estimated" ? "estimated" : "derived",+ }];+}++function proportionalWords(region: TimedCaptionRegion): CaptionPresentationUnit[] {+ const words = displayWords(region.display);+ if (!words.length) return [];+ const exact = new Map(region.refinements.map((item) => [`${item.displayStart}:${item.displayEnd}`, item]));+ const units: Array<CaptionPresentationUnit | undefined> = words.map((word, index) => {+ const refinement = exact.get(`${word.start}:${word.end}`);+ return refinement ? {+ id: `${region.id}:word:${index + 1}`,+ regionId: region.id,+ display: word.display,+ displayStart: word.start,+ displayEnd: word.end,+ startSec: refinement.startSec,+ endSec: refinement.endSec,+ basis: "exact-correspondence",+ timingQuality: refinement.startQuality === "estimated" || refinement.endQuality === "estimated" ? "estimated" : "derived",+ } : undefined;+ });++ let cursor = 0;+ while (cursor < words.length) {+ if (units[cursor]) {+ cursor += 1;+ continue;+ }+ const runStart = cursor;+ while (cursor < words.length && !units[cursor]) cursor += 1;+ const runEnd = cursor;+ const left = runStart === 0 ? region.startSec : units[runStart - 1]!.endSec;+ const right = runEnd === words.length ? region.endSec : Math.max(left, units[runEnd]!.startSec);+ const weights = words.slice(runStart, runEnd).map((word) => Math.max(1, [...word.display].length));+ const total = weights.reduce((sum, weight) => sum + weight, 0);+ let used = 0;+ for (let index = runStart; index < runEnd; index += 1) {+ const weight = weights[index - runStart]!;+ const startSec = left + (right - left) * used / total;+ used += weight;+ const endSec = left + (right - left) * used / total;+ const word = words[index]!;+ units[index] = {+ id: `${region.id}:word:${index + 1}`,+ regionId: region.id,+ display: word.display,+ displayStart: word.start,+ displayEnd: word.end,+ startSec,+ endSec,+ basis: "presentation-estimate",+ timingQuality: "estimated",+ };+ }+ }+ return units as CaptionPresentationUnit[];+}++function characterFlow(region: TimedCaptionRegion): CaptionPresentationUnit[] {+ const characters = [...region.display];+ if (!characters.length) return [];+ const duration = region.endSec - region.startSec;+ const units: CaptionPresentationUnit[] = [];+ let sourceOffset = 0;+ for (const [index, character] of characters.entries()) {+ const start = sourceOffset;+ sourceOffset += character.length;+ if (!character.trim()) continue;+ units.push({+ id: `${region.id}:character:${index + 1}`,+ regionId: region.id,+ display: character,+ displayStart: start,+ displayEnd: sourceOffset,+ startSec: region.startSec + duration * index / characters.length,+ endSec: region.startSec + duration * (index + 1) / characters.length,+ basis: "presentation-estimate",+ timingQuality: "estimated",+ });+ }+ return units;+}++export function planCaptionPresentation(+ projection: { readonly projectionDigest: string; readonly regions: readonly TimedCaptionRegion[] },+ mode: CaptionPresentationMode = "whole",+): CaptionPresentationPlan {+ const units = projection.regions.flatMap((region) =>+ mode === "whole"+ ? wholeRegion(region)+ : mode === "proportional-word"+ ? proportionalWords(region)+ : characterFlow(region));+ const payload = {+ contract: "svml.caption-presentation-plan@0" as const,+ sourceProjectionDigest: projection.projectionDigest,+ mode,+ units,+ };+ return { ...payload, planDigest: digestOf(payload) };+}diff --git a/packages/caption/src/temporalize.ts b/packages/caption/src/temporalize.tsnew file mode 100644index 00000000..f198c806--- /dev/null+++ b/packages/caption/src/temporalize.ts@@ -0,0 +1,79 @@+import { digestOf } from "@svml/core";+import type { ParsedNarrative } from "@svml/script";+import type { CompleteSpeechTimeMap, TimingQuality } from "@svml/speech-align";++import { CaptionProjectionError } from "./error.js";+import type {+ TimedCaptionProjection,+ TimedCaptionRefinement,+ TimedCaptionRegion,+} from "./types.js";++const QUALITY_RANK: Readonly<Record<TimingQuality, number>> = {+ measured: 0,+ derived: 1,+ estimated: 2,+};++function composedQuality(value: TimingQuality): TimingQuality {+ return QUALITY_RANK[value] >= QUALITY_RANK.derived ? value : "derived";+}++export function temporalizeCaption(+ narrative: ParsedNarrative,+ map: CompleteSpeechTimeMap,+): TimedCaptionProjection {+ if (map.semanticIndexDigest !== narrative.semanticIndex.digest) {+ throw new CaptionProjectionError(+ "CAPTION_SEMANTIC_INDEX",+ "Caption Projection and speech time map use different Semantic Indexes.",+ );+ }+ const timingByToken = new Map(map.tokens.map((token) => [token.tokenId, token]));+ const sourceTiming = (startToken: number, endTokenExclusive: number, owner: string) => {+ const source = narrative.tokens.slice(startToken, endTokenExclusive);+ const timed = source.map((token) => timingByToken.get(token.id));+ if (!timed.length || timed.some((token) => token === undefined)) {+ throw new CaptionProjectionError(+ "CAPTION_SPEECH_COVERAGE",+ `${owner} is not covered by the complete speech time map.`,+ );+ }+ const first = timed[0]!;+ const last = timed.at(-1)!;+ return {+ sourceTokenIds: source.map((token) => token.id),+ startSec: first.startSec,+ endSec: last.endSec,+ startQuality: composedQuality(first.startQuality),+ endQuality: composedQuality(last.endQuality),+ };+ };++ const regions: TimedCaptionRegion[] = narrative.captionProjection.regions.map((region) => {+ const refinements: TimedCaptionRefinement[] = region.refinements.map((refinement) => ({+ id: refinement.id,+ display: refinement.display,+ displayStart: refinement.displayStart,+ displayEnd: refinement.displayEnd,+ ...sourceTiming(refinement.startToken, refinement.endTokenExclusive, refinement.id),+ relation: refinement.relation,+ }));+ return {+ id: region.id,+ display: region.display,+ segmentId: region.segmentId,+ kind: region.kind,+ ...sourceTiming(region.startToken, region.endTokenExclusive, region.id),+ refinements,+ };+ });+ const payload = {+ contract: "svml.timed-caption-projection@0" as const,+ semanticIndexDigest: narrative.semanticIndex.digest,+ speechTimeMapDigest: map.mapDigest,+ text: narrative.captionProjection.text,+ regions,+ };+ return { ...payload, projectionDigest: digestOf(payload) };+}diff --git a/packages/caption/src/types.ts b/packages/caption/src/types.tsnew file mode 100644index 00000000..9a698181--- /dev/null+++ b/packages/caption/src/types.ts@@ -0,0 +1,58 @@+import type { TimingQuality } from "@svml/speech-align";++export type TimedCaptionRefinement = {+ readonly id: string;+ readonly display: string;+ readonly displayStart: number;+ readonly displayEnd: number;+ readonly sourceTokenIds: readonly string[];+ readonly startSec: number;+ readonly endSec: number;+ readonly startQuality: TimingQuality;+ readonly endQuality: TimingQuality;+ readonly relation: "exact";+};++export type TimedCaptionRegion = {+ readonly id: string;+ readonly display: string;+ readonly segmentId: string;+ readonly kind: "identity" | "alias" | "hidden";+ readonly sourceTokenIds: readonly string[];+ readonly startSec: number;+ readonly endSec: number;+ readonly startQuality: TimingQuality;+ readonly endQuality: TimingQuality;+ readonly refinements: readonly TimedCaptionRefinement[];+};++export type TimedCaptionProjection = {+ readonly contract: "svml.timed-caption-projection@0";+ readonly semanticIndexDigest: string;+ readonly speechTimeMapDigest: string;+ readonly text: string;+ readonly regions: readonly TimedCaptionRegion[];+ readonly projectionDigest: string;+};++export type CaptionPresentationMode = "whole" | "proportional-word" | "character-flow";++export type CaptionPresentationUnit = {+ readonly id: string;+ readonly regionId: string;+ readonly display: string;+ readonly displayStart: number;+ readonly displayEnd: number;+ readonly startSec: number;+ readonly endSec: number;+ readonly basis: "region-envelope" | "exact-correspondence" | "presentation-estimate";+ readonly timingQuality: TimingQuality;+};++export type CaptionPresentationPlan = {+ readonly contract: "svml.caption-presentation-plan@0";+ readonly sourceProjectionDigest: string;+ readonly mode: CaptionPresentationMode;+ readonly units: readonly CaptionPresentationUnit[];+ readonly planDigest: string;+};diff --git a/packages/caption/test/caption.test.ts b/packages/caption/test/caption.test.tsnew file mode 100644index 00000000..516e2394--- /dev/null+++ b/packages/caption/test/caption.test.ts@@ -0,0 +1,79 @@+import assert from "node:assert/strict";+import test from "node:test";++import { planCaptionPresentation, temporalizeCaption } from "@svml/caption";+import { parseScript } from "@svml/script";+import { locateSpeechTiming } from "@svml/speech-align";++test("Caption temporalization preserves evidence envelopes and labels local estimates", () => {+ const narrative = parseScript(+ "caption.svml",+ "<line><that was insane | what the fuck> <15% off | fifteen percent off></line>",+ );+ const map = locateSpeechTiming(narrative, {+ contract: "svml.aligned-transcript-evidence@0",+ durationSec: 2,+ segments: [{+ sourceSegmentId: "line",+ startSec: 0,+ endSec: 2,+ words: [+ { text: "what", startSec: 0.1, endSec: 0.25 },+ { text: "the", startSec: 0.3, endSec: 0.42 },+ { text: "fuck", startSec: 0.48, endSec: 0.72 },+ { text: "fifteen", startSec: 0.9, endSec: 1.15 },+ { text: "percent", startSec: 1.2, endSec: 1.42 },+ { text: "off", startSec: 1.48, endSec: 1.65 },+ ],+ chars: [],+ }],+ });+ const originalMap = structuredClone(map);+ const timed = temporalizeCaption(narrative, map);++ assert.equal(timed.regions[0]?.display, "that was insane");+ assert.deepEqual(+ [timed.regions[0]?.startSec, timed.regions[0]?.endSec, timed.regions[0]?.refinements.length],+ [0.1, 0.72, 0],+ );+ assert.deepEqual(timed.regions[1]?.refinements.map((item) => item.display), ["off"]);++ const whole = planCaptionPresentation(timed, "whole");+ assert.deepEqual(+ [whole.units[0]?.display, whole.units[0]?.startSec, whole.units[0]?.endSec, whole.units[0]?.basis],+ ["that was insane", 0.1, 0.72, "region-envelope"],+ );++ const words = planCaptionPresentation(timed, "proportional-word");+ assert.deepEqual(words.units.slice(0, 3).map((unit) => unit.basis), [+ "presentation-estimate",+ "presentation-estimate",+ "presentation-estimate",+ ]);+ assert.equal(words.units.find((unit) => unit.display === "off")?.basis, "exact-correspondence");+ assert.equal(words.units.find((unit) => unit.display === "15")?.timingQuality, "estimated");+ assert.deepEqual(map, originalMap, "caption presentation must not modify the global speech map");+});++test("hidden speech owns time but emits no visible presentation unit", () => {+ const narrative = parseScript("hidden.svml", "<line>Hello < | um> world.</line>");+ const map = locateSpeechTiming(narrative, {+ contract: "svml.aligned-transcript-evidence@0",+ durationSec: 1,+ segments: [{+ sourceSegmentId: "line",+ startSec: 0,+ endSec: 1,+ words: [+ { text: "Hello", startSec: 0.1, endSec: 0.25 },+ { text: "um", startSec: 0.3, endSec: 0.4 },+ { text: "world", startSec: 0.45, endSec: 0.7 },+ ],+ chars: [],+ }],+ });+ const timed = temporalizeCaption(narrative, map);+ const hidden = timed.regions.find((region) => region.kind === "hidden");+ assert.deepEqual([hidden?.startSec, hidden?.endSec], [0.3, 0.4]);+ assert.equal(planCaptionPresentation(timed).units.some((unit) => unit.regionId === hidden?.id), false);+});diff --git a/packages/driver-node/README.md b/packages/driver-node/README.mdindex 5f4b8672..e3d75615 100644--- a/packages/driver-node/README.md+++ b/packages/driver-node/README.md@@ -1,11 +1,16 @@# `@svml/driver-node`-Experimental Node Host for the v2 Core.+Node runtime for executing Core commands without adding workflow semantics to Core.-Driver v0 reads explicitly supplied static `svml.module.json` files, runs Host-registered producer-functions, dispatches typed Needs to registered handlers, stores content-addressed bytes and can-serialize or resume Core state.+- `HostRegistry` registers deterministic Producer implementations locked by implementation digest.+- `ProviderRegistry` registers external capabilities that may use APIs, credentials, queues or local runtimes.+- A Producer emits a typed `Need`; it never reads provider configuration directly.+- One matching Provider may fulfill a Need. Multiple matches require an explicit runtime `bind()`.+- Provider identity becomes the Receipt fulfiller; handlers return only value, conformance, delivery and metadata.-It deliberately does not load implementation code from a package locator. A future sandboxed-executor must be designed and audited before third-party implementations can run. Frontend and-Import Prologue behavior are also outside this package version.+Build state is serializable. Missing providers, generation latency and transient provider errors pause a+build without replaying completed Producers.++The Driver still does not load implementation code from a package locator. A future sandboxed executor+must be designed and audited before third-party implementations can run. Frontend and Import Prologue+behavior remain outside this package.diff --git a/packages/driver-node/src/driver.ts b/packages/driver-node/src/driver.tsindex dd876bf5..1266a32c 100644--- a/packages/driver-node/src/driver.ts+++ b/packages/driver-node/src/driver.ts@@ -9,18 +9,19 @@ import type {} from "@svml/protocol";import { MemoryArtifactStore } from "./artifacts.js";-import { HostRegistry, producerRegistryKey, requirementRegistryKey } from "./registry.js";+import { HostRegistry, ProviderRegistry, producerRegistryKey, providerCapabilityKey } from "./registry.js";import type {ArtifactStore,BlockedCommand,DriverJournalEntry,DriverRunResult,ProducerHandlerResult,- RequirementHandlerResult,+ ProviderHandlerResult,} from "./types.js";export type NodeDriverOptions = {readonly registry?: HostRegistry;+ readonly providers?: ProviderRegistry;readonly artifacts?: ArtifactStore;readonly maxEvents?: number;};@@ -32,16 +33,19 @@ type Executable =}| {readonly command: FulfillNeedCommand;- readonly run: () => Promise<RequirementHandlerResult>;+ readonly providerId: string;+ readonly run: () => Promise<ProviderHandlerResult>;};export class NodeDriver {readonly registry: HostRegistry;+ readonly providers: ProviderRegistry;readonly artifacts: ArtifactStore;readonly maxEvents: number;constructor(options: NodeDriverOptions = {}) {this.registry = options.registry ?? new HostRegistry();+ this.providers = options.providers ?? new ProviderRegistry();this.artifacts = options.artifacts ?? new MemoryArtifactStore();this.maxEvents = options.maxEvents ?? 1_000;}@@ -101,19 +105,32 @@ export class NodeDriver {};}- const registration = this.registry.requirement(command.need.wants);- if (registration === undefined) {+ const resolution = this.providers.resolve(command.need);+ if (resolution.status === "missing") {return {blocked: {command: command.id,- reason: "missing-handler",- subject: requirementRegistryKey(command.need.wants),+ reason: "missing-provider",+ subject: resolution.providerId === undefined+ ? providerCapabilityKey(command.need.wants)+ : `${providerCapabilityKey(command.need.wants)} -> ${resolution.providerId}`,},};}+ if (resolution.status === "ambiguous") {+ return {+ blocked: {+ command: command.id,+ reason: "ambiguous-provider",+ subject: `${providerCapabilityKey(command.need.wants)} -> ${resolution.providerIds.join(", ")}`,+ },+ };+ }+ const registration = resolution.registration;return {executable: {command,+ providerId: registration.id,run: async () =>registration.handler({command: structuredClone(command),@@ -125,7 +142,7 @@ export class NodeDriver {}async #execute(executable: Executable): Promise<BuildEvent> {- if (executable.command.kind === "invoke-producer") {+ if (!("providerId" in executable)) {const result = (await executable.run()) as ProducerHandlerResult;return {kind: "producer-completed",@@ -140,7 +157,7 @@ export class NodeDriver {};}- const result = (await executable.run()) as RequirementHandlerResult;+ const result = (await executable.run()) as ProviderHandlerResult;return {kind: "need-fulfilled",id: `event:${digestOf({@@ -151,7 +168,7 @@ export class NodeDriver {command: executable.command.id,value: result.value,requestDigest: executable.command.need.requestDigest,- fulfiller: result.fulfiller,+ fulfiller: executable.providerId,conformance: result.conformance,delivery: result.delivery,metadata: result.metadata,diff --git a/packages/driver-node/src/index.ts b/packages/driver-node/src/index.tsindex b97338e2..859833ae 100644--- a/packages/driver-node/src/index.ts+++ b/packages/driver-node/src/index.ts@@ -9,7 +9,8 @@ export {export { parseBuildState, serializeBuildState } from "./persistence.js";export {HostRegistry,+ ProviderRegistry,producerRegistryKey,- requirementRegistryKey,+ providerCapabilityKey,} from "./registry.js";export type * from "./types.js";diff --git a/packages/driver-node/src/registry.ts b/packages/driver-node/src/registry.tsindex da418a10..78025211 100644--- a/packages/driver-node/src/registry.ts+++ b/packages/driver-node/src/registry.ts@@ -1,10 +1,11 @@-import type { Digest, ProducerRef, TypeRef } from "@svml/protocol";+import type { Digest, Need, ProducerRef, TypeRef } from "@svml/protocol";import type {ProducerHandler,ProducerRegistration,- RequirementHandler,- RequirementRegistration,+ ProviderHandler,+ ProviderRegistration,+ ProviderResolution,} from "./types.js";function moduleKey(ref: { readonly name: string; readonly version: string }): string {@@ -15,13 +16,12 @@ export function producerRegistryKey(ref: ProducerRef): string {return `${moduleKey(ref.module)}#${ref.name}`;}-export function requirementRegistryKey(ref: TypeRef): string {+export function providerCapabilityKey(ref: TypeRef): string {return `${moduleKey(ref.module)}#${ref.name}`;}export class HostRegistry {readonly #producers = new Map<string, ProducerRegistration>();- readonly #requirements = new Map<string, RequirementRegistration>();registerProducer(producer: ProducerRef,@@ -33,17 +33,61 @@ export class HostRegistry {this.#producers.set(key, { producer, implementationDigest, handler });}- registerRequirement(type: TypeRef, handler: RequirementHandler): void {- const key = requirementRegistryKey(type);- if (this.#requirements.has(key)) throw new Error(`requirement ${key} is already registered`);- this.#requirements.set(key, { type, handler });- }-producer(ref: ProducerRef): ProducerRegistration | undefined {return this.#producers.get(producerRegistryKey(ref));}+}++export class ProviderRegistry {+ readonly #capabilities = new Map<string, ProviderRegistration[]>();+ readonly #bindings = new Map<string, string>();++ registerProvider(+ id: string,+ wants: TypeRef,+ handler: ProviderHandler,+ options: { readonly supports?: (need: Need) => boolean } = {},+ ): void {+ if (!id.trim()) throw new Error("provider id must not be empty");+ const key = providerCapabilityKey(wants);+ const registrations = this.#capabilities.get(key) ?? [];+ if (registrations.some((registration) => registration.id === id)) {+ throw new Error(`provider ${id} already registers capability ${key}`);+ }+ registrations.push({ id, wants, handler, ...options });+ this.#capabilities.set(key, registrations);+ }++ bind(wants: TypeRef, providerId: string): void {+ const key = providerCapabilityKey(wants);+ if (!(this.#capabilities.get(key) ?? []).some((registration) => registration.id === providerId)) {+ throw new Error(`provider ${providerId} does not register capability ${key}`);+ }+ this.#bindings.set(key, providerId);+ }++ resolve(need: Need): ProviderResolution {+ const key = providerCapabilityKey(need.wants);+ const bound = this.#bindings.get(key);+ const registrations = (this.#capabilities.get(key) ?? [])+ .filter((registration) => registration.supports?.(need) ?? true);+ if (bound !== undefined) {+ const registration = registrations.find((candidate) => candidate.id === bound);+ return registration === undefined+ ? { status: "missing", providerId: bound }+ : { status: "resolved", registration };+ }+ if (registrations.length === 0) return { status: "missing" };+ if (registrations.length > 1) {+ return {+ status: "ambiguous",+ providerIds: registrations.map((registration) => registration.id).sort(),+ };+ }+ return { status: "resolved", registration: registrations[0]! };+ }- requirement(ref: TypeRef): RequirementRegistration | undefined {- return this.#requirements.get(requirementRegistryKey(ref));+ providers(wants: TypeRef): readonly ProviderRegistration[] {+ return [...(this.#capabilities.get(providerCapabilityKey(wants)) ?? [])];}}diff --git a/packages/driver-node/src/types.ts b/packages/driver-node/src/types.tsindex eee874c6..63d89bfe 100644--- a/packages/driver-node/src/types.ts+++ b/packages/driver-node/src/types.ts@@ -31,23 +31,22 @@ export type ProducerHandler = (context: ProducerHandlerContext,) => ProducerHandlerResult | Promise<ProducerHandlerResult>;-export type RequirementHandlerResult = {+export type ProviderHandlerResult = {readonly value: StoredValue;- readonly fulfiller: string;readonly conformance: Conformance;readonly delivery: Delivery;readonly metadata: CanonicalValue;};-export type RequirementHandlerContext = {+export type ProviderHandlerContext = {readonly command: FulfillNeedCommand;readonly need: Need;readonly artifacts: ArtifactStore;};-export type RequirementHandler = (- context: RequirementHandlerContext,-) => RequirementHandlerResult | Promise<RequirementHandlerResult>;+export type ProviderHandler = (+ context: ProviderHandlerContext,+) => ProviderHandlerResult | Promise<ProviderHandlerResult>;export type ArtifactStore = {put(bytes: Uint8Array, mediaType: string): Promise<BlobRef>;@@ -65,7 +64,11 @@ export type DriverJournalEntry = {export type BlockedCommand = {readonly command: string;- readonly reason: "missing-producer" | "implementation-mismatch" | "missing-handler";+ readonly reason:+ | "missing-producer"+ | "implementation-mismatch"+ | "missing-provider"+ | "ambiguous-provider";readonly subject: string;};@@ -82,7 +85,14 @@ export type ProducerRegistration = {readonly handler: ProducerHandler;};-export type RequirementRegistration = {- readonly type: TypeRef;- readonly handler: RequirementHandler;+export type ProviderRegistration = {+ readonly id: string;+ readonly wants: TypeRef;+ readonly handler: ProviderHandler;+ readonly supports?: (need: Need) => boolean;};++export type ProviderResolution =+ | { readonly status: "resolved"; readonly registration: ProviderRegistration }+ | { readonly status: "missing"; readonly providerId?: string }+ | { readonly status: "ambiguous"; readonly providerIds: readonly string[] };diff --git a/packages/driver-node/test/driver.test.ts b/packages/driver-node/test/driver.test.tsindex 8c66c4ab..4c3f7888 100644--- a/packages/driver-node/test/driver.test.ts+++ b/packages/driver-node/test/driver.test.ts@@ -9,6 +9,7 @@ import {HostRegistry,MemoryArtifactStore,NodeDriver,+ ProviderRegistry,loadResolvedClosure,parseBuildState,serializeBuildState,@@ -36,9 +37,11 @@ type GreetingCalls = {function configuredRegistry(): {registry: HostRegistry;+ providers: ProviderRegistry;calls: GreetingCalls;} {const registry = new HostRegistry();+ const providers = new ProviderRegistry();const calls = { prompt: 0, request: 0, assemble: 0, fulfill: 0 };registry.registerProducer(producers.makePrompt, implementationDigests.makePrompt, ({ inputs }) => {@@ -81,25 +84,24 @@ function configuredRegistry(): {};});- return { registry, calls };+ return { registry, providers, calls };}test("Driver pauses at an unbound Need, serializes, then resumes without rerunning producers", async () => {- const { registry, calls } = configuredRegistry();- const driver = new NodeDriver({ registry });+ const { registry, providers, calls } = configuredRegistry();+ const driver = new NodeDriver({ registry, providers });const paused = await driver.run(createGreetingBuild());assert.equal(paused.status, "paused");- assert.equal(paused.blocked[0]?.reason, "missing-handler");+ assert.equal(paused.blocked[0]?.reason, "missing-provider");assert.deepEqual(calls, { prompt: 1, request: 1, assemble: 0, fulfill: 0 });const restored = parseBuildState(serializeBuildState(paused.state));- registry.registerRequirement(types.generated, ({ need }) => {+ providers.registerProvider("example:generation", types.generated, ({ need }) => {calls.fulfill += 1;assert.deepEqual(need.constraints, { prompt: "Greet Ada" });return {value: { kind: "inline", value: "Hello, Ada!" },- fulfiller: "example:generation",conformance: "exact",delivery: "executed",metadata: { provider: "fixture" },@@ -110,19 +112,72 @@ test("Driver pauses at an unbound Need, serializes, then resumes without rerunniassert.equal(completed.status, "complete");assert.deepEqual(calls, { prompt: 1, request: 1, assemble: 1, fulfill: 1 });assert.equal(completed.state.receipts[0]?.delivery, "executed");+ assert.equal(completed.state.receipts[0]?.fulfiller, "example:generation");assert.equal(completed.state.records.find((record) => record.id === "document:root")?.conformance, "exact");});+test("Provider Registry rejects ambiguity until the Runtime binds one provider", async () => {+ const { registry, providers } = configuredRegistry();+ providers.registerProvider("example:alpha", types.generated, () => ({+ value: { kind: "inline", value: "Alpha" },+ conformance: "exact",+ delivery: "executed",+ metadata: {},+ }));+ providers.registerProvider("example:beta", types.generated, () => ({+ value: { kind: "inline", value: "Beta" },+ conformance: "exact",+ delivery: "executed",+ metadata: {},+ }));++ const driver = new NodeDriver({ registry, providers });+ const ambiguous = await driver.run(createGreetingBuild());+ assert.equal(ambiguous.status, "paused");+ assert.equal(ambiguous.blocked[0]?.reason, "ambiguous-provider");+ assert.match(ambiguous.blocked[0]?.subject ?? "", /example:alpha, example:beta/u);++ providers.bind(types.generated, "example:beta");+ const completed = await driver.run(ambiguous.state);+ assert.equal(completed.status, "complete");+ assert.equal(completed.state.receipts[0]?.fulfiller, "example:beta");+ assert.deepEqual(+ completed.state.records.find((record) => record.id === "document:root")?.value,+ { kind: "inline", value: { text: "Beta" } },+ );+});++test("Provider capabilities may narrow themselves with typed Need constraints", async () => {+ const { registry, providers } = configuredRegistry();+ providers.registerProvider("example:wrong-model", types.generated, () => {+ throw new Error("unsupported provider must never run");+ }, { supports: () => false });+ providers.registerProvider("example:compatible", types.generated, () => ({+ value: { kind: "inline", value: "Compatible" },+ conformance: "exact",+ delivery: "executed",+ metadata: {},+ }), {+ supports: (need) => {+ const constraints = need.constraints as Readonly<Record<string, unknown>>;+ return constraints.prompt === "Greet Ada";+ },+ });++ const completed = await new NodeDriver({ registry, providers }).run(createGreetingBuild());+ assert.equal(completed.status, "complete");+ assert.equal(completed.state.receipts[0]?.fulfiller, "example:compatible");+});+test("substitute conformance propagates through later producer outputs", async () => {- const { registry } = configuredRegistry();- registry.registerRequirement(types.generated, () => ({+ const { registry, providers } = configuredRegistry();+ providers.registerProvider("example:placeholder", types.generated, () => ({value: { kind: "inline", value: "Placeholder" },- fulfiller: "example:placeholder",conformance: "substitute",delivery: "provided",metadata: { reason: "preview" },}));- const result = await new NodeDriver({ registry }).run(+ const result = await new NodeDriver({ registry, providers }).run(createGreetingBuild({ needAccepts: "substitute", goalAccepts: "substitute" }),);assert.equal(result.status, "complete");@@ -186,31 +241,29 @@ test("Driver reads static manifests without executing package code", async () =>});test("Core still owns scheduling when Driver has every implementation", async () => {- const { registry } = configuredRegistry();- registry.registerRequirement(types.generated, () => ({+ const { registry, providers } = configuredRegistry();+ providers.registerProvider("example:cache", types.generated, () => ({value: { kind: "inline", value: "Hello, Ada!" },- fulfiller: "example:cache",conformance: "exact",delivery: "cache",metadata: {},}));const start = createGreetingBuild();assert.equal(reduce(start).commands[0]?.kind, "invoke-producer");- const result = await new NodeDriver({ registry }).run(start);+ const result = await new NodeDriver({ registry, providers }).run(start);assert.equal(result.status, "complete");assert.equal(result.state.receipts[0]?.delivery, "cache");});test("receipt metadata is covered by its identity during resume validation", async () => {- const { registry } = configuredRegistry();- registry.registerRequirement(types.generated, () => ({+ const { registry, providers } = configuredRegistry();+ providers.registerProvider("example:cache", types.generated, () => ({value: { kind: "inline", value: "Hello, Ada!" },- fulfiller: "example:cache",conformance: "exact",delivery: "cache",metadata: { cacheKey: "stable" },}));- const result = await new NodeDriver({ registry }).run(createGreetingBuild());+ const result = await new NodeDriver({ registry, providers }).run(createGreetingBuild());assert.equal(result.status, "complete");const tampered = structuredClone(result.state);const receipt = tampered.receipts[0];@@ -220,22 +273,21 @@ test("receipt metadata is covered by its identity during resume validation", asy});test("a transient Handler failure pauses and can resume without replaying completed producers", async () => {- const { registry, calls } = configuredRegistry();+ const { registry, providers, calls } = configuredRegistry();let attempts = 0;- registry.registerRequirement(types.generated, () => {+ providers.registerProvider("example:unstable", types.generated, () => {calls.fulfill += 1;attempts += 1;if (attempts === 1) throw new Error("temporary outage");return {value: { kind: "inline", value: "Hello after retry" },- fulfiller: "example:unstable",conformance: "exact",delivery: "executed",metadata: { attempt: attempts },};});- const driver = new NodeDriver({ registry });+ const driver = new NodeDriver({ registry, providers });const paused = await driver.run(createGreetingBuild());assert.equal(paused.status, "paused");assert.match(paused.journal.at(-1)?.message ?? "", /temporary outage/u);diff --git a/packages/driver-node/test/video-pipeline-fixture.ts b/packages/driver-node/test/video-pipeline-fixture.tsnew file mode 100644index 00000000..78015284--- /dev/null+++ b/packages/driver-node/test/video-pipeline-fixture.ts@@ -0,0 +1,195 @@+import {+ createResolvedClosure,+ digestOf,+ link,+ sealRecord,+ sealTypedModule,+ start,+} from "@svml/core";+import type {+ BuildPlan,+ BuildState,+ ModuleManifest,+ ProducerRef,+ TypeRef,+ ValueSchema,+} from "@svml/protocol";++export const videoModule = { name: "example.video-pipeline", version: "0.0.0" } as const;++export const videoTypes = {+ narrative: { module: videoModule, name: "Narrative" },+ schedule: { module: videoModule, name: "SpeechSchedule" },+ basis: { module: videoModule, name: "SpeechBasis" },+ evidence: { module: videoModule, name: "AlignedTranscriptEvidence" },+ speechMap: { module: videoModule, name: "CompleteSpeechTimeMap" },+ caption: { module: videoModule, name: "TimedCaption" },+} satisfies Record<string, TypeRef>;++export const videoProducers = {+ requestEstimate: { module: videoModule, name: "request-estimate" },+ requestGeneration: { module: videoModule, name: "request-generation" },+ requestRecognition: { module: videoModule, name: "request-recognition" },+ locateSpeech: { module: videoModule, name: "locate-speech" },+ temporalizeCaption: { module: videoModule, name: "temporalize-caption" },+} satisfies Record<string, ProducerRef>;++export const videoImplementations = Object.fromEntries(+ Object.entries(videoProducers).map(([name]) => [name, digestOf(`example.video-pipeline/${name}@0`)]),+) as Readonly<Record<keyof typeof videoProducers, ReturnType<typeof digestOf>>>;++const string = { kind: "string", minLength: 1 } as const;+const number = { kind: "number", minimum: 0 } as const;+const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema }>>): ValueSchema => ({+ kind: "object",+ fields,+});++export const videoManifest: ModuleManifest = {+ format: "svml.module@0",+ name: videoModule.name,+ version: videoModule.version,+ dependencies: [],+ types: [+ { name: videoTypes.narrative.name, schema: object({ speech: { schema: string }, display: { schema: string } }) },+ { name: videoTypes.schedule.name, schema: object({ durationSec: { schema: number } }) },+ { name: videoTypes.basis.name, schema: object({ audio: { schema: string }, visual: { schema: string } }) },+ { name: videoTypes.evidence.name, schema: object({ transcript: { schema: string } }) },+ { name: videoTypes.speechMap.name, schema: object({ startSec: { schema: number }, endSec: { schema: number } }) },+ {+ name: videoTypes.caption.name,+ schema: object({ text: { schema: string }, startSec: { schema: number }, endSec: { schema: number } }),+ },+ ],+ surfaces: [],+ producers: [+ {+ name: videoProducers.requestEstimate.name,+ inputs: [{ name: "narrative", type: videoTypes.narrative }],+ outputs: [],+ needs: [{ name: "schedule", wants: videoTypes.schedule }],+ implementation: {+ kind: "registered",+ locator: "example.video-pipeline/request-estimate",+ digest: videoImplementations.requestEstimate,+ },+ },+ {+ name: videoProducers.requestGeneration.name,+ inputs: [+ { name: "narrative", type: videoTypes.narrative },+ { name: "schedule", type: videoTypes.schedule },+ ],+ outputs: [],+ needs: [{ name: "basis", wants: videoTypes.basis }],+ implementation: {+ kind: "registered",+ locator: "example.video-pipeline/request-generation",+ digest: videoImplementations.requestGeneration,+ },+ },+ {+ name: videoProducers.requestRecognition.name,+ inputs: [{ name: "basis", type: videoTypes.basis }],+ outputs: [],+ needs: [{ name: "evidence", wants: videoTypes.evidence }],+ implementation: {+ kind: "registered",+ locator: "example.video-pipeline/request-recognition",+ digest: videoImplementations.requestRecognition,+ },+ },+ {+ name: videoProducers.locateSpeech.name,+ inputs: [+ { name: "narrative", type: videoTypes.narrative },+ { name: "evidence", type: videoTypes.evidence },+ ],+ outputs: [{ name: "map", type: videoTypes.speechMap }],+ needs: [],+ implementation: {+ kind: "registered",+ locator: "example.video-pipeline/locate-speech",+ digest: videoImplementations.locateSpeech,+ },+ },+ {+ name: videoProducers.temporalizeCaption.name,+ inputs: [+ { name: "narrative", type: videoTypes.narrative },+ { name: "map", type: videoTypes.speechMap },+ ],+ outputs: [{ name: "caption", type: videoTypes.caption }],+ needs: [],+ implementation: {+ kind: "registered",+ locator: "example.video-pipeline/temporalize-caption",+ digest: videoImplementations.temporalizeCaption,+ },+ },+ ],+};++export const videoPlan: BuildPlan = {+ format: "svml.plan@0",+ id: "video-pipeline",+ steps: [+ {+ id: "request-estimate",+ producer: videoProducers.requestEstimate,+ inputs: { narrative: "narrative:root" },+ outputs: {},+ needs: { schedule: { id: "need:schedule", result: "schedule:root", accepts: "exact" } },+ },+ {+ id: "request-generation",+ producer: videoProducers.requestGeneration,+ inputs: { narrative: "narrative:root", schedule: "schedule:root" },+ outputs: {},+ needs: { basis: { id: "need:basis", result: "basis:root", accepts: "exact" } },+ },+ {+ id: "request-recognition",+ producer: videoProducers.requestRecognition,+ inputs: { basis: "basis:root" },+ outputs: {},+ needs: { evidence: { id: "need:evidence", result: "evidence:root", accepts: "exact" } },+ },+ {+ id: "locate-speech",+ producer: videoProducers.locateSpeech,+ inputs: { narrative: "narrative:root", evidence: "evidence:root" },+ outputs: { map: "speech-map:root" },+ needs: {},+ },+ {+ id: "temporalize-caption",+ producer: videoProducers.temporalizeCaption,+ inputs: { narrative: "narrative:root", map: "speech-map:root" },+ outputs: { caption: "caption:root" },+ needs: {},+ },+ ],+ goals: [{ record: "caption:root", type: videoTypes.caption, accepts: "exact" }],+};++export function createVideoBuild(): BuildState {+ const closure = createResolvedClosure([videoManifest]);+ const narrative = sealRecord({+ id: "narrative:root",+ type: videoTypes.narrative,+ value: { kind: "inline", value: { speech: "what the fuck", display: "that was insane" } },+ conformance: "exact",+ origin: {+ kind: "authored",+ sourceDigest: digestOf("source:video-pipeline"),+ frontendClosureDigest: digestOf("frontend:script"),+ sourceName: "pipeline.svml",+ },+ });+ return start(link(closure, [sealTypedModule({+ id: "author:video-pipeline",+ closureDigest: closure.digest,+ records: [narrative],+ })]), videoPlan);+}diff --git a/packages/driver-node/test/video-pipeline.test.ts b/packages/driver-node/test/video-pipeline.test.tsnew file mode 100644index 00000000..85144a68--- /dev/null+++ b/packages/driver-node/test/video-pipeline.test.ts@@ -0,0 +1,128 @@+import assert from "node:assert/strict";+import test from "node:test";++import {+ HostRegistry,+ NodeDriver,+ ProviderRegistry,+ parseBuildState,+ serializeBuildState,+} from "@svml/driver-node";++import {+ createVideoBuild,+ videoImplementations,+ videoProducers,+ videoTypes,+} from "./video-pipeline-fixture.js";++function inlineObject(value: unknown): Readonly<Record<string, unknown>> {+ if (value === null || Array.isArray(value) || typeof value !== "object") throw new Error("expected object");+ return value as Readonly<Record<string, unknown>>;+}++function inlineString(value: unknown): string {+ if (typeof value !== "string") throw new Error("expected string");+ return value;+}++function inlineNumber(value: unknown): number {+ if (typeof value !== "number" || !Number.isFinite(value)) throw new Error("expected number");+ return value;+}++test("estimate, generation and recognition pause independently without replaying the immutable DAG", async () => {+ const host = new HostRegistry();+ const providers = new ProviderRegistry();+ const calls = { estimate: 0, generate: 0, recognize: 0, locate: 0, caption: 0 };++ host.registerProducer(videoProducers.requestEstimate, videoImplementations.requestEstimate, ({ inputs }) => {+ calls.estimate += 1;+ const narrative = inputs.narrative;+ assert.equal(narrative?.value.kind, "inline");+ return { outputs: {}, needs: { schedule: { speech: inlineString(inlineObject(narrative.value.value).speech) } } };+ });+ host.registerProducer(videoProducers.requestGeneration, videoImplementations.requestGeneration, ({ inputs }) => {+ calls.generate += 1;+ assert.equal(inputs.schedule?.value.kind, "inline");+ return { outputs: {}, needs: { basis: { durationSec: inlineNumber(inlineObject(inputs.schedule.value.value).durationSec) } } };+ });+ host.registerProducer(videoProducers.requestRecognition, videoImplementations.requestRecognition, ({ inputs }) => {+ calls.recognize += 1;+ assert.equal(inputs.basis?.value.kind, "inline");+ return { outputs: {}, needs: { evidence: { audio: inlineString(inlineObject(inputs.basis.value.value).audio) } } };+ });+ host.registerProducer(videoProducers.locateSpeech, videoImplementations.locateSpeech, ({ inputs }) => {+ calls.locate += 1;+ assert.equal(inputs.evidence?.value.kind, "inline");+ return { outputs: { map: { kind: "inline", value: { startSec: 0.1, endSec: 0.72 } } }, needs: {} };+ });+ host.registerProducer(videoProducers.temporalizeCaption, videoImplementations.temporalizeCaption, ({ inputs }) => {+ calls.caption += 1;+ assert.equal(inputs.narrative?.value.kind, "inline");+ assert.equal(inputs.map?.value.kind, "inline");+ const narrative = inlineObject(inputs.narrative.value.value);+ const map = inlineObject(inputs.map.value.value);+ return {+ outputs: {+ caption: {+ kind: "inline",+ value: {+ text: inlineString(narrative.display),+ startSec: inlineNumber(map.startSec),+ endSec: inlineNumber(map.endSec),+ },+ },+ },+ needs: {},+ };+ });++ const driver = new NodeDriver({ registry: host, providers });+ const atEstimate = await driver.run(createVideoBuild());+ assert.equal(atEstimate.blocked[0]?.reason, "missing-provider");+ assert.deepEqual(calls, { estimate: 1, generate: 0, recognize: 0, locate: 0, caption: 0 });++ providers.registerProvider("runtime:estimate", videoTypes.schedule, () => ({+ value: { kind: "inline", value: { durationSec: 1 } },+ conformance: "exact",+ delivery: "executed",+ metadata: {},+ }));+ const atGeneration = await driver.run(parseBuildState(serializeBuildState(atEstimate.state)));+ assert.equal(atGeneration.blocked[0]?.reason, "missing-provider");+ assert.deepEqual(calls, { estimate: 1, generate: 1, recognize: 0, locate: 0, caption: 0 });++ providers.registerProvider("runtime:seedance-mini", videoTypes.basis, ({ need }) => {+ assert.deepEqual(need.constraints, { durationSec: 1 });+ return {+ value: { kind: "inline", value: { audio: "artifact:audio", visual: "artifact:visual" } },+ conformance: "exact",+ delivery: "executed",+ metadata: { model: "mini" },+ };+ });+ const atRecognition = await driver.run(parseBuildState(serializeBuildState(atGeneration.state)));+ assert.equal(atRecognition.blocked[0]?.reason, "missing-provider");+ assert.deepEqual(calls, { estimate: 1, generate: 1, recognize: 1, locate: 0, caption: 0 });++ providers.registerProvider("runtime:whisperx", videoTypes.evidence, () => ({+ value: { kind: "inline", value: { transcript: "what the fuck" } },+ conformance: "exact",+ delivery: "executed",+ metadata: { adapter: "whisperx" },+ }));+ const completed = await driver.run(parseBuildState(serializeBuildState(atRecognition.state)));++ assert.equal(completed.status, "complete");+ assert.deepEqual(calls, { estimate: 1, generate: 1, recognize: 1, locate: 1, caption: 1 });+ assert.deepEqual(completed.state.receipts.map((receipt) => receipt.fulfiller), [+ "runtime:estimate",+ "runtime:seedance-mini",+ "runtime:whisperx",+ ]);+ assert.deepEqual(+ completed.state.records.find((record) => record.id === "caption:root")?.value,+ { kind: "inline", value: { text: "that was insane", startSec: 0.1, endSec: 0.72 } },+ );+});diff --git a/packages/script/src/index.ts b/packages/script/src/index.tsindex 66af6162..39a567de 100644--- a/packages/script/src/index.ts+++ b/packages/script/src/index.ts@@ -7,7 +7,13 @@ export {scriptModuleRef,scriptSurfaceImplementationDigest,} from "./manifest.js";-export { narrativeSourceMap, narrativeValue } from "./narrative.js";+export {+ narrativeSourceMap,+ narrativeValue,+ serializeCaption,+ serializeDialogue,+ serializeSpeech,+} from "./narrative.js";export { parseScript } from "./parser.js";export { decodeScriptSurface } from "./surface.js";export type * from "./types.js";diff --git a/packages/script/src/manifest.ts b/packages/script/src/manifest.tsindex cdb9564a..bf84d853 100644--- a/packages/script/src/manifest.ts+++ b/packages/script/src/manifest.ts@@ -132,17 +132,44 @@ export const narrativeSchema: ValueSchema = {},},},- captionAtoms: {+ captionProjection: {schema: {- kind: "array",- items: {- kind: "object",- fields: {- id: { schema: stringSchema },- display: { schema: { kind: "string" } },- segmentId: { schema: stringSchema },- startToken: { schema: integerSchema },- endTokenExclusive: { schema: integerSchema },+ kind: "object",+ fields: {+ contract: { schema: { kind: "literal", value: "svml.caption-projection@0" } },+ text: { schema: { kind: "string" } },+ regions: {+ schema: {+ kind: "array",+ items: {+ kind: "object",+ fields: {+ id: { schema: stringSchema },+ display: { schema: { kind: "string" } },+ segmentId: { schema: stringSchema },+ startToken: { schema: integerSchema },+ endTokenExclusive: { schema: integerSchema },+ kind: { schema: { kind: "string", enum: ["identity", "alias", "hidden"] } },+ refinements: {+ schema: {+ kind: "array",+ items: {+ kind: "object",+ fields: {+ id: { schema: stringSchema },+ display: { schema: stringSchema },+ displayStart: { schema: integerSchema },+ displayEnd: { schema: integerSchema },+ startToken: { schema: integerSchema },+ endTokenExclusive: { schema: integerSchema },+ relation: { schema: { kind: "literal", value: "exact" } },+ },+ },+ },+ },+ },+ },+ },},},},@@ -177,13 +204,12 @@ export const narrativeSchema: ValueSchema = {},},},- projections: {+ serializations: {schema: {kind: "object",fields: {dialogue: { schema: { kind: "string" } },speech: { schema: { kind: "string" } },- caption: { schema: { kind: "string" } },},},},@@ -199,7 +225,7 @@ export const scriptManifest: ModuleManifest = {{name: narrativeType.name,schema: narrativeSchema,- description: "Authoritative narrative, projections and 2M + 2N semantic identities.",+ description: "Authoritative narrative, pure serializations, Caption Projection and 2M + 2N identities.",},],surfaces: [diff --git a/packages/script/src/narrative.ts b/packages/script/src/narrative.tsindex 3491171b..de465549 100644--- a/packages/script/src/narrative.ts+++ b/packages/script/src/narrative.ts@@ -61,18 +61,44 @@ export function narrativeValue(parsed: ParsedNarrative): CanonicalValue {boundary: semanticBoundary(occurrence.boundary),})),})),- captionAtoms: parsed.captionAtoms.map((atom) => ({- id: atom.id,- display: atom.display,- segmentId: atom.segmentId,- startToken: atom.startToken,- endTokenExclusive: atom.endTokenExclusive,- })),+ captionProjection: {+ contract: parsed.captionProjection.contract,+ text: parsed.captionProjection.text,+ regions: parsed.captionProjection.regions.map((region) => ({+ id: region.id,+ display: region.display,+ segmentId: region.segmentId,+ startToken: region.startToken,+ endTokenExclusive: region.endTokenExclusive,+ kind: region.kind,+ refinements: region.refinements.map((refinement) => ({+ id: refinement.id,+ display: refinement.display,+ displayStart: refinement.displayStart,+ displayEnd: refinement.displayEnd,+ startToken: refinement.startToken,+ endTokenExclusive: refinement.endTokenExclusive,+ relation: refinement.relation,+ })),+ })),+ },semanticIndex: parsed.semanticIndex,- projections: parsed.projections,+ serializations: parsed.serializations,});}+export function serializeSpeech(parsed: ParsedNarrative): string {+ return parsed.serializations.speech;+}++export function serializeDialogue(parsed: ParsedNarrative): string {+ return parsed.serializations.dialogue;+}++export function serializeCaption(parsed: ParsedNarrative): string {+ return parsed.captionProjection.text;+}+export function narrativeSourceMap(recordId: string, parsed: ParsedNarrative): CanonicalValue {return canonicalize({format: "svml.script-source-map@0",diff --git a/packages/script/src/parser.ts b/packages/script/src/parser.tsindex 7b385e7b..43b747fe 100644--- a/packages/script/src/parser.ts+++ b/packages/script/src/parser.ts@@ -5,7 +5,8 @@ import type {Affinity,MarkerBoundary,ParsedAtom,- ParsedCaptionAtom,+ ParsedCaptionRefinement,+ ParsedCaptionRegion,ParsedMoment,ParsedNarrative,ParsedSegment,@@ -46,6 +47,89 @@ function normalizeWord(value: string): string {.replace(/[^\p{L}\p{M}\p{N}]+/gu, "");}+type WordLexeme = {+ readonly text: string;+ readonly normalized: string;+ readonly start: number;+ readonly end: number;+};++function wordLexemes(value: string): WordLexeme[] {+ const lexemes: WordLexeme[] = [];+ WORD.lastIndex = 0;+ let match: RegExpExecArray | null;+ while ((match = WORD.exec(value))) {+ const normalized = normalizeWord(match[0]);+ if (!normalized) continue;+ lexemes.push({+ text: match[0],+ normalized,+ start: match.index,+ end: match.index + match[0].length,+ });+ }+ return lexemes;+}++function exactCaptionRefinements(+ regionId: string,+ display: string,+ speechTokens: readonly ParsedToken[],+): ParsedCaptionRefinement[] {+ const displayWords = wordLexemes(display);+ if (!displayWords.length || !speechTokens.length) return [];++ let matches: Array<{ readonly displayIndex: number; readonly speechIndex: number }>;+ if (+ displayWords.length === speechTokens.length+ && displayWords.every((word, index) => word.normalized === speechTokens[index]!.normalized)+ ) {+ matches = displayWords.map((_, index) => ({ displayIndex: index, speechIndex: index }));+ } else {+ const displayCounts = new Map<string, number>();+ const speechCounts = new Map<string, number>();+ for (const word of displayWords) displayCounts.set(word.normalized, (displayCounts.get(word.normalized) ?? 0) + 1);+ for (const token of speechTokens) speechCounts.set(token.normalized, (speechCounts.get(token.normalized) ?? 0) + 1);+ const candidates = displayWords.flatMap((word, displayIndex) => {+ if (displayCounts.get(word.normalized) !== 1 || speechCounts.get(word.normalized) !== 1) return [];+ const speechIndex = speechTokens.findIndex((token) => token.normalized === word.normalized);+ return speechIndex < 0 ? [] : [{ displayIndex, speechIndex }];+ });++ // Keep a longest monotonic chain. Exact words that cross after an alias+ // rewrite are individually plausible, but cannot form a temporal caption+ // refinement without reversing display order.+ const chains: Array<Array<{ readonly displayIndex: number; readonly speechIndex: number }>> = [];+ for (let index = 0; index < candidates.length; index += 1) {+ let best: Array<{ readonly displayIndex: number; readonly speechIndex: number }> = [];+ for (let before = 0; before < index; before += 1) {+ if (candidates[before]!.speechIndex < candidates[index]!.speechIndex && chains[before]!.length > best.length) {+ best = chains[before]!;+ }+ }+ chains.push([...best, candidates[index]!]);+ }+ matches = chains.reduce<typeof candidates>(+ (best, chain) => chain.length > best.length ? chain : best,+ [],+ );+ }++ return matches.map(({ displayIndex, speechIndex }, index) => {+ const word = displayWords[displayIndex]!;+ const token = speechTokens[speechIndex]!;+ return {+ id: `${regionId}:exact:${index + 1}`,+ display: word.text,+ displayStart: word.start,+ displayEnd: word.end,+ startToken: token.index,+ endTokenExclusive: token.index + 1,+ relation: "exact",+ };+ });+}+function cleanProjection(value: string): string {return value.replace(/\s+/gu, " ")@@ -124,7 +208,7 @@ export function parseScript(const segments: ParsedSegment[] = [];const tokens: ParsedToken[] = [];- const captionAtoms: ParsedCaptionAtom[] = [];+ const captionRegions: ParsedCaptionRegion[] = [];const selections = new Map<string, ParsedSelection["occurrences"] extends readonly (infer T)[] ? T[] : never>();const moments = new Map<string, ParsedMoment["occurrences"] extends readonly (infer T)[] ? T[] : never>();const openSelections = new Map<string, {@@ -267,7 +351,13 @@ export function parseScript(const literalString = (raw: string, start: number, dual = false): string =>literalPieces(raw, start, dual).map((piece) => piece.value).join("");- const addText = (speech: string, caption: string, start: number, end: number): void => {+ const addText = (+ speech: string,+ caption: string,+ start: number,+ end: number,+ captureCaptionRegion = true,+ ): void => {if (!current) {if (speech.trim() || caption.trim()) {fail("SCRIPT_TEXT_OUTSIDE_SEGMENT", "Natural-language text is only allowed inside a named Segment.", start);@@ -307,6 +397,20 @@ export function parseScript(tokenEndExclusive: tokens.length,range: { start: sourceOffset + start, end: sourceOffset + end },});+ const display = cleanProjection(caption);+ if (captureCaptionRegion && display && tokens.length > tokenStart) {+ const id = `caption-region:${captionRegions.length + 1}`;+ captionRegions.push({+ id,+ display,+ segmentId: current.id,+ startToken: tokenStart,+ endTokenExclusive: tokens.length,+ kind: "identity",+ refinements: exactCaptionRefinements(id, display, tokens.slice(tokenStart)),+ range: { start: sourceOffset + start, end: sourceOffset + end },+ });+ }};const consumeSpeechSide = (raw: string, absoluteStart: number, caption: string): void => {@@ -316,7 +420,7 @@ export function parseScript(let emittedCaption = false;const addLiteral = (part: string, start: number): void => {for (const piece of literalPieces(part, start, true)) {- addText(piece.value, emittedCaption ? "" : caption, piece.start, piece.end);+ addText(piece.value, emittedCaption ? "" : caption, piece.start, piece.end, false);emittedCaption = true;}};@@ -344,12 +448,17 @@ export function parseScript(}addLiteral(raw.slice(partStart), absoluteStart + partStart);if (tokens.length > startToken) {- captionAtoms.push({- id: `caption-atom:${captionAtoms.length + 1}`,- display: cleanProjection(caption),+ const display = cleanProjection(caption);+ const id = `caption-region:${captionRegions.length + 1}`;+ const speechTokens = tokens.slice(startToken);+ captionRegions.push({+ id,+ display,segmentId: current!.id,startToken,endTokenExclusive: tokens.length,+ kind: display ? (cleanProjection(speechTokens.map((token) => token.text).join(" ")) === display ? "identity" : "alias") : "hidden",+ refinements: exactCaptionRefinements(id, display, speechTokens),range: { start: sourceOffset + absoluteStart, end: sourceOffset + absoluteStart + raw.length },});}@@ -590,16 +699,19 @@ export function parseScript(turns,selections: [...selections].sort(([left], [right]) => left.localeCompare(right)).map(([id, occurrences]) => ({ id, occurrences })),moments: [...moments].sort(([left], [right]) => left.localeCompare(right)).map(([id, occurrences]) => ({ id, occurrences })),- captionAtoms,+ captionProjection: {+ contract: "svml.caption-projection@0",+ text: captionSegments.join("\n"),+ regions: captionRegions,+ },semanticIndex: {contract: "svml.semantic-index@0",anchors,digest: digestOf(semanticPayload),},- projections: {+ serializations: {dialogue: dialogueTurns.join("\n"),speech: speechSegments.join("\n"),- caption: captionSegments.join("\n"),},};}diff --git a/packages/script/src/types.ts b/packages/script/src/types.tsindex 3bbd10ff..f96ae2d5 100644--- a/packages/script/src/types.ts+++ b/packages/script/src/types.ts@@ -89,15 +89,33 @@ export type ParsedMoment = {readonly occurrences: readonly ParsedMomentOccurrence[];};-export type ParsedCaptionAtom = {+export type ParsedCaptionRefinement = {+ readonly id: string;+ readonly display: string;+ readonly displayStart: number;+ readonly displayEnd: number;+ readonly startToken: number;+ readonly endTokenExclusive: number;+ readonly relation: "exact";+};++export type ParsedCaptionRegion = {readonly id: string;readonly display: string;readonly segmentId: string;readonly startToken: number;readonly endTokenExclusive: number;+ readonly kind: "identity" | "alias" | "hidden";+ readonly refinements: readonly ParsedCaptionRefinement[];readonly range: SourceRange;};+export type CaptionProjection = {+ readonly contract: "svml.caption-projection@0";+ readonly text: string;+ readonly regions: readonly ParsedCaptionRegion[];+};+export type SemanticAnchor = {readonly id: string;readonly kind: "segment-start" | "token-start" | "token-end" | "segment-end";@@ -112,16 +130,15 @@ export type ParsedNarrative = {readonly turns: readonly ParsedTurn[];readonly selections: readonly ParsedSelection[];readonly moments: readonly ParsedMoment[];- readonly captionAtoms: readonly ParsedCaptionAtom[];+ readonly captionProjection: CaptionProjection;readonly semanticIndex: {readonly contract: "svml.semantic-index@0";readonly anchors: readonly SemanticAnchor[];readonly digest: string;};- readonly projections: {+ readonly serializations: {readonly dialogue: string;readonly speech: string;- readonly caption: string;};};diff --git a/packages/script/test/script.test.ts b/packages/script/test/script.test.tsindex b5151f68..82d354aa 100644--- a/packages/script/test/script.test.ts+++ b/packages/script/test/script.test.ts@@ -7,6 +7,9 @@ import {formatScript,narrativeValue,parseScript,+ serializeCaption,+ serializeDialogue,+ serializeSpeech,} from "@svml/script";test("named blocks are Segments and Role Cues do not depend on line breaks", () => {@@ -29,7 +32,7 @@ test("named blocks are Segments and Role Cues do not depend on line breaks", ()assert.equal(digestOf(narrativeValue(compact)), digestOf(narrativeValue(multiline)));assert.deepEqual(compact.segments.map((segment) => segment.id), ["opening", "pause"]);assert.deepEqual(compact.turns.map((turn) => turn.role), ["ALICE", "BOB"]);- assert.equal(compact.projections.dialogue, "ALICE: Hello there.\nBOB: Good morning.");+ assert.equal(serializeDialogue(compact), "ALICE: Hello there.\nBOB: Good morning.");});test("Script produces exactly 2M + 2N independent semantic anchors", () => {@@ -58,12 +61,37 @@ test("selections, moments and Dual Text preserve separate semantic projections",@/whole~`,);- assert.equal(parsed.projections.speech, "I really laughed my ass off there.");- assert.equal(parsed.projections.caption, "I really laughed there.");- assert.equal(parsed.projections.dialogue, "BOB: I really laughed my ass off there.");+ assert.equal(serializeSpeech(parsed), "I really laughed my ass off there.");+ assert.equal(serializeCaption(parsed), "I really laughed there.");+ assert.equal(serializeDialogue(parsed), "BOB: I really laughed my ass off there.");assert.deepEqual(parsed.selections.map((selection) => selection.id), ["whole"]);assert.deepEqual(parsed.moments.map((moment) => moment.id), ["beat"]);- assert.equal(parsed.captionAtoms[0]?.display, "laughed");+ assert.equal(parsed.captionProjection.regions.find((region) => region.kind === "alias")?.display, "laughed");+});++test("Caption Projection owns speech ranges without inventing alias word timing", () => {+ const parsed = parseScript(+ "caption.svml",+ `<line>+ <test this | test this>+ <15% off | fifteen percent off>+ <that was insane | what the fuck>+ <what the— | what the fuck>+ < | um>+ </line>`,+ );+ const [identity, discount, replacement, interrupted, hidden] = parsed.captionProjection.regions;++ assert.equal(parsed.captionProjection.contract, "svml.caption-projection@0");+ assert.equal(identity?.kind, "identity");+ assert.deepEqual(identity?.refinements.map((item) => item.display), ["test", "this"]);+ assert.equal(discount?.kind, "alias");+ assert.deepEqual(discount?.refinements.map((item) => item.display), ["off"]);+ assert.deepEqual(replacement?.refinements, []);+ assert.deepEqual(interrupted?.refinements.map((item) => item.display), ["what", "the"]);+ assert.equal(hidden?.kind, "hidden");+ assert.equal(hidden?.display, "");+ assert.equal(hidden?.endTokenExclusive! - hidden?.startToken!, 1);});test("mismatched named Segment closes are rejected", () => {diff --git a/packages/speech-align/README.md b/packages/speech-align/README.mdnew file mode 100644index 00000000..0051fbad--- /dev/null+++ b/packages/speech-align/README.md@@ -0,0 +1,15 @@+# `@svml/speech-align`++Provider-neutral deterministic timing locator for one aligned transcript. The package does not+invoke Python, read audio, call a speech provider, use an LLM, infer speakers or retain multiple+candidate paths. A runtime adapter preserves its raw provider artifact and supplies the normalized+word, character, score, speech-activity and Segment evidence used here.++The locator aligns each known Script Segment independently. A bounded monotonic M:N dynamic+program accepts exact, split, merge, replacement, source-omission and evidence-insertion groups.+Within a selected group, timed evidence characters determine Script token boundaries; word times,+speech-activity bounds and deterministic zero-width estimates are fallbacks in that order. The result contains+one timing for every Script token and all `2M + 2N` semantic anchors.++Caption display is outside this package. `@svml/caption` composes the resulting complete speech+map with Script's authored Caption Projection and keeps presentation estimates local to the track.diff --git a/packages/whisperx/package.json b/packages/speech-align/package.jsonsimilarity index 86%rename from packages/whisperx/package.jsonrename to packages/speech-align/package.jsonindex 849dc588..3466e5b1 100644--- a/packages/whisperx/package.json+++ b/packages/speech-align/package.json@@ -1,5 +1,5 @@{- "name": "@svml/whisperx",+ "name": "@svml/speech-align","version": "0.0.0-dev","private": true,"type": "module",diff --git a/packages/whisperx/src/align.ts b/packages/speech-align/src/align.tssimilarity index 97%rename from packages/whisperx/src/align.tsrename to packages/speech-align/src/align.tsindex 006bd26d..ba7f9563 100644--- a/packages/whisperx/src/align.ts+++ b/packages/speech-align/src/align.ts@@ -4,7 +4,7 @@ import { editDistance, normalizeForAlignment } from "./normalize.js";import type {AlignmentGroup,AlignmentRelation,- WhisperXWordEvidence,+ SpeechWordEvidence,} from "./types.js";const EPSILON = 1e-9;@@ -25,7 +25,7 @@ function comparisonText(values: readonly { readonly text: string }[]): string {return values.map((value) => normalizeForAlignment(value.text)).join("");}-function evidenceReliability(words: readonly WhisperXWordEvidence[]): number {+function evidenceReliability(words: readonly SpeechWordEvidence[]): number {const scores = words.map((word) => word.score).filter((score): score is number => score !== undefined && Number.isFinite(score));@@ -36,7 +36,7 @@ function evidenceReliability(words: readonly WhisperXWordEvidence[]): number {function exactWordLcs(source: readonly ParsedToken[],- evidence: readonly WhisperXWordEvidence[],+ evidence: readonly SpeechWordEvidence[],): number {const rows = Array.from({ length: source.length + 1 }, () =>Array<number>(evidence.length + 1).fill(0));@@ -65,7 +65,7 @@ function relation(function pairedCost(source: readonly ParsedToken[],- evidence: readonly WhisperXWordEvidence[],+ evidence: readonly SpeechWordEvidence[],): number {const sourceText = comparisonText(source);const evidenceText = comparisonText(evidence);@@ -109,7 +109,7 @@ function extend(export function alignWordGroups(sourceSegmentId: string,source: readonly ParsedToken[],- evidence: readonly WhisperXWordEvidence[],+ evidence: readonly SpeechWordEvidence[],maxGroupSize = DEFAULT_MAX_GROUP,): AlignmentGroup[] {const rows = Array.from({ length: source.length + 1 }, () =>diff --git a/packages/speech-align/src/error.ts b/packages/speech-align/src/error.tsnew file mode 100644index 00000000..19d966ca--- /dev/null+++ b/packages/speech-align/src/error.ts@@ -0,0 +1,9 @@+export class SpeechAlignmentError extends Error {+ readonly code: string;++ constructor(code: string, message: string) {+ super(message);+ this.name = "SpeechAlignmentError";+ this.code = code;+ }+}diff --git a/packages/whisperx/src/index.ts b/packages/speech-align/src/index.tssimilarity index 51%rename from packages/whisperx/src/index.tsrename to packages/speech-align/src/index.tsindex b6ad78d4..d13913e8 100644--- a/packages/whisperx/src/index.ts+++ b/packages/speech-align/src/index.ts@@ -1,6 +1,5 @@export { alignWordGroups } from "./align.js";-export { deriveCaptionAtomTiming } from "./caption.js";-export { WhisperXAlignmentError } from "./error.js";-export { locateWhisperXTiming } from "./locate.js";+export { SpeechAlignmentError } from "./error.js";+export { locateSpeechTiming } from "./locate.js";export { alignCharacters, alignmentCharacters, normalizeForAlignment } from "./normalize.js";export type * from "./types.js";diff --git a/packages/whisperx/src/locate.ts b/packages/speech-align/src/locate.tssimilarity index 82%rename from packages/whisperx/src/locate.tsrename to packages/speech-align/src/locate.tsindex d4137905..030aab6b 100644--- a/packages/whisperx/src/locate.ts+++ b/packages/speech-align/src/locate.ts@@ -2,7 +2,7 @@ import { digestOf } from "@svml/core";import type { ParsedNarrative, ParsedToken } from "@svml/script";import { alignWordGroups } from "./align.js";-import { WhisperXAlignmentError } from "./error.js";+import { SpeechAlignmentError } from "./error.js";import { alignCharacters, alignmentCharacters } from "./normalize.js";import type {AlignmentGroup,@@ -11,10 +11,10 @@ import type {TimedSpeechSegment,TimedSpeechToken,TimingQuality,- WhisperXCharEvidence,- WhisperXEvidence,- WhisperXSegmentEvidence,- WhisperXWordEvidence,+ AlignedTranscriptEvidence,+ AlignedTranscriptSegment,+ SpeechCharacterEvidence,+ SpeechWordEvidence,} from "./types.js";const EPSILON = 1e-6;@@ -34,7 +34,7 @@ type TimedEvidenceChar = {};function fail(code: string, message: string): never {- throw new WhisperXAlignmentError(code, message);+ throw new SpeechAlignmentError(code, message);}function finite(value: number | undefined): value is number {@@ -43,33 +43,33 @@ function finite(value: number | undefined): value is number {function validateWindow(start: number, end: number, limit: number, label: string): void {if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end < start || end > limit + EPSILON) {- fail("WHISPERX_WINDOW", `${label} has invalid time window ${start}..${end}.`);+ fail("SPEECH_WINDOW", `${label} has invalid time window ${start}..${end}.`);}}-function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidence): void {- if (evidence.contract !== "svml.whisperx-evidence@0") {- fail("WHISPERX_CONTRACT", `Unsupported WhisperX evidence contract ${evidence.contract}.`);+function validateEvidence(narrative: ParsedNarrative, evidence: AlignedTranscriptEvidence): void {+ if (evidence.contract !== "svml.aligned-transcript-evidence@0") {+ fail("SPEECH_CONTRACT", `Unsupported aligned-transcript contract ${evidence.contract}.`);}if (!Number.isFinite(evidence.durationSec) || evidence.durationSec <= 0) {- fail("WHISPERX_DURATION", "WhisperX evidence duration must be positive and finite.");+ fail("SPEECH_DURATION", "Aligned-transcript duration must be positive and finite.");}const expected = new Set(narrative.segments.map((segment) => segment.id));const seen = new Set<string>();for (const segment of evidence.segments) {if (!expected.has(segment.sourceSegmentId)) {- fail("WHISPERX_SEGMENT_UNKNOWN", `WhisperX evidence references unknown Segment ${segment.sourceSegmentId}.`);+ fail("SPEECH_SEGMENT_UNKNOWN", `Aligned transcript references unknown Segment ${segment.sourceSegmentId}.`);}if (seen.has(segment.sourceSegmentId)) {- fail("WHISPERX_SEGMENT_DUPLICATE", `WhisperX evidence repeats Segment ${segment.sourceSegmentId}.`);+ fail("SPEECH_SEGMENT_DUPLICATE", `Aligned transcript repeats Segment ${segment.sourceSegmentId}.`);}seen.add(segment.sourceSegmentId);validateWindow(segment.startSec, segment.endSec, evidence.durationSec, `Segment ${segment.sourceSegmentId}`);let previousEnd = segment.startSec;for (const [index, word] of segment.words.entries()) {- if (typeof word.text !== "string") fail("WHISPERX_WORD_TEXT", `Word ${index + 1} has no text.`);+ if (typeof word.text !== "string") fail("SPEECH_WORD_TEXT", `Word ${index + 1} has no text.`);if (finite(word.startSec) !== finite(word.endSec)) {- fail("WHISPERX_WORD_PARTIAL_TIME", `Word ${index + 1} must provide both start and end or neither.`);+ fail("SPEECH_WORD_PARTIAL_TIME", `Word ${index + 1} must provide both start and end or neither.`);}if (finite(word.startSec) && finite(word.endSec)) {validateWindow(word.startSec, word.endSec, evidence.durationSec, `Word ${index + 1}`);@@ -78,20 +78,20 @@ function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidence|| word.endSec > segment.endSec + EPSILON|| word.startSec < previousEnd - EPSILON) {- fail("WHISPERX_WORD_ORDER", `Word ${index + 1} is outside or overlaps its Segment window.`);+ fail("SPEECH_WORD_ORDER", `Word ${index + 1} is outside or overlaps its Segment window.`);}previousEnd = word.endSec;}if (word.score !== undefined && (!Number.isFinite(word.score) || word.score < 0 || word.score > 1)) {- fail("WHISPERX_SCORE", `Word ${index + 1} score must be between zero and one.`);+ fail("SPEECH_SCORE", `Word ${index + 1} score must be between zero and one.`);}}for (const [index, char] of segment.chars.entries()) {if (!Number.isInteger(char.wordIndex) || char.wordIndex < 0 || char.wordIndex >= segment.words.length) {- fail("WHISPERX_CHAR_WORD", `Character ${index + 1} has an invalid wordIndex.`);+ fail("SPEECH_CHAR_WORD", `Character ${index + 1} has an invalid wordIndex.`);}if (finite(char.startSec) !== finite(char.endSec)) {- fail("WHISPERX_CHAR_PARTIAL_TIME", `Character ${index + 1} must provide both start and end or neither.`);+ fail("SPEECH_CHAR_PARTIAL_TIME", `Character ${index + 1} must provide both start and end or neither.`);}if (finite(char.startSec) && finite(char.endSec)) {validateWindow(char.startSec, char.endSec, evidence.durationSec, `Character ${index + 1}`);@@ -102,28 +102,28 @@ function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidence|| (finite(word.startSec) && char.startSec < word.startSec - EPSILON)|| (finite(word.endSec) && char.endSec > word.endSec + EPSILON)) {- fail("WHISPERX_CHAR_WINDOW", `Character ${index + 1} falls outside its Word or Segment window.`);+ fail("SPEECH_CHAR_WINDOW", `Character ${index + 1} falls outside its Word or Segment window.`);}}if (char.score !== undefined && (!Number.isFinite(char.score) || char.score < 0 || char.score > 1)) {- fail("WHISPERX_SCORE", `Character ${index + 1} score must be between zero and one.`);+ fail("SPEECH_SCORE", `Character ${index + 1} score must be between zero and one.`);}}let previousVadEnd = segment.startSec;- for (const [index, span] of (segment.vadSpans ?? []).entries()) {+ for (const [index, span] of (segment.speechActivity ?? []).entries()) {validateWindow(span.startSec, span.endSec, evidence.durationSec, `VAD span ${index + 1}`);if (span.startSec < segment.startSec - EPSILON|| span.endSec > segment.endSec + EPSILON|| span.startSec < previousVadEnd - EPSILON) {- fail("WHISPERX_VAD_WINDOW", `VAD span ${index + 1} falls outside Segment ${segment.sourceSegmentId}.`);+ fail("SPEECH_ACTIVITY_WINDOW", `Speech activity ${index + 1} falls outside Segment ${segment.sourceSegmentId}.`);}previousVadEnd = span.endSec;}}for (const id of expected) {- if (!seen.has(id)) fail("WHISPERX_SEGMENT_MISSING", `WhisperX evidence is missing Segment ${id}.`);+ if (!seen.has(id)) fail("SPEECH_SEGMENT_MISSING", `Aligned transcript is missing Segment ${id}.`);}const segmentsById = new Map(evidence.segments.map((segment) => [segment.sourceSegmentId, segment]));let previousSegmentStart = -Infinity;@@ -134,7 +134,7 @@ function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidencesegment.startSec < previousSegmentStart - EPSILON|| segment.endSec < previousSegmentEnd - EPSILON) {- fail("WHISPERX_SEGMENT_ORDER", `Segment ${segment.sourceSegmentId} violates Script source order.`);+ fail("SPEECH_SEGMENT_ORDER", `Segment ${segment.sourceSegmentId} violates Script source order.`);}previousSegmentStart = segment.startSec;previousSegmentEnd = segment.endSec;@@ -144,7 +144,7 @@ function validateEvidence(narrative: ParsedNarrative, evidence: WhisperXEvidencefunction syntheticCharTimes(characters: readonly string[],wordIndex: number,- word: WhisperXWordEvidence,+ word: SpeechWordEvidence,): TimedEvidenceChar[] {const hasWindow = finite(word.startSec) && finite(word.endSec);return characters.map((value, index) => ({@@ -160,9 +160,9 @@ function syntheticCharTimes(}function timedCharsForWord(- word: WhisperXWordEvidence,+ word: SpeechWordEvidence,wordIndex: number,- rawChars: readonly WhisperXCharEvidence[],+ rawChars: readonly SpeechCharacterEvidence[],): TimedEvidenceChar[] {const expected = alignmentCharacters(word.text);if (!expected.length) return [];@@ -192,11 +192,11 @@ function timedCharsForWord(return fallback;}-function timedChars(segment: WhisperXSegmentEvidence): TimedEvidenceChar[] {+function timedChars(segment: AlignedTranscriptSegment): TimedEvidenceChar[] {return segment.words.flatMap((word, index) => timedCharsForWord(word, index, segment.chars));}-function scoredWords(segment: WhisperXSegmentEvidence): WhisperXWordEvidence[] {+function scoredWords(segment: AlignedTranscriptSegment): SpeechWordEvidence[] {return segment.words.map((word, wordIndex) => {const scores = [...(word.score === undefined ? [] : [word.score]),@@ -212,7 +212,7 @@ function scoredWords(segment: WhisperXSegmentEvidence): WhisperXWordEvidence[] {function groupWindow(group: AlignmentGroup,- words: readonly WhisperXWordEvidence[],+ words: readonly SpeechWordEvidence[],chars: readonly TimedEvidenceChar[],): { readonly startSec: number; readonly endSec: number } | undefined {const wordRun = words.slice(group.evidenceWordStart, group.evidenceWordEndExclusive);@@ -233,7 +233,7 @@ function groupWindow(function locatePairedGroup(group: AlignmentGroup,source: readonly ParsedToken[],- words: readonly WhisperXWordEvidence[],+ words: readonly SpeechWordEvidence[],chars: readonly TimedEvidenceChar[],output: Array<MutableTiming | undefined>,): void {@@ -276,8 +276,8 @@ function locatePairedGroup(}}-function speechBounds(segment: WhisperXSegmentEvidence): { readonly start: number; readonly end: number } {- const spans = segment.vadSpans ?? [];+function speechBounds(segment: AlignedTranscriptSegment): { readonly start: number; readonly end: number } {+ const spans = segment.speechActivity ?? [];return spans.length? { start: spans[0]!.startSec, end: spans.at(-1)!.endSec }: { start: segment.startSec, end: segment.endSec };@@ -317,7 +317,7 @@ function assertMonotonic(tokens: readonly TimedSpeechToken[], segmentId: string)let previousEnd = -Infinity;for (const token of tokens) {if (token.startSec < previousEnd - EPSILON || token.endSec < token.startSec - EPSILON) {- fail("WHISPERX_MAP_ORDER", `Located token ${token.tokenId} violates time order in Segment ${segmentId}.`);+ fail("SPEECH_MAP_ORDER", `Located token ${token.tokenId} violates time order in Segment ${segmentId}.`);}previousEnd = token.endSec;}@@ -327,9 +327,9 @@ function mapDigest(map: Omit<CompleteSpeechTimeMap, "mapDigest">): string {return digestOf(map);}-export function locateWhisperXTiming(+export function locateSpeechTiming(narrative: ParsedNarrative,- evidence: WhisperXEvidence,+ evidence: AlignedTranscriptEvidence,): CompleteSpeechTimeMap {validateEvidence(narrative, evidence);const evidenceBySegment = new Map(evidence.segments.map((segment) => [segment.sourceSegmentId, segment]));@@ -378,7 +378,7 @@ export function locateWhisperXTiming(: { identity: anchor.id, timeSec: token.endSec, quality: token.endQuality };});const payload = {- contract: "svml.whisperx-speech-time-map@0" as const,+ contract: "svml.speech-time-map@0" as const,semanticIndexDigest: narrative.semanticIndex.digest,durationSec: evidence.durationSec,segments: timedSegments,diff --git a/packages/whisperx/src/normalize.ts b/packages/speech-align/src/normalize.tssimilarity index 100%rename from packages/whisperx/src/normalize.tsrename to packages/speech-align/src/normalize.tsdiff --git a/packages/whisperx/src/types.ts b/packages/speech-align/src/types.tssimilarity index 66%rename from packages/whisperx/src/types.tsrename to packages/speech-align/src/types.tsindex 11b42328..d312b99a 100644--- a/packages/whisperx/src/types.ts+++ b/packages/speech-align/src/types.ts@@ -1,13 +1,13 @@export type TimingQuality = "measured" | "derived" | "estimated";-export type WhisperXWordEvidence = {+export type SpeechWordEvidence = {readonly text: string;readonly startSec?: number;readonly endSec?: number;readonly score?: number;};-export type WhisperXCharEvidence = {+export type SpeechCharacterEvidence = {readonly char: string;readonly wordIndex: number;readonly startSec?: number;@@ -15,29 +15,29 @@ export type WhisperXCharEvidence = {readonly score?: number;};-export type WhisperXVadSpan = {+export type SpeechActivitySpan = {readonly startSec: number;readonly endSec: number;};/**- * Normalized evidence from one ordinary WhisperX transcription + alignment run.- * A host adapter keeps the raw provider result as an artifact and lowers the+ * Provider-neutral evidence produced by a speech-recognition adapter. The+ * adapter keeps its raw provider result as an artifact and lowers only the* fields used by this deterministic locator into this value.*/-export type WhisperXSegmentEvidence = {+export type AlignedTranscriptSegment = {readonly sourceSegmentId: string;readonly startSec: number;readonly endSec: number;- readonly words: readonly WhisperXWordEvidence[];- readonly chars: readonly WhisperXCharEvidence[];- readonly vadSpans?: readonly WhisperXVadSpan[];+ readonly words: readonly SpeechWordEvidence[];+ readonly chars: readonly SpeechCharacterEvidence[];+ readonly speechActivity?: readonly SpeechActivitySpan[];};-export type WhisperXEvidence = {- readonly contract: "svml.whisperx-evidence@0";+export type AlignedTranscriptEvidence = {+ readonly contract: "svml.aligned-transcript-evidence@0";readonly durationSec: number;- readonly segments: readonly WhisperXSegmentEvidence[];+ readonly segments: readonly AlignedTranscriptSegment[];};export type AlignmentRelation =@@ -81,7 +81,7 @@ export type SemanticTimePoint = {};export type CompleteSpeechTimeMap = {- readonly contract: "svml.whisperx-speech-time-map@0";+ readonly contract: "svml.speech-time-map@0";readonly semanticIndexDigest: string;readonly durationSec: number;readonly segments: readonly TimedSpeechSegment[];@@ -90,14 +90,3 @@ export type CompleteSpeechTimeMap = {readonly groups: readonly AlignmentGroup[];readonly mapDigest: string;};--export type TimedCaptionAtom = {- readonly id: string;- readonly display: string;- readonly segmentId: string;- readonly sourceTokenIds: readonly string[];- readonly startSec: number;- readonly endSec: number;- readonly startQuality: TimingQuality;- readonly endQuality: TimingQuality;-};diff --git a/packages/whisperx/test/whisperx.test.ts b/packages/speech-align/test/speech-align.test.tssimilarity index 71%rename from packages/whisperx/test/whisperx.test.tsrename to packages/speech-align/test/speech-align.test.tsindex 70022338..6d4f21a6 100644--- a/packages/whisperx/test/whisperx.test.ts+++ b/packages/speech-align/test/speech-align.test.ts@@ -3,27 +3,26 @@ import test from "node:test";import { parseScript } from "@svml/script";import {- WhisperXAlignmentError,- deriveCaptionAtomTiming,- locateWhisperXTiming,-} from "@svml/whisperx";+ SpeechAlignmentError,+ locateSpeechTiming,+} from "@svml/speech-align";import type {- WhisperXCharEvidence,- WhisperXEvidence,- WhisperXWordEvidence,-} from "@svml/whisperx";+ AlignedTranscriptEvidence,+ SpeechCharacterEvidence,+ SpeechWordEvidence,+} from "@svml/speech-align";function evidence(args: {readonly segmentId?: string;readonly durationSec?: number;readonly startSec?: number;readonly endSec?: number;- readonly words: readonly WhisperXWordEvidence[];- readonly chars?: readonly WhisperXCharEvidence[];+ readonly words: readonly SpeechWordEvidence[];+ readonly chars?: readonly SpeechCharacterEvidence[];readonly vad?: readonly { readonly startSec: number; readonly endSec: number }[];-}): WhisperXEvidence {+}): AlignedTranscriptEvidence {return {- contract: "svml.whisperx-evidence@0",+ contract: "svml.aligned-transcript-evidence@0",durationSec: args.durationSec ?? 2,segments: [{@@ -32,7 +31,7 @@ function evidence(args: {endSec: args.endSec ?? 2,words: args.words,chars: args.chars ?? [],- ...(args.vad === undefined ? {} : { vadSpans: args.vad }),+ ...(args.vad === undefined ? {} : { speechActivity: args.vad }),},],};@@ -43,7 +42,7 @@ function characters(starts: readonly number[],ends: readonly number[],wordIndex = 0,-): WhisperXCharEvidence[] {+): SpeechCharacterEvidence[] {return [...text].map((char, index) => ({char,wordIndex,@@ -53,9 +52,9 @@ function characters(}));}-test("exact WhisperX words cover every Script and Segment anchor", () => {+test("exact transcript words cover every Script and Segment anchor", () => {const narrative = parseScript("exact.svml", "<line>Hello world.</line>");- const map = locateWhisperXTiming(narrative, evidence({+ const map = locateSpeechTiming(narrative, evidence({words: [{ text: "Hello", startSec: 0.1, endSec: 0.4, score: 0.97 },{ text: "world", startSec: 0.5, endSec: 0.9, score: 0.96 },@@ -75,9 +74,9 @@ test("exact WhisperX words cover every Script and Segment anchor", () => {assert.match(map.mapDigest, /^sha256:[a-f0-9]{64}$/u);});-test("M:1 uses WhisperX character times instead of dividing a merged word by length", () => {+test("M:1 uses evidence character times instead of dividing a merged word by length", () => {const narrative = parseScript("merge.svml", "<line>can not</line>");- const map = locateWhisperXTiming(narrative, evidence({+ const map = locateSpeechTiming(narrative, evidence({endSec: 1,durationSec: 1,words: [{ text: "cannot", startSec: 0.1, endSec: 0.78, score: 0.93 }],@@ -98,9 +97,9 @@ test("M:1 uses WhisperX character times instead of dividing a merged word by len);});-test("1:N wraps all WhisperX words in one Script token", () => {+test("1:N wraps all evidence words in one Script token", () => {const narrative = parseScript("split.svml", "<line>website</line>");- const map = locateWhisperXTiming(narrative, evidence({+ const map = locateSpeechTiming(narrative, evidence({words: [{ text: "web", startSec: 0.2, endSec: 0.45, score: 0.9 },{ text: "site", startSec: 0.5, endSec: 0.82, score: 0.91 },@@ -116,7 +115,7 @@ test("1:N wraps all WhisperX words in one Script token", () => {test("a recognized filler stays an insertion and does not absorb neighboring Script words", () => {const narrative = parseScript("insertion.svml", "<line>I really like it.</line>");- const map = locateWhisperXTiming(narrative, evidence({+ const map = locateSpeechTiming(narrative, evidence({words: [{ text: "I", startSec: 0.1, endSec: 0.2, score: 0.98 },{ text: "uh", startSec: 0.24, endSec: 0.34, score: 0.88 },@@ -146,7 +145,7 @@ test("a recognized filler stays an insertion and does not absorb neighboring Scrtest("an omitted Script word gets a zero-width estimated point between measured neighbors", () => {const narrative = parseScript("omission.svml", "<line>This is very good.</line>");- const map = locateWhisperXTiming(narrative, evidence({+ const map = locateSpeechTiming(narrative, evidence({words: [{ text: "This", startSec: 0.1, endSec: 0.25, score: 0.98 },{ text: "is", startSec: 0.3, endSec: 0.4, score: 0.97 },@@ -168,7 +167,7 @@ test("an omitted Script word gets a zero-width estimated point between measuredtest("VAD bounds contain estimates when an entire Script Segment has no recognized words", () => {const narrative = parseScript("vad.svml", "<line>One two.</line>");- const map = locateWhisperXTiming(narrative, evidence({+ const map = locateSpeechTiming(narrative, evidence({words: [],vad: [{ startSec: 0.4, endSec: 1.2 }],}));@@ -182,44 +181,10 @@ test("VAD bounds contain estimates when an entire Script Segment has no recogniz);});-test("Dual Text display inherits one speech envelope without linear display-word timing", () => {- const narrative = parseScript(- "dual.svml",- "<line>That was <really funny | absolutely hilarious>.</line>",- );- const map = locateWhisperXTiming(narrative, evidence({- words: [- { text: "That", startSec: 0.1, endSec: 0.22 },- { text: "was", startSec: 0.25, endSec: 0.34 },- { text: "absolutely", startSec: 0.4, endSec: 0.72 },- { text: "hilarious", startSec: 0.75, endSec: 1.1 },- ],- }));- const captions = deriveCaptionAtomTiming(narrative, map);-- assert.equal(captions.length, 1);- assert.deepEqual(- {- display: captions[0]?.display,- sourceTokenIds: captions[0]?.sourceTokenIds,- startSec: captions[0]?.startSec,- endSec: captions[0]?.endSec,- startQuality: captions[0]?.startQuality,- },- {- display: "really funny",- sourceTokenIds: narrative.tokens.slice(2, 4).map((token) => token.id),- startSec: 0.4,- endSec: 1.1,- startQuality: "derived",- },- );-});-test("multiple Script Segments stay independent even when evidence records arrive out of order", () => {const narrative = parseScript("segments.svml", "<one>Hello.</one><two>Goodbye.</two>");- const map = locateWhisperXTiming(narrative, {- contract: "svml.whisperx-evidence@0",+ const map = locateSpeechTiming(narrative, {+ contract: "svml.aligned-transcript-evidence@0",durationSec: 2,segments: [{@@ -246,15 +211,15 @@ test("multiple Script Segments stay independent even when evidence records arrivassert.deepEqual(map.groups.map((group) => group.sourceSegmentId), ["one", "two"]);});-test("invalid overlapping WhisperX word windows fail instead of producing a reversed map", () => {+test("invalid overlapping evidence word windows fail instead of producing a reversed map", () => {const narrative = parseScript("bad.svml", "<line>one two</line>");assert.throws(- () => locateWhisperXTiming(narrative, evidence({+ () => locateSpeechTiming(narrative, evidence({words: [{ text: "one", startSec: 0.1, endSec: 0.5 },{ text: "two", startSec: 0.4, endSec: 0.8 },],})),- (error: unknown) => error instanceof WhisperXAlignmentError && error.code === "WHISPERX_WORD_ORDER",+ (error: unknown) => error instanceof SpeechAlignmentError && error.code === "SPEECH_WORD_ORDER",);});diff --git a/packages/whisperx/README.md b/packages/whisperx/README.mddeleted file mode 100644index beb2ccfe..00000000--- a/packages/whisperx/README.md+++ /dev/null@@ -1,16 +0,0 @@-# `@svml/whisperx`--Deterministic timing locator for the result of one ordinary WhisperX run. The package does not-invoke Python, read audio, rerun forced alignment against Script, use an LLM, infer speakers or-retain multiple candidate paths. A Host adapter preserves the raw WhisperX artifact and supplies-the normalized word, character, score, VAD and Segment fields used here.--The locator aligns each known Script Segment independently. A bounded monotonic M:N dynamic-program accepts exact, split, merge, replacement, source-omission and evidence-insertion groups.-Within a selected group, timed WhisperX characters determine Script token boundaries; word times,-VAD bounds and deterministic zero-width estimates are fallbacks in that order. The result contains-one timing for every Script token and all `2M + 2N` semantic anchors.--Dual Text is not located a second time. `deriveCaptionAtomTiming()` composes the speech map with-the authored `CaptionAtom -> speech token range` projection. The complete display atom inherits-the speech envelope; this package never invents linear timings for words inside its display side.diff --git a/packages/whisperx/src/caption.ts b/packages/whisperx/src/caption.tsdeleted file mode 100644index 4b406a83..00000000--- a/packages/whisperx/src/caption.ts+++ /dev/null@@ -1,54 +0,0 @@-import type { ParsedNarrative } from "@svml/script";--import { WhisperXAlignmentError } from "./error.js";-import type { CompleteSpeechTimeMap, TimedCaptionAtom, TimingQuality } from "./types.js";--const QUALITY_RANK: Readonly<Record<TimingQuality, number>> = {- measured: 0,- derived: 1,- estimated: 2,-};--function worse(left: TimingQuality, right: TimingQuality): TimingQuality {- return QUALITY_RANK[left] >= QUALITY_RANK[right] ? left : right;-}--/**- * Derives only explicit Dual Text atoms. Ordinary caption text reuses its- * corresponding speech token timing directly. A Dual Text display side stays- * atomic and inherits the complete speech-side envelope; it is never linearly- * split into invented display-word times.- */-export function deriveCaptionAtomTiming(- narrative: ParsedNarrative,- map: CompleteSpeechTimeMap,-): TimedCaptionAtom[] {- if (map.semanticIndexDigest !== narrative.semanticIndex.digest) {- throw new WhisperXAlignmentError(- "WHISPERX_CAPTION_MAP",- "Caption projection and speech map use different Semantic Indexes.",- );- }- return narrative.captionAtoms.map((atom) => {- const source = narrative.tokens.slice(atom.startToken, atom.endTokenExclusive);- const timed = source.map((token) => map.tokens.find((value) => value.tokenId === token.id));- if (!timed.length || timed.some((token) => token === undefined)) {- throw new WhisperXAlignmentError(- "WHISPERX_CAPTION_COVERAGE",- `Caption atom ${atom.id} is not covered by the complete speech map.`,- );- }- const first = timed[0]!;- const last = timed.at(-1)!;- return {- id: atom.id,- display: atom.display,- segmentId: atom.segmentId,- sourceTokenIds: source.map((token) => token.id),- startSec: first.startSec,- endSec: last.endSec,- startQuality: worse(first.startQuality, "derived"),- endQuality: worse(last.endQuality, "derived"),- };- });-}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex d0f61f46..f0e1de69 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -22,6 +22,18 @@ importers:specifier: 5.9.3version: 5.9.3+ packages/caption:+ dependencies:+ '@svml/core':+ specifier: workspace:*+ version: link:../core+ '@svml/script':+ specifier: workspace:*+ version: link:../script+ '@svml/speech-align':+ specifier: workspace:*+ version: link:../speech-align+packages/core:dependencies:'@svml/protocol':@@ -48,24 +60,24 @@ importers:specifier: workspace:*version: link:../protocol- packages/text:+ packages/speech-align:dependencies:'@svml/core':specifier: workspace:*version: link:../core- '@svml/protocol':- specifier: workspace:*- version: link:../protocol- devDependencies:'@svml/script':specifier: workspace:*version: link:../script- packages/whisperx:+ packages/text:dependencies:'@svml/core':specifier: workspace:*version: link:../core+ '@svml/protocol':+ specifier: workspace:*+ version: link:../protocol+ devDependencies:'@svml/script':specifier: workspace:*version: link:../scriptdiff --git a/tsconfig.v2.json b/tsconfig.v2.jsonindex a5348d84..7c9cc4f4 100644--- a/tsconfig.v2.json+++ b/tsconfig.v2.json@@ -10,7 +10,8 @@"@svml/driver-node": ["packages/driver-node/src/index.ts"],"@svml/script": ["packages/script/src/index.ts"],"@svml/text": ["packages/text/src/index.ts"],- "@svml/whisperx": ["packages/whisperx/src/index.ts"]+ "@svml/speech-align": ["packages/speech-align/src/index.ts"],+ "@svml/caption": ["packages/caption/src/index.ts"]}},"include": ["packages/**/*.ts"],